diff options
Diffstat (limited to 'arch/x86')
-rw-r--r-- | arch/x86/include/asm/local.h | 37 | ||||
-rw-r--r-- | arch/x86/include/asm/percpu.h | 119 | ||||
-rw-r--r-- | arch/x86/include/asm/system.h | 8 | ||||
-rw-r--r-- | arch/x86/kernel/apic/nmi.c | 6 | ||||
-rw-r--r-- | arch/x86/kernel/head_32.S | 6 | ||||
-rw-r--r-- | arch/x86/kernel/vmlinux.lds.S | 4 | ||||
-rw-r--r-- | arch/x86/xen/xen-asm_32.S | 4 |
7 files changed, 101 insertions, 83 deletions
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h index 47b9b6f19057..2e9972468a5d 100644 --- a/arch/x86/include/asm/local.h +++ b/arch/x86/include/asm/local.h | |||
@@ -195,41 +195,4 @@ static inline long local_sub_return(long i, local_t *l) | |||
195 | #define __local_add(i, l) local_add((i), (l)) | 195 | #define __local_add(i, l) local_add((i), (l)) |
196 | #define __local_sub(i, l) local_sub((i), (l)) | 196 | #define __local_sub(i, l) local_sub((i), (l)) |
197 | 197 | ||
198 | /* Use these for per-cpu local_t variables: on some archs they are | ||
199 | * much more efficient than these naive implementations. Note they take | ||
200 | * a variable, not an address. | ||
201 | * | ||
202 | * X86_64: This could be done better if we moved the per cpu data directly | ||
203 | * after GS. | ||
204 | */ | ||
205 | |||
206 | /* Need to disable preemption for the cpu local counters otherwise we could | ||
207 | still access a variable of a previous CPU in a non atomic way. */ | ||
208 | #define cpu_local_wrap_v(l) \ | ||
209 | ({ \ | ||
210 | local_t res__; \ | ||
211 | preempt_disable(); \ | ||
212 | res__ = (l); \ | ||
213 | preempt_enable(); \ | ||
214 | res__; \ | ||
215 | }) | ||
216 | #define cpu_local_wrap(l) \ | ||
217 | ({ \ | ||
218 | preempt_disable(); \ | ||
219 | (l); \ | ||
220 | preempt_enable(); \ | ||
221 | }) \ | ||
222 | |||
223 | #define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var((l)))) | ||
224 | #define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var((l)), (i))) | ||
225 | #define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var((l)))) | ||
226 | #define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var((l)))) | ||
227 | #define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var((l)))) | ||
228 | #define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var((l)))) | ||
229 | |||
230 | #define __cpu_local_inc(l) cpu_local_inc((l)) | ||
231 | #define __cpu_local_dec(l) cpu_local_dec((l)) | ||
232 | #define __cpu_local_add(i, l) cpu_local_add((i), (l)) | ||
233 | #define __cpu_local_sub(i, l) cpu_local_sub((i), (l)) | ||
234 | |||
235 | #endif /* _ASM_X86_LOCAL_H */ | 198 | #endif /* _ASM_X86_LOCAL_H */ |
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 0c44196b78ac..66a272dfd8b8 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h | |||
@@ -25,19 +25,18 @@ | |||
25 | */ | 25 | */ |
26 | #ifdef CONFIG_SMP | 26 | #ifdef CONFIG_SMP |
27 | #define PER_CPU(var, reg) \ | 27 | #define PER_CPU(var, reg) \ |
28 | __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg; \ | 28 | __percpu_mov_op %__percpu_seg:this_cpu_off, reg; \ |
29 | lea per_cpu__##var(reg), reg | 29 | lea var(reg), reg |
30 | #define PER_CPU_VAR(var) %__percpu_seg:per_cpu__##var | 30 | #define PER_CPU_VAR(var) %__percpu_seg:var |
31 | #else /* ! SMP */ | 31 | #else /* ! SMP */ |
32 | #define PER_CPU(var, reg) \ | 32 | #define PER_CPU(var, reg) __percpu_mov_op $var, reg |
33 | __percpu_mov_op $per_cpu__##var, reg | 33 | #define PER_CPU_VAR(var) var |
34 | #define PER_CPU_VAR(var) per_cpu__##var | ||
35 | #endif /* SMP */ | 34 | #endif /* SMP */ |
36 | 35 | ||
37 | #ifdef CONFIG_X86_64_SMP | 36 | #ifdef CONFIG_X86_64_SMP |
38 | #define INIT_PER_CPU_VAR(var) init_per_cpu__##var | 37 | #define INIT_PER_CPU_VAR(var) init_per_cpu__##var |
39 | #else | 38 | #else |
40 | #define INIT_PER_CPU_VAR(var) per_cpu__##var | 39 | #define INIT_PER_CPU_VAR(var) var |
41 | #endif | 40 | #endif |
42 | 41 | ||
43 | #else /* ...!ASSEMBLY */ | 42 | #else /* ...!ASSEMBLY */ |
@@ -60,12 +59,12 @@ | |||
60 | * There also must be an entry in vmlinux_64.lds.S | 59 | * There also must be an entry in vmlinux_64.lds.S |
61 | */ | 60 | */ |
62 | #define DECLARE_INIT_PER_CPU(var) \ | 61 | #define DECLARE_INIT_PER_CPU(var) \ |
63 | extern typeof(per_cpu_var(var)) init_per_cpu_var(var) | 62 | extern typeof(var) init_per_cpu_var(var) |
64 | 63 | ||
65 | #ifdef CONFIG_X86_64_SMP | 64 | #ifdef CONFIG_X86_64_SMP |
66 | #define init_per_cpu_var(var) init_per_cpu__##var | 65 | #define init_per_cpu_var(var) init_per_cpu__##var |
67 | #else | 66 | #else |
68 | #define init_per_cpu_var(var) per_cpu_var(var) | 67 | #define init_per_cpu_var(var) var |
69 | #endif | 68 | #endif |
70 | 69 | ||
71 | /* For arch-specific code, we can use direct single-insn ops (they | 70 | /* For arch-specific code, we can use direct single-insn ops (they |
@@ -104,6 +103,64 @@ do { \ | |||
104 | } \ | 103 | } \ |
105 | } while (0) | 104 | } while (0) |
106 | 105 | ||
106 | /* | ||
107 | * Generate a percpu add to memory instruction and optimize code | ||
108 | * if a one is added or subtracted. | ||
109 | */ | ||
110 | #define percpu_add_op(var, val) \ | ||
111 | do { \ | ||
112 | typedef typeof(var) pao_T__; \ | ||
113 | const int pao_ID__ = (__builtin_constant_p(val) && \ | ||
114 | ((val) == 1 || (val) == -1)) ? (val) : 0; \ | ||
115 | if (0) { \ | ||
116 | pao_T__ pao_tmp__; \ | ||
117 | pao_tmp__ = (val); \ | ||
118 | } \ | ||
119 | switch (sizeof(var)) { \ | ||
120 | case 1: \ | ||
121 | if (pao_ID__ == 1) \ | ||
122 | asm("incb "__percpu_arg(0) : "+m" (var)); \ | ||
123 | else if (pao_ID__ == -1) \ | ||
124 | asm("decb "__percpu_arg(0) : "+m" (var)); \ | ||
125 | else \ | ||
126 | asm("addb %1, "__percpu_arg(0) \ | ||
127 | : "+m" (var) \ | ||
128 | : "qi" ((pao_T__)(val))); \ | ||
129 | break; \ | ||
130 | case 2: \ | ||
131 | if (pao_ID__ == 1) \ | ||
132 | asm("incw "__percpu_arg(0) : "+m" (var)); \ | ||
133 | else if (pao_ID__ == -1) \ | ||
134 | asm("decw "__percpu_arg(0) : "+m" (var)); \ | ||
135 | else \ | ||
136 | asm("addw %1, "__percpu_arg(0) \ | ||
137 | : "+m" (var) \ | ||
138 | : "ri" ((pao_T__)(val))); \ | ||
139 | break; \ | ||
140 | case 4: \ | ||
141 | if (pao_ID__ == 1) \ | ||
142 | asm("incl "__percpu_arg(0) : "+m" (var)); \ | ||
143 | else if (pao_ID__ == -1) \ | ||
144 | asm("decl "__percpu_arg(0) : "+m" (var)); \ | ||
145 | else \ | ||
146 | asm("addl %1, "__percpu_arg(0) \ | ||
147 | : "+m" (var) \ | ||
148 | : "ri" ((pao_T__)(val))); \ | ||
149 | break; \ | ||
150 | case 8: \ | ||
151 | if (pao_ID__ == 1) \ | ||
152 | asm("incq "__percpu_arg(0) : "+m" (var)); \ | ||
153 | else if (pao_ID__ == -1) \ | ||
154 | asm("decq "__percpu_arg(0) : "+m" (var)); \ | ||
155 | else \ | ||
156 | asm("addq %1, "__percpu_arg(0) \ | ||
157 | : "+m" (var) \ | ||
158 | : "re" ((pao_T__)(val))); \ | ||
159 | break; \ | ||
160 | default: __bad_percpu_size(); \ | ||
161 | } \ | ||
162 | } while (0) | ||
163 | |||
107 | #define percpu_from_op(op, var, constraint) \ | 164 | #define percpu_from_op(op, var, constraint) \ |
108 | ({ \ | 165 | ({ \ |
109 | typeof(var) pfo_ret__; \ | 166 | typeof(var) pfo_ret__; \ |
@@ -142,16 +199,14 @@ do { \ | |||
142 | * per-thread variables implemented as per-cpu variables and thus | 199 | * per-thread variables implemented as per-cpu variables and thus |
143 | * stable for the duration of the respective task. | 200 | * stable for the duration of the respective task. |
144 | */ | 201 | */ |
145 | #define percpu_read(var) percpu_from_op("mov", per_cpu__##var, \ | 202 | #define percpu_read(var) percpu_from_op("mov", var, "m" (var)) |
146 | "m" (per_cpu__##var)) | 203 | #define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var))) |
147 | #define percpu_read_stable(var) percpu_from_op("mov", per_cpu__##var, \ | 204 | #define percpu_write(var, val) percpu_to_op("mov", var, val) |
148 | "p" (&per_cpu__##var)) | 205 | #define percpu_add(var, val) percpu_add_op(var, val) |
149 | #define percpu_write(var, val) percpu_to_op("mov", per_cpu__##var, val) | 206 | #define percpu_sub(var, val) percpu_add_op(var, -(val)) |
150 | #define percpu_add(var, val) percpu_to_op("add", per_cpu__##var, val) | 207 | #define percpu_and(var, val) percpu_to_op("and", var, val) |
151 | #define percpu_sub(var, val) percpu_to_op("sub", per_cpu__##var, val) | 208 | #define percpu_or(var, val) percpu_to_op("or", var, val) |
152 | #define percpu_and(var, val) percpu_to_op("and", per_cpu__##var, val) | 209 | #define percpu_xor(var, val) percpu_to_op("xor", var, val) |
153 | #define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val) | ||
154 | #define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val) | ||
155 | 210 | ||
156 | #define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 211 | #define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
157 | #define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 212 | #define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
@@ -160,9 +215,9 @@ do { \ | |||
160 | #define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) | 215 | #define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) |
161 | #define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) | 216 | #define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) |
162 | #define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) | 217 | #define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) |
163 | #define __this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) | 218 | #define __this_cpu_add_1(pcp, val) percpu_add_op((pcp), val) |
164 | #define __this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) | 219 | #define __this_cpu_add_2(pcp, val) percpu_add_op((pcp), val) |
165 | #define __this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) | 220 | #define __this_cpu_add_4(pcp, val) percpu_add_op((pcp), val) |
166 | #define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) | 221 | #define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) |
167 | #define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) | 222 | #define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) |
168 | #define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) | 223 | #define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) |
@@ -179,9 +234,9 @@ do { \ | |||
179 | #define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) | 234 | #define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) |
180 | #define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) | 235 | #define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) |
181 | #define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) | 236 | #define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) |
182 | #define this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) | 237 | #define this_cpu_add_1(pcp, val) percpu_add_op((pcp), val) |
183 | #define this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) | 238 | #define this_cpu_add_2(pcp, val) percpu_add_op((pcp), val) |
184 | #define this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) | 239 | #define this_cpu_add_4(pcp, val) percpu_add_op((pcp), val) |
185 | #define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) | 240 | #define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) |
186 | #define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) | 241 | #define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) |
187 | #define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) | 242 | #define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) |
@@ -192,9 +247,9 @@ do { \ | |||
192 | #define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) | 247 | #define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) |
193 | #define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) | 248 | #define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) |
194 | 249 | ||
195 | #define irqsafe_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) | 250 | #define irqsafe_cpu_add_1(pcp, val) percpu_add_op((pcp), val) |
196 | #define irqsafe_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) | 251 | #define irqsafe_cpu_add_2(pcp, val) percpu_add_op((pcp), val) |
197 | #define irqsafe_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) | 252 | #define irqsafe_cpu_add_4(pcp, val) percpu_add_op((pcp), val) |
198 | #define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) | 253 | #define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) |
199 | #define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) | 254 | #define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) |
200 | #define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) | 255 | #define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) |
@@ -212,19 +267,19 @@ do { \ | |||
212 | #ifdef CONFIG_X86_64 | 267 | #ifdef CONFIG_X86_64 |
213 | #define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 268 | #define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
214 | #define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) | 269 | #define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) |
215 | #define __this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) | 270 | #define __this_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
216 | #define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) | 271 | #define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
217 | #define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) | 272 | #define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
218 | #define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) | 273 | #define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) |
219 | 274 | ||
220 | #define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 275 | #define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
221 | #define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) | 276 | #define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) |
222 | #define this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) | 277 | #define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
223 | #define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) | 278 | #define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
224 | #define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) | 279 | #define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
225 | #define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) | 280 | #define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) |
226 | 281 | ||
227 | #define irqsafe_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) | 282 | #define irqsafe_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
228 | #define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) | 283 | #define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
229 | #define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) | 284 | #define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
230 | #define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) | 285 | #define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) |
@@ -236,7 +291,7 @@ do { \ | |||
236 | ({ \ | 291 | ({ \ |
237 | int old__; \ | 292 | int old__; \ |
238 | asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \ | 293 | asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \ |
239 | : "=r" (old__), "+m" (per_cpu__##var) \ | 294 | : "=r" (old__), "+m" (var) \ |
240 | : "dIr" (bit)); \ | 295 | : "dIr" (bit)); \ |
241 | old__; \ | 296 | old__; \ |
242 | }) | 297 | }) |
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h index e04740f7a0bb..b8fe48ee2ed9 100644 --- a/arch/x86/include/asm/system.h +++ b/arch/x86/include/asm/system.h | |||
@@ -32,7 +32,7 @@ extern void show_regs_common(void); | |||
32 | "movl %P[task_canary](%[next]), %%ebx\n\t" \ | 32 | "movl %P[task_canary](%[next]), %%ebx\n\t" \ |
33 | "movl %%ebx, "__percpu_arg([stack_canary])"\n\t" | 33 | "movl %%ebx, "__percpu_arg([stack_canary])"\n\t" |
34 | #define __switch_canary_oparam \ | 34 | #define __switch_canary_oparam \ |
35 | , [stack_canary] "=m" (per_cpu_var(stack_canary.canary)) | 35 | , [stack_canary] "=m" (stack_canary.canary) |
36 | #define __switch_canary_iparam \ | 36 | #define __switch_canary_iparam \ |
37 | , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) | 37 | , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) |
38 | #else /* CC_STACKPROTECTOR */ | 38 | #else /* CC_STACKPROTECTOR */ |
@@ -114,7 +114,7 @@ do { \ | |||
114 | "movq %P[task_canary](%%rsi),%%r8\n\t" \ | 114 | "movq %P[task_canary](%%rsi),%%r8\n\t" \ |
115 | "movq %%r8,"__percpu_arg([gs_canary])"\n\t" | 115 | "movq %%r8,"__percpu_arg([gs_canary])"\n\t" |
116 | #define __switch_canary_oparam \ | 116 | #define __switch_canary_oparam \ |
117 | , [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary)) | 117 | , [gs_canary] "=m" (irq_stack_union.stack_canary) |
118 | #define __switch_canary_iparam \ | 118 | #define __switch_canary_iparam \ |
119 | , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) | 119 | , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) |
120 | #else /* CC_STACKPROTECTOR */ | 120 | #else /* CC_STACKPROTECTOR */ |
@@ -133,7 +133,7 @@ do { \ | |||
133 | __switch_canary \ | 133 | __switch_canary \ |
134 | "movq %P[thread_info](%%rsi),%%r8\n\t" \ | 134 | "movq %P[thread_info](%%rsi),%%r8\n\t" \ |
135 | "movq %%rax,%%rdi\n\t" \ | 135 | "movq %%rax,%%rdi\n\t" \ |
136 | "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \ | 136 | "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \ |
137 | "jnz ret_from_fork\n\t" \ | 137 | "jnz ret_from_fork\n\t" \ |
138 | RESTORE_CONTEXT \ | 138 | RESTORE_CONTEXT \ |
139 | : "=a" (last) \ | 139 | : "=a" (last) \ |
@@ -143,7 +143,7 @@ do { \ | |||
143 | [ti_flags] "i" (offsetof(struct thread_info, flags)), \ | 143 | [ti_flags] "i" (offsetof(struct thread_info, flags)), \ |
144 | [_tif_fork] "i" (_TIF_FORK), \ | 144 | [_tif_fork] "i" (_TIF_FORK), \ |
145 | [thread_info] "i" (offsetof(struct task_struct, stack)), \ | 145 | [thread_info] "i" (offsetof(struct task_struct, stack)), \ |
146 | [current_task] "m" (per_cpu_var(current_task)) \ | 146 | [current_task] "m" (current_task) \ |
147 | __switch_canary_iparam \ | 147 | __switch_canary_iparam \ |
148 | : "memory", "cc" __EXTRA_CLOBBER) | 148 | : "memory", "cc" __EXTRA_CLOBBER) |
149 | #endif | 149 | #endif |
diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c index 0159a69396cb..4ada42c3dabb 100644 --- a/arch/x86/kernel/apic/nmi.c +++ b/arch/x86/kernel/apic/nmi.c | |||
@@ -438,8 +438,8 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) | |||
438 | * Ayiee, looks like this CPU is stuck ... | 438 | * Ayiee, looks like this CPU is stuck ... |
439 | * wait a few IRQs (5 seconds) before doing the oops ... | 439 | * wait a few IRQs (5 seconds) before doing the oops ... |
440 | */ | 440 | */ |
441 | __this_cpu_inc(per_cpu_var(alert_counter)); | 441 | __this_cpu_inc(alert_counter); |
442 | if (__this_cpu_read(per_cpu_var(alert_counter)) == 5 * nmi_hz) | 442 | if (__this_cpu_read(alert_counter) == 5 * nmi_hz) |
443 | /* | 443 | /* |
444 | * die_nmi will return ONLY if NOTIFY_STOP happens.. | 444 | * die_nmi will return ONLY if NOTIFY_STOP happens.. |
445 | */ | 445 | */ |
@@ -447,7 +447,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) | |||
447 | regs, panic_on_timeout); | 447 | regs, panic_on_timeout); |
448 | } else { | 448 | } else { |
449 | __get_cpu_var(last_irq_sum) = sum; | 449 | __get_cpu_var(last_irq_sum) = sum; |
450 | __this_cpu_write(per_cpu_var(alert_counter), 0); | 450 | __this_cpu_write(alert_counter, 0); |
451 | } | 451 | } |
452 | 452 | ||
453 | /* see if the nmi watchdog went off */ | 453 | /* see if the nmi watchdog went off */ |
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index 7fd318bac59c..37c3d4b17d85 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S | |||
@@ -442,8 +442,8 @@ is386: movl $2,%ecx # set MP | |||
442 | */ | 442 | */ |
443 | cmpb $0,ready | 443 | cmpb $0,ready |
444 | jne 1f | 444 | jne 1f |
445 | movl $per_cpu__gdt_page,%eax | 445 | movl $gdt_page,%eax |
446 | movl $per_cpu__stack_canary,%ecx | 446 | movl $stack_canary,%ecx |
447 | movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax) | 447 | movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax) |
448 | shrl $16, %ecx | 448 | shrl $16, %ecx |
449 | movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax) | 449 | movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax) |
@@ -706,7 +706,7 @@ idt_descr: | |||
706 | .word 0 # 32 bit align gdt_desc.address | 706 | .word 0 # 32 bit align gdt_desc.address |
707 | ENTRY(early_gdt_descr) | 707 | ENTRY(early_gdt_descr) |
708 | .word GDT_ENTRIES*8-1 | 708 | .word GDT_ENTRIES*8-1 |
709 | .long per_cpu__gdt_page /* Overwritten for secondary CPUs */ | 709 | .long gdt_page /* Overwritten for secondary CPUs */ |
710 | 710 | ||
711 | /* | 711 | /* |
712 | * The boot_gdt must mirror the equivalent in setup.S and is | 712 | * The boot_gdt must mirror the equivalent in setup.S and is |
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index f92a0da608cb..44879df55696 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S | |||
@@ -341,7 +341,7 @@ SECTIONS | |||
341 | * Per-cpu symbols which need to be offset from __per_cpu_load | 341 | * Per-cpu symbols which need to be offset from __per_cpu_load |
342 | * for the boot processor. | 342 | * for the boot processor. |
343 | */ | 343 | */ |
344 | #define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load | 344 | #define INIT_PER_CPU(x) init_per_cpu__##x = x + __per_cpu_load |
345 | INIT_PER_CPU(gdt_page); | 345 | INIT_PER_CPU(gdt_page); |
346 | INIT_PER_CPU(irq_stack_union); | 346 | INIT_PER_CPU(irq_stack_union); |
347 | 347 | ||
@@ -352,7 +352,7 @@ INIT_PER_CPU(irq_stack_union); | |||
352 | "kernel image bigger than KERNEL_IMAGE_SIZE"); | 352 | "kernel image bigger than KERNEL_IMAGE_SIZE"); |
353 | 353 | ||
354 | #ifdef CONFIG_SMP | 354 | #ifdef CONFIG_SMP |
355 | . = ASSERT((per_cpu__irq_stack_union == 0), | 355 | . = ASSERT((irq_stack_union == 0), |
356 | "irq_stack_union is not at start of per-cpu area"); | 356 | "irq_stack_union is not at start of per-cpu area"); |
357 | #endif | 357 | #endif |
358 | 358 | ||
diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S index 88e15deb8b82..22a2093b5862 100644 --- a/arch/x86/xen/xen-asm_32.S +++ b/arch/x86/xen/xen-asm_32.S | |||
@@ -90,9 +90,9 @@ ENTRY(xen_iret) | |||
90 | GET_THREAD_INFO(%eax) | 90 | GET_THREAD_INFO(%eax) |
91 | movl TI_cpu(%eax), %eax | 91 | movl TI_cpu(%eax), %eax |
92 | movl __per_cpu_offset(,%eax,4), %eax | 92 | movl __per_cpu_offset(,%eax,4), %eax |
93 | mov per_cpu__xen_vcpu(%eax), %eax | 93 | mov xen_vcpu(%eax), %eax |
94 | #else | 94 | #else |
95 | movl per_cpu__xen_vcpu, %eax | 95 | movl xen_vcpu, %eax |
96 | #endif | 96 | #endif |
97 | 97 | ||
98 | /* check IF state we're restoring */ | 98 | /* check IF state we're restoring */ |