aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/alpha/include/asm/local.h17
-rw-r--r--arch/blackfin/mach-common/entry.S4
-rw-r--r--arch/cris/arch-v10/kernel/entry.S2
-rw-r--r--arch/cris/arch-v32/mm/mmu.S2
-rw-r--r--arch/ia64/include/asm/percpu.h4
-rw-r--r--arch/ia64/kernel/ia64_ksyms.c4
-rw-r--r--arch/ia64/mm/discontig.c2
-rw-r--r--arch/m32r/include/asm/local.h25
-rw-r--r--arch/microblaze/include/asm/entry.h2
-rw-r--r--arch/mips/include/asm/local.h25
-rw-r--r--arch/parisc/lib/fixup.S8
-rw-r--r--arch/powerpc/include/asm/local.h25
-rw-r--r--arch/sparc/kernel/nmi.c7
-rw-r--r--arch/sparc/kernel/rtrap_64.S8
-rw-r--r--arch/x86/include/asm/local.h37
-rw-r--r--arch/x86/include/asm/percpu.h119
-rw-r--r--arch/x86/include/asm/system.h8
-rw-r--r--arch/x86/kernel/apic/nmi.c6
-rw-r--r--arch/x86/kernel/head_32.S6
-rw-r--r--arch/x86/kernel/vmlinux.lds.S4
-rw-r--r--arch/x86/xen/xen-asm_32.S4
21 files changed, 122 insertions, 197 deletions
diff --git a/arch/alpha/include/asm/local.h b/arch/alpha/include/asm/local.h
index 6ad3ea69642..b9e3e331837 100644
--- a/arch/alpha/include/asm/local.h
+++ b/arch/alpha/include/asm/local.h
@@ -98,21 +98,4 @@ static __inline__ long local_sub_return(long i, local_t * l)
98#define __local_add(i,l) ((l)->a.counter+=(i)) 98#define __local_add(i,l) ((l)->a.counter+=(i))
99#define __local_sub(i,l) ((l)->a.counter-=(i)) 99#define __local_sub(i,l) ((l)->a.counter-=(i))
100 100
101/* Use these for per-cpu local_t variables: on some archs they are
102 * much more efficient than these naive implementations. Note they take
103 * a variable, not an address.
104 */
105#define cpu_local_read(l) local_read(&__get_cpu_var(l))
106#define cpu_local_set(l, i) local_set(&__get_cpu_var(l), (i))
107
108#define cpu_local_inc(l) local_inc(&__get_cpu_var(l))
109#define cpu_local_dec(l) local_dec(&__get_cpu_var(l))
110#define cpu_local_add(i, l) local_add((i), &__get_cpu_var(l))
111#define cpu_local_sub(i, l) local_sub((i), &__get_cpu_var(l))
112
113#define __cpu_local_inc(l) __local_inc(&__get_cpu_var(l))
114#define __cpu_local_dec(l) __local_dec(&__get_cpu_var(l))
115#define __cpu_local_add(i, l) __local_add((i), &__get_cpu_var(l))
116#define __cpu_local_sub(i, l) __local_sub((i), &__get_cpu_var(l))
117
118#endif /* _ALPHA_LOCAL_H */ 101#endif /* _ALPHA_LOCAL_H */
diff --git a/arch/blackfin/mach-common/entry.S b/arch/blackfin/mach-common/entry.S
index b0ed0b487ff..01b2f58dfb9 100644
--- a/arch/blackfin/mach-common/entry.S
+++ b/arch/blackfin/mach-common/entry.S
@@ -816,8 +816,8 @@ ENDPROC(_resume)
816 816
817ENTRY(_ret_from_exception) 817ENTRY(_ret_from_exception)
818#ifdef CONFIG_IPIPE 818#ifdef CONFIG_IPIPE
819 p2.l = _per_cpu__ipipe_percpu_domain; 819 p2.l = _ipipe_percpu_domain;
820 p2.h = _per_cpu__ipipe_percpu_domain; 820 p2.h = _ipipe_percpu_domain;
821 r0.l = _ipipe_root; 821 r0.l = _ipipe_root;
822 r0.h = _ipipe_root; 822 r0.h = _ipipe_root;
823 r2 = [p2]; 823 r2 = [p2];
diff --git a/arch/cris/arch-v10/kernel/entry.S b/arch/cris/arch-v10/kernel/entry.S
index 2c18d08cd91..c52bef39e25 100644
--- a/arch/cris/arch-v10/kernel/entry.S
+++ b/arch/cris/arch-v10/kernel/entry.S
@@ -358,7 +358,7 @@ mmu_bus_fault:
3581: btstq 12, $r1 ; Refill? 3581: btstq 12, $r1 ; Refill?
359 bpl 2f 359 bpl 2f
360 lsrq 24, $r1 ; Get PGD index (bit 24-31) 360 lsrq 24, $r1 ; Get PGD index (bit 24-31)
361 move.d [per_cpu__current_pgd], $r0 ; PGD for the current process 361 move.d [current_pgd], $r0 ; PGD for the current process
362 move.d [$r0+$r1.d], $r0 ; Get PMD 362 move.d [$r0+$r1.d], $r0 ; Get PMD
363 beq 2f 363 beq 2f
364 nop 364 nop
diff --git a/arch/cris/arch-v32/mm/mmu.S b/arch/cris/arch-v32/mm/mmu.S
index 2238d154bde..f125d912e14 100644
--- a/arch/cris/arch-v32/mm/mmu.S
+++ b/arch/cris/arch-v32/mm/mmu.S
@@ -115,7 +115,7 @@
115#ifdef CONFIG_SMP 115#ifdef CONFIG_SMP
116 move $s7, $acr ; PGD 116 move $s7, $acr ; PGD
117#else 117#else
118 move.d per_cpu__current_pgd, $acr ; PGD 118 move.d current_pgd, $acr ; PGD
119#endif 119#endif
120 ; Look up PMD in PGD 120 ; Look up PMD in PGD
121 lsrq 24, $r0 ; Get PMD index into PGD (bit 24-31) 121 lsrq 24, $r0 ; Get PMD index into PGD (bit 24-31)
diff --git a/arch/ia64/include/asm/percpu.h b/arch/ia64/include/asm/percpu.h
index 30cf46534dd..f7c00a5e0e2 100644
--- a/arch/ia64/include/asm/percpu.h
+++ b/arch/ia64/include/asm/percpu.h
@@ -9,7 +9,7 @@
9#define PERCPU_ENOUGH_ROOM PERCPU_PAGE_SIZE 9#define PERCPU_ENOUGH_ROOM PERCPU_PAGE_SIZE
10 10
11#ifdef __ASSEMBLY__ 11#ifdef __ASSEMBLY__
12# define THIS_CPU(var) (per_cpu__##var) /* use this to mark accesses to per-CPU variables... */ 12# define THIS_CPU(var) (var) /* use this to mark accesses to per-CPU variables... */
13#else /* !__ASSEMBLY__ */ 13#else /* !__ASSEMBLY__ */
14 14
15 15
@@ -39,7 +39,7 @@ extern void *per_cpu_init(void);
39 * On the positive side, using __ia64_per_cpu_var() instead of __get_cpu_var() is slightly 39 * On the positive side, using __ia64_per_cpu_var() instead of __get_cpu_var() is slightly
40 * more efficient. 40 * more efficient.
41 */ 41 */
42#define __ia64_per_cpu_var(var) per_cpu__##var 42#define __ia64_per_cpu_var(var) var
43 43
44#include <asm-generic/percpu.h> 44#include <asm-generic/percpu.h>
45 45
diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c
index 461b99902bf..7f4a0ed2415 100644
--- a/arch/ia64/kernel/ia64_ksyms.c
+++ b/arch/ia64/kernel/ia64_ksyms.c
@@ -30,9 +30,9 @@ EXPORT_SYMBOL(max_low_pfn); /* defined by bootmem.c, but not exported by generic
30#endif 30#endif
31 31
32#include <asm/processor.h> 32#include <asm/processor.h>
33EXPORT_SYMBOL(per_cpu__ia64_cpu_info); 33EXPORT_SYMBOL(ia64_cpu_info);
34#ifdef CONFIG_SMP 34#ifdef CONFIG_SMP
35EXPORT_SYMBOL(per_cpu__local_per_cpu_offset); 35EXPORT_SYMBOL(local_per_cpu_offset);
36#endif 36#endif
37 37
38#include <asm/uaccess.h> 38#include <asm/uaccess.h>
diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c
index 19c4b2195dc..8d586d1e251 100644
--- a/arch/ia64/mm/discontig.c
+++ b/arch/ia64/mm/discontig.c
@@ -459,7 +459,7 @@ static void __init initialize_pernode_data(void)
459 cpu = 0; 459 cpu = 0;
460 node = node_cpuid[cpu].nid; 460 node = node_cpuid[cpu].nid;
461 cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start + 461 cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start +
462 ((char *)&per_cpu__ia64_cpu_info - __per_cpu_start)); 462 ((char *)&ia64_cpu_info - __per_cpu_start));
463 cpu0_cpu_info->node_data = mem_data[node].node_data; 463 cpu0_cpu_info->node_data = mem_data[node].node_data;
464 } 464 }
465#endif /* CONFIG_SMP */ 465#endif /* CONFIG_SMP */
diff --git a/arch/m32r/include/asm/local.h b/arch/m32r/include/asm/local.h
index 22256d13863..734bca87018 100644
--- a/arch/m32r/include/asm/local.h
+++ b/arch/m32r/include/asm/local.h
@@ -338,29 +338,4 @@ static inline void local_set_mask(unsigned long mask, local_t *addr)
338 * a variable, not an address. 338 * a variable, not an address.
339 */ 339 */
340 340
341/* Need to disable preemption for the cpu local counters otherwise we could
342 still access a variable of a previous CPU in a non local way. */
343#define cpu_local_wrap_v(l) \
344 ({ local_t res__; \
345 preempt_disable(); \
346 res__ = (l); \
347 preempt_enable(); \
348 res__; })
349#define cpu_local_wrap(l) \
350 ({ preempt_disable(); \
351 l; \
352 preempt_enable(); }) \
353
354#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
355#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
356#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var(l)))
357#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var(l)))
358#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
359#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
360
361#define __cpu_local_inc(l) cpu_local_inc(l)
362#define __cpu_local_dec(l) cpu_local_dec(l)
363#define __cpu_local_add(i, l) cpu_local_add((i), (l))
364#define __cpu_local_sub(i, l) cpu_local_sub((i), (l))
365
366#endif /* __M32R_LOCAL_H */ 341#endif /* __M32R_LOCAL_H */
diff --git a/arch/microblaze/include/asm/entry.h b/arch/microblaze/include/asm/entry.h
index 61abbd23264..ec89f2ad0fe 100644
--- a/arch/microblaze/include/asm/entry.h
+++ b/arch/microblaze/include/asm/entry.h
@@ -21,7 +21,7 @@
21 * places 21 * places
22 */ 22 */
23 23
24#define PER_CPU(var) per_cpu__##var 24#define PER_CPU(var) var
25 25
26# ifndef __ASSEMBLY__ 26# ifndef __ASSEMBLY__
27DECLARE_PER_CPU(unsigned int, KSP); /* Saved kernel stack pointer */ 27DECLARE_PER_CPU(unsigned int, KSP); /* Saved kernel stack pointer */
diff --git a/arch/mips/include/asm/local.h b/arch/mips/include/asm/local.h
index 361f4f16c30..bdcdef02d14 100644
--- a/arch/mips/include/asm/local.h
+++ b/arch/mips/include/asm/local.h
@@ -193,29 +193,4 @@ static __inline__ long local_sub_return(long i, local_t * l)
193#define __local_add(i, l) ((l)->a.counter+=(i)) 193#define __local_add(i, l) ((l)->a.counter+=(i))
194#define __local_sub(i, l) ((l)->a.counter-=(i)) 194#define __local_sub(i, l) ((l)->a.counter-=(i))
195 195
196/* Need to disable preemption for the cpu local counters otherwise we could
197 still access a variable of a previous CPU in a non atomic way. */
198#define cpu_local_wrap_v(l) \
199 ({ local_t res__; \
200 preempt_disable(); \
201 res__ = (l); \
202 preempt_enable(); \
203 res__; })
204#define cpu_local_wrap(l) \
205 ({ preempt_disable(); \
206 l; \
207 preempt_enable(); }) \
208
209#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
210#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
211#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var(l)))
212#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var(l)))
213#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
214#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
215
216#define __cpu_local_inc(l) cpu_local_inc(l)
217#define __cpu_local_dec(l) cpu_local_dec(l)
218#define __cpu_local_add(i, l) cpu_local_add((i), (l))
219#define __cpu_local_sub(i, l) cpu_local_sub((i), (l))
220
221#endif /* _ARCH_MIPS_LOCAL_H */ 196#endif /* _ARCH_MIPS_LOCAL_H */
diff --git a/arch/parisc/lib/fixup.S b/arch/parisc/lib/fixup.S
index d172d4245cd..f8c45cc2947 100644
--- a/arch/parisc/lib/fixup.S
+++ b/arch/parisc/lib/fixup.S
@@ -36,8 +36,8 @@
36#endif 36#endif
37 /* t2 = &__per_cpu_offset[smp_processor_id()]; */ 37 /* t2 = &__per_cpu_offset[smp_processor_id()]; */
38 LDREGX \t2(\t1),\t2 38 LDREGX \t2(\t1),\t2
39 addil LT%per_cpu__exception_data,%r27 39 addil LT%exception_data,%r27
40 LDREG RT%per_cpu__exception_data(%r1),\t1 40 LDREG RT%exception_data(%r1),\t1
41 /* t1 = &__get_cpu_var(exception_data) */ 41 /* t1 = &__get_cpu_var(exception_data) */
42 add,l \t1,\t2,\t1 42 add,l \t1,\t2,\t1
43 /* t1 = t1->fault_ip */ 43 /* t1 = t1->fault_ip */
@@ -46,8 +46,8 @@
46#else 46#else
47 .macro get_fault_ip t1 t2 47 .macro get_fault_ip t1 t2
48 /* t1 = &__get_cpu_var(exception_data) */ 48 /* t1 = &__get_cpu_var(exception_data) */
49 addil LT%per_cpu__exception_data,%r27 49 addil LT%exception_data,%r27
50 LDREG RT%per_cpu__exception_data(%r1),\t2 50 LDREG RT%exception_data(%r1),\t2
51 /* t1 = t2->fault_ip */ 51 /* t1 = t2->fault_ip */
52 LDREG EXCDATA_IP(\t2), \t1 52 LDREG EXCDATA_IP(\t2), \t1
53 .endm 53 .endm
diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h
index ce58c80e1bc..c2410af6bfd 100644
--- a/arch/powerpc/include/asm/local.h
+++ b/arch/powerpc/include/asm/local.h
@@ -172,29 +172,4 @@ static __inline__ long local_dec_if_positive(local_t *l)
172#define __local_add(i,l) ((l)->a.counter+=(i)) 172#define __local_add(i,l) ((l)->a.counter+=(i))
173#define __local_sub(i,l) ((l)->a.counter-=(i)) 173#define __local_sub(i,l) ((l)->a.counter-=(i))
174 174
175/* Need to disable preemption for the cpu local counters otherwise we could
176 still access a variable of a previous CPU in a non atomic way. */
177#define cpu_local_wrap_v(l) \
178 ({ local_t res__; \
179 preempt_disable(); \
180 res__ = (l); \
181 preempt_enable(); \
182 res__; })
183#define cpu_local_wrap(l) \
184 ({ preempt_disable(); \
185 l; \
186 preempt_enable(); }) \
187
188#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
189#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
190#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var(l)))
191#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var(l)))
192#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
193#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
194
195#define __cpu_local_inc(l) cpu_local_inc(l)
196#define __cpu_local_dec(l) cpu_local_dec(l)
197#define __cpu_local_add(i, l) cpu_local_add((i), (l))
198#define __cpu_local_sub(i, l) cpu_local_sub((i), (l))
199
200#endif /* _ARCH_POWERPC_LOCAL_H */ 175#endif /* _ARCH_POWERPC_LOCAL_H */
diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c
index d242a734054..b287b62c7ea 100644
--- a/arch/sparc/kernel/nmi.c
+++ b/arch/sparc/kernel/nmi.c
@@ -21,7 +21,6 @@
21 21
22#include <asm/perf_event.h> 22#include <asm/perf_event.h>
23#include <asm/ptrace.h> 23#include <asm/ptrace.h>
24#include <asm/local.h>
25#include <asm/pcr.h> 24#include <asm/pcr.h>
26 25
27/* We don't have a real NMI on sparc64, but we can fake one 26/* We don't have a real NMI on sparc64, but we can fake one
@@ -113,13 +112,13 @@ notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs)
113 touched = 1; 112 touched = 1;
114 } 113 }
115 if (!touched && __get_cpu_var(last_irq_sum) == sum) { 114 if (!touched && __get_cpu_var(last_irq_sum) == sum) {
116 __this_cpu_inc(per_cpu_var(alert_counter)); 115 __this_cpu_inc(alert_counter);
117 if (__this_cpu_read(per_cpu_var(alert_counter)) == 30 * nmi_hz) 116 if (__this_cpu_read(alert_counter) == 30 * nmi_hz)
118 die_nmi("BUG: NMI Watchdog detected LOCKUP", 117 die_nmi("BUG: NMI Watchdog detected LOCKUP",
119 regs, panic_on_timeout); 118 regs, panic_on_timeout);
120 } else { 119 } else {
121 __get_cpu_var(last_irq_sum) = sum; 120 __get_cpu_var(last_irq_sum) = sum;
122 __this_cpu_write(per_cpu_var(alert_counter), 0); 121 __this_cpu_write(alert_counter, 0);
123 } 122 }
124 if (__get_cpu_var(wd_enabled)) { 123 if (__get_cpu_var(wd_enabled)) {
125 write_pic(picl_value(nmi_hz)); 124 write_pic(picl_value(nmi_hz));
diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S
index fd3cee4d117..1ddec403f51 100644
--- a/arch/sparc/kernel/rtrap_64.S
+++ b/arch/sparc/kernel/rtrap_64.S
@@ -149,11 +149,11 @@ rtrap_nmi: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
149rtrap_irq: 149rtrap_irq:
150rtrap: 150rtrap:
151#ifndef CONFIG_SMP 151#ifndef CONFIG_SMP
152 sethi %hi(per_cpu____cpu_data), %l0 152 sethi %hi(__cpu_data), %l0
153 lduw [%l0 + %lo(per_cpu____cpu_data)], %l1 153 lduw [%l0 + %lo(__cpu_data)], %l1
154#else 154#else
155 sethi %hi(per_cpu____cpu_data), %l0 155 sethi %hi(__cpu_data), %l0
156 or %l0, %lo(per_cpu____cpu_data), %l0 156 or %l0, %lo(__cpu_data), %l0
157 lduw [%l0 + %g5], %l1 157 lduw [%l0 + %g5], %l1
158#endif 158#endif
159 cmp %l1, 0 159 cmp %l1, 0
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 47b9b6f1905..2e9972468a5 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -195,41 +195,4 @@ static inline long local_sub_return(long i, local_t *l)
195#define __local_add(i, l) local_add((i), (l)) 195#define __local_add(i, l) local_add((i), (l))
196#define __local_sub(i, l) local_sub((i), (l)) 196#define __local_sub(i, l) local_sub((i), (l))
197 197
198/* Use these for per-cpu local_t variables: on some archs they are
199 * much more efficient than these naive implementations. Note they take
200 * a variable, not an address.
201 *
202 * X86_64: This could be done better if we moved the per cpu data directly
203 * after GS.
204 */
205
206/* Need to disable preemption for the cpu local counters otherwise we could
207 still access a variable of a previous CPU in a non atomic way. */
208#define cpu_local_wrap_v(l) \
209({ \
210 local_t res__; \
211 preempt_disable(); \
212 res__ = (l); \
213 preempt_enable(); \
214 res__; \
215})
216#define cpu_local_wrap(l) \
217({ \
218 preempt_disable(); \
219 (l); \
220 preempt_enable(); \
221}) \
222
223#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var((l))))
224#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var((l)), (i)))
225#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var((l))))
226#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var((l))))
227#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var((l))))
228#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var((l))))
229
230#define __cpu_local_inc(l) cpu_local_inc((l))
231#define __cpu_local_dec(l) cpu_local_dec((l))
232#define __cpu_local_add(i, l) cpu_local_add((i), (l))
233#define __cpu_local_sub(i, l) cpu_local_sub((i), (l))
234
235#endif /* _ASM_X86_LOCAL_H */ 198#endif /* _ASM_X86_LOCAL_H */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 0c44196b78a..66a272dfd8b 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -25,19 +25,18 @@
25 */ 25 */
26#ifdef CONFIG_SMP 26#ifdef CONFIG_SMP
27#define PER_CPU(var, reg) \ 27#define PER_CPU(var, reg) \
28 __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg; \ 28 __percpu_mov_op %__percpu_seg:this_cpu_off, reg; \
29 lea per_cpu__##var(reg), reg 29 lea var(reg), reg
30#define PER_CPU_VAR(var) %__percpu_seg:per_cpu__##var 30#define PER_CPU_VAR(var) %__percpu_seg:var
31#else /* ! SMP */ 31#else /* ! SMP */
32#define PER_CPU(var, reg) \ 32#define PER_CPU(var, reg) __percpu_mov_op $var, reg
33 __percpu_mov_op $per_cpu__##var, reg 33#define PER_CPU_VAR(var) var
34#define PER_CPU_VAR(var) per_cpu__##var
35#endif /* SMP */ 34#endif /* SMP */
36 35
37#ifdef CONFIG_X86_64_SMP 36#ifdef CONFIG_X86_64_SMP
38#define INIT_PER_CPU_VAR(var) init_per_cpu__##var 37#define INIT_PER_CPU_VAR(var) init_per_cpu__##var
39#else 38#else
40#define INIT_PER_CPU_VAR(var) per_cpu__##var 39#define INIT_PER_CPU_VAR(var) var
41#endif 40#endif
42 41
43#else /* ...!ASSEMBLY */ 42#else /* ...!ASSEMBLY */
@@ -60,12 +59,12 @@
60 * There also must be an entry in vmlinux_64.lds.S 59 * There also must be an entry in vmlinux_64.lds.S
61 */ 60 */
62#define DECLARE_INIT_PER_CPU(var) \ 61#define DECLARE_INIT_PER_CPU(var) \
63 extern typeof(per_cpu_var(var)) init_per_cpu_var(var) 62 extern typeof(var) init_per_cpu_var(var)
64 63
65#ifdef CONFIG_X86_64_SMP 64#ifdef CONFIG_X86_64_SMP
66#define init_per_cpu_var(var) init_per_cpu__##var 65#define init_per_cpu_var(var) init_per_cpu__##var
67#else 66#else
68#define init_per_cpu_var(var) per_cpu_var(var) 67#define init_per_cpu_var(var) var
69#endif 68#endif
70 69
71/* For arch-specific code, we can use direct single-insn ops (they 70/* For arch-specific code, we can use direct single-insn ops (they
@@ -104,6 +103,64 @@ do { \
104 } \ 103 } \
105} while (0) 104} while (0)
106 105
106/*
107 * Generate a percpu add to memory instruction and optimize code
108 * if a one is added or subtracted.
109 */
110#define percpu_add_op(var, val) \
111do { \
112 typedef typeof(var) pao_T__; \
113 const int pao_ID__ = (__builtin_constant_p(val) && \
114 ((val) == 1 || (val) == -1)) ? (val) : 0; \
115 if (0) { \
116 pao_T__ pao_tmp__; \
117 pao_tmp__ = (val); \
118 } \
119 switch (sizeof(var)) { \
120 case 1: \
121 if (pao_ID__ == 1) \
122 asm("incb "__percpu_arg(0) : "+m" (var)); \
123 else if (pao_ID__ == -1) \
124 asm("decb "__percpu_arg(0) : "+m" (var)); \
125 else \
126 asm("addb %1, "__percpu_arg(0) \
127 : "+m" (var) \
128 : "qi" ((pao_T__)(val))); \
129 break; \
130 case 2: \
131 if (pao_ID__ == 1) \
132 asm("incw "__percpu_arg(0) : "+m" (var)); \
133 else if (pao_ID__ == -1) \
134 asm("decw "__percpu_arg(0) : "+m" (var)); \
135 else \
136 asm("addw %1, "__percpu_arg(0) \
137 : "+m" (var) \
138 : "ri" ((pao_T__)(val))); \
139 break; \
140 case 4: \
141 if (pao_ID__ == 1) \
142 asm("incl "__percpu_arg(0) : "+m" (var)); \
143 else if (pao_ID__ == -1) \
144 asm("decl "__percpu_arg(0) : "+m" (var)); \
145 else \
146 asm("addl %1, "__percpu_arg(0) \
147 : "+m" (var) \
148 : "ri" ((pao_T__)(val))); \
149 break; \
150 case 8: \
151 if (pao_ID__ == 1) \
152 asm("incq "__percpu_arg(0) : "+m" (var)); \
153 else if (pao_ID__ == -1) \
154 asm("decq "__percpu_arg(0) : "+m" (var)); \
155 else \
156 asm("addq %1, "__percpu_arg(0) \
157 : "+m" (var) \
158 : "re" ((pao_T__)(val))); \
159 break; \
160 default: __bad_percpu_size(); \
161 } \
162} while (0)
163
107#define percpu_from_op(op, var, constraint) \ 164#define percpu_from_op(op, var, constraint) \
108({ \ 165({ \
109 typeof(var) pfo_ret__; \ 166 typeof(var) pfo_ret__; \
@@ -142,16 +199,14 @@ do { \
142 * per-thread variables implemented as per-cpu variables and thus 199 * per-thread variables implemented as per-cpu variables and thus
143 * stable for the duration of the respective task. 200 * stable for the duration of the respective task.
144 */ 201 */
145#define percpu_read(var) percpu_from_op("mov", per_cpu__##var, \ 202#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
146 "m" (per_cpu__##var)) 203#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
147#define percpu_read_stable(var) percpu_from_op("mov", per_cpu__##var, \ 204#define percpu_write(var, val) percpu_to_op("mov", var, val)
148 "p" (&per_cpu__##var)) 205#define percpu_add(var, val) percpu_add_op(var, val)
149#define percpu_write(var, val) percpu_to_op("mov", per_cpu__##var, val) 206#define percpu_sub(var, val) percpu_add_op(var, -(val))
150#define percpu_add(var, val) percpu_to_op("add", per_cpu__##var, val) 207#define percpu_and(var, val) percpu_to_op("and", var, val)
151#define percpu_sub(var, val) percpu_to_op("sub", per_cpu__##var, val) 208#define percpu_or(var, val) percpu_to_op("or", var, val)
152#define percpu_and(var, val) percpu_to_op("and", per_cpu__##var, val) 209#define percpu_xor(var, val) percpu_to_op("xor", var, val)
153#define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val)
154#define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val)
155 210
156#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 211#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
157#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 212#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -160,9 +215,9 @@ do { \
160#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) 215#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
161#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) 216#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
162#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) 217#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
163#define __this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) 218#define __this_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
164#define __this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) 219#define __this_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
165#define __this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) 220#define __this_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
166#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) 221#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
167#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) 222#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
168#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) 223#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
@@ -179,9 +234,9 @@ do { \
179#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) 234#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
180#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) 235#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
181#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) 236#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
182#define this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) 237#define this_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
183#define this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) 238#define this_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
184#define this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) 239#define this_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
185#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) 240#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
186#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) 241#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
187#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) 242#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
@@ -192,9 +247,9 @@ do { \
192#define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) 247#define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val)
193#define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) 248#define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val)
194 249
195#define irqsafe_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) 250#define irqsafe_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
196#define irqsafe_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) 251#define irqsafe_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
197#define irqsafe_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) 252#define irqsafe_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
198#define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) 253#define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
199#define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) 254#define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
200#define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) 255#define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
@@ -212,19 +267,19 @@ do { \
212#ifdef CONFIG_X86_64 267#ifdef CONFIG_X86_64
213#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 268#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
214#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) 269#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
215#define __this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) 270#define __this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
216#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) 271#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
217#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) 272#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
218#define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) 273#define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
219 274
220#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 275#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
221#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) 276#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
222#define this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) 277#define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
223#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) 278#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
224#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) 279#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
225#define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) 280#define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
226 281
227#define irqsafe_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) 282#define irqsafe_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
228#define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) 283#define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
229#define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) 284#define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
230#define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) 285#define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
@@ -236,7 +291,7 @@ do { \
236({ \ 291({ \
237 int old__; \ 292 int old__; \
238 asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \ 293 asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \
239 : "=r" (old__), "+m" (per_cpu__##var) \ 294 : "=r" (old__), "+m" (var) \
240 : "dIr" (bit)); \ 295 : "dIr" (bit)); \
241 old__; \ 296 old__; \
242}) 297})
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index e04740f7a0b..b8fe48ee2ed 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -32,7 +32,7 @@ extern void show_regs_common(void);
32 "movl %P[task_canary](%[next]), %%ebx\n\t" \ 32 "movl %P[task_canary](%[next]), %%ebx\n\t" \
33 "movl %%ebx, "__percpu_arg([stack_canary])"\n\t" 33 "movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
34#define __switch_canary_oparam \ 34#define __switch_canary_oparam \
35 , [stack_canary] "=m" (per_cpu_var(stack_canary.canary)) 35 , [stack_canary] "=m" (stack_canary.canary)
36#define __switch_canary_iparam \ 36#define __switch_canary_iparam \
37 , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) 37 , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
38#else /* CC_STACKPROTECTOR */ 38#else /* CC_STACKPROTECTOR */
@@ -114,7 +114,7 @@ do { \
114 "movq %P[task_canary](%%rsi),%%r8\n\t" \ 114 "movq %P[task_canary](%%rsi),%%r8\n\t" \
115 "movq %%r8,"__percpu_arg([gs_canary])"\n\t" 115 "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
116#define __switch_canary_oparam \ 116#define __switch_canary_oparam \
117 , [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary)) 117 , [gs_canary] "=m" (irq_stack_union.stack_canary)
118#define __switch_canary_iparam \ 118#define __switch_canary_iparam \
119 , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) 119 , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
120#else /* CC_STACKPROTECTOR */ 120#else /* CC_STACKPROTECTOR */
@@ -133,7 +133,7 @@ do { \
133 __switch_canary \ 133 __switch_canary \
134 "movq %P[thread_info](%%rsi),%%r8\n\t" \ 134 "movq %P[thread_info](%%rsi),%%r8\n\t" \
135 "movq %%rax,%%rdi\n\t" \ 135 "movq %%rax,%%rdi\n\t" \
136 "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \ 136 "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \
137 "jnz ret_from_fork\n\t" \ 137 "jnz ret_from_fork\n\t" \
138 RESTORE_CONTEXT \ 138 RESTORE_CONTEXT \
139 : "=a" (last) \ 139 : "=a" (last) \
@@ -143,7 +143,7 @@ do { \
143 [ti_flags] "i" (offsetof(struct thread_info, flags)), \ 143 [ti_flags] "i" (offsetof(struct thread_info, flags)), \
144 [_tif_fork] "i" (_TIF_FORK), \ 144 [_tif_fork] "i" (_TIF_FORK), \
145 [thread_info] "i" (offsetof(struct task_struct, stack)), \ 145 [thread_info] "i" (offsetof(struct task_struct, stack)), \
146 [current_task] "m" (per_cpu_var(current_task)) \ 146 [current_task] "m" (current_task) \
147 __switch_canary_iparam \ 147 __switch_canary_iparam \
148 : "memory", "cc" __EXTRA_CLOBBER) 148 : "memory", "cc" __EXTRA_CLOBBER)
149#endif 149#endif
diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c
index 0159a69396c..4ada42c3dab 100644
--- a/arch/x86/kernel/apic/nmi.c
+++ b/arch/x86/kernel/apic/nmi.c
@@ -438,8 +438,8 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
438 * Ayiee, looks like this CPU is stuck ... 438 * Ayiee, looks like this CPU is stuck ...
439 * wait a few IRQs (5 seconds) before doing the oops ... 439 * wait a few IRQs (5 seconds) before doing the oops ...
440 */ 440 */
441 __this_cpu_inc(per_cpu_var(alert_counter)); 441 __this_cpu_inc(alert_counter);
442 if (__this_cpu_read(per_cpu_var(alert_counter)) == 5 * nmi_hz) 442 if (__this_cpu_read(alert_counter) == 5 * nmi_hz)
443 /* 443 /*
444 * die_nmi will return ONLY if NOTIFY_STOP happens.. 444 * die_nmi will return ONLY if NOTIFY_STOP happens..
445 */ 445 */
@@ -447,7 +447,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
447 regs, panic_on_timeout); 447 regs, panic_on_timeout);
448 } else { 448 } else {
449 __get_cpu_var(last_irq_sum) = sum; 449 __get_cpu_var(last_irq_sum) = sum;
450 __this_cpu_write(per_cpu_var(alert_counter), 0); 450 __this_cpu_write(alert_counter, 0);
451 } 451 }
452 452
453 /* see if the nmi watchdog went off */ 453 /* see if the nmi watchdog went off */
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 7fd318bac59..37c3d4b17d8 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -442,8 +442,8 @@ is386: movl $2,%ecx # set MP
442 */ 442 */
443 cmpb $0,ready 443 cmpb $0,ready
444 jne 1f 444 jne 1f
445 movl $per_cpu__gdt_page,%eax 445 movl $gdt_page,%eax
446 movl $per_cpu__stack_canary,%ecx 446 movl $stack_canary,%ecx
447 movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax) 447 movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
448 shrl $16, %ecx 448 shrl $16, %ecx
449 movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax) 449 movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
@@ -706,7 +706,7 @@ idt_descr:
706 .word 0 # 32 bit align gdt_desc.address 706 .word 0 # 32 bit align gdt_desc.address
707ENTRY(early_gdt_descr) 707ENTRY(early_gdt_descr)
708 .word GDT_ENTRIES*8-1 708 .word GDT_ENTRIES*8-1
709 .long per_cpu__gdt_page /* Overwritten for secondary CPUs */ 709 .long gdt_page /* Overwritten for secondary CPUs */
710 710
711/* 711/*
712 * The boot_gdt must mirror the equivalent in setup.S and is 712 * The boot_gdt must mirror the equivalent in setup.S and is
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index f92a0da608c..44879df5569 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -341,7 +341,7 @@ SECTIONS
341 * Per-cpu symbols which need to be offset from __per_cpu_load 341 * Per-cpu symbols which need to be offset from __per_cpu_load
342 * for the boot processor. 342 * for the boot processor.
343 */ 343 */
344#define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load 344#define INIT_PER_CPU(x) init_per_cpu__##x = x + __per_cpu_load
345INIT_PER_CPU(gdt_page); 345INIT_PER_CPU(gdt_page);
346INIT_PER_CPU(irq_stack_union); 346INIT_PER_CPU(irq_stack_union);
347 347
@@ -352,7 +352,7 @@ INIT_PER_CPU(irq_stack_union);
352 "kernel image bigger than KERNEL_IMAGE_SIZE"); 352 "kernel image bigger than KERNEL_IMAGE_SIZE");
353 353
354#ifdef CONFIG_SMP 354#ifdef CONFIG_SMP
355. = ASSERT((per_cpu__irq_stack_union == 0), 355. = ASSERT((irq_stack_union == 0),
356 "irq_stack_union is not at start of per-cpu area"); 356 "irq_stack_union is not at start of per-cpu area");
357#endif 357#endif
358 358
diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S
index 88e15deb8b8..22a2093b586 100644
--- a/arch/x86/xen/xen-asm_32.S
+++ b/arch/x86/xen/xen-asm_32.S
@@ -90,9 +90,9 @@ ENTRY(xen_iret)
90 GET_THREAD_INFO(%eax) 90 GET_THREAD_INFO(%eax)
91 movl TI_cpu(%eax), %eax 91 movl TI_cpu(%eax), %eax
92 movl __per_cpu_offset(,%eax,4), %eax 92 movl __per_cpu_offset(,%eax,4), %eax
93 mov per_cpu__xen_vcpu(%eax), %eax 93 mov xen_vcpu(%eax), %eax
94#else 94#else
95 movl per_cpu__xen_vcpu, %eax 95 movl xen_vcpu, %eax
96#endif 96#endif
97 97
98 /* check IF state we're restoring */ 98 /* check IF state we're restoring */
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-23 06:22:59 -0400