aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/alpha/include/asm/local.h17
-rw-r--r--arch/blackfin/mach-common/entry.S4
-rw-r--r--arch/cris/arch-v10/kernel/entry.S2
-rw-r--r--arch/cris/arch-v32/mm/mmu.S2
-rw-r--r--arch/ia64/include/asm/percpu.h4
-rw-r--r--arch/ia64/kernel/ia64_ksyms.c4
-rw-r--r--arch/ia64/mm/discontig.c2
-rw-r--r--arch/m32r/include/asm/local.h25
-rw-r--r--arch/microblaze/include/asm/entry.h2
-rw-r--r--arch/mips/include/asm/local.h25
-rw-r--r--arch/parisc/lib/fixup.S8
-rw-r--r--arch/powerpc/include/asm/local.h25
-rw-r--r--arch/sparc/kernel/nmi.c7
-rw-r--r--arch/sparc/kernel/rtrap_64.S8
-rw-r--r--arch/x86/include/asm/local.h37
-rw-r--r--arch/x86/include/asm/percpu.h119
-rw-r--r--arch/x86/include/asm/system.h8
-rw-r--r--arch/x86/kernel/apic/nmi.c6
-rw-r--r--arch/x86/kernel/head_32.S6
-rw-r--r--arch/x86/kernel/vmlinux.lds.S4
-rw-r--r--arch/x86/xen/xen-asm_32.S4
-rw-r--r--crypto/cryptd.c2
-rw-r--r--drivers/acpi/processor_perflib.c2
-rw-r--r--drivers/dma/dmaengine.c2
-rw-r--r--drivers/edac/amd64_edac.c2
-rw-r--r--drivers/md/raid5.c2
-rw-r--r--drivers/md/raid5.h2
-rw-r--r--fs/ext4/ext4.h2
-rw-r--r--fs/nfs/iostat.h4
-rw-r--r--fs/xfs/xfs_mount.h2
-rw-r--r--include/acpi/processor.h2
-rw-r--r--include/asm-generic/local.h19
-rw-r--r--include/asm-generic/percpu.h18
-rw-r--r--include/linux/blktrace_api.h4
-rw-r--r--include/linux/compiler.h2
-rw-r--r--include/linux/dmaengine.h2
-rw-r--r--include/linux/genhd.h2
-rw-r--r--include/linux/kexec.h2
-rw-r--r--include/linux/mm.h4
-rw-r--r--include/linux/mmzone.h12
-rw-r--r--include/linux/module.h37
-rw-r--r--include/linux/mount.h2
-rw-r--r--include/linux/nfs_fs_sb.h2
-rw-r--r--include/linux/percpu-defs.h40
-rw-r--r--include/linux/percpu.h44
-rw-r--r--include/linux/percpu_counter.h2
-rw-r--r--include/linux/srcu.h2
-rw-r--r--include/linux/vmstat.h8
-rw-r--r--kernel/kexec.c2
-rw-r--r--kernel/module.c29
-rw-r--r--kernel/rcutorture.c8
-rw-r--r--kernel/sched.c4
-rw-r--r--kernel/stop_machine.c2
-rw-r--r--kernel/trace/ring_buffer.c1
-rw-r--r--kernel/trace/ring_buffer_benchmark.c1
-rw-r--r--kernel/trace/trace.c6
-rw-r--r--kernel/trace/trace_functions_graph.c4
-rw-r--r--mm/page_alloc.c202
-rw-r--r--mm/percpu.c36
-rw-r--r--mm/vmstat.c15
60 files changed, 354 insertions, 499 deletions
diff --git a/arch/alpha/include/asm/local.h b/arch/alpha/include/asm/local.h
index 6ad3ea696421..b9e3e3318371 100644
--- a/arch/alpha/include/asm/local.h
+++ b/arch/alpha/include/asm/local.h
@@ -98,21 +98,4 @@ static __inline__ long local_sub_return(long i, local_t * l)
98#define __local_add(i,l) ((l)->a.counter+=(i)) 98#define __local_add(i,l) ((l)->a.counter+=(i))
99#define __local_sub(i,l) ((l)->a.counter-=(i)) 99#define __local_sub(i,l) ((l)->a.counter-=(i))
100 100
101/* Use these for per-cpu local_t variables: on some archs they are
102 * much more efficient than these naive implementations. Note they take
103 * a variable, not an address.
104 */
105#define cpu_local_read(l) local_read(&__get_cpu_var(l))
106#define cpu_local_set(l, i) local_set(&__get_cpu_var(l), (i))
107
108#define cpu_local_inc(l) local_inc(&__get_cpu_var(l))
109#define cpu_local_dec(l) local_dec(&__get_cpu_var(l))
110#define cpu_local_add(i, l) local_add((i), &__get_cpu_var(l))
111#define cpu_local_sub(i, l) local_sub((i), &__get_cpu_var(l))
112
113#define __cpu_local_inc(l) __local_inc(&__get_cpu_var(l))
114#define __cpu_local_dec(l) __local_dec(&__get_cpu_var(l))
115#define __cpu_local_add(i, l) __local_add((i), &__get_cpu_var(l))
116#define __cpu_local_sub(i, l) __local_sub((i), &__get_cpu_var(l))
117
118#endif /* _ALPHA_LOCAL_H */ 101#endif /* _ALPHA_LOCAL_H */
diff --git a/arch/blackfin/mach-common/entry.S b/arch/blackfin/mach-common/entry.S
index b0ed0b487ff2..01b2f58dfb95 100644
--- a/arch/blackfin/mach-common/entry.S
+++ b/arch/blackfin/mach-common/entry.S
@@ -816,8 +816,8 @@ ENDPROC(_resume)
816 816
817ENTRY(_ret_from_exception) 817ENTRY(_ret_from_exception)
818#ifdef CONFIG_IPIPE 818#ifdef CONFIG_IPIPE
819 p2.l = _per_cpu__ipipe_percpu_domain; 819 p2.l = _ipipe_percpu_domain;
820 p2.h = _per_cpu__ipipe_percpu_domain; 820 p2.h = _ipipe_percpu_domain;
821 r0.l = _ipipe_root; 821 r0.l = _ipipe_root;
822 r0.h = _ipipe_root; 822 r0.h = _ipipe_root;
823 r2 = [p2]; 823 r2 = [p2];
diff --git a/arch/cris/arch-v10/kernel/entry.S b/arch/cris/arch-v10/kernel/entry.S
index 2c18d08cd913..c52bef39e250 100644
--- a/arch/cris/arch-v10/kernel/entry.S
+++ b/arch/cris/arch-v10/kernel/entry.S
@@ -358,7 +358,7 @@ mmu_bus_fault:
3581: btstq 12, $r1 ; Refill? 3581: btstq 12, $r1 ; Refill?
359 bpl 2f 359 bpl 2f
360 lsrq 24, $r1 ; Get PGD index (bit 24-31) 360 lsrq 24, $r1 ; Get PGD index (bit 24-31)
361 move.d [per_cpu__current_pgd], $r0 ; PGD for the current process 361 move.d [current_pgd], $r0 ; PGD for the current process
362 move.d [$r0+$r1.d], $r0 ; Get PMD 362 move.d [$r0+$r1.d], $r0 ; Get PMD
363 beq 2f 363 beq 2f
364 nop 364 nop
diff --git a/arch/cris/arch-v32/mm/mmu.S b/arch/cris/arch-v32/mm/mmu.S
index 2238d154bde3..f125d912e140 100644
--- a/arch/cris/arch-v32/mm/mmu.S
+++ b/arch/cris/arch-v32/mm/mmu.S
@@ -115,7 +115,7 @@
115#ifdef CONFIG_SMP 115#ifdef CONFIG_SMP
116 move $s7, $acr ; PGD 116 move $s7, $acr ; PGD
117#else 117#else
118 move.d per_cpu__current_pgd, $acr ; PGD 118 move.d current_pgd, $acr ; PGD
119#endif 119#endif
120 ; Look up PMD in PGD 120 ; Look up PMD in PGD
121 lsrq 24, $r0 ; Get PMD index into PGD (bit 24-31) 121 lsrq 24, $r0 ; Get PMD index into PGD (bit 24-31)
diff --git a/arch/ia64/include/asm/percpu.h b/arch/ia64/include/asm/percpu.h
index 30cf46534dd2..f7c00a5e0e2b 100644
--- a/arch/ia64/include/asm/percpu.h
+++ b/arch/ia64/include/asm/percpu.h
@@ -9,7 +9,7 @@
9#define PERCPU_ENOUGH_ROOM PERCPU_PAGE_SIZE 9#define PERCPU_ENOUGH_ROOM PERCPU_PAGE_SIZE
10 10
11#ifdef __ASSEMBLY__ 11#ifdef __ASSEMBLY__
12# define THIS_CPU(var) (per_cpu__##var) /* use this to mark accesses to per-CPU variables... */ 12# define THIS_CPU(var) (var) /* use this to mark accesses to per-CPU variables... */
13#else /* !__ASSEMBLY__ */ 13#else /* !__ASSEMBLY__ */
14 14
15 15
@@ -39,7 +39,7 @@ extern void *per_cpu_init(void);
39 * On the positive side, using __ia64_per_cpu_var() instead of __get_cpu_var() is slightly 39 * On the positive side, using __ia64_per_cpu_var() instead of __get_cpu_var() is slightly
40 * more efficient. 40 * more efficient.
41 */ 41 */
42#define __ia64_per_cpu_var(var) per_cpu__##var 42#define __ia64_per_cpu_var(var) var
43 43
44#include <asm-generic/percpu.h> 44#include <asm-generic/percpu.h>
45 45
diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c
index 461b99902bf6..7f4a0ed24152 100644
--- a/arch/ia64/kernel/ia64_ksyms.c
+++ b/arch/ia64/kernel/ia64_ksyms.c
@@ -30,9 +30,9 @@ EXPORT_SYMBOL(max_low_pfn); /* defined by bootmem.c, but not exported by generic
30#endif 30#endif
31 31
32#include <asm/processor.h> 32#include <asm/processor.h>
33EXPORT_SYMBOL(per_cpu__ia64_cpu_info); 33EXPORT_SYMBOL(ia64_cpu_info);
34#ifdef CONFIG_SMP 34#ifdef CONFIG_SMP
35EXPORT_SYMBOL(per_cpu__local_per_cpu_offset); 35EXPORT_SYMBOL(local_per_cpu_offset);
36#endif 36#endif
37 37
38#include <asm/uaccess.h> 38#include <asm/uaccess.h>
diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c
index 19c4b2195dce..8d586d1e2515 100644
--- a/arch/ia64/mm/discontig.c
+++ b/arch/ia64/mm/discontig.c
@@ -459,7 +459,7 @@ static void __init initialize_pernode_data(void)
459 cpu = 0; 459 cpu = 0;
460 node = node_cpuid[cpu].nid; 460 node = node_cpuid[cpu].nid;
461 cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start + 461 cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start +
462 ((char *)&per_cpu__ia64_cpu_info - __per_cpu_start)); 462 ((char *)&ia64_cpu_info - __per_cpu_start));
463 cpu0_cpu_info->node_data = mem_data[node].node_data; 463 cpu0_cpu_info->node_data = mem_data[node].node_data;
464 } 464 }
465#endif /* CONFIG_SMP */ 465#endif /* CONFIG_SMP */
diff --git a/arch/m32r/include/asm/local.h b/arch/m32r/include/asm/local.h
index 22256d138630..734bca87018a 100644
--- a/arch/m32r/include/asm/local.h
+++ b/arch/m32r/include/asm/local.h
@@ -338,29 +338,4 @@ static inline void local_set_mask(unsigned long mask, local_t *addr)
338 * a variable, not an address. 338 * a variable, not an address.
339 */ 339 */
340 340
341/* Need to disable preemption for the cpu local counters otherwise we could
342 still access a variable of a previous CPU in a non local way. */
343#define cpu_local_wrap_v(l) \
344 ({ local_t res__; \
345 preempt_disable(); \
346 res__ = (l); \
347 preempt_enable(); \
348 res__; })
349#define cpu_local_wrap(l) \
350 ({ preempt_disable(); \
351 l; \
352 preempt_enable(); }) \
353
354#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
355#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
356#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var(l)))
357#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var(l)))
358#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
359#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
360
361#define __cpu_local_inc(l) cpu_local_inc(l)
362#define __cpu_local_dec(l) cpu_local_dec(l)
363#define __cpu_local_add(i, l) cpu_local_add((i), (l))
364#define __cpu_local_sub(i, l) cpu_local_sub((i), (l))
365
366#endif /* __M32R_LOCAL_H */ 341#endif /* __M32R_LOCAL_H */
diff --git a/arch/microblaze/include/asm/entry.h b/arch/microblaze/include/asm/entry.h
index 61abbd232640..ec89f2ad0fe1 100644
--- a/arch/microblaze/include/asm/entry.h
+++ b/arch/microblaze/include/asm/entry.h
@@ -21,7 +21,7 @@
21 * places 21 * places
22 */ 22 */
23 23
24#define PER_CPU(var) per_cpu__##var 24#define PER_CPU(var) var
25 25
26# ifndef __ASSEMBLY__ 26# ifndef __ASSEMBLY__
27DECLARE_PER_CPU(unsigned int, KSP); /* Saved kernel stack pointer */ 27DECLARE_PER_CPU(unsigned int, KSP); /* Saved kernel stack pointer */
diff --git a/arch/mips/include/asm/local.h b/arch/mips/include/asm/local.h
index 361f4f16c30c..bdcdef02d147 100644
--- a/arch/mips/include/asm/local.h
+++ b/arch/mips/include/asm/local.h
@@ -193,29 +193,4 @@ static __inline__ long local_sub_return(long i, local_t * l)
193#define __local_add(i, l) ((l)->a.counter+=(i)) 193#define __local_add(i, l) ((l)->a.counter+=(i))
194#define __local_sub(i, l) ((l)->a.counter-=(i)) 194#define __local_sub(i, l) ((l)->a.counter-=(i))
195 195
196/* Need to disable preemption for the cpu local counters otherwise we could
197 still access a variable of a previous CPU in a non atomic way. */
198#define cpu_local_wrap_v(l) \
199 ({ local_t res__; \
200 preempt_disable(); \
201 res__ = (l); \
202 preempt_enable(); \
203 res__; })
204#define cpu_local_wrap(l) \
205 ({ preempt_disable(); \
206 l; \
207 preempt_enable(); }) \
208
209#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
210#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
211#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var(l)))
212#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var(l)))
213#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
214#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
215
216#define __cpu_local_inc(l) cpu_local_inc(l)
217#define __cpu_local_dec(l) cpu_local_dec(l)
218#define __cpu_local_add(i, l) cpu_local_add((i), (l))
219#define __cpu_local_sub(i, l) cpu_local_sub((i), (l))
220
221#endif /* _ARCH_MIPS_LOCAL_H */ 196#endif /* _ARCH_MIPS_LOCAL_H */
diff --git a/arch/parisc/lib/fixup.S b/arch/parisc/lib/fixup.S
index d172d4245cdc..f8c45cc2947d 100644
--- a/arch/parisc/lib/fixup.S
+++ b/arch/parisc/lib/fixup.S
@@ -36,8 +36,8 @@
36#endif 36#endif
37 /* t2 = &__per_cpu_offset[smp_processor_id()]; */ 37 /* t2 = &__per_cpu_offset[smp_processor_id()]; */
38 LDREGX \t2(\t1),\t2 38 LDREGX \t2(\t1),\t2
39 addil LT%per_cpu__exception_data,%r27 39 addil LT%exception_data,%r27
40 LDREG RT%per_cpu__exception_data(%r1),\t1 40 LDREG RT%exception_data(%r1),\t1
41 /* t1 = &__get_cpu_var(exception_data) */ 41 /* t1 = &__get_cpu_var(exception_data) */
42 add,l \t1,\t2,\t1 42 add,l \t1,\t2,\t1
43 /* t1 = t1->fault_ip */ 43 /* t1 = t1->fault_ip */
@@ -46,8 +46,8 @@
46#else 46#else
47 .macro get_fault_ip t1 t2 47 .macro get_fault_ip t1 t2
48 /* t1 = &__get_cpu_var(exception_data) */ 48 /* t1 = &__get_cpu_var(exception_data) */
49 addil LT%per_cpu__exception_data,%r27 49 addil LT%exception_data,%r27
50 LDREG RT%per_cpu__exception_data(%r1),\t2 50 LDREG RT%exception_data(%r1),\t2
51 /* t1 = t2->fault_ip */ 51 /* t1 = t2->fault_ip */
52 LDREG EXCDATA_IP(\t2), \t1 52 LDREG EXCDATA_IP(\t2), \t1
53 .endm 53 .endm
diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h
index ce58c80e1bcf..c2410af6bfd9 100644
--- a/arch/powerpc/include/asm/local.h
+++ b/arch/powerpc/include/asm/local.h
@@ -172,29 +172,4 @@ static __inline__ long local_dec_if_positive(local_t *l)
172#define __local_add(i,l) ((l)->a.counter+=(i)) 172#define __local_add(i,l) ((l)->a.counter+=(i))
173#define __local_sub(i,l) ((l)->a.counter-=(i)) 173#define __local_sub(i,l) ((l)->a.counter-=(i))
174 174
175/* Need to disable preemption for the cpu local counters otherwise we could
176 still access a variable of a previous CPU in a non atomic way. */
177#define cpu_local_wrap_v(l) \
178 ({ local_t res__; \
179 preempt_disable(); \
180 res__ = (l); \
181 preempt_enable(); \
182 res__; })
183#define cpu_local_wrap(l) \
184 ({ preempt_disable(); \
185 l; \
186 preempt_enable(); }) \
187
188#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
189#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
190#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var(l)))
191#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var(l)))
192#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
193#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
194
195#define __cpu_local_inc(l) cpu_local_inc(l)
196#define __cpu_local_dec(l) cpu_local_dec(l)
197#define __cpu_local_add(i, l) cpu_local_add((i), (l))
198#define __cpu_local_sub(i, l) cpu_local_sub((i), (l))
199
200#endif /* _ARCH_POWERPC_LOCAL_H */ 175#endif /* _ARCH_POWERPC_LOCAL_H */
diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c
index d242a7340541..b287b62c7ea3 100644
--- a/arch/sparc/kernel/nmi.c
+++ b/arch/sparc/kernel/nmi.c
@@ -21,7 +21,6 @@
21 21
22#include <asm/perf_event.h> 22#include <asm/perf_event.h>
23#include <asm/ptrace.h> 23#include <asm/ptrace.h>
24#include <asm/local.h>
25#include <asm/pcr.h> 24#include <asm/pcr.h>
26 25
27/* We don't have a real NMI on sparc64, but we can fake one 26/* We don't have a real NMI on sparc64, but we can fake one
@@ -113,13 +112,13 @@ notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs)
113 touched = 1; 112 touched = 1;
114 } 113 }
115 if (!touched && __get_cpu_var(last_irq_sum) == sum) { 114 if (!touched && __get_cpu_var(last_irq_sum) == sum) {
116 __this_cpu_inc(per_cpu_var(alert_counter)); 115 __this_cpu_inc(alert_counter);
117 if (__this_cpu_read(per_cpu_var(alert_counter)) == 30 * nmi_hz) 116 if (__this_cpu_read(alert_counter) == 30 * nmi_hz)
118 die_nmi("BUG: NMI Watchdog detected LOCKUP", 117 die_nmi("BUG: NMI Watchdog detected LOCKUP",
119 regs, panic_on_timeout); 118 regs, panic_on_timeout);
120 } else { 119 } else {
121 __get_cpu_var(last_irq_sum) = sum; 120 __get_cpu_var(last_irq_sum) = sum;
122 __this_cpu_write(per_cpu_var(alert_counter), 0); 121 __this_cpu_write(alert_counter, 0);
123 } 122 }
124 if (__get_cpu_var(wd_enabled)) { 123 if (__get_cpu_var(wd_enabled)) {
125 write_pic(picl_value(nmi_hz)); 124 write_pic(picl_value(nmi_hz));
diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S
index fd3cee4d117c..1ddec403f512 100644
--- a/arch/sparc/kernel/rtrap_64.S
+++ b/arch/sparc/kernel/rtrap_64.S
@@ -149,11 +149,11 @@ rtrap_nmi: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
149rtrap_irq: 149rtrap_irq:
150rtrap: 150rtrap:
151#ifndef CONFIG_SMP 151#ifndef CONFIG_SMP
152 sethi %hi(per_cpu____cpu_data), %l0 152 sethi %hi(__cpu_data), %l0
153 lduw [%l0 + %lo(per_cpu____cpu_data)], %l1 153 lduw [%l0 + %lo(__cpu_data)], %l1
154#else 154#else
155 sethi %hi(per_cpu____cpu_data), %l0 155 sethi %hi(__cpu_data), %l0
156 or %l0, %lo(per_cpu____cpu_data), %l0 156 or %l0, %lo(__cpu_data), %l0
157 lduw [%l0 + %g5], %l1 157 lduw [%l0 + %g5], %l1
158#endif 158#endif
159 cmp %l1, 0 159 cmp %l1, 0
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 47b9b6f19057..2e9972468a5d 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -195,41 +195,4 @@ static inline long local_sub_return(long i, local_t *l)
195#define __local_add(i, l) local_add((i), (l)) 195#define __local_add(i, l) local_add((i), (l))
196#define __local_sub(i, l) local_sub((i), (l)) 196#define __local_sub(i, l) local_sub((i), (l))
197 197
198/* Use these for per-cpu local_t variables: on some archs they are
199 * much more efficient than these naive implementations. Note they take
200 * a variable, not an address.
201 *
202 * X86_64: This could be done better if we moved the per cpu data directly
203 * after GS.
204 */
205
206/* Need to disable preemption for the cpu local counters otherwise we could
207 still access a variable of a previous CPU in a non atomic way. */
208#define cpu_local_wrap_v(l) \
209({ \
210 local_t res__; \
211 preempt_disable(); \
212 res__ = (l); \
213 preempt_enable(); \
214 res__; \
215})
216#define cpu_local_wrap(l) \
217({ \
218 preempt_disable(); \
219 (l); \
220 preempt_enable(); \
221}) \
222
223#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var((l))))
224#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var((l)), (i)))
225#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var((l))))
226#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var((l))))
227#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var((l))))
228#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var((l))))
229
230#define __cpu_local_inc(l) cpu_local_inc((l))
231#define __cpu_local_dec(l) cpu_local_dec((l))
232#define __cpu_local_add(i, l) cpu_local_add((i), (l))
233#define __cpu_local_sub(i, l) cpu_local_sub((i), (l))
234
235#endif /* _ASM_X86_LOCAL_H */ 198#endif /* _ASM_X86_LOCAL_H */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 0c44196b78ac..66a272dfd8b8 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -25,19 +25,18 @@
25 */ 25 */
26#ifdef CONFIG_SMP 26#ifdef CONFIG_SMP
27#define PER_CPU(var, reg) \ 27#define PER_CPU(var, reg) \
28 __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg; \ 28 __percpu_mov_op %__percpu_seg:this_cpu_off, reg; \
29 lea per_cpu__##var(reg), reg 29 lea var(reg), reg
30#define PER_CPU_VAR(var) %__percpu_seg:per_cpu__##var 30#define PER_CPU_VAR(var) %__percpu_seg:var
31#else /* ! SMP */ 31#else /* ! SMP */
32#define PER_CPU(var, reg) \ 32#define PER_CPU(var, reg) __percpu_mov_op $var, reg
33 __percpu_mov_op $per_cpu__##var, reg 33#define PER_CPU_VAR(var) var
34#define PER_CPU_VAR(var) per_cpu__##var
35#endif /* SMP */ 34#endif /* SMP */
36 35
37#ifdef CONFIG_X86_64_SMP 36#ifdef CONFIG_X86_64_SMP
38#define INIT_PER_CPU_VAR(var) init_per_cpu__##var 37#define INIT_PER_CPU_VAR(var) init_per_cpu__##var
39#else 38#else
40#define INIT_PER_CPU_VAR(var) per_cpu__##var 39#define INIT_PER_CPU_VAR(var) var
41#endif 40#endif
42 41
43#else /* ...!ASSEMBLY */ 42#else /* ...!ASSEMBLY */
@@ -60,12 +59,12 @@
60 * There also must be an entry in vmlinux_64.lds.S 59 * There also must be an entry in vmlinux_64.lds.S
61 */ 60 */
62#define DECLARE_INIT_PER_CPU(var) \ 61#define DECLARE_INIT_PER_CPU(var) \
63 extern typeof(per_cpu_var(var)) init_per_cpu_var(var) 62 extern typeof(var) init_per_cpu_var(var)
64 63
65#ifdef CONFIG_X86_64_SMP 64#ifdef CONFIG_X86_64_SMP
66#define init_per_cpu_var(var) init_per_cpu__##var 65#define init_per_cpu_var(var) init_per_cpu__##var
67#else 66#else
68#define init_per_cpu_var(var) per_cpu_var(var) 67#define init_per_cpu_var(var) var
69#endif 68#endif
70 69
71/* For arch-specific code, we can use direct single-insn ops (they 70/* For arch-specific code, we can use direct single-insn ops (they
@@ -104,6 +103,64 @@ do { \
104 } \ 103 } \
105} while (0) 104} while (0)
106 105
106/*
107 * Generate a percpu add to memory instruction and optimize code
108 * if a one is added or subtracted.
109 */
110#define percpu_add_op(var, val) \
111do { \
112 typedef typeof(var) pao_T__; \
113 const int pao_ID__ = (__builtin_constant_p(val) && \
114 ((val) == 1 || (val) == -1)) ? (val) : 0; \
115 if (0) { \
116 pao_T__ pao_tmp__; \
117 pao_tmp__ = (val); \
118 } \
119 switch (sizeof(var)) { \
120 case 1: \
121 if (pao_ID__ == 1) \
122 asm("incb "__percpu_arg(0) : "+m" (var)); \
123 else if (pao_ID__ == -1) \
124 asm("decb "__percpu_arg(0) : "+m" (var)); \
125 else \
126 asm("addb %1, "__percpu_arg(0) \
127 : "+m" (var) \
128 : "qi" ((pao_T__)(val))); \
129 break; \
130 case 2: \
131 if (pao_ID__ == 1) \
132 asm("incw "__percpu_arg(0) : "+m" (var)); \
133 else if (pao_ID__ == -1) \
134 asm("decw "__percpu_arg(0) : "+m" (var)); \
135 else \
136 asm("addw %1, "__percpu_arg(0) \
137 : "+m" (var) \
138 : "ri" ((pao_T__)(val))); \
139 break; \
140 case 4: \
141 if (pao_ID__ == 1) \
142 asm("incl "__percpu_arg(0) : "+m" (var)); \
143 else if (pao_ID__ == -1) \
144 asm("decl "__percpu_arg(0) : "+m" (var)); \
145 else \
146 asm("addl %1, "__percpu_arg(0) \
147 : "+m" (var) \
148 : "ri" ((pao_T__)(val))); \
149 break; \
150 case 8: \
151 if (pao_ID__ == 1) \
152 asm("incq "__percpu_arg(0) : "+m" (var)); \
153 else if (pao_ID__ == -1) \
154 asm("decq "__percpu_arg(0) : "+m" (var)); \
155 else \
156 asm("addq %1, "__percpu_arg(0) \
157 : "+m" (var) \
158 : "re" ((pao_T__)(val))); \
159 break; \
160 default: __bad_percpu_size(); \
161 } \
162} while (0)
163
107#define percpu_from_op(op, var, constraint) \ 164#define percpu_from_op(op, var, constraint) \
108({ \ 165({ \
109 typeof(var) pfo_ret__; \ 166 typeof(var) pfo_ret__; \
@@ -142,16 +199,14 @@ do { \
142 * per-thread variables implemented as per-cpu variables and thus 199 * per-thread variables implemented as per-cpu variables and thus
143 * stable for the duration of the respective task. 200 * stable for the duration of the respective task.
144 */ 201 */
145#define percpu_read(var) percpu_from_op("mov", per_cpu__##var, \ 202#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
146 "m" (per_cpu__##var)) 203#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
147#define percpu_read_stable(var) percpu_from_op("mov", per_cpu__##var, \ 204#define percpu_write(var, val) percpu_to_op("mov", var, val)
148 "p" (&per_cpu__##var)) 205#define percpu_add(var, val) percpu_add_op(var, val)
149#define percpu_write(var, val) percpu_to_op("mov", per_cpu__##var, val) 206#define percpu_sub(var, val) percpu_add_op(var, -(val))
150#define percpu_add(var, val) percpu_to_op("add", per_cpu__##var, val) 207#define percpu_and(var, val) percpu_to_op("and", var, val)
151#define percpu_sub(var, val) percpu_to_op("sub", per_cpu__##var, val) 208#define percpu_or(var, val) percpu_to_op("or", var, val)
152#define percpu_and(var, val) percpu_to_op("and", per_cpu__##var, val) 209#define percpu_xor(var, val) percpu_to_op("xor", var, val)
153#define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val)
154#define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val)
155 210
156#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 211#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
157#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 212#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -160,9 +215,9 @@ do { \
160#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) 215#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
161#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) 216#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
162#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) 217#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
163#define __this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) 218#define __this_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
164#define __this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) 219#define __this_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
165#define __this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) 220#define __this_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
166#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) 221#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
167#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) 222#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
168#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) 223#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
@@ -179,9 +234,9 @@ do { \
179#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) 234#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
180#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) 235#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
181#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) 236#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
182#define this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) 237#define this_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
183#define this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) 238#define this_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
184#define this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) 239#define this_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
185#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) 240#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
186#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) 241#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
187#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) 242#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
@@ -192,9 +247,9 @@ do { \
192#define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) 247#define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val)
193#define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) 248#define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val)
194 249
195#define irqsafe_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) 250#define irqsafe_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
196#define irqsafe_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) 251#define irqsafe_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
197#define irqsafe_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) 252#define irqsafe_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
198#define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) 253#define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
199#define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) 254#define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
200#define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) 255#define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
@@ -212,19 +267,19 @@ do { \
212#ifdef CONFIG_X86_64 267#ifdef CONFIG_X86_64
213#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 268#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
214#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) 269#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
215#define __this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) 270#define __this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
216#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) 271#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
217#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) 272#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
218#define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) 273#define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
219 274
220#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 275#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
221#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) 276#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
222#define this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) 277#define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
223#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) 278#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
224#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) 279#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
225#define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) 280#define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
226 281
227#define irqsafe_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) 282#define irqsafe_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
228#define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) 283#define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
229#define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) 284#define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
230#define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) 285#define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
@@ -236,7 +291,7 @@ do { \
236({ \ 291({ \
237 int old__; \ 292 int old__; \
238 asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \ 293 asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \
239 : "=r" (old__), "+m" (per_cpu__##var) \ 294 : "=r" (old__), "+m" (var) \
240 : "dIr" (bit)); \ 295 : "dIr" (bit)); \
241 old__; \ 296 old__; \
242}) 297})
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index e04740f7a0bb..b8fe48ee2ed9 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -32,7 +32,7 @@ extern void show_regs_common(void);
32 "movl %P[task_canary](%[next]), %%ebx\n\t" \ 32 "movl %P[task_canary](%[next]), %%ebx\n\t" \
33 "movl %%ebx, "__percpu_arg([stack_canary])"\n\t" 33 "movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
34#define __switch_canary_oparam \ 34#define __switch_canary_oparam \
35 , [stack_canary] "=m" (per_cpu_var(stack_canary.canary)) 35 , [stack_canary] "=m" (stack_canary.canary)
36#define __switch_canary_iparam \ 36#define __switch_canary_iparam \
37 , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) 37 , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
38#else /* CC_STACKPROTECTOR */ 38#else /* CC_STACKPROTECTOR */
@@ -114,7 +114,7 @@ do { \
114 "movq %P[task_canary](%%rsi),%%r8\n\t" \ 114 "movq %P[task_canary](%%rsi),%%r8\n\t" \
115 "movq %%r8,"__percpu_arg([gs_canary])"\n\t" 115 "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
116#define __switch_canary_oparam \ 116#define __switch_canary_oparam \
117 , [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary)) 117 , [gs_canary] "=m" (irq_stack_union.stack_canary)
118#define __switch_canary_iparam \ 118#define __switch_canary_iparam \
119 , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) 119 , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
120#else /* CC_STACKPROTECTOR */ 120#else /* CC_STACKPROTECTOR */
@@ -133,7 +133,7 @@ do { \
133 __switch_canary \ 133 __switch_canary \
134 "movq %P[thread_info](%%rsi),%%r8\n\t" \ 134 "movq %P[thread_info](%%rsi),%%r8\n\t" \
135 "movq %%rax,%%rdi\n\t" \ 135 "movq %%rax,%%rdi\n\t" \
136 "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \ 136 "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \
137 "jnz ret_from_fork\n\t" \ 137 "jnz ret_from_fork\n\t" \
138 RESTORE_CONTEXT \ 138 RESTORE_CONTEXT \
139 : "=a" (last) \ 139 : "=a" (last) \
@@ -143,7 +143,7 @@ do { \
143 [ti_flags] "i" (offsetof(struct thread_info, flags)), \ 143 [ti_flags] "i" (offsetof(struct thread_info, flags)), \
144 [_tif_fork] "i" (_TIF_FORK), \ 144 [_tif_fork] "i" (_TIF_FORK), \
145 [thread_info] "i" (offsetof(struct task_struct, stack)), \ 145 [thread_info] "i" (offsetof(struct task_struct, stack)), \
146 [current_task] "m" (per_cpu_var(current_task)) \ 146 [current_task] "m" (current_task) \
147 __switch_canary_iparam \ 147 __switch_canary_iparam \
148 : "memory", "cc" __EXTRA_CLOBBER) 148 : "memory", "cc" __EXTRA_CLOBBER)
149#endif 149#endif
diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c
index 0159a69396cb..4ada42c3dabb 100644
--- a/arch/x86/kernel/apic/nmi.c
+++ b/arch/x86/kernel/apic/nmi.c
@@ -438,8 +438,8 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
438 * Ayiee, looks like this CPU is stuck ... 438 * Ayiee, looks like this CPU is stuck ...
439 * wait a few IRQs (5 seconds) before doing the oops ... 439 * wait a few IRQs (5 seconds) before doing the oops ...
440 */ 440 */
441 __this_cpu_inc(per_cpu_var(alert_counter)); 441 __this_cpu_inc(alert_counter);
442 if (__this_cpu_read(per_cpu_var(alert_counter)) == 5 * nmi_hz) 442 if (__this_cpu_read(alert_counter) == 5 * nmi_hz)
443 /* 443 /*
444 * die_nmi will return ONLY if NOTIFY_STOP happens.. 444 * die_nmi will return ONLY if NOTIFY_STOP happens..
445 */ 445 */
@@ -447,7 +447,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
447 regs, panic_on_timeout); 447 regs, panic_on_timeout);
448 } else { 448 } else {
449 __get_cpu_var(last_irq_sum) = sum; 449 __get_cpu_var(last_irq_sum) = sum;
450 __this_cpu_write(per_cpu_var(alert_counter), 0); 450 __this_cpu_write(alert_counter, 0);
451 } 451 }
452 452
453 /* see if the nmi watchdog went off */ 453 /* see if the nmi watchdog went off */
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 7fd318bac59c..37c3d4b17d85 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -442,8 +442,8 @@ is386: movl $2,%ecx # set MP
442 */ 442 */
443 cmpb $0,ready 443 cmpb $0,ready
444 jne 1f 444 jne 1f
445 movl $per_cpu__gdt_page,%eax 445 movl $gdt_page,%eax
446 movl $per_cpu__stack_canary,%ecx 446 movl $stack_canary,%ecx
447 movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax) 447 movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
448 shrl $16, %ecx 448 shrl $16, %ecx
449 movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax) 449 movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
@@ -706,7 +706,7 @@ idt_descr:
706 .word 0 # 32 bit align gdt_desc.address 706 .word 0 # 32 bit align gdt_desc.address
707ENTRY(early_gdt_descr) 707ENTRY(early_gdt_descr)
708 .word GDT_ENTRIES*8-1 708 .word GDT_ENTRIES*8-1
709 .long per_cpu__gdt_page /* Overwritten for secondary CPUs */ 709 .long gdt_page /* Overwritten for secondary CPUs */
710 710
711/* 711/*
712 * The boot_gdt must mirror the equivalent in setup.S and is 712 * The boot_gdt must mirror the equivalent in setup.S and is
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index f92a0da608cb..44879df55696 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -341,7 +341,7 @@ SECTIONS
341 * Per-cpu symbols which need to be offset from __per_cpu_load 341 * Per-cpu symbols which need to be offset from __per_cpu_load
342 * for the boot processor. 342 * for the boot processor.
343 */ 343 */
344#define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load 344#define INIT_PER_CPU(x) init_per_cpu__##x = x + __per_cpu_load
345INIT_PER_CPU(gdt_page); 345INIT_PER_CPU(gdt_page);
346INIT_PER_CPU(irq_stack_union); 346INIT_PER_CPU(irq_stack_union);
347 347
@@ -352,7 +352,7 @@ INIT_PER_CPU(irq_stack_union);
352 "kernel image bigger than KERNEL_IMAGE_SIZE"); 352 "kernel image bigger than KERNEL_IMAGE_SIZE");
353 353
354#ifdef CONFIG_SMP 354#ifdef CONFIG_SMP
355. = ASSERT((per_cpu__irq_stack_union == 0), 355. = ASSERT((irq_stack_union == 0),
356 "irq_stack_union is not at start of per-cpu area"); 356 "irq_stack_union is not at start of per-cpu area");
357#endif 357#endif
358 358
diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S
index 88e15deb8b82..22a2093b5862 100644
--- a/arch/x86/xen/xen-asm_32.S
+++ b/arch/x86/xen/xen-asm_32.S
@@ -90,9 +90,9 @@ ENTRY(xen_iret)
90 GET_THREAD_INFO(%eax) 90 GET_THREAD_INFO(%eax)
91 movl TI_cpu(%eax), %eax 91 movl TI_cpu(%eax), %eax
92 movl __per_cpu_offset(,%eax,4), %eax 92 movl __per_cpu_offset(,%eax,4), %eax
93 mov per_cpu__xen_vcpu(%eax), %eax 93 mov xen_vcpu(%eax), %eax
94#else 94#else
95 movl per_cpu__xen_vcpu, %eax 95 movl xen_vcpu, %eax
96#endif 96#endif
97 97
98 /* check IF state we're restoring */ 98 /* check IF state we're restoring */
diff --git a/crypto/cryptd.c b/crypto/cryptd.c
index 704c14115323..ef71318976c7 100644
--- a/crypto/cryptd.c
+++ b/crypto/cryptd.c
@@ -31,7 +31,7 @@ struct cryptd_cpu_queue {
31}; 31};
32 32
33struct cryptd_queue { 33struct cryptd_queue {
34 struct cryptd_cpu_queue *cpu_queue; 34 struct cryptd_cpu_queue __percpu *cpu_queue;
35}; 35};
36 36
37struct cryptd_instance_ctx { 37struct cryptd_instance_ctx {
diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c
index a959f6a07508..d648a9860b88 100644
--- a/drivers/acpi/processor_perflib.c
+++ b/drivers/acpi/processor_perflib.c
@@ -561,7 +561,7 @@ end:
561} 561}
562 562
563int acpi_processor_preregister_performance( 563int acpi_processor_preregister_performance(
564 struct acpi_processor_performance *performance) 564 struct acpi_processor_performance __percpu *performance)
565{ 565{
566 int count, count_target; 566 int count, count_target;
567 int retval = 0; 567 int retval = 0;
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index e7a3230fb7d5..87399cafce37 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -284,7 +284,7 @@ struct dma_chan_tbl_ent {
284/** 284/**
285 * channel_table - percpu lookup table for memory-to-memory offload providers 285 * channel_table - percpu lookup table for memory-to-memory offload providers
286 */ 286 */
287static struct dma_chan_tbl_ent *channel_table[DMA_TX_TYPE_END]; 287static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];
288 288
289static int __init dma_channel_table_init(void) 289static int __init dma_channel_table_init(void)
290{ 290{
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index 3391e6739d06..7cd1cdcbe0f5 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -13,7 +13,7 @@ module_param(report_gart_errors, int, 0644);
13static int ecc_enable_override; 13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644); 14module_param(ecc_enable_override, int, 0644);
15 15
16static struct msr *msrs; 16static struct msr __percpu *msrs;
17 17
18/* Lookup table for all possible MC control instances */ 18/* Lookup table for all possible MC control instances */
19struct amd64_pvt; 19struct amd64_pvt;
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 509c8f3dd9a5..70ffbd071b2e 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -4680,7 +4680,7 @@ static int raid5_alloc_percpu(raid5_conf_t *conf)
4680{ 4680{
4681 unsigned long cpu; 4681 unsigned long cpu;
4682 struct page *spare_page; 4682 struct page *spare_page;
4683 struct raid5_percpu *allcpus; 4683 struct raid5_percpu __percpu *allcpus;
4684 void *scribble; 4684 void *scribble;
4685 int err; 4685 int err;
4686 4686
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index dd708359b451..0f86f5e36724 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -405,7 +405,7 @@ struct raid5_private_data {
405 * lists and performing address 405 * lists and performing address
406 * conversions 406 * conversions
407 */ 407 */
408 } *percpu; 408 } __percpu *percpu;
409 size_t scribble_len; /* size of scribble region must be 409 size_t scribble_len; /* size of scribble region must be
410 * associated with conf to handle 410 * associated with conf to handle
411 * cpu hotplug while reshaping 411 * cpu hotplug while reshaping
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index 874d169a193e..4cedc91ec59d 100644
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -1014,7 +1014,7 @@ struct ext4_sb_info {
1014 atomic_t s_lock_busy; 1014 atomic_t s_lock_busy;
1015 1015
1016 /* locality groups */ 1016 /* locality groups */
1017 struct ext4_locality_group *s_locality_groups; 1017 struct ext4_locality_group __percpu *s_locality_groups;
1018 1018
1019 /* for write statistics */ 1019 /* for write statistics */
1020 unsigned long s_sectors_written_start; 1020 unsigned long s_sectors_written_start;
diff --git a/fs/nfs/iostat.h b/fs/nfs/iostat.h
index 46d779abafd3..1d8d5c813b01 100644
--- a/fs/nfs/iostat.h
+++ b/fs/nfs/iostat.h
@@ -57,12 +57,12 @@ static inline void nfs_add_fscache_stats(struct inode *inode,
57} 57}
58#endif 58#endif
59 59
60static inline struct nfs_iostats *nfs_alloc_iostats(void) 60static inline struct nfs_iostats __percpu *nfs_alloc_iostats(void)
61{ 61{
62 return alloc_percpu(struct nfs_iostats); 62 return alloc_percpu(struct nfs_iostats);
63} 63}
64 64
65static inline void nfs_free_iostats(struct nfs_iostats *stats) 65static inline void nfs_free_iostats(struct nfs_iostats __percpu *stats)
66{ 66{
67 if (stats != NULL) 67 if (stats != NULL)
68 free_percpu(stats); 68 free_percpu(stats);
diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h
index 70504fcf14cd..14dafd608230 100644
--- a/fs/xfs/xfs_mount.h
+++ b/fs/xfs/xfs_mount.h
@@ -245,7 +245,7 @@ typedef struct xfs_mount {
245 struct xfs_qmops *m_qm_ops; /* vector of XQM ops */ 245 struct xfs_qmops *m_qm_ops; /* vector of XQM ops */
246 atomic_t m_active_trans; /* number trans frozen */ 246 atomic_t m_active_trans; /* number trans frozen */
247#ifdef HAVE_PERCPU_SB 247#ifdef HAVE_PERCPU_SB
248 xfs_icsb_cnts_t *m_sb_cnts; /* per-cpu superblock counters */ 248 xfs_icsb_cnts_t __percpu *m_sb_cnts; /* per-cpu superblock counters */
249 unsigned long m_icsb_counters; /* disabled per-cpu counters */ 249 unsigned long m_icsb_counters; /* disabled per-cpu counters */
250 struct notifier_block m_icsb_notifier; /* hotplug cpu notifier */ 250 struct notifier_block m_icsb_notifier; /* hotplug cpu notifier */
251 struct mutex m_icsb_mutex; /* balancer sync lock */ 251 struct mutex m_icsb_mutex; /* balancer sync lock */
diff --git a/include/acpi/processor.h b/include/acpi/processor.h
index 29831768c0e6..1172c27adadf 100644
--- a/include/acpi/processor.h
+++ b/include/acpi/processor.h
@@ -238,7 +238,7 @@ struct acpi_processor_errata {
238 238
239extern int acpi_processor_preregister_performance(struct 239extern int acpi_processor_preregister_performance(struct
240 acpi_processor_performance 240 acpi_processor_performance
241 *performance); 241 __percpu *performance);
242 242
243extern int acpi_processor_register_performance(struct acpi_processor_performance 243extern int acpi_processor_register_performance(struct acpi_processor_performance
244 *performance, unsigned int cpu); 244 *performance, unsigned int cpu);
diff --git a/include/asm-generic/local.h b/include/asm-generic/local.h
index fc218444e315..c8a5d68541d7 100644
--- a/include/asm-generic/local.h
+++ b/include/asm-generic/local.h
@@ -52,23 +52,4 @@ typedef struct
52#define __local_add(i,l) local_set((l), local_read(l) + (i)) 52#define __local_add(i,l) local_set((l), local_read(l) + (i))
53#define __local_sub(i,l) local_set((l), local_read(l) - (i)) 53#define __local_sub(i,l) local_set((l), local_read(l) - (i))
54 54
55/* Use these for per-cpu local_t variables: on some archs they are
56 * much more efficient than these naive implementations. Note they take
57 * a variable (eg. mystruct.foo), not an address.
58 */
59#define cpu_local_read(l) local_read(&__get_cpu_var(l))
60#define cpu_local_set(l, i) local_set(&__get_cpu_var(l), (i))
61#define cpu_local_inc(l) local_inc(&__get_cpu_var(l))
62#define cpu_local_dec(l) local_dec(&__get_cpu_var(l))
63#define cpu_local_add(i, l) local_add((i), &__get_cpu_var(l))
64#define cpu_local_sub(i, l) local_sub((i), &__get_cpu_var(l))
65
66/* Non-atomic increments, ie. preemption disabled and won't be touched
67 * in interrupt, etc. Some archs can optimize this case well.
68 */
69#define __cpu_local_inc(l) __local_inc(&__get_cpu_var(l))
70#define __cpu_local_dec(l) __local_dec(&__get_cpu_var(l))
71#define __cpu_local_add(i, l) __local_add((i), &__get_cpu_var(l))
72#define __cpu_local_sub(i, l) __local_sub((i), &__get_cpu_var(l))
73
74#endif /* _ASM_GENERIC_LOCAL_H */ 55#endif /* _ASM_GENERIC_LOCAL_H */
diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index 8087b90d4673..04f91c2d3f7b 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -41,7 +41,11 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
41 * Only S390 provides its own means of moving the pointer. 41 * Only S390 provides its own means of moving the pointer.
42 */ 42 */
43#ifndef SHIFT_PERCPU_PTR 43#ifndef SHIFT_PERCPU_PTR
44#define SHIFT_PERCPU_PTR(__p, __offset) RELOC_HIDE((__p), (__offset)) 44/* Weird cast keeps both GCC and sparse happy. */
45#define SHIFT_PERCPU_PTR(__p, __offset) ({ \
46 __verify_pcpu_ptr((__p)); \
47 RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \
48})
45#endif 49#endif
46 50
47/* 51/*
@@ -50,11 +54,11 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
50 * offset. 54 * offset.
51 */ 55 */
52#define per_cpu(var, cpu) \ 56#define per_cpu(var, cpu) \
53 (*SHIFT_PERCPU_PTR(&per_cpu_var(var), per_cpu_offset(cpu))) 57 (*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu)))
54#define __get_cpu_var(var) \ 58#define __get_cpu_var(var) \
55 (*SHIFT_PERCPU_PTR(&per_cpu_var(var), my_cpu_offset)) 59 (*SHIFT_PERCPU_PTR(&(var), my_cpu_offset))
56#define __raw_get_cpu_var(var) \ 60#define __raw_get_cpu_var(var) \
57 (*SHIFT_PERCPU_PTR(&per_cpu_var(var), __my_cpu_offset)) 61 (*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset))
58 62
59#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) 63#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
60#define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) 64#define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
@@ -66,9 +70,9 @@ extern void setup_per_cpu_areas(void);
66 70
67#else /* ! SMP */ 71#else /* ! SMP */
68 72
69#define per_cpu(var, cpu) (*((void)(cpu), &per_cpu_var(var))) 73#define per_cpu(var, cpu) (*((void)(cpu), &(var)))
70#define __get_cpu_var(var) per_cpu_var(var) 74#define __get_cpu_var(var) (var)
71#define __raw_get_cpu_var(var) per_cpu_var(var) 75#define __raw_get_cpu_var(var) (var)
72#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0) 76#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
73#define __this_cpu_ptr(ptr) this_cpu_ptr(ptr) 77#define __this_cpu_ptr(ptr) this_cpu_ptr(ptr)
74 78
diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h
index 3b73b9992b26..416bf62d6d46 100644
--- a/include/linux/blktrace_api.h
+++ b/include/linux/blktrace_api.h
@@ -150,8 +150,8 @@ struct blk_user_trace_setup {
150struct blk_trace { 150struct blk_trace {
151 int trace_state; 151 int trace_state;
152 struct rchan *rchan; 152 struct rchan *rchan;
153 unsigned long *sequence; 153 unsigned long __percpu *sequence;
154 unsigned char *msg_data; 154 unsigned char __percpu *msg_data;
155 u16 act_mask; 155 u16 act_mask;
156 u64 start_lba; 156 u64 start_lba;
157 u64 end_lba; 157 u64 end_lba;
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 188fcae10a99..a5a472b10746 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -5,7 +5,7 @@
5 5
6#ifdef __CHECKER__ 6#ifdef __CHECKER__
7# define __user __attribute__((noderef, address_space(1))) 7# define __user __attribute__((noderef, address_space(1)))
8# define __kernel /* default address space */ 8# define __kernel __attribute__((address_space(0)))
9# define __safe __attribute__((safe)) 9# define __safe __attribute__((safe))
10# define __force __attribute__((force)) 10# define __force __attribute__((force))
11# define __nocast __attribute__((nocast)) 11# define __nocast __attribute__((nocast))
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index 78784982b33e..21fd9b7c6a40 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -162,7 +162,7 @@ struct dma_chan {
162 struct dma_chan_dev *dev; 162 struct dma_chan_dev *dev;
163 163
164 struct list_head device_node; 164 struct list_head device_node;
165 struct dma_chan_percpu *local; 165 struct dma_chan_percpu __percpu *local;
166 int client_count; 166 int client_count;
167 int table_count; 167 int table_count;
168 void *private; 168 void *private;
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 9717081c75ad..56b50514ab25 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -101,7 +101,7 @@ struct hd_struct {
101 unsigned long stamp; 101 unsigned long stamp;
102 int in_flight[2]; 102 int in_flight[2];
103#ifdef CONFIG_SMP 103#ifdef CONFIG_SMP
104 struct disk_stats *dkstats; 104 struct disk_stats __percpu *dkstats;
105#else 105#else
106 struct disk_stats dkstats; 106 struct disk_stats dkstats;
107#endif 107#endif
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index c356b6914ffd..03e8e8dbc577 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -199,7 +199,7 @@ extern struct kimage *kexec_crash_image;
199 */ 199 */
200extern struct resource crashk_res; 200extern struct resource crashk_res;
201typedef u32 note_buf_t[KEXEC_NOTE_BYTES/4]; 201typedef u32 note_buf_t[KEXEC_NOTE_BYTES/4];
202extern note_buf_t *crash_notes; 202extern note_buf_t __percpu *crash_notes;
203extern u32 vmcoreinfo_note[VMCOREINFO_NOTE_SIZE/4]; 203extern u32 vmcoreinfo_note[VMCOREINFO_NOTE_SIZE/4];
204extern size_t vmcoreinfo_size; 204extern size_t vmcoreinfo_size;
205extern size_t vmcoreinfo_max_size; 205extern size_t vmcoreinfo_max_size;
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8b2fa8593c61..2e724c877ec1 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1081,11 +1081,7 @@ extern void si_meminfo(struct sysinfo * val);
1081extern void si_meminfo_node(struct sysinfo *val, int nid); 1081extern void si_meminfo_node(struct sysinfo *val, int nid);
1082extern int after_bootmem; 1082extern int after_bootmem;
1083 1083
1084#ifdef CONFIG_NUMA
1085extern void setup_per_cpu_pageset(void); 1084extern void setup_per_cpu_pageset(void);
1086#else
1087static inline void setup_per_cpu_pageset(void) {}
1088#endif
1089 1085
1090extern void zone_pcp_update(struct zone *zone); 1086extern void zone_pcp_update(struct zone *zone);
1091 1087
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 30fe668c2542..41acd4bf7664 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -184,13 +184,7 @@ struct per_cpu_pageset {
184 s8 stat_threshold; 184 s8 stat_threshold;
185 s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; 185 s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
186#endif 186#endif
187} ____cacheline_aligned_in_smp; 187};
188
189#ifdef CONFIG_NUMA
190#define zone_pcp(__z, __cpu) ((__z)->pageset[(__cpu)])
191#else
192#define zone_pcp(__z, __cpu) (&(__z)->pageset[(__cpu)])
193#endif
194 188
195#endif /* !__GENERATING_BOUNDS.H */ 189#endif /* !__GENERATING_BOUNDS.H */
196 190
@@ -306,10 +300,8 @@ struct zone {
306 */ 300 */
307 unsigned long min_unmapped_pages; 301 unsigned long min_unmapped_pages;
308 unsigned long min_slab_pages; 302 unsigned long min_slab_pages;
309 struct per_cpu_pageset *pageset[NR_CPUS];
310#else
311 struct per_cpu_pageset pageset[NR_CPUS];
312#endif 303#endif
304 struct per_cpu_pageset __percpu *pageset;
313 /* 305 /*
314 * free areas of different sizes 306 * free areas of different sizes
315 */ 307 */
diff --git a/include/linux/module.h b/include/linux/module.h
index 6cb1a3cab5d3..dd618eb026aa 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -17,7 +17,7 @@
17#include <linux/moduleparam.h> 17#include <linux/moduleparam.h>
18#include <linux/tracepoint.h> 18#include <linux/tracepoint.h>
19 19
20#include <asm/local.h> 20#include <linux/percpu.h>
21#include <asm/module.h> 21#include <asm/module.h>
22 22
23#include <trace/events/module.h> 23#include <trace/events/module.h>
@@ -363,11 +363,9 @@ struct module
363 /* Destruction function. */ 363 /* Destruction function. */
364 void (*exit)(void); 364 void (*exit)(void);
365 365
366#ifdef CONFIG_SMP 366 struct module_ref {
367 char *refptr; 367 int count;
368#else 368 } __percpu *refptr;
369 local_t ref;
370#endif
371#endif 369#endif
372 370
373#ifdef CONFIG_CONSTRUCTORS 371#ifdef CONFIG_CONSTRUCTORS
@@ -454,25 +452,16 @@ void __symbol_put(const char *symbol);
454#define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x) 452#define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x)
455void symbol_put_addr(void *addr); 453void symbol_put_addr(void *addr);
456 454
457static inline local_t *__module_ref_addr(struct module *mod, int cpu)
458{
459#ifdef CONFIG_SMP
460 return (local_t *) (mod->refptr + per_cpu_offset(cpu));
461#else
462 return &mod->ref;
463#endif
464}
465
466/* Sometimes we know we already have a refcount, and it's easier not 455/* Sometimes we know we already have a refcount, and it's easier not
467 to handle the error case (which only happens with rmmod --wait). */ 456 to handle the error case (which only happens with rmmod --wait). */
468static inline void __module_get(struct module *module) 457static inline void __module_get(struct module *module)
469{ 458{
470 if (module) { 459 if (module) {
471 unsigned int cpu = get_cpu(); 460 preempt_disable();
472 local_inc(__module_ref_addr(module, cpu)); 461 __this_cpu_inc(module->refptr->count);
473 trace_module_get(module, _THIS_IP_, 462 trace_module_get(module, _THIS_IP_,
474 local_read(__module_ref_addr(module, cpu))); 463 __this_cpu_read(module->refptr->count));
475 put_cpu(); 464 preempt_enable();
476 } 465 }
477} 466}
478 467
@@ -481,15 +470,17 @@ static inline int try_module_get(struct module *module)
481 int ret = 1; 470 int ret = 1;
482 471
483 if (module) { 472 if (module) {
484 unsigned int cpu = get_cpu(); 473 preempt_disable();
474
485 if (likely(module_is_live(module))) { 475 if (likely(module_is_live(module))) {
486 local_inc(__module_ref_addr(module, cpu)); 476 __this_cpu_inc(module->refptr->count);
487 trace_module_get(module, _THIS_IP_, 477 trace_module_get(module, _THIS_IP_,
488 local_read(__module_ref_addr(module, cpu))); 478 __this_cpu_read(module->refptr->count));
489 } 479 }
490 else 480 else
491 ret = 0; 481 ret = 0;
492 put_cpu(); 482
483 preempt_enable();
493 } 484 }
494 return ret; 485 return ret;
495} 486}
diff --git a/include/linux/mount.h b/include/linux/mount.h
index 5d5275364867..b5f43a34ef88 100644
--- a/include/linux/mount.h
+++ b/include/linux/mount.h
@@ -66,7 +66,7 @@ struct vfsmount {
66 int mnt_pinned; 66 int mnt_pinned;
67 int mnt_ghosts; 67 int mnt_ghosts;
68#ifdef CONFIG_SMP 68#ifdef CONFIG_SMP
69 int *mnt_writers; 69 int __percpu *mnt_writers;
70#else 70#else
71 int mnt_writers; 71 int mnt_writers;
72#endif 72#endif
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index 34fc6be5bfcf..6a2e44fd75e2 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -105,7 +105,7 @@ struct nfs_server {
105 struct rpc_clnt * client; /* RPC client handle */ 105 struct rpc_clnt * client; /* RPC client handle */
106 struct rpc_clnt * client_acl; /* ACL RPC client handle */ 106 struct rpc_clnt * client_acl; /* ACL RPC client handle */
107 struct nlm_host *nlm_host; /* NLM client handle */ 107 struct nlm_host *nlm_host; /* NLM client handle */
108 struct nfs_iostats * io_stats; /* I/O statistics */ 108 struct nfs_iostats __percpu *io_stats; /* I/O statistics */
109 struct backing_dev_info backing_dev_info; 109 struct backing_dev_info backing_dev_info;
110 atomic_long_t writeback; /* number of writeback pages */ 110 atomic_long_t writeback; /* number of writeback pages */
111 int flags; /* various flags */ 111 int flags; /* various flags */
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 5a5d6ce4bd55..68567c0b3a5d 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -2,12 +2,6 @@
2#define _LINUX_PERCPU_DEFS_H 2#define _LINUX_PERCPU_DEFS_H
3 3
4/* 4/*
5 * Determine the real variable name from the name visible in the
6 * kernel sources.
7 */
8#define per_cpu_var(var) per_cpu__##var
9
10/*
11 * Base implementations of per-CPU variable declarations and definitions, where 5 * Base implementations of per-CPU variable declarations and definitions, where
12 * the section in which the variable is to be placed is provided by the 6 * the section in which the variable is to be placed is provided by the
13 * 'sec' argument. This may be used to affect the parameters governing the 7 * 'sec' argument. This may be used to affect the parameters governing the
@@ -18,13 +12,23 @@
18 * that section. 12 * that section.
19 */ 13 */
20#define __PCPU_ATTRS(sec) \ 14#define __PCPU_ATTRS(sec) \
21 __attribute__((section(PER_CPU_BASE_SECTION sec))) \ 15 __percpu __attribute__((section(PER_CPU_BASE_SECTION sec))) \
22 PER_CPU_ATTRIBUTES 16 PER_CPU_ATTRIBUTES
23 17
24#define __PCPU_DUMMY_ATTRS \ 18#define __PCPU_DUMMY_ATTRS \
25 __attribute__((section(".discard"), unused)) 19 __attribute__((section(".discard"), unused))
26 20
27/* 21/*
22 * Macro which verifies @ptr is a percpu pointer without evaluating
23 * @ptr. This is to be used in percpu accessors to verify that the
24 * input parameter is a percpu pointer.
25 */
26#define __verify_pcpu_ptr(ptr) do { \
27 const void __percpu *__vpp_verify = (typeof(ptr))NULL; \
28 (void)__vpp_verify; \
29} while (0)
30
31/*
28 * s390 and alpha modules require percpu variables to be defined as 32 * s390 and alpha modules require percpu variables to be defined as
29 * weak to force the compiler to generate GOT based external 33 * weak to force the compiler to generate GOT based external
30 * references for them. This is necessary because percpu sections 34 * references for them. This is necessary because percpu sections
@@ -56,24 +60,24 @@
56 */ 60 */
57#define DECLARE_PER_CPU_SECTION(type, name, sec) \ 61#define DECLARE_PER_CPU_SECTION(type, name, sec) \
58 extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \ 62 extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \
59 extern __PCPU_ATTRS(sec) __typeof__(type) per_cpu__##name 63 extern __PCPU_ATTRS(sec) __typeof__(type) name
60 64
61#define DEFINE_PER_CPU_SECTION(type, name, sec) \ 65#define DEFINE_PER_CPU_SECTION(type, name, sec) \
62 __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \ 66 __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \
63 extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ 67 extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
64 __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ 68 __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
65 __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak \ 69 __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak \
66 __typeof__(type) per_cpu__##name 70 __typeof__(type) name
67#else 71#else
68/* 72/*
69 * Normal declaration and definition macros. 73 * Normal declaration and definition macros.
70 */ 74 */
71#define DECLARE_PER_CPU_SECTION(type, name, sec) \ 75#define DECLARE_PER_CPU_SECTION(type, name, sec) \
72 extern __PCPU_ATTRS(sec) __typeof__(type) per_cpu__##name 76 extern __PCPU_ATTRS(sec) __typeof__(type) name
73 77
74#define DEFINE_PER_CPU_SECTION(type, name, sec) \ 78#define DEFINE_PER_CPU_SECTION(type, name, sec) \
75 __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES \ 79 __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES \
76 __typeof__(type) per_cpu__##name 80 __typeof__(type) name
77#endif 81#endif
78 82
79/* 83/*
@@ -135,10 +139,16 @@
135 __aligned(PAGE_SIZE) 139 __aligned(PAGE_SIZE)
136 140
137/* 141/*
138 * Intermodule exports for per-CPU variables. 142 * Intermodule exports for per-CPU variables. sparse forgets about
143 * address space across EXPORT_SYMBOL(), change EXPORT_SYMBOL() to
144 * noop if __CHECKER__.
139 */ 145 */
140#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) 146#ifndef __CHECKER__
141#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) 147#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var)
142 148#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var)
149#else
150#define EXPORT_PER_CPU_SYMBOL(var)
151#define EXPORT_PER_CPU_SYMBOL_GPL(var)
152#endif
143 153
144#endif /* _LINUX_PERCPU_DEFS_H */ 154#endif /* _LINUX_PERCPU_DEFS_H */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index cf5efbcf716c..a93e5bfdccb8 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -27,10 +27,17 @@
27 * we force a syntax error here if it isn't. 27 * we force a syntax error here if it isn't.
28 */ 28 */
29#define get_cpu_var(var) (*({ \ 29#define get_cpu_var(var) (*({ \
30 extern int simple_identifier_##var(void); \
31 preempt_disable(); \ 30 preempt_disable(); \
32 &__get_cpu_var(var); })) 31 &__get_cpu_var(var); }))
33#define put_cpu_var(var) preempt_enable() 32
33/*
34 * The weird & is necessary because sparse considers (void)(var) to be
35 * a direct dereference of percpu variable (var).
36 */
37#define put_cpu_var(var) do { \
38 (void)&(var); \
39 preempt_enable(); \
40} while (0)
34 41
35#ifdef CONFIG_SMP 42#ifdef CONFIG_SMP
36 43
@@ -127,9 +134,9 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
127 */ 134 */
128#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) 135#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
129 136
130extern void *__alloc_reserved_percpu(size_t size, size_t align); 137extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
131extern void *__alloc_percpu(size_t size, size_t align); 138extern void __percpu *__alloc_percpu(size_t size, size_t align);
132extern void free_percpu(void *__pdata); 139extern void free_percpu(void __percpu *__pdata);
133extern phys_addr_t per_cpu_ptr_to_phys(void *addr); 140extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
134 141
135#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA 142#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
@@ -140,7 +147,7 @@ extern void __init setup_per_cpu_areas(void);
140 147
141#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) 148#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
142 149
143static inline void *__alloc_percpu(size_t size, size_t align) 150static inline void __percpu *__alloc_percpu(size_t size, size_t align)
144{ 151{
145 /* 152 /*
146 * Can't easily make larger alignment work with kmalloc. WARN 153 * Can't easily make larger alignment work with kmalloc. WARN
@@ -151,7 +158,7 @@ static inline void *__alloc_percpu(size_t size, size_t align)
151 return kzalloc(size, GFP_KERNEL); 158 return kzalloc(size, GFP_KERNEL);
152} 159}
153 160
154static inline void free_percpu(void *p) 161static inline void free_percpu(void __percpu *p)
155{ 162{
156 kfree(p); 163 kfree(p);
157} 164}
@@ -171,7 +178,7 @@ static inline void *pcpu_lpage_remapped(void *kaddr)
171#endif /* CONFIG_SMP */ 178#endif /* CONFIG_SMP */
172 179
173#define alloc_percpu(type) \ 180#define alloc_percpu(type) \
174 (typeof(type) *)__alloc_percpu(sizeof(type), __alignof__(type)) 181 (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
175 182
176/* 183/*
177 * Optional methods for optimized non-lvalue per-cpu variable access. 184 * Optional methods for optimized non-lvalue per-cpu variable access.
@@ -188,17 +195,19 @@ static inline void *pcpu_lpage_remapped(void *kaddr)
188#ifndef percpu_read 195#ifndef percpu_read
189# define percpu_read(var) \ 196# define percpu_read(var) \
190 ({ \ 197 ({ \
191 typeof(per_cpu_var(var)) __tmp_var__; \ 198 typeof(var) *pr_ptr__ = &(var); \
192 __tmp_var__ = get_cpu_var(var); \ 199 typeof(var) pr_ret__; \
193 put_cpu_var(var); \ 200 pr_ret__ = get_cpu_var(*pr_ptr__); \
194 __tmp_var__; \ 201 put_cpu_var(*pr_ptr__); \
202 pr_ret__; \
195 }) 203 })
196#endif 204#endif
197 205
198#define __percpu_generic_to_op(var, val, op) \ 206#define __percpu_generic_to_op(var, val, op) \
199do { \ 207do { \
200 get_cpu_var(var) op val; \ 208 typeof(var) *pgto_ptr__ = &(var); \
201 put_cpu_var(var); \ 209 get_cpu_var(*pgto_ptr__) op val; \
210 put_cpu_var(*pgto_ptr__); \
202} while (0) 211} while (0)
203 212
204#ifndef percpu_write 213#ifndef percpu_write
@@ -234,6 +243,7 @@ extern void __bad_size_call_parameter(void);
234 243
235#define __pcpu_size_call_return(stem, variable) \ 244#define __pcpu_size_call_return(stem, variable) \
236({ typeof(variable) pscr_ret__; \ 245({ typeof(variable) pscr_ret__; \
246 __verify_pcpu_ptr(&(variable)); \
237 switch(sizeof(variable)) { \ 247 switch(sizeof(variable)) { \
238 case 1: pscr_ret__ = stem##1(variable);break; \ 248 case 1: pscr_ret__ = stem##1(variable);break; \
239 case 2: pscr_ret__ = stem##2(variable);break; \ 249 case 2: pscr_ret__ = stem##2(variable);break; \
@@ -247,6 +257,7 @@ extern void __bad_size_call_parameter(void);
247 257
248#define __pcpu_size_call(stem, variable, ...) \ 258#define __pcpu_size_call(stem, variable, ...) \
249do { \ 259do { \
260 __verify_pcpu_ptr(&(variable)); \
250 switch(sizeof(variable)) { \ 261 switch(sizeof(variable)) { \
251 case 1: stem##1(variable, __VA_ARGS__);break; \ 262 case 1: stem##1(variable, __VA_ARGS__);break; \
252 case 2: stem##2(variable, __VA_ARGS__);break; \ 263 case 2: stem##2(variable, __VA_ARGS__);break; \
@@ -259,8 +270,7 @@ do { \
259 270
260/* 271/*
261 * Optimized manipulation for memory allocated through the per cpu 272 * Optimized manipulation for memory allocated through the per cpu
262 * allocator or for addresses of per cpu variables (can be determined 273 * allocator or for addresses of per cpu variables.
263 * using per_cpu_var(xx).
264 * 274 *
265 * These operation guarantee exclusivity of access for other operations 275 * These operation guarantee exclusivity of access for other operations
266 * on the *same* processor. The assumption is that per cpu data is only 276 * on the *same* processor. The assumption is that per cpu data is only
@@ -311,7 +321,7 @@ do { \
311#define _this_cpu_generic_to_op(pcp, val, op) \ 321#define _this_cpu_generic_to_op(pcp, val, op) \
312do { \ 322do { \
313 preempt_disable(); \ 323 preempt_disable(); \
314 *__this_cpu_ptr(&pcp) op val; \ 324 *__this_cpu_ptr(&(pcp)) op val; \
315 preempt_enable(); \ 325 preempt_enable(); \
316} while (0) 326} while (0)
317 327
diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h
index 794662b2be5d..c88d67b59394 100644
--- a/include/linux/percpu_counter.h
+++ b/include/linux/percpu_counter.h
@@ -21,7 +21,7 @@ struct percpu_counter {
21#ifdef CONFIG_HOTPLUG_CPU 21#ifdef CONFIG_HOTPLUG_CPU
22 struct list_head list; /* All percpu_counters are on a list */ 22 struct list_head list; /* All percpu_counters are on a list */
23#endif 23#endif
24 s32 *counters; 24 s32 __percpu *counters;
25}; 25};
26 26
27extern int percpu_counter_batch; 27extern int percpu_counter_batch;
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 3084f80909cd..4d5ecb222af9 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -33,7 +33,7 @@ struct srcu_struct_array {
33 33
34struct srcu_struct { 34struct srcu_struct {
35 int completed; 35 int completed;
36 struct srcu_struct_array *per_cpu_ref; 36 struct srcu_struct_array __percpu *per_cpu_ref;
37 struct mutex mutex; 37 struct mutex mutex;
38#ifdef CONFIG_DEBUG_LOCK_ALLOC 38#ifdef CONFIG_DEBUG_LOCK_ALLOC
39 struct lockdep_map dep_map; 39 struct lockdep_map dep_map;
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index ee03bba9c5df..117f0dd8ad03 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -78,22 +78,22 @@ DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
78 78
79static inline void __count_vm_event(enum vm_event_item item) 79static inline void __count_vm_event(enum vm_event_item item)
80{ 80{
81 __this_cpu_inc(per_cpu_var(vm_event_states).event[item]); 81 __this_cpu_inc(vm_event_states.event[item]);
82} 82}
83 83
84static inline void count_vm_event(enum vm_event_item item) 84static inline void count_vm_event(enum vm_event_item item)
85{ 85{
86 this_cpu_inc(per_cpu_var(vm_event_states).event[item]); 86 this_cpu_inc(vm_event_states.event[item]);
87} 87}
88 88
89static inline void __count_vm_events(enum vm_event_item item, long delta) 89static inline void __count_vm_events(enum vm_event_item item, long delta)
90{ 90{
91 __this_cpu_add(per_cpu_var(vm_event_states).event[item], delta); 91 __this_cpu_add(vm_event_states.event[item], delta);
92} 92}
93 93
94static inline void count_vm_events(enum vm_event_item item, long delta) 94static inline void count_vm_events(enum vm_event_item item, long delta)
95{ 95{
96 this_cpu_add(per_cpu_var(vm_event_states).event[item], delta); 96 this_cpu_add(vm_event_states.event[item], delta);
97} 97}
98 98
99extern void all_vm_events(unsigned long *); 99extern void all_vm_events(unsigned long *);
diff --git a/kernel/kexec.c b/kernel/kexec.c
index ef077fb73155..87ebe8adc474 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -41,7 +41,7 @@
41#include <asm/sections.h> 41#include <asm/sections.h>
42 42
43/* Per cpu memory for storing cpu states in case of system crash. */ 43/* Per cpu memory for storing cpu states in case of system crash. */
44note_buf_t* crash_notes; 44note_buf_t __percpu *crash_notes;
45 45
46/* vmcoreinfo stuff */ 46/* vmcoreinfo stuff */
47static unsigned char vmcoreinfo_data[VMCOREINFO_BYTES]; 47static unsigned char vmcoreinfo_data[VMCOREINFO_BYTES];
diff --git a/kernel/module.c b/kernel/module.c
index f82386bd9ee9..e5538d5f00ad 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -474,9 +474,10 @@ static void module_unload_init(struct module *mod)
474 474
475 INIT_LIST_HEAD(&mod->modules_which_use_me); 475 INIT_LIST_HEAD(&mod->modules_which_use_me);
476 for_each_possible_cpu(cpu) 476 for_each_possible_cpu(cpu)
477 local_set(__module_ref_addr(mod, cpu), 0); 477 per_cpu_ptr(mod->refptr, cpu)->count = 0;
478
478 /* Hold reference count during initialization. */ 479 /* Hold reference count during initialization. */
479 local_set(__module_ref_addr(mod, raw_smp_processor_id()), 1); 480 __this_cpu_write(mod->refptr->count, 1);
480 /* Backwards compatibility macros put refcount during init. */ 481 /* Backwards compatibility macros put refcount during init. */
481 mod->waiter = current; 482 mod->waiter = current;
482} 483}
@@ -619,7 +620,7 @@ unsigned int module_refcount(struct module *mod)
619 int cpu; 620 int cpu;
620 621
621 for_each_possible_cpu(cpu) 622 for_each_possible_cpu(cpu)
622 total += local_read(__module_ref_addr(mod, cpu)); 623 total += per_cpu_ptr(mod->refptr, cpu)->count;
623 return total; 624 return total;
624} 625}
625EXPORT_SYMBOL(module_refcount); 626EXPORT_SYMBOL(module_refcount);
@@ -796,14 +797,15 @@ static struct module_attribute refcnt = {
796void module_put(struct module *module) 797void module_put(struct module *module)
797{ 798{
798 if (module) { 799 if (module) {
799 unsigned int cpu = get_cpu(); 800 preempt_disable();
800 local_dec(__module_ref_addr(module, cpu)); 801 __this_cpu_dec(module->refptr->count);
802
801 trace_module_put(module, _RET_IP_, 803 trace_module_put(module, _RET_IP_,
802 local_read(__module_ref_addr(module, cpu))); 804 __this_cpu_read(module->refptr->count));
803 /* Maybe they're waiting for us to drop reference? */ 805 /* Maybe they're waiting for us to drop reference? */
804 if (unlikely(!module_is_live(module))) 806 if (unlikely(!module_is_live(module)))
805 wake_up_process(module->waiter); 807 wake_up_process(module->waiter);
806 put_cpu(); 808 preempt_enable();
807 } 809 }
808} 810}
809EXPORT_SYMBOL(module_put); 811EXPORT_SYMBOL(module_put);
@@ -1397,9 +1399,9 @@ static void free_module(struct module *mod)
1397 kfree(mod->args); 1399 kfree(mod->args);
1398 if (mod->percpu) 1400 if (mod->percpu)
1399 percpu_modfree(mod->percpu); 1401 percpu_modfree(mod->percpu);
1400#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP) 1402#if defined(CONFIG_MODULE_UNLOAD)
1401 if (mod->refptr) 1403 if (mod->refptr)
1402 percpu_modfree(mod->refptr); 1404 free_percpu(mod->refptr);
1403#endif 1405#endif
1404 /* Free lock-classes: */ 1406 /* Free lock-classes: */
1405 lockdep_free_key_range(mod->module_core, mod->core_size); 1407 lockdep_free_key_range(mod->module_core, mod->core_size);
@@ -2162,9 +2164,8 @@ static noinline struct module *load_module(void __user *umod,
2162 mod = (void *)sechdrs[modindex].sh_addr; 2164 mod = (void *)sechdrs[modindex].sh_addr;
2163 kmemleak_load_module(mod, hdr, sechdrs, secstrings); 2165 kmemleak_load_module(mod, hdr, sechdrs, secstrings);
2164 2166
2165#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP) 2167#if defined(CONFIG_MODULE_UNLOAD)
2166 mod->refptr = percpu_modalloc(sizeof(local_t), __alignof__(local_t), 2168 mod->refptr = alloc_percpu(struct module_ref);
2167 mod->name);
2168 if (!mod->refptr) { 2169 if (!mod->refptr) {
2169 err = -ENOMEM; 2170 err = -ENOMEM;
2170 goto free_init; 2171 goto free_init;
@@ -2396,8 +2397,8 @@ static noinline struct module *load_module(void __user *umod,
2396 kobject_put(&mod->mkobj.kobj); 2397 kobject_put(&mod->mkobj.kobj);
2397 free_unload: 2398 free_unload:
2398 module_unload_free(mod); 2399 module_unload_free(mod);
2399#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP) 2400#if defined(CONFIG_MODULE_UNLOAD)
2400 percpu_modfree(mod->refptr); 2401 free_percpu(mod->refptr);
2401 free_init: 2402 free_init:
2402#endif 2403#endif
2403 module_free(mod, mod->module_init); 2404 module_free(mod, mod->module_init);
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 258cdf0a91eb..58df55bf83ed 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -818,13 +818,13 @@ static void rcu_torture_timer(unsigned long unused)
818 /* Should not happen, but... */ 818 /* Should not happen, but... */
819 pipe_count = RCU_TORTURE_PIPE_LEN; 819 pipe_count = RCU_TORTURE_PIPE_LEN;
820 } 820 }
821 __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]); 821 __this_cpu_inc(rcu_torture_count[pipe_count]);
822 completed = cur_ops->completed() - completed; 822 completed = cur_ops->completed() - completed;
823 if (completed > RCU_TORTURE_PIPE_LEN) { 823 if (completed > RCU_TORTURE_PIPE_LEN) {
824 /* Should not happen, but... */ 824 /* Should not happen, but... */
825 completed = RCU_TORTURE_PIPE_LEN; 825 completed = RCU_TORTURE_PIPE_LEN;
826 } 826 }
827 __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]); 827 __this_cpu_inc(rcu_torture_batch[completed]);
828 preempt_enable(); 828 preempt_enable();
829 cur_ops->readunlock(idx); 829 cur_ops->readunlock(idx);
830} 830}
@@ -877,13 +877,13 @@ rcu_torture_reader(void *arg)
877 /* Should not happen, but... */ 877 /* Should not happen, but... */
878 pipe_count = RCU_TORTURE_PIPE_LEN; 878 pipe_count = RCU_TORTURE_PIPE_LEN;
879 } 879 }
880 __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]); 880 __this_cpu_inc(rcu_torture_count[pipe_count]);
881 completed = cur_ops->completed() - completed; 881 completed = cur_ops->completed() - completed;
882 if (completed > RCU_TORTURE_PIPE_LEN) { 882 if (completed > RCU_TORTURE_PIPE_LEN) {
883 /* Should not happen, but... */ 883 /* Should not happen, but... */
884 completed = RCU_TORTURE_PIPE_LEN; 884 completed = RCU_TORTURE_PIPE_LEN;
885 } 885 }
886 __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]); 886 __this_cpu_inc(rcu_torture_batch[completed]);
887 preempt_enable(); 887 preempt_enable();
888 cur_ops->readunlock(idx); 888 cur_ops->readunlock(idx);
889 schedule(); 889 schedule();
diff --git a/kernel/sched.c b/kernel/sched.c
index 6a212c97f523..abb36b16b93b 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -1521,7 +1521,7 @@ static unsigned long cpu_avg_load_per_task(int cpu)
1521 1521
1522#ifdef CONFIG_FAIR_GROUP_SCHED 1522#ifdef CONFIG_FAIR_GROUP_SCHED
1523 1523
1524static __read_mostly unsigned long *update_shares_data; 1524static __read_mostly unsigned long __percpu *update_shares_data;
1525 1525
1526static void __set_se_shares(struct sched_entity *se, unsigned long shares); 1526static void __set_se_shares(struct sched_entity *se, unsigned long shares);
1527 1527
@@ -8813,7 +8813,7 @@ struct cgroup_subsys cpu_cgroup_subsys = {
8813struct cpuacct { 8813struct cpuacct {
8814 struct cgroup_subsys_state css; 8814 struct cgroup_subsys_state css;
8815 /* cpuusage holds pointer to a u64-type object on every cpu */ 8815 /* cpuusage holds pointer to a u64-type object on every cpu */
8816 u64 *cpuusage; 8816 u64 __percpu *cpuusage;
8817 struct percpu_counter cpustat[CPUACCT_STAT_NSTATS]; 8817 struct percpu_counter cpustat[CPUACCT_STAT_NSTATS];
8818 struct cpuacct *parent; 8818 struct cpuacct *parent;
8819}; 8819};
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 912823e2a11b..9bb9fb1bd79c 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -45,7 +45,7 @@ static int refcount;
45static struct workqueue_struct *stop_machine_wq; 45static struct workqueue_struct *stop_machine_wq;
46static struct stop_machine_data active, idle; 46static struct stop_machine_data active, idle;
47static const struct cpumask *active_cpus; 47static const struct cpumask *active_cpus;
48static void *stop_machine_work; 48static void __percpu *stop_machine_work;
49 49
50static void set_state(enum stopmachine_state newstate) 50static void set_state(enum stopmachine_state newstate)
51{ 51{
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 8c1b2d290718..0287f9f52f5a 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -20,6 +20,7 @@
20#include <linux/cpu.h> 20#include <linux/cpu.h>
21#include <linux/fs.h> 21#include <linux/fs.h>
22 22
23#include <asm/local.h>
23#include "trace.h" 24#include "trace.h"
24 25
25/* 26/*
diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c
index b2477caf09c2..df74c7982255 100644
--- a/kernel/trace/ring_buffer_benchmark.c
+++ b/kernel/trace/ring_buffer_benchmark.c
@@ -8,6 +8,7 @@
8#include <linux/kthread.h> 8#include <linux/kthread.h>
9#include <linux/module.h> 9#include <linux/module.h>
10#include <linux/time.h> 10#include <linux/time.h>
11#include <asm/local.h>
11 12
12struct rb_page { 13struct rb_page {
13 u64 ts; 14 u64 ts;
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 032c57ca6502..ed01fdba4a55 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -92,12 +92,12 @@ DEFINE_PER_CPU(int, ftrace_cpu_disabled);
92static inline void ftrace_disable_cpu(void) 92static inline void ftrace_disable_cpu(void)
93{ 93{
94 preempt_disable(); 94 preempt_disable();
95 __this_cpu_inc(per_cpu_var(ftrace_cpu_disabled)); 95 __this_cpu_inc(ftrace_cpu_disabled);
96} 96}
97 97
98static inline void ftrace_enable_cpu(void) 98static inline void ftrace_enable_cpu(void)
99{ 99{
100 __this_cpu_dec(per_cpu_var(ftrace_cpu_disabled)); 100 __this_cpu_dec(ftrace_cpu_disabled);
101 preempt_enable(); 101 preempt_enable();
102} 102}
103 103
@@ -1166,7 +1166,7 @@ trace_function(struct trace_array *tr,
1166 struct ftrace_entry *entry; 1166 struct ftrace_entry *entry;
1167 1167
1168 /* If we are reading the ring buffer, don't trace */ 1168 /* If we are reading the ring buffer, don't trace */
1169 if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) 1169 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1170 return; 1170 return;
1171 1171
1172 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 1172 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index e998a824e9db..3fc2a575664f 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -188,7 +188,7 @@ static int __trace_graph_entry(struct trace_array *tr,
188 struct ring_buffer *buffer = tr->buffer; 188 struct ring_buffer *buffer = tr->buffer;
189 struct ftrace_graph_ent_entry *entry; 189 struct ftrace_graph_ent_entry *entry;
190 190
191 if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) 191 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
192 return 0; 192 return 0;
193 193
194 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, 194 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT,
@@ -247,7 +247,7 @@ static void __trace_graph_return(struct trace_array *tr,
247 struct ring_buffer *buffer = tr->buffer; 247 struct ring_buffer *buffer = tr->buffer;
248 struct ftrace_graph_ret_entry *entry; 248 struct ftrace_graph_ret_entry *entry;
249 249
250 if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) 250 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
251 return; 251 return;
252 252
253 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, 253 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET,
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 8deb9d0fd5b1..9a7aaae07ab4 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1009,10 +1009,10 @@ static void drain_pages(unsigned int cpu)
1009 struct per_cpu_pageset *pset; 1009 struct per_cpu_pageset *pset;
1010 struct per_cpu_pages *pcp; 1010 struct per_cpu_pages *pcp;
1011 1011
1012 pset = zone_pcp(zone, cpu); 1012 local_irq_save(flags);
1013 pset = per_cpu_ptr(zone->pageset, cpu);
1013 1014
1014 pcp = &pset->pcp; 1015 pcp = &pset->pcp;
1015 local_irq_save(flags);
1016 free_pcppages_bulk(zone, pcp->count, pcp); 1016 free_pcppages_bulk(zone, pcp->count, pcp);
1017 pcp->count = 0; 1017 pcp->count = 0;
1018 local_irq_restore(flags); 1018 local_irq_restore(flags);
@@ -1096,7 +1096,6 @@ static void free_hot_cold_page(struct page *page, int cold)
1096 arch_free_page(page, 0); 1096 arch_free_page(page, 0);
1097 kernel_map_pages(page, 1, 0); 1097 kernel_map_pages(page, 1, 0);
1098 1098
1099 pcp = &zone_pcp(zone, get_cpu())->pcp;
1100 migratetype = get_pageblock_migratetype(page); 1099 migratetype = get_pageblock_migratetype(page);
1101 set_page_private(page, migratetype); 1100 set_page_private(page, migratetype);
1102 local_irq_save(flags); 1101 local_irq_save(flags);
@@ -1119,6 +1118,7 @@ static void free_hot_cold_page(struct page *page, int cold)
1119 migratetype = MIGRATE_MOVABLE; 1118 migratetype = MIGRATE_MOVABLE;
1120 } 1119 }
1121 1120
1121 pcp = &this_cpu_ptr(zone->pageset)->pcp;
1122 if (cold) 1122 if (cold)
1123 list_add_tail(&page->lru, &pcp->lists[migratetype]); 1123 list_add_tail(&page->lru, &pcp->lists[migratetype]);
1124 else 1124 else
@@ -1131,7 +1131,6 @@ static void free_hot_cold_page(struct page *page, int cold)
1131 1131
1132out: 1132out:
1133 local_irq_restore(flags); 1133 local_irq_restore(flags);
1134 put_cpu();
1135} 1134}
1136 1135
1137void free_hot_page(struct page *page) 1136void free_hot_page(struct page *page)
@@ -1181,17 +1180,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
1181 unsigned long flags; 1180 unsigned long flags;
1182 struct page *page; 1181 struct page *page;
1183 int cold = !!(gfp_flags & __GFP_COLD); 1182 int cold = !!(gfp_flags & __GFP_COLD);
1184 int cpu;
1185 1183
1186again: 1184again:
1187 cpu = get_cpu();
1188 if (likely(order == 0)) { 1185 if (likely(order == 0)) {
1189 struct per_cpu_pages *pcp; 1186 struct per_cpu_pages *pcp;
1190 struct list_head *list; 1187 struct list_head *list;
1191 1188
1192 pcp = &zone_pcp(zone, cpu)->pcp;
1193 list = &pcp->lists[migratetype];
1194 local_irq_save(flags); 1189 local_irq_save(flags);
1190 pcp = &this_cpu_ptr(zone->pageset)->pcp;
1191 list = &pcp->lists[migratetype];
1195 if (list_empty(list)) { 1192 if (list_empty(list)) {
1196 pcp->count += rmqueue_bulk(zone, 0, 1193 pcp->count += rmqueue_bulk(zone, 0,
1197 pcp->batch, list, 1194 pcp->batch, list,
@@ -1232,7 +1229,6 @@ again:
1232 __count_zone_vm_events(PGALLOC, zone, 1 << order); 1229 __count_zone_vm_events(PGALLOC, zone, 1 << order);
1233 zone_statistics(preferred_zone, zone); 1230 zone_statistics(preferred_zone, zone);
1234 local_irq_restore(flags); 1231 local_irq_restore(flags);
1235 put_cpu();
1236 1232
1237 VM_BUG_ON(bad_range(zone, page)); 1233 VM_BUG_ON(bad_range(zone, page));
1238 if (prep_new_page(page, order, gfp_flags)) 1234 if (prep_new_page(page, order, gfp_flags))
@@ -1241,7 +1237,6 @@ again:
1241 1237
1242failed: 1238failed:
1243 local_irq_restore(flags); 1239 local_irq_restore(flags);
1244 put_cpu();
1245 return NULL; 1240 return NULL;
1246} 1241}
1247 1242
@@ -2180,7 +2175,7 @@ void show_free_areas(void)
2180 for_each_online_cpu(cpu) { 2175 for_each_online_cpu(cpu) {
2181 struct per_cpu_pageset *pageset; 2176 struct per_cpu_pageset *pageset;
2182 2177
2183 pageset = zone_pcp(zone, cpu); 2178 pageset = per_cpu_ptr(zone->pageset, cpu);
2184 2179
2185 printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n", 2180 printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
2186 cpu, pageset->pcp.high, 2181 cpu, pageset->pcp.high,
@@ -2745,10 +2740,29 @@ static void build_zonelist_cache(pg_data_t *pgdat)
2745 2740
2746#endif /* CONFIG_NUMA */ 2741#endif /* CONFIG_NUMA */
2747 2742
2743/*
2744 * Boot pageset table. One per cpu which is going to be used for all
2745 * zones and all nodes. The parameters will be set in such a way
2746 * that an item put on a list will immediately be handed over to
2747 * the buddy list. This is safe since pageset manipulation is done
2748 * with interrupts disabled.
2749 *
2750 * The boot_pagesets must be kept even after bootup is complete for
2751 * unused processors and/or zones. They do play a role for bootstrapping
2752 * hotplugged processors.
2753 *
2754 * zoneinfo_show() and maybe other functions do
2755 * not check if the processor is online before following the pageset pointer.
2756 * Other parts of the kernel may not check if the zone is available.
2757 */
2758static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
2759static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
2760
2748/* return values int ....just for stop_machine() */ 2761/* return values int ....just for stop_machine() */
2749static int __build_all_zonelists(void *dummy) 2762static int __build_all_zonelists(void *dummy)
2750{ 2763{
2751 int nid; 2764 int nid;
2765 int cpu;
2752 2766
2753#ifdef CONFIG_NUMA 2767#ifdef CONFIG_NUMA
2754 memset(node_load, 0, sizeof(node_load)); 2768 memset(node_load, 0, sizeof(node_load));
@@ -2759,6 +2773,23 @@ static int __build_all_zonelists(void *dummy)
2759 build_zonelists(pgdat); 2773 build_zonelists(pgdat);
2760 build_zonelist_cache(pgdat); 2774 build_zonelist_cache(pgdat);
2761 } 2775 }
2776
2777 /*
2778 * Initialize the boot_pagesets that are going to be used
2779 * for bootstrapping processors. The real pagesets for
2780 * each zone will be allocated later when the per cpu
2781 * allocator is available.
2782 *
2783 * boot_pagesets are used also for bootstrapping offline
2784 * cpus if the system is already booted because the pagesets
2785 * are needed to initialize allocators on a specific cpu too.
2786 * F.e. the percpu allocator needs the page allocator which
2787 * needs the percpu allocator in order to allocate its pagesets
2788 * (a chicken-egg dilemma).
2789 */
2790 for_each_possible_cpu(cpu)
2791 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
2792
2762 return 0; 2793 return 0;
2763} 2794}
2764 2795
@@ -3096,121 +3127,33 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p,
3096 pcp->batch = PAGE_SHIFT * 8; 3127 pcp->batch = PAGE_SHIFT * 8;
3097} 3128}
3098 3129
3099
3100#ifdef CONFIG_NUMA
3101/*
3102 * Boot pageset table. One per cpu which is going to be used for all
3103 * zones and all nodes. The parameters will be set in such a way
3104 * that an item put on a list will immediately be handed over to
3105 * the buddy list. This is safe since pageset manipulation is done
3106 * with interrupts disabled.
3107 *
3108 * Some NUMA counter updates may also be caught by the boot pagesets.
3109 *
3110 * The boot_pagesets must be kept even after bootup is complete for
3111 * unused processors and/or zones. They do play a role for bootstrapping
3112 * hotplugged processors.
3113 *
3114 * zoneinfo_show() and maybe other functions do
3115 * not check if the processor is online before following the pageset pointer.
3116 * Other parts of the kernel may not check if the zone is available.
3117 */
3118static struct per_cpu_pageset boot_pageset[NR_CPUS];
3119
3120/* 3130/*
3121 * Dynamically allocate memory for the 3131 * Allocate per cpu pagesets and initialize them.
3122 * per cpu pageset array in struct zone. 3132 * Before this call only boot pagesets were available.
3133 * Boot pagesets will no longer be used by this processorr
3134 * after setup_per_cpu_pageset().
3123 */ 3135 */
3124static int __cpuinit process_zones(int cpu) 3136void __init setup_per_cpu_pageset(void)
3125{ 3137{
3126 struct zone *zone, *dzone; 3138 struct zone *zone;
3127 int node = cpu_to_node(cpu); 3139 int cpu;
3128
3129 node_set_state(node, N_CPU); /* this node has a cpu */
3130 3140
3131 for_each_populated_zone(zone) { 3141 for_each_populated_zone(zone) {
3132 zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset), 3142 zone->pageset = alloc_percpu(struct per_cpu_pageset);
3133 GFP_KERNEL, node);
3134 if (!zone_pcp(zone, cpu))
3135 goto bad;
3136
3137 setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone));
3138
3139 if (percpu_pagelist_fraction)
3140 setup_pagelist_highmark(zone_pcp(zone, cpu),
3141 (zone->present_pages / percpu_pagelist_fraction));
3142 }
3143
3144 return 0;
3145bad:
3146 for_each_zone(dzone) {
3147 if (!populated_zone(dzone))
3148 continue;
3149 if (dzone == zone)
3150 break;
3151 kfree(zone_pcp(dzone, cpu));
3152 zone_pcp(dzone, cpu) = &boot_pageset[cpu];
3153 }
3154 return -ENOMEM;
3155}
3156 3143
3157static inline void free_zone_pagesets(int cpu) 3144 for_each_possible_cpu(cpu) {
3158{ 3145 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
3159 struct zone *zone;
3160
3161 for_each_zone(zone) {
3162 struct per_cpu_pageset *pset = zone_pcp(zone, cpu);
3163 3146
3164 /* Free per_cpu_pageset if it is slab allocated */ 3147 setup_pageset(pcp, zone_batchsize(zone));
3165 if (pset != &boot_pageset[cpu])
3166 kfree(pset);
3167 zone_pcp(zone, cpu) = &boot_pageset[cpu];
3168 }
3169}
3170 3148
3171static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb, 3149 if (percpu_pagelist_fraction)
3172 unsigned long action, 3150 setup_pagelist_highmark(pcp,
3173 void *hcpu) 3151 (zone->present_pages /
3174{ 3152 percpu_pagelist_fraction));
3175 int cpu = (long)hcpu; 3153 }
3176 int ret = NOTIFY_OK;
3177
3178 switch (action) {
3179 case CPU_UP_PREPARE:
3180 case CPU_UP_PREPARE_FROZEN:
3181 if (process_zones(cpu))
3182 ret = NOTIFY_BAD;
3183 break;
3184 case CPU_UP_CANCELED:
3185 case CPU_UP_CANCELED_FROZEN:
3186 case CPU_DEAD:
3187 case CPU_DEAD_FROZEN:
3188 free_zone_pagesets(cpu);
3189 break;
3190 default:
3191 break;
3192 } 3154 }
3193 return ret;
3194} 3155}
3195 3156
3196static struct notifier_block __cpuinitdata pageset_notifier =
3197 { &pageset_cpuup_callback, NULL, 0 };
3198
3199void __init setup_per_cpu_pageset(void)
3200{
3201 int err;
3202
3203 /* Initialize per_cpu_pageset for cpu 0.
3204 * A cpuup callback will do this for every cpu
3205 * as it comes online
3206 */
3207 err = process_zones(smp_processor_id());
3208 BUG_ON(err);
3209 register_cpu_notifier(&pageset_notifier);
3210}
3211
3212#endif
3213
3214static noinline __init_refok 3157static noinline __init_refok
3215int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) 3158int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
3216{ 3159{
@@ -3264,7 +3207,7 @@ static int __zone_pcp_update(void *data)
3264 struct per_cpu_pageset *pset; 3207 struct per_cpu_pageset *pset;
3265 struct per_cpu_pages *pcp; 3208 struct per_cpu_pages *pcp;
3266 3209
3267 pset = zone_pcp(zone, cpu); 3210 pset = per_cpu_ptr(zone->pageset, cpu);
3268 pcp = &pset->pcp; 3211 pcp = &pset->pcp;
3269 3212
3270 local_irq_save(flags); 3213 local_irq_save(flags);
@@ -3282,21 +3225,17 @@ void zone_pcp_update(struct zone *zone)
3282 3225
3283static __meminit void zone_pcp_init(struct zone *zone) 3226static __meminit void zone_pcp_init(struct zone *zone)
3284{ 3227{
3285 int cpu; 3228 /*
3286 unsigned long batch = zone_batchsize(zone); 3229 * per cpu subsystem is not up at this point. The following code
3230 * relies on the ability of the linker to provide the
3231 * offset of a (static) per cpu variable into the per cpu area.
3232 */
3233 zone->pageset = &boot_pageset;
3287 3234
3288 for (cpu = 0; cpu < NR_CPUS; cpu++) {
3289#ifdef CONFIG_NUMA
3290 /* Early boot. Slab allocator not functional yet */
3291 zone_pcp(zone, cpu) = &boot_pageset[cpu];
3292 setup_pageset(&boot_pageset[cpu],0);
3293#else
3294 setup_pageset(zone_pcp(zone,cpu), batch);
3295#endif
3296 }
3297 if (zone->present_pages) 3235 if (zone->present_pages)
3298 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", 3236 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
3299 zone->name, zone->present_pages, batch); 3237 zone->name, zone->present_pages,
3238 zone_batchsize(zone));
3300} 3239}
3301 3240
3302__meminit int init_currently_empty_zone(struct zone *zone, 3241__meminit int init_currently_empty_zone(struct zone *zone,
@@ -4810,10 +4749,11 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
4810 if (!write || (ret == -EINVAL)) 4749 if (!write || (ret == -EINVAL))
4811 return ret; 4750 return ret;
4812 for_each_populated_zone(zone) { 4751 for_each_populated_zone(zone) {
4813 for_each_online_cpu(cpu) { 4752 for_each_possible_cpu(cpu) {
4814 unsigned long high; 4753 unsigned long high;
4815 high = zone->present_pages / percpu_pagelist_fraction; 4754 high = zone->present_pages / percpu_pagelist_fraction;
4816 setup_pagelist_highmark(zone_pcp(zone, cpu), high); 4755 setup_pagelist_highmark(
4756 per_cpu_ptr(zone->pageset, cpu), high);
4817 } 4757 }
4818 } 4758 }
4819 return 0; 4759 return 0;
diff --git a/mm/percpu.c b/mm/percpu.c
index 083e7c91e5f6..768419d44ad7 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -80,13 +80,15 @@
80/* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */ 80/* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
81#ifndef __addr_to_pcpu_ptr 81#ifndef __addr_to_pcpu_ptr
82#define __addr_to_pcpu_ptr(addr) \ 82#define __addr_to_pcpu_ptr(addr) \
83 (void *)((unsigned long)(addr) - (unsigned long)pcpu_base_addr \ 83 (void __percpu *)((unsigned long)(addr) - \
84 + (unsigned long)__per_cpu_start) 84 (unsigned long)pcpu_base_addr + \
85 (unsigned long)__per_cpu_start)
85#endif 86#endif
86#ifndef __pcpu_ptr_to_addr 87#ifndef __pcpu_ptr_to_addr
87#define __pcpu_ptr_to_addr(ptr) \ 88#define __pcpu_ptr_to_addr(ptr) \
88 (void *)((unsigned long)(ptr) + (unsigned long)pcpu_base_addr \ 89 (void __force *)((unsigned long)(ptr) + \
89 - (unsigned long)__per_cpu_start) 90 (unsigned long)pcpu_base_addr - \
91 (unsigned long)__per_cpu_start)
90#endif 92#endif
91 93
92struct pcpu_chunk { 94struct pcpu_chunk {
@@ -913,11 +915,10 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
913 int rs, re; 915 int rs, re;
914 916
915 /* quick path, check whether it's empty already */ 917 /* quick path, check whether it's empty already */
916 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) { 918 rs = page_start;
917 if (rs == page_start && re == page_end) 919 pcpu_next_unpop(chunk, &rs, &re, page_end);
918 return; 920 if (rs == page_start && re == page_end)
919 break; 921 return;
920 }
921 922
922 /* immutable chunks can't be depopulated */ 923 /* immutable chunks can't be depopulated */
923 WARN_ON(chunk->immutable); 924 WARN_ON(chunk->immutable);
@@ -968,11 +969,10 @@ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
968 int rs, re, rc; 969 int rs, re, rc;
969 970
970 /* quick path, check whether all pages are already there */ 971 /* quick path, check whether all pages are already there */
971 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end) { 972 rs = page_start;
972 if (rs == page_start && re == page_end) 973 pcpu_next_pop(chunk, &rs, &re, page_end);
973 goto clear; 974 if (rs == page_start && re == page_end)
974 break; 975 goto clear;
975 }
976 976
977 /* need to allocate and map pages, this chunk can't be immutable */ 977 /* need to allocate and map pages, this chunk can't be immutable */
978 WARN_ON(chunk->immutable); 978 WARN_ON(chunk->immutable);
@@ -1067,7 +1067,7 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
1067 * RETURNS: 1067 * RETURNS:
1068 * Percpu pointer to the allocated area on success, NULL on failure. 1068 * Percpu pointer to the allocated area on success, NULL on failure.
1069 */ 1069 */
1070static void *pcpu_alloc(size_t size, size_t align, bool reserved) 1070static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
1071{ 1071{
1072 static int warn_limit = 10; 1072 static int warn_limit = 10;
1073 struct pcpu_chunk *chunk; 1073 struct pcpu_chunk *chunk;
@@ -1196,7 +1196,7 @@ fail_unlock_mutex:
1196 * RETURNS: 1196 * RETURNS:
1197 * Percpu pointer to the allocated area on success, NULL on failure. 1197 * Percpu pointer to the allocated area on success, NULL on failure.
1198 */ 1198 */
1199void *__alloc_percpu(size_t size, size_t align) 1199void __percpu *__alloc_percpu(size_t size, size_t align)
1200{ 1200{
1201 return pcpu_alloc(size, align, false); 1201 return pcpu_alloc(size, align, false);
1202} 1202}
@@ -1217,7 +1217,7 @@ EXPORT_SYMBOL_GPL(__alloc_percpu);
1217 * RETURNS: 1217 * RETURNS:
1218 * Percpu pointer to the allocated area on success, NULL on failure. 1218 * Percpu pointer to the allocated area on success, NULL on failure.
1219 */ 1219 */
1220void *__alloc_reserved_percpu(size_t size, size_t align) 1220void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
1221{ 1221{
1222 return pcpu_alloc(size, align, true); 1222 return pcpu_alloc(size, align, true);
1223} 1223}
@@ -1269,7 +1269,7 @@ static void pcpu_reclaim(struct work_struct *work)
1269 * CONTEXT: 1269 * CONTEXT:
1270 * Can be called from atomic context. 1270 * Can be called from atomic context.
1271 */ 1271 */
1272void free_percpu(void *ptr) 1272void free_percpu(void __percpu *ptr)
1273{ 1273{
1274 void *addr; 1274 void *addr;
1275 struct pcpu_chunk *chunk; 1275 struct pcpu_chunk *chunk;
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 6051fbab67ba..fc5aa183bc45 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -139,7 +139,8 @@ static void refresh_zone_stat_thresholds(void)
139 threshold = calculate_threshold(zone); 139 threshold = calculate_threshold(zone);
140 140
141 for_each_online_cpu(cpu) 141 for_each_online_cpu(cpu)
142 zone_pcp(zone, cpu)->stat_threshold = threshold; 142 per_cpu_ptr(zone->pageset, cpu)->stat_threshold
143 = threshold;
143 } 144 }
144} 145}
145 146
@@ -149,7 +150,8 @@ static void refresh_zone_stat_thresholds(void)
149void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 150void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
150 int delta) 151 int delta)
151{ 152{
152 struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); 153 struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
154
153 s8 *p = pcp->vm_stat_diff + item; 155 s8 *p = pcp->vm_stat_diff + item;
154 long x; 156 long x;
155 157
@@ -202,7 +204,7 @@ EXPORT_SYMBOL(mod_zone_page_state);
202 */ 204 */
203void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 205void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
204{ 206{
205 struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); 207 struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
206 s8 *p = pcp->vm_stat_diff + item; 208 s8 *p = pcp->vm_stat_diff + item;
207 209
208 (*p)++; 210 (*p)++;
@@ -223,7 +225,7 @@ EXPORT_SYMBOL(__inc_zone_page_state);
223 225
224void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 226void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
225{ 227{
226 struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); 228 struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
227 s8 *p = pcp->vm_stat_diff + item; 229 s8 *p = pcp->vm_stat_diff + item;
228 230
229 (*p)--; 231 (*p)--;
@@ -300,7 +302,7 @@ void refresh_cpu_vm_stats(int cpu)
300 for_each_populated_zone(zone) { 302 for_each_populated_zone(zone) {
301 struct per_cpu_pageset *p; 303 struct per_cpu_pageset *p;
302 304
303 p = zone_pcp(zone, cpu); 305 p = per_cpu_ptr(zone->pageset, cpu);
304 306
305 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 307 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
306 if (p->vm_stat_diff[i]) { 308 if (p->vm_stat_diff[i]) {
@@ -741,7 +743,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
741 for_each_online_cpu(i) { 743 for_each_online_cpu(i) {
742 struct per_cpu_pageset *pageset; 744 struct per_cpu_pageset *pageset;
743 745
744 pageset = zone_pcp(zone, i); 746 pageset = per_cpu_ptr(zone->pageset, i);
745 seq_printf(m, 747 seq_printf(m,
746 "\n cpu: %i" 748 "\n cpu: %i"
747 "\n count: %i" 749 "\n count: %i"
@@ -906,6 +908,7 @@ static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb,
906 case CPU_ONLINE: 908 case CPU_ONLINE:
907 case CPU_ONLINE_FROZEN: 909 case CPU_ONLINE_FROZEN:
908 start_cpu_timer(cpu); 910 start_cpu_timer(cpu);
911 node_set_state(cpu_to_node(cpu), N_CPU);
909 break; 912 break;
910 case CPU_DOWN_PREPARE: 913 case CPU_DOWN_PREPARE:
911 case CPU_DOWN_PREPARE_FROZEN: 914 case CPU_DOWN_PREPARE_FROZEN:
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-22 08:36:44 -0500