21 files changed, 122 insertions, 197 deletions

diff --git a/arch/alpha/include/asm/local.h b/arch/alpha/include/asm/local.h
index 6ad3ea696421..b9e3e3318371 100644
--- a/arch/alpha/include/asm/local.h
+++ b/arch/alpha/include/asm/local.h
@@ -98,21 +98,4 @@ static __inline__ long local_sub_return(long i, local_t * l)
 #define __local_add(i,l)        ((l)->a.counter+=(i))
 #define __local_sub(i,l)        ((l)->a.counter-=(i))
 
-/* Use these for per-cpu local_t variables: on some archs they are
- * much more efficient than these naive implementations.  Note they take
- * a variable, not an address.
- */
-#define cpu_local_read(l)       local_read(&__get_cpu_var(l))
-#define cpu_local_set(l, i)     local_set(&__get_cpu_var(l), (i))
-
-#define cpu_local_inc(l)        local_inc(&__get_cpu_var(l))
-#define cpu_local_dec(l)        local_dec(&__get_cpu_var(l))
-#define cpu_local_add(i, l)     local_add((i), &__get_cpu_var(l))
-#define cpu_local_sub(i, l)     local_sub((i), &__get_cpu_var(l))
-
-#define __cpu_local_inc(l)      __local_inc(&__get_cpu_var(l))
-#define __cpu_local_dec(l)      __local_dec(&__get_cpu_var(l))
-#define __cpu_local_add(i, l)   __local_add((i), &__get_cpu_var(l))
-#define __cpu_local_sub(i, l)   __local_sub((i), &__get_cpu_var(l))
-
 #endif /* _ALPHA_LOCAL_H */
diff --git a/arch/blackfin/mach-common/entry.S b/arch/blackfin/mach-common/entry.S
index b0ed0b487ff2..01b2f58dfb95 100644
--- a/arch/blackfin/mach-common/entry.S
+++ b/arch/blackfin/mach-common/entry.S
@@ -816,8 +816,8 @@ ENDPROC(_resume)
 
 ENTRY(_ret_from_exception)
 #ifdef CONFIG_IPIPE
-        p2.l = _per_cpu__ipipe_percpu_domain;
-        p2.h = _per_cpu__ipipe_percpu_domain;
+        p2.l = _ipipe_percpu_domain;
+        p2.h = _ipipe_percpu_domain;
         r0.l = _ipipe_root;
         r0.h = _ipipe_root;
         r2 = [p2];
diff --git a/arch/cris/arch-v10/kernel/entry.S b/arch/cris/arch-v10/kernel/entry.S
index 2c18d08cd913..c52bef39e250 100644
--- a/arch/cris/arch-v10/kernel/entry.S
+++ b/arch/cris/arch-v10/kernel/entry.S
@@ -358,7 +358,7 @@ mmu_bus_fault:
 1:      btstq   12, $r1            ; Refill?
         bpl     2f
         lsrq    24, $r1     ; Get PGD index (bit 24-31)
-        move.d  [per_cpu__current_pgd], $r0 ; PGD for the current process
+        move.d  [current_pgd], $r0 ; PGD for the current process
         move.d  [$r0+$r1.d], $r0   ; Get PMD
         beq     2f
         nop
diff --git a/arch/cris/arch-v32/mm/mmu.S b/arch/cris/arch-v32/mm/mmu.S
index 2238d154bde3..f125d912e140 100644
--- a/arch/cris/arch-v32/mm/mmu.S
+++ b/arch/cris/arch-v32/mm/mmu.S
@@ -115,7 +115,7 @@
 #ifdef CONFIG_SMP
         move    $s7, $acr       ; PGD
 #else
-        move.d  per_cpu__current_pgd, $acr ; PGD
+        move.d  current_pgd, $acr ; PGD
 #endif
         ; Look up PMD in PGD
         lsrq    24, $r0 ; Get PMD index into PGD (bit 24-31)
diff --git a/arch/ia64/include/asm/percpu.h b/arch/ia64/include/asm/percpu.h
index 30cf46534dd2..f7c00a5e0e2b 100644
--- a/arch/ia64/include/asm/percpu.h
+++ b/arch/ia64/include/asm/percpu.h
@@ -9,7 +9,7 @@
 #define PERCPU_ENOUGH_ROOM PERCPU_PAGE_SIZE
 
 #ifdef __ASSEMBLY__
-# define THIS_CPU(var)  (per_cpu__##var)  /* use this to mark accesses to per-CPU variables... */
+# define THIS_CPU(var)  (var)  /* use this to mark accesses to per-CPU variables... */
 #else /* !__ASSEMBLY__ */
 
 
@@ -39,7 +39,7 @@ extern void *per_cpu_init(void);
  * On the positive side, using __ia64_per_cpu_var() instead of __get_cpu_var() is slightly
  * more efficient.
  */
-#define __ia64_per_cpu_var(var) per_cpu__##var
+#define __ia64_per_cpu_var(var) var
 
 #include <asm-generic/percpu.h>
 
diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c
index 461b99902bf6..7f4a0ed24152 100644
--- a/arch/ia64/kernel/ia64_ksyms.c
+++ b/arch/ia64/kernel/ia64_ksyms.c
@@ -30,9 +30,9 @@ EXPORT_SYMBOL(max_low_pfn);	/* defined by bootmem.c, but not exported by generic
 #endif
 
 #include <asm/processor.h>
-EXPORT_SYMBOL(per_cpu__ia64_cpu_info);
+EXPORT_SYMBOL(ia64_cpu_info);
 #ifdef CONFIG_SMP
-EXPORT_SYMBOL(per_cpu__local_per_cpu_offset);
+EXPORT_SYMBOL(local_per_cpu_offset);
 #endif
 
 #include <asm/uaccess.h>
diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c
index 19c4b2195dce..8d586d1e2515 100644
--- a/arch/ia64/mm/discontig.c
+++ b/arch/ia64/mm/discontig.c
@@ -459,7 +459,7 @@ static void __init initialize_pernode_data(void)
                 cpu = 0;
                 node = node_cpuid[cpu].nid;
                 cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start +
-                        ((char *)&per_cpu__ia64_cpu_info - __per_cpu_start));
+                        ((char *)&ia64_cpu_info - __per_cpu_start));
                 cpu0_cpu_info->node_data = mem_data[node].node_data;
         }
 #endif /* CONFIG_SMP */
diff --git a/arch/m32r/include/asm/local.h b/arch/m32r/include/asm/local.h
index 22256d138630..734bca87018a 100644
--- a/arch/m32r/include/asm/local.h
+++ b/arch/m32r/include/asm/local.h
@@ -338,29 +338,4 @@ static inline void local_set_mask(unsigned long  mask, local_t *addr)
  * a variable, not an address.
  */
 
-/* Need to disable preemption for the cpu local counters otherwise we could
-   still access a variable of a previous CPU in a non local way. */
-#define cpu_local_wrap_v(l)             \
-        ({ local_t res__;               \
-           preempt_disable();           \
-           res__ = (l);                 \
-           preempt_enable();            \
-           res__; })
-#define cpu_local_wrap(l)               \
-        ({ preempt_disable();           \
-           l;                           \
-           preempt_enable(); })         \
-
-#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
-#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
-#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var(l)))
-#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var(l)))
-#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
-#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
-
-#define __cpu_local_inc(l)      cpu_local_inc(l)
-#define __cpu_local_dec(l)      cpu_local_dec(l)
-#define __cpu_local_add(i, l)   cpu_local_add((i), (l))
-#define __cpu_local_sub(i, l)   cpu_local_sub((i), (l))
-
 #endif /* __M32R_LOCAL_H */
diff --git a/arch/microblaze/include/asm/entry.h b/arch/microblaze/include/asm/entry.h
index 61abbd232640..ec89f2ad0fe1 100644
--- a/arch/microblaze/include/asm/entry.h
+++ b/arch/microblaze/include/asm/entry.h
@@ -21,7 +21,7 @@
  * places
  */
 
-#define PER_CPU(var) per_cpu__##var
+#define PER_CPU(var) var
 
 # ifndef __ASSEMBLY__
 DECLARE_PER_CPU(unsigned int, KSP); /* Saved kernel stack pointer */
diff --git a/arch/mips/include/asm/local.h b/arch/mips/include/asm/local.h
index 361f4f16c30c..bdcdef02d147 100644
--- a/arch/mips/include/asm/local.h
+++ b/arch/mips/include/asm/local.h
@@ -193,29 +193,4 @@ static __inline__ long local_sub_return(long i, local_t * l)
 #define __local_add(i, l)       ((l)->a.counter+=(i))
 #define __local_sub(i, l)       ((l)->a.counter-=(i))
 
-/* Need to disable preemption for the cpu local counters otherwise we could
-   still access a variable of a previous CPU in a non atomic way. */
-#define cpu_local_wrap_v(l)             \
-        ({ local_t res__;               \
-           preempt_disable();           \
-           res__ = (l);                 \
-           preempt_enable();            \
-           res__; })
-#define cpu_local_wrap(l)               \
-        ({ preempt_disable();           \
-           l;                           \
-           preempt_enable(); })         \
-
-#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
-#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
-#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var(l)))
-#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var(l)))
-#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
-#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
-
-#define __cpu_local_inc(l)      cpu_local_inc(l)
-#define __cpu_local_dec(l)      cpu_local_dec(l)
-#define __cpu_local_add(i, l)   cpu_local_add((i), (l))
-#define __cpu_local_sub(i, l)   cpu_local_sub((i), (l))
-
 #endif /* _ARCH_MIPS_LOCAL_H */
diff --git a/arch/parisc/lib/fixup.S b/arch/parisc/lib/fixup.S
index d172d4245cdc..f8c45cc2947d 100644
--- a/arch/parisc/lib/fixup.S
+++ b/arch/parisc/lib/fixup.S
@@ -36,8 +36,8 @@
 #endif
         /* t2 = &__per_cpu_offset[smp_processor_id()]; */
         LDREGX \t2(\t1),\t2 
-        addil LT%per_cpu__exception_data,%r27
-        LDREG RT%per_cpu__exception_data(%r1),\t1
+        addil LT%exception_data,%r27
+        LDREG RT%exception_data(%r1),\t1
         /* t1 = &__get_cpu_var(exception_data) */
         add,l \t1,\t2,\t1
         /* t1 = t1->fault_ip */
@@ -46,8 +46,8 @@
 #else
         .macro  get_fault_ip t1 t2
         /* t1 = &__get_cpu_var(exception_data) */
-        addil LT%per_cpu__exception_data,%r27
-        LDREG RT%per_cpu__exception_data(%r1),\t2
+        addil LT%exception_data,%r27
+        LDREG RT%exception_data(%r1),\t2
         /* t1 = t2->fault_ip */
         LDREG EXCDATA_IP(\t2), \t1
         .endm
diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h
index 84b457a3c1bc..227753d288f6 100644
--- a/arch/powerpc/include/asm/local.h
+++ b/arch/powerpc/include/asm/local.h
@@ -172,29 +172,4 @@ static __inline__ long local_dec_if_positive(local_t *l)
 #define __local_add(i,l)        ((l)->a.counter+=(i))
 #define __local_sub(i,l)        ((l)->a.counter-=(i))
 
-/* Need to disable preemption for the cpu local counters otherwise we could
-   still access a variable of a previous CPU in a non atomic way. */
-#define cpu_local_wrap_v(l)             \
-        ({ local_t res__;               \
-           preempt_disable();           \
-           res__ = (l);                 \
-           preempt_enable();            \
-           res__; })
-#define cpu_local_wrap(l)               \
-        ({ preempt_disable();           \
-           l;                           \
-           preempt_enable(); })         \
-
-#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
-#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
-#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var(l)))
-#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var(l)))
-#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
-#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
-
-#define __cpu_local_inc(l)      cpu_local_inc(l)
-#define __cpu_local_dec(l)      cpu_local_dec(l)
-#define __cpu_local_add(i, l)   cpu_local_add((i), (l))
-#define __cpu_local_sub(i, l)   cpu_local_sub((i), (l))
-
 #endif /* _ARCH_POWERPC_LOCAL_H */
diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c
index d242a7340541..b287b62c7ea3 100644
--- a/arch/sparc/kernel/nmi.c
+++ b/arch/sparc/kernel/nmi.c
@@ -21,7 +21,6 @@
 
 #include <asm/perf_event.h>
 #include <asm/ptrace.h>
-#include <asm/local.h>
 #include <asm/pcr.h>
 
 /* We don't have a real NMI on sparc64, but we can fake one
@@ -113,13 +112,13 @@ notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs)
                 touched = 1;
         }
         if (!touched && __get_cpu_var(last_irq_sum) == sum) {
-                __this_cpu_inc(per_cpu_var(alert_counter));
-                if (__this_cpu_read(per_cpu_var(alert_counter)) == 30 * nmi_hz)
+                __this_cpu_inc(alert_counter);
+                if (__this_cpu_read(alert_counter) == 30 * nmi_hz)
                         die_nmi("BUG: NMI Watchdog detected LOCKUP",
                                 regs, panic_on_timeout);
         } else {
                 __get_cpu_var(last_irq_sum) = sum;
-                __this_cpu_write(per_cpu_var(alert_counter), 0);
+                __this_cpu_write(alert_counter, 0);
         }
         if (__get_cpu_var(wd_enabled)) {
                 write_pic(picl_value(nmi_hz));
diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S
index fd3cee4d117c..1ddec403f512 100644
--- a/arch/sparc/kernel/rtrap_64.S
+++ b/arch/sparc/kernel/rtrap_64.S
@@ -149,11 +149,11 @@ rtrap_nmi:	ldx			[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
 rtrap_irq:
 rtrap:
 #ifndef CONFIG_SMP
-                sethi                   %hi(per_cpu____cpu_data), %l0
-                lduw                    [%l0 + %lo(per_cpu____cpu_data)], %l1
+                sethi                   %hi(__cpu_data), %l0
+                lduw                    [%l0 + %lo(__cpu_data)], %l1
 #else
-                sethi                   %hi(per_cpu____cpu_data), %l0
-                or                      %l0, %lo(per_cpu____cpu_data), %l0
+                sethi                   %hi(__cpu_data), %l0
+                or                      %l0, %lo(__cpu_data), %l0
                 lduw                    [%l0 + %g5], %l1
 #endif
                 cmp                     %l1, 0
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 47b9b6f19057..2e9972468a5d 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -195,41 +195,4 @@ static inline long local_sub_return(long i, local_t *l)
 #define __local_add(i, l)       local_add((i), (l))
 #define __local_sub(i, l)       local_sub((i), (l))
 
-/* Use these for per-cpu local_t variables: on some archs they are
- * much more efficient than these naive implementations.  Note they take
- * a variable, not an address.
- *
- * X86_64: This could be done better if we moved the per cpu data directly
- * after GS.
- */
-
-/* Need to disable preemption for the cpu local counters otherwise we could
-   still access a variable of a previous CPU in a non atomic way. */
-#define cpu_local_wrap_v(l)             \
-({                                      \
-        local_t res__;                  \
-        preempt_disable();              \
-        res__ = (l);                    \
-        preempt_enable();               \
-        res__;                          \
-})
-#define cpu_local_wrap(l)               \
-({                                      \
-        preempt_disable();              \
-        (l);                            \
-        preempt_enable();               \
-})                                      \
-
-#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var((l))))
-#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var((l)), (i)))
-#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var((l))))
-#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var((l))))
-#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var((l))))
-#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var((l))))
-
-#define __cpu_local_inc(l)      cpu_local_inc((l))
-#define __cpu_local_dec(l)      cpu_local_dec((l))
-#define __cpu_local_add(i, l)   cpu_local_add((i), (l))
-#define __cpu_local_sub(i, l)   cpu_local_sub((i), (l))
-
 #endif /* _ASM_X86_LOCAL_H */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 0c44196b78ac..66a272dfd8b8 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -25,19 +25,18 @@
  */
 #ifdef CONFIG_SMP
 #define PER_CPU(var, reg)                                               \
-        __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg;       \
-        lea per_cpu__##var(reg), reg
-#define PER_CPU_VAR(var)        %__percpu_seg:per_cpu__##var
+        __percpu_mov_op %__percpu_seg:this_cpu_off, reg;                \
+        lea var(reg), reg
+#define PER_CPU_VAR(var)        %__percpu_seg:var
 #else /* ! SMP */
-#define PER_CPU(var, reg)                                               \
-        __percpu_mov_op $per_cpu__##var, reg
-#define PER_CPU_VAR(var)        per_cpu__##var
+#define PER_CPU(var, reg)       __percpu_mov_op $var, reg
+#define PER_CPU_VAR(var)        var
 #endif  /* SMP */
 
 #ifdef CONFIG_X86_64_SMP
 #define INIT_PER_CPU_VAR(var)  init_per_cpu__##var
 #else
-#define INIT_PER_CPU_VAR(var)  per_cpu__##var
+#define INIT_PER_CPU_VAR(var)  var
 #endif
 
 #else /* ...!ASSEMBLY */
@@ -60,12 +59,12 @@
  * There also must be an entry in vmlinux_64.lds.S
  */
 #define DECLARE_INIT_PER_CPU(var) \
-       extern typeof(per_cpu_var(var)) init_per_cpu_var(var)
+       extern typeof(var) init_per_cpu_var(var)
 
 #ifdef CONFIG_X86_64_SMP
 #define init_per_cpu_var(var)  init_per_cpu__##var
 #else
-#define init_per_cpu_var(var)  per_cpu_var(var)
+#define init_per_cpu_var(var)  var
 #endif
 
 /* For arch-specific code, we can use direct single-insn ops (they
@@ -104,6 +103,64 @@ do {							\
         }                                               \
 } while (0)
 
+/*
+ * Generate a percpu add to memory instruction and optimize code
+ * if a one is added or subtracted.
+ */
+#define percpu_add_op(var, val)                                         \
+do {                                                                    \
+        typedef typeof(var) pao_T__;                                    \
+        const int pao_ID__ = (__builtin_constant_p(val) &&              \
+                              ((val) == 1 || (val) == -1)) ? (val) : 0; \
+        if (0) {                                                        \
+                pao_T__ pao_tmp__;                                      \
+                pao_tmp__ = (val);                                      \
+        }                                                               \
+        switch (sizeof(var)) {                                          \
+        case 1:                                                         \
+                if (pao_ID__ == 1)                                      \
+                        asm("incb "__percpu_arg(0) : "+m" (var));       \
+                else if (pao_ID__ == -1)                                \
+                        asm("decb "__percpu_arg(0) : "+m" (var));       \
+                else                                                    \
+                        asm("addb %1, "__percpu_arg(0)                  \
+                            : "+m" (var)                                \
+                            : "qi" ((pao_T__)(val)));                   \
+                break;                                                  \
+        case 2:                                                         \
+                if (pao_ID__ == 1)                                      \
+                        asm("incw "__percpu_arg(0) : "+m" (var));       \
+                else if (pao_ID__ == -1)                                \
+                        asm("decw "__percpu_arg(0) : "+m" (var));       \
+                else                                                    \
+                        asm("addw %1, "__percpu_arg(0)                  \
+                            : "+m" (var)                                \
+                            : "ri" ((pao_T__)(val)));                   \
+                break;                                                  \
+        case 4:                                                         \
+                if (pao_ID__ == 1)                                      \
+                        asm("incl "__percpu_arg(0) : "+m" (var));       \
+                else if (pao_ID__ == -1)                                \
+                        asm("decl "__percpu_arg(0) : "+m" (var));       \
+                else                                                    \
+                        asm("addl %1, "__percpu_arg(0)                  \
+                            : "+m" (var)                                \
+                            : "ri" ((pao_T__)(val)));                   \
+                break;                                                  \
+        case 8:                                                         \
+                if (pao_ID__ == 1)                                      \
+                        asm("incq "__percpu_arg(0) : "+m" (var));       \
+                else if (pao_ID__ == -1)                                \
+                        asm("decq "__percpu_arg(0) : "+m" (var));       \
+                else                                                    \
+                        asm("addq %1, "__percpu_arg(0)                  \
+                            : "+m" (var)                                \
+                            : "re" ((pao_T__)(val)));                   \
+                break;                                                  \
+        default: __bad_percpu_size();                                   \
+        }                                                               \
+} while (0)
+
 #define percpu_from_op(op, var, constraint)             \
 ({                                                      \
         typeof(var) pfo_ret__;                          \
@@ -142,16 +199,14 @@ do {							\
  * per-thread variables implemented as per-cpu variables and thus
  * stable for the duration of the respective task.
  */
-#define percpu_read(var)        percpu_from_op("mov", per_cpu__##var,   \
-                                               "m" (per_cpu__##var))
-#define percpu_read_stable(var) percpu_from_op("mov", per_cpu__##var,   \
-                                               "p" (&per_cpu__##var))
-#define percpu_write(var, val)  percpu_to_op("mov", per_cpu__##var, val)
-#define percpu_add(var, val)    percpu_to_op("add", per_cpu__##var, val)
-#define percpu_sub(var, val)    percpu_to_op("sub", per_cpu__##var, val)
-#define percpu_and(var, val)    percpu_to_op("and", per_cpu__##var, val)
-#define percpu_or(var, val)     percpu_to_op("or", per_cpu__##var, val)
-#define percpu_xor(var, val)    percpu_to_op("xor", per_cpu__##var, val)
+#define percpu_read(var)                percpu_from_op("mov", var, "m" (var))
+#define percpu_read_stable(var)         percpu_from_op("mov", var, "p" (&(var)))
+#define percpu_write(var, val)          percpu_to_op("mov", var, val)
+#define percpu_add(var, val)            percpu_add_op(var, val)
+#define percpu_sub(var, val)            percpu_add_op(var, -(val))
+#define percpu_and(var, val)            percpu_to_op("and", var, val)
+#define percpu_or(var, val)             percpu_to_op("or", var, val)
+#define percpu_xor(var, val)            percpu_to_op("xor", var, val)
 
 #define __this_cpu_read_1(pcp)          percpu_from_op("mov", (pcp), "m"(pcp))
 #define __this_cpu_read_2(pcp)          percpu_from_op("mov", (pcp), "m"(pcp))
@@ -160,9 +215,9 @@ do {							\
 #define __this_cpu_write_1(pcp, val)    percpu_to_op("mov", (pcp), val)
 #define __this_cpu_write_2(pcp, val)    percpu_to_op("mov", (pcp), val)
 #define __this_cpu_write_4(pcp, val)    percpu_to_op("mov", (pcp), val)
-#define __this_cpu_add_1(pcp, val)      percpu_to_op("add", (pcp), val)
-#define __this_cpu_add_2(pcp, val)      percpu_to_op("add", (pcp), val)
-#define __this_cpu_add_4(pcp, val)      percpu_to_op("add", (pcp), val)
+#define __this_cpu_add_1(pcp, val)      percpu_add_op((pcp), val)
+#define __this_cpu_add_2(pcp, val)      percpu_add_op((pcp), val)
+#define __this_cpu_add_4(pcp, val)      percpu_add_op((pcp), val)
 #define __this_cpu_and_1(pcp, val)      percpu_to_op("and", (pcp), val)
 #define __this_cpu_and_2(pcp, val)      percpu_to_op("and", (pcp), val)
 #define __this_cpu_and_4(pcp, val)      percpu_to_op("and", (pcp), val)
@@ -179,9 +234,9 @@ do {							\
 #define this_cpu_write_1(pcp, val)      percpu_to_op("mov", (pcp), val)
 #define this_cpu_write_2(pcp, val)      percpu_to_op("mov", (pcp), val)
 #define this_cpu_write_4(pcp, val)      percpu_to_op("mov", (pcp), val)
-#define this_cpu_add_1(pcp, val)        percpu_to_op("add", (pcp), val)
-#define this_cpu_add_2(pcp, val)        percpu_to_op("add", (pcp), val)
-#define this_cpu_add_4(pcp, val)        percpu_to_op("add", (pcp), val)
+#define this_cpu_add_1(pcp, val)        percpu_add_op((pcp), val)
+#define this_cpu_add_2(pcp, val)        percpu_add_op((pcp), val)
+#define this_cpu_add_4(pcp, val)        percpu_add_op((pcp), val)
 #define this_cpu_and_1(pcp, val)        percpu_to_op("and", (pcp), val)
 #define this_cpu_and_2(pcp, val)        percpu_to_op("and", (pcp), val)
 #define this_cpu_and_4(pcp, val)        percpu_to_op("and", (pcp), val)
@@ -192,9 +247,9 @@ do {							\
 #define this_cpu_xor_2(pcp, val)        percpu_to_op("xor", (pcp), val)
 #define this_cpu_xor_4(pcp, val)        percpu_to_op("xor", (pcp), val)
 
-#define irqsafe_cpu_add_1(pcp, val)     percpu_to_op("add", (pcp), val)
-#define irqsafe_cpu_add_2(pcp, val)     percpu_to_op("add", (pcp), val)
-#define irqsafe_cpu_add_4(pcp, val)     percpu_to_op("add", (pcp), val)
+#define irqsafe_cpu_add_1(pcp, val)     percpu_add_op((pcp), val)
+#define irqsafe_cpu_add_2(pcp, val)     percpu_add_op((pcp), val)
+#define irqsafe_cpu_add_4(pcp, val)     percpu_add_op((pcp), val)
 #define irqsafe_cpu_and_1(pcp, val)     percpu_to_op("and", (pcp), val)
 #define irqsafe_cpu_and_2(pcp, val)     percpu_to_op("and", (pcp), val)
 #define irqsafe_cpu_and_4(pcp, val)     percpu_to_op("and", (pcp), val)
@@ -212,19 +267,19 @@ do {							\
 #ifdef CONFIG_X86_64
 #define __this_cpu_read_8(pcp)          percpu_from_op("mov", (pcp), "m"(pcp))
 #define __this_cpu_write_8(pcp, val)    percpu_to_op("mov", (pcp), val)
-#define __this_cpu_add_8(pcp, val)      percpu_to_op("add", (pcp), val)
+#define __this_cpu_add_8(pcp, val)      percpu_add_op((pcp), val)
 #define __this_cpu_and_8(pcp, val)      percpu_to_op("and", (pcp), val)
 #define __this_cpu_or_8(pcp, val)       percpu_to_op("or", (pcp), val)
 #define __this_cpu_xor_8(pcp, val)      percpu_to_op("xor", (pcp), val)
 
 #define this_cpu_read_8(pcp)            percpu_from_op("mov", (pcp), "m"(pcp))
 #define this_cpu_write_8(pcp, val)      percpu_to_op("mov", (pcp), val)
-#define this_cpu_add_8(pcp, val)        percpu_to_op("add", (pcp), val)
+#define this_cpu_add_8(pcp, val)        percpu_add_op((pcp), val)
 #define this_cpu_and_8(pcp, val)        percpu_to_op("and", (pcp), val)
 #define this_cpu_or_8(pcp, val)         percpu_to_op("or", (pcp), val)
 #define this_cpu_xor_8(pcp, val)        percpu_to_op("xor", (pcp), val)
 
-#define irqsafe_cpu_add_8(pcp, val)     percpu_to_op("add", (pcp), val)
+#define irqsafe_cpu_add_8(pcp, val)     percpu_add_op((pcp), val)
 #define irqsafe_cpu_and_8(pcp, val)     percpu_to_op("and", (pcp), val)
 #define irqsafe_cpu_or_8(pcp, val)      percpu_to_op("or", (pcp), val)
 #define irqsafe_cpu_xor_8(pcp, val)     percpu_to_op("xor", (pcp), val)
@@ -236,7 +291,7 @@ do {							\
 ({                                                                      \
         int old__;                                                      \
         asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0"           \
-                     : "=r" (old__), "+m" (per_cpu__##var)              \
+                     : "=r" (old__), "+m" (var)                         \
                      : "dIr" (bit));                                    \
         old__;                                                          \
 })
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index ecb544e65382..e529f26c3292 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -32,7 +32,7 @@ extern void show_regs_common(void);
         "movl %P[task_canary](%[next]), %%ebx\n\t"                      \
         "movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
 #define __switch_canary_oparam                                          \
-        , [stack_canary] "=m" (per_cpu_var(stack_canary.canary))
+        , [stack_canary] "=m" (stack_canary.canary)
 #define __switch_canary_iparam                                          \
         , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
 #else   /* CC_STACKPROTECTOR */
@@ -114,7 +114,7 @@ do {									\
         "movq %P[task_canary](%%rsi),%%r8\n\t"                            \
         "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
 #define __switch_canary_oparam                                            \
-        , [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary))
+        , [gs_canary] "=m" (irq_stack_union.stack_canary)
 #define __switch_canary_iparam                                            \
         , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
 #else   /* CC_STACKPROTECTOR */
@@ -133,7 +133,7 @@ do {									\
              __switch_canary                                              \
              "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
              "movq %%rax,%%rdi\n\t"                                       \
-             "testl  %[_tif_fork],%P[ti_flags](%%r8)\n\t"         \
+             "testl  %[_tif_fork],%P[ti_flags](%%r8)\n\t"                 \
              "jnz   ret_from_fork\n\t"                                    \
              RESTORE_CONTEXT                                              \
              : "=a" (last)                                                \
@@ -143,7 +143,7 @@ do {									\
                [ti_flags] "i" (offsetof(struct thread_info, flags)),      \
                [_tif_fork] "i" (_TIF_FORK),                               \
                [thread_info] "i" (offsetof(struct task_struct, stack)),   \
-               [current_task] "m" (per_cpu_var(current_task))             \
+               [current_task] "m" (current_task)                          \
                __switch_canary_iparam                                     \
              : "memory", "cc" __EXTRA_CLOBBER)
 #endif
diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c
index 0159a69396cb..4ada42c3dabb 100644
--- a/arch/x86/kernel/apic/nmi.c
+++ b/arch/x86/kernel/apic/nmi.c
@@ -438,8 +438,8 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
                  * Ayiee, looks like this CPU is stuck ...
                  * wait a few IRQs (5 seconds) before doing the oops ...
                  */
-                __this_cpu_inc(per_cpu_var(alert_counter));
-                if (__this_cpu_read(per_cpu_var(alert_counter)) == 5 * nmi_hz)
+                __this_cpu_inc(alert_counter);
+                if (__this_cpu_read(alert_counter) == 5 * nmi_hz)
                         /*
                          * die_nmi will return ONLY if NOTIFY_STOP happens..
                          */
@@ -447,7 +447,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
                                 regs, panic_on_timeout);
         } else {
                 __get_cpu_var(last_irq_sum) = sum;
-                __this_cpu_write(per_cpu_var(alert_counter), 0);
+                __this_cpu_write(alert_counter, 0);
         }
 
         /* see if the nmi watchdog went off */
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 7fd318bac59c..37c3d4b17d85 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -442,8 +442,8 @@ is386:	movl $2,%ecx		# set MP
          */
         cmpb $0,ready
         jne 1f
-        movl $per_cpu__gdt_page,%eax
-        movl $per_cpu__stack_canary,%ecx
+        movl $gdt_page,%eax
+        movl $stack_canary,%ecx
         movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
         shrl $16, %ecx
         movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
@@ -706,7 +706,7 @@ idt_descr:
         .word 0                         # 32 bit align gdt_desc.address
 ENTRY(early_gdt_descr)
         .word GDT_ENTRIES*8-1
-        .long per_cpu__gdt_page         /* Overwritten for secondary CPUs */
+        .long gdt_page                  /* Overwritten for secondary CPUs */
 
 /*
  * The boot_gdt must mirror the equivalent in setup.S and is
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index f92a0da608cb..44879df55696 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -341,7 +341,7 @@ SECTIONS
  * Per-cpu symbols which need to be offset from __per_cpu_load
  * for the boot processor.
  */
-#define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load
+#define INIT_PER_CPU(x) init_per_cpu__##x = x + __per_cpu_load
 INIT_PER_CPU(gdt_page);
 INIT_PER_CPU(irq_stack_union);
 
@@ -352,7 +352,7 @@ INIT_PER_CPU(irq_stack_union);
            "kernel image bigger than KERNEL_IMAGE_SIZE");
 
 #ifdef CONFIG_SMP
-. = ASSERT((per_cpu__irq_stack_union == 0),
+. = ASSERT((irq_stack_union == 0),
            "irq_stack_union is not at start of per-cpu area");
 #endif
 
diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S
index 88e15deb8b82..22a2093b5862 100644
--- a/arch/x86/xen/xen-asm_32.S
+++ b/arch/x86/xen/xen-asm_32.S
@@ -90,9 +90,9 @@ ENTRY(xen_iret)
         GET_THREAD_INFO(%eax)
         movl TI_cpu(%eax), %eax
         movl __per_cpu_offset(,%eax,4), %eax
-        mov per_cpu__xen_vcpu(%eax), %eax
+        mov xen_vcpu(%eax), %eax
 #else
-        movl per_cpu__xen_vcpu, %eax
+        movl xen_vcpu, %eax
 #endif
 
         /* check IF state we're restoring */