percpu: add optimized generic percpu accessors

It is an optimization and a cleanup, and adds the following new
generic percpu methods:

  percpu_read()
  percpu_write()
  percpu_add()
  percpu_sub()
  percpu_and()
  percpu_or()
  percpu_xor()

and implements support for them on x86. (other architectures will fall
back to a default implementation)

The advantage is that for example to read a local percpu variable,
instead of this sequence:

 return __get_cpu_var(var);

 ffffffff8102ca2b:	48 8b 14 fd 80 09 74 	mov    -0x7e8bf680(,%rdi,8),%rdx
 ffffffff8102ca32:	81
 ffffffff8102ca33:	48 c7 c0 d8 59 00 00 	mov    $0x59d8,%rax
 ffffffff8102ca3a:	48 8b 04 10          	mov    (%rax,%rdx,1),%rax

We can get a single instruction by using the optimized variants:

 return percpu_read(var);

 ffffffff8102ca3f:	65 48 8b 05 91 8f fd 	mov    %gs:0x7efd8f91(%rip),%rax

I also cleaned up the x86-specific APIs and made the x86 code use
these new generic percpu primitives.

tj: * fixed generic percpu_sub() definition as Roel Kluin pointed out
    * added percpu_and() for completeness's sake
    * made generic percpu ops atomic against preemption

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Tejun Heo <tj@kernel.org>

5 files changed, 14 insertions, 14 deletions

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 312414ef9365..75b94139e1f2 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -695,17 +695,17 @@ static void xen_write_cr0(unsigned long cr0)
 
 static void xen_write_cr2(unsigned long cr2)
 {
-        x86_read_percpu(xen_vcpu)->arch.cr2 = cr2;
+        percpu_read(xen_vcpu)->arch.cr2 = cr2;
 }
 
 static unsigned long xen_read_cr2(void)
 {
-        return x86_read_percpu(xen_vcpu)->arch.cr2;
+        return percpu_read(xen_vcpu)->arch.cr2;
 }
 
 static unsigned long xen_read_cr2_direct(void)
 {
-        return x86_read_percpu(xen_vcpu_info.arch.cr2);
+        return percpu_read(xen_vcpu_info.arch.cr2);
 }
 
 static void xen_write_cr4(unsigned long cr4)
@@ -718,12 +718,12 @@ static void xen_write_cr4(unsigned long cr4)
 
 static unsigned long xen_read_cr3(void)
 {
-        return x86_read_percpu(xen_cr3);
+        return percpu_read(xen_cr3);
 }
 
 static void set_current_cr3(void *v)
 {
-        x86_write_percpu(xen_current_cr3, (unsigned long)v);
+        percpu_write(xen_current_cr3, (unsigned long)v);
 }
 
 static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -748,7 +748,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
 
         if (kernel) {
-                x86_write_percpu(xen_cr3, cr3);
+                percpu_write(xen_cr3, cr3);
 
                 /* Update xen_current_cr3 once the batch has actually
                    been submitted. */
@@ -764,7 +764,7 @@ static void xen_write_cr3(unsigned long cr3)
 
         /* Update while interrupts are disabled, so its atomic with
            respect to ipis */
-        x86_write_percpu(xen_cr3, cr3);
+        percpu_write(xen_cr3, cr3);
 
         __xen_write_cr3(true, cr3);
 
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index bb042608c602..2e8271431e1a 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -39,7 +39,7 @@ static unsigned long xen_save_fl(void)
         struct vcpu_info *vcpu;
         unsigned long flags;
 
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
 
         /* flag has opposite sense of mask */
         flags = !vcpu->evtchn_upcall_mask;
@@ -62,7 +62,7 @@ static void xen_restore_fl(unsigned long flags)
            make sure we're don't switch CPUs between getting the vcpu
            pointer and updating the mask. */
         preempt_disable();
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
         vcpu->evtchn_upcall_mask = flags;
         preempt_enable_no_resched();
 
@@ -83,7 +83,7 @@ static void xen_irq_disable(void)
            make sure we're don't switch CPUs between getting the vcpu
            pointer and updating the mask. */
         preempt_disable();
-        x86_read_percpu(xen_vcpu)->evtchn_upcall_mask = 1;
+        percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
         preempt_enable_no_resched();
 }
 
@@ -96,7 +96,7 @@ static void xen_irq_enable(void)
            the caller is confused and is trying to re-enable interrupts
            on an indeterminate processor. */
 
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
         vcpu->evtchn_upcall_mask = 0;
 
         /* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 503c240e26c7..7bc7852cc5c4 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1074,7 +1074,7 @@ static void drop_other_mm_ref(void *info)
 
         /* If this cpu still has a stale cr3 reference, then make sure
            it has been flushed. */
-        if (x86_read_percpu(xen_current_cr3) == __pa(mm->pgd)) {
+        if (percpu_read(xen_current_cr3) == __pa(mm->pgd)) {
                 load_cr3(swapper_pg_dir);
                 arch_flush_lazy_cpu_mode();
         }
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index 858938241616..e786fa7f2615 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -39,7 +39,7 @@ static inline void xen_mc_issue(unsigned mode)
                 xen_mc_flush();
 
         /* restore flags saved in xen_mc_batch */
-        local_irq_restore(x86_read_percpu(xen_mc_irq_flags));
+        local_irq_restore(percpu_read(xen_mc_irq_flags));
 }
 
 /* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 83fa4236477d..3bfd6dd0b47c 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -78,7 +78,7 @@ static __cpuinit void cpu_bringup(void)
         xen_setup_cpu_clockevents();
 
         cpu_set(cpu, cpu_online_map);
-        x86_write_percpu(cpu_state, CPU_ONLINE);
+        percpu_write(cpu_state, CPU_ONLINE);
         wmb();
 
         /* We can take interrupts now: we're officially "up". */