core: Replace __get_cpu_var with __this_cpu_read if not used for an address.

__get_cpu_var() can be replaced with this_cpu_read and will then use a
single read instruction with implied address calculation to access the
correct per cpu instance.

However, the address of a per cpu variable passed to __this_cpu_read()
cannot be determined (since it's an implied address conversion through
segment prefixes).  Therefore apply this only to uses of __get_cpu_var
where the address of the variable is not used.

Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Hugh Dickins <hughd@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@zytor.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

3 files changed, 8 insertions, 14 deletions

diff --git a/include/asm-generic/irq_regs.h b/include/asm-generic/irq_regs.h
index 5ae1d07d4a12..6bf9355fa7eb 100644
--- a/include/asm-generic/irq_regs.h
+++ b/include/asm-generic/irq_regs.h
@@ -22,15 +22,15 @@ DECLARE_PER_CPU(struct pt_regs *, __irq_regs);
 
 static inline struct pt_regs *get_irq_regs(void)
 {
-        return __get_cpu_var(__irq_regs);
+        return __this_cpu_read(__irq_regs);
 }
 
 static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
 {
-        struct pt_regs *old_regs, **pp_regs = &__get_cpu_var(__irq_regs);
+        struct pt_regs *old_regs;
 
-        old_regs = *pp_regs;
-        *pp_regs = new_regs;
+        old_regs = __this_cpu_read(__irq_regs);
+        __this_cpu_write(__irq_regs, new_regs);
         return old_regs;
 }
 
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index 4fd978e7eb83..4d857973d2c9 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -195,15 +195,9 @@ enum {
 /*
  * io context count accounting
  */
-#define elv_ioc_count_mod(name, __val)                          \
-        do {                                                    \
-                preempt_disable();                              \
-                __get_cpu_var(name) += (__val);                 \
-                preempt_enable();                               \
-        } while (0)
-
-#define elv_ioc_count_inc(name) elv_ioc_count_mod(name, 1)
-#define elv_ioc_count_dec(name) elv_ioc_count_mod(name, -1)
+#define elv_ioc_count_mod(name, __val) this_cpu_add(name, __val)
+#define elv_ioc_count_inc(name) this_cpu_inc(name)
+#define elv_ioc_count_dec(name) this_cpu_dec(name)
 
 #define elv_ioc_count_read(name)                                \
 ({                                                              \
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index ad54c846911b..44e83ba12b5b 100644
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -47,7 +47,7 @@ extern unsigned long long nr_context_switches(void);
 
 #ifndef CONFIG_GENERIC_HARDIRQS
 #define kstat_irqs_this_cpu(irq) \
-        (kstat_this_cpu.irqs[irq])
+        (this_cpu_read(kstat.irqs[irq])
 
 struct irq_desc;