percpu,x86: relocate this_cpu_add_return() and friends

- include/linux/percpu.h: this_cpu_add_return() and friends were
  located next to __this_cpu_add_return().  However, the overall
  organization is to first group by preemption safeness.  Relocate
  this_cpu_add_return() and friends to preemption-safe area.

- arch/x86/include/asm/percpu.h: Relocate percpu_add_return_op() after
  other more basic operations.  Relocate [__]this_cpu_add_return_8()
  so that they're first grouped by preemption safeness.

Signed-off-by: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>

2 files changed, 65 insertions, 66 deletions

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 38f9e965ff96..dd0cd4b6a76f 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -177,39 +177,6 @@ do {									\
         }                                                               \
 } while (0)
 
-/*
- * Add return operation
- */
-#define percpu_add_return_op(var, val)                                  \
-({                                                                      \
-        typeof(var) paro_ret__ = val;                                   \
-        switch (sizeof(var)) {                                          \
-        case 1:                                                         \
-                asm("xaddb %0, "__percpu_arg(1)                         \
-                            : "+q" (paro_ret__), "+m" (var)             \
-                            : : "memory");                              \
-                break;                                                  \
-        case 2:                                                         \
-                asm("xaddw %0, "__percpu_arg(1)                         \
-                            : "+r" (paro_ret__), "+m" (var)             \
-                            : : "memory");                              \
-                break;                                                  \
-        case 4:                                                         \
-                asm("xaddl %0, "__percpu_arg(1)                         \
-                            : "+r" (paro_ret__), "+m" (var)             \
-                            : : "memory");                              \
-                break;                                                  \
-        case 8:                                                         \
-                asm("xaddq %0, "__percpu_arg(1)                         \
-                            : "+re" (paro_ret__), "+m" (var)            \
-                            : : "memory");                              \
-                break;                                                  \
-        default: __bad_percpu_size();                                   \
-        }                                                               \
-        paro_ret__ += val;                                              \
-        paro_ret__;                                                     \
-})
-
 #define percpu_from_op(op, var, constraint)             \
 ({                                                      \
         typeof(var) pfo_ret__;                          \
@@ -263,6 +230,39 @@ do {									\
 })
 
 /*
+ * Add return operation
+ */
+#define percpu_add_return_op(var, val)                                  \
+({                                                                      \
+        typeof(var) paro_ret__ = val;                                   \
+        switch (sizeof(var)) {                                          \
+        case 1:                                                         \
+                asm("xaddb %0, "__percpu_arg(1)                         \
+                            : "+q" (paro_ret__), "+m" (var)             \
+                            : : "memory");                              \
+                break;                                                  \
+        case 2:                                                         \
+                asm("xaddw %0, "__percpu_arg(1)                         \
+                            : "+r" (paro_ret__), "+m" (var)             \
+                            : : "memory");                              \
+                break;                                                  \
+        case 4:                                                         \
+                asm("xaddl %0, "__percpu_arg(1)                         \
+                            : "+r" (paro_ret__), "+m" (var)             \
+                            : : "memory");                              \
+                break;                                                  \
+        case 8:                                                         \
+                asm("xaddq %0, "__percpu_arg(1)                         \
+                            : "+re" (paro_ret__), "+m" (var)            \
+                            : : "memory");                              \
+                break;                                                  \
+        default: __bad_percpu_size();                                   \
+        }                                                               \
+        paro_ret__ += val;                                              \
+        paro_ret__;                                                     \
+})
+
+/*
  * percpu_read() makes gcc load the percpu variable every time it is
  * accessed while percpu_read_stable() allows the value to be cached.
  * percpu_read_stable() is more efficient and can be used if its value
@@ -352,6 +352,7 @@ do {									\
 #define __this_cpu_and_8(pcp, val)      percpu_to_op("and", (pcp), val)
 #define __this_cpu_or_8(pcp, val)       percpu_to_op("or", (pcp), val)
 #define __this_cpu_xor_8(pcp, val)      percpu_to_op("xor", (pcp), val)
+#define __this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
 
 #define this_cpu_read_8(pcp)            percpu_from_op("mov", (pcp), "m"(pcp))
 #define this_cpu_write_8(pcp, val)      percpu_to_op("mov", (pcp), val)
@@ -359,14 +360,12 @@ do {									\
 #define this_cpu_and_8(pcp, val)        percpu_to_op("and", (pcp), val)
 #define this_cpu_or_8(pcp, val)         percpu_to_op("or", (pcp), val)
 #define this_cpu_xor_8(pcp, val)        percpu_to_op("xor", (pcp), val)
+#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
 
 #define irqsafe_cpu_add_8(pcp, val)     percpu_add_op((pcp), val)
 #define irqsafe_cpu_and_8(pcp, val)     percpu_to_op("and", (pcp), val)
 #define irqsafe_cpu_or_8(pcp, val)      percpu_to_op("or", (pcp), val)
 #define irqsafe_cpu_xor_8(pcp, val)     percpu_to_op("xor", (pcp), val)
-
-#define __this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
-#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
 #endif
 
 /* This is not atomic against other CPUs -- CPU preemption needs to be off */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 4d593defc47d..3484e88d93f8 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -417,6 +417,36 @@ do {									\
 # define this_cpu_xor(pcp, val)         __pcpu_size_call(this_cpu_or_, (pcp), (val))
 #endif
 
+#define _this_cpu_generic_add_return(pcp, val)                          \
+({                                                                      \
+        typeof(pcp) ret__;                                              \
+        preempt_disable();                                              \
+        __this_cpu_add(pcp, val);                                       \
+        ret__ = __this_cpu_read(pcp);                                   \
+        preempt_enable();                                               \
+        ret__;                                                          \
+})
+
+#ifndef this_cpu_add_return
+# ifndef this_cpu_add_return_1
+#  define this_cpu_add_return_1(pcp, val)       _this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_2
+#  define this_cpu_add_return_2(pcp, val)       _this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_4
+#  define this_cpu_add_return_4(pcp, val)       _this_cpu_generic_add_return(pcp, val)
+# endif
+# ifndef this_cpu_add_return_8
+#  define this_cpu_add_return_8(pcp, val)       _this_cpu_generic_add_return(pcp, val)
+# endif
+# define this_cpu_add_return(pcp, val)  __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
+#endif
+
+#define this_cpu_sub_return(pcp, val)   this_cpu_add_return(pcp, -(val))
+#define this_cpu_inc_return(pcp)        this_cpu_add_return(pcp, 1)
+#define this_cpu_dec_return(pcp)        this_cpu_add_return(pcp, -1)
+
 /*
  * Generic percpu operations that do not require preemption handling.
  * Either we do not care about races or the caller has the
@@ -544,36 +574,6 @@ do {									\
 # define __this_cpu_xor(pcp, val)       __pcpu_size_call(__this_cpu_xor_, (pcp), (val))
 #endif
 
-#define _this_cpu_generic_add_return(pcp, val)                          \
-({                                                                      \
-        typeof(pcp) ret__;                                              \
-        preempt_disable();                                              \
-        __this_cpu_add(pcp, val);                                       \
-        ret__ = __this_cpu_read(pcp);                                   \
-        preempt_enable();                                               \
-        ret__;                                                          \
-})
-
-#ifndef this_cpu_add_return
-# ifndef this_cpu_add_return_1
-#  define this_cpu_add_return_1(pcp, val)       _this_cpu_generic_add_return(pcp, val)
-# endif
-# ifndef this_cpu_add_return_2
-#  define this_cpu_add_return_2(pcp, val)       _this_cpu_generic_add_return(pcp, val)
-# endif
-# ifndef this_cpu_add_return_4
-#  define this_cpu_add_return_4(pcp, val)       _this_cpu_generic_add_return(pcp, val)
-# endif
-# ifndef this_cpu_add_return_8
-#  define this_cpu_add_return_8(pcp, val)       _this_cpu_generic_add_return(pcp, val)
-# endif
-# define this_cpu_add_return(pcp, val)  __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
-#endif
-
-#define this_cpu_sub_return(pcp, val)   this_cpu_add_return(pcp, -(val))
-#define this_cpu_inc_return(pcp)        this_cpu_add_return(pcp, 1)
-#define this_cpu_dec_return(pcp)        this_cpu_add_return(pcp, -1)
-
 #define __this_cpu_generic_add_return(pcp, val)                         \
 ({                                                                      \
         __this_cpu_add(pcp, val);                                       \