Merge branch 'for-4.9' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu

Pull percpu updates from Tejun Heo:

 - Nick improved generic implementations of percpu operations which
   modify the variable and return so that they calculate the physical
   address only once.

 - percpu_ref percpu <-> atomic mode switching improvements. The
   patchset was originally posted about a year ago but fell through the
   crack.

 - misc non-critical fixes.

* 'for-4.9' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu:
  mm/percpu.c: fix potential memory leakage for pcpu_embed_first_chunk()
  mm/percpu.c: correct max_distance calculation for pcpu_embed_first_chunk()
  percpu: eliminate two sparse warnings
  percpu: improve generic percpu modify-return implementation
  percpu-refcount: init ->confirm_switch member properly
  percpu_ref: allow operation mode switching operations to be called concurrently
  percpu_ref: restructure operation mode switching
  percpu_ref: unify staggered atomic switching wait behavior
  percpu_ref: reorganize __percpu_ref_switch_to_atomic() and relocate percpu_ref_switch_to_atomic()
  percpu_ref: remove unnecessary RCU grace period for staggered atomic switching confirmation

1 files changed, 30 insertions, 23 deletions

diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index 4d9f233c4ba8..40e887068da2 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -65,6 +65,11 @@ extern void setup_per_cpu_areas(void);
 #define PER_CPU_DEF_ATTRIBUTES
 #endif
 
+#define raw_cpu_generic_read(pcp)                                       \
+({                                                                      \
+        *raw_cpu_ptr(&(pcp));                                           \
+})
+
 #define raw_cpu_generic_to_op(pcp, val, op)                             \
 do {                                                                    \
         *raw_cpu_ptr(&(pcp)) op val;                                    \
@@ -72,34 +77,39 @@ do {									\
 
 #define raw_cpu_generic_add_return(pcp, val)                            \
 ({                                                                      \
-        raw_cpu_add(pcp, val);                                          \
-        raw_cpu_read(pcp);                                              \
+        typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
+                                                                        \
+        *__p += val;                                                    \
+        *__p;                                                           \
 })
 
 #define raw_cpu_generic_xchg(pcp, nval)                                 \
 ({                                                                      \
+        typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
         typeof(pcp) __ret;                                              \
-        __ret = raw_cpu_read(pcp);                                      \
-        raw_cpu_write(pcp, nval);                                       \
+        __ret = *__p;                                                   \
+        *__p = nval;                                                    \
         __ret;                                                          \
 })
 
 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)                        \
 ({                                                                      \
+        typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
         typeof(pcp) __ret;                                              \
-        __ret = raw_cpu_read(pcp);                                      \
+        __ret = *__p;                                                   \
         if (__ret == (oval))                                            \
-                raw_cpu_write(pcp, nval);                               \
+                *__p = nval;                                            \
         __ret;                                                          \
 })
 
 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 ({                                                                      \
+        typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1));                    \
+        typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2));                    \
         int __ret = 0;                                                  \
-        if (raw_cpu_read(pcp1) == (oval1) &&                            \
-                         raw_cpu_read(pcp2)  == (oval2)) {              \
-                raw_cpu_write(pcp1, nval1);                             \
-                raw_cpu_write(pcp2, nval2);                             \
+        if (*__p1 == (oval1) && *__p2  == (oval2)) {                    \
+                *__p1 = nval1;                                          \
+                *__p2 = nval2;                                          \
                 __ret = 1;                                              \
         }                                                               \
         (__ret);                                                        \
@@ -109,7 +119,7 @@ do {									\
 ({                                                                      \
         typeof(pcp) __ret;                                              \
         preempt_disable();                                              \
-        __ret = *this_cpu_ptr(&(pcp));                                  \
+        __ret = raw_cpu_generic_read(pcp);                              \
         preempt_enable();                                               \
         __ret;                                                          \
 })
@@ -118,17 +128,17 @@ do {									\
 do {                                                                    \
         unsigned long __flags;                                          \
         raw_local_irq_save(__flags);                                    \
-        *raw_cpu_ptr(&(pcp)) op val;                                    \
+        raw_cpu_generic_to_op(pcp, val, op);                            \
         raw_local_irq_restore(__flags);                                 \
 } while (0)
 
+
 #define this_cpu_generic_add_return(pcp, val)                           \
 ({                                                                      \
         typeof(pcp) __ret;                                              \
         unsigned long __flags;                                          \
         raw_local_irq_save(__flags);                                    \
-        raw_cpu_add(pcp, val);                                          \
-        __ret = raw_cpu_read(pcp);                                      \
+        __ret = raw_cpu_generic_add_return(pcp, val);                   \
         raw_local_irq_restore(__flags);                                 \
         __ret;                                                          \
 })
@@ -138,8 +148,7 @@ do {									\
         typeof(pcp) __ret;                                              \
         unsigned long __flags;                                          \
         raw_local_irq_save(__flags);                                    \
-        __ret = raw_cpu_read(pcp);                                      \
-        raw_cpu_write(pcp, nval);                                       \
+        __ret = raw_cpu_generic_xchg(pcp, nval);                        \
         raw_local_irq_restore(__flags);                                 \
         __ret;                                                          \
 })
@@ -149,9 +158,7 @@ do {									\
         typeof(pcp) __ret;                                              \
         unsigned long __flags;                                          \
         raw_local_irq_save(__flags);                                    \
-        __ret = raw_cpu_read(pcp);                                      \
-        if (__ret == (oval))                                            \
-                raw_cpu_write(pcp, nval);                               \
+        __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);               \
         raw_local_irq_restore(__flags);                                 \
         __ret;                                                          \
 })
@@ -168,16 +175,16 @@ do {									\
 })
 
 #ifndef raw_cpu_read_1
-#define raw_cpu_read_1(pcp)             (*raw_cpu_ptr(&(pcp)))
+#define raw_cpu_read_1(pcp)             raw_cpu_generic_read(pcp)
 #endif
 #ifndef raw_cpu_read_2
-#define raw_cpu_read_2(pcp)             (*raw_cpu_ptr(&(pcp)))
+#define raw_cpu_read_2(pcp)             raw_cpu_generic_read(pcp)
 #endif
 #ifndef raw_cpu_read_4
-#define raw_cpu_read_4(pcp)             (*raw_cpu_ptr(&(pcp)))
+#define raw_cpu_read_4(pcp)             raw_cpu_generic_read(pcp)
 #endif
 #ifndef raw_cpu_read_8
-#define raw_cpu_read_8(pcp)             (*raw_cpu_ptr(&(pcp)))
+#define raw_cpu_read_8(pcp)             raw_cpu_generic_read(pcp)
 #endif
 
 #ifndef raw_cpu_write_1