[ARM] barriers: improve xchg, bitops and atomic SMP barriers

Mathieu Desnoyers pointed out that the ARM barriers were lacking:

- cmpxchg, xchg and atomic add return need memory barriers on
  architectures which can reorder the relative order in which memory
  read/writes can be seen between CPUs, which seems to include recent
  ARM architectures. Those barriers are currently missing on ARM.

- test_and_xxx_bit were missing SMP barriers.

So put these barriers in.  Provide separate atomic_add/atomic_sub
operations which do not require barriers.

Reported-Reviewed-and-Acked-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

5 files changed, 71 insertions, 13 deletions

diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h
index 6116e4893c0a..15f8a092b700 100644
--- a/arch/arm/include/asm/assembler.h
+++ b/arch/arm/include/asm/assembler.h
@@ -114,3 +114,16 @@
         .align  3;                              \
         .long   9999b,9001f;                    \
         .previous
+
+/*
+ * SMP data memory barrier
+ */
+        .macro  smp_dmb
+#ifdef CONFIG_SMP
+#if __LINUX_ARM_ARCH__ >= 7
+        dmb
+#elif __LINUX_ARM_ARCH__ == 6
+        mcr     p15, 0, r0, c7, c10, 5  @ dmb
+#endif
+#endif
+        .endm
diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h
index ee99723b3a6c..16b52f397983 100644
--- a/arch/arm/include/asm/atomic.h
+++ b/arch/arm/include/asm/atomic.h
@@ -44,11 +44,29 @@ static inline void atomic_set(atomic_t *v, int i)
         : "cc");
 }
 
+static inline void atomic_add(int i, atomic_t *v)
+{
+        unsigned long tmp;
+        int result;
+
+        __asm__ __volatile__("@ atomic_add\n"
+"1:     ldrex   %0, [%2]\n"
+"       add     %0, %0, %3\n"
+"       strex   %1, %0, [%2]\n"
+"       teq     %1, #0\n"
+"       bne     1b"
+        : "=&r" (result), "=&r" (tmp)
+        : "r" (&v->counter), "Ir" (i)
+        : "cc");
+}
+
 static inline int atomic_add_return(int i, atomic_t *v)
 {
         unsigned long tmp;
         int result;
 
+        smp_mb();
+
         __asm__ __volatile__("@ atomic_add_return\n"
 "1:     ldrex   %0, [%2]\n"
 "       add     %0, %0, %3\n"
@@ -59,14 +77,34 @@ static inline int atomic_add_return(int i, atomic_t *v)
         : "r" (&v->counter), "Ir" (i)
         : "cc");
 
+        smp_mb();
+
         return result;
 }
 
+static inline void atomic_sub(int i, atomic_t *v)
+{
+        unsigned long tmp;
+        int result;
+
+        __asm__ __volatile__("@ atomic_sub\n"
+"1:     ldrex   %0, [%2]\n"
+"       sub     %0, %0, %3\n"
+"       strex   %1, %0, [%2]\n"
+"       teq     %1, #0\n"
+"       bne     1b"
+        : "=&r" (result), "=&r" (tmp)
+        : "r" (&v->counter), "Ir" (i)
+        : "cc");
+}
+
 static inline int atomic_sub_return(int i, atomic_t *v)
 {
         unsigned long tmp;
         int result;
 
+        smp_mb();
+
         __asm__ __volatile__("@ atomic_sub_return\n"
 "1:     ldrex   %0, [%2]\n"
 "       sub     %0, %0, %3\n"
@@ -77,6 +115,8 @@ static inline int atomic_sub_return(int i, atomic_t *v)
         : "r" (&v->counter), "Ir" (i)
         : "cc");
 
+        smp_mb();
+
         return result;
 }
 
@@ -84,6 +124,8 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
 {
         unsigned long oldval, res;
 
+        smp_mb();
+
         do {
                 __asm__ __volatile__("@ atomic_cmpxchg\n"
                 "ldrex  %1, [%2]\n"
@@ -95,6 +137,8 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
                     : "cc");
         } while (res);
 
+        smp_mb();
+
         return oldval;
 }
 
@@ -135,6 +179,7 @@ static inline int atomic_add_return(int i, atomic_t *v)
 
         return val;
 }
+#define atomic_add(i, v)        (void) atomic_add_return(i, v)
 
 static inline int atomic_sub_return(int i, atomic_t *v)
 {
@@ -148,6 +193,7 @@ static inline int atomic_sub_return(int i, atomic_t *v)
 
         return val;
 }
+#define atomic_sub(i, v)        (void) atomic_sub_return(i, v)
 
 static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
 {
@@ -187,10 +233,8 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u)
 }
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
 
-#define atomic_add(i, v)        (void) atomic_add_return(i, v)
-#define atomic_inc(v)           (void) atomic_add_return(1, v)
-#define atomic_sub(i, v)        (void) atomic_sub_return(i, v)
-#define atomic_dec(v)           (void) atomic_sub_return(1, v)
+#define atomic_inc(v)           atomic_add(1, v)
+#define atomic_dec(v)           atomic_sub(1, v)
 
 #define atomic_inc_and_test(v)  (atomic_add_return(1, v) == 0)
 #define atomic_dec_and_test(v)  (atomic_sub_return(1, v) == 0)
@@ -200,11 +244,10 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u)
 
 #define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)
 
-/* Atomic operations are already serializing on ARM */
-#define smp_mb__before_atomic_dec()     barrier()
-#define smp_mb__after_atomic_dec()      barrier()
-#define smp_mb__before_atomic_inc()     barrier()
-#define smp_mb__after_atomic_inc()      barrier()
+#define smp_mb__before_atomic_dec()     smp_mb()
+#define smp_mb__after_atomic_dec()      smp_mb()
+#define smp_mb__before_atomic_inc()     smp_mb()
+#define smp_mb__after_atomic_inc()      smp_mb()
 
 #include <asm-generic/atomic.h>
 #endif
diff --git a/arch/arm/include/asm/system.h b/arch/arm/include/asm/system.h
index bd4dc8ed53d5..7fce8f3b391d 100644
--- a/arch/arm/include/asm/system.h
+++ b/arch/arm/include/asm/system.h
@@ -248,6 +248,8 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
         unsigned int tmp;
 #endif
 
+        smp_mb();
+
         switch (size) {
 #if __LINUX_ARM_ARCH__ >= 6
         case 1:
@@ -307,6 +309,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
                 __bad_xchg(ptr, size), ret = 0;
                 break;
         }
+        smp_mb();
 
         return ret;
 }
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index d662a2f1fd85..83b1da6b7baa 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -815,10 +815,7 @@ __kuser_helper_start:
  */
 
 __kuser_memory_barrier:                         @ 0xffff0fa0
-
-#if __LINUX_ARM_ARCH__ >= 6 && defined(CONFIG_SMP)
-        mcr     p15, 0, r0, c7, c10, 5  @ dmb
-#endif
+        smp_dmb
         usr_ret lr
 
         .align  5
diff --git a/arch/arm/lib/bitops.h b/arch/arm/lib/bitops.h
index 2e787d40d599..c7f2627385e7 100644
--- a/arch/arm/lib/bitops.h
+++ b/arch/arm/lib/bitops.h
@@ -18,12 +18,14 @@
         mov     r2, #1
         add     r1, r1, r0, lsr #3      @ Get byte offset
         mov     r3, r2, lsl r3          @ create mask
+        smp_dmb
 1:      ldrexb  r2, [r1]
         ands    r0, r2, r3              @ save old value of bit
         \instr  r2, r2, r3                      @ toggle bit
         strexb  ip, r2, [r1]
         cmp     ip, #0
         bne     1b
+        smp_dmb
         cmp     r0, #0
         movne   r0, #1
 2:      mov     pc, lr