47 files changed, 1032 insertions, 979 deletions

diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h
index fa9aaf7144b7..1d4706114a45 100644
--- a/arch/s390/include/asm/atomic.h
+++ b/arch/s390/include/asm/atomic.h
@@ -15,23 +15,29 @@
 
 #include <linux/compiler.h>
 #include <linux/types.h>
+#include <asm/barrier.h>
 #include <asm/cmpxchg.h>
 
 #define ATOMIC_INIT(i)  { (i) }
 
+#define __ATOMIC_NO_BARRIER     "\n"
+
 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 
 #define __ATOMIC_OR     "lao"
 #define __ATOMIC_AND    "lan"
 #define __ATOMIC_ADD    "laa"
+#define __ATOMIC_BARRIER "bcr   14,0\n"
 
-#define __ATOMIC_LOOP(ptr, op_val, op_string)                           \
+#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)                \
 ({                                                                      \
         int old_val;                                                    \
                                                                         \
         typecheck(atomic_t *, ptr);                                     \
         asm volatile(                                                   \
+                __barrier                                               \
                 op_string "     %0,%2,%1\n"                             \
+                __barrier                                               \
                 : "=d" (old_val), "+Q" ((ptr)->counter)                 \
                 : "d" (op_val)                                          \
                 : "cc", "memory");                                      \
@@ -43,8 +49,9 @@
 #define __ATOMIC_OR     "or"
 #define __ATOMIC_AND    "nr"
 #define __ATOMIC_ADD    "ar"
+#define __ATOMIC_BARRIER "\n"
 
-#define __ATOMIC_LOOP(ptr, op_val, op_string)                           \
+#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)                \
 ({                                                                      \
         int old_val, new_val;                                           \
                                                                         \
@@ -82,7 +89,7 @@ static inline void atomic_set(atomic_t *v, int i)
 
 static inline int atomic_add_return(int i, atomic_t *v)
 {
-        return __ATOMIC_LOOP(v, i, __ATOMIC_ADD) + i;
+        return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i;
 }
 
 static inline void atomic_add(int i, atomic_t *v)
@@ -94,12 +101,10 @@ static inline void atomic_add(int i, atomic_t *v)
                         : "+Q" (v->counter)
                         : "i" (i)
                         : "cc", "memory");
-        } else {
-                atomic_add_return(i, v);
+                return;
         }
-#else
-        atomic_add_return(i, v);
 #endif
+        __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER);
 }
 
 #define atomic_add_negative(_i, _v)     (atomic_add_return(_i, _v) < 0)
@@ -115,12 +120,12 @@ static inline void atomic_add(int i, atomic_t *v)
 
 static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)
 {
-        __ATOMIC_LOOP(v, ~mask, __ATOMIC_AND);
+        __ATOMIC_LOOP(v, ~mask, __ATOMIC_AND, __ATOMIC_NO_BARRIER);
 }
 
 static inline void atomic_set_mask(unsigned int mask, atomic_t *v)
 {
-        __ATOMIC_LOOP(v, mask, __ATOMIC_OR);
+        __ATOMIC_LOOP(v, mask, __ATOMIC_OR, __ATOMIC_NO_BARRIER);
 }
 
 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
@@ -157,19 +162,24 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u)
 
 #ifdef CONFIG_64BIT
 
+#define __ATOMIC64_NO_BARRIER   "\n"
+
 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 
 #define __ATOMIC64_OR   "laog"
 #define __ATOMIC64_AND  "lang"
 #define __ATOMIC64_ADD  "laag"
+#define __ATOMIC64_BARRIER "bcr 14,0\n"