Merge branch 'x86-atomic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86/atomic changes from Ingo Molnar.

* 'x86-atomic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86: atomic64 assembly improvements
  x86: Adjust asm constraints in atomic64 wrappers

5 files changed, 101 insertions, 145 deletions

diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 37ad100a2210..49331bedc158 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -145,6 +145,12 @@ static inline int alternatives_text_reserved(void *start, void *end)
  */
 #define ASM_OUTPUT2(a...) a
 
+/*
+ * use this macro if you need clobbers but no inputs in
+ * alternative_{input,io,call}()
+ */
+#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
+
 struct paravirt_patch_site;
 #ifdef CONFIG_PARAVIRT
 void apply_paravirt(struct paravirt_patch_site *start,
diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h
index fa13f0ec2874..198119910da5 100644
--- a/arch/x86/include/asm/atomic64_32.h
+++ b/arch/x86/include/asm/atomic64_32.h
@@ -14,13 +14,52 @@ typedef struct {
 
 #define ATOMIC64_INIT(val)      { (val) }
 
+#define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...)
+#ifndef ATOMIC64_EXPORT
+#define ATOMIC64_DECL_ONE __ATOMIC64_DECL
+#else
+#define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \
+        ATOMIC64_EXPORT(atomic64_##sym)
+#endif
+
 #ifdef CONFIG_X86_CMPXCHG64
-#define ATOMIC64_ALTERNATIVE_(f, g) "call atomic64_" #g "_cx8"
+#define __alternative_atomic64(f, g, out, in...) \
+        asm volatile("call %P[func]" \
+                     : out : [func] "i" (atomic64_##g##_cx8), ## in)
+
+#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8)
 #else
-#define ATOMIC64_ALTERNATIVE_(f, g) ALTERNATIVE("call atomic64_" #f "_386", "call atomic64_" #g "_cx8", X86_FEATURE_CX8)
+#define __alternative_atomic64(f, g, out, in...) \
+        alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \
+                         X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in)
+
+#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \
+        ATOMIC64_DECL_ONE(sym##_386)
+
+ATOMIC64_DECL_ONE(add_386);
+ATOMIC64_DECL_ONE(sub_386);
+ATOMIC64_DECL_ONE(inc_386);
+ATOMIC64_DECL_ONE(dec_386);
 #endif
 
-#define ATOMIC64_ALTERNATIVE(f) ATOMIC64_ALTERNATIVE_(f, f)
+#define alternative_atomic64(f, out, in...) \
+        __alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in)
+
+ATOMIC64_DECL(read);
+ATOMIC64_DECL(set);
+ATOMIC64_DECL(xchg);
+ATOMIC64_DECL(add_return);
+ATOMIC64_DECL(sub_return);
+ATOMIC64_DECL(inc_return);
+ATOMIC64_DECL(dec_return);
+ATOMIC64_DECL(dec_if_positive);
+ATOMIC64_DECL(inc_not_zero);
+ATOMIC64_DECL(add_unless);
+
+#undef ATOMIC64_DECL
+#undef ATOMIC64_DECL_ONE
+#undef __ATOMIC64_DECL
+#undef ATOMIC64_EXPORT
 
 /**
  * atomic64_cmpxchg - cmpxchg atomic64 variable
@@ -50,11 +89,9 @@ static inline long long atomic64_xchg(atomic64_t *v, long long n)
         long long o;
         unsigned high = (unsigned)(n >> 32);
         unsigned low = (unsigned)n;
-        asm volatile(ATOMIC64_ALTERNATIVE(xchg)
-                     : "=A" (o), "+b" (low), "+c" (high)
-                     : "S" (v)
-                     : "memory"
-                     );
+        alternative_atomic64(xchg, "=&A" (o),
+                             "S" (v), "b" (low), "c" (high)
+                             : "memory");
         return o;
 }
 
@@ -69,11 +106,9 @@ static inline void atomic64_set(atomic64_t *v, long long i)
 {
         unsigned high = (unsigned)(i >> 32);
         unsigned low = (unsigned)i;
-        asm volatile(ATOMIC64_ALTERNATIVE(set)
-                     : "+b" (low), "+c" (high)
-                     : "S" (v)
-                     : "eax", "edx", "memory"
-                     );
+        alternative_atomic64(set, /* no output */,
+                             "S" (v), "b" (low), "c" (high)
+                             : "eax", "edx", "memory");
 }
 
 /**
@@ -85,10 +120,7 @@ static inline void atomic64_set(atomic64_t *v, long long i)
 static inline long long atomic64_read(const atomic64_t *v)
 {
         long long r;
-        asm volatile(ATOMIC64_ALTERNATIVE(read)
-                     : "=A" (r), "+c" (v)
-                     : : "memory"
-                     );
+        alternative_atomic64(read, "=&A" (r), "c" (v) : "memory");
         return r;
  }
 
@@ -101,10 +133,9 @@ static inline long long atomic64_read(const atomic64_t *v)
  */
 static inline long long atomic64_add_return(long long i, atomic64_t *v)
 {
-        asm volatile(ATOMIC64_ALTERNATIVE(add_return)
-                     : "+A" (i), "+c" (v)
-                     : : "memory"
-                     );
+        alternative_atomic64(add_return,
+                             ASM_OUTPUT2("+A" (i), "+c" (v)),
+                             ASM_NO_INPUT_CLOBBER("memory"));
         return i;
 }
 
@@ -113,32 +144,25 @@ static inline long long atomic64_add_return(long long i, atomic64_t *v)
  */
 static inline long long atomic64_sub_return(long long i, atomic64_t *v)
 {
-        asm volatile(ATOMIC64_ALTERNATIVE(sub_return)
-                     : "+A" (i), "+c" (v)
-                     : : "memory"
-                     );
+        alternative_atomic64(sub_return,
+                             ASM_OUTPUT2("+A" (i), "+c" (v)),
+                             ASM_NO_INPUT_CLOBBER("memory"));
         return i;
 }
 
 static inline long long atomic64_inc_return(atomic64_t *v)
 {
         long long a;
-        asm volatile(ATOMIC64_ALTERNATIVE(inc_return)
-                     : "=A" (a)
-                     : "S" (v)
-                     : "memory", "ecx"
-                     );
+        alternative_atomic64(inc_return, "=&A" (a),
+                             "S" (v) : "memory", "ecx");
         return a;
 }
 
 static inline long long atomic64_dec_return(atomic64_t *v)
 {
         long long a;
-        asm volatile(ATOMIC64_ALTERNATIVE(dec_return)
-                     : "=A" (a)
-                     : "S" (v)
-                     : "memory", "ecx"
-                     );
+        alternative_atomic64(dec_return, "=&A" (a),
+                             "S" (v) : "memory", "ecx");
         return a;