4 files changed, 25 insertions, 91 deletions

diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h
index 5488e10b9dba..0c9fa2745f13 100644
--- a/arch/x86/include/asm/cmpxchg.h
+++ b/arch/x86/include/asm/cmpxchg.h
@@ -207,4 +207,27 @@ extern void __add_wrong_size(void)
 #define add_smp(ptr, inc)       __add((ptr), (inc), LOCK_PREFIX)
 #define add_sync(ptr, inc)      __add((ptr), (inc), "lock; ")
 
+#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2)                   \
+({                                                                      \
+        bool __ret;                                                     \
+        __typeof__(*(p1)) __old1 = (o1), __new1 = (n1);                 \
+        __typeof__(*(p2)) __old2 = (o2), __new2 = (n2);                 \
+        BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long));                    \
+        BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long));                    \
+        VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long)));            \
+        VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2));    \
+        asm volatile(pfx "cmpxchg%c4b %2; sete %0"                      \
+                     : "=a" (__ret), "+d" (__old2),                     \
+                       "+m" (*(p1)), "+m" (*(p2))                       \
+                     : "i" (2 * sizeof(long)), "a" (__old1),            \
+                       "b" (__new1), "c" (__new2));                     \
+        __ret;                                                          \
+})
+
+#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
+        __cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)
+
+#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
+        __cmpxchg_double(, p1, p2, o1, o2, n1, n2)
+
 #endif  /* ASM_X86_CMPXCHG_H */
diff --git a/arch/x86/include/asm/cmpxchg_32.h b/arch/x86/include/asm/cmpxchg_32.h
index fbebb07dd80b..53f4b219336b 100644
--- a/arch/x86/include/asm/cmpxchg_32.h
+++ b/arch/x86/include/asm/cmpxchg_32.h
@@ -166,52 +166,6 @@ static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
 
 #endif
 
-#define cmpxchg8b(ptr, o1, o2, n1, n2)                          \
-({                                                              \
-        char __ret;                                             \
-        __typeof__(o2) __dummy;                                 \
-        __typeof__(*(ptr)) __old1 = (o1);                       \
-        __typeof__(o2) __old2 = (o2);                           \
-        __typeof__(*(ptr)) __new1 = (n1);                       \
-        __typeof__(o2) __new2 = (n2);                           \
-        asm volatile(LOCK_PREFIX "cmpxchg8b %2; setz %1"        \
-                       : "=d"(__dummy), "=a" (__ret), "+m" (*ptr)\
-                       : "a" (__old1), "d"(__old2),             \
-                         "b" (__new1), "c" (__new2)             \
-                       : "memory");                             \
-        __ret; })
-
-
-#define cmpxchg8b_local(ptr, o1, o2, n1, n2)                    \
-({                                                              \
-        char __ret;                                             \
-        __typeof__(o2) __dummy;                                 \
-        __typeof__(*(ptr)) __old1 = (o1);                       \
-        __typeof__(o2) __old2 = (o2);                           \
-        __typeof__(*(ptr)) __new1 = (n1);                       \
-        __typeof__(o2) __new2 = (n2);                           \
-        asm volatile("cmpxchg8b %2; setz %1"                    \
-                       : "=d"(__dummy), "=a"(__ret), "+m" (*ptr)\
-                       : "a" (__old), "d"(__old2),              \
-                         "b" (__new1), "c" (__new2),            \
-                       : "memory");                             \
-        __ret; })
-
-
-#define cmpxchg_double(ptr, o1, o2, n1, n2)                             \
-({                                                                      \
-        BUILD_BUG_ON(sizeof(*(ptr)) != 4);                              \
-        VM_BUG_ON((unsigned long)(ptr) % 8);                            \
-        cmpxchg8b((ptr), (o1), (o2), (n1), (n2));                       \
-})
-
-#define cmpxchg_double_local(ptr, o1, o2, n1, n2)                       \
-({                                                                      \
-       BUILD_BUG_ON(sizeof(*(ptr)) != 4);                               \
-       VM_BUG_ON((unsigned long)(ptr) % 8);                             \
-       cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2));                 \
-})
-
 #define system_has_cmpxchg_double() cpu_has_cx8
 
 #endif /* _ASM_X86_CMPXCHG_32_H */
diff --git a/arch/x86/include/asm/cmpxchg_64.h b/arch/x86/include/asm/cmpxchg_64.h
index 285da02c38fa..614be87f1a9b 100644
--- a/arch/x86/include/asm/cmpxchg_64.h
+++ b/arch/x86/include/asm/cmpxchg_64.h
@@ -20,49 +20,6 @@ static inline void set_64bit(volatile u64 *ptr, u64 val)
         cmpxchg_local((ptr), (o), (n));                                 \
 })
 
-#define cmpxchg16b(ptr, o1, o2, n1, n2)                         \
-({                                                              \
-        char __ret;                                             \
-        __typeof__(o2) __junk;                                  \
-        __typeof__(*(ptr)) __old1 = (o1);                       \
-        __typeof__(o2) __old2 = (o2);                           \
-        __typeof__(*(ptr)) __new1 = (n1);                       \
-        __typeof__(o2) __new2 = (n2);                           \
-        asm volatile(LOCK_PREFIX "cmpxchg16b %2;setz %1"        \
-                       : "=d"(__junk), "=a"(__ret), "+m" (*ptr) \
-                       : "b"(__new1), "c"(__new2),              \
-                         "a"(__old1), "d"(__old2));             \
-        __ret; })
-
-
-#define cmpxchg16b_local(ptr, o1, o2, n1, n2)                   \
-({                                                              \
-        char __ret;                                             \
-        __typeof__(o2) __junk;                                  \
-        __typeof__(*(ptr)) __old1 = (o1);                       \
-        __typeof__(o2) __old2 = (o2);                           \
-        __typeof__(*(ptr)) __new1 = (n1);                       \
-        __typeof__(o2) __new2 = (n2);                           \
-        asm volatile("cmpxchg16b %2;setz %1"                    \
-                       : "=d"(__junk), "=a"(__ret), "+m" (*ptr) \
-                       : "b"(__new1), "c"(__new2),              \
-                         "a"(__old1), "d"(__old2));             \
-        __ret; })
-
-#define cmpxchg_double(ptr, o1, o2, n1, n2)                             \
-({                                                                      \
-        BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
-        VM_BUG_ON((unsigned long)(ptr) % 16);                           \
-        cmpxchg16b((ptr), (o1), (o2), (n1), (n2));                      \
-})
-
-#define cmpxchg_double_local(ptr, o1, o2, n1, n2)                       \
-({                                                                      \
-        BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
-        VM_BUG_ON((unsigned long)(ptr) % 16);                           \
-        cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2));                \
-})
-
 #define system_has_cmpxchg_double() cpu_has_cx16
 
 #endif /* _ASM_X86_CMPXCHG_64_H */
diff --git a/mm/slub.c b/mm/slub.c
index ed3334d9b6da..09ccee8fb58e 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -368,7 +368,7 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page
         VM_BUG_ON(!irqs_disabled());
 #ifdef CONFIG_CMPXCHG_DOUBLE
         if (s->flags & __CMPXCHG_DOUBLE) {
-                if (cmpxchg_double(&page->freelist,
+                if (cmpxchg_double(&page->freelist, &page->counters,
                         freelist_old, counters_old,
                         freelist_new, counters_new))
                 return 1;
@@ -402,7 +402,7 @@ static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
 {
 #ifdef CONFIG_CMPXCHG_DOUBLE
         if (s->flags & __CMPXCHG_DOUBLE) {
-                if (cmpxchg_double(&page->freelist,
+                if (cmpxchg_double(&page->freelist, &page->counters,
                         freelist_old, counters_old,
                         freelist_new, counters_new))
                 return 1;