Merge branch 'x86-atomic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'x86-atomic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
  x86: Fix LOCK_PREFIX_HERE for uniprocessor build
  x86, atomic64: In selftest, distinguish x86-64 from 586+
  x86-32: Fix atomic64_inc_not_zero return value convention
  lib: Fix atomic64_inc_not_zero test
  lib: Fix atomic64_add_unless return value convention
  x86-32: Fix atomic64_add_unless return value convention
  lib: Fix atomic64_add_unless test
  x86: Implement atomic[64]_dec_if_positive()
  lib: Only test atomic64_dec_if_positive on archs having it
  x86-32: Rewrite 32-bit atomic64 functions in assembly
  lib: Add self-test for atomic64_t
  x86-32: Allow UP/SMP lock replacement in cmpxchg64
  x86: Add support for lock prefix in alternatives

4 files changed, 451 insertions, 223 deletions

diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index cbaf8f2b83df..f871e04b6965 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,11 +26,12 @@ obj-y += msr.o msr-reg.o msr-reg-export.o
 
 ifeq ($(CONFIG_X86_32),y)
         obj-y += atomic64_32.o
+        lib-y += atomic64_cx8_32.o
         lib-y += checksum_32.o
         lib-y += strstr_32.o
         lib-y += semaphore_32.o string_32.o
 ifneq ($(CONFIG_X86_CMPXCHG64),y)
-        lib-y += cmpxchg8b_emu.o
+        lib-y += cmpxchg8b_emu.o atomic64_386_32.o
 endif
         lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o
 else
diff --git a/arch/x86/lib/atomic64_32.c b/arch/x86/lib/atomic64_32.c
index 824fa0be55a3..540179e8e9fa 100644
--- a/arch/x86/lib/atomic64_32.c
+++ b/arch/x86/lib/atomic64_32.c
@@ -6,225 +6,54 @@
 #include <asm/cmpxchg.h>
 #include <asm/atomic.h>
 
-static noinline u64 cmpxchg8b(u64 *ptr, u64 old, u64 new)
-{
-        u32 low = new;
-        u32 high = new >> 32;
-
-        asm volatile(
-                LOCK_PREFIX "cmpxchg8b %1\n"
-                     : "+A" (old), "+m" (*ptr)
-                     :  "b" (low),  "c" (high)
-                     );
-        return old;
-}
-
-u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val)
-{
-        return cmpxchg8b(&ptr->counter, old_val, new_val);
-}
-EXPORT_SYMBOL(atomic64_cmpxchg);
-
-/**
- * atomic64_xchg - xchg atomic64 variable
- * @ptr:      pointer to type atomic64_t
- * @new_val:  value to assign
- *
- * Atomically xchgs the value of @ptr to @new_val and returns
- * the old value.
- */
-u64 atomic64_xchg(atomic64_t *ptr, u64 new_val)
-{
-        /*
-         * Try first with a (possibly incorrect) assumption about
-         * what we have there. We'll do two loops most likely,
-         * but we'll get an ownership MESI transaction straight away
-         * instead of a read transaction followed by a
-         * flush-for-ownership transaction:
-         */
-        u64 old_val, real_val = 0;
-
-        do {
-                old_val = real_val;
-
-                real_val = atomic64_cmpxchg(ptr, old_val, new_val);
-
-        } while (real_val != old_val);
-
-        return old_val;
-}
-EXPORT_SYMBOL(atomic64_xchg);
-
-/**
- * atomic64_set - set atomic64 variable
- * @ptr:      pointer to type atomic64_t
- * @new_val:  value to assign
- *
- * Atomically sets the value of @ptr to @new_val.
- */
-void atomic64_set(atomic64_t *ptr, u64 new_val)
-{
-        atomic64_xchg(ptr, new_val);
-}
-EXPORT_SYMBOL(atomic64_set);
-
-/**
-EXPORT_SYMBOL(atomic64_read);
- * atomic64_add_return - add and return
- * @delta: integer value to add
- * @ptr:   pointer to type atomic64_t
- *
- * Atomically adds @delta to @ptr and returns @delta + *@ptr
- */
-noinline u64 atomic64_add_return(u64 delta, atomic64_t *ptr)
-{
-        /*
-         * Try first with a (possibly incorrect) assumption about
-         * what we have there. We'll do two loops most likely,
-         * but we'll get an ownership MESI transaction straight away
-         * instead of a read transaction followed by a
-         * flush-for-ownership transaction:
-         */
-        u64 old_val, new_val, real_val = 0;
-
-        do {
-                old_val = real_val;
-                new_val = old_val + delta;
-
-                real_val = atomic64_cmpxchg(ptr, old_val, new_val);
-
-        } while (real_val != old_val);
-
-        return new_val;
-}
-EXPORT_SYMBOL(atomic64_add_return);
-
-u64 atomic64_sub_return(u64 delta, atomic64_t *ptr)
-{
-        return atomic64_add_return(-delta, ptr);
-}
-EXPORT_SYMBOL(atomic64_sub_return);
-
-u64 atomic64_inc_return(atomic64_t *ptr)
-{
-        return atomic64_add_return(1, ptr);
-}
-EXPORT_SYMBOL(atomic64_inc_return);
-
-u64 atomic64_dec_return(atomic64_t *ptr)
-{
-        return atomic64_sub_return(1, ptr);
-}
-EXPORT_SYMBOL(atomic64_dec_return);
-
-/**
- * atomic64_add - add integer to atomic64 variable
- * @delta: integer value to add
- * @ptr:   pointer to type atomic64_t
- *
- * Atomically adds @delta to @ptr.
- */
-void atomic64_add(u64 delta, atomic64_t *ptr)
-{
-        atomic64_add_return(delta, ptr);
-}
-EXPORT_SYMBOL(atomic64_add);
-
-/**
- * atomic64_sub - subtract the atomic64 variable
- * @delta: integer value to subtract
- * @ptr:   pointer to type atomic64_t
- *
- * Atomically subtracts @delta from @ptr.
- */
-void atomic64_sub(u64 delta, atomic64_t *ptr)
-{
-        atomic64_add(-delta, ptr);
-}
-EXPORT_SYMBOL(atomic64_sub);
-
-/**
- * atomic64_sub_and_test - subtract value from variable and test result
- * @delta: integer value to subtract
- * @ptr:   pointer to type atomic64_t
- *
- * Atomically subtracts @delta from @ptr and returns
- * true if the result is zero, or false for all
- * other cases.
- */
-int atomic64_sub_and_test(u64 delta, atomic64_t *ptr)
-{
-        u64 new_val = atomic64_sub_return(delta, ptr);
-
-        return new_val == 0;
-}
-EXPORT_SYMBOL(atomic64_sub_and_test);
-
-/**
- * atomic64_inc - increment atomic64 variable
- * @ptr: pointer to type atomic64_t
- *
- * Atomically increments @ptr by 1.
- */
-void atomic64_inc(atomic64_t *ptr)
-{
-        atomic64_add(1, ptr);
-}
-EXPORT_SYMBOL(atomic64_inc);
-
-/**
- * atomic64_dec - decrement atomic64 variable
- * @ptr: pointer to type atomic64_t
- *
- * Atomically decrements @ptr by 1.
- */
-void atomic64_dec(atomic64_t *ptr)
-{
-        atomic64_sub(1, ptr);
-}
-EXPORT_SYMBOL(atomic64_dec);
-
-/**
- * atomic64_dec_and_test - decrement and test
- * @ptr: pointer to type atomic64_t
- *
- * Atomically decrements @ptr by 1 and
- * returns true if the result is 0, or false for all other
- * cases.
- */
-int atomic64_dec_and_test(atomic64_t *ptr)
-{
-        return atomic64_sub_and_test(1, ptr);
-}
-EXPORT_SYMBOL(atomic64_dec_and_test);
-
-/**
- * atomic64_inc_and_test - increment and test
- * @ptr: pointer to type atomic64_t
- *
- * Atomically increments @ptr by 1
- * and returns true if the result is zero, or false for all
- * other cases.
- */
-int atomic64_inc_and_test(atomic64_t *ptr)
-{
-        return atomic64_sub_and_test(-1, ptr);
-}
-EXPORT_SYMBOL(atomic64_inc_and_test);
-
-/**
- * atomic64_add_negative - add and test if negative
- * @delta: integer value to add
- * @ptr:   pointer to type atomic64_t
- *
- * Atomically adds @delta to @ptr and returns true
- * if the result is negative, or false when
- * result is greater than or equal to zero.
- */
-int atomic64_add_negative(u64 delta, atomic64_t *ptr)
-{
-        s64 new_val = atomic64_add_return(delta, ptr);
-
-        return new_val < 0;
-}
-EXPORT_SYMBOL(atomic64_add_negative);
+long long atomic64_read_cx8(long long, const atomic64_t *v);
+EXPORT_SYMBOL(atomic64_read_cx8);
+long long atomic64_set_cx8(long long, const atomic64_t *v);
+EXPORT_SYMBOL(atomic64_set_cx8);
+long long atomic64_xchg_cx8(long long, unsigned high);
+EXPORT_SYMBOL(atomic64_xchg_cx8);
+long long atomic64_add_return_cx8(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_add_return_cx8);
+long long atomic64_sub_return_cx8(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_sub_return_cx8);
+long long atomic64_inc_return_cx8(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_inc_return_cx8);
+long long atomic64_dec_return_cx8(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_dec_return_cx8);
+long long atomic64_dec_if_positive_cx8(atomic64_t *v);
+EXPORT_SYMBOL(atomic64_dec_if_positive_cx8);
+int atomic64_inc_not_zero_cx8(atomic64_t *v);
+EXPORT_SYMBOL(atomic64_inc_not_zero_cx8);
+int atomic64_add_unless_cx8(atomic64_t *v, long long a, long long u);
+EXPORT_SYMBOL(atomic64_add_unless_cx8);
+
+#ifndef CONFIG_X86_CMPXCHG64
+long long atomic64_read_386(long long, const atomic64_t *v);
+EXPORT_SYMBOL(atomic64_read_386);
+long long atomic64_set_386(long long, const atomic64_t *v);
+EXPORT_SYMBOL(atomic64_set_386);
+long long atomic64_xchg_386(long long, unsigned high);
+EXPORT_SYMBOL(atomic64_xchg_386);
+long long atomic64_add_return_386(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_add_return_386);
+long long atomic64_sub_return_386(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_sub_return_386);
+long long atomic64_inc_return_386(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_inc_return_386);
+long long atomic64_dec_return_386(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_dec_return_386);
+long long atomic64_add_386(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_add_386);
+long long atomic64_sub_386(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_sub_386);
+long long atomic64_inc_386(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_inc_386);
+long long atomic64_dec_386(long long a, atomic64_t *v);
+EXPORT_SYMBOL(atomic64_dec_386);
+long long atomic64_dec_if_positive_386(atomic64_t *v);
+EXPORT_SYMBOL(atomic64_dec_if_positive_386);
+int atomic64_inc_not_zero_386(atomic64_t *v);
+EXPORT_SYMBOL(atomic64_inc_not_zero_386);
+int atomic64_add_unless_386(atomic64_t *v, long long a, long long u);
+EXPORT_SYMBOL(atomic64_add_unless_386);
+#endif
diff --git a/arch/x86/lib/atomic64_386_32.S b/arch/x86/lib/atomic64_386_32.S
new file mode 100644
index 000000000000..4a5979aa6883
--- /dev/null
+++ b/arch/x86/lib/atomic64_386_32.S
@@ -0,0 +1,174 @@
+/*
+ * atomic64_t for 386/486
+ *
+ * Copyright © 2010  Luca Barbieri
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
+#include <asm/dwarf2.h>
+
+/* if you want SMP support, implement these with real spinlocks */
+.macro LOCK reg
+        pushfl
+        CFI_ADJUST_CFA_OFFSET 4
+        cli
+.endm
+
+.macro UNLOCK reg
+        popfl
+        CFI_ADJUST_CFA_OFFSET -4
+.endm
+
+.macro BEGIN func reg
+$v = \reg
+
+ENTRY(atomic64_\func\()_386)
+        CFI_STARTPROC
+        LOCK $v
+
+.macro RETURN
+        UNLOCK $v
+        ret
+.endm
+
+.macro END_
+        CFI_ENDPROC
+ENDPROC(atomic64_\func\()_386)
+.purgem RETURN
+.purgem END_
+.purgem END
+.endm
+
+.macro END
+RETURN
+END_
+.endm
+.endm
+
+BEGIN read %ecx
+        movl  ($v), %eax
+        movl 4($v), %edx
+END
+
+BEGIN set %esi
+        movl %ebx,  ($v)
+        movl %ecx, 4($v)
+END
+
+BEGIN xchg %esi
+        movl  ($v), %eax
+        movl 4($v), %edx
+        movl %ebx,  ($v)
+        movl %ecx, 4($v)
+END
+
+BEGIN add %ecx
+        addl %eax,  ($v)
+        adcl %edx, 4($v)
+END
+
+BEGIN add_return %ecx
+        addl  ($v), %eax
+        adcl 4($v), %edx
+        movl %eax,  ($v)
+        movl %edx, 4($v)
+END
+
+BEGIN sub %ecx
+        subl %eax,  ($v)
+        sbbl %edx, 4($v)
+END
+
+BEGIN sub_return %ecx
+        negl %edx
+        negl %eax
+        sbbl $0, %edx
+        addl  ($v), %eax
+        adcl 4($v), %edx
+        movl %eax,  ($v)
+        movl %edx, 4($v)
+END
+
+BEGIN inc %esi
+        addl $1,  ($v)
+        adcl $0, 4($v)
+END
+
+BEGIN inc_return %esi
+        movl  ($v), %eax
+        movl 4($v), %edx
+        addl $1, %eax
+        adcl $0, %edx
+        movl %eax,  ($v)
+        movl %edx, 4($v)
+END
+
+BEGIN dec %esi
+        subl $1,  ($v)
+        sbbl $0, 4($v)
+END
+
+BEGIN dec_return %esi
+        movl  ($v), %eax
+        movl 4($v), %edx
+        subl $1, %eax
+        sbbl $0, %edx
+        movl %eax,  ($v)
+        movl %edx, 4($v)
+END
+
+BEGIN add_unless %ecx
+        addl %eax, %esi
+        adcl %edx, %edi
+        addl  ($v), %eax
+        adcl 4($v), %edx
+        cmpl %eax, %esi
+        je 3f
+1:
+        movl %eax,  ($v)
+        movl %edx, 4($v)
+        movl $1, %eax
+2:
+RETURN
+3:
+        cmpl %edx, %edi
+        jne 1b
+        xorl %eax, %eax
+        jmp 2b
+END_
+
+BEGIN inc_not_zero %esi
+        movl  ($v), %eax
+        movl 4($v), %edx
+        testl %eax, %eax
+        je 3f
+1:
+        addl $1, %eax
+        adcl $0, %edx
+        movl %eax,  ($v)
+        movl %edx, 4($v)
+        movl $1, %eax
+2:
+RETURN
+3:
+        testl %edx, %edx
+        jne 1b
+        jmp 2b
+END_
+
+BEGIN dec_if_positive %esi
+        movl  ($v), %eax
+        movl 4($v), %edx
+        subl $1, %eax
+        sbbl $0, %edx
+        js 1f
+        movl %eax,  ($v)
+        movl %edx, 4($v)
+1:
+END
diff --git a/arch/x86/lib/atomic64_cx8_32.S b/arch/x86/lib/atomic64_cx8_32.S
new file mode 100644
index 000000000000..71e080de3352
--- /dev/null
+++ b/arch/x86/lib/atomic64_cx8_32.S
@@ -0,0 +1,224 @@
+/*
+ * atomic64_t for 586+
+ *
+ * Copyright © 2010  Luca Barbieri
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
+#include <asm/dwarf2.h>
+
+.macro SAVE reg
+        pushl %\reg
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET \reg, 0
+.endm
+
+.macro RESTORE reg
+        popl %\reg
+        CFI_ADJUST_CFA_OFFSET -4
+        CFI_RESTORE \reg
+.endm
+
+.macro read64 reg
+        movl %ebx, %eax
+        movl %ecx, %edx
+/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */
+        LOCK_PREFIX
+        cmpxchg8b (\reg)
+.endm
+
+ENTRY(atomic64_read_cx8)
+        CFI_STARTPROC
+
+        read64 %ecx
+        ret
+        CFI_ENDPROC
+ENDPROC(atomic64_read_cx8)
+
+ENTRY(atomic64_set_cx8)
+        CFI_STARTPROC
+
+1:
+/* we don't need LOCK_PREFIX since aligned 64-bit writes
+ * are atomic on 586 and newer */
+        cmpxchg8b (%esi)
+        jne 1b
+
+        ret
+        CFI_ENDPROC
+ENDPROC(atomic64_set_cx8)
+
+ENTRY(atomic64_xchg_cx8)
+        CFI_STARTPROC
+
+        movl %ebx, %eax
+        movl %ecx, %edx
+1:
+        LOCK_PREFIX
+        cmpxchg8b (%esi)
+        jne 1b
+
+        ret
+        CFI_ENDPROC
+ENDPROC(atomic64_xchg_cx8)
+
+.macro addsub_return func ins insc
+ENTRY(atomic64_\func\()_return_cx8)
+        CFI_STARTPROC
+        SAVE ebp
+        SAVE ebx
+        SAVE esi
+        SAVE edi
+
+        movl %eax, %esi
+        movl %edx, %edi
+        movl %ecx, %ebp
+
+        read64 %ebp
+1:
+        movl %eax, %ebx
+        movl %edx, %ecx
+        \ins\()l %esi, %ebx
+        \insc\()l %edi, %ecx
+        LOCK_PREFIX
+        cmpxchg8b (%ebp)
+        jne 1b
+
+10:
+        movl %ebx, %eax
+        movl %ecx, %edx
+        RESTORE edi
+        RESTORE esi
+        RESTORE ebx
+        RESTORE ebp
+        ret
+        CFI_ENDPROC
+ENDPROC(atomic64_\func\()_return_cx8)
+.endm
+
+addsub_return add add adc
+addsub_return sub sub sbb
+
+.macro incdec_return func ins insc
+ENTRY(atomic64_\func\()_return_cx8)
+        CFI_STARTPROC
+        SAVE ebx
+
+        read64 %esi
+1:
+        movl %eax, %ebx
+        movl %edx, %ecx
+        \ins\()l $1, %ebx
+        \insc\()l $0, %ecx
+        LOCK_PREFIX
+        cmpxchg8b (%esi)
+        jne 1b
+
+10:
+        movl %ebx, %eax
+        movl %ecx, %edx
+        RESTORE ebx
+        ret
+        CFI_ENDPROC
+ENDPROC(atomic64_\func\()_return_cx8)
+.endm
+
+incdec_return inc add adc
+incdec_return dec sub sbb
+
+ENTRY(atomic64_dec_if_positive_cx8)
+        CFI_STARTPROC
+        SAVE ebx
+
+        read64 %esi
+1:
+        movl %eax, %ebx
+        movl %edx, %ecx
+        subl $1, %ebx
+        sbb $0, %ecx
+        js 2f
+        LOCK_PREFIX
+        cmpxchg8b (%esi)
+        jne 1b
+
+2:
+        movl %ebx, %eax
+        movl %ecx, %edx
+        RESTORE ebx
+        ret
+        CFI_ENDPROC
+ENDPROC(atomic64_dec_if_positive_cx8)
+
+ENTRY(atomic64_add_unless_cx8)
+        CFI_STARTPROC
+        SAVE ebp
+        SAVE ebx
+/* these just push these two parameters on the stack */
+        SAVE edi
+        SAVE esi
+
+        movl %ecx, %ebp
+        movl %eax, %esi
+        movl %edx, %edi
+
+        read64 %ebp
+1:
+        cmpl %eax, 0(%esp)
+        je 4f
+2:
+        movl %eax, %ebx
+        movl %edx, %ecx
+        addl %esi, %ebx
+        adcl %edi, %ecx
+        LOCK_PREFIX
+        cmpxchg8b (%ebp)
+        jne 1b
+
+        movl $1, %eax
+3:
+        addl $8, %esp
+        CFI_ADJUST_CFA_OFFSET -8
+        RESTORE ebx
+        RESTORE ebp
+        ret
+4:
+        cmpl %edx, 4(%esp)
+        jne 2b
+        xorl %eax, %eax
+        jmp 3b
+        CFI_ENDPROC
+ENDPROC(atomic64_add_unless_cx8)
+
+ENTRY(atomic64_inc_not_zero_cx8)
+        CFI_STARTPROC
+        SAVE ebx
+
+        read64 %esi
+1:
+        testl %eax, %eax
+        je 4f
+2:
+        movl %eax, %ebx
+        movl %edx, %ecx
+        addl $1, %ebx
+        adcl $0, %ecx
+        LOCK_PREFIX
+        cmpxchg8b (%esi)
+        jne 1b
+
+        movl $1, %eax
+3:
+        RESTORE ebx
+        ret
+4:
+        testl %edx, %edx
+        jne 2b
+        jmp 3b
+        CFI_ENDPROC
+ENDPROC(atomic64_inc_not_zero_cx8)