5 files changed, 123 insertions, 23 deletions

diff --git a/arch/um/sys-x86_64/Makefile b/arch/um/sys-x86_64/Makefile
index 2201e9c20e4a..c1ea9eb04466 100644
--- a/arch/um/sys-x86_64/Makefile
+++ b/arch/um/sys-x86_64/Makefile
@@ -8,7 +8,8 @@ obj-y = bug.o bugs.o delay.o fault.o ldt.o mem.o ptrace.o ptrace_user.o \
         setjmp.o signal.o stub.o stub_segv.o syscalls.o syscall_table.o \
         sysrq.o ksyms.o tls.o
 
-subarch-obj-y = lib/csum-partial_64.o lib/memcpy_64.o lib/thunk_64.o
+subarch-obj-y = lib/csum-partial_64.o lib/memcpy_64.o lib/thunk_64.o \
+                lib/rwsem_64.o
 subarch-obj-$(CONFIG_MODULES) += kernel/module.o
 
 ldt-y = ../sys-i386/ldt.o
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index f20ddf84a893..a19829374e6a 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -319,7 +319,7 @@ config X86_L1_CACHE_SHIFT
 
 config X86_XADD
         def_bool y
-        depends on X86_32 && !M386
+        depends on X86_64 || !M386
 
 config X86_PPRO_FENCE
         bool "PentiumPro memory ordering errata workaround"
diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index 413620024768..606ede126972 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -41,6 +41,7 @@
 #include <linux/list.h>
 #include <linux/spinlock.h>
 #include <linux/lockdep.h>
+#include <asm/asm.h>
 
 struct rwsem_waiter;
 
@@ -55,17 +56,28 @@ extern asmregparm struct rw_semaphore *
 
 /*
  * the semaphore definition
+ *
+ * The bias values and the counter type limits the number of
+ * potential readers/writers to 32767 for 32 bits and 2147483647
+ * for 64 bits.
  */
 
-#define RWSEM_UNLOCKED_VALUE            0x00000000
-#define RWSEM_ACTIVE_BIAS               0x00000001
-#define RWSEM_ACTIVE_MASK               0x0000ffff
-#define RWSEM_WAITING_BIAS              (-0x00010000)
+#ifdef CONFIG_X86_64
+# define RWSEM_ACTIVE_MASK              0xffffffffL
+#else
+# define RWSEM_ACTIVE_MASK              0x0000ffffL
+#endif
+
+#define RWSEM_UNLOCKED_VALUE            0x00000000L
+#define RWSEM_ACTIVE_BIAS               0x00000001L
+#define RWSEM_WAITING_BIAS              (-RWSEM_ACTIVE_MASK-1)
 #define RWSEM_ACTIVE_READ_BIAS          RWSEM_ACTIVE_BIAS
 #define RWSEM_ACTIVE_WRITE_BIAS         (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
 
+typedef signed long rwsem_count_t;
+
 struct rw_semaphore {
-        signed long             count;
+        rwsem_count_t           count;
         spinlock_t              wait_lock;
         struct list_head        wait_list;
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -105,7 +117,7 @@ do {								\
 static inline void __down_read(struct rw_semaphore *sem)
 {
         asm volatile("# beginning down_read\n\t"
-                     LOCK_PREFIX "  inc%z0      (%1)\n\t"
+                     LOCK_PREFIX _ASM_INC "(%1)\n\t"
                      /* adds 0x00000001, returns the old value */
                      "  jns        1f\n"
                      "  call call_rwsem_down_read_failed\n"
@@ -121,7 +133,7 @@ static inline void __down_read(struct rw_semaphore *sem)
  */
 static inline int __down_read_trylock(struct rw_semaphore *sem)
 {
-        __s32 result, tmp;
+        rwsem_count_t result, tmp;
         asm volatile("# beginning __down_read_trylock\n\t"
                      "  mov          %0,%1\n\t"
                      "1:\n\t"
@@ -143,7 +155,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
  */
 static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
 {
-        int tmp;
+        rwsem_count_t tmp;
 
         tmp = RWSEM_ACTIVE_WRITE_BIAS;
         asm volatile("# beginning down_write\n\t"
@@ -170,9 +182,9 @@ static inline void __down_write(struct rw_semaphore *sem)
  */
 static inline int __down_write_trylock(struct rw_semaphore *sem)
 {
-        signed long ret = cmpxchg(&sem->count,
-                                  RWSEM_UNLOCKED_VALUE,
-                                  RWSEM_ACTIVE_WRITE_BIAS);
+        rwsem_count_t ret = cmpxchg(&sem->count,
+                                    RWSEM_UNLOCKED_VALUE,
+                                    RWSEM_ACTIVE_WRITE_BIAS);
         if (ret == RWSEM_UNLOCKED_VALUE)
                 return 1;
         return 0;
@@ -183,7 +195,7 @@ static inline int __down_write_trylock(struct rw_semaphore *sem)
  */
 static inline void __up_read(struct rw_semaphore *sem)
 {
-        __s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
+        rwsem_count_t tmp = -RWSEM_ACTIVE_READ_BIAS;
         asm volatile("# beginning __up_read\n\t"
                      LOCK_PREFIX "  xadd      %1,(%2)\n\t"
                      /* subtracts 1, returns the old value */
@@ -201,7 +213,7 @@ static inline void __up_read(struct rw_semaphore *sem)
  */
 static inline void __up_write(struct rw_semaphore *sem)
 {
-        unsigned long tmp;
+        rwsem_count_t tmp;
         asm volatile("# beginning __up_write\n\t"
                      LOCK_PREFIX "  xadd      %1,(%2)\n\t"
                      /* tries to transition
@@ -221,33 +233,38 @@ static inline void __up_write(struct rw_semaphore *sem)
 static inline void __downgrade_write(struct rw_semaphore *sem)
 {
         asm volatile("# beginning __downgrade_write\n\t"
-                     LOCK_PREFIX "  add%z0    %2,(%1)\n\t"
-                     /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
+                     LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t"
+                     /*
+                      * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386)
+                      *     0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64)
+                      */
                      "  jns       1f\n\t"
                      "  call call_rwsem_downgrade_wake\n"
                      "1:\n\t"
                      "# ending __downgrade_write\n"
                      : "+m" (sem->count)
-                     : "a" (sem), "i" (-RWSEM_WAITING_BIAS)
+                     : "a" (sem), "er" (-RWSEM_WAITING_BIAS)
                      : "memory", "cc");
 }
 
 /*
  * implement atomic add functionality
  */
-static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
+static inline void rwsem_atomic_add(rwsem_count_t delta,
+                                    struct rw_semaphore *sem)
 {
-        asm volatile(LOCK_PREFIX "add%z0 %1,%0"
+        asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0"
                      : "+m" (sem->count)
-                     : "ir" (delta));
+                     : "er" (delta));
 }
 
 /*
  * implement exchange and add functionality
  */
-static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
+static inline rwsem_count_t rwsem_atomic_update(rwsem_count_t delta,
+                                                struct rw_semaphore *sem)
 {
-        int tmp = delta;
+        rwsem_count_t tmp = delta;
 
         asm volatile(LOCK_PREFIX "xadd %0,%1"
                      : "+r" (tmp), "+m" (sem->count)
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 710cc1405866..419386c24b82 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -39,4 +39,5 @@ else
         lib-y += thunk_64.o clear_page_64.o copy_page_64.o
         lib-y += memmove_64.o memset_64.o
         lib-y += copy_user_64.o rwlock_64.o copy_user_nocache_64.o
+        lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem_64.o
 endif
diff --git a/arch/x86/lib/rwsem_64.S b/arch/x86/lib/rwsem_64.S
new file mode 100644
index 000000000000..15acecf0d7aa
--- /dev/null
+++ b/arch/x86/lib/rwsem_64.S
@@ -0,0 +1,81 @@
+/*
+ * x86-64 rwsem wrappers
+ *
+ * This interfaces the inline asm code to the slow-path
+ * C routines. We need to save the call-clobbered regs
+ * that the asm does not mark as clobbered, and move the
+ * argument from %rax to %rdi.
+ *
+ * NOTE! We don't need to save %rax, because the functions
+ * will always return the semaphore pointer in %rax (which
+ * is also the input argument to these helpers)
+ *
+ * The following can clobber %rdx because the asm clobbers it:
+ *   call_rwsem_down_write_failed
+ *   call_rwsem_wake
+ * but %rdi, %rsi, %rcx, %r8-r11 always need saving.
+ */
+
+#include <linux/linkage.h>
+#include <asm/rwlock.h>
+#include <asm/alternative-asm.h>
+#include <asm/frame.h>
+#include <asm/dwarf2.h>
+
+#define save_common_regs \
+        pushq %rdi; \
+        pushq %rsi; \
+        pushq %rcx; \
+        pushq %r8; \
+        pushq %r9; \
+        pushq %r10; \
+        pushq %r11
+
+#define restore_common_regs \
+        popq %r11; \
+        popq %r10; \
+        popq %r9; \
+        popq %r8; \
+        popq %rcx; \
+        popq %rsi; \
+        popq %rdi
+
+/* Fix up special calling conventions */
+ENTRY(call_rwsem_down_read_failed)
+        save_common_regs
+        pushq %rdx
+        movq %rax,%rdi
+        call rwsem_down_read_failed
+        popq %rdx
+        restore_common_regs
+        ret
+        ENDPROC(call_rwsem_down_read_failed)
+
+ENTRY(call_rwsem_down_write_failed)
+        save_common_regs
+        movq %rax,%rdi
+        call rwsem_down_write_failed
+        restore_common_regs
+        ret
+        ENDPROC(call_rwsem_down_write_failed)
+
+ENTRY(call_rwsem_wake)
+        decw %dx    /* do nothing if still outstanding active readers */
+        jnz 1f
+        save_common_regs
+        movq %rax,%rdi
+        call rwsem_wake
+        restore_common_regs
+1:      ret
+        ENDPROC(call_rwsem_wake)
+
+/* Fix up special calling conventions */
+ENTRY(call_rwsem_downgrade_wake)
+        save_common_regs
+        pushq %rdx
+        movq %rax,%rdi
+        call rwsem_downgrade_wake
+        popq %rdx
+        restore_common_regs
+        ret
+        ENDPROC(call_rwsem_downgrade_wake)