6 files changed, 73 insertions, 28 deletions

diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index 5890beb021c4..9412d6558c88 100644
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -3,9 +3,11 @@
 
 #ifdef __ASSEMBLY__
 # define __ASM_FORM(x)  x
+# define __ASM_FORM_COMMA(x) x,
 # define __ASM_EX_SEC   .section __ex_table, "a"
 #else
 # define __ASM_FORM(x)  " " #x " "
+# define __ASM_FORM_COMMA(x) " " #x ","
 # define __ASM_EX_SEC   " .section __ex_table,\"a\"\n"
 #endif
 
diff --git a/arch/x86/include/asm/rwlock.h b/arch/x86/include/asm/rwlock.h
index 6a8c0d645108..a5370a03d90c 100644
--- a/arch/x86/include/asm/rwlock.h
+++ b/arch/x86/include/asm/rwlock.h
@@ -1,7 +1,48 @@
 #ifndef _ASM_X86_RWLOCK_H
 #define _ASM_X86_RWLOCK_H
 
-#define RW_LOCK_BIAS             0x01000000
+#include <asm/asm.h>
+
+#if CONFIG_NR_CPUS <= 2048
+
+#ifndef __ASSEMBLY__
+typedef union {
+        s32 lock;
+        s32 write;
+} arch_rwlock_t;
+#endif
+
+#define RW_LOCK_BIAS            0x00100000
+#define READ_LOCK_SIZE(insn)    __ASM_FORM(insn##l)
+#define READ_LOCK_ATOMIC(n)     atomic_##n
+#define WRITE_LOCK_ADD(n)       __ASM_FORM_COMMA(addl n)
+#define WRITE_LOCK_SUB(n)       __ASM_FORM_COMMA(subl n)
+#define WRITE_LOCK_CMP          RW_LOCK_BIAS
+
+#else /* CONFIG_NR_CPUS > 2048 */
+
+#include <linux/const.h>
+
+#ifndef __ASSEMBLY__
+typedef union {
+        s64 lock;
+        struct {
+                u32 read;
+                s32 write;
+        };
+} arch_rwlock_t;
+#endif
+
+#define RW_LOCK_BIAS            (_AC(1,L) << 32)
+#define READ_LOCK_SIZE(insn)    __ASM_FORM(insn##q)
+#define READ_LOCK_ATOMIC(n)     atomic64_##n
+#define WRITE_LOCK_ADD(n)       __ASM_FORM(incl)
+#define WRITE_LOCK_SUB(n)       __ASM_FORM(decl)
+#define WRITE_LOCK_CMP          1
+
+#endif /* CONFIG_NR_CPUS */
+
+#define __ARCH_RW_LOCK_UNLOCKED         { RW_LOCK_BIAS }
 
 /* Actual code is in asm/spinlock.h or in arch/x86/lib/rwlock.S */
 
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 3089f70c0c52..e9e51f710e6c 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -2,7 +2,6 @@
 #define _ASM_X86_SPINLOCK_H
 
 #include <asm/atomic.h>
-#include <asm/rwlock.h>
 #include <asm/page.h>
 #include <asm/processor.h>
 #include <linux/compiler.h>
@@ -234,7 +233,7 @@ static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
  */
 static inline int arch_read_can_lock(arch_rwlock_t *lock)
 {
-        return (int)(lock)->lock > 0;
+        return lock->lock > 0;
 }
 
 /**
@@ -243,12 +242,12 @@ static inline int arch_read_can_lock(arch_rwlock_t *lock)
  */
 static inline int arch_write_can_lock(arch_rwlock_t *lock)
 {
-        return (lock)->lock == RW_LOCK_BIAS;
+        return lock->write == WRITE_LOCK_CMP;
 }
 
 static inline void arch_read_lock(arch_rwlock_t *rw)
 {
-        asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t"
+        asm volatile(LOCK_PREFIX READ_LOCK_SIZE(dec) " (%0)\n\t"
                      "jns 1f\n"
                      "call __read_lock_failed\n\t"
                      "1:\n"
@@ -257,47 +256,55 @@ static inline void arch_read_lock(arch_rwlock_t *rw)
 
 static inline void arch_write_lock(arch_rwlock_t *rw)
 {
-        asm volatile(LOCK_PREFIX " subl %1,(%0)\n\t"
+        asm volatile(LOCK_PREFIX WRITE_LOCK_SUB(%1) "(%0)\n\t"
                      "jz 1f\n"
                      "call __write_lock_failed\n\t"
                      "1:\n"
-                     ::LOCK_PTR_REG (rw), "i" (RW_LOCK_BIAS) : "memory");
+                     ::LOCK_PTR_REG (&rw->write), "i" (RW_LOCK_BIAS)
+                     : "memory");
 }
 
 static inline int arch_read_trylock(arch_rwlock_t *lock)
 {
-        atomic_t *count = (atomic_t *)lock;
+        READ_LOCK_ATOMIC(t) *count = (READ_LOCK_ATOMIC(t) *)lock;
 
-        if (atomic_dec_return(count) >= 0)
+        if (READ_LOCK_ATOMIC(dec_return)(count) >= 0)
                 return 1;
-        atomic_inc(count);
+        READ_LOCK_ATOMIC(inc)(count);
         return 0;
 }
 
 static inline int arch_write_trylock(arch_rwlock_t *lock)
 {
-        atomic_t *count = (atomic_t *)lock;
+        atomic_t *count = (atomic_t *)&lock->write;
 
-        if (atomic_sub_and_test(RW_LOCK_BIAS, count))
+        if (atomic_sub_and_test(WRITE_LOCK_CMP, count))
                 return 1;
-        atomic_add(RW_LOCK_BIAS, count);
+        atomic_add(WRITE_LOCK_CMP, count);
         return 0;
 }
 
 static inline void arch_read_unlock(arch_rwlock_t *rw)
 {
-        asm volatile(LOCK_PREFIX "incl %0" :"+m" (rw->lock) : : "memory");
+        asm volatile(LOCK_PREFIX READ_LOCK_SIZE(inc) " %0"
+                     :"+m" (rw->lock) : : "memory");
 }
 
 static inline void arch_write_unlock(arch_rwlock_t *rw)
 {
-        asm volatile(LOCK_PREFIX "addl %1, %0"
-                     : "+m" (rw->lock) : "i" (RW_LOCK_BIAS) : "memory");
+        asm volatile(LOCK_PREFIX WRITE_LOCK_ADD(%1) "%0"
+                     : "+m" (rw->write) : "i" (RW_LOCK_BIAS) : "memory");
 }
 
 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
 
+#undef READ_LOCK_SIZE
+#undef READ_LOCK_ATOMIC
+#undef WRITE_LOCK_ADD
+#undef WRITE_LOCK_SUB
+#undef WRITE_LOCK_CMP
+
 #define arch_spin_relax(lock)   cpu_relax()
 #define arch_read_relax(lock)   cpu_relax()
 #define arch_write_relax(lock)  cpu_relax()
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index dcb48b2edc11..7c7a486fcb68 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -11,10 +11,6 @@ typedef struct arch_spinlock {
 
 #define __ARCH_SPIN_LOCK_UNLOCKED       { 0 }
 
-typedef struct {
-        unsigned int lock;
-} arch_rwlock_t;
-
-#define __ARCH_RW_LOCK_UNLOCKED         { RW_LOCK_BIAS }
+#include <asm/rwlock.h>
 
 #endif /* _ASM_X86_SPINLOCK_TYPES_H */
diff --git a/arch/x86/lib/rwlock.S b/arch/x86/lib/rwlock.S
index fca17829caa8..1cad22139c88 100644
--- a/arch/x86/lib/rwlock.S
+++ b/arch/x86/lib/rwlock.S
@@ -15,12 +15,12 @@ ENTRY(__write_lock_failed)
         CFI_STARTPROC
         FRAME
 0:      LOCK_PREFIX
-        addl    $RW_LOCK_BIAS, (%__lock_ptr)
+        WRITE_LOCK_ADD($RW_LOCK_BIAS) (%__lock_ptr)
 1:      rep; nop
-        cmpl    $RW_LOCK_BIAS, (%__lock_ptr)
+        cmpl    $WRITE_LOCK_CMP, (%__lock_ptr)
         jne     1b
         LOCK_PREFIX
-        subl    $RW_LOCK_BIAS, (%__lock_ptr)
+        WRITE_LOCK_SUB($RW_LOCK_BIAS) (%__lock_ptr)
         jnz     0b
         ENDFRAME
         ret
@@ -31,12 +31,12 @@ ENTRY(__read_lock_failed)
         CFI_STARTPROC
         FRAME
 0:      LOCK_PREFIX
-        incl    (%__lock_ptr)
+        READ_LOCK_SIZE(inc) (%__lock_ptr)
 1:      rep; nop
-        cmpl    $1, (%__lock_ptr)
+        READ_LOCK_SIZE(cmp) $1, (%__lock_ptr)
         js      1b
         LOCK_PREFIX
-        decl    (%__lock_ptr)
+        READ_LOCK_SIZE(dec) (%__lock_ptr)
         js      0b
         ENDFRAME
         ret
diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
index d5b088b3ab81..a63efd6bb6a5 100644
--- a/arch/x86/lib/thunk_64.S
+++ b/arch/x86/lib/thunk_64.S
@@ -8,7 +8,6 @@
 #include <linux/linkage.h>
 #include <asm/dwarf2.h>
 #include <asm/calling.h>
-#include <asm/rwlock.h>
 
         /* rdi: arg1 ... normal C conventions. rax is saved/restored. */
         .macro THUNK name, func, put_ret_addr_in_rdi=0