5 files changed, 84 insertions, 3 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index b32ebf92b0ce..67e00740531c 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -16,6 +16,7 @@ config X86_64
         def_bool y
         depends on 64BIT
         select X86_DEV_DMA_OPS
+        select ARCH_USE_CMPXCHG_LOCKREF
 
 ### Arch settings
 config X86
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index e3ddd7db723f..e0e668422c75 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -34,6 +34,11 @@
 # define UNLOCK_LOCK_PREFIX
 #endif
 
+static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
+{
+        return lock.tickets.head == lock.tickets.tail;
+}
+
 /*
  * Ticket locks are conceptually two parts, one indicating the current head of
  * the queue, and the other indicating the current tail. The lock is acquired
diff --git a/include/linux/lockref.h b/include/linux/lockref.h
index 4c0af31c8d47..ca07b5028b01 100644
--- a/include/linux/lockref.h
+++ b/include/linux/lockref.h
@@ -17,8 +17,15 @@
 #include <linux/spinlock.h>
 
 struct lockref {
-        spinlock_t lock;
-        unsigned int count;
+        union {
+#ifdef CONFIG_CMPXCHG_LOCKREF
+                aligned_u64 lock_count;
+#endif
+                struct {
+                        spinlock_t lock;
+                        unsigned int count;
+                };
+        };
 };
 
 extern void lockref_get(struct lockref *);
diff --git a/lib/Kconfig b/lib/Kconfig
index 71d9f81f6eed..65561716c16c 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -48,6 +48,16 @@ config STMP_DEVICE
 config PERCPU_RWSEM
         boolean
 
+config ARCH_USE_CMPXCHG_LOCKREF
+        bool
+
+config CMPXCHG_LOCKREF
+        def_bool y if ARCH_USE_CMPXCHG_LOCKREF
+        depends on SMP
+        depends on !GENERIC_LOCKBREAK
+        depends on !DEBUG_SPINLOCK
+        depends on !DEBUG_LOCK_ALLOC
+
 config CRC_CCITT
         tristate "CRC-CCITT functions"
         help
diff --git a/lib/lockref.c b/lib/lockref.c
index a9a4f4e1eff5..7819c2d1d315 100644
--- a/lib/lockref.c
+++ b/lib/lockref.c
@@ -1,6 +1,33 @@
 #include <linux/export.h>
 #include <linux/lockref.h>
 
+#ifdef CONFIG_CMPXCHG_LOCKREF
+
+/*
+ * Note that the "cmpxchg()" reloads the "old" value for the
+ * failure case.
+ */
+#define CMPXCHG_LOOP(CODE, SUCCESS) do {                                        \
+        struct lockref old;                                                     \
+        BUILD_BUG_ON(sizeof(old) != 8);                                         \
+        old.lock_count = ACCESS_ONCE(lockref->lock_count);                      \
+        while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {     \
+                struct lockref new = old, prev = old;                           \
+                CODE                                                            \
+                old.lock_count = cmpxchg(&lockref->lock_count,                  \
+                                         old.lock_count, new.lock_count);       \
+                if (likely(old.lock_count == prev.lock_count)) {                \
+                        SUCCESS;                                                \
+                }                                                               \
+        }                                                                       \
+} while (0)
+
+#else
+
+#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
+
+#endif
+
 /**
  * lockref_get - Increments reference count unconditionally
  * @lockcnt: pointer to lockref structure
@@ -10,6 +37,12 @@
  */
 void lockref_get(struct lockref *lockref)
 {
+        CMPXCHG_LOOP(
+                new.count++;
+        ,
+                return;
+        );
+
         spin_lock(&lockref->lock);
         lockref->count++;
         spin_unlock(&lockref->lock);
@@ -23,9 +56,18 @@ EXPORT_SYMBOL(lockref_get);
  */
 int lockref_get_not_zero(struct lockref *lockref)
 {
-        int retval = 0;
+        int retval;
+
+        CMPXCHG_LOOP(
+                new.count++;
+                if (!old.count)
+                        return 0;
+        ,
+                return 1;
+        );
 
         spin_lock(&lockref->lock);
+        retval = 0;
         if (lockref->count) {
                 lockref->count++;
                 retval = 1;
@@ -43,6 +85,14 @@ EXPORT_SYMBOL(lockref_get_not_zero);
  */
 int lockref_get_or_lock(struct lockref *lockref)
 {
+        CMPXCHG_LOOP(
+                new.count++;
+                if (!old.count)
+                        break;
+        ,
+                return 1;
+        );
+
         spin_lock(&lockref->lock);
         if (!lockref->count)
                 return 0;
@@ -59,6 +109,14 @@ EXPORT_SYMBOL(lockref_get_or_lock);
  */
 int lockref_put_or_lock(struct lockref *lockref)
 {
+        CMPXCHG_LOOP(
+                new.count--;
+                if (old.count <= 1)
+                        break;
+        ,
+                return 1;
+        );
+
         spin_lock(&lockref->lock);
         if (lockref->count <= 1)
                 return 0;