3 files changed, 248 insertions, 33 deletions

diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 03f3b05e8ec1..3e108f154cb6 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -16,6 +16,7 @@
 
 #include <linux/atomic.h>
 
+struct optimistic_spin_queue;
 struct rw_semaphore;
 
 #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK
@@ -23,9 +24,17 @@ struct rw_semaphore;
 #else
 /* All arch specific implementations share the same struct */
 struct rw_semaphore {
-        long                    count;
-        raw_spinlock_t          wait_lock;
-        struct list_head        wait_list;
+        long count;
+        raw_spinlock_t wait_lock;
+        struct list_head wait_list;
+#ifdef CONFIG_SMP
+        /*
+         * Write owner. Used as a speculative check to see
+         * if the owner is running on the cpu.
+         */
+        struct task_struct *owner;
+        struct optimistic_spin_queue *osq; /* spinner MCS lock */
+#endif
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
         struct lockdep_map      dep_map;
 #endif
@@ -55,11 +64,21 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem)
 # define __RWSEM_DEP_MAP_INIT(lockname)
 #endif
 
+#ifdef CONFIG_SMP
+#define __RWSEM_INITIALIZER(name)                       \
+        { RWSEM_UNLOCKED_VALUE,                         \
+          __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),     \
+          LIST_HEAD_INIT((name).wait_list),             \
+          NULL, /* owner */                             \
+          NULL /* mcs lock */                           \
+          __RWSEM_DEP_MAP_INIT(name) }
+#else
 #define __RWSEM_INITIALIZER(name)                       \
         { RWSEM_UNLOCKED_VALUE,                         \
           __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),     \
           LIST_HEAD_INIT((name).wait_list)              \
           __RWSEM_DEP_MAP_INIT(name) }
+#endif
 
 #define DECLARE_RWSEM(name) \
         struct rw_semaphore name = __RWSEM_INITIALIZER(name)
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index b4219ff87b8c..4a75278142cd 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -5,11 +5,17 @@
  *
  * Writer lock-stealing by Alex Shi <alex.shi@intel.com>
  * and Michel Lespinasse <walken@google.com>
+ *
+ * Optimistic spinning by Tim Chen <tim.c.chen@intel.com>
+ * and Davidlohr Bueso <davidlohr@hp.com>. Based on mutexes.
  */
 #include <linux/rwsem.h>
 #include <linux/sched.h>
 #include <linux/init.h>
 #include <linux/export.h>
+#include <linux/sched/rt.h>
+
+#include "mcs_spinlock.h"
 
 /*
  * Guide to the rw_semaphore's count field for common values.
@@ -76,6 +82,10 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
         sem->count = RWSEM_UNLOCKED_VALUE;
         raw_spin_lock_init(&sem->wait_lock);
         INIT_LIST_HEAD(&sem->wait_list);
+#ifdef CONFIG_SMP
+        sem->owner = NULL;
+        sem->osq = NULL;
+#endif
 }
 
 EXPORT_SYMBOL(__init_rwsem);
@@ -190,7 +200,7 @@ __rwsem_do_wake(struct rw_semaphore *sem, enum rwsem_wake_type wake_type)
 }
 
 /*
- * wait for the read lock to be granted
+ * Wait for the read lock to be granted
  */
 __visible
 struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)
@@ -237,64 +247,221 @@ struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)
         return sem;
 }
 
+static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)
+{
+        if (!(count & RWSEM_ACTIVE_MASK)) {
+                /* try acquiring the write lock */
+                if (sem->count == RWSEM_WAITING_BIAS &&
+                    cmpxchg(&sem->count, RWSEM_WAITING_BIAS,
+                            RWSEM_ACTIVE_WRITE_BIAS) == RWSEM_WAITING_BIAS) {
+                        if (!list_is_singular(&sem->wait_list))
+                                rwsem_atomic_update(RWSEM_WAITING_BIAS, sem);
+                        return true;
+                }
+        }
+        return false;
+}
+
+#ifdef CONFIG_SMP
 /*
- * wait until we successfully acquire the write lock
+ * Try to acquire write lock before the writer has been put on wait queue.
+ */
+static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)
+{
+        long old, count = ACCESS_ONCE(sem->count);
+
+        while (true) {
+                if (!(count == 0 || count == RWSEM_WAITING_BIAS))
+                        return false;
+
+                old = cmpxchg(&sem->count, count, count + RWSEM_ACTIVE_WRITE_BIAS);
+                if (old == count)
+                        return true;
+
+                count = old;
+        }
+}
+
+static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
+{
+        struct task_struct *owner;
+        bool on_cpu = true;
+
+        if (need_resched())
+                return 0;
+
+        rcu_read_lock();
+        owner = ACCESS_ONCE(sem->owner);
+        if (owner)
+                on_cpu = owner->on_cpu;
+        rcu_read_unlock();
+
+        /*
+         * If sem->owner is not set, the rwsem owner may have
+         * just acquired it and not set the owner yet or the rwsem
+         * has been released.
+         */
+        return on_cpu;
+}
+
+static inline bool owner_running(struct rw_semaphore *sem,
+                                 struct task_struct *owner)
+{
+        if (sem->owner != owner)
+                return false;
+
+        /*
+         * Ensure we emit the owner->on_cpu, dereference _after_ checking
+         * sem->owner still matches owner, if that fails, owner might
+         * point to free()d memory, if it still matches, the rcu_read_lock()
+         * ensures the memory stays valid.
+         */
+        barrier();
+
+        return owner->on_cpu;
+}
+
+static noinline
+bool rwsem_spin_on_owner(struct rw_semaphore *sem, struct task_struct *owner)
+{
+        rcu_read_lock();
+        while (owner_running(sem, owner)) {
+                if (need_resched())
+                        break;
+
+                arch_mutex_cpu_relax();
+        }