6 files changed, 204 insertions, 38 deletions

diff --git a/kernel/mutex-debug.c b/kernel/mutex-debug.c
index 1d94160eb532..50d022e5a560 100644
--- a/kernel/mutex-debug.c
+++ b/kernel/mutex-debug.c
@@ -26,11 +26,6 @@
 /*
  * Must be called with lock->wait_lock held.
  */
-void debug_mutex_set_owner(struct mutex *lock, struct thread_info *new_owner)
-{
-        lock->owner = new_owner;
-}
-
 void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter)
 {
         memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter));
@@ -59,7 +54,6 @@ void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
 
         /* Mark the current thread as blocked on the lock: */
         ti->task->blocked_on = waiter;
-        waiter->lock = lock;
 }
 
 void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
@@ -82,7 +76,7 @@ void debug_mutex_unlock(struct mutex *lock)
         DEBUG_LOCKS_WARN_ON(lock->magic != lock);
         DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
         DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
-        DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
+        mutex_clear_owner(lock);
 }
 
 void debug_mutex_init(struct mutex *lock, const char *name,
@@ -95,7 +89,6 @@ void debug_mutex_init(struct mutex *lock, const char *name,
         debug_check_no_locks_freed((void *)lock, sizeof(*lock));
         lockdep_init_map(&lock->dep_map, name, key, 0);
 #endif
-        lock->owner = NULL;
         lock->magic = lock;
 }
 
diff --git a/kernel/mutex-debug.h b/kernel/mutex-debug.h
index babfbdfc534b..6b2d735846a5 100644
--- a/kernel/mutex-debug.h
+++ b/kernel/mutex-debug.h
@@ -13,14 +13,6 @@
 /*
  * This must be called with lock->wait_lock held.
  */
-extern void
-debug_mutex_set_owner(struct mutex *lock, struct thread_info *new_owner);
-
-static inline void debug_mutex_clear_owner(struct mutex *lock)
-{
-        lock->owner = NULL;
-}
-
 extern void debug_mutex_lock_common(struct mutex *lock,
                                     struct mutex_waiter *waiter);
 extern void debug_mutex_wake_waiter(struct mutex *lock,
@@ -35,6 +27,16 @@ extern void debug_mutex_unlock(struct mutex *lock);
 extern void debug_mutex_init(struct mutex *lock, const char *name,
                              struct lock_class_key *key);
 
+static inline void mutex_set_owner(struct mutex *lock)
+{
+        lock->owner = current_thread_info();
+}
+
+static inline void mutex_clear_owner(struct mutex *lock)
+{
+        lock->owner = NULL;
+}
+
 #define spin_lock_mutex(lock, flags)                    \
         do {                                            \
                 struct mutex *l = container_of(lock, struct mutex, wait_lock); \
diff --git a/kernel/mutex.c b/kernel/mutex.c
index 4f45d4b658ef..5d79781394a3 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -10,6 +10,11 @@
  * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
  * David Howells for suggestions and improvements.
  *
+ *  - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
+ *    from the -rt tree, where it was originally implemented for rtmutexes
+ *    by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
+ *    and Sven Dietrich.
+ *
  * Also see Documentation/mutex-design.txt.
  */
 #include <linux/mutex.h>
@@ -46,6 +51,7 @@ __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
         atomic_set(&lock->count, 1);
         spin_lock_init(&lock->wait_lock);
         INIT_LIST_HEAD(&lock->wait_list);
+        mutex_clear_owner(lock);
 
         debug_mutex_init(lock, name, key);
 }
@@ -91,6 +97,7 @@ void inline __sched mutex_lock(struct mutex *lock)
          * 'unlocked' into 'locked' state.
          */
         __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
+        mutex_set_owner(lock);
 }
 
 EXPORT_SYMBOL(mutex_lock);
@@ -115,6 +122,14 @@ void __sched mutex_unlock(struct mutex *lock)
          * The unlocking fastpath is the 0->1 transition from 'locked'
          * into 'unlocked' state:
          */
+#ifndef CONFIG_DEBUG_MUTEXES
+        /*
+         * When debugging is enabled we must not clear the owner before time,
+         * the slow path will always be taken, and that clears the owner field
+         * after verifying that it was indeed current.
+         */
+        mutex_clear_owner(lock);
+#endif
         __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
 }
 
@@ -129,21 +144,75 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
 {
         struct task_struct *task = current;
         struct mutex_waiter waiter;
-        unsigned int old_val;
         unsigned long flags;
 
+        preempt_disable();
+        mutex_acquire(&lock->dep_map, subclass, 0, ip);
+#if defined(CONFIG_SMP) && !defined(CONFIG_DEBUG_MUTEXES)
+        /*
+         * Optimistic spinning.
+         *
+         * We try to spin for acquisition when we find that there are no
+         * pending waiters and the lock owner is currently running on a
+         * (different) CPU.
+         *
+         * The rationale is that if the lock owner is running, it is likely to
+         * release the lock soon.
+         *
+         * Since this needs the lock owner, and this mutex implementation
+         * doesn't track the owner atomically in the lock field, we need to
+         * track it non-atomically.
+         *
+         * We can't do this for DEBUG_MUTEXES because that relies on wait_lock
+         * to serialize everything.
+         */
+
+        for (;;) {
+                struct thread_info *owner;
+
+                /*
+                 * If there's an owner, wait for it to either
+                 * release the lock or go to sleep.
+                 */
+                owner = ACCESS_ONCE(lock->owner);
+                if (owner && !mutex_spin_on_owner(lock, owner))
+                        break;
+
+                if (atomic_cmpxchg(&lock->count, 1, 0) == 1) {
+                        lock_acquired(&lock->dep_map, ip);
+                        mutex_set_owner(lock);
+                        preempt_enable();
+                        return 0;
+                }
+
+                /*
+                 * When there's no owner, we might have preempted between the
+                 * owner acquiring the lock and setting the owner field. If
+                 * we're an RT task that will live-lock because we won't let
+                 * the owner complete.
+                 */
+                if (!owner && (need_resched() || rt_task(task)))
+                        break;
+
+                /*
+                 * The cpu_relax() call is a compiler barrier which forces
+                 * everything in this loop to be re-loaded. We don't need
+                 * memory barriers as we'll eventually observe the right
+                 * values at the cost of a few extra spins.
+                 */
+                cpu_relax();
+        }
+#endif
         spin_lock_mutex(&lock->wait_lock, flags);
 
         debug_mutex_lock_common(lock, &waiter);
-        mutex_acquire(&lock->dep_map, subclass, 0, ip);
         debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
 
         /* add waiting tasks to the end of the waitqueue (FIFO): */
         list_add_tail(&waiter.list, &lock->wait_list);
         waiter.task = task;
 
-        old_val = atomic_xchg(&lock->count, -1);
-        if (old_val == 1)
+        if (atomic_xchg(&lock->count, -1) == 1)
                 goto done;
 
         lock_contended(&lock->dep_map, ip);
@@ -158,8 +227,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                  * that when we release the lock, we properly wake up the
                  * other waiters:
                  */
-                old_val = atomic_xchg(&lock->count, -1);
-                if (old_val == 1)
+                if (atomic_xchg(&lock->count, -1) == 1)
                         break;
 
                 /*
@@ -173,21 +241,22 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                         spin_unlock_mutex(&lock->wait_lock, flags);
 
                         debug_mutex_free_waiter(&waiter);
+                        preempt_enable();
                         return -EINTR;
                 }
                 __set_task_state(task, state);
 
                 /* didnt get the lock, go to sleep: */
                 spin_unlock_mutex(&lock->wait_lock, flags);
-                schedule();
+                __schedule();
                 spin_lock_mutex(&lock->wait_lock, flags);
         }
 
 done:
         lock_acquired(&lock->dep_map, ip);
         /* got the lock - rejoice! */
-        mutex_remove_waiter(lock, &waiter, task_thread_info(task));
-        debug_mutex_set_owner(lock, task_thread_info(task));
+        mutex_remove_waiter(lock, &waiter, current_thread_info());
+        mutex_set_owner(lock);
 
         /* set it to 0 if there are no waiters left: */
         if (likely(list_empty(&lock->wait_list)))
@@ -196,6 +265,7 @@ done:
         spin_unlock_mutex(&lock->wait_lock, flags);
 
         debug_mutex_free_waiter(&waiter);
+        preempt_enable();
 
         return 0;
 }
@@ -222,7 +292,8 @@ int __sched
 mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
 {
         might_sleep();
-        return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, _RET_IP_);
+        return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
+                                   subclass, _RET_IP_);
 }
 
 EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
@@ -260,8 +331,6 @@ __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
                 wake_up_process(waiter->task);
         }
 
-        debug_mutex_clear_owner(lock);
-
         spin_unlock_mutex(&lock->wait_lock, flags);
 }
 
@@ -298,18 +367,30 @@ __mutex_lock_interruptible_slowpath(atomic_t *lock_count);
  */
 int __sched mutex_lock_interruptible(struct mutex *lock)
 {
+        int ret;
+
         might_sleep();
-        return __mutex_fastpath_lock_retval
+        ret =  __mutex_fastpath_lock_retval
                         (&lock->count, __mutex_lock_interruptible_slowpath);
+        if (!ret)
+                mutex_set_owner(lock);
+
+        return ret;
 }
 
 EXPORT_SYMBOL(mutex_lock_interruptible);
 
 int __sched mutex_lock_killable(struct mutex *lock)
 {
+        int ret;
+
         might_sleep();
-        return __mutex_fastpath_lock_retval
+        ret = __mutex_fastpath_lock_retval
                         (&lock->count, __mutex_lock_killable_slowpath);
+        if (!ret)
+                mutex_set_owner(lock);
+
+        return ret;
 }
 EXPORT_SYMBOL(mutex_lock_killable);
 
@@ -352,9 +433,10 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
 
         prev = atomic_xchg(&lock->count, -1);
         if (likely(prev == 1)) {
-                debug_mutex_set_owner(lock, current_thread_info());
+                mutex_set_owner(lock);
                 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
         }
+
         /* Set it back to 0 if there are no waiters: */
         if (likely(list_empty(&lock->wait_list)))
                 atomic_set(&lock->count, 0);
@@ -380,8 +462,13 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
  */
 int __sched mutex_trylock(struct mutex *lock)
 {
-        return __mutex_fastpath_trylock(&lock->count,
-                                        __mutex_trylock_slowpath);
+        int ret;
+
+        ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath);
+        if (ret)
+                mutex_set_owner(lock);
+
+        return ret;
 }
 
 EXPORT_SYMBOL(mutex_trylock);
diff --git a/kernel/mutex.h b/kernel/mutex.h
index a075dafbb290..67578ca48f94 100644
--- a/kernel/mutex.h
+++ b/kernel/mutex.h
@@ -16,8 +16,26 @@
 #define mutex_remove_waiter(lock, waiter, ti) \
                 __list_del((waiter)->list.prev, (waiter)->list.next)
 
-#define debug_mutex_set_owner(lock, new_owner)          do { } while (0)
-#define debug_mutex_clear_owner(lock)                   do { } while (0)
+#ifdef CONFIG_SMP
+static inline void mutex_set_owner(struct mutex *lock)
+{
+        lock->owner = current_thread_info();
+}
+
+static inline void mutex_clear_owner(struct mutex *lock)
+{
+        lock->owner = NULL;
+}
+#else
+static inline void mutex_set_owner(struct mutex *lock)
+{
+}
+
+static inline void mutex_clear_owner(struct mutex *lock)
+{
+}
+#endif
+
 #define debug_mutex_wake_waiter(lock, waiter)           do { } while (0)
 #define debug_mutex_free_waiter(waiter)                 do { } while (0)
 #define debug_mutex_add_waiter(lock, waiter, ti)        do { } while (0)
diff --git a/kernel/sched.c b/kernel/sched.c
index 8ee437a5ec1d..186c6fd08acf 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -4548,15 +4548,13 @@ pick_next_task(struct rq *rq, struct task_struct *prev)
 /*
  * schedule() is the main scheduler function.
  */
-asmlinkage void __sched schedule(void)
+asmlinkage void __sched __schedule(void)
 {
         struct task_struct *prev, *next;
         unsigned long *switch_count;
         struct rq *rq;
         int cpu;
 
-need_resched:
-        preempt_disable();
         cpu = smp_processor_id();
         rq = cpu_rq(cpu);
         rcu_qsctr_inc(cpu);
@@ -4613,13 +4611,80 @@ need_resched_nonpreemptible:
 
         if (unlikely(reacquire_kernel_lock(current) < 0))
                 goto need_resched_nonpreemptible;
+}
 
+asmlinkage void __sched schedule(void)
+{
+need_resched:
+        preempt_disable();
+        __schedule();
         preempt_enable_no_resched();
         if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
                 goto need_resched;
 }
 EXPORT_SYMBOL(schedule);
 
+#ifdef CONFIG_SMP
+/*
+ * Look out! "owner" is an entirely speculative pointer
+ * access and not reliable.
+ */
+int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner)
+{
+        unsigned int cpu;
+        struct rq *rq;
+
+        if (!sched_feat(OWNER_SPIN))
+                return 0;
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+        /*
+         * Need to access the cpu field knowing that
+         * DEBUG_PAGEALLOC could have unmapped it if
+         * the mutex owner just released it and exited.
+         */
+        if (probe_kernel_address(&owner->cpu, cpu))
+                goto out;
+#else
+        cpu = owner->cpu;
+#endif
+
+        /*
+         * Even if the access succeeded (likely case),
+         * the cpu field may no longer be valid.
+         */
+        if (cpu >= nr_cpumask_bits)
+                goto out;
+
+        /*
+         * We need to validate that we can do a
+         * get_cpu() and that we have the percpu area.
+         */
+        if (!cpu_online(cpu))
+                goto out;
+
+        rq = cpu_rq(cpu);
+
+        for (;;) {
+                /*
+                 * Owner changed, break to re-assess state.
+                 */
+                if (lock->owner != owner)
+                        break;
+
+                /*
+                 * Is that owner really running on that cpu?
+                 */
+                if (task_thread_info(rq->curr) != owner || need_resched())
+                        return 0;
+
+                cpu_relax();
+        }
+out:
+        return 1;
+}
+#endif
+
 #ifdef CONFIG_PREEMPT
 /*
  * this is the entry point to schedule() from in-kernel preemption
diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index da5d93b5d2c6..07bc02e99ab1 100644
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -13,3 +13,4 @@ SCHED_FEAT(LB_WAKEUP_UPDATE, 1)
 SCHED_FEAT(ASYM_EFF_LOAD, 1)
 SCHED_FEAT(WAKEUP_OVERLAP, 0)
 SCHED_FEAT(LAST_BUDDY, 1)
+SCHED_FEAT(OWNER_SPIN, 1)