Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Ingo Molnar:
 "Here are the locking changes in this cycle:

   - rwsem unification and simpler micro-optimizations to prepare for
     more intrusive (and more lucrative) scalability improvements in
     v5.3 (Waiman Long)

   - Lockdep irq state tracking flag usage cleanups (Frederic
     Weisbecker)

   - static key improvements (Jakub Kicinski, Peter Zijlstra)

   - misc updates, cleanups and smaller fixes"

* 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (26 commits)
  locking/lockdep: Remove unnecessary unlikely()
  locking/static_key: Don't take sleeping locks in __static_key_slow_dec_deferred()
  locking/static_key: Factor out the fast path of static_key_slow_dec()
  locking/static_key: Add support for deferred static branches
  locking/lockdep: Test all incompatible scenarios at once in check_irq_usage()
  locking/lockdep: Avoid bogus Clang warning
  locking/lockdep: Generate LOCKF_ bit composites
  locking/lockdep: Use expanded masks on find_usage_*() functions
  locking/lockdep: Map remaining magic numbers to lock usage mask names
  locking/lockdep: Move valid_state() inside CONFIG_TRACE_IRQFLAGS && CONFIG_PROVE_LOCKING
  locking/rwsem: Prevent unneeded warning during locking selftest
  locking/rwsem: Optimize rwsem structure for uncontended lock acquisition
  locking/rwsem: Enable lock event counting
  locking/lock_events: Don't show pvqspinlock events on bare metal
  locking/lock_events: Make lock_events available for all archs & other locks
  locking/qspinlock_stat: Introduce generic lockevent_*() counting APIs
  locking/rwsem: Enhance DEBUG_RWSEMS_WARN_ON() macro
  locking/rwsem: Add debug check for __down_read*()
  locking/rwsem: Micro-optimize rwsem_try_read_lock_unqueued()
  locking/rwsem: Move rwsem internal function declarations to rwsem-xadd.h
  ...

1 files changed, 105 insertions, 99 deletions

diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index fbe96341beee..6b3ee9948bf1 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -147,6 +147,7 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem,
                          * will notice the queued writer.
                          */
                         wake_q_add(wake_q, waiter->task);
+                        lockevent_inc(rwsem_wake_writer);
                 }
 
                 return;
@@ -176,9 +177,8 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem,
                         goto try_reader_grant;
                 }
                 /*
-                 * It is not really necessary to set it to reader-owned here,
-                 * but it gives the spinners an early indication that the
-                 * readers now have the lock.
+                 * Set it to reader-owned to give spinners an early
+                 * indication that readers now have the lock.
                  */
                 __rwsem_set_reader_owned(sem, waiter->task);
         }
@@ -215,6 +215,7 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem,
         }
 
         adjustment = woken * RWSEM_ACTIVE_READ_BIAS - adjustment;
+        lockevent_cond_inc(rwsem_wake_reader, woken);
         if (list_empty(&sem->wait_list)) {
                 /* hit end of list above */
                 adjustment -= RWSEM_WAITING_BIAS;
@@ -225,92 +226,6 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem,
 }
 
 /*
- * Wait for the read lock to be granted
- */
-static inline struct rw_semaphore __sched *
-__rwsem_down_read_failed_common(struct rw_semaphore *sem, int state)
-{
-        long count, adjustment = -RWSEM_ACTIVE_READ_BIAS;
-        struct rwsem_waiter waiter;
-        DEFINE_WAKE_Q(wake_q);
-
-        waiter.task = current;
-        waiter.type = RWSEM_WAITING_FOR_READ;
-
-        raw_spin_lock_irq(&sem->wait_lock);
-        if (list_empty(&sem->wait_list)) {
-                /*
-                 * In case the wait queue is empty and the lock isn't owned
-                 * by a writer, this reader can exit the slowpath and return
-                 * immediately as its RWSEM_ACTIVE_READ_BIAS has already
-                 * been set in the count.
-                 */
-                if (atomic_long_read(&sem->count) >= 0) {
-                        raw_spin_unlock_irq(&sem->wait_lock);
-                        return sem;
-                }
-                adjustment += RWSEM_WAITING_BIAS;
-        }
-        list_add_tail(&waiter.list, &sem->wait_list);
-
-        /* we're now waiting on the lock, but no longer actively locking */
-        count = atomic_long_add_return(adjustment, &sem->count);
-
-        /*
-         * If there are no active locks, wake the front queued process(es).
-         *
-         * If there are no writers and we are first in the queue,
-         * wake our own waiter to join the existing active readers !
-         */
-        if (count == RWSEM_WAITING_BIAS ||
-            (count > RWSEM_WAITING_BIAS &&
-             adjustment != -RWSEM_ACTIVE_READ_BIAS))
-                __rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
-
-        raw_spin_unlock_irq(&sem->wait_lock);
-        wake_up_q(&wake_q);
-
-        /* wait to be given the lock */
-        while (true) {
-                set_current_state(state);
-                if (!waiter.task)
-                        break;
-                if (signal_pending_state(state, current)) {
-                        raw_spin_lock_irq(&sem->wait_lock);
-                        if (waiter.task)
-                                goto out_nolock;
-                        raw_spin_unlock_irq(&sem->wait_lock);
-                        break;
-                }
-                schedule();
-        }
-
-        __set_current_state(TASK_RUNNING);
-        return sem;
-out_nolock:
-        list_del(&waiter.list);
-        if (list_empty(&sem->wait_list))
-                atomic_long_add(-RWSEM_WAITING_BIAS, &sem->count);
-        raw_spin_unlock_irq(&sem->wait_lock);
-        __set_current_state(TASK_RUNNING);
-        return ERR_PTR(-EINTR);
-}
-
-__visible struct rw_semaphore * __sched
-rwsem_down_read_failed(struct rw_semaphore *sem)
-{
-        return __rwsem_down_read_failed_common(sem, TASK_UNINTERRUPTIBLE);
-}
-EXPORT_SYMBOL(rwsem_down_read_failed);
-
-__visible struct rw_semaphore * __sched
-rwsem_down_read_failed_killable(struct rw_semaphore *sem)
-{
-        return __rwsem_down_read_failed_common(sem, TASK_KILLABLE);
-}
-EXPORT_SYMBOL(rwsem_down_read_failed_killable);
-
-/*
  * This function must be called with the sem->wait_lock held to prevent
  * race conditions between checking the rwsem wait list and setting the
  * sem->count accordingly.
@@ -346,21 +261,17 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)
  */
 static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)
 {
-        long old, count = atomic_long_read(&sem->count);
-
-        while (true) {
-                if (!(count == 0 || count == RWSEM_WAITING_BIAS))
-                        return false;
+        long count = atomic_long_read(&sem->count);
 
-                old = atomic_long_cmpxchg_acquire(&sem->count, count,
-                                      count + RWSEM_ACTIVE_WRITE_BIAS);
-                if (old == count) {
+        while (!count || count == RWSEM_WAITING_BIAS) {
+                if (atomic_long_try_cmpxchg_acquire(&sem->count, &count,
+                                        count + RWSEM_ACTIVE_WRITE_BIAS)) {
                         rwsem_set_owner(sem);
+                        lockevent_inc(rwsem_opt_wlock);
                         return true;
                 }
-
-                count = old;
         }
+        return false;
 }
 
 static inline bool owner_on_cpu(struct task_struct *owner)
@@ -481,6 +392,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
         osq_unlock(&sem->osq);
 done:
         preempt_enable();
+        lockevent_cond_inc(rwsem_opt_fail, !taken);
         return taken;
 }
 
@@ -505,6 +417,97 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)
 #endif
 
 /*
+ * Wait for the read lock to be granted
+ */
+static inline struct rw_semaphore __sched *
+__rwsem_down_read_failed_common(struct rw_semaphore *sem, int state)
+{
+        long count, adjustment = -RWSEM_ACTIVE_READ_BIAS;
+        struct rwsem_waiter waiter;
+        DEFINE_WAKE_Q(wake_q);
+
+        waiter.task = current;
+        waiter.type = RWSEM_WAITING_FOR_READ;
+
+        raw_spin_lock_irq(&sem->wait_lock);
+        if (list_empty(&sem->wait_list)) {
+                /*
+                 * In case the wait queue is empty and the lock isn't owned
+                 * by a writer, this reader can exit the slowpath and return
+                 * immediately as its RWSEM_ACTIVE_READ_BIAS has already
+                 * been set in the count.
+                 */
+                if (atomic_long_read(&sem->count) >= 0) {
+                        raw_spin_unlock_irq(&sem->wait_lock);
+                        rwsem_set_reader_owned(sem);
+                        lockevent_inc(rwsem_rlock_fast);
+                        return sem;
+                }
+                adjustment += RWSEM_WAITING_BIAS;
+        }
+        list_add_tail(&waiter.list, &sem->wait_list);
+
+        /* we're now waiting on the lock, but no longer actively locking */
+        count = atomic_long_add_return(adjustment, &sem->count);
+
+        /*
+         * If there are no active locks, wake the front queued process(es).
+         *
+         * If there are no writers and we are first in the queue,
+         * wake our own waiter to join the existing active readers !
+         */
+        if (count == RWSEM_WAITING_BIAS ||
+            (count > RWSEM_WAITING_BIAS &&
+             adjustment != -RWSEM_ACTIVE_READ_BIAS))
+                __rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
+
+        raw_spin_unlock_irq(&sem->wait_lock);
+        wake_up_q(&wake_q);
+
+        /* wait to be given the lock */
+        while (true) {
+                set_current_state(state);
+                if (!waiter.task)
+                        break;
+                if (signal_pending_state(state, current)) {
+                        raw_spin_lock_irq(&sem->wait_lock);
+                        if (waiter.task)
+                                goto out_nolock;
+                        raw_spin_unlock_irq(&sem->wait_lock);
+                        break;
+                }
+                schedule();
+                lockevent_inc(rwsem_sleep_reader);
+        }
+
+        __set_current_state(TASK_RUNNING);
+        lockevent_inc(rwsem_rlock);
+        return sem;
+out_nolock:
+        list_del(&waiter.list);
+        if (list_empty(&sem->wait_list))
+                atomic_long_add(-RWSEM_WAITING_BIAS, &sem->count);
+        raw_spin_unlock_irq(&sem->wait_lock);
+        __set_current_state(TASK_RUNNING);
+        lockevent_inc(rwsem_rlock_fail);
+        return ERR_PTR(-EINTR);
+}
+
+__visible struct rw_semaphore * __sched
+rwsem_down_read_failed(struct rw_semaphore *sem)
+{
+        return __rwsem_down_read_failed_common(sem, TASK_UNINTERRUPTIBLE);
+}
+EXPORT_SYMBOL(rwsem_down_read_failed);
+
+__visible struct rw_semaphore * __sched
+rwsem_down_read_failed_killable(struct rw_semaphore *sem)
+{
+        return __rwsem_down_read_failed_common(sem, TASK_KILLABLE);
+}
+EXPORT_SYMBOL(rwsem_down_read_failed_killable);
+
+/*
  * Wait until we successfully acquire the write lock
  */
 static inline struct rw_semaphore *
@@ -580,6 +583,7 @@ __rwsem_down_write_failed_common(struct rw_semaphore *sem, int state)
                                 goto out_nolock;
 
                         schedule();
+                        lockevent_inc(rwsem_sleep_writer);
                         set_current_state(state);
                 } while ((count = atomic_long_read(&sem->count)) & RWSEM_ACTIVE_MASK);
 
@@ -588,6 +592,7 @@ __rwsem_down_write_failed_common(struct rw_semaphore *sem, int state)
         __set_current_state(TASK_RUNNING);
         list_del(&waiter.list);
         raw_spin_unlock_irq(&sem->wait_lock);
+        lockevent_inc(rwsem_wlock);
 
         return ret;
 
@@ -601,6 +606,7 @@ out_nolock:
                 __rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
         raw_spin_unlock_irq(&sem->wait_lock);
         wake_up_q(&wake_q);
+        lockevent_inc(rwsem_wlock_fail);
 
         return ERR_PTR(-EINTR);
 }