14 files changed, 342 insertions, 225 deletions

diff --git a/kernel/cpu.c b/kernel/cpu.c
index 0097acec1c71..be4859f07153 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -315,6 +315,16 @@ void lockdep_assert_cpus_held(void)
         percpu_rwsem_assert_held(&cpu_hotplug_lock);
 }
 
+static void lockdep_acquire_cpus_lock(void)
+{
+        rwsem_acquire(&cpu_hotplug_lock.rw_sem.dep_map, 0, 0, _THIS_IP_);
+}
+
+static void lockdep_release_cpus_lock(void)
+{
+        rwsem_release(&cpu_hotplug_lock.rw_sem.dep_map, 1, _THIS_IP_);
+}
+
 /*
  * Wait for currently running CPU hotplug operations to complete (if any) and
  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
@@ -344,6 +354,17 @@ void cpu_hotplug_enable(void)
         cpu_maps_update_done();
 }
 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
+
+#else
+
+static void lockdep_acquire_cpus_lock(void)
+{
+}
+
+static void lockdep_release_cpus_lock(void)
+{
+}
+
 #endif  /* CONFIG_HOTPLUG_CPU */
 
 #ifdef CONFIG_HOTPLUG_SMT
@@ -616,6 +637,12 @@ static void cpuhp_thread_fun(unsigned int cpu)
          */
         smp_mb();
 
+        /*
+         * The BP holds the hotplug lock, but we're now running on the AP,
+         * ensure that anybody asserting the lock is held, will actually find
+         * it so.
+         */
+        lockdep_acquire_cpus_lock();
         cpuhp_lock_acquire(bringup);
 
         if (st->single) {
@@ -661,6 +688,7 @@ static void cpuhp_thread_fun(unsigned int cpu)
         }
 
         cpuhp_lock_release(bringup);
+        lockdep_release_cpus_lock();
 
         if (!st->should_run)
                 complete_ap_thread(st, bringup);
diff --git a/kernel/futex.c b/kernel/futex.c
index 11fc3bb456d6..3e2de8fc1891 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1365,9 +1365,9 @@ static void __unqueue_futex(struct futex_q *q)
 {
         struct futex_hash_bucket *hb;
 
-        if (WARN_ON_SMP(!q->lock_ptr || !spin_is_locked(q->lock_ptr))
-            || WARN_ON(plist_node_empty(&q->list)))
+        if (WARN_ON_SMP(!q->lock_ptr) || WARN_ON(plist_node_empty(&q->list)))
                 return;
+        lockdep_assert_held(q->lock_ptr);
 
         hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);
         plist_del(&q->list, &hb->chain);
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 2e62503bea0d..b28028b08d44 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -38,23 +38,43 @@ static int jump_label_cmp(const void *a, const void *b)
         const struct jump_entry *jea = a;
         const struct jump_entry *jeb = b;
 
-        if (jea->key < jeb->key)
+        if (jump_entry_key(jea) < jump_entry_key(jeb))
                 return -1;
 
-        if (jea->key > jeb->key)
+        if (jump_entry_key(jea) > jump_entry_key(jeb))
                 return 1;
 
         return 0;
 }
 
+static void jump_label_swap(void *a, void *b, int size)
+{
+        long delta = (unsigned long)a - (unsigned long)b;
+        struct jump_entry *jea = a;
+        struct jump_entry *jeb = b;
+        struct jump_entry tmp = *jea;
+
+        jea->code       = jeb->code - delta;
+        jea->target     = jeb->target - delta;
+        jea->key        = jeb->key - delta;
+
+        jeb->code       = tmp.code + delta;
+        jeb->target     = tmp.target + delta;
+        jeb->key        = tmp.key + delta;
+}
+
 static void
 jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
 {
         unsigned long size;
+        void *swapfn = NULL;
+
+        if (IS_ENABLED(CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE))
+                swapfn = jump_label_swap;
 
         size = (((unsigned long)stop - (unsigned long)start)
                                         / sizeof(struct jump_entry));
-        sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
+        sort(start, size, sizeof(struct jump_entry), jump_label_cmp, swapfn);
 }
 
 static void jump_label_update(struct static_key *key);
@@ -85,6 +105,7 @@ void static_key_slow_inc_cpuslocked(struct static_key *key)
         int v, v1;
 
         STATIC_KEY_CHECK_USE(key);
+        lockdep_assert_cpus_held();
 
         /*
          * Careful if we get concurrent static_key_slow_inc() calls;
@@ -130,6 +151,7 @@ EXPORT_SYMBOL_GPL(static_key_slow_inc);
 void static_key_enable_cpuslocked(struct static_key *key)
 {
         STATIC_KEY_CHECK_USE(key);
+        lockdep_assert_cpus_held();
 
         if (atomic_read(&key->enabled) > 0) {
                 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
@@ -160,6 +182,7 @@ EXPORT_SYMBOL_GPL(static_key_enable);
 void static_key_disable_cpuslocked(struct static_key *key)
 {
         STATIC_KEY_CHECK_USE(key);
+        lockdep_assert_cpus_held();
 
         if (atomic_read(&key->enabled) != 1) {
                 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
@@ -185,6 +208,8 @@ static void __static_key_slow_dec_cpuslocked(struct static_key *key,
                                            unsigned long rate_limit,
                                            struct delayed_work *work)
 {
+        lockdep_assert_cpus_held();
+
         /*
          * The negative count check is valid even when a negative
          * key->enabled is in use by static_key_slow_inc(); a
@@ -261,8 +286,8 @@ EXPORT_SYMBOL_GPL(jump_label_rate_limit);
 
 static int addr_conflict(struct jump_entry *entry, void *start, void *end)
 {
-        if (entry->code <= (unsigned long)end &&
-                entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
+        if (jump_entry_code(entry) <= (unsigned long)end &&
+            jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
                 return 1;
 
         return 0;
@@ -321,16 +346,6 @@ static inline void static_key_set_linked(struct static_key *key)
         key->type |= JUMP_TYPE_LINKED;
 }
 
-static inline struct static_key *jump_entry_key(struct jump_entry *entry)
-{
-        return (struct static_key *)((unsigned long)entry->key & ~1UL);
-}
-
-static bool jump_entry_branch(struct jump_entry *entry)
-{
-        return (unsigned long)entry->key & 1UL;
-}
-
 /***
  * A 'struct static_key' uses a union such that it either points directly
  * to a table of 'struct jump_entry' or to a linked list of modules which in
@@ -355,7 +370,7 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry)
 {
         struct static_key *key = jump_entry_key(entry);
         bool enabled = static_key_enabled(key);
-        bool branch = jump_entry_branch(entry);
+        bool branch = jump_entry_is_branch(entry);
 
         /* See the comment in linux/jump_label.h */
         return enabled ^ branch;
@@ -363,19 +378,20 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry)
 
 static void __jump_label_update(struct static_key *key,
                                 struct jump_entry *entry,
-                                struct jump_entry *stop)
+                                struct jump_entry *stop,
+                                bool init)
 {
         for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
                 /*
                  * An entry->code of 0 indicates an entry which has been
                  * disabled because it was in an init text area.
                  */
-                if (entry->code) {
-                        if (kernel_text_address(entry->code))
+                if (init || !jump_entry_is_init(entry)) {
+                        if (kernel_text_address(jump_entry_code(entry)))
                                 arch_jump_label_transform(entry, jump_label_type(entry));
                         else
                                 WARN_ONCE(1, "can't patch jump_label at %pS",
-                                          (void *)(unsigned long)entry->code);
+                                          (void *)jump_entry_code(entry));
                 }
         }
 }
@@ -410,6 +426,9 @@ void __init jump_label_init(void)
                 if (jump_label_type(iter) == JUMP_LABEL_NOP)
                         arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
 
+                if (init_section_contains((void *)jump_entry_code(iter), 1))
+                        jump_entry_set_init(iter);
+
                 iterk = jump_entry_key(iter);
                 if (iterk == key)
                         continue;
@@ -422,26 +441,13 @@ void __init jump_label_init(void)
         cpus_read_unlock();
 }
 
-/* Disable any jump label entries in __init/__exit code */
-void __init jump_label_invalidate_initmem(void)
-{
-        struct jump_entry *iter_start = __start___jump_table;
-        struct jump_entry *iter_stop = __stop___jump_table;
-        struct jump_entry *iter;
-
-        for (iter = iter_start; iter < iter_stop; iter++) {
-                if (init_section_contains((void *)(unsigned long)iter->code, 1))
-                        iter->code = 0;
-        }
-}
-
 #ifdef CONFIG_MODULES
 
 static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
 {
         struct static_key *key = jump_entry_key(entry);
         bool type = static_key_type(key);
-        bool branch = jump_entry_branch(entry);
+        bool branch = jump_entry_is_branch(entry);
 
         /* See the comment in linux/jump_label.h */
         return type ^ branch;
@@ -455,7 +461,7 @@ struct static_key_mod {
 
 static inline struct static_key_mod *static_key_mod(struct static_key *key)
 {
-        WARN_ON_ONCE(!(key->type & JUMP_TYPE_LINKED));
+        WARN_ON_ONCE(!static_key_linked(key));
         return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);
 }
 
@@ -514,7 +520,8 @@ static void __jump_label_mod_update(struct static_key *key)
                         stop = __stop___jump_table;
                 else
                         stop = m->jump_entries + m->num_jump_entries;
-                __jump_label_update(key, mod->entries, stop);
+                __jump_label_update(key, mod->entries, stop,
+                                    m && m->state == MODULE_STATE_COMING);
         }
 }
 
@@ -560,12 +567,15 @@ static int jump_label_add_module(struct module *mod)
         for (iter = iter_start; iter < iter_stop; iter++) {
                 struct static_key *iterk;
 
+                if (within_module_init(jump_entry_code(iter), mod))
+                        jump_entry_set_init(iter);
+
                 iterk = jump_entry_key(iter);
                 if (iterk == key)
                         continue;
 
                 key = iterk;
-                if (within_module(iter->key, mod)) {
+                if (within_module((unsigned long)key, mod)) {
                         static_key_set_entries(key, iter);
                         continue;
                 }
@@ -595,7 +605,7 @@ static int jump_label_add_module(struct module *mod)
 
                 /* Only update if we've changed from our initial state */
                 if (jump_label_type(iter) != jump_label_init_type(iter))
-                        __jump_label_update(key, iter, iter_stop);
+                        __jump_label_update(key, iter, iter_stop, true);
         }
 
         return 0;
@@ -615,7 +625,7 @@ static void jump_label_del_module(struct module *mod)
 
                 key = jump_entry_key(iter);
 
-                if (within_module(iter->key, mod))
+                if (within_module((unsigned long)key, mod))
                         continue;
 
                 /* No memory during module load */
@@ -651,19 +661,6 @@ static void jump_label_del_module(struct module *mod)
         }
 }
 
-/* Disable any jump label entries in module init code */
-static void jump_label_invalidate_module_init(struct module *mod)
-{
-        struct jump_entry *iter_start = mod->jump_entries;
-        struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
-        struct jump_entry *iter;
-
-        for (iter = iter_start; iter < iter_stop; iter++) {
-                if (within_module_init(iter->code, mod))
-                        iter->code = 0;
-        }
-}
-
 static int
 jump_label_module_notify(struct notifier_block *self, unsigned long val,
                          void *data)
@@ -685,9 +682,6 @@ jump_label_module_notify(struct notifier_block *self, unsigned long val,
         case MODULE_STATE_GOING:
                 jump_label_del_module(mod);
                 break;
-        case MODULE_STATE_LIVE:
-                jump_label_invalidate_module_init(mod);
-                break;
         }
 
         jump_label_unlock();
@@ -757,7 +751,8 @@ static void jump_label_update(struct static_key *key)
         entry = static_key_entries(key);
         /* if there are no users, entry can be NULL */
         if (entry)
-                __jump_label_update(key, entry, stop);
+                __jump_label_update(key, entry, stop,
+                                    system_state < SYSTEM_RUNNING);
 }
 
 #ifdef CONFIG_STATIC_KEYS_SELFTEST
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index dd13f865ad40..1efada2dd9dd 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -138,7 +138,7 @@ static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];
  * get freed - this significantly simplifies the debugging code.
  */
 unsigned long nr_lock_classes;
-static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
+struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
 
 static inline struct lock_class *hlock_class(struct held_lock *hlock)
 {
@@ -1391,7 +1391,9 @@ static void print_lock_class_header(struct lock_class *class, int depth)
 
         printk("%*s->", depth, "");
         print_lock_name(class);
-        printk(KERN_CONT " ops: %lu", class->ops);
+#ifdef CONFIG_DEBUG_LOCKDEP
+        printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));
+#endif
         printk(KERN_CONT " {\n");
 
         for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
@@ -2148,76 +2150,6 @@ static int check_no_collision(struct task_struct *curr,
 }
 
 /*
- * This is for building a chain between just two different classes,
- * instead of adding a new hlock upon current, which is done by
- * add_chain_cache().
- *
- * This can be called in any context with two classes, while
- * add_chain_cache() must be done within the lock owener's context
- * since it uses hlock which might be racy in another context.
- */
-static inline int add_chain_cache_classes(unsigned int prev,
-                                          unsigned int next,
-                                          unsigned int irq_context,
-                                          u64 chain_key)
-{
-        struct hlist_head *hash_head = chainhashentry(chain_key);
-        struct lock_chain *chain;
-
-        /*
-         * Allocate a new chain entry from the static array, and add
-         * it to the hash:
-         */
-
-        /*
-         * We might need to take the graph lock, ensure we've got IRQs
-         * disabled to make this an IRQ-safe lock.. for recursion reasons
-         * lockdep won't complain about its own locking errors.
-         */
-        if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-                return 0;
-
-        if (unlikely(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) {
-                if (!debug_locks_off_graph_unlock())
-                        return 0;
-
-                print_lockdep_off("BUG: MAX_LOCKDEP_CHAINS too low!");
-                dump_stack();
-                return 0;
-        }
-
-        chain = lock_chains + nr_lock_chains++;
-        chain->chain_key = chain_key;
-        chain->irq_context = irq_context;
-        chain->depth = 2;
-        if (likely(nr_chain_hlocks + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) {
-                chain->base = nr_chain_hlocks;
-                nr_chain_hlocks += chain->depth;
-                chain_hlocks[chain->base] = prev - 1;
-                chain_hlocks[chain->base + 1] = next -1;
-        }
-#ifdef CONFIG_DEBUG_LOCKDEP
-        /*
-         * Important for check_no_collision().
-         */
-        else {
-                if (!debug_locks_off_graph_unlock())
-                        return 0;
-
-                print_lockdep_off("BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!");
-                dump_stack();
-                return 0;
-        }
-#endif
-
-        hlist_add_head_rcu(&chain->entry, hash_head);
-        debug_atomic_inc(chain_lookup_misses);
-        inc_chains();
-
-        return 1;
-}
-
-/*
  * Adds a dependency chain into chain hashtable. And must be called with
  * graph_lock held.
  *
@@ -3262,6 +3194,10 @@ static int __lock_is_held(const struct lockdep_map *lock, int read);
 /*
  * This gets called for every mutex_lock*()/spin_lock*() operation.
  * We maintain the dependency maps and validate the locking attempt:
+ *
+ * The callers must make sure that IRQs are disabled before calling it,
+ * otherwise we could get an interrupt which would want to take locks,
+ * which would end up in lockdep again.
  */
 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
                           int trylock, int read, int check, int hardirqs_off,
@@ -3279,14 +3215,6 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
         if (unlikely(!debug_locks))
                 return 0;
 
-        /*
-         * Lockdep should run with IRQs disabled, otherwise we could
-         * get an interrupt which would want to take locks, which would
-         * end up in lockdep and have you got a head-ache already?
-         */
-        if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-                return 0;
-
         if (!prove_locking || lock->key == &__lockdep_no_validate__)
                 check = 0;
 
@@ -3300,7 +3228,9 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
                 if (!class)
                         return 0;
         }
-        atomic_inc((atomic_t *)&class->ops);
+
+        debug_class_ops_inc(class);
+
         if (very_verbose(class)) {
                 printk("\nacquire class [%px] %s", class->key, class->name);
                 if (class->name_version > 1)
@@ -3543,6 +3473,9 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth,
 {
         struct held_lock *hlock;
 
+        if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+                return 0;
+
         for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) {
                 if (!__lock_acquire(hlock->instance,
                                     hlock_class(hlock)->subclass,
@@ -3696,6 +3629,13 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
         curr->lockdep_depth = i;
         curr->curr_chain_key = hlock->prev_chain_key;
 
+        /*
+         * The most likely case is when the unlock is on the innermost
+         * lock. In this case, we are done!
+         */
+        if (i == depth-1)
+                return 1;
+
         if (reacquire_held_locks(curr, depth, i + 1))
                 return 0;
 
@@ -3703,10 +3643,14 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
          * We had N bottles of beer on the wall, we drank one, but now
          * there's not N-1 bottles of beer left on the wall...
          */
-        if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1))
-                return 0;
+        DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth-1);
 
-        return 1;
+        /*
+         * Since reacquire_held_locks() would have called check_chain_key()
+         * indirectly via __lock_acquire(), we don't need to do it again
+         * on return.
+         */
+        return 0;
 }
 
 static int __lock_is_held(const struct lockdep_map *lock, int read)
@@ -4122,7 +4066,7 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip)
 {
         unsigned long flags;
 
-        if (unlikely(!lock_stat))
+        if (unlikely(!lock_stat || !debug_locks))
                 return;
 
         if (unlikely(current->lockdep_recursion))
@@ -4142,7 +4086,7 @@ void lock_acquired(struct lockdep_map *lock, unsigned long ip)
 {
         unsigned long flags;
 
-        if (unlikely(!lock_stat))
+        if (unlikely(!lock_stat || !debug_locks))
                 return;
 
         if (unlikely(current->lockdep_recursion))
diff --git a/kernel/locking/lockdep_internals.h b/kernel/locking/lockdep_internals.h
index d459d624ba2a..88c847a41c8a 100644
--- a/kernel/locking/lockdep_internals.h
+++ b/kernel/locking/lockdep_internals.h
@@ -152,9 +152,15 @@ struct lockdep_stats {
         int     nr_find_usage_forwards_recursions;
         int     nr_find_usage_backwards_checks;
         int     nr_find_usage_backwards_recursions;
+
+        /*
+         * Per lock class locking operation stat counts
+         */
+        unsigned long lock_class_ops[MAX_LOCKDEP_KEYS];
 };
 
 DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
+extern struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
 
 #define __debug_atomic_inc(ptr)                                 \
         this_cpu_inc(lockdep_stats.ptr);
@@ -179,9 +185,30 @@ DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
         }                                                               \
         __total;                                                        \
 })
+
+static inline void debug_class_ops_inc(struct lock_class *class)
+{
+        int idx;
+
+        idx = class - lock_classes;
+        __debug_atomic_inc(lock_class_ops[idx]);
+}
+
+static inline unsigned long debug_class_ops_read(struct lock_class *class)
+{
+        int idx, cpu;
+        unsigned long ops = 0;
+
+        idx = class - lock_classes;
+        for_each_possible_cpu(cpu)
+                ops += per_cpu(lockdep_stats.lock_class_ops[idx], cpu);
+        return ops;
+}
+
 #else
 # define __debug_atomic_inc(ptr)        do { } while (0)
 # define debug_atomic_inc(ptr)          do { } while (0)
 # define debug_atomic_dec(ptr)          do { } while (0)
 # define debug_atomic_read(ptr)         0
+# define debug_class_ops_inc(ptr)       do { } while (0)
 #endif
diff --git a/kernel/locking/lockdep_proc.c b/kernel/locking/lockdep_proc.c
index 3dd980dfba2d..3d31f9b0059e 100644
--- a/kernel/locking/lockdep_proc.c
+++ b/kernel/locking/lockdep_proc.c
@@ -68,7 +68,7 @@ static int l_show(struct seq_file *m, void *v)
 
         seq_printf(m, "%p", class->key);
 #ifdef CONFIG_DEBUG_LOCKDEP
-        seq_printf(m, " OPS:%8ld", class->ops);
+        seq_printf(m, " OPS:%8ld", debug_class_ops_read(class));
 #endif
 #ifdef CONFIG_PROVE_LOCKING
         seq_printf(m, " FD:%5ld", lockdep_count_forward_deps(class));
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index bfaeb05123ff..8a8c3c208c5e 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -74,12 +74,24 @@
  */
 
 #include "mcs_spinlock.h"
+#define MAX_NODES       4
 
+/*
+ * On 64-bit architectures, the mcs_spinlock structure will be 16 bytes in
+ * size and four of them will fit nicely in one 64-byte cacheline. For
+ * pvqspinlock, however, we need more space for extra data. To accommodate
+ * that, we insert two more long words to pad it up to 32 bytes. IOW, only
+ * two of them can fit in a cacheline in this case. That is OK as it is rare
+ * to have more than 2 levels of slowpath nesting in actual use. We don't
+ * want to penalize pvqspinlocks to optimize for a rare case in native
+ * qspinlocks.
+ */
+struct qnode {
+        struct mcs_spinlock mcs;
 #ifdef CONFIG_PARAVIRT_SPINLOCKS
-#define MAX_NODES       8
-#else
-#define MAX_NODES       4
+        long reserved[2];
 #endif
+};
 
 /*
  * The pending bit spinning loop count.
@@ -101,7 +113,7 @@
  *
  * PV doubles the storage and uses the second cacheline for PV state.
  */
-static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[MAX_NODES]);
+static DEFINE_PER_CPU_ALIGNED(struct qnode, qnodes[MAX_NODES]);
 
 /*
  * We must be able to distinguish between no-tail and the tail at 0:0,
@@ -126,7 +138,13 @@ static inline __pure struct mcs_spinlock *decode_tail(u32 tail)
         int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
         int idx = (tail &  _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
 
-        return per_cpu_ptr(&mcs_nodes[idx], cpu);
+        return per_cpu_ptr(&qnodes[idx].mcs, cpu);
+}
+
+static inline __pure
+struct mcs_spinlock *grab_mcs_node(struct mcs_spinlock *base, int idx)
+{
+        return &((struct qnode *)base + idx)->mcs;
 }
 
 #define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
@@ -232,6 +250,20 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
 #endif /* _Q_PENDING_BITS == 8 */
 
 /**
+ * queued_fetch_set_pending_acquire - fetch the whole lock value and set pending
+ * @lock : Pointer to queued spinlock structure
+ * Return: The previous lock value
+ *
+ * *,*,* -> *,1,*
+ */
+#ifndef queued_fetch_set_pending_acquire
+static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lock)
+{
+        return atomic_fetch_or_acquire(_Q_PENDING_VAL, &lock->val);
+}
+#endif
+
+/**
  * set_locked - Set the lock bit and own the lock
  * @lock: Pointer to queued spinlock structure
  *
@@ -326,43 +358,48 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
         /*
          * trylock || pending
          *
-         * 0,0,0 -> 0,0,1 ; trylock
-         * 0,0,1 -> 0,1,1 ; pending
+         * 0,0,* -> 0,1,* -> 0,0,1 pending, trylock
          */
-        val = atomic_fetch_or_acquire(_Q_PENDING_VAL, &lock->val);
-        if (!(val & ~_Q_LOCKED_MASK)) {
-                /*
-                 * We're pending, wait for the owner to go away.
-                 *
-                 * *,1,1 -> *,1,0
-                 *
-                 * this wait loop must be a load-acquire such that we match the
-                 * store-release that clears the locked bit and create lock
-                 * sequentiality; this is because not all
-                 * clear_pending_set_locked() implementations imply full
-                 * barriers.
-                 */
-                if (val & _Q_LOCKED_MASK) {
-                        atomic_cond_read_acquire(&lock->val,
-                                                 !(VAL & _Q_LOCKED_MASK));
-                }
+        val = queued_fetch_set_pending_acquire(lock);
 
-                /*
-                 * take ownership and clear the pending bit.
-                 *
-                 * *,1,0 -> *,0,1
-                 */
-                clear_pending_set_locked(lock);
-                qstat_inc(qstat_lock_pending, true);
-                return;
+        /*
+         * If we observe contention, there is a concurrent locker.
+         *
+         * Undo and queue; our setting of PENDING might have made the
+         * n,0,0 -> 0,0,0 transition fail and it will now be waiting
+         * on @next to become !NULL.
+         */
+        if (unlikely(val & ~_Q_LOCKED_MASK)) {
+
+                /* Undo PENDING if we set it. */
+                if (!(val & _Q_PENDING_MASK))
+                        clear_pending(lock);
+
+                goto queue;
         }
 
         /*
-         * If pending was clear but there are waiters in the queue, then
-         * we need to undo our setting of pending before we queue ourselves.
+         * We're pending, wait for the owner to go away.
+         *
+         * 0,1,1 -> 0,1,0
+         *
+         * this wait loop must be a load-acquire such that we match the
+         * store-release that clears the locked bit and create lock
+         * sequentiality; this is because not all
+         * clear_pending_set_locked() implementations imply full
+         * barriers.
+         */
+        if (val & _Q_LOCKED_MASK)
+                atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_MASK));
+
+        /*
+         * take ownership and clear the pending bit.
+         *
+         * 0,1,0 -> 0,0,1
          */
-        if (!(val & _Q_PENDING_MASK))
-                clear_pending(lock);
+        clear_pending_set_locked(lock);
+        qstat_inc(qstat_lock_pending, true);
+        return;
 
         /*
          * End of pending bit optimistic spinning and beginning of MCS
@@ -371,11 +408,16 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
 queue:
         qstat_inc(qstat_lock_slowpath, true);
 pv_queue:
-        node = this_cpu_ptr(&mcs_nodes[0]);
+        node = this_cpu_ptr(&qnodes[0].mcs);
         idx = node->count++;
         tail = encode_tail(smp_processor_id(), idx);
 
-        node += idx;
+        node = grab_mcs_node(node, idx);
+
+        /*
+         * Keep counts of non-zero index values:
+         */
+        qstat_inc(qstat_lock_idx1 + idx - 1, idx);
 
         /*
          * Ensure that we increment the head node->count before initialising
@@ -476,16 +518,25 @@ locked:
          */
 
         /*
-         * In the PV case we might already have _Q_LOCKED_VAL set.
+         * In the PV case we might already have _Q_LOCKED_VAL set, because
+         * of lock stealing; therefore we must also allow:
+         *
+         * n,0,1 -> 0,0,1
          *
-         * The atomic_cond_read_acquire() call above has provided the
-         * necessary acquire semantics required for locking.
+         * Note: at this point: (val & _Q_PENDING_MASK) == 0, because of the
+         *       above wait condition, therefore any concurrent setting of
+         *       PENDING will make the uncontended transition fail.
          */
-        if (((val & _Q_TAIL_MASK) == tail) &&
-            atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
-                goto release; /* No contention */
+        if ((val & _Q_TAIL_MASK) == tail) {
+                if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
+                        goto release; /* No contention */
+        }
 
-        /* Either somebody is queued behind us or _Q_PENDING_VAL is set */
+        /*
+         * Either somebody is queued behind us or _Q_PENDING_VAL got set
+         * which will then detect the remaining tail and queue behind us
+         * ensuring we'll see a @next.
+         */
         set_locked(lock);
 
         /*
@@ -501,7 +552,7 @@ release:
         /*
          * release the node
          */
-        __this_cpu_dec(mcs_nodes[0].count);
+        __this_cpu_dec(qnodes[0].mcs.count);
 }
 EXPORT_SYMBOL(queued_spin_lock_slowpath);
 
diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
index 5a0cf5f9008c..0130e488ebfe 100644
--- a/kernel/locking/qspinlock_paravirt.h
+++ b/kernel/locking/qspinlock_paravirt.h
@@ -49,8 +49,6 @@ enum vcpu_state {
 
 struct pv_node {
         struct mcs_spinlock     mcs;
-        struct mcs_spinlock     __res[3];
-
         int                     cpu;
         u8                      state;
 };
@@ -281,7 +279,7 @@ static void pv_init_node(struct mcs_spinlock *node)
 {
         struct pv_node *pn = (struct pv_node *)node;
 
-        BUILD_BUG_ON(sizeof(struct pv_node) > 5*sizeof(struct mcs_spinlock));
+        BUILD_BUG_ON(sizeof(struct pv_node) > sizeof(struct qnode));
 
         pn->cpu = smp_processor_id();
         pn->state = vcpu_running;
diff --git a/kernel/locking/qspinlock_stat.h b/kernel/locking/qspinlock_stat.h
index 6bd78c0740fc..42d3d8dc8f49 100644
--- a/kernel/locking/qspinlock_stat.h
+++ b/kernel/locking/qspinlock_stat.h
@@ -55,6 +55,9 @@ enum qlock_stats {
         qstat_pv_wait_node,
         qstat_lock_pending,
         qstat_lock_slowpath,
+        qstat_lock_idx1,
+        qstat_lock_idx2,
+        qstat_lock_idx3,
         qstat_num,      /* Total number of statistical counters */
         qstat_reset_cnts = qstat_num,
 };
@@ -82,6 +85,9 @@ static const char * const qstat_names[qstat_num + 1] = {
         [qstat_pv_wait_node]       = "pv_wait_node",
         [qstat_lock_pending]       = "lock_pending",
         [qstat_lock_slowpath]      = "lock_slowpath",
+        [qstat_lock_idx1]          = "lock_index1",
+        [qstat_lock_idx2]          = "lock_index2",
+        [qstat_lock_idx3]          = "lock_index3",
         [qstat_reset_cnts]         = "reset_counters",
 };
 
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 2823d4163a37..581edcc63c26 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1485,9 +1485,9 @@ void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
         __rt_mutex_lock(lock, subclass);
 }
 EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
-#endif
 
-#ifndef CONFIG_DEBUG_LOCK_ALLOC
+#else /* !CONFIG_DEBUG_LOCK_ALLOC */
+
 /**
  * rt_mutex_lock - lock a rt_mutex
  *
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index 3064c50e181e..09b180063ee1 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -180,7 +180,7 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem,
                  * but it gives the spinners an early indication that the
                  * readers now have the lock.
                  */
-                rwsem_set_reader_owned(sem);
+                __rwsem_set_reader_owned(sem, waiter->task);
         }
 
         /*
@@ -233,8 +233,19 @@ __rwsem_down_read_failed_common(struct rw_semaphore *sem, int state)
         waiter.type = RWSEM_WAITING_FOR_READ;
 
         raw_spin_lock_irq(&sem->wait_lock);
-        if (list_empty(&sem->wait_list))
+        if (list_empty(&sem->wait_list)) {
+                /*
+                 * In case the wait queue is empty and the lock isn't owned
+                 * by a writer, this reader can exit the slowpath and return
+                 * immediately as its RWSEM_ACTIVE_READ_BIAS has already
+                 * been set in the count.
+                 */
+                if (atomic_long_read(&sem->count) >= 0) {
+                        raw_spin_unlock_irq(&sem->wait_lock);
+                        return sem;
+                }
                 adjustment += RWSEM_WAITING_BIAS;
+        }
         list_add_tail(&waiter.list, &sem->wait_list);
 
         /* we're now waiting on the lock, but no longer actively locking */
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 776308d2fa9e..e586f0d03ad3 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -117,8 +117,9 @@ EXPORT_SYMBOL(down_write_trylock);
 void up_read(struct rw_semaphore *sem)
 {
         rwsem_release(&sem->dep_map, 1, _RET_IP_);
-        DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_READER_OWNED);
+        DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner & RWSEM_READER_OWNED));
 
+        rwsem_clear_reader_owned(sem);
         __up_read(sem);
 }
 
@@ -181,7 +182,7 @@ void down_read_non_owner(struct rw_semaphore *sem)
         might_sleep();
 
         __down_read(sem);
-        rwsem_set_reader_owned(sem);
+        __rwsem_set_reader_owned(sem, NULL);
 }
 
 EXPORT_SYMBOL(down_read_non_owner);
@@ -215,7 +216,7 @@ EXPORT_SYMBOL(down_write_killable_nested);
 
 void up_read_non_owner(struct rw_semaphore *sem)
 {
-        DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_READER_OWNED);
+        DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner & RWSEM_READER_OWNED));
         __up_read(sem);
 }
 
diff --git a/kernel/locking/rwsem.h b/kernel/locking/rwsem.h
index b9d0e72aa80f..bad2bca0268b 100644
--- a/kernel/locking/rwsem.h
+++ b/kernel/locking/rwsem.h
@@ -1,24 +1,30 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * The owner field of the rw_semaphore structure will be set to
- * RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear
- * the owner field when it unlocks. A reader, on the other hand, will
- * not touch the owner field when it unlocks.
+ * The least significant 2 bits of the owner value has the following
+ * meanings when set.
+ *  - RWSEM_READER_OWNED (bit 0): The rwsem is owned by readers
+ *  - RWSEM_ANONYMOUSLY_OWNED (bit 1): The rwsem is anonymously owned,
+ *    i.e. the owner(s) cannot be readily determined. It can be reader
+ *    owned or the owning writer is indeterminate.
  *
- * In essence, the owner field now has the following 4 states:
- *  1) 0
- *     - lock is free or the owner hasn't set the field yet
- *  2) RWSEM_READER_OWNED
- *     - lock is currently or previously owned by readers (lock is free
- *       or not set by owner yet)
- *  3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well
- *     - lock is owned by an anonymous writer, so spinning on the lock
- *       owner should be disabled.
- *  4) Other non-zero value
- *     - a writer owns the lock and other writers can spin on the lock owner.
+ * When a writer acquires a rwsem, it puts its task_struct pointer
+ * into the owner field. It is cleared after an unlock.
+ *
+ * When a reader acquires a rwsem, it will also puts its task_struct
+ * pointer into the owner field with both the RWSEM_READER_OWNED and
+ * RWSEM_ANONYMOUSLY_OWNED bits set. On unlock, the owner field will
+ * largely be left untouched. So for a free or reader-owned rwsem,
+ * the owner value may contain information about the last reader that
+ * acquires the rwsem. The anonymous bit is set because that particular
+ * reader may or may not still own the lock.
+ *
+ * That information may be helpful in debugging cases where the system
+ * seems to hang on a reader owned rwsem especially if only one reader
+ * is involved. Ideally we would like to track all the readers that own
+ * a rwsem, but the overhead is simply too big.
  */
-#define RWSEM_ANONYMOUSLY_OWNED (1UL << 0)
-#define RWSEM_READER_OWNED      ((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED)
+#define RWSEM_READER_OWNED      (1UL << 0)
+#define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)
 
 #ifdef CONFIG_DEBUG_RWSEMS
 # define DEBUG_RWSEMS_WARN_ON(c)        DEBUG_LOCKS_WARN_ON(c)
@@ -44,15 +50,26 @@ static inline void rwsem_clear_owner(struct rw_semaphore *sem)
         WRITE_ONCE(sem->owner, NULL);
 }
 
+/*
+ * The task_struct pointer of the last owning reader will be left in
+ * the owner field.
+ *
+ * Note that the owner value just indicates the task has owned the rwsem
+ * previously, it may not be the real owner or one of the real owners
+ * anymore when that field is examined, so take it with a grain of salt.
+ */
+static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
+                                            struct task_struct *owner)
+{
+        unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED
+                                                 | RWSEM_ANONYMOUSLY_OWNED;
+
+        WRITE_ONCE(sem->owner, (struct task_struct *)val);
+}
+
 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
 {
-        /*
-         * We check the owner value first to make sure that we will only
-         * do a write to the rwsem cacheline when it is really necessary
-         * to minimize cacheline contention.
-         */
-        if (READ_ONCE(sem->owner) != RWSEM_READER_OWNED)
-                WRITE_ONCE(sem->owner, RWSEM_READER_OWNED);
+        __rwsem_set_reader_owned(sem, current);
 }
 
 /*
@@ -72,6 +89,25 @@ static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)
 {
         return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;
 }
+
+#ifdef CONFIG_DEBUG_RWSEMS
+/*
+ * With CONFIG_DEBUG_RWSEMS configured, it will make sure that if there
+ * is a task pointer in owner of a reader-owned rwsem, it will be the
+ * real owner or one of the real owners. The only exception is when the
+ * unlock is done by up_read_non_owner().
+ */
+#define rwsem_clear_reader_owned rwsem_clear_reader_owned
+static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
+{
+        unsigned long val = (unsigned long)current | RWSEM_READER_OWNED
+                                                   | RWSEM_ANONYMOUSLY_OWNED;
+        if (READ_ONCE(sem->owner) == (struct task_struct *)val)
+                cmpxchg_relaxed((unsigned long *)&sem->owner, val,
+                                RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED);
+}
+#endif
+
 #else
 static inline void rwsem_set_owner(struct rw_semaphore *sem)
 {
@@ -81,7 +117,18 @@ static inline void rwsem_clear_owner(struct rw_semaphore *sem)
 {
 }
 
+static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
+                                           struct task_struct *owner)
+{
+}
+
 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
 {
 }
 #endif
+
+#ifndef rwsem_clear_reader_owned
+static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
+{
+}
+#endif
diff --git a/kernel/module.c b/kernel/module.c
index 6746c85511fe..49a405891587 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -3317,6 +3317,15 @@ static struct module *layout_and_allocate(struct load_info *info, int flags)
         ndx = find_sec(info, ".data..ro_after_init");
         if (ndx)
                 info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
+        /*
+         * Mark the __jump_table section as ro_after_init as well: these data
+         * structures are never modified, with the exception of entries that
+         * refer to code in the __init section, which are annotated as such
+         * at module load time.
+         */
+        ndx = find_sec(info, "__jump_table");
+        if (ndx)
+                info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
 
         /* Determine total sizes, and put offsets in sh_entsize.  For now
            this is done generically; there doesn't appear to be any