Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking and misc x86 updates from Ingo Molnar:
 "Lots of changes in this cycle - in part because locking/core attracted
  a number of related x86 low level work which was easier to handle in a
  single tree:

   - Linux Kernel Memory Consistency Model updates (Alan Stern, Paul E.
     McKenney, Andrea Parri)

   - lockdep scalability improvements and micro-optimizations (Waiman
     Long)

   - rwsem improvements (Waiman Long)

   - spinlock micro-optimization (Matthew Wilcox)

   - qspinlocks: Provide a liveness guarantee (more fairness) on x86.
     (Peter Zijlstra)

   - Add support for relative references in jump tables on arm64, x86
     and s390 to optimize jump labels (Ard Biesheuvel, Heiko Carstens)

   - Be a lot less permissive on weird (kernel address) uaccess faults
     on x86: BUG() when uaccess helpers fault on kernel addresses (Jann
     Horn)

   - macrofy x86 asm statements to un-confuse the GCC inliner. (Nadav
     Amit)

   - ... and a handful of other smaller changes as well"

* 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (57 commits)
  locking/lockdep: Make global debug_locks* variables read-mostly
  locking/lockdep: Fix debug_locks off performance problem
  locking/pvqspinlock: Extend node size when pvqspinlock is configured
  locking/qspinlock_stat: Count instances of nested lock slowpaths
  locking/qspinlock, x86: Provide liveness guarantee
  x86/asm: 'Simplify' GEN_*_RMWcc() macros
  locking/qspinlock: Rework some comments
  locking/qspinlock: Re-order code
  locking/lockdep: Remove duplicated 'lock_class_ops' percpu array
  x86/defconfig: Enable CONFIG_USB_XHCI_HCD=y
  futex: Replace spin_is_locked() with lockdep
  locking/lockdep: Make class->ops a percpu counter and move it under CONFIG_DEBUG_LOCKDEP=y
  x86/jump-labels: Macrofy inline assembly code to work around GCC inlining bugs
  x86/cpufeature: Macrofy inline assembly code to work around GCC inlining bugs
  x86/extable: Macrofy inline assembly code to work around GCC inlining bugs
  x86/paravirt: Work around GCC inlining bugs when compiling paravirt ops
  x86/bug: Macrofy the BUG table section handling, to work around GCC inlining bugs
  x86/alternatives: Macrofy lock prefixes to work around GCC inlining bugs
  x86/refcount: Work around GCC inlining bug
  x86/objtool: Use asm macros to work around GCC inlining bugs
  ...

1 files changed, 30 insertions, 86 deletions

diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index dd13f865ad40..1efada2dd9dd 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -138,7 +138,7 @@ static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];
  * get freed - this significantly simplifies the debugging code.
  */
 unsigned long nr_lock_classes;
-static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
+struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
 
 static inline struct lock_class *hlock_class(struct held_lock *hlock)
 {
@@ -1391,7 +1391,9 @@ static void print_lock_class_header(struct lock_class *class, int depth)
 
         printk("%*s->", depth, "");
         print_lock_name(class);
-        printk(KERN_CONT " ops: %lu", class->ops);
+#ifdef CONFIG_DEBUG_LOCKDEP
+        printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));
+#endif
         printk(KERN_CONT " {\n");
 
         for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
@@ -2148,76 +2150,6 @@ static int check_no_collision(struct task_struct *curr,
 }
 
 /*
- * This is for building a chain between just two different classes,
- * instead of adding a new hlock upon current, which is done by
- * add_chain_cache().
- *
- * This can be called in any context with two classes, while
- * add_chain_cache() must be done within the lock owener's context
- * since it uses hlock which might be racy in another context.
- */
-static inline int add_chain_cache_classes(unsigned int prev,
-                                          unsigned int next,
-                                          unsigned int irq_context,
-                                          u64 chain_key)
-{
-        struct hlist_head *hash_head = chainhashentry(chain_key);
-        struct lock_chain *chain;
-
-        /*
-         * Allocate a new chain entry from the static array, and add
-         * it to the hash:
-         */
-
-        /*
-         * We might need to take the graph lock, ensure we've got IRQs
-         * disabled to make this an IRQ-safe lock.. for recursion reasons
-         * lockdep won't complain about its own locking errors.
-         */
-        if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-                return 0;
-
-        if (unlikely(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) {
-                if (!debug_locks_off_graph_unlock())
-                        return 0;
-
-                print_lockdep_off("BUG: MAX_LOCKDEP_CHAINS too low!");
-                dump_stack();
-                return 0;
-        }
-
-        chain = lock_chains + nr_lock_chains++;
-        chain->chain_key = chain_key;
-        chain->irq_context = irq_context;
-        chain->depth = 2;
-        if (likely(nr_chain_hlocks + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) {
-                chain->base = nr_chain_hlocks;
-                nr_chain_hlocks += chain->depth;
-                chain_hlocks[chain->base] = prev - 1;
-                chain_hlocks[chain->base + 1] = next -1;
-        }
-#ifdef CONFIG_DEBUG_LOCKDEP
-        /*
-         * Important for check_no_collision().
-         */
-        else {
-                if (!debug_locks_off_graph_unlock())
-                        return 0;
-
-                print_lockdep_off("BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!");
-                dump_stack();
-                return 0;
-        }
-#endif
-
-        hlist_add_head_rcu(&chain->entry, hash_head);
-        debug_atomic_inc(chain_lookup_misses);
-        inc_chains();
-
-        return 1;
-}
-
-/*
  * Adds a dependency chain into chain hashtable. And must be called with
  * graph_lock held.
  *
@@ -3262,6 +3194,10 @@ static int __lock_is_held(const struct lockdep_map *lock, int read);
 /*
  * This gets called for every mutex_lock*()/spin_lock*() operation.
  * We maintain the dependency maps and validate the locking attempt:
+ *
+ * The callers must make sure that IRQs are disabled before calling it,
+ * otherwise we could get an interrupt which would want to take locks,
+ * which would end up in lockdep again.
  */
 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
                           int trylock, int read, int check, int hardirqs_off,
@@ -3279,14 +3215,6 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
         if (unlikely(!debug_locks))
                 return 0;
 
-        /*
-         * Lockdep should run with IRQs disabled, otherwise we could
-         * get an interrupt which would want to take locks, which would
-         * end up in lockdep and have you got a head-ache already?
-         */
-        if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-                return 0;
-
         if (!prove_locking || lock->key == &__lockdep_no_validate__)
                 check = 0;
 
@@ -3300,7 +3228,9 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
                 if (!class)
                         return 0;
         }
-        atomic_inc((atomic_t *)&class->ops);
+
+        debug_class_ops_inc(class);
+
         if (very_verbose(class)) {
                 printk("\nacquire class [%px] %s", class->key, class->name);
                 if (class->name_version > 1)
@@ -3543,6 +3473,9 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth,
 {
         struct held_lock *hlock;
 
+        if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+                return 0;
+
         for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) {
                 if (!__lock_acquire(hlock->instance,
                                     hlock_class(hlock)->subclass,
@@ -3696,6 +3629,13 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
         curr->lockdep_depth = i;
         curr->curr_chain_key = hlock->prev_chain_key;
 
+        /*
+         * The most likely case is when the unlock is on the innermost
+         * lock. In this case, we are done!
+         */
+        if (i == depth-1)
+                return 1;
+
         if (reacquire_held_locks(curr, depth, i + 1))
                 return 0;
 
@@ -3703,10 +3643,14 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
          * We had N bottles of beer on the wall, we drank one, but now
          * there's not N-1 bottles of beer left on the wall...
          */
-        if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1))
-                return 0;
+        DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth-1);
 
-        return 1;
+        /*
+         * Since reacquire_held_locks() would have called check_chain_key()
+         * indirectly via __lock_acquire(), we don't need to do it again
+         * on return.
+         */
+        return 0;
 }
 
 static int __lock_is_held(const struct lockdep_map *lock, int read)
@@ -4122,7 +4066,7 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip)
 {
         unsigned long flags;
 
-        if (unlikely(!lock_stat))
+        if (unlikely(!lock_stat || !debug_locks))
                 return;
 
         if (unlikely(current->lockdep_recursion))
@@ -4142,7 +4086,7 @@ void lock_acquired(struct lockdep_map *lock, unsigned long ip)
 {
         unsigned long flags;
 
-        if (unlikely(!lock_stat))
+        if (unlikely(!lock_stat || !debug_locks))
                 return;
 
         if (unlikely(current->lockdep_recursion))