Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip into next

Pull core locking updates from Ingo Molnar:
 "The main changes in this cycle were:

   - reduced/streamlined smp_mb__*() interface that allows more usecases
     and makes the existing ones less buggy, especially in rarer
     architectures

   - add rwsem implementation comments

   - bump up lockdep limits"

* 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (33 commits)
  rwsem: Add comments to explain the meaning of the rwsem's count field
  lockdep: Increase static allocations
  arch: Mass conversion of smp_mb__*()
  arch,doc: Convert smp_mb__*()
  arch,xtensa: Convert smp_mb__*()
  arch,x86: Convert smp_mb__*()
  arch,tile: Convert smp_mb__*()
  arch,sparc: Convert smp_mb__*()
  arch,sh: Convert smp_mb__*()
  arch,score: Convert smp_mb__*()
  arch,s390: Convert smp_mb__*()
  arch,powerpc: Convert smp_mb__*()
  arch,parisc: Convert smp_mb__*()
  arch,openrisc: Convert smp_mb__*()
  arch,mn10300: Convert smp_mb__*()
  arch,mips: Convert smp_mb__*()
  arch,metag: Convert smp_mb__*()
  arch,m68k: Convert smp_mb__*()
  arch,m32r: Convert smp_mb__*()
  arch,ia64: Convert smp_mb__*()
  ...

10 files changed, 92 insertions, 27 deletions

diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
index 2956c8da1605..1adf62b39b96 100644
--- a/kernel/debug/debug_core.c
+++ b/kernel/debug/debug_core.c
@@ -534,7 +534,7 @@ return_normal:
                         kgdb_info[cpu].exception_state &=
                                 ~(DCPU_WANT_MASTER | DCPU_IS_SLAVE);
                         kgdb_info[cpu].enter_kgdb--;
-                        smp_mb__before_atomic_dec();
+                        smp_mb__before_atomic();
                         atomic_dec(&slaves_in_kgdb);
                         dbg_touch_watchdogs();
                         local_irq_restore(flags);
@@ -662,7 +662,7 @@ kgdb_restore:
         kgdb_info[cpu].exception_state &=
                 ~(DCPU_WANT_MASTER | DCPU_IS_SLAVE);
         kgdb_info[cpu].enter_kgdb--;
-        smp_mb__before_atomic_dec();
+        smp_mb__before_atomic();
         atomic_dec(&masters_in_kgdb);
         /* Free kgdb_active */
         atomic_set(&kgdb_active, -1);
diff --git a/kernel/futex.c b/kernel/futex.c
index 81dbe773ce4c..89bc9d59ac65 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -267,7 +267,7 @@ static inline void futex_get_mm(union futex_key *key)
          * get_futex_key() implies a full barrier. This is relied upon
          * as full barrier (B), see the ordering comment above.
          */
-        smp_mb__after_atomic_inc();
+        smp_mb__after_atomic();
 }
 
 /*
@@ -280,7 +280,7 @@ static inline void hb_waiters_inc(struct futex_hash_bucket *hb)
         /*
          * Full barrier (A), see the ordering comment above.
          */
-        smp_mb__after_atomic_inc();
+        smp_mb__after_atomic();
 #endif
 }
 
diff --git a/kernel/kmod.c b/kernel/kmod.c
index 6b375af4958d..0ac67a5861c5 100644
--- a/kernel/kmod.c
+++ b/kernel/kmod.c
@@ -498,7 +498,7 @@ int __usermodehelper_disable(enum umh_disable_depth depth)
 static void helper_lock(void)
 {
         atomic_inc(&running_helpers);
-        smp_mb__after_atomic_inc();
+        smp_mb__after_atomic();
 }
 
 static void helper_unlock(void)
diff --git a/kernel/locking/lockdep_internals.h b/kernel/locking/lockdep_internals.h
index 4f560cfedc8f..51c4b24b6328 100644
--- a/kernel/locking/lockdep_internals.h
+++ b/kernel/locking/lockdep_internals.h
@@ -54,9 +54,9 @@ enum {
  * table (if it's not there yet), and we check it for lock order
  * conflicts and deadlocks.
  */
-#define MAX_LOCKDEP_ENTRIES     16384UL
+#define MAX_LOCKDEP_ENTRIES     32768UL
 
-#define MAX_LOCKDEP_CHAINS_BITS 15
+#define MAX_LOCKDEP_CHAINS_BITS 16
 #define MAX_LOCKDEP_CHAINS      (1UL << MAX_LOCKDEP_CHAINS_BITS)
 
 #define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS*5)
@@ -65,7 +65,7 @@ enum {
  * Stack-trace: tightly packed array of stack backtrace
  * addresses. Protected by the hash_lock.
  */
-#define MAX_STACK_TRACE_ENTRIES 262144UL
+#define MAX_STACK_TRACE_ENTRIES 524288UL
 
 extern struct list_head all_lock_classes;
 extern struct lock_chain lock_chains[];
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index 1d66e08e897d..b4219ff87b8c 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -12,6 +12,55 @@
 #include <linux/export.h>
 
 /*
+ * Guide to the rw_semaphore's count field for common values.
+ * (32-bit case illustrated, similar for 64-bit)
+ *
+ * 0x0000000X   (1) X readers active or attempting lock, no writer waiting
+ *                  X = #active_readers + #readers attempting to lock
+ *                  (X*ACTIVE_BIAS)
+ *
+ * 0x00000000   rwsem is unlocked, and no one is waiting for the lock or
+ *              attempting to read lock or write lock.
+ *
+ * 0xffff000X   (1) X readers active or attempting lock, with waiters for lock
+ *                  X = #active readers + # readers attempting lock
+ *                  (X*ACTIVE_BIAS + WAITING_BIAS)
+ *              (2) 1 writer attempting lock, no waiters for lock
+ *                  X-1 = #active readers + #readers attempting lock
+ *                  ((X-1)*ACTIVE_BIAS + ACTIVE_WRITE_BIAS)
+ *              (3) 1 writer active, no waiters for lock
+ *                  X-1 = #active readers + #readers attempting lock
+ *                  ((X-1)*ACTIVE_BIAS + ACTIVE_WRITE_BIAS)
+ *
+ * 0xffff0001   (1) 1 reader active or attempting lock, waiters for lock
+ *                  (WAITING_BIAS + ACTIVE_BIAS)
+ *              (2) 1 writer active or attempting lock, no waiters for lock
+ *                  (ACTIVE_WRITE_BIAS)
+ *
+ * 0xffff0000   (1) There are writers or readers queued but none active
+ *                  or in the process of attempting lock.
+ *                  (WAITING_BIAS)
+ *              Note: writer can attempt to steal lock for this count by adding
+ *              ACTIVE_WRITE_BIAS in cmpxchg and checking the old count
+ *
+ * 0xfffe0001   (1) 1 writer active, or attempting lock. Waiters on queue.
+ *                  (ACTIVE_WRITE_BIAS + WAITING_BIAS)
+ *
+ * Note: Readers attempt to lock by adding ACTIVE_BIAS in down_read and checking
+ *       the count becomes more than 0 for successful lock acquisition,
+ *       i.e. the case where there are only readers or nobody has lock.
+ *       (1st and 2nd case above).
+ *
+ *       Writers attempt to lock by adding ACTIVE_WRITE_BIAS in down_write and
+ *       checking the count becomes ACTIVE_WRITE_BIAS for successful lock
+ *       acquisition (i.e. nobody else has lock or attempts lock).  If
+ *       unsuccessful, in rwsem_down_write_failed, we'll check to see if there
+ *       are only waiters but none active (5th case above), and attempt to
+ *       steal the lock.
+ *
+ */
+
+/*
  * Initialize an rwsem:
  */
 void __init_rwsem(struct rw_semaphore *sem, const char *name,
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 3e3f13e8b429..f1ba77363fbb 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -458,9 +458,9 @@ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
         }
         rcu_prepare_for_idle(smp_processor_id());
         /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
-        smp_mb__before_atomic_inc();  /* See above. */
+        smp_mb__before_atomic();  /* See above. */
         atomic_inc(&rdtp->dynticks);
-        smp_mb__after_atomic_inc();  /* Force ordering with next sojourn. */
+        smp_mb__after_atomic();  /* Force ordering with next sojourn. */
         WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
 
         /*
@@ -578,10 +578,10 @@ void rcu_irq_exit(void)
 static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,
                                int user)
 {
-        smp_mb__before_atomic_inc();  /* Force ordering w/previous sojourn. */
+        smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
         atomic_inc(&rdtp->dynticks);
         /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
-        smp_mb__after_atomic_inc();  /* See above. */
+        smp_mb__after_atomic();  /* See above. */
         WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
         rcu_cleanup_after_idle(smp_processor_id());
         trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
@@ -706,10 +706,10 @@ void rcu_nmi_enter(void)
             (atomic_read(&rdtp->dynticks) & 0x1))
                 return;
         rdtp->dynticks_nmi_nesting++;
-        smp_mb__before_atomic_inc();  /* Force delay from prior write. */
+        smp_mb__before_atomic();  /* Force delay from prior write. */
         atomic_inc(&rdtp->dynticks);
         /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
-        smp_mb__after_atomic_inc();  /* See above. */
+        smp_mb__after_atomic();  /* See above. */
         WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
 }
 
@@ -728,9 +728,9 @@ void rcu_nmi_exit(void)
             --rdtp->dynticks_nmi_nesting != 0)
                 return;
         /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
-        smp_mb__before_atomic_inc();  /* See above. */
+        smp_mb__before_atomic();  /* See above. */
         atomic_inc(&rdtp->dynticks);
-        smp_mb__after_atomic_inc();  /* Force delay to next write. */
+        smp_mb__after_atomic();  /* Force delay to next write. */
         WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
 }
 
@@ -2918,7 +2918,7 @@ void synchronize_sched_expedited(void)
                 s = atomic_long_read(&rsp->expedited_done);
                 if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
                         /* ensure test happens before caller kfree */
-                        smp_mb__before_atomic_inc(); /* ^^^ */
+                        smp_mb__before_atomic(); /* ^^^ */
                         atomic_long_inc(&rsp->expedited_workdone1);
                         return;
                 }
@@ -2936,7 +2936,7 @@ void synchronize_sched_expedited(void)
                 s = atomic_long_read(&rsp->expedited_done);
                 if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
                         /* ensure test happens before caller kfree */
-                        smp_mb__before_atomic_inc(); /* ^^^ */
+                        smp_mb__before_atomic(); /* ^^^ */
                         atomic_long_inc(&rsp->expedited_workdone2);
                         return;
                 }
@@ -2965,7 +2965,7 @@ void synchronize_sched_expedited(void)
                 s = atomic_long_read(&rsp->expedited_done);
                 if (ULONG_CMP_GE((ulong)s, (ulong)snap)) {
                         /* ensure test happens before caller kfree */
-                        smp_mb__before_atomic_inc(); /* ^^^ */
+                        smp_mb__before_atomic(); /* ^^^ */
                         atomic_long_inc(&rsp->expedited_done_lost);
                         break;
                 }
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 29977ae84e7e..cbc2c45265e2 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2462,9 +2462,9 @@ static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
         /* Record start of fully idle period. */
         j = jiffies;
         ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j;
-        smp_mb__before_atomic_inc();
+        smp_mb__before_atomic();
         atomic_inc(&rdtp->dynticks_idle);
-        smp_mb__after_atomic_inc();
+        smp_mb__after_atomic();
         WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
 }
 
@@ -2529,9 +2529,9 @@ static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
         }
 
         /* Record end of idle period. */
-        smp_mb__before_atomic_inc();
+        smp_mb__before_atomic();
         atomic_inc(&rdtp->dynticks_idle);
-        smp_mb__after_atomic_inc();
+        smp_mb__after_atomic();
         WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
 
         /*
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 71d9a9c93954..a62a7dec3986 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -90,6 +90,22 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/sched.h>
 
+#ifdef smp_mb__before_atomic
+void __smp_mb__before_atomic(void)
+{
+        smp_mb__before_atomic();
+}
+EXPORT_SYMBOL(__smp_mb__before_atomic);
+#endif
+
+#ifdef smp_mb__after_atomic
+void __smp_mb__after_atomic(void)
+{
+        smp_mb__after_atomic();
+}
+EXPORT_SYMBOL(__smp_mb__after_atomic);
+#endif
+
 void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
 {
         unsigned long delta;
diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c
index 8834243abee2..981fcd7dc394 100644
--- a/kernel/sched/cpupri.c
+++ b/kernel/sched/cpupri.c
@@ -165,7 +165,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
                  * do a write memory barrier, and then update the count, to
                  * make sure the vector is visible when count is set.
                  */
-                smp_mb__before_atomic_inc();
+                smp_mb__before_atomic();
                 atomic_inc(&(vec)->count);
                 do_mb = 1;
         }
@@ -185,14 +185,14 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
                  * the new priority vec.
                  */
                 if (do_mb)
-                        smp_mb__after_atomic_inc();
+                        smp_mb__after_atomic();
 
                 /*
                  * When removing from the vector, we decrement the counter first
                  * do a memory barrier and then clear the mask.
                  */
                 atomic_dec(&(vec)->count);
-                smp_mb__after_atomic_inc();
+                smp_mb__after_atomic();
                 cpumask_clear_cpu(cpu, vec->mask);
         }
 
diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
index 7d50f794e248..0ffa20ae657b 100644
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -394,7 +394,7 @@ EXPORT_SYMBOL(__wake_up_bit);
  *
  * In order for this to function properly, as it uses waitqueue_active()
  * internally, some kind of memory barrier must be done prior to calling
- * this. Typically, this will be smp_mb__after_clear_bit(), but in some
+ * this. Typically, this will be smp_mb__after_atomic(), but in some
  * cases where bitflags are manipulated non-atomically under a lock, one
  * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
  * because spin_unlock() does not guarantee a memory barrier.