Merge branch 'core-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull core locking changes from Ingo Molnar:
 - futex performance increases: larger hashes, smarter wakeups
 - mutex debugging improvements
 - lots of SMP ordering documentation updates
 - introduce the smp_load_acquire(), smp_store_release() primitives.
   (There are WIP patches that make use of them - not yet merged)
 - lockdep micro-optimizations
 - lockdep improvement: better cover IRQ contexts
 - liblockdep at last. We'll continue to monitor how useful this is

* 'core-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (34 commits)
  futexes: Fix futex_hashsize initialization
  arch: Re-sort some Kbuild files to hopefully help avoid some conflicts
  futexes: Avoid taking the hb->lock if there's nothing to wake up
  futexes: Document multiprocessor ordering guarantees
  futexes: Increase hash table size for better performance
  futexes: Clean up various details
  arch: Introduce smp_load_acquire(), smp_store_release()
  arch: Clean up asm/barrier.h implementations using asm-generic/barrier.h
  arch: Move smp_mb__{before,after}_atomic_{inc,dec}.h into asm/atomic.h
  locking/doc: Rename LOCK/UNLOCK to ACQUIRE/RELEASE
  mutexes: Give more informative mutex warning in the !lock->owner case
  powerpc: Full barrier for smp_mb__after_unlock_lock()
  rcu: Apply smp_mb__after_unlock_lock() to preserve grace periods
  Documentation/memory-barriers.txt: Downgrade UNLOCK+BLOCK
  locking: Add an smp_mb__after_unlock_lock() for UNLOCK+BLOCK barrier
  Documentation/memory-barriers.txt: Document ACCESS_ONCE()
  Documentation/memory-barriers.txt: Prohibit speculative writes
  Documentation/memory-barriers.txt: Add long atomic examples to memory-barriers.txt
  Documentation/memory-barriers.txt: Add needed ACCESS_ONCE() calls to memory-barriers.txt
  Revert "smp/cpumask: Make CONFIG_CPUMASK_OFFSTACK=y usable without debug dependency"
  ...

1 files changed, 17 insertions, 1 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index dd081987a8ec..a6205a05b5e4 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1133,8 +1133,10 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
          * hold it, acquire the root rcu_node structure's lock in order to
          * start one (if needed).
          */
-        if (rnp != rnp_root)
+        if (rnp != rnp_root) {
                 raw_spin_lock(&rnp_root->lock);
+                smp_mb__after_unlock_lock();
+        }
 
         /*
          * Get a new grace-period number.  If there really is no grace
@@ -1354,6 +1356,7 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
                 local_irq_restore(flags);
                 return;
         }
+        smp_mb__after_unlock_lock();
         __note_gp_changes(rsp, rnp, rdp);
         raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
@@ -1368,6 +1371,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
 
         rcu_bind_gp_kthread();
         raw_spin_lock_irq(&rnp->lock);
+        smp_mb__after_unlock_lock();
         if (rsp->gp_flags == 0) {
                 /* Spurious wakeup, tell caller to go back to sleep.  */
                 raw_spin_unlock_irq(&rnp->lock);
@@ -1409,6 +1413,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
          */
         rcu_for_each_node_breadth_first(rsp, rnp) {
                 raw_spin_lock_irq(&rnp->lock);
+                smp_mb__after_unlock_lock();
                 rdp = this_cpu_ptr(rsp->rda);
                 rcu_preempt_check_blocked_tasks(rnp);
                 rnp->qsmask = rnp->qsmaskinit;
@@ -1463,6 +1468,7 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
         /* Clear flag to prevent immediate re-entry. */
         if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
                 raw_spin_lock_irq(&rnp->lock);
+                smp_mb__after_unlock_lock();
                 rsp->gp_flags &= ~RCU_GP_FLAG_FQS;
                 raw_spin_unlock_irq(&rnp->lock);
         }
@@ -1480,6 +1486,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
         struct rcu_node *rnp = rcu_get_root(rsp);
 
         raw_spin_lock_irq(&rnp->lock);
+        smp_mb__after_unlock_lock();
         gp_duration = jiffies - rsp->gp_start;
         if (gp_duration > rsp->gp_max)
                 rsp->gp_max = gp_duration;
@@ -1505,6 +1512,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
          */
         rcu_for_each_node_breadth_first(rsp, rnp) {
                 raw_spin_lock_irq(&rnp->lock);
+                smp_mb__after_unlock_lock();
                 ACCESS_ONCE(rnp->completed) = rsp->gpnum;
                 rdp = this_cpu_ptr(rsp->rda);
                 if (rnp == rdp->mynode)
@@ -1515,6 +1523,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
         }
         rnp = rcu_get_root(rsp);
         raw_spin_lock_irq(&rnp->lock);
+        smp_mb__after_unlock_lock();
         rcu_nocb_gp_set(rnp, nocb);
 
         rsp->completed = rsp->gpnum; /* Declare grace period done. */
@@ -1749,6 +1758,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
                 rnp_c = rnp;
                 rnp = rnp->parent;
                 raw_spin_lock_irqsave(&rnp->lock, flags);
+                smp_mb__after_unlock_lock();
                 WARN_ON_ONCE(rnp_c->qsmask);
         }
 
@@ -1778,6 +1788,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
 
         rnp = rdp->mynode;
         raw_spin_lock_irqsave(&rnp->lock, flags);
+        smp_mb__after_unlock_lock();
         if (rdp->passed_quiesce == 0 || rdp->gpnum != rnp->gpnum ||
             rnp->completed == rnp->gpnum) {
 
@@ -1992,6 +2003,7 @@ static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
         mask = rdp->grpmask;    /* rnp->grplo is constant. */
         do {
                 raw_spin_lock(&rnp->lock);      /* irqs already disabled. */
+                smp_mb__after_unlock_lock();
                 rnp->qsmaskinit &= ~mask;
                 if (rnp->qsmaskinit != 0) {
                         if (rnp != rdp->mynode)
@@ -2202,6 +2214,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
                 cond_resched();
                 mask = 0;
                 raw_spin_lock_irqsave(&rnp->lock, flags);
+                smp_mb__after_unlock_lock();
                 if (!rcu_gp_in_progress(rsp)) {
                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
                         return;
@@ -2231,6 +2244,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
         rnp = rcu_get_root(rsp);
         if (rnp->qsmask == 0) {
                 raw_spin_lock_irqsave(&rnp->lock, flags);
+                smp_mb__after_unlock_lock();
                 rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */
         }
 }
@@ -2263,6 +2277,7 @@ static void force_quiescent_state(struct rcu_state *rsp)
 
         /* Reached the root of the rcu_node tree, acquire lock. */
         raw_spin_lock_irqsave(&rnp_old->lock, flags);
+        smp_mb__after_unlock_lock();
         raw_spin_unlock(&rnp_old->fqslock);
         if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
                 rsp->n_force_qs_lh++;
@@ -2378,6 +2393,7 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
                         struct rcu_node *rnp_root = rcu_get_root(rsp);
 
                         raw_spin_lock(&rnp_root->lock);
+                        smp_mb__after_unlock_lock();
                         rcu_start_gp(rsp);
                         raw_spin_unlock(&rnp_root->lock);
                 } else {