rcu: Add WARN_ON_ONCE() consistency checks covering state transitions

o Verify that qsmask bits stay clear through GP
  initialization.

o Verify that cpu_quiet_msk_finish() is never invoked unless
  there actually is an RCU grace period in progress.

o Verify that all internal-node rcu_node structures have empty
  blocked_tasks[] lists.

o Verify that child rcu_node structure's bits remain clear after
  acquiring parent's lock.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: akpm@linux-foundation.org
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
LKML-Reference: <12532926191947-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 9 insertions, 4 deletions

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 6c99553e9f15..e8624ebf2320 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -628,8 +628,8 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
 
         /* Special-case the common single-level case. */
         if (NUM_RCU_NODES == 1) {
-                rnp->qsmask = rnp->qsmaskinit;
                 rcu_preempt_check_blocked_tasks(rnp);
+                rnp->qsmask = rnp->qsmaskinit;
                 rnp->gpnum = rsp->gpnum;
                 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */
                 spin_unlock_irqrestore(&rnp->lock, flags);
@@ -662,8 +662,8 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
         rnp_end = &rsp->node[NUM_RCU_NODES];
         for (rnp_cur = &rsp->node[0]; rnp_cur < rnp_end; rnp_cur++) {
                 spin_lock(&rnp_cur->lock);      /* irqs already disabled. */
-                rnp_cur->qsmask = rnp_cur->qsmaskinit;
                 rcu_preempt_check_blocked_tasks(rnp);
+                rnp_cur->qsmask = rnp_cur->qsmaskinit;
                 rnp->gpnum = rsp->gpnum;
                 spin_unlock(&rnp_cur->lock);    /* irqs already disabled. */
         }
@@ -708,6 +708,7 @@ rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp)
 static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags)
         __releases(rnp->lock)
 {
+        WARN_ON_ONCE(rsp->completed == rsp->gpnum);
         rsp->completed = rsp->gpnum;
         rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
         rcu_start_gp(rsp, flags);  /* releases root node's rnp->lock. */
@@ -725,6 +726,8 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
               unsigned long flags)
         __releases(rnp->lock)
 {
+        struct rcu_node *rnp_c;
+
         /* Walk up the rcu_node hierarchy. */
         for (;;) {
                 if (!(rnp->qsmask & mask)) {
@@ -748,8 +751,10 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
                         break;
                 }
                 spin_unlock_irqrestore(&rnp->lock, flags);
+                rnp_c = rnp;
                 rnp = rnp->parent;
                 spin_lock_irqsave(&rnp->lock, flags);
+                WARN_ON_ONCE(rnp_c->qsmask);
         }
 
         /*
@@ -858,7 +863,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
         spin_lock_irqsave(&rsp->onofflock, flags);
 
         /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */
-        rnp = rdp->mynode;
+        rnp = rdp->mynode;      /* this is the outgoing CPU's rnp. */
         mask = rdp->grpmask;    /* rnp->grplo is constant. */
         do {
                 spin_lock(&rnp->lock);          /* irqs already disabled. */
@@ -867,7 +872,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
                         spin_unlock(&rnp->lock); /* irqs remain disabled. */
                         break;
                 }
-                rcu_preempt_offline_tasks(rsp, rnp);
+                rcu_preempt_offline_tasks(rsp, rnp, rdp);
                 mask = rnp->grpmask;
                 spin_unlock(&rnp->lock);        /* irqs remain disabled. */
                 rnp = rnp->parent;