rcu: Add expedited grace-period support for preemptible RCU

Implement an synchronize_rcu_expedited() for preemptible RCU
that actually is expedited.  This uses
synchronize_sched_expedited() to force all threads currently
running in a preemptible-RCU read-side critical section onto the
appropriate ->blocked_tasks[] list, then takes a snapshot of all
of these lists and waits for them to drain.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1259784616158-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 7 insertions, 3 deletions

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index d47e03e5792a..53ae9598f798 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -948,7 +948,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
 {
         unsigned long flags;
         unsigned long mask;
-        int need_quiet = 0;
+        int need_report = 0;
         struct rcu_data *rdp = rsp->rda[cpu];
         struct rcu_node *rnp;
 
@@ -967,7 +967,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
                         break;
                 }
                 if (rnp == rdp->mynode)
-                        need_quiet = rcu_preempt_offline_tasks(rsp, rnp, rdp);
+                        need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp);
                 else
                         spin_unlock(&rnp->lock); /* irqs remain disabled. */
                 mask = rnp->grpmask;
@@ -982,10 +982,12 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
          */
         spin_unlock(&rsp->onofflock); /* irqs remain disabled. */
         rnp = rdp->mynode;
-        if (need_quiet)
+        if (need_report & RCU_OFL_TASKS_NORM_GP)
                 rcu_report_unblock_qs_rnp(rnp, flags);
         else
                 spin_unlock_irqrestore(&rnp->lock, flags);
+        if (need_report & RCU_OFL_TASKS_EXP_GP)
+                rcu_report_exp_rnp(rsp, rnp);
 
         rcu_adopt_orphan_cbs(rsp);
 }
@@ -1843,6 +1845,8 @@ static void __init rcu_init_one(struct rcu_state *rsp)
                         rnp->level = i;
                         INIT_LIST_HEAD(&rnp->blocked_tasks[0]);
                         INIT_LIST_HEAD(&rnp->blocked_tasks[1]);
+                        INIT_LIST_HEAD(&rnp->blocked_tasks[2]);
+                        INIT_LIST_HEAD(&rnp->blocked_tasks[3]);
                 }
         }
 }