rcu: Provide cond_resched_rcu_qs() to force quiescent states in long loops

RCU-tasks requires the occasional voluntary context switch
from CPU-bound in-kernel tasks.  In some cases, this requires
instrumenting cond_resched().  However, there is some reluctance
to countenance unconditionally instrumenting cond_resched() (see
http://lwn.net/Articles/603252/), so this commit creates a separate
cond_resched_rcu_qs() that may be used in place of cond_resched() in
locations prone to long-duration in-kernel looping.

This commit currently instruments only RCU-tasks.  Future possibilities
include also instrumenting RCU, RCU-bh, and RCU-sched in order to reduce
IPI usage.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

3 files changed, 9 insertions, 9 deletions

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 948a7693748e..178716713e11 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -667,7 +667,7 @@ static int rcu_torture_boost(void *arg)
                                 }
                                 call_rcu_time = jiffies;
                         }
-                        cond_resched();
+                        cond_resched_rcu_qs();
                         stutter_wait("rcu_torture_boost");
                         if (torture_must_stop())
                                 goto checkwait;
@@ -1019,7 +1019,7 @@ rcu_torture_reader(void *arg)
                 __this_cpu_inc(rcu_torture_batch[completed]);
                 preempt_enable();
                 cur_ops->readunlock(idx);
-                cond_resched();
+                cond_resched_rcu_qs();
                 stutter_wait("rcu_torture_reader");
         } while (!torture_must_stop());
         if (irqreader && cur_ops->irq_capable) {
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 8ad91d1e317d..e23dad0661e2 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1647,7 +1647,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
                                             rnp->level, rnp->grplo,
                                             rnp->grphi, rnp->qsmask);
                 raw_spin_unlock_irq(&rnp->lock);
-                cond_resched();
+                cond_resched_rcu_qs();
         }
 
         mutex_unlock(&rsp->onoff_mutex);
@@ -1736,7 +1736,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
                 /* smp_mb() provided by prior unlock-lock pair. */
                 nocb += rcu_future_gp_cleanup(rsp, rnp);
                 raw_spin_unlock_irq(&rnp->lock);
-                cond_resched();
+                cond_resched_rcu_qs();
         }
         rnp = rcu_get_root(rsp);
         raw_spin_lock_irq(&rnp->lock);
@@ -1785,7 +1785,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
                         /* Locking provides needed memory barrier. */
                         if (rcu_gp_init(rsp))
                                 break;
-                        cond_resched();
+                        cond_resched_rcu_qs();
                         flush_signals(current);
                         trace_rcu_grace_period(rsp->name,
                                                ACCESS_ONCE(rsp->gpnum),
@@ -1828,10 +1828,10 @@ static int __noreturn rcu_gp_kthread(void *arg)
                                 trace_rcu_grace_period(rsp->name,
                                                        ACCESS_ONCE(rsp->gpnum),
                                                        TPS("fqsend"));
-                                cond_resched();
+                                cond_resched_rcu_qs();
                         } else {
                                 /* Deal with stray signal. */
-                                cond_resched();
+                                cond_resched_rcu_qs();
                                 flush_signals(current);
                                 trace_rcu_grace_period(rsp->name,
                                                        ACCESS_ONCE(rsp->gpnum),
@@ -2434,7 +2434,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
         struct rcu_node *rnp;
 
         rcu_for_each_leaf_node(rsp, rnp) {
-                cond_resched();
+                cond_resched_rcu_qs();
                 mask = 0;
                 raw_spin_lock_irqsave(&rnp->lock, flags);
                 smp_mb__after_unlock_lock();
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index a7997e272564..7672586d3920 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1848,7 +1848,7 @@ static int rcu_oom_notify(struct notifier_block *self,
         get_online_cpus();
         for_each_online_cpu(cpu) {
                 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
-                cond_resched();
+                cond_resched_rcu_qs();
         }
         put_online_cpus();