rcu: Make cond_resched_rcu_qs() apply to normal RCU flavors

Although cond_resched_rcu_qs() only applies to TASKS_RCU, it is used
in places where it would be useful for it to apply to the normal RCU
flavors, rcu_preempt, rcu_sched, and rcu_bh.  This is especially the
case for workloads that aggressively overload the system, particularly
those that generate large numbers of RCU updates on systems running
NO_HZ_FULL CPUs.  This commit therefore communicates quiescent states
from cond_resched_rcu_qs() to the normal RCU flavors.

Note that it is unfortunately necessary to leave the old ->passed_quiesce
mechanism in place to allow quiescent states that apply to only one
flavor to be recorded.  (Yes, we could decrement ->rcu_qs_ctr_snap in
that case, but that is not so good for debugging of RCU internals.)
In addition, if one of the RCU flavor's grace period has stalled, this
will invoke rcu_momentary_dyntick_idle(), resulting in a heavy-weight
quiescent state visible from other CPUs.

Reported-by: Sasha Levin <sasha.levin@oracle.com>
Reported-by: Dave Jones <davej@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Merge commit from Sasha Levin fixing a bug where __this_cpu()
  was used in preemptible code. ]

1 files changed, 33 insertions, 5 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 75ce12316b4c..cb00e038c2f2 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -219,6 +219,9 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 
+DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
+EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
+
 /*
  * Let the RCU core know that this CPU has gone through the scheduler,
  * which is a quiescent state.  This is called when the need for a
@@ -288,6 +291,22 @@ void rcu_note_context_switch(void)
 }
 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 
+/*
+ * Register a quiesecent state for all RCU flavors.  If there is an
+ * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
+ * dyntick-idle quiescent state visible to other CPUs (but only for those
+ * RCU flavors in desparate need of a quiescent state, which will normally
+ * be none of them).  Either way, do a lightweight quiescent state for
+ * all RCU flavors.
+ */
+void rcu_all_qs(void)
+{
+        if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))
+                rcu_momentary_dyntick_idle();
+        this_cpu_inc(rcu_qs_ctr);
+}
+EXPORT_SYMBOL_GPL(rcu_all_qs);
+
 static long blimit = 10;        /* Maximum callbacks per rcu_do_batch. */
 static long qhimark = 10000;    /* If this many pending, ignore blimit. */
 static long qlowmark = 100;     /* Once only this many pending, use blimit. */
@@ -1609,6 +1628,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
                 rdp->gpnum = rnp->gpnum;
                 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
                 rdp->passed_quiesce = 0;
+                rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
                 rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
                 zero_cpu_stall_ticks(rdp);
                 ACCESS_ONCE(rdp->gpwrap) = false;
@@ -2075,8 +2095,10 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
         rnp = rdp->mynode;
         raw_spin_lock_irqsave(&rnp->lock, flags);
         smp_mb__after_unlock_lock();
-        if (rdp->passed_quiesce == 0 || rdp->gpnum != rnp->gpnum ||
-            rnp->completed == rnp->gpnum || rdp->gpwrap) {
+        if ((rdp->passed_quiesce == 0 &&
+             rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
+            rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
+            rdp->gpwrap) {
 
                 /*
                  * The grace period in which this quiescent state was
@@ -2085,6 +2107,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
                  * within the current grace period.
                  */
                 rdp->passed_quiesce = 0;        /* need qs for new gp. */
+                rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
                 return;
         }
@@ -2129,7 +2152,8 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
          * Was there a quiescent state since the beginning of the grace
          * period? If no, then exit and wait for the next call.
          */
-        if (!rdp->passed_quiesce)
+        if (!rdp->passed_quiesce &&
+            rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
                 return;
 
         /*
@@ -3174,9 +3198,12 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
 
         /* Is the RCU core waiting for a quiescent state from this CPU? */
         if (rcu_scheduler_fully_active &&
-            rdp->qs_pending && !rdp->passed_quiesce) {
+            rdp->qs_pending && !rdp->passed_quiesce &&
+            rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
                 rdp->n_rp_qs_pending++;
-        } else if (rdp->qs_pending && rdp->passed_quiesce) {
+        } else if (rdp->qs_pending &&
+                   (rdp->passed_quiesce ||
+                    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
                 rdp->n_rp_report_qs++;
                 return 1;
         }
@@ -3510,6 +3537,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
                         rdp->gpnum = rnp->completed;
                         rdp->completed = rnp->completed;
                         rdp->passed_quiesce = 0;
+                        rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
                         rdp->qs_pending = 0;
                         trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
                 }