rcu: Check cond_resched_rcu_qs() state less often to reduce GP overhead

Commit 4a81e8328d37 ("rcu: Reduce overhead of cond_resched() checks
for RCU") moved quiescent-state generation out of cond_resched()
and commit bde6c3aa9930 ("rcu: Provide cond_resched_rcu_qs() to force
quiescent states in long loops") introduced cond_resched_rcu_qs(), and
commit 5cd37193ce85 ("rcu: Make cond_resched_rcu_qs() apply to normal RCU
flavors") introduced the per-CPU rcu_qs_ctr variable, which is frequently
polled by the RCU core state machine.

This frequent polling can increase grace-period rate, which in turn
increases grace-period overhead, which is visible in some benchmarks
(for example, the "open1" benchmark in Anton Blanchard's "will it scale"
suite).  This commit therefore reduces the rate at which rcu_qs_ctr
is polled by moving that polling into the force-quiescent-state (FQS)
machinery, and by further polling it only after the grace period has
been in effect for at least jiffies_till_sched_qs jiffies.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

1 files changed, 34 insertions, 12 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 8b970319c75b..d8245cbd08f9 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1232,7 +1232,10 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
                                     bool *isidle, unsigned long *maxj)
 {
+        unsigned long jtsq;
         int *rcrmp;
+        unsigned long rjtsc;
+        struct rcu_node *rnp;
 
         /*
          * If the CPU passed through or entered a dynticks idle phase with
@@ -1248,6 +1251,31 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
                 return 1;
         }
 
+        /* Compute and saturate jiffies_till_sched_qs. */
+        jtsq = jiffies_till_sched_qs;
+        rjtsc = rcu_jiffies_till_stall_check();
+        if (jtsq > rjtsc / 2) {
+                WRITE_ONCE(jiffies_till_sched_qs, rjtsc);
+                jtsq = rjtsc / 2;
+        } else if (jtsq < 1) {
+                WRITE_ONCE(jiffies_till_sched_qs, 1);
+                jtsq = 1;
+        }
+
+        /*
+         * Has this CPU encountered a cond_resched_rcu_qs() since the
+         * beginning of the grace period?  For this to be the case,
+         * the CPU has to have noticed the current grace period.  This
+         * might not be the case for nohz_full CPUs looping in the kernel.
+         */
+        rnp = rdp->mynode;
+        if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
+            READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_qs_ctr, rdp->cpu) &&
+            READ_ONCE(rdp->gpnum) == rnp->gpnum && !rdp->gpwrap) {
+                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("rqc"));
+                return 1;
+        }
+
         /*
          * Check for the CPU being offline, but only if the grace period
          * is old enough.  We don't need to worry about the CPU changing
@@ -1290,9 +1318,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
          * warning delay.
          */
         rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu);
-        if (ULONG_CMP_GE(jiffies,
-                         rdp->rsp->gp_start + jiffies_till_sched_qs) ||
-            ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
+        if (time_after(jiffies, rdp->rsp->gp_start + jtsq) ||
+            time_after(jiffies, rdp->rsp->jiffies_resched)) {
                 if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) {
                         WRITE_ONCE(rdp->cond_resched_completed,
                                    READ_ONCE(rdp->mynode->completed));
@@ -2550,10 +2577,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
 
         rnp = rdp->mynode;
         raw_spin_lock_irqsave_rcu_node(rnp, flags);
-        if ((rdp->cpu_no_qs.b.norm &&
-             rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
-            rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
-            rdp->gpwrap) {
+        if (rdp->cpu_no_qs.b.norm || rdp->gpnum != rnp->gpnum ||
+            rnp->completed == rnp->gpnum || rdp->gpwrap) {
 
                 /*
                  * The grace period in which this quiescent state was
@@ -2608,8 +2633,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
          * Was there a quiescent state since the beginning of the grace
          * period? If no, then exit and wait for the next call.
          */
-        if (rdp->cpu_no_qs.b.norm &&
-            rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
+        if (rdp->cpu_no_qs.b.norm)
                 return;
 
         /*
@@ -3563,9 +3587,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
             rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
             rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
                 rdp->n_rp_core_needs_qs++;
-        } else if (rdp->core_needs_qs &&
-                   (!rdp->cpu_no_qs.b.norm ||
-                    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
+        } else if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm) {
                 rdp->n_rp_report_qs++;
                 return 1;
         }