rcu: reduce the number of spurious RCU_SOFTIRQ invocations

Lai Jiangshan noted that up to 10% of the RCU_SOFTIRQ are spurious, and
traced this down to the fact that the current grace-period machinery
will uselessly raise RCU_SOFTIRQ when a given CPU needs to go through
a quiescent state, but has not yet done so.  In this situation, there
might well be nothing that RCU_SOFTIRQ can do, and the overhead can be
worth worrying about in the ksoftirqd case.  This patch therefore avoids
raising RCU_SOFTIRQ in this situation.

Changes since v1 (http://lkml.org/lkml/2010/3/30/122 from Lai Jiangshan):

o	Omit the rcu_qs_pending() prechecks, as they aren't that
	much less expensive than the quiescent-state checks.

o	Merge with the set_need_resched() patch that reduces IPIs.

o	Add the new n_rp_report_qs field to the rcu_pending tracing output.

o	Update the tracing documentation accordingly.

Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

3 files changed, 10 insertions, 6 deletions

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index c60fd74e7ec9..ba6996943e28 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1161,8 +1161,6 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
  */
 void rcu_check_callbacks(int cpu, int user)
 {
-        if (!rcu_pending(cpu))
-                return; /* if nothing for RCU to do. */
         if (user ||
             (idle_cpu(cpu) && rcu_scheduler_active &&
              !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
@@ -1194,7 +1192,8 @@ void rcu_check_callbacks(int cpu, int user)
                 rcu_bh_qs(cpu);
         }
         rcu_preempt_check_callbacks(cpu);
-        raise_softirq(RCU_SOFTIRQ);
+        if (rcu_pending(cpu))
+                raise_softirq(RCU_SOFTIRQ);
 }
 
 #ifdef CONFIG_SMP
@@ -1534,18 +1533,20 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
         check_cpu_stall(rsp, rdp);
 
         /* Is the RCU core waiting for a quiescent state from this CPU? */
-        if (rdp->qs_pending) {
+        if (rdp->qs_pending && !rdp->passed_quiesc) {
 
                 /*
                  * If force_quiescent_state() coming soon and this CPU
                  * needs a quiescent state, and this is either RCU-sched
                  * or RCU-bh, force a local reschedule.
                  */
+                rdp->n_rp_qs_pending++;
                 if (!rdp->preemptable &&
                     ULONG_CMP_LT(ACCESS_ONCE(rsp->jiffies_force_qs) - 1,
                                  jiffies))
                         set_need_resched();
-                rdp->n_rp_qs_pending++;
+        } else if (rdp->qs_pending && rdp->passed_quiesc) {
+                rdp->n_rp_report_qs++;
                 return 1;
         }
 
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 11f171121ad9..14c040b18ed0 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -223,6 +223,7 @@ struct rcu_data {
         /* 5) __rcu_pending() statistics. */
         unsigned long n_rcu_pending;    /* rcu_pending() calls since boot. */
         unsigned long n_rp_qs_pending;
+        unsigned long n_rp_report_qs;
         unsigned long n_rp_cb_ready;
         unsigned long n_rp_cpu_needs_gp;
         unsigned long n_rp_gp_completed;
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
index d45db2e35d27..36c95b45738e 100644
--- a/kernel/rcutree_trace.c
+++ b/kernel/rcutree_trace.c
@@ -241,11 +241,13 @@ static const struct file_operations rcugp_fops = {
 static void print_one_rcu_pending(struct seq_file *m, struct rcu_data *rdp)
 {
         seq_printf(m, "%3d%cnp=%ld "
-                   "qsp=%ld cbr=%ld cng=%ld gpc=%ld gps=%ld nf=%ld nn=%ld\n",
+                   "qsp=%ld rpq=%ld cbr=%ld cng=%ld "
+                   "gpc=%ld gps=%ld nf=%ld nn=%ld\n",
                    rdp->cpu,
                    cpu_is_offline(rdp->cpu) ? '!' : ' ',
                    rdp->n_rcu_pending,
                    rdp->n_rp_qs_pending,
+                   rdp->n_rp_report_qs,
                    rdp->n_rp_cb_ready,
                    rdp->n_rp_cpu_needs_gp,
                    rdp->n_rp_gp_completed,