rcu: Add event-tracing for RCU callback invocation

There was recently some controversy about the overhead of invoking RCU
callbacks.  Add TRACE_EVENT()s to obtain fine-grained timings for the
start and stop of a batch of callbacks and also for each callback invoked.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

1 files changed, 12 insertions, 3 deletions

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index a7c6bce1af83..45dcc2036a1e 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -52,6 +52,9 @@
 #include <linux/prefetch.h>
 
 #include "rcutree.h"
+#include <trace/events/rcu.h>
+
+#include "rcu.h"
 
 /* Data structures. */
 
@@ -1190,17 +1193,22 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
 {
         unsigned long flags;
         struct rcu_head *next, *list, **tail;
-        int count;
+        int bl, count;
 
         /* If no callbacks are ready, just return.*/
-        if (!cpu_has_callbacks_ready_to_invoke(rdp))
+        if (!cpu_has_callbacks_ready_to_invoke(rdp)) {
+                trace_rcu_batch_start(0, 0);
+                trace_rcu_batch_end(0);
                 return;
+        }
 
         /*
          * Extract the list of ready callbacks, disabling to prevent
          * races with call_rcu() from interrupt handlers.
          */
         local_irq_save(flags);
+        bl = rdp->blimit;
+        trace_rcu_batch_start(rdp->qlen, bl);
         list = rdp->nxtlist;
         rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL];
         *rdp->nxttail[RCU_DONE_TAIL] = NULL;
@@ -1218,11 +1226,12 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
                 debug_rcu_head_unqueue(list);
                 __rcu_reclaim(list);
                 list = next;
-                if (++count >= rdp->blimit)
+                if (++count >= bl)
                         break;
         }
 
         local_irq_save(flags);
+        trace_rcu_batch_end(count);
 
         /* Update count, and requeue any remaining callbacks. */
         rdp->qlen -= count;