1 files changed, 185 insertions, 105 deletions

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index ba06207b1dd3..e234eb92a177 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -52,13 +52,16 @@
 #include <linux/prefetch.h>
 
 #include "rcutree.h"
+#include <trace/events/rcu.h>
+
+#include "rcu.h"
 
 /* Data structures. */
 
 static struct lock_class_key rcu_node_class[NUM_RCU_LVLS];
 
 #define RCU_STATE_INITIALIZER(structname) { \
-        .level = { &structname.node[0] }, \
+        .level = { &structname##_state.node[0] }, \
         .levelcnt = { \
                 NUM_RCU_LVL_0,  /* root of hierarchy. */ \
                 NUM_RCU_LVL_1, \
@@ -69,17 +72,17 @@ static struct lock_class_key rcu_node_class[NUM_RCU_LVLS];
         .signaled = RCU_GP_IDLE, \
         .gpnum = -300, \
         .completed = -300, \
-        .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname.onofflock), \
-        .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname.fqslock), \
+        .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname##_state.onofflock), \
+        .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname##_state.fqslock), \
         .n_force_qs = 0, \
         .n_force_qs_ngp = 0, \
         .name = #structname, \
 }
 
-struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched_state);
+struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched);
 DEFINE_PER_CPU(struct rcu_data, rcu_sched_data);
 
-struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);
+struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh);
 DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
 
 static struct rcu_state *rcu_state;
@@ -128,8 +131,6 @@ static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
 static void invoke_rcu_core(void);
 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
 
-#define RCU_KTHREAD_PRIO 1      /* RT priority for per-CPU kthreads. */
-
 /*
  * Track the rcutorture test sequence number and the update version
  * number within a given test.  The rcutorture_testseq is incremented
@@ -156,33 +157,41 @@ static int rcu_gp_in_progress(struct rcu_state *rsp)
  * Note a quiescent state.  Because we do not need to know
  * how many quiescent states passed, just if there was at least
  * one since the start of the grace period, this just sets a flag.
+ * The caller must have disabled preemption.
  */
 void rcu_sched_qs(int cpu)
 {
         struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu);
 
-        rdp->passed_quiesc_completed = rdp->gpnum - 1;
+        rdp->passed_quiesce_gpnum = rdp->gpnum;
         barrier();
-        rdp->passed_quiesc = 1;
+        if (rdp->passed_quiesce == 0)
+                trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs");
+        rdp->passed_quiesce = 1;
 }
 
 void rcu_bh_qs(int cpu)
 {
         struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
 
-        rdp->passed_quiesc_completed = rdp->gpnum - 1;
+        rdp->passed_quiesce_gpnum = rdp->gpnum;
         barrier();
-        rdp->passed_quiesc = 1;
+        if (rdp->passed_quiesce == 0)
+                trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs");
+        rdp->passed_quiesce = 1;
 }
 
 /*
  * Note a context switch.  This is a quiescent state for RCU-sched,
  * and requires special handling for preemptible RCU.
+ * The caller must have disabled preemption.
  */
 void rcu_note_context_switch(int cpu)
 {
+        trace_rcu_utilization("Start context switch");
         rcu_sched_qs(cpu);
         rcu_preempt_note_context_switch(cpu);
+        trace_rcu_utilization("End context switch");
 }
 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 
@@ -193,7 +202,7 @@ DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 };
 #endif /* #ifdef CONFIG_NO_HZ */
 
-static int blimit = 10;         /* Maximum callbacks per softirq. */
+static int blimit = 10;         /* Maximum callbacks per rcu_do_batch. */
 static int qhimark = 10000;     /* If this many pending, ignore blimit. */
 static int qlowmark = 100;      /* Once only this many pending, use blimit. */
 
@@ -314,6 +323,7 @@ static int rcu_implicit_offline_qs(struct rcu_data *rdp)
          * trust its state not to change because interrupts are disabled.
          */
         if (cpu_is_offline(rdp->cpu)) {
+                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "ofl");
                 rdp->offline_fqs++;
                 return 1;
         }
@@ -354,19 +364,13 @@ void rcu_enter_nohz(void)
                 local_irq_restore(flags);
                 return;
         }
+        trace_rcu_dyntick("Start");
         /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
         smp_mb__before_atomic_inc();  /* See above. */
         atomic_inc(&rdtp->dynticks);
         smp_mb__after_atomic_inc();  /* Force ordering with next sojourn. */
         WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
         local_irq_restore(flags);
-
-        /* If the interrupt queued a callback, get out of dyntick mode. */
-        if (in_irq() &&
-            (__get_cpu_var(rcu_sched_data).nxtlist ||
-             __get_cpu_var(rcu_bh_data).nxtlist ||
-             rcu_preempt_needs_cpu(smp_processor_id())))
-                set_need_resched();
 }
 
 /*
@@ -391,6 +395,7 @@ void rcu_exit_nohz(void)
         /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
         smp_mb__after_atomic_inc();  /* See above. */
         WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+        trace_rcu_dyntick("End");
         local_irq_restore(flags);
 }
 
@@ -481,11 +486,11 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp)
  */
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
 {
-        unsigned long curr;
-        unsigned long snap;
+        unsigned int curr;
+        unsigned int snap;
 
-        curr = (unsigned long)atomic_add_return(0, &rdp->dynticks->dynticks);
-        snap = (unsigned long)rdp->dynticks_snap;
+        curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks);
+        snap = (unsigned int)rdp->dynticks_snap;
 
         /*
          * If the CPU passed through or entered a dynticks idle phase with
@@ -495,7 +500,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
          * read-side critical section that started before the beginning
          * of the current RCU grace period.
          */
-        if ((curr & 0x1) == 0 || ULONG_CMP_GE(curr, snap + 2)) {
+        if ((curr & 0x1) == 0 || UINT_CMP_GE(curr, snap + 2)) {
+                trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "dti");
                 rdp->dynticks_fqs++;
                 return 1;
         }
@@ -537,6 +543,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
         int cpu;
         long delta;
         unsigned long flags;
+        int ndetected;
         struct rcu_node *rnp = rcu_get_root(rsp);
 
         /* Only let one CPU complain about others per time interval. */
@@ -553,7 +560,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
          * Now rat on any tasks that got kicked up to the root rcu_node
          * due to CPU offlining.
          */
-        rcu_print_task_stall(rnp);
+        ndetected = rcu_print_task_stall(rnp);
         raw_spin_unlock_irqrestore(&rnp->lock, flags);
 
         /*
@@ -565,17 +572,22 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
                rsp->name);
         rcu_for_each_leaf_node(rsp, rnp) {
                 raw_spin_lock_irqsave(&rnp->lock, flags);
-                rcu_print_task_stall(rnp);
+                ndetected += rcu_print_task_stall(rnp);
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
                 if (rnp->qsmask == 0)
                         continue;
                 for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)
-                        if (rnp->qsmask & (1UL << cpu))
+                        if (rnp->qsmask & (1UL << cpu)) {
                                 printk(" %d", rnp->grplo + cpu);
+                                ndetected++;
+                        }
         }
         printk("} (detected by %d, t=%ld jiffies)\n",
                smp_processor_id(), (long)(jiffies - rsp->gp_start));
-        trigger_all_cpu_backtrace();
+        if (ndetected == 0)
+                printk(KERN_ERR "INFO: Stall ended before state dump start\n");
+        else if (!trigger_all_cpu_backtrace())
+                dump_stack();
 
         /* If so configured, complain about tasks blocking the grace period. */
 
@@ -596,7 +608,8 @@ static void print_cpu_stall(struct rcu_state *rsp)
          */
         printk(KERN_ERR "INFO: %s detected stall on CPU %d (t=%lu jiffies)\n",
                rsp->name, smp_processor_id(), jiffies - rsp->gp_start);
-        trigger_all_cpu_backtrace();
+        if (!trigger_all_cpu_backtrace())
+                dump_stack();
 
         raw_spin_lock_irqsave(&rnp->lock, flags);
         if (ULONG_CMP_GE(jiffies, rsp->jiffies_stall))
@@ -678,9 +691,10 @@ static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct
                  * go looking for one.
                  */
                 rdp->gpnum = rnp->gpnum;
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
                 if (rnp->qsmask & rdp->grpmask) {
                         rdp->qs_pending = 1;
-                        rdp->passed_quiesc = 0;
+                        rdp->passed_quiesce = 0;
                 } else
                         rdp->qs_pending = 0;
         }
@@ -741,6 +755,7 @@ __rcu_process_gp_end(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat
 
                 /* Remember that we saw this grace-period completion. */
                 rdp->completed = rnp->completed;
+                trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuend");
 
                 /*
                  * If we were in an extended quiescent state, we may have
@@ -826,31 +841,31 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
         struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
         struct rcu_node *rnp = rcu_get_root(rsp);
 
-        if (!cpu_needs_another_gp(rsp, rdp) || rsp->fqs_active) {
-                if (cpu_needs_another_gp(rsp, rdp))
-                        rsp->fqs_need_gp = 1;
-                if (rnp->completed == rsp->completed) {
-                        raw_spin_unlock_irqrestore(&rnp->lock, flags);
-                        return;
-                }
-                raw_spin_unlock(&rnp->lock);     /* irqs remain disabled. */
+        if (!rcu_scheduler_fully_active ||
+            !cpu_needs_another_gp(rsp, rdp)) {
+                /*
+                 * Either the scheduler hasn't yet spawned the first
+                 * non-idle task or this CPU does not need another
+                 * grace period.  Either way, don't start a new grace
+                 * period.
+                 */
+                raw_spin_unlock_irqrestore(&rnp->lock, flags);
+                return;
+        }
 
+        if (rsp->fqs_active) {
                 /*
-                 * Propagate new ->completed value to rcu_node structures
-                 * so that other CPUs don't have to wait until the start
-                 * of the next grace period to process their callbacks.
+                 * This CPU needs a grace period, but force_quiescent_state()
+                 * is running.  Tell it to start one on this CPU's behalf.
                  */
-                rcu_for_each_node_breadth_first(rsp, rnp) {
-                        raw_spin_lock(&rnp->lock); /* irqs already disabled. */
-                        rnp->completed = rsp->completed;
-                        raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
-                }
-                local_irq_restore(flags);
+                rsp->fqs_need_gp = 1;
+                raw_spin_unlock_irqrestore(&rnp->lock, flags);
                 return;
         }
 
         /* Advance to a new grace period and initialize state. */
         rsp->gpnum++;
+        trace_rcu_grace_period(rsp->name, rsp->gpnum, "start");
         WARN_ON_ONCE(rsp->signaled == RCU_GP_INIT);
         rsp->signaled = RCU_GP_INIT; /* Hold off force_quiescent_state. */
         rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
@@ -865,6 +880,9 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
                 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */
                 rcu_start_gp_per_cpu(rsp, rnp, rdp);
                 rcu_preempt_boost_start_gp(rnp);
+                trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
+                                            rnp->level, rnp->grplo,
+                                            rnp->grphi, rnp->qsmask);
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
                 return;
         }
@@ -901,6 +919,9 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
                 if (rnp == rdp->mynode)
                         rcu_start_gp_per_cpu(rsp, rnp, rdp);
                 rcu_preempt_boost_start_gp(rnp);
+                trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
+                                            rnp->level, rnp->grplo,
+                                            rnp->grphi, rnp->qsmask);
                 raw_spin_unlock(&rnp->lock);    /* irqs remain disabled. */
         }
 
@@ -922,6 +943,8 @@ static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
         __releases(rcu_get_root(rsp)->lock)
 {
         unsigned long gp_duration;
+        struct rcu_node *rnp = rcu_get_root(rsp);
+        struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
 
         WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
 
@@ -933,7 +956,41 @@ static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
         gp_duration = jiffies - rsp->gp_start;
         if (gp_duration > rsp->gp_max)
                 rsp->gp_max = gp_duration;
-        rsp->completed = rsp->gpnum;
+
+        /*
+         * We know the grace period is complete, but to everyone else
+         * it appears to still be ongoing.  But it is also the case
+         * that to everyone else it looks like there is nothing that
+         * they can do to advance the grace period.  It is therefore
+         * safe for us to drop the lock in order to mark the grace
+         * period as completed in all of the rcu_node structures.
+         *
+         * But if this CPU needs another grace period, it will take
+         * care of this while initializing the next grace period.
+         * We use RCU_WAIT_TAIL instead of the usual RCU_DONE_TAIL
+         * because the callbacks have not yet been advanced: Those
+         * callbacks are waiting on the grace period that just now
+         * completed.
+         */
+        if (*rdp->nxttail[RCU_WAIT_TAIL] == NULL) {
+                raw_spin_unlock(&rnp->lock);     /* irqs remain disabled. */
+
+                /*
+                 * Propagate new ->completed value to rcu_node structures
+                 * so that other CPUs don't have to wait until the start
+                 * of the next grace period to process their callbacks.
+                 */
+                rcu_for_each_node_breadth_first(rsp, rnp) {
+                        raw_spin_lock(&rnp->lock); /* irqs already disabled. */
+                        rnp->completed = rsp->gpnum;
+                        raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
+                }
+                rnp = rcu_get_root(rsp);
+                raw_spin_lock(&rnp->lock); /* irqs already disabled. */
+        }
+
+        rsp->completed = rsp->gpnum;  /* Declare the grace period complete. */
+        trace_rcu_grace_period(rsp->name, rsp->completed, "end");
         rsp->signaled = RCU_GP_IDLE;
         rcu_start_gp(rsp, flags);  /* releases root node's rnp->lock. */
 }
@@ -962,6 +1019,10 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
                         return;
                 }
                 rnp->qsmask &= ~mask;
+                trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum,
+                                                 mask, rnp->qsmask, rnp->level,
+                                                 rnp->grplo, rnp->grphi,
+                                                 !!rnp->gp_tasks);
                 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
 
                         /* Other bits still set at this level, so done. */
@@ -1000,7 +1061,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
  * based on quiescent states detected in an earlier grace period!
  */
 static void
-rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp)
+rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastgp)
 {
         unsigned long flags;
         unsigned long mask;
@@ -1008,17 +1069,15 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long las
 
         rnp = rdp->mynode;
         raw_spin_lock_irqsave(&rnp->lock, flags);
-        if (lastcomp != rnp->completed) {
+        if (lastgp != rnp->gpnum || rnp->completed == rnp->gpnum) {
 
                 /*
-                 * Someone beat us to it for this grace period, so leave.
-                 * The race with GP start is resolved by the fact that we
-                 * hold the leaf rcu_node lock, so that the per-CPU bits
-                 * cannot yet be initialized -- so we would simply find our
-                 * CPU's bit already cleared in rcu_report_qs_rnp() if this
-                 * race occurred.
+                 * The grace period in which this quiescent state was
+                 * recorded has ended, so don't report it upwards.
+                 * We will instead need a new quiescent state that lies
+                 * within the current grace period.
                  */
-                rdp->passed_quiesc = 0; /* try again later! */
+                rdp->passed_quiesce = 0;        /* need qs for new gp. */
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
                 return;
         }
@@ -1062,14 +1121,14 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
          * Was there a quiescent state since the beginning of the grace
          * period? If no, then exit and wait for the next call.
          */
-        if (!rdp->passed_quiesc)
+        if (!rdp->passed_quiesce)
                 return;
 
         /*
          * Tell RCU we are done (but rcu_report_qs_rdp() will be the
          * judge of that).
          */
-        rcu_report_qs_rdp(rdp->cpu, rsp, rdp, rdp->passed_quiesc_completed);
+        rcu_report_qs_rdp(rdp->cpu, rsp, rdp, rdp->passed_quiesce_gpnum);
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
@@ -1130,11 +1189,20 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
                 if (rnp->qsmaskinit != 0) {
                         if (rnp != rdp->mynode)
                                 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
+                        else
+                                trace_rcu_grace_period(rsp->name,
+                                                       rnp->gpnum + 1 -
+                                                       !!(rnp->qsmask & mask),
+                                                       "cpuofl");
                         break;
                 }
-                if (rnp == rdp->mynode)
+                if (rnp == rdp->mynode) {
+                        trace_rcu_grace_period(rsp->name,
+                                               rnp->gpnum + 1 -
+                                               !!(rnp->qsmask & mask),
+                                               "cpuofl");
                         need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp);
-                else
+                } else
                         raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
                 mask = rnp->grpmask;
                 rnp = rnp->parent;
@@ -1190,17 +1258,22 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
 {
         unsigned long flags;
         struct rcu_head *next, *list, **tail;
-        int count;
+        int bl, count;
 
         /* If no callbacks are ready, just return.*/
-        if (!cpu_has_callbacks_ready_to_invoke(rdp))
+        if (!cpu_has_callbacks_ready_to_invoke(rdp)) {
+                trace_rcu_batch_start(rsp->name, 0, 0);
+                trace_rcu_batch_end(rsp->name, 0);
                 return;
+        }
 
         /*
          * Extract the list of ready callbacks, disabling to prevent
          * races with call_rcu() from interrupt handlers.
          */
         local_irq_save(flags);
+        bl = rdp->blimit;
+        trace_rcu_batch_start(rsp->name, rdp->qlen, bl);
         list = rdp->nxtlist;
         rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL];
         *rdp->nxttail[RCU_DONE_TAIL] = NULL;
@@ -1216,13 +1289,14 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
                 next = list->next;
                 prefetch(next);
                 debug_rcu_head_unqueue(list);
-                __rcu_reclaim(list);
+                __rcu_reclaim(rsp->name, list);
                 list = next;
-                if (++count >= rdp->blimit)
+                if (++count >= bl)
                         break;
         }
 
         local_irq_save(flags);
+        trace_rcu_batch_end(rsp->name, count);
 
         /* Update count, and requeue any remaining callbacks. */
         rdp->qlen -= count;
@@ -1250,7 +1324,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
 
         local_irq_restore(flags);
 
-        /* Re-raise the RCU softirq if there are callbacks remaining. */
+        /* Re-invoke RCU core processing if there are callbacks remaining. */
         if (cpu_has_callbacks_ready_to_invoke(rdp))
                 invoke_rcu_core();
 }
@@ -1258,7 +1332,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
 /*
  * Check to see if this CPU is in a non-context-switch quiescent state
  * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
- * Also schedule the RCU softirq handler.
+ * Also schedule RCU core processing.
  *
  * This function must be called with hardirqs disabled.  It is normally
  * invoked from the scheduling-clock interrupt.  If rcu_pending returns
@@ -1266,6 +1340,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
  */
 void rcu_check_callbacks(int cpu, int user)
 {
+        trace_rcu_utilization("Start scheduler-tick");
         if (user ||
             (idle_cpu(cpu) && rcu_scheduler_active &&
              !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
@@ -1299,6 +1374,7 @@ void rcu_check_callbacks(int cpu, int user)
         rcu_preempt_check_callbacks(cpu);
         if (rcu_pending(cpu))
                 invoke_rcu_core();
+        trace_rcu_utilization("End scheduler-tick");
 }
 
 #ifdef CONFIG_SMP
@@ -1360,10 +1436,14 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
         unsigned long flags;
         struct rcu_node *rnp = rcu_get_root(rsp);
 
-        if (!rcu_gp_in_progress(rsp))
+        trace_rcu_utilization("Start fqs");
+        if (!rcu_gp_in_progress(rsp)) {
+                trace_rcu_utilization("End fqs");
                 return;  /* No grace period in progress, nothing to force. */
+        }
         if (!raw_spin_trylock_irqsave(&rsp->fqslock, flags)) {
                 rsp->n_force_qs_lh++; /* Inexact, can lose counts.  Tough! */
+                trace_rcu_utilization("End fqs");
                 return; /* Someone else is already on the job. */
         }
         if (relaxed && ULONG_CMP_GE(rsp->jiffies_force_qs, jiffies))
@@ -1412,11 +1492,13 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
                 raw_spin_unlock(&rsp->fqslock); /* irqs remain disabled */
                 rsp->fqs_need_gp = 0;
                 rcu_start_gp(rsp, flags); /* releases rnp->lock */
+                trace_rcu_utilization("End fqs");
                 return;
         }
         raw_spin_unlock(&rnp->lock);  /* irqs remain disabled */
 unlock_fqs_ret:
         raw_spin_unlock_irqrestore(&rsp->fqslock, flags);
+        trace_rcu_utilization("End fqs");
 }
 
 #else /* #ifdef CONFIG_SMP */
@@ -1429,9 +1511,9 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
 #endif /* #else #ifdef CONFIG_SMP */
 
 /*
- * This does the RCU processing work from softirq context for the
- * specified rcu_state and rcu_data structures.  This may be called
- * only from the CPU to whom the rdp belongs.
+ * This does the RCU core processing work for the specified rcu_state
+ * and rcu_data structures.  This may be called only from the CPU to
+ * whom the rdp belongs.
  */
 static void
 __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
@@ -1468,24 +1550,24 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
 }
 
 /*
- * Do softirq processing for the current CPU.
+ * Do RCU core processing for the current CPU.
  */
 static void rcu_process_callbacks(struct softirq_action *unused)
 {
+        trace_rcu_utilization("Start RCU core");
         __rcu_process_callbacks(&rcu_sched_state,
                                 &__get_cpu_var(rcu_sched_data));
         __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
         rcu_preempt_process_callbacks();
-
-        /* If we are last CPU on way to dyntick-idle mode, accelerate it. */
-        rcu_needs_cpu_flush();
+        trace_rcu_utilization("End RCU core");
 }
 
 /*
- * Wake up the current CPU's kthread.  This replaces raise_softirq()
- * in earlier versions of RCU.  Note that because we are running on
- * the current CPU with interrupts disabled, the rcu_cpu_kthread_task
- * cannot disappear out from under us.
+ * Schedule RCU callback invocation.  If the specified type of RCU
+ * does not support RCU priority boosting, just do a direct call,
+ * otherwise wake up the per-CPU kernel kthread.  Note that because we
+ * are running on the current CPU with interrupts disabled, the
+ * rcu_cpu_kthread_task cannot disappear out from under us.
  */
 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
 {
@@ -1530,6 +1612,12 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
         rdp->nxttail[RCU_NEXT_TAIL] = &head->next;
         rdp->qlen++;
 
+        if (__is_kfree_rcu_offset((unsigned long)func))
+                trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
+                                         rdp->qlen);
+        else
+                trace_rcu_callback(rsp->name, head, rdp->qlen);
+
         /* If interrupts were disabled, don't dive into RCU core. */
         if (irqs_disabled_flags(flags)) {
                 local_irq_restore(flags);
@@ -1613,18 +1701,9 @@ EXPORT_SYMBOL_GPL(call_rcu_bh);
  */
 void synchronize_sched(void)
 {
-        struct rcu_synchronize rcu;
-
         if (rcu_blocking_is_gp())
                 return;
-
-        init_rcu_head_on_stack(&rcu.head);
-        init_completion(&rcu.completion);
-        /* Will wake me after RCU finished. */
-        call_rcu_sched(&rcu.head, wakeme_after_rcu);
-        /* Wait for it. */
-        wait_for_completion(&rcu.completion);
-        destroy_rcu_head_on_stack(&rcu.head);
+        wait_rcu_gp(call_rcu_sched);
 }
 EXPORT_SYMBOL_GPL(synchronize_sched);
 
@@ -1639,18 +1718,9 @@ EXPORT_SYMBOL_GPL(synchronize_sched);
  */
 void synchronize_rcu_bh(void)
 {
-        struct rcu_synchronize rcu;
-
         if (rcu_blocking_is_gp())
                 return;
-
-        init_rcu_head_on_stack(&rcu.head);
-        init_completion(&rcu.completion);
-        /* Will wake me after RCU finished. */
-        call_rcu_bh(&rcu.head, wakeme_after_rcu);
-        /* Wait for it. */
-        wait_for_completion(&rcu.completion);
-        destroy_rcu_head_on_stack(&rcu.head);
+        wait_rcu_gp(call_rcu_bh);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
 
@@ -1671,7 +1741,8 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
         check_cpu_stall(rsp, rdp);
 
         /* Is the RCU core waiting for a quiescent state from this CPU? */
-        if (rdp->qs_pending && !rdp->passed_quiesc) {
+        if (rcu_scheduler_fully_active &&
+            rdp->qs_pending && !rdp->passed_quiesce) {
 
                 /*
                  * If force_quiescent_state() coming soon and this CPU
@@ -1683,7 +1754,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
                     ULONG_CMP_LT(ACCESS_ONCE(rsp->jiffies_force_qs) - 1,
                                  jiffies))
                         set_need_resched();
-        } else if (rdp->qs_pending && rdp->passed_quiesc) {
+        } else if (rdp->qs_pending && rdp->passed_quiesce) {
                 rdp->n_rp_report_qs++;
                 return 1;
         }
@@ -1846,6 +1917,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
         rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
 #endif /* #ifdef CONFIG_NO_HZ */
         rdp->cpu = cpu;
+        rdp->rsp = rsp;
         raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
 
@@ -1865,8 +1937,6 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
 
         /* Set up local state, ensuring consistent view of global state. */
         raw_spin_lock_irqsave(&rnp->lock, flags);
-        rdp->passed_quiesc = 0;  /* We could be racing with new GP, */
-        rdp->qs_pending = 1;     /*  so set up to respond to current GP. */
         rdp->beenonline = 1;     /* We have now been online. */
         rdp->preemptible = preemptible;
         rdp->qlen_last_fqs_check = 0;
@@ -1891,9 +1961,17 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
                 rnp->qsmaskinit |= mask;
                 mask = rnp->grpmask;
                 if (rnp == rdp->mynode) {
-                        rdp->gpnum = rnp->completed; /* if GP in progress... */
+                        /*
+                         * If there is a grace period in progress, we will
+                         * set up to wait for it next time we run the
+                         * RCU core code.
+                         */
+                        rdp->gpnum = rnp->completed;
                         rdp->completed = rnp->completed;
-                        rdp->passed_quiesc_completed = rnp->completed - 1;
+                        rdp->passed_quiesce = 0;
+                        rdp->qs_pending = 0;
+                        rdp->passed_quiesce_gpnum = rnp->gpnum - 1;
+                        trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuonl");
                 }
                 raw_spin_unlock(&rnp->lock); /* irqs already disabled. */
                 rnp = rnp->parent;
@@ -1919,6 +1997,7 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
         struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
         struct rcu_node *rnp = rdp->mynode;
 
+        trace_rcu_utilization("Start CPU hotplug");
         switch (action) {
         case CPU_UP_PREPARE:
         case CPU_UP_PREPARE_FROZEN:
@@ -1954,6 +2033,7 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
         default:
                 break;
         }
+        trace_rcu_utilization("End CPU hotplug");
         return NOTIFY_OK;
 }