1 files changed, 146 insertions, 83 deletions

diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index c023464816be..3e4899459f3d 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -969,22 +969,6 @@ static void __init __rcu_init_preempt(void)
         rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
 }
 
-/*
- * Check for a task exiting while in a preemptible-RCU read-side
- * critical section, clean up if so.  No need to issue warnings,
- * as debug_check_no_locks_held() already does this if lockdep
- * is enabled.
- */
-void exit_rcu(void)
-{
-        struct task_struct *t = current;
-
-        if (t->rcu_read_lock_nesting == 0)
-                return;
-        t->rcu_read_lock_nesting = 1;
-        __rcu_read_unlock();
-}
-
 #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static struct rcu_state *rcu_state = &rcu_sched_state;
@@ -1910,8 +1894,9 @@ static void __cpuinit rcu_prepare_kthreads(int cpu)
  * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
  * any flavor of RCU.
  */
-int rcu_needs_cpu(int cpu)
+int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
 {
+        *delta_jiffies = ULONG_MAX;
         return rcu_cpu_has_callbacks(cpu);
 }
 
@@ -1938,6 +1923,14 @@ static void rcu_prepare_for_idle(int cpu)
 {
 }
 
+/*
+ * Don't bother keeping a running count of the number of RCU callbacks
+ * posted because CONFIG_RCU_FAST_NO_HZ=n.
+ */
+static void rcu_idle_count_callbacks_posted(void)
+{
+}
+
 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
 
 /*
@@ -1978,30 +1971,6 @@ static void rcu_prepare_for_idle(int cpu)
 #define RCU_IDLE_GP_DELAY 6             /* Roughly one grace period. */
 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
 
-static DEFINE_PER_CPU(int, rcu_dyntick_drain);
-static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
-static DEFINE_PER_CPU(struct hrtimer, rcu_idle_gp_timer);
-static ktime_t rcu_idle_gp_wait;        /* If some non-lazy callbacks. */
-static ktime_t rcu_idle_lazy_gp_wait;   /* If only lazy callbacks. */
-
-/*
- * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
- * callbacks on this CPU, (2) this CPU has not yet attempted to enter
- * dyntick-idle mode, or (3) this CPU is in the process of attempting to
- * enter dyntick-idle mode.  Otherwise, if we have recently tried and failed
- * to enter dyntick-idle mode, we refuse to try to enter it.  After all,
- * it is better to incur scheduling-clock interrupts than to spin
- * continuously for the same time duration!
- */
-int rcu_needs_cpu(int cpu)
-{
-        /* If no callbacks, RCU doesn't need the CPU. */
-        if (!rcu_cpu_has_callbacks(cpu))
-                return 0;
-        /* Otherwise, RCU needs the CPU only if it recently tried and failed. */
-        return per_cpu(rcu_dyntick_holdoff, cpu) == jiffies;
-}
-
 /*
  * Does the specified flavor of RCU have non-lazy callbacks pending on
  * the specified CPU?  Both RCU flavor and CPU are specified by the
@@ -2045,16 +2014,75 @@ static bool rcu_cpu_has_nonlazy_callbacks(int cpu)
 }
 
 /*
+ * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
+ * callbacks on this CPU, (2) this CPU has not yet attempted to enter
+ * dyntick-idle mode, or (3) this CPU is in the process of attempting to
+ * enter dyntick-idle mode.  Otherwise, if we have recently tried and failed
+ * to enter dyntick-idle mode, we refuse to try to enter it.  After all,
+ * it is better to incur scheduling-clock interrupts than to spin
+ * continuously for the same time duration!
+ *
+ * The delta_jiffies argument is used to store the time when RCU is
+ * going to need the CPU again if it still has callbacks.  The reason
+ * for this is that rcu_prepare_for_idle() might need to post a timer,
+ * but if so, it will do so after tick_nohz_stop_sched_tick() has set
+ * the wakeup time for this CPU.  This means that RCU's timer can be
+ * delayed until the wakeup time, which defeats the purpose of posting
+ * a timer.
+ */
+int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
+{
+        struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
+
+        /* Flag a new idle sojourn to the idle-entry state machine. */
+        rdtp->idle_first_pass = 1;
+        /* If no callbacks, RCU doesn't need the CPU. */
+        if (!rcu_cpu_has_callbacks(cpu)) {
+                *delta_jiffies = ULONG_MAX;
+                return 0;
+        }
+        if (rdtp->dyntick_holdoff == jiffies) {
+                /* RCU recently tried and failed, so don't try again. */
+                *delta_jiffies = 1;
+                return 1;
+        }
+        /* Set up for the possibility that RCU will post a timer. */
+        if (rcu_cpu_has_nonlazy_callbacks(cpu))
+                *delta_jiffies = RCU_IDLE_GP_DELAY;
+        else
+                *delta_jiffies = RCU_IDLE_LAZY_GP_DELAY;
+        return 0;
+}
+
+/*
+ * Handler for smp_call_function_single().  The only point of this
+ * handler is to wake the CPU up, so the handler does only tracing.
+ */
+void rcu_idle_demigrate(void *unused)
+{
+        trace_rcu_prep_idle("Demigrate");
+}
+
+/*
  * Timer handler used to force CPU to start pushing its remaining RCU
  * callbacks in the case where it entered dyntick-idle mode with callbacks
  * pending.  The hander doesn't really need to do anything because the
  * real work is done upon re-entry to idle, or by the next scheduling-clock
  * interrupt should idle not be re-entered.
+ *
+ * One special case: the timer gets migrated without awakening the CPU
+ * on which the timer was scheduled on.  In this case, we must wake up
+ * that CPU.  We do so with smp_call_function_single().
  */
-static enum hrtimer_restart rcu_idle_gp_timer_func(struct hrtimer *hrtp)
+static void rcu_idle_gp_timer_func(unsigned long cpu_in)
 {
+        int cpu = (int)cpu_in;
+
         trace_rcu_prep_idle("Timer");
-        return HRTIMER_NORESTART;
+        if (cpu != smp_processor_id())
+                smp_call_function_single(cpu, rcu_idle_demigrate, NULL, 0);
+        else
+                WARN_ON_ONCE(1); /* Getting here can hang the system... */
 }
 
 /*
@@ -2062,29 +2090,25 @@ static enum hrtimer_restart rcu_idle_gp_timer_func(struct hrtimer *hrtp)
  */
 static void rcu_prepare_for_idle_init(int cpu)
 {
-        static int firsttime = 1;
-        struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu);
+        struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
 
-        hrtimer_init(hrtp, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
-        hrtp->function = rcu_idle_gp_timer_func;
-        if (firsttime) {
-                unsigned int upj = jiffies_to_usecs(RCU_IDLE_GP_DELAY);
-
-                rcu_idle_gp_wait = ns_to_ktime(upj * (u64)1000);
-                upj = jiffies_to_usecs(RCU_IDLE_LAZY_GP_DELAY);
-                rcu_idle_lazy_gp_wait = ns_to_ktime(upj * (u64)1000);
-                firsttime = 0;
-        }
+        rdtp->dyntick_holdoff = jiffies - 1;
+        setup_timer(&rdtp->idle_gp_timer, rcu_idle_gp_timer_func, cpu);
+        rdtp->idle_gp_timer_expires = jiffies - 1;
+        rdtp->idle_first_pass = 1;
 }
 
 /*
  * Clean up for exit from idle.  Because we are exiting from idle, there
- * is no longer any point to rcu_idle_gp_timer, so cancel it.  This will
+ * is no longer any point to ->idle_gp_timer, so cancel it.  This will
  * do nothing if this timer is not active, so just cancel it unconditionally.
  */
 static void rcu_cleanup_after_idle(int cpu)
 {
-        hrtimer_cancel(&per_cpu(rcu_idle_gp_timer, cpu));
+        struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
+
+        del_timer(&rdtp->idle_gp_timer);
+        trace_rcu_prep_idle("Cleanup after idle");
 }
 
 /*
@@ -2102,19 +2126,41 @@ static void rcu_cleanup_after_idle(int cpu)
  * Because it is not legal to invoke rcu_process_callbacks() with irqs
  * disabled, we do one pass of force_quiescent_state(), then do a
  * invoke_rcu_core() to cause rcu_process_callbacks() to be invoked
- * later.  The per-cpu rcu_dyntick_drain variable controls the sequencing.
+ * later.  The ->dyntick_drain field controls the sequencing.
  *
  * The caller must have disabled interrupts.
  */
 static void rcu_prepare_for_idle(int cpu)
 {
+        struct timer_list *tp;
+        struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
+
+        /*
+         * If this is an idle re-entry, for example, due to use of
+         * RCU_NONIDLE() or the new idle-loop tracing API within the idle
+         * loop, then don't take any state-machine actions, unless the
+         * momentary exit from idle queued additional non-lazy callbacks.
+         * Instead, repost the ->idle_gp_timer if this CPU has callbacks
+         * pending.
+         */
+        if (!rdtp->idle_first_pass &&
+            (rdtp->nonlazy_posted == rdtp->nonlazy_posted_snap)) {
+                if (rcu_cpu_has_callbacks(cpu)) {
+                        tp = &rdtp->idle_gp_timer;
+                        mod_timer_pinned(tp, rdtp->idle_gp_timer_expires);
+                }
+                return;
+        }
+        rdtp->idle_first_pass = 0;
+        rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted - 1;
+
         /*
          * If there are no callbacks on this CPU, enter dyntick-idle mode.
          * Also reset state to avoid prejudicing later attempts.
          */
         if (!rcu_cpu_has_callbacks(cpu)) {
-                per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
-                per_cpu(rcu_dyntick_drain, cpu) = 0;
+                rdtp->dyntick_holdoff = jiffies - 1;
+                rdtp->dyntick_drain = 0;
                 trace_rcu_prep_idle("No callbacks");
                 return;
         }
@@ -2123,32 +2169,37 @@ static void rcu_prepare_for_idle(int cpu)
          * If in holdoff mode, just return.  We will presumably have
          * refrained from disabling the scheduling-clock tick.
          */
-        if (per_cpu(rcu_dyntick_holdoff, cpu) == jiffies) {
+        if (rdtp->dyntick_holdoff == jiffies) {
                 trace_rcu_prep_idle("In holdoff");
                 return;
         }
 
-        /* Check and update the rcu_dyntick_drain sequencing. */
-        if (per_cpu(rcu_dyntick_drain, cpu) <= 0) {
+        /* Check and update the ->dyntick_drain sequencing. */
+        if (rdtp->dyntick_drain <= 0) {
                 /* First time through, initialize the counter. */
-                per_cpu(rcu_dyntick_drain, cpu) = RCU_IDLE_FLUSHES;
-        } else if (per_cpu(rcu_dyntick_drain, cpu) <= RCU_IDLE_OPT_FLUSHES &&
+                rdtp->dyntick_drain = RCU_IDLE_FLUSHES;
+        } else if (rdtp->dyntick_drain <= RCU_IDLE_OPT_FLUSHES &&
                    !rcu_pending(cpu) &&
                    !local_softirq_pending()) {
                 /* Can we go dyntick-idle despite still having callbacks? */
-                trace_rcu_prep_idle("Dyntick with callbacks");
-                per_cpu(rcu_dyntick_drain, cpu) = 0;
-                per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
-                if (rcu_cpu_has_nonlazy_callbacks(cpu))
-                        hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
-                                      rcu_idle_gp_wait, HRTIMER_MODE_REL);
-                else
-                        hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
-                                      rcu_idle_lazy_gp_wait, HRTIMER_MODE_REL);
+                rdtp->dyntick_drain = 0;
+                rdtp->dyntick_holdoff = jiffies;
+                if (rcu_cpu_has_nonlazy_callbacks(cpu)) {
+                        trace_rcu_prep_idle("Dyntick with callbacks");
+                        rdtp->idle_gp_timer_expires =
+                                           jiffies + RCU_IDLE_GP_DELAY;
+                } else {
+                        rdtp->idle_gp_timer_expires =
+                                           jiffies + RCU_IDLE_LAZY_GP_DELAY;
+                        trace_rcu_prep_idle("Dyntick with lazy callbacks");
+                }
+                tp = &rdtp->idle_gp_timer;
+                mod_timer_pinned(tp, rdtp->idle_gp_timer_expires);
+                rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
                 return; /* Nothing more to do immediately. */
-        } else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
+        } else if (--(rdtp->dyntick_drain) <= 0) {
                 /* We have hit the limit, so time to give up. */
-                per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
+                rdtp->dyntick_holdoff = jiffies;
                 trace_rcu_prep_idle("Begin holdoff");
                 invoke_rcu_core();  /* Force the CPU out of dyntick-idle. */
                 return;
@@ -2184,6 +2235,19 @@ static void rcu_prepare_for_idle(int cpu)
                 trace_rcu_prep_idle("Callbacks drained");
 }
 
+/*
+ * Keep a running count of the number of non-lazy callbacks posted
+ * on this CPU.  This running counter (which is never decremented) allows
+ * rcu_prepare_for_idle() to detect when something out of the idle loop
+ * posts a callback, even if an equal number of callbacks are invoked.
+ * Of course, callbacks should only be posted from within a trace event
+ * designed to be called from idle or from within RCU_NONIDLE().
+ */
+static void rcu_idle_count_callbacks_posted(void)
+{
+        __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
+}
+
 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
 
 #ifdef CONFIG_RCU_CPU_STALL_INFO
@@ -2192,14 +2256,13 @@ static void rcu_prepare_for_idle(int cpu)
 
 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
 {
-        struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu);
+        struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
+        struct timer_list *tltp = &rdtp->idle_gp_timer;
 
-        sprintf(cp, "drain=%d %c timer=%lld",
-                per_cpu(rcu_dyntick_drain, cpu),
-                per_cpu(rcu_dyntick_holdoff, cpu) == jiffies ? 'H' : '.',
-                hrtimer_active(hrtp)
-                        ? ktime_to_us(hrtimer_get_remaining(hrtp))
-                        : -1);
+        sprintf(cp, "drain=%d %c timer=%lu",
+                rdtp->dyntick_drain,
+                rdtp->dyntick_holdoff == jiffies ? 'H' : '.',
+                timer_pending(tltp) ? tltp->expires - jiffies : -1);
 }
 
 #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */