Merge branch 'rcu/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into core/urgent

Merge RCU fixes from Paul E. McKenney:

 " This series has four patches, the major point of which is to eliminate
   some slowdowns (including boot-time slowdowns) resulting from some
   RCU_FAST_NO_HZ changes.  The issue with the changes is that posting timers
   from the idle loop has no effect if the CPU has entered dyntick-idle
   mode because the CPU has already computed its wakeup time, and posting
   a timer does not cause it to be recomputed.  The short-term fix is for
   RCU to precompute the timeout value so that the CPU's calculation is
   correct. "

Signed-off-by: Ingo Molnar <mingo@kernel.org>

7 files changed, 116 insertions, 81 deletions

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index adb5e5a38ca..854dc4c5c27 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -87,8 +87,9 @@ static inline void kfree_call_rcu(struct rcu_head *head,
 
 #ifdef CONFIG_TINY_RCU
 
-static inline int rcu_needs_cpu(int cpu)
+static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
 {
+        *delta_jiffies = ULONG_MAX;
         return 0;
 }
 
@@ -96,8 +97,9 @@ static inline int rcu_needs_cpu(int cpu)
 
 int rcu_preempt_needs_cpu(void);
 
-static inline int rcu_needs_cpu(int cpu)
+static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
 {
+        *delta_jiffies = ULONG_MAX;
         return rcu_preempt_needs_cpu();
 }
 
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 3c6083cde4f..952b7933930 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -32,7 +32,7 @@
 
 extern void rcu_init(void);
 extern void rcu_note_context_switch(int cpu);
-extern int rcu_needs_cpu(int cpu);
+extern int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies);
 extern void rcu_cpu_stall_reset(void);
 
 /*
diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index 1480900c511..d274734b2aa 100644
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -289,6 +289,7 @@ TRACE_EVENT(rcu_dyntick,
  *      "In holdoff": Nothing to do, holding off after unsuccessful attempt.
  *      "Begin holdoff": Attempt failed, don't retry until next jiffy.
  *      "Dyntick with callbacks": Entering dyntick-idle despite callbacks.
+ *      "Dyntick with lazy callbacks": Entering dyntick-idle w/lazy callbacks.
  *      "More callbacks": Still more callbacks, try again to clear them out.
  *      "Callbacks drained": All callbacks processed, off to dyntick idle!
  *      "Timer": Timer fired to cause CPU to continue processing callbacks.
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 0da7b88d92d..3b0f1337f75 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1397,6 +1397,8 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
         rdp->qlen_lazy += rsp->qlen_lazy;
         rdp->qlen += rsp->qlen;
         rdp->n_cbs_adopted += rsp->qlen;
+        if (rsp->qlen_lazy != rsp->qlen)
+                rcu_idle_count_callbacks_posted();
         rsp->qlen_lazy = 0;
         rsp->qlen = 0;
 
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 7f5d138dedf..ea056495783 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -84,6 +84,20 @@ struct rcu_dynticks {
                                     /* Process level is worth LLONG_MAX/2. */
         int dynticks_nmi_nesting;   /* Track NMI nesting level. */
         atomic_t dynticks;          /* Even value for idle, else odd. */
+#ifdef CONFIG_RCU_FAST_NO_HZ
+        int dyntick_drain;          /* Prepare-for-idle state variable. */
+        unsigned long dyntick_holdoff;
+                                    /* No retries for the jiffy of failure. */
+        struct timer_list idle_gp_timer;
+                                    /* Wake up CPU sleeping with callbacks. */
+        unsigned long idle_gp_timer_expires;
+                                    /* When to wake up CPU (for repost). */
+        bool idle_first_pass;       /* First pass of attempt to go idle? */
+        unsigned long nonlazy_posted;
+                                    /* # times non-lazy CBs posted to CPU. */
+        unsigned long nonlazy_posted_snap;
+                                    /* idle-period nonlazy_posted snapshot. */
+#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
 };
 
 /* RCU's kthread states for tracing. */
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 2411000d986..5271a020887 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -1886,8 +1886,9 @@ static void __cpuinit rcu_prepare_kthreads(int cpu)
  * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
  * any flavor of RCU.
  */
-int rcu_needs_cpu(int cpu)
+int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
 {
+        *delta_jiffies = ULONG_MAX;
         return rcu_cpu_has_callbacks(cpu);
 }
 
@@ -1962,41 +1963,6 @@ static void rcu_idle_count_callbacks_posted(void)
 #define RCU_IDLE_GP_DELAY 6             /* Roughly one grace period. */
 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
 
-/* Loop counter for rcu_prepare_for_idle(). */
-static DEFINE_PER_CPU(int, rcu_dyntick_drain);
-/* If rcu_dyntick_holdoff==jiffies, don't try to enter dyntick-idle mode. */
-static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
-/* Timer to awaken the CPU if it enters dyntick-idle mode with callbacks. */
-static DEFINE_PER_CPU(struct timer_list, rcu_idle_gp_timer);
-/* Scheduled expiry time for rcu_idle_gp_timer to allow reposting. */
-static DEFINE_PER_CPU(unsigned long, rcu_idle_gp_timer_expires);
-/* Enable special processing on first attempt to enter dyntick-idle mode. */
-static DEFINE_PER_CPU(bool, rcu_idle_first_pass);
-/* Running count of non-lazy callbacks posted, never decremented. */
-static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted);
-/* Snapshot of rcu_nonlazy_posted to detect meaningful exits from idle. */
-static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted_snap);
-
-/*
- * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
- * callbacks on this CPU, (2) this CPU has not yet attempted to enter
- * dyntick-idle mode, or (3) this CPU is in the process of attempting to
- * enter dyntick-idle mode.  Otherwise, if we have recently tried and failed
- * to enter dyntick-idle mode, we refuse to try to enter it.  After all,
- * it is better to incur scheduling-clock interrupts than to spin
- * continuously for the same time duration!
- */
-int rcu_needs_cpu(int cpu)
-{
-        /* Flag a new idle sojourn to the idle-entry state machine. */
-        per_cpu(rcu_idle_first_pass, cpu) = 1;
-        /* If no callbacks, RCU doesn't need the CPU. */
-        if (!rcu_cpu_has_callbacks(cpu))
-                return 0;
-        /* Otherwise, RCU needs the CPU only if it recently tried and failed. */
-        return per_cpu(rcu_dyntick_holdoff, cpu) == jiffies;
-}
-
 /*
  * Does the specified flavor of RCU have non-lazy callbacks pending on
  * the specified CPU?  Both RCU flavor and CPU are specified by the
@@ -2040,6 +2006,47 @@ static bool rcu_cpu_has_nonlazy_callbacks(int cpu)
 }
 
 /*
+ * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
+ * callbacks on this CPU, (2) this CPU has not yet attempted to enter
+ * dyntick-idle mode, or (3) this CPU is in the process of attempting to
+ * enter dyntick-idle mode.  Otherwise, if we have recently tried and failed
+ * to enter dyntick-idle mode, we refuse to try to enter it.  After all,
+ * it is better to incur scheduling-clock interrupts than to spin
+ * continuously for the same time duration!
+ *
+ * The delta_jiffies argument is used to store the time when RCU is
+ * going to need the CPU again if it still has callbacks.  The reason
+ * for this is that rcu_prepare_for_idle() might need to post a timer,
+ * but if so, it will do so after tick_nohz_stop_sched_tick() has set
+ * the wakeup time for this CPU.  This means that RCU's timer can be
+ * delayed until the wakeup time, which defeats the purpose of posting
+ * a timer.
+ */
+int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
+{
+        struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
+
+        /* Flag a new idle sojourn to the idle-entry state machine. */
+        rdtp->idle_first_pass = 1;
+        /* If no callbacks, RCU doesn't need the CPU. */
+        if (!rcu_cpu_has_callbacks(cpu)) {
+                *delta_jiffies = ULONG_MAX;
+                return 0;
+        }
+        if (rdtp->dyntick_holdoff == jiffies) {
+                /* RCU recently tried and failed, so don't try again. */
+                *delta_jiffies = 1;
+                return 1;
+        }
+        /* Set up for the possibility that RCU will post a timer. */
+        if (rcu_cpu_has_nonlazy_callbacks(cpu))
+                *delta_jiffies = RCU_IDLE_GP_DELAY;
+        else
+                *delta_jiffies = RCU_IDLE_LAZY_GP_DELAY;
+        return 0;
+}