Merge branch 'idle.2013.09.25a' into HEAD

idle.2013.09.25a: Topic branch for idle entry-/exit-related changes.

6 files changed, 41 insertions, 24 deletions

diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index e16c45b9ee77..4e8e14c34e42 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -4224,7 +4224,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
         printk("\n%srcu_scheduler_active = %d, debug_locks = %d\n",
                !rcu_lockdep_current_cpu_online()
                         ? "RCU used illegally from offline CPU!\n"
-                        : rcu_is_cpu_idle()
+                        : !rcu_is_watching()
                                 ? "RCU used illegally from idle CPU!\n"
                                 : "",
                rcu_scheduler_active, debug_locks);
@@ -4247,7 +4247,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
          * So complain bitterly if someone does call rcu_read_lock(),
          * rcu_read_lock_bh() and so on from extended quiescent states.
          */
-        if (rcu_is_cpu_idle())
+        if (!rcu_is_watching())
                 printk("RCU used illegally from extended quiescent state!\n");
 
         lockdep_print_held_locks(curr);
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 3260a1074b48..c07af1c4e1bb 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -148,7 +148,7 @@ int rcu_read_lock_bh_held(void)
 {
         if (!debug_lockdep_rcu_enabled())
                 return 1;
-        if (rcu_is_cpu_idle())
+        if (!rcu_is_watching())
                 return 0;
         if (!rcu_lockdep_current_cpu_online())
                 return 0;
diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c
index 7e3b0d6fc6e2..312e9709713f 100644
--- a/kernel/rcutiny.c
+++ b/kernel/rcutiny.c
@@ -176,18 +176,18 @@ void rcu_irq_enter(void)
 }
 EXPORT_SYMBOL_GPL(rcu_irq_enter);
 
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
+#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE)
 
 /*
  * Test whether RCU thinks that the current CPU is idle.
  */
-int rcu_is_cpu_idle(void)
+bool __rcu_is_watching(void)
 {
-        return !rcu_dynticks_nesting;
+        return rcu_dynticks_nesting;
 }
-EXPORT_SYMBOL(rcu_is_cpu_idle);
+EXPORT_SYMBOL(__rcu_is_watching);
 
-#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
+#endif /* defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
 
 /*
  * Test whether the current CPU was interrupted from idle.  Nested
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index a06d172c75e0..240604aa3f70 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -650,21 +650,34 @@ void rcu_nmi_exit(void)
 }
 
 /**
- * rcu_is_cpu_idle - see if RCU thinks that the current CPU is idle
+ * __rcu_is_watching - are RCU read-side critical sections safe?
+ *
+ * Return true if RCU is watching the running CPU, which means that
+ * this CPU can safely enter RCU read-side critical sections.  Unlike
+ * rcu_is_watching(), the caller of __rcu_is_watching() must have at
+ * least disabled preemption.
+ */
+bool __rcu_is_watching(void)
+{
+        return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
+}
+
+/**
+ * rcu_is_watching - see if RCU thinks that the current CPU is idle
  *
  * If the current CPU is in its idle loop and is neither in an interrupt
  * or NMI handler, return true.
  */
-int rcu_is_cpu_idle(void)
+bool rcu_is_watching(void)
 {
         int ret;
 
         preempt_disable();
-        ret = (atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1) == 0;
+        ret = __rcu_is_watching();
         preempt_enable();
         return ret;
 }
-EXPORT_SYMBOL(rcu_is_cpu_idle);
+EXPORT_SYMBOL_GPL(rcu_is_watching);
 
 #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
 
@@ -2321,7 +2334,7 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
          * If called from an extended quiescent state, invoke the RCU
          * core in order to force a re-evaluation of RCU's idleness.
          */
-        if (rcu_is_cpu_idle() && cpu_online(smp_processor_id()))
+        if (!rcu_is_watching() && cpu_online(smp_processor_id()))
                 invoke_rcu_core();
 
         /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 5f97eab602cd..52be957c9fe2 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -104,6 +104,8 @@ struct rcu_dynticks {
                                     /* idle-period nonlazy_posted snapshot. */
         unsigned long last_accelerate;
                                     /* Last jiffy CBs were accelerated. */
+        unsigned long last_advance_all;
+                                    /* Last jiffy CBs were all advanced. */
         int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */
 #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
 };
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index cd95efa1da48..8d85a5ce093a 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -1635,17 +1635,23 @@ module_param(rcu_idle_lazy_gp_delay, int, 0644);
 extern int tick_nohz_enabled;
 
 /*
- * Try to advance callbacks for all flavors of RCU on the current CPU.
- * Afterwards, if there are any callbacks ready for immediate invocation,
- * return true.
+ * Try to advance callbacks for all flavors of RCU on the current CPU, but
+ * only if it has been awhile since the last time we did so.  Afterwards,
+ * if there are any callbacks ready for immediate invocation, return true.
  */
 static bool rcu_try_advance_all_cbs(void)
 {
         bool cbs_ready = false;
         struct rcu_data *rdp;
+        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
         struct rcu_node *rnp;
         struct rcu_state *rsp;
 
+        /* Exit early if we advanced recently. */
+        if (jiffies == rdtp->last_advance_all)
+                return 0;
+        rdtp->last_advance_all = jiffies;
+
         for_each_rcu_flavor(rsp) {
                 rdp = this_cpu_ptr(rsp->rda);
                 rnp = rdp->mynode;
@@ -1744,6 +1750,8 @@ static void rcu_prepare_for_idle(int cpu)
          */
         if (rdtp->all_lazy &&
             rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
+                rdtp->all_lazy = false;
+                rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
                 invoke_rcu_core();
                 return;
         }
@@ -1773,17 +1781,11 @@ static void rcu_prepare_for_idle(int cpu)
  */
 static void rcu_cleanup_after_idle(int cpu)
 {
-        struct rcu_data *rdp;
-        struct rcu_state *rsp;
 
         if (rcu_is_nocb_cpu(cpu))
                 return;
-        rcu_try_advance_all_cbs();
-        for_each_rcu_flavor(rsp) {
-                rdp = per_cpu_ptr(rsp->rda, cpu);
-                if (cpu_has_callbacks_ready_to_invoke(rdp))
-                        invoke_rcu_core();
-        }
+        if (rcu_try_advance_all_cbs())
+                invoke_rcu_core();
 }
 
 /*