Merge git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched

* git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched:
  softlockup: fix task state setting
  rcu: add support for dynamic ticks and preempt rcu

7 files changed, 266 insertions, 10 deletions

diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 2961ec788046..49829988bfa0 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -109,6 +109,14 @@ static inline void account_system_vtime(struct task_struct *tsk)
 }
 #endif
 
+#if defined(CONFIG_PREEMPT_RCU) && defined(CONFIG_NO_HZ)
+extern void rcu_irq_enter(void);
+extern void rcu_irq_exit(void);
+#else
+# define rcu_irq_enter() do { } while (0)
+# define rcu_irq_exit() do { } while (0)
+#endif /* CONFIG_PREEMPT_RCU */
+
 /*
  * It is safe to do non-atomic ops on ->hardirq_context,
  * because NMI handlers may not preempt and the ops are
@@ -117,6 +125,7 @@ static inline void account_system_vtime(struct task_struct *tsk)
  */
 #define __irq_enter()                                   \
         do {                                            \
+                rcu_irq_enter();                        \
                 account_system_vtime(current);          \
                 add_preempt_count(HARDIRQ_OFFSET);      \
                 trace_hardirq_enter();                  \
@@ -135,6 +144,7 @@ extern void irq_enter(void);
                 trace_hardirq_exit();                   \
                 account_system_vtime(current);          \
                 sub_preempt_count(HARDIRQ_OFFSET);      \
+                rcu_irq_exit();                         \
         } while (0)
 
 /*
diff --git a/include/linux/rcuclassic.h b/include/linux/rcuclassic.h
index 4d6624260b4c..b3dccd68629e 100644
--- a/include/linux/rcuclassic.h
+++ b/include/linux/rcuclassic.h
@@ -160,5 +160,8 @@ extern void rcu_restart_cpu(int cpu);
 extern long rcu_batches_completed(void);
 extern long rcu_batches_completed_bh(void);
 
+#define rcu_enter_nohz()        do { } while (0)
+#define rcu_exit_nohz()         do { } while (0)
+
 #endif /* __KERNEL__ */
 #endif /* __LINUX_RCUCLASSIC_H */
diff --git a/include/linux/rcupreempt.h b/include/linux/rcupreempt.h
index 60c2a033b19e..01152ed532c8 100644
--- a/include/linux/rcupreempt.h
+++ b/include/linux/rcupreempt.h
@@ -82,5 +82,27 @@ extern struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu);
 
 struct softirq_action;
 
+#ifdef CONFIG_NO_HZ
+DECLARE_PER_CPU(long, dynticks_progress_counter);
+
+static inline void rcu_enter_nohz(void)
+{
+        __get_cpu_var(dynticks_progress_counter)++;
+        WARN_ON(__get_cpu_var(dynticks_progress_counter) & 0x1);
+        mb();
+}
+
+static inline void rcu_exit_nohz(void)
+{
+        mb();
+        __get_cpu_var(dynticks_progress_counter)++;
+        WARN_ON(!(__get_cpu_var(dynticks_progress_counter) & 0x1));
+}
+
+#else /* CONFIG_NO_HZ */
+#define rcu_enter_nohz()        do { } while (0)
+#define rcu_exit_nohz()         do { } while (0)
+#endif /* CONFIG_NO_HZ */
+
 #endif /* __KERNEL__ */
 #endif /* __LINUX_RCUPREEMPT_H */
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
index 987cfb7ade89..c7c52096df48 100644
--- a/kernel/rcupreempt.c
+++ b/kernel/rcupreempt.c
@@ -23,6 +23,10 @@
  *              to Suparna Bhattacharya for pushing me completely away
  *              from atomic instructions on the read side.
  *
+ *  - Added handling of Dynamic Ticks
+ *      Copyright 2007 - Paul E. Mckenney <paulmck@us.ibm.com>
+ *                     - Steven Rostedt <srostedt@redhat.com>
+ *
  * Papers:  http://www.rdrop.com/users/paulmck/RCU
  *
  * Design Document: http://lwn.net/Articles/253651/
@@ -409,6 +413,212 @@ static void __rcu_advance_callbacks(struct rcu_data *rdp)
         }
 }
 
+#ifdef CONFIG_NO_HZ
+
+DEFINE_PER_CPU(long, dynticks_progress_counter) = 1;
+static DEFINE_PER_CPU(long, rcu_dyntick_snapshot);
+static DEFINE_PER_CPU(int, rcu_update_flag);
+
+/**
+ * rcu_irq_enter - Called from Hard irq handlers and NMI/SMI.
+ *
+ * If the CPU was idle with dynamic ticks active, this updates the
+ * dynticks_progress_counter to let the RCU handling know that the
+ * CPU is active.
+ */
+void rcu_irq_enter(void)
+{
+        int cpu = smp_processor_id();
+
+        if (per_cpu(rcu_update_flag, cpu))
+                per_cpu(rcu_update_flag, cpu)++;
+
+        /*
+         * Only update if we are coming from a stopped ticks mode
+         * (dynticks_progress_counter is even).
+         */
+        if (!in_interrupt() &&
+            (per_cpu(dynticks_progress_counter, cpu) & 0x1) == 0) {
+                /*
+                 * The following might seem like we could have a race
+                 * with NMI/SMIs. But this really isn't a problem.
+                 * Here we do a read/modify/write, and the race happens
+                 * when an NMI/SMI comes in after the read and before
+                 * the write. But NMI/SMIs will increment this counter
+                 * twice before returning, so the zero bit will not
+                 * be corrupted by the NMI/SMI which is the most important
+                 * part.
+                 *
+                 * The only thing is that we would bring back the counter
+                 * to a postion that it was in during the NMI/SMI.
+                 * But the zero bit would be set, so the rest of the
+                 * counter would again be ignored.
+                 *
+                 * On return from the IRQ, the counter may have the zero
+                 * bit be 0 and the counter the same as the return from
+                 * the NMI/SMI. If the state machine was so unlucky to
+                 * see that, it still doesn't matter, since all
+                 * RCU read-side critical sections on this CPU would
+                 * have already completed.
+                 */
+                per_cpu(dynticks_progress_counter, cpu)++;
+                /*
+                 * The following memory barrier ensures that any
+                 * rcu_read_lock() primitives in the irq handler
+                 * are seen by other CPUs to follow the above
+                 * increment to dynticks_progress_counter. This is
+                 * required in order for other CPUs to correctly
+                 * determine when it is safe to advance the RCU
+                 * grace-period state machine.
+                 */
+                smp_mb(); /* see above block comment. */
+                /*
+                 * Since we can't determine the dynamic tick mode from
+                 * the dynticks_progress_counter after this routine,
+                 * we use a second flag to acknowledge that we came
+                 * from an idle state with ticks stopped.
+                 */
+                per_cpu(rcu_update_flag, cpu)++;
+                /*
+                 * If we take an NMI/SMI now, they will also increment
+                 * the rcu_update_flag, and will not update the
+                 * dynticks_progress_counter on exit. That is for
+                 * this IRQ to do.
+                 */
+        }
+}
+
+/**
+ * rcu_irq_exit - Called from exiting Hard irq context.
+ *
+ * If the CPU was idle with dynamic ticks active, update the
+ * dynticks_progress_counter to put let the RCU handling be
+ * aware that the CPU is going back to idle with no ticks.
+ */