[PATCH] tick-management: dyntick / highres functionality

With Ingo Molnar <mingo@elte.hu>

Add functions to provide dynamic ticks and high resolution timers.  The code
which keeps track of jiffies and handles the long idle periods is shared
between tick based and high resolution timer based dynticks.  The dyntick
functionality can be disabled on the kernel commandline.  Provide also the
infrastructure to support high resolution timers.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

11 files changed, 959 insertions, 14 deletions

diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index a2310d1bebe1..e04ef38ea3be 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -2,8 +2,8 @@
  *  linux/kernel/hrtimer.c
  *
  *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
- *  Copyright(C) 2005-2006, Red Hat, Inc., Ingo Molnar
- *  Copyright(C) 2006       Timesys Corp., Thomas Gleixner <tglx@timesys.com>
+ *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
+ *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  *
  *  High-resolution kernel timers
  *
@@ -38,6 +38,7 @@
 #include <linux/notifier.h>
 #include <linux/syscalls.h>
 #include <linux/interrupt.h>
+#include <linux/tick.h>
 
 #include <asm/uaccess.h>
 
@@ -288,7 +289,7 @@ ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
 /*
  * Divide a ktime value by a nanosecond value
  */
-static unsigned long ktime_divns(const ktime_t kt, s64 div)
+unsigned long ktime_divns(const ktime_t kt, s64 div)
 {
         u64 dclc, inc, dns;
         int sft = 0;
@@ -305,9 +306,6 @@ static unsigned long ktime_divns(const ktime_t kt, s64 div)
 
         return (unsigned long) dclc;
 }
-
-#else /* BITS_PER_LONG < 64 */
-# define ktime_divns(kt, div)           (unsigned long)((kt).tv64 / (div))
 #endif /* BITS_PER_LONG >= 64 */
 
 /*
@@ -682,6 +680,16 @@ void hrtimer_run_queues(void)
         struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
         int i;
 
+        /*
+         * This _is_ ugly: We have to check in the softirq context,
+         * whether we can switch to highres and / or nohz mode. The
+         * clocksource switch happens in the timer interrupt with
+         * xtime_lock held. Notification from there only sets the
+         * check bit in the tick_oneshot code, otherwise we might
+         * deadlock vs. xtime_lock.
+         */
+        tick_check_oneshot_change(1);
+
         hrtimer_get_softirq_time(cpu_base);
 
         for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 14e1a14f94d2..8b75008e2bd8 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -17,6 +17,7 @@
 #include <linux/kthread.h>
 #include <linux/rcupdate.h>
 #include <linux/smp.h>
+#include <linux/tick.h>
 
 #include <asm/irq.h>
 /*
@@ -278,9 +279,11 @@ EXPORT_SYMBOL(do_softirq);
  */
 void irq_enter(void)
 {
-        account_system_vtime(current);
-        add_preempt_count(HARDIRQ_OFFSET);
-        trace_hardirq_enter();
+        __irq_enter();
+#ifdef CONFIG_NO_HZ
+        if (idle_cpu(smp_processor_id()))
+                tick_nohz_update_jiffies();
+#endif
 }
 
 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
@@ -299,6 +302,12 @@ void irq_exit(void)
         sub_preempt_count(IRQ_EXIT_OFFSET);
         if (!in_interrupt() && local_softirq_pending())
                 invoke_softirq();
+
+#ifdef CONFIG_NO_HZ
+        /* Make sure that timer wheel updates are propagated */
+        if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
+                tick_nohz_stop_sched_tick();
+#endif
         preempt_enable_no_resched();
 }
 
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
new file mode 100644
index 000000000000..9ec54eb3667f
--- /dev/null
+++ b/kernel/time/Kconfig
@@ -0,0 +1,15 @@
+#
+# Timer subsystem related configuration options
+#
+config TICK_ONESHOT
+        bool
+        default n
+
+config NO_HZ
+        bool "Tickless System (Dynamic Ticks)"
+        depends on GENERIC_TIME && GENERIC_CLOCKEVENTS
+        select TICK_ONESHOT
+        help
+          This option enables a tickless system: timer interrupts will
+          only trigger on an as-needed basis both when the system is
+          busy and when the system is idle.
diff --git a/kernel/time/Makefile b/kernel/time/Makefile
index a941743c3ff8..f246bc836b9a 100644
--- a/kernel/time/Makefile
+++ b/kernel/time/Makefile
@@ -3,3 +3,5 @@ obj-y += ntp.o clocksource.o jiffies.o
 obj-$(CONFIG_GENERIC_CLOCKEVENTS)               += clockevents.o
 obj-$(CONFIG_GENERIC_CLOCKEVENTS)               += tick-common.o
 obj-$(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST)     += tick-broadcast.o
+obj-$(CONFIG_TICK_ONESHOT)                      += tick-oneshot.o
+obj-$(CONFIG_TICK_ONESHOT)                      += tick-sched.o
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 3cb8ac978270..193a0793af95 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -29,6 +29,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
+#include <linux/tick.h>
 
 /* XXX - Would like a better way for initializing curr_clocksource */
 extern struct clocksource clocksource_jiffies;
@@ -109,6 +110,13 @@ static void clocksource_watchdog(unsigned long data)
                         if ((cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&
                             (watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {
                                 cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
+                                /*
+                                 * We just marked the clocksource as
+                                 * highres-capable, notify the rest of the
+                                 * system as well so that we transition
+                                 * into high-res mode:
+                                 */
+                                tick_clock_notify();
                         }
                         cs->flags |= CLOCK_SOURCE_WATCHDOG;
                         cs->wd_last = csnow;
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 0ee4968ff791..8314ecb32d33 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -29,7 +29,7 @@
 
 struct tick_device tick_broadcast_device;
 static cpumask_t tick_broadcast_mask;
-DEFINE_SPINLOCK(tick_broadcast_lock);
+static DEFINE_SPINLOCK(tick_broadcast_lock);
 
 /*
  * Start the device in periodic mode
@@ -215,6 +215,8 @@ static void tick_do_broadcast_on_off(void *why)
         else {