[PATCH] tick-management: dyntick / highres functionality

With Ingo Molnar <mingo@elte.hu>

Add functions to provide dynamic ticks and high resolution timers.  The code
which keeps track of jiffies and handles the long idle periods is shared
between tick based and high resolution timer based dynticks.  The dyntick
functionality can be disabled on the kernel commandline.  Provide also the
infrastructure to support high resolution timers.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

8 files changed, 930 insertions, 3 deletions

diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
new file mode 100644
index 000000000000..9ec54eb3667f
--- /dev/null
+++ b/kernel/time/Kconfig
@@ -0,0 +1,15 @@
+#
+# Timer subsystem related configuration options
+#
+config TICK_ONESHOT
+        bool
+        default n
+
+config NO_HZ
+        bool "Tickless System (Dynamic Ticks)"
+        depends on GENERIC_TIME && GENERIC_CLOCKEVENTS
+        select TICK_ONESHOT
+        help
+          This option enables a tickless system: timer interrupts will
+          only trigger on an as-needed basis both when the system is
+          busy and when the system is idle.
diff --git a/kernel/time/Makefile b/kernel/time/Makefile
index a941743c3ff8..f246bc836b9a 100644
--- a/kernel/time/Makefile
+++ b/kernel/time/Makefile
@@ -3,3 +3,5 @@ obj-y += ntp.o clocksource.o jiffies.o
 obj-$(CONFIG_GENERIC_CLOCKEVENTS)               += clockevents.o
 obj-$(CONFIG_GENERIC_CLOCKEVENTS)               += tick-common.o
 obj-$(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST)     += tick-broadcast.o
+obj-$(CONFIG_TICK_ONESHOT)                      += tick-oneshot.o
+obj-$(CONFIG_TICK_ONESHOT)                      += tick-sched.o
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 3cb8ac978270..193a0793af95 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -29,6 +29,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
+#include <linux/tick.h>
 
 /* XXX - Would like a better way for initializing curr_clocksource */
 extern struct clocksource clocksource_jiffies;
@@ -109,6 +110,13 @@ static void clocksource_watchdog(unsigned long data)
                         if ((cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&
                             (watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {
                                 cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
+                                /*
+                                 * We just marked the clocksource as
+                                 * highres-capable, notify the rest of the
+                                 * system as well so that we transition
+                                 * into high-res mode:
+                                 */
+                                tick_clock_notify();
                         }
                         cs->flags |= CLOCK_SOURCE_WATCHDOG;
                         cs->wd_last = csnow;
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 0ee4968ff791..8314ecb32d33 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -29,7 +29,7 @@
 
 struct tick_device tick_broadcast_device;
 static cpumask_t tick_broadcast_mask;
-DEFINE_SPINLOCK(tick_broadcast_lock);
+static DEFINE_SPINLOCK(tick_broadcast_lock);
 
 /*
  * Start the device in periodic mode
@@ -215,6 +215,8 @@ static void tick_do_broadcast_on_off(void *why)
         else {
                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
                         tick_broadcast_start_periodic(bc);
+                else
+                        tick_broadcast_setup_oneshot(bc);
         }
 out:
         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
@@ -268,3 +270,190 @@ void tick_shutdown_broadcast(unsigned int *cpup)
 
         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 }
+
+#ifdef CONFIG_TICK_ONESHOT
+
+static cpumask_t tick_broadcast_oneshot_mask;
+
+static int tick_broadcast_set_event(ktime_t expires, int force)
+{
+        struct clock_event_device *bc = tick_broadcast_device.evtdev;
+        ktime_t now = ktime_get();
+        int res;
+
+        for(;;) {
+                res = clockevents_program_event(bc, expires, now);
+                if (!res || !force)
+                        return res;
+                now = ktime_get();
+                expires = ktime_add(now, ktime_set(0, bc->min_delta_ns));
+        }
+}
+
+/*
+ * Reprogram the broadcast device:
+ *
+ * Called with tick_broadcast_lock held and interrupts disabled.
+ */
+static int tick_broadcast_reprogram(void)
+{
+        ktime_t expires = { .tv64 = KTIME_MAX };
+        struct tick_device *td;
+        int cpu;
+
+        /*
+         * Find the event which expires next:
+         */
+        for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
+             cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
+                td = &per_cpu(tick_cpu_device, cpu);
+                if (td->evtdev->next_event.tv64 < expires.tv64)
+                        expires = td->evtdev->next_event;
+        }
+
+        if (expires.tv64 == KTIME_MAX)
+                return 0;
+
+        return tick_broadcast_set_event(expires, 0);
+}
+
+/*
+ * Handle oneshot mode broadcasting
+ */
+static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
+{
+        struct tick_device *td;
+        cpumask_t mask;
+        ktime_t now;
+        int cpu;
+
+        spin_lock(&tick_broadcast_lock);
+again:
+        dev->next_event.tv64 = KTIME_MAX;
+        mask = CPU_MASK_NONE;
+        now = ktime_get();
+        /* Find all expired events */
+        for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
+             cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
+                td = &per_cpu(tick_cpu_device, cpu);
+                if (td->evtdev->next_event.tv64 <= now.tv64)
+                        cpu_set(cpu, mask);
+        }
+
+        /*
+         * Wakeup the cpus which have an expired event. The broadcast
+         * device is reprogrammed in the return from idle code.
+         */
+        if (!tick_do_broadcast(mask)) {
+                /*
+                 * The global event did not expire any CPU local
+                 * events. This happens in dyntick mode, as the
+                 * maximum PIT delta is quite small.
+                 */
+                if (tick_broadcast_reprogram())
+                        goto again;
+        }
+        spin_unlock(&tick_broadcast_lock);
+}
+
+/*
+ * Powerstate information: The system enters/leaves a state, where
+ * affected devices might stop
+ */
+void tick_broadcast_oneshot_control(unsigned long reason)
+{
+        struct clock_event_device *bc, *dev;
+        struct tick_device *td;
+        unsigned long flags;
+        int cpu;