sched: sched_clock_idle_[sleep|wakeup]_event()

construct a more or less wall-clock time out of sched_clock(), by
using ACPI-idle's existing knowledge about how much time we spent
idling. This allows the rq clock to work around TSC-stops-in-C2,
TSC-gets-corrupted-in-C3 type of problems.

( Besides the scheduler's statistics this also benefits blktrace and
  printk-timestamps as well. )

Furthermore, the precise before-C2/C3-sleep and after-C2/C3-wakeup
callbacks allow the scheduler to get out the most of the period where
the CPU has a reliable TSC. This results in slightly more precise
task statistics.

the ACPI bits were acked by Len.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Acked-by: Len Brown <len.brown@intel.com>

2 files changed, 34 insertions, 10 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index 45e17b83b7f1..48e7586168ef 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -262,7 +262,8 @@ struct rq {
         s64 clock_max_delta;
 
         unsigned int clock_warps, clock_overflows;
-        unsigned int clock_unstable_events;
+        u64 idle_clock;
+        unsigned int clock_deep_idle_events;
         u64 tick_timestamp;
 
         atomic_t nr_iowait;
@@ -556,18 +557,40 @@ static inline struct rq *this_rq_lock(void)
 }
 
 /*
- * CPU frequency is/was unstable - start new by setting prev_clock_raw:
+ * We are going deep-idle (irqs are disabled):
  */
-void sched_clock_unstable_event(void)
+void sched_clock_idle_sleep_event(void)
 {
-        unsigned long flags;
-        struct rq *rq;
+        struct rq *rq = cpu_rq(smp_processor_id());
 
-        rq = task_rq_lock(current, &flags);
-        rq->prev_clock_raw = sched_clock();
-        rq->clock_unstable_events++;
-        task_rq_unlock(rq, &flags);
+        spin_lock(&rq->lock);
+        __update_rq_clock(rq);
+        spin_unlock(&rq->lock);
+        rq->clock_deep_idle_events++;
+}
+EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event);
+
+/*
+ * We just idled delta nanoseconds (called with irqs disabled):
+ */
+void sched_clock_idle_wakeup_event(u64 delta_ns)
+{
+        struct rq *rq = cpu_rq(smp_processor_id());
+        u64 now = sched_clock();
+
+        rq->idle_clock += delta_ns;
+        /*
+         * Override the previous timestamp and ignore all
+         * sched_clock() deltas that occured while we idled,
+         * and use the PM-provided delta_ns to advance the
+         * rq clock:
+         */
+        spin_lock(&rq->lock);
+        rq->prev_clock_raw = now;
+        rq->clock += delta_ns;
+        spin_unlock(&rq->lock);
 }
+EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
 
 /*
  * resched_task - mark a task 'to be rescheduled now'.
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 87e524762b85..ab18f45f2ab2 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -154,10 +154,11 @@ static void print_cpu(struct seq_file *m, int cpu)
         P(next_balance);
         P(curr->pid);
         P(clock);
+        P(idle_clock);
         P(prev_clock_raw);
         P(clock_warps);
         P(clock_overflows);
-        P(clock_unstable_events);
+        P(clock_deep_idle_events);
         P(clock_max_delta);
         P(cpu_load[0]);
         P(cpu_load[1]);