x86: idle wakeup event in the HLT loop

do a proper idle-wakeup event on HLT as well - some CPUs stop the TSC
in HLT too, not just when going through the ACPI methods.

(the ACPI idle code already does this.)

[ update the 64-bit side too, as noticed by Jiri Slaby. ]

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

2 files changed, 22 insertions, 6 deletions

diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 46d391d49de8..a63d2d2556ee 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -113,10 +113,19 @@ void default_idle(void)
                 smp_mb();
 
                 local_irq_disable();
-                if (!need_resched())
+                if (!need_resched()) {
+                        ktime_t t0, t1;
+                        u64 t0n, t1n;
+
+                        t0 = ktime_get();
+                        t0n = ktime_to_ns(t0);
                         safe_halt();    /* enables interrupts racelessly */
-                else
-                        local_irq_enable();
+                        local_irq_disable();
+                        t1 = ktime_get();
+                        t1n = ktime_to_ns(t1);
+                        sched_clock_idle_wakeup_event(t1n - t0n);
+                }
+                local_irq_enable();
                 current_thread_info()->status |= TS_POLLING;
         } else {
                 /* loop is done by the caller */
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index c2db7ef93565..40fed477f3e5 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -116,9 +116,16 @@ static void default_idle(void)
         smp_mb();
         local_irq_disable();
         if (!need_resched()) {
-                /* Enables interrupts one instruction before HLT.
-                   x86 special cases this so there is no race. */
-                safe_halt();
+                ktime_t t0, t1;
+                u64 t0n, t1n;
+
+                t0 = ktime_get();
+                t0n = ktime_to_ns(t0);
+                safe_halt();    /* enables interrupts racelessly */
+                local_irq_disable();
+                t1 = ktime_get();
+                t1n = ktime_to_ns(t1);
+                sched_clock_idle_wakeup_event(t1n - t0n);
         } else
                 local_irq_enable();
         current_thread_info()->status |= TS_POLLING;