4 files changed, 59 insertions, 17 deletions

diff --git a/include/linux/tick.h b/include/linux/tick.h
index ef0717e5e526..e8e7ff16b929 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -116,6 +116,7 @@ extern bool tick_nohz_enabled;
 extern bool tick_nohz_tick_stopped(void);
 extern bool tick_nohz_tick_stopped_cpu(int cpu);
 extern void tick_nohz_idle_stop_tick(void);
+extern void tick_nohz_idle_retain_tick(void);
 extern void tick_nohz_idle_restart_tick(void);
 extern void tick_nohz_idle_enter(void);
 extern void tick_nohz_idle_exit(void);
@@ -139,6 +140,7 @@ static inline void tick_nohz_idle_stop_tick_protected(void)
 static inline int tick_nohz_tick_stopped(void) { return 0; }
 static inline int tick_nohz_tick_stopped_cpu(int cpu) { return 0; }
 static inline void tick_nohz_idle_stop_tick(void) { }
+static inline void tick_nohz_idle_retain_tick(void) { }
 static inline void tick_nohz_idle_restart_tick(void) { }
 static inline void tick_nohz_idle_enter(void) { }
 static inline void tick_nohz_idle_exit(void) { }
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index a966bd2a6fa0..1a3e9bddd17b 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -185,13 +185,18 @@ static void cpuidle_idle_call(void)
         } else {
                 bool stop_tick = true;
 
-                tick_nohz_idle_stop_tick();
-                rcu_idle_enter();
-
                 /*
                  * Ask the cpuidle framework to choose a convenient idle state.
                  */
                 next_state = cpuidle_select(drv, dev, &stop_tick);
+
+                if (stop_tick)
+                        tick_nohz_idle_stop_tick();
+                else
+                        tick_nohz_idle_retain_tick();
+
+                rcu_idle_enter();
+
                 entered_state = call_cpuidle(drv, dev, next_state);
                 /*
                  * Give the governor an opportunity to reflect on the outcome
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index f56d2c695712..c57c98c7e953 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -913,16 +913,19 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
 
 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
 {
-        struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
         ktime_t expires;
         int cpu = smp_processor_id();
 
-        WARN_ON_ONCE(ts->timer_expires_base);
-
-        if (!can_stop_idle_tick(cpu, ts))
-                goto out;
-
-        expires = tick_nohz_next_event(ts, cpu);
+        /*
+         * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
+         * tick timer expiration time is known already.
+         */
+        if (ts->timer_expires_base)
+                expires = ts->timer_expires;
+        else if (can_stop_idle_tick(cpu, ts))
+                expires = tick_nohz_next_event(ts, cpu);
+        else
+                return;
 
         ts->idle_calls++;
 
@@ -941,9 +944,6 @@ static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
         } else {
                 tick_nohz_retain_tick(ts);
         }
-
-out:
-        ts->sleep_length = ktime_sub(dev->next_event, ts->idle_entrytime);
 }
 
 /**
@@ -956,6 +956,16 @@ void tick_nohz_idle_stop_tick(void)
         __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
 }
 
+void tick_nohz_idle_retain_tick(void)
+{
+        tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
+        /*
+         * Undo the effect of get_next_timer_interrupt() called from
+         * tick_nohz_next_event().
+         */
+        timer_clear_idle();
+}
+
 /**
  * tick_nohz_idle_enter - prepare for entering idle on the current CPU
  *
@@ -1012,15 +1022,42 @@ bool tick_nohz_idle_got_tick(void)
 }
 
 /**
- * tick_nohz_get_sleep_length - return the length of the current sleep
+ * tick_nohz_get_sleep_length - return the expected length of the current sleep
  *
  * Called from power state control code with interrupts disabled
  */
 ktime_t tick_nohz_get_sleep_length(void)
 {
+        struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
+        int cpu = smp_processor_id();
+        /*
+         * The idle entry time is expected to be a sufficient approximation of
+         * the current time at this point.
+         */
+        ktime_t now = ts->idle_entrytime;
+        ktime_t next_event;
+
+        WARN_ON_ONCE(!ts->inidle);
+
+        if (!can_stop_idle_tick(cpu, ts))
+                goto out_dev;
+
+        next_event = tick_nohz_next_event(ts, cpu);
+        if (!next_event)
+                goto out_dev;
+
+        /*
+         * If the next highres timer to expire is earlier than next_event, the
+         * idle governor needs to know that.
+         */
+        next_event = min_t(u64, next_event,
+                           hrtimer_next_event_without(&ts->sched_timer));
+
+        return ktime_sub(next_event, now);
 
-        return ts->sleep_length;
+out_dev:
+        return ktime_sub(dev->next_event, now);
 }
 
 /**
diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h
index 53e45a39bdbc..2b845f2c44b1 100644
--- a/kernel/time/tick-sched.h
+++ b/kernel/time/tick-sched.h
@@ -38,7 +38,6 @@ enum tick_nohz_mode {
  * @idle_exittime:      Time when the idle state was left
  * @idle_sleeptime:     Sum of the time slept in idle with sched tick stopped
  * @iowait_sleeptime:   Sum of the time slept in idle with sched tick stopped, with IO outstanding
- * @sleep_length:       Duration of the current idle sleep
  * @timer_expires:      Anticipated timer expiration time (in case sched tick is stopped)
  * @timer_expires_base: Base time clock monotonic for @timer_expires
  * @do_timer_lst:       CPU was the last one doing do_timer before going idle
@@ -60,7 +59,6 @@ struct tick_sched {
         ktime_t                         idle_exittime;
         ktime_t                         idle_sleeptime;
         ktime_t                         iowait_sleeptime;
-        ktime_t                         sleep_length;
         unsigned long                   last_jiffies;
         u64                             timer_expires;
         u64                             timer_expires_base;