2 files changed, 82 insertions, 46 deletions

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 69fe113cfc7f..f56d2c695712 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -641,13 +641,10 @@ static inline bool local_timer_softirq_pending(void)
         return local_softirq_pending() & TIMER_SOFTIRQ;
 }
 
-static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
-                                         ktime_t now, int cpu)
+static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 {
-        struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
         u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
         unsigned long seq, basejiff;
-        ktime_t tick;
 
         /* Read jiffies and the time when jiffies were updated last */
         do {
@@ -656,6 +653,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
                 basejiff = jiffies;
         } while (read_seqretry(&jiffies_lock, seq));
         ts->last_jiffies = basejiff;
+        ts->timer_expires_base = basemono;
 
         /*
          * Keep the periodic tick, when RCU, architecture or irq_work
@@ -700,47 +698,63 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
                  * next period, so no point in stopping it either, bail.
                  */
                 if (!ts->tick_stopped) {
-                        tick = 0;
+                        ts->timer_expires = 0;
                         goto out;
                 }
         }
 
         /*
+         * If this CPU is the one which had the do_timer() duty last, we limit
+         * the sleep time to the timekeeping max_deferment value.
+         * Otherwise we can sleep as long as we want.
+         */
+        delta = timekeeping_max_deferment();
+        if (cpu != tick_do_timer_cpu &&
+            (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))
+                delta = KTIME_MAX;
+
+        /* Calculate the next expiry time */
+        if (delta < (KTIME_MAX - basemono))
+                expires = basemono + delta;
+        else
+                expires = KTIME_MAX;
+
+        ts->timer_expires = min_t(u64, expires, next_tick);
+
+out:
+        return ts->timer_expires;
+}
+
+static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
+{
+        struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
+        u64 basemono = ts->timer_expires_base;
+        u64 expires = ts->timer_expires;
+        ktime_t tick = expires;
+
+        /* Make sure we won't be trying to stop it twice in a row. */
+        ts->timer_expires_base = 0;
+
+        /*
          * If this CPU is the one which updates jiffies, then give up
          * the assignment and let it be taken by the CPU which runs
          * the tick timer next, which might be this CPU as well. If we
          * don't drop this here the jiffies might be stale and
          * do_timer() never invoked. Keep track of the fact that it
-         * was the one which had the do_timer() duty last. If this CPU
-         * is the one which had the do_timer() duty last, we limit the
-         * sleep time to the timekeeping max_deferment value.
-         * Otherwise we can sleep as long as we want.
+         * was the one which had the do_timer() duty last.
          */
-        delta = timekeeping_max_deferment();
         if (cpu == tick_do_timer_cpu) {
                 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
                 ts->do_timer_last = 1;
         } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
-                delta = KTIME_MAX;
                 ts->do_timer_last = 0;
-        } else if (!ts->do_timer_last) {
-                delta = KTIME_MAX;
         }
 
-        /* Calculate the next expiry time */
-        if (delta < (KTIME_MAX - basemono))
-                expires = basemono + delta;
-        else
-                expires = KTIME_MAX;
-
-        expires = min_t(u64, expires, next_tick);
-        tick = expires;
-
         /* Skip reprogram of event if its not changed */
         if (ts->tick_stopped && (expires == ts->next_tick)) {
                 /* Sanity check: make sure clockevent is actually programmed */
                 if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
-                        goto out;
+                        return;
 
                 WARN_ON_ONCE(1);
                 printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
@@ -774,7 +788,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
         if (unlikely(expires == KTIME_MAX)) {
                 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
                         hrtimer_cancel(&ts->sched_timer);
-                goto out;
+                return;
         }
 
         hrtimer_set_expires(&ts->sched_timer, tick);
@@ -783,15 +797,23 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
                 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
         else
                 tick_program_event(tick, 1);
-out:
-        /*
-         * Update the estimated sleep length until the next timer
-         * (not only the tick).
-         */
-        ts->sleep_length = ktime_sub(dev->next_event, now);
-        return tick;
 }
 
+static void tick_nohz_retain_tick(struct tick_sched *ts)
+{
+        ts->timer_expires_base = 0;
+}
+
+#ifdef CONFIG_NO_HZ_FULL
+static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)
+{
+        if (tick_nohz_next_event(ts, cpu))
+                tick_nohz_stop_tick(ts, cpu);
+        else
+                tick_nohz_retain_tick(ts);
+}
+#endif /* CONFIG_NO_HZ_FULL */
+
 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
 {
         /* Update jiffies first */
@@ -827,7 +849,7 @@ static void tick_nohz_full_update_tick(struct tick_sched *ts)
                 return;
 
         if (can_stop_full_tick(cpu, ts))
-                tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);
+                tick_nohz_stop_sched_tick(ts, cpu);
         else if (ts->tick_stopped)
                 tick_nohz_restart_sched_tick(ts, ktime_get());
 #endif
@@ -853,10 +875,8 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
                 return false;
         }
 
-        if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
-                ts->sleep_length = NSEC_PER_SEC / HZ;
+        if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
                 return false;
-        }
 
         if (need_resched())
                 return false;
@@ -893,29 +913,37 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
 
 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
 {
+        struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
         ktime_t expires;
         int cpu = smp_processor_id();
 
-        if (can_stop_idle_tick(cpu, ts)) {
+        WARN_ON_ONCE(ts->timer_expires_base);
+
+        if (!can_stop_idle_tick(cpu, ts))
+                goto out;
+
+        expires = tick_nohz_next_event(ts, cpu);
+
+        ts->idle_calls++;
+
+        if (expires > 0LL) {
                 int was_stopped = ts->tick_stopped;
 
-                ts->idle_calls++;
+                tick_nohz_stop_tick(ts, cpu);
 
-                /*
-                 * The idle entry time should be a sufficient approximation of
-                 * the current time at this point.
-                 */
-                expires = tick_nohz_stop_sched_tick(ts, ts->idle_entrytime, cpu);
-                if (expires > 0LL) {
-                        ts->idle_sleeps++;
-                        ts->idle_expires = expires;
-                }
+                ts->idle_sleeps++;
+                ts->idle_expires = expires;
 
                 if (!was_stopped && ts->tick_stopped) {
                         ts->idle_jiffies = ts->last_jiffies;
                         nohz_balance_enter_idle(cpu);
                 }
+        } else {
+                tick_nohz_retain_tick(ts);
         }
+
+out:
+        ts->sleep_length = ktime_sub(dev->next_event, ts->idle_entrytime);
 }
 
 /**
@@ -942,6 +970,9 @@ void tick_nohz_idle_enter(void)
         local_irq_disable();
 
         ts = this_cpu_ptr(&tick_cpu_sched);
+
+        WARN_ON_ONCE(ts->timer_expires_base);
+
         ts->inidle = 1;
         tick_nohz_start_idle(ts);
 
@@ -1067,6 +1098,7 @@ void tick_nohz_idle_exit(void)
         local_irq_disable();
 
         WARN_ON_ONCE(!ts->inidle);
+        WARN_ON_ONCE(ts->timer_expires_base);
 
         ts->inidle = 0;
 
diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h
index 954b43dbf21c..53e45a39bdbc 100644
--- a/kernel/time/tick-sched.h
+++ b/kernel/time/tick-sched.h
@@ -39,6 +39,8 @@ enum tick_nohz_mode {
  * @idle_sleeptime:     Sum of the time slept in idle with sched tick stopped
  * @iowait_sleeptime:   Sum of the time slept in idle with sched tick stopped, with IO outstanding
  * @sleep_length:       Duration of the current idle sleep
+ * @timer_expires:      Anticipated timer expiration time (in case sched tick is stopped)
+ * @timer_expires_base: Base time clock monotonic for @timer_expires
  * @do_timer_lst:       CPU was the last one doing do_timer before going idle
  */
 struct tick_sched {
@@ -60,6 +62,8 @@ struct tick_sched {
         ktime_t                         iowait_sleeptime;
         ktime_t                         sleep_length;
         unsigned long                   last_jiffies;
+        u64                             timer_expires;
+        u64                             timer_expires_base;
         u64                             next_timer;
         ktime_t                         idle_expires;
         int                             do_timer_last;