1 files changed, 109 insertions, 85 deletions

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 4a08472c3ca7..024540f97f74 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -105,7 +105,7 @@ static ktime_t tick_init_jiffy_update(void)
 /*
  * NO HZ enabled ?
  */
-static int tick_nohz_enabled __read_mostly  = 1;
+int tick_nohz_enabled __read_mostly  = 1;
 
 /*
  * Enable / Disable tickless mode
@@ -271,50 +271,15 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 }
 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 
-static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
+static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
+                                         ktime_t now, int cpu)
 {
         unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
+        ktime_t last_update, expires, ret = { .tv64 = 0 };
         unsigned long rcu_delta_jiffies;
-        ktime_t last_update, expires, now;
         struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
         u64 time_delta;
-        int cpu;
-
-        cpu = smp_processor_id();
-        ts = &per_cpu(tick_cpu_sched, cpu);
-
-        now = tick_nohz_start_idle(cpu, ts);
-
-        /*
-         * If this cpu is offline and it is the one which updates
-         * jiffies, then give up the assignment and let it be taken by
-         * the cpu which runs the tick timer next. If we don't drop
-         * this here the jiffies might be stale and do_timer() never
-         * invoked.
-         */
-        if (unlikely(!cpu_online(cpu))) {
-                if (cpu == tick_do_timer_cpu)
-                        tick_do_timer_cpu = TICK_DO_TIMER_NONE;
-        }
-
-        if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
-                return;
 
-        if (need_resched())
-                return;
-
-        if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
-                static int ratelimit;
-
-                if (ratelimit < 10) {
-                        printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
-                               (unsigned int) local_softirq_pending());
-                        ratelimit++;
-                }
-                return;
-        }
-
-        ts->idle_calls++;
         /* Read jiffies and the time when jiffies were updated last */
         do {
                 seq = read_seqbegin(&xtime_lock);
@@ -397,6 +362,8 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
                 if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
                         goto out;
 
+                ret = expires;
+
                 /*
                  * nohz_stop_sched_tick can be called several times before
                  * the nohz_restart_sched_tick is called. This happens when
@@ -408,16 +375,10 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
                         select_nohz_load_balancer(1);
                         calc_load_enter_idle();
 
-                        ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
+                        ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
                         ts->tick_stopped = 1;
-                        ts->idle_jiffies = last_jiffies;
                 }
 
-                ts->idle_sleeps++;
-
-                /* Mark expires */
-                ts->idle_expires = expires;
-
                 /*
                  * If the expiration time == KTIME_MAX, then
                  * in this case we simply stop the tick timer.
@@ -448,6 +409,65 @@ out:
         ts->next_jiffies = next_jiffies;
         ts->last_jiffies = last_jiffies;
         ts->sleep_length = ktime_sub(dev->next_event, now);
+
+        return ret;
+}
+
+static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
+{
+        /*
+         * If this cpu is offline and it is the one which updates
+         * jiffies, then give up the assignment and let it be taken by
+         * the cpu which runs the tick timer next. If we don't drop
+         * this here the jiffies might be stale and do_timer() never
+         * invoked.
+         */
+        if (unlikely(!cpu_online(cpu))) {
+                if (cpu == tick_do_timer_cpu)
+                        tick_do_timer_cpu = TICK_DO_TIMER_NONE;
+        }
+
+        if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
+                return false;
+
+        if (need_resched())
+                return false;
+
+        if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
+                static int ratelimit;
+
+                if (ratelimit < 10) {
+                        printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
+                               (unsigned int) local_softirq_pending());
+                        ratelimit++;
+                }
+                return false;
+        }
+
+        return true;
+}
+
+static void __tick_nohz_idle_enter(struct tick_sched *ts)
+{
+        ktime_t now, expires;
+        int cpu = smp_processor_id();
+
+        now = tick_nohz_start_idle(cpu, ts);
+
+        if (can_stop_idle_tick(cpu, ts)) {
+                int was_stopped = ts->tick_stopped;
+
+                ts->idle_calls++;
+
+                expires = tick_nohz_stop_sched_tick(ts, now, cpu);
+                if (expires.tv64 > 0LL) {
+                        ts->idle_sleeps++;
+                        ts->idle_expires = expires;
+                }
+
+                if (!was_stopped && ts->tick_stopped)
+                        ts->idle_jiffies = ts->last_jiffies;
+        }
 }
 
 /**
@@ -485,7 +505,7 @@ void tick_nohz_idle_enter(void)
          * update of the idle time accounting in tick_nohz_start_idle().
          */
         ts->inidle = 1;
-        tick_nohz_stop_sched_tick(ts);
+        __tick_nohz_idle_enter(ts);
 
         local_irq_enable();
 }
@@ -505,7 +525,7 @@ void tick_nohz_irq_exit(void)
         if (!ts->inidle)
                 return;
 
-        tick_nohz_stop_sched_tick(ts);
+        __tick_nohz_idle_enter(ts);
 }
 
 /**
@@ -523,7 +543,7 @@ ktime_t tick_nohz_get_sleep_length(void)
 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 {
         hrtimer_cancel(&ts->sched_timer);
-        hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
+        hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
 
         while (1) {
                 /* Forward the time to expire in the future */
@@ -546,6 +566,41 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
         }
 }
 
+static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
+{
+        /* Update jiffies first */
+        select_nohz_load_balancer(0);
+        tick_do_update_jiffies64(now);
+        update_cpu_load_nohz();
+
+        touch_softlockup_watchdog();
+        /*
+         * Cancel the scheduled timer and restore the tick
+         */
+        ts->tick_stopped  = 0;
+        ts->idle_exittime = now;
+
+        tick_nohz_restart(ts, now);
+}
+
+static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
+{
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING
+        unsigned long ticks;
+        /*
+         * We stopped the tick in idle. Update process times would miss the
+         * time we slept as update_process_times does only a 1 tick
+         * accounting. Enforce that this is accounted to idle !
+         */
+        ticks = jiffies - ts->idle_jiffies;
+        /*
+         * We might be one off. Do not randomly account a huge number of ticks!
+         */
+        if (ticks && ticks < LONG_MAX)
+                account_idle_ticks(ticks);
+#endif
+}
+
 /**
  * tick_nohz_idle_exit - restart the idle tick from the idle task
  *
@@ -557,9 +612,6 @@ void tick_nohz_idle_exit(void)
 {
         int cpu = smp_processor_id();
         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
-#ifndef CONFIG_VIRT_CPU_ACCOUNTING
-        unsigned long ticks;
-#endif
         ktime_t now;
 
         local_irq_disable();
@@ -574,40 +626,11 @@ void tick_nohz_idle_exit(void)
         if (ts->idle_active)
                 tick_nohz_stop_idle(cpu, now);
 
-        if (!ts->tick_stopped) {
-                local_irq_enable();
-                return;
+        if (ts->tick_stopped) {
+                tick_nohz_restart_sched_tick(ts, now);
+                tick_nohz_account_idle_ticks(ts);
         }
 
-        /* Update jiffies first */
-        select_nohz_load_balancer(0);
-        tick_do_update_jiffies64(now);
-        update_cpu_load_nohz();
-
-#ifndef CONFIG_VIRT_CPU_ACCOUNTING
-        /*
-         * We stopped the tick in idle. Update process times would miss the
-         * time we slept as update_process_times does only a 1 tick
-         * accounting. Enforce that this is accounted to idle !
-         */
-        ticks = jiffies - ts->idle_jiffies;
-        /*
-         * We might be one off. Do not randomly account a huge number of ticks!
-         */
-        if (ticks && ticks < LONG_MAX)
-                account_idle_ticks(ticks);
-#endif
-
-        calc_load_exit_idle();
-        touch_softlockup_watchdog();
-        /*
-         * Cancel the scheduled timer and restore the tick
-         */
-        ts->tick_stopped  = 0;
-        ts->idle_exittime = now;
-
-        tick_nohz_restart(ts, now);
-
         local_irq_enable();
 }
 
@@ -811,7 +834,8 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
                  */
                 if (ts->tick_stopped) {
                         touch_softlockup_watchdog();
-                        ts->idle_jiffies++;
+                        if (idle_cpu(cpu))
+                                ts->idle_jiffies++;
                 }
                 update_process_times(user_mode(regs));
                 profile_tick(CPU_PROFILING);