4 files changed, 81 insertions, 44 deletions

diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c
index c436e790b21b..8a46f5d64504 100644
--- a/kernel/time/alarmtimer.c
+++ b/kernel/time/alarmtimer.c
@@ -195,7 +195,7 @@ static enum hrtimer_restart alarmtimer_fired(struct hrtimer *timer)
                 struct alarm *alarm;
                 ktime_t expired = next->expires;
 
-                if (expired.tv64 >= now.tv64)
+                if (expired.tv64 > now.tv64)
                         break;
 
                 alarm = container_of(next, struct alarm, node);
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index cfc65e1eb9fb..d3ad022136e5 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -548,7 +548,7 @@ static u64 clocksource_max_deferment(struct clocksource *cs)
          * note a margin of 12.5% is used because this can be computed with
          * a shift, versus say 10% which would require division.
          */
-        return max_nsecs - (max_nsecs >> 5);
+        return max_nsecs - (max_nsecs >> 3);
 }
 
 #ifndef CONFIG_ARCH_USES_GETTIMEOFFSET
@@ -647,7 +647,7 @@ static void clocksource_enqueue(struct clocksource *cs)
 
 /**
  * __clocksource_updatefreq_scale - Used update clocksource with new freq
- * @t:          clocksource to be registered
+ * @cs:         clocksource to be registered
  * @scale:      Scale factor multiplied against freq to get clocksource hz
  * @freq:       clocksource frequency (cycles per second) divided by scale
  *
@@ -669,7 +669,7 @@ void __clocksource_updatefreq_scale(struct clocksource *cs, u32 scale, u32 freq)
          * ~ 0.06ppm granularity for NTP. We apply the same 12.5%
          * margin as we do in clocksource_max_deferment()
          */
-        sec = (cs->mask - (cs->mask >> 5));
+        sec = (cs->mask - (cs->mask >> 3));
         do_div(sec, freq);
         do_div(sec, scale);
         if (!sec)
@@ -699,7 +699,7 @@ EXPORT_SYMBOL_GPL(__clocksource_updatefreq_scale);
 
 /**
  * __clocksource_register_scale - Used to install new clocksources
- * @t:          clocksource to be registered
+ * @cs:         clocksource to be registered
  * @scale:      Scale factor multiplied against freq to get clocksource hz
  * @freq:       clocksource frequency (cycles per second) divided by scale
  *
@@ -727,7 +727,7 @@ EXPORT_SYMBOL_GPL(__clocksource_register_scale);
 
 /**
  * clocksource_register - Used to install new clocksources
- * @t:          clocksource to be registered
+ * @cs:         clocksource to be registered
  *
  * Returns -EBUSY if registration fails, zero otherwise.
  */
@@ -761,6 +761,8 @@ static void __clocksource_change_rating(struct clocksource *cs, int rating)
 
 /**
  * clocksource_change_rating - Change the rating of a registered clocksource
+ * @cs:         clocksource to be changed
+ * @rating:     new rating
  */
 void clocksource_change_rating(struct clocksource *cs, int rating)
 {
@@ -772,6 +774,7 @@ EXPORT_SYMBOL(clocksource_change_rating);
 
 /**
  * clocksource_unregister - remove a registered clocksource
+ * @cs: clocksource to be unregistered
  */
 void clocksource_unregister(struct clocksource *cs)
 {
@@ -787,6 +790,7 @@ EXPORT_SYMBOL(clocksource_unregister);
 /**
  * sysfs_show_current_clocksources - sysfs interface for current clocksource
  * @dev:        unused
+ * @attr:       unused
  * @buf:        char buffer to be filled with clocksource list
  *
  * Provides sysfs interface for listing current clocksource.
@@ -807,6 +811,7 @@ sysfs_show_current_clocksources(struct sys_device *dev,
 /**
  * sysfs_override_clocksource - interface for manually overriding clocksource
  * @dev:        unused
+ * @attr:       unused
  * @buf:        name of override clocksource
  * @count:      length of buffer
  *
@@ -842,6 +847,7 @@ static ssize_t sysfs_override_clocksource(struct sys_device *dev,
 /**
  * sysfs_show_available_clocksources - sysfs interface for listing clocksource
  * @dev:        unused
+ * @attr:       unused
  * @buf:        char buffer to be filled with clocksource list
  *
  * Provides sysfs interface for listing registered clocksources
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index f954282d9a82..fd4a7b1625a2 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -71,7 +71,7 @@ int tick_check_broadcast_device(struct clock_event_device *dev)
              (dev->features & CLOCK_EVT_FEAT_C3STOP))
                 return 0;
 
-        clockevents_exchange_device(NULL, dev);
+        clockevents_exchange_device(tick_broadcast_device.evtdev, dev);
         tick_broadcast_device.evtdev = dev;
         if (!cpumask_empty(tick_get_broadcast_mask()))
                 tick_broadcast_start_periodic(dev);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 40420644d0ba..7656642e4b8e 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -275,42 +275,17 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 }
 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 
-/**
- * tick_nohz_stop_sched_tick - stop the idle tick from the idle task
- *
- * When the next event is more than a tick into the future, stop the idle tick
- * Called either from the idle loop or from irq_exit() when an idle period was
- * just interrupted by an interrupt which did not cause a reschedule.
- */
-void tick_nohz_stop_sched_tick(int inidle)
+static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
 {
-        unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
-        struct tick_sched *ts;
+        unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
         ktime_t last_update, expires, now;
         struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
         u64 time_delta;
         int cpu;
 
-        local_irq_save(flags);
-
         cpu = smp_processor_id();
         ts = &per_cpu(tick_cpu_sched, cpu);
 
-        /*
-         * Call to tick_nohz_start_idle stops the last_update_time from being
-         * updated. Thus, it must not be called in the event we are called from
-         * irq_exit() with the prior state different than idle.
-         */
-        if (!inidle && !ts->inidle)
-                goto end;
-
-        /*
-         * Set ts->inidle unconditionally. Even if the system did not
-         * switch to NOHZ mode the cpu frequency governers rely on the
-         * update of the idle time accounting in tick_nohz_start_idle().
-         */
-        ts->inidle = 1;
-
         now = tick_nohz_start_idle(cpu, ts);
 
         /*
@@ -326,10 +301,10 @@ void tick_nohz_stop_sched_tick(int inidle)
         }
 
         if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
-                goto end;
+                return;
 
         if (need_resched())
-                goto end;
+                return;
 
         if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
                 static int ratelimit;
@@ -339,7 +314,7 @@ void tick_nohz_stop_sched_tick(int inidle)
                                (unsigned int) local_softirq_pending());
                         ratelimit++;
                 }
-                goto end;
+                return;
         }
 
         ts->idle_calls++;
@@ -434,7 +409,6 @@ void tick_nohz_stop_sched_tick(int inidle)
                         ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
                         ts->tick_stopped = 1;
                         ts->idle_jiffies = last_jiffies;
-                        rcu_enter_nohz();
                 }
 
                 ts->idle_sleeps++;
@@ -472,8 +446,64 @@ out:
         ts->next_jiffies = next_jiffies;
         ts->last_jiffies = last_jiffies;
         ts->sleep_length = ktime_sub(dev->next_event, now);
-end:
-        local_irq_restore(flags);
+}
+
+/**
+ * tick_nohz_idle_enter - stop the idle tick from the idle task
+ *
+ * When the next event is more than a tick into the future, stop the idle tick
+ * Called when we start the idle loop.
+ *
+ * The arch is responsible of calling:
+ *
+ * - rcu_idle_enter() after its last use of RCU before the CPU is put
+ *  to sleep.
+ * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
+ */
+void tick_nohz_idle_enter(void)
+{
+        struct tick_sched *ts;
+
+        WARN_ON_ONCE(irqs_disabled());
+
+        /*
+         * Update the idle state in the scheduler domain hierarchy
+         * when tick_nohz_stop_sched_tick() is called from the idle loop.
+         * State will be updated to busy during the first busy tick after
+         * exiting idle.
+         */
+        set_cpu_sd_state_idle();
+
+        local_irq_disable();
+
+        ts = &__get_cpu_var(tick_cpu_sched);
+        /*
+         * set ts->inidle unconditionally. even if the system did not
+         * switch to nohz mode the cpu frequency governers rely on the
+         * update of the idle time accounting in tick_nohz_start_idle().
+         */
+        ts->inidle = 1;
+        tick_nohz_stop_sched_tick(ts);
+
+        local_irq_enable();
+}
+
+/**
+ * tick_nohz_irq_exit - update next tick event from interrupt exit
+ *
+ * When an interrupt fires while we are idle and it doesn't cause
+ * a reschedule, it may still add, modify or delete a timer, enqueue
+ * an RCU callback, etc...
+ * So we need to re-calculate and reprogram the next tick event.
+ */
+void tick_nohz_irq_exit(void)
+{
+        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+
+        if (!ts->inidle)
+                return;
+
+        tick_nohz_stop_sched_tick(ts);
 }
 
 /**
@@ -515,11 +545,13 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 }
 
 /**
- * tick_nohz_restart_sched_tick - restart the idle tick from the idle task
+ * tick_nohz_idle_exit - restart the idle tick from the idle task
  *
  * Restart the idle tick when the CPU is woken up from idle
+ * This also exit the RCU extended quiescent state. The CPU
+ * can use RCU again after this function is called.
  */
-void tick_nohz_restart_sched_tick(void)
+void tick_nohz_idle_exit(void)
 {
         int cpu = smp_processor_id();
         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
@@ -529,6 +561,7 @@ void tick_nohz_restart_sched_tick(void)
         ktime_t now;
 
         local_irq_disable();
+
         if (ts->idle_active || (ts->inidle && ts->tick_stopped))
                 now = ktime_get();
 
@@ -543,8 +576,6 @@ void tick_nohz_restart_sched_tick(void)
 
         ts->inidle = 0;
 
-        rcu_exit_nohz();
-
         /* Update jiffies first */
         select_nohz_load_balancer(0);
         tick_do_update_jiffies64(now);