7 files changed, 100 insertions, 85 deletions

diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index 7a925ba456fb..a6a5bf53e86d 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -51,7 +51,13 @@
  * HZ shrinks, so values greater than 8 overflow 32bits when
  * HZ=100.
  */
+#if HZ < 34
+#define JIFFIES_SHIFT   6
+#elif HZ < 67
+#define JIFFIES_SHIFT   7
+#else
 #define JIFFIES_SHIFT   8
+#endif
 
 static cycle_t jiffies_read(struct clocksource *cs)
 {
diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c
index 68b799375981..4d23dc4d8139 100644
--- a/kernel/time/sched_clock.c
+++ b/kernel/time/sched_clock.c
@@ -74,7 +74,7 @@ unsigned long long notrace sched_clock(void)
                 return cd.epoch_ns;
 
         do {
-                seq = read_seqcount_begin(&cd.seq);
+                seq = raw_read_seqcount_begin(&cd.seq);
                 epoch_cyc = cd.epoch_cyc;
                 epoch_ns = cd.epoch_ns;
         } while (read_seqcount_retry(&cd.seq, seq));
@@ -99,10 +99,10 @@ static void notrace update_sched_clock(void)
                           cd.mult, cd.shift);
 
         raw_local_irq_save(flags);
-        write_seqcount_begin(&cd.seq);
+        raw_write_seqcount_begin(&cd.seq);
         cd.epoch_ns = ns;
         cd.epoch_cyc = cyc;
-        write_seqcount_end(&cd.seq);
+        raw_write_seqcount_end(&cd.seq);
         raw_local_irq_restore(flags);
 }
 
@@ -116,20 +116,42 @@ static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)
 void __init sched_clock_register(u64 (*read)(void), int bits,
                                  unsigned long rate)
 {
+        u64 res, wrap, new_mask, new_epoch, cyc, ns;
+        u32 new_mult, new_shift;
+        ktime_t new_wrap_kt;
         unsigned long r;
-        u64 res, wrap;
         char r_unit;
 
         if (cd.rate > rate)
                 return;
 
         WARN_ON(!irqs_disabled());
-        read_sched_clock = read;
-        sched_clock_mask = CLOCKSOURCE_MASK(bits);
-        cd.rate = rate;
 
         /* calculate the mult/shift to convert counter ticks to ns. */
-        clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 3600);
+        clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600);
+
+        new_mask = CLOCKSOURCE_MASK(bits);
+
+        /* calculate how many ns until we wrap */
+        wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask);
+        new_wrap_kt = ns_to_ktime(wrap - (wrap >> 3));
+
+        /* update epoch for new counter and update epoch_ns from old counter*/
+        new_epoch = read();
+        cyc = read_sched_clock();
+        ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask,
+                          cd.mult, cd.shift);
+
+        raw_write_seqcount_begin(&cd.seq);
+        read_sched_clock = read;
+        sched_clock_mask = new_mask;
+        cd.rate = rate;
+        cd.wrap_kt = new_wrap_kt;
+        cd.mult = new_mult;
+        cd.shift = new_shift;
+        cd.epoch_cyc = new_epoch;
+        cd.epoch_ns = ns;
+        raw_write_seqcount_end(&cd.seq);
 
         r = rate;
         if (r >= 4000000) {
@@ -141,22 +163,12 @@ void __init sched_clock_register(u64 (*read)(void), int bits,
         } else
                 r_unit = ' ';
 
-        /* calculate how many ns until we wrap */
-        wrap = clocks_calc_max_nsecs(cd.mult, cd.shift, 0, sched_clock_mask);
-        cd.wrap_kt = ns_to_ktime(wrap - (wrap >> 3));
-
         /* calculate the ns resolution of this counter */
-        res = cyc_to_ns(1ULL, cd.mult, cd.shift);
+        res = cyc_to_ns(1ULL, new_mult, new_shift);
+
         pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n",
                 bits, r, r_unit, res, wrap);
 
-        update_sched_clock();
-
-        /*
-         * Ensure that sched_clock() starts off at 0ns
-         */
-        cd.epoch_ns = 0;
-
         /* Enable IRQ time accounting if we have a fast enough sched_clock */
         if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
                 enable_sched_clock_irqtime();
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 9532690daaa9..98977a57ac72 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -538,10 +538,10 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
  * Called from irq_enter() when idle was interrupted to reenable the
  * per cpu device.
  */
-void tick_check_oneshot_broadcast(int cpu)
+void tick_check_oneshot_broadcast_this_cpu(void)
 {
-        if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) {
-                struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
+        if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
+                struct tick_device *td = &__get_cpu_var(tick_cpu_device);
 
                 /*
                  * We might be in the middle of switching over from
@@ -756,6 +756,7 @@ out:
 static void tick_broadcast_clear_oneshot(int cpu)
 {
         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
+        cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 }
 
 static void tick_broadcast_init_next_event(struct cpumask *mask,
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 162b03ab0ad2..20b2fe37d105 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -85,6 +85,7 @@ static void tick_periodic(int cpu)
 
                 do_timer(1);
                 write_sequnlock(&jiffies_lock);
+                update_wall_time();
         }
 
         update_process_times(user_mode(get_irq_regs()));
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 18e71f7fbc2a..8329669b51ec 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -51,7 +51,7 @@ extern void tick_broadcast_switch_to_oneshot(void);
 extern void tick_shutdown_broadcast_oneshot(unsigned int *cpup);
 extern int tick_resume_broadcast_oneshot(struct clock_event_device *bc);
 extern int tick_broadcast_oneshot_active(void);
-extern void tick_check_oneshot_broadcast(int cpu);
+extern void tick_check_oneshot_broadcast_this_cpu(void);
 bool tick_broadcast_oneshot_available(void);
 # else /* BROADCAST */
 static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
@@ -62,7 +62,7 @@ static inline void tick_broadcast_oneshot_control(unsigned long reason) { }
 static inline void tick_broadcast_switch_to_oneshot(void) { }
 static inline void tick_shutdown_broadcast_oneshot(unsigned int *cpup) { }
 static inline int tick_broadcast_oneshot_active(void) { return 0; }
-static inline void tick_check_oneshot_broadcast(int cpu) { }
+static inline void tick_check_oneshot_broadcast_this_cpu(void) { }
 static inline bool tick_broadcast_oneshot_available(void) { return true; }
 # endif /* !BROADCAST */
 
@@ -155,3 +155,4 @@ static inline int tick_device_is_functional(struct clock_event_device *dev)
 #endif
 
 extern void do_timer(unsigned long ticks);
+extern void update_wall_time(void);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index ea20f7d1ac2c..9f8af69c67ec 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -86,6 +86,7 @@ static void tick_do_update_jiffies64(ktime_t now)
                 tick_next_period = ktime_add(last_jiffies_update, tick_period);
         }
         write_sequnlock(&jiffies_lock);
+        update_wall_time();
 }
 
 /*
@@ -177,7 +178,7 @@ static bool can_stop_full_tick(void)
          * TODO: kick full dynticks CPUs when
          * sched_clock_stable is set.
          */
-        if (!sched_clock_stable) {
+        if (!sched_clock_stable()) {
                 trace_tick_stop(0, "unstable sched clock\n");
                 /*
                  * Don't allow the user to think they can get
@@ -391,11 +392,9 @@ __setup("nohz=", setup_tick_nohz);
  */
 static void tick_nohz_update_jiffies(ktime_t now)
 {
-        int cpu = smp_processor_id();
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
         unsigned long flags;
 
-        ts->idle_waketime = now;
+        __this_cpu_write(tick_cpu_sched.idle_waketime, now);
 
         local_irq_save(flags);
         tick_do_update_jiffies64(now);
@@ -426,17 +425,15 @@ update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_upda
 
 }
 
-static void tick_nohz_stop_idle(int cpu, ktime_t now)
+static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
 {
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
-
-        update_ts_time_stats(cpu, ts, now, NULL);
+        update_ts_time_stats(smp_processor_id(), ts, now, NULL);
         ts->idle_active = 0;
 
         sched_clock_idle_wakeup_event(0);
 }
 
-static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts)
+static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
 {
         ktime_t now = ktime_get();
 
@@ -536,12 +533,13 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
         struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
         u64 time_delta;
 
+        time_delta = timekeeping_max_deferment();
+
         /* Read jiffies and the time when jiffies were updated last */
         do {
                 seq = read_seqbegin(&jiffies_lock);
                 last_update = last_jiffies_update;
                 last_jiffies = jiffies;
-                time_delta = timekeeping_max_deferment();
         } while (read_seqretry(&jiffies_lock, seq));
 
         if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) ||
@@ -681,18 +679,18 @@ out:
 static void tick_nohz_full_stop_tick(struct tick_sched *ts)
 {
 #ifdef CONFIG_NO_HZ_FULL
-       int cpu = smp_processor_id();
+        int cpu = smp_processor_id();
 
-       if (!tick_nohz_full_cpu(cpu) || is_idle_task(current))
-               return;
+        if (!tick_nohz_full_cpu(cpu) || is_idle_task(current))
+                return;
 
-       if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
-               return;
+        if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
+                return;
 
-       if (!can_stop_full_tick())
-               return;
+        if (!can_stop_full_tick())
+                return;
 
-       tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);
+        tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);
 #endif
 }
 
@@ -754,7 +752,7 @@ static void __tick_nohz_idle_enter(struct tick_sched *ts)
         ktime_t now, expires;
         int cpu = smp_processor_id();
 
-        now = tick_nohz_start_idle(cpu, ts);
+        now = tick_nohz_start_idle(ts);
 
         if (can_stop_idle_tick(cpu, ts)) {
                 int was_stopped = ts->tick_stopped;
@@ -911,8 +909,7 @@ static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
  */
 void tick_nohz_idle_exit(void)
 {
-        int cpu = smp_processor_id();
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
         ktime_t now;
 
         local_irq_disable();
@@ -925,7 +922,7 @@ void tick_nohz_idle_exit(void)
                 now = ktime_get();
 
         if (ts->idle_active)
-                tick_nohz_stop_idle(cpu, now);
+                tick_nohz_stop_idle(ts, now);
 
         if (ts->tick_stopped) {
                 tick_nohz_restart_sched_tick(ts, now);
@@ -1009,12 +1006,10 @@ static void tick_nohz_switch_to_nohz(void)
  * timer and do not touch the other magic bits which need to be done
  * when idle is left.
  */
-static void tick_nohz_kick_tick(int cpu, ktime_t now)
+static void tick_nohz_kick_tick(struct tick_sched *ts, ktime_t now)
 {
 #if 0
         /* Switch back to 2.6.27 behaviour */
-
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
         ktime_t delta;
 
         /*
@@ -1029,36 +1024,36 @@ static void tick_nohz_kick_tick(int cpu, ktime_t now)
 #endif
 }
 
-static inline void tick_check_nohz(int cpu)
+static inline void tick_nohz_irq_enter(void)
 {
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
         ktime_t now;
 
         if (!ts->idle_active && !ts->tick_stopped)
                 return;
         now = ktime_get();
         if (ts->idle_active)
-                tick_nohz_stop_idle(cpu, now);
+                tick_nohz_stop_idle(ts, now);
         if (ts->tick_stopped) {
                 tick_nohz_update_jiffies(now);
-                tick_nohz_kick_tick(cpu, now);
+                tick_nohz_kick_tick(ts, now);
         }
 }
 
 #else
 
 static inline void tick_nohz_switch_to_nohz(void) { }
-static inline void tick_check_nohz(int cpu) { }
+static inline void tick_nohz_irq_enter(void) { }
 
 #endif /* CONFIG_NO_HZ_COMMON */
 
 /*
  * Called from irq_enter to notify about the possible interruption of idle()
  */
-void tick_check_idle(int cpu)
+void tick_irq_enter(void)
 {
-        tick_check_oneshot_broadcast(cpu);
-        tick_check_nohz(cpu);
+        tick_check_oneshot_broadcast_this_cpu();
+        tick_nohz_irq_enter();
 }
 
 /*
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 87b4f00284c9..0aa4ce81bc16 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -77,7 +77,7 @@ static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm)
         tk->wall_to_monotonic = wtm;
         set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
         tk->offs_real = timespec_to_ktime(tmp);
-        tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tk->tai_offset, 0));
+        tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
 }
 
 static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t)
@@ -90,8 +90,9 @@ static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t)
 }
 
 /**
- * timekeeper_setup_internals - Set up internals to use clocksource clock.
+ * tk_setup_internals - Set up internals to use clocksource clock.
  *
+ * @tk:         The target timekeeper to setup.
  * @clock:              Pointer to clocksource.
  *
  * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
@@ -595,7 +596,7 @@ s32 timekeeping_get_tai_offset(void)
 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
 {
         tk->tai_offset = tai_offset;
-        tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tai_offset, 0));
+        tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
 }
 
 /**
@@ -610,6 +611,7 @@ void timekeeping_set_tai_offset(s32 tai_offset)
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
         write_seqcount_begin(&timekeeper_seq);
         __timekeeping_set_tai_offset(tk, tai_offset);
+        timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
         write_seqcount_end(&timekeeper_seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
         clock_was_set();
@@ -1023,6 +1025,8 @@ static int timekeeping_suspend(void)
                 timekeeping_suspend_time =
                         timespec_add(timekeeping_suspend_time, delta_delta);
         }
+
+        timekeeping_update(tk, TK_MIRROR);
         write_seqcount_end(&timekeeper_seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
@@ -1130,16 +1134,6 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
                  * we can adjust by 1.
                  */
                 error >>= 2;
-                /*
-                 * XXX - In update_wall_time, we round up to the next
-                 * nanosecond, and store the amount rounded up into
-                 * the error. This causes the likely below to be unlikely.
-                 *
-                 * The proper fix is to avoid rounding up by using
-                 * the high precision tk->xtime_nsec instead of
-                 * xtime.tv_nsec everywhere. Fixing this will take some
-                 * time.
-                 */
                 if (likely(error <= interval))
                         adj = 1;
                 else
@@ -1255,7 +1249,7 @@ out_adjust:
 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
 {
         u64 nsecps = (u64)NSEC_PER_SEC << tk->shift;
-        unsigned int action = 0;
+        unsigned int clock_set = 0;
 
         while (tk->xtime_nsec >= nsecps) {
                 int leap;
@@ -1277,11 +1271,10 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
 
                         __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
 
-                        clock_was_set_delayed();
-                        action = TK_CLOCK_WAS_SET;
+                        clock_set = TK_CLOCK_WAS_SET;
                 }
         }
-        return action;
+        return clock_set;
 }
 
 /**
@@ -1294,7 +1287,8 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
  * Returns the unconsumed cycles.
  */
 static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
-                                                u32 shift)
+                                                u32 shift,
+                                                unsigned int *clock_set)
 {
         cycle_t interval = tk->cycle_interval << shift;
         u64 raw_nsecs;
@@ -1308,7 +1302,7 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
         tk->cycle_last += interval;
 
         tk->xtime_nsec += tk->xtime_interval << shift;
-        accumulate_nsecs_to_secs(tk);
+        *clock_set |= accumulate_nsecs_to_secs(tk);
 
         /* Accumulate raw time */
         raw_nsecs = (u64)tk->raw_interval << shift;
@@ -1359,14 +1353,14 @@ static inline void old_vsyscall_fixup(struct timekeeper *tk)
  * update_wall_time - Uses the current clocksource to increment the wall time
  *
  */
-static void update_wall_time(void)
+void update_wall_time(void)
 {
         struct clocksource *clock;
         struct timekeeper *real_tk = &timekeeper;
         struct timekeeper *tk = &shadow_timekeeper;
         cycle_t offset;
         int shift = 0, maxshift;
-        unsigned int action;
+        unsigned int clock_set = 0;
         unsigned long flags;
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
@@ -1401,7 +1395,8 @@ static void update_wall_time(void)
         maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
         shift = min(shift, maxshift);
         while (offset >= tk->cycle_interval) {
-                offset = logarithmic_accumulation(tk, offset, shift);
+                offset = logarithmic_accumulation(tk, offset, shift,
+                                                        &clock_set);
                 if (offset < tk->cycle_interval<<shift)
                         shift--;
         }
@@ -1419,7 +1414,7 @@ static void update_wall_time(void)
          * Finally, make sure that after the rounding
          * xtime_nsec isn't larger than NSEC_PER_SEC
          */
-        action = accumulate_nsecs_to_secs(tk);
+        clock_set |= accumulate_nsecs_to_secs(tk);
 
         write_seqcount_begin(&timekeeper_seq);
         /* Update clock->cycle_last with the new value */
@@ -1435,10 +1430,12 @@ static void update_wall_time(void)
          * updating.
          */
         memcpy(real_tk, tk, sizeof(*tk));
-        timekeeping_update(real_tk, action);
+        timekeeping_update(real_tk, clock_set);
         write_seqcount_end(&timekeeper_seq);
 out:
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+        if (clock_set)
+                clock_was_set();
 }
 
 /**
@@ -1583,7 +1580,6 @@ struct timespec get_monotonic_coarse(void)
 void do_timer(unsigned long ticks)
 {
         jiffies_64 += ticks;
-        update_wall_time();
         calc_global_load(ticks);
 }
 
@@ -1698,12 +1694,14 @@ int do_adjtimex(struct timex *txc)
 
         if (tai != orig_tai) {
                 __timekeeping_set_tai_offset(tk, tai);
-                update_pvclock_gtod(tk, true);
-                clock_was_set_delayed();
+                timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
         }
         write_seqcount_end(&timekeeper_seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
+        if (tai != orig_tai)
+                clock_was_set();
+
         ntp_notify_cmos_timer();
 
         return ret;
@@ -1739,4 +1737,5 @@ void xtime_update(unsigned long ticks)
         write_seqlock(&jiffies_lock);
         do_timer(ticks);
         write_sequnlock(&jiffies_lock);
+        update_wall_time();
 }