Merge branch 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull timer changes from Ingo Molnar:
  - ARM clocksource/clockevent improvements and fixes
  - generic timekeeping updates: TAI fixes/improvements, cleanups
  - Posix cpu timer cleanups and improvements
  - dynticks updates: full dynticks bugfixes, optimizations and cleanups

* 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (46 commits)
  clocksource: Timer-sun5i: Switch to sched_clock_register()
  timekeeping: Remove comment that's mostly out of date
  rtc-cmos: Add an alarm disable quirk
  timekeeper: fix comment typo for tk_setup_internals()
  timekeeping: Fix missing timekeeping_update in suspend path
  timekeeping: Fix CLOCK_TAI timer/nanosleep delays
  tick/timekeeping: Call update_wall_time outside the jiffies lock
  timekeeping: Avoid possible deadlock from clock_was_set_delayed
  timekeeping: Fix potential lost pv notification of time change
  timekeeping: Fix lost updates to tai adjustment
  clocksource: sh_cmt: Add clk_prepare/unprepare support
  clocksource: bcm_kona_timer: Remove unused bcm_timer_ids
  clocksource: vt8500: Remove deprecated IRQF_DISABLED
  clocksource: tegra: Remove deprecated IRQF_DISABLED
  clocksource: misc drivers: Remove deprecated IRQF_DISABLED
  clocksource: sh_mtu2: Remove unnecessary platform_set_drvdata()
  clocksource: sh_tmu: Remove unnecessary platform_set_drvdata()
  clocksource: armada-370-xp: Enable timer divider only when needed
  clocksource: clksrc-of: Warn if no clock sources are found
  clocksource: orion: Switch to sched_clock_register()
  ...

8 files changed, 190 insertions, 254 deletions

diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index e5f3917aa05b..6cb20d2e7ee0 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -53,10 +53,10 @@ void context_tracking_user_enter(void)
         /*
          * Repeat the user_enter() check here because some archs may be calling
          * this from asm and if no CPU needs context tracking, they shouldn't
-         * go further. Repeat the check here until they support the static key
-         * check.
+         * go further. Repeat the check here until they support the inline static
+         * key check.
          */
-        if (!static_key_false(&context_tracking_enabled))
+        if (!context_tracking_is_enabled())
                 return;
 
         /*
@@ -160,7 +160,7 @@ void context_tracking_user_exit(void)
 {
         unsigned long flags;
 
-        if (!static_key_false(&context_tracking_enabled))
+        if (!context_tracking_is_enabled())
                 return;
 
         if (in_interrupt())
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index c7f31aa272f7..3b8946416a5f 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -233,7 +233,8 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times)
 
 /*
  * Sample a process (thread group) clock for the given group_leader task.
- * Must be called with tasklist_lock held for reading.
+ * Must be called with task sighand lock held for safe while_each_thread()
+ * traversal.
  */
 static int cpu_clock_sample_group(const clockid_t which_clock,
                                   struct task_struct *p,
@@ -260,30 +261,53 @@ static int cpu_clock_sample_group(const clockid_t which_clock,
         return 0;
 }
 
+static int posix_cpu_clock_get_task(struct task_struct *tsk,
+                                    const clockid_t which_clock,
+                                    struct timespec *tp)
+{
+        int err = -EINVAL;
+        unsigned long long rtn;
+
+        if (CPUCLOCK_PERTHREAD(which_clock)) {
+                if (same_thread_group(tsk, current))
+                        err = cpu_clock_sample(which_clock, tsk, &rtn);
+        } else {
+                unsigned long flags;
+                struct sighand_struct *sighand;
+
+                /*
+                 * while_each_thread() is not yet entirely RCU safe,
+                 * keep locking the group while sampling process
+                 * clock for now.
+                 */
+                sighand = lock_task_sighand(tsk, &flags);
+                if (!sighand)
+                        return err;
+
+                if (tsk == current || thread_group_leader(tsk))
+                        err = cpu_clock_sample_group(which_clock, tsk, &rtn);
+
+                unlock_task_sighand(tsk, &flags);
+        }
+
+        if (!err)
+                sample_to_timespec(which_clock, rtn, tp);
+
+        return err;
+}
+
 
 static int posix_cpu_clock_get(const clockid_t which_clock, struct timespec *tp)
 {
         const pid_t pid = CPUCLOCK_PID(which_clock);
-        int error = -EINVAL;
-        unsigned long long rtn;
+        int err = -EINVAL;
 
         if (pid == 0) {
                 /*
                  * Special case constant value for our own clocks.
                  * We don't have to do any lookup to find ourselves.
                  */
-                if (CPUCLOCK_PERTHREAD(which_clock)) {
-                        /*
-                         * Sampling just ourselves we can do with no locking.
-                         */
-                        error = cpu_clock_sample(which_clock,
-                                                 current, &rtn);
-                } else {
-                        read_lock(&tasklist_lock);
-                        error = cpu_clock_sample_group(which_clock,
-                                                       current, &rtn);
-                        read_unlock(&tasklist_lock);
-                }
+                err = posix_cpu_clock_get_task(current, which_clock, tp);
         } else {
                 /*
                  * Find the given PID, and validate that the caller
@@ -292,29 +316,12 @@ static int posix_cpu_clock_get(const clockid_t which_clock, struct timespec *tp)
                 struct task_struct *p;
                 rcu_read_lock();
                 p = find_task_by_vpid(pid);
-                if (p) {
-                        if (CPUCLOCK_PERTHREAD(which_clock)) {
-                                if (same_thread_group(p, current)) {
-                                        error = cpu_clock_sample(which_clock,
-                                                                 p, &rtn);
-                                }
-                        } else {
-                                read_lock(&tasklist_lock);
-                                if (thread_group_leader(p) && p->sighand) {
-                                        error =
-                                            cpu_clock_sample_group(which_clock,
-                                                                   p, &rtn);
-                                }
-                                read_unlock(&tasklist_lock);
-                        }
-                }
+                if (p)
+                        err = posix_cpu_clock_get_task(p, which_clock, tp);
                 rcu_read_unlock();
         }
 
-        if (error)
-                return error;
-        sample_to_timespec(which_clock, rtn, tp);
-        return 0;
+        return err;
 }
 
 
@@ -371,36 +378,40 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer)
  */
 static int posix_cpu_timer_del(struct k_itimer *timer)
 {
-        struct task_struct *p = timer->it.cpu.task;
         int ret = 0;
+        unsigned long flags;
+        struct sighand_struct *sighand;
+        struct task_struct *p = timer->it.cpu.task;
 
-        if (likely(p != NULL)) {
-                read_lock(&tasklist_lock);
-                if (unlikely(p->sighand == NULL)) {
-                        /*
-                         * We raced with the reaping of the task.
-                         * The deletion should have cleared us off the list.
-                         */
-                        BUG_ON(!list_empty(&timer->it.cpu.entry));
-                } else {
-                        spin_lock(&p->sighand->siglock);
-                        if (timer->it.cpu.firing)
-                                ret = TIMER_RETRY;
-                        else
-                                list_del(&timer->it.cpu.entry);
-                        spin_unlock(&p->sighand->siglock);
-                }
-                read_unlock(&tasklist_lock);
+        WARN_ON_ONCE(p == NULL);
 
-                if (!ret)
-                        put_task_struct(p);
+        /*
+         * Protect against sighand release/switch in exit/exec and process/
+         * thread timer list entry concurrent read/writes.
+         */
+        sighand = lock_task_sighand(p, &flags);
+        if (unlikely(sighand == NULL)) {
+                /*
+                 * We raced with the reaping of the task.
+                 * The deletion should have cleared us off the list.
+                 */
+                WARN_ON_ONCE(!list_empty(&timer->it.cpu.entry));
+        } else {
+                if (timer->it.cpu.firing)
+                        ret = TIMER_RETRY;
+                else
+                        list_del(&timer->it.cpu.entry);
+
+                unlock_task_sighand(p, &flags);
         }
 
+        if (!ret)
+                put_task_struct(p);
+
         return ret;
 }
 
-static void cleanup_timers_list(struct list_head *head,
-                                unsigned long long curr)
+static void cleanup_timers_list(struct list_head *head)
 {
         struct cpu_timer_list *timer, *next;
 
@@ -414,16 +425,11 @@ static void cleanup_timers_list(struct list_head *head,
  * time for later timer_gettime calls to return.
  * This must be called with the siglock held.
  */
-static void cleanup_timers(struct list_head *head,
-                           cputime_t utime, cputime_t stime,
-                           unsigned long long sum_exec_runtime)
+static void cleanup_timers(struct list_head *head)
 {
-
-        cputime_t ptime = utime + stime;
-
-        cleanup_timers_list(head, cputime_to_expires(ptime));
-        cleanup_timers_list(++head, cputime_to_expires(utime));
-        cleanup_timers_list(++head, sum_exec_runtime);
+        cleanup_timers_list(head);
+        cleanup_timers_list(++head);
+        cleanup_timers_list(++head);
 }
 
 /*
@@ -433,41 +439,14 @@ static void cleanup_timers(struct list_head *head,
  */
 void posix_cpu_timers_exit(struct task_struct *tsk)
 {
-        cputime_t utime, stime;
-
         add_device_randomness((const void*) &tsk->se.sum_exec_runtime,
                                                 sizeof(unsigned long long));
-        task_cputime(tsk, &utime, &stime);
-        cleanup_timers(tsk->cpu_timers,
-                       utime, stime, tsk->se.sum_exec_runtime);
+        cleanup_timers(tsk->cpu_timers);
 
 }
 void posix_cpu_timers_exit_group(struct task_struct *tsk)
 {
-        struct signal_struct *const sig = tsk->signal;
-        cputime_t utime, stime;
-
-        task_cputime(tsk, &utime, &stime);
-        cleanup_timers(tsk->signal->cpu_timers,
-                       utime + sig->utime, stime + sig->stime,
-                       tsk->se.sum_exec_runtime + sig->sum_sched_runtime);
-}
-
-static void clear_dead_task(struct k_itimer *itimer, unsigned long long now)
-{
-        struct cpu_timer_list *timer = &itimer->it.cpu;
-
-        /*
-         * That's all for this thread or process.
-         * We leave our residual in expires to be reported.
-         */
-        put_task_struct(timer->task);
-        timer->task = NULL;
-        if (timer->expires < now) {
-                timer->expires = 0;
-        } else {
-                timer->expires -= now;
-        }
+        cleanup_timers(tsk->signal->cpu_timers);
 }
 
 static inline int expires_gt(cputime_t expires, cputime_t new_exp)
@@ -477,8 +456,7 @@ static inline int expires_gt(cputime_t expires, cputime_t new_exp)
 
 /*
  * Insert the timer on the appropriate list before any timers that
- * expire later.  This must be called with the tasklist_lock held
- * for reading, interrupts disabled and p->sighand->siglock taken.
+ * expire later.  This must be called with the sighand lock held.
  */
 static void arm_timer(struct k_itimer *timer)
 {
@@ -569,7 +547,8 @@ static void cpu_timer_fire(struct k_itimer *timer)
 
 /*
  * Sample a process (thread group) timer for the given group_leader task.
- * Must be called with tasklist_lock held for reading.
+ * Must be called with task sighand lock held for safe while_each_thread()
+ * traversal.
  */
 static int cpu_timer_sample_group(const clockid_t which_clock,
                                   struct task_struct *p,
@@ -608,7 +587,8 @@ static DECLARE_WORK(nohz_kick_work, nohz_kick_work_fn);
  */
 static void posix_cpu_timer_kick_nohz(void)
 {
-        schedule_work(&nohz_kick_work);
+        if (context_tracking_is_enabled())
+                schedule_work(&nohz_kick_work);
 }
 
 bool posix_cpu_timers_can_stop_tick(struct task_struct *tsk)
@@ -631,43 +611,39 @@ static inline void posix_cpu_timer_kick_nohz(void) { }
  * If we return TIMER_RETRY, it's necessary to release the timer's lock
  * and try again.  (This happens when the timer is in the middle of firing.)
  */
-static int posix_cpu_timer_set(struct k_itimer *timer, int flags,
+static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
                                struct itimerspec *new, struct itimerspec *old)
 {
+        unsigned long flags;
+        struct sighand_struct *sighand;
         struct task_struct *p = timer->it.cpu.task;
         unsigned long long old_expires, new_expires, old_incr, val;
         int ret;
 
-        if (unlikely(p == NULL)) {
-                /*
-                 * Timer refers to a dead task's clock.
-                 */
-                return -ESRCH;
-        }
+        WARN_ON_ONCE(p == NULL);
 
         new_expires = timespec_to_sample(timer->it_clock, &new->it_value);
 
-        read_lock(&tasklist_lock);
         /*
-         * We need the tasklist_lock to protect against reaping that
-         * clears p->sighand.  If p has just been reaped, we can no
+         * Protect against sighand release/switch in exit/exec and p->cpu_timers
+         * and p->signal->cpu_timers read/write in arm_timer()
+         */
+        sighand = lock_task_sighand(p, &flags);
+        /*
+         * If p has just been reaped, we can no
          * longer get any information about it at all.
          */
-        if (unlikely(p->sighand == NULL)) {
-                read_unlock(&tasklist_lock);
-                put_task_struct(p);
-                timer->it.cpu.task = NULL;
+        if (unlikely(sighand == NULL)) {
                 return -ESRCH;
         }
 
         /*
          * Disarm any old timer after extracting its expiry time.
          */
-        BUG_ON(!irqs_disabled());
+        WARN_ON_ONCE(!irqs_disabled());
 
         ret = 0;
         old_incr = timer->it.cpu.incr;
-        spin_lock(&p->sighand->siglock);
         old_expires = timer->it.cpu.expires;
         if (unlikely(timer->it.cpu.firing)) {
                 timer->it.cpu.firing = -1;
@@ -724,12 +700,11 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int flags,
                  * disable this firing since we are already reporting
                  * it as an overrun (thanks to bump_cpu_timer above).
                  */
-                spin_unlock(&p->sighand->siglock);
-                read_unlock(&tasklist_lock);
+                unlock_task_sighand(p, &flags);
                 goto out;
         }
 
-        if (new_expires != 0 && !(flags & TIMER_ABSTIME)) {
+        if (new_expires != 0 && !(timer_flags & TIMER_ABSTIME)) {
                 new_expires += val;
         }
 
@@ -743,9 +718,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int flags,
                 arm_timer(timer);
         }
 
-        spin_unlock(&p->sighand->siglock);
-        read_unlock(&tasklist_lock);
-
+        unlock_task_sighand(p, &flags);
         /*
          * Install the new reload setting, and
          * set up the signal and overrun bookkeeping.
@@ -787,7 +760,8 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp)
 {
         unsigned long long now;
         struct task_struct *p = timer->it.cpu.task;
-        int clear_dead;
+
+        WARN_ON_ONCE(p == NULL);
 
         /*
          * Easy part: convert the reload time.
@@ -800,52 +774,34 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp)
                 return;
         }
 
-        if (unlikely(p == NULL)) {
-                /*
-                 * This task already died and the timer will never fire.
-                 * In this case, expires is actually the dead value.
-                 */
-        dead:
-                sample_to_timespec(timer->it_clock, timer->it.cpu.expires,
-                                   &itp->it_value);
-                return;
-        }
-
         /*
          * Sample the clock to take the difference with the expiry time.
          */
         if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
                 cpu_clock_sample(timer->it_clock, p, &now);
-                clear_dead = p->exit_state;
         } else {
-                read_lock(&tasklist_lock);
-                if (unlikely(p->sighand == NULL)) {
+                struct sighand_struct *sighand;
+                unsigned long flags;
+
+                /*
+                 * Protect against sighand release/switch in exit/exec and
+                 * also make timer sampling safe if it ends up calling
+                 * thread_group_cputime().
+                 */
+                sighand = lock_task_sighand(p, &flags);
+                if (unlikely(sighand == NULL)) {
                         /*
                          * The process has been reaped.
                          * We can't even collect a sample any more.
                          * Call the timer disarmed, nothing else to do.
                          */
-                        put_task_struct(p);
-                        timer->it.cpu.task = NULL;
                         timer->it.cpu.expires = 0;
-                        read_unlock(&tasklist_lock);
-                        goto dead;
+                        sample_to_timespec(timer->it_clock, timer->it.cpu.expires,
+                                           &itp->it_value);
                 } else {
                         cpu_timer_sample_group(timer->it_clock, p, &now);
-                        clear_dead = (unlikely(p->exit_state) &&
-                                      thread_group_empty(p));
+                        unlock_task_sighand(p, &flags);
                 }
-                read_unlock(&tasklist_lock);
-        }
-
-        if (unlikely(clear_dead)) {
-                /*
-                 * We've noticed that the thread is dead, but
-                 * not yet reaped.  Take this opportunity to
-                 * drop our task ref.
-                 */
-                clear_dead_task(timer, now);
-                goto dead;
         }
 
         if (now < timer->it.cpu.expires) {
@@ -1059,14 +1015,12 @@ static void check_process_timers(struct task_struct *tsk,
  */
 void posix_cpu_timer_schedule(struct k_itimer *timer)
 {
+        struct sighand_struct *sighand;
+        unsigned long flags;
         struct task_struct *p = timer->it.cpu.task;
         unsigned long long now;
 
-        if (unlikely(p == NULL))
-                /*
-                 * The task was cleaned up already, no future firings.
-                 */
-                goto out;
+        WARN_ON_ONCE(p == NULL);
 
         /*
          * Fetch the current sample and update the timer's expiry time.
@@ -1074,49 +1028,45 @@ void posix_cpu_timer_schedule(struct k_itimer *timer)
         if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
                 cpu_clock_sample(timer->it_clock, p, &now);
                 bump_cpu_timer(timer, now);
-                if (unlikely(p->exit_state)) {
-                        clear_dead_task(timer, now);
+                if (unlikely(p->exit_state))
+                        goto out;
+
+                /* Protect timer list r/w in arm_timer() */
+                sighand = lock_task_sighand(p, &flags);
+                if (!sighand)
                         goto out;
-                }
-                read_lock(&tasklist_lock); /* arm_timer needs it.  */
-                spin_lock(&p->sighand->siglock);
         } else {
-                read_lock(&tasklist_lock);
-                if (unlikely(p->sighand == NULL)) {
+                /*
+                 * Protect arm_timer() and timer sampling in case of call to
+                 * thread_group_cputime().
+                 */
+                sighand = lock_task_sighand(p, &flags);
+                if (unlikely(sighand == NULL)) {
                         /*
                          * The process has been reaped.
                          * We can't even collect a sample any more.
                          */
-                        put_task_struct(p);
-                        timer->it.cpu.task = p = NULL;
                         timer->it.cpu.expires = 0;
-                        goto out_unlock;
+                        goto out;
                 } else if (unlikely(p->exit_state) && thread_group_empty(p)) {
-                        /*
-                         * We've noticed that the thread is dead, but
-                         * not yet reaped.  Take this opportunity to
-                         * drop our task ref.
-                         */
-                        cpu_timer_sample_group(timer->it_clock, p, &now);
-                        clear_dead_task(timer, now);
-                        goto out_unlock;
+                        unlock_task_sighand(p, &flags);
+                        /* Optimizations: if the process is dying, no need to rearm */
+                        goto out;
                 }
-                spin_lock(&p->sighand->siglock);
                 cpu_timer_sample_group(timer->it_clock, p, &now);
                 bump_cpu_timer(timer, now);
-                /* Leave the tasklist_lock locked for the call below.  */
+                /* Leave the sighand locked for the call below.  */
         }
 
         /*
          * Now re-arm for the new expiry time.
          */
-        BUG_ON(!irqs_disabled());
+        WARN_ON_ONCE(!irqs_disabled());
         arm_timer(timer);
-        spin_unlock(&p->sighand->siglock);
-
-out_unlock:
-        read_unlock(&tasklist_lock);
+        unlock_task_sighand(p, &flags);
 
+        /* Kick full dynticks CPUs in case they need to tick on the new timer */
+        posix_cpu_timer_kick_nohz();
 out:
         timer->it_overrun_last = timer->it_overrun;
         timer->it_overrun = -1;
@@ -1200,7 +1150,7 @@ void run_posix_cpu_timers(struct task_struct *tsk)
         struct k_itimer *timer, *next;
         unsigned long flags;
 
-        BUG_ON(!irqs_disabled());
+        WARN_ON_ONCE(!irqs_disabled());
 
         /*
          * The fast path checks that there are no expired thread or thread
@@ -1256,13 +1206,6 @@ void run_posix_cpu_timers(struct task_struct *tsk)
                         cpu_timer_fire(timer);
                 spin_unlock(&timer->it_lock);
         }
-
-        /*
-         * In case some timers were rescheduled after the queue got emptied,
-         * wake up full dynticks CPUs.
-         */
-        if (tsk->signal->cputimer.running)
-                posix_cpu_timer_kick_nohz();
 }
 
 /*
@@ -1274,7 +1217,7 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx,
 {
         unsigned long long now;
 
-        BUG_ON(clock_idx == CPUCLOCK_SCHED);
+        WARN_ON_ONCE(clock_idx == CPUCLOCK_SCHED);
         cpu_timer_sample_group(clock_idx, tsk, &now);
 
         if (oldval) {
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 8b93b3770f85..8a1e6e104892 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -319,8 +319,6 @@ asmlinkage void do_softirq(void)
  */
 void irq_enter(void)
 {
-        int cpu = smp_processor_id();
-
         rcu_irq_enter();
         if (is_idle_task(current) && !in_interrupt()) {
                 /*
@@ -328,7 +326,7 @@ void irq_enter(void)
                  * here, as softirq will be serviced on return from interrupt.
                  */
                 local_bh_disable();
-                tick_check_idle(cpu);
+                tick_check_idle();
                 _local_bh_enable();
         }
 
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 9532690daaa9..43780ab5e279 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -538,10 +538,10 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
  * Called from irq_enter() when idle was interrupted to reenable the
  * per cpu device.
  */
-void tick_check_oneshot_broadcast(int cpu)
+void tick_check_oneshot_broadcast_this_cpu(void)
 {
-        if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) {
-                struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
+        if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
+                struct tick_device *td = &__get_cpu_var(tick_cpu_device);
 
                 /*
                  * We might be in the middle of switching over from
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 162b03ab0ad2..20b2fe37d105 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -85,6 +85,7 @@ static void tick_periodic(int cpu)
 
                 do_timer(1);
                 write_sequnlock(&jiffies_lock);
+                update_wall_time();
         }
 
         update_process_times(user_mode(get_irq_regs()));
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 18e71f7fbc2a..8329669b51ec 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -51,7 +51,7 @@ extern void tick_broadcast_switch_to_oneshot(void);
 extern void tick_shutdown_broadcast_oneshot(unsigned int *cpup);
 extern int tick_resume_broadcast_oneshot(struct clock_event_device *bc);
 extern int tick_broadcast_oneshot_active(void);
-extern void tick_check_oneshot_broadcast(int cpu);
+extern void tick_check_oneshot_broadcast_this_cpu(void);
 bool tick_broadcast_oneshot_available(void);
 # else /* BROADCAST */
 static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
@@ -62,7 +62,7 @@ static inline void tick_broadcast_oneshot_control(unsigned long reason) { }
 static inline void tick_broadcast_switch_to_oneshot(void) { }
 static inline void tick_shutdown_broadcast_oneshot(unsigned int *cpup) { }
 static inline int tick_broadcast_oneshot_active(void) { return 0; }
-static inline void tick_check_oneshot_broadcast(int cpu) { }
+static inline void tick_check_oneshot_broadcast_this_cpu(void) { }
 static inline bool tick_broadcast_oneshot_available(void) { return true; }
 # endif /* !BROADCAST */
 
@@ -155,3 +155,4 @@ static inline int tick_device_is_functional(struct clock_event_device *dev)
 #endif
 
 extern void do_timer(unsigned long ticks);
+extern void update_wall_time(void);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index c833249ab0fb..08cb0c3b8ccb 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -86,6 +86,7 @@ static void tick_do_update_jiffies64(ktime_t now)
                 tick_next_period = ktime_add(last_jiffies_update, tick_period);
         }
         write_sequnlock(&jiffies_lock);
+        update_wall_time();
 }
 
 /*
@@ -391,11 +392,9 @@ __setup("nohz=", setup_tick_nohz);
  */
 static void tick_nohz_update_jiffies(ktime_t now)
 {
-        int cpu = smp_processor_id();
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
         unsigned long flags;
 
-        ts->idle_waketime = now;
+        __this_cpu_write(tick_cpu_sched.idle_waketime, now);
 
         local_irq_save(flags);
         tick_do_update_jiffies64(now);
@@ -426,17 +425,15 @@ update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_upda
 
 }
 
-static void tick_nohz_stop_idle(int cpu, ktime_t now)
+static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
 {
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
-
-        update_ts_time_stats(cpu, ts, now, NULL);
+        update_ts_time_stats(smp_processor_id(), ts, now, NULL);
         ts->idle_active = 0;
 
         sched_clock_idle_wakeup_event(0);
 }
 
-static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts)
+static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
 {
         ktime_t now = ktime_get();
 
@@ -754,7 +751,7 @@ static void __tick_nohz_idle_enter(struct tick_sched *ts)
         ktime_t now, expires;
         int cpu = smp_processor_id();
 
-        now = tick_nohz_start_idle(cpu, ts);
+        now = tick_nohz_start_idle(ts);
 
         if (can_stop_idle_tick(cpu, ts)) {
                 int was_stopped = ts->tick_stopped;
@@ -911,8 +908,7 @@ static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
  */
 void tick_nohz_idle_exit(void)
 {
-        int cpu = smp_processor_id();
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
         ktime_t now;
 
         local_irq_disable();
@@ -925,7 +921,7 @@ void tick_nohz_idle_exit(void)
                 now = ktime_get();
 
         if (ts->idle_active)
-                tick_nohz_stop_idle(cpu, now);
+                tick_nohz_stop_idle(ts, now);
 
         if (ts->tick_stopped) {
                 tick_nohz_restart_sched_tick(ts, now);
@@ -1009,12 +1005,10 @@ static void tick_nohz_switch_to_nohz(void)
  * timer and do not touch the other magic bits which need to be done
  * when idle is left.
  */
-static void tick_nohz_kick_tick(int cpu, ktime_t now)
+static void tick_nohz_kick_tick(struct tick_sched *ts, ktime_t now)
 {
 #if 0
         /* Switch back to 2.6.27 behaviour */
-
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
         ktime_t delta;
 
         /*
@@ -1029,36 +1023,36 @@ static void tick_nohz_kick_tick(int cpu, ktime_t now)
 #endif
 }
 
-static inline void tick_check_nohz(int cpu)
+static inline void tick_check_nohz_this_cpu(void)
 {
-        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
         ktime_t now;
 
         if (!ts->idle_active && !ts->tick_stopped)
                 return;
         now = ktime_get();
         if (ts->idle_active)
-                tick_nohz_stop_idle(cpu, now);
+                tick_nohz_stop_idle(ts, now);
         if (ts->tick_stopped) {
                 tick_nohz_update_jiffies(now);
-                tick_nohz_kick_tick(cpu, now);
+                tick_nohz_kick_tick(ts, now);
         }
 }
 
 #else
 
 static inline void tick_nohz_switch_to_nohz(void) { }
-static inline void tick_check_nohz(int cpu) { }
+static inline void tick_check_nohz_this_cpu(void) { }
 
 #endif /* CONFIG_NO_HZ_COMMON */
 
 /*
  * Called from irq_enter to notify about the possible interruption of idle()
  */
-void tick_check_idle(int cpu)
+void tick_check_idle(void)
 {
-        tick_check_oneshot_broadcast(cpu);
-        tick_check_nohz(cpu);
+        tick_check_oneshot_broadcast_this_cpu();
+        tick_check_nohz_this_cpu();
 }
 
 /*
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 87b4f00284c9..0aa4ce81bc16 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -77,7 +77,7 @@ static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm)
         tk->wall_to_monotonic = wtm;
         set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
         tk->offs_real = timespec_to_ktime(tmp);
-        tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tk->tai_offset, 0));
+        tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
 }
 
 static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t)
@@ -90,8 +90,9 @@ static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t)
 }
 
 /**
- * timekeeper_setup_internals - Set up internals to use clocksource clock.
+ * tk_setup_internals - Set up internals to use clocksource clock.
  *
+ * @tk:         The target timekeeper to setup.
  * @clock:              Pointer to clocksource.
  *
  * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
@@ -595,7 +596,7 @@ s32 timekeeping_get_tai_offset(void)
 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
 {
         tk->tai_offset = tai_offset;
-        tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tai_offset, 0));
+        tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
 }
 
 /**
@@ -610,6 +611,7 @@ void timekeeping_set_tai_offset(s32 tai_offset)
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
         write_seqcount_begin(&timekeeper_seq);
         __timekeeping_set_tai_offset(tk, tai_offset);
+        timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
         write_seqcount_end(&timekeeper_seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
         clock_was_set();
@@ -1023,6 +1025,8 @@ static int timekeeping_suspend(void)
                 timekeeping_suspend_time =
                         timespec_add(timekeeping_suspend_time, delta_delta);
         }
+
+        timekeeping_update(tk, TK_MIRROR);
         write_seqcount_end(&timekeeper_seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
@@ -1130,16 +1134,6 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
                  * we can adjust by 1.
                  */
                 error >>= 2;
-                /*
-                 * XXX - In update_wall_time, we round up to the next
-                 * nanosecond, and store the amount rounded up into
-                 * the error. This causes the likely below to be unlikely.
-                 *
-                 * The proper fix is to avoid rounding up by using
-                 * the high precision tk->xtime_nsec instead of
-                 * xtime.tv_nsec everywhere. Fixing this will take some
-                 * time.
-                 */
                 if (likely(error <= interval))
                         adj = 1;
                 else
@@ -1255,7 +1249,7 @@ out_adjust:
 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
 {
         u64 nsecps = (u64)NSEC_PER_SEC << tk->shift;
-        unsigned int action = 0;
+        unsigned int clock_set = 0;
 
         while (tk->xtime_nsec >= nsecps) {
                 int leap;
@@ -1277,11 +1271,10 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
 
                         __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
 
-                        clock_was_set_delayed();
-                        action = TK_CLOCK_WAS_SET;
+                        clock_set = TK_CLOCK_WAS_SET;
                 }
         }
-        return action;
+        return clock_set;
 }
 
 /**
@@ -1294,7 +1287,8 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
  * Returns the unconsumed cycles.
  */
 static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
-                                                u32 shift)
+                                                u32 shift,
+                                                unsigned int *clock_set)
 {
         cycle_t interval = tk->cycle_interval << shift;
         u64 raw_nsecs;
@@ -1308,7 +1302,7 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
         tk->cycle_last += interval;
 
         tk->xtime_nsec += tk->xtime_interval << shift;
-        accumulate_nsecs_to_secs(tk);
+        *clock_set |= accumulate_nsecs_to_secs(tk);
 
         /* Accumulate raw time */
         raw_nsecs = (u64)tk->raw_interval << shift;
@@ -1359,14 +1353,14 @@ static inline void old_vsyscall_fixup(struct timekeeper *tk)
  * update_wall_time - Uses the current clocksource to increment the wall time
  *
  */
-static void update_wall_time(void)
+void update_wall_time(void)
 {
         struct clocksource *clock;
         struct timekeeper *real_tk = &timekeeper;
         struct timekeeper *tk = &shadow_timekeeper;
         cycle_t offset;
         int shift = 0, maxshift;
-        unsigned int action;
+        unsigned int clock_set = 0;
         unsigned long flags;
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
@@ -1401,7 +1395,8 @@ static void update_wall_time(void)
         maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
         shift = min(shift, maxshift);
         while (offset >= tk->cycle_interval) {
-                offset = logarithmic_accumulation(tk, offset, shift);
+                offset = logarithmic_accumulation(tk, offset, shift,
+                                                        &clock_set);
                 if (offset < tk->cycle_interval<<shift)
                         shift--;
         }
@@ -1419,7 +1414,7 @@ static void update_wall_time(void)
          * Finally, make sure that after the rounding
          * xtime_nsec isn't larger than NSEC_PER_SEC
          */
-        action = accumulate_nsecs_to_secs(tk);
+        clock_set |= accumulate_nsecs_to_secs(tk);
 
         write_seqcount_begin(&timekeeper_seq);
         /* Update clock->cycle_last with the new value */
@@ -1435,10 +1430,12 @@ static void update_wall_time(void)
          * updating.
          */
         memcpy(real_tk, tk, sizeof(*tk));
-        timekeeping_update(real_tk, action);
+        timekeeping_update(real_tk, clock_set);
         write_seqcount_end(&timekeeper_seq);
 out:
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+        if (clock_set)
+                clock_was_set();
 }
 
 /**
@@ -1583,7 +1580,6 @@ struct timespec get_monotonic_coarse(void)
 void do_timer(unsigned long ticks)
 {
         jiffies_64 += ticks;
-        update_wall_time();
         calc_global_load(ticks);
 }
 
@@ -1698,12 +1694,14 @@ int do_adjtimex(struct timex *txc)
 
         if (tai != orig_tai) {
                 __timekeeping_set_tai_offset(tk, tai);
-                update_pvclock_gtod(tk, true);
-                clock_was_set_delayed();
+                timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
         }
         write_seqcount_end(&timekeeper_seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
+        if (tai != orig_tai)
+                clock_was_set();
+
         ntp_notify_cmos_timer();
 
         return ret;
@@ -1739,4 +1737,5 @@ void xtime_update(unsigned long ticks)
         write_seqlock(&jiffies_lock);
         do_timer(ticks);
         write_sequnlock(&jiffies_lock);
+        update_wall_time();
 }