timekeeping: Cache optimize struct timekeeper

struct timekeeper is quite badly sorted for the hot readout path. Most
time access functions need to load two cache lines.

Rearrange it so ktime_get() and getnstimeofday() are happy with a
single cache line.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: John Stultz <john.stultz@linaro.org>

1 files changed, 97 insertions, 88 deletions

diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 983d67b388d7..7ca150ad387d 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -32,9 +32,16 @@
 #define TK_MIRROR               (1 << 1)
 #define TK_CLOCK_WAS_SET        (1 << 2)
 
-static struct timekeeper timekeeper;
+/*
+ * The most important data for readout fits into a single 64 byte
+ * cache line.
+ */
+static struct {
+        seqcount_t              seq;
+        struct timekeeper       timekeeper;
+} tk_core ____cacheline_aligned;
+
 static DEFINE_RAW_SPINLOCK(timekeeper_lock);
-static seqcount_t timekeeper_seq;
 static struct timekeeper shadow_timekeeper;
 
 /* flag for if timekeeping is suspended */
@@ -254,7 +261,7 @@ static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
  */
 int pvclock_gtod_register_notifier(struct notifier_block *nb)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long flags;
         int ret;
 
@@ -295,7 +302,8 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action)
         update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
 
         if (action & TK_MIRROR)
-                memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper));
+                memcpy(&shadow_timekeeper, &tk_core.timekeeper,
+                       sizeof(tk_core.timekeeper));
 }
 
 /**
@@ -336,17 +344,17 @@ static void timekeeping_forward_now(struct timekeeper *tk)
  */
 int __getnstimeofday64(struct timespec64 *ts)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long seq;
         s64 nsecs = 0;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 ts->tv_sec = tk->xtime_sec;
                 nsecs = timekeeping_get_ns(tk);
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         ts->tv_nsec = 0;
         timespec64_add_ns(ts, nsecs);
@@ -375,18 +383,18 @@ EXPORT_SYMBOL(getnstimeofday64);
 
 ktime_t ktime_get(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned int seq;
         s64 secs, nsecs;
 
         WARN_ON(timekeeping_suspended);
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
                 secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
                 nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec;
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         return ktime_set(secs, nsecs);
 }
@@ -402,7 +410,7 @@ EXPORT_SYMBOL_GPL(ktime_get);
  */
 void ktime_get_ts64(struct timespec64 *ts)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 tomono;
         s64 nsec;
         unsigned int seq;
@@ -410,12 +418,12 @@ void ktime_get_ts64(struct timespec64 *ts)
         WARN_ON(timekeeping_suspended);
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
                 ts->tv_sec = tk->xtime_sec;
                 nsec = timekeeping_get_ns(tk);
                 tomono = tk->wall_to_monotonic;
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         ts->tv_sec += tomono.tv_sec;
         ts->tv_nsec = 0;
@@ -432,7 +440,7 @@ EXPORT_SYMBOL_GPL(ktime_get_ts64);
  */
 void timekeeping_clocktai(struct timespec *ts)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 ts64;
         unsigned long seq;
         u64 nsecs;
@@ -440,12 +448,12 @@ void timekeeping_clocktai(struct timespec *ts)
         WARN_ON(timekeeping_suspended);
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 ts64.tv_sec = tk->xtime_sec + tk->tai_offset;
                 nsecs = timekeeping_get_ns(tk);
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         ts64.tv_nsec = 0;
         timespec64_add_ns(&ts64, nsecs);
@@ -482,14 +490,14 @@ EXPORT_SYMBOL(ktime_get_clocktai);
  */
 void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long seq;
         s64 nsecs_raw, nsecs_real;
 
         WARN_ON_ONCE(timekeeping_suspended);
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 *ts_raw = timespec64_to_timespec(tk->raw_time);
                 ts_real->tv_sec = tk->xtime_sec;
@@ -498,7 +506,7 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
                 nsecs_raw = timekeeping_get_ns_raw(tk);
                 nsecs_real = timekeeping_get_ns(tk);
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         timespec_add_ns(ts_raw, nsecs_raw);
         timespec_add_ns(ts_real, nsecs_real);
@@ -531,7 +539,7 @@ EXPORT_SYMBOL(do_gettimeofday);
  */
 int do_settimeofday(const struct timespec *tv)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 ts_delta, xt, tmp;
         unsigned long flags;
 
@@ -539,7 +547,7 @@ int do_settimeofday(const struct timespec *tv)
                 return -EINVAL;
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
 
         timekeeping_forward_now(tk);
 
@@ -554,7 +562,7 @@ int do_settimeofday(const struct timespec *tv)
 
         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
 
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
         /* signal hrtimers about time change */
@@ -572,7 +580,7 @@ EXPORT_SYMBOL(do_settimeofday);
  */
 int timekeeping_inject_offset(struct timespec *ts)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long flags;
         struct timespec64 ts64, tmp;
         int ret = 0;
@@ -583,7 +591,7 @@ int timekeeping_inject_offset(struct timespec *ts)
         ts64 = timespec_to_timespec64(*ts);
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
 
         timekeeping_forward_now(tk);
 
@@ -600,7 +608,7 @@ int timekeeping_inject_offset(struct timespec *ts)
 error: /* even if we error out, we forwarded the time, so call update */
         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
 
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
         /* signal hrtimers about time change */
@@ -617,14 +625,14 @@ EXPORT_SYMBOL(timekeeping_inject_offset);
  */
 s32 timekeeping_get_tai_offset(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned int seq;
         s32 ret;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
                 ret = tk->tai_offset;
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         return ret;
 }
@@ -645,14 +653,14 @@ static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
  */
 void timekeeping_set_tai_offset(s32 tai_offset)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long flags;
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
         __timekeeping_set_tai_offset(tk, tai_offset);
         timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
         clock_was_set();
 }
@@ -664,14 +672,14 @@ void timekeeping_set_tai_offset(s32 tai_offset)
  */
 static int change_clocksource(void *data)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct clocksource *new, *old;
         unsigned long flags;
 
         new = (struct clocksource *) data;
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
 
         timekeeping_forward_now(tk);
         /*
@@ -691,7 +699,7 @@ static int change_clocksource(void *data)
         }
         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
 
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
         return 0;
@@ -706,7 +714,7 @@ static int change_clocksource(void *data)
  */
 int timekeeping_notify(struct clocksource *clock)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
 
         if (tk->clock == clock)
                 return 0;
@@ -738,17 +746,17 @@ EXPORT_SYMBOL_GPL(ktime_get_real);
  */
 void getrawmonotonic(struct timespec *ts)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 ts64;
         unsigned long seq;
         s64 nsecs;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
                 nsecs = timekeeping_get_ns_raw(tk);
                 ts64 = tk->raw_time;
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         timespec64_add_ns(&ts64, nsecs);
         *ts = timespec64_to_timespec(ts64);
@@ -760,16 +768,16 @@ EXPORT_SYMBOL(getrawmonotonic);
  */
 int timekeeping_valid_for_hres(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long seq;
         int ret;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         return ret;
 }
@@ -779,16 +787,16 @@ int timekeeping_valid_for_hres(void)
  */
 u64 timekeeping_max_deferment(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long seq;
         u64 ret;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 ret = tk->clock->max_idle_ns;
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         return ret;
 }
@@ -828,7 +836,7 @@ void __weak read_boot_clock(struct timespec *ts)
  */
 void __init timekeeping_init(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct clocksource *clock;
         unsigned long flags;
         struct timespec64 now, boot, tmp;
@@ -854,7 +862,7 @@ void __init timekeeping_init(void)
         }
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
         ntp_init();
 
         clock = clocksource_default_clock();
@@ -875,9 +883,10 @@ void __init timekeeping_init(void)
         tmp.tv_nsec = 0;
         tk_set_sleep_time(tk, tmp);
 
-        memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper));
+        memcpy(&shadow_timekeeper, &tk_core.timekeeper,
+               sizeof(tk_core.timekeeper));
 
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 }
 
@@ -918,7 +927,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
  */
 void timekeeping_inject_sleeptime(struct timespec *delta)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 tmp;
         unsigned long flags;
 
@@ -930,7 +939,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
                 return;
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
 
         timekeeping_forward_now(tk);
 
@@ -939,7 +948,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
 
         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
 
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
         /* signal hrtimers about time change */
@@ -955,7 +964,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
  */
 static void timekeeping_resume(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct clocksource *clock = tk->clock;
         unsigned long flags;
         struct timespec64 ts_new, ts_delta;
@@ -970,7 +979,7 @@ static void timekeeping_resume(void)
         clocksource_resume();
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
 
         /*
          * After system resumes, we need to calculate the suspended time and
@@ -1022,7 +1031,7 @@ static void timekeeping_resume(void)
         tk->ntp_error = 0;
         timekeeping_suspended = 0;
         timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
         touch_softlockup_watchdog();
@@ -1035,7 +1044,7 @@ static void timekeeping_resume(void)
 
 static int timekeeping_suspend(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long flags;
         struct timespec64               delta, delta_delta;
         static struct timespec64        old_delta;
@@ -1053,7 +1062,7 @@ static int timekeeping_suspend(void)
                 persistent_clock_exist = true;
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
         timekeeping_forward_now(tk);
         timekeeping_suspended = 1;
 
@@ -1078,7 +1087,7 @@ static int timekeeping_suspend(void)
         }
 
         timekeeping_update(tk, TK_MIRROR);
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
         clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
@@ -1380,7 +1389,7 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
 void update_wall_time(void)
 {
         struct clocksource *clock;
-        struct timekeeper *real_tk = &timekeeper;
+        struct timekeeper *real_tk = &tk_core.timekeeper;
         struct timekeeper *tk = &shadow_timekeeper;
         cycle_t offset;
         int shift = 0, maxshift;
@@ -1440,7 +1449,7 @@ void update_wall_time(void)
          */
         clock_set |= accumulate_nsecs_to_secs(tk);
 
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
         /* Update clock->cycle_last with the new value */
         clock->cycle_last = tk->cycle_last;
         /*
@@ -1450,12 +1459,12 @@ void update_wall_time(void)
          * requires changes to all other timekeeper usage sites as
          * well, i.e. move the timekeeper pointer getter into the
          * spinlocked/seqcount protected sections. And we trade this
-         * memcpy under the timekeeper_seq against one before we start
+         * memcpy under the tk_core.seq against one before we start
          * updating.
          */
         memcpy(real_tk, tk, sizeof(*tk));
         timekeeping_update(real_tk, clock_set);
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
 out:
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
         if (clock_set)
@@ -1476,7 +1485,7 @@ out:
  */
 void getboottime(struct timespec *ts)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec boottime = {
                 .tv_sec = tk->wall_to_monotonic.tv_sec +
                                 tk->total_sleep_time.tv_sec,
@@ -1499,7 +1508,7 @@ EXPORT_SYMBOL_GPL(getboottime);
  */
 void get_monotonic_boottime(struct timespec *ts)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 tomono, sleep, ret;
         s64 nsec;
         unsigned int seq;
@@ -1507,13 +1516,13 @@ void get_monotonic_boottime(struct timespec *ts)
         WARN_ON(timekeeping_suspended);
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
                 ret.tv_sec = tk->xtime_sec;
                 nsec = timekeeping_get_ns(tk);
                 tomono = tk->wall_to_monotonic;
                 sleep = tk->total_sleep_time;
 
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         ret.tv_sec += tomono.tv_sec + sleep.tv_sec;
         ret.tv_nsec = 0;
@@ -1545,7 +1554,7 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime);
  */
 void monotonic_to_bootbased(struct timespec *ts)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 ts64;
 
         ts64 = timespec_to_timespec64(*ts);
@@ -1556,7 +1565,7 @@ EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
 
 unsigned long get_seconds(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
 
         return tk->xtime_sec;
 }
@@ -1564,22 +1573,22 @@ EXPORT_SYMBOL(get_seconds);
 
 struct timespec __current_kernel_time(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
 
         return timespec64_to_timespec(tk_xtime(tk));
 }
 
 struct timespec current_kernel_time(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 now;
         unsigned long seq;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 now = tk_xtime(tk);
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         return timespec64_to_timespec(now);
 }
@@ -1587,16 +1596,16 @@ EXPORT_SYMBOL(current_kernel_time);
 
 struct timespec get_monotonic_coarse(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 now, mono;
         unsigned long seq;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 now = tk_xtime(tk);
                 mono = tk->wall_to_monotonic;
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec,
                                 now.tv_nsec + mono.tv_nsec);
@@ -1624,19 +1633,19 @@ void do_timer(unsigned long ticks)
 ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot,
                                                         ktime_t *offs_tai)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         struct timespec64 ts;
         ktime_t now;
         unsigned int seq;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 ts = tk_xtime(tk);
                 *offs_real = tk->offs_real;
                 *offs_boot = tk->offs_boot;
                 *offs_tai = tk->offs_tai;
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         now = ktime_set(ts.tv_sec, ts.tv_nsec);
         now = ktime_sub(now, *offs_real);
@@ -1656,13 +1665,13 @@ ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot,
 ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot,
                                                         ktime_t *offs_tai)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         ktime_t now;
         unsigned int seq;
         u64 secs, nsecs;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
 
                 secs = tk->xtime_sec;
                 nsecs = timekeeping_get_ns(tk);
@@ -1670,7 +1679,7 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot,
                 *offs_real = tk->offs_real;
                 *offs_boot = tk->offs_boot;
                 *offs_tai = tk->offs_tai;
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         now = ktime_add_ns(ktime_set(secs, 0), nsecs);
         now = ktime_sub(now, *offs_real);
@@ -1683,14 +1692,14 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot,
  */
 ktime_t ktime_get_monotonic_offset(void)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long seq;
         struct timespec64 wtom;
 
         do {
-                seq = read_seqcount_begin(&timekeeper_seq);
+                seq = read_seqcount_begin(&tk_core.seq);
                 wtom = tk->wall_to_monotonic;
-        } while (read_seqcount_retry(&timekeeper_seq, seq));
+        } while (read_seqcount_retry(&tk_core.seq, seq));
 
         return timespec64_to_ktime(wtom);
 }
@@ -1701,7 +1710,7 @@ EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset);
  */
 int do_adjtimex(struct timex *txc)
 {
-        struct timekeeper *tk = &timekeeper;
+        struct timekeeper *tk = &tk_core.timekeeper;
         unsigned long flags;
         struct timespec64 ts;
         s32 orig_tai, tai;
@@ -1726,7 +1735,7 @@ int do_adjtimex(struct timex *txc)
         getnstimeofday64(&ts);
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
 
         orig_tai = tai = tk->tai_offset;
         ret = __do_adjtimex(txc, &ts, &tai);
@@ -1735,7 +1744,7 @@ int do_adjtimex(struct timex *txc)
                 __timekeeping_set_tai_offset(tk, tai);
                 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
         }
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 
         if (tai != orig_tai)
@@ -1755,11 +1764,11 @@ void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts)
         unsigned long flags;
 
         raw_spin_lock_irqsave(&timekeeper_lock, flags);
-        write_seqcount_begin(&timekeeper_seq);
+        write_seqcount_begin(&tk_core.seq);
 
         __hardpps(phase_ts, raw_ts);
 
-        write_seqcount_end(&timekeeper_seq);
+        write_seqcount_end(&tk_core.seq);
         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
 }
 EXPORT_SYMBOL(hardpps);