aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Makefile25
-rw-r--r--kernel/acct.c10
-rw-r--r--kernel/debug/kdb/kdb_main.c2
-rw-r--r--kernel/delayacct.c16
-rw-r--r--kernel/fork.c5
-rw-r--r--kernel/time/Kconfig4
-rw-r--r--kernel/time/Makefile19
-rw-r--r--kernel/time/hrtimer.c (renamed from kernel/hrtimer.c)125
-rw-r--r--kernel/time/itimer.c (renamed from kernel/itimer.c)0
-rw-r--r--kernel/time/ntp.c15
-rw-r--r--kernel/time/ntp_internal.h2
-rw-r--r--kernel/time/posix-cpu-timers.c (renamed from kernel/posix-cpu-timers.c)0
-rw-r--r--kernel/time/posix-timers.c (renamed from kernel/posix-timers.c)2
-rw-r--r--kernel/time/tick-internal.h2
-rw-r--r--kernel/time/time.c (renamed from kernel/time.c)64
-rw-r--r--kernel/time/timeconst.bc (renamed from kernel/timeconst.bc)0
-rw-r--r--kernel/time/timekeeping.c641
-rw-r--r--kernel/time/timekeeping.h20
-rw-r--r--kernel/time/timekeeping_debug.c2
-rw-r--r--kernel/time/timekeeping_internal.h2
-rw-r--r--kernel/time/timer.c (renamed from kernel/timer.c)34
-rw-r--r--kernel/time/udelay_test.c168
-rw-r--r--kernel/tsacct.c19
23 files changed, 708 insertions, 469 deletions
diff --git a/kernel/Makefile b/kernel/Makefile
index f2a8b6246ce9..973a40cf8068 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -3,12 +3,11 @@
3# 3#
4 4
5obj-y = fork.o exec_domain.o panic.o \ 5obj-y = fork.o exec_domain.o panic.o \
6 cpu.o exit.o itimer.o time.o softirq.o resource.o \ 6 cpu.o exit.o softirq.o resource.o \
7 sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \ 7 sysctl.o sysctl_binary.o capability.o ptrace.o user.o \
8 signal.o sys.o kmod.o workqueue.o pid.o task_work.o \ 8 signal.o sys.o kmod.o workqueue.o pid.o task_work.o \
9 extable.o params.o posix-timers.o \ 9 extable.o params.o \
10 kthread.o sys_ni.o posix-cpu-timers.o \ 10 kthread.o sys_ni.o nsproxy.o \
11 hrtimer.o nsproxy.o \
12 notifier.o ksysfs.o cred.o reboot.o \ 11 notifier.o ksysfs.o cred.o reboot.o \
13 async.o range.o groups.o smpboot.o 12 async.o range.o groups.o smpboot.o
14 13
@@ -110,22 +109,6 @@ targets += config_data.h
110$(obj)/config_data.h: $(obj)/config_data.gz FORCE 109$(obj)/config_data.h: $(obj)/config_data.gz FORCE
111 $(call filechk,ikconfiggz) 110 $(call filechk,ikconfiggz)
112 111
113$(obj)/time.o: $(obj)/timeconst.h
114
115quiet_cmd_hzfile = HZFILE $@
116 cmd_hzfile = echo "hz=$(CONFIG_HZ)" > $@
117
118targets += hz.bc
119$(obj)/hz.bc: $(objtree)/include/config/hz.h FORCE
120 $(call if_changed,hzfile)
121
122quiet_cmd_bc = BC $@
123 cmd_bc = bc -q $(filter-out FORCE,$^) > $@
124
125targets += timeconst.h
126$(obj)/timeconst.h: $(obj)/hz.bc $(src)/timeconst.bc FORCE
127 $(call if_changed,bc)
128
129############################################################################### 112###############################################################################
130# 113#
131# Roll all the X.509 certificates that we can find together and pull them into 114# Roll all the X.509 certificates that we can find together and pull them into
diff --git a/kernel/acct.c b/kernel/acct.c
index 808a86ff229d..a1844f14c6d6 100644
--- a/kernel/acct.c
+++ b/kernel/acct.c
@@ -458,9 +458,7 @@ static void do_acct_process(struct bsd_acct_struct *acct,
458 acct_t ac; 458 acct_t ac;
459 mm_segment_t fs; 459 mm_segment_t fs;
460 unsigned long flim; 460 unsigned long flim;
461 u64 elapsed; 461 u64 elapsed, run_time;
462 u64 run_time;
463 struct timespec uptime;
464 struct tty_struct *tty; 462 struct tty_struct *tty;
465 const struct cred *orig_cred; 463 const struct cred *orig_cred;
466 464
@@ -484,10 +482,8 @@ static void do_acct_process(struct bsd_acct_struct *acct,
484 strlcpy(ac.ac_comm, current->comm, sizeof(ac.ac_comm)); 482 strlcpy(ac.ac_comm, current->comm, sizeof(ac.ac_comm));
485 483
486 /* calculate run_time in nsec*/ 484 /* calculate run_time in nsec*/
487 do_posix_clock_monotonic_gettime(&uptime); 485 run_time = ktime_get_ns();
488 run_time = (u64)uptime.tv_sec*NSEC_PER_SEC + uptime.tv_nsec; 486 run_time -= current->group_leader->start_time;
489 run_time -= (u64)current->group_leader->start_time.tv_sec * NSEC_PER_SEC
490 + current->group_leader->start_time.tv_nsec;
491 /* convert nsec -> AHZ */ 487 /* convert nsec -> AHZ */
492 elapsed = nsec_to_AHZ(run_time); 488 elapsed = nsec_to_AHZ(run_time);
493#if ACCT_VERSION==3 489#if ACCT_VERSION==3
diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c
index 2f7c760305ca..379650b984f8 100644
--- a/kernel/debug/kdb/kdb_main.c
+++ b/kernel/debug/kdb/kdb_main.c
@@ -2472,7 +2472,7 @@ static void kdb_gmtime(struct timespec *tv, struct kdb_tm *tm)
2472static void kdb_sysinfo(struct sysinfo *val) 2472static void kdb_sysinfo(struct sysinfo *val)
2473{ 2473{
2474 struct timespec uptime; 2474 struct timespec uptime;
2475 do_posix_clock_monotonic_gettime(&uptime); 2475 ktime_get_ts(&uptime);
2476 memset(val, 0, sizeof(*val)); 2476 memset(val, 0, sizeof(*val));
2477 val->uptime = uptime.tv_sec; 2477 val->uptime = uptime.tv_sec;
2478 val->loads[0] = avenrun[0]; 2478 val->loads[0] = avenrun[0];
diff --git a/kernel/delayacct.c b/kernel/delayacct.c
index 54996b71e66d..de699f42f9bc 100644
--- a/kernel/delayacct.c
+++ b/kernel/delayacct.c
@@ -46,16 +46,6 @@ void __delayacct_tsk_init(struct task_struct *tsk)
46} 46}
47 47
48/* 48/*
49 * Start accounting for a delay statistic using
50 * its starting timestamp (@start)
51 */
52
53static inline void delayacct_start(struct timespec *start)
54{
55 do_posix_clock_monotonic_gettime(start);
56}
57
58/*
59 * Finish delay accounting for a statistic using 49 * Finish delay accounting for a statistic using
60 * its timestamps (@start, @end), accumalator (@total) and @count 50 * its timestamps (@start, @end), accumalator (@total) and @count
61 */ 51 */
@@ -67,7 +57,7 @@ static void delayacct_end(struct timespec *start, struct timespec *end,
67 s64 ns; 57 s64 ns;
68 unsigned long flags; 58 unsigned long flags;
69 59
70 do_posix_clock_monotonic_gettime(end); 60 ktime_get_ts(end);
71 ts = timespec_sub(*end, *start); 61 ts = timespec_sub(*end, *start);
72 ns = timespec_to_ns(&ts); 62 ns = timespec_to_ns(&ts);
73 if (ns < 0) 63 if (ns < 0)
@@ -81,7 +71,7 @@ static void delayacct_end(struct timespec *start, struct timespec *end,
81 71
82void __delayacct_blkio_start(void) 72void __delayacct_blkio_start(void)
83{ 73{
84 delayacct_start(&current->delays->blkio_start); 74 ktime_get_ts(&current->delays->blkio_start);
85} 75}
86 76
87void __delayacct_blkio_end(void) 77void __delayacct_blkio_end(void)
@@ -169,7 +159,7 @@ __u64 __delayacct_blkio_ticks(struct task_struct *tsk)
169 159
170void __delayacct_freepages_start(void) 160void __delayacct_freepages_start(void)
171{ 161{
172 delayacct_start(&current->delays->freepages_start); 162 ktime_get_ts(&current->delays->freepages_start);
173} 163}
174 164
175void __delayacct_freepages_end(void) 165void __delayacct_freepages_end(void)
diff --git a/kernel/fork.c b/kernel/fork.c
index 6a13c46cd87d..627b7f80afb0 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1262,9 +1262,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1262 1262
1263 posix_cpu_timers_init(p); 1263 posix_cpu_timers_init(p);
1264 1264
1265 do_posix_clock_monotonic_gettime(&p->start_time); 1265 p->start_time = ktime_get_ns();
1266 p->real_start_time = p->start_time; 1266 p->real_start_time = ktime_get_boot_ns();
1267 monotonic_to_bootbased(&p->real_start_time);
1268 p->io_context = NULL; 1267 p->io_context = NULL;
1269 p->audit_context = NULL; 1268 p->audit_context = NULL;
1270 if (clone_flags & CLONE_THREAD) 1269 if (clone_flags & CLONE_THREAD)
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index f448513a45ed..feccfd888732 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -20,10 +20,6 @@ config GENERIC_TIME_VSYSCALL
20config GENERIC_TIME_VSYSCALL_OLD 20config GENERIC_TIME_VSYSCALL_OLD
21 bool 21 bool
22 22
23# ktime_t scalar 64bit nsec representation
24config KTIME_SCALAR
25 bool
26
27# Old style timekeeping 23# Old style timekeeping
28config ARCH_USES_GETTIMEOFFSET 24config ARCH_USES_GETTIMEOFFSET
29 bool 25 bool
diff --git a/kernel/time/Makefile b/kernel/time/Makefile
index 57a413fd0ebf..7347426fa68d 100644
--- a/kernel/time/Makefile
+++ b/kernel/time/Makefile
@@ -1,3 +1,4 @@
1obj-y += time.o timer.o hrtimer.o itimer.o posix-timers.o posix-cpu-timers.o
1obj-y += timekeeping.o ntp.o clocksource.o jiffies.o timer_list.o 2obj-y += timekeeping.o ntp.o clocksource.o jiffies.o timer_list.o
2obj-y += timeconv.o posix-clock.o alarmtimer.o 3obj-y += timeconv.o posix-clock.o alarmtimer.o
3 4
@@ -12,3 +13,21 @@ obj-$(CONFIG_TICK_ONESHOT) += tick-oneshot.o
12obj-$(CONFIG_TICK_ONESHOT) += tick-sched.o 13obj-$(CONFIG_TICK_ONESHOT) += tick-sched.o
13obj-$(CONFIG_TIMER_STATS) += timer_stats.o 14obj-$(CONFIG_TIMER_STATS) += timer_stats.o
14obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o 15obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o
16obj-$(CONFIG_TEST_UDELAY) += udelay_test.o
17
18$(obj)/time.o: $(obj)/timeconst.h
19
20quiet_cmd_hzfile = HZFILE $@
21 cmd_hzfile = echo "hz=$(CONFIG_HZ)" > $@
22
23targets += hz.bc
24$(obj)/hz.bc: $(objtree)/include/config/hz.h FORCE
25 $(call if_changed,hzfile)
26
27quiet_cmd_bc = BC $@
28 cmd_bc = bc -q $(filter-out FORCE,$^) > $@
29
30targets += timeconst.h
31$(obj)/timeconst.h: $(obj)/hz.bc $(src)/timeconst.bc FORCE
32 $(call if_changed,bc)
33
diff --git a/kernel/hrtimer.c b/kernel/time/hrtimer.c
index 3ab28993f6e0..1c2fe7de2842 100644
--- a/kernel/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -54,6 +54,8 @@
54 54
55#include <trace/events/timer.h> 55#include <trace/events/timer.h>
56 56
57#include "timekeeping.h"
58
57/* 59/*
58 * The timer bases: 60 * The timer bases:
59 * 61 *
@@ -114,21 +116,18 @@ static inline int hrtimer_clockid_to_base(clockid_t clock_id)
114 */ 116 */
115static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) 117static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base)
116{ 118{
117 ktime_t xtim, mono, boot; 119 ktime_t xtim, mono, boot, tai;
118 struct timespec xts, tom, slp; 120 ktime_t off_real, off_boot, off_tai;
119 s32 tai_offset;
120 121
121 get_xtime_and_monotonic_and_sleep_offset(&xts, &tom, &slp); 122 mono = ktime_get_update_offsets_tick(&off_real, &off_boot, &off_tai);
122 tai_offset = timekeeping_get_tai_offset(); 123 boot = ktime_add(mono, off_boot);
124 xtim = ktime_add(mono, off_real);
125 tai = ktime_add(xtim, off_tai);
123 126
124 xtim = timespec_to_ktime(xts);
125 mono = ktime_add(xtim, timespec_to_ktime(tom));
126 boot = ktime_add(mono, timespec_to_ktime(slp));
127 base->clock_base[HRTIMER_BASE_REALTIME].softirq_time = xtim; 127 base->clock_base[HRTIMER_BASE_REALTIME].softirq_time = xtim;
128 base->clock_base[HRTIMER_BASE_MONOTONIC].softirq_time = mono; 128 base->clock_base[HRTIMER_BASE_MONOTONIC].softirq_time = mono;
129 base->clock_base[HRTIMER_BASE_BOOTTIME].softirq_time = boot; 129 base->clock_base[HRTIMER_BASE_BOOTTIME].softirq_time = boot;
130 base->clock_base[HRTIMER_BASE_TAI].softirq_time = 130 base->clock_base[HRTIMER_BASE_TAI].softirq_time = tai;
131 ktime_add(xtim, ktime_set(tai_offset, 0));
132} 131}
133 132
134/* 133/*
@@ -264,60 +263,6 @@ lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
264 * too large for inlining: 263 * too large for inlining:
265 */ 264 */
266#if BITS_PER_LONG < 64 265#if BITS_PER_LONG < 64
267# ifndef CONFIG_KTIME_SCALAR
268/**
269 * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable
270 * @kt: addend
271 * @nsec: the scalar nsec value to add
272 *
273 * Returns the sum of kt and nsec in ktime_t format
274 */
275ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
276{
277 ktime_t tmp;
278
279 if (likely(nsec < NSEC_PER_SEC)) {
280 tmp.tv64 = nsec;
281 } else {
282 unsigned long rem = do_div(nsec, NSEC_PER_SEC);
283
284 /* Make sure nsec fits into long */
285 if (unlikely(nsec > KTIME_SEC_MAX))
286 return (ktime_t){ .tv64 = KTIME_MAX };
287
288 tmp = ktime_set((long)nsec, rem);
289 }
290
291 return ktime_add(kt, tmp);
292}
293
294EXPORT_SYMBOL_GPL(ktime_add_ns);
295
296/**
297 * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable
298 * @kt: minuend
299 * @nsec: the scalar nsec value to subtract
300 *
301 * Returns the subtraction of @nsec from @kt in ktime_t format
302 */
303ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec)
304{
305 ktime_t tmp;
306
307 if (likely(nsec < NSEC_PER_SEC)) {
308 tmp.tv64 = nsec;
309 } else {
310 unsigned long rem = do_div(nsec, NSEC_PER_SEC);
311
312 tmp = ktime_set((long)nsec, rem);
313 }
314
315 return ktime_sub(kt, tmp);
316}
317
318EXPORT_SYMBOL_GPL(ktime_sub_ns);
319# endif /* !CONFIG_KTIME_SCALAR */
320
321/* 266/*
322 * Divide a ktime value by a nanosecond value 267 * Divide a ktime value by a nanosecond value
323 */ 268 */
@@ -337,6 +282,7 @@ u64 ktime_divns(const ktime_t kt, s64 div)
337 282
338 return dclc; 283 return dclc;
339} 284}
285EXPORT_SYMBOL_GPL(ktime_divns);
340#endif /* BITS_PER_LONG >= 64 */ 286#endif /* BITS_PER_LONG >= 64 */
341 287
342/* 288/*
@@ -602,6 +548,11 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
602 * timers, we have to check, whether it expires earlier than the timer for 548 * timers, we have to check, whether it expires earlier than the timer for
603 * which the clock event device was armed. 549 * which the clock event device was armed.
604 * 550 *
551 * Note, that in case the state has HRTIMER_STATE_CALLBACK set, no reprogramming
552 * and no expiry check happens. The timer gets enqueued into the rbtree. The
553 * reprogramming and expiry check is done in the hrtimer_interrupt or in the
554 * softirq.
555 *
605 * Called with interrupts disabled and base->cpu_base.lock held 556 * Called with interrupts disabled and base->cpu_base.lock held
606 */ 557 */
607static int hrtimer_reprogram(struct hrtimer *timer, 558static int hrtimer_reprogram(struct hrtimer *timer,
@@ -662,25 +613,13 @@ static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base)
662 base->hres_active = 0; 613 base->hres_active = 0;
663} 614}
664 615
665/*
666 * When High resolution timers are active, try to reprogram. Note, that in case
667 * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry
668 * check happens. The timer gets enqueued into the rbtree. The reprogramming
669 * and expiry check is done in the hrtimer_interrupt or in the softirq.
670 */
671static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
672 struct hrtimer_clock_base *base)
673{
674 return base->cpu_base->hres_active && hrtimer_reprogram(timer, base);
675}
676
677static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) 616static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
678{ 617{
679 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; 618 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
680 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; 619 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
681 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; 620 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
682 621
683 return ktime_get_update_offsets(offs_real, offs_boot, offs_tai); 622 return ktime_get_update_offsets_now(offs_real, offs_boot, offs_tai);
684} 623}
685 624
686/* 625/*
@@ -755,8 +694,8 @@ static inline int hrtimer_is_hres_enabled(void) { return 0; }
755static inline int hrtimer_switch_to_hres(void) { return 0; } 694static inline int hrtimer_switch_to_hres(void) { return 0; }
756static inline void 695static inline void
757hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } 696hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }
758static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, 697static inline int hrtimer_reprogram(struct hrtimer *timer,
759 struct hrtimer_clock_base *base) 698 struct hrtimer_clock_base *base)
760{ 699{
761 return 0; 700 return 0;
762} 701}
@@ -1013,14 +952,25 @@ int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1013 952
1014 leftmost = enqueue_hrtimer(timer, new_base); 953 leftmost = enqueue_hrtimer(timer, new_base);
1015 954
1016 /* 955 if (!leftmost) {
1017 * Only allow reprogramming if the new base is on this CPU. 956 unlock_hrtimer_base(timer, &flags);
1018 * (it might still be on another CPU if the timer was pending) 957 return ret;
1019 * 958 }
1020 * XXX send_remote_softirq() ? 959
1021 */ 960 if (!hrtimer_is_hres_active(timer)) {
1022 if (leftmost && new_base->cpu_base == &__get_cpu_var(hrtimer_bases) 961 /*
1023 && hrtimer_enqueue_reprogram(timer, new_base)) { 962 * Kick to reschedule the next tick to handle the new timer
963 * on dynticks target.
964 */
965 wake_up_nohz_cpu(new_base->cpu_base->cpu);
966 } else if (new_base->cpu_base == &__get_cpu_var(hrtimer_bases) &&
967 hrtimer_reprogram(timer, new_base)) {
968 /*
969 * Only allow reprogramming if the new base is on this CPU.
970 * (it might still be on another CPU if the timer was pending)
971 *
972 * XXX send_remote_softirq() ?
973 */
1024 if (wakeup) { 974 if (wakeup) {
1025 /* 975 /*
1026 * We need to drop cpu_base->lock to avoid a 976 * We need to drop cpu_base->lock to avoid a
@@ -1680,6 +1630,7 @@ static void init_hrtimers_cpu(int cpu)
1680 timerqueue_init_head(&cpu_base->clock_base[i].active); 1630 timerqueue_init_head(&cpu_base->clock_base[i].active);
1681 } 1631 }
1682 1632
1633 cpu_base->cpu = cpu;
1683 hrtimer_init_hres(cpu_base); 1634 hrtimer_init_hres(cpu_base);
1684} 1635}
1685 1636
diff --git a/kernel/itimer.c b/kernel/time/itimer.c
index 8d262b467573..8d262b467573 100644
--- a/kernel/itimer.c
+++ b/kernel/time/itimer.c
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 33db43a39515..87a346fd6d61 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -466,7 +466,8 @@ static DECLARE_DELAYED_WORK(sync_cmos_work, sync_cmos_clock);
466 466
467static void sync_cmos_clock(struct work_struct *work) 467static void sync_cmos_clock(struct work_struct *work)
468{ 468{
469 struct timespec now, next; 469 struct timespec64 now;
470 struct timespec next;
470 int fail = 1; 471 int fail = 1;
471 472
472 /* 473 /*
@@ -485,9 +486,9 @@ static void sync_cmos_clock(struct work_struct *work)
485 return; 486 return;
486 } 487 }
487 488
488 getnstimeofday(&now); 489 getnstimeofday64(&now);
489 if (abs(now.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec * 5) { 490 if (abs(now.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec * 5) {
490 struct timespec adjust = now; 491 struct timespec adjust = timespec64_to_timespec(now);
491 492
492 fail = -ENODEV; 493 fail = -ENODEV;
493 if (persistent_clock_is_local) 494 if (persistent_clock_is_local)
@@ -531,7 +532,7 @@ void ntp_notify_cmos_timer(void) { }
531/* 532/*
532 * Propagate a new txc->status value into the NTP state: 533 * Propagate a new txc->status value into the NTP state:
533 */ 534 */
534static inline void process_adj_status(struct timex *txc, struct timespec *ts) 535static inline void process_adj_status(struct timex *txc, struct timespec64 *ts)
535{ 536{
536 if ((time_status & STA_PLL) && !(txc->status & STA_PLL)) { 537 if ((time_status & STA_PLL) && !(txc->status & STA_PLL)) {
537 time_state = TIME_OK; 538 time_state = TIME_OK;
@@ -554,7 +555,7 @@ static inline void process_adj_status(struct timex *txc, struct timespec *ts)
554 555
555 556
556static inline void process_adjtimex_modes(struct timex *txc, 557static inline void process_adjtimex_modes(struct timex *txc,
557 struct timespec *ts, 558 struct timespec64 *ts,
558 s32 *time_tai) 559 s32 *time_tai)
559{ 560{
560 if (txc->modes & ADJ_STATUS) 561 if (txc->modes & ADJ_STATUS)
@@ -640,7 +641,7 @@ int ntp_validate_timex(struct timex *txc)
640 * adjtimex mainly allows reading (and writing, if superuser) of 641 * adjtimex mainly allows reading (and writing, if superuser) of
641 * kernel time-keeping variables. used by xntpd. 642 * kernel time-keeping variables. used by xntpd.
642 */ 643 */
643int __do_adjtimex(struct timex *txc, struct timespec *ts, s32 *time_tai) 644int __do_adjtimex(struct timex *txc, struct timespec64 *ts, s32 *time_tai)
644{ 645{
645 int result; 646 int result;
646 647
@@ -684,7 +685,7 @@ int __do_adjtimex(struct timex *txc, struct timespec *ts, s32 *time_tai)
684 /* fill PPS status fields */ 685 /* fill PPS status fields */
685 pps_fill_timex(txc); 686 pps_fill_timex(txc);
686 687
687 txc->time.tv_sec = ts->tv_sec; 688 txc->time.tv_sec = (time_t)ts->tv_sec;
688 txc->time.tv_usec = ts->tv_nsec; 689 txc->time.tv_usec = ts->tv_nsec;
689 if (!(time_status & STA_NANO)) 690 if (!(time_status & STA_NANO))
690 txc->time.tv_usec /= NSEC_PER_USEC; 691 txc->time.tv_usec /= NSEC_PER_USEC;
diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h
index 1950cb4ca2a4..bbd102ad9df7 100644
--- a/kernel/time/ntp_internal.h
+++ b/kernel/time/ntp_internal.h
@@ -7,6 +7,6 @@ extern void ntp_clear(void);
7extern u64 ntp_tick_length(void); 7extern u64 ntp_tick_length(void);
8extern int second_overflow(unsigned long secs); 8extern int second_overflow(unsigned long secs);
9extern int ntp_validate_timex(struct timex *); 9extern int ntp_validate_timex(struct timex *);
10extern int __do_adjtimex(struct timex *, struct timespec *, s32 *); 10extern int __do_adjtimex(struct timex *, struct timespec64 *, s32 *);
11extern void __hardpps(const struct timespec *, const struct timespec *); 11extern void __hardpps(const struct timespec *, const struct timespec *);
12#endif /* _LINUX_NTP_INTERNAL_H */ 12#endif /* _LINUX_NTP_INTERNAL_H */
diff --git a/kernel/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 3b8946416a5f..3b8946416a5f 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
diff --git a/kernel/posix-timers.c b/kernel/time/posix-timers.c
index 424c2d4265c9..42b463ad90f2 100644
--- a/kernel/posix-timers.c
+++ b/kernel/time/posix-timers.c
@@ -49,6 +49,8 @@
49#include <linux/export.h> 49#include <linux/export.h>
50#include <linux/hashtable.h> 50#include <linux/hashtable.h>
51 51
52#include "timekeeping.h"
53
52/* 54/*
53 * Management arrays for POSIX timers. Timers are now kept in static hash table 55 * Management arrays for POSIX timers. Timers are now kept in static hash table
54 * with 512 entries. 56 * with 512 entries.
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 7ab92b19965a..c19c1d84b6f3 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -4,6 +4,8 @@
4#include <linux/hrtimer.h> 4#include <linux/hrtimer.h>
5#include <linux/tick.h> 5#include <linux/tick.h>
6 6
7#include "timekeeping.h"
8
7extern seqlock_t jiffies_lock; 9extern seqlock_t jiffies_lock;
8 10
9#define CS_NAME_LEN 32 11#define CS_NAME_LEN 32
diff --git a/kernel/time.c b/kernel/time/time.c
index 7c7964c33ae7..f0294ba14634 100644
--- a/kernel/time.c
+++ b/kernel/time/time.c
@@ -42,6 +42,7 @@
42#include <asm/unistd.h> 42#include <asm/unistd.h>
43 43
44#include "timeconst.h" 44#include "timeconst.h"
45#include "timekeeping.h"
45 46
46/* 47/*
47 * The timezone where the local system is located. Used as a default by some 48 * The timezone where the local system is located. Used as a default by some
@@ -420,6 +421,68 @@ struct timeval ns_to_timeval(const s64 nsec)
420} 421}
421EXPORT_SYMBOL(ns_to_timeval); 422EXPORT_SYMBOL(ns_to_timeval);
422 423
424#if BITS_PER_LONG == 32
425/**
426 * set_normalized_timespec - set timespec sec and nsec parts and normalize
427 *
428 * @ts: pointer to timespec variable to be set
429 * @sec: seconds to set
430 * @nsec: nanoseconds to set
431 *
432 * Set seconds and nanoseconds field of a timespec variable and
433 * normalize to the timespec storage format
434 *
435 * Note: The tv_nsec part is always in the range of
436 * 0 <= tv_nsec < NSEC_PER_SEC
437 * For negative values only the tv_sec field is negative !
438 */
439void set_normalized_timespec64(struct timespec64 *ts, time64_t sec, s64 nsec)
440{
441 while (nsec >= NSEC_PER_SEC) {
442 /*
443 * The following asm() prevents the compiler from
444 * optimising this loop into a modulo operation. See
445 * also __iter_div_u64_rem() in include/linux/time.h
446 */
447 asm("" : "+rm"(nsec));
448 nsec -= NSEC_PER_SEC;
449 ++sec;
450 }
451 while (nsec < 0) {
452 asm("" : "+rm"(nsec));
453 nsec += NSEC_PER_SEC;
454 --sec;
455 }
456 ts->tv_sec = sec;
457 ts->tv_nsec = nsec;
458}
459EXPORT_SYMBOL(set_normalized_timespec64);
460
461/**
462 * ns_to_timespec64 - Convert nanoseconds to timespec64
463 * @nsec: the nanoseconds value to be converted
464 *
465 * Returns the timespec64 representation of the nsec parameter.
466 */
467struct timespec64 ns_to_timespec64(const s64 nsec)
468{
469 struct timespec64 ts;
470 s32 rem;
471
472 if (!nsec)
473 return (struct timespec64) {0, 0};
474
475 ts.tv_sec = div_s64_rem(nsec, NSEC_PER_SEC, &rem);
476 if (unlikely(rem < 0)) {
477 ts.tv_sec--;
478 rem += NSEC_PER_SEC;
479 }
480 ts.tv_nsec = rem;
481
482 return ts;
483}
484EXPORT_SYMBOL(ns_to_timespec64);
485#endif
423/* 486/*
424 * When we convert to jiffies then we interpret incoming values 487 * When we convert to jiffies then we interpret incoming values
425 * the following way: 488 * the following way:
@@ -694,6 +757,7 @@ unsigned long nsecs_to_jiffies(u64 n)
694{ 757{
695 return (unsigned long)nsecs_to_jiffies64(n); 758 return (unsigned long)nsecs_to_jiffies64(n);
696} 759}
760EXPORT_SYMBOL_GPL(nsecs_to_jiffies);
697 761
698/* 762/*
699 * Add two timespec values and do a safety check for overflow. 763 * Add two timespec values and do a safety check for overflow.
diff --git a/kernel/timeconst.bc b/kernel/time/timeconst.bc
index 511bdf2cafda..511bdf2cafda 100644
--- a/kernel/timeconst.bc
+++ b/kernel/time/timeconst.bc
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 32d8d6aaedb8..f7378eaebe67 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -32,9 +32,16 @@
32#define TK_MIRROR (1 << 1) 32#define TK_MIRROR (1 << 1)
33#define TK_CLOCK_WAS_SET (1 << 2) 33#define TK_CLOCK_WAS_SET (1 << 2)
34 34
35static struct timekeeper timekeeper; 35/*
36 * The most important data for readout fits into a single 64 byte
37 * cache line.
38 */
39static struct {
40 seqcount_t seq;
41 struct timekeeper timekeeper;
42} tk_core ____cacheline_aligned;
43
36static DEFINE_RAW_SPINLOCK(timekeeper_lock); 44static DEFINE_RAW_SPINLOCK(timekeeper_lock);
37static seqcount_t timekeeper_seq;
38static struct timekeeper shadow_timekeeper; 45static struct timekeeper shadow_timekeeper;
39 46
40/* flag for if timekeeping is suspended */ 47/* flag for if timekeeping is suspended */
@@ -51,43 +58,52 @@ static inline void tk_normalize_xtime(struct timekeeper *tk)
51 } 58 }
52} 59}
53 60
54static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) 61static inline struct timespec64 tk_xtime(struct timekeeper *tk)
62{
63 struct timespec64 ts;
64
65 ts.tv_sec = tk->xtime_sec;
66 ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
67 return ts;
68}
69
70static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
55{ 71{
56 tk->xtime_sec = ts->tv_sec; 72 tk->xtime_sec = ts->tv_sec;
57 tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift; 73 tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift;
58} 74}
59 75
60static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts) 76static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
61{ 77{
62 tk->xtime_sec += ts->tv_sec; 78 tk->xtime_sec += ts->tv_sec;
63 tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift; 79 tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift;
64 tk_normalize_xtime(tk); 80 tk_normalize_xtime(tk);
65} 81}
66 82
67static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm) 83static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
68{ 84{
69 struct timespec tmp; 85 struct timespec64 tmp;
70 86
71 /* 87 /*
72 * Verify consistency of: offset_real = -wall_to_monotonic 88 * Verify consistency of: offset_real = -wall_to_monotonic
73 * before modifying anything 89 * before modifying anything
74 */ 90 */
75 set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec, 91 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
76 -tk->wall_to_monotonic.tv_nsec); 92 -tk->wall_to_monotonic.tv_nsec);
77 WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64); 93 WARN_ON_ONCE(tk->offs_real.tv64 != timespec64_to_ktime(tmp).tv64);
78 tk->wall_to_monotonic = wtm; 94 tk->wall_to_monotonic = wtm;
79 set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec); 95 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
80 tk->offs_real = timespec_to_ktime(tmp); 96 tk->offs_real = timespec64_to_ktime(tmp);
81 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)); 97 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
82} 98}
83 99
84static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t) 100static void tk_set_sleep_time(struct timekeeper *tk, struct timespec64 t)
85{ 101{
86 /* Verify consistency before modifying */ 102 /* Verify consistency before modifying */
87 WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64); 103 WARN_ON_ONCE(tk->offs_boot.tv64 != timespec64_to_ktime(tk->total_sleep_time).tv64);
88 104
89 tk->total_sleep_time = t; 105 tk->total_sleep_time = t;
90 tk->offs_boot = timespec_to_ktime(t); 106 tk->offs_boot = timespec64_to_ktime(t);
91} 107}
92 108
93/** 109/**
@@ -153,16 +169,10 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
153/* Timekeeper helper functions. */ 169/* Timekeeper helper functions. */
154 170
155#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET 171#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
156u32 (*arch_gettimeoffset)(void); 172static u32 default_arch_gettimeoffset(void) { return 0; }
157 173u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
158u32 get_arch_timeoffset(void)
159{
160 if (likely(arch_gettimeoffset))
161 return arch_gettimeoffset();
162 return 0;
163}
164#else 174#else
165static inline u32 get_arch_timeoffset(void) { return 0; } 175static inline u32 arch_gettimeoffset(void) { return 0; }
166#endif 176#endif
167 177
168static inline s64 timekeeping_get_ns(struct timekeeper *tk) 178static inline s64 timekeeping_get_ns(struct timekeeper *tk)
@@ -182,7 +192,7 @@ static inline s64 timekeeping_get_ns(struct timekeeper *tk)
182 nsec >>= tk->shift; 192 nsec >>= tk->shift;
183 193
184 /* If arch requires, add in get_arch_timeoffset() */ 194 /* If arch requires, add in get_arch_timeoffset() */
185 return nsec + get_arch_timeoffset(); 195 return nsec + arch_gettimeoffset();
186} 196}
187 197
188static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) 198static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
@@ -202,9 +212,43 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
202 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); 212 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
203 213
204 /* If arch requires, add in get_arch_timeoffset() */ 214 /* If arch requires, add in get_arch_timeoffset() */
205 return nsec + get_arch_timeoffset(); 215 return nsec + arch_gettimeoffset();
206} 216}
207 217
218#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
219
220static inline void update_vsyscall(struct timekeeper *tk)
221{
222 struct timespec xt;
223
224 xt = tk_xtime(tk);
225 update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult);
226}
227
228static inline void old_vsyscall_fixup(struct timekeeper *tk)
229{
230 s64 remainder;
231
232 /*
233 * Store only full nanoseconds into xtime_nsec after rounding
234 * it up and add the remainder to the error difference.
235 * XXX - This is necessary to avoid small 1ns inconsistnecies caused
236 * by truncating the remainder in vsyscalls. However, it causes
237 * additional work to be done in timekeeping_adjust(). Once
238 * the vsyscall implementations are converted to use xtime_nsec
239 * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD
240 * users are removed, this can be killed.
241 */
242 remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1);
243 tk->xtime_nsec -= remainder;
244 tk->xtime_nsec += 1ULL << tk->shift;
245 tk->ntp_error += remainder << tk->ntp_error_shift;
246 tk->ntp_error -= (1ULL << tk->shift) << tk->ntp_error_shift;
247}
248#else
249#define old_vsyscall_fixup(tk)
250#endif
251
208static RAW_NOTIFIER_HEAD(pvclock_gtod_chain); 252static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
209 253
210static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) 254static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
@@ -217,7 +261,7 @@ static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
217 */ 261 */
218int pvclock_gtod_register_notifier(struct notifier_block *nb) 262int pvclock_gtod_register_notifier(struct notifier_block *nb)
219{ 263{
220 struct timekeeper *tk = &timekeeper; 264 struct timekeeper *tk = &tk_core.timekeeper;
221 unsigned long flags; 265 unsigned long flags;
222 int ret; 266 int ret;
223 267
@@ -247,6 +291,26 @@ int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
247} 291}
248EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier); 292EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
249 293
294/*
295 * Update the ktime_t based scalar nsec members of the timekeeper
296 */
297static inline void tk_update_ktime_data(struct timekeeper *tk)
298{
299 s64 nsec;
300
301 /*
302 * The xtime based monotonic readout is:
303 * nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
304 * The ktime based monotonic readout is:
305 * nsec = base_mono + now();
306 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
307 */
308 nsec = (s64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
309 nsec *= NSEC_PER_SEC;
310 nsec += tk->wall_to_monotonic.tv_nsec;
311 tk->base_mono = ns_to_ktime(nsec);
312}
313
250/* must hold timekeeper_lock */ 314/* must hold timekeeper_lock */
251static void timekeeping_update(struct timekeeper *tk, unsigned int action) 315static void timekeeping_update(struct timekeeper *tk, unsigned int action)
252{ 316{
@@ -257,8 +321,11 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action)
257 update_vsyscall(tk); 321 update_vsyscall(tk);
258 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); 322 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
259 323
324 tk_update_ktime_data(tk);
325
260 if (action & TK_MIRROR) 326 if (action & TK_MIRROR)
261 memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper)); 327 memcpy(&shadow_timekeeper, &tk_core.timekeeper,
328 sizeof(tk_core.timekeeper));
262} 329}
263 330
264/** 331/**
@@ -282,37 +349,37 @@ static void timekeeping_forward_now(struct timekeeper *tk)
282 tk->xtime_nsec += cycle_delta * tk->mult; 349 tk->xtime_nsec += cycle_delta * tk->mult;
283 350
284 /* If arch requires, add in get_arch_timeoffset() */ 351 /* If arch requires, add in get_arch_timeoffset() */
285 tk->xtime_nsec += (u64)get_arch_timeoffset() << tk->shift; 352 tk->xtime_nsec += (u64)arch_gettimeoffset() << tk->shift;
286 353
287 tk_normalize_xtime(tk); 354 tk_normalize_xtime(tk);
288 355
289 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); 356 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
290 timespec_add_ns(&tk->raw_time, nsec); 357 timespec64_add_ns(&tk->raw_time, nsec);
291} 358}
292 359
293/** 360/**
294 * __getnstimeofday - Returns the time of day in a timespec. 361 * __getnstimeofday64 - Returns the time of day in a timespec64.
295 * @ts: pointer to the timespec to be set 362 * @ts: pointer to the timespec to be set
296 * 363 *
297 * Updates the time of day in the timespec. 364 * Updates the time of day in the timespec.
298 * Returns 0 on success, or -ve when suspended (timespec will be undefined). 365 * Returns 0 on success, or -ve when suspended (timespec will be undefined).
299 */ 366 */
300int __getnstimeofday(struct timespec *ts) 367int __getnstimeofday64(struct timespec64 *ts)
301{ 368{
302 struct timekeeper *tk = &timekeeper; 369 struct timekeeper *tk = &tk_core.timekeeper;
303 unsigned long seq; 370 unsigned long seq;
304 s64 nsecs = 0; 371 s64 nsecs = 0;
305 372
306 do { 373 do {
307 seq = read_seqcount_begin(&timekeeper_seq); 374 seq = read_seqcount_begin(&tk_core.seq);
308 375
309 ts->tv_sec = tk->xtime_sec; 376 ts->tv_sec = tk->xtime_sec;
310 nsecs = timekeeping_get_ns(tk); 377 nsecs = timekeeping_get_ns(tk);
311 378
312 } while (read_seqcount_retry(&timekeeper_seq, seq)); 379 } while (read_seqcount_retry(&tk_core.seq, seq));
313 380
314 ts->tv_nsec = 0; 381 ts->tv_nsec = 0;
315 timespec_add_ns(ts, nsecs); 382 timespec64_add_ns(ts, nsecs);
316 383
317 /* 384 /*
318 * Do not bail out early, in case there were callers still using 385 * Do not bail out early, in case there were callers still using
@@ -322,72 +389,117 @@ int __getnstimeofday(struct timespec *ts)
322 return -EAGAIN; 389 return -EAGAIN;
323 return 0; 390 return 0;
324} 391}
325EXPORT_SYMBOL(__getnstimeofday); 392EXPORT_SYMBOL(__getnstimeofday64);
326 393
327/** 394/**
328 * getnstimeofday - Returns the time of day in a timespec. 395 * getnstimeofday64 - Returns the time of day in a timespec64.
329 * @ts: pointer to the timespec to be set 396 * @ts: pointer to the timespec to be set
330 * 397 *
331 * Returns the time of day in a timespec (WARN if suspended). 398 * Returns the time of day in a timespec (WARN if suspended).
332 */ 399 */
333void getnstimeofday(struct timespec *ts) 400void getnstimeofday64(struct timespec64 *ts)
334{ 401{
335 WARN_ON(__getnstimeofday(ts)); 402 WARN_ON(__getnstimeofday64(ts));
336} 403}
337EXPORT_SYMBOL(getnstimeofday); 404EXPORT_SYMBOL(getnstimeofday64);
338 405
339ktime_t ktime_get(void) 406ktime_t ktime_get(void)
340{ 407{
341 struct timekeeper *tk = &timekeeper; 408 struct timekeeper *tk = &tk_core.timekeeper;
342 unsigned int seq; 409 unsigned int seq;
343 s64 secs, nsecs; 410 ktime_t base;
411 s64 nsecs;
344 412
345 WARN_ON(timekeeping_suspended); 413 WARN_ON(timekeeping_suspended);
346 414
347 do { 415 do {
348 seq = read_seqcount_begin(&timekeeper_seq); 416 seq = read_seqcount_begin(&tk_core.seq);
349 secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; 417 base = tk->base_mono;
350 nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; 418 nsecs = timekeeping_get_ns(tk);
351 419
352 } while (read_seqcount_retry(&timekeeper_seq, seq)); 420 } while (read_seqcount_retry(&tk_core.seq, seq));
353 /* 421
354 * Use ktime_set/ktime_add_ns to create a proper ktime on 422 return ktime_add_ns(base, nsecs);
355 * 32-bit architectures without CONFIG_KTIME_SCALAR.
356 */
357 return ktime_add_ns(ktime_set(secs, 0), nsecs);
358} 423}
359EXPORT_SYMBOL_GPL(ktime_get); 424EXPORT_SYMBOL_GPL(ktime_get);
360 425
426static ktime_t *offsets[TK_OFFS_MAX] = {
427 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
428 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
429 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
430};
431
432ktime_t ktime_get_with_offset(enum tk_offsets offs)
433{
434 struct timekeeper *tk = &tk_core.timekeeper;
435 unsigned int seq;
436 ktime_t base, *offset = offsets[offs];
437 s64 nsecs;
438
439 WARN_ON(timekeeping_suspended);
440
441 do {
442 seq = read_seqcount_begin(&tk_core.seq);
443 base = ktime_add(tk->base_mono, *offset);
444 nsecs = timekeeping_get_ns(tk);
445
446 } while (read_seqcount_retry(&tk_core.seq, seq));
447
448 return ktime_add_ns(base, nsecs);
449
450}
451EXPORT_SYMBOL_GPL(ktime_get_with_offset);
452
361/** 453/**
362 * ktime_get_ts - get the monotonic clock in timespec format 454 * ktime_mono_to_any() - convert mononotic time to any other time
455 * @tmono: time to convert.
456 * @offs: which offset to use
457 */
458ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
459{
460 ktime_t *offset = offsets[offs];
461 unsigned long seq;
462 ktime_t tconv;
463
464 do {
465 seq = read_seqcount_begin(&tk_core.seq);
466 tconv = ktime_add(tmono, *offset);
467 } while (read_seqcount_retry(&tk_core.seq, seq));
468
469 return tconv;
470}
471EXPORT_SYMBOL_GPL(ktime_mono_to_any);
472
473/**
474 * ktime_get_ts64 - get the monotonic clock in timespec64 format
363 * @ts: pointer to timespec variable 475 * @ts: pointer to timespec variable
364 * 476 *
365 * The function calculates the monotonic clock from the realtime 477 * The function calculates the monotonic clock from the realtime
366 * clock and the wall_to_monotonic offset and stores the result 478 * clock and the wall_to_monotonic offset and stores the result
367 * in normalized timespec format in the variable pointed to by @ts. 479 * in normalized timespec format in the variable pointed to by @ts.
368 */ 480 */
369void ktime_get_ts(struct timespec *ts) 481void ktime_get_ts64(struct timespec64 *ts)
370{ 482{
371 struct timekeeper *tk = &timekeeper; 483 struct timekeeper *tk = &tk_core.timekeeper;
372 struct timespec tomono; 484 struct timespec64 tomono;
373 s64 nsec; 485 s64 nsec;
374 unsigned int seq; 486 unsigned int seq;
375 487
376 WARN_ON(timekeeping_suspended); 488 WARN_ON(timekeeping_suspended);
377 489
378 do { 490 do {
379 seq = read_seqcount_begin(&timekeeper_seq); 491 seq = read_seqcount_begin(&tk_core.seq);
380 ts->tv_sec = tk->xtime_sec; 492 ts->tv_sec = tk->xtime_sec;
381 nsec = timekeeping_get_ns(tk); 493 nsec = timekeeping_get_ns(tk);
382 tomono = tk->wall_to_monotonic; 494 tomono = tk->wall_to_monotonic;
383 495
384 } while (read_seqcount_retry(&timekeeper_seq, seq)); 496 } while (read_seqcount_retry(&tk_core.seq, seq));
385 497
386 ts->tv_sec += tomono.tv_sec; 498 ts->tv_sec += tomono.tv_sec;
387 ts->tv_nsec = 0; 499 ts->tv_nsec = 0;
388 timespec_add_ns(ts, nsec + tomono.tv_nsec); 500 timespec64_add_ns(ts, nsec + tomono.tv_nsec);
389} 501}
390EXPORT_SYMBOL_GPL(ktime_get_ts); 502EXPORT_SYMBOL_GPL(ktime_get_ts64);
391 503
392 504
393/** 505/**
@@ -398,41 +510,28 @@ EXPORT_SYMBOL_GPL(ktime_get_ts);
398 */ 510 */
399void timekeeping_clocktai(struct timespec *ts) 511void timekeeping_clocktai(struct timespec *ts)
400{ 512{
401 struct timekeeper *tk = &timekeeper; 513 struct timekeeper *tk = &tk_core.timekeeper;
514 struct timespec64 ts64;
402 unsigned long seq; 515 unsigned long seq;
403 u64 nsecs; 516 u64 nsecs;
404 517
405 WARN_ON(timekeeping_suspended); 518 WARN_ON(timekeeping_suspended);
406 519
407 do { 520 do {
408 seq = read_seqcount_begin(&timekeeper_seq); 521 seq = read_seqcount_begin(&tk_core.seq);
409 522
410 ts->tv_sec = tk->xtime_sec + tk->tai_offset; 523 ts64.tv_sec = tk->xtime_sec + tk->tai_offset;
411 nsecs = timekeeping_get_ns(tk); 524 nsecs = timekeeping_get_ns(tk);
412 525
413 } while (read_seqcount_retry(&timekeeper_seq, seq)); 526 } while (read_seqcount_retry(&tk_core.seq, seq));
414 527
415 ts->tv_nsec = 0; 528 ts64.tv_nsec = 0;
416 timespec_add_ns(ts, nsecs); 529 timespec64_add_ns(&ts64, nsecs);
530 *ts = timespec64_to_timespec(ts64);
417 531
418} 532}
419EXPORT_SYMBOL(timekeeping_clocktai); 533EXPORT_SYMBOL(timekeeping_clocktai);
420 534
421
422/**
423 * ktime_get_clocktai - Returns the TAI time of day in a ktime
424 *
425 * Returns the time of day in a ktime.
426 */
427ktime_t ktime_get_clocktai(void)
428{
429 struct timespec ts;
430
431 timekeeping_clocktai(&ts);
432 return timespec_to_ktime(ts);
433}
434EXPORT_SYMBOL(ktime_get_clocktai);
435
436#ifdef CONFIG_NTP_PPS 535#ifdef CONFIG_NTP_PPS
437 536
438/** 537/**
@@ -446,23 +545,23 @@ EXPORT_SYMBOL(ktime_get_clocktai);
446 */ 545 */
447void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) 546void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
448{ 547{
449 struct timekeeper *tk = &timekeeper; 548 struct timekeeper *tk = &tk_core.timekeeper;
450 unsigned long seq; 549 unsigned long seq;
451 s64 nsecs_raw, nsecs_real; 550 s64 nsecs_raw, nsecs_real;
452 551
453 WARN_ON_ONCE(timekeeping_suspended); 552 WARN_ON_ONCE(timekeeping_suspended);
454 553
455 do { 554 do {
456 seq = read_seqcount_begin(&timekeeper_seq); 555 seq = read_seqcount_begin(&tk_core.seq);
457 556
458 *ts_raw = tk->raw_time; 557 *ts_raw = timespec64_to_timespec(tk->raw_time);
459 ts_real->tv_sec = tk->xtime_sec; 558 ts_real->tv_sec = tk->xtime_sec;
460 ts_real->tv_nsec = 0; 559 ts_real->tv_nsec = 0;
461 560
462 nsecs_raw = timekeeping_get_ns_raw(tk); 561 nsecs_raw = timekeeping_get_ns_raw(tk);
463 nsecs_real = timekeeping_get_ns(tk); 562 nsecs_real = timekeeping_get_ns(tk);
464 563
465 } while (read_seqcount_retry(&timekeeper_seq, seq)); 564 } while (read_seqcount_retry(&tk_core.seq, seq));
466 565
467 timespec_add_ns(ts_raw, nsecs_raw); 566 timespec_add_ns(ts_raw, nsecs_raw);
468 timespec_add_ns(ts_real, nsecs_real); 567 timespec_add_ns(ts_real, nsecs_real);
@@ -479,9 +578,9 @@ EXPORT_SYMBOL(getnstime_raw_and_real);
479 */ 578 */
480void do_gettimeofday(struct timeval *tv) 579void do_gettimeofday(struct timeval *tv)
481{ 580{
482 struct timespec now; 581 struct timespec64 now;
483 582
484 getnstimeofday(&now); 583 getnstimeofday64(&now);
485 tv->tv_sec = now.tv_sec; 584 tv->tv_sec = now.tv_sec;
486 tv->tv_usec = now.tv_nsec/1000; 585 tv->tv_usec = now.tv_nsec/1000;
487} 586}
@@ -495,15 +594,15 @@ EXPORT_SYMBOL(do_gettimeofday);
495 */ 594 */
496int do_settimeofday(const struct timespec *tv) 595int do_settimeofday(const struct timespec *tv)
497{ 596{
498 struct timekeeper *tk = &timekeeper; 597 struct timekeeper *tk = &tk_core.timekeeper;
499 struct timespec ts_delta, xt; 598 struct timespec64 ts_delta, xt, tmp;
500 unsigned long flags; 599 unsigned long flags;
501 600
502 if (!timespec_valid_strict(tv)) 601 if (!timespec_valid_strict(tv))
503 return -EINVAL; 602 return -EINVAL;
504 603
505 raw_spin_lock_irqsave(&timekeeper_lock, flags); 604 raw_spin_lock_irqsave(&timekeeper_lock, flags);
506 write_seqcount_begin(&timekeeper_seq); 605 write_seqcount_begin(&tk_core.seq);
507 606
508 timekeeping_forward_now(tk); 607 timekeeping_forward_now(tk);
509 608
@@ -511,13 +610,14 @@ int do_settimeofday(const struct timespec *tv)
511 ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; 610 ts_delta.tv_sec = tv->tv_sec - xt.tv_sec;
512 ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; 611 ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec;
513 612
514 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta)); 613 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
515 614
516 tk_set_xtime(tk, tv); 615 tmp = timespec_to_timespec64(*tv);
616 tk_set_xtime(tk, &tmp);
517 617
518 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); 618 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
519 619
520 write_seqcount_end(&timekeeper_seq); 620 write_seqcount_end(&tk_core.seq);
521 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 621 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
522 622
523 /* signal hrtimers about time change */ 623 /* signal hrtimers about time change */
@@ -535,33 +635,35 @@ EXPORT_SYMBOL(do_settimeofday);
535 */ 635 */
536int timekeeping_inject_offset(struct timespec *ts) 636int timekeeping_inject_offset(struct timespec *ts)
537{ 637{
538 struct timekeeper *tk = &timekeeper; 638 struct timekeeper *tk = &tk_core.timekeeper;
539 unsigned long flags; 639 unsigned long flags;
540 struct timespec tmp; 640 struct timespec64 ts64, tmp;
541 int ret = 0; 641 int ret = 0;
542 642
543 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) 643 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
544 return -EINVAL; 644 return -EINVAL;
545 645
646 ts64 = timespec_to_timespec64(*ts);
647
546 raw_spin_lock_irqsave(&timekeeper_lock, flags); 648 raw_spin_lock_irqsave(&timekeeper_lock, flags);
547 write_seqcount_begin(&timekeeper_seq); 649 write_seqcount_begin(&tk_core.seq);
548 650
549 timekeeping_forward_now(tk); 651 timekeeping_forward_now(tk);
550 652
551 /* Make sure the proposed value is valid */ 653 /* Make sure the proposed value is valid */
552 tmp = timespec_add(tk_xtime(tk), *ts); 654 tmp = timespec64_add(tk_xtime(tk), ts64);
553 if (!timespec_valid_strict(&tmp)) { 655 if (!timespec64_valid_strict(&tmp)) {
554 ret = -EINVAL; 656 ret = -EINVAL;
555 goto error; 657 goto error;
556 } 658 }
557 659
558 tk_xtime_add(tk, ts); 660 tk_xtime_add(tk, &ts64);
559 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts)); 661 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts64));
560 662
561error: /* even if we error out, we forwarded the time, so call update */ 663error: /* even if we error out, we forwarded the time, so call update */
562 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); 664 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
563 665
564 write_seqcount_end(&timekeeper_seq); 666 write_seqcount_end(&tk_core.seq);
565 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 667 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
566 668
567 /* signal hrtimers about time change */ 669 /* signal hrtimers about time change */
@@ -578,14 +680,14 @@ EXPORT_SYMBOL(timekeeping_inject_offset);
578 */ 680 */
579s32 timekeeping_get_tai_offset(void) 681s32 timekeeping_get_tai_offset(void)
580{ 682{
581 struct timekeeper *tk = &timekeeper; 683 struct timekeeper *tk = &tk_core.timekeeper;
582 unsigned int seq; 684 unsigned int seq;
583 s32 ret; 685 s32 ret;
584 686
585 do { 687 do {
586 seq = read_seqcount_begin(&timekeeper_seq); 688 seq = read_seqcount_begin(&tk_core.seq);
587 ret = tk->tai_offset; 689 ret = tk->tai_offset;
588 } while (read_seqcount_retry(&timekeeper_seq, seq)); 690 } while (read_seqcount_retry(&tk_core.seq, seq));
589 691
590 return ret; 692 return ret;
591} 693}
@@ -606,14 +708,14 @@ static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
606 */ 708 */
607void timekeeping_set_tai_offset(s32 tai_offset) 709void timekeeping_set_tai_offset(s32 tai_offset)
608{ 710{
609 struct timekeeper *tk = &timekeeper; 711 struct timekeeper *tk = &tk_core.timekeeper;
610 unsigned long flags; 712 unsigned long flags;
611 713
612 raw_spin_lock_irqsave(&timekeeper_lock, flags); 714 raw_spin_lock_irqsave(&timekeeper_lock, flags);
613 write_seqcount_begin(&timekeeper_seq); 715 write_seqcount_begin(&tk_core.seq);
614 __timekeeping_set_tai_offset(tk, tai_offset); 716 __timekeeping_set_tai_offset(tk, tai_offset);
615 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); 717 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
616 write_seqcount_end(&timekeeper_seq); 718 write_seqcount_end(&tk_core.seq);
617 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 719 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
618 clock_was_set(); 720 clock_was_set();
619} 721}
@@ -625,14 +727,14 @@ void timekeeping_set_tai_offset(s32 tai_offset)
625 */ 727 */
626static int change_clocksource(void *data) 728static int change_clocksource(void *data)
627{ 729{
628 struct timekeeper *tk = &timekeeper; 730 struct timekeeper *tk = &tk_core.timekeeper;
629 struct clocksource *new, *old; 731 struct clocksource *new, *old;
630 unsigned long flags; 732 unsigned long flags;
631 733
632 new = (struct clocksource *) data; 734 new = (struct clocksource *) data;
633 735
634 raw_spin_lock_irqsave(&timekeeper_lock, flags); 736 raw_spin_lock_irqsave(&timekeeper_lock, flags);
635 write_seqcount_begin(&timekeeper_seq); 737 write_seqcount_begin(&tk_core.seq);
636 738
637 timekeeping_forward_now(tk); 739 timekeeping_forward_now(tk);
638 /* 740 /*
@@ -652,7 +754,7 @@ static int change_clocksource(void *data)
652 } 754 }
653 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); 755 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
654 756
655 write_seqcount_end(&timekeeper_seq); 757 write_seqcount_end(&tk_core.seq);
656 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 758 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
657 759
658 return 0; 760 return 0;
@@ -667,7 +769,7 @@ static int change_clocksource(void *data)
667 */ 769 */
668int timekeeping_notify(struct clocksource *clock) 770int timekeeping_notify(struct clocksource *clock)
669{ 771{
670 struct timekeeper *tk = &timekeeper; 772 struct timekeeper *tk = &tk_core.timekeeper;
671 773
672 if (tk->clock == clock) 774 if (tk->clock == clock)
673 return 0; 775 return 0;
@@ -677,21 +779,6 @@ int timekeeping_notify(struct clocksource *clock)
677} 779}
678 780
679/** 781/**
680 * ktime_get_real - get the real (wall-) time in ktime_t format
681 *
682 * returns the time in ktime_t format
683 */
684ktime_t ktime_get_real(void)
685{
686 struct timespec now;
687
688 getnstimeofday(&now);
689
690 return timespec_to_ktime(now);
691}
692EXPORT_SYMBOL_GPL(ktime_get_real);
693
694/**
695 * getrawmonotonic - Returns the raw monotonic time in a timespec 782 * getrawmonotonic - Returns the raw monotonic time in a timespec
696 * @ts: pointer to the timespec to be set 783 * @ts: pointer to the timespec to be set
697 * 784 *
@@ -699,18 +786,20 @@ EXPORT_SYMBOL_GPL(ktime_get_real);
699 */ 786 */
700void getrawmonotonic(struct timespec *ts) 787void getrawmonotonic(struct timespec *ts)
701{ 788{
702 struct timekeeper *tk = &timekeeper; 789 struct timekeeper *tk = &tk_core.timekeeper;
790 struct timespec64 ts64;
703 unsigned long seq; 791 unsigned long seq;
704 s64 nsecs; 792 s64 nsecs;
705 793
706 do { 794 do {
707 seq = read_seqcount_begin(&timekeeper_seq); 795 seq = read_seqcount_begin(&tk_core.seq);
708 nsecs = timekeeping_get_ns_raw(tk); 796 nsecs = timekeeping_get_ns_raw(tk);
709 *ts = tk->raw_time; 797 ts64 = tk->raw_time;
710 798
711 } while (read_seqcount_retry(&timekeeper_seq, seq)); 799 } while (read_seqcount_retry(&tk_core.seq, seq));
712 800
713 timespec_add_ns(ts, nsecs); 801 timespec64_add_ns(&ts64, nsecs);
802 *ts = timespec64_to_timespec(ts64);
714} 803}
715EXPORT_SYMBOL(getrawmonotonic); 804EXPORT_SYMBOL(getrawmonotonic);
716 805
@@ -719,16 +808,16 @@ EXPORT_SYMBOL(getrawmonotonic);
719 */ 808 */
720int timekeeping_valid_for_hres(void) 809int timekeeping_valid_for_hres(void)
721{ 810{
722 struct timekeeper *tk = &timekeeper; 811 struct timekeeper *tk = &tk_core.timekeeper;
723 unsigned long seq; 812 unsigned long seq;
724 int ret; 813 int ret;
725 814
726 do { 815 do {
727 seq = read_seqcount_begin(&timekeeper_seq); 816 seq = read_seqcount_begin(&tk_core.seq);
728 817
729 ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 818 ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
730 819
731 } while (read_seqcount_retry(&timekeeper_seq, seq)); 820 } while (read_seqcount_retry(&tk_core.seq, seq));
732 821
733 return ret; 822 return ret;
734} 823}
@@ -738,16 +827,16 @@ int timekeeping_valid_for_hres(void)
738 */ 827 */
739u64 timekeeping_max_deferment(void) 828u64 timekeeping_max_deferment(void)
740{ 829{
741 struct timekeeper *tk = &timekeeper; 830 struct timekeeper *tk = &tk_core.timekeeper;
742 unsigned long seq; 831 unsigned long seq;
743 u64 ret; 832 u64 ret;
744 833
745 do { 834 do {
746 seq = read_seqcount_begin(&timekeeper_seq); 835 seq = read_seqcount_begin(&tk_core.seq);
747 836
748 ret = tk->clock->max_idle_ns; 837 ret = tk->clock->max_idle_ns;
749 838
750 } while (read_seqcount_retry(&timekeeper_seq, seq)); 839 } while (read_seqcount_retry(&tk_core.seq, seq));
751 840
752 return ret; 841 return ret;
753} 842}
@@ -787,14 +876,15 @@ void __weak read_boot_clock(struct timespec *ts)
787 */ 876 */
788void __init timekeeping_init(void) 877void __init timekeeping_init(void)
789{ 878{
790 struct timekeeper *tk = &timekeeper; 879 struct timekeeper *tk = &tk_core.timekeeper;
791 struct clocksource *clock; 880 struct clocksource *clock;
792 unsigned long flags; 881 unsigned long flags;
793 struct timespec now, boot, tmp; 882 struct timespec64 now, boot, tmp;
794 883 struct timespec ts;
795 read_persistent_clock(&now);
796 884
797 if (!timespec_valid_strict(&now)) { 885 read_persistent_clock(&ts);
886 now = timespec_to_timespec64(ts);
887 if (!timespec64_valid_strict(&now)) {
798 pr_warn("WARNING: Persistent clock returned invalid value!\n" 888 pr_warn("WARNING: Persistent clock returned invalid value!\n"
799 " Check your CMOS/BIOS settings.\n"); 889 " Check your CMOS/BIOS settings.\n");
800 now.tv_sec = 0; 890 now.tv_sec = 0;
@@ -802,8 +892,9 @@ void __init timekeeping_init(void)
802 } else if (now.tv_sec || now.tv_nsec) 892 } else if (now.tv_sec || now.tv_nsec)
803 persistent_clock_exist = true; 893 persistent_clock_exist = true;
804 894
805 read_boot_clock(&boot); 895 read_boot_clock(&ts);
806 if (!timespec_valid_strict(&boot)) { 896 boot = timespec_to_timespec64(ts);
897 if (!timespec64_valid_strict(&boot)) {
807 pr_warn("WARNING: Boot clock returned invalid value!\n" 898 pr_warn("WARNING: Boot clock returned invalid value!\n"
808 " Check your CMOS/BIOS settings.\n"); 899 " Check your CMOS/BIOS settings.\n");
809 boot.tv_sec = 0; 900 boot.tv_sec = 0;
@@ -811,7 +902,7 @@ void __init timekeeping_init(void)
811 } 902 }
812 903
813 raw_spin_lock_irqsave(&timekeeper_lock, flags); 904 raw_spin_lock_irqsave(&timekeeper_lock, flags);
814 write_seqcount_begin(&timekeeper_seq); 905 write_seqcount_begin(&tk_core.seq);
815 ntp_init(); 906 ntp_init();
816 907
817 clock = clocksource_default_clock(); 908 clock = clocksource_default_clock();
@@ -825,21 +916,21 @@ void __init timekeeping_init(void)
825 if (boot.tv_sec == 0 && boot.tv_nsec == 0) 916 if (boot.tv_sec == 0 && boot.tv_nsec == 0)
826 boot = tk_xtime(tk); 917 boot = tk_xtime(tk);
827 918
828 set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec); 919 set_normalized_timespec64(&tmp, -boot.tv_sec, -boot.tv_nsec);
829 tk_set_wall_to_mono(tk, tmp); 920 tk_set_wall_to_mono(tk, tmp);
830 921
831 tmp.tv_sec = 0; 922 tmp.tv_sec = 0;
832 tmp.tv_nsec = 0; 923 tmp.tv_nsec = 0;
833 tk_set_sleep_time(tk, tmp); 924 tk_set_sleep_time(tk, tmp);
834 925
835 memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper)); 926 timekeeping_update(tk, TK_MIRROR);
836 927
837 write_seqcount_end(&timekeeper_seq); 928 write_seqcount_end(&tk_core.seq);
838 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 929 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
839} 930}
840 931
841/* time in seconds when suspend began */ 932/* time in seconds when suspend began */
842static struct timespec timekeeping_suspend_time; 933static struct timespec64 timekeeping_suspend_time;
843 934
844/** 935/**
845 * __timekeeping_inject_sleeptime - Internal function to add sleep interval 936 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
@@ -849,17 +940,17 @@ static struct timespec timekeeping_suspend_time;
849 * adds the sleep offset to the timekeeping variables. 940 * adds the sleep offset to the timekeeping variables.
850 */ 941 */
851static void __timekeeping_inject_sleeptime(struct timekeeper *tk, 942static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
852 struct timespec *delta) 943 struct timespec64 *delta)
853{ 944{
854 if (!timespec_valid_strict(delta)) { 945 if (!timespec64_valid_strict(delta)) {
855 printk_deferred(KERN_WARNING 946 printk_deferred(KERN_WARNING
856 "__timekeeping_inject_sleeptime: Invalid " 947 "__timekeeping_inject_sleeptime: Invalid "
857 "sleep delta value!\n"); 948 "sleep delta value!\n");
858 return; 949 return;
859 } 950 }
860 tk_xtime_add(tk, delta); 951 tk_xtime_add(tk, delta);
861 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta)); 952 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
862 tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta)); 953 tk_set_sleep_time(tk, timespec64_add(tk->total_sleep_time, *delta));
863 tk_debug_account_sleep_time(delta); 954 tk_debug_account_sleep_time(delta);
864} 955}
865 956
@@ -875,7 +966,8 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
875 */ 966 */
876void timekeeping_inject_sleeptime(struct timespec *delta) 967void timekeeping_inject_sleeptime(struct timespec *delta)
877{ 968{
878 struct timekeeper *tk = &timekeeper; 969 struct timekeeper *tk = &tk_core.timekeeper;
970 struct timespec64 tmp;
879 unsigned long flags; 971 unsigned long flags;
880 972
881 /* 973 /*
@@ -886,15 +978,16 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
886 return; 978 return;
887 979
888 raw_spin_lock_irqsave(&timekeeper_lock, flags); 980 raw_spin_lock_irqsave(&timekeeper_lock, flags);
889 write_seqcount_begin(&timekeeper_seq); 981 write_seqcount_begin(&tk_core.seq);
890 982
891 timekeeping_forward_now(tk); 983 timekeeping_forward_now(tk);
892 984
893 __timekeeping_inject_sleeptime(tk, delta); 985 tmp = timespec_to_timespec64(*delta);
986 __timekeeping_inject_sleeptime(tk, &tmp);
894 987
895 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); 988 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
896 989
897 write_seqcount_end(&timekeeper_seq); 990 write_seqcount_end(&tk_core.seq);
898 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 991 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
899 992
900 /* signal hrtimers about time change */ 993 /* signal hrtimers about time change */
@@ -910,20 +1003,22 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
910 */ 1003 */
911static void timekeeping_resume(void) 1004static void timekeeping_resume(void)
912{ 1005{
913 struct timekeeper *tk = &timekeeper; 1006 struct timekeeper *tk = &tk_core.timekeeper;
914 struct clocksource *clock = tk->clock; 1007 struct clocksource *clock = tk->clock;
915 unsigned long flags; 1008 unsigned long flags;
916 struct timespec ts_new, ts_delta; 1009 struct timespec64 ts_new, ts_delta;
1010 struct timespec tmp;
917 cycle_t cycle_now, cycle_delta; 1011 cycle_t cycle_now, cycle_delta;
918 bool suspendtime_found = false; 1012 bool suspendtime_found = false;
919 1013
920 read_persistent_clock(&ts_new); 1014 read_persistent_clock(&tmp);
1015 ts_new = timespec_to_timespec64(tmp);
921 1016
922 clockevents_resume(); 1017 clockevents_resume();
923 clocksource_resume(); 1018 clocksource_resume();
924 1019
925 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1020 raw_spin_lock_irqsave(&timekeeper_lock, flags);
926 write_seqcount_begin(&timekeeper_seq); 1021 write_seqcount_begin(&tk_core.seq);
927 1022
928 /* 1023 /*
929 * After system resumes, we need to calculate the suspended time and 1024 * After system resumes, we need to calculate the suspended time and
@@ -960,10 +1055,10 @@ static void timekeeping_resume(void)
960 } 1055 }
961 nsec += ((u64) cycle_delta * mult) >> shift; 1056 nsec += ((u64) cycle_delta * mult) >> shift;
962 1057
963 ts_delta = ns_to_timespec(nsec); 1058 ts_delta = ns_to_timespec64(nsec);
964 suspendtime_found = true; 1059 suspendtime_found = true;
965 } else if (timespec_compare(&ts_new, &timekeeping_suspend_time) > 0) { 1060 } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
966 ts_delta = timespec_sub(ts_new, timekeeping_suspend_time); 1061 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
967 suspendtime_found = true; 1062 suspendtime_found = true;
968 } 1063 }
969 1064
@@ -975,7 +1070,7 @@ static void timekeeping_resume(void)
975 tk->ntp_error = 0; 1070 tk->ntp_error = 0;
976 timekeeping_suspended = 0; 1071 timekeeping_suspended = 0;
977 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); 1072 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
978 write_seqcount_end(&timekeeper_seq); 1073 write_seqcount_end(&tk_core.seq);
979 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1074 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
980 1075
981 touch_softlockup_watchdog(); 1076 touch_softlockup_watchdog();
@@ -988,12 +1083,14 @@ static void timekeeping_resume(void)
988 1083
989static int timekeeping_suspend(void) 1084static int timekeeping_suspend(void)
990{ 1085{
991 struct timekeeper *tk = &timekeeper; 1086 struct timekeeper *tk = &tk_core.timekeeper;
992 unsigned long flags; 1087 unsigned long flags;
993 struct timespec delta, delta_delta; 1088 struct timespec64 delta, delta_delta;
994 static struct timespec old_delta; 1089 static struct timespec64 old_delta;
1090 struct timespec tmp;
995 1091
996 read_persistent_clock(&timekeeping_suspend_time); 1092 read_persistent_clock(&tmp);
1093 timekeeping_suspend_time = timespec_to_timespec64(tmp);
997 1094
998 /* 1095 /*
999 * On some systems the persistent_clock can not be detected at 1096 * On some systems the persistent_clock can not be detected at
@@ -1004,7 +1101,7 @@ static int timekeeping_suspend(void)
1004 persistent_clock_exist = true; 1101 persistent_clock_exist = true;
1005 1102
1006 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1103 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1007 write_seqcount_begin(&timekeeper_seq); 1104 write_seqcount_begin(&tk_core.seq);
1008 timekeeping_forward_now(tk); 1105 timekeeping_forward_now(tk);
1009 timekeeping_suspended = 1; 1106 timekeeping_suspended = 1;
1010 1107
@@ -1014,8 +1111,8 @@ static int timekeeping_suspend(void)
1014 * try to compensate so the difference in system time 1111 * try to compensate so the difference in system time
1015 * and persistent_clock time stays close to constant. 1112 * and persistent_clock time stays close to constant.
1016 */ 1113 */
1017 delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time); 1114 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
1018 delta_delta = timespec_sub(delta, old_delta); 1115 delta_delta = timespec64_sub(delta, old_delta);
1019 if (abs(delta_delta.tv_sec) >= 2) { 1116 if (abs(delta_delta.tv_sec) >= 2) {
1020 /* 1117 /*
1021 * if delta_delta is too large, assume time correction 1118 * if delta_delta is too large, assume time correction
@@ -1025,11 +1122,11 @@ static int timekeeping_suspend(void)
1025 } else { 1122 } else {
1026 /* Otherwise try to adjust old_system to compensate */ 1123 /* Otherwise try to adjust old_system to compensate */
1027 timekeeping_suspend_time = 1124 timekeeping_suspend_time =
1028 timespec_add(timekeeping_suspend_time, delta_delta); 1125 timespec64_add(timekeeping_suspend_time, delta_delta);
1029 } 1126 }
1030 1127
1031 timekeeping_update(tk, TK_MIRROR); 1128 timekeeping_update(tk, TK_MIRROR);
1032 write_seqcount_end(&timekeeper_seq); 1129 write_seqcount_end(&tk_core.seq);
1033 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1130 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1034 1131
1035 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); 1132 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
@@ -1262,14 +1359,14 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
1262 /* Figure out if its a leap sec and apply if needed */ 1359 /* Figure out if its a leap sec and apply if needed */
1263 leap = second_overflow(tk->xtime_sec); 1360 leap = second_overflow(tk->xtime_sec);
1264 if (unlikely(leap)) { 1361 if (unlikely(leap)) {
1265 struct timespec ts; 1362 struct timespec64 ts;
1266 1363
1267 tk->xtime_sec += leap; 1364 tk->xtime_sec += leap;
1268 1365
1269 ts.tv_sec = leap; 1366 ts.tv_sec = leap;
1270 ts.tv_nsec = 0; 1367 ts.tv_nsec = 0;
1271 tk_set_wall_to_mono(tk, 1368 tk_set_wall_to_mono(tk,
1272 timespec_sub(tk->wall_to_monotonic, ts)); 1369 timespec64_sub(tk->wall_to_monotonic, ts));
1273 1370
1274 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); 1371 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
1275 1372
@@ -1324,33 +1421,6 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1324 return offset; 1421 return offset;
1325} 1422}
1326 1423
1327#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
1328static inline void old_vsyscall_fixup(struct timekeeper *tk)
1329{
1330 s64 remainder;
1331
1332 /*
1333 * Store only full nanoseconds into xtime_nsec after rounding
1334 * it up and add the remainder to the error difference.
1335 * XXX - This is necessary to avoid small 1ns inconsistnecies caused
1336 * by truncating the remainder in vsyscalls. However, it causes
1337 * additional work to be done in timekeeping_adjust(). Once
1338 * the vsyscall implementations are converted to use xtime_nsec
1339 * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD
1340 * users are removed, this can be killed.
1341 */
1342 remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1);
1343 tk->xtime_nsec -= remainder;
1344 tk->xtime_nsec += 1ULL << tk->shift;
1345 tk->ntp_error += remainder << tk->ntp_error_shift;
1346 tk->ntp_error -= (1ULL << tk->shift) << tk->ntp_error_shift;
1347}
1348#else
1349#define old_vsyscall_fixup(tk)
1350#endif
1351
1352
1353
1354/** 1424/**
1355 * update_wall_time - Uses the current clocksource to increment the wall time 1425 * update_wall_time - Uses the current clocksource to increment the wall time
1356 * 1426 *
@@ -1358,7 +1428,7 @@ static inline void old_vsyscall_fixup(struct timekeeper *tk)
1358void update_wall_time(void) 1428void update_wall_time(void)
1359{ 1429{
1360 struct clocksource *clock; 1430 struct clocksource *clock;
1361 struct timekeeper *real_tk = &timekeeper; 1431 struct timekeeper *real_tk = &tk_core.timekeeper;
1362 struct timekeeper *tk = &shadow_timekeeper; 1432 struct timekeeper *tk = &shadow_timekeeper;
1363 cycle_t offset; 1433 cycle_t offset;
1364 int shift = 0, maxshift; 1434 int shift = 0, maxshift;
@@ -1418,7 +1488,7 @@ void update_wall_time(void)
1418 */ 1488 */
1419 clock_set |= accumulate_nsecs_to_secs(tk); 1489 clock_set |= accumulate_nsecs_to_secs(tk);
1420 1490
1421 write_seqcount_begin(&timekeeper_seq); 1491 write_seqcount_begin(&tk_core.seq);
1422 /* Update clock->cycle_last with the new value */ 1492 /* Update clock->cycle_last with the new value */
1423 clock->cycle_last = tk->cycle_last; 1493 clock->cycle_last = tk->cycle_last;
1424 /* 1494 /*
@@ -1428,12 +1498,12 @@ void update_wall_time(void)
1428 * requires changes to all other timekeeper usage sites as 1498 * requires changes to all other timekeeper usage sites as
1429 * well, i.e. move the timekeeper pointer getter into the 1499 * well, i.e. move the timekeeper pointer getter into the
1430 * spinlocked/seqcount protected sections. And we trade this 1500 * spinlocked/seqcount protected sections. And we trade this
1431 * memcpy under the timekeeper_seq against one before we start 1501 * memcpy under the tk_core.seq against one before we start
1432 * updating. 1502 * updating.
1433 */ 1503 */
1434 memcpy(real_tk, tk, sizeof(*tk)); 1504 memcpy(real_tk, tk, sizeof(*tk));
1435 timekeeping_update(real_tk, clock_set); 1505 timekeeping_update(real_tk, clock_set);
1436 write_seqcount_end(&timekeeper_seq); 1506 write_seqcount_end(&tk_core.seq);
1437out: 1507out:
1438 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1508 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1439 if (clock_set) 1509 if (clock_set)
@@ -1454,7 +1524,7 @@ out:
1454 */ 1524 */
1455void getboottime(struct timespec *ts) 1525void getboottime(struct timespec *ts)
1456{ 1526{
1457 struct timekeeper *tk = &timekeeper; 1527 struct timekeeper *tk = &tk_core.timekeeper;
1458 struct timespec boottime = { 1528 struct timespec boottime = {
1459 .tv_sec = tk->wall_to_monotonic.tv_sec + 1529 .tv_sec = tk->wall_to_monotonic.tv_sec +
1460 tk->total_sleep_time.tv_sec, 1530 tk->total_sleep_time.tv_sec,
@@ -1477,60 +1547,47 @@ EXPORT_SYMBOL_GPL(getboottime);
1477 */ 1547 */
1478void get_monotonic_boottime(struct timespec *ts) 1548void get_monotonic_boottime(struct timespec *ts)
1479{ 1549{
1480 struct timekeeper *tk = &timekeeper; 1550 struct timekeeper *tk = &tk_core.timekeeper;
1481 struct timespec tomono, sleep; 1551 struct timespec64 tomono, sleep, ret;
1482 s64 nsec; 1552 s64 nsec;
1483 unsigned int seq; 1553 unsigned int seq;
1484 1554
1485 WARN_ON(timekeeping_suspended); 1555 WARN_ON(timekeeping_suspended);
1486 1556
1487 do { 1557 do {
1488 seq = read_seqcount_begin(&timekeeper_seq); 1558 seq = read_seqcount_begin(&tk_core.seq);
1489 ts->tv_sec = tk->xtime_sec; 1559 ret.tv_sec = tk->xtime_sec;
1490 nsec = timekeeping_get_ns(tk); 1560 nsec = timekeeping_get_ns(tk);
1491 tomono = tk->wall_to_monotonic; 1561 tomono = tk->wall_to_monotonic;
1492 sleep = tk->total_sleep_time; 1562 sleep = tk->total_sleep_time;
1493 1563
1494 } while (read_seqcount_retry(&timekeeper_seq, seq)); 1564 } while (read_seqcount_retry(&tk_core.seq, seq));
1495 1565
1496 ts->tv_sec += tomono.tv_sec + sleep.tv_sec; 1566 ret.tv_sec += tomono.tv_sec + sleep.tv_sec;
1497 ts->tv_nsec = 0; 1567 ret.tv_nsec = 0;
1498 timespec_add_ns(ts, nsec + tomono.tv_nsec + sleep.tv_nsec); 1568 timespec64_add_ns(&ret, nsec + tomono.tv_nsec + sleep.tv_nsec);
1569 *ts = timespec64_to_timespec(ret);
1499} 1570}
1500EXPORT_SYMBOL_GPL(get_monotonic_boottime); 1571EXPORT_SYMBOL_GPL(get_monotonic_boottime);
1501 1572
1502/** 1573/**
1503 * ktime_get_boottime - Returns monotonic time since boot in a ktime
1504 *
1505 * Returns the monotonic time since boot in a ktime
1506 *
1507 * This is similar to CLOCK_MONTONIC/ktime_get, but also
1508 * includes the time spent in suspend.
1509 */
1510ktime_t ktime_get_boottime(void)
1511{
1512 struct timespec ts;
1513
1514 get_monotonic_boottime(&ts);
1515 return timespec_to_ktime(ts);
1516}
1517EXPORT_SYMBOL_GPL(ktime_get_boottime);
1518
1519/**
1520 * monotonic_to_bootbased - Convert the monotonic time to boot based. 1574 * monotonic_to_bootbased - Convert the monotonic time to boot based.
1521 * @ts: pointer to the timespec to be converted 1575 * @ts: pointer to the timespec to be converted
1522 */ 1576 */
1523void monotonic_to_bootbased(struct timespec *ts) 1577void monotonic_to_bootbased(struct timespec *ts)
1524{ 1578{
1525 struct timekeeper *tk = &timekeeper; 1579 struct timekeeper *tk = &tk_core.timekeeper;
1580 struct timespec64 ts64;
1526 1581
1527 *ts = timespec_add(*ts, tk->total_sleep_time); 1582 ts64 = timespec_to_timespec64(*ts);
1583 ts64 = timespec64_add(ts64, tk->total_sleep_time);
1584 *ts = timespec64_to_timespec(ts64);
1528} 1585}
1529EXPORT_SYMBOL_GPL(monotonic_to_bootbased); 1586EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
1530 1587
1531unsigned long get_seconds(void) 1588unsigned long get_seconds(void)
1532{ 1589{
1533 struct timekeeper *tk = &timekeeper; 1590 struct timekeeper *tk = &tk_core.timekeeper;
1534 1591
1535 return tk->xtime_sec; 1592 return tk->xtime_sec;
1536} 1593}
@@ -1538,43 +1595,44 @@ EXPORT_SYMBOL(get_seconds);
1538 1595
1539struct timespec __current_kernel_time(void) 1596struct timespec __current_kernel_time(void)
1540{ 1597{
1541 struct timekeeper *tk = &timekeeper; 1598 struct timekeeper *tk = &tk_core.timekeeper;
1542 1599
1543 return tk_xtime(tk); 1600 return timespec64_to_timespec(tk_xtime(tk));
1544} 1601}
1545 1602
1546struct timespec current_kernel_time(void) 1603struct timespec current_kernel_time(void)
1547{ 1604{
1548 struct timekeeper *tk = &timekeeper; 1605 struct timekeeper *tk = &tk_core.timekeeper;
1549 struct timespec now; 1606 struct timespec64 now;
1550 unsigned long seq; 1607 unsigned long seq;
1551 1608
1552 do { 1609 do {
1553 seq = read_seqcount_begin(&timekeeper_seq); 1610 seq = read_seqcount_begin(&tk_core.seq);
1554 1611
1555 now = tk_xtime(tk); 1612 now = tk_xtime(tk);
1556 } while (read_seqcount_retry(&timekeeper_seq, seq)); 1613 } while (read_seqcount_retry(&tk_core.seq, seq));
1557 1614
1558 return now; 1615 return timespec64_to_timespec(now);
1559} 1616}
1560EXPORT_SYMBOL(current_kernel_time); 1617EXPORT_SYMBOL(current_kernel_time);
1561 1618
1562struct timespec get_monotonic_coarse(void) 1619struct timespec get_monotonic_coarse(void)
1563{ 1620{
1564 struct timekeeper *tk = &timekeeper; 1621 struct timekeeper *tk = &tk_core.timekeeper;
1565 struct timespec now, mono; 1622 struct timespec64 now, mono;
1566 unsigned long seq; 1623 unsigned long seq;
1567 1624
1568 do { 1625 do {
1569 seq = read_seqcount_begin(&timekeeper_seq); 1626 seq = read_seqcount_begin(&tk_core.seq);
1570 1627
1571 now = tk_xtime(tk); 1628 now = tk_xtime(tk);
1572 mono = tk->wall_to_monotonic; 1629 mono = tk->wall_to_monotonic;
1573 } while (read_seqcount_retry(&timekeeper_seq, seq)); 1630 } while (read_seqcount_retry(&tk_core.seq, seq));
1574 1631
1575 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, 1632 set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec,
1576 now.tv_nsec + mono.tv_nsec); 1633 now.tv_nsec + mono.tv_nsec);
1577 return now; 1634
1635 return timespec64_to_timespec(now);
1578} 1636}
1579 1637
1580/* 1638/*
@@ -1587,29 +1645,38 @@ void do_timer(unsigned long ticks)
1587} 1645}
1588 1646
1589/** 1647/**
1590 * get_xtime_and_monotonic_and_sleep_offset() - get xtime, wall_to_monotonic, 1648 * ktime_get_update_offsets_tick - hrtimer helper
1591 * and sleep offsets. 1649 * @offs_real: pointer to storage for monotonic -> realtime offset
1592 * @xtim: pointer to timespec to be set with xtime 1650 * @offs_boot: pointer to storage for monotonic -> boottime offset
1593 * @wtom: pointer to timespec to be set with wall_to_monotonic 1651 * @offs_tai: pointer to storage for monotonic -> clock tai offset
1594 * @sleep: pointer to timespec to be set with time in suspend 1652 *
1653 * Returns monotonic time at last tick and various offsets
1595 */ 1654 */
1596void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, 1655ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot,
1597 struct timespec *wtom, struct timespec *sleep) 1656 ktime_t *offs_tai)
1598{ 1657{
1599 struct timekeeper *tk = &timekeeper; 1658 struct timekeeper *tk = &tk_core.timekeeper;
1600 unsigned long seq; 1659 unsigned int seq;
1660 ktime_t base;
1661 u64 nsecs;
1601 1662
1602 do { 1663 do {
1603 seq = read_seqcount_begin(&timekeeper_seq); 1664 seq = read_seqcount_begin(&tk_core.seq);
1604 *xtim = tk_xtime(tk); 1665
1605 *wtom = tk->wall_to_monotonic; 1666 base = tk->base_mono;
1606 *sleep = tk->total_sleep_time; 1667 nsecs = tk->xtime_nsec >> tk->shift;
1607 } while (read_seqcount_retry(&timekeeper_seq, seq)); 1668
1669 *offs_real = tk->offs_real;
1670 *offs_boot = tk->offs_boot;
1671 *offs_tai = tk->offs_tai;
1672 } while (read_seqcount_retry(&tk_core.seq, seq));
1673
1674 return ktime_add_ns(base, nsecs);
1608} 1675}
1609 1676
1610#ifdef CONFIG_HIGH_RES_TIMERS 1677#ifdef CONFIG_HIGH_RES_TIMERS
1611/** 1678/**
1612 * ktime_get_update_offsets - hrtimer helper 1679 * ktime_get_update_offsets_now - hrtimer helper
1613 * @offs_real: pointer to storage for monotonic -> realtime offset 1680 * @offs_real: pointer to storage for monotonic -> realtime offset
1614 * @offs_boot: pointer to storage for monotonic -> boottime offset 1681 * @offs_boot: pointer to storage for monotonic -> boottime offset
1615 * @offs_tai: pointer to storage for monotonic -> clock tai offset 1682 * @offs_tai: pointer to storage for monotonic -> clock tai offset
@@ -1617,57 +1684,37 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1617 * Returns current monotonic time and updates the offsets 1684 * Returns current monotonic time and updates the offsets
1618 * Called from hrtimer_interrupt() or retrigger_next_event() 1685 * Called from hrtimer_interrupt() or retrigger_next_event()
1619 */ 1686 */
1620ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot, 1687ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot,
1621 ktime_t *offs_tai) 1688 ktime_t *offs_tai)
1622{ 1689{
1623 struct timekeeper *tk = &timekeeper; 1690 struct timekeeper *tk = &tk_core.timekeeper;
1624 ktime_t now;
1625 unsigned int seq; 1691 unsigned int seq;
1626 u64 secs, nsecs; 1692 ktime_t base;
1693 u64 nsecs;
1627 1694
1628 do { 1695 do {
1629 seq = read_seqcount_begin(&timekeeper_seq); 1696 seq = read_seqcount_begin(&tk_core.seq);
1630 1697
1631 secs = tk->xtime_sec; 1698 base = tk->base_mono;
1632 nsecs = timekeeping_get_ns(tk); 1699 nsecs = timekeeping_get_ns(tk);
1633 1700
1634 *offs_real = tk->offs_real; 1701 *offs_real = tk->offs_real;
1635 *offs_boot = tk->offs_boot; 1702 *offs_boot = tk->offs_boot;
1636 *offs_tai = tk->offs_tai; 1703 *offs_tai = tk->offs_tai;
1637 } while (read_seqcount_retry(&timekeeper_seq, seq)); 1704 } while (read_seqcount_retry(&tk_core.seq, seq));
1638 1705
1639 now = ktime_add_ns(ktime_set(secs, 0), nsecs); 1706 return ktime_add_ns(base, nsecs);
1640 now = ktime_sub(now, *offs_real);
1641 return now;
1642} 1707}
1643#endif 1708#endif
1644 1709
1645/** 1710/**
1646 * ktime_get_monotonic_offset() - get wall_to_monotonic in ktime_t format
1647 */
1648ktime_t ktime_get_monotonic_offset(void)
1649{
1650 struct timekeeper *tk = &timekeeper;
1651 unsigned long seq;
1652 struct timespec wtom;
1653
1654 do {
1655 seq = read_seqcount_begin(&timekeeper_seq);
1656 wtom = tk->wall_to_monotonic;
1657 } while (read_seqcount_retry(&timekeeper_seq, seq));
1658
1659 return timespec_to_ktime(wtom);
1660}
1661EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset);
1662
1663/**
1664 * do_adjtimex() - Accessor function to NTP __do_adjtimex function 1711 * do_adjtimex() - Accessor function to NTP __do_adjtimex function
1665 */ 1712 */
1666int do_adjtimex(struct timex *txc) 1713int do_adjtimex(struct timex *txc)
1667{ 1714{
1668 struct timekeeper *tk = &timekeeper; 1715 struct timekeeper *tk = &tk_core.timekeeper;
1669 unsigned long flags; 1716 unsigned long flags;
1670 struct timespec ts; 1717 struct timespec64 ts;
1671 s32 orig_tai, tai; 1718 s32 orig_tai, tai;
1672 int ret; 1719 int ret;
1673 1720
@@ -1687,10 +1734,10 @@ int do_adjtimex(struct timex *txc)
1687 return ret; 1734 return ret;
1688 } 1735 }
1689 1736
1690 getnstimeofday(&ts); 1737 getnstimeofday64(&ts);
1691 1738
1692 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1739 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1693 write_seqcount_begin(&timekeeper_seq); 1740 write_seqcount_begin(&tk_core.seq);
1694 1741
1695 orig_tai = tai = tk->tai_offset; 1742 orig_tai = tai = tk->tai_offset;
1696 ret = __do_adjtimex(txc, &ts, &tai); 1743 ret = __do_adjtimex(txc, &ts, &tai);
@@ -1699,7 +1746,7 @@ int do_adjtimex(struct timex *txc)
1699 __timekeeping_set_tai_offset(tk, tai); 1746 __timekeeping_set_tai_offset(tk, tai);
1700 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); 1747 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1701 } 1748 }
1702 write_seqcount_end(&timekeeper_seq); 1749 write_seqcount_end(&tk_core.seq);
1703 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1750 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1704 1751
1705 if (tai != orig_tai) 1752 if (tai != orig_tai)
@@ -1719,11 +1766,11 @@ void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts)
1719 unsigned long flags; 1766 unsigned long flags;
1720 1767
1721 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1768 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1722 write_seqcount_begin(&timekeeper_seq); 1769 write_seqcount_begin(&tk_core.seq);
1723 1770
1724 __hardpps(phase_ts, raw_ts); 1771 __hardpps(phase_ts, raw_ts);
1725 1772
1726 write_seqcount_end(&timekeeper_seq); 1773 write_seqcount_end(&tk_core.seq);
1727 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1774 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1728} 1775}
1729EXPORT_SYMBOL(hardpps); 1776EXPORT_SYMBOL(hardpps);
diff --git a/kernel/time/timekeeping.h b/kernel/time/timekeeping.h
new file mode 100644
index 000000000000..adc1fc98bde3
--- /dev/null
+++ b/kernel/time/timekeeping.h
@@ -0,0 +1,20 @@
1#ifndef _KERNEL_TIME_TIMEKEEPING_H
2#define _KERNEL_TIME_TIMEKEEPING_H
3/*
4 * Internal interfaces for kernel/time/
5 */
6extern ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real,
7 ktime_t *offs_boot,
8 ktime_t *offs_tai);
9extern ktime_t ktime_get_update_offsets_now(ktime_t *offs_real,
10 ktime_t *offs_boot,
11 ktime_t *offs_tai);
12
13extern int timekeeping_valid_for_hres(void);
14extern u64 timekeeping_max_deferment(void);
15extern int timekeeping_inject_offset(struct timespec *ts);
16extern s32 timekeeping_get_tai_offset(void);
17extern void timekeeping_set_tai_offset(s32 tai_offset);
18extern void timekeeping_clocktai(struct timespec *ts);
19
20#endif
diff --git a/kernel/time/timekeeping_debug.c b/kernel/time/timekeeping_debug.c
index 4d54f97558df..f6bd65236712 100644
--- a/kernel/time/timekeeping_debug.c
+++ b/kernel/time/timekeeping_debug.c
@@ -67,7 +67,7 @@ static int __init tk_debug_sleep_time_init(void)
67} 67}
68late_initcall(tk_debug_sleep_time_init); 68late_initcall(tk_debug_sleep_time_init);
69 69
70void tk_debug_account_sleep_time(struct timespec *t) 70void tk_debug_account_sleep_time(struct timespec64 *t)
71{ 71{
72 sleep_time_bin[fls(t->tv_sec)]++; 72 sleep_time_bin[fls(t->tv_sec)]++;
73} 73}
diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h
index 13323ea08ffa..e3d28ad236f9 100644
--- a/kernel/time/timekeeping_internal.h
+++ b/kernel/time/timekeeping_internal.h
@@ -6,7 +6,7 @@
6#include <linux/time.h> 6#include <linux/time.h>
7 7
8#ifdef CONFIG_DEBUG_FS 8#ifdef CONFIG_DEBUG_FS
9extern void tk_debug_account_sleep_time(struct timespec *t); 9extern void tk_debug_account_sleep_time(struct timespec64 *t);
10#else 10#else
11#define tk_debug_account_sleep_time(x) 11#define tk_debug_account_sleep_time(x)
12#endif 12#endif
diff --git a/kernel/timer.c b/kernel/time/timer.c
index 3bb01a323b2a..aca5dfe2fa3d 100644
--- a/kernel/timer.c
+++ b/kernel/time/timer.c
@@ -82,6 +82,7 @@ struct tvec_base {
82 unsigned long next_timer; 82 unsigned long next_timer;
83 unsigned long active_timers; 83 unsigned long active_timers;
84 unsigned long all_timers; 84 unsigned long all_timers;
85 int cpu;
85 struct tvec_root tv1; 86 struct tvec_root tv1;
86 struct tvec tv2; 87 struct tvec tv2;
87 struct tvec tv3; 88 struct tvec tv3;
@@ -409,6 +410,22 @@ static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
409 base->next_timer = timer->expires; 410 base->next_timer = timer->expires;
410 } 411 }
411 base->all_timers++; 412 base->all_timers++;
413
414 /*
415 * Check whether the other CPU is in dynticks mode and needs
416 * to be triggered to reevaluate the timer wheel.
417 * We are protected against the other CPU fiddling
418 * with the timer by holding the timer base lock. This also
419 * makes sure that a CPU on the way to stop its tick can not
420 * evaluate the timer wheel.
421 *
422 * Spare the IPI for deferrable timers on idle targets though.
423 * The next busy ticks will take care of it. Except full dynticks
424 * require special care against races with idle_cpu(), lets deal
425 * with that later.
426 */
427 if (!tbase_get_deferrable(base) || tick_nohz_full_cpu(base->cpu))
428 wake_up_nohz_cpu(base->cpu);
412} 429}
413 430
414#ifdef CONFIG_TIMER_STATS 431#ifdef CONFIG_TIMER_STATS
@@ -948,22 +965,6 @@ void add_timer_on(struct timer_list *timer, int cpu)
948 timer_set_base(timer, base); 965 timer_set_base(timer, base);
949 debug_activate(timer, timer->expires); 966 debug_activate(timer, timer->expires);
950 internal_add_timer(base, timer); 967 internal_add_timer(base, timer);
951 /*
952 * Check whether the other CPU is in dynticks mode and needs
953 * to be triggered to reevaluate the timer wheel.
954 * We are protected against the other CPU fiddling
955 * with the timer by holding the timer base lock. This also
956 * makes sure that a CPU on the way to stop its tick can not
957 * evaluate the timer wheel.
958 *
959 * Spare the IPI for deferrable timers on idle targets though.
960 * The next busy ticks will take care of it. Except full dynticks
961 * require special care against races with idle_cpu(), lets deal
962 * with that later.
963 */
964 if (!tbase_get_deferrable(timer->base) || tick_nohz_full_cpu(cpu))
965 wake_up_nohz_cpu(cpu);
966
967 spin_unlock_irqrestore(&base->lock, flags); 968 spin_unlock_irqrestore(&base->lock, flags);
968} 969}
969EXPORT_SYMBOL_GPL(add_timer_on); 970EXPORT_SYMBOL_GPL(add_timer_on);
@@ -1568,6 +1569,7 @@ static int init_timers_cpu(int cpu)
1568 } 1569 }
1569 spin_lock_init(&base->lock); 1570 spin_lock_init(&base->lock);
1570 tvec_base_done[cpu] = 1; 1571 tvec_base_done[cpu] = 1;
1572 base->cpu = cpu;
1571 } else { 1573 } else {
1572 base = per_cpu(tvec_bases, cpu); 1574 base = per_cpu(tvec_bases, cpu);
1573 } 1575 }
diff --git a/kernel/time/udelay_test.c b/kernel/time/udelay_test.c
new file mode 100644
index 000000000000..e622ba365a13
--- /dev/null
+++ b/kernel/time/udelay_test.c
@@ -0,0 +1,168 @@
1/*
2 * udelay() test kernel module
3 *
4 * Test is executed by writing and reading to /sys/kernel/debug/udelay_test
5 * Tests are configured by writing: USECS ITERATIONS
6 * Tests are executed by reading from the same file.
7 * Specifying usecs of 0 or negative values will run multiples tests.
8 *
9 * Copyright (C) 2014 Google, Inc.
10 *
11 * This software is licensed under the terms of the GNU General Public
12 * License version 2, as published by the Free Software Foundation, and
13 * may be copied, distributed, and modified under those terms.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 */
20
21#include <linux/debugfs.h>
22#include <linux/delay.h>
23#include <linux/ktime.h>
24#include <linux/module.h>
25#include <linux/uaccess.h>
26
27#define DEFAULT_ITERATIONS 100
28
29#define DEBUGFS_FILENAME "udelay_test"
30
31static DEFINE_MUTEX(udelay_test_lock);
32static struct dentry *udelay_test_debugfs_file;
33static int udelay_test_usecs;
34static int udelay_test_iterations = DEFAULT_ITERATIONS;
35
36static int udelay_test_single(struct seq_file *s, int usecs, uint32_t iters)
37{
38 int min = 0, max = 0, fail_count = 0;
39 uint64_t sum = 0;
40 uint64_t avg;
41 int i;
42 /* Allow udelay to be up to 0.5% fast */
43 int allowed_error_ns = usecs * 5;
44
45 for (i = 0; i < iters; ++i) {
46 struct timespec ts1, ts2;
47 int time_passed;
48
49 ktime_get_ts(&ts1);
50 udelay(usecs);
51 ktime_get_ts(&ts2);
52 time_passed = timespec_to_ns(&ts2) - timespec_to_ns(&ts1);
53
54 if (i == 0 || time_passed < min)
55 min = time_passed;
56 if (i == 0 || time_passed > max)
57 max = time_passed;
58 if ((time_passed + allowed_error_ns) / 1000 < usecs)
59 ++fail_count;
60 WARN_ON(time_passed < 0);
61 sum += time_passed;
62 }
63
64 avg = sum;
65 do_div(avg, iters);
66 seq_printf(s, "%d usecs x %d: exp=%d allowed=%d min=%d avg=%lld max=%d",
67 usecs, iters, usecs * 1000,
68 (usecs * 1000) - allowed_error_ns, min, avg, max);
69 if (fail_count)
70 seq_printf(s, " FAIL=%d", fail_count);
71 seq_puts(s, "\n");
72
73 return 0;
74}
75
76static int udelay_test_show(struct seq_file *s, void *v)
77{
78 int usecs;
79 int iters;
80 int ret = 0;
81
82 mutex_lock(&udelay_test_lock);
83 usecs = udelay_test_usecs;
84 iters = udelay_test_iterations;
85 mutex_unlock(&udelay_test_lock);
86
87 if (usecs > 0 && iters > 0) {
88 return udelay_test_single(s, usecs, iters);
89 } else if (usecs == 0) {
90 struct timespec ts;
91
92 ktime_get_ts(&ts);
93 seq_printf(s, "udelay() test (lpj=%ld kt=%ld.%09ld)\n",
94 loops_per_jiffy, ts.tv_sec, ts.tv_nsec);
95 seq_puts(s, "usage:\n");
96 seq_puts(s, "echo USECS [ITERS] > " DEBUGFS_FILENAME "\n");
97 seq_puts(s, "cat " DEBUGFS_FILENAME "\n");
98 }
99
100 return ret;
101}
102
103static int udelay_test_open(struct inode *inode, struct file *file)
104{
105 return single_open(file, udelay_test_show, inode->i_private);
106}
107
108static ssize_t udelay_test_write(struct file *file, const char __user *buf,
109 size_t count, loff_t *pos)
110{
111 char lbuf[32];
112 int ret;
113 int usecs;
114 int iters;
115
116 if (count >= sizeof(lbuf))
117 return -EINVAL;
118
119 if (copy_from_user(lbuf, buf, count))
120 return -EFAULT;
121 lbuf[count] = '\0';
122
123 ret = sscanf(lbuf, "%d %d", &usecs, &iters);
124 if (ret < 1)
125 return -EINVAL;
126 else if (ret < 2)
127 iters = DEFAULT_ITERATIONS;
128
129 mutex_lock(&udelay_test_lock);
130 udelay_test_usecs = usecs;
131 udelay_test_iterations = iters;
132 mutex_unlock(&udelay_test_lock);
133
134 return count;
135}
136
137static const struct file_operations udelay_test_debugfs_ops = {
138 .owner = THIS_MODULE,
139 .open = udelay_test_open,
140 .read = seq_read,
141 .write = udelay_test_write,
142 .llseek = seq_lseek,
143 .release = single_release,
144};
145
146static int __init udelay_test_init(void)
147{
148 mutex_lock(&udelay_test_lock);
149 udelay_test_debugfs_file = debugfs_create_file(DEBUGFS_FILENAME,
150 S_IRUSR, NULL, NULL, &udelay_test_debugfs_ops);
151 mutex_unlock(&udelay_test_lock);
152
153 return 0;
154}
155
156module_init(udelay_test_init);
157
158static void __exit udelay_test_exit(void)
159{
160 mutex_lock(&udelay_test_lock);
161 debugfs_remove(udelay_test_debugfs_file);
162 mutex_unlock(&udelay_test_lock);
163}
164
165module_exit(udelay_test_exit);
166
167MODULE_AUTHOR("David Riley <davidriley@chromium.org>");
168MODULE_LICENSE("GPL");
diff --git a/kernel/tsacct.c b/kernel/tsacct.c
index a1dd9a1b1327..975cb49e32bf 100644
--- a/kernel/tsacct.c
+++ b/kernel/tsacct.c
@@ -31,20 +31,19 @@ void bacct_add_tsk(struct user_namespace *user_ns,
31 struct taskstats *stats, struct task_struct *tsk) 31 struct taskstats *stats, struct task_struct *tsk)
32{ 32{
33 const struct cred *tcred; 33 const struct cred *tcred;
34 struct timespec uptime, ts;
35 cputime_t utime, stime, utimescaled, stimescaled; 34 cputime_t utime, stime, utimescaled, stimescaled;
36 u64 ac_etime; 35 u64 delta;
37 36
38 BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN); 37 BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN);
39 38
40 /* calculate task elapsed time in timespec */ 39 /* calculate task elapsed time in nsec */
41 do_posix_clock_monotonic_gettime(&uptime); 40 delta = ktime_get_ns() - tsk->start_time;
42 ts = timespec_sub(uptime, tsk->start_time); 41 /* Convert to micro seconds */
43 /* rebase elapsed time to usec (should never be negative) */ 42 do_div(delta, NSEC_PER_USEC);
44 ac_etime = timespec_to_ns(&ts); 43 stats->ac_etime = delta;
45 do_div(ac_etime, NSEC_PER_USEC); 44 /* Convert to seconds for btime */
46 stats->ac_etime = ac_etime; 45 do_div(delta, USEC_PER_SEC);
47 stats->ac_btime = get_seconds() - ts.tv_sec; 46 stats->ac_btime = get_seconds() - delta;
48 if (thread_group_leader(tsk)) { 47 if (thread_group_leader(tsk)) {
49 stats->ac_exitcode = tsk->exit_code; 48 stats->ac_exitcode = tsk->exit_code;
50 if (tsk->flags & PF_FORKNOEXEC) 49 if (tsk->flags & PF_FORKNOEXEC)
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-29 08:28:11 -0500