aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/trace/ftrace.rst14
-rw-r--r--drivers/input/evdev.c7
-rw-r--r--include/linux/hrtimer.h2
-rw-r--r--include/linux/timekeeper_internal.h2
-rw-r--r--include/linux/timekeeping.h37
-rw-r--r--include/uapi/linux/time.h1
-rw-r--r--kernel/time/hrtimer.c16
-rw-r--r--kernel/time/posix-stubs.c2
-rw-r--r--kernel/time/posix-timers.c26
-rw-r--r--kernel/time/tick-common.c15
-rw-r--r--kernel/time/tick-internal.h6
-rw-r--r--kernel/time/tick-sched.c19
-rw-r--r--kernel/time/timekeeping.c78
-rw-r--r--kernel/time/timekeeping.h1
-rw-r--r--kernel/trace/trace.c2
15 files changed, 119 insertions, 109 deletions
diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst
index e45f0786f3f9..67d9c38e95eb 100644
--- a/Documentation/trace/ftrace.rst
+++ b/Documentation/trace/ftrace.rst
@@ -461,9 +461,17 @@ of ftrace. Here is a list of some of the key files:
461 and ticks at the same rate as the hardware clocksource. 461 and ticks at the same rate as the hardware clocksource.
462 462
463 boot: 463 boot:
464 Same as mono. Used to be a separate clock which accounted 464 This is the boot clock (CLOCK_BOOTTIME) and is based on the
465 for the time spent in suspend while CLOCK_MONOTONIC did 465 fast monotonic clock, but also accounts for time spent in
466 not. 466 suspend. Since the clock access is designed for use in
467 tracing in the suspend path, some side effects are possible
468 if clock is accessed after the suspend time is accounted before
469 the fast mono clock is updated. In this case, the clock update
470 appears to happen slightly sooner than it normally would have.
471 Also on 32-bit systems, it's possible that the 64-bit boot offset
472 sees a partial update. These effects are rare and post
473 processing should be able to handle them. See comments in the
474 ktime_get_boot_fast_ns() function for more information.
467 475
468 To set a clock, simply echo the clock name into this file:: 476 To set a clock, simply echo the clock name into this file::
469 477
diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c
index 46115a392098..c81c79d01d93 100644
--- a/drivers/input/evdev.c
+++ b/drivers/input/evdev.c
@@ -31,6 +31,7 @@
31enum evdev_clock_type { 31enum evdev_clock_type {
32 EV_CLK_REAL = 0, 32 EV_CLK_REAL = 0,
33 EV_CLK_MONO, 33 EV_CLK_MONO,
34 EV_CLK_BOOT,
34 EV_CLK_MAX 35 EV_CLK_MAX
35}; 36};
36 37
@@ -197,10 +198,12 @@ static int evdev_set_clk_type(struct evdev_client *client, unsigned int clkid)
197 case CLOCK_REALTIME: 198 case CLOCK_REALTIME:
198 clk_type = EV_CLK_REAL; 199 clk_type = EV_CLK_REAL;
199 break; 200 break;
200 case CLOCK_BOOTTIME:
201 case CLOCK_MONOTONIC: 201 case CLOCK_MONOTONIC:
202 clk_type = EV_CLK_MONO; 202 clk_type = EV_CLK_MONO;
203 break; 203 break;
204 case CLOCK_BOOTTIME:
205 clk_type = EV_CLK_BOOT;
206 break;
204 default: 207 default:
205 return -EINVAL; 208 return -EINVAL;
206 } 209 }
@@ -311,6 +314,8 @@ static void evdev_events(struct input_handle *handle,
311 314
312 ev_time[EV_CLK_MONO] = ktime_get(); 315 ev_time[EV_CLK_MONO] = ktime_get();
313 ev_time[EV_CLK_REAL] = ktime_mono_to_real(ev_time[EV_CLK_MONO]); 316 ev_time[EV_CLK_REAL] = ktime_mono_to_real(ev_time[EV_CLK_MONO]);
317 ev_time[EV_CLK_BOOT] = ktime_mono_to_any(ev_time[EV_CLK_MONO],
318 TK_OFFS_BOOT);
314 319
315 rcu_read_lock(); 320 rcu_read_lock();
316 321
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index a2656c3ebe81..3892e9c8b2de 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -161,9 +161,11 @@ struct hrtimer_clock_base {
161enum hrtimer_base_type { 161enum hrtimer_base_type {
162 HRTIMER_BASE_MONOTONIC, 162 HRTIMER_BASE_MONOTONIC,
163 HRTIMER_BASE_REALTIME, 163 HRTIMER_BASE_REALTIME,
164 HRTIMER_BASE_BOOTTIME,
164 HRTIMER_BASE_TAI, 165 HRTIMER_BASE_TAI,
165 HRTIMER_BASE_MONOTONIC_SOFT, 166 HRTIMER_BASE_MONOTONIC_SOFT,
166 HRTIMER_BASE_REALTIME_SOFT, 167 HRTIMER_BASE_REALTIME_SOFT,
168 HRTIMER_BASE_BOOTTIME_SOFT,
167 HRTIMER_BASE_TAI_SOFT, 169 HRTIMER_BASE_TAI_SOFT,
168 HRTIMER_MAX_CLOCK_BASES, 170 HRTIMER_MAX_CLOCK_BASES,
169}; 171};
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
index 4b3dca173e89..7acb953298a7 100644
--- a/include/linux/timekeeper_internal.h
+++ b/include/linux/timekeeper_internal.h
@@ -52,7 +52,6 @@ struct tk_read_base {
52 * @offs_real: Offset clock monotonic -> clock realtime 52 * @offs_real: Offset clock monotonic -> clock realtime
53 * @offs_boot: Offset clock monotonic -> clock boottime 53 * @offs_boot: Offset clock monotonic -> clock boottime
54 * @offs_tai: Offset clock monotonic -> clock tai 54 * @offs_tai: Offset clock monotonic -> clock tai
55 * @time_suspended: Accumulated suspend time
56 * @tai_offset: The current UTC to TAI offset in seconds 55 * @tai_offset: The current UTC to TAI offset in seconds
57 * @clock_was_set_seq: The sequence number of clock was set events 56 * @clock_was_set_seq: The sequence number of clock was set events
58 * @cs_was_changed_seq: The sequence number of clocksource change events 57 * @cs_was_changed_seq: The sequence number of clocksource change events
@@ -95,7 +94,6 @@ struct timekeeper {
95 ktime_t offs_real; 94 ktime_t offs_real;
96 ktime_t offs_boot; 95 ktime_t offs_boot;
97 ktime_t offs_tai; 96 ktime_t offs_tai;
98 ktime_t time_suspended;
99 s32 tai_offset; 97 s32 tai_offset;
100 unsigned int clock_was_set_seq; 98 unsigned int clock_was_set_seq;
101 u8 cs_was_changed_seq; 99 u8 cs_was_changed_seq;
diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h
index 9737fbec7019..588a0e4b1ab9 100644
--- a/include/linux/timekeeping.h
+++ b/include/linux/timekeeping.h
@@ -33,25 +33,20 @@ extern void ktime_get_ts64(struct timespec64 *ts);
33extern time64_t ktime_get_seconds(void); 33extern time64_t ktime_get_seconds(void);
34extern time64_t __ktime_get_real_seconds(void); 34extern time64_t __ktime_get_real_seconds(void);
35extern time64_t ktime_get_real_seconds(void); 35extern time64_t ktime_get_real_seconds(void);
36extern void ktime_get_active_ts64(struct timespec64 *ts);
37 36
38extern int __getnstimeofday64(struct timespec64 *tv); 37extern int __getnstimeofday64(struct timespec64 *tv);
39extern void getnstimeofday64(struct timespec64 *tv); 38extern void getnstimeofday64(struct timespec64 *tv);
40extern void getboottime64(struct timespec64 *ts); 39extern void getboottime64(struct timespec64 *ts);
41 40
42#define ktime_get_real_ts64(ts) getnstimeofday64(ts) 41#define ktime_get_real_ts64(ts) getnstimeofday64(ts)
43
44/* Clock BOOTTIME compatibility wrappers */
45static inline void get_monotonic_boottime64(struct timespec64 *ts)
46{
47 ktime_get_ts64(ts);
48}
49 42
50/* 43/*
51 * ktime_t based interfaces 44 * ktime_t based interfaces
52 */ 45 */
46
53enum tk_offsets { 47enum tk_offsets {
54 TK_OFFS_REAL, 48 TK_OFFS_REAL,
49 TK_OFFS_BOOT,
55 TK_OFFS_TAI, 50 TK_OFFS_TAI,
56 TK_OFFS_MAX, 51 TK_OFFS_MAX,
57}; 52};
@@ -62,10 +57,6 @@ extern ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs);
62extern ktime_t ktime_get_raw(void); 57extern ktime_t ktime_get_raw(void);
63extern u32 ktime_get_resolution_ns(void); 58extern u32 ktime_get_resolution_ns(void);
64 59
65/* Clock BOOTTIME compatibility wrappers */
66static inline ktime_t ktime_get_boottime(void) { return ktime_get(); }
67static inline u64 ktime_get_boot_ns(void) { return ktime_get(); }
68
69/** 60/**
70 * ktime_get_real - get the real (wall-) time in ktime_t format 61 * ktime_get_real - get the real (wall-) time in ktime_t format
71 */ 62 */
@@ -75,6 +66,17 @@ static inline ktime_t ktime_get_real(void)
75} 66}
76 67
77/** 68/**
69 * ktime_get_boottime - Returns monotonic time since boot in ktime_t format
70 *
71 * This is similar to CLOCK_MONTONIC/ktime_get, but also includes the
72 * time spent in suspend.
73 */
74static inline ktime_t ktime_get_boottime(void)
75{
76 return ktime_get_with_offset(TK_OFFS_BOOT);
77}
78
79/**
78 * ktime_get_clocktai - Returns the TAI time of day in ktime_t format 80 * ktime_get_clocktai - Returns the TAI time of day in ktime_t format
79 */ 81 */
80static inline ktime_t ktime_get_clocktai(void) 82static inline ktime_t ktime_get_clocktai(void)
@@ -100,6 +102,11 @@ static inline u64 ktime_get_real_ns(void)
100 return ktime_to_ns(ktime_get_real()); 102 return ktime_to_ns(ktime_get_real());
101} 103}
102 104
105static inline u64 ktime_get_boot_ns(void)
106{
107 return ktime_to_ns(ktime_get_boottime());
108}
109
103static inline u64 ktime_get_tai_ns(void) 110static inline u64 ktime_get_tai_ns(void)
104{ 111{
105 return ktime_to_ns(ktime_get_clocktai()); 112 return ktime_to_ns(ktime_get_clocktai());
@@ -112,11 +119,17 @@ static inline u64 ktime_get_raw_ns(void)
112 119
113extern u64 ktime_get_mono_fast_ns(void); 120extern u64 ktime_get_mono_fast_ns(void);
114extern u64 ktime_get_raw_fast_ns(void); 121extern u64 ktime_get_raw_fast_ns(void);
122extern u64 ktime_get_boot_fast_ns(void);
115extern u64 ktime_get_real_fast_ns(void); 123extern u64 ktime_get_real_fast_ns(void);
116 124
117/* 125/*
118 * timespec64 interfaces utilizing the ktime based ones 126 * timespec64 interfaces utilizing the ktime based ones
119 */ 127 */
128static inline void get_monotonic_boottime64(struct timespec64 *ts)
129{
130 *ts = ktime_to_timespec64(ktime_get_boottime());
131}
132
120static inline void timekeeping_clocktai64(struct timespec64 *ts) 133static inline void timekeeping_clocktai64(struct timespec64 *ts)
121{ 134{
122 *ts = ktime_to_timespec64(ktime_get_clocktai()); 135 *ts = ktime_to_timespec64(ktime_get_clocktai());
diff --git a/include/uapi/linux/time.h b/include/uapi/linux/time.h
index 16a296612ba4..4c0338ea308a 100644
--- a/include/uapi/linux/time.h
+++ b/include/uapi/linux/time.h
@@ -73,7 +73,6 @@ struct __kernel_old_timeval {
73 */ 73 */
74#define CLOCK_SGI_CYCLE 10 74#define CLOCK_SGI_CYCLE 10
75#define CLOCK_TAI 11 75#define CLOCK_TAI 11
76#define CLOCK_MONOTONIC_ACTIVE 12
77 76
78#define MAX_CLOCKS 16 77#define MAX_CLOCKS 16
79#define CLOCKS_MASK (CLOCK_REALTIME | CLOCK_MONOTONIC) 78#define CLOCKS_MASK (CLOCK_REALTIME | CLOCK_MONOTONIC)
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index eda1210ce50f..14e858753d76 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -91,6 +91,11 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
91 .get_time = &ktime_get_real, 91 .get_time = &ktime_get_real,
92 }, 92 },
93 { 93 {
94 .index = HRTIMER_BASE_BOOTTIME,
95 .clockid = CLOCK_BOOTTIME,
96 .get_time = &ktime_get_boottime,
97 },
98 {
94 .index = HRTIMER_BASE_TAI, 99 .index = HRTIMER_BASE_TAI,
95 .clockid = CLOCK_TAI, 100 .clockid = CLOCK_TAI,
96 .get_time = &ktime_get_clocktai, 101 .get_time = &ktime_get_clocktai,
@@ -106,6 +111,11 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
106 .get_time = &ktime_get_real, 111 .get_time = &ktime_get_real,
107 }, 112 },
108 { 113 {
114 .index = HRTIMER_BASE_BOOTTIME_SOFT,
115 .clockid = CLOCK_BOOTTIME,
116 .get_time = &ktime_get_boottime,
117 },
118 {
109 .index = HRTIMER_BASE_TAI_SOFT, 119 .index = HRTIMER_BASE_TAI_SOFT,
110 .clockid = CLOCK_TAI, 120 .clockid = CLOCK_TAI,
111 .get_time = &ktime_get_clocktai, 121 .get_time = &ktime_get_clocktai,
@@ -119,7 +129,7 @@ static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
119 129
120 [CLOCK_REALTIME] = HRTIMER_BASE_REALTIME, 130 [CLOCK_REALTIME] = HRTIMER_BASE_REALTIME,
121 [CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC, 131 [CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC,
122 [CLOCK_BOOTTIME] = HRTIMER_BASE_MONOTONIC, 132 [CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME,
123 [CLOCK_TAI] = HRTIMER_BASE_TAI, 133 [CLOCK_TAI] = HRTIMER_BASE_TAI,
124}; 134};
125 135
@@ -571,12 +581,14 @@ __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_
571static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) 581static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
572{ 582{
573 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; 583 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
584 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
574 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; 585 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
575 586
576 ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 587 ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
577 offs_real, offs_tai); 588 offs_real, offs_boot, offs_tai);
578 589
579 base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; 590 base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
591 base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
580 base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; 592 base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
581 593
582 return now; 594 return now;
diff --git a/kernel/time/posix-stubs.c b/kernel/time/posix-stubs.c
index e0dbae98db9d..69a937c3cd81 100644
--- a/kernel/time/posix-stubs.c
+++ b/kernel/time/posix-stubs.c
@@ -83,8 +83,6 @@ int do_clock_gettime(clockid_t which_clock, struct timespec64 *tp)
83 case CLOCK_BOOTTIME: 83 case CLOCK_BOOTTIME:
84 get_monotonic_boottime64(tp); 84 get_monotonic_boottime64(tp);
85 break; 85 break;
86 case CLOCK_MONOTONIC_ACTIVE:
87 ktime_get_active_ts64(tp);
88 default: 86 default:
89 return -EINVAL; 87 return -EINVAL;
90 } 88 }
diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c
index b6899b5060bd..10b7186d0638 100644
--- a/kernel/time/posix-timers.c
+++ b/kernel/time/posix-timers.c
@@ -252,16 +252,15 @@ static int posix_get_coarse_res(const clockid_t which_clock, struct timespec64 *
252 return 0; 252 return 0;
253} 253}
254 254
255static int posix_get_tai(clockid_t which_clock, struct timespec64 *tp) 255static int posix_get_boottime(const clockid_t which_clock, struct timespec64 *tp)
256{ 256{
257 timekeeping_clocktai64(tp); 257 get_monotonic_boottime64(tp);
258 return 0; 258 return 0;
259} 259}
260 260
261static int posix_get_monotonic_active(clockid_t which_clock, 261static int posix_get_tai(clockid_t which_clock, struct timespec64 *tp)
262 struct timespec64 *tp)
263{ 262{
264 ktime_get_active_ts64(tp); 263 timekeeping_clocktai64(tp);
265 return 0; 264 return 0;
266} 265}
267 266
@@ -1317,9 +1316,19 @@ static const struct k_clock clock_tai = {
1317 .timer_arm = common_hrtimer_arm, 1316 .timer_arm = common_hrtimer_arm,
1318}; 1317};
1319 1318
1320static const struct k_clock clock_monotonic_active = { 1319static const struct k_clock clock_boottime = {
1321 .clock_getres = posix_get_hrtimer_res, 1320 .clock_getres = posix_get_hrtimer_res,
1322 .clock_get = posix_get_monotonic_active, 1321 .clock_get = posix_get_boottime,
1322 .nsleep = common_nsleep,
1323 .timer_create = common_timer_create,
1324 .timer_set = common_timer_set,
1325 .timer_get = common_timer_get,
1326 .timer_del = common_timer_del,
1327 .timer_rearm = common_hrtimer_rearm,
1328 .timer_forward = common_hrtimer_forward,
1329 .timer_remaining = common_hrtimer_remaining,
1330 .timer_try_to_cancel = common_hrtimer_try_to_cancel,
1331 .timer_arm = common_hrtimer_arm,
1323}; 1332};
1324 1333
1325static const struct k_clock * const posix_clocks[] = { 1334static const struct k_clock * const posix_clocks[] = {
@@ -1330,11 +1339,10 @@ static const struct k_clock * const posix_clocks[] = {
1330 [CLOCK_MONOTONIC_RAW] = &clock_monotonic_raw, 1339 [CLOCK_MONOTONIC_RAW] = &clock_monotonic_raw,
1331 [CLOCK_REALTIME_COARSE] = &clock_realtime_coarse, 1340 [CLOCK_REALTIME_COARSE] = &clock_realtime_coarse,
1332 [CLOCK_MONOTONIC_COARSE] = &clock_monotonic_coarse, 1341 [CLOCK_MONOTONIC_COARSE] = &clock_monotonic_coarse,
1333 [CLOCK_BOOTTIME] = &clock_monotonic, 1342 [CLOCK_BOOTTIME] = &clock_boottime,
1334 [CLOCK_REALTIME_ALARM] = &alarm_clock, 1343 [CLOCK_REALTIME_ALARM] = &alarm_clock,
1335 [CLOCK_BOOTTIME_ALARM] = &alarm_clock, 1344 [CLOCK_BOOTTIME_ALARM] = &alarm_clock,
1336 [CLOCK_TAI] = &clock_tai, 1345 [CLOCK_TAI] = &clock_tai,
1337 [CLOCK_MONOTONIC_ACTIVE] = &clock_monotonic_active,
1338}; 1346};
1339 1347
1340static const struct k_clock *clockid_to_kclock(const clockid_t id) 1348static const struct k_clock *clockid_to_kclock(const clockid_t id)
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 099572ca4a8f..49edc1c4f3e6 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -419,19 +419,6 @@ void tick_suspend_local(void)
419 clockevents_shutdown(td->evtdev); 419 clockevents_shutdown(td->evtdev);
420} 420}
421 421
422static void tick_forward_next_period(void)
423{
424 ktime_t delta, now = ktime_get();
425 u64 n;
426
427 delta = ktime_sub(now, tick_next_period);
428 n = ktime_divns(delta, tick_period);
429 tick_next_period += n * tick_period;
430 if (tick_next_period < now)
431 tick_next_period += tick_period;
432 tick_sched_forward_next_period();
433}
434
435/** 422/**
436 * tick_resume_local - Resume the local tick device 423 * tick_resume_local - Resume the local tick device
437 * 424 *
@@ -444,8 +431,6 @@ void tick_resume_local(void)
444 struct tick_device *td = this_cpu_ptr(&tick_cpu_device); 431 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
445 bool broadcast = tick_resume_check_broadcast(); 432 bool broadcast = tick_resume_check_broadcast();
446 433
447 tick_forward_next_period();
448
449 clockevents_tick_resume(td->evtdev); 434 clockevents_tick_resume(td->evtdev);
450 if (!broadcast) { 435 if (!broadcast) {
451 if (td->mode == TICKDEV_MODE_PERIODIC) 436 if (td->mode == TICKDEV_MODE_PERIODIC)
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 21efab7485ca..e277284c2831 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -141,12 +141,6 @@ static inline void tick_check_oneshot_broadcast_this_cpu(void) { }
141static inline bool tick_broadcast_oneshot_available(void) { return tick_oneshot_possible(); } 141static inline bool tick_broadcast_oneshot_available(void) { return tick_oneshot_possible(); }
142#endif /* !(BROADCAST && ONESHOT) */ 142#endif /* !(BROADCAST && ONESHOT) */
143 143
144#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
145extern void tick_sched_forward_next_period(void);
146#else
147static inline void tick_sched_forward_next_period(void) { }
148#endif
149
150/* NO_HZ_FULL internal */ 144/* NO_HZ_FULL internal */
151#ifdef CONFIG_NO_HZ_FULL 145#ifdef CONFIG_NO_HZ_FULL
152extern void tick_nohz_init(void); 146extern void tick_nohz_init(void);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 646645e981f9..da9455a6b42b 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -52,15 +52,6 @@ struct tick_sched *tick_get_tick_sched(int cpu)
52static ktime_t last_jiffies_update; 52static ktime_t last_jiffies_update;
53 53
54/* 54/*
55 * Called after resume. Make sure that jiffies are not fast forwarded due to
56 * clock monotonic being forwarded by the suspended time.
57 */
58void tick_sched_forward_next_period(void)
59{
60 last_jiffies_update = tick_next_period;
61}
62
63/*
64 * Must be called with interrupts disabled ! 55 * Must be called with interrupts disabled !
65 */ 56 */
66static void tick_do_update_jiffies64(ktime_t now) 57static void tick_do_update_jiffies64(ktime_t now)
@@ -804,12 +795,12 @@ static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
804 return; 795 return;
805 } 796 }
806 797
807 hrtimer_set_expires(&ts->sched_timer, tick); 798 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
808 799 hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED);
809 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) 800 } else {
810 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED); 801 hrtimer_set_expires(&ts->sched_timer, tick);
811 else
812 tick_program_event(tick, 1); 802 tick_program_event(tick, 1);
803 }
813} 804}
814 805
815static void tick_nohz_retain_tick(struct tick_sched *ts) 806static void tick_nohz_retain_tick(struct tick_sched *ts)
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index dcf7f20fcd12..49cbceef5deb 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -138,12 +138,7 @@ static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
138 138
139static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) 139static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
140{ 140{
141 /* Update both bases so mono and raw stay coupled. */ 141 tk->offs_boot = ktime_add(tk->offs_boot, delta);
142 tk->tkr_mono.base += delta;
143 tk->tkr_raw.base += delta;
144
145 /* Accumulate time spent in suspend */
146 tk->time_suspended += delta;
147} 142}
148 143
149/* 144/*
@@ -473,6 +468,36 @@ u64 ktime_get_raw_fast_ns(void)
473} 468}
474EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns); 469EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
475 470
471/**
472 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
473 *
474 * To keep it NMI safe since we're accessing from tracing, we're not using a
475 * separate timekeeper with updates to monotonic clock and boot offset
476 * protected with seqlocks. This has the following minor side effects:
477 *
478 * (1) Its possible that a timestamp be taken after the boot offset is updated
479 * but before the timekeeper is updated. If this happens, the new boot offset
480 * is added to the old timekeeping making the clock appear to update slightly
481 * earlier:
482 * CPU 0 CPU 1
483 * timekeeping_inject_sleeptime64()
484 * __timekeeping_inject_sleeptime(tk, delta);
485 * timestamp();
486 * timekeeping_update(tk, TK_CLEAR_NTP...);
487 *
488 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
489 * partially updated. Since the tk->offs_boot update is a rare event, this
490 * should be a rare occurrence which postprocessing should be able to handle.
491 */
492u64 notrace ktime_get_boot_fast_ns(void)
493{
494 struct timekeeper *tk = &tk_core.timekeeper;
495
496 return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot));
497}
498EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
499
500
476/* 501/*
477 * See comment for __ktime_get_fast_ns() vs. timestamp ordering 502 * See comment for __ktime_get_fast_ns() vs. timestamp ordering
478 */ 503 */
@@ -764,6 +789,7 @@ EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
764 789
765static ktime_t *offsets[TK_OFFS_MAX] = { 790static ktime_t *offsets[TK_OFFS_MAX] = {
766 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real, 791 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
792 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
767 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai, 793 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
768}; 794};
769 795
@@ -861,39 +887,6 @@ void ktime_get_ts64(struct timespec64 *ts)
861EXPORT_SYMBOL_GPL(ktime_get_ts64); 887EXPORT_SYMBOL_GPL(ktime_get_ts64);
862 888
863/** 889/**
864 * ktime_get_active_ts64 - Get the active non-suspended monotonic clock
865 * @ts: pointer to timespec variable
866 *
867 * The function calculates the monotonic clock from the realtime clock and
868 * the wall_to_monotonic offset, subtracts the accumulated suspend time and
869 * stores the result in normalized timespec64 format in the variable
870 * pointed to by @ts.
871 */
872void ktime_get_active_ts64(struct timespec64 *ts)
873{
874 struct timekeeper *tk = &tk_core.timekeeper;
875 struct timespec64 tomono, tsusp;
876 u64 nsec, nssusp;
877 unsigned int seq;
878
879 WARN_ON(timekeeping_suspended);
880
881 do {
882 seq = read_seqcount_begin(&tk_core.seq);
883 ts->tv_sec = tk->xtime_sec;
884 nsec = timekeeping_get_ns(&tk->tkr_mono);
885 tomono = tk->wall_to_monotonic;
886 nssusp = tk->time_suspended;
887 } while (read_seqcount_retry(&tk_core.seq, seq));
888
889 ts->tv_sec += tomono.tv_sec;
890 ts->tv_nsec = 0;
891 timespec64_add_ns(ts, nsec + tomono.tv_nsec);
892 tsusp = ns_to_timespec64(nssusp);
893 *ts = timespec64_sub(*ts, tsusp);
894}
895
896/**
897 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC 890 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
898 * 891 *
899 * Returns the seconds portion of CLOCK_MONOTONIC with a single non 892 * Returns the seconds portion of CLOCK_MONOTONIC with a single non
@@ -1593,6 +1586,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1593 return; 1586 return;
1594 } 1587 }
1595 tk_xtime_add(tk, delta); 1588 tk_xtime_add(tk, delta);
1589 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1596 tk_update_sleep_time(tk, timespec64_to_ktime(*delta)); 1590 tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1597 tk_debug_account_sleep_time(delta); 1591 tk_debug_account_sleep_time(delta);
1598} 1592}
@@ -2125,7 +2119,7 @@ out:
2125void getboottime64(struct timespec64 *ts) 2119void getboottime64(struct timespec64 *ts)
2126{ 2120{
2127 struct timekeeper *tk = &tk_core.timekeeper; 2121 struct timekeeper *tk = &tk_core.timekeeper;
2128 ktime_t t = ktime_sub(tk->offs_real, tk->time_suspended); 2122 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
2129 2123
2130 *ts = ktime_to_timespec64(t); 2124 *ts = ktime_to_timespec64(t);
2131} 2125}
@@ -2188,6 +2182,7 @@ void do_timer(unsigned long ticks)
2188 * ktime_get_update_offsets_now - hrtimer helper 2182 * ktime_get_update_offsets_now - hrtimer helper
2189 * @cwsseq: pointer to check and store the clock was set sequence number 2183 * @cwsseq: pointer to check and store the clock was set sequence number
2190 * @offs_real: pointer to storage for monotonic -> realtime offset 2184 * @offs_real: pointer to storage for monotonic -> realtime offset
2185 * @offs_boot: pointer to storage for monotonic -> boottime offset
2191 * @offs_tai: pointer to storage for monotonic -> clock tai offset 2186 * @offs_tai: pointer to storage for monotonic -> clock tai offset
2192 * 2187 *
2193 * Returns current monotonic time and updates the offsets if the 2188 * Returns current monotonic time and updates the offsets if the
@@ -2197,7 +2192,7 @@ void do_timer(unsigned long ticks)
2197 * Called from hrtimer_interrupt() or retrigger_next_event() 2192 * Called from hrtimer_interrupt() or retrigger_next_event()
2198 */ 2193 */
2199ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real, 2194ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
2200 ktime_t *offs_tai) 2195 ktime_t *offs_boot, ktime_t *offs_tai)
2201{ 2196{
2202 struct timekeeper *tk = &tk_core.timekeeper; 2197 struct timekeeper *tk = &tk_core.timekeeper;
2203 unsigned int seq; 2198 unsigned int seq;
@@ -2214,6 +2209,7 @@ ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
2214 if (*cwsseq != tk->clock_was_set_seq) { 2209 if (*cwsseq != tk->clock_was_set_seq) {
2215 *cwsseq = tk->clock_was_set_seq; 2210 *cwsseq = tk->clock_was_set_seq;
2216 *offs_real = tk->offs_real; 2211 *offs_real = tk->offs_real;
2212 *offs_boot = tk->offs_boot;
2217 *offs_tai = tk->offs_tai; 2213 *offs_tai = tk->offs_tai;
2218 } 2214 }
2219 2215
diff --git a/kernel/time/timekeeping.h b/kernel/time/timekeeping.h
index 79b67f5e0343..7a9b4eb7a1d5 100644
--- a/kernel/time/timekeeping.h
+++ b/kernel/time/timekeeping.h
@@ -6,6 +6,7 @@
6 */ 6 */
7extern ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, 7extern ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq,
8 ktime_t *offs_real, 8 ktime_t *offs_real,
9 ktime_t *offs_boot,
9 ktime_t *offs_tai); 10 ktime_t *offs_tai);
10 11
11extern int timekeeping_valid_for_hres(void); 12extern int timekeeping_valid_for_hres(void);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index dfbcf9ee1447..414d7210b2ec 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1165,7 +1165,7 @@ static struct {
1165 { trace_clock, "perf", 1 }, 1165 { trace_clock, "perf", 1 },
1166 { ktime_get_mono_fast_ns, "mono", 1 }, 1166 { ktime_get_mono_fast_ns, "mono", 1 },
1167 { ktime_get_raw_fast_ns, "mono_raw", 1 }, 1167 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1168 { ktime_get_mono_fast_ns, "boot", 1 }, 1168 { ktime_get_boot_fast_ns, "boot", 1 },
1169 ARCH_TRACE_CLOCKS 1169 ARCH_TRACE_CLOCKS
1170}; 1170};
1171 1171