aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/time/timekeeping.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/time/timekeeping.c')
-rw-r--r--kernel/time/timekeeping.c755
1 files changed, 414 insertions, 341 deletions
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 269b1fe5f2ae..e16af197a2bc 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -24,32 +24,32 @@
24/* Structure holding internal timekeeping values. */ 24/* Structure holding internal timekeeping values. */
25struct timekeeper { 25struct timekeeper {
26 /* Current clocksource used for timekeeping. */ 26 /* Current clocksource used for timekeeping. */
27 struct clocksource *clock; 27 struct clocksource *clock;
28 /* NTP adjusted clock multiplier */ 28 /* NTP adjusted clock multiplier */
29 u32 mult; 29 u32 mult;
30 /* The shift value of the current clocksource. */ 30 /* The shift value of the current clocksource. */
31 int shift; 31 u32 shift;
32
33 /* Number of clock cycles in one NTP interval. */ 32 /* Number of clock cycles in one NTP interval. */
34 cycle_t cycle_interval; 33 cycle_t cycle_interval;
35 /* Number of clock shifted nano seconds in one NTP interval. */ 34 /* Number of clock shifted nano seconds in one NTP interval. */
36 u64 xtime_interval; 35 u64 xtime_interval;
37 /* shifted nano seconds left over when rounding cycle_interval */ 36 /* shifted nano seconds left over when rounding cycle_interval */
38 s64 xtime_remainder; 37 s64 xtime_remainder;
39 /* Raw nano seconds accumulated per NTP interval. */ 38 /* Raw nano seconds accumulated per NTP interval. */
40 u32 raw_interval; 39 u32 raw_interval;
40
41 /* Current CLOCK_REALTIME time in seconds */
42 u64 xtime_sec;
43 /* Clock shifted nano seconds */
44 u64 xtime_nsec;
41 45
42 /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */
43 u64 xtime_nsec;
44 /* Difference between accumulated time and NTP time in ntp 46 /* Difference between accumulated time and NTP time in ntp
45 * shifted nano seconds. */ 47 * shifted nano seconds. */
46 s64 ntp_error; 48 s64 ntp_error;
47 /* Shift conversion between clock shifted nano seconds and 49 /* Shift conversion between clock shifted nano seconds and
48 * ntp shifted nano seconds. */ 50 * ntp shifted nano seconds. */
49 int ntp_error_shift; 51 u32 ntp_error_shift;
50 52
51 /* The current time */
52 struct timespec xtime;
53 /* 53 /*
54 * wall_to_monotonic is what we need to add to xtime (or xtime corrected 54 * wall_to_monotonic is what we need to add to xtime (or xtime corrected
55 * for sub jiffie times) to get to monotonic time. Monotonic is pegged 55 * for sub jiffie times) to get to monotonic time. Monotonic is pegged
@@ -64,20 +64,17 @@ struct timekeeper {
64 * - wall_to_monotonic is no longer the boot time, getboottime must be 64 * - wall_to_monotonic is no longer the boot time, getboottime must be
65 * used instead. 65 * used instead.
66 */ 66 */
67 struct timespec wall_to_monotonic; 67 struct timespec wall_to_monotonic;
68 /* time spent in suspend */
69 struct timespec total_sleep_time;
70 /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
71 struct timespec raw_time;
72
73 /* Offset clock monotonic -> clock realtime */ 68 /* Offset clock monotonic -> clock realtime */
74 ktime_t offs_real; 69 ktime_t offs_real;
75 70 /* time spent in suspend */
71 struct timespec total_sleep_time;
76 /* Offset clock monotonic -> clock boottime */ 72 /* Offset clock monotonic -> clock boottime */
77 ktime_t offs_boot; 73 ktime_t offs_boot;
78 74 /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
75 struct timespec raw_time;
79 /* Seqlock for all timekeeper values */ 76 /* Seqlock for all timekeeper values */
80 seqlock_t lock; 77 seqlock_t lock;
81}; 78};
82 79
83static struct timekeeper timekeeper; 80static struct timekeeper timekeeper;
@@ -88,11 +85,62 @@ static struct timekeeper timekeeper;
88 */ 85 */
89__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); 86__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
90 87
91
92/* flag for if timekeeping is suspended */ 88/* flag for if timekeeping is suspended */
93int __read_mostly timekeeping_suspended; 89int __read_mostly timekeeping_suspended;
94 90
91static inline void tk_normalize_xtime(struct timekeeper *tk)
92{
93 while (tk->xtime_nsec >= ((u64)NSEC_PER_SEC << tk->shift)) {
94 tk->xtime_nsec -= (u64)NSEC_PER_SEC << tk->shift;
95 tk->xtime_sec++;
96 }
97}
95 98
99static struct timespec tk_xtime(struct timekeeper *tk)
100{
101 struct timespec ts;
102
103 ts.tv_sec = tk->xtime_sec;
104 ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
105 return ts;
106}
107
108static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts)
109{
110 tk->xtime_sec = ts->tv_sec;
111 tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift;
112}
113
114static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts)
115{
116 tk->xtime_sec += ts->tv_sec;
117 tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift;
118}
119
120static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm)
121{
122 struct timespec tmp;
123
124 /*
125 * Verify consistency of: offset_real = -wall_to_monotonic
126 * before modifying anything
127 */
128 set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec,
129 -tk->wall_to_monotonic.tv_nsec);
130 WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64);
131 tk->wall_to_monotonic = wtm;
132 set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
133 tk->offs_real = timespec_to_ktime(tmp);
134}
135
136static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t)
137{
138 /* Verify consistency before modifying */
139 WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64);
140
141 tk->total_sleep_time = t;
142 tk->offs_boot = timespec_to_ktime(t);
143}
96 144
97/** 145/**
98 * timekeeper_setup_internals - Set up internals to use clocksource clock. 146 * timekeeper_setup_internals - Set up internals to use clocksource clock.
@@ -104,12 +152,14 @@ int __read_mostly timekeeping_suspended;
104 * 152 *
105 * Unless you're the timekeeping code, you should not be using this! 153 * Unless you're the timekeeping code, you should not be using this!
106 */ 154 */
107static void timekeeper_setup_internals(struct clocksource *clock) 155static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
108{ 156{
109 cycle_t interval; 157 cycle_t interval;
110 u64 tmp, ntpinterval; 158 u64 tmp, ntpinterval;
159 struct clocksource *old_clock;
111 160
112 timekeeper.clock = clock; 161 old_clock = tk->clock;
162 tk->clock = clock;
113 clock->cycle_last = clock->read(clock); 163 clock->cycle_last = clock->read(clock);
114 164
115 /* Do the ns -> cycle conversion first, using original mult */ 165 /* Do the ns -> cycle conversion first, using original mult */
@@ -122,83 +172,89 @@ static void timekeeper_setup_internals(struct clocksource *clock)
122 tmp = 1; 172 tmp = 1;
123 173
124 interval = (cycle_t) tmp; 174 interval = (cycle_t) tmp;
125 timekeeper.cycle_interval = interval; 175 tk->cycle_interval = interval;
126 176
127 /* Go back from cycles -> shifted ns */ 177 /* Go back from cycles -> shifted ns */
128 timekeeper.xtime_interval = (u64) interval * clock->mult; 178 tk->xtime_interval = (u64) interval * clock->mult;
129 timekeeper.xtime_remainder = ntpinterval - timekeeper.xtime_interval; 179 tk->xtime_remainder = ntpinterval - tk->xtime_interval;
130 timekeeper.raw_interval = 180 tk->raw_interval =
131 ((u64) interval * clock->mult) >> clock->shift; 181 ((u64) interval * clock->mult) >> clock->shift;
132 182
133 timekeeper.xtime_nsec = 0; 183 /* if changing clocks, convert xtime_nsec shift units */
134 timekeeper.shift = clock->shift; 184 if (old_clock) {
185 int shift_change = clock->shift - old_clock->shift;
186 if (shift_change < 0)
187 tk->xtime_nsec >>= -shift_change;
188 else
189 tk->xtime_nsec <<= shift_change;
190 }
191 tk->shift = clock->shift;
135 192
136 timekeeper.ntp_error = 0; 193 tk->ntp_error = 0;
137 timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; 194 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
138 195
139 /* 196 /*
140 * The timekeeper keeps its own mult values for the currently 197 * The timekeeper keeps its own mult values for the currently
141 * active clocksource. These value will be adjusted via NTP 198 * active clocksource. These value will be adjusted via NTP
142 * to counteract clock drifting. 199 * to counteract clock drifting.
143 */ 200 */
144 timekeeper.mult = clock->mult; 201 tk->mult = clock->mult;
145} 202}
146 203
147/* Timekeeper helper functions. */ 204/* Timekeeper helper functions. */
148static inline s64 timekeeping_get_ns(void) 205static inline s64 timekeeping_get_ns(struct timekeeper *tk)
149{ 206{
150 cycle_t cycle_now, cycle_delta; 207 cycle_t cycle_now, cycle_delta;
151 struct clocksource *clock; 208 struct clocksource *clock;
209 s64 nsec;
152 210
153 /* read clocksource: */ 211 /* read clocksource: */
154 clock = timekeeper.clock; 212 clock = tk->clock;
155 cycle_now = clock->read(clock); 213 cycle_now = clock->read(clock);
156 214
157 /* calculate the delta since the last update_wall_time: */ 215 /* calculate the delta since the last update_wall_time: */
158 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 216 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
159 217
160 /* return delta convert to nanoseconds using ntp adjusted mult. */ 218 nsec = cycle_delta * tk->mult + tk->xtime_nsec;
161 return clocksource_cyc2ns(cycle_delta, timekeeper.mult, 219 nsec >>= tk->shift;
162 timekeeper.shift); 220
221 /* If arch requires, add in gettimeoffset() */
222 return nsec + arch_gettimeoffset();
163} 223}
164 224
165static inline s64 timekeeping_get_ns_raw(void) 225static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
166{ 226{
167 cycle_t cycle_now, cycle_delta; 227 cycle_t cycle_now, cycle_delta;
168 struct clocksource *clock; 228 struct clocksource *clock;
229 s64 nsec;
169 230
170 /* read clocksource: */ 231 /* read clocksource: */
171 clock = timekeeper.clock; 232 clock = tk->clock;
172 cycle_now = clock->read(clock); 233 cycle_now = clock->read(clock);
173 234
174 /* calculate the delta since the last update_wall_time: */ 235 /* calculate the delta since the last update_wall_time: */
175 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 236 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
176 237
177 /* return delta convert to nanoseconds. */ 238 /* convert delta to nanoseconds. */
178 return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); 239 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
179}
180
181static void update_rt_offset(void)
182{
183 struct timespec tmp, *wtm = &timekeeper.wall_to_monotonic;
184 240
185 set_normalized_timespec(&tmp, -wtm->tv_sec, -wtm->tv_nsec); 241 /* If arch requires, add in gettimeoffset() */
186 timekeeper.offs_real = timespec_to_ktime(tmp); 242 return nsec + arch_gettimeoffset();
187} 243}
188 244
189/* must hold write on timekeeper.lock */ 245/* must hold write on timekeeper.lock */
190static void timekeeping_update(bool clearntp) 246static void timekeeping_update(struct timekeeper *tk, bool clearntp)
191{ 247{
248 struct timespec xt;
249
192 if (clearntp) { 250 if (clearntp) {
193 timekeeper.ntp_error = 0; 251 tk->ntp_error = 0;
194 ntp_clear(); 252 ntp_clear();
195 } 253 }
196 update_rt_offset(); 254 xt = tk_xtime(tk);
197 update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, 255 update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult);
198 timekeeper.clock, timekeeper.mult);
199} 256}
200 257
201
202/** 258/**
203 * timekeeping_forward_now - update clock to the current time 259 * timekeeping_forward_now - update clock to the current time
204 * 260 *
@@ -206,27 +262,26 @@ static void timekeeping_update(bool clearntp)
206 * update_wall_time(). This is useful before significant clock changes, 262 * update_wall_time(). This is useful before significant clock changes,
207 * as it avoids having to deal with this time offset explicitly. 263 * as it avoids having to deal with this time offset explicitly.
208 */ 264 */
209static void timekeeping_forward_now(void) 265static void timekeeping_forward_now(struct timekeeper *tk)
210{ 266{
211 cycle_t cycle_now, cycle_delta; 267 cycle_t cycle_now, cycle_delta;
212 struct clocksource *clock; 268 struct clocksource *clock;
213 s64 nsec; 269 s64 nsec;
214 270
215 clock = timekeeper.clock; 271 clock = tk->clock;
216 cycle_now = clock->read(clock); 272 cycle_now = clock->read(clock);
217 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 273 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
218 clock->cycle_last = cycle_now; 274 clock->cycle_last = cycle_now;
219 275
220 nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult, 276 tk->xtime_nsec += cycle_delta * tk->mult;
221 timekeeper.shift);
222 277
223 /* If arch requires, add in gettimeoffset() */ 278 /* If arch requires, add in gettimeoffset() */
224 nsec += arch_gettimeoffset(); 279 tk->xtime_nsec += arch_gettimeoffset() << tk->shift;
225 280
226 timespec_add_ns(&timekeeper.xtime, nsec); 281 tk_normalize_xtime(tk);
227 282
228 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); 283 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
229 timespec_add_ns(&timekeeper.raw_time, nsec); 284 timespec_add_ns(&tk->raw_time, nsec);
230} 285}
231 286
232/** 287/**
@@ -237,21 +292,19 @@ static void timekeeping_forward_now(void)
237 */ 292 */
238void getnstimeofday(struct timespec *ts) 293void getnstimeofday(struct timespec *ts)
239{ 294{
295 struct timekeeper *tk = &timekeeper;
240 unsigned long seq; 296 unsigned long seq;
241 s64 nsecs; 297 s64 nsecs = 0;
242 298
243 WARN_ON(timekeeping_suspended); 299 WARN_ON(timekeeping_suspended);
244 300
245 do { 301 do {
246 seq = read_seqbegin(&timekeeper.lock); 302 seq = read_seqbegin(&tk->lock);
247 303
248 *ts = timekeeper.xtime; 304 ts->tv_sec = tk->xtime_sec;
249 nsecs = timekeeping_get_ns(); 305 ts->tv_nsec = timekeeping_get_ns(tk);
250 306
251 /* If arch requires, add in gettimeoffset() */ 307 } while (read_seqretry(&tk->lock, seq));
252 nsecs += arch_gettimeoffset();
253
254 } while (read_seqretry(&timekeeper.lock, seq));
255 308
256 timespec_add_ns(ts, nsecs); 309 timespec_add_ns(ts, nsecs);
257} 310}
@@ -259,22 +312,18 @@ EXPORT_SYMBOL(getnstimeofday);
259 312
260ktime_t ktime_get(void) 313ktime_t ktime_get(void)
261{ 314{
315 struct timekeeper *tk = &timekeeper;
262 unsigned int seq; 316 unsigned int seq;
263 s64 secs, nsecs; 317 s64 secs, nsecs;
264 318
265 WARN_ON(timekeeping_suspended); 319 WARN_ON(timekeeping_suspended);
266 320
267 do { 321 do {
268 seq = read_seqbegin(&timekeeper.lock); 322 seq = read_seqbegin(&tk->lock);
269 secs = timekeeper.xtime.tv_sec + 323 secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
270 timekeeper.wall_to_monotonic.tv_sec; 324 nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec;
271 nsecs = timekeeper.xtime.tv_nsec + 325
272 timekeeper.wall_to_monotonic.tv_nsec; 326 } while (read_seqretry(&tk->lock, seq));
273 nsecs += timekeeping_get_ns();
274 /* If arch requires, add in gettimeoffset() */
275 nsecs += arch_gettimeoffset();
276
277 } while (read_seqretry(&timekeeper.lock, seq));
278 /* 327 /*
279 * Use ktime_set/ktime_add_ns to create a proper ktime on 328 * Use ktime_set/ktime_add_ns to create a proper ktime on
280 * 32-bit architectures without CONFIG_KTIME_SCALAR. 329 * 32-bit architectures without CONFIG_KTIME_SCALAR.
@@ -293,24 +342,22 @@ EXPORT_SYMBOL_GPL(ktime_get);
293 */ 342 */
294void ktime_get_ts(struct timespec *ts) 343void ktime_get_ts(struct timespec *ts)
295{ 344{
345 struct timekeeper *tk = &timekeeper;
296 struct timespec tomono; 346 struct timespec tomono;
297 unsigned int seq; 347 unsigned int seq;
298 s64 nsecs;
299 348
300 WARN_ON(timekeeping_suspended); 349 WARN_ON(timekeeping_suspended);
301 350
302 do { 351 do {
303 seq = read_seqbegin(&timekeeper.lock); 352 seq = read_seqbegin(&tk->lock);
304 *ts = timekeeper.xtime; 353 ts->tv_sec = tk->xtime_sec;
305 tomono = timekeeper.wall_to_monotonic; 354 ts->tv_nsec = timekeeping_get_ns(tk);
306 nsecs = timekeeping_get_ns(); 355 tomono = tk->wall_to_monotonic;
307 /* If arch requires, add in gettimeoffset() */
308 nsecs += arch_gettimeoffset();
309 356
310 } while (read_seqretry(&timekeeper.lock, seq)); 357 } while (read_seqretry(&tk->lock, seq));
311 358
312 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, 359 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
313 ts->tv_nsec + tomono.tv_nsec + nsecs); 360 ts->tv_nsec + tomono.tv_nsec);
314} 361}
315EXPORT_SYMBOL_GPL(ktime_get_ts); 362EXPORT_SYMBOL_GPL(ktime_get_ts);
316 363
@@ -327,28 +374,23 @@ EXPORT_SYMBOL_GPL(ktime_get_ts);
327 */ 374 */
328void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) 375void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
329{ 376{
377 struct timekeeper *tk = &timekeeper;
330 unsigned long seq; 378 unsigned long seq;
331 s64 nsecs_raw, nsecs_real; 379 s64 nsecs_raw, nsecs_real;
332 380
333 WARN_ON_ONCE(timekeeping_suspended); 381 WARN_ON_ONCE(timekeeping_suspended);
334 382
335 do { 383 do {
336 u32 arch_offset; 384 seq = read_seqbegin(&tk->lock);
337
338 seq = read_seqbegin(&timekeeper.lock);
339 385
340 *ts_raw = timekeeper.raw_time; 386 *ts_raw = tk->raw_time;
341 *ts_real = timekeeper.xtime; 387 ts_real->tv_sec = tk->xtime_sec;
388 ts_real->tv_nsec = 0;
342 389
343 nsecs_raw = timekeeping_get_ns_raw(); 390 nsecs_raw = timekeeping_get_ns_raw(tk);
344 nsecs_real = timekeeping_get_ns(); 391 nsecs_real = timekeeping_get_ns(tk);
345 392
346 /* If arch requires, add in gettimeoffset() */ 393 } while (read_seqretry(&tk->lock, seq));
347 arch_offset = arch_gettimeoffset();
348 nsecs_raw += arch_offset;
349 nsecs_real += arch_offset;
350
351 } while (read_seqretry(&timekeeper.lock, seq));
352 394
353 timespec_add_ns(ts_raw, nsecs_raw); 395 timespec_add_ns(ts_raw, nsecs_raw);
354 timespec_add_ns(ts_real, nsecs_real); 396 timespec_add_ns(ts_real, nsecs_real);
@@ -381,25 +423,28 @@ EXPORT_SYMBOL(do_gettimeofday);
381 */ 423 */
382int do_settimeofday(const struct timespec *tv) 424int do_settimeofday(const struct timespec *tv)
383{ 425{
384 struct timespec ts_delta; 426 struct timekeeper *tk = &timekeeper;
427 struct timespec ts_delta, xt;
385 unsigned long flags; 428 unsigned long flags;
386 429
387 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) 430 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
388 return -EINVAL; 431 return -EINVAL;
389 432
390 write_seqlock_irqsave(&timekeeper.lock, flags); 433 write_seqlock_irqsave(&tk->lock, flags);
434
435 timekeeping_forward_now(tk);
391 436
392 timekeeping_forward_now(); 437 xt = tk_xtime(tk);
438 ts_delta.tv_sec = tv->tv_sec - xt.tv_sec;
439 ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec;
393 440
394 ts_delta.tv_sec = tv->tv_sec - timekeeper.xtime.tv_sec; 441 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta));
395 ts_delta.tv_nsec = tv->tv_nsec - timekeeper.xtime.tv_nsec;
396 timekeeper.wall_to_monotonic =
397 timespec_sub(timekeeper.wall_to_monotonic, ts_delta);
398 442
399 timekeeper.xtime = *tv; 443 tk_set_xtime(tk, tv);
400 timekeeping_update(true);
401 444
402 write_sequnlock_irqrestore(&timekeeper.lock, flags); 445 timekeeping_update(tk, true);
446
447 write_sequnlock_irqrestore(&tk->lock, flags);
403 448
404 /* signal hrtimers about time change */ 449 /* signal hrtimers about time change */
405 clock_was_set(); 450 clock_was_set();
@@ -408,7 +453,6 @@ int do_settimeofday(const struct timespec *tv)
408} 453}
409EXPORT_SYMBOL(do_settimeofday); 454EXPORT_SYMBOL(do_settimeofday);
410 455
411
412/** 456/**
413 * timekeeping_inject_offset - Adds or subtracts from the current time. 457 * timekeeping_inject_offset - Adds or subtracts from the current time.
414 * @tv: pointer to the timespec variable containing the offset 458 * @tv: pointer to the timespec variable containing the offset
@@ -417,22 +461,23 @@ EXPORT_SYMBOL(do_settimeofday);
417 */ 461 */
418int timekeeping_inject_offset(struct timespec *ts) 462int timekeeping_inject_offset(struct timespec *ts)
419{ 463{
464 struct timekeeper *tk = &timekeeper;
420 unsigned long flags; 465 unsigned long flags;
421 466
422 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) 467 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
423 return -EINVAL; 468 return -EINVAL;
424 469
425 write_seqlock_irqsave(&timekeeper.lock, flags); 470 write_seqlock_irqsave(&tk->lock, flags);
426 471
427 timekeeping_forward_now(); 472 timekeeping_forward_now(tk);
428 473
429 timekeeper.xtime = timespec_add(timekeeper.xtime, *ts);
430 timekeeper.wall_to_monotonic =
431 timespec_sub(timekeeper.wall_to_monotonic, *ts);
432 474
433 timekeeping_update(true); 475 tk_xtime_add(tk, ts);
476 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts));
434 477
435 write_sequnlock_irqrestore(&timekeeper.lock, flags); 478 timekeeping_update(tk, true);
479
480 write_sequnlock_irqrestore(&tk->lock, flags);
436 481
437 /* signal hrtimers about time change */ 482 /* signal hrtimers about time change */
438 clock_was_set(); 483 clock_was_set();
@@ -448,23 +493,24 @@ EXPORT_SYMBOL(timekeeping_inject_offset);
448 */ 493 */
449static int change_clocksource(void *data) 494static int change_clocksource(void *data)
450{ 495{
496 struct timekeeper *tk = &timekeeper;
451 struct clocksource *new, *old; 497 struct clocksource *new, *old;
452 unsigned long flags; 498 unsigned long flags;
453 499
454 new = (struct clocksource *) data; 500 new = (struct clocksource *) data;
455 501
456 write_seqlock_irqsave(&timekeeper.lock, flags); 502 write_seqlock_irqsave(&tk->lock, flags);
457 503
458 timekeeping_forward_now(); 504 timekeeping_forward_now(tk);
459 if (!new->enable || new->enable(new) == 0) { 505 if (!new->enable || new->enable(new) == 0) {
460 old = timekeeper.clock; 506 old = tk->clock;
461 timekeeper_setup_internals(new); 507 tk_setup_internals(tk, new);
462 if (old->disable) 508 if (old->disable)
463 old->disable(old); 509 old->disable(old);
464 } 510 }
465 timekeeping_update(true); 511 timekeeping_update(tk, true);
466 512
467 write_sequnlock_irqrestore(&timekeeper.lock, flags); 513 write_sequnlock_irqrestore(&tk->lock, flags);
468 514
469 return 0; 515 return 0;
470} 516}
@@ -478,7 +524,9 @@ static int change_clocksource(void *data)
478 */ 524 */
479void timekeeping_notify(struct clocksource *clock) 525void timekeeping_notify(struct clocksource *clock)
480{ 526{
481 if (timekeeper.clock == clock) 527 struct timekeeper *tk = &timekeeper;
528
529 if (tk->clock == clock)
482 return; 530 return;
483 stop_machine(change_clocksource, clock, NULL); 531 stop_machine(change_clocksource, clock, NULL);
484 tick_clock_notify(); 532 tick_clock_notify();
@@ -507,35 +555,36 @@ EXPORT_SYMBOL_GPL(ktime_get_real);
507 */ 555 */
508void getrawmonotonic(struct timespec *ts) 556void getrawmonotonic(struct timespec *ts)
509{ 557{
558 struct timekeeper *tk = &timekeeper;
510 unsigned long seq; 559 unsigned long seq;
511 s64 nsecs; 560 s64 nsecs;
512 561
513 do { 562 do {
514 seq = read_seqbegin(&timekeeper.lock); 563 seq = read_seqbegin(&tk->lock);
515 nsecs = timekeeping_get_ns_raw(); 564 nsecs = timekeeping_get_ns_raw(tk);
516 *ts = timekeeper.raw_time; 565 *ts = tk->raw_time;
517 566
518 } while (read_seqretry(&timekeeper.lock, seq)); 567 } while (read_seqretry(&tk->lock, seq));
519 568
520 timespec_add_ns(ts, nsecs); 569 timespec_add_ns(ts, nsecs);
521} 570}
522EXPORT_SYMBOL(getrawmonotonic); 571EXPORT_SYMBOL(getrawmonotonic);
523 572
524
525/** 573/**
526 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres 574 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
527 */ 575 */
528int timekeeping_valid_for_hres(void) 576int timekeeping_valid_for_hres(void)
529{ 577{
578 struct timekeeper *tk = &timekeeper;
530 unsigned long seq; 579 unsigned long seq;
531 int ret; 580 int ret;
532 581
533 do { 582 do {
534 seq = read_seqbegin(&timekeeper.lock); 583 seq = read_seqbegin(&tk->lock);
535 584
536 ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 585 ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
537 586
538 } while (read_seqretry(&timekeeper.lock, seq)); 587 } while (read_seqretry(&tk->lock, seq));
539 588
540 return ret; 589 return ret;
541} 590}
@@ -545,14 +594,16 @@ int timekeeping_valid_for_hres(void)
545 */ 594 */
546u64 timekeeping_max_deferment(void) 595u64 timekeeping_max_deferment(void)
547{ 596{
597 struct timekeeper *tk = &timekeeper;
548 unsigned long seq; 598 unsigned long seq;
549 u64 ret; 599 u64 ret;
600
550 do { 601 do {
551 seq = read_seqbegin(&timekeeper.lock); 602 seq = read_seqbegin(&tk->lock);
552 603
553 ret = timekeeper.clock->max_idle_ns; 604 ret = tk->clock->max_idle_ns;
554 605
555 } while (read_seqretry(&timekeeper.lock, seq)); 606 } while (read_seqretry(&tk->lock, seq));
556 607
557 return ret; 608 return ret;
558} 609}
@@ -592,48 +643,43 @@ void __attribute__((weak)) read_boot_clock(struct timespec *ts)
592 */ 643 */
593void __init timekeeping_init(void) 644void __init timekeeping_init(void)
594{ 645{
646 struct timekeeper *tk = &timekeeper;
595 struct clocksource *clock; 647 struct clocksource *clock;
596 unsigned long flags; 648 unsigned long flags;
597 struct timespec now, boot; 649 struct timespec now, boot, tmp;
598 650
599 read_persistent_clock(&now); 651 read_persistent_clock(&now);
600 read_boot_clock(&boot); 652 read_boot_clock(&boot);
601 653
602 seqlock_init(&timekeeper.lock); 654 seqlock_init(&tk->lock);
603 655
604 ntp_init(); 656 ntp_init();
605 657
606 write_seqlock_irqsave(&timekeeper.lock, flags); 658 write_seqlock_irqsave(&tk->lock, flags);
607 clock = clocksource_default_clock(); 659 clock = clocksource_default_clock();
608 if (clock->enable) 660 if (clock->enable)
609 clock->enable(clock); 661 clock->enable(clock);
610 timekeeper_setup_internals(clock); 662 tk_setup_internals(tk, clock);
611 663
612 timekeeper.xtime.tv_sec = now.tv_sec; 664 tk_set_xtime(tk, &now);
613 timekeeper.xtime.tv_nsec = now.tv_nsec; 665 tk->raw_time.tv_sec = 0;
614 timekeeper.raw_time.tv_sec = 0; 666 tk->raw_time.tv_nsec = 0;
615 timekeeper.raw_time.tv_nsec = 0; 667 if (boot.tv_sec == 0 && boot.tv_nsec == 0)
616 if (boot.tv_sec == 0 && boot.tv_nsec == 0) { 668 boot = tk_xtime(tk);
617 boot.tv_sec = timekeeper.xtime.tv_sec; 669
618 boot.tv_nsec = timekeeper.xtime.tv_nsec; 670 set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec);
619 } 671 tk_set_wall_to_mono(tk, tmp);
620 set_normalized_timespec(&timekeeper.wall_to_monotonic, 672
621 -boot.tv_sec, -boot.tv_nsec); 673 tmp.tv_sec = 0;
622 update_rt_offset(); 674 tmp.tv_nsec = 0;
623 timekeeper.total_sleep_time.tv_sec = 0; 675 tk_set_sleep_time(tk, tmp);
624 timekeeper.total_sleep_time.tv_nsec = 0; 676
625 write_sequnlock_irqrestore(&timekeeper.lock, flags); 677 write_sequnlock_irqrestore(&tk->lock, flags);
626} 678}
627 679
628/* time in seconds when suspend began */ 680/* time in seconds when suspend began */
629static struct timespec timekeeping_suspend_time; 681static struct timespec timekeeping_suspend_time;
630 682
631static void update_sleep_time(struct timespec t)
632{
633 timekeeper.total_sleep_time = t;
634 timekeeper.offs_boot = timespec_to_ktime(t);
635}
636
637/** 683/**
638 * __timekeeping_inject_sleeptime - Internal function to add sleep interval 684 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
639 * @delta: pointer to a timespec delta value 685 * @delta: pointer to a timespec delta value
@@ -641,21 +687,19 @@ static void update_sleep_time(struct timespec t)
641 * Takes a timespec offset measuring a suspend interval and properly 687 * Takes a timespec offset measuring a suspend interval and properly
642 * adds the sleep offset to the timekeeping variables. 688 * adds the sleep offset to the timekeeping variables.
643 */ 689 */
644static void __timekeeping_inject_sleeptime(struct timespec *delta) 690static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
691 struct timespec *delta)
645{ 692{
646 if (!timespec_valid(delta)) { 693 if (!timespec_valid(delta)) {
647 printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid " 694 printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid "
648 "sleep delta value!\n"); 695 "sleep delta value!\n");
649 return; 696 return;
650 } 697 }
651 698 tk_xtime_add(tk, delta);
652 timekeeper.xtime = timespec_add(timekeeper.xtime, *delta); 699 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta));
653 timekeeper.wall_to_monotonic = 700 tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta));
654 timespec_sub(timekeeper.wall_to_monotonic, *delta);
655 update_sleep_time(timespec_add(timekeeper.total_sleep_time, *delta));
656} 701}
657 702
658
659/** 703/**
660 * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values 704 * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values
661 * @delta: pointer to a timespec delta value 705 * @delta: pointer to a timespec delta value
@@ -668,6 +712,7 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta)
668 */ 712 */
669void timekeeping_inject_sleeptime(struct timespec *delta) 713void timekeeping_inject_sleeptime(struct timespec *delta)
670{ 714{
715 struct timekeeper *tk = &timekeeper;
671 unsigned long flags; 716 unsigned long flags;
672 struct timespec ts; 717 struct timespec ts;
673 718
@@ -676,21 +721,20 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
676 if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) 721 if (!(ts.tv_sec == 0 && ts.tv_nsec == 0))
677 return; 722 return;
678 723
679 write_seqlock_irqsave(&timekeeper.lock, flags); 724 write_seqlock_irqsave(&tk->lock, flags);
680 725
681 timekeeping_forward_now(); 726 timekeeping_forward_now(tk);
682 727
683 __timekeeping_inject_sleeptime(delta); 728 __timekeeping_inject_sleeptime(tk, delta);
684 729
685 timekeeping_update(true); 730 timekeeping_update(tk, true);
686 731
687 write_sequnlock_irqrestore(&timekeeper.lock, flags); 732 write_sequnlock_irqrestore(&tk->lock, flags);
688 733
689 /* signal hrtimers about time change */ 734 /* signal hrtimers about time change */
690 clock_was_set(); 735 clock_was_set();
691} 736}
692 737
693
694/** 738/**
695 * timekeeping_resume - Resumes the generic timekeeping subsystem. 739 * timekeeping_resume - Resumes the generic timekeeping subsystem.
696 * 740 *
@@ -700,6 +744,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
700 */ 744 */
701static void timekeeping_resume(void) 745static void timekeeping_resume(void)
702{ 746{
747 struct timekeeper *tk = &timekeeper;
703 unsigned long flags; 748 unsigned long flags;
704 struct timespec ts; 749 struct timespec ts;
705 750
@@ -707,17 +752,18 @@ static void timekeeping_resume(void)
707 752
708 clocksource_resume(); 753 clocksource_resume();
709 754
710 write_seqlock_irqsave(&timekeeper.lock, flags); 755 write_seqlock_irqsave(&tk->lock, flags);
711 756
712 if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { 757 if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
713 ts = timespec_sub(ts, timekeeping_suspend_time); 758 ts = timespec_sub(ts, timekeeping_suspend_time);
714 __timekeeping_inject_sleeptime(&ts); 759 __timekeeping_inject_sleeptime(tk, &ts);
715 } 760 }
716 /* re-base the last cycle value */ 761 /* re-base the last cycle value */
717 timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); 762 tk->clock->cycle_last = tk->clock->read(tk->clock);
718 timekeeper.ntp_error = 0; 763 tk->ntp_error = 0;
719 timekeeping_suspended = 0; 764 timekeeping_suspended = 0;
720 write_sequnlock_irqrestore(&timekeeper.lock, flags); 765 timekeeping_update(tk, false);
766 write_sequnlock_irqrestore(&tk->lock, flags);
721 767
722 touch_softlockup_watchdog(); 768 touch_softlockup_watchdog();
723 769
@@ -729,14 +775,15 @@ static void timekeeping_resume(void)
729 775
730static int timekeeping_suspend(void) 776static int timekeeping_suspend(void)
731{ 777{
778 struct timekeeper *tk = &timekeeper;
732 unsigned long flags; 779 unsigned long flags;
733 struct timespec delta, delta_delta; 780 struct timespec delta, delta_delta;
734 static struct timespec old_delta; 781 static struct timespec old_delta;
735 782
736 read_persistent_clock(&timekeeping_suspend_time); 783 read_persistent_clock(&timekeeping_suspend_time);
737 784
738 write_seqlock_irqsave(&timekeeper.lock, flags); 785 write_seqlock_irqsave(&tk->lock, flags);
739 timekeeping_forward_now(); 786 timekeeping_forward_now(tk);
740 timekeeping_suspended = 1; 787 timekeeping_suspended = 1;
741 788
742 /* 789 /*
@@ -745,7 +792,7 @@ static int timekeeping_suspend(void)
745 * try to compensate so the difference in system time 792 * try to compensate so the difference in system time
746 * and persistent_clock time stays close to constant. 793 * and persistent_clock time stays close to constant.
747 */ 794 */
748 delta = timespec_sub(timekeeper.xtime, timekeeping_suspend_time); 795 delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time);
749 delta_delta = timespec_sub(delta, old_delta); 796 delta_delta = timespec_sub(delta, old_delta);
750 if (abs(delta_delta.tv_sec) >= 2) { 797 if (abs(delta_delta.tv_sec) >= 2) {
751 /* 798 /*
@@ -758,7 +805,7 @@ static int timekeeping_suspend(void)
758 timekeeping_suspend_time = 805 timekeeping_suspend_time =
759 timespec_add(timekeeping_suspend_time, delta_delta); 806 timespec_add(timekeeping_suspend_time, delta_delta);
760 } 807 }
761 write_sequnlock_irqrestore(&timekeeper.lock, flags); 808 write_sequnlock_irqrestore(&tk->lock, flags);
762 809
763 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); 810 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
764 clocksource_suspend(); 811 clocksource_suspend();
@@ -784,7 +831,8 @@ device_initcall(timekeeping_init_ops);
784 * If the error is already larger, we look ahead even further 831 * If the error is already larger, we look ahead even further
785 * to compensate for late or lost adjustments. 832 * to compensate for late or lost adjustments.
786 */ 833 */
787static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval, 834static __always_inline int timekeeping_bigadjust(struct timekeeper *tk,
835 s64 error, s64 *interval,
788 s64 *offset) 836 s64 *offset)
789{ 837{
790 s64 tick_error, i; 838 s64 tick_error, i;
@@ -800,7 +848,7 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
800 * here. This is tuned so that an error of about 1 msec is adjusted 848 * here. This is tuned so that an error of about 1 msec is adjusted
801 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks). 849 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
802 */ 850 */
803 error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ); 851 error2 = tk->ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
804 error2 = abs(error2); 852 error2 = abs(error2);
805 for (look_ahead = 0; error2 > 0; look_ahead++) 853 for (look_ahead = 0; error2 > 0; look_ahead++)
806 error2 >>= 2; 854 error2 >>= 2;
@@ -809,8 +857,8 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
809 * Now calculate the error in (1 << look_ahead) ticks, but first 857 * Now calculate the error in (1 << look_ahead) ticks, but first
810 * remove the single look ahead already included in the error. 858 * remove the single look ahead already included in the error.
811 */ 859 */
812 tick_error = ntp_tick_length() >> (timekeeper.ntp_error_shift + 1); 860 tick_error = ntp_tick_length() >> (tk->ntp_error_shift + 1);
813 tick_error -= timekeeper.xtime_interval >> 1; 861 tick_error -= tk->xtime_interval >> 1;
814 error = ((error - tick_error) >> look_ahead) + tick_error; 862 error = ((error - tick_error) >> look_ahead) + tick_error;
815 863
816 /* Finally calculate the adjustment shift value. */ 864 /* Finally calculate the adjustment shift value. */
@@ -835,9 +883,9 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
835 * this is optimized for the most common adjustments of -1,0,1, 883 * this is optimized for the most common adjustments of -1,0,1,
836 * for other values we can do a bit more work. 884 * for other values we can do a bit more work.
837 */ 885 */
838static void timekeeping_adjust(s64 offset) 886static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
839{ 887{
840 s64 error, interval = timekeeper.cycle_interval; 888 s64 error, interval = tk->cycle_interval;
841 int adj; 889 int adj;
842 890
843 /* 891 /*
@@ -853,7 +901,7 @@ static void timekeeping_adjust(s64 offset)
853 * 901 *
854 * Note: It does not "save" on aggravation when reading the code. 902 * Note: It does not "save" on aggravation when reading the code.
855 */ 903 */
856 error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1); 904 error = tk->ntp_error >> (tk->ntp_error_shift - 1);
857 if (error > interval) { 905 if (error > interval) {
858 /* 906 /*
859 * We now divide error by 4(via shift), which checks if 907 * We now divide error by 4(via shift), which checks if
@@ -868,34 +916,36 @@ static void timekeeping_adjust(s64 offset)
868 * the error. This causes the likely below to be unlikely. 916 * the error. This causes the likely below to be unlikely.
869 * 917 *
870 * The proper fix is to avoid rounding up by using 918 * The proper fix is to avoid rounding up by using
871 * the high precision timekeeper.xtime_nsec instead of 919 * the high precision tk->xtime_nsec instead of
872 * xtime.tv_nsec everywhere. Fixing this will take some 920 * xtime.tv_nsec everywhere. Fixing this will take some
873 * time. 921 * time.
874 */ 922 */
875 if (likely(error <= interval)) 923 if (likely(error <= interval))
876 adj = 1; 924 adj = 1;
877 else 925 else
878 adj = timekeeping_bigadjust(error, &interval, &offset); 926 adj = timekeeping_bigadjust(tk, error, &interval, &offset);
879 } else if (error < -interval) { 927 } else {
880 /* See comment above, this is just switched for the negative */ 928 if (error < -interval) {
881 error >>= 2; 929 /* See comment above, this is just switched for the negative */
882 if (likely(error >= -interval)) { 930 error >>= 2;
883 adj = -1; 931 if (likely(error >= -interval)) {
884 interval = -interval; 932 adj = -1;
885 offset = -offset; 933 interval = -interval;
886 } else 934 offset = -offset;
887 adj = timekeeping_bigadjust(error, &interval, &offset); 935 } else {
888 } else /* No adjustment needed */ 936 adj = timekeeping_bigadjust(tk, error, &interval, &offset);
889 return; 937 }
938 } else {
939 goto out_adjust;
940 }
941 }
890 942
891 if (unlikely(timekeeper.clock->maxadj && 943 if (unlikely(tk->clock->maxadj &&
892 (timekeeper.mult + adj > 944 (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) {
893 timekeeper.clock->mult + timekeeper.clock->maxadj))) {
894 printk_once(KERN_WARNING 945 printk_once(KERN_WARNING
895 "Adjusting %s more than 11%% (%ld vs %ld)\n", 946 "Adjusting %s more than 11%% (%ld vs %ld)\n",
896 timekeeper.clock->name, (long)timekeeper.mult + adj, 947 tk->clock->name, (long)tk->mult + adj,
897 (long)timekeeper.clock->mult + 948 (long)tk->clock->mult + tk->clock->maxadj);
898 timekeeper.clock->maxadj);
899 } 949 }
900 /* 950 /*
901 * So the following can be confusing. 951 * So the following can be confusing.
@@ -946,13 +996,68 @@ static void timekeeping_adjust(s64 offset)
946 * 996 *
947 * XXX - TODO: Doc ntp_error calculation. 997 * XXX - TODO: Doc ntp_error calculation.
948 */ 998 */
949 timekeeper.mult += adj; 999 tk->mult += adj;
950 timekeeper.xtime_interval += interval; 1000 tk->xtime_interval += interval;
951 timekeeper.xtime_nsec -= offset; 1001 tk->xtime_nsec -= offset;
952 timekeeper.ntp_error -= (interval - offset) << 1002 tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
953 timekeeper.ntp_error_shift; 1003
1004out_adjust:
1005 /*
1006 * It may be possible that when we entered this function, xtime_nsec
1007 * was very small. Further, if we're slightly speeding the clocksource
1008 * in the code above, its possible the required corrective factor to
1009 * xtime_nsec could cause it to underflow.
1010 *
1011 * Now, since we already accumulated the second, cannot simply roll
1012 * the accumulated second back, since the NTP subsystem has been
1013 * notified via second_overflow. So instead we push xtime_nsec forward
1014 * by the amount we underflowed, and add that amount into the error.
1015 *
1016 * We'll correct this error next time through this function, when
1017 * xtime_nsec is not as small.
1018 */
1019 if (unlikely((s64)tk->xtime_nsec < 0)) {
1020 s64 neg = -(s64)tk->xtime_nsec;
1021 tk->xtime_nsec = 0;
1022 tk->ntp_error += neg << tk->ntp_error_shift;
1023 }
1024
954} 1025}
955 1026
1027/**
1028 * accumulate_nsecs_to_secs - Accumulates nsecs into secs
1029 *
1030 * Helper function that accumulates a the nsecs greater then a second
1031 * from the xtime_nsec field to the xtime_secs field.
1032 * It also calls into the NTP code to handle leapsecond processing.
1033 *
1034 */
1035static inline void accumulate_nsecs_to_secs(struct timekeeper *tk)
1036{
1037 u64 nsecps = (u64)NSEC_PER_SEC << tk->shift;
1038
1039 while (tk->xtime_nsec >= nsecps) {
1040 int leap;
1041
1042 tk->xtime_nsec -= nsecps;
1043 tk->xtime_sec++;
1044
1045 /* Figure out if its a leap sec and apply if needed */
1046 leap = second_overflow(tk->xtime_sec);
1047 if (unlikely(leap)) {
1048 struct timespec ts;
1049
1050 tk->xtime_sec += leap;
1051
1052 ts.tv_sec = leap;
1053 ts.tv_nsec = 0;
1054 tk_set_wall_to_mono(tk,
1055 timespec_sub(tk->wall_to_monotonic, ts));
1056
1057 clock_was_set_delayed();
1058 }
1059 }
1060}
956 1061
957/** 1062/**
958 * logarithmic_accumulation - shifted accumulation of cycles 1063 * logarithmic_accumulation - shifted accumulation of cycles
@@ -963,51 +1068,40 @@ static void timekeeping_adjust(s64 offset)
963 * 1068 *
964 * Returns the unconsumed cycles. 1069 * Returns the unconsumed cycles.
965 */ 1070 */
966static cycle_t logarithmic_accumulation(cycle_t offset, int shift) 1071static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1072 u32 shift)
967{ 1073{
968 u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
969 u64 raw_nsecs; 1074 u64 raw_nsecs;
970 1075
971 /* If the offset is smaller than a shifted interval, do nothing */ 1076 /* If the offset is smaller then a shifted interval, do nothing */
972 if (offset < timekeeper.cycle_interval<<shift) 1077 if (offset < tk->cycle_interval<<shift)
973 return offset; 1078 return offset;
974 1079
975 /* Accumulate one shifted interval */ 1080 /* Accumulate one shifted interval */
976 offset -= timekeeper.cycle_interval << shift; 1081 offset -= tk->cycle_interval << shift;
977 timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift; 1082 tk->clock->cycle_last += tk->cycle_interval << shift;
978 1083
979 timekeeper.xtime_nsec += timekeeper.xtime_interval << shift; 1084 tk->xtime_nsec += tk->xtime_interval << shift;
980 while (timekeeper.xtime_nsec >= nsecps) { 1085 accumulate_nsecs_to_secs(tk);
981 int leap;
982 timekeeper.xtime_nsec -= nsecps;
983 timekeeper.xtime.tv_sec++;
984 leap = second_overflow(timekeeper.xtime.tv_sec);
985 timekeeper.xtime.tv_sec += leap;
986 timekeeper.wall_to_monotonic.tv_sec -= leap;
987 if (leap)
988 clock_was_set_delayed();
989 }
990 1086
991 /* Accumulate raw time */ 1087 /* Accumulate raw time */
992 raw_nsecs = timekeeper.raw_interval << shift; 1088 raw_nsecs = tk->raw_interval << shift;
993 raw_nsecs += timekeeper.raw_time.tv_nsec; 1089 raw_nsecs += tk->raw_time.tv_nsec;
994 if (raw_nsecs >= NSEC_PER_SEC) { 1090 if (raw_nsecs >= NSEC_PER_SEC) {
995 u64 raw_secs = raw_nsecs; 1091 u64 raw_secs = raw_nsecs;
996 raw_nsecs = do_div(raw_secs, NSEC_PER_SEC); 1092 raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
997 timekeeper.raw_time.tv_sec += raw_secs; 1093 tk->raw_time.tv_sec += raw_secs;
998 } 1094 }
999 timekeeper.raw_time.tv_nsec = raw_nsecs; 1095 tk->raw_time.tv_nsec = raw_nsecs;
1000 1096
1001 /* Accumulate error between NTP and clock interval */ 1097 /* Accumulate error between NTP and clock interval */
1002 timekeeper.ntp_error += ntp_tick_length() << shift; 1098 tk->ntp_error += ntp_tick_length() << shift;
1003 timekeeper.ntp_error -= 1099 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
1004 (timekeeper.xtime_interval + timekeeper.xtime_remainder) << 1100 (tk->ntp_error_shift + shift);
1005 (timekeeper.ntp_error_shift + shift);
1006 1101
1007 return offset; 1102 return offset;
1008} 1103}
1009 1104
1010
1011/** 1105/**
1012 * update_wall_time - Uses the current clocksource to increment the wall time 1106 * update_wall_time - Uses the current clocksource to increment the wall time
1013 * 1107 *
@@ -1015,25 +1109,25 @@ static cycle_t logarithmic_accumulation(cycle_t offset, int shift)
1015static void update_wall_time(void) 1109static void update_wall_time(void)
1016{ 1110{
1017 struct clocksource *clock; 1111 struct clocksource *clock;
1112 struct timekeeper *tk = &timekeeper;
1018 cycle_t offset; 1113 cycle_t offset;
1019 int shift = 0, maxshift; 1114 int shift = 0, maxshift;
1020 unsigned long flags; 1115 unsigned long flags;
1116 s64 remainder;
1021 1117
1022 write_seqlock_irqsave(&timekeeper.lock, flags); 1118 write_seqlock_irqsave(&tk->lock, flags);
1023 1119
1024 /* Make sure we're fully resumed: */ 1120 /* Make sure we're fully resumed: */
1025 if (unlikely(timekeeping_suspended)) 1121 if (unlikely(timekeeping_suspended))
1026 goto out; 1122 goto out;
1027 1123
1028 clock = timekeeper.clock; 1124 clock = tk->clock;
1029 1125
1030#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET 1126#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
1031 offset = timekeeper.cycle_interval; 1127 offset = tk->cycle_interval;
1032#else 1128#else
1033 offset = (clock->read(clock) - clock->cycle_last) & clock->mask; 1129 offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
1034#endif 1130#endif
1035 timekeeper.xtime_nsec = (s64)timekeeper.xtime.tv_nsec <<
1036 timekeeper.shift;
1037 1131
1038 /* 1132 /*
1039 * With NO_HZ we may have to accumulate many cycle_intervals 1133 * With NO_HZ we may have to accumulate many cycle_intervals
@@ -1043,73 +1137,45 @@ static void update_wall_time(void)
1043 * chunk in one go, and then try to consume the next smaller 1137 * chunk in one go, and then try to consume the next smaller
1044 * doubled multiple. 1138 * doubled multiple.
1045 */ 1139 */
1046 shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); 1140 shift = ilog2(offset) - ilog2(tk->cycle_interval);
1047 shift = max(0, shift); 1141 shift = max(0, shift);
1048 /* Bound shift to one less than what overflows tick_length */ 1142 /* Bound shift to one less than what overflows tick_length */
1049 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; 1143 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
1050 shift = min(shift, maxshift); 1144 shift = min(shift, maxshift);
1051 while (offset >= timekeeper.cycle_interval) { 1145 while (offset >= tk->cycle_interval) {
1052 offset = logarithmic_accumulation(offset, shift); 1146 offset = logarithmic_accumulation(tk, offset, shift);
1053 if(offset < timekeeper.cycle_interval<<shift) 1147 if (offset < tk->cycle_interval<<shift)
1054 shift--; 1148 shift--;
1055 } 1149 }
1056 1150
1057 /* correct the clock when NTP error is too big */ 1151 /* correct the clock when NTP error is too big */
1058 timekeeping_adjust(offset); 1152 timekeeping_adjust(tk, offset);
1059
1060 /*
1061 * Since in the loop above, we accumulate any amount of time
1062 * in xtime_nsec over a second into xtime.tv_sec, its possible for
1063 * xtime_nsec to be fairly small after the loop. Further, if we're
1064 * slightly speeding the clocksource up in timekeeping_adjust(),
1065 * its possible the required corrective factor to xtime_nsec could
1066 * cause it to underflow.
1067 *
1068 * Now, we cannot simply roll the accumulated second back, since
1069 * the NTP subsystem has been notified via second_overflow. So
1070 * instead we push xtime_nsec forward by the amount we underflowed,
1071 * and add that amount into the error.
1072 *
1073 * We'll correct this error next time through this function, when
1074 * xtime_nsec is not as small.
1075 */
1076 if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
1077 s64 neg = -(s64)timekeeper.xtime_nsec;
1078 timekeeper.xtime_nsec = 0;
1079 timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
1080 }
1081 1153
1082 1154
1083 /* 1155 /*
1084 * Store full nanoseconds into xtime after rounding it up and 1156 * Store only full nanoseconds into xtime_nsec after rounding
1085 * add the remainder to the error difference. 1157 * it up and add the remainder to the error difference.
1086 */ 1158 * XXX - This is necessary to avoid small 1ns inconsistnecies caused
1087 timekeeper.xtime.tv_nsec = ((s64)timekeeper.xtime_nsec >> 1159 * by truncating the remainder in vsyscalls. However, it causes
1088 timekeeper.shift) + 1; 1160 * additional work to be done in timekeeping_adjust(). Once
1089 timekeeper.xtime_nsec -= (s64)timekeeper.xtime.tv_nsec << 1161 * the vsyscall implementations are converted to use xtime_nsec
1090 timekeeper.shift; 1162 * (shifted nanoseconds), this can be killed.
1091 timekeeper.ntp_error += timekeeper.xtime_nsec << 1163 */
1092 timekeeper.ntp_error_shift; 1164 remainder = tk->xtime_nsec & ((1 << tk->shift) - 1);
1165 tk->xtime_nsec -= remainder;
1166 tk->xtime_nsec += 1 << tk->shift;
1167 tk->ntp_error += remainder << tk->ntp_error_shift;
1093 1168
1094 /* 1169 /*
1095 * Finally, make sure that after the rounding 1170 * Finally, make sure that after the rounding
1096 * xtime.tv_nsec isn't larger than NSEC_PER_SEC 1171 * xtime_nsec isn't larger than NSEC_PER_SEC
1097 */ 1172 */
1098 if (unlikely(timekeeper.xtime.tv_nsec >= NSEC_PER_SEC)) { 1173 accumulate_nsecs_to_secs(tk);
1099 int leap;
1100 timekeeper.xtime.tv_nsec -= NSEC_PER_SEC;
1101 timekeeper.xtime.tv_sec++;
1102 leap = second_overflow(timekeeper.xtime.tv_sec);
1103 timekeeper.xtime.tv_sec += leap;
1104 timekeeper.wall_to_monotonic.tv_sec -= leap;
1105 if (leap)
1106 clock_was_set_delayed();
1107 }
1108 1174
1109 timekeeping_update(false); 1175 timekeeping_update(tk, false);
1110 1176
1111out: 1177out:
1112 write_sequnlock_irqrestore(&timekeeper.lock, flags); 1178 write_sequnlock_irqrestore(&tk->lock, flags);
1113 1179
1114} 1180}
1115 1181
@@ -1126,18 +1192,18 @@ out:
1126 */ 1192 */
1127void getboottime(struct timespec *ts) 1193void getboottime(struct timespec *ts)
1128{ 1194{
1195 struct timekeeper *tk = &timekeeper;
1129 struct timespec boottime = { 1196 struct timespec boottime = {
1130 .tv_sec = timekeeper.wall_to_monotonic.tv_sec + 1197 .tv_sec = tk->wall_to_monotonic.tv_sec +
1131 timekeeper.total_sleep_time.tv_sec, 1198 tk->total_sleep_time.tv_sec,
1132 .tv_nsec = timekeeper.wall_to_monotonic.tv_nsec + 1199 .tv_nsec = tk->wall_to_monotonic.tv_nsec +
1133 timekeeper.total_sleep_time.tv_nsec 1200 tk->total_sleep_time.tv_nsec
1134 }; 1201 };
1135 1202
1136 set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); 1203 set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec);
1137} 1204}
1138EXPORT_SYMBOL_GPL(getboottime); 1205EXPORT_SYMBOL_GPL(getboottime);
1139 1206
1140
1141/** 1207/**
1142 * get_monotonic_boottime - Returns monotonic time since boot 1208 * get_monotonic_boottime - Returns monotonic time since boot
1143 * @ts: pointer to the timespec to be set 1209 * @ts: pointer to the timespec to be set
@@ -1149,23 +1215,23 @@ EXPORT_SYMBOL_GPL(getboottime);
1149 */ 1215 */
1150void get_monotonic_boottime(struct timespec *ts) 1216void get_monotonic_boottime(struct timespec *ts)
1151{ 1217{
1218 struct timekeeper *tk = &timekeeper;
1152 struct timespec tomono, sleep; 1219 struct timespec tomono, sleep;
1153 unsigned int seq; 1220 unsigned int seq;
1154 s64 nsecs;
1155 1221
1156 WARN_ON(timekeeping_suspended); 1222 WARN_ON(timekeeping_suspended);
1157 1223
1158 do { 1224 do {
1159 seq = read_seqbegin(&timekeeper.lock); 1225 seq = read_seqbegin(&tk->lock);
1160 *ts = timekeeper.xtime; 1226 ts->tv_sec = tk->xtime_sec;
1161 tomono = timekeeper.wall_to_monotonic; 1227 ts->tv_nsec = timekeeping_get_ns(tk);
1162 sleep = timekeeper.total_sleep_time; 1228 tomono = tk->wall_to_monotonic;
1163 nsecs = timekeeping_get_ns(); 1229 sleep = tk->total_sleep_time;
1164 1230
1165 } while (read_seqretry(&timekeeper.lock, seq)); 1231 } while (read_seqretry(&tk->lock, seq));
1166 1232
1167 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, 1233 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec,
1168 ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs); 1234 ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec);
1169} 1235}
1170EXPORT_SYMBOL_GPL(get_monotonic_boottime); 1236EXPORT_SYMBOL_GPL(get_monotonic_boottime);
1171 1237
@@ -1192,31 +1258,38 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime);
1192 */ 1258 */
1193void monotonic_to_bootbased(struct timespec *ts) 1259void monotonic_to_bootbased(struct timespec *ts)
1194{ 1260{
1195 *ts = timespec_add(*ts, timekeeper.total_sleep_time); 1261 struct timekeeper *tk = &timekeeper;
1262
1263 *ts = timespec_add(*ts, tk->total_sleep_time);
1196} 1264}
1197EXPORT_SYMBOL_GPL(monotonic_to_bootbased); 1265EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
1198 1266
1199unsigned long get_seconds(void) 1267unsigned long get_seconds(void)
1200{ 1268{
1201 return timekeeper.xtime.tv_sec; 1269 struct timekeeper *tk = &timekeeper;
1270
1271 return tk->xtime_sec;
1202} 1272}
1203EXPORT_SYMBOL(get_seconds); 1273EXPORT_SYMBOL(get_seconds);
1204 1274
1205struct timespec __current_kernel_time(void) 1275struct timespec __current_kernel_time(void)
1206{ 1276{
1207 return timekeeper.xtime; 1277 struct timekeeper *tk = &timekeeper;
1278
1279 return tk_xtime(tk);
1208} 1280}
1209 1281
1210struct timespec current_kernel_time(void) 1282struct timespec current_kernel_time(void)
1211{ 1283{
1284 struct timekeeper *tk = &timekeeper;
1212 struct timespec now; 1285 struct timespec now;
1213 unsigned long seq; 1286 unsigned long seq;
1214 1287
1215 do { 1288 do {
1216 seq = read_seqbegin(&timekeeper.lock); 1289 seq = read_seqbegin(&tk->lock);
1217 1290
1218 now = timekeeper.xtime; 1291 now = tk_xtime(tk);
1219 } while (read_seqretry(&timekeeper.lock, seq)); 1292 } while (read_seqretry(&tk->lock, seq));
1220 1293
1221 return now; 1294 return now;
1222} 1295}
@@ -1224,15 +1297,16 @@ EXPORT_SYMBOL(current_kernel_time);
1224 1297
1225struct timespec get_monotonic_coarse(void) 1298struct timespec get_monotonic_coarse(void)
1226{ 1299{
1300 struct timekeeper *tk = &timekeeper;
1227 struct timespec now, mono; 1301 struct timespec now, mono;
1228 unsigned long seq; 1302 unsigned long seq;
1229 1303
1230 do { 1304 do {
1231 seq = read_seqbegin(&timekeeper.lock); 1305 seq = read_seqbegin(&tk->lock);
1232 1306
1233 now = timekeeper.xtime; 1307 now = tk_xtime(tk);
1234 mono = timekeeper.wall_to_monotonic; 1308 mono = tk->wall_to_monotonic;
1235 } while (read_seqretry(&timekeeper.lock, seq)); 1309 } while (read_seqretry(&tk->lock, seq));
1236 1310
1237 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, 1311 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
1238 now.tv_nsec + mono.tv_nsec); 1312 now.tv_nsec + mono.tv_nsec);
@@ -1261,14 +1335,15 @@ void do_timer(unsigned long ticks)
1261void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, 1335void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1262 struct timespec *wtom, struct timespec *sleep) 1336 struct timespec *wtom, struct timespec *sleep)
1263{ 1337{
1338 struct timekeeper *tk = &timekeeper;
1264 unsigned long seq; 1339 unsigned long seq;
1265 1340
1266 do { 1341 do {
1267 seq = read_seqbegin(&timekeeper.lock); 1342 seq = read_seqbegin(&tk->lock);
1268 *xtim = timekeeper.xtime; 1343 *xtim = tk_xtime(tk);
1269 *wtom = timekeeper.wall_to_monotonic; 1344 *wtom = tk->wall_to_monotonic;
1270 *sleep = timekeeper.total_sleep_time; 1345 *sleep = tk->total_sleep_time;
1271 } while (read_seqretry(&timekeeper.lock, seq)); 1346 } while (read_seqretry(&tk->lock, seq));
1272} 1347}
1273 1348
1274#ifdef CONFIG_HIGH_RES_TIMERS 1349#ifdef CONFIG_HIGH_RES_TIMERS
@@ -1282,22 +1357,20 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1282 */ 1357 */
1283ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) 1358ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot)
1284{ 1359{
1360 struct timekeeper *tk = &timekeeper;
1285 ktime_t now; 1361 ktime_t now;
1286 unsigned int seq; 1362 unsigned int seq;
1287 u64 secs, nsecs; 1363 u64 secs, nsecs;
1288 1364
1289 do { 1365 do {
1290 seq = read_seqbegin(&timekeeper.lock); 1366 seq = read_seqbegin(&tk->lock);
1291 1367
1292 secs = timekeeper.xtime.tv_sec; 1368 secs = tk->xtime_sec;
1293 nsecs = timekeeper.xtime.tv_nsec; 1369 nsecs = timekeeping_get_ns(tk);
1294 nsecs += timekeeping_get_ns();
1295 /* If arch requires, add in gettimeoffset() */
1296 nsecs += arch_gettimeoffset();
1297 1370
1298 *offs_real = timekeeper.offs_real; 1371 *offs_real = tk->offs_real;
1299 *offs_boot = timekeeper.offs_boot; 1372 *offs_boot = tk->offs_boot;
1300 } while (read_seqretry(&timekeeper.lock, seq)); 1373 } while (read_seqretry(&tk->lock, seq));
1301 1374
1302 now = ktime_add_ns(ktime_set(secs, 0), nsecs); 1375 now = ktime_add_ns(ktime_set(secs, 0), nsecs);
1303 now = ktime_sub(now, *offs_real); 1376 now = ktime_sub(now, *offs_real);
@@ -1310,19 +1383,19 @@ ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot)
1310 */ 1383 */
1311ktime_t ktime_get_monotonic_offset(void) 1384ktime_t ktime_get_monotonic_offset(void)
1312{ 1385{
1386 struct timekeeper *tk = &timekeeper;
1313 unsigned long seq; 1387 unsigned long seq;
1314 struct timespec wtom; 1388 struct timespec wtom;
1315 1389
1316 do { 1390 do {
1317 seq = read_seqbegin(&timekeeper.lock); 1391 seq = read_seqbegin(&tk->lock);
1318 wtom = timekeeper.wall_to_monotonic; 1392 wtom = tk->wall_to_monotonic;
1319 } while (read_seqretry(&timekeeper.lock, seq)); 1393 } while (read_seqretry(&tk->lock, seq));
1320 1394
1321 return timespec_to_ktime(wtom); 1395 return timespec_to_ktime(wtom);
1322} 1396}
1323EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); 1397EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset);
1324 1398
1325
1326/** 1399/**
1327 * xtime_update() - advances the timekeeping infrastructure 1400 * xtime_update() - advances the timekeeping infrastructure
1328 * @ticks: number of ticks, that have elapsed since the last call. 1401 * @ticks: number of ticks, that have elapsed since the last call.