diff options
Diffstat (limited to 'kernel/time/timekeeping.c')
-rw-r--r-- | kernel/time/timekeeping.c | 442 |
1 files changed, 253 insertions, 189 deletions
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index f045cc50832d..34e5eac81424 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
@@ -65,14 +65,14 @@ struct timekeeper { | |||
65 | * used instead. | 65 | * used instead. |
66 | */ | 66 | */ |
67 | struct timespec wall_to_monotonic; | 67 | struct timespec wall_to_monotonic; |
68 | /* time spent in suspend */ | ||
69 | struct timespec total_sleep_time; | ||
70 | /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ | ||
71 | struct timespec raw_time; | ||
72 | /* Offset clock monotonic -> clock realtime */ | 68 | /* Offset clock monotonic -> clock realtime */ |
73 | ktime_t offs_real; | 69 | ktime_t offs_real; |
70 | /* time spent in suspend */ | ||
71 | struct timespec total_sleep_time; | ||
74 | /* Offset clock monotonic -> clock boottime */ | 72 | /* Offset clock monotonic -> clock boottime */ |
75 | ktime_t offs_boot; | 73 | ktime_t offs_boot; |
74 | /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ | ||
75 | struct timespec raw_time; | ||
76 | /* Seqlock for all timekeeper values */ | 76 | /* Seqlock for all timekeeper values */ |
77 | seqlock_t lock; | 77 | seqlock_t lock; |
78 | }; | 78 | }; |
@@ -108,13 +108,39 @@ static struct timespec tk_xtime(struct timekeeper *tk) | |||
108 | static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) | 108 | static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) |
109 | { | 109 | { |
110 | tk->xtime_sec = ts->tv_sec; | 110 | tk->xtime_sec = ts->tv_sec; |
111 | tk->xtime_nsec = ts->tv_nsec << tk->shift; | 111 | tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift; |
112 | } | 112 | } |
113 | 113 | ||
114 | static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts) | 114 | static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts) |
115 | { | 115 | { |
116 | tk->xtime_sec += ts->tv_sec; | 116 | tk->xtime_sec += ts->tv_sec; |
117 | tk->xtime_nsec += ts->tv_nsec << tk->shift; | 117 | tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift; |
118 | tk_normalize_xtime(tk); | ||
119 | } | ||
120 | |||
121 | static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm) | ||
122 | { | ||
123 | struct timespec tmp; | ||
124 | |||
125 | /* | ||
126 | * Verify consistency of: offset_real = -wall_to_monotonic | ||
127 | * before modifying anything | ||
128 | */ | ||
129 | set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec, | ||
130 | -tk->wall_to_monotonic.tv_nsec); | ||
131 | WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64); | ||
132 | tk->wall_to_monotonic = wtm; | ||
133 | set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec); | ||
134 | tk->offs_real = timespec_to_ktime(tmp); | ||
135 | } | ||
136 | |||
137 | static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t) | ||
138 | { | ||
139 | /* Verify consistency before modifying */ | ||
140 | WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64); | ||
141 | |||
142 | tk->total_sleep_time = t; | ||
143 | tk->offs_boot = timespec_to_ktime(t); | ||
118 | } | 144 | } |
119 | 145 | ||
120 | /** | 146 | /** |
@@ -217,14 +243,6 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) | |||
217 | return nsec + arch_gettimeoffset(); | 243 | return nsec + arch_gettimeoffset(); |
218 | } | 244 | } |
219 | 245 | ||
220 | static void update_rt_offset(struct timekeeper *tk) | ||
221 | { | ||
222 | struct timespec tmp, *wtm = &tk->wall_to_monotonic; | ||
223 | |||
224 | set_normalized_timespec(&tmp, -wtm->tv_sec, -wtm->tv_nsec); | ||
225 | tk->offs_real = timespec_to_ktime(tmp); | ||
226 | } | ||
227 | |||
228 | /* must hold write on timekeeper.lock */ | 246 | /* must hold write on timekeeper.lock */ |
229 | static void timekeeping_update(struct timekeeper *tk, bool clearntp) | 247 | static void timekeeping_update(struct timekeeper *tk, bool clearntp) |
230 | { | 248 | { |
@@ -234,12 +252,10 @@ static void timekeeping_update(struct timekeeper *tk, bool clearntp) | |||
234 | tk->ntp_error = 0; | 252 | tk->ntp_error = 0; |
235 | ntp_clear(); | 253 | ntp_clear(); |
236 | } | 254 | } |
237 | update_rt_offset(tk); | ||
238 | xt = tk_xtime(tk); | 255 | xt = tk_xtime(tk); |
239 | update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); | 256 | update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); |
240 | } | 257 | } |
241 | 258 | ||
242 | |||
243 | /** | 259 | /** |
244 | * timekeeping_forward_now - update clock to the current time | 260 | * timekeeping_forward_now - update clock to the current time |
245 | * | 261 | * |
@@ -261,7 +277,7 @@ static void timekeeping_forward_now(struct timekeeper *tk) | |||
261 | tk->xtime_nsec += cycle_delta * tk->mult; | 277 | tk->xtime_nsec += cycle_delta * tk->mult; |
262 | 278 | ||
263 | /* If arch requires, add in gettimeoffset() */ | 279 | /* If arch requires, add in gettimeoffset() */ |
264 | tk->xtime_nsec += arch_gettimeoffset() << tk->shift; | 280 | tk->xtime_nsec += (u64)arch_gettimeoffset() << tk->shift; |
265 | 281 | ||
266 | tk_normalize_xtime(tk); | 282 | tk_normalize_xtime(tk); |
267 | 283 | ||
@@ -277,18 +293,19 @@ static void timekeeping_forward_now(struct timekeeper *tk) | |||
277 | */ | 293 | */ |
278 | void getnstimeofday(struct timespec *ts) | 294 | void getnstimeofday(struct timespec *ts) |
279 | { | 295 | { |
296 | struct timekeeper *tk = &timekeeper; | ||
280 | unsigned long seq; | 297 | unsigned long seq; |
281 | s64 nsecs = 0; | 298 | s64 nsecs = 0; |
282 | 299 | ||
283 | WARN_ON(timekeeping_suspended); | 300 | WARN_ON(timekeeping_suspended); |
284 | 301 | ||
285 | do { | 302 | do { |
286 | seq = read_seqbegin(&timekeeper.lock); | 303 | seq = read_seqbegin(&tk->lock); |
287 | 304 | ||
288 | ts->tv_sec = timekeeper.xtime_sec; | 305 | ts->tv_sec = tk->xtime_sec; |
289 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 306 | ts->tv_nsec = timekeeping_get_ns(tk); |
290 | 307 | ||
291 | } while (read_seqretry(&timekeeper.lock, seq)); | 308 | } while (read_seqretry(&tk->lock, seq)); |
292 | 309 | ||
293 | timespec_add_ns(ts, nsecs); | 310 | timespec_add_ns(ts, nsecs); |
294 | } | 311 | } |
@@ -296,19 +313,18 @@ EXPORT_SYMBOL(getnstimeofday); | |||
296 | 313 | ||
297 | ktime_t ktime_get(void) | 314 | ktime_t ktime_get(void) |
298 | { | 315 | { |
316 | struct timekeeper *tk = &timekeeper; | ||
299 | unsigned int seq; | 317 | unsigned int seq; |
300 | s64 secs, nsecs; | 318 | s64 secs, nsecs; |
301 | 319 | ||
302 | WARN_ON(timekeeping_suspended); | 320 | WARN_ON(timekeeping_suspended); |
303 | 321 | ||
304 | do { | 322 | do { |
305 | seq = read_seqbegin(&timekeeper.lock); | 323 | seq = read_seqbegin(&tk->lock); |
306 | secs = timekeeper.xtime_sec + | 324 | secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; |
307 | timekeeper.wall_to_monotonic.tv_sec; | 325 | nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; |
308 | nsecs = timekeeping_get_ns(&timekeeper) + | ||
309 | timekeeper.wall_to_monotonic.tv_nsec; | ||
310 | 326 | ||
311 | } while (read_seqretry(&timekeeper.lock, seq)); | 327 | } while (read_seqretry(&tk->lock, seq)); |
312 | /* | 328 | /* |
313 | * Use ktime_set/ktime_add_ns to create a proper ktime on | 329 | * Use ktime_set/ktime_add_ns to create a proper ktime on |
314 | * 32-bit architectures without CONFIG_KTIME_SCALAR. | 330 | * 32-bit architectures without CONFIG_KTIME_SCALAR. |
@@ -327,18 +343,19 @@ EXPORT_SYMBOL_GPL(ktime_get); | |||
327 | */ | 343 | */ |
328 | void ktime_get_ts(struct timespec *ts) | 344 | void ktime_get_ts(struct timespec *ts) |
329 | { | 345 | { |
346 | struct timekeeper *tk = &timekeeper; | ||
330 | struct timespec tomono; | 347 | struct timespec tomono; |
331 | unsigned int seq; | 348 | unsigned int seq; |
332 | 349 | ||
333 | WARN_ON(timekeeping_suspended); | 350 | WARN_ON(timekeeping_suspended); |
334 | 351 | ||
335 | do { | 352 | do { |
336 | seq = read_seqbegin(&timekeeper.lock); | 353 | seq = read_seqbegin(&tk->lock); |
337 | ts->tv_sec = timekeeper.xtime_sec; | 354 | ts->tv_sec = tk->xtime_sec; |
338 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 355 | ts->tv_nsec = timekeeping_get_ns(tk); |
339 | tomono = timekeeper.wall_to_monotonic; | 356 | tomono = tk->wall_to_monotonic; |
340 | 357 | ||
341 | } while (read_seqretry(&timekeeper.lock, seq)); | 358 | } while (read_seqretry(&tk->lock, seq)); |
342 | 359 | ||
343 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, | 360 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, |
344 | ts->tv_nsec + tomono.tv_nsec); | 361 | ts->tv_nsec + tomono.tv_nsec); |
@@ -358,22 +375,23 @@ EXPORT_SYMBOL_GPL(ktime_get_ts); | |||
358 | */ | 375 | */ |
359 | void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) | 376 | void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) |
360 | { | 377 | { |
378 | struct timekeeper *tk = &timekeeper; | ||
361 | unsigned long seq; | 379 | unsigned long seq; |
362 | s64 nsecs_raw, nsecs_real; | 380 | s64 nsecs_raw, nsecs_real; |
363 | 381 | ||
364 | WARN_ON_ONCE(timekeeping_suspended); | 382 | WARN_ON_ONCE(timekeeping_suspended); |
365 | 383 | ||
366 | do { | 384 | do { |
367 | seq = read_seqbegin(&timekeeper.lock); | 385 | seq = read_seqbegin(&tk->lock); |
368 | 386 | ||
369 | *ts_raw = timekeeper.raw_time; | 387 | *ts_raw = tk->raw_time; |
370 | ts_real->tv_sec = timekeeper.xtime_sec; | 388 | ts_real->tv_sec = tk->xtime_sec; |
371 | ts_real->tv_nsec = 0; | 389 | ts_real->tv_nsec = 0; |
372 | 390 | ||
373 | nsecs_raw = timekeeping_get_ns_raw(&timekeeper); | 391 | nsecs_raw = timekeeping_get_ns_raw(tk); |
374 | nsecs_real = timekeeping_get_ns(&timekeeper); | 392 | nsecs_real = timekeeping_get_ns(tk); |
375 | 393 | ||
376 | } while (read_seqretry(&timekeeper.lock, seq)); | 394 | } while (read_seqretry(&tk->lock, seq)); |
377 | 395 | ||
378 | timespec_add_ns(ts_raw, nsecs_raw); | 396 | timespec_add_ns(ts_raw, nsecs_raw); |
379 | timespec_add_ns(ts_real, nsecs_real); | 397 | timespec_add_ns(ts_real, nsecs_real); |
@@ -406,28 +424,28 @@ EXPORT_SYMBOL(do_gettimeofday); | |||
406 | */ | 424 | */ |
407 | int do_settimeofday(const struct timespec *tv) | 425 | int do_settimeofday(const struct timespec *tv) |
408 | { | 426 | { |
427 | struct timekeeper *tk = &timekeeper; | ||
409 | struct timespec ts_delta, xt; | 428 | struct timespec ts_delta, xt; |
410 | unsigned long flags; | 429 | unsigned long flags; |
411 | 430 | ||
412 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | 431 | if (!timespec_valid_strict(tv)) |
413 | return -EINVAL; | 432 | return -EINVAL; |
414 | 433 | ||
415 | write_seqlock_irqsave(&timekeeper.lock, flags); | 434 | write_seqlock_irqsave(&tk->lock, flags); |
416 | 435 | ||
417 | timekeeping_forward_now(&timekeeper); | 436 | timekeeping_forward_now(tk); |
418 | 437 | ||
419 | xt = tk_xtime(&timekeeper); | 438 | xt = tk_xtime(tk); |
420 | ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; | 439 | ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; |
421 | ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; | 440 | ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; |
422 | 441 | ||
423 | timekeeper.wall_to_monotonic = | 442 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta)); |
424 | timespec_sub(timekeeper.wall_to_monotonic, ts_delta); | ||
425 | 443 | ||
426 | tk_set_xtime(&timekeeper, tv); | 444 | tk_set_xtime(tk, tv); |
427 | 445 | ||
428 | timekeeping_update(&timekeeper, true); | 446 | timekeeping_update(tk, true); |
429 | 447 | ||
430 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 448 | write_sequnlock_irqrestore(&tk->lock, flags); |
431 | 449 | ||
432 | /* signal hrtimers about time change */ | 450 | /* signal hrtimers about time change */ |
433 | clock_was_set(); | 451 | clock_was_set(); |
@@ -436,7 +454,6 @@ int do_settimeofday(const struct timespec *tv) | |||
436 | } | 454 | } |
437 | EXPORT_SYMBOL(do_settimeofday); | 455 | EXPORT_SYMBOL(do_settimeofday); |
438 | 456 | ||
439 | |||
440 | /** | 457 | /** |
441 | * timekeeping_inject_offset - Adds or subtracts from the current time. | 458 | * timekeeping_inject_offset - Adds or subtracts from the current time. |
442 | * @tv: pointer to the timespec variable containing the offset | 459 | * @tv: pointer to the timespec variable containing the offset |
@@ -445,28 +462,37 @@ EXPORT_SYMBOL(do_settimeofday); | |||
445 | */ | 462 | */ |
446 | int timekeeping_inject_offset(struct timespec *ts) | 463 | int timekeeping_inject_offset(struct timespec *ts) |
447 | { | 464 | { |
465 | struct timekeeper *tk = &timekeeper; | ||
448 | unsigned long flags; | 466 | unsigned long flags; |
467 | struct timespec tmp; | ||
468 | int ret = 0; | ||
449 | 469 | ||
450 | if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) | 470 | if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) |
451 | return -EINVAL; | 471 | return -EINVAL; |
452 | 472 | ||
453 | write_seqlock_irqsave(&timekeeper.lock, flags); | 473 | write_seqlock_irqsave(&tk->lock, flags); |
454 | 474 | ||
455 | timekeeping_forward_now(&timekeeper); | 475 | timekeeping_forward_now(tk); |
456 | 476 | ||
477 | /* Make sure the proposed value is valid */ | ||
478 | tmp = timespec_add(tk_xtime(tk), *ts); | ||
479 | if (!timespec_valid_strict(&tmp)) { | ||
480 | ret = -EINVAL; | ||
481 | goto error; | ||
482 | } | ||
457 | 483 | ||
458 | tk_xtime_add(&timekeeper, ts); | 484 | tk_xtime_add(tk, ts); |
459 | timekeeper.wall_to_monotonic = | 485 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts)); |
460 | timespec_sub(timekeeper.wall_to_monotonic, *ts); | ||
461 | 486 | ||
462 | timekeeping_update(&timekeeper, true); | 487 | error: /* even if we error out, we forwarded the time, so call update */ |
488 | timekeeping_update(tk, true); | ||
463 | 489 | ||
464 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 490 | write_sequnlock_irqrestore(&tk->lock, flags); |
465 | 491 | ||
466 | /* signal hrtimers about time change */ | 492 | /* signal hrtimers about time change */ |
467 | clock_was_set(); | 493 | clock_was_set(); |
468 | 494 | ||
469 | return 0; | 495 | return ret; |
470 | } | 496 | } |
471 | EXPORT_SYMBOL(timekeeping_inject_offset); | 497 | EXPORT_SYMBOL(timekeeping_inject_offset); |
472 | 498 | ||
@@ -477,23 +503,24 @@ EXPORT_SYMBOL(timekeeping_inject_offset); | |||
477 | */ | 503 | */ |
478 | static int change_clocksource(void *data) | 504 | static int change_clocksource(void *data) |
479 | { | 505 | { |
506 | struct timekeeper *tk = &timekeeper; | ||
480 | struct clocksource *new, *old; | 507 | struct clocksource *new, *old; |
481 | unsigned long flags; | 508 | unsigned long flags; |
482 | 509 | ||
483 | new = (struct clocksource *) data; | 510 | new = (struct clocksource *) data; |
484 | 511 | ||
485 | write_seqlock_irqsave(&timekeeper.lock, flags); | 512 | write_seqlock_irqsave(&tk->lock, flags); |
486 | 513 | ||
487 | timekeeping_forward_now(&timekeeper); | 514 | timekeeping_forward_now(tk); |
488 | if (!new->enable || new->enable(new) == 0) { | 515 | if (!new->enable || new->enable(new) == 0) { |
489 | old = timekeeper.clock; | 516 | old = tk->clock; |
490 | tk_setup_internals(&timekeeper, new); | 517 | tk_setup_internals(tk, new); |
491 | if (old->disable) | 518 | if (old->disable) |
492 | old->disable(old); | 519 | old->disable(old); |
493 | } | 520 | } |
494 | timekeeping_update(&timekeeper, true); | 521 | timekeeping_update(tk, true); |
495 | 522 | ||
496 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 523 | write_sequnlock_irqrestore(&tk->lock, flags); |
497 | 524 | ||
498 | return 0; | 525 | return 0; |
499 | } | 526 | } |
@@ -507,7 +534,9 @@ static int change_clocksource(void *data) | |||
507 | */ | 534 | */ |
508 | void timekeeping_notify(struct clocksource *clock) | 535 | void timekeeping_notify(struct clocksource *clock) |
509 | { | 536 | { |
510 | if (timekeeper.clock == clock) | 537 | struct timekeeper *tk = &timekeeper; |
538 | |||
539 | if (tk->clock == clock) | ||
511 | return; | 540 | return; |
512 | stop_machine(change_clocksource, clock, NULL); | 541 | stop_machine(change_clocksource, clock, NULL); |
513 | tick_clock_notify(); | 542 | tick_clock_notify(); |
@@ -536,35 +565,36 @@ EXPORT_SYMBOL_GPL(ktime_get_real); | |||
536 | */ | 565 | */ |
537 | void getrawmonotonic(struct timespec *ts) | 566 | void getrawmonotonic(struct timespec *ts) |
538 | { | 567 | { |
568 | struct timekeeper *tk = &timekeeper; | ||
539 | unsigned long seq; | 569 | unsigned long seq; |
540 | s64 nsecs; | 570 | s64 nsecs; |
541 | 571 | ||
542 | do { | 572 | do { |
543 | seq = read_seqbegin(&timekeeper.lock); | 573 | seq = read_seqbegin(&tk->lock); |
544 | nsecs = timekeeping_get_ns_raw(&timekeeper); | 574 | nsecs = timekeeping_get_ns_raw(tk); |
545 | *ts = timekeeper.raw_time; | 575 | *ts = tk->raw_time; |
546 | 576 | ||
547 | } while (read_seqretry(&timekeeper.lock, seq)); | 577 | } while (read_seqretry(&tk->lock, seq)); |
548 | 578 | ||
549 | timespec_add_ns(ts, nsecs); | 579 | timespec_add_ns(ts, nsecs); |
550 | } | 580 | } |
551 | EXPORT_SYMBOL(getrawmonotonic); | 581 | EXPORT_SYMBOL(getrawmonotonic); |
552 | 582 | ||
553 | |||
554 | /** | 583 | /** |
555 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres | 584 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres |
556 | */ | 585 | */ |
557 | int timekeeping_valid_for_hres(void) | 586 | int timekeeping_valid_for_hres(void) |
558 | { | 587 | { |
588 | struct timekeeper *tk = &timekeeper; | ||
559 | unsigned long seq; | 589 | unsigned long seq; |
560 | int ret; | 590 | int ret; |
561 | 591 | ||
562 | do { | 592 | do { |
563 | seq = read_seqbegin(&timekeeper.lock); | 593 | seq = read_seqbegin(&tk->lock); |
564 | 594 | ||
565 | ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; | 595 | ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; |
566 | 596 | ||
567 | } while (read_seqretry(&timekeeper.lock, seq)); | 597 | } while (read_seqretry(&tk->lock, seq)); |
568 | 598 | ||
569 | return ret; | 599 | return ret; |
570 | } | 600 | } |
@@ -574,15 +604,16 @@ int timekeeping_valid_for_hres(void) | |||
574 | */ | 604 | */ |
575 | u64 timekeeping_max_deferment(void) | 605 | u64 timekeeping_max_deferment(void) |
576 | { | 606 | { |
607 | struct timekeeper *tk = &timekeeper; | ||
577 | unsigned long seq; | 608 | unsigned long seq; |
578 | u64 ret; | 609 | u64 ret; |
579 | 610 | ||
580 | do { | 611 | do { |
581 | seq = read_seqbegin(&timekeeper.lock); | 612 | seq = read_seqbegin(&tk->lock); |
582 | 613 | ||
583 | ret = timekeeper.clock->max_idle_ns; | 614 | ret = tk->clock->max_idle_ns; |
584 | 615 | ||
585 | } while (read_seqretry(&timekeeper.lock, seq)); | 616 | } while (read_seqretry(&tk->lock, seq)); |
586 | 617 | ||
587 | return ret; | 618 | return ret; |
588 | } | 619 | } |
@@ -622,46 +653,56 @@ void __attribute__((weak)) read_boot_clock(struct timespec *ts) | |||
622 | */ | 653 | */ |
623 | void __init timekeeping_init(void) | 654 | void __init timekeeping_init(void) |
624 | { | 655 | { |
656 | struct timekeeper *tk = &timekeeper; | ||
625 | struct clocksource *clock; | 657 | struct clocksource *clock; |
626 | unsigned long flags; | 658 | unsigned long flags; |
627 | struct timespec now, boot; | 659 | struct timespec now, boot, tmp; |
628 | 660 | ||
629 | read_persistent_clock(&now); | 661 | read_persistent_clock(&now); |
662 | if (!timespec_valid_strict(&now)) { | ||
663 | pr_warn("WARNING: Persistent clock returned invalid value!\n" | ||
664 | " Check your CMOS/BIOS settings.\n"); | ||
665 | now.tv_sec = 0; | ||
666 | now.tv_nsec = 0; | ||
667 | } | ||
668 | |||
630 | read_boot_clock(&boot); | 669 | read_boot_clock(&boot); |
670 | if (!timespec_valid_strict(&boot)) { | ||
671 | pr_warn("WARNING: Boot clock returned invalid value!\n" | ||
672 | " Check your CMOS/BIOS settings.\n"); | ||
673 | boot.tv_sec = 0; | ||
674 | boot.tv_nsec = 0; | ||
675 | } | ||
631 | 676 | ||
632 | seqlock_init(&timekeeper.lock); | 677 | seqlock_init(&tk->lock); |
633 | 678 | ||
634 | ntp_init(); | 679 | ntp_init(); |
635 | 680 | ||
636 | write_seqlock_irqsave(&timekeeper.lock, flags); | 681 | write_seqlock_irqsave(&tk->lock, flags); |
637 | clock = clocksource_default_clock(); | 682 | clock = clocksource_default_clock(); |
638 | if (clock->enable) | 683 | if (clock->enable) |
639 | clock->enable(clock); | 684 | clock->enable(clock); |
640 | tk_setup_internals(&timekeeper, clock); | 685 | tk_setup_internals(tk, clock); |
641 | 686 | ||
642 | tk_set_xtime(&timekeeper, &now); | 687 | tk_set_xtime(tk, &now); |
643 | timekeeper.raw_time.tv_sec = 0; | 688 | tk->raw_time.tv_sec = 0; |
644 | timekeeper.raw_time.tv_nsec = 0; | 689 | tk->raw_time.tv_nsec = 0; |
645 | if (boot.tv_sec == 0 && boot.tv_nsec == 0) | 690 | if (boot.tv_sec == 0 && boot.tv_nsec == 0) |
646 | boot = tk_xtime(&timekeeper); | 691 | boot = tk_xtime(tk); |
647 | 692 | ||
648 | set_normalized_timespec(&timekeeper.wall_to_monotonic, | 693 | set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec); |
649 | -boot.tv_sec, -boot.tv_nsec); | 694 | tk_set_wall_to_mono(tk, tmp); |
650 | update_rt_offset(&timekeeper); | 695 | |
651 | timekeeper.total_sleep_time.tv_sec = 0; | 696 | tmp.tv_sec = 0; |
652 | timekeeper.total_sleep_time.tv_nsec = 0; | 697 | tmp.tv_nsec = 0; |
653 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 698 | tk_set_sleep_time(tk, tmp); |
699 | |||
700 | write_sequnlock_irqrestore(&tk->lock, flags); | ||
654 | } | 701 | } |
655 | 702 | ||
656 | /* time in seconds when suspend began */ | 703 | /* time in seconds when suspend began */ |
657 | static struct timespec timekeeping_suspend_time; | 704 | static struct timespec timekeeping_suspend_time; |
658 | 705 | ||
659 | static void update_sleep_time(struct timespec t) | ||
660 | { | ||
661 | timekeeper.total_sleep_time = t; | ||
662 | timekeeper.offs_boot = timespec_to_ktime(t); | ||
663 | } | ||
664 | |||
665 | /** | 706 | /** |
666 | * __timekeeping_inject_sleeptime - Internal function to add sleep interval | 707 | * __timekeeping_inject_sleeptime - Internal function to add sleep interval |
667 | * @delta: pointer to a timespec delta value | 708 | * @delta: pointer to a timespec delta value |
@@ -672,18 +713,16 @@ static void update_sleep_time(struct timespec t) | |||
672 | static void __timekeeping_inject_sleeptime(struct timekeeper *tk, | 713 | static void __timekeeping_inject_sleeptime(struct timekeeper *tk, |
673 | struct timespec *delta) | 714 | struct timespec *delta) |
674 | { | 715 | { |
675 | if (!timespec_valid(delta)) { | 716 | if (!timespec_valid_strict(delta)) { |
676 | printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid " | 717 | printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid " |
677 | "sleep delta value!\n"); | 718 | "sleep delta value!\n"); |
678 | return; | 719 | return; |
679 | } | 720 | } |
680 | |||
681 | tk_xtime_add(tk, delta); | 721 | tk_xtime_add(tk, delta); |
682 | tk->wall_to_monotonic = timespec_sub(tk->wall_to_monotonic, *delta); | 722 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta)); |
683 | update_sleep_time(timespec_add(tk->total_sleep_time, *delta)); | 723 | tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta)); |
684 | } | 724 | } |
685 | 725 | ||
686 | |||
687 | /** | 726 | /** |
688 | * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values | 727 | * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values |
689 | * @delta: pointer to a timespec delta value | 728 | * @delta: pointer to a timespec delta value |
@@ -696,6 +735,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk, | |||
696 | */ | 735 | */ |
697 | void timekeeping_inject_sleeptime(struct timespec *delta) | 736 | void timekeeping_inject_sleeptime(struct timespec *delta) |
698 | { | 737 | { |
738 | struct timekeeper *tk = &timekeeper; | ||
699 | unsigned long flags; | 739 | unsigned long flags; |
700 | struct timespec ts; | 740 | struct timespec ts; |
701 | 741 | ||
@@ -704,21 +744,20 @@ void timekeeping_inject_sleeptime(struct timespec *delta) | |||
704 | if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) | 744 | if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) |
705 | return; | 745 | return; |
706 | 746 | ||
707 | write_seqlock_irqsave(&timekeeper.lock, flags); | 747 | write_seqlock_irqsave(&tk->lock, flags); |
708 | 748 | ||
709 | timekeeping_forward_now(&timekeeper); | 749 | timekeeping_forward_now(tk); |
710 | 750 | ||
711 | __timekeeping_inject_sleeptime(&timekeeper, delta); | 751 | __timekeeping_inject_sleeptime(tk, delta); |
712 | 752 | ||
713 | timekeeping_update(&timekeeper, true); | 753 | timekeeping_update(tk, true); |
714 | 754 | ||
715 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 755 | write_sequnlock_irqrestore(&tk->lock, flags); |
716 | 756 | ||
717 | /* signal hrtimers about time change */ | 757 | /* signal hrtimers about time change */ |
718 | clock_was_set(); | 758 | clock_was_set(); |
719 | } | 759 | } |
720 | 760 | ||
721 | |||
722 | /** | 761 | /** |
723 | * timekeeping_resume - Resumes the generic timekeeping subsystem. | 762 | * timekeeping_resume - Resumes the generic timekeeping subsystem. |
724 | * | 763 | * |
@@ -728,6 +767,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) | |||
728 | */ | 767 | */ |
729 | static void timekeeping_resume(void) | 768 | static void timekeeping_resume(void) |
730 | { | 769 | { |
770 | struct timekeeper *tk = &timekeeper; | ||
731 | unsigned long flags; | 771 | unsigned long flags; |
732 | struct timespec ts; | 772 | struct timespec ts; |
733 | 773 | ||
@@ -735,18 +775,18 @@ static void timekeeping_resume(void) | |||
735 | 775 | ||
736 | clocksource_resume(); | 776 | clocksource_resume(); |
737 | 777 | ||
738 | write_seqlock_irqsave(&timekeeper.lock, flags); | 778 | write_seqlock_irqsave(&tk->lock, flags); |
739 | 779 | ||
740 | if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { | 780 | if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { |
741 | ts = timespec_sub(ts, timekeeping_suspend_time); | 781 | ts = timespec_sub(ts, timekeeping_suspend_time); |
742 | __timekeeping_inject_sleeptime(&timekeeper, &ts); | 782 | __timekeeping_inject_sleeptime(tk, &ts); |
743 | } | 783 | } |
744 | /* re-base the last cycle value */ | 784 | /* re-base the last cycle value */ |
745 | timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); | 785 | tk->clock->cycle_last = tk->clock->read(tk->clock); |
746 | timekeeper.ntp_error = 0; | 786 | tk->ntp_error = 0; |
747 | timekeeping_suspended = 0; | 787 | timekeeping_suspended = 0; |
748 | timekeeping_update(&timekeeper, false); | 788 | timekeeping_update(tk, false); |
749 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 789 | write_sequnlock_irqrestore(&tk->lock, flags); |
750 | 790 | ||
751 | touch_softlockup_watchdog(); | 791 | touch_softlockup_watchdog(); |
752 | 792 | ||
@@ -758,14 +798,15 @@ static void timekeeping_resume(void) | |||
758 | 798 | ||
759 | static int timekeeping_suspend(void) | 799 | static int timekeeping_suspend(void) |
760 | { | 800 | { |
801 | struct timekeeper *tk = &timekeeper; | ||
761 | unsigned long flags; | 802 | unsigned long flags; |
762 | struct timespec delta, delta_delta; | 803 | struct timespec delta, delta_delta; |
763 | static struct timespec old_delta; | 804 | static struct timespec old_delta; |
764 | 805 | ||
765 | read_persistent_clock(&timekeeping_suspend_time); | 806 | read_persistent_clock(&timekeeping_suspend_time); |
766 | 807 | ||
767 | write_seqlock_irqsave(&timekeeper.lock, flags); | 808 | write_seqlock_irqsave(&tk->lock, flags); |
768 | timekeeping_forward_now(&timekeeper); | 809 | timekeeping_forward_now(tk); |
769 | timekeeping_suspended = 1; | 810 | timekeeping_suspended = 1; |
770 | 811 | ||
771 | /* | 812 | /* |
@@ -774,7 +815,7 @@ static int timekeeping_suspend(void) | |||
774 | * try to compensate so the difference in system time | 815 | * try to compensate so the difference in system time |
775 | * and persistent_clock time stays close to constant. | 816 | * and persistent_clock time stays close to constant. |
776 | */ | 817 | */ |
777 | delta = timespec_sub(tk_xtime(&timekeeper), timekeeping_suspend_time); | 818 | delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time); |
778 | delta_delta = timespec_sub(delta, old_delta); | 819 | delta_delta = timespec_sub(delta, old_delta); |
779 | if (abs(delta_delta.tv_sec) >= 2) { | 820 | if (abs(delta_delta.tv_sec) >= 2) { |
780 | /* | 821 | /* |
@@ -787,7 +828,7 @@ static int timekeeping_suspend(void) | |||
787 | timekeeping_suspend_time = | 828 | timekeeping_suspend_time = |
788 | timespec_add(timekeeping_suspend_time, delta_delta); | 829 | timespec_add(timekeeping_suspend_time, delta_delta); |
789 | } | 830 | } |
790 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 831 | write_sequnlock_irqrestore(&tk->lock, flags); |
791 | 832 | ||
792 | clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); | 833 | clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); |
793 | clocksource_suspend(); | 834 | clocksource_suspend(); |
@@ -898,27 +939,29 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) | |||
898 | * the error. This causes the likely below to be unlikely. | 939 | * the error. This causes the likely below to be unlikely. |
899 | * | 940 | * |
900 | * The proper fix is to avoid rounding up by using | 941 | * The proper fix is to avoid rounding up by using |
901 | * the high precision timekeeper.xtime_nsec instead of | 942 | * the high precision tk->xtime_nsec instead of |
902 | * xtime.tv_nsec everywhere. Fixing this will take some | 943 | * xtime.tv_nsec everywhere. Fixing this will take some |
903 | * time. | 944 | * time. |
904 | */ | 945 | */ |
905 | if (likely(error <= interval)) | 946 | if (likely(error <= interval)) |
906 | adj = 1; | 947 | adj = 1; |
907 | else | 948 | else |
908 | adj = timekeeping_bigadjust(tk, error, &interval, | 949 | adj = timekeeping_bigadjust(tk, error, &interval, &offset); |
909 | &offset); | 950 | } else { |
910 | } else if (error < -interval) { | 951 | if (error < -interval) { |
911 | /* See comment above, this is just switched for the negative */ | 952 | /* See comment above, this is just switched for the negative */ |
912 | error >>= 2; | 953 | error >>= 2; |
913 | if (likely(error >= -interval)) { | 954 | if (likely(error >= -interval)) { |
914 | adj = -1; | 955 | adj = -1; |
915 | interval = -interval; | 956 | interval = -interval; |
916 | offset = -offset; | 957 | offset = -offset; |
917 | } else | 958 | } else { |
918 | adj = timekeeping_bigadjust(tk, error, &interval, | 959 | adj = timekeeping_bigadjust(tk, error, &interval, &offset); |
919 | &offset); | 960 | } |
920 | } else | 961 | } else { |
921 | return; | 962 | goto out_adjust; |
963 | } | ||
964 | } | ||
922 | 965 | ||
923 | if (unlikely(tk->clock->maxadj && | 966 | if (unlikely(tk->clock->maxadj && |
924 | (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) { | 967 | (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) { |
@@ -981,6 +1024,7 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) | |||
981 | tk->xtime_nsec -= offset; | 1024 | tk->xtime_nsec -= offset; |
982 | tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; | 1025 | tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; |
983 | 1026 | ||
1027 | out_adjust: | ||
984 | /* | 1028 | /* |
985 | * It may be possible that when we entered this function, xtime_nsec | 1029 | * It may be possible that when we entered this function, xtime_nsec |
986 | * was very small. Further, if we're slightly speeding the clocksource | 1030 | * was very small. Further, if we're slightly speeding the clocksource |
@@ -1003,7 +1047,6 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) | |||
1003 | 1047 | ||
1004 | } | 1048 | } |
1005 | 1049 | ||
1006 | |||
1007 | /** | 1050 | /** |
1008 | * accumulate_nsecs_to_secs - Accumulates nsecs into secs | 1051 | * accumulate_nsecs_to_secs - Accumulates nsecs into secs |
1009 | * | 1052 | * |
@@ -1024,15 +1067,21 @@ static inline void accumulate_nsecs_to_secs(struct timekeeper *tk) | |||
1024 | 1067 | ||
1025 | /* Figure out if its a leap sec and apply if needed */ | 1068 | /* Figure out if its a leap sec and apply if needed */ |
1026 | leap = second_overflow(tk->xtime_sec); | 1069 | leap = second_overflow(tk->xtime_sec); |
1027 | tk->xtime_sec += leap; | 1070 | if (unlikely(leap)) { |
1028 | tk->wall_to_monotonic.tv_sec -= leap; | 1071 | struct timespec ts; |
1029 | if (leap) | 1072 | |
1030 | clock_was_set_delayed(); | 1073 | tk->xtime_sec += leap; |
1074 | |||
1075 | ts.tv_sec = leap; | ||
1076 | ts.tv_nsec = 0; | ||
1077 | tk_set_wall_to_mono(tk, | ||
1078 | timespec_sub(tk->wall_to_monotonic, ts)); | ||
1031 | 1079 | ||
1080 | clock_was_set_delayed(); | ||
1081 | } | ||
1032 | } | 1082 | } |
1033 | } | 1083 | } |
1034 | 1084 | ||
1035 | |||
1036 | /** | 1085 | /** |
1037 | * logarithmic_accumulation - shifted accumulation of cycles | 1086 | * logarithmic_accumulation - shifted accumulation of cycles |
1038 | * | 1087 | * |
@@ -1076,7 +1125,6 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, | |||
1076 | return offset; | 1125 | return offset; |
1077 | } | 1126 | } |
1078 | 1127 | ||
1079 | |||
1080 | /** | 1128 | /** |
1081 | * update_wall_time - Uses the current clocksource to increment the wall time | 1129 | * update_wall_time - Uses the current clocksource to increment the wall time |
1082 | * | 1130 | * |
@@ -1084,25 +1132,30 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, | |||
1084 | static void update_wall_time(void) | 1132 | static void update_wall_time(void) |
1085 | { | 1133 | { |
1086 | struct clocksource *clock; | 1134 | struct clocksource *clock; |
1135 | struct timekeeper *tk = &timekeeper; | ||
1087 | cycle_t offset; | 1136 | cycle_t offset; |
1088 | int shift = 0, maxshift; | 1137 | int shift = 0, maxshift; |
1089 | unsigned long flags; | 1138 | unsigned long flags; |
1090 | s64 remainder; | 1139 | s64 remainder; |
1091 | 1140 | ||
1092 | write_seqlock_irqsave(&timekeeper.lock, flags); | 1141 | write_seqlock_irqsave(&tk->lock, flags); |
1093 | 1142 | ||
1094 | /* Make sure we're fully resumed: */ | 1143 | /* Make sure we're fully resumed: */ |
1095 | if (unlikely(timekeeping_suspended)) | 1144 | if (unlikely(timekeeping_suspended)) |
1096 | goto out; | 1145 | goto out; |
1097 | 1146 | ||
1098 | clock = timekeeper.clock; | 1147 | clock = tk->clock; |
1099 | 1148 | ||
1100 | #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET | 1149 | #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET |
1101 | offset = timekeeper.cycle_interval; | 1150 | offset = tk->cycle_interval; |
1102 | #else | 1151 | #else |
1103 | offset = (clock->read(clock) - clock->cycle_last) & clock->mask; | 1152 | offset = (clock->read(clock) - clock->cycle_last) & clock->mask; |
1104 | #endif | 1153 | #endif |
1105 | 1154 | ||
1155 | /* Check if there's really nothing to do */ | ||
1156 | if (offset < tk->cycle_interval) | ||
1157 | goto out; | ||
1158 | |||
1106 | /* | 1159 | /* |
1107 | * With NO_HZ we may have to accumulate many cycle_intervals | 1160 | * With NO_HZ we may have to accumulate many cycle_intervals |
1108 | * (think "ticks") worth of time at once. To do this efficiently, | 1161 | * (think "ticks") worth of time at once. To do this efficiently, |
@@ -1111,19 +1164,19 @@ static void update_wall_time(void) | |||
1111 | * chunk in one go, and then try to consume the next smaller | 1164 | * chunk in one go, and then try to consume the next smaller |
1112 | * doubled multiple. | 1165 | * doubled multiple. |
1113 | */ | 1166 | */ |
1114 | shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); | 1167 | shift = ilog2(offset) - ilog2(tk->cycle_interval); |
1115 | shift = max(0, shift); | 1168 | shift = max(0, shift); |
1116 | /* Bound shift to one less than what overflows tick_length */ | 1169 | /* Bound shift to one less than what overflows tick_length */ |
1117 | maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; | 1170 | maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; |
1118 | shift = min(shift, maxshift); | 1171 | shift = min(shift, maxshift); |
1119 | while (offset >= timekeeper.cycle_interval) { | 1172 | while (offset >= tk->cycle_interval) { |
1120 | offset = logarithmic_accumulation(&timekeeper, offset, shift); | 1173 | offset = logarithmic_accumulation(tk, offset, shift); |
1121 | if(offset < timekeeper.cycle_interval<<shift) | 1174 | if (offset < tk->cycle_interval<<shift) |
1122 | shift--; | 1175 | shift--; |
1123 | } | 1176 | } |
1124 | 1177 | ||
1125 | /* correct the clock when NTP error is too big */ | 1178 | /* correct the clock when NTP error is too big */ |
1126 | timekeeping_adjust(&timekeeper, offset); | 1179 | timekeeping_adjust(tk, offset); |
1127 | 1180 | ||
1128 | 1181 | ||
1129 | /* | 1182 | /* |
@@ -1135,21 +1188,21 @@ static void update_wall_time(void) | |||
1135 | * the vsyscall implementations are converted to use xtime_nsec | 1188 | * the vsyscall implementations are converted to use xtime_nsec |
1136 | * (shifted nanoseconds), this can be killed. | 1189 | * (shifted nanoseconds), this can be killed. |
1137 | */ | 1190 | */ |
1138 | remainder = timekeeper.xtime_nsec & ((1 << timekeeper.shift) - 1); | 1191 | remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1); |
1139 | timekeeper.xtime_nsec -= remainder; | 1192 | tk->xtime_nsec -= remainder; |
1140 | timekeeper.xtime_nsec += 1 << timekeeper.shift; | 1193 | tk->xtime_nsec += 1ULL << tk->shift; |
1141 | timekeeper.ntp_error += remainder << timekeeper.ntp_error_shift; | 1194 | tk->ntp_error += remainder << tk->ntp_error_shift; |
1142 | 1195 | ||
1143 | /* | 1196 | /* |
1144 | * Finally, make sure that after the rounding | 1197 | * Finally, make sure that after the rounding |
1145 | * xtime_nsec isn't larger than NSEC_PER_SEC | 1198 | * xtime_nsec isn't larger than NSEC_PER_SEC |
1146 | */ | 1199 | */ |
1147 | accumulate_nsecs_to_secs(&timekeeper); | 1200 | accumulate_nsecs_to_secs(tk); |
1148 | 1201 | ||
1149 | timekeeping_update(&timekeeper, false); | 1202 | timekeeping_update(tk, false); |
1150 | 1203 | ||
1151 | out: | 1204 | out: |
1152 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 1205 | write_sequnlock_irqrestore(&tk->lock, flags); |
1153 | 1206 | ||
1154 | } | 1207 | } |
1155 | 1208 | ||
@@ -1166,18 +1219,18 @@ out: | |||
1166 | */ | 1219 | */ |
1167 | void getboottime(struct timespec *ts) | 1220 | void getboottime(struct timespec *ts) |
1168 | { | 1221 | { |
1222 | struct timekeeper *tk = &timekeeper; | ||
1169 | struct timespec boottime = { | 1223 | struct timespec boottime = { |
1170 | .tv_sec = timekeeper.wall_to_monotonic.tv_sec + | 1224 | .tv_sec = tk->wall_to_monotonic.tv_sec + |
1171 | timekeeper.total_sleep_time.tv_sec, | 1225 | tk->total_sleep_time.tv_sec, |
1172 | .tv_nsec = timekeeper.wall_to_monotonic.tv_nsec + | 1226 | .tv_nsec = tk->wall_to_monotonic.tv_nsec + |
1173 | timekeeper.total_sleep_time.tv_nsec | 1227 | tk->total_sleep_time.tv_nsec |
1174 | }; | 1228 | }; |
1175 | 1229 | ||
1176 | set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); | 1230 | set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); |
1177 | } | 1231 | } |
1178 | EXPORT_SYMBOL_GPL(getboottime); | 1232 | EXPORT_SYMBOL_GPL(getboottime); |
1179 | 1233 | ||
1180 | |||
1181 | /** | 1234 | /** |
1182 | * get_monotonic_boottime - Returns monotonic time since boot | 1235 | * get_monotonic_boottime - Returns monotonic time since boot |
1183 | * @ts: pointer to the timespec to be set | 1236 | * @ts: pointer to the timespec to be set |
@@ -1189,19 +1242,20 @@ EXPORT_SYMBOL_GPL(getboottime); | |||
1189 | */ | 1242 | */ |
1190 | void get_monotonic_boottime(struct timespec *ts) | 1243 | void get_monotonic_boottime(struct timespec *ts) |
1191 | { | 1244 | { |
1245 | struct timekeeper *tk = &timekeeper; | ||
1192 | struct timespec tomono, sleep; | 1246 | struct timespec tomono, sleep; |
1193 | unsigned int seq; | 1247 | unsigned int seq; |
1194 | 1248 | ||
1195 | WARN_ON(timekeeping_suspended); | 1249 | WARN_ON(timekeeping_suspended); |
1196 | 1250 | ||
1197 | do { | 1251 | do { |
1198 | seq = read_seqbegin(&timekeeper.lock); | 1252 | seq = read_seqbegin(&tk->lock); |
1199 | ts->tv_sec = timekeeper.xtime_sec; | 1253 | ts->tv_sec = tk->xtime_sec; |
1200 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 1254 | ts->tv_nsec = timekeeping_get_ns(tk); |
1201 | tomono = timekeeper.wall_to_monotonic; | 1255 | tomono = tk->wall_to_monotonic; |
1202 | sleep = timekeeper.total_sleep_time; | 1256 | sleep = tk->total_sleep_time; |
1203 | 1257 | ||
1204 | } while (read_seqretry(&timekeeper.lock, seq)); | 1258 | } while (read_seqretry(&tk->lock, seq)); |
1205 | 1259 | ||
1206 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, | 1260 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, |
1207 | ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec); | 1261 | ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec); |
@@ -1231,31 +1285,38 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime); | |||
1231 | */ | 1285 | */ |
1232 | void monotonic_to_bootbased(struct timespec *ts) | 1286 | void monotonic_to_bootbased(struct timespec *ts) |
1233 | { | 1287 | { |
1234 | *ts = timespec_add(*ts, timekeeper.total_sleep_time); | 1288 | struct timekeeper *tk = &timekeeper; |
1289 | |||
1290 | *ts = timespec_add(*ts, tk->total_sleep_time); | ||
1235 | } | 1291 | } |
1236 | EXPORT_SYMBOL_GPL(monotonic_to_bootbased); | 1292 | EXPORT_SYMBOL_GPL(monotonic_to_bootbased); |
1237 | 1293 | ||
1238 | unsigned long get_seconds(void) | 1294 | unsigned long get_seconds(void) |
1239 | { | 1295 | { |
1240 | return timekeeper.xtime_sec; | 1296 | struct timekeeper *tk = &timekeeper; |
1297 | |||
1298 | return tk->xtime_sec; | ||
1241 | } | 1299 | } |
1242 | EXPORT_SYMBOL(get_seconds); | 1300 | EXPORT_SYMBOL(get_seconds); |
1243 | 1301 | ||
1244 | struct timespec __current_kernel_time(void) | 1302 | struct timespec __current_kernel_time(void) |
1245 | { | 1303 | { |
1246 | return tk_xtime(&timekeeper); | 1304 | struct timekeeper *tk = &timekeeper; |
1305 | |||
1306 | return tk_xtime(tk); | ||
1247 | } | 1307 | } |
1248 | 1308 | ||
1249 | struct timespec current_kernel_time(void) | 1309 | struct timespec current_kernel_time(void) |
1250 | { | 1310 | { |
1311 | struct timekeeper *tk = &timekeeper; | ||
1251 | struct timespec now; | 1312 | struct timespec now; |
1252 | unsigned long seq; | 1313 | unsigned long seq; |
1253 | 1314 | ||
1254 | do { | 1315 | do { |
1255 | seq = read_seqbegin(&timekeeper.lock); | 1316 | seq = read_seqbegin(&tk->lock); |
1256 | 1317 | ||
1257 | now = tk_xtime(&timekeeper); | 1318 | now = tk_xtime(tk); |
1258 | } while (read_seqretry(&timekeeper.lock, seq)); | 1319 | } while (read_seqretry(&tk->lock, seq)); |
1259 | 1320 | ||
1260 | return now; | 1321 | return now; |
1261 | } | 1322 | } |
@@ -1263,15 +1324,16 @@ EXPORT_SYMBOL(current_kernel_time); | |||
1263 | 1324 | ||
1264 | struct timespec get_monotonic_coarse(void) | 1325 | struct timespec get_monotonic_coarse(void) |
1265 | { | 1326 | { |
1327 | struct timekeeper *tk = &timekeeper; | ||
1266 | struct timespec now, mono; | 1328 | struct timespec now, mono; |
1267 | unsigned long seq; | 1329 | unsigned long seq; |
1268 | 1330 | ||
1269 | do { | 1331 | do { |
1270 | seq = read_seqbegin(&timekeeper.lock); | 1332 | seq = read_seqbegin(&tk->lock); |
1271 | 1333 | ||
1272 | now = tk_xtime(&timekeeper); | 1334 | now = tk_xtime(tk); |
1273 | mono = timekeeper.wall_to_monotonic; | 1335 | mono = tk->wall_to_monotonic; |
1274 | } while (read_seqretry(&timekeeper.lock, seq)); | 1336 | } while (read_seqretry(&tk->lock, seq)); |
1275 | 1337 | ||
1276 | set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, | 1338 | set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, |
1277 | now.tv_nsec + mono.tv_nsec); | 1339 | now.tv_nsec + mono.tv_nsec); |
@@ -1300,14 +1362,15 @@ void do_timer(unsigned long ticks) | |||
1300 | void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, | 1362 | void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, |
1301 | struct timespec *wtom, struct timespec *sleep) | 1363 | struct timespec *wtom, struct timespec *sleep) |
1302 | { | 1364 | { |
1365 | struct timekeeper *tk = &timekeeper; | ||
1303 | unsigned long seq; | 1366 | unsigned long seq; |
1304 | 1367 | ||
1305 | do { | 1368 | do { |
1306 | seq = read_seqbegin(&timekeeper.lock); | 1369 | seq = read_seqbegin(&tk->lock); |
1307 | *xtim = tk_xtime(&timekeeper); | 1370 | *xtim = tk_xtime(tk); |
1308 | *wtom = timekeeper.wall_to_monotonic; | 1371 | *wtom = tk->wall_to_monotonic; |
1309 | *sleep = timekeeper.total_sleep_time; | 1372 | *sleep = tk->total_sleep_time; |
1310 | } while (read_seqretry(&timekeeper.lock, seq)); | 1373 | } while (read_seqretry(&tk->lock, seq)); |
1311 | } | 1374 | } |
1312 | 1375 | ||
1313 | #ifdef CONFIG_HIGH_RES_TIMERS | 1376 | #ifdef CONFIG_HIGH_RES_TIMERS |
@@ -1321,19 +1384,20 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, | |||
1321 | */ | 1384 | */ |
1322 | ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) | 1385 | ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) |
1323 | { | 1386 | { |
1387 | struct timekeeper *tk = &timekeeper; | ||
1324 | ktime_t now; | 1388 | ktime_t now; |
1325 | unsigned int seq; | 1389 | unsigned int seq; |
1326 | u64 secs, nsecs; | 1390 | u64 secs, nsecs; |
1327 | 1391 | ||
1328 | do { | 1392 | do { |
1329 | seq = read_seqbegin(&timekeeper.lock); | 1393 | seq = read_seqbegin(&tk->lock); |
1330 | 1394 | ||
1331 | secs = timekeeper.xtime_sec; | 1395 | secs = tk->xtime_sec; |
1332 | nsecs = timekeeping_get_ns(&timekeeper); | 1396 | nsecs = timekeeping_get_ns(tk); |
1333 | 1397 | ||
1334 | *offs_real = timekeeper.offs_real; | 1398 | *offs_real = tk->offs_real; |
1335 | *offs_boot = timekeeper.offs_boot; | 1399 | *offs_boot = tk->offs_boot; |
1336 | } while (read_seqretry(&timekeeper.lock, seq)); | 1400 | } while (read_seqretry(&tk->lock, seq)); |
1337 | 1401 | ||
1338 | now = ktime_add_ns(ktime_set(secs, 0), nsecs); | 1402 | now = ktime_add_ns(ktime_set(secs, 0), nsecs); |
1339 | now = ktime_sub(now, *offs_real); | 1403 | now = ktime_sub(now, *offs_real); |
@@ -1346,19 +1410,19 @@ ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) | |||
1346 | */ | 1410 | */ |
1347 | ktime_t ktime_get_monotonic_offset(void) | 1411 | ktime_t ktime_get_monotonic_offset(void) |
1348 | { | 1412 | { |
1413 | struct timekeeper *tk = &timekeeper; | ||
1349 | unsigned long seq; | 1414 | unsigned long seq; |
1350 | struct timespec wtom; | 1415 | struct timespec wtom; |
1351 | 1416 | ||
1352 | do { | 1417 | do { |
1353 | seq = read_seqbegin(&timekeeper.lock); | 1418 | seq = read_seqbegin(&tk->lock); |
1354 | wtom = timekeeper.wall_to_monotonic; | 1419 | wtom = tk->wall_to_monotonic; |
1355 | } while (read_seqretry(&timekeeper.lock, seq)); | 1420 | } while (read_seqretry(&tk->lock, seq)); |
1356 | 1421 | ||
1357 | return timespec_to_ktime(wtom); | 1422 | return timespec_to_ktime(wtom); |
1358 | } | 1423 | } |
1359 | EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); | 1424 | EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); |
1360 | 1425 | ||
1361 | |||
1362 | /** | 1426 | /** |
1363 | * xtime_update() - advances the timekeeping infrastructure | 1427 | * xtime_update() - advances the timekeeping infrastructure |
1364 | * @ticks: number of ticks, that have elapsed since the last call. | 1428 | * @ticks: number of ticks, that have elapsed since the last call. |