aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/time/timekeeping.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/time/timekeeping.c')
-rw-r--r--kernel/time/timekeeping.c282
1 files changed, 154 insertions, 128 deletions
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 4da65592b4d9..2988bc819187 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -292,18 +292,19 @@ static void timekeeping_forward_now(struct timekeeper *tk)
292 */ 292 */
293void getnstimeofday(struct timespec *ts) 293void getnstimeofday(struct timespec *ts)
294{ 294{
295 struct timekeeper *tk = &timekeeper;
295 unsigned long seq; 296 unsigned long seq;
296 s64 nsecs = 0; 297 s64 nsecs = 0;
297 298
298 WARN_ON(timekeeping_suspended); 299 WARN_ON(timekeeping_suspended);
299 300
300 do { 301 do {
301 seq = read_seqbegin(&timekeeper.lock); 302 seq = read_seqbegin(&tk->lock);
302 303
303 ts->tv_sec = timekeeper.xtime_sec; 304 ts->tv_sec = tk->xtime_sec;
304 ts->tv_nsec = timekeeping_get_ns(&timekeeper); 305 ts->tv_nsec = timekeeping_get_ns(tk);
305 306
306 } while (read_seqretry(&timekeeper.lock, seq)); 307 } while (read_seqretry(&tk->lock, seq));
307 308
308 timespec_add_ns(ts, nsecs); 309 timespec_add_ns(ts, nsecs);
309} 310}
@@ -311,19 +312,18 @@ EXPORT_SYMBOL(getnstimeofday);
311 312
312ktime_t ktime_get(void) 313ktime_t ktime_get(void)
313{ 314{
315 struct timekeeper *tk = &timekeeper;
314 unsigned int seq; 316 unsigned int seq;
315 s64 secs, nsecs; 317 s64 secs, nsecs;
316 318
317 WARN_ON(timekeeping_suspended); 319 WARN_ON(timekeeping_suspended);
318 320
319 do { 321 do {
320 seq = read_seqbegin(&timekeeper.lock); 322 seq = read_seqbegin(&tk->lock);
321 secs = timekeeper.xtime_sec + 323 secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
322 timekeeper.wall_to_monotonic.tv_sec; 324 nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec;
323 nsecs = timekeeping_get_ns(&timekeeper) +
324 timekeeper.wall_to_monotonic.tv_nsec;
325 325
326 } while (read_seqretry(&timekeeper.lock, seq)); 326 } while (read_seqretry(&tk->lock, seq));
327 /* 327 /*
328 * Use ktime_set/ktime_add_ns to create a proper ktime on 328 * Use ktime_set/ktime_add_ns to create a proper ktime on
329 * 32-bit architectures without CONFIG_KTIME_SCALAR. 329 * 32-bit architectures without CONFIG_KTIME_SCALAR.
@@ -342,18 +342,19 @@ EXPORT_SYMBOL_GPL(ktime_get);
342 */ 342 */
343void ktime_get_ts(struct timespec *ts) 343void ktime_get_ts(struct timespec *ts)
344{ 344{
345 struct timekeeper *tk = &timekeeper;
345 struct timespec tomono; 346 struct timespec tomono;
346 unsigned int seq; 347 unsigned int seq;
347 348
348 WARN_ON(timekeeping_suspended); 349 WARN_ON(timekeeping_suspended);
349 350
350 do { 351 do {
351 seq = read_seqbegin(&timekeeper.lock); 352 seq = read_seqbegin(&tk->lock);
352 ts->tv_sec = timekeeper.xtime_sec; 353 ts->tv_sec = tk->xtime_sec;
353 ts->tv_nsec = timekeeping_get_ns(&timekeeper); 354 ts->tv_nsec = timekeeping_get_ns(tk);
354 tomono = timekeeper.wall_to_monotonic; 355 tomono = tk->wall_to_monotonic;
355 356
356 } while (read_seqretry(&timekeeper.lock, seq)); 357 } while (read_seqretry(&tk->lock, seq));
357 358
358 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, 359 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
359 ts->tv_nsec + tomono.tv_nsec); 360 ts->tv_nsec + tomono.tv_nsec);
@@ -373,22 +374,23 @@ EXPORT_SYMBOL_GPL(ktime_get_ts);
373 */ 374 */
374void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) 375void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
375{ 376{
377 struct timekeeper *tk = &timekeeper;
376 unsigned long seq; 378 unsigned long seq;
377 s64 nsecs_raw, nsecs_real; 379 s64 nsecs_raw, nsecs_real;
378 380
379 WARN_ON_ONCE(timekeeping_suspended); 381 WARN_ON_ONCE(timekeeping_suspended);
380 382
381 do { 383 do {
382 seq = read_seqbegin(&timekeeper.lock); 384 seq = read_seqbegin(&tk->lock);
383 385
384 *ts_raw = timekeeper.raw_time; 386 *ts_raw = tk->raw_time;
385 ts_real->tv_sec = timekeeper.xtime_sec; 387 ts_real->tv_sec = tk->xtime_sec;
386 ts_real->tv_nsec = 0; 388 ts_real->tv_nsec = 0;
387 389
388 nsecs_raw = timekeeping_get_ns_raw(&timekeeper); 390 nsecs_raw = timekeeping_get_ns_raw(tk);
389 nsecs_real = timekeeping_get_ns(&timekeeper); 391 nsecs_real = timekeeping_get_ns(tk);
390 392
391 } while (read_seqretry(&timekeeper.lock, seq)); 393 } while (read_seqretry(&tk->lock, seq));
392 394
393 timespec_add_ns(ts_raw, nsecs_raw); 395 timespec_add_ns(ts_raw, nsecs_raw);
394 timespec_add_ns(ts_real, nsecs_real); 396 timespec_add_ns(ts_real, nsecs_real);
@@ -421,28 +423,28 @@ EXPORT_SYMBOL(do_gettimeofday);
421 */ 423 */
422int do_settimeofday(const struct timespec *tv) 424int do_settimeofday(const struct timespec *tv)
423{ 425{
426 struct timekeeper *tk = &timekeeper;
424 struct timespec ts_delta, xt; 427 struct timespec ts_delta, xt;
425 unsigned long flags; 428 unsigned long flags;
426 429
427 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) 430 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
428 return -EINVAL; 431 return -EINVAL;
429 432
430 write_seqlock_irqsave(&timekeeper.lock, flags); 433 write_seqlock_irqsave(&tk->lock, flags);
431 434
432 timekeeping_forward_now(&timekeeper); 435 timekeeping_forward_now(tk);
433 436
434 xt = tk_xtime(&timekeeper); 437 xt = tk_xtime(tk);
435 ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; 438 ts_delta.tv_sec = tv->tv_sec - xt.tv_sec;
436 ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; 439 ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec;
437 440
438 tk_set_wall_to_mono(&timekeeper, 441 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta));
439 timespec_sub(timekeeper.wall_to_monotonic, ts_delta));
440 442
441 tk_set_xtime(&timekeeper, tv); 443 tk_set_xtime(tk, tv);
442 444
443 timekeeping_update(&timekeeper, true); 445 timekeeping_update(tk, true);
444 446
445 write_sequnlock_irqrestore(&timekeeper.lock, flags); 447 write_sequnlock_irqrestore(&tk->lock, flags);
446 448
447 /* signal hrtimers about time change */ 449 /* signal hrtimers about time change */
448 clock_was_set(); 450 clock_was_set();
@@ -459,23 +461,23 @@ EXPORT_SYMBOL(do_settimeofday);
459 */ 461 */
460int timekeeping_inject_offset(struct timespec *ts) 462int timekeeping_inject_offset(struct timespec *ts)
461{ 463{
464 struct timekeeper *tk = &timekeeper;
462 unsigned long flags; 465 unsigned long flags;
463 466
464 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) 467 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
465 return -EINVAL; 468 return -EINVAL;
466 469
467 write_seqlock_irqsave(&timekeeper.lock, flags); 470 write_seqlock_irqsave(&tk->lock, flags);
468 471
469 timekeeping_forward_now(&timekeeper); 472 timekeeping_forward_now(tk);
470 473
471 474
472 tk_xtime_add(&timekeeper, ts); 475 tk_xtime_add(tk, ts);
473 tk_set_wall_to_mono(&timekeeper, 476 tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts));
474 timespec_sub(timekeeper.wall_to_monotonic, *ts));
475 477
476 timekeeping_update(&timekeeper, true); 478 timekeeping_update(tk, true);
477 479
478 write_sequnlock_irqrestore(&timekeeper.lock, flags); 480 write_sequnlock_irqrestore(&tk->lock, flags);
479 481
480 /* signal hrtimers about time change */ 482 /* signal hrtimers about time change */
481 clock_was_set(); 483 clock_was_set();
@@ -491,23 +493,24 @@ EXPORT_SYMBOL(timekeeping_inject_offset);
491 */ 493 */
492static int change_clocksource(void *data) 494static int change_clocksource(void *data)
493{ 495{
496 struct timekeeper *tk = &timekeeper;
494 struct clocksource *new, *old; 497 struct clocksource *new, *old;
495 unsigned long flags; 498 unsigned long flags;
496 499
497 new = (struct clocksource *) data; 500 new = (struct clocksource *) data;
498 501
499 write_seqlock_irqsave(&timekeeper.lock, flags); 502 write_seqlock_irqsave(&tk->lock, flags);
500 503
501 timekeeping_forward_now(&timekeeper); 504 timekeeping_forward_now(tk);
502 if (!new->enable || new->enable(new) == 0) { 505 if (!new->enable || new->enable(new) == 0) {
503 old = timekeeper.clock; 506 old = tk->clock;
504 tk_setup_internals(&timekeeper, new); 507 tk_setup_internals(tk, new);
505 if (old->disable) 508 if (old->disable)
506 old->disable(old); 509 old->disable(old);
507 } 510 }
508 timekeeping_update(&timekeeper, true); 511 timekeeping_update(tk, true);
509 512
510 write_sequnlock_irqrestore(&timekeeper.lock, flags); 513 write_sequnlock_irqrestore(&tk->lock, flags);
511 514
512 return 0; 515 return 0;
513} 516}
@@ -521,7 +524,9 @@ static int change_clocksource(void *data)
521 */ 524 */
522void timekeeping_notify(struct clocksource *clock) 525void timekeeping_notify(struct clocksource *clock)
523{ 526{
524 if (timekeeper.clock == clock) 527 struct timekeeper *tk = &timekeeper;
528
529 if (tk->clock == clock)
525 return; 530 return;
526 stop_machine(change_clocksource, clock, NULL); 531 stop_machine(change_clocksource, clock, NULL);
527 tick_clock_notify(); 532 tick_clock_notify();
@@ -550,15 +555,16 @@ EXPORT_SYMBOL_GPL(ktime_get_real);
550 */ 555 */
551void getrawmonotonic(struct timespec *ts) 556void getrawmonotonic(struct timespec *ts)
552{ 557{
558 struct timekeeper *tk = &timekeeper;
553 unsigned long seq; 559 unsigned long seq;
554 s64 nsecs; 560 s64 nsecs;
555 561
556 do { 562 do {
557 seq = read_seqbegin(&timekeeper.lock); 563 seq = read_seqbegin(&tk->lock);
558 nsecs = timekeeping_get_ns_raw(&timekeeper); 564 nsecs = timekeeping_get_ns_raw(tk);
559 *ts = timekeeper.raw_time; 565 *ts = tk->raw_time;
560 566
561 } while (read_seqretry(&timekeeper.lock, seq)); 567 } while (read_seqretry(&tk->lock, seq));
562 568
563 timespec_add_ns(ts, nsecs); 569 timespec_add_ns(ts, nsecs);
564} 570}
@@ -569,15 +575,16 @@ EXPORT_SYMBOL(getrawmonotonic);
569 */ 575 */
570int timekeeping_valid_for_hres(void) 576int timekeeping_valid_for_hres(void)
571{ 577{
578 struct timekeeper *tk = &timekeeper;
572 unsigned long seq; 579 unsigned long seq;
573 int ret; 580 int ret;
574 581
575 do { 582 do {
576 seq = read_seqbegin(&timekeeper.lock); 583 seq = read_seqbegin(&tk->lock);
577 584
578 ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 585 ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
579 586
580 } while (read_seqretry(&timekeeper.lock, seq)); 587 } while (read_seqretry(&tk->lock, seq));
581 588
582 return ret; 589 return ret;
583} 590}
@@ -587,15 +594,16 @@ int timekeeping_valid_for_hres(void)
587 */ 594 */
588u64 timekeeping_max_deferment(void) 595u64 timekeeping_max_deferment(void)
589{ 596{
597 struct timekeeper *tk = &timekeeper;
590 unsigned long seq; 598 unsigned long seq;
591 u64 ret; 599 u64 ret;
592 600
593 do { 601 do {
594 seq = read_seqbegin(&timekeeper.lock); 602 seq = read_seqbegin(&tk->lock);
595 603
596 ret = timekeeper.clock->max_idle_ns; 604 ret = tk->clock->max_idle_ns;
597 605
598 } while (read_seqretry(&timekeeper.lock, seq)); 606 } while (read_seqretry(&tk->lock, seq));
599 607
600 return ret; 608 return ret;
601} 609}
@@ -635,6 +643,7 @@ void __attribute__((weak)) read_boot_clock(struct timespec *ts)
635 */ 643 */
636void __init timekeeping_init(void) 644void __init timekeeping_init(void)
637{ 645{
646 struct timekeeper *tk = &timekeeper;
638 struct clocksource *clock; 647 struct clocksource *clock;
639 unsigned long flags; 648 unsigned long flags;
640 struct timespec now, boot, tmp; 649 struct timespec now, boot, tmp;
@@ -642,30 +651,30 @@ void __init timekeeping_init(void)
642 read_persistent_clock(&now); 651 read_persistent_clock(&now);
643 read_boot_clock(&boot); 652 read_boot_clock(&boot);
644 653
645 seqlock_init(&timekeeper.lock); 654 seqlock_init(&tk->lock);
646 655
647 ntp_init(); 656 ntp_init();
648 657
649 write_seqlock_irqsave(&timekeeper.lock, flags); 658 write_seqlock_irqsave(&tk->lock, flags);
650 clock = clocksource_default_clock(); 659 clock = clocksource_default_clock();
651 if (clock->enable) 660 if (clock->enable)
652 clock->enable(clock); 661 clock->enable(clock);
653 tk_setup_internals(&timekeeper, clock); 662 tk_setup_internals(tk, clock);
654 663
655 tk_set_xtime(&timekeeper, &now); 664 tk_set_xtime(tk, &now);
656 timekeeper.raw_time.tv_sec = 0; 665 tk->raw_time.tv_sec = 0;
657 timekeeper.raw_time.tv_nsec = 0; 666 tk->raw_time.tv_nsec = 0;
658 if (boot.tv_sec == 0 && boot.tv_nsec == 0) 667 if (boot.tv_sec == 0 && boot.tv_nsec == 0)
659 boot = tk_xtime(&timekeeper); 668 boot = tk_xtime(tk);
660 669
661 set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec); 670 set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec);
662 tk_set_wall_to_mono(&timekeeper, tmp); 671 tk_set_wall_to_mono(tk, tmp);
663 672
664 tmp.tv_sec = 0; 673 tmp.tv_sec = 0;
665 tmp.tv_nsec = 0; 674 tmp.tv_nsec = 0;
666 tk_set_sleep_time(&timekeeper, tmp); 675 tk_set_sleep_time(tk, tmp);
667 676
668 write_sequnlock_irqrestore(&timekeeper.lock, flags); 677 write_sequnlock_irqrestore(&tk->lock, flags);
669} 678}
670 679
671/* time in seconds when suspend began */ 680/* time in seconds when suspend began */
@@ -703,6 +712,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
703 */ 712 */
704void timekeeping_inject_sleeptime(struct timespec *delta) 713void timekeeping_inject_sleeptime(struct timespec *delta)
705{ 714{
715 struct timekeeper *tk = &timekeeper;
706 unsigned long flags; 716 unsigned long flags;
707 struct timespec ts; 717 struct timespec ts;
708 718
@@ -711,15 +721,15 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
711 if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) 721 if (!(ts.tv_sec == 0 && ts.tv_nsec == 0))
712 return; 722 return;
713 723
714 write_seqlock_irqsave(&timekeeper.lock, flags); 724 write_seqlock_irqsave(&tk->lock, flags);
715 725
716 timekeeping_forward_now(&timekeeper); 726 timekeeping_forward_now(tk);
717 727
718 __timekeeping_inject_sleeptime(&timekeeper, delta); 728 __timekeeping_inject_sleeptime(tk, delta);
719 729
720 timekeeping_update(&timekeeper, true); 730 timekeeping_update(tk, true);
721 731
722 write_sequnlock_irqrestore(&timekeeper.lock, flags); 732 write_sequnlock_irqrestore(&tk->lock, flags);
723 733
724 /* signal hrtimers about time change */ 734 /* signal hrtimers about time change */
725 clock_was_set(); 735 clock_was_set();
@@ -734,6 +744,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
734 */ 744 */
735static void timekeeping_resume(void) 745static void timekeeping_resume(void)
736{ 746{
747 struct timekeeper *tk = &timekeeper;
737 unsigned long flags; 748 unsigned long flags;
738 struct timespec ts; 749 struct timespec ts;
739 750
@@ -741,18 +752,18 @@ static void timekeeping_resume(void)
741 752
742 clocksource_resume(); 753 clocksource_resume();
743 754
744 write_seqlock_irqsave(&timekeeper.lock, flags); 755 write_seqlock_irqsave(&tk->lock, flags);
745 756
746 if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { 757 if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
747 ts = timespec_sub(ts, timekeeping_suspend_time); 758 ts = timespec_sub(ts, timekeeping_suspend_time);
748 __timekeeping_inject_sleeptime(&timekeeper, &ts); 759 __timekeeping_inject_sleeptime(tk, &ts);
749 } 760 }
750 /* re-base the last cycle value */ 761 /* re-base the last cycle value */
751 timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); 762 tk->clock->cycle_last = tk->clock->read(tk->clock);
752 timekeeper.ntp_error = 0; 763 tk->ntp_error = 0;
753 timekeeping_suspended = 0; 764 timekeeping_suspended = 0;
754 timekeeping_update(&timekeeper, false); 765 timekeeping_update(tk, false);
755 write_sequnlock_irqrestore(&timekeeper.lock, flags); 766 write_sequnlock_irqrestore(&tk->lock, flags);
756 767
757 touch_softlockup_watchdog(); 768 touch_softlockup_watchdog();
758 769
@@ -764,14 +775,15 @@ static void timekeeping_resume(void)
764 775
765static int timekeeping_suspend(void) 776static int timekeeping_suspend(void)
766{ 777{
778 struct timekeeper *tk = &timekeeper;
767 unsigned long flags; 779 unsigned long flags;
768 struct timespec delta, delta_delta; 780 struct timespec delta, delta_delta;
769 static struct timespec old_delta; 781 static struct timespec old_delta;
770 782
771 read_persistent_clock(&timekeeping_suspend_time); 783 read_persistent_clock(&timekeeping_suspend_time);
772 784
773 write_seqlock_irqsave(&timekeeper.lock, flags); 785 write_seqlock_irqsave(&tk->lock, flags);
774 timekeeping_forward_now(&timekeeper); 786 timekeeping_forward_now(tk);
775 timekeeping_suspended = 1; 787 timekeeping_suspended = 1;
776 788
777 /* 789 /*
@@ -780,7 +792,7 @@ static int timekeeping_suspend(void)
780 * try to compensate so the difference in system time 792 * try to compensate so the difference in system time
781 * and persistent_clock time stays close to constant. 793 * and persistent_clock time stays close to constant.
782 */ 794 */
783 delta = timespec_sub(tk_xtime(&timekeeper), timekeeping_suspend_time); 795 delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time);
784 delta_delta = timespec_sub(delta, old_delta); 796 delta_delta = timespec_sub(delta, old_delta);
785 if (abs(delta_delta.tv_sec) >= 2) { 797 if (abs(delta_delta.tv_sec) >= 2) {
786 /* 798 /*
@@ -793,7 +805,7 @@ static int timekeeping_suspend(void)
793 timekeeping_suspend_time = 805 timekeeping_suspend_time =
794 timespec_add(timekeeping_suspend_time, delta_delta); 806 timespec_add(timekeeping_suspend_time, delta_delta);
795 } 807 }
796 write_sequnlock_irqrestore(&timekeeper.lock, flags); 808 write_sequnlock_irqrestore(&tk->lock, flags);
797 809
798 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); 810 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
799 clocksource_suspend(); 811 clocksource_suspend();
@@ -904,7 +916,7 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
904 * the error. This causes the likely below to be unlikely. 916 * the error. This causes the likely below to be unlikely.
905 * 917 *
906 * The proper fix is to avoid rounding up by using 918 * The proper fix is to avoid rounding up by using
907 * the high precision timekeeper.xtime_nsec instead of 919 * the high precision tk->xtime_nsec instead of
908 * xtime.tv_nsec everywhere. Fixing this will take some 920 * xtime.tv_nsec everywhere. Fixing this will take some
909 * time. 921 * time.
910 */ 922 */
@@ -1094,21 +1106,22 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1094static void update_wall_time(void) 1106static void update_wall_time(void)
1095{ 1107{
1096 struct clocksource *clock; 1108 struct clocksource *clock;
1109 struct timekeeper *tk = &timekeeper;
1097 cycle_t offset; 1110 cycle_t offset;
1098 int shift = 0, maxshift; 1111 int shift = 0, maxshift;
1099 unsigned long flags; 1112 unsigned long flags;
1100 s64 remainder; 1113 s64 remainder;
1101 1114
1102 write_seqlock_irqsave(&timekeeper.lock, flags); 1115 write_seqlock_irqsave(&tk->lock, flags);
1103 1116
1104 /* Make sure we're fully resumed: */ 1117 /* Make sure we're fully resumed: */
1105 if (unlikely(timekeeping_suspended)) 1118 if (unlikely(timekeeping_suspended))
1106 goto out; 1119 goto out;
1107 1120
1108 clock = timekeeper.clock; 1121 clock = tk->clock;
1109 1122
1110#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET 1123#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
1111 offset = timekeeper.cycle_interval; 1124 offset = tk->cycle_interval;
1112#else 1125#else
1113 offset = (clock->read(clock) - clock->cycle_last) & clock->mask; 1126 offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
1114#endif 1127#endif
@@ -1121,19 +1134,19 @@ static void update_wall_time(void)
1121 * chunk in one go, and then try to consume the next smaller 1134 * chunk in one go, and then try to consume the next smaller
1122 * doubled multiple. 1135 * doubled multiple.
1123 */ 1136 */
1124 shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); 1137 shift = ilog2(offset) - ilog2(tk->cycle_interval);
1125 shift = max(0, shift); 1138 shift = max(0, shift);
1126 /* Bound shift to one less than what overflows tick_length */ 1139 /* Bound shift to one less than what overflows tick_length */
1127 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; 1140 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
1128 shift = min(shift, maxshift); 1141 shift = min(shift, maxshift);
1129 while (offset >= timekeeper.cycle_interval) { 1142 while (offset >= tk->cycle_interval) {
1130 offset = logarithmic_accumulation(&timekeeper, offset, shift); 1143 offset = logarithmic_accumulation(tk, offset, shift);
1131 if(offset < timekeeper.cycle_interval<<shift) 1144 if (offset < tk->cycle_interval<<shift)
1132 shift--; 1145 shift--;
1133 } 1146 }
1134 1147
1135 /* correct the clock when NTP error is too big */ 1148 /* correct the clock when NTP error is too big */
1136 timekeeping_adjust(&timekeeper, offset); 1149 timekeeping_adjust(tk, offset);
1137 1150
1138 1151
1139 /* 1152 /*
@@ -1145,21 +1158,21 @@ static void update_wall_time(void)
1145 * the vsyscall implementations are converted to use xtime_nsec 1158 * the vsyscall implementations are converted to use xtime_nsec
1146 * (shifted nanoseconds), this can be killed. 1159 * (shifted nanoseconds), this can be killed.
1147 */ 1160 */
1148 remainder = timekeeper.xtime_nsec & ((1 << timekeeper.shift) - 1); 1161 remainder = tk->xtime_nsec & ((1 << tk->shift) - 1);
1149 timekeeper.xtime_nsec -= remainder; 1162 tk->xtime_nsec -= remainder;
1150 timekeeper.xtime_nsec += 1 << timekeeper.shift; 1163 tk->xtime_nsec += 1 << tk->shift;
1151 timekeeper.ntp_error += remainder << timekeeper.ntp_error_shift; 1164 tk->ntp_error += remainder << tk->ntp_error_shift;
1152 1165
1153 /* 1166 /*
1154 * Finally, make sure that after the rounding 1167 * Finally, make sure that after the rounding
1155 * xtime_nsec isn't larger than NSEC_PER_SEC 1168 * xtime_nsec isn't larger than NSEC_PER_SEC
1156 */ 1169 */
1157 accumulate_nsecs_to_secs(&timekeeper); 1170 accumulate_nsecs_to_secs(tk);
1158 1171
1159 timekeeping_update(&timekeeper, false); 1172 timekeeping_update(tk, false);
1160 1173
1161out: 1174out:
1162 write_sequnlock_irqrestore(&timekeeper.lock, flags); 1175 write_sequnlock_irqrestore(&tk->lock, flags);
1163 1176
1164} 1177}
1165 1178
@@ -1176,11 +1189,12 @@ out:
1176 */ 1189 */
1177void getboottime(struct timespec *ts) 1190void getboottime(struct timespec *ts)
1178{ 1191{
1192 struct timekeeper *tk = &timekeeper;
1179 struct timespec boottime = { 1193 struct timespec boottime = {
1180 .tv_sec = timekeeper.wall_to_monotonic.tv_sec + 1194 .tv_sec = tk->wall_to_monotonic.tv_sec +
1181 timekeeper.total_sleep_time.tv_sec, 1195 tk->total_sleep_time.tv_sec,
1182 .tv_nsec = timekeeper.wall_to_monotonic.tv_nsec + 1196 .tv_nsec = tk->wall_to_monotonic.tv_nsec +
1183 timekeeper.total_sleep_time.tv_nsec 1197 tk->total_sleep_time.tv_nsec
1184 }; 1198 };
1185 1199
1186 set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); 1200 set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec);
@@ -1198,19 +1212,20 @@ EXPORT_SYMBOL_GPL(getboottime);
1198 */ 1212 */
1199void get_monotonic_boottime(struct timespec *ts) 1213void get_monotonic_boottime(struct timespec *ts)
1200{ 1214{
1215 struct timekeeper *tk = &timekeeper;
1201 struct timespec tomono, sleep; 1216 struct timespec tomono, sleep;
1202 unsigned int seq; 1217 unsigned int seq;
1203 1218
1204 WARN_ON(timekeeping_suspended); 1219 WARN_ON(timekeeping_suspended);
1205 1220
1206 do { 1221 do {
1207 seq = read_seqbegin(&timekeeper.lock); 1222 seq = read_seqbegin(&tk->lock);
1208 ts->tv_sec = timekeeper.xtime_sec; 1223 ts->tv_sec = tk->xtime_sec;
1209 ts->tv_nsec = timekeeping_get_ns(&timekeeper); 1224 ts->tv_nsec = timekeeping_get_ns(tk);
1210 tomono = timekeeper.wall_to_monotonic; 1225 tomono = tk->wall_to_monotonic;
1211 sleep = timekeeper.total_sleep_time; 1226 sleep = tk->total_sleep_time;
1212 1227
1213 } while (read_seqretry(&timekeeper.lock, seq)); 1228 } while (read_seqretry(&tk->lock, seq));
1214 1229
1215 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, 1230 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec,
1216 ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec); 1231 ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec);
@@ -1240,31 +1255,38 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime);
1240 */ 1255 */
1241void monotonic_to_bootbased(struct timespec *ts) 1256void monotonic_to_bootbased(struct timespec *ts)
1242{ 1257{
1243 *ts = timespec_add(*ts, timekeeper.total_sleep_time); 1258 struct timekeeper *tk = &timekeeper;
1259
1260 *ts = timespec_add(*ts, tk->total_sleep_time);
1244} 1261}
1245EXPORT_SYMBOL_GPL(monotonic_to_bootbased); 1262EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
1246 1263
1247unsigned long get_seconds(void) 1264unsigned long get_seconds(void)
1248{ 1265{
1249 return timekeeper.xtime_sec; 1266 struct timekeeper *tk = &timekeeper;
1267
1268 return tk->xtime_sec;
1250} 1269}
1251EXPORT_SYMBOL(get_seconds); 1270EXPORT_SYMBOL(get_seconds);
1252 1271
1253struct timespec __current_kernel_time(void) 1272struct timespec __current_kernel_time(void)
1254{ 1273{
1255 return tk_xtime(&timekeeper); 1274 struct timekeeper *tk = &timekeeper;
1275
1276 return tk_xtime(tk);
1256} 1277}
1257 1278
1258struct timespec current_kernel_time(void) 1279struct timespec current_kernel_time(void)
1259{ 1280{
1281 struct timekeeper *tk = &timekeeper;
1260 struct timespec now; 1282 struct timespec now;
1261 unsigned long seq; 1283 unsigned long seq;
1262 1284
1263 do { 1285 do {
1264 seq = read_seqbegin(&timekeeper.lock); 1286 seq = read_seqbegin(&tk->lock);
1265 1287
1266 now = tk_xtime(&timekeeper); 1288 now = tk_xtime(tk);
1267 } while (read_seqretry(&timekeeper.lock, seq)); 1289 } while (read_seqretry(&tk->lock, seq));
1268 1290
1269 return now; 1291 return now;
1270} 1292}
@@ -1272,15 +1294,16 @@ EXPORT_SYMBOL(current_kernel_time);
1272 1294
1273struct timespec get_monotonic_coarse(void) 1295struct timespec get_monotonic_coarse(void)
1274{ 1296{
1297 struct timekeeper *tk = &timekeeper;
1275 struct timespec now, mono; 1298 struct timespec now, mono;
1276 unsigned long seq; 1299 unsigned long seq;
1277 1300
1278 do { 1301 do {
1279 seq = read_seqbegin(&timekeeper.lock); 1302 seq = read_seqbegin(&tk->lock);
1280 1303
1281 now = tk_xtime(&timekeeper); 1304 now = tk_xtime(tk);
1282 mono = timekeeper.wall_to_monotonic; 1305 mono = tk->wall_to_monotonic;
1283 } while (read_seqretry(&timekeeper.lock, seq)); 1306 } while (read_seqretry(&tk->lock, seq));
1284 1307
1285 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, 1308 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
1286 now.tv_nsec + mono.tv_nsec); 1309 now.tv_nsec + mono.tv_nsec);
@@ -1309,14 +1332,15 @@ void do_timer(unsigned long ticks)
1309void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, 1332void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1310 struct timespec *wtom, struct timespec *sleep) 1333 struct timespec *wtom, struct timespec *sleep)
1311{ 1334{
1335 struct timekeeper *tk = &timekeeper;
1312 unsigned long seq; 1336 unsigned long seq;
1313 1337
1314 do { 1338 do {
1315 seq = read_seqbegin(&timekeeper.lock); 1339 seq = read_seqbegin(&tk->lock);
1316 *xtim = tk_xtime(&timekeeper); 1340 *xtim = tk_xtime(tk);
1317 *wtom = timekeeper.wall_to_monotonic; 1341 *wtom = tk->wall_to_monotonic;
1318 *sleep = timekeeper.total_sleep_time; 1342 *sleep = tk->total_sleep_time;
1319 } while (read_seqretry(&timekeeper.lock, seq)); 1343 } while (read_seqretry(&tk->lock, seq));
1320} 1344}
1321 1345
1322#ifdef CONFIG_HIGH_RES_TIMERS 1346#ifdef CONFIG_HIGH_RES_TIMERS
@@ -1330,19 +1354,20 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1330 */ 1354 */
1331ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) 1355ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot)
1332{ 1356{
1357 struct timekeeper *tk = &timekeeper;
1333 ktime_t now; 1358 ktime_t now;
1334 unsigned int seq; 1359 unsigned int seq;
1335 u64 secs, nsecs; 1360 u64 secs, nsecs;
1336 1361
1337 do { 1362 do {
1338 seq = read_seqbegin(&timekeeper.lock); 1363 seq = read_seqbegin(&tk->lock);
1339 1364
1340 secs = timekeeper.xtime_sec; 1365 secs = tk->xtime_sec;
1341 nsecs = timekeeping_get_ns(&timekeeper); 1366 nsecs = timekeeping_get_ns(tk);
1342 1367
1343 *offs_real = timekeeper.offs_real; 1368 *offs_real = tk->offs_real;
1344 *offs_boot = timekeeper.offs_boot; 1369 *offs_boot = tk->offs_boot;
1345 } while (read_seqretry(&timekeeper.lock, seq)); 1370 } while (read_seqretry(&tk->lock, seq));
1346 1371
1347 now = ktime_add_ns(ktime_set(secs, 0), nsecs); 1372 now = ktime_add_ns(ktime_set(secs, 0), nsecs);
1348 now = ktime_sub(now, *offs_real); 1373 now = ktime_sub(now, *offs_real);
@@ -1355,13 +1380,14 @@ ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot)
1355 */ 1380 */
1356ktime_t ktime_get_monotonic_offset(void) 1381ktime_t ktime_get_monotonic_offset(void)
1357{ 1382{
1383 struct timekeeper *tk = &timekeeper;
1358 unsigned long seq; 1384 unsigned long seq;
1359 struct timespec wtom; 1385 struct timespec wtom;
1360 1386
1361 do { 1387 do {
1362 seq = read_seqbegin(&timekeeper.lock); 1388 seq = read_seqbegin(&tk->lock);
1363 wtom = timekeeper.wall_to_monotonic; 1389 wtom = tk->wall_to_monotonic;
1364 } while (read_seqretry(&timekeeper.lock, seq)); 1390 } while (read_seqretry(&tk->lock, seq));
1365 1391
1366 return timespec_to_ktime(wtom); 1392 return timespec_to_ktime(wtom);
1367} 1393}