diff options
Diffstat (limited to 'kernel/time/timekeeping.c')
-rw-r--r-- | kernel/time/timekeeping.c | 282 |
1 files changed, 154 insertions, 128 deletions
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 4da65592b4d9..2988bc819187 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
@@ -292,18 +292,19 @@ static void timekeeping_forward_now(struct timekeeper *tk) | |||
292 | */ | 292 | */ |
293 | void getnstimeofday(struct timespec *ts) | 293 | void getnstimeofday(struct timespec *ts) |
294 | { | 294 | { |
295 | struct timekeeper *tk = &timekeeper; | ||
295 | unsigned long seq; | 296 | unsigned long seq; |
296 | s64 nsecs = 0; | 297 | s64 nsecs = 0; |
297 | 298 | ||
298 | WARN_ON(timekeeping_suspended); | 299 | WARN_ON(timekeeping_suspended); |
299 | 300 | ||
300 | do { | 301 | do { |
301 | seq = read_seqbegin(&timekeeper.lock); | 302 | seq = read_seqbegin(&tk->lock); |
302 | 303 | ||
303 | ts->tv_sec = timekeeper.xtime_sec; | 304 | ts->tv_sec = tk->xtime_sec; |
304 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 305 | ts->tv_nsec = timekeeping_get_ns(tk); |
305 | 306 | ||
306 | } while (read_seqretry(&timekeeper.lock, seq)); | 307 | } while (read_seqretry(&tk->lock, seq)); |
307 | 308 | ||
308 | timespec_add_ns(ts, nsecs); | 309 | timespec_add_ns(ts, nsecs); |
309 | } | 310 | } |
@@ -311,19 +312,18 @@ EXPORT_SYMBOL(getnstimeofday); | |||
311 | 312 | ||
312 | ktime_t ktime_get(void) | 313 | ktime_t ktime_get(void) |
313 | { | 314 | { |
315 | struct timekeeper *tk = &timekeeper; | ||
314 | unsigned int seq; | 316 | unsigned int seq; |
315 | s64 secs, nsecs; | 317 | s64 secs, nsecs; |
316 | 318 | ||
317 | WARN_ON(timekeeping_suspended); | 319 | WARN_ON(timekeeping_suspended); |
318 | 320 | ||
319 | do { | 321 | do { |
320 | seq = read_seqbegin(&timekeeper.lock); | 322 | seq = read_seqbegin(&tk->lock); |
321 | secs = timekeeper.xtime_sec + | 323 | secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; |
322 | timekeeper.wall_to_monotonic.tv_sec; | 324 | nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; |
323 | nsecs = timekeeping_get_ns(&timekeeper) + | ||
324 | timekeeper.wall_to_monotonic.tv_nsec; | ||
325 | 325 | ||
326 | } while (read_seqretry(&timekeeper.lock, seq)); | 326 | } while (read_seqretry(&tk->lock, seq)); |
327 | /* | 327 | /* |
328 | * Use ktime_set/ktime_add_ns to create a proper ktime on | 328 | * Use ktime_set/ktime_add_ns to create a proper ktime on |
329 | * 32-bit architectures without CONFIG_KTIME_SCALAR. | 329 | * 32-bit architectures without CONFIG_KTIME_SCALAR. |
@@ -342,18 +342,19 @@ EXPORT_SYMBOL_GPL(ktime_get); | |||
342 | */ | 342 | */ |
343 | void ktime_get_ts(struct timespec *ts) | 343 | void ktime_get_ts(struct timespec *ts) |
344 | { | 344 | { |
345 | struct timekeeper *tk = &timekeeper; | ||
345 | struct timespec tomono; | 346 | struct timespec tomono; |
346 | unsigned int seq; | 347 | unsigned int seq; |
347 | 348 | ||
348 | WARN_ON(timekeeping_suspended); | 349 | WARN_ON(timekeeping_suspended); |
349 | 350 | ||
350 | do { | 351 | do { |
351 | seq = read_seqbegin(&timekeeper.lock); | 352 | seq = read_seqbegin(&tk->lock); |
352 | ts->tv_sec = timekeeper.xtime_sec; | 353 | ts->tv_sec = tk->xtime_sec; |
353 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 354 | ts->tv_nsec = timekeeping_get_ns(tk); |
354 | tomono = timekeeper.wall_to_monotonic; | 355 | tomono = tk->wall_to_monotonic; |
355 | 356 | ||
356 | } while (read_seqretry(&timekeeper.lock, seq)); | 357 | } while (read_seqretry(&tk->lock, seq)); |
357 | 358 | ||
358 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, | 359 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, |
359 | ts->tv_nsec + tomono.tv_nsec); | 360 | ts->tv_nsec + tomono.tv_nsec); |
@@ -373,22 +374,23 @@ EXPORT_SYMBOL_GPL(ktime_get_ts); | |||
373 | */ | 374 | */ |
374 | void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) | 375 | void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) |
375 | { | 376 | { |
377 | struct timekeeper *tk = &timekeeper; | ||
376 | unsigned long seq; | 378 | unsigned long seq; |
377 | s64 nsecs_raw, nsecs_real; | 379 | s64 nsecs_raw, nsecs_real; |
378 | 380 | ||
379 | WARN_ON_ONCE(timekeeping_suspended); | 381 | WARN_ON_ONCE(timekeeping_suspended); |
380 | 382 | ||
381 | do { | 383 | do { |
382 | seq = read_seqbegin(&timekeeper.lock); | 384 | seq = read_seqbegin(&tk->lock); |
383 | 385 | ||
384 | *ts_raw = timekeeper.raw_time; | 386 | *ts_raw = tk->raw_time; |
385 | ts_real->tv_sec = timekeeper.xtime_sec; | 387 | ts_real->tv_sec = tk->xtime_sec; |
386 | ts_real->tv_nsec = 0; | 388 | ts_real->tv_nsec = 0; |
387 | 389 | ||
388 | nsecs_raw = timekeeping_get_ns_raw(&timekeeper); | 390 | nsecs_raw = timekeeping_get_ns_raw(tk); |
389 | nsecs_real = timekeeping_get_ns(&timekeeper); | 391 | nsecs_real = timekeeping_get_ns(tk); |
390 | 392 | ||
391 | } while (read_seqretry(&timekeeper.lock, seq)); | 393 | } while (read_seqretry(&tk->lock, seq)); |
392 | 394 | ||
393 | timespec_add_ns(ts_raw, nsecs_raw); | 395 | timespec_add_ns(ts_raw, nsecs_raw); |
394 | timespec_add_ns(ts_real, nsecs_real); | 396 | timespec_add_ns(ts_real, nsecs_real); |
@@ -421,28 +423,28 @@ EXPORT_SYMBOL(do_gettimeofday); | |||
421 | */ | 423 | */ |
422 | int do_settimeofday(const struct timespec *tv) | 424 | int do_settimeofday(const struct timespec *tv) |
423 | { | 425 | { |
426 | struct timekeeper *tk = &timekeeper; | ||
424 | struct timespec ts_delta, xt; | 427 | struct timespec ts_delta, xt; |
425 | unsigned long flags; | 428 | unsigned long flags; |
426 | 429 | ||
427 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | 430 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) |
428 | return -EINVAL; | 431 | return -EINVAL; |
429 | 432 | ||
430 | write_seqlock_irqsave(&timekeeper.lock, flags); | 433 | write_seqlock_irqsave(&tk->lock, flags); |
431 | 434 | ||
432 | timekeeping_forward_now(&timekeeper); | 435 | timekeeping_forward_now(tk); |
433 | 436 | ||
434 | xt = tk_xtime(&timekeeper); | 437 | xt = tk_xtime(tk); |
435 | ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; | 438 | ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; |
436 | ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; | 439 | ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; |
437 | 440 | ||
438 | tk_set_wall_to_mono(&timekeeper, | 441 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta)); |
439 | timespec_sub(timekeeper.wall_to_monotonic, ts_delta)); | ||
440 | 442 | ||
441 | tk_set_xtime(&timekeeper, tv); | 443 | tk_set_xtime(tk, tv); |
442 | 444 | ||
443 | timekeeping_update(&timekeeper, true); | 445 | timekeeping_update(tk, true); |
444 | 446 | ||
445 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 447 | write_sequnlock_irqrestore(&tk->lock, flags); |
446 | 448 | ||
447 | /* signal hrtimers about time change */ | 449 | /* signal hrtimers about time change */ |
448 | clock_was_set(); | 450 | clock_was_set(); |
@@ -459,23 +461,23 @@ EXPORT_SYMBOL(do_settimeofday); | |||
459 | */ | 461 | */ |
460 | int timekeeping_inject_offset(struct timespec *ts) | 462 | int timekeeping_inject_offset(struct timespec *ts) |
461 | { | 463 | { |
464 | struct timekeeper *tk = &timekeeper; | ||
462 | unsigned long flags; | 465 | unsigned long flags; |
463 | 466 | ||
464 | if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) | 467 | if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) |
465 | return -EINVAL; | 468 | return -EINVAL; |
466 | 469 | ||
467 | write_seqlock_irqsave(&timekeeper.lock, flags); | 470 | write_seqlock_irqsave(&tk->lock, flags); |
468 | 471 | ||
469 | timekeeping_forward_now(&timekeeper); | 472 | timekeeping_forward_now(tk); |
470 | 473 | ||
471 | 474 | ||
472 | tk_xtime_add(&timekeeper, ts); | 475 | tk_xtime_add(tk, ts); |
473 | tk_set_wall_to_mono(&timekeeper, | 476 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts)); |
474 | timespec_sub(timekeeper.wall_to_monotonic, *ts)); | ||
475 | 477 | ||
476 | timekeeping_update(&timekeeper, true); | 478 | timekeeping_update(tk, true); |
477 | 479 | ||
478 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 480 | write_sequnlock_irqrestore(&tk->lock, flags); |
479 | 481 | ||
480 | /* signal hrtimers about time change */ | 482 | /* signal hrtimers about time change */ |
481 | clock_was_set(); | 483 | clock_was_set(); |
@@ -491,23 +493,24 @@ EXPORT_SYMBOL(timekeeping_inject_offset); | |||
491 | */ | 493 | */ |
492 | static int change_clocksource(void *data) | 494 | static int change_clocksource(void *data) |
493 | { | 495 | { |
496 | struct timekeeper *tk = &timekeeper; | ||
494 | struct clocksource *new, *old; | 497 | struct clocksource *new, *old; |
495 | unsigned long flags; | 498 | unsigned long flags; |
496 | 499 | ||
497 | new = (struct clocksource *) data; | 500 | new = (struct clocksource *) data; |
498 | 501 | ||
499 | write_seqlock_irqsave(&timekeeper.lock, flags); | 502 | write_seqlock_irqsave(&tk->lock, flags); |
500 | 503 | ||
501 | timekeeping_forward_now(&timekeeper); | 504 | timekeeping_forward_now(tk); |
502 | if (!new->enable || new->enable(new) == 0) { | 505 | if (!new->enable || new->enable(new) == 0) { |
503 | old = timekeeper.clock; | 506 | old = tk->clock; |
504 | tk_setup_internals(&timekeeper, new); | 507 | tk_setup_internals(tk, new); |
505 | if (old->disable) | 508 | if (old->disable) |
506 | old->disable(old); | 509 | old->disable(old); |
507 | } | 510 | } |
508 | timekeeping_update(&timekeeper, true); | 511 | timekeeping_update(tk, true); |
509 | 512 | ||
510 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 513 | write_sequnlock_irqrestore(&tk->lock, flags); |
511 | 514 | ||
512 | return 0; | 515 | return 0; |
513 | } | 516 | } |
@@ -521,7 +524,9 @@ static int change_clocksource(void *data) | |||
521 | */ | 524 | */ |
522 | void timekeeping_notify(struct clocksource *clock) | 525 | void timekeeping_notify(struct clocksource *clock) |
523 | { | 526 | { |
524 | if (timekeeper.clock == clock) | 527 | struct timekeeper *tk = &timekeeper; |
528 | |||
529 | if (tk->clock == clock) | ||
525 | return; | 530 | return; |
526 | stop_machine(change_clocksource, clock, NULL); | 531 | stop_machine(change_clocksource, clock, NULL); |
527 | tick_clock_notify(); | 532 | tick_clock_notify(); |
@@ -550,15 +555,16 @@ EXPORT_SYMBOL_GPL(ktime_get_real); | |||
550 | */ | 555 | */ |
551 | void getrawmonotonic(struct timespec *ts) | 556 | void getrawmonotonic(struct timespec *ts) |
552 | { | 557 | { |
558 | struct timekeeper *tk = &timekeeper; | ||
553 | unsigned long seq; | 559 | unsigned long seq; |
554 | s64 nsecs; | 560 | s64 nsecs; |
555 | 561 | ||
556 | do { | 562 | do { |
557 | seq = read_seqbegin(&timekeeper.lock); | 563 | seq = read_seqbegin(&tk->lock); |
558 | nsecs = timekeeping_get_ns_raw(&timekeeper); | 564 | nsecs = timekeeping_get_ns_raw(tk); |
559 | *ts = timekeeper.raw_time; | 565 | *ts = tk->raw_time; |
560 | 566 | ||
561 | } while (read_seqretry(&timekeeper.lock, seq)); | 567 | } while (read_seqretry(&tk->lock, seq)); |
562 | 568 | ||
563 | timespec_add_ns(ts, nsecs); | 569 | timespec_add_ns(ts, nsecs); |
564 | } | 570 | } |
@@ -569,15 +575,16 @@ EXPORT_SYMBOL(getrawmonotonic); | |||
569 | */ | 575 | */ |
570 | int timekeeping_valid_for_hres(void) | 576 | int timekeeping_valid_for_hres(void) |
571 | { | 577 | { |
578 | struct timekeeper *tk = &timekeeper; | ||
572 | unsigned long seq; | 579 | unsigned long seq; |
573 | int ret; | 580 | int ret; |
574 | 581 | ||
575 | do { | 582 | do { |
576 | seq = read_seqbegin(&timekeeper.lock); | 583 | seq = read_seqbegin(&tk->lock); |
577 | 584 | ||
578 | ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; | 585 | ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; |
579 | 586 | ||
580 | } while (read_seqretry(&timekeeper.lock, seq)); | 587 | } while (read_seqretry(&tk->lock, seq)); |
581 | 588 | ||
582 | return ret; | 589 | return ret; |
583 | } | 590 | } |
@@ -587,15 +594,16 @@ int timekeeping_valid_for_hres(void) | |||
587 | */ | 594 | */ |
588 | u64 timekeeping_max_deferment(void) | 595 | u64 timekeeping_max_deferment(void) |
589 | { | 596 | { |
597 | struct timekeeper *tk = &timekeeper; | ||
590 | unsigned long seq; | 598 | unsigned long seq; |
591 | u64 ret; | 599 | u64 ret; |
592 | 600 | ||
593 | do { | 601 | do { |
594 | seq = read_seqbegin(&timekeeper.lock); | 602 | seq = read_seqbegin(&tk->lock); |
595 | 603 | ||
596 | ret = timekeeper.clock->max_idle_ns; | 604 | ret = tk->clock->max_idle_ns; |
597 | 605 | ||
598 | } while (read_seqretry(&timekeeper.lock, seq)); | 606 | } while (read_seqretry(&tk->lock, seq)); |
599 | 607 | ||
600 | return ret; | 608 | return ret; |
601 | } | 609 | } |
@@ -635,6 +643,7 @@ void __attribute__((weak)) read_boot_clock(struct timespec *ts) | |||
635 | */ | 643 | */ |
636 | void __init timekeeping_init(void) | 644 | void __init timekeeping_init(void) |
637 | { | 645 | { |
646 | struct timekeeper *tk = &timekeeper; | ||
638 | struct clocksource *clock; | 647 | struct clocksource *clock; |
639 | unsigned long flags; | 648 | unsigned long flags; |
640 | struct timespec now, boot, tmp; | 649 | struct timespec now, boot, tmp; |
@@ -642,30 +651,30 @@ void __init timekeeping_init(void) | |||
642 | read_persistent_clock(&now); | 651 | read_persistent_clock(&now); |
643 | read_boot_clock(&boot); | 652 | read_boot_clock(&boot); |
644 | 653 | ||
645 | seqlock_init(&timekeeper.lock); | 654 | seqlock_init(&tk->lock); |
646 | 655 | ||
647 | ntp_init(); | 656 | ntp_init(); |
648 | 657 | ||
649 | write_seqlock_irqsave(&timekeeper.lock, flags); | 658 | write_seqlock_irqsave(&tk->lock, flags); |
650 | clock = clocksource_default_clock(); | 659 | clock = clocksource_default_clock(); |
651 | if (clock->enable) | 660 | if (clock->enable) |
652 | clock->enable(clock); | 661 | clock->enable(clock); |
653 | tk_setup_internals(&timekeeper, clock); | 662 | tk_setup_internals(tk, clock); |
654 | 663 | ||
655 | tk_set_xtime(&timekeeper, &now); | 664 | tk_set_xtime(tk, &now); |
656 | timekeeper.raw_time.tv_sec = 0; | 665 | tk->raw_time.tv_sec = 0; |
657 | timekeeper.raw_time.tv_nsec = 0; | 666 | tk->raw_time.tv_nsec = 0; |
658 | if (boot.tv_sec == 0 && boot.tv_nsec == 0) | 667 | if (boot.tv_sec == 0 && boot.tv_nsec == 0) |
659 | boot = tk_xtime(&timekeeper); | 668 | boot = tk_xtime(tk); |
660 | 669 | ||
661 | set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec); | 670 | set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec); |
662 | tk_set_wall_to_mono(&timekeeper, tmp); | 671 | tk_set_wall_to_mono(tk, tmp); |
663 | 672 | ||
664 | tmp.tv_sec = 0; | 673 | tmp.tv_sec = 0; |
665 | tmp.tv_nsec = 0; | 674 | tmp.tv_nsec = 0; |
666 | tk_set_sleep_time(&timekeeper, tmp); | 675 | tk_set_sleep_time(tk, tmp); |
667 | 676 | ||
668 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 677 | write_sequnlock_irqrestore(&tk->lock, flags); |
669 | } | 678 | } |
670 | 679 | ||
671 | /* time in seconds when suspend began */ | 680 | /* time in seconds when suspend began */ |
@@ -703,6 +712,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk, | |||
703 | */ | 712 | */ |
704 | void timekeeping_inject_sleeptime(struct timespec *delta) | 713 | void timekeeping_inject_sleeptime(struct timespec *delta) |
705 | { | 714 | { |
715 | struct timekeeper *tk = &timekeeper; | ||
706 | unsigned long flags; | 716 | unsigned long flags; |
707 | struct timespec ts; | 717 | struct timespec ts; |
708 | 718 | ||
@@ -711,15 +721,15 @@ void timekeeping_inject_sleeptime(struct timespec *delta) | |||
711 | if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) | 721 | if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) |
712 | return; | 722 | return; |
713 | 723 | ||
714 | write_seqlock_irqsave(&timekeeper.lock, flags); | 724 | write_seqlock_irqsave(&tk->lock, flags); |
715 | 725 | ||
716 | timekeeping_forward_now(&timekeeper); | 726 | timekeeping_forward_now(tk); |
717 | 727 | ||
718 | __timekeeping_inject_sleeptime(&timekeeper, delta); | 728 | __timekeeping_inject_sleeptime(tk, delta); |
719 | 729 | ||
720 | timekeeping_update(&timekeeper, true); | 730 | timekeeping_update(tk, true); |
721 | 731 | ||
722 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 732 | write_sequnlock_irqrestore(&tk->lock, flags); |
723 | 733 | ||
724 | /* signal hrtimers about time change */ | 734 | /* signal hrtimers about time change */ |
725 | clock_was_set(); | 735 | clock_was_set(); |
@@ -734,6 +744,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) | |||
734 | */ | 744 | */ |
735 | static void timekeeping_resume(void) | 745 | static void timekeeping_resume(void) |
736 | { | 746 | { |
747 | struct timekeeper *tk = &timekeeper; | ||
737 | unsigned long flags; | 748 | unsigned long flags; |
738 | struct timespec ts; | 749 | struct timespec ts; |
739 | 750 | ||
@@ -741,18 +752,18 @@ static void timekeeping_resume(void) | |||
741 | 752 | ||
742 | clocksource_resume(); | 753 | clocksource_resume(); |
743 | 754 | ||
744 | write_seqlock_irqsave(&timekeeper.lock, flags); | 755 | write_seqlock_irqsave(&tk->lock, flags); |
745 | 756 | ||
746 | if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { | 757 | if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { |
747 | ts = timespec_sub(ts, timekeeping_suspend_time); | 758 | ts = timespec_sub(ts, timekeeping_suspend_time); |
748 | __timekeeping_inject_sleeptime(&timekeeper, &ts); | 759 | __timekeeping_inject_sleeptime(tk, &ts); |
749 | } | 760 | } |
750 | /* re-base the last cycle value */ | 761 | /* re-base the last cycle value */ |
751 | timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); | 762 | tk->clock->cycle_last = tk->clock->read(tk->clock); |
752 | timekeeper.ntp_error = 0; | 763 | tk->ntp_error = 0; |
753 | timekeeping_suspended = 0; | 764 | timekeeping_suspended = 0; |
754 | timekeeping_update(&timekeeper, false); | 765 | timekeeping_update(tk, false); |
755 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 766 | write_sequnlock_irqrestore(&tk->lock, flags); |
756 | 767 | ||
757 | touch_softlockup_watchdog(); | 768 | touch_softlockup_watchdog(); |
758 | 769 | ||
@@ -764,14 +775,15 @@ static void timekeeping_resume(void) | |||
764 | 775 | ||
765 | static int timekeeping_suspend(void) | 776 | static int timekeeping_suspend(void) |
766 | { | 777 | { |
778 | struct timekeeper *tk = &timekeeper; | ||
767 | unsigned long flags; | 779 | unsigned long flags; |
768 | struct timespec delta, delta_delta; | 780 | struct timespec delta, delta_delta; |
769 | static struct timespec old_delta; | 781 | static struct timespec old_delta; |
770 | 782 | ||
771 | read_persistent_clock(&timekeeping_suspend_time); | 783 | read_persistent_clock(&timekeeping_suspend_time); |
772 | 784 | ||
773 | write_seqlock_irqsave(&timekeeper.lock, flags); | 785 | write_seqlock_irqsave(&tk->lock, flags); |
774 | timekeeping_forward_now(&timekeeper); | 786 | timekeeping_forward_now(tk); |
775 | timekeeping_suspended = 1; | 787 | timekeeping_suspended = 1; |
776 | 788 | ||
777 | /* | 789 | /* |
@@ -780,7 +792,7 @@ static int timekeeping_suspend(void) | |||
780 | * try to compensate so the difference in system time | 792 | * try to compensate so the difference in system time |
781 | * and persistent_clock time stays close to constant. | 793 | * and persistent_clock time stays close to constant. |
782 | */ | 794 | */ |
783 | delta = timespec_sub(tk_xtime(&timekeeper), timekeeping_suspend_time); | 795 | delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time); |
784 | delta_delta = timespec_sub(delta, old_delta); | 796 | delta_delta = timespec_sub(delta, old_delta); |
785 | if (abs(delta_delta.tv_sec) >= 2) { | 797 | if (abs(delta_delta.tv_sec) >= 2) { |
786 | /* | 798 | /* |
@@ -793,7 +805,7 @@ static int timekeeping_suspend(void) | |||
793 | timekeeping_suspend_time = | 805 | timekeeping_suspend_time = |
794 | timespec_add(timekeeping_suspend_time, delta_delta); | 806 | timespec_add(timekeeping_suspend_time, delta_delta); |
795 | } | 807 | } |
796 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 808 | write_sequnlock_irqrestore(&tk->lock, flags); |
797 | 809 | ||
798 | clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); | 810 | clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); |
799 | clocksource_suspend(); | 811 | clocksource_suspend(); |
@@ -904,7 +916,7 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) | |||
904 | * the error. This causes the likely below to be unlikely. | 916 | * the error. This causes the likely below to be unlikely. |
905 | * | 917 | * |
906 | * The proper fix is to avoid rounding up by using | 918 | * The proper fix is to avoid rounding up by using |
907 | * the high precision timekeeper.xtime_nsec instead of | 919 | * the high precision tk->xtime_nsec instead of |
908 | * xtime.tv_nsec everywhere. Fixing this will take some | 920 | * xtime.tv_nsec everywhere. Fixing this will take some |
909 | * time. | 921 | * time. |
910 | */ | 922 | */ |
@@ -1094,21 +1106,22 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, | |||
1094 | static void update_wall_time(void) | 1106 | static void update_wall_time(void) |
1095 | { | 1107 | { |
1096 | struct clocksource *clock; | 1108 | struct clocksource *clock; |
1109 | struct timekeeper *tk = &timekeeper; | ||
1097 | cycle_t offset; | 1110 | cycle_t offset; |
1098 | int shift = 0, maxshift; | 1111 | int shift = 0, maxshift; |
1099 | unsigned long flags; | 1112 | unsigned long flags; |
1100 | s64 remainder; | 1113 | s64 remainder; |
1101 | 1114 | ||
1102 | write_seqlock_irqsave(&timekeeper.lock, flags); | 1115 | write_seqlock_irqsave(&tk->lock, flags); |
1103 | 1116 | ||
1104 | /* Make sure we're fully resumed: */ | 1117 | /* Make sure we're fully resumed: */ |
1105 | if (unlikely(timekeeping_suspended)) | 1118 | if (unlikely(timekeeping_suspended)) |
1106 | goto out; | 1119 | goto out; |
1107 | 1120 | ||
1108 | clock = timekeeper.clock; | 1121 | clock = tk->clock; |
1109 | 1122 | ||
1110 | #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET | 1123 | #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET |
1111 | offset = timekeeper.cycle_interval; | 1124 | offset = tk->cycle_interval; |
1112 | #else | 1125 | #else |
1113 | offset = (clock->read(clock) - clock->cycle_last) & clock->mask; | 1126 | offset = (clock->read(clock) - clock->cycle_last) & clock->mask; |
1114 | #endif | 1127 | #endif |
@@ -1121,19 +1134,19 @@ static void update_wall_time(void) | |||
1121 | * chunk in one go, and then try to consume the next smaller | 1134 | * chunk in one go, and then try to consume the next smaller |
1122 | * doubled multiple. | 1135 | * doubled multiple. |
1123 | */ | 1136 | */ |
1124 | shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); | 1137 | shift = ilog2(offset) - ilog2(tk->cycle_interval); |
1125 | shift = max(0, shift); | 1138 | shift = max(0, shift); |
1126 | /* Bound shift to one less than what overflows tick_length */ | 1139 | /* Bound shift to one less than what overflows tick_length */ |
1127 | maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; | 1140 | maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; |
1128 | shift = min(shift, maxshift); | 1141 | shift = min(shift, maxshift); |
1129 | while (offset >= timekeeper.cycle_interval) { | 1142 | while (offset >= tk->cycle_interval) { |
1130 | offset = logarithmic_accumulation(&timekeeper, offset, shift); | 1143 | offset = logarithmic_accumulation(tk, offset, shift); |
1131 | if(offset < timekeeper.cycle_interval<<shift) | 1144 | if (offset < tk->cycle_interval<<shift) |
1132 | shift--; | 1145 | shift--; |
1133 | } | 1146 | } |
1134 | 1147 | ||
1135 | /* correct the clock when NTP error is too big */ | 1148 | /* correct the clock when NTP error is too big */ |
1136 | timekeeping_adjust(&timekeeper, offset); | 1149 | timekeeping_adjust(tk, offset); |
1137 | 1150 | ||
1138 | 1151 | ||
1139 | /* | 1152 | /* |
@@ -1145,21 +1158,21 @@ static void update_wall_time(void) | |||
1145 | * the vsyscall implementations are converted to use xtime_nsec | 1158 | * the vsyscall implementations are converted to use xtime_nsec |
1146 | * (shifted nanoseconds), this can be killed. | 1159 | * (shifted nanoseconds), this can be killed. |
1147 | */ | 1160 | */ |
1148 | remainder = timekeeper.xtime_nsec & ((1 << timekeeper.shift) - 1); | 1161 | remainder = tk->xtime_nsec & ((1 << tk->shift) - 1); |
1149 | timekeeper.xtime_nsec -= remainder; | 1162 | tk->xtime_nsec -= remainder; |
1150 | timekeeper.xtime_nsec += 1 << timekeeper.shift; | 1163 | tk->xtime_nsec += 1 << tk->shift; |
1151 | timekeeper.ntp_error += remainder << timekeeper.ntp_error_shift; | 1164 | tk->ntp_error += remainder << tk->ntp_error_shift; |
1152 | 1165 | ||
1153 | /* | 1166 | /* |
1154 | * Finally, make sure that after the rounding | 1167 | * Finally, make sure that after the rounding |
1155 | * xtime_nsec isn't larger than NSEC_PER_SEC | 1168 | * xtime_nsec isn't larger than NSEC_PER_SEC |
1156 | */ | 1169 | */ |
1157 | accumulate_nsecs_to_secs(&timekeeper); | 1170 | accumulate_nsecs_to_secs(tk); |
1158 | 1171 | ||
1159 | timekeeping_update(&timekeeper, false); | 1172 | timekeeping_update(tk, false); |
1160 | 1173 | ||
1161 | out: | 1174 | out: |
1162 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 1175 | write_sequnlock_irqrestore(&tk->lock, flags); |
1163 | 1176 | ||
1164 | } | 1177 | } |
1165 | 1178 | ||
@@ -1176,11 +1189,12 @@ out: | |||
1176 | */ | 1189 | */ |
1177 | void getboottime(struct timespec *ts) | 1190 | void getboottime(struct timespec *ts) |
1178 | { | 1191 | { |
1192 | struct timekeeper *tk = &timekeeper; | ||
1179 | struct timespec boottime = { | 1193 | struct timespec boottime = { |
1180 | .tv_sec = timekeeper.wall_to_monotonic.tv_sec + | 1194 | .tv_sec = tk->wall_to_monotonic.tv_sec + |
1181 | timekeeper.total_sleep_time.tv_sec, | 1195 | tk->total_sleep_time.tv_sec, |
1182 | .tv_nsec = timekeeper.wall_to_monotonic.tv_nsec + | 1196 | .tv_nsec = tk->wall_to_monotonic.tv_nsec + |
1183 | timekeeper.total_sleep_time.tv_nsec | 1197 | tk->total_sleep_time.tv_nsec |
1184 | }; | 1198 | }; |
1185 | 1199 | ||
1186 | set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); | 1200 | set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); |
@@ -1198,19 +1212,20 @@ EXPORT_SYMBOL_GPL(getboottime); | |||
1198 | */ | 1212 | */ |
1199 | void get_monotonic_boottime(struct timespec *ts) | 1213 | void get_monotonic_boottime(struct timespec *ts) |
1200 | { | 1214 | { |
1215 | struct timekeeper *tk = &timekeeper; | ||
1201 | struct timespec tomono, sleep; | 1216 | struct timespec tomono, sleep; |
1202 | unsigned int seq; | 1217 | unsigned int seq; |
1203 | 1218 | ||
1204 | WARN_ON(timekeeping_suspended); | 1219 | WARN_ON(timekeeping_suspended); |
1205 | 1220 | ||
1206 | do { | 1221 | do { |
1207 | seq = read_seqbegin(&timekeeper.lock); | 1222 | seq = read_seqbegin(&tk->lock); |
1208 | ts->tv_sec = timekeeper.xtime_sec; | 1223 | ts->tv_sec = tk->xtime_sec; |
1209 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 1224 | ts->tv_nsec = timekeeping_get_ns(tk); |
1210 | tomono = timekeeper.wall_to_monotonic; | 1225 | tomono = tk->wall_to_monotonic; |
1211 | sleep = timekeeper.total_sleep_time; | 1226 | sleep = tk->total_sleep_time; |
1212 | 1227 | ||
1213 | } while (read_seqretry(&timekeeper.lock, seq)); | 1228 | } while (read_seqretry(&tk->lock, seq)); |
1214 | 1229 | ||
1215 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, | 1230 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, |
1216 | ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec); | 1231 | ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec); |
@@ -1240,31 +1255,38 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime); | |||
1240 | */ | 1255 | */ |
1241 | void monotonic_to_bootbased(struct timespec *ts) | 1256 | void monotonic_to_bootbased(struct timespec *ts) |
1242 | { | 1257 | { |
1243 | *ts = timespec_add(*ts, timekeeper.total_sleep_time); | 1258 | struct timekeeper *tk = &timekeeper; |
1259 | |||
1260 | *ts = timespec_add(*ts, tk->total_sleep_time); | ||
1244 | } | 1261 | } |
1245 | EXPORT_SYMBOL_GPL(monotonic_to_bootbased); | 1262 | EXPORT_SYMBOL_GPL(monotonic_to_bootbased); |
1246 | 1263 | ||
1247 | unsigned long get_seconds(void) | 1264 | unsigned long get_seconds(void) |
1248 | { | 1265 | { |
1249 | return timekeeper.xtime_sec; | 1266 | struct timekeeper *tk = &timekeeper; |
1267 | |||
1268 | return tk->xtime_sec; | ||
1250 | } | 1269 | } |
1251 | EXPORT_SYMBOL(get_seconds); | 1270 | EXPORT_SYMBOL(get_seconds); |
1252 | 1271 | ||
1253 | struct timespec __current_kernel_time(void) | 1272 | struct timespec __current_kernel_time(void) |
1254 | { | 1273 | { |
1255 | return tk_xtime(&timekeeper); | 1274 | struct timekeeper *tk = &timekeeper; |
1275 | |||
1276 | return tk_xtime(tk); | ||
1256 | } | 1277 | } |
1257 | 1278 | ||
1258 | struct timespec current_kernel_time(void) | 1279 | struct timespec current_kernel_time(void) |
1259 | { | 1280 | { |
1281 | struct timekeeper *tk = &timekeeper; | ||
1260 | struct timespec now; | 1282 | struct timespec now; |
1261 | unsigned long seq; | 1283 | unsigned long seq; |
1262 | 1284 | ||
1263 | do { | 1285 | do { |
1264 | seq = read_seqbegin(&timekeeper.lock); | 1286 | seq = read_seqbegin(&tk->lock); |
1265 | 1287 | ||
1266 | now = tk_xtime(&timekeeper); | 1288 | now = tk_xtime(tk); |
1267 | } while (read_seqretry(&timekeeper.lock, seq)); | 1289 | } while (read_seqretry(&tk->lock, seq)); |
1268 | 1290 | ||
1269 | return now; | 1291 | return now; |
1270 | } | 1292 | } |
@@ -1272,15 +1294,16 @@ EXPORT_SYMBOL(current_kernel_time); | |||
1272 | 1294 | ||
1273 | struct timespec get_monotonic_coarse(void) | 1295 | struct timespec get_monotonic_coarse(void) |
1274 | { | 1296 | { |
1297 | struct timekeeper *tk = &timekeeper; | ||
1275 | struct timespec now, mono; | 1298 | struct timespec now, mono; |
1276 | unsigned long seq; | 1299 | unsigned long seq; |
1277 | 1300 | ||
1278 | do { | 1301 | do { |
1279 | seq = read_seqbegin(&timekeeper.lock); | 1302 | seq = read_seqbegin(&tk->lock); |
1280 | 1303 | ||
1281 | now = tk_xtime(&timekeeper); | 1304 | now = tk_xtime(tk); |
1282 | mono = timekeeper.wall_to_monotonic; | 1305 | mono = tk->wall_to_monotonic; |
1283 | } while (read_seqretry(&timekeeper.lock, seq)); | 1306 | } while (read_seqretry(&tk->lock, seq)); |
1284 | 1307 | ||
1285 | set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, | 1308 | set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, |
1286 | now.tv_nsec + mono.tv_nsec); | 1309 | now.tv_nsec + mono.tv_nsec); |
@@ -1309,14 +1332,15 @@ void do_timer(unsigned long ticks) | |||
1309 | void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, | 1332 | void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, |
1310 | struct timespec *wtom, struct timespec *sleep) | 1333 | struct timespec *wtom, struct timespec *sleep) |
1311 | { | 1334 | { |
1335 | struct timekeeper *tk = &timekeeper; | ||
1312 | unsigned long seq; | 1336 | unsigned long seq; |
1313 | 1337 | ||
1314 | do { | 1338 | do { |
1315 | seq = read_seqbegin(&timekeeper.lock); | 1339 | seq = read_seqbegin(&tk->lock); |
1316 | *xtim = tk_xtime(&timekeeper); | 1340 | *xtim = tk_xtime(tk); |
1317 | *wtom = timekeeper.wall_to_monotonic; | 1341 | *wtom = tk->wall_to_monotonic; |
1318 | *sleep = timekeeper.total_sleep_time; | 1342 | *sleep = tk->total_sleep_time; |
1319 | } while (read_seqretry(&timekeeper.lock, seq)); | 1343 | } while (read_seqretry(&tk->lock, seq)); |
1320 | } | 1344 | } |
1321 | 1345 | ||
1322 | #ifdef CONFIG_HIGH_RES_TIMERS | 1346 | #ifdef CONFIG_HIGH_RES_TIMERS |
@@ -1330,19 +1354,20 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, | |||
1330 | */ | 1354 | */ |
1331 | ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) | 1355 | ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) |
1332 | { | 1356 | { |
1357 | struct timekeeper *tk = &timekeeper; | ||
1333 | ktime_t now; | 1358 | ktime_t now; |
1334 | unsigned int seq; | 1359 | unsigned int seq; |
1335 | u64 secs, nsecs; | 1360 | u64 secs, nsecs; |
1336 | 1361 | ||
1337 | do { | 1362 | do { |
1338 | seq = read_seqbegin(&timekeeper.lock); | 1363 | seq = read_seqbegin(&tk->lock); |
1339 | 1364 | ||
1340 | secs = timekeeper.xtime_sec; | 1365 | secs = tk->xtime_sec; |
1341 | nsecs = timekeeping_get_ns(&timekeeper); | 1366 | nsecs = timekeeping_get_ns(tk); |
1342 | 1367 | ||
1343 | *offs_real = timekeeper.offs_real; | 1368 | *offs_real = tk->offs_real; |
1344 | *offs_boot = timekeeper.offs_boot; | 1369 | *offs_boot = tk->offs_boot; |
1345 | } while (read_seqretry(&timekeeper.lock, seq)); | 1370 | } while (read_seqretry(&tk->lock, seq)); |
1346 | 1371 | ||
1347 | now = ktime_add_ns(ktime_set(secs, 0), nsecs); | 1372 | now = ktime_add_ns(ktime_set(secs, 0), nsecs); |
1348 | now = ktime_sub(now, *offs_real); | 1373 | now = ktime_sub(now, *offs_real); |
@@ -1355,13 +1380,14 @@ ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) | |||
1355 | */ | 1380 | */ |
1356 | ktime_t ktime_get_monotonic_offset(void) | 1381 | ktime_t ktime_get_monotonic_offset(void) |
1357 | { | 1382 | { |
1383 | struct timekeeper *tk = &timekeeper; | ||
1358 | unsigned long seq; | 1384 | unsigned long seq; |
1359 | struct timespec wtom; | 1385 | struct timespec wtom; |
1360 | 1386 | ||
1361 | do { | 1387 | do { |
1362 | seq = read_seqbegin(&timekeeper.lock); | 1388 | seq = read_seqbegin(&tk->lock); |
1363 | wtom = timekeeper.wall_to_monotonic; | 1389 | wtom = tk->wall_to_monotonic; |
1364 | } while (read_seqretry(&timekeeper.lock, seq)); | 1390 | } while (read_seqretry(&tk->lock, seq)); |
1365 | 1391 | ||
1366 | return timespec_to_ktime(wtom); | 1392 | return timespec_to_ktime(wtom); |
1367 | } | 1393 | } |