aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/hrtimer.h16
-rw-r--r--kernel/hrtimer.c125
-rw-r--r--kernel/time/timekeeping.c2
3 files changed, 65 insertions, 78 deletions
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 62f500c724f9..4135c88fe4fa 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -148,9 +148,7 @@ struct hrtimer_clock_base {
148 ktime_t resolution; 148 ktime_t resolution;
149 ktime_t (*get_time)(void); 149 ktime_t (*get_time)(void);
150 ktime_t softirq_time; 150 ktime_t softirq_time;
151#ifdef CONFIG_HIGH_RES_TIMERS
152 ktime_t offset; 151 ktime_t offset;
153#endif
154}; 152};
155 153
156enum hrtimer_base_type { 154enum hrtimer_base_type {
@@ -256,8 +254,6 @@ static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
256#ifdef CONFIG_HIGH_RES_TIMERS 254#ifdef CONFIG_HIGH_RES_TIMERS
257struct clock_event_device; 255struct clock_event_device;
258 256
259extern void clock_was_set(void);
260extern void hres_timers_resume(void);
261extern void hrtimer_interrupt(struct clock_event_device *dev); 257extern void hrtimer_interrupt(struct clock_event_device *dev);
262 258
263/* 259/*
@@ -291,16 +287,8 @@ extern void hrtimer_peek_ahead_timers(void);
291# define MONOTONIC_RES_NSEC LOW_RES_NSEC 287# define MONOTONIC_RES_NSEC LOW_RES_NSEC
292# define KTIME_MONOTONIC_RES KTIME_LOW_RES 288# define KTIME_MONOTONIC_RES KTIME_LOW_RES
293 289
294/*
295 * clock_was_set() is a NOP for non- high-resolution systems. The
296 * time-sorted order guarantees that a timer does not expire early and
297 * is expired in the next softirq when the clock was advanced.
298 */
299static inline void clock_was_set(void) { }
300static inline void hrtimer_peek_ahead_timers(void) { } 290static inline void hrtimer_peek_ahead_timers(void) { }
301 291
302static inline void hres_timers_resume(void) { }
303
304/* 292/*
305 * In non high resolution mode the time reference is taken from 293 * In non high resolution mode the time reference is taken from
306 * the base softirq time variable. 294 * the base softirq time variable.
@@ -316,11 +304,13 @@ static inline int hrtimer_is_hres_active(struct hrtimer *timer)
316} 304}
317#endif 305#endif
318 306
307extern void clock_was_set(void);
308extern void hrtimers_resume(void);
309
319extern ktime_t ktime_get(void); 310extern ktime_t ktime_get(void);
320extern ktime_t ktime_get_real(void); 311extern ktime_t ktime_get_real(void);
321extern ktime_t ktime_get_boottime(void); 312extern ktime_t ktime_get_boottime(void);
322 313
323
324DECLARE_PER_CPU(struct tick_device, tick_cpu_device); 314DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
325 315
326 316
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index dbbbf7d43080..c145ed643bca 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -621,66 +621,6 @@ static int hrtimer_reprogram(struct hrtimer *timer,
621 return res; 621 return res;
622} 622}
623 623
624
625/*
626 * Retrigger next event is called after clock was set
627 *
628 * Called with interrupts disabled via on_each_cpu()
629 */
630static void retrigger_next_event(void *arg)
631{
632 struct hrtimer_cpu_base *base;
633 struct timespec realtime_offset, wtm, sleep;
634
635 if (!hrtimer_hres_active())
636 return;
637
638 get_xtime_and_monotonic_and_sleep_offset(&realtime_offset, &wtm,
639 &sleep);
640 set_normalized_timespec(&realtime_offset, -wtm.tv_sec, -wtm.tv_nsec);
641
642 base = &__get_cpu_var(hrtimer_bases);
643
644 /* Adjust CLOCK_REALTIME offset */
645 raw_spin_lock(&base->lock);
646 base->clock_base[HRTIMER_BASE_REALTIME].offset =
647 timespec_to_ktime(realtime_offset);
648 base->clock_base[HRTIMER_BASE_BOOTTIME].offset =
649 timespec_to_ktime(sleep);
650
651 hrtimer_force_reprogram(base, 0);
652 raw_spin_unlock(&base->lock);
653}
654
655/*
656 * Clock realtime was set
657 *
658 * Change the offset of the realtime clock vs. the monotonic
659 * clock.
660 *
661 * We might have to reprogram the high resolution timer interrupt. On
662 * SMP we call the architecture specific code to retrigger _all_ high
663 * resolution timer interrupts. On UP we just disable interrupts and
664 * call the high resolution interrupt code.
665 */
666void clock_was_set(void)
667{
668 /* Retrigger the CPU local events everywhere */
669 on_each_cpu(retrigger_next_event, NULL, 1);
670}
671
672/*
673 * During resume we might have to reprogram the high resolution timer
674 * interrupt (on the local CPU):
675 */
676void hres_timers_resume(void)
677{
678 WARN_ONCE(!irqs_disabled(),
679 KERN_INFO "hres_timers_resume() called with IRQs enabled!");
680
681 retrigger_next_event(NULL);
682}
683
684/* 624/*
685 * Initialize the high resolution related parts of cpu_base 625 * Initialize the high resolution related parts of cpu_base
686 */ 626 */
@@ -714,12 +654,14 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
714 return 0; 654 return 0;
715} 655}
716 656
657static void retrigger_next_event(void *arg);
658
717/* 659/*
718 * Switch to high resolution mode 660 * Switch to high resolution mode
719 */ 661 */
720static int hrtimer_switch_to_hres(void) 662static int hrtimer_switch_to_hres(void)
721{ 663{
722 int cpu = smp_processor_id(); 664 int i, cpu = smp_processor_id();
723 struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu); 665 struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu);
724 unsigned long flags; 666 unsigned long flags;
725 667
@@ -735,9 +677,8 @@ static int hrtimer_switch_to_hres(void)
735 return 0; 677 return 0;
736 } 678 }
737 base->hres_active = 1; 679 base->hres_active = 1;
738 base->clock_base[HRTIMER_BASE_REALTIME].resolution = KTIME_HIGH_RES; 680 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
739 base->clock_base[HRTIMER_BASE_MONOTONIC].resolution = KTIME_HIGH_RES; 681 base->clock_base[i].resolution = KTIME_HIGH_RES;
740 base->clock_base[HRTIMER_BASE_BOOTTIME].resolution = KTIME_HIGH_RES;
741 682
742 tick_setup_sched_timer(); 683 tick_setup_sched_timer();
743 684
@@ -764,6 +705,62 @@ static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
764 705
765#endif /* CONFIG_HIGH_RES_TIMERS */ 706#endif /* CONFIG_HIGH_RES_TIMERS */
766 707
708/*
709 * Retrigger next event is called after clock was set
710 *
711 * Called with interrupts disabled via on_each_cpu()
712 */
713static void retrigger_next_event(void *arg)
714{
715 struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases);
716 struct timespec realtime_offset, xtim, wtm, sleep;
717
718 if (!hrtimer_hres_active())
719 return;
720
721 get_xtime_and_monotonic_and_sleep_offset(&xtim, &wtm, &sleep);
722 set_normalized_timespec(&realtime_offset, -wtm.tv_sec, -wtm.tv_nsec);
723
724 /* Adjust CLOCK_REALTIME offset */
725 raw_spin_lock(&base->lock);
726 base->clock_base[HRTIMER_BASE_REALTIME].offset =
727 timespec_to_ktime(realtime_offset);
728 base->clock_base[HRTIMER_BASE_BOOTTIME].offset =
729 timespec_to_ktime(sleep);
730
731 hrtimer_force_reprogram(base, 0);
732 raw_spin_unlock(&base->lock);
733}
734
735/*
736 * Clock realtime was set
737 *
738 * Change the offset of the realtime clock vs. the monotonic
739 * clock.
740 *
741 * We might have to reprogram the high resolution timer interrupt. On
742 * SMP we call the architecture specific code to retrigger _all_ high
743 * resolution timer interrupts. On UP we just disable interrupts and
744 * call the high resolution interrupt code.
745 */
746void clock_was_set(void)
747{
748 /* Retrigger the CPU local events everywhere */
749 on_each_cpu(retrigger_next_event, NULL, 1);
750}
751
752/*
753 * During resume we might have to reprogram the high resolution timer
754 * interrupt (on the local CPU):
755 */
756void hrtimers_resume(void)
757{
758 WARN_ONCE(!irqs_disabled(),
759 KERN_INFO "hrtimers_resume() called with IRQs enabled!");
760
761 retrigger_next_event(NULL);
762}
763
767static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer) 764static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer)
768{ 765{
769#ifdef CONFIG_TIMER_STATS 766#ifdef CONFIG_TIMER_STATS
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 8e6a05a5915a..a61b8fa2d39a 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -680,7 +680,7 @@ static void timekeeping_resume(void)
680 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL); 680 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
681 681
682 /* Resume hrtimers */ 682 /* Resume hrtimers */
683 hres_timers_resume(); 683 hrtimers_resume();
684} 684}
685 685
686static int timekeeping_suspend(void) 686static int timekeeping_suspend(void)