aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2012-10-12 09:17:48 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2012-10-12 09:17:48 -0400
commit03d3602a833715f83ea53b9feb078b9c4c5f6c1a (patch)
tree7b63ec6753dd65e61392017e1aba2ee1e6b4bcd1
parent0588f1f934791b79d0a1e9b327be9b6eb361d2b8 (diff)
parent5b3900cd409466c0070b234d941650685ad0c791 (diff)
Merge branch 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull timer core update from Thomas Gleixner: - Bug fixes (one for a longstanding dead loop issue) - Rework of time related vsyscalls - Alarm timer updates - Jiffies updates to remove compile time dependencies * 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: timekeeping: Cast raw_interval to u64 to avoid shift overflow timers: Fix endless looping between cascade() and internal_add_timer() time/jiffies: bring back unconditional LATCH definition time: Convert x86_64 to using new update_vsyscall time: Only do nanosecond rounding on GENERIC_TIME_VSYSCALL_OLD systems time: Introduce new GENERIC_TIME_VSYSCALL time: Convert CONFIG_GENERIC_TIME_VSYSCALL to CONFIG_GENERIC_TIME_VSYSCALL_OLD time: Move update_vsyscall definitions to timekeeper_internal.h time: Move timekeeper structure to timekeeper_internal.h for vsyscall changes jiffies: Remove compile time assumptions about CLOCK_TICK_RATE jiffies: Kill unused TICK_USEC_TO_NSEC alarmtimer: Rename alarmtimer_remove to alarmtimer_dequeue alarmtimer: Remove unused helpers & defines alarmtimer: Use hrtimer per-alarm instead of per-base alarmtimer: Implement minimum alarm interval for allowing suspend
-rw-r--r--arch/ia64/Kconfig2
-rw-r--r--arch/ia64/kernel/time.c4
-rw-r--r--arch/powerpc/Kconfig2
-rw-r--r--arch/powerpc/kernel/time.c4
-rw-r--r--arch/s390/Kconfig2
-rw-r--r--arch/s390/kernel/time.c4
-rw-r--r--arch/x86/include/asm/vgtod.h4
-rw-r--r--arch/x86/kernel/setup.c3
-rw-r--r--arch/x86/kernel/vsyscall_64.c49
-rw-r--r--arch/x86/vdso/vclock_gettime.c22
-rw-r--r--include/linux/alarmtimer.h31
-rw-r--r--include/linux/clocksource.h16
-rw-r--r--include/linux/jiffies.h20
-rw-r--r--include/linux/timekeeper_internal.h108
-rw-r--r--kernel/time.c2
-rw-r--r--kernel/time/Kconfig4
-rw-r--r--kernel/time/alarmtimer.c118
-rw-r--r--kernel/time/jiffies.c32
-rw-r--r--kernel/time/timekeeping.c117
-rw-r--r--kernel/timer.c10
20 files changed, 289 insertions, 265 deletions
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 3c720ef6c32d..4c10e607c908 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -39,7 +39,7 @@ config IA64
39 select ARCH_TASK_STRUCT_ALLOCATOR 39 select ARCH_TASK_STRUCT_ALLOCATOR
40 select ARCH_THREAD_INFO_ALLOCATOR 40 select ARCH_THREAD_INFO_ALLOCATOR
41 select ARCH_CLOCKSOURCE_DATA 41 select ARCH_CLOCKSOURCE_DATA
42 select GENERIC_TIME_VSYSCALL 42 select GENERIC_TIME_VSYSCALL_OLD
43 default y 43 default y
44 help 44 help
45 The Itanium Processor Family is Intel's 64-bit successor to 45 The Itanium Processor Family is Intel's 64-bit successor to
diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c
index 80ff9acc5edf..f6388216080d 100644
--- a/arch/ia64/kernel/time.c
+++ b/arch/ia64/kernel/time.c
@@ -19,7 +19,7 @@
19#include <linux/interrupt.h> 19#include <linux/interrupt.h>
20#include <linux/efi.h> 20#include <linux/efi.h>
21#include <linux/timex.h> 21#include <linux/timex.h>
22#include <linux/clocksource.h> 22#include <linux/timekeeper_internal.h>
23#include <linux/platform_device.h> 23#include <linux/platform_device.h>
24 24
25#include <asm/machvec.h> 25#include <asm/machvec.h>
@@ -454,7 +454,7 @@ void update_vsyscall_tz(void)
454{ 454{
455} 455}
456 456
457void update_vsyscall(struct timespec *wall, struct timespec *wtm, 457void update_vsyscall_old(struct timespec *wall, struct timespec *wtm,
458 struct clocksource *c, u32 mult) 458 struct clocksource *c, u32 mult)
459{ 459{
460 write_seqcount_begin(&fsyscall_gtod_data.seq); 460 write_seqcount_begin(&fsyscall_gtod_data.seq);
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 78d6588b6e86..969f3d9ded91 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -137,7 +137,7 @@ config PPC
137 select ARCH_HAVE_NMI_SAFE_CMPXCHG 137 select ARCH_HAVE_NMI_SAFE_CMPXCHG
138 select GENERIC_SMP_IDLE_THREAD 138 select GENERIC_SMP_IDLE_THREAD
139 select GENERIC_CMOS_UPDATE 139 select GENERIC_CMOS_UPDATE
140 select GENERIC_TIME_VSYSCALL 140 select GENERIC_TIME_VSYSCALL_OLD
141 select GENERIC_CLOCKEVENTS 141 select GENERIC_CLOCKEVENTS
142 select GENERIC_STRNCPY_FROM_USER 142 select GENERIC_STRNCPY_FROM_USER
143 select GENERIC_STRNLEN_USER 143 select GENERIC_STRNLEN_USER
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index c9986fd400d8..ce4cb772dc78 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -73,7 +73,7 @@
73/* powerpc clocksource/clockevent code */ 73/* powerpc clocksource/clockevent code */
74 74
75#include <linux/clockchips.h> 75#include <linux/clockchips.h>
76#include <linux/clocksource.h> 76#include <linux/timekeeper_internal.h>
77 77
78static cycle_t rtc_read(struct clocksource *); 78static cycle_t rtc_read(struct clocksource *);
79static struct clocksource clocksource_rtc = { 79static struct clocksource clocksource_rtc = {
@@ -727,7 +727,7 @@ static cycle_t timebase_read(struct clocksource *cs)
727 return (cycle_t)get_tb(); 727 return (cycle_t)get_tb();
728} 728}
729 729
730void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, 730void update_vsyscall_old(struct timespec *wall_time, struct timespec *wtm,
731 struct clocksource *clock, u32 mult) 731 struct clocksource *clock, u32 mult)
732{ 732{
733 u64 new_tb_to_xs, new_stamp_xsec; 733 u64 new_tb_to_xs, new_stamp_xsec;
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 99d2d790d152..e5dac1236185 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -131,7 +131,7 @@ config S390
131 select HAVE_UID16 if 32BIT 131 select HAVE_UID16 if 32BIT
132 select ARCH_WANT_IPC_PARSE_VERSION 132 select ARCH_WANT_IPC_PARSE_VERSION
133 select GENERIC_SMP_IDLE_THREAD 133 select GENERIC_SMP_IDLE_THREAD
134 select GENERIC_TIME_VSYSCALL 134 select GENERIC_TIME_VSYSCALL_OLD
135 select GENERIC_CLOCKEVENTS 135 select GENERIC_CLOCKEVENTS
136 select KTIME_SCALAR if 32BIT 136 select KTIME_SCALAR if 32BIT
137 select HAVE_ARCH_SECCOMP_FILTER 137 select HAVE_ARCH_SECCOMP_FILTER
diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c
index 2db1011b8b19..7fcd690d42c7 100644
--- a/arch/s390/kernel/time.c
+++ b/arch/s390/kernel/time.c
@@ -34,7 +34,7 @@
34#include <linux/profile.h> 34#include <linux/profile.h>
35#include <linux/timex.h> 35#include <linux/timex.h>
36#include <linux/notifier.h> 36#include <linux/notifier.h>
37#include <linux/clocksource.h> 37#include <linux/timekeeper_internal.h>
38#include <linux/clockchips.h> 38#include <linux/clockchips.h>
39#include <linux/gfp.h> 39#include <linux/gfp.h>
40#include <linux/kprobes.h> 40#include <linux/kprobes.h>
@@ -219,7 +219,7 @@ struct clocksource * __init clocksource_default_clock(void)
219 return &clocksource_tod; 219 return &clocksource_tod;
220} 220}
221 221
222void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, 222void update_vsyscall_old(struct timespec *wall_time, struct timespec *wtm,
223 struct clocksource *clock, u32 mult) 223 struct clocksource *clock, u32 mult)
224{ 224{
225 if (clock != &clocksource_tod) 225 if (clock != &clocksource_tod)
diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h
index 8b38be2de9e1..46e24d36b7da 100644
--- a/arch/x86/include/asm/vgtod.h
+++ b/arch/x86/include/asm/vgtod.h
@@ -17,8 +17,8 @@ struct vsyscall_gtod_data {
17 17
18 /* open coded 'struct timespec' */ 18 /* open coded 'struct timespec' */
19 time_t wall_time_sec; 19 time_t wall_time_sec;
20 u32 wall_time_nsec; 20 u64 wall_time_snsec;
21 u32 monotonic_time_nsec; 21 u64 monotonic_time_snsec;
22 time_t monotonic_time_sec; 22 time_t monotonic_time_sec;
23 23
24 struct timezone sys_tz; 24 struct timezone sys_tz;
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index d609be046b57..a2bb18e02839 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -68,6 +68,7 @@
68#include <linux/percpu.h> 68#include <linux/percpu.h>
69#include <linux/crash_dump.h> 69#include <linux/crash_dump.h>
70#include <linux/tboot.h> 70#include <linux/tboot.h>
71#include <linux/jiffies.h>
71 72
72#include <video/edid.h> 73#include <video/edid.h>
73 74
@@ -1032,6 +1033,8 @@ void __init setup_arch(char **cmdline_p)
1032 mcheck_init(); 1033 mcheck_init();
1033 1034
1034 arch_init_ideal_nops(); 1035 arch_init_ideal_nops();
1036
1037 register_refined_jiffies(CLOCK_TICK_RATE);
1035} 1038}
1036 1039
1037#ifdef CONFIG_X86_32 1040#ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index 8d141b309046..3a3e8c9e280d 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -28,7 +28,7 @@
28#include <linux/jiffies.h> 28#include <linux/jiffies.h>
29#include <linux/sysctl.h> 29#include <linux/sysctl.h>
30#include <linux/topology.h> 30#include <linux/topology.h>
31#include <linux/clocksource.h> 31#include <linux/timekeeper_internal.h>
32#include <linux/getcpu.h> 32#include <linux/getcpu.h>
33#include <linux/cpu.h> 33#include <linux/cpu.h>
34#include <linux/smp.h> 34#include <linux/smp.h>
@@ -82,32 +82,41 @@ void update_vsyscall_tz(void)
82 vsyscall_gtod_data.sys_tz = sys_tz; 82 vsyscall_gtod_data.sys_tz = sys_tz;
83} 83}
84 84
85void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, 85void update_vsyscall(struct timekeeper *tk)
86 struct clocksource *clock, u32 mult)
87{ 86{
88 struct timespec monotonic; 87 struct vsyscall_gtod_data *vdata = &vsyscall_gtod_data;
89 88
90 write_seqcount_begin(&vsyscall_gtod_data.seq); 89 write_seqcount_begin(&vdata->seq);
91 90
92 /* copy vsyscall data */ 91 /* copy vsyscall data */
93 vsyscall_gtod_data.clock.vclock_mode = clock->archdata.vclock_mode; 92 vdata->clock.vclock_mode = tk->clock->archdata.vclock_mode;
94 vsyscall_gtod_data.clock.cycle_last = clock->cycle_last; 93 vdata->clock.cycle_last = tk->clock->cycle_last;
95 vsyscall_gtod_data.clock.mask = clock->mask; 94 vdata->clock.mask = tk->clock->mask;
96 vsyscall_gtod_data.clock.mult = mult; 95 vdata->clock.mult = tk->mult;
97 vsyscall_gtod_data.clock.shift = clock->shift; 96 vdata->clock.shift = tk->shift;
98 97
99 vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec; 98 vdata->wall_time_sec = tk->xtime_sec;
100 vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec; 99 vdata->wall_time_snsec = tk->xtime_nsec;
100
101 vdata->monotonic_time_sec = tk->xtime_sec
102 + tk->wall_to_monotonic.tv_sec;
103 vdata->monotonic_time_snsec = tk->xtime_nsec
104 + (tk->wall_to_monotonic.tv_nsec
105 << tk->shift);
106 while (vdata->monotonic_time_snsec >=
107 (((u64)NSEC_PER_SEC) << tk->shift)) {
108 vdata->monotonic_time_snsec -=
109 ((u64)NSEC_PER_SEC) << tk->shift;
110 vdata->monotonic_time_sec++;
111 }
101 112
102 monotonic = timespec_add(*wall_time, *wtm); 113 vdata->wall_time_coarse.tv_sec = tk->xtime_sec;
103 vsyscall_gtod_data.monotonic_time_sec = monotonic.tv_sec; 114 vdata->wall_time_coarse.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
104 vsyscall_gtod_data.monotonic_time_nsec = monotonic.tv_nsec;
105 115
106 vsyscall_gtod_data.wall_time_coarse = __current_kernel_time(); 116 vdata->monotonic_time_coarse = timespec_add(vdata->wall_time_coarse,
107 vsyscall_gtod_data.monotonic_time_coarse = 117 tk->wall_to_monotonic);
108 timespec_add(vsyscall_gtod_data.wall_time_coarse, *wtm);
109 118
110 write_seqcount_end(&vsyscall_gtod_data.seq); 119 write_seqcount_end(&vdata->seq);
111} 120}
112 121
113static void warn_bad_vsyscall(const char *level, struct pt_regs *regs, 122static void warn_bad_vsyscall(const char *level, struct pt_regs *regs,
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 885eff49d6ab..4df6c373421a 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -80,7 +80,7 @@ notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
80} 80}
81 81
82 82
83notrace static inline long vgetns(void) 83notrace static inline u64 vgetsns(void)
84{ 84{
85 long v; 85 long v;
86 cycles_t cycles; 86 cycles_t cycles;
@@ -91,21 +91,24 @@ notrace static inline long vgetns(void)
91 else 91 else
92 return 0; 92 return 0;
93 v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask; 93 v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask;
94 return (v * gtod->clock.mult) >> gtod->clock.shift; 94 return v * gtod->clock.mult;
95} 95}
96 96
97/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */ 97/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
98notrace static int __always_inline do_realtime(struct timespec *ts) 98notrace static int __always_inline do_realtime(struct timespec *ts)
99{ 99{
100 unsigned long seq, ns; 100 unsigned long seq;
101 u64 ns;
101 int mode; 102 int mode;
102 103
104 ts->tv_nsec = 0;
103 do { 105 do {
104 seq = read_seqcount_begin(&gtod->seq); 106 seq = read_seqcount_begin(&gtod->seq);
105 mode = gtod->clock.vclock_mode; 107 mode = gtod->clock.vclock_mode;
106 ts->tv_sec = gtod->wall_time_sec; 108 ts->tv_sec = gtod->wall_time_sec;
107 ts->tv_nsec = gtod->wall_time_nsec; 109 ns = gtod->wall_time_snsec;
108 ns = vgetns(); 110 ns += vgetsns();
111 ns >>= gtod->clock.shift;
109 } while (unlikely(read_seqcount_retry(&gtod->seq, seq))); 112 } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
110 113
111 timespec_add_ns(ts, ns); 114 timespec_add_ns(ts, ns);
@@ -114,15 +117,18 @@ notrace static int __always_inline do_realtime(struct timespec *ts)
114 117
115notrace static int do_monotonic(struct timespec *ts) 118notrace static int do_monotonic(struct timespec *ts)
116{ 119{
117 unsigned long seq, ns; 120 unsigned long seq;
121 u64 ns;
118 int mode; 122 int mode;
119 123
124 ts->tv_nsec = 0;
120 do { 125 do {
121 seq = read_seqcount_begin(&gtod->seq); 126 seq = read_seqcount_begin(&gtod->seq);
122 mode = gtod->clock.vclock_mode; 127 mode = gtod->clock.vclock_mode;
123 ts->tv_sec = gtod->monotonic_time_sec; 128 ts->tv_sec = gtod->monotonic_time_sec;
124 ts->tv_nsec = gtod->monotonic_time_nsec; 129 ns = gtod->monotonic_time_snsec;
125 ns = vgetns(); 130 ns += vgetsns();
131 ns >>= gtod->clock.shift;
126 } while (unlikely(read_seqcount_retry(&gtod->seq, seq))); 132 } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
127 timespec_add_ns(ts, ns); 133 timespec_add_ns(ts, ns);
128 134
diff --git a/include/linux/alarmtimer.h b/include/linux/alarmtimer.h
index 96c5c249b086..9069694e70eb 100644
--- a/include/linux/alarmtimer.h
+++ b/include/linux/alarmtimer.h
@@ -21,7 +21,6 @@ enum alarmtimer_restart {
21 21
22#define ALARMTIMER_STATE_INACTIVE 0x00 22#define ALARMTIMER_STATE_INACTIVE 0x00
23#define ALARMTIMER_STATE_ENQUEUED 0x01 23#define ALARMTIMER_STATE_ENQUEUED 0x01
24#define ALARMTIMER_STATE_CALLBACK 0x02
25 24
26/** 25/**
27 * struct alarm - Alarm timer structure 26 * struct alarm - Alarm timer structure
@@ -35,6 +34,7 @@ enum alarmtimer_restart {
35 */ 34 */
36struct alarm { 35struct alarm {
37 struct timerqueue_node node; 36 struct timerqueue_node node;
37 struct hrtimer timer;
38 enum alarmtimer_restart (*function)(struct alarm *, ktime_t now); 38 enum alarmtimer_restart (*function)(struct alarm *, ktime_t now);
39 enum alarmtimer_type type; 39 enum alarmtimer_type type;
40 int state; 40 int state;
@@ -43,39 +43,12 @@ struct alarm {
43 43
44void alarm_init(struct alarm *alarm, enum alarmtimer_type type, 44void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
45 enum alarmtimer_restart (*function)(struct alarm *, ktime_t)); 45 enum alarmtimer_restart (*function)(struct alarm *, ktime_t));
46void alarm_start(struct alarm *alarm, ktime_t start); 46int alarm_start(struct alarm *alarm, ktime_t start);
47int alarm_try_to_cancel(struct alarm *alarm); 47int alarm_try_to_cancel(struct alarm *alarm);
48int alarm_cancel(struct alarm *alarm); 48int alarm_cancel(struct alarm *alarm);
49 49
50u64 alarm_forward(struct alarm *alarm, ktime_t now, ktime_t interval); 50u64 alarm_forward(struct alarm *alarm, ktime_t now, ktime_t interval);
51 51
52/*
53 * A alarmtimer is active, when it is enqueued into timerqueue or the
54 * callback function is running.
55 */
56static inline int alarmtimer_active(const struct alarm *timer)
57{
58 return timer->state != ALARMTIMER_STATE_INACTIVE;
59}
60
61/*
62 * Helper function to check, whether the timer is on one of the queues
63 */
64static inline int alarmtimer_is_queued(struct alarm *timer)
65{
66 return timer->state & ALARMTIMER_STATE_ENQUEUED;
67}
68
69/*
70 * Helper function to check, whether the timer is running the callback
71 * function
72 */
73static inline int alarmtimer_callback_running(struct alarm *timer)
74{
75 return timer->state & ALARMTIMER_STATE_CALLBACK;
76}
77
78
79/* Provide way to access the rtc device being used by alarmtimers */ 52/* Provide way to access the rtc device being used by alarmtimers */
80struct rtc_device *alarmtimer_get_rtcdev(void); 53struct rtc_device *alarmtimer_get_rtcdev(void);
81 54
diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index fbe89e17124e..4dceaf8ae152 100644
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -319,22 +319,6 @@ static inline void __clocksource_updatefreq_khz(struct clocksource *cs, u32 khz)
319 __clocksource_updatefreq_scale(cs, 1000, khz); 319 __clocksource_updatefreq_scale(cs, 1000, khz);
320} 320}
321 321
322#ifdef CONFIG_GENERIC_TIME_VSYSCALL
323extern void
324update_vsyscall(struct timespec *ts, struct timespec *wtm,
325 struct clocksource *c, u32 mult);
326extern void update_vsyscall_tz(void);
327#else
328static inline void
329update_vsyscall(struct timespec *ts, struct timespec *wtm,
330 struct clocksource *c, u32 mult)
331{
332}
333
334static inline void update_vsyscall_tz(void)
335{
336}
337#endif
338 322
339extern void timekeeping_notify(struct clocksource *clock); 323extern void timekeeping_notify(struct clocksource *clock);
340 324
diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h
index 05e3c2c7a8cf..6b87413da9d6 100644
--- a/include/linux/jiffies.h
+++ b/include/linux/jiffies.h
@@ -51,31 +51,17 @@
51#define SH_DIV(NOM,DEN,LSH) ( (((NOM) / (DEN)) << (LSH)) \ 51#define SH_DIV(NOM,DEN,LSH) ( (((NOM) / (DEN)) << (LSH)) \
52 + ((((NOM) % (DEN)) << (LSH)) + (DEN) / 2) / (DEN)) 52 + ((((NOM) % (DEN)) << (LSH)) + (DEN) / 2) / (DEN))
53 53
54#ifdef CLOCK_TICK_RATE
55/* LATCH is used in the interval timer and ftape setup. */ 54/* LATCH is used in the interval timer and ftape setup. */
56# define LATCH ((CLOCK_TICK_RATE + HZ/2) / HZ) /* For divider */ 55#define LATCH ((CLOCK_TICK_RATE + HZ/2) / HZ) /* For divider */
57 56
58/* 57extern int register_refined_jiffies(long clock_tick_rate);
59 * HZ is the requested value. However the CLOCK_TICK_RATE may not allow
60 * for exactly HZ. So SHIFTED_HZ is high res HZ ("<< 8" is for accuracy)
61 */
62# define SHIFTED_HZ (SH_DIV(CLOCK_TICK_RATE, LATCH, 8))
63#else
64# define SHIFTED_HZ (HZ << 8)
65#endif
66 58
67/* TICK_NSEC is the time between ticks in nsec assuming SHIFTED_HZ */ 59/* TICK_NSEC is the time between ticks in nsec assuming SHIFTED_HZ */
68#define TICK_NSEC (SH_DIV(1000000UL * 1000, SHIFTED_HZ, 8)) 60#define TICK_NSEC ((NSEC_PER_SEC+HZ/2)/HZ)
69 61
70/* TICK_USEC is the time between ticks in usec assuming fake USER_HZ */ 62/* TICK_USEC is the time between ticks in usec assuming fake USER_HZ */
71#define TICK_USEC ((1000000UL + USER_HZ/2) / USER_HZ) 63#define TICK_USEC ((1000000UL + USER_HZ/2) / USER_HZ)
72 64
73/*
74 * TICK_USEC_TO_NSEC is the time between ticks in nsec assuming SHIFTED_HZ and
75 * a value TUSEC for TICK_USEC (can be set bij adjtimex)
76 */
77#define TICK_USEC_TO_NSEC(TUSEC) (SH_DIV(TUSEC * USER_HZ * 1000, SHIFTED_HZ, 8))
78
79/* some arch's have a small-data section that can be accessed register-relative 65/* some arch's have a small-data section that can be accessed register-relative
80 * but that can only take up to, say, 4-byte variables. jiffies being part of 66 * but that can only take up to, say, 4-byte variables. jiffies being part of
81 * an 8-byte variable may not be correctly accessed unless we force the issue 67 * an 8-byte variable may not be correctly accessed unless we force the issue
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
new file mode 100644
index 000000000000..e1d558e237ec
--- /dev/null
+++ b/include/linux/timekeeper_internal.h
@@ -0,0 +1,108 @@
1/*
2 * You SHOULD NOT be including this unless you're vsyscall
3 * handling code or timekeeping internal code!
4 */
5
6#ifndef _LINUX_TIMEKEEPER_INTERNAL_H
7#define _LINUX_TIMEKEEPER_INTERNAL_H
8
9#include <linux/clocksource.h>
10#include <linux/jiffies.h>
11#include <linux/time.h>
12
13/* Structure holding internal timekeeping values. */
14struct timekeeper {
15 /* Current clocksource used for timekeeping. */
16 struct clocksource *clock;
17 /* NTP adjusted clock multiplier */
18 u32 mult;
19 /* The shift value of the current clocksource. */
20 u32 shift;
21 /* Number of clock cycles in one NTP interval. */
22 cycle_t cycle_interval;
23 /* Number of clock shifted nano seconds in one NTP interval. */
24 u64 xtime_interval;
25 /* shifted nano seconds left over when rounding cycle_interval */
26 s64 xtime_remainder;
27 /* Raw nano seconds accumulated per NTP interval. */
28 u32 raw_interval;
29
30 /* Current CLOCK_REALTIME time in seconds */
31 u64 xtime_sec;
32 /* Clock shifted nano seconds */
33 u64 xtime_nsec;
34
35 /* Difference between accumulated time and NTP time in ntp
36 * shifted nano seconds. */
37 s64 ntp_error;
38 /* Shift conversion between clock shifted nano seconds and
39 * ntp shifted nano seconds. */
40 u32 ntp_error_shift;
41
42 /*
43 * wall_to_monotonic is what we need to add to xtime (or xtime corrected
44 * for sub jiffie times) to get to monotonic time. Monotonic is pegged
45 * at zero at system boot time, so wall_to_monotonic will be negative,
46 * however, we will ALWAYS keep the tv_nsec part positive so we can use
47 * the usual normalization.
48 *
49 * wall_to_monotonic is moved after resume from suspend for the
50 * monotonic time not to jump. We need to add total_sleep_time to
51 * wall_to_monotonic to get the real boot based time offset.
52 *
53 * - wall_to_monotonic is no longer the boot time, getboottime must be
54 * used instead.
55 */
56 struct timespec wall_to_monotonic;
57 /* Offset clock monotonic -> clock realtime */
58 ktime_t offs_real;
59 /* time spent in suspend */
60 struct timespec total_sleep_time;
61 /* Offset clock monotonic -> clock boottime */
62 ktime_t offs_boot;
63 /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
64 struct timespec raw_time;
65 /* Seqlock for all timekeeper values */
66 seqlock_t lock;
67};
68
69static inline struct timespec tk_xtime(struct timekeeper *tk)
70{
71 struct timespec ts;
72
73 ts.tv_sec = tk->xtime_sec;
74 ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
75 return ts;
76}
77
78
79#ifdef CONFIG_GENERIC_TIME_VSYSCALL
80
81extern void update_vsyscall(struct timekeeper *tk);
82extern void update_vsyscall_tz(void);
83
84#elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD)
85
86extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm,
87 struct clocksource *c, u32 mult);
88extern void update_vsyscall_tz(void);
89
90static inline void update_vsyscall(struct timekeeper *tk)
91{
92 struct timespec xt;
93
94 xt = tk_xtime(tk);
95 update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult);
96}
97
98#else
99
100static inline void update_vsyscall(struct timekeeper *tk)
101{
102}
103static inline void update_vsyscall_tz(void)
104{
105}
106#endif
107
108#endif /* _LINUX_TIMEKEEPER_INTERNAL_H */
diff --git a/kernel/time.c b/kernel/time.c
index ba744cf80696..d226c6a3fd28 100644
--- a/kernel/time.c
+++ b/kernel/time.c
@@ -30,7 +30,7 @@
30#include <linux/export.h> 30#include <linux/export.h>
31#include <linux/timex.h> 31#include <linux/timex.h>
32#include <linux/capability.h> 32#include <linux/capability.h>
33#include <linux/clocksource.h> 33#include <linux/timekeeper_internal.h>
34#include <linux/errno.h> 34#include <linux/errno.h>
35#include <linux/syscalls.h> 35#include <linux/syscalls.h>
36#include <linux/security.h> 36#include <linux/security.h>
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index fd42bd452b75..8601f0db1261 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -16,6 +16,10 @@ config ARCH_CLOCKSOURCE_DATA
16config GENERIC_TIME_VSYSCALL 16config GENERIC_TIME_VSYSCALL
17 bool 17 bool
18 18
19# Timekeeping vsyscall support
20config GENERIC_TIME_VSYSCALL_OLD
21 bool
22
19# ktime_t scalar 64bit nsec representation 23# ktime_t scalar 64bit nsec representation
20config KTIME_SCALAR 24config KTIME_SCALAR
21 bool 25 bool
diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c
index aa27d391bfc8..f11d83b12949 100644
--- a/kernel/time/alarmtimer.c
+++ b/kernel/time/alarmtimer.c
@@ -37,7 +37,6 @@
37static struct alarm_base { 37static struct alarm_base {
38 spinlock_t lock; 38 spinlock_t lock;
39 struct timerqueue_head timerqueue; 39 struct timerqueue_head timerqueue;
40 struct hrtimer timer;
41 ktime_t (*gettime)(void); 40 ktime_t (*gettime)(void);
42 clockid_t base_clockid; 41 clockid_t base_clockid;
43} alarm_bases[ALARM_NUMTYPE]; 42} alarm_bases[ALARM_NUMTYPE];
@@ -46,6 +45,8 @@ static struct alarm_base {
46static ktime_t freezer_delta; 45static ktime_t freezer_delta;
47static DEFINE_SPINLOCK(freezer_delta_lock); 46static DEFINE_SPINLOCK(freezer_delta_lock);
48 47
48static struct wakeup_source *ws;
49
49#ifdef CONFIG_RTC_CLASS 50#ifdef CONFIG_RTC_CLASS
50/* rtc timer and device for setting alarm wakeups at suspend */ 51/* rtc timer and device for setting alarm wakeups at suspend */
51static struct rtc_timer rtctimer; 52static struct rtc_timer rtctimer;
@@ -130,50 +131,35 @@ static inline void alarmtimer_rtc_timer_init(void) { }
130 * @base: pointer to the base where the timer is being run 131 * @base: pointer to the base where the timer is being run
131 * @alarm: pointer to alarm being enqueued. 132 * @alarm: pointer to alarm being enqueued.
132 * 133 *
133 * Adds alarm to a alarm_base timerqueue and if necessary sets 134 * Adds alarm to a alarm_base timerqueue
134 * an hrtimer to run.
135 * 135 *
136 * Must hold base->lock when calling. 136 * Must hold base->lock when calling.
137 */ 137 */
138static void alarmtimer_enqueue(struct alarm_base *base, struct alarm *alarm) 138static void alarmtimer_enqueue(struct alarm_base *base, struct alarm *alarm)
139{ 139{
140 if (alarm->state & ALARMTIMER_STATE_ENQUEUED)
141 timerqueue_del(&base->timerqueue, &alarm->node);
142
140 timerqueue_add(&base->timerqueue, &alarm->node); 143 timerqueue_add(&base->timerqueue, &alarm->node);
141 alarm->state |= ALARMTIMER_STATE_ENQUEUED; 144 alarm->state |= ALARMTIMER_STATE_ENQUEUED;
142
143 if (&alarm->node == timerqueue_getnext(&base->timerqueue)) {
144 hrtimer_try_to_cancel(&base->timer);
145 hrtimer_start(&base->timer, alarm->node.expires,
146 HRTIMER_MODE_ABS);
147 }
148} 145}
149 146
150/** 147/**
151 * alarmtimer_remove - Removes an alarm timer from an alarm_base timerqueue 148 * alarmtimer_dequeue - Removes an alarm timer from an alarm_base timerqueue
152 * @base: pointer to the base where the timer is running 149 * @base: pointer to the base where the timer is running
153 * @alarm: pointer to alarm being removed 150 * @alarm: pointer to alarm being removed
154 * 151 *
155 * Removes alarm to a alarm_base timerqueue and if necessary sets 152 * Removes alarm to a alarm_base timerqueue
156 * a new timer to run.
157 * 153 *
158 * Must hold base->lock when calling. 154 * Must hold base->lock when calling.
159 */ 155 */
160static void alarmtimer_remove(struct alarm_base *base, struct alarm *alarm) 156static void alarmtimer_dequeue(struct alarm_base *base, struct alarm *alarm)
161{ 157{
162 struct timerqueue_node *next = timerqueue_getnext(&base->timerqueue);
163
164 if (!(alarm->state & ALARMTIMER_STATE_ENQUEUED)) 158 if (!(alarm->state & ALARMTIMER_STATE_ENQUEUED))
165 return; 159 return;
166 160
167 timerqueue_del(&base->timerqueue, &alarm->node); 161 timerqueue_del(&base->timerqueue, &alarm->node);
168 alarm->state &= ~ALARMTIMER_STATE_ENQUEUED; 162 alarm->state &= ~ALARMTIMER_STATE_ENQUEUED;
169
170 if (next == &alarm->node) {
171 hrtimer_try_to_cancel(&base->timer);
172 next = timerqueue_getnext(&base->timerqueue);
173 if (!next)
174 return;
175 hrtimer_start(&base->timer, next->expires, HRTIMER_MODE_ABS);
176 }
177} 163}
178 164
179 165
@@ -188,42 +174,23 @@ static void alarmtimer_remove(struct alarm_base *base, struct alarm *alarm)
188 */ 174 */
189static enum hrtimer_restart alarmtimer_fired(struct hrtimer *timer) 175static enum hrtimer_restart alarmtimer_fired(struct hrtimer *timer)
190{ 176{
191 struct alarm_base *base = container_of(timer, struct alarm_base, timer); 177 struct alarm *alarm = container_of(timer, struct alarm, timer);
192 struct timerqueue_node *next; 178 struct alarm_base *base = &alarm_bases[alarm->type];
193 unsigned long flags; 179 unsigned long flags;
194 ktime_t now;
195 int ret = HRTIMER_NORESTART; 180 int ret = HRTIMER_NORESTART;
196 int restart = ALARMTIMER_NORESTART; 181 int restart = ALARMTIMER_NORESTART;
197 182
198 spin_lock_irqsave(&base->lock, flags); 183 spin_lock_irqsave(&base->lock, flags);
199 now = base->gettime(); 184 alarmtimer_dequeue(base, alarm);
200 while ((next = timerqueue_getnext(&base->timerqueue))) { 185 spin_unlock_irqrestore(&base->lock, flags);
201 struct alarm *alarm;
202 ktime_t expired = next->expires;
203
204 if (expired.tv64 > now.tv64)
205 break;
206
207 alarm = container_of(next, struct alarm, node);
208
209 timerqueue_del(&base->timerqueue, &alarm->node);
210 alarm->state &= ~ALARMTIMER_STATE_ENQUEUED;
211
212 alarm->state |= ALARMTIMER_STATE_CALLBACK;
213 spin_unlock_irqrestore(&base->lock, flags);
214 if (alarm->function)
215 restart = alarm->function(alarm, now);
216 spin_lock_irqsave(&base->lock, flags);
217 alarm->state &= ~ALARMTIMER_STATE_CALLBACK;
218 186
219 if (restart != ALARMTIMER_NORESTART) { 187 if (alarm->function)
220 timerqueue_add(&base->timerqueue, &alarm->node); 188 restart = alarm->function(alarm, base->gettime());
221 alarm->state |= ALARMTIMER_STATE_ENQUEUED;
222 }
223 }
224 189
225 if (next) { 190 spin_lock_irqsave(&base->lock, flags);
226 hrtimer_set_expires(&base->timer, next->expires); 191 if (restart != ALARMTIMER_NORESTART) {
192 hrtimer_set_expires(&alarm->timer, alarm->node.expires);
193 alarmtimer_enqueue(base, alarm);
227 ret = HRTIMER_RESTART; 194 ret = HRTIMER_RESTART;
228 } 195 }
229 spin_unlock_irqrestore(&base->lock, flags); 196 spin_unlock_irqrestore(&base->lock, flags);
@@ -250,6 +217,7 @@ static int alarmtimer_suspend(struct device *dev)
250 unsigned long flags; 217 unsigned long flags;
251 struct rtc_device *rtc; 218 struct rtc_device *rtc;
252 int i; 219 int i;
220 int ret;
253 221
254 spin_lock_irqsave(&freezer_delta_lock, flags); 222 spin_lock_irqsave(&freezer_delta_lock, flags);
255 min = freezer_delta; 223 min = freezer_delta;
@@ -279,8 +247,10 @@ static int alarmtimer_suspend(struct device *dev)
279 if (min.tv64 == 0) 247 if (min.tv64 == 0)
280 return 0; 248 return 0;
281 249
282 /* XXX - Should we enforce a minimum sleep time? */ 250 if (ktime_to_ns(min) < 2 * NSEC_PER_SEC) {
283 WARN_ON(min.tv64 < NSEC_PER_SEC); 251 __pm_wakeup_event(ws, 2 * MSEC_PER_SEC);
252 return -EBUSY;
253 }
284 254
285 /* Setup an rtc timer to fire that far in the future */ 255 /* Setup an rtc timer to fire that far in the future */
286 rtc_timer_cancel(rtc, &rtctimer); 256 rtc_timer_cancel(rtc, &rtctimer);
@@ -288,9 +258,11 @@ static int alarmtimer_suspend(struct device *dev)
288 now = rtc_tm_to_ktime(tm); 258 now = rtc_tm_to_ktime(tm);
289 now = ktime_add(now, min); 259 now = ktime_add(now, min);
290 260
291 rtc_timer_start(rtc, &rtctimer, now, ktime_set(0, 0)); 261 /* Set alarm, if in the past reject suspend briefly to handle */
292 262 ret = rtc_timer_start(rtc, &rtctimer, now, ktime_set(0, 0));
293 return 0; 263 if (ret < 0)
264 __pm_wakeup_event(ws, MSEC_PER_SEC);
265 return ret;
294} 266}
295#else 267#else
296static int alarmtimer_suspend(struct device *dev) 268static int alarmtimer_suspend(struct device *dev)
@@ -324,6 +296,9 @@ void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
324 enum alarmtimer_restart (*function)(struct alarm *, ktime_t)) 296 enum alarmtimer_restart (*function)(struct alarm *, ktime_t))
325{ 297{
326 timerqueue_init(&alarm->node); 298 timerqueue_init(&alarm->node);
299 hrtimer_init(&alarm->timer, alarm_bases[type].base_clockid,
300 HRTIMER_MODE_ABS);
301 alarm->timer.function = alarmtimer_fired;
327 alarm->function = function; 302 alarm->function = function;
328 alarm->type = type; 303 alarm->type = type;
329 alarm->state = ALARMTIMER_STATE_INACTIVE; 304 alarm->state = ALARMTIMER_STATE_INACTIVE;
@@ -334,17 +309,19 @@ void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
334 * @alarm: ptr to alarm to set 309 * @alarm: ptr to alarm to set
335 * @start: time to run the alarm 310 * @start: time to run the alarm
336 */ 311 */
337void alarm_start(struct alarm *alarm, ktime_t start) 312int alarm_start(struct alarm *alarm, ktime_t start)
338{ 313{
339 struct alarm_base *base = &alarm_bases[alarm->type]; 314 struct alarm_base *base = &alarm_bases[alarm->type];
340 unsigned long flags; 315 unsigned long flags;
316 int ret;
341 317
342 spin_lock_irqsave(&base->lock, flags); 318 spin_lock_irqsave(&base->lock, flags);
343 if (alarmtimer_active(alarm))
344 alarmtimer_remove(base, alarm);
345 alarm->node.expires = start; 319 alarm->node.expires = start;
346 alarmtimer_enqueue(base, alarm); 320 alarmtimer_enqueue(base, alarm);
321 ret = hrtimer_start(&alarm->timer, alarm->node.expires,
322 HRTIMER_MODE_ABS);
347 spin_unlock_irqrestore(&base->lock, flags); 323 spin_unlock_irqrestore(&base->lock, flags);
324 return ret;
348} 325}
349 326
350/** 327/**
@@ -358,18 +335,12 @@ int alarm_try_to_cancel(struct alarm *alarm)
358{ 335{
359 struct alarm_base *base = &alarm_bases[alarm->type]; 336 struct alarm_base *base = &alarm_bases[alarm->type];
360 unsigned long flags; 337 unsigned long flags;
361 int ret = -1; 338 int ret;
362 spin_lock_irqsave(&base->lock, flags);
363
364 if (alarmtimer_callback_running(alarm))
365 goto out;
366 339
367 if (alarmtimer_is_queued(alarm)) { 340 spin_lock_irqsave(&base->lock, flags);
368 alarmtimer_remove(base, alarm); 341 ret = hrtimer_try_to_cancel(&alarm->timer);
369 ret = 1; 342 if (ret >= 0)
370 } else 343 alarmtimer_dequeue(base, alarm);
371 ret = 0;
372out:
373 spin_unlock_irqrestore(&base->lock, flags); 344 spin_unlock_irqrestore(&base->lock, flags);
374 return ret; 345 return ret;
375} 346}
@@ -802,10 +773,6 @@ static int __init alarmtimer_init(void)
802 for (i = 0; i < ALARM_NUMTYPE; i++) { 773 for (i = 0; i < ALARM_NUMTYPE; i++) {
803 timerqueue_init_head(&alarm_bases[i].timerqueue); 774 timerqueue_init_head(&alarm_bases[i].timerqueue);
804 spin_lock_init(&alarm_bases[i].lock); 775 spin_lock_init(&alarm_bases[i].lock);
805 hrtimer_init(&alarm_bases[i].timer,
806 alarm_bases[i].base_clockid,
807 HRTIMER_MODE_ABS);
808 alarm_bases[i].timer.function = alarmtimer_fired;
809 } 776 }
810 777
811 error = alarmtimer_rtc_interface_setup(); 778 error = alarmtimer_rtc_interface_setup();
@@ -821,6 +788,7 @@ static int __init alarmtimer_init(void)
821 error = PTR_ERR(pdev); 788 error = PTR_ERR(pdev);
822 goto out_drv; 789 goto out_drv;
823 } 790 }
791 ws = wakeup_source_register("alarmtimer");
824 return 0; 792 return 0;
825 793
826out_drv: 794out_drv:
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index 46da0537c10b..6629bf7b5285 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -37,7 +37,7 @@
37 * requested HZ value. It is also not recommended 37 * requested HZ value. It is also not recommended
38 * for "tick-less" systems. 38 * for "tick-less" systems.
39 */ 39 */
40#define NSEC_PER_JIFFY ((u32)((((u64)NSEC_PER_SEC)<<8)/SHIFTED_HZ)) 40#define NSEC_PER_JIFFY ((NSEC_PER_SEC+HZ/2)/HZ)
41 41
42/* Since jiffies uses a simple NSEC_PER_JIFFY multiplier 42/* Since jiffies uses a simple NSEC_PER_JIFFY multiplier
43 * conversion, the .shift value could be zero. However 43 * conversion, the .shift value could be zero. However
@@ -95,3 +95,33 @@ struct clocksource * __init __weak clocksource_default_clock(void)
95{ 95{
96 return &clocksource_jiffies; 96 return &clocksource_jiffies;
97} 97}
98
99struct clocksource refined_jiffies;
100
101int register_refined_jiffies(long cycles_per_second)
102{
103 u64 nsec_per_tick, shift_hz;
104 long cycles_per_tick;
105
106
107
108 refined_jiffies = clocksource_jiffies;
109 refined_jiffies.name = "refined-jiffies";
110 refined_jiffies.rating++;
111
112 /* Calc cycles per tick */
113 cycles_per_tick = (cycles_per_second + HZ/2)/HZ;
114 /* shift_hz stores hz<<8 for extra accuracy */
115 shift_hz = (u64)cycles_per_second << 8;
116 shift_hz += cycles_per_tick/2;
117 do_div(shift_hz, cycles_per_tick);
118 /* Calculate nsec_per_tick using shift_hz */
119 nsec_per_tick = (u64)NSEC_PER_SEC << 8;
120 nsec_per_tick += (u32)shift_hz/2;
121 do_div(nsec_per_tick, (u32)shift_hz);
122
123 refined_jiffies.mult = ((u32)nsec_per_tick) << JIFFIES_SHIFT;
124
125 clocksource_register(&refined_jiffies);
126 return 0;
127}
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 5ce06a3fa91e..e424970bb562 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -8,6 +8,7 @@
8 * 8 *
9 */ 9 */
10 10
11#include <linux/timekeeper_internal.h>
11#include <linux/module.h> 12#include <linux/module.h>
12#include <linux/interrupt.h> 13#include <linux/interrupt.h>
13#include <linux/percpu.h> 14#include <linux/percpu.h>
@@ -21,61 +22,6 @@
21#include <linux/tick.h> 22#include <linux/tick.h>
22#include <linux/stop_machine.h> 23#include <linux/stop_machine.h>
23 24
24/* Structure holding internal timekeeping values. */
25struct timekeeper {
26 /* Current clocksource used for timekeeping. */
27 struct clocksource *clock;
28 /* NTP adjusted clock multiplier */
29 u32 mult;
30 /* The shift value of the current clocksource. */
31 u32 shift;
32 /* Number of clock cycles in one NTP interval. */
33 cycle_t cycle_interval;
34 /* Number of clock shifted nano seconds in one NTP interval. */
35 u64 xtime_interval;
36 /* shifted nano seconds left over when rounding cycle_interval */
37 s64 xtime_remainder;
38 /* Raw nano seconds accumulated per NTP interval. */
39 u32 raw_interval;
40
41 /* Current CLOCK_REALTIME time in seconds */
42 u64 xtime_sec;
43 /* Clock shifted nano seconds */
44 u64 xtime_nsec;
45
46 /* Difference between accumulated time and NTP time in ntp
47 * shifted nano seconds. */
48 s64 ntp_error;
49 /* Shift conversion between clock shifted nano seconds and
50 * ntp shifted nano seconds. */
51 u32 ntp_error_shift;
52
53 /*
54 * wall_to_monotonic is what we need to add to xtime (or xtime corrected
55 * for sub jiffie times) to get to monotonic time. Monotonic is pegged
56 * at zero at system boot time, so wall_to_monotonic will be negative,
57 * however, we will ALWAYS keep the tv_nsec part positive so we can use
58 * the usual normalization.
59 *
60 * wall_to_monotonic is moved after resume from suspend for the
61 * monotonic time not to jump. We need to add total_sleep_time to
62 * wall_to_monotonic to get the real boot based time offset.
63 *
64 * - wall_to_monotonic is no longer the boot time, getboottime must be
65 * used instead.
66 */
67 struct timespec wall_to_monotonic;
68 /* Offset clock monotonic -> clock realtime */
69 ktime_t offs_real;
70 /* time spent in suspend */
71 struct timespec total_sleep_time;
72 /* Offset clock monotonic -> clock boottime */
73 ktime_t offs_boot;
74 /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
75 struct timespec raw_time;
76 /* Seqlock for all timekeeper values */
77 seqlock_t lock;
78};
79 25
80static struct timekeeper timekeeper; 26static struct timekeeper timekeeper;
81 27
@@ -96,15 +42,6 @@ static inline void tk_normalize_xtime(struct timekeeper *tk)
96 } 42 }
97} 43}
98 44
99static struct timespec tk_xtime(struct timekeeper *tk)
100{
101 struct timespec ts;
102
103 ts.tv_sec = tk->xtime_sec;
104 ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
105 return ts;
106}
107
108static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) 45static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts)
109{ 46{
110 tk->xtime_sec = ts->tv_sec; 47 tk->xtime_sec = ts->tv_sec;
@@ -246,14 +183,11 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
246/* must hold write on timekeeper.lock */ 183/* must hold write on timekeeper.lock */
247static void timekeeping_update(struct timekeeper *tk, bool clearntp) 184static void timekeeping_update(struct timekeeper *tk, bool clearntp)
248{ 185{
249 struct timespec xt;
250
251 if (clearntp) { 186 if (clearntp) {
252 tk->ntp_error = 0; 187 tk->ntp_error = 0;
253 ntp_clear(); 188 ntp_clear();
254 } 189 }
255 xt = tk_xtime(tk); 190 update_vsyscall(tk);
256 update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult);
257} 191}
258 192
259/** 193/**
@@ -1113,7 +1047,7 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1113 accumulate_nsecs_to_secs(tk); 1047 accumulate_nsecs_to_secs(tk);
1114 1048
1115 /* Accumulate raw time */ 1049 /* Accumulate raw time */
1116 raw_nsecs = tk->raw_interval << shift; 1050 raw_nsecs = (u64)tk->raw_interval << shift;
1117 raw_nsecs += tk->raw_time.tv_nsec; 1051 raw_nsecs += tk->raw_time.tv_nsec;
1118 if (raw_nsecs >= NSEC_PER_SEC) { 1052 if (raw_nsecs >= NSEC_PER_SEC) {
1119 u64 raw_secs = raw_nsecs; 1053 u64 raw_secs = raw_nsecs;
@@ -1130,6 +1064,33 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1130 return offset; 1064 return offset;
1131} 1065}
1132 1066
1067#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
1068static inline void old_vsyscall_fixup(struct timekeeper *tk)
1069{
1070 s64 remainder;
1071
1072 /*
1073 * Store only full nanoseconds into xtime_nsec after rounding
1074 * it up and add the remainder to the error difference.
1075 * XXX - This is necessary to avoid small 1ns inconsistnecies caused
1076 * by truncating the remainder in vsyscalls. However, it causes
1077 * additional work to be done in timekeeping_adjust(). Once
1078 * the vsyscall implementations are converted to use xtime_nsec
1079 * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD
1080 * users are removed, this can be killed.
1081 */
1082 remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1);
1083 tk->xtime_nsec -= remainder;
1084 tk->xtime_nsec += 1ULL << tk->shift;
1085 tk->ntp_error += remainder << tk->ntp_error_shift;
1086
1087}
1088#else
1089#define old_vsyscall_fixup(tk)
1090#endif
1091
1092
1093
1133/** 1094/**
1134 * update_wall_time - Uses the current clocksource to increment the wall time 1095 * update_wall_time - Uses the current clocksource to increment the wall time
1135 * 1096 *
@@ -1141,7 +1102,6 @@ static void update_wall_time(void)
1141 cycle_t offset; 1102 cycle_t offset;
1142 int shift = 0, maxshift; 1103 int shift = 0, maxshift;
1143 unsigned long flags; 1104 unsigned long flags;
1144 s64 remainder;
1145 1105
1146 write_seqlock_irqsave(&tk->lock, flags); 1106 write_seqlock_irqsave(&tk->lock, flags);
1147 1107
@@ -1183,20 +1143,11 @@ static void update_wall_time(void)
1183 /* correct the clock when NTP error is too big */ 1143 /* correct the clock when NTP error is too big */
1184 timekeeping_adjust(tk, offset); 1144 timekeeping_adjust(tk, offset);
1185 1145
1186
1187 /* 1146 /*
1188 * Store only full nanoseconds into xtime_nsec after rounding 1147 * XXX This can be killed once everyone converts
1189 * it up and add the remainder to the error difference. 1148 * to the new update_vsyscall.
1190 * XXX - This is necessary to avoid small 1ns inconsistnecies caused 1149 */
1191 * by truncating the remainder in vsyscalls. However, it causes 1150 old_vsyscall_fixup(tk);
1192 * additional work to be done in timekeeping_adjust(). Once
1193 * the vsyscall implementations are converted to use xtime_nsec
1194 * (shifted nanoseconds), this can be killed.
1195 */
1196 remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1);
1197 tk->xtime_nsec -= remainder;
1198 tk->xtime_nsec += 1ULL << tk->shift;
1199 tk->ntp_error += remainder << tk->ntp_error_shift;
1200 1151
1201 /* 1152 /*
1202 * Finally, make sure that after the rounding 1153 * Finally, make sure that after the rounding
diff --git a/kernel/timer.c b/kernel/timer.c
index d5de1b2292aa..367d00858482 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -63,6 +63,7 @@ EXPORT_SYMBOL(jiffies_64);
63#define TVR_SIZE (1 << TVR_BITS) 63#define TVR_SIZE (1 << TVR_BITS)
64#define TVN_MASK (TVN_SIZE - 1) 64#define TVN_MASK (TVN_SIZE - 1)
65#define TVR_MASK (TVR_SIZE - 1) 65#define TVR_MASK (TVR_SIZE - 1)
66#define MAX_TVAL ((unsigned long)((1ULL << (TVR_BITS + 4*TVN_BITS)) - 1))
66 67
67struct tvec { 68struct tvec {
68 struct list_head vec[TVN_SIZE]; 69 struct list_head vec[TVN_SIZE];
@@ -359,11 +360,12 @@ __internal_add_timer(struct tvec_base *base, struct timer_list *timer)
359 vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK); 360 vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);
360 } else { 361 } else {
361 int i; 362 int i;
362 /* If the timeout is larger than 0xffffffff on 64-bit 363 /* If the timeout is larger than MAX_TVAL (on 64-bit
363 * architectures then we use the maximum timeout: 364 * architectures or with CONFIG_BASE_SMALL=1) then we
365 * use the maximum timeout.
364 */ 366 */
365 if (idx > 0xffffffffUL) { 367 if (idx > MAX_TVAL) {
366 idx = 0xffffffffUL; 368 idx = MAX_TVAL;
367 expires = idx + base->timer_jiffies; 369 expires = idx + base->timer_jiffies;
368 } 370 }
369 i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK; 371 i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;