aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-05-23 14:30:28 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2011-05-23 14:30:28 -0400
commit30cb6d5f2eb24d15d20139d5ceefaccc68734bd7 (patch)
tree773c5a98645e4b945343caddcfe5af365566ccc5
parent4867faab1e3eb8cc3f74e390357615d9b8e8cda6 (diff)
parent68fa61c026057a39d6ccb850aa8785043afbee02 (diff)
Merge branch 'timers-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'timers-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: hrtimers: Reorder clock bases hrtimers: Avoid touching inactive timer bases hrtimers: Make struct hrtimer_cpu_base layout less stupid timerfd: Manage cancelable timers in timerfd clockevents: Move C3 stop test outside lock alarmtimer: Drop device refcount after rtc_open() alarmtimer: Check return value of class_find_device() timerfd: Allow timers to be cancelled when clock was set hrtimers: Prepare for cancel on clock was set timers
-rw-r--r--fs/timerfd.c102
-rw-r--r--include/linux/hrtimer.h33
-rw-r--r--include/linux/thread_info.h2
-rw-r--r--include/linux/timerfd.h3
-rw-r--r--kernel/hrtimer.c162
-rw-r--r--kernel/posix-cpu-timers.c4
-rw-r--r--kernel/posix-timers.c2
-rw-r--r--kernel/time/alarmtimer.c16
-rw-r--r--kernel/time/tick-broadcast.c16
-rw-r--r--kernel/time/timekeeping.c17
10 files changed, 239 insertions, 118 deletions
diff --git a/fs/timerfd.c b/fs/timerfd.c
index 8c4fc1425b3e..f67acbdda5e8 100644
--- a/fs/timerfd.c
+++ b/fs/timerfd.c
@@ -22,16 +22,24 @@
22#include <linux/anon_inodes.h> 22#include <linux/anon_inodes.h>
23#include <linux/timerfd.h> 23#include <linux/timerfd.h>
24#include <linux/syscalls.h> 24#include <linux/syscalls.h>
25#include <linux/rcupdate.h>
25 26
26struct timerfd_ctx { 27struct timerfd_ctx {
27 struct hrtimer tmr; 28 struct hrtimer tmr;
28 ktime_t tintv; 29 ktime_t tintv;
30 ktime_t moffs;
29 wait_queue_head_t wqh; 31 wait_queue_head_t wqh;
30 u64 ticks; 32 u64 ticks;
31 int expired; 33 int expired;
32 int clockid; 34 int clockid;
35 struct rcu_head rcu;
36 struct list_head clist;
37 bool might_cancel;
33}; 38};
34 39
40static LIST_HEAD(cancel_list);
41static DEFINE_SPINLOCK(cancel_lock);
42
35/* 43/*
36 * This gets called when the timer event triggers. We set the "expired" 44 * This gets called when the timer event triggers. We set the "expired"
37 * flag, but we do not re-arm the timer (in case it's necessary, 45 * flag, but we do not re-arm the timer (in case it's necessary,
@@ -51,6 +59,63 @@ static enum hrtimer_restart timerfd_tmrproc(struct hrtimer *htmr)
51 return HRTIMER_NORESTART; 59 return HRTIMER_NORESTART;
52} 60}
53 61
62/*
63 * Called when the clock was set to cancel the timers in the cancel
64 * list.
65 */
66void timerfd_clock_was_set(void)
67{
68 ktime_t moffs = ktime_get_monotonic_offset();
69 struct timerfd_ctx *ctx;
70 unsigned long flags;
71
72 rcu_read_lock();
73 list_for_each_entry_rcu(ctx, &cancel_list, clist) {
74 if (!ctx->might_cancel)
75 continue;
76 spin_lock_irqsave(&ctx->wqh.lock, flags);
77 if (ctx->moffs.tv64 != moffs.tv64) {
78 ctx->moffs.tv64 = KTIME_MAX;
79 wake_up_locked(&ctx->wqh);
80 }
81 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
82 }
83 rcu_read_unlock();
84}
85
86static void timerfd_remove_cancel(struct timerfd_ctx *ctx)
87{
88 if (ctx->might_cancel) {
89 ctx->might_cancel = false;
90 spin_lock(&cancel_lock);
91 list_del_rcu(&ctx->clist);
92 spin_unlock(&cancel_lock);
93 }
94}
95
96static bool timerfd_canceled(struct timerfd_ctx *ctx)
97{
98 if (!ctx->might_cancel || ctx->moffs.tv64 != KTIME_MAX)
99 return false;
100 ctx->moffs = ktime_get_monotonic_offset();
101 return true;
102}
103
104static void timerfd_setup_cancel(struct timerfd_ctx *ctx, int flags)
105{
106 if (ctx->clockid == CLOCK_REALTIME && (flags & TFD_TIMER_ABSTIME) &&
107 (flags & TFD_TIMER_CANCEL_ON_SET)) {
108 if (!ctx->might_cancel) {
109 ctx->might_cancel = true;
110 spin_lock(&cancel_lock);
111 list_add_rcu(&ctx->clist, &cancel_list);
112 spin_unlock(&cancel_lock);
113 }
114 } else if (ctx->might_cancel) {
115 timerfd_remove_cancel(ctx);
116 }
117}
118
54static ktime_t timerfd_get_remaining(struct timerfd_ctx *ctx) 119static ktime_t timerfd_get_remaining(struct timerfd_ctx *ctx)
55{ 120{
56 ktime_t remaining; 121 ktime_t remaining;
@@ -59,11 +124,12 @@ static ktime_t timerfd_get_remaining(struct timerfd_ctx *ctx)
59 return remaining.tv64 < 0 ? ktime_set(0, 0): remaining; 124 return remaining.tv64 < 0 ? ktime_set(0, 0): remaining;
60} 125}
61 126
62static void timerfd_setup(struct timerfd_ctx *ctx, int flags, 127static int timerfd_setup(struct timerfd_ctx *ctx, int flags,
63 const struct itimerspec *ktmr) 128 const struct itimerspec *ktmr)
64{ 129{
65 enum hrtimer_mode htmode; 130 enum hrtimer_mode htmode;
66 ktime_t texp; 131 ktime_t texp;
132 int clockid = ctx->clockid;
67 133
68 htmode = (flags & TFD_TIMER_ABSTIME) ? 134 htmode = (flags & TFD_TIMER_ABSTIME) ?
69 HRTIMER_MODE_ABS: HRTIMER_MODE_REL; 135 HRTIMER_MODE_ABS: HRTIMER_MODE_REL;
@@ -72,19 +138,24 @@ static void timerfd_setup(struct timerfd_ctx *ctx, int flags,
72 ctx->expired = 0; 138 ctx->expired = 0;
73 ctx->ticks = 0; 139 ctx->ticks = 0;
74 ctx->tintv = timespec_to_ktime(ktmr->it_interval); 140 ctx->tintv = timespec_to_ktime(ktmr->it_interval);
75 hrtimer_init(&ctx->tmr, ctx->clockid, htmode); 141 hrtimer_init(&ctx->tmr, clockid, htmode);
76 hrtimer_set_expires(&ctx->tmr, texp); 142 hrtimer_set_expires(&ctx->tmr, texp);
77 ctx->tmr.function = timerfd_tmrproc; 143 ctx->tmr.function = timerfd_tmrproc;
78 if (texp.tv64 != 0) 144 if (texp.tv64 != 0) {
79 hrtimer_start(&ctx->tmr, texp, htmode); 145 hrtimer_start(&ctx->tmr, texp, htmode);
146 if (timerfd_canceled(ctx))
147 return -ECANCELED;
148 }
149 return 0;
80} 150}
81 151
82static int timerfd_release(struct inode *inode, struct file *file) 152static int timerfd_release(struct inode *inode, struct file *file)
83{ 153{
84 struct timerfd_ctx *ctx = file->private_data; 154 struct timerfd_ctx *ctx = file->private_data;
85 155
156 timerfd_remove_cancel(ctx);
86 hrtimer_cancel(&ctx->tmr); 157 hrtimer_cancel(&ctx->tmr);
87 kfree(ctx); 158 kfree_rcu(ctx, rcu);
88 return 0; 159 return 0;
89} 160}
90 161
@@ -118,8 +189,21 @@ static ssize_t timerfd_read(struct file *file, char __user *buf, size_t count,
118 res = -EAGAIN; 189 res = -EAGAIN;
119 else 190 else
120 res = wait_event_interruptible_locked_irq(ctx->wqh, ctx->ticks); 191 res = wait_event_interruptible_locked_irq(ctx->wqh, ctx->ticks);
192
193 /*
194 * If clock has changed, we do not care about the
195 * ticks and we do not rearm the timer. Userspace must
196 * reevaluate anyway.
197 */
198 if (timerfd_canceled(ctx)) {
199 ctx->ticks = 0;
200 ctx->expired = 0;
201 res = -ECANCELED;
202 }
203
121 if (ctx->ticks) { 204 if (ctx->ticks) {
122 ticks = ctx->ticks; 205 ticks = ctx->ticks;
206
123 if (ctx->expired && ctx->tintv.tv64) { 207 if (ctx->expired && ctx->tintv.tv64) {
124 /* 208 /*
125 * If tintv.tv64 != 0, this is a periodic timer that 209 * If tintv.tv64 != 0, this is a periodic timer that
@@ -183,6 +267,7 @@ SYSCALL_DEFINE2(timerfd_create, int, clockid, int, flags)
183 init_waitqueue_head(&ctx->wqh); 267 init_waitqueue_head(&ctx->wqh);
184 ctx->clockid = clockid; 268 ctx->clockid = clockid;
185 hrtimer_init(&ctx->tmr, clockid, HRTIMER_MODE_ABS); 269 hrtimer_init(&ctx->tmr, clockid, HRTIMER_MODE_ABS);
270 ctx->moffs = ktime_get_monotonic_offset();
186 271
187 ufd = anon_inode_getfd("[timerfd]", &timerfd_fops, ctx, 272 ufd = anon_inode_getfd("[timerfd]", &timerfd_fops, ctx,
188 O_RDWR | (flags & TFD_SHARED_FCNTL_FLAGS)); 273 O_RDWR | (flags & TFD_SHARED_FCNTL_FLAGS));
@@ -199,6 +284,7 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags,
199 struct file *file; 284 struct file *file;
200 struct timerfd_ctx *ctx; 285 struct timerfd_ctx *ctx;
201 struct itimerspec ktmr, kotmr; 286 struct itimerspec ktmr, kotmr;
287 int ret;
202 288
203 if (copy_from_user(&ktmr, utmr, sizeof(ktmr))) 289 if (copy_from_user(&ktmr, utmr, sizeof(ktmr)))
204 return -EFAULT; 290 return -EFAULT;
@@ -213,6 +299,8 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags,
213 return PTR_ERR(file); 299 return PTR_ERR(file);
214 ctx = file->private_data; 300 ctx = file->private_data;
215 301
302 timerfd_setup_cancel(ctx, flags);
303
216 /* 304 /*
217 * We need to stop the existing timer before reprogramming 305 * We need to stop the existing timer before reprogramming
218 * it to the new values. 306 * it to the new values.
@@ -240,14 +328,14 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags,
240 /* 328 /*
241 * Re-program the timer to the new value ... 329 * Re-program the timer to the new value ...
242 */ 330 */
243 timerfd_setup(ctx, flags, &ktmr); 331 ret = timerfd_setup(ctx, flags, &ktmr);
244 332
245 spin_unlock_irq(&ctx->wqh.lock); 333 spin_unlock_irq(&ctx->wqh.lock);
246 fput(file); 334 fput(file);
247 if (otmr && copy_to_user(otmr, &kotmr, sizeof(kotmr))) 335 if (otmr && copy_to_user(otmr, &kotmr, sizeof(kotmr)))
248 return -EFAULT; 336 return -EFAULT;
249 337
250 return 0; 338 return ret;
251} 339}
252 340
253SYSCALL_DEFINE2(timerfd_gettime, int, ufd, struct itimerspec __user *, otmr) 341SYSCALL_DEFINE2(timerfd_gettime, int, ufd, struct itimerspec __user *, otmr)
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 62f500c724f9..51932e5acf7c 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -143,19 +143,18 @@ struct hrtimer_sleeper {
143 */ 143 */
144struct hrtimer_clock_base { 144struct hrtimer_clock_base {
145 struct hrtimer_cpu_base *cpu_base; 145 struct hrtimer_cpu_base *cpu_base;
146 clockid_t index; 146 int index;
147 clockid_t clockid;
147 struct timerqueue_head active; 148 struct timerqueue_head active;
148 ktime_t resolution; 149 ktime_t resolution;
149 ktime_t (*get_time)(void); 150 ktime_t (*get_time)(void);
150 ktime_t softirq_time; 151 ktime_t softirq_time;
151#ifdef CONFIG_HIGH_RES_TIMERS
152 ktime_t offset; 152 ktime_t offset;
153#endif
154}; 153};
155 154
156enum hrtimer_base_type { 155enum hrtimer_base_type {
157 HRTIMER_BASE_REALTIME,
158 HRTIMER_BASE_MONOTONIC, 156 HRTIMER_BASE_MONOTONIC,
157 HRTIMER_BASE_REALTIME,
159 HRTIMER_BASE_BOOTTIME, 158 HRTIMER_BASE_BOOTTIME,
160 HRTIMER_MAX_CLOCK_BASES, 159 HRTIMER_MAX_CLOCK_BASES,
161}; 160};
@@ -164,7 +163,7 @@ enum hrtimer_base_type {
164 * struct hrtimer_cpu_base - the per cpu clock bases 163 * struct hrtimer_cpu_base - the per cpu clock bases
165 * @lock: lock protecting the base and associated clock bases 164 * @lock: lock protecting the base and associated clock bases
166 * and timers 165 * and timers
167 * @clock_base: array of clock bases for this cpu 166 * @active_bases: Bitfield to mark bases with active timers
168 * @expires_next: absolute time of the next event which was scheduled 167 * @expires_next: absolute time of the next event which was scheduled
169 * via clock_set_next_event() 168 * via clock_set_next_event()
170 * @hres_active: State of high resolution mode 169 * @hres_active: State of high resolution mode
@@ -173,10 +172,11 @@ enum hrtimer_base_type {
173 * @nr_retries: Total number of hrtimer interrupt retries 172 * @nr_retries: Total number of hrtimer interrupt retries
174 * @nr_hangs: Total number of hrtimer interrupt hangs 173 * @nr_hangs: Total number of hrtimer interrupt hangs
175 * @max_hang_time: Maximum time spent in hrtimer_interrupt 174 * @max_hang_time: Maximum time spent in hrtimer_interrupt
175 * @clock_base: array of clock bases for this cpu
176 */ 176 */
177struct hrtimer_cpu_base { 177struct hrtimer_cpu_base {
178 raw_spinlock_t lock; 178 raw_spinlock_t lock;
179 struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; 179 unsigned long active_bases;
180#ifdef CONFIG_HIGH_RES_TIMERS 180#ifdef CONFIG_HIGH_RES_TIMERS
181 ktime_t expires_next; 181 ktime_t expires_next;
182 int hres_active; 182 int hres_active;
@@ -186,6 +186,7 @@ struct hrtimer_cpu_base {
186 unsigned long nr_hangs; 186 unsigned long nr_hangs;
187 ktime_t max_hang_time; 187 ktime_t max_hang_time;
188#endif 188#endif
189 struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
189}; 190};
190 191
191static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) 192static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
@@ -256,8 +257,6 @@ static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
256#ifdef CONFIG_HIGH_RES_TIMERS 257#ifdef CONFIG_HIGH_RES_TIMERS
257struct clock_event_device; 258struct clock_event_device;
258 259
259extern void clock_was_set(void);
260extern void hres_timers_resume(void);
261extern void hrtimer_interrupt(struct clock_event_device *dev); 260extern void hrtimer_interrupt(struct clock_event_device *dev);
262 261
263/* 262/*
@@ -291,16 +290,8 @@ extern void hrtimer_peek_ahead_timers(void);
291# define MONOTONIC_RES_NSEC LOW_RES_NSEC 290# define MONOTONIC_RES_NSEC LOW_RES_NSEC
292# define KTIME_MONOTONIC_RES KTIME_LOW_RES 291# define KTIME_MONOTONIC_RES KTIME_LOW_RES
293 292
294/*
295 * clock_was_set() is a NOP for non- high-resolution systems. The
296 * time-sorted order guarantees that a timer does not expire early and
297 * is expired in the next softirq when the clock was advanced.
298 */
299static inline void clock_was_set(void) { }
300static inline void hrtimer_peek_ahead_timers(void) { } 293static inline void hrtimer_peek_ahead_timers(void) { }
301 294
302static inline void hres_timers_resume(void) { }
303
304/* 295/*
305 * In non high resolution mode the time reference is taken from 296 * In non high resolution mode the time reference is taken from
306 * the base softirq time variable. 297 * the base softirq time variable.
@@ -316,10 +307,18 @@ static inline int hrtimer_is_hres_active(struct hrtimer *timer)
316} 307}
317#endif 308#endif
318 309
310extern void clock_was_set(void);
311#ifdef CONFIG_TIMERFD
312extern void timerfd_clock_was_set(void);
313#else
314static inline void timerfd_clock_was_set(void) { }
315#endif
316extern void hrtimers_resume(void);
317
319extern ktime_t ktime_get(void); 318extern ktime_t ktime_get(void);
320extern ktime_t ktime_get_real(void); 319extern ktime_t ktime_get_real(void);
321extern ktime_t ktime_get_boottime(void); 320extern ktime_t ktime_get_boottime(void);
322 321extern ktime_t ktime_get_monotonic_offset(void);
323 322
324DECLARE_PER_CPU(struct tick_device, tick_cpu_device); 323DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
325 324
diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h
index 20fc303947d3..8d03f079688c 100644
--- a/include/linux/thread_info.h
+++ b/include/linux/thread_info.h
@@ -29,7 +29,7 @@ struct restart_block {
29 } futex; 29 } futex;
30 /* For nanosleep */ 30 /* For nanosleep */
31 struct { 31 struct {
32 clockid_t index; 32 clockid_t clockid;
33 struct timespec __user *rmtp; 33 struct timespec __user *rmtp;
34#ifdef CONFIG_COMPAT 34#ifdef CONFIG_COMPAT
35 struct compat_timespec __user *compat_rmtp; 35 struct compat_timespec __user *compat_rmtp;
diff --git a/include/linux/timerfd.h b/include/linux/timerfd.h
index 2d0792983f8c..d3b57fa12225 100644
--- a/include/linux/timerfd.h
+++ b/include/linux/timerfd.h
@@ -19,6 +19,7 @@
19 * shared O_* flags. 19 * shared O_* flags.
20 */ 20 */
21#define TFD_TIMER_ABSTIME (1 << 0) 21#define TFD_TIMER_ABSTIME (1 << 0)
22#define TFD_TIMER_CANCEL_ON_SET (1 << 1)
22#define TFD_CLOEXEC O_CLOEXEC 23#define TFD_CLOEXEC O_CLOEXEC
23#define TFD_NONBLOCK O_NONBLOCK 24#define TFD_NONBLOCK O_NONBLOCK
24 25
@@ -26,6 +27,6 @@
26/* Flags for timerfd_create. */ 27/* Flags for timerfd_create. */
27#define TFD_CREATE_FLAGS TFD_SHARED_FCNTL_FLAGS 28#define TFD_CREATE_FLAGS TFD_SHARED_FCNTL_FLAGS
28/* Flags for timerfd_settime. */ 29/* Flags for timerfd_settime. */
29#define TFD_SETTIME_FLAGS TFD_TIMER_ABSTIME 30#define TFD_SETTIME_FLAGS (TFD_TIMER_ABSTIME | TFD_TIMER_CANCEL_ON_SET)
30 31
31#endif /* _LINUX_TIMERFD_H */ 32#endif /* _LINUX_TIMERFD_H */
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index dbbbf7d43080..c541ee527ecb 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -64,17 +64,20 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
64 .clock_base = 64 .clock_base =
65 { 65 {
66 { 66 {
67 .index = CLOCK_REALTIME, 67 .index = HRTIMER_BASE_MONOTONIC,
68 .get_time = &ktime_get_real, 68 .clockid = CLOCK_MONOTONIC,
69 .get_time = &ktime_get,
69 .resolution = KTIME_LOW_RES, 70 .resolution = KTIME_LOW_RES,
70 }, 71 },
71 { 72 {
72 .index = CLOCK_MONOTONIC, 73 .index = HRTIMER_BASE_REALTIME,
73 .get_time = &ktime_get, 74 .clockid = CLOCK_REALTIME,
75 .get_time = &ktime_get_real,
74 .resolution = KTIME_LOW_RES, 76 .resolution = KTIME_LOW_RES,
75 }, 77 },
76 { 78 {
77 .index = CLOCK_BOOTTIME, 79 .index = HRTIMER_BASE_BOOTTIME,
80 .clockid = CLOCK_BOOTTIME,
78 .get_time = &ktime_get_boottime, 81 .get_time = &ktime_get_boottime,
79 .resolution = KTIME_LOW_RES, 82 .resolution = KTIME_LOW_RES,
80 }, 83 },
@@ -196,7 +199,7 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
196 struct hrtimer_cpu_base *new_cpu_base; 199 struct hrtimer_cpu_base *new_cpu_base;
197 int this_cpu = smp_processor_id(); 200 int this_cpu = smp_processor_id();
198 int cpu = hrtimer_get_target(this_cpu, pinned); 201 int cpu = hrtimer_get_target(this_cpu, pinned);
199 int basenum = hrtimer_clockid_to_base(base->index); 202 int basenum = base->index;
200 203
201again: 204again:
202 new_cpu_base = &per_cpu(hrtimer_bases, cpu); 205 new_cpu_base = &per_cpu(hrtimer_bases, cpu);
@@ -621,66 +624,6 @@ static int hrtimer_reprogram(struct hrtimer *timer,
621 return res; 624 return res;
622} 625}
623 626
624
625/*
626 * Retrigger next event is called after clock was set
627 *
628 * Called with interrupts disabled via on_each_cpu()
629 */
630static void retrigger_next_event(void *arg)
631{
632 struct hrtimer_cpu_base *base;
633 struct timespec realtime_offset, wtm, sleep;
634
635 if (!hrtimer_hres_active())
636 return;
637
638 get_xtime_and_monotonic_and_sleep_offset(&realtime_offset, &wtm,
639 &sleep);
640 set_normalized_timespec(&realtime_offset, -wtm.tv_sec, -wtm.tv_nsec);
641
642 base = &__get_cpu_var(hrtimer_bases);
643
644 /* Adjust CLOCK_REALTIME offset */
645 raw_spin_lock(&base->lock);
646 base->clock_base[HRTIMER_BASE_REALTIME].offset =
647 timespec_to_ktime(realtime_offset);
648 base->clock_base[HRTIMER_BASE_BOOTTIME].offset =
649 timespec_to_ktime(sleep);
650
651 hrtimer_force_reprogram(base, 0);
652 raw_spin_unlock(&base->lock);
653}
654
655/*
656 * Clock realtime was set
657 *
658 * Change the offset of the realtime clock vs. the monotonic
659 * clock.
660 *
661 * We might have to reprogram the high resolution timer interrupt. On
662 * SMP we call the architecture specific code to retrigger _all_ high
663 * resolution timer interrupts. On UP we just disable interrupts and
664 * call the high resolution interrupt code.
665 */
666void clock_was_set(void)
667{
668 /* Retrigger the CPU local events everywhere */
669 on_each_cpu(retrigger_next_event, NULL, 1);
670}
671
672/*
673 * During resume we might have to reprogram the high resolution timer
674 * interrupt (on the local CPU):
675 */
676void hres_timers_resume(void)
677{
678 WARN_ONCE(!irqs_disabled(),
679 KERN_INFO "hres_timers_resume() called with IRQs enabled!");
680
681 retrigger_next_event(NULL);
682}
683
684/* 627/*
685 * Initialize the high resolution related parts of cpu_base 628 * Initialize the high resolution related parts of cpu_base
686 */ 629 */
@@ -715,11 +658,39 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
715} 658}
716 659
717/* 660/*
661 * Retrigger next event is called after clock was set
662 *
663 * Called with interrupts disabled via on_each_cpu()
664 */
665static void retrigger_next_event(void *arg)
666{
667 struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases);
668 struct timespec realtime_offset, xtim, wtm, sleep;
669
670 if (!hrtimer_hres_active())
671 return;
672
673 /* Optimized out for !HIGH_RES */
674 get_xtime_and_monotonic_and_sleep_offset(&xtim, &wtm, &sleep);
675 set_normalized_timespec(&realtime_offset, -wtm.tv_sec, -wtm.tv_nsec);
676
677 /* Adjust CLOCK_REALTIME offset */
678 raw_spin_lock(&base->lock);
679 base->clock_base[HRTIMER_BASE_REALTIME].offset =
680 timespec_to_ktime(realtime_offset);
681 base->clock_base[HRTIMER_BASE_BOOTTIME].offset =
682 timespec_to_ktime(sleep);
683
684 hrtimer_force_reprogram(base, 0);
685 raw_spin_unlock(&base->lock);
686}
687
688/*
718 * Switch to high resolution mode 689 * Switch to high resolution mode
719 */ 690 */
720static int hrtimer_switch_to_hres(void) 691static int hrtimer_switch_to_hres(void)
721{ 692{
722 int cpu = smp_processor_id(); 693 int i, cpu = smp_processor_id();
723 struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu); 694 struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu);
724 unsigned long flags; 695 unsigned long flags;
725 696
@@ -735,9 +706,8 @@ static int hrtimer_switch_to_hres(void)
735 return 0; 706 return 0;
736 } 707 }
737 base->hres_active = 1; 708 base->hres_active = 1;
738 base->clock_base[HRTIMER_BASE_REALTIME].resolution = KTIME_HIGH_RES; 709 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
739 base->clock_base[HRTIMER_BASE_MONOTONIC].resolution = KTIME_HIGH_RES; 710 base->clock_base[i].resolution = KTIME_HIGH_RES;
740 base->clock_base[HRTIMER_BASE_BOOTTIME].resolution = KTIME_HIGH_RES;
741 711
742 tick_setup_sched_timer(); 712 tick_setup_sched_timer();
743 713
@@ -761,9 +731,43 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
761 return 0; 731 return 0;
762} 732}
763static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } 733static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
734static inline void retrigger_next_event(void *arg) { }
764 735
765#endif /* CONFIG_HIGH_RES_TIMERS */ 736#endif /* CONFIG_HIGH_RES_TIMERS */
766 737
738/*
739 * Clock realtime was set
740 *
741 * Change the offset of the realtime clock vs. the monotonic
742 * clock.
743 *
744 * We might have to reprogram the high resolution timer interrupt. On
745 * SMP we call the architecture specific code to retrigger _all_ high
746 * resolution timer interrupts. On UP we just disable interrupts and
747 * call the high resolution interrupt code.
748 */
749void clock_was_set(void)
750{
751#ifdef CONFIG_HIGHRES_TIMERS
752 /* Retrigger the CPU local events everywhere */
753 on_each_cpu(retrigger_next_event, NULL, 1);
754#endif
755 timerfd_clock_was_set();
756}
757
758/*
759 * During resume we might have to reprogram the high resolution timer
760 * interrupt (on the local CPU):
761 */
762void hrtimers_resume(void)
763{
764 WARN_ONCE(!irqs_disabled(),
765 KERN_INFO "hrtimers_resume() called with IRQs enabled!");
766
767 retrigger_next_event(NULL);
768 timerfd_clock_was_set();
769}
770
767static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer) 771static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer)
768{ 772{
769#ifdef CONFIG_TIMER_STATS 773#ifdef CONFIG_TIMER_STATS
@@ -856,6 +860,7 @@ static int enqueue_hrtimer(struct hrtimer *timer,
856 debug_activate(timer); 860 debug_activate(timer);
857 861
858 timerqueue_add(&base->active, &timer->node); 862 timerqueue_add(&base->active, &timer->node);
863 base->cpu_base->active_bases |= 1 << base->index;
859 864
860 /* 865 /*
861 * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the 866 * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the
@@ -897,6 +902,8 @@ static void __remove_hrtimer(struct hrtimer *timer,
897#endif 902#endif
898 } 903 }
899 timerqueue_del(&base->active, &timer->node); 904 timerqueue_del(&base->active, &timer->node);
905 if (!timerqueue_getnext(&base->active))
906 base->cpu_base->active_bases &= ~(1 << base->index);
900out: 907out:
901 timer->state = newstate; 908 timer->state = newstate;
902} 909}
@@ -1234,7 +1241,6 @@ static void __run_hrtimer(struct hrtimer *timer, ktime_t *now)
1234void hrtimer_interrupt(struct clock_event_device *dev) 1241void hrtimer_interrupt(struct clock_event_device *dev)
1235{ 1242{
1236 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); 1243 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1237 struct hrtimer_clock_base *base;
1238 ktime_t expires_next, now, entry_time, delta; 1244 ktime_t expires_next, now, entry_time, delta;
1239 int i, retries = 0; 1245 int i, retries = 0;
1240 1246
@@ -1256,12 +1262,15 @@ retry:
1256 */ 1262 */
1257 cpu_base->expires_next.tv64 = KTIME_MAX; 1263 cpu_base->expires_next.tv64 = KTIME_MAX;
1258 1264
1259 base = cpu_base->clock_base;
1260
1261 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { 1265 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1262 ktime_t basenow; 1266 struct hrtimer_clock_base *base;
1263 struct timerqueue_node *node; 1267 struct timerqueue_node *node;
1268 ktime_t basenow;
1269
1270 if (!(cpu_base->active_bases & (1 << i)))
1271 continue;
1264 1272
1273 base = cpu_base->clock_base + i;
1265 basenow = ktime_add(now, base->offset); 1274 basenow = ktime_add(now, base->offset);
1266 1275
1267 while ((node = timerqueue_getnext(&base->active))) { 1276 while ((node = timerqueue_getnext(&base->active))) {
@@ -1294,7 +1303,6 @@ retry:
1294 1303
1295 __run_hrtimer(timer, &basenow); 1304 __run_hrtimer(timer, &basenow);
1296 } 1305 }
1297 base++;
1298 } 1306 }
1299 1307
1300 /* 1308 /*
@@ -1525,7 +1533,7 @@ long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
1525 struct timespec __user *rmtp; 1533 struct timespec __user *rmtp;
1526 int ret = 0; 1534 int ret = 0;
1527 1535
1528 hrtimer_init_on_stack(&t.timer, restart->nanosleep.index, 1536 hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,
1529 HRTIMER_MODE_ABS); 1537 HRTIMER_MODE_ABS);
1530 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); 1538 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
1531 1539
@@ -1577,7 +1585,7 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
1577 1585
1578 restart = &current_thread_info()->restart_block; 1586 restart = &current_thread_info()->restart_block;
1579 restart->fn = hrtimer_nanosleep_restart; 1587 restart->fn = hrtimer_nanosleep_restart;
1580 restart->nanosleep.index = t.timer.base->index; 1588 restart->nanosleep.clockid = t.timer.base->clockid;
1581 restart->nanosleep.rmtp = rmtp; 1589 restart->nanosleep.rmtp = rmtp;
1582 restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); 1590 restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
1583 1591
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index 0791b13df7bf..58f405b581e7 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -1514,7 +1514,7 @@ static int posix_cpu_nsleep(const clockid_t which_clock, int flags,
1514 return -EFAULT; 1514 return -EFAULT;
1515 1515
1516 restart_block->fn = posix_cpu_nsleep_restart; 1516 restart_block->fn = posix_cpu_nsleep_restart;
1517 restart_block->nanosleep.index = which_clock; 1517 restart_block->nanosleep.clockid = which_clock;
1518 restart_block->nanosleep.rmtp = rmtp; 1518 restart_block->nanosleep.rmtp = rmtp;
1519 restart_block->nanosleep.expires = timespec_to_ns(rqtp); 1519 restart_block->nanosleep.expires = timespec_to_ns(rqtp);
1520 } 1520 }
@@ -1523,7 +1523,7 @@ static int posix_cpu_nsleep(const clockid_t which_clock, int flags,
1523 1523
1524static long posix_cpu_nsleep_restart(struct restart_block *restart_block) 1524static long posix_cpu_nsleep_restart(struct restart_block *restart_block)
1525{ 1525{
1526 clockid_t which_clock = restart_block->nanosleep.index; 1526 clockid_t which_clock = restart_block->nanosleep.clockid;
1527 struct timespec t; 1527 struct timespec t;
1528 struct itimerspec it; 1528 struct itimerspec it;
1529 int error; 1529 int error;
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c
index e5498d7405c3..a1b5edf1bf92 100644
--- a/kernel/posix-timers.c
+++ b/kernel/posix-timers.c
@@ -1056,7 +1056,7 @@ SYSCALL_DEFINE4(clock_nanosleep, const clockid_t, which_clock, int, flags,
1056 */ 1056 */
1057long clock_nanosleep_restart(struct restart_block *restart_block) 1057long clock_nanosleep_restart(struct restart_block *restart_block)
1058{ 1058{
1059 clockid_t which_clock = restart_block->nanosleep.index; 1059 clockid_t which_clock = restart_block->nanosleep.clockid;
1060 struct k_clock *kc = clockid_to_kclock(which_clock); 1060 struct k_clock *kc = clockid_to_kclock(which_clock);
1061 1061
1062 if (WARN_ON_ONCE(!kc || !kc->nsleep_restart)) 1062 if (WARN_ON_ONCE(!kc || !kc->nsleep_restart))
diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c
index 9265014cb4db..2d966244ea60 100644
--- a/kernel/time/alarmtimer.c
+++ b/kernel/time/alarmtimer.c
@@ -494,7 +494,7 @@ static int update_rmtp(ktime_t exp, enum alarmtimer_type type,
494 */ 494 */
495static long __sched alarm_timer_nsleep_restart(struct restart_block *restart) 495static long __sched alarm_timer_nsleep_restart(struct restart_block *restart)
496{ 496{
497 enum alarmtimer_type type = restart->nanosleep.index; 497 enum alarmtimer_type type = restart->nanosleep.clockid;
498 ktime_t exp; 498 ktime_t exp;
499 struct timespec __user *rmtp; 499 struct timespec __user *rmtp;
500 struct alarm alarm; 500 struct alarm alarm;
@@ -573,7 +573,7 @@ static int alarm_timer_nsleep(const clockid_t which_clock, int flags,
573 573
574 restart = &current_thread_info()->restart_block; 574 restart = &current_thread_info()->restart_block;
575 restart->fn = alarm_timer_nsleep_restart; 575 restart->fn = alarm_timer_nsleep_restart;
576 restart->nanosleep.index = type; 576 restart->nanosleep.clockid = type;
577 restart->nanosleep.expires = exp.tv64; 577 restart->nanosleep.expires = exp.tv64;
578 restart->nanosleep.rmtp = rmtp; 578 restart->nanosleep.rmtp = rmtp;
579 ret = -ERESTART_RESTARTBLOCK; 579 ret = -ERESTART_RESTARTBLOCK;
@@ -669,12 +669,20 @@ static int __init has_wakealarm(struct device *dev, void *name_ptr)
669 */ 669 */
670static int __init alarmtimer_init_late(void) 670static int __init alarmtimer_init_late(void)
671{ 671{
672 struct device *dev;
672 char *str; 673 char *str;
673 674
674 /* Find an rtc device and init the rtc_timer */ 675 /* Find an rtc device and init the rtc_timer */
675 class_find_device(rtc_class, NULL, &str, has_wakealarm); 676 dev = class_find_device(rtc_class, NULL, &str, has_wakealarm);
676 if (str) 677 /* If we have a device then str is valid. See has_wakealarm() */
678 if (dev) {
677 rtcdev = rtc_class_open(str); 679 rtcdev = rtc_class_open(str);
680 /*
681 * Drop the reference we got in class_find_device,
682 * rtc_open takes its own.
683 */
684 put_device(dev);
685 }
678 if (!rtcdev) { 686 if (!rtcdev) {
679 printk(KERN_WARNING "No RTC device found, ALARM timers will" 687 printk(KERN_WARNING "No RTC device found, ALARM timers will"
680 " not wake from suspend"); 688 " not wake from suspend");
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 723c7637e55a..c7218d132738 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -456,23 +456,27 @@ void tick_broadcast_oneshot_control(unsigned long reason)
456 unsigned long flags; 456 unsigned long flags;
457 int cpu; 457 int cpu;
458 458
459 raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
460
461 /* 459 /*
462 * Periodic mode does not care about the enter/exit of power 460 * Periodic mode does not care about the enter/exit of power
463 * states 461 * states
464 */ 462 */
465 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 463 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
466 goto out; 464 return;
467 465
468 bc = tick_broadcast_device.evtdev; 466 /*
467 * We are called with preemtion disabled from the depth of the
468 * idle code, so we can't be moved away.
469 */
469 cpu = smp_processor_id(); 470 cpu = smp_processor_id();
470 td = &per_cpu(tick_cpu_device, cpu); 471 td = &per_cpu(tick_cpu_device, cpu);
471 dev = td->evtdev; 472 dev = td->evtdev;
472 473
473 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) 474 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
474 goto out; 475 return;
476
477 bc = tick_broadcast_device.evtdev;
475 478
479 raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
476 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { 480 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
477 if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) { 481 if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
478 cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask()); 482 cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask());
@@ -489,8 +493,6 @@ void tick_broadcast_oneshot_control(unsigned long reason)
489 tick_program_event(dev->next_event, 1); 493 tick_program_event(dev->next_event, 1);
490 } 494 }
491 } 495 }
492
493out:
494 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 496 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
495} 497}
496 498
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 8e6a05a5915a..342408cf68dd 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -680,7 +680,7 @@ static void timekeeping_resume(void)
680 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL); 680 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
681 681
682 /* Resume hrtimers */ 682 /* Resume hrtimers */
683 hres_timers_resume(); 683 hrtimers_resume();
684} 684}
685 685
686static int timekeeping_suspend(void) 686static int timekeeping_suspend(void)
@@ -1099,6 +1099,21 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1099} 1099}
1100 1100
1101/** 1101/**
1102 * ktime_get_monotonic_offset() - get wall_to_monotonic in ktime_t format
1103 */
1104ktime_t ktime_get_monotonic_offset(void)
1105{
1106 unsigned long seq;
1107 struct timespec wtom;
1108
1109 do {
1110 seq = read_seqbegin(&xtime_lock);
1111 wtom = wall_to_monotonic;
1112 } while (read_seqretry(&xtime_lock, seq));
1113 return timespec_to_ktime(wtom);
1114}
1115
1116/**
1102 * xtime_update() - advances the timekeeping infrastructure 1117 * xtime_update() - advances the timekeeping infrastructure
1103 * @ticks: number of ticks, that have elapsed since the last call. 1118 * @ticks: number of ticks, that have elapsed since the last call.
1104 * 1119 *