aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/time
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/time')
-rw-r--r--kernel/time/clocksource.c3
-rw-r--r--kernel/time/jiffies.c1
-rw-r--r--kernel/time/ntp.c93
-rw-r--r--kernel/time/tick-broadcast.c13
-rw-r--r--kernel/time/tick-internal.h2
-rw-r--r--kernel/time/tick-sched.c101
-rw-r--r--kernel/time/timekeeping.c122
-rw-r--r--kernel/time/timer_list.c20
8 files changed, 234 insertions, 121 deletions
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 093d4acf993b..9ed2eec97526 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -325,6 +325,9 @@ int clocksource_register(struct clocksource *c)
325 unsigned long flags; 325 unsigned long flags;
326 int ret; 326 int ret;
327 327
328 /* save mult_orig on registration */
329 c->mult_orig = c->mult;
330
328 spin_lock_irqsave(&clocksource_lock, flags); 331 spin_lock_irqsave(&clocksource_lock, flags);
329 ret = clocksource_enqueue(c); 332 ret = clocksource_enqueue(c);
330 if (!ret) 333 if (!ret)
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index 4c256fdb8875..1ca99557e929 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -61,6 +61,7 @@ struct clocksource clocksource_jiffies = {
61 .read = jiffies_read, 61 .read = jiffies_read,
62 .mask = 0xffffffff, /*32bits*/ 62 .mask = 0xffffffff, /*32bits*/
63 .mult = NSEC_PER_JIFFY << JIFFIES_SHIFT, /* details above */ 63 .mult = NSEC_PER_JIFFY << JIFFIES_SHIFT, /* details above */
64 .mult_orig = NSEC_PER_JIFFY << JIFFIES_SHIFT,
64 .shift = JIFFIES_SHIFT, 65 .shift = JIFFIES_SHIFT,
65}; 66};
66 67
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 9c114b726ab3..8ff15e5d486b 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -10,13 +10,13 @@
10 10
11#include <linux/mm.h> 11#include <linux/mm.h>
12#include <linux/time.h> 12#include <linux/time.h>
13#include <linux/timer.h>
14#include <linux/timex.h> 13#include <linux/timex.h>
15#include <linux/jiffies.h> 14#include <linux/jiffies.h>
16#include <linux/hrtimer.h> 15#include <linux/hrtimer.h>
17#include <linux/capability.h> 16#include <linux/capability.h>
18#include <linux/math64.h> 17#include <linux/math64.h>
19#include <linux/clocksource.h> 18#include <linux/clocksource.h>
19#include <linux/workqueue.h>
20#include <asm/timex.h> 20#include <asm/timex.h>
21 21
22/* 22/*
@@ -217,11 +217,11 @@ void second_overflow(void)
217/* Disable the cmos update - used by virtualization and embedded */ 217/* Disable the cmos update - used by virtualization and embedded */
218int no_sync_cmos_clock __read_mostly; 218int no_sync_cmos_clock __read_mostly;
219 219
220static void sync_cmos_clock(unsigned long dummy); 220static void sync_cmos_clock(struct work_struct *work);
221 221
222static DEFINE_TIMER(sync_cmos_timer, sync_cmos_clock, 0, 0); 222static DECLARE_DELAYED_WORK(sync_cmos_work, sync_cmos_clock);
223 223
224static void sync_cmos_clock(unsigned long dummy) 224static void sync_cmos_clock(struct work_struct *work)
225{ 225{
226 struct timespec now, next; 226 struct timespec now, next;
227 int fail = 1; 227 int fail = 1;
@@ -257,13 +257,13 @@ static void sync_cmos_clock(unsigned long dummy)
257 next.tv_sec++; 257 next.tv_sec++;
258 next.tv_nsec -= NSEC_PER_SEC; 258 next.tv_nsec -= NSEC_PER_SEC;
259 } 259 }
260 mod_timer(&sync_cmos_timer, jiffies + timespec_to_jiffies(&next)); 260 schedule_delayed_work(&sync_cmos_work, timespec_to_jiffies(&next));
261} 261}
262 262
263static void notify_cmos_timer(void) 263static void notify_cmos_timer(void)
264{ 264{
265 if (!no_sync_cmos_clock) 265 if (!no_sync_cmos_clock)
266 mod_timer(&sync_cmos_timer, jiffies + 1); 266 schedule_delayed_work(&sync_cmos_work, 0);
267} 267}
268 268
269#else 269#else
@@ -276,38 +276,50 @@ static inline void notify_cmos_timer(void) { }
276int do_adjtimex(struct timex *txc) 276int do_adjtimex(struct timex *txc)
277{ 277{
278 struct timespec ts; 278 struct timespec ts;
279 long save_adjust, sec;
280 int result; 279 int result;
281 280
282 /* In order to modify anything, you gotta be super-user! */ 281 /* Validate the data before disabling interrupts */
283 if (txc->modes && !capable(CAP_SYS_TIME)) 282 if (txc->modes & ADJ_ADJTIME) {
284 return -EPERM;
285
286 /* Now we validate the data before disabling interrupts */
287
288 if ((txc->modes & ADJ_OFFSET_SINGLESHOT) == ADJ_OFFSET_SINGLESHOT) {
289 /* singleshot must not be used with any other mode bits */ 283 /* singleshot must not be used with any other mode bits */
290 if (txc->modes & ~ADJ_OFFSET_SS_READ) 284 if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
291 return -EINVAL; 285 return -EINVAL;
286 if (!(txc->modes & ADJ_OFFSET_READONLY) &&
287 !capable(CAP_SYS_TIME))
288 return -EPERM;
289 } else {
290 /* In order to modify anything, you gotta be super-user! */
291 if (txc->modes && !capable(CAP_SYS_TIME))
292 return -EPERM;
293
294 /* if the quartz is off by more than 10% something is VERY wrong! */
295 if (txc->modes & ADJ_TICK &&
296 (txc->tick < 900000/USER_HZ ||
297 txc->tick > 1100000/USER_HZ))
298 return -EINVAL;
299
300 if (txc->modes & ADJ_STATUS && time_state != TIME_OK)
301 hrtimer_cancel(&leap_timer);
292 } 302 }
293 303
294 /* if the quartz is off by more than 10% something is VERY wrong ! */
295 if (txc->modes & ADJ_TICK)
296 if (txc->tick < 900000/USER_HZ ||
297 txc->tick > 1100000/USER_HZ)
298 return -EINVAL;
299
300 if (time_state != TIME_OK && txc->modes & ADJ_STATUS)
301 hrtimer_cancel(&leap_timer);
302 getnstimeofday(&ts); 304 getnstimeofday(&ts);
303 305
304 write_seqlock_irq(&xtime_lock); 306 write_seqlock_irq(&xtime_lock);
305 307
306 /* Save for later - semantics of adjtime is to return old value */
307 save_adjust = time_adjust;
308
309 /* If there are input parameters, then process them */ 308 /* If there are input parameters, then process them */
309 if (txc->modes & ADJ_ADJTIME) {
310 long save_adjust = time_adjust;
311
312 if (!(txc->modes & ADJ_OFFSET_READONLY)) {
313 /* adjtime() is independent from ntp_adjtime() */
314 time_adjust = txc->offset;
315 ntp_update_frequency();
316 }
317 txc->offset = save_adjust;
318 goto adj_done;
319 }
310 if (txc->modes) { 320 if (txc->modes) {
321 long sec;
322
311 if (txc->modes & ADJ_STATUS) { 323 if (txc->modes & ADJ_STATUS) {
312 if ((time_status & STA_PLL) && 324 if ((time_status & STA_PLL) &&
313 !(txc->status & STA_PLL)) { 325 !(txc->status & STA_PLL)) {
@@ -374,13 +386,8 @@ int do_adjtimex(struct timex *txc)
374 if (txc->modes & ADJ_TAI && txc->constant > 0) 386 if (txc->modes & ADJ_TAI && txc->constant > 0)
375 time_tai = txc->constant; 387 time_tai = txc->constant;
376 388
377 if (txc->modes & ADJ_OFFSET) { 389 if (txc->modes & ADJ_OFFSET)
378 if (txc->modes == ADJ_OFFSET_SINGLESHOT) 390 ntp_update_offset(txc->offset);
379 /* adjtime() is independent from ntp_adjtime() */
380 time_adjust = txc->offset;
381 else
382 ntp_update_offset(txc->offset);
383 }
384 if (txc->modes & ADJ_TICK) 391 if (txc->modes & ADJ_TICK)
385 tick_usec = txc->tick; 392 tick_usec = txc->tick;
386 393
@@ -388,22 +395,18 @@ int do_adjtimex(struct timex *txc)
388 ntp_update_frequency(); 395 ntp_update_frequency();
389 } 396 }
390 397
398 txc->offset = shift_right(time_offset * NTP_INTERVAL_FREQ,
399 NTP_SCALE_SHIFT);
400 if (!(time_status & STA_NANO))
401 txc->offset /= NSEC_PER_USEC;
402
403adj_done:
391 result = time_state; /* mostly `TIME_OK' */ 404 result = time_state; /* mostly `TIME_OK' */
392 if (time_status & (STA_UNSYNC|STA_CLOCKERR)) 405 if (time_status & (STA_UNSYNC|STA_CLOCKERR))
393 result = TIME_ERROR; 406 result = TIME_ERROR;
394 407
395 if ((txc->modes == ADJ_OFFSET_SINGLESHOT) || 408 txc->freq = shift_right((time_freq >> PPM_SCALE_INV_SHIFT) *
396 (txc->modes == ADJ_OFFSET_SS_READ)) 409 (s64)PPM_SCALE_INV, NTP_SCALE_SHIFT);
397 txc->offset = save_adjust;
398 else {
399 txc->offset = shift_right(time_offset * NTP_INTERVAL_FREQ,
400 NTP_SCALE_SHIFT);
401 if (!(time_status & STA_NANO))
402 txc->offset /= NSEC_PER_USEC;
403 }
404 txc->freq = shift_right((s32)(time_freq >> PPM_SCALE_INV_SHIFT) *
405 (s64)PPM_SCALE_INV,
406 NTP_SCALE_SHIFT);
407 txc->maxerror = time_maxerror; 410 txc->maxerror = time_maxerror;
408 txc->esterror = time_esterror; 411 txc->esterror = time_esterror;
409 txc->status = time_status; 412 txc->status = time_status;
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index cb01cd8f919b..f98a1b7b16e9 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -384,6 +384,19 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
384} 384}
385 385
386/* 386/*
387 * Called from irq_enter() when idle was interrupted to reenable the
388 * per cpu device.
389 */
390void tick_check_oneshot_broadcast(int cpu)
391{
392 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
393 struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
394
395 clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
396 }
397}
398
399/*
387 * Handle oneshot mode broadcasting 400 * Handle oneshot mode broadcasting
388 */ 401 */
389static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) 402static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 469248782c23..b1c05bf75ee0 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -36,6 +36,7 @@ extern void tick_broadcast_switch_to_oneshot(void);
36extern void tick_shutdown_broadcast_oneshot(unsigned int *cpup); 36extern void tick_shutdown_broadcast_oneshot(unsigned int *cpup);
37extern int tick_resume_broadcast_oneshot(struct clock_event_device *bc); 37extern int tick_resume_broadcast_oneshot(struct clock_event_device *bc);
38extern int tick_broadcast_oneshot_active(void); 38extern int tick_broadcast_oneshot_active(void);
39extern void tick_check_oneshot_broadcast(int cpu);
39# else /* BROADCAST */ 40# else /* BROADCAST */
40static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) 41static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
41{ 42{
@@ -45,6 +46,7 @@ static inline void tick_broadcast_oneshot_control(unsigned long reason) { }
45static inline void tick_broadcast_switch_to_oneshot(void) { } 46static inline void tick_broadcast_switch_to_oneshot(void) { }
46static inline void tick_shutdown_broadcast_oneshot(unsigned int *cpup) { } 47static inline void tick_shutdown_broadcast_oneshot(unsigned int *cpup) { }
47static inline int tick_broadcast_oneshot_active(void) { return 0; } 48static inline int tick_broadcast_oneshot_active(void) { return 0; }
49static inline void tick_check_oneshot_broadcast(int cpu) { }
48# endif /* !BROADCAST */ 50# endif /* !BROADCAST */
49 51
50#else /* !ONESHOT */ 52#else /* !ONESHOT */
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index a547be11cf97..5bbb1044f847 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -155,7 +155,7 @@ void tick_nohz_update_jiffies(void)
155 touch_softlockup_watchdog(); 155 touch_softlockup_watchdog();
156} 156}
157 157
158void tick_nohz_stop_idle(int cpu) 158static void tick_nohz_stop_idle(int cpu)
159{ 159{
160 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 160 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
161 161
@@ -377,6 +377,32 @@ ktime_t tick_nohz_get_sleep_length(void)
377 return ts->sleep_length; 377 return ts->sleep_length;
378} 378}
379 379
380static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
381{
382 hrtimer_cancel(&ts->sched_timer);
383 hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
384
385 while (1) {
386 /* Forward the time to expire in the future */
387 hrtimer_forward(&ts->sched_timer, now, tick_period);
388
389 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
390 hrtimer_start_expires(&ts->sched_timer,
391 HRTIMER_MODE_ABS);
392 /* Check, if the timer was already in the past */
393 if (hrtimer_active(&ts->sched_timer))
394 break;
395 } else {
396 if (!tick_program_event(
397 hrtimer_get_expires(&ts->sched_timer), 0))
398 break;
399 }
400 /* Update jiffies and reread time */
401 tick_do_update_jiffies64(now);
402 now = ktime_get();
403 }
404}
405
380/** 406/**
381 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task 407 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task
382 * 408 *
@@ -430,28 +456,9 @@ void tick_nohz_restart_sched_tick(void)
430 */ 456 */
431 ts->tick_stopped = 0; 457 ts->tick_stopped = 0;
432 ts->idle_exittime = now; 458 ts->idle_exittime = now;
433 hrtimer_cancel(&ts->sched_timer);
434 hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
435 459
436 while (1) { 460 tick_nohz_restart(ts, now);
437 /* Forward the time to expire in the future */
438 hrtimer_forward(&ts->sched_timer, now, tick_period);
439 461
440 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
441 hrtimer_start_expires(&ts->sched_timer,
442 HRTIMER_MODE_ABS);
443 /* Check, if the timer was already in the past */
444 if (hrtimer_active(&ts->sched_timer))
445 break;
446 } else {
447 if (!tick_program_event(
448 hrtimer_get_expires(&ts->sched_timer), 0))
449 break;
450 }
451 /* Update jiffies and reread time */
452 tick_do_update_jiffies64(now);
453 now = ktime_get();
454 }
455 local_irq_enable(); 462 local_irq_enable();
456} 463}
457 464
@@ -503,10 +510,6 @@ static void tick_nohz_handler(struct clock_event_device *dev)
503 update_process_times(user_mode(regs)); 510 update_process_times(user_mode(regs));
504 profile_tick(CPU_PROFILING); 511 profile_tick(CPU_PROFILING);
505 512
506 /* Do not restart, when we are in the idle loop */
507 if (ts->tick_stopped)
508 return;
509
510 while (tick_nohz_reprogram(ts, now)) { 513 while (tick_nohz_reprogram(ts, now)) {
511 now = ktime_get(); 514 now = ktime_get();
512 tick_do_update_jiffies64(now); 515 tick_do_update_jiffies64(now);
@@ -552,6 +555,37 @@ static void tick_nohz_switch_to_nohz(void)
552 smp_processor_id()); 555 smp_processor_id());
553} 556}
554 557
558/*
559 * When NOHZ is enabled and the tick is stopped, we need to kick the
560 * tick timer from irq_enter() so that the jiffies update is kept
561 * alive during long running softirqs. That's ugly as hell, but
562 * correctness is key even if we need to fix the offending softirq in
563 * the first place.
564 *
565 * Note, this is different to tick_nohz_restart. We just kick the
566 * timer and do not touch the other magic bits which need to be done
567 * when idle is left.
568 */
569static void tick_nohz_kick_tick(int cpu)
570{
571 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
572 ktime_t delta, now;
573
574 if (!ts->tick_stopped)
575 return;
576
577 /*
578 * Do not touch the tick device, when the next expiry is either
579 * already reached or less/equal than the tick period.
580 */
581 now = ktime_get();
582 delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
583 if (delta.tv64 <= tick_period.tv64)
584 return;
585
586 tick_nohz_restart(ts, now);
587}
588
555#else 589#else
556 590
557static inline void tick_nohz_switch_to_nohz(void) { } 591static inline void tick_nohz_switch_to_nohz(void) { }
@@ -559,6 +593,19 @@ static inline void tick_nohz_switch_to_nohz(void) { }
559#endif /* NO_HZ */ 593#endif /* NO_HZ */
560 594
561/* 595/*
596 * Called from irq_enter to notify about the possible interruption of idle()
597 */
598void tick_check_idle(int cpu)
599{
600 tick_check_oneshot_broadcast(cpu);
601#ifdef CONFIG_NO_HZ
602 tick_nohz_stop_idle(cpu);
603 tick_nohz_update_jiffies();
604 tick_nohz_kick_tick(cpu);
605#endif
606}
607
608/*
562 * High resolution timer specific code 609 * High resolution timer specific code
563 */ 610 */
564#ifdef CONFIG_HIGH_RES_TIMERS 611#ifdef CONFIG_HIGH_RES_TIMERS
@@ -611,10 +658,6 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
611 profile_tick(CPU_PROFILING); 658 profile_tick(CPU_PROFILING);
612 } 659 }
613 660
614 /* Do not restart, when we are in the idle loop */
615 if (ts->tick_stopped)
616 return HRTIMER_NORESTART;
617
618 hrtimer_forward(timer, now, tick_period); 661 hrtimer_forward(timer, now, tick_period);
619 662
620 return HRTIMER_RESTART; 663 return HRTIMER_RESTART;
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index e91c29f961c9..e7acfb482a68 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -58,27 +58,26 @@ struct clocksource *clock;
58 58
59#ifdef CONFIG_GENERIC_TIME 59#ifdef CONFIG_GENERIC_TIME
60/** 60/**
61 * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook 61 * clocksource_forward_now - update clock to the current time
62 * 62 *
63 * private function, must hold xtime_lock lock when being 63 * Forward the current clock to update its state since the last call to
64 * called. Returns the number of nanoseconds since the 64 * update_wall_time(). This is useful before significant clock changes,
65 * last call to update_wall_time() (adjusted by NTP scaling) 65 * as it avoids having to deal with this time offset explicitly.
66 */ 66 */
67static inline s64 __get_nsec_offset(void) 67static void clocksource_forward_now(void)
68{ 68{
69 cycle_t cycle_now, cycle_delta; 69 cycle_t cycle_now, cycle_delta;
70 s64 ns_offset; 70 s64 nsec;
71 71
72 /* read clocksource: */
73 cycle_now = clocksource_read(clock); 72 cycle_now = clocksource_read(clock);
74
75 /* calculate the delta since the last update_wall_time: */
76 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 73 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
74 clock->cycle_last = cycle_now;
77 75
78 /* convert to nanoseconds: */ 76 nsec = cyc2ns(clock, cycle_delta);
79 ns_offset = cyc2ns(clock, cycle_delta); 77 timespec_add_ns(&xtime, nsec);
80 78
81 return ns_offset; 79 nsec = ((s64)cycle_delta * clock->mult_orig) >> clock->shift;
80 clock->raw_time.tv_nsec += nsec;
82} 81}
83 82
84/** 83/**
@@ -89,6 +88,7 @@ static inline s64 __get_nsec_offset(void)
89 */ 88 */
90void getnstimeofday(struct timespec *ts) 89void getnstimeofday(struct timespec *ts)
91{ 90{
91 cycle_t cycle_now, cycle_delta;
92 unsigned long seq; 92 unsigned long seq;
93 s64 nsecs; 93 s64 nsecs;
94 94
@@ -96,7 +96,15 @@ void getnstimeofday(struct timespec *ts)
96 seq = read_seqbegin(&xtime_lock); 96 seq = read_seqbegin(&xtime_lock);
97 97
98 *ts = xtime; 98 *ts = xtime;
99 nsecs = __get_nsec_offset(); 99
100 /* read clocksource: */
101 cycle_now = clocksource_read(clock);
102
103 /* calculate the delta since the last update_wall_time: */
104 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
105
106 /* convert to nanoseconds: */
107 nsecs = cyc2ns(clock, cycle_delta);
100 108
101 } while (read_seqretry(&xtime_lock, seq)); 109 } while (read_seqretry(&xtime_lock, seq));
102 110
@@ -129,22 +137,22 @@ EXPORT_SYMBOL(do_gettimeofday);
129 */ 137 */
130int do_settimeofday(struct timespec *tv) 138int do_settimeofday(struct timespec *tv)
131{ 139{
140 struct timespec ts_delta;
132 unsigned long flags; 141 unsigned long flags;
133 time_t wtm_sec, sec = tv->tv_sec;
134 long wtm_nsec, nsec = tv->tv_nsec;
135 142
136 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) 143 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
137 return -EINVAL; 144 return -EINVAL;
138 145
139 write_seqlock_irqsave(&xtime_lock, flags); 146 write_seqlock_irqsave(&xtime_lock, flags);
140 147
141 nsec -= __get_nsec_offset(); 148 clocksource_forward_now();
149
150 ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec;
151 ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec;
152 wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta);
142 153
143 wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec); 154 xtime = *tv;
144 wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec);
145 155
146 set_normalized_timespec(&xtime, sec, nsec);
147 set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec);
148 update_xtime_cache(0); 156 update_xtime_cache(0);
149 157
150 clock->error = 0; 158 clock->error = 0;
@@ -170,22 +178,19 @@ EXPORT_SYMBOL(do_settimeofday);
170static void change_clocksource(void) 178static void change_clocksource(void)
171{ 179{
172 struct clocksource *new; 180 struct clocksource *new;
173 cycle_t now;
174 u64 nsec;
175 181
176 new = clocksource_get_next(); 182 new = clocksource_get_next();
177 183
178 if (clock == new) 184 if (clock == new)
179 return; 185 return;
180 186
181 new->cycle_last = 0; 187 clocksource_forward_now();
182 now = clocksource_read(new);
183 nsec = __get_nsec_offset();
184 timespec_add_ns(&xtime, nsec);
185 188
186 clock = new; 189 new->raw_time = clock->raw_time;
187 clock->cycle_last = now;
188 190
191 clock = new;
192 clock->cycle_last = 0;
193 clock->cycle_last = clocksource_read(new);
189 clock->error = 0; 194 clock->error = 0;
190 clock->xtime_nsec = 0; 195 clock->xtime_nsec = 0;
191 clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); 196 clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH);
@@ -200,11 +205,44 @@ static void change_clocksource(void)
200 */ 205 */
201} 206}
202#else 207#else
208static inline void clocksource_forward_now(void) { }
203static inline void change_clocksource(void) { } 209static inline void change_clocksource(void) { }
204static inline s64 __get_nsec_offset(void) { return 0; }
205#endif 210#endif
206 211
207/** 212/**
213 * getrawmonotonic - Returns the raw monotonic time in a timespec
214 * @ts: pointer to the timespec to be set
215 *
216 * Returns the raw monotonic time (completely un-modified by ntp)
217 */
218void getrawmonotonic(struct timespec *ts)
219{
220 unsigned long seq;
221 s64 nsecs;
222 cycle_t cycle_now, cycle_delta;
223
224 do {
225 seq = read_seqbegin(&xtime_lock);
226
227 /* read clocksource: */
228 cycle_now = clocksource_read(clock);
229
230 /* calculate the delta since the last update_wall_time: */
231 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
232
233 /* convert to nanoseconds: */
234 nsecs = ((s64)cycle_delta * clock->mult_orig) >> clock->shift;
235
236 *ts = clock->raw_time;
237
238 } while (read_seqretry(&xtime_lock, seq));
239
240 timespec_add_ns(ts, nsecs);
241}
242EXPORT_SYMBOL(getrawmonotonic);
243
244
245/**
208 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres 246 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
209 */ 247 */
210int timekeeping_valid_for_hres(void) 248int timekeeping_valid_for_hres(void)
@@ -265,8 +303,6 @@ void __init timekeeping_init(void)
265static int timekeeping_suspended; 303static int timekeeping_suspended;
266/* time in seconds when suspend began */ 304/* time in seconds when suspend began */
267static unsigned long timekeeping_suspend_time; 305static unsigned long timekeeping_suspend_time;
268/* xtime offset when we went into suspend */
269static s64 timekeeping_suspend_nsecs;
270 306
271/** 307/**
272 * timekeeping_resume - Resumes the generic timekeeping subsystem. 308 * timekeeping_resume - Resumes the generic timekeeping subsystem.
@@ -292,8 +328,6 @@ static int timekeeping_resume(struct sys_device *dev)
292 wall_to_monotonic.tv_sec -= sleep_length; 328 wall_to_monotonic.tv_sec -= sleep_length;
293 total_sleep_time += sleep_length; 329 total_sleep_time += sleep_length;
294 } 330 }
295 /* Make sure that we have the correct xtime reference */
296 timespec_add_ns(&xtime, timekeeping_suspend_nsecs);
297 update_xtime_cache(0); 331 update_xtime_cache(0);
298 /* re-base the last cycle value */ 332 /* re-base the last cycle value */
299 clock->cycle_last = 0; 333 clock->cycle_last = 0;
@@ -319,8 +353,7 @@ static int timekeeping_suspend(struct sys_device *dev, pm_message_t state)
319 timekeeping_suspend_time = read_persistent_clock(); 353 timekeeping_suspend_time = read_persistent_clock();
320 354
321 write_seqlock_irqsave(&xtime_lock, flags); 355 write_seqlock_irqsave(&xtime_lock, flags);
322 /* Get the current xtime offset */ 356 clocksource_forward_now();
323 timekeeping_suspend_nsecs = __get_nsec_offset();
324 timekeeping_suspended = 1; 357 timekeeping_suspended = 1;
325 write_sequnlock_irqrestore(&xtime_lock, flags); 358 write_sequnlock_irqrestore(&xtime_lock, flags);
326 359
@@ -454,23 +487,29 @@ void update_wall_time(void)
454#else 487#else
455 offset = clock->cycle_interval; 488 offset = clock->cycle_interval;
456#endif 489#endif
457 clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift; 490 clock->xtime_nsec = (s64)xtime.tv_nsec << clock->shift;
458 491
459 /* normally this loop will run just once, however in the 492 /* normally this loop will run just once, however in the
460 * case of lost or late ticks, it will accumulate correctly. 493 * case of lost or late ticks, it will accumulate correctly.
461 */ 494 */
462 while (offset >= clock->cycle_interval) { 495 while (offset >= clock->cycle_interval) {
463 /* accumulate one interval */ 496 /* accumulate one interval */
464 clock->xtime_nsec += clock->xtime_interval;
465 clock->cycle_last += clock->cycle_interval;
466 offset -= clock->cycle_interval; 497 offset -= clock->cycle_interval;
498 clock->cycle_last += clock->cycle_interval;
467 499
500 clock->xtime_nsec += clock->xtime_interval;
468 if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { 501 if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) {
469 clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; 502 clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift;
470 xtime.tv_sec++; 503 xtime.tv_sec++;
471 second_overflow(); 504 second_overflow();
472 } 505 }
473 506
507 clock->raw_time.tv_nsec += clock->raw_interval;
508 if (clock->raw_time.tv_nsec >= NSEC_PER_SEC) {
509 clock->raw_time.tv_nsec -= NSEC_PER_SEC;
510 clock->raw_time.tv_sec++;
511 }
512
474 /* accumulate error between NTP and clock interval */ 513 /* accumulate error between NTP and clock interval */
475 clock->error += tick_length; 514 clock->error += tick_length;
476 clock->error -= clock->xtime_interval << (NTP_SCALE_SHIFT - clock->shift); 515 clock->error -= clock->xtime_interval << (NTP_SCALE_SHIFT - clock->shift);
@@ -479,9 +518,12 @@ void update_wall_time(void)
479 /* correct the clock when NTP error is too big */ 518 /* correct the clock when NTP error is too big */
480 clocksource_adjust(offset); 519 clocksource_adjust(offset);
481 520
482 /* store full nanoseconds into xtime */ 521 /* store full nanoseconds into xtime after rounding it up and
483 xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift; 522 * add the remainder to the error difference.
523 */
524 xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
484 clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; 525 clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
526 clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
485 527
486 update_xtime_cache(cyc2ns(clock, offset)); 528 update_xtime_cache(cyc2ns(clock, offset));
487 529
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 122ee751d2d1..a999b92a1277 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -47,13 +47,14 @@ static void print_name_offset(struct seq_file *m, void *sym)
47} 47}
48 48
49static void 49static void
50print_timer(struct seq_file *m, struct hrtimer *timer, int idx, u64 now) 50print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer,
51 int idx, u64 now)
51{ 52{
52#ifdef CONFIG_TIMER_STATS 53#ifdef CONFIG_TIMER_STATS
53 char tmp[TASK_COMM_LEN + 1]; 54 char tmp[TASK_COMM_LEN + 1];
54#endif 55#endif
55 SEQ_printf(m, " #%d: ", idx); 56 SEQ_printf(m, " #%d: ", idx);
56 print_name_offset(m, timer); 57 print_name_offset(m, taddr);
57 SEQ_printf(m, ", "); 58 SEQ_printf(m, ", ");
58 print_name_offset(m, timer->function); 59 print_name_offset(m, timer->function);
59 SEQ_printf(m, ", S:%02lx", timer->state); 60 SEQ_printf(m, ", S:%02lx", timer->state);
@@ -101,7 +102,7 @@ next_one:
101 tmp = *timer; 102 tmp = *timer;
102 spin_unlock_irqrestore(&base->cpu_base->lock, flags); 103 spin_unlock_irqrestore(&base->cpu_base->lock, flags);
103 104
104 print_timer(m, &tmp, i, now); 105 print_timer(m, timer, &tmp, i, now);
105 next++; 106 next++;
106 goto next_one; 107 goto next_one;
107 } 108 }
@@ -111,6 +112,7 @@ next_one:
111static void 112static void
112print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now) 113print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
113{ 114{
115 SEQ_printf(m, " .base: %p\n", base);
114 SEQ_printf(m, " .index: %d\n", 116 SEQ_printf(m, " .index: %d\n",
115 base->index); 117 base->index);
116 SEQ_printf(m, " .resolution: %Lu nsecs\n", 118 SEQ_printf(m, " .resolution: %Lu nsecs\n",
@@ -185,12 +187,16 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
185 187
186#ifdef CONFIG_GENERIC_CLOCKEVENTS 188#ifdef CONFIG_GENERIC_CLOCKEVENTS
187static void 189static void
188print_tickdevice(struct seq_file *m, struct tick_device *td) 190print_tickdevice(struct seq_file *m, struct tick_device *td, int cpu)
189{ 191{
190 struct clock_event_device *dev = td->evtdev; 192 struct clock_event_device *dev = td->evtdev;
191 193
192 SEQ_printf(m, "\n"); 194 SEQ_printf(m, "\n");
193 SEQ_printf(m, "Tick Device: mode: %d\n", td->mode); 195 SEQ_printf(m, "Tick Device: mode: %d\n", td->mode);
196 if (cpu < 0)
197 SEQ_printf(m, "Broadcast device\n");
198 else
199 SEQ_printf(m, "Per CPU device: %d\n", cpu);
194 200
195 SEQ_printf(m, "Clock Event Device: "); 201 SEQ_printf(m, "Clock Event Device: ");
196 if (!dev) { 202 if (!dev) {
@@ -224,7 +230,7 @@ static void timer_list_show_tickdevices(struct seq_file *m)
224 int cpu; 230 int cpu;
225 231
226#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 232#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
227 print_tickdevice(m, tick_get_broadcast_device()); 233 print_tickdevice(m, tick_get_broadcast_device(), -1);
228 SEQ_printf(m, "tick_broadcast_mask: %08lx\n", 234 SEQ_printf(m, "tick_broadcast_mask: %08lx\n",
229 tick_get_broadcast_mask()->bits[0]); 235 tick_get_broadcast_mask()->bits[0]);
230#ifdef CONFIG_TICK_ONESHOT 236#ifdef CONFIG_TICK_ONESHOT
@@ -234,7 +240,7 @@ static void timer_list_show_tickdevices(struct seq_file *m)
234 SEQ_printf(m, "\n"); 240 SEQ_printf(m, "\n");
235#endif 241#endif
236 for_each_online_cpu(cpu) 242 for_each_online_cpu(cpu)
237 print_tickdevice(m, tick_get_device(cpu)); 243 print_tickdevice(m, tick_get_device(cpu), cpu);
238 SEQ_printf(m, "\n"); 244 SEQ_printf(m, "\n");
239} 245}
240#else 246#else
@@ -246,7 +252,7 @@ static int timer_list_show(struct seq_file *m, void *v)
246 u64 now = ktime_to_ns(ktime_get()); 252 u64 now = ktime_to_ns(ktime_get());
247 int cpu; 253 int cpu;
248 254
249 SEQ_printf(m, "Timer List Version: v0.3\n"); 255 SEQ_printf(m, "Timer List Version: v0.4\n");
250 SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES); 256 SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES);
251 SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now); 257 SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now);
252 258