diff options
Diffstat (limited to 'kernel/time')
-rw-r--r-- | kernel/time/clocksource.c | 3 | ||||
-rw-r--r-- | kernel/time/jiffies.c | 1 | ||||
-rw-r--r-- | kernel/time/ntp.c | 93 | ||||
-rw-r--r-- | kernel/time/tick-broadcast.c | 13 | ||||
-rw-r--r-- | kernel/time/tick-internal.h | 2 | ||||
-rw-r--r-- | kernel/time/tick-sched.c | 101 | ||||
-rw-r--r-- | kernel/time/timekeeping.c | 122 | ||||
-rw-r--r-- | kernel/time/timer_list.c | 20 |
8 files changed, 234 insertions, 121 deletions
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 093d4acf993b..9ed2eec97526 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c | |||
@@ -325,6 +325,9 @@ int clocksource_register(struct clocksource *c) | |||
325 | unsigned long flags; | 325 | unsigned long flags; |
326 | int ret; | 326 | int ret; |
327 | 327 | ||
328 | /* save mult_orig on registration */ | ||
329 | c->mult_orig = c->mult; | ||
330 | |||
328 | spin_lock_irqsave(&clocksource_lock, flags); | 331 | spin_lock_irqsave(&clocksource_lock, flags); |
329 | ret = clocksource_enqueue(c); | 332 | ret = clocksource_enqueue(c); |
330 | if (!ret) | 333 | if (!ret) |
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 4c256fdb8875..1ca99557e929 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c | |||
@@ -61,6 +61,7 @@ struct clocksource clocksource_jiffies = { | |||
61 | .read = jiffies_read, | 61 | .read = jiffies_read, |
62 | .mask = 0xffffffff, /*32bits*/ | 62 | .mask = 0xffffffff, /*32bits*/ |
63 | .mult = NSEC_PER_JIFFY << JIFFIES_SHIFT, /* details above */ | 63 | .mult = NSEC_PER_JIFFY << JIFFIES_SHIFT, /* details above */ |
64 | .mult_orig = NSEC_PER_JIFFY << JIFFIES_SHIFT, | ||
64 | .shift = JIFFIES_SHIFT, | 65 | .shift = JIFFIES_SHIFT, |
65 | }; | 66 | }; |
66 | 67 | ||
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 9c114b726ab3..8ff15e5d486b 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c | |||
@@ -10,13 +10,13 @@ | |||
10 | 10 | ||
11 | #include <linux/mm.h> | 11 | #include <linux/mm.h> |
12 | #include <linux/time.h> | 12 | #include <linux/time.h> |
13 | #include <linux/timer.h> | ||
14 | #include <linux/timex.h> | 13 | #include <linux/timex.h> |
15 | #include <linux/jiffies.h> | 14 | #include <linux/jiffies.h> |
16 | #include <linux/hrtimer.h> | 15 | #include <linux/hrtimer.h> |
17 | #include <linux/capability.h> | 16 | #include <linux/capability.h> |
18 | #include <linux/math64.h> | 17 | #include <linux/math64.h> |
19 | #include <linux/clocksource.h> | 18 | #include <linux/clocksource.h> |
19 | #include <linux/workqueue.h> | ||
20 | #include <asm/timex.h> | 20 | #include <asm/timex.h> |
21 | 21 | ||
22 | /* | 22 | /* |
@@ -217,11 +217,11 @@ void second_overflow(void) | |||
217 | /* Disable the cmos update - used by virtualization and embedded */ | 217 | /* Disable the cmos update - used by virtualization and embedded */ |
218 | int no_sync_cmos_clock __read_mostly; | 218 | int no_sync_cmos_clock __read_mostly; |
219 | 219 | ||
220 | static void sync_cmos_clock(unsigned long dummy); | 220 | static void sync_cmos_clock(struct work_struct *work); |
221 | 221 | ||
222 | static DEFINE_TIMER(sync_cmos_timer, sync_cmos_clock, 0, 0); | 222 | static DECLARE_DELAYED_WORK(sync_cmos_work, sync_cmos_clock); |
223 | 223 | ||
224 | static void sync_cmos_clock(unsigned long dummy) | 224 | static void sync_cmos_clock(struct work_struct *work) |
225 | { | 225 | { |
226 | struct timespec now, next; | 226 | struct timespec now, next; |
227 | int fail = 1; | 227 | int fail = 1; |
@@ -257,13 +257,13 @@ static void sync_cmos_clock(unsigned long dummy) | |||
257 | next.tv_sec++; | 257 | next.tv_sec++; |
258 | next.tv_nsec -= NSEC_PER_SEC; | 258 | next.tv_nsec -= NSEC_PER_SEC; |
259 | } | 259 | } |
260 | mod_timer(&sync_cmos_timer, jiffies + timespec_to_jiffies(&next)); | 260 | schedule_delayed_work(&sync_cmos_work, timespec_to_jiffies(&next)); |
261 | } | 261 | } |
262 | 262 | ||
263 | static void notify_cmos_timer(void) | 263 | static void notify_cmos_timer(void) |
264 | { | 264 | { |
265 | if (!no_sync_cmos_clock) | 265 | if (!no_sync_cmos_clock) |
266 | mod_timer(&sync_cmos_timer, jiffies + 1); | 266 | schedule_delayed_work(&sync_cmos_work, 0); |
267 | } | 267 | } |
268 | 268 | ||
269 | #else | 269 | #else |
@@ -276,38 +276,50 @@ static inline void notify_cmos_timer(void) { } | |||
276 | int do_adjtimex(struct timex *txc) | 276 | int do_adjtimex(struct timex *txc) |
277 | { | 277 | { |
278 | struct timespec ts; | 278 | struct timespec ts; |
279 | long save_adjust, sec; | ||
280 | int result; | 279 | int result; |
281 | 280 | ||
282 | /* In order to modify anything, you gotta be super-user! */ | 281 | /* Validate the data before disabling interrupts */ |
283 | if (txc->modes && !capable(CAP_SYS_TIME)) | 282 | if (txc->modes & ADJ_ADJTIME) { |
284 | return -EPERM; | ||
285 | |||
286 | /* Now we validate the data before disabling interrupts */ | ||
287 | |||
288 | if ((txc->modes & ADJ_OFFSET_SINGLESHOT) == ADJ_OFFSET_SINGLESHOT) { | ||
289 | /* singleshot must not be used with any other mode bits */ | 283 | /* singleshot must not be used with any other mode bits */ |
290 | if (txc->modes & ~ADJ_OFFSET_SS_READ) | 284 | if (!(txc->modes & ADJ_OFFSET_SINGLESHOT)) |
291 | return -EINVAL; | 285 | return -EINVAL; |
286 | if (!(txc->modes & ADJ_OFFSET_READONLY) && | ||
287 | !capable(CAP_SYS_TIME)) | ||
288 | return -EPERM; | ||
289 | } else { | ||
290 | /* In order to modify anything, you gotta be super-user! */ | ||
291 | if (txc->modes && !capable(CAP_SYS_TIME)) | ||
292 | return -EPERM; | ||
293 | |||
294 | /* if the quartz is off by more than 10% something is VERY wrong! */ | ||
295 | if (txc->modes & ADJ_TICK && | ||
296 | (txc->tick < 900000/USER_HZ || | ||
297 | txc->tick > 1100000/USER_HZ)) | ||
298 | return -EINVAL; | ||
299 | |||
300 | if (txc->modes & ADJ_STATUS && time_state != TIME_OK) | ||
301 | hrtimer_cancel(&leap_timer); | ||
292 | } | 302 | } |
293 | 303 | ||
294 | /* if the quartz is off by more than 10% something is VERY wrong ! */ | ||
295 | if (txc->modes & ADJ_TICK) | ||
296 | if (txc->tick < 900000/USER_HZ || | ||
297 | txc->tick > 1100000/USER_HZ) | ||
298 | return -EINVAL; | ||
299 | |||
300 | if (time_state != TIME_OK && txc->modes & ADJ_STATUS) | ||
301 | hrtimer_cancel(&leap_timer); | ||
302 | getnstimeofday(&ts); | 304 | getnstimeofday(&ts); |
303 | 305 | ||
304 | write_seqlock_irq(&xtime_lock); | 306 | write_seqlock_irq(&xtime_lock); |
305 | 307 | ||
306 | /* Save for later - semantics of adjtime is to return old value */ | ||
307 | save_adjust = time_adjust; | ||
308 | |||
309 | /* If there are input parameters, then process them */ | 308 | /* If there are input parameters, then process them */ |
309 | if (txc->modes & ADJ_ADJTIME) { | ||
310 | long save_adjust = time_adjust; | ||
311 | |||
312 | if (!(txc->modes & ADJ_OFFSET_READONLY)) { | ||
313 | /* adjtime() is independent from ntp_adjtime() */ | ||
314 | time_adjust = txc->offset; | ||
315 | ntp_update_frequency(); | ||
316 | } | ||
317 | txc->offset = save_adjust; | ||
318 | goto adj_done; | ||
319 | } | ||
310 | if (txc->modes) { | 320 | if (txc->modes) { |
321 | long sec; | ||
322 | |||
311 | if (txc->modes & ADJ_STATUS) { | 323 | if (txc->modes & ADJ_STATUS) { |
312 | if ((time_status & STA_PLL) && | 324 | if ((time_status & STA_PLL) && |
313 | !(txc->status & STA_PLL)) { | 325 | !(txc->status & STA_PLL)) { |
@@ -374,13 +386,8 @@ int do_adjtimex(struct timex *txc) | |||
374 | if (txc->modes & ADJ_TAI && txc->constant > 0) | 386 | if (txc->modes & ADJ_TAI && txc->constant > 0) |
375 | time_tai = txc->constant; | 387 | time_tai = txc->constant; |
376 | 388 | ||
377 | if (txc->modes & ADJ_OFFSET) { | 389 | if (txc->modes & ADJ_OFFSET) |
378 | if (txc->modes == ADJ_OFFSET_SINGLESHOT) | 390 | ntp_update_offset(txc->offset); |
379 | /* adjtime() is independent from ntp_adjtime() */ | ||
380 | time_adjust = txc->offset; | ||
381 | else | ||
382 | ntp_update_offset(txc->offset); | ||
383 | } | ||
384 | if (txc->modes & ADJ_TICK) | 391 | if (txc->modes & ADJ_TICK) |
385 | tick_usec = txc->tick; | 392 | tick_usec = txc->tick; |
386 | 393 | ||
@@ -388,22 +395,18 @@ int do_adjtimex(struct timex *txc) | |||
388 | ntp_update_frequency(); | 395 | ntp_update_frequency(); |
389 | } | 396 | } |
390 | 397 | ||
398 | txc->offset = shift_right(time_offset * NTP_INTERVAL_FREQ, | ||
399 | NTP_SCALE_SHIFT); | ||
400 | if (!(time_status & STA_NANO)) | ||
401 | txc->offset /= NSEC_PER_USEC; | ||
402 | |||
403 | adj_done: | ||
391 | result = time_state; /* mostly `TIME_OK' */ | 404 | result = time_state; /* mostly `TIME_OK' */ |
392 | if (time_status & (STA_UNSYNC|STA_CLOCKERR)) | 405 | if (time_status & (STA_UNSYNC|STA_CLOCKERR)) |
393 | result = TIME_ERROR; | 406 | result = TIME_ERROR; |
394 | 407 | ||
395 | if ((txc->modes == ADJ_OFFSET_SINGLESHOT) || | 408 | txc->freq = shift_right((time_freq >> PPM_SCALE_INV_SHIFT) * |
396 | (txc->modes == ADJ_OFFSET_SS_READ)) | 409 | (s64)PPM_SCALE_INV, NTP_SCALE_SHIFT); |
397 | txc->offset = save_adjust; | ||
398 | else { | ||
399 | txc->offset = shift_right(time_offset * NTP_INTERVAL_FREQ, | ||
400 | NTP_SCALE_SHIFT); | ||
401 | if (!(time_status & STA_NANO)) | ||
402 | txc->offset /= NSEC_PER_USEC; | ||
403 | } | ||
404 | txc->freq = shift_right((s32)(time_freq >> PPM_SCALE_INV_SHIFT) * | ||
405 | (s64)PPM_SCALE_INV, | ||
406 | NTP_SCALE_SHIFT); | ||
407 | txc->maxerror = time_maxerror; | 410 | txc->maxerror = time_maxerror; |
408 | txc->esterror = time_esterror; | 411 | txc->esterror = time_esterror; |
409 | txc->status = time_status; | 412 | txc->status = time_status; |
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index cb01cd8f919b..f98a1b7b16e9 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c | |||
@@ -384,6 +384,19 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc) | |||
384 | } | 384 | } |
385 | 385 | ||
386 | /* | 386 | /* |
387 | * Called from irq_enter() when idle was interrupted to reenable the | ||
388 | * per cpu device. | ||
389 | */ | ||
390 | void tick_check_oneshot_broadcast(int cpu) | ||
391 | { | ||
392 | if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) { | ||
393 | struct tick_device *td = &per_cpu(tick_cpu_device, cpu); | ||
394 | |||
395 | clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT); | ||
396 | } | ||
397 | } | ||
398 | |||
399 | /* | ||
387 | * Handle oneshot mode broadcasting | 400 | * Handle oneshot mode broadcasting |
388 | */ | 401 | */ |
389 | static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) | 402 | static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) |
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 469248782c23..b1c05bf75ee0 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h | |||
@@ -36,6 +36,7 @@ extern void tick_broadcast_switch_to_oneshot(void); | |||
36 | extern void tick_shutdown_broadcast_oneshot(unsigned int *cpup); | 36 | extern void tick_shutdown_broadcast_oneshot(unsigned int *cpup); |
37 | extern int tick_resume_broadcast_oneshot(struct clock_event_device *bc); | 37 | extern int tick_resume_broadcast_oneshot(struct clock_event_device *bc); |
38 | extern int tick_broadcast_oneshot_active(void); | 38 | extern int tick_broadcast_oneshot_active(void); |
39 | extern void tick_check_oneshot_broadcast(int cpu); | ||
39 | # else /* BROADCAST */ | 40 | # else /* BROADCAST */ |
40 | static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) | 41 | static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) |
41 | { | 42 | { |
@@ -45,6 +46,7 @@ static inline void tick_broadcast_oneshot_control(unsigned long reason) { } | |||
45 | static inline void tick_broadcast_switch_to_oneshot(void) { } | 46 | static inline void tick_broadcast_switch_to_oneshot(void) { } |
46 | static inline void tick_shutdown_broadcast_oneshot(unsigned int *cpup) { } | 47 | static inline void tick_shutdown_broadcast_oneshot(unsigned int *cpup) { } |
47 | static inline int tick_broadcast_oneshot_active(void) { return 0; } | 48 | static inline int tick_broadcast_oneshot_active(void) { return 0; } |
49 | static inline void tick_check_oneshot_broadcast(int cpu) { } | ||
48 | # endif /* !BROADCAST */ | 50 | # endif /* !BROADCAST */ |
49 | 51 | ||
50 | #else /* !ONESHOT */ | 52 | #else /* !ONESHOT */ |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index a547be11cf97..5bbb1044f847 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
@@ -155,7 +155,7 @@ void tick_nohz_update_jiffies(void) | |||
155 | touch_softlockup_watchdog(); | 155 | touch_softlockup_watchdog(); |
156 | } | 156 | } |
157 | 157 | ||
158 | void tick_nohz_stop_idle(int cpu) | 158 | static void tick_nohz_stop_idle(int cpu) |
159 | { | 159 | { |
160 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | 160 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); |
161 | 161 | ||
@@ -377,6 +377,32 @@ ktime_t tick_nohz_get_sleep_length(void) | |||
377 | return ts->sleep_length; | 377 | return ts->sleep_length; |
378 | } | 378 | } |
379 | 379 | ||
380 | static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) | ||
381 | { | ||
382 | hrtimer_cancel(&ts->sched_timer); | ||
383 | hrtimer_set_expires(&ts->sched_timer, ts->idle_tick); | ||
384 | |||
385 | while (1) { | ||
386 | /* Forward the time to expire in the future */ | ||
387 | hrtimer_forward(&ts->sched_timer, now, tick_period); | ||
388 | |||
389 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { | ||
390 | hrtimer_start_expires(&ts->sched_timer, | ||
391 | HRTIMER_MODE_ABS); | ||
392 | /* Check, if the timer was already in the past */ | ||
393 | if (hrtimer_active(&ts->sched_timer)) | ||
394 | break; | ||
395 | } else { | ||
396 | if (!tick_program_event( | ||
397 | hrtimer_get_expires(&ts->sched_timer), 0)) | ||
398 | break; | ||
399 | } | ||
400 | /* Update jiffies and reread time */ | ||
401 | tick_do_update_jiffies64(now); | ||
402 | now = ktime_get(); | ||
403 | } | ||
404 | } | ||
405 | |||
380 | /** | 406 | /** |
381 | * tick_nohz_restart_sched_tick - restart the idle tick from the idle task | 407 | * tick_nohz_restart_sched_tick - restart the idle tick from the idle task |
382 | * | 408 | * |
@@ -430,28 +456,9 @@ void tick_nohz_restart_sched_tick(void) | |||
430 | */ | 456 | */ |
431 | ts->tick_stopped = 0; | 457 | ts->tick_stopped = 0; |
432 | ts->idle_exittime = now; | 458 | ts->idle_exittime = now; |
433 | hrtimer_cancel(&ts->sched_timer); | ||
434 | hrtimer_set_expires(&ts->sched_timer, ts->idle_tick); | ||
435 | 459 | ||
436 | while (1) { | 460 | tick_nohz_restart(ts, now); |
437 | /* Forward the time to expire in the future */ | ||
438 | hrtimer_forward(&ts->sched_timer, now, tick_period); | ||
439 | 461 | ||
440 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { | ||
441 | hrtimer_start_expires(&ts->sched_timer, | ||
442 | HRTIMER_MODE_ABS); | ||
443 | /* Check, if the timer was already in the past */ | ||
444 | if (hrtimer_active(&ts->sched_timer)) | ||
445 | break; | ||
446 | } else { | ||
447 | if (!tick_program_event( | ||
448 | hrtimer_get_expires(&ts->sched_timer), 0)) | ||
449 | break; | ||
450 | } | ||
451 | /* Update jiffies and reread time */ | ||
452 | tick_do_update_jiffies64(now); | ||
453 | now = ktime_get(); | ||
454 | } | ||
455 | local_irq_enable(); | 462 | local_irq_enable(); |
456 | } | 463 | } |
457 | 464 | ||
@@ -503,10 +510,6 @@ static void tick_nohz_handler(struct clock_event_device *dev) | |||
503 | update_process_times(user_mode(regs)); | 510 | update_process_times(user_mode(regs)); |
504 | profile_tick(CPU_PROFILING); | 511 | profile_tick(CPU_PROFILING); |
505 | 512 | ||
506 | /* Do not restart, when we are in the idle loop */ | ||
507 | if (ts->tick_stopped) | ||
508 | return; | ||
509 | |||
510 | while (tick_nohz_reprogram(ts, now)) { | 513 | while (tick_nohz_reprogram(ts, now)) { |
511 | now = ktime_get(); | 514 | now = ktime_get(); |
512 | tick_do_update_jiffies64(now); | 515 | tick_do_update_jiffies64(now); |
@@ -552,6 +555,37 @@ static void tick_nohz_switch_to_nohz(void) | |||
552 | smp_processor_id()); | 555 | smp_processor_id()); |
553 | } | 556 | } |
554 | 557 | ||
558 | /* | ||
559 | * When NOHZ is enabled and the tick is stopped, we need to kick the | ||
560 | * tick timer from irq_enter() so that the jiffies update is kept | ||
561 | * alive during long running softirqs. That's ugly as hell, but | ||
562 | * correctness is key even if we need to fix the offending softirq in | ||
563 | * the first place. | ||
564 | * | ||
565 | * Note, this is different to tick_nohz_restart. We just kick the | ||
566 | * timer and do not touch the other magic bits which need to be done | ||
567 | * when idle is left. | ||
568 | */ | ||
569 | static void tick_nohz_kick_tick(int cpu) | ||
570 | { | ||
571 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | ||
572 | ktime_t delta, now; | ||
573 | |||
574 | if (!ts->tick_stopped) | ||
575 | return; | ||
576 | |||
577 | /* | ||
578 | * Do not touch the tick device, when the next expiry is either | ||
579 | * already reached or less/equal than the tick period. | ||
580 | */ | ||
581 | now = ktime_get(); | ||
582 | delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now); | ||
583 | if (delta.tv64 <= tick_period.tv64) | ||
584 | return; | ||
585 | |||
586 | tick_nohz_restart(ts, now); | ||
587 | } | ||
588 | |||
555 | #else | 589 | #else |
556 | 590 | ||
557 | static inline void tick_nohz_switch_to_nohz(void) { } | 591 | static inline void tick_nohz_switch_to_nohz(void) { } |
@@ -559,6 +593,19 @@ static inline void tick_nohz_switch_to_nohz(void) { } | |||
559 | #endif /* NO_HZ */ | 593 | #endif /* NO_HZ */ |
560 | 594 | ||
561 | /* | 595 | /* |
596 | * Called from irq_enter to notify about the possible interruption of idle() | ||
597 | */ | ||
598 | void tick_check_idle(int cpu) | ||
599 | { | ||
600 | tick_check_oneshot_broadcast(cpu); | ||
601 | #ifdef CONFIG_NO_HZ | ||
602 | tick_nohz_stop_idle(cpu); | ||
603 | tick_nohz_update_jiffies(); | ||
604 | tick_nohz_kick_tick(cpu); | ||
605 | #endif | ||
606 | } | ||
607 | |||
608 | /* | ||
562 | * High resolution timer specific code | 609 | * High resolution timer specific code |
563 | */ | 610 | */ |
564 | #ifdef CONFIG_HIGH_RES_TIMERS | 611 | #ifdef CONFIG_HIGH_RES_TIMERS |
@@ -611,10 +658,6 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) | |||
611 | profile_tick(CPU_PROFILING); | 658 | profile_tick(CPU_PROFILING); |
612 | } | 659 | } |
613 | 660 | ||
614 | /* Do not restart, when we are in the idle loop */ | ||
615 | if (ts->tick_stopped) | ||
616 | return HRTIMER_NORESTART; | ||
617 | |||
618 | hrtimer_forward(timer, now, tick_period); | 661 | hrtimer_forward(timer, now, tick_period); |
619 | 662 | ||
620 | return HRTIMER_RESTART; | 663 | return HRTIMER_RESTART; |
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index e91c29f961c9..e7acfb482a68 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
@@ -58,27 +58,26 @@ struct clocksource *clock; | |||
58 | 58 | ||
59 | #ifdef CONFIG_GENERIC_TIME | 59 | #ifdef CONFIG_GENERIC_TIME |
60 | /** | 60 | /** |
61 | * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook | 61 | * clocksource_forward_now - update clock to the current time |
62 | * | 62 | * |
63 | * private function, must hold xtime_lock lock when being | 63 | * Forward the current clock to update its state since the last call to |
64 | * called. Returns the number of nanoseconds since the | 64 | * update_wall_time(). This is useful before significant clock changes, |
65 | * last call to update_wall_time() (adjusted by NTP scaling) | 65 | * as it avoids having to deal with this time offset explicitly. |
66 | */ | 66 | */ |
67 | static inline s64 __get_nsec_offset(void) | 67 | static void clocksource_forward_now(void) |
68 | { | 68 | { |
69 | cycle_t cycle_now, cycle_delta; | 69 | cycle_t cycle_now, cycle_delta; |
70 | s64 ns_offset; | 70 | s64 nsec; |
71 | 71 | ||
72 | /* read clocksource: */ | ||
73 | cycle_now = clocksource_read(clock); | 72 | cycle_now = clocksource_read(clock); |
74 | |||
75 | /* calculate the delta since the last update_wall_time: */ | ||
76 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | 73 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; |
74 | clock->cycle_last = cycle_now; | ||
77 | 75 | ||
78 | /* convert to nanoseconds: */ | 76 | nsec = cyc2ns(clock, cycle_delta); |
79 | ns_offset = cyc2ns(clock, cycle_delta); | 77 | timespec_add_ns(&xtime, nsec); |
80 | 78 | ||
81 | return ns_offset; | 79 | nsec = ((s64)cycle_delta * clock->mult_orig) >> clock->shift; |
80 | clock->raw_time.tv_nsec += nsec; | ||
82 | } | 81 | } |
83 | 82 | ||
84 | /** | 83 | /** |
@@ -89,6 +88,7 @@ static inline s64 __get_nsec_offset(void) | |||
89 | */ | 88 | */ |
90 | void getnstimeofday(struct timespec *ts) | 89 | void getnstimeofday(struct timespec *ts) |
91 | { | 90 | { |
91 | cycle_t cycle_now, cycle_delta; | ||
92 | unsigned long seq; | 92 | unsigned long seq; |
93 | s64 nsecs; | 93 | s64 nsecs; |
94 | 94 | ||
@@ -96,7 +96,15 @@ void getnstimeofday(struct timespec *ts) | |||
96 | seq = read_seqbegin(&xtime_lock); | 96 | seq = read_seqbegin(&xtime_lock); |
97 | 97 | ||
98 | *ts = xtime; | 98 | *ts = xtime; |
99 | nsecs = __get_nsec_offset(); | 99 | |
100 | /* read clocksource: */ | ||
101 | cycle_now = clocksource_read(clock); | ||
102 | |||
103 | /* calculate the delta since the last update_wall_time: */ | ||
104 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | ||
105 | |||
106 | /* convert to nanoseconds: */ | ||
107 | nsecs = cyc2ns(clock, cycle_delta); | ||
100 | 108 | ||
101 | } while (read_seqretry(&xtime_lock, seq)); | 109 | } while (read_seqretry(&xtime_lock, seq)); |
102 | 110 | ||
@@ -129,22 +137,22 @@ EXPORT_SYMBOL(do_gettimeofday); | |||
129 | */ | 137 | */ |
130 | int do_settimeofday(struct timespec *tv) | 138 | int do_settimeofday(struct timespec *tv) |
131 | { | 139 | { |
140 | struct timespec ts_delta; | ||
132 | unsigned long flags; | 141 | unsigned long flags; |
133 | time_t wtm_sec, sec = tv->tv_sec; | ||
134 | long wtm_nsec, nsec = tv->tv_nsec; | ||
135 | 142 | ||
136 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | 143 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) |
137 | return -EINVAL; | 144 | return -EINVAL; |
138 | 145 | ||
139 | write_seqlock_irqsave(&xtime_lock, flags); | 146 | write_seqlock_irqsave(&xtime_lock, flags); |
140 | 147 | ||
141 | nsec -= __get_nsec_offset(); | 148 | clocksource_forward_now(); |
149 | |||
150 | ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec; | ||
151 | ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec; | ||
152 | wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta); | ||
142 | 153 | ||
143 | wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec); | 154 | xtime = *tv; |
144 | wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec); | ||
145 | 155 | ||
146 | set_normalized_timespec(&xtime, sec, nsec); | ||
147 | set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec); | ||
148 | update_xtime_cache(0); | 156 | update_xtime_cache(0); |
149 | 157 | ||
150 | clock->error = 0; | 158 | clock->error = 0; |
@@ -170,22 +178,19 @@ EXPORT_SYMBOL(do_settimeofday); | |||
170 | static void change_clocksource(void) | 178 | static void change_clocksource(void) |
171 | { | 179 | { |
172 | struct clocksource *new; | 180 | struct clocksource *new; |
173 | cycle_t now; | ||
174 | u64 nsec; | ||
175 | 181 | ||
176 | new = clocksource_get_next(); | 182 | new = clocksource_get_next(); |
177 | 183 | ||
178 | if (clock == new) | 184 | if (clock == new) |
179 | return; | 185 | return; |
180 | 186 | ||
181 | new->cycle_last = 0; | 187 | clocksource_forward_now(); |
182 | now = clocksource_read(new); | ||
183 | nsec = __get_nsec_offset(); | ||
184 | timespec_add_ns(&xtime, nsec); | ||
185 | 188 | ||
186 | clock = new; | 189 | new->raw_time = clock->raw_time; |
187 | clock->cycle_last = now; | ||
188 | 190 | ||
191 | clock = new; | ||
192 | clock->cycle_last = 0; | ||
193 | clock->cycle_last = clocksource_read(new); | ||
189 | clock->error = 0; | 194 | clock->error = 0; |
190 | clock->xtime_nsec = 0; | 195 | clock->xtime_nsec = 0; |
191 | clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); | 196 | clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); |
@@ -200,11 +205,44 @@ static void change_clocksource(void) | |||
200 | */ | 205 | */ |
201 | } | 206 | } |
202 | #else | 207 | #else |
208 | static inline void clocksource_forward_now(void) { } | ||
203 | static inline void change_clocksource(void) { } | 209 | static inline void change_clocksource(void) { } |
204 | static inline s64 __get_nsec_offset(void) { return 0; } | ||
205 | #endif | 210 | #endif |
206 | 211 | ||
207 | /** | 212 | /** |
213 | * getrawmonotonic - Returns the raw monotonic time in a timespec | ||
214 | * @ts: pointer to the timespec to be set | ||
215 | * | ||
216 | * Returns the raw monotonic time (completely un-modified by ntp) | ||
217 | */ | ||
218 | void getrawmonotonic(struct timespec *ts) | ||
219 | { | ||
220 | unsigned long seq; | ||
221 | s64 nsecs; | ||
222 | cycle_t cycle_now, cycle_delta; | ||
223 | |||
224 | do { | ||
225 | seq = read_seqbegin(&xtime_lock); | ||
226 | |||
227 | /* read clocksource: */ | ||
228 | cycle_now = clocksource_read(clock); | ||
229 | |||
230 | /* calculate the delta since the last update_wall_time: */ | ||
231 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | ||
232 | |||
233 | /* convert to nanoseconds: */ | ||
234 | nsecs = ((s64)cycle_delta * clock->mult_orig) >> clock->shift; | ||
235 | |||
236 | *ts = clock->raw_time; | ||
237 | |||
238 | } while (read_seqretry(&xtime_lock, seq)); | ||
239 | |||
240 | timespec_add_ns(ts, nsecs); | ||
241 | } | ||
242 | EXPORT_SYMBOL(getrawmonotonic); | ||
243 | |||
244 | |||
245 | /** | ||
208 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres | 246 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres |
209 | */ | 247 | */ |
210 | int timekeeping_valid_for_hres(void) | 248 | int timekeeping_valid_for_hres(void) |
@@ -265,8 +303,6 @@ void __init timekeeping_init(void) | |||
265 | static int timekeeping_suspended; | 303 | static int timekeeping_suspended; |
266 | /* time in seconds when suspend began */ | 304 | /* time in seconds when suspend began */ |
267 | static unsigned long timekeeping_suspend_time; | 305 | static unsigned long timekeeping_suspend_time; |
268 | /* xtime offset when we went into suspend */ | ||
269 | static s64 timekeeping_suspend_nsecs; | ||
270 | 306 | ||
271 | /** | 307 | /** |
272 | * timekeeping_resume - Resumes the generic timekeeping subsystem. | 308 | * timekeeping_resume - Resumes the generic timekeeping subsystem. |
@@ -292,8 +328,6 @@ static int timekeeping_resume(struct sys_device *dev) | |||
292 | wall_to_monotonic.tv_sec -= sleep_length; | 328 | wall_to_monotonic.tv_sec -= sleep_length; |
293 | total_sleep_time += sleep_length; | 329 | total_sleep_time += sleep_length; |
294 | } | 330 | } |
295 | /* Make sure that we have the correct xtime reference */ | ||
296 | timespec_add_ns(&xtime, timekeeping_suspend_nsecs); | ||
297 | update_xtime_cache(0); | 331 | update_xtime_cache(0); |
298 | /* re-base the last cycle value */ | 332 | /* re-base the last cycle value */ |
299 | clock->cycle_last = 0; | 333 | clock->cycle_last = 0; |
@@ -319,8 +353,7 @@ static int timekeeping_suspend(struct sys_device *dev, pm_message_t state) | |||
319 | timekeeping_suspend_time = read_persistent_clock(); | 353 | timekeeping_suspend_time = read_persistent_clock(); |
320 | 354 | ||
321 | write_seqlock_irqsave(&xtime_lock, flags); | 355 | write_seqlock_irqsave(&xtime_lock, flags); |
322 | /* Get the current xtime offset */ | 356 | clocksource_forward_now(); |
323 | timekeeping_suspend_nsecs = __get_nsec_offset(); | ||
324 | timekeeping_suspended = 1; | 357 | timekeeping_suspended = 1; |
325 | write_sequnlock_irqrestore(&xtime_lock, flags); | 358 | write_sequnlock_irqrestore(&xtime_lock, flags); |
326 | 359 | ||
@@ -454,23 +487,29 @@ void update_wall_time(void) | |||
454 | #else | 487 | #else |
455 | offset = clock->cycle_interval; | 488 | offset = clock->cycle_interval; |
456 | #endif | 489 | #endif |
457 | clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift; | 490 | clock->xtime_nsec = (s64)xtime.tv_nsec << clock->shift; |
458 | 491 | ||
459 | /* normally this loop will run just once, however in the | 492 | /* normally this loop will run just once, however in the |
460 | * case of lost or late ticks, it will accumulate correctly. | 493 | * case of lost or late ticks, it will accumulate correctly. |
461 | */ | 494 | */ |
462 | while (offset >= clock->cycle_interval) { | 495 | while (offset >= clock->cycle_interval) { |
463 | /* accumulate one interval */ | 496 | /* accumulate one interval */ |
464 | clock->xtime_nsec += clock->xtime_interval; | ||
465 | clock->cycle_last += clock->cycle_interval; | ||
466 | offset -= clock->cycle_interval; | 497 | offset -= clock->cycle_interval; |
498 | clock->cycle_last += clock->cycle_interval; | ||
467 | 499 | ||
500 | clock->xtime_nsec += clock->xtime_interval; | ||
468 | if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { | 501 | if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { |
469 | clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; | 502 | clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; |
470 | xtime.tv_sec++; | 503 | xtime.tv_sec++; |
471 | second_overflow(); | 504 | second_overflow(); |
472 | } | 505 | } |
473 | 506 | ||
507 | clock->raw_time.tv_nsec += clock->raw_interval; | ||
508 | if (clock->raw_time.tv_nsec >= NSEC_PER_SEC) { | ||
509 | clock->raw_time.tv_nsec -= NSEC_PER_SEC; | ||
510 | clock->raw_time.tv_sec++; | ||
511 | } | ||
512 | |||
474 | /* accumulate error between NTP and clock interval */ | 513 | /* accumulate error between NTP and clock interval */ |
475 | clock->error += tick_length; | 514 | clock->error += tick_length; |
476 | clock->error -= clock->xtime_interval << (NTP_SCALE_SHIFT - clock->shift); | 515 | clock->error -= clock->xtime_interval << (NTP_SCALE_SHIFT - clock->shift); |
@@ -479,9 +518,12 @@ void update_wall_time(void) | |||
479 | /* correct the clock when NTP error is too big */ | 518 | /* correct the clock when NTP error is too big */ |
480 | clocksource_adjust(offset); | 519 | clocksource_adjust(offset); |
481 | 520 | ||
482 | /* store full nanoseconds into xtime */ | 521 | /* store full nanoseconds into xtime after rounding it up and |
483 | xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift; | 522 | * add the remainder to the error difference. |
523 | */ | ||
524 | xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1; | ||
484 | clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; | 525 | clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; |
526 | clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift); | ||
485 | 527 | ||
486 | update_xtime_cache(cyc2ns(clock, offset)); | 528 | update_xtime_cache(cyc2ns(clock, offset)); |
487 | 529 | ||
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 122ee751d2d1..a999b92a1277 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c | |||
@@ -47,13 +47,14 @@ static void print_name_offset(struct seq_file *m, void *sym) | |||
47 | } | 47 | } |
48 | 48 | ||
49 | static void | 49 | static void |
50 | print_timer(struct seq_file *m, struct hrtimer *timer, int idx, u64 now) | 50 | print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer, |
51 | int idx, u64 now) | ||
51 | { | 52 | { |
52 | #ifdef CONFIG_TIMER_STATS | 53 | #ifdef CONFIG_TIMER_STATS |
53 | char tmp[TASK_COMM_LEN + 1]; | 54 | char tmp[TASK_COMM_LEN + 1]; |
54 | #endif | 55 | #endif |
55 | SEQ_printf(m, " #%d: ", idx); | 56 | SEQ_printf(m, " #%d: ", idx); |
56 | print_name_offset(m, timer); | 57 | print_name_offset(m, taddr); |
57 | SEQ_printf(m, ", "); | 58 | SEQ_printf(m, ", "); |
58 | print_name_offset(m, timer->function); | 59 | print_name_offset(m, timer->function); |
59 | SEQ_printf(m, ", S:%02lx", timer->state); | 60 | SEQ_printf(m, ", S:%02lx", timer->state); |
@@ -101,7 +102,7 @@ next_one: | |||
101 | tmp = *timer; | 102 | tmp = *timer; |
102 | spin_unlock_irqrestore(&base->cpu_base->lock, flags); | 103 | spin_unlock_irqrestore(&base->cpu_base->lock, flags); |
103 | 104 | ||
104 | print_timer(m, &tmp, i, now); | 105 | print_timer(m, timer, &tmp, i, now); |
105 | next++; | 106 | next++; |
106 | goto next_one; | 107 | goto next_one; |
107 | } | 108 | } |
@@ -111,6 +112,7 @@ next_one: | |||
111 | static void | 112 | static void |
112 | print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now) | 113 | print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now) |
113 | { | 114 | { |
115 | SEQ_printf(m, " .base: %p\n", base); | ||
114 | SEQ_printf(m, " .index: %d\n", | 116 | SEQ_printf(m, " .index: %d\n", |
115 | base->index); | 117 | base->index); |
116 | SEQ_printf(m, " .resolution: %Lu nsecs\n", | 118 | SEQ_printf(m, " .resolution: %Lu nsecs\n", |
@@ -185,12 +187,16 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now) | |||
185 | 187 | ||
186 | #ifdef CONFIG_GENERIC_CLOCKEVENTS | 188 | #ifdef CONFIG_GENERIC_CLOCKEVENTS |
187 | static void | 189 | static void |
188 | print_tickdevice(struct seq_file *m, struct tick_device *td) | 190 | print_tickdevice(struct seq_file *m, struct tick_device *td, int cpu) |
189 | { | 191 | { |
190 | struct clock_event_device *dev = td->evtdev; | 192 | struct clock_event_device *dev = td->evtdev; |
191 | 193 | ||
192 | SEQ_printf(m, "\n"); | 194 | SEQ_printf(m, "\n"); |
193 | SEQ_printf(m, "Tick Device: mode: %d\n", td->mode); | 195 | SEQ_printf(m, "Tick Device: mode: %d\n", td->mode); |
196 | if (cpu < 0) | ||
197 | SEQ_printf(m, "Broadcast device\n"); | ||
198 | else | ||
199 | SEQ_printf(m, "Per CPU device: %d\n", cpu); | ||
194 | 200 | ||
195 | SEQ_printf(m, "Clock Event Device: "); | 201 | SEQ_printf(m, "Clock Event Device: "); |
196 | if (!dev) { | 202 | if (!dev) { |
@@ -224,7 +230,7 @@ static void timer_list_show_tickdevices(struct seq_file *m) | |||
224 | int cpu; | 230 | int cpu; |
225 | 231 | ||
226 | #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST | 232 | #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST |
227 | print_tickdevice(m, tick_get_broadcast_device()); | 233 | print_tickdevice(m, tick_get_broadcast_device(), -1); |
228 | SEQ_printf(m, "tick_broadcast_mask: %08lx\n", | 234 | SEQ_printf(m, "tick_broadcast_mask: %08lx\n", |
229 | tick_get_broadcast_mask()->bits[0]); | 235 | tick_get_broadcast_mask()->bits[0]); |
230 | #ifdef CONFIG_TICK_ONESHOT | 236 | #ifdef CONFIG_TICK_ONESHOT |
@@ -234,7 +240,7 @@ static void timer_list_show_tickdevices(struct seq_file *m) | |||
234 | SEQ_printf(m, "\n"); | 240 | SEQ_printf(m, "\n"); |
235 | #endif | 241 | #endif |
236 | for_each_online_cpu(cpu) | 242 | for_each_online_cpu(cpu) |
237 | print_tickdevice(m, tick_get_device(cpu)); | 243 | print_tickdevice(m, tick_get_device(cpu), cpu); |
238 | SEQ_printf(m, "\n"); | 244 | SEQ_printf(m, "\n"); |
239 | } | 245 | } |
240 | #else | 246 | #else |
@@ -246,7 +252,7 @@ static int timer_list_show(struct seq_file *m, void *v) | |||
246 | u64 now = ktime_to_ns(ktime_get()); | 252 | u64 now = ktime_to_ns(ktime_get()); |
247 | int cpu; | 253 | int cpu; |
248 | 254 | ||
249 | SEQ_printf(m, "Timer List Version: v0.3\n"); | 255 | SEQ_printf(m, "Timer List Version: v0.4\n"); |
250 | SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES); | 256 | SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES); |
251 | SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now); | 257 | SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now); |
252 | 258 | ||