diff options
Diffstat (limited to 'kernel/time/timekeeping.c')
-rw-r--r-- | kernel/time/timekeeping.c | 122 |
1 files changed, 82 insertions, 40 deletions
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index e91c29f961c9..e7acfb482a68 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
@@ -58,27 +58,26 @@ struct clocksource *clock; | |||
58 | 58 | ||
59 | #ifdef CONFIG_GENERIC_TIME | 59 | #ifdef CONFIG_GENERIC_TIME |
60 | /** | 60 | /** |
61 | * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook | 61 | * clocksource_forward_now - update clock to the current time |
62 | * | 62 | * |
63 | * private function, must hold xtime_lock lock when being | 63 | * Forward the current clock to update its state since the last call to |
64 | * called. Returns the number of nanoseconds since the | 64 | * update_wall_time(). This is useful before significant clock changes, |
65 | * last call to update_wall_time() (adjusted by NTP scaling) | 65 | * as it avoids having to deal with this time offset explicitly. |
66 | */ | 66 | */ |
67 | static inline s64 __get_nsec_offset(void) | 67 | static void clocksource_forward_now(void) |
68 | { | 68 | { |
69 | cycle_t cycle_now, cycle_delta; | 69 | cycle_t cycle_now, cycle_delta; |
70 | s64 ns_offset; | 70 | s64 nsec; |
71 | 71 | ||
72 | /* read clocksource: */ | ||
73 | cycle_now = clocksource_read(clock); | 72 | cycle_now = clocksource_read(clock); |
74 | |||
75 | /* calculate the delta since the last update_wall_time: */ | ||
76 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | 73 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; |
74 | clock->cycle_last = cycle_now; | ||
77 | 75 | ||
78 | /* convert to nanoseconds: */ | 76 | nsec = cyc2ns(clock, cycle_delta); |
79 | ns_offset = cyc2ns(clock, cycle_delta); | 77 | timespec_add_ns(&xtime, nsec); |
80 | 78 | ||
81 | return ns_offset; | 79 | nsec = ((s64)cycle_delta * clock->mult_orig) >> clock->shift; |
80 | clock->raw_time.tv_nsec += nsec; | ||
82 | } | 81 | } |
83 | 82 | ||
84 | /** | 83 | /** |
@@ -89,6 +88,7 @@ static inline s64 __get_nsec_offset(void) | |||
89 | */ | 88 | */ |
90 | void getnstimeofday(struct timespec *ts) | 89 | void getnstimeofday(struct timespec *ts) |
91 | { | 90 | { |
91 | cycle_t cycle_now, cycle_delta; | ||
92 | unsigned long seq; | 92 | unsigned long seq; |
93 | s64 nsecs; | 93 | s64 nsecs; |
94 | 94 | ||
@@ -96,7 +96,15 @@ void getnstimeofday(struct timespec *ts) | |||
96 | seq = read_seqbegin(&xtime_lock); | 96 | seq = read_seqbegin(&xtime_lock); |
97 | 97 | ||
98 | *ts = xtime; | 98 | *ts = xtime; |
99 | nsecs = __get_nsec_offset(); | 99 | |
100 | /* read clocksource: */ | ||
101 | cycle_now = clocksource_read(clock); | ||
102 | |||
103 | /* calculate the delta since the last update_wall_time: */ | ||
104 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | ||
105 | |||
106 | /* convert to nanoseconds: */ | ||
107 | nsecs = cyc2ns(clock, cycle_delta); | ||
100 | 108 | ||
101 | } while (read_seqretry(&xtime_lock, seq)); | 109 | } while (read_seqretry(&xtime_lock, seq)); |
102 | 110 | ||
@@ -129,22 +137,22 @@ EXPORT_SYMBOL(do_gettimeofday); | |||
129 | */ | 137 | */ |
130 | int do_settimeofday(struct timespec *tv) | 138 | int do_settimeofday(struct timespec *tv) |
131 | { | 139 | { |
140 | struct timespec ts_delta; | ||
132 | unsigned long flags; | 141 | unsigned long flags; |
133 | time_t wtm_sec, sec = tv->tv_sec; | ||
134 | long wtm_nsec, nsec = tv->tv_nsec; | ||
135 | 142 | ||
136 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | 143 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) |
137 | return -EINVAL; | 144 | return -EINVAL; |
138 | 145 | ||
139 | write_seqlock_irqsave(&xtime_lock, flags); | 146 | write_seqlock_irqsave(&xtime_lock, flags); |
140 | 147 | ||
141 | nsec -= __get_nsec_offset(); | 148 | clocksource_forward_now(); |
149 | |||
150 | ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec; | ||
151 | ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec; | ||
152 | wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta); | ||
142 | 153 | ||
143 | wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec); | 154 | xtime = *tv; |
144 | wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec); | ||
145 | 155 | ||
146 | set_normalized_timespec(&xtime, sec, nsec); | ||
147 | set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec); | ||
148 | update_xtime_cache(0); | 156 | update_xtime_cache(0); |
149 | 157 | ||
150 | clock->error = 0; | 158 | clock->error = 0; |
@@ -170,22 +178,19 @@ EXPORT_SYMBOL(do_settimeofday); | |||
170 | static void change_clocksource(void) | 178 | static void change_clocksource(void) |
171 | { | 179 | { |
172 | struct clocksource *new; | 180 | struct clocksource *new; |
173 | cycle_t now; | ||
174 | u64 nsec; | ||
175 | 181 | ||
176 | new = clocksource_get_next(); | 182 | new = clocksource_get_next(); |
177 | 183 | ||
178 | if (clock == new) | 184 | if (clock == new) |
179 | return; | 185 | return; |
180 | 186 | ||
181 | new->cycle_last = 0; | 187 | clocksource_forward_now(); |
182 | now = clocksource_read(new); | ||
183 | nsec = __get_nsec_offset(); | ||
184 | timespec_add_ns(&xtime, nsec); | ||
185 | 188 | ||
186 | clock = new; | 189 | new->raw_time = clock->raw_time; |
187 | clock->cycle_last = now; | ||
188 | 190 | ||
191 | clock = new; | ||
192 | clock->cycle_last = 0; | ||
193 | clock->cycle_last = clocksource_read(new); | ||
189 | clock->error = 0; | 194 | clock->error = 0; |
190 | clock->xtime_nsec = 0; | 195 | clock->xtime_nsec = 0; |
191 | clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); | 196 | clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); |
@@ -200,11 +205,44 @@ static void change_clocksource(void) | |||
200 | */ | 205 | */ |
201 | } | 206 | } |
202 | #else | 207 | #else |
208 | static inline void clocksource_forward_now(void) { } | ||
203 | static inline void change_clocksource(void) { } | 209 | static inline void change_clocksource(void) { } |
204 | static inline s64 __get_nsec_offset(void) { return 0; } | ||
205 | #endif | 210 | #endif |
206 | 211 | ||
207 | /** | 212 | /** |
213 | * getrawmonotonic - Returns the raw monotonic time in a timespec | ||
214 | * @ts: pointer to the timespec to be set | ||
215 | * | ||
216 | * Returns the raw monotonic time (completely un-modified by ntp) | ||
217 | */ | ||
218 | void getrawmonotonic(struct timespec *ts) | ||
219 | { | ||
220 | unsigned long seq; | ||
221 | s64 nsecs; | ||
222 | cycle_t cycle_now, cycle_delta; | ||
223 | |||
224 | do { | ||
225 | seq = read_seqbegin(&xtime_lock); | ||
226 | |||
227 | /* read clocksource: */ | ||
228 | cycle_now = clocksource_read(clock); | ||
229 | |||
230 | /* calculate the delta since the last update_wall_time: */ | ||
231 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | ||
232 | |||
233 | /* convert to nanoseconds: */ | ||
234 | nsecs = ((s64)cycle_delta * clock->mult_orig) >> clock->shift; | ||
235 | |||
236 | *ts = clock->raw_time; | ||
237 | |||
238 | } while (read_seqretry(&xtime_lock, seq)); | ||
239 | |||
240 | timespec_add_ns(ts, nsecs); | ||
241 | } | ||
242 | EXPORT_SYMBOL(getrawmonotonic); | ||
243 | |||
244 | |||
245 | /** | ||
208 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres | 246 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres |
209 | */ | 247 | */ |
210 | int timekeeping_valid_for_hres(void) | 248 | int timekeeping_valid_for_hres(void) |
@@ -265,8 +303,6 @@ void __init timekeeping_init(void) | |||
265 | static int timekeeping_suspended; | 303 | static int timekeeping_suspended; |
266 | /* time in seconds when suspend began */ | 304 | /* time in seconds when suspend began */ |
267 | static unsigned long timekeeping_suspend_time; | 305 | static unsigned long timekeeping_suspend_time; |
268 | /* xtime offset when we went into suspend */ | ||
269 | static s64 timekeeping_suspend_nsecs; | ||
270 | 306 | ||
271 | /** | 307 | /** |
272 | * timekeeping_resume - Resumes the generic timekeeping subsystem. | 308 | * timekeeping_resume - Resumes the generic timekeeping subsystem. |
@@ -292,8 +328,6 @@ static int timekeeping_resume(struct sys_device *dev) | |||
292 | wall_to_monotonic.tv_sec -= sleep_length; | 328 | wall_to_monotonic.tv_sec -= sleep_length; |
293 | total_sleep_time += sleep_length; | 329 | total_sleep_time += sleep_length; |
294 | } | 330 | } |
295 | /* Make sure that we have the correct xtime reference */ | ||
296 | timespec_add_ns(&xtime, timekeeping_suspend_nsecs); | ||
297 | update_xtime_cache(0); | 331 | update_xtime_cache(0); |
298 | /* re-base the last cycle value */ | 332 | /* re-base the last cycle value */ |
299 | clock->cycle_last = 0; | 333 | clock->cycle_last = 0; |
@@ -319,8 +353,7 @@ static int timekeeping_suspend(struct sys_device *dev, pm_message_t state) | |||
319 | timekeeping_suspend_time = read_persistent_clock(); | 353 | timekeeping_suspend_time = read_persistent_clock(); |
320 | 354 | ||
321 | write_seqlock_irqsave(&xtime_lock, flags); | 355 | write_seqlock_irqsave(&xtime_lock, flags); |
322 | /* Get the current xtime offset */ | 356 | clocksource_forward_now(); |
323 | timekeeping_suspend_nsecs = __get_nsec_offset(); | ||
324 | timekeeping_suspended = 1; | 357 | timekeeping_suspended = 1; |
325 | write_sequnlock_irqrestore(&xtime_lock, flags); | 358 | write_sequnlock_irqrestore(&xtime_lock, flags); |
326 | 359 | ||
@@ -454,23 +487,29 @@ void update_wall_time(void) | |||
454 | #else | 487 | #else |
455 | offset = clock->cycle_interval; | 488 | offset = clock->cycle_interval; |
456 | #endif | 489 | #endif |
457 | clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift; | 490 | clock->xtime_nsec = (s64)xtime.tv_nsec << clock->shift; |
458 | 491 | ||
459 | /* normally this loop will run just once, however in the | 492 | /* normally this loop will run just once, however in the |
460 | * case of lost or late ticks, it will accumulate correctly. | 493 | * case of lost or late ticks, it will accumulate correctly. |
461 | */ | 494 | */ |
462 | while (offset >= clock->cycle_interval) { | 495 | while (offset >= clock->cycle_interval) { |
463 | /* accumulate one interval */ | 496 | /* accumulate one interval */ |
464 | clock->xtime_nsec += clock->xtime_interval; | ||
465 | clock->cycle_last += clock->cycle_interval; | ||
466 | offset -= clock->cycle_interval; | 497 | offset -= clock->cycle_interval; |
498 | clock->cycle_last += clock->cycle_interval; | ||
467 | 499 | ||
500 | clock->xtime_nsec += clock->xtime_interval; | ||
468 | if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { | 501 | if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { |
469 | clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; | 502 | clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; |
470 | xtime.tv_sec++; | 503 | xtime.tv_sec++; |
471 | second_overflow(); | 504 | second_overflow(); |
472 | } | 505 | } |
473 | 506 | ||
507 | clock->raw_time.tv_nsec += clock->raw_interval; | ||
508 | if (clock->raw_time.tv_nsec >= NSEC_PER_SEC) { | ||
509 | clock->raw_time.tv_nsec -= NSEC_PER_SEC; | ||
510 | clock->raw_time.tv_sec++; | ||
511 | } | ||
512 | |||
474 | /* accumulate error between NTP and clock interval */ | 513 | /* accumulate error between NTP and clock interval */ |
475 | clock->error += tick_length; | 514 | clock->error += tick_length; |
476 | clock->error -= clock->xtime_interval << (NTP_SCALE_SHIFT - clock->shift); | 515 | clock->error -= clock->xtime_interval << (NTP_SCALE_SHIFT - clock->shift); |
@@ -479,9 +518,12 @@ void update_wall_time(void) | |||
479 | /* correct the clock when NTP error is too big */ | 518 | /* correct the clock when NTP error is too big */ |
480 | clocksource_adjust(offset); | 519 | clocksource_adjust(offset); |
481 | 520 | ||
482 | /* store full nanoseconds into xtime */ | 521 | /* store full nanoseconds into xtime after rounding it up and |
483 | xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift; | 522 | * add the remainder to the error difference. |
523 | */ | ||
524 | xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1; | ||
484 | clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; | 525 | clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; |
526 | clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift); | ||
485 | 527 | ||
486 | update_xtime_cache(cyc2ns(clock, offset)); | 528 | update_xtime_cache(cyc2ns(clock, offset)); |
487 | 529 | ||