diff options
Diffstat (limited to 'kernel/time/timekeeping.c')
| -rw-r--r-- | kernel/time/timekeeping.c | 755 |
1 files changed, 414 insertions, 341 deletions
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 269b1fe5f2ae..e16af197a2bc 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
| @@ -24,32 +24,32 @@ | |||
| 24 | /* Structure holding internal timekeeping values. */ | 24 | /* Structure holding internal timekeeping values. */ |
| 25 | struct timekeeper { | 25 | struct timekeeper { |
| 26 | /* Current clocksource used for timekeeping. */ | 26 | /* Current clocksource used for timekeeping. */ |
| 27 | struct clocksource *clock; | 27 | struct clocksource *clock; |
| 28 | /* NTP adjusted clock multiplier */ | 28 | /* NTP adjusted clock multiplier */ |
| 29 | u32 mult; | 29 | u32 mult; |
| 30 | /* The shift value of the current clocksource. */ | 30 | /* The shift value of the current clocksource. */ |
| 31 | int shift; | 31 | u32 shift; |
| 32 | |||
| 33 | /* Number of clock cycles in one NTP interval. */ | 32 | /* Number of clock cycles in one NTP interval. */ |
| 34 | cycle_t cycle_interval; | 33 | cycle_t cycle_interval; |
| 35 | /* Number of clock shifted nano seconds in one NTP interval. */ | 34 | /* Number of clock shifted nano seconds in one NTP interval. */ |
| 36 | u64 xtime_interval; | 35 | u64 xtime_interval; |
| 37 | /* shifted nano seconds left over when rounding cycle_interval */ | 36 | /* shifted nano seconds left over when rounding cycle_interval */ |
| 38 | s64 xtime_remainder; | 37 | s64 xtime_remainder; |
| 39 | /* Raw nano seconds accumulated per NTP interval. */ | 38 | /* Raw nano seconds accumulated per NTP interval. */ |
| 40 | u32 raw_interval; | 39 | u32 raw_interval; |
| 40 | |||
| 41 | /* Current CLOCK_REALTIME time in seconds */ | ||
| 42 | u64 xtime_sec; | ||
| 43 | /* Clock shifted nano seconds */ | ||
| 44 | u64 xtime_nsec; | ||
| 41 | 45 | ||
| 42 | /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */ | ||
| 43 | u64 xtime_nsec; | ||
| 44 | /* Difference between accumulated time and NTP time in ntp | 46 | /* Difference between accumulated time and NTP time in ntp |
| 45 | * shifted nano seconds. */ | 47 | * shifted nano seconds. */ |
| 46 | s64 ntp_error; | 48 | s64 ntp_error; |
| 47 | /* Shift conversion between clock shifted nano seconds and | 49 | /* Shift conversion between clock shifted nano seconds and |
| 48 | * ntp shifted nano seconds. */ | 50 | * ntp shifted nano seconds. */ |
| 49 | int ntp_error_shift; | 51 | u32 ntp_error_shift; |
| 50 | 52 | ||
| 51 | /* The current time */ | ||
| 52 | struct timespec xtime; | ||
| 53 | /* | 53 | /* |
| 54 | * wall_to_monotonic is what we need to add to xtime (or xtime corrected | 54 | * wall_to_monotonic is what we need to add to xtime (or xtime corrected |
| 55 | * for sub jiffie times) to get to monotonic time. Monotonic is pegged | 55 | * for sub jiffie times) to get to monotonic time. Monotonic is pegged |
| @@ -64,20 +64,17 @@ struct timekeeper { | |||
| 64 | * - wall_to_monotonic is no longer the boot time, getboottime must be | 64 | * - wall_to_monotonic is no longer the boot time, getboottime must be |
| 65 | * used instead. | 65 | * used instead. |
| 66 | */ | 66 | */ |
| 67 | struct timespec wall_to_monotonic; | 67 | struct timespec wall_to_monotonic; |
| 68 | /* time spent in suspend */ | ||
| 69 | struct timespec total_sleep_time; | ||
| 70 | /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ | ||
| 71 | struct timespec raw_time; | ||
| 72 | |||
| 73 | /* Offset clock monotonic -> clock realtime */ | 68 | /* Offset clock monotonic -> clock realtime */ |
| 74 | ktime_t offs_real; | 69 | ktime_t offs_real; |
| 75 | 70 | /* time spent in suspend */ | |
| 71 | struct timespec total_sleep_time; | ||
| 76 | /* Offset clock monotonic -> clock boottime */ | 72 | /* Offset clock monotonic -> clock boottime */ |
| 77 | ktime_t offs_boot; | 73 | ktime_t offs_boot; |
| 78 | 74 | /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ | |
| 75 | struct timespec raw_time; | ||
| 79 | /* Seqlock for all timekeeper values */ | 76 | /* Seqlock for all timekeeper values */ |
| 80 | seqlock_t lock; | 77 | seqlock_t lock; |
| 81 | }; | 78 | }; |
| 82 | 79 | ||
| 83 | static struct timekeeper timekeeper; | 80 | static struct timekeeper timekeeper; |
| @@ -88,11 +85,62 @@ static struct timekeeper timekeeper; | |||
| 88 | */ | 85 | */ |
| 89 | __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); | 86 | __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); |
| 90 | 87 | ||
| 91 | |||
| 92 | /* flag for if timekeeping is suspended */ | 88 | /* flag for if timekeeping is suspended */ |
| 93 | int __read_mostly timekeeping_suspended; | 89 | int __read_mostly timekeeping_suspended; |
| 94 | 90 | ||
| 91 | static inline void tk_normalize_xtime(struct timekeeper *tk) | ||
| 92 | { | ||
| 93 | while (tk->xtime_nsec >= ((u64)NSEC_PER_SEC << tk->shift)) { | ||
| 94 | tk->xtime_nsec -= (u64)NSEC_PER_SEC << tk->shift; | ||
| 95 | tk->xtime_sec++; | ||
| 96 | } | ||
| 97 | } | ||
| 95 | 98 | ||
| 99 | static struct timespec tk_xtime(struct timekeeper *tk) | ||
| 100 | { | ||
| 101 | struct timespec ts; | ||
| 102 | |||
| 103 | ts.tv_sec = tk->xtime_sec; | ||
| 104 | ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift); | ||
| 105 | return ts; | ||
| 106 | } | ||
| 107 | |||
| 108 | static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) | ||
| 109 | { | ||
| 110 | tk->xtime_sec = ts->tv_sec; | ||
| 111 | tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift; | ||
| 112 | } | ||
| 113 | |||
| 114 | static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts) | ||
| 115 | { | ||
| 116 | tk->xtime_sec += ts->tv_sec; | ||
| 117 | tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift; | ||
| 118 | } | ||
| 119 | |||
| 120 | static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm) | ||
| 121 | { | ||
| 122 | struct timespec tmp; | ||
| 123 | |||
| 124 | /* | ||
| 125 | * Verify consistency of: offset_real = -wall_to_monotonic | ||
| 126 | * before modifying anything | ||
| 127 | */ | ||
| 128 | set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec, | ||
| 129 | -tk->wall_to_monotonic.tv_nsec); | ||
| 130 | WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64); | ||
| 131 | tk->wall_to_monotonic = wtm; | ||
| 132 | set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec); | ||
| 133 | tk->offs_real = timespec_to_ktime(tmp); | ||
| 134 | } | ||
| 135 | |||
| 136 | static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t) | ||
| 137 | { | ||
| 138 | /* Verify consistency before modifying */ | ||
| 139 | WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64); | ||
| 140 | |||
| 141 | tk->total_sleep_time = t; | ||
| 142 | tk->offs_boot = timespec_to_ktime(t); | ||
| 143 | } | ||
| 96 | 144 | ||
| 97 | /** | 145 | /** |
| 98 | * timekeeper_setup_internals - Set up internals to use clocksource clock. | 146 | * timekeeper_setup_internals - Set up internals to use clocksource clock. |
| @@ -104,12 +152,14 @@ int __read_mostly timekeeping_suspended; | |||
| 104 | * | 152 | * |
| 105 | * Unless you're the timekeeping code, you should not be using this! | 153 | * Unless you're the timekeeping code, you should not be using this! |
| 106 | */ | 154 | */ |
| 107 | static void timekeeper_setup_internals(struct clocksource *clock) | 155 | static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) |
| 108 | { | 156 | { |
| 109 | cycle_t interval; | 157 | cycle_t interval; |
| 110 | u64 tmp, ntpinterval; | 158 | u64 tmp, ntpinterval; |
| 159 | struct clocksource *old_clock; | ||
| 111 | 160 | ||
| 112 | timekeeper.clock = clock; | 161 | old_clock = tk->clock; |
| 162 | tk->clock = clock; | ||
| 113 | clock->cycle_last = clock->read(clock); | 163 | clock->cycle_last = clock->read(clock); |
| 114 | 164 | ||
| 115 | /* Do the ns -> cycle conversion first, using original mult */ | 165 | /* Do the ns -> cycle conversion first, using original mult */ |
| @@ -122,83 +172,89 @@ static void timekeeper_setup_internals(struct clocksource *clock) | |||
| 122 | tmp = 1; | 172 | tmp = 1; |
| 123 | 173 | ||
| 124 | interval = (cycle_t) tmp; | 174 | interval = (cycle_t) tmp; |
| 125 | timekeeper.cycle_interval = interval; | 175 | tk->cycle_interval = interval; |
| 126 | 176 | ||
| 127 | /* Go back from cycles -> shifted ns */ | 177 | /* Go back from cycles -> shifted ns */ |
| 128 | timekeeper.xtime_interval = (u64) interval * clock->mult; | 178 | tk->xtime_interval = (u64) interval * clock->mult; |
| 129 | timekeeper.xtime_remainder = ntpinterval - timekeeper.xtime_interval; | 179 | tk->xtime_remainder = ntpinterval - tk->xtime_interval; |
| 130 | timekeeper.raw_interval = | 180 | tk->raw_interval = |
| 131 | ((u64) interval * clock->mult) >> clock->shift; | 181 | ((u64) interval * clock->mult) >> clock->shift; |
| 132 | 182 | ||
| 133 | timekeeper.xtime_nsec = 0; | 183 | /* if changing clocks, convert xtime_nsec shift units */ |
| 134 | timekeeper.shift = clock->shift; | 184 | if (old_clock) { |
| 185 | int shift_change = clock->shift - old_clock->shift; | ||
| 186 | if (shift_change < 0) | ||
| 187 | tk->xtime_nsec >>= -shift_change; | ||
| 188 | else | ||
| 189 | tk->xtime_nsec <<= shift_change; | ||
| 190 | } | ||
| 191 | tk->shift = clock->shift; | ||
| 135 | 192 | ||
| 136 | timekeeper.ntp_error = 0; | 193 | tk->ntp_error = 0; |
| 137 | timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; | 194 | tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; |
| 138 | 195 | ||
| 139 | /* | 196 | /* |
| 140 | * The timekeeper keeps its own mult values for the currently | 197 | * The timekeeper keeps its own mult values for the currently |
| 141 | * active clocksource. These value will be adjusted via NTP | 198 | * active clocksource. These value will be adjusted via NTP |
| 142 | * to counteract clock drifting. | 199 | * to counteract clock drifting. |
| 143 | */ | 200 | */ |
| 144 | timekeeper.mult = clock->mult; | 201 | tk->mult = clock->mult; |
| 145 | } | 202 | } |
| 146 | 203 | ||
| 147 | /* Timekeeper helper functions. */ | 204 | /* Timekeeper helper functions. */ |
| 148 | static inline s64 timekeeping_get_ns(void) | 205 | static inline s64 timekeeping_get_ns(struct timekeeper *tk) |
| 149 | { | 206 | { |
| 150 | cycle_t cycle_now, cycle_delta; | 207 | cycle_t cycle_now, cycle_delta; |
| 151 | struct clocksource *clock; | 208 | struct clocksource *clock; |
| 209 | s64 nsec; | ||
| 152 | 210 | ||
| 153 | /* read clocksource: */ | 211 | /* read clocksource: */ |
| 154 | clock = timekeeper.clock; | 212 | clock = tk->clock; |
| 155 | cycle_now = clock->read(clock); | 213 | cycle_now = clock->read(clock); |
| 156 | 214 | ||
| 157 | /* calculate the delta since the last update_wall_time: */ | 215 | /* calculate the delta since the last update_wall_time: */ |
| 158 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | 216 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; |
| 159 | 217 | ||
| 160 | /* return delta convert to nanoseconds using ntp adjusted mult. */ | 218 | nsec = cycle_delta * tk->mult + tk->xtime_nsec; |
| 161 | return clocksource_cyc2ns(cycle_delta, timekeeper.mult, | 219 | nsec >>= tk->shift; |
| 162 | timekeeper.shift); | 220 | |
| 221 | /* If arch requires, add in gettimeoffset() */ | ||
| 222 | return nsec + arch_gettimeoffset(); | ||
| 163 | } | 223 | } |
| 164 | 224 | ||
| 165 | static inline s64 timekeeping_get_ns_raw(void) | 225 | static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) |
| 166 | { | 226 | { |
| 167 | cycle_t cycle_now, cycle_delta; | 227 | cycle_t cycle_now, cycle_delta; |
| 168 | struct clocksource *clock; | 228 | struct clocksource *clock; |
| 229 | s64 nsec; | ||
| 169 | 230 | ||
| 170 | /* read clocksource: */ | 231 | /* read clocksource: */ |
| 171 | clock = timekeeper.clock; | 232 | clock = tk->clock; |
| 172 | cycle_now = clock->read(clock); | 233 | cycle_now = clock->read(clock); |
| 173 | 234 | ||
| 174 | /* calculate the delta since the last update_wall_time: */ | 235 | /* calculate the delta since the last update_wall_time: */ |
| 175 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | 236 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; |
| 176 | 237 | ||
| 177 | /* return delta convert to nanoseconds. */ | 238 | /* convert delta to nanoseconds. */ |
| 178 | return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); | 239 | nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); |
| 179 | } | ||
| 180 | |||
| 181 | static void update_rt_offset(void) | ||
| 182 | { | ||
| 183 | struct timespec tmp, *wtm = &timekeeper.wall_to_monotonic; | ||
| 184 | 240 | ||
| 185 | set_normalized_timespec(&tmp, -wtm->tv_sec, -wtm->tv_nsec); | 241 | /* If arch requires, add in gettimeoffset() */ |
| 186 | timekeeper.offs_real = timespec_to_ktime(tmp); | 242 | return nsec + arch_gettimeoffset(); |
| 187 | } | 243 | } |
| 188 | 244 | ||
| 189 | /* must hold write on timekeeper.lock */ | 245 | /* must hold write on timekeeper.lock */ |
| 190 | static void timekeeping_update(bool clearntp) | 246 | static void timekeeping_update(struct timekeeper *tk, bool clearntp) |
| 191 | { | 247 | { |
| 248 | struct timespec xt; | ||
| 249 | |||
| 192 | if (clearntp) { | 250 | if (clearntp) { |
| 193 | timekeeper.ntp_error = 0; | 251 | tk->ntp_error = 0; |
| 194 | ntp_clear(); | 252 | ntp_clear(); |
| 195 | } | 253 | } |
| 196 | update_rt_offset(); | 254 | xt = tk_xtime(tk); |
| 197 | update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic, | 255 | update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); |
| 198 | timekeeper.clock, timekeeper.mult); | ||
| 199 | } | 256 | } |
| 200 | 257 | ||
| 201 | |||
| 202 | /** | 258 | /** |
| 203 | * timekeeping_forward_now - update clock to the current time | 259 | * timekeeping_forward_now - update clock to the current time |
| 204 | * | 260 | * |
| @@ -206,27 +262,26 @@ static void timekeeping_update(bool clearntp) | |||
| 206 | * update_wall_time(). This is useful before significant clock changes, | 262 | * update_wall_time(). This is useful before significant clock changes, |
| 207 | * as it avoids having to deal with this time offset explicitly. | 263 | * as it avoids having to deal with this time offset explicitly. |
| 208 | */ | 264 | */ |
| 209 | static void timekeeping_forward_now(void) | 265 | static void timekeeping_forward_now(struct timekeeper *tk) |
| 210 | { | 266 | { |
| 211 | cycle_t cycle_now, cycle_delta; | 267 | cycle_t cycle_now, cycle_delta; |
| 212 | struct clocksource *clock; | 268 | struct clocksource *clock; |
| 213 | s64 nsec; | 269 | s64 nsec; |
| 214 | 270 | ||
| 215 | clock = timekeeper.clock; | 271 | clock = tk->clock; |
| 216 | cycle_now = clock->read(clock); | 272 | cycle_now = clock->read(clock); |
| 217 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | 273 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; |
| 218 | clock->cycle_last = cycle_now; | 274 | clock->cycle_last = cycle_now; |
| 219 | 275 | ||
| 220 | nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult, | 276 | tk->xtime_nsec += cycle_delta * tk->mult; |
| 221 | timekeeper.shift); | ||
| 222 | 277 | ||
| 223 | /* If arch requires, add in gettimeoffset() */ | 278 | /* If arch requires, add in gettimeoffset() */ |
| 224 | nsec += arch_gettimeoffset(); | 279 | tk->xtime_nsec += arch_gettimeoffset() << tk->shift; |
| 225 | 280 | ||
| 226 | timespec_add_ns(&timekeeper.xtime, nsec); | 281 | tk_normalize_xtime(tk); |
| 227 | 282 | ||
| 228 | nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); | 283 | nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); |
| 229 | timespec_add_ns(&timekeeper.raw_time, nsec); | 284 | timespec_add_ns(&tk->raw_time, nsec); |
| 230 | } | 285 | } |
| 231 | 286 | ||
| 232 | /** | 287 | /** |
| @@ -237,21 +292,19 @@ static void timekeeping_forward_now(void) | |||
| 237 | */ | 292 | */ |
| 238 | void getnstimeofday(struct timespec *ts) | 293 | void getnstimeofday(struct timespec *ts) |
| 239 | { | 294 | { |
| 295 | struct timekeeper *tk = &timekeeper; | ||
| 240 | unsigned long seq; | 296 | unsigned long seq; |
| 241 | s64 nsecs; | 297 | s64 nsecs = 0; |
| 242 | 298 | ||
| 243 | WARN_ON(timekeeping_suspended); | 299 | WARN_ON(timekeeping_suspended); |
| 244 | 300 | ||
| 245 | do { | 301 | do { |
| 246 | seq = read_seqbegin(&timekeeper.lock); | 302 | seq = read_seqbegin(&tk->lock); |
| 247 | 303 | ||
| 248 | *ts = timekeeper.xtime; | 304 | ts->tv_sec = tk->xtime_sec; |
| 249 | nsecs = timekeeping_get_ns(); | 305 | ts->tv_nsec = timekeeping_get_ns(tk); |
| 250 | 306 | ||
| 251 | /* If arch requires, add in gettimeoffset() */ | 307 | } while (read_seqretry(&tk->lock, seq)); |
| 252 | nsecs += arch_gettimeoffset(); | ||
| 253 | |||
| 254 | } while (read_seqretry(&timekeeper.lock, seq)); | ||
| 255 | 308 | ||
| 256 | timespec_add_ns(ts, nsecs); | 309 | timespec_add_ns(ts, nsecs); |
| 257 | } | 310 | } |
| @@ -259,22 +312,18 @@ EXPORT_SYMBOL(getnstimeofday); | |||
| 259 | 312 | ||
| 260 | ktime_t ktime_get(void) | 313 | ktime_t ktime_get(void) |
| 261 | { | 314 | { |
| 315 | struct timekeeper *tk = &timekeeper; | ||
| 262 | unsigned int seq; | 316 | unsigned int seq; |
| 263 | s64 secs, nsecs; | 317 | s64 secs, nsecs; |
| 264 | 318 | ||
| 265 | WARN_ON(timekeeping_suspended); | 319 | WARN_ON(timekeeping_suspended); |
| 266 | 320 | ||
| 267 | do { | 321 | do { |
| 268 | seq = read_seqbegin(&timekeeper.lock); | 322 | seq = read_seqbegin(&tk->lock); |
| 269 | secs = timekeeper.xtime.tv_sec + | 323 | secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; |
| 270 | timekeeper.wall_to_monotonic.tv_sec; | 324 | nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; |
| 271 | nsecs = timekeeper.xtime.tv_nsec + | 325 | |
| 272 | timekeeper.wall_to_monotonic.tv_nsec; | 326 | } while (read_seqretry(&tk->lock, seq)); |
| 273 | nsecs += timekeeping_get_ns(); | ||
| 274 | /* If arch requires, add in gettimeoffset() */ | ||
| 275 | nsecs += arch_gettimeoffset(); | ||
| 276 | |||
| 277 | } while (read_seqretry(&timekeeper.lock, seq)); | ||
| 278 | /* | 327 | /* |
| 279 | * Use ktime_set/ktime_add_ns to create a proper ktime on | 328 | * Use ktime_set/ktime_add_ns to create a proper ktime on |
| 280 | * 32-bit architectures without CONFIG_KTIME_SCALAR. | 329 | * 32-bit architectures without CONFIG_KTIME_SCALAR. |
| @@ -293,24 +342,22 @@ EXPORT_SYMBOL_GPL(ktime_get); | |||
| 293 | */ | 342 | */ |
| 294 | void ktime_get_ts(struct timespec *ts) | 343 | void ktime_get_ts(struct timespec *ts) |
| 295 | { | 344 | { |
| 345 | struct timekeeper *tk = &timekeeper; | ||
| 296 | struct timespec tomono; | 346 | struct timespec tomono; |
| 297 | unsigned int seq; | 347 | unsigned int seq; |
| 298 | s64 nsecs; | ||
| 299 | 348 | ||
| 300 | WARN_ON(timekeeping_suspended); | 349 | WARN_ON(timekeeping_suspended); |
| 301 | 350 | ||
| 302 | do { | 351 | do { |
| 303 | seq = read_seqbegin(&timekeeper.lock); | 352 | seq = read_seqbegin(&tk->lock); |
| 304 | *ts = timekeeper.xtime; | 353 | ts->tv_sec = tk->xtime_sec; |
| 305 | tomono = timekeeper.wall_to_monotonic; | 354 | ts->tv_nsec = timekeeping_get_ns(tk); |
| 306 | nsecs = timekeeping_get_ns(); | 355 | tomono = tk->wall_to_monotonic; |
| 307 | /* If arch requires, add in gettimeoffset() */ | ||
| 308 | nsecs += arch_gettimeoffset(); | ||
| 309 | 356 | ||
| 310 | } while (read_seqretry(&timekeeper.lock, seq)); | 357 | } while (read_seqretry(&tk->lock, seq)); |
| 311 | 358 | ||
| 312 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, | 359 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, |
| 313 | ts->tv_nsec + tomono.tv_nsec + nsecs); | 360 | ts->tv_nsec + tomono.tv_nsec); |
| 314 | } | 361 | } |
| 315 | EXPORT_SYMBOL_GPL(ktime_get_ts); | 362 | EXPORT_SYMBOL_GPL(ktime_get_ts); |
| 316 | 363 | ||
| @@ -327,28 +374,23 @@ EXPORT_SYMBOL_GPL(ktime_get_ts); | |||
| 327 | */ | 374 | */ |
| 328 | void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) | 375 | void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) |
| 329 | { | 376 | { |
| 377 | struct timekeeper *tk = &timekeeper; | ||
| 330 | unsigned long seq; | 378 | unsigned long seq; |
| 331 | s64 nsecs_raw, nsecs_real; | 379 | s64 nsecs_raw, nsecs_real; |
| 332 | 380 | ||
| 333 | WARN_ON_ONCE(timekeeping_suspended); | 381 | WARN_ON_ONCE(timekeeping_suspended); |
| 334 | 382 | ||
| 335 | do { | 383 | do { |
| 336 | u32 arch_offset; | 384 | seq = read_seqbegin(&tk->lock); |
| 337 | |||
| 338 | seq = read_seqbegin(&timekeeper.lock); | ||
| 339 | 385 | ||
| 340 | *ts_raw = timekeeper.raw_time; | 386 | *ts_raw = tk->raw_time; |
| 341 | *ts_real = timekeeper.xtime; | 387 | ts_real->tv_sec = tk->xtime_sec; |
| 388 | ts_real->tv_nsec = 0; | ||
| 342 | 389 | ||
| 343 | nsecs_raw = timekeeping_get_ns_raw(); | 390 | nsecs_raw = timekeeping_get_ns_raw(tk); |
| 344 | nsecs_real = timekeeping_get_ns(); | 391 | nsecs_real = timekeeping_get_ns(tk); |
| 345 | 392 | ||
| 346 | /* If arch requires, add in gettimeoffset() */ | 393 | } while (read_seqretry(&tk->lock, seq)); |
| 347 | arch_offset = arch_gettimeoffset(); | ||
| 348 | nsecs_raw += arch_offset; | ||
| 349 | nsecs_real += arch_offset; | ||
| 350 | |||
| 351 | } while (read_seqretry(&timekeeper.lock, seq)); | ||
| 352 | 394 | ||
| 353 | timespec_add_ns(ts_raw, nsecs_raw); | 395 | timespec_add_ns(ts_raw, nsecs_raw); |
| 354 | timespec_add_ns(ts_real, nsecs_real); | 396 | timespec_add_ns(ts_real, nsecs_real); |
| @@ -381,25 +423,28 @@ EXPORT_SYMBOL(do_gettimeofday); | |||
| 381 | */ | 423 | */ |
| 382 | int do_settimeofday(const struct timespec *tv) | 424 | int do_settimeofday(const struct timespec *tv) |
| 383 | { | 425 | { |
| 384 | struct timespec ts_delta; | 426 | struct timekeeper *tk = &timekeeper; |
| 427 | struct timespec ts_delta, xt; | ||
| 385 | unsigned long flags; | 428 | unsigned long flags; |
| 386 | 429 | ||
| 387 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | 430 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) |
| 388 | return -EINVAL; | 431 | return -EINVAL; |
| 389 | 432 | ||
| 390 | write_seqlock_irqsave(&timekeeper.lock, flags); | 433 | write_seqlock_irqsave(&tk->lock, flags); |
| 434 | |||
| 435 | timekeeping_forward_now(tk); | ||
| 391 | 436 | ||
| 392 | timekeeping_forward_now(); | 437 | xt = tk_xtime(tk); |
| 438 | ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; | ||
| 439 | ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; | ||
| 393 | 440 | ||
| 394 | ts_delta.tv_sec = tv->tv_sec - timekeeper.xtime.tv_sec; | 441 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta)); |
| 395 | ts_delta.tv_nsec = tv->tv_nsec - timekeeper.xtime.tv_nsec; | ||
| 396 | timekeeper.wall_to_monotonic = | ||
| 397 | timespec_sub(timekeeper.wall_to_monotonic, ts_delta); | ||
| 398 | 442 | ||
| 399 | timekeeper.xtime = *tv; | 443 | tk_set_xtime(tk, tv); |
| 400 | timekeeping_update(true); | ||
| 401 | 444 | ||
| 402 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 445 | timekeeping_update(tk, true); |
| 446 | |||
| 447 | write_sequnlock_irqrestore(&tk->lock, flags); | ||
| 403 | 448 | ||
| 404 | /* signal hrtimers about time change */ | 449 | /* signal hrtimers about time change */ |
| 405 | clock_was_set(); | 450 | clock_was_set(); |
| @@ -408,7 +453,6 @@ int do_settimeofday(const struct timespec *tv) | |||
| 408 | } | 453 | } |
| 409 | EXPORT_SYMBOL(do_settimeofday); | 454 | EXPORT_SYMBOL(do_settimeofday); |
| 410 | 455 | ||
| 411 | |||
| 412 | /** | 456 | /** |
| 413 | * timekeeping_inject_offset - Adds or subtracts from the current time. | 457 | * timekeeping_inject_offset - Adds or subtracts from the current time. |
| 414 | * @tv: pointer to the timespec variable containing the offset | 458 | * @tv: pointer to the timespec variable containing the offset |
| @@ -417,22 +461,23 @@ EXPORT_SYMBOL(do_settimeofday); | |||
| 417 | */ | 461 | */ |
| 418 | int timekeeping_inject_offset(struct timespec *ts) | 462 | int timekeeping_inject_offset(struct timespec *ts) |
| 419 | { | 463 | { |
| 464 | struct timekeeper *tk = &timekeeper; | ||
| 420 | unsigned long flags; | 465 | unsigned long flags; |
| 421 | 466 | ||
| 422 | if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) | 467 | if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) |
| 423 | return -EINVAL; | 468 | return -EINVAL; |
| 424 | 469 | ||
| 425 | write_seqlock_irqsave(&timekeeper.lock, flags); | 470 | write_seqlock_irqsave(&tk->lock, flags); |
| 426 | 471 | ||
| 427 | timekeeping_forward_now(); | 472 | timekeeping_forward_now(tk); |
| 428 | 473 | ||
| 429 | timekeeper.xtime = timespec_add(timekeeper.xtime, *ts); | ||
| 430 | timekeeper.wall_to_monotonic = | ||
| 431 | timespec_sub(timekeeper.wall_to_monotonic, *ts); | ||
| 432 | 474 | ||
| 433 | timekeeping_update(true); | 475 | tk_xtime_add(tk, ts); |
| 476 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts)); | ||
| 434 | 477 | ||
| 435 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 478 | timekeeping_update(tk, true); |
| 479 | |||
| 480 | write_sequnlock_irqrestore(&tk->lock, flags); | ||
| 436 | 481 | ||
| 437 | /* signal hrtimers about time change */ | 482 | /* signal hrtimers about time change */ |
| 438 | clock_was_set(); | 483 | clock_was_set(); |
| @@ -448,23 +493,24 @@ EXPORT_SYMBOL(timekeeping_inject_offset); | |||
| 448 | */ | 493 | */ |
| 449 | static int change_clocksource(void *data) | 494 | static int change_clocksource(void *data) |
| 450 | { | 495 | { |
| 496 | struct timekeeper *tk = &timekeeper; | ||
| 451 | struct clocksource *new, *old; | 497 | struct clocksource *new, *old; |
| 452 | unsigned long flags; | 498 | unsigned long flags; |
| 453 | 499 | ||
| 454 | new = (struct clocksource *) data; | 500 | new = (struct clocksource *) data; |
| 455 | 501 | ||
| 456 | write_seqlock_irqsave(&timekeeper.lock, flags); | 502 | write_seqlock_irqsave(&tk->lock, flags); |
| 457 | 503 | ||
| 458 | timekeeping_forward_now(); | 504 | timekeeping_forward_now(tk); |
| 459 | if (!new->enable || new->enable(new) == 0) { | 505 | if (!new->enable || new->enable(new) == 0) { |
| 460 | old = timekeeper.clock; | 506 | old = tk->clock; |
| 461 | timekeeper_setup_internals(new); | 507 | tk_setup_internals(tk, new); |
| 462 | if (old->disable) | 508 | if (old->disable) |
| 463 | old->disable(old); | 509 | old->disable(old); |
| 464 | } | 510 | } |
| 465 | timekeeping_update(true); | 511 | timekeeping_update(tk, true); |
| 466 | 512 | ||
| 467 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 513 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 468 | 514 | ||
| 469 | return 0; | 515 | return 0; |
| 470 | } | 516 | } |
| @@ -478,7 +524,9 @@ static int change_clocksource(void *data) | |||
| 478 | */ | 524 | */ |
| 479 | void timekeeping_notify(struct clocksource *clock) | 525 | void timekeeping_notify(struct clocksource *clock) |
| 480 | { | 526 | { |
| 481 | if (timekeeper.clock == clock) | 527 | struct timekeeper *tk = &timekeeper; |
| 528 | |||
| 529 | if (tk->clock == clock) | ||
| 482 | return; | 530 | return; |
| 483 | stop_machine(change_clocksource, clock, NULL); | 531 | stop_machine(change_clocksource, clock, NULL); |
| 484 | tick_clock_notify(); | 532 | tick_clock_notify(); |
| @@ -507,35 +555,36 @@ EXPORT_SYMBOL_GPL(ktime_get_real); | |||
| 507 | */ | 555 | */ |
| 508 | void getrawmonotonic(struct timespec *ts) | 556 | void getrawmonotonic(struct timespec *ts) |
| 509 | { | 557 | { |
| 558 | struct timekeeper *tk = &timekeeper; | ||
| 510 | unsigned long seq; | 559 | unsigned long seq; |
| 511 | s64 nsecs; | 560 | s64 nsecs; |
| 512 | 561 | ||
| 513 | do { | 562 | do { |
| 514 | seq = read_seqbegin(&timekeeper.lock); | 563 | seq = read_seqbegin(&tk->lock); |
| 515 | nsecs = timekeeping_get_ns_raw(); | 564 | nsecs = timekeeping_get_ns_raw(tk); |
| 516 | *ts = timekeeper.raw_time; | 565 | *ts = tk->raw_time; |
| 517 | 566 | ||
| 518 | } while (read_seqretry(&timekeeper.lock, seq)); | 567 | } while (read_seqretry(&tk->lock, seq)); |
| 519 | 568 | ||
| 520 | timespec_add_ns(ts, nsecs); | 569 | timespec_add_ns(ts, nsecs); |
| 521 | } | 570 | } |
| 522 | EXPORT_SYMBOL(getrawmonotonic); | 571 | EXPORT_SYMBOL(getrawmonotonic); |
| 523 | 572 | ||
| 524 | |||
| 525 | /** | 573 | /** |
| 526 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres | 574 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres |
| 527 | */ | 575 | */ |
| 528 | int timekeeping_valid_for_hres(void) | 576 | int timekeeping_valid_for_hres(void) |
| 529 | { | 577 | { |
| 578 | struct timekeeper *tk = &timekeeper; | ||
| 530 | unsigned long seq; | 579 | unsigned long seq; |
| 531 | int ret; | 580 | int ret; |
| 532 | 581 | ||
| 533 | do { | 582 | do { |
| 534 | seq = read_seqbegin(&timekeeper.lock); | 583 | seq = read_seqbegin(&tk->lock); |
| 535 | 584 | ||
| 536 | ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; | 585 | ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; |
| 537 | 586 | ||
| 538 | } while (read_seqretry(&timekeeper.lock, seq)); | 587 | } while (read_seqretry(&tk->lock, seq)); |
| 539 | 588 | ||
| 540 | return ret; | 589 | return ret; |
| 541 | } | 590 | } |
| @@ -545,14 +594,16 @@ int timekeeping_valid_for_hres(void) | |||
| 545 | */ | 594 | */ |
| 546 | u64 timekeeping_max_deferment(void) | 595 | u64 timekeeping_max_deferment(void) |
| 547 | { | 596 | { |
| 597 | struct timekeeper *tk = &timekeeper; | ||
| 548 | unsigned long seq; | 598 | unsigned long seq; |
| 549 | u64 ret; | 599 | u64 ret; |
| 600 | |||
| 550 | do { | 601 | do { |
| 551 | seq = read_seqbegin(&timekeeper.lock); | 602 | seq = read_seqbegin(&tk->lock); |
| 552 | 603 | ||
| 553 | ret = timekeeper.clock->max_idle_ns; | 604 | ret = tk->clock->max_idle_ns; |
| 554 | 605 | ||
| 555 | } while (read_seqretry(&timekeeper.lock, seq)); | 606 | } while (read_seqretry(&tk->lock, seq)); |
| 556 | 607 | ||
| 557 | return ret; | 608 | return ret; |
| 558 | } | 609 | } |
| @@ -592,48 +643,43 @@ void __attribute__((weak)) read_boot_clock(struct timespec *ts) | |||
| 592 | */ | 643 | */ |
| 593 | void __init timekeeping_init(void) | 644 | void __init timekeeping_init(void) |
| 594 | { | 645 | { |
| 646 | struct timekeeper *tk = &timekeeper; | ||
| 595 | struct clocksource *clock; | 647 | struct clocksource *clock; |
| 596 | unsigned long flags; | 648 | unsigned long flags; |
| 597 | struct timespec now, boot; | 649 | struct timespec now, boot, tmp; |
| 598 | 650 | ||
| 599 | read_persistent_clock(&now); | 651 | read_persistent_clock(&now); |
| 600 | read_boot_clock(&boot); | 652 | read_boot_clock(&boot); |
| 601 | 653 | ||
| 602 | seqlock_init(&timekeeper.lock); | 654 | seqlock_init(&tk->lock); |
| 603 | 655 | ||
| 604 | ntp_init(); | 656 | ntp_init(); |
| 605 | 657 | ||
| 606 | write_seqlock_irqsave(&timekeeper.lock, flags); | 658 | write_seqlock_irqsave(&tk->lock, flags); |
| 607 | clock = clocksource_default_clock(); | 659 | clock = clocksource_default_clock(); |
| 608 | if (clock->enable) | 660 | if (clock->enable) |
| 609 | clock->enable(clock); | 661 | clock->enable(clock); |
| 610 | timekeeper_setup_internals(clock); | 662 | tk_setup_internals(tk, clock); |
| 611 | 663 | ||
| 612 | timekeeper.xtime.tv_sec = now.tv_sec; | 664 | tk_set_xtime(tk, &now); |
| 613 | timekeeper.xtime.tv_nsec = now.tv_nsec; | 665 | tk->raw_time.tv_sec = 0; |
| 614 | timekeeper.raw_time.tv_sec = 0; | 666 | tk->raw_time.tv_nsec = 0; |
| 615 | timekeeper.raw_time.tv_nsec = 0; | 667 | if (boot.tv_sec == 0 && boot.tv_nsec == 0) |
| 616 | if (boot.tv_sec == 0 && boot.tv_nsec == 0) { | 668 | boot = tk_xtime(tk); |
| 617 | boot.tv_sec = timekeeper.xtime.tv_sec; | 669 | |
| 618 | boot.tv_nsec = timekeeper.xtime.tv_nsec; | 670 | set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec); |
| 619 | } | 671 | tk_set_wall_to_mono(tk, tmp); |
| 620 | set_normalized_timespec(&timekeeper.wall_to_monotonic, | 672 | |
| 621 | -boot.tv_sec, -boot.tv_nsec); | 673 | tmp.tv_sec = 0; |
| 622 | update_rt_offset(); | 674 | tmp.tv_nsec = 0; |
| 623 | timekeeper.total_sleep_time.tv_sec = 0; | 675 | tk_set_sleep_time(tk, tmp); |
| 624 | timekeeper.total_sleep_time.tv_nsec = 0; | 676 | |
| 625 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 677 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 626 | } | 678 | } |
| 627 | 679 | ||
| 628 | /* time in seconds when suspend began */ | 680 | /* time in seconds when suspend began */ |
| 629 | static struct timespec timekeeping_suspend_time; | 681 | static struct timespec timekeeping_suspend_time; |
| 630 | 682 | ||
| 631 | static void update_sleep_time(struct timespec t) | ||
| 632 | { | ||
| 633 | timekeeper.total_sleep_time = t; | ||
| 634 | timekeeper.offs_boot = timespec_to_ktime(t); | ||
| 635 | } | ||
| 636 | |||
| 637 | /** | 683 | /** |
| 638 | * __timekeeping_inject_sleeptime - Internal function to add sleep interval | 684 | * __timekeeping_inject_sleeptime - Internal function to add sleep interval |
| 639 | * @delta: pointer to a timespec delta value | 685 | * @delta: pointer to a timespec delta value |
| @@ -641,21 +687,19 @@ static void update_sleep_time(struct timespec t) | |||
| 641 | * Takes a timespec offset measuring a suspend interval and properly | 687 | * Takes a timespec offset measuring a suspend interval and properly |
| 642 | * adds the sleep offset to the timekeeping variables. | 688 | * adds the sleep offset to the timekeeping variables. |
| 643 | */ | 689 | */ |
| 644 | static void __timekeeping_inject_sleeptime(struct timespec *delta) | 690 | static void __timekeeping_inject_sleeptime(struct timekeeper *tk, |
| 691 | struct timespec *delta) | ||
| 645 | { | 692 | { |
| 646 | if (!timespec_valid(delta)) { | 693 | if (!timespec_valid(delta)) { |
| 647 | printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid " | 694 | printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid " |
| 648 | "sleep delta value!\n"); | 695 | "sleep delta value!\n"); |
| 649 | return; | 696 | return; |
| 650 | } | 697 | } |
| 651 | 698 | tk_xtime_add(tk, delta); | |
| 652 | timekeeper.xtime = timespec_add(timekeeper.xtime, *delta); | 699 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta)); |
| 653 | timekeeper.wall_to_monotonic = | 700 | tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta)); |
| 654 | timespec_sub(timekeeper.wall_to_monotonic, *delta); | ||
| 655 | update_sleep_time(timespec_add(timekeeper.total_sleep_time, *delta)); | ||
| 656 | } | 701 | } |
| 657 | 702 | ||
| 658 | |||
| 659 | /** | 703 | /** |
| 660 | * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values | 704 | * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values |
| 661 | * @delta: pointer to a timespec delta value | 705 | * @delta: pointer to a timespec delta value |
| @@ -668,6 +712,7 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta) | |||
| 668 | */ | 712 | */ |
| 669 | void timekeeping_inject_sleeptime(struct timespec *delta) | 713 | void timekeeping_inject_sleeptime(struct timespec *delta) |
| 670 | { | 714 | { |
| 715 | struct timekeeper *tk = &timekeeper; | ||
| 671 | unsigned long flags; | 716 | unsigned long flags; |
| 672 | struct timespec ts; | 717 | struct timespec ts; |
| 673 | 718 | ||
| @@ -676,21 +721,20 @@ void timekeeping_inject_sleeptime(struct timespec *delta) | |||
| 676 | if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) | 721 | if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) |
| 677 | return; | 722 | return; |
| 678 | 723 | ||
| 679 | write_seqlock_irqsave(&timekeeper.lock, flags); | 724 | write_seqlock_irqsave(&tk->lock, flags); |
| 680 | 725 | ||
| 681 | timekeeping_forward_now(); | 726 | timekeeping_forward_now(tk); |
| 682 | 727 | ||
| 683 | __timekeeping_inject_sleeptime(delta); | 728 | __timekeeping_inject_sleeptime(tk, delta); |
| 684 | 729 | ||
| 685 | timekeeping_update(true); | 730 | timekeeping_update(tk, true); |
| 686 | 731 | ||
| 687 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 732 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 688 | 733 | ||
| 689 | /* signal hrtimers about time change */ | 734 | /* signal hrtimers about time change */ |
| 690 | clock_was_set(); | 735 | clock_was_set(); |
| 691 | } | 736 | } |
| 692 | 737 | ||
| 693 | |||
| 694 | /** | 738 | /** |
| 695 | * timekeeping_resume - Resumes the generic timekeeping subsystem. | 739 | * timekeeping_resume - Resumes the generic timekeeping subsystem. |
| 696 | * | 740 | * |
| @@ -700,6 +744,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) | |||
| 700 | */ | 744 | */ |
| 701 | static void timekeeping_resume(void) | 745 | static void timekeeping_resume(void) |
| 702 | { | 746 | { |
| 747 | struct timekeeper *tk = &timekeeper; | ||
| 703 | unsigned long flags; | 748 | unsigned long flags; |
| 704 | struct timespec ts; | 749 | struct timespec ts; |
| 705 | 750 | ||
| @@ -707,17 +752,18 @@ static void timekeeping_resume(void) | |||
| 707 | 752 | ||
| 708 | clocksource_resume(); | 753 | clocksource_resume(); |
| 709 | 754 | ||
| 710 | write_seqlock_irqsave(&timekeeper.lock, flags); | 755 | write_seqlock_irqsave(&tk->lock, flags); |
| 711 | 756 | ||
| 712 | if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { | 757 | if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { |
| 713 | ts = timespec_sub(ts, timekeeping_suspend_time); | 758 | ts = timespec_sub(ts, timekeeping_suspend_time); |
| 714 | __timekeeping_inject_sleeptime(&ts); | 759 | __timekeeping_inject_sleeptime(tk, &ts); |
| 715 | } | 760 | } |
| 716 | /* re-base the last cycle value */ | 761 | /* re-base the last cycle value */ |
| 717 | timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); | 762 | tk->clock->cycle_last = tk->clock->read(tk->clock); |
| 718 | timekeeper.ntp_error = 0; | 763 | tk->ntp_error = 0; |
| 719 | timekeeping_suspended = 0; | 764 | timekeeping_suspended = 0; |
| 720 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 765 | timekeeping_update(tk, false); |
| 766 | write_sequnlock_irqrestore(&tk->lock, flags); | ||
| 721 | 767 | ||
| 722 | touch_softlockup_watchdog(); | 768 | touch_softlockup_watchdog(); |
| 723 | 769 | ||
| @@ -729,14 +775,15 @@ static void timekeeping_resume(void) | |||
| 729 | 775 | ||
| 730 | static int timekeeping_suspend(void) | 776 | static int timekeeping_suspend(void) |
| 731 | { | 777 | { |
| 778 | struct timekeeper *tk = &timekeeper; | ||
| 732 | unsigned long flags; | 779 | unsigned long flags; |
| 733 | struct timespec delta, delta_delta; | 780 | struct timespec delta, delta_delta; |
| 734 | static struct timespec old_delta; | 781 | static struct timespec old_delta; |
| 735 | 782 | ||
| 736 | read_persistent_clock(&timekeeping_suspend_time); | 783 | read_persistent_clock(&timekeeping_suspend_time); |
| 737 | 784 | ||
| 738 | write_seqlock_irqsave(&timekeeper.lock, flags); | 785 | write_seqlock_irqsave(&tk->lock, flags); |
| 739 | timekeeping_forward_now(); | 786 | timekeeping_forward_now(tk); |
| 740 | timekeeping_suspended = 1; | 787 | timekeeping_suspended = 1; |
| 741 | 788 | ||
| 742 | /* | 789 | /* |
| @@ -745,7 +792,7 @@ static int timekeeping_suspend(void) | |||
| 745 | * try to compensate so the difference in system time | 792 | * try to compensate so the difference in system time |
| 746 | * and persistent_clock time stays close to constant. | 793 | * and persistent_clock time stays close to constant. |
| 747 | */ | 794 | */ |
| 748 | delta = timespec_sub(timekeeper.xtime, timekeeping_suspend_time); | 795 | delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time); |
| 749 | delta_delta = timespec_sub(delta, old_delta); | 796 | delta_delta = timespec_sub(delta, old_delta); |
| 750 | if (abs(delta_delta.tv_sec) >= 2) { | 797 | if (abs(delta_delta.tv_sec) >= 2) { |
| 751 | /* | 798 | /* |
| @@ -758,7 +805,7 @@ static int timekeeping_suspend(void) | |||
| 758 | timekeeping_suspend_time = | 805 | timekeeping_suspend_time = |
| 759 | timespec_add(timekeeping_suspend_time, delta_delta); | 806 | timespec_add(timekeeping_suspend_time, delta_delta); |
| 760 | } | 807 | } |
| 761 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 808 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 762 | 809 | ||
| 763 | clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); | 810 | clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); |
| 764 | clocksource_suspend(); | 811 | clocksource_suspend(); |
| @@ -784,7 +831,8 @@ device_initcall(timekeeping_init_ops); | |||
| 784 | * If the error is already larger, we look ahead even further | 831 | * If the error is already larger, we look ahead even further |
| 785 | * to compensate for late or lost adjustments. | 832 | * to compensate for late or lost adjustments. |
| 786 | */ | 833 | */ |
| 787 | static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval, | 834 | static __always_inline int timekeeping_bigadjust(struct timekeeper *tk, |
| 835 | s64 error, s64 *interval, | ||
| 788 | s64 *offset) | 836 | s64 *offset) |
| 789 | { | 837 | { |
| 790 | s64 tick_error, i; | 838 | s64 tick_error, i; |
| @@ -800,7 +848,7 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval, | |||
| 800 | * here. This is tuned so that an error of about 1 msec is adjusted | 848 | * here. This is tuned so that an error of about 1 msec is adjusted |
| 801 | * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks). | 849 | * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks). |
| 802 | */ | 850 | */ |
| 803 | error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ); | 851 | error2 = tk->ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ); |
| 804 | error2 = abs(error2); | 852 | error2 = abs(error2); |
| 805 | for (look_ahead = 0; error2 > 0; look_ahead++) | 853 | for (look_ahead = 0; error2 > 0; look_ahead++) |
| 806 | error2 >>= 2; | 854 | error2 >>= 2; |
| @@ -809,8 +857,8 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval, | |||
| 809 | * Now calculate the error in (1 << look_ahead) ticks, but first | 857 | * Now calculate the error in (1 << look_ahead) ticks, but first |
| 810 | * remove the single look ahead already included in the error. | 858 | * remove the single look ahead already included in the error. |
| 811 | */ | 859 | */ |
| 812 | tick_error = ntp_tick_length() >> (timekeeper.ntp_error_shift + 1); | 860 | tick_error = ntp_tick_length() >> (tk->ntp_error_shift + 1); |
| 813 | tick_error -= timekeeper.xtime_interval >> 1; | 861 | tick_error -= tk->xtime_interval >> 1; |
| 814 | error = ((error - tick_error) >> look_ahead) + tick_error; | 862 | error = ((error - tick_error) >> look_ahead) + tick_error; |
| 815 | 863 | ||
| 816 | /* Finally calculate the adjustment shift value. */ | 864 | /* Finally calculate the adjustment shift value. */ |
| @@ -835,9 +883,9 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval, | |||
| 835 | * this is optimized for the most common adjustments of -1,0,1, | 883 | * this is optimized for the most common adjustments of -1,0,1, |
| 836 | * for other values we can do a bit more work. | 884 | * for other values we can do a bit more work. |
| 837 | */ | 885 | */ |
| 838 | static void timekeeping_adjust(s64 offset) | 886 | static void timekeeping_adjust(struct timekeeper *tk, s64 offset) |
| 839 | { | 887 | { |
| 840 | s64 error, interval = timekeeper.cycle_interval; | 888 | s64 error, interval = tk->cycle_interval; |
| 841 | int adj; | 889 | int adj; |
| 842 | 890 | ||
| 843 | /* | 891 | /* |
| @@ -853,7 +901,7 @@ static void timekeeping_adjust(s64 offset) | |||
| 853 | * | 901 | * |
| 854 | * Note: It does not "save" on aggravation when reading the code. | 902 | * Note: It does not "save" on aggravation when reading the code. |
| 855 | */ | 903 | */ |
| 856 | error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1); | 904 | error = tk->ntp_error >> (tk->ntp_error_shift - 1); |
| 857 | if (error > interval) { | 905 | if (error > interval) { |
| 858 | /* | 906 | /* |
| 859 | * We now divide error by 4(via shift), which checks if | 907 | * We now divide error by 4(via shift), which checks if |
| @@ -868,34 +916,36 @@ static void timekeeping_adjust(s64 offset) | |||
| 868 | * the error. This causes the likely below to be unlikely. | 916 | * the error. This causes the likely below to be unlikely. |
| 869 | * | 917 | * |
| 870 | * The proper fix is to avoid rounding up by using | 918 | * The proper fix is to avoid rounding up by using |
| 871 | * the high precision timekeeper.xtime_nsec instead of | 919 | * the high precision tk->xtime_nsec instead of |
| 872 | * xtime.tv_nsec everywhere. Fixing this will take some | 920 | * xtime.tv_nsec everywhere. Fixing this will take some |
| 873 | * time. | 921 | * time. |
| 874 | */ | 922 | */ |
| 875 | if (likely(error <= interval)) | 923 | if (likely(error <= interval)) |
| 876 | adj = 1; | 924 | adj = 1; |
| 877 | else | 925 | else |
| 878 | adj = timekeeping_bigadjust(error, &interval, &offset); | 926 | adj = timekeeping_bigadjust(tk, error, &interval, &offset); |
| 879 | } else if (error < -interval) { | 927 | } else { |
| 880 | /* See comment above, this is just switched for the negative */ | 928 | if (error < -interval) { |
| 881 | error >>= 2; | 929 | /* See comment above, this is just switched for the negative */ |
| 882 | if (likely(error >= -interval)) { | 930 | error >>= 2; |
| 883 | adj = -1; | 931 | if (likely(error >= -interval)) { |
| 884 | interval = -interval; | 932 | adj = -1; |
| 885 | offset = -offset; | 933 | interval = -interval; |
| 886 | } else | 934 | offset = -offset; |
| 887 | adj = timekeeping_bigadjust(error, &interval, &offset); | 935 | } else { |
| 888 | } else /* No adjustment needed */ | 936 | adj = timekeeping_bigadjust(tk, error, &interval, &offset); |
| 889 | return; | 937 | } |
| 938 | } else { | ||
| 939 | goto out_adjust; | ||
| 940 | } | ||
| 941 | } | ||
| 890 | 942 | ||
| 891 | if (unlikely(timekeeper.clock->maxadj && | 943 | if (unlikely(tk->clock->maxadj && |
| 892 | (timekeeper.mult + adj > | 944 | (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) { |
| 893 | timekeeper.clock->mult + timekeeper.clock->maxadj))) { | ||
| 894 | printk_once(KERN_WARNING | 945 | printk_once(KERN_WARNING |
| 895 | "Adjusting %s more than 11%% (%ld vs %ld)\n", | 946 | "Adjusting %s more than 11%% (%ld vs %ld)\n", |
| 896 | timekeeper.clock->name, (long)timekeeper.mult + adj, | 947 | tk->clock->name, (long)tk->mult + adj, |
| 897 | (long)timekeeper.clock->mult + | 948 | (long)tk->clock->mult + tk->clock->maxadj); |
| 898 | timekeeper.clock->maxadj); | ||
| 899 | } | 949 | } |
| 900 | /* | 950 | /* |
| 901 | * So the following can be confusing. | 951 | * So the following can be confusing. |
| @@ -946,13 +996,68 @@ static void timekeeping_adjust(s64 offset) | |||
| 946 | * | 996 | * |
| 947 | * XXX - TODO: Doc ntp_error calculation. | 997 | * XXX - TODO: Doc ntp_error calculation. |
| 948 | */ | 998 | */ |
| 949 | timekeeper.mult += adj; | 999 | tk->mult += adj; |
| 950 | timekeeper.xtime_interval += interval; | 1000 | tk->xtime_interval += interval; |
| 951 | timekeeper.xtime_nsec -= offset; | 1001 | tk->xtime_nsec -= offset; |
| 952 | timekeeper.ntp_error -= (interval - offset) << | 1002 | tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; |
| 953 | timekeeper.ntp_error_shift; | 1003 | |
| 1004 | out_adjust: | ||
| 1005 | /* | ||
| 1006 | * It may be possible that when we entered this function, xtime_nsec | ||
| 1007 | * was very small. Further, if we're slightly speeding the clocksource | ||
| 1008 | * in the code above, its possible the required corrective factor to | ||
| 1009 | * xtime_nsec could cause it to underflow. | ||
| 1010 | * | ||
| 1011 | * Now, since we already accumulated the second, cannot simply roll | ||
| 1012 | * the accumulated second back, since the NTP subsystem has been | ||
| 1013 | * notified via second_overflow. So instead we push xtime_nsec forward | ||
| 1014 | * by the amount we underflowed, and add that amount into the error. | ||
| 1015 | * | ||
| 1016 | * We'll correct this error next time through this function, when | ||
| 1017 | * xtime_nsec is not as small. | ||
| 1018 | */ | ||
| 1019 | if (unlikely((s64)tk->xtime_nsec < 0)) { | ||
| 1020 | s64 neg = -(s64)tk->xtime_nsec; | ||
| 1021 | tk->xtime_nsec = 0; | ||
| 1022 | tk->ntp_error += neg << tk->ntp_error_shift; | ||
| 1023 | } | ||
| 1024 | |||
| 954 | } | 1025 | } |
| 955 | 1026 | ||
| 1027 | /** | ||
| 1028 | * accumulate_nsecs_to_secs - Accumulates nsecs into secs | ||
| 1029 | * | ||
| 1030 | * Helper function that accumulates a the nsecs greater then a second | ||
| 1031 | * from the xtime_nsec field to the xtime_secs field. | ||
| 1032 | * It also calls into the NTP code to handle leapsecond processing. | ||
| 1033 | * | ||
| 1034 | */ | ||
| 1035 | static inline void accumulate_nsecs_to_secs(struct timekeeper *tk) | ||
| 1036 | { | ||
| 1037 | u64 nsecps = (u64)NSEC_PER_SEC << tk->shift; | ||
| 1038 | |||
| 1039 | while (tk->xtime_nsec >= nsecps) { | ||
| 1040 | int leap; | ||
| 1041 | |||
| 1042 | tk->xtime_nsec -= nsecps; | ||
| 1043 | tk->xtime_sec++; | ||
| 1044 | |||
| 1045 | /* Figure out if its a leap sec and apply if needed */ | ||
| 1046 | leap = second_overflow(tk->xtime_sec); | ||
| 1047 | if (unlikely(leap)) { | ||
| 1048 | struct timespec ts; | ||
| 1049 | |||
| 1050 | tk->xtime_sec += leap; | ||
| 1051 | |||
| 1052 | ts.tv_sec = leap; | ||
| 1053 | ts.tv_nsec = 0; | ||
| 1054 | tk_set_wall_to_mono(tk, | ||
| 1055 | timespec_sub(tk->wall_to_monotonic, ts)); | ||
| 1056 | |||
| 1057 | clock_was_set_delayed(); | ||
| 1058 | } | ||
| 1059 | } | ||
| 1060 | } | ||
| 956 | 1061 | ||
| 957 | /** | 1062 | /** |
| 958 | * logarithmic_accumulation - shifted accumulation of cycles | 1063 | * logarithmic_accumulation - shifted accumulation of cycles |
| @@ -963,51 +1068,40 @@ static void timekeeping_adjust(s64 offset) | |||
| 963 | * | 1068 | * |
| 964 | * Returns the unconsumed cycles. | 1069 | * Returns the unconsumed cycles. |
| 965 | */ | 1070 | */ |
| 966 | static cycle_t logarithmic_accumulation(cycle_t offset, int shift) | 1071 | static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, |
| 1072 | u32 shift) | ||
| 967 | { | 1073 | { |
| 968 | u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift; | ||
| 969 | u64 raw_nsecs; | 1074 | u64 raw_nsecs; |
| 970 | 1075 | ||
| 971 | /* If the offset is smaller than a shifted interval, do nothing */ | 1076 | /* If the offset is smaller then a shifted interval, do nothing */ |
| 972 | if (offset < timekeeper.cycle_interval<<shift) | 1077 | if (offset < tk->cycle_interval<<shift) |
| 973 | return offset; | 1078 | return offset; |
| 974 | 1079 | ||
| 975 | /* Accumulate one shifted interval */ | 1080 | /* Accumulate one shifted interval */ |
| 976 | offset -= timekeeper.cycle_interval << shift; | 1081 | offset -= tk->cycle_interval << shift; |
| 977 | timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift; | 1082 | tk->clock->cycle_last += tk->cycle_interval << shift; |
| 978 | 1083 | ||
| 979 | timekeeper.xtime_nsec += timekeeper.xtime_interval << shift; | 1084 | tk->xtime_nsec += tk->xtime_interval << shift; |
| 980 | while (timekeeper.xtime_nsec >= nsecps) { | 1085 | accumulate_nsecs_to_secs(tk); |
| 981 | int leap; | ||
| 982 | timekeeper.xtime_nsec -= nsecps; | ||
| 983 | timekeeper.xtime.tv_sec++; | ||
| 984 | leap = second_overflow(timekeeper.xtime.tv_sec); | ||
| 985 | timekeeper.xtime.tv_sec += leap; | ||
| 986 | timekeeper.wall_to_monotonic.tv_sec -= leap; | ||
| 987 | if (leap) | ||
| 988 | clock_was_set_delayed(); | ||
| 989 | } | ||
| 990 | 1086 | ||
| 991 | /* Accumulate raw time */ | 1087 | /* Accumulate raw time */ |
| 992 | raw_nsecs = timekeeper.raw_interval << shift; | 1088 | raw_nsecs = tk->raw_interval << shift; |
| 993 | raw_nsecs += timekeeper.raw_time.tv_nsec; | 1089 | raw_nsecs += tk->raw_time.tv_nsec; |
| 994 | if (raw_nsecs >= NSEC_PER_SEC) { | 1090 | if (raw_nsecs >= NSEC_PER_SEC) { |
| 995 | u64 raw_secs = raw_nsecs; | 1091 | u64 raw_secs = raw_nsecs; |
| 996 | raw_nsecs = do_div(raw_secs, NSEC_PER_SEC); | 1092 | raw_nsecs = do_div(raw_secs, NSEC_PER_SEC); |
| 997 | timekeeper.raw_time.tv_sec += raw_secs; | 1093 | tk->raw_time.tv_sec += raw_secs; |
| 998 | } | 1094 | } |
| 999 | timekeeper.raw_time.tv_nsec = raw_nsecs; | 1095 | tk->raw_time.tv_nsec = raw_nsecs; |
| 1000 | 1096 | ||
| 1001 | /* Accumulate error between NTP and clock interval */ | 1097 | /* Accumulate error between NTP and clock interval */ |
| 1002 | timekeeper.ntp_error += ntp_tick_length() << shift; | 1098 | tk->ntp_error += ntp_tick_length() << shift; |
| 1003 | timekeeper.ntp_error -= | 1099 | tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << |
| 1004 | (timekeeper.xtime_interval + timekeeper.xtime_remainder) << | 1100 | (tk->ntp_error_shift + shift); |
| 1005 | (timekeeper.ntp_error_shift + shift); | ||
| 1006 | 1101 | ||
| 1007 | return offset; | 1102 | return offset; |
| 1008 | } | 1103 | } |
| 1009 | 1104 | ||
| 1010 | |||
| 1011 | /** | 1105 | /** |
| 1012 | * update_wall_time - Uses the current clocksource to increment the wall time | 1106 | * update_wall_time - Uses the current clocksource to increment the wall time |
| 1013 | * | 1107 | * |
| @@ -1015,25 +1109,25 @@ static cycle_t logarithmic_accumulation(cycle_t offset, int shift) | |||
| 1015 | static void update_wall_time(void) | 1109 | static void update_wall_time(void) |
| 1016 | { | 1110 | { |
| 1017 | struct clocksource *clock; | 1111 | struct clocksource *clock; |
| 1112 | struct timekeeper *tk = &timekeeper; | ||
| 1018 | cycle_t offset; | 1113 | cycle_t offset; |
| 1019 | int shift = 0, maxshift; | 1114 | int shift = 0, maxshift; |
| 1020 | unsigned long flags; | 1115 | unsigned long flags; |
| 1116 | s64 remainder; | ||
| 1021 | 1117 | ||
| 1022 | write_seqlock_irqsave(&timekeeper.lock, flags); | 1118 | write_seqlock_irqsave(&tk->lock, flags); |
| 1023 | 1119 | ||
| 1024 | /* Make sure we're fully resumed: */ | 1120 | /* Make sure we're fully resumed: */ |
| 1025 | if (unlikely(timekeeping_suspended)) | 1121 | if (unlikely(timekeeping_suspended)) |
| 1026 | goto out; | 1122 | goto out; |
| 1027 | 1123 | ||
| 1028 | clock = timekeeper.clock; | 1124 | clock = tk->clock; |
| 1029 | 1125 | ||
| 1030 | #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET | 1126 | #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET |
| 1031 | offset = timekeeper.cycle_interval; | 1127 | offset = tk->cycle_interval; |
| 1032 | #else | 1128 | #else |
| 1033 | offset = (clock->read(clock) - clock->cycle_last) & clock->mask; | 1129 | offset = (clock->read(clock) - clock->cycle_last) & clock->mask; |
| 1034 | #endif | 1130 | #endif |
| 1035 | timekeeper.xtime_nsec = (s64)timekeeper.xtime.tv_nsec << | ||
| 1036 | timekeeper.shift; | ||
| 1037 | 1131 | ||
| 1038 | /* | 1132 | /* |
| 1039 | * With NO_HZ we may have to accumulate many cycle_intervals | 1133 | * With NO_HZ we may have to accumulate many cycle_intervals |
| @@ -1043,73 +1137,45 @@ static void update_wall_time(void) | |||
| 1043 | * chunk in one go, and then try to consume the next smaller | 1137 | * chunk in one go, and then try to consume the next smaller |
| 1044 | * doubled multiple. | 1138 | * doubled multiple. |
| 1045 | */ | 1139 | */ |
| 1046 | shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); | 1140 | shift = ilog2(offset) - ilog2(tk->cycle_interval); |
| 1047 | shift = max(0, shift); | 1141 | shift = max(0, shift); |
| 1048 | /* Bound shift to one less than what overflows tick_length */ | 1142 | /* Bound shift to one less than what overflows tick_length */ |
| 1049 | maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; | 1143 | maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; |
| 1050 | shift = min(shift, maxshift); | 1144 | shift = min(shift, maxshift); |
| 1051 | while (offset >= timekeeper.cycle_interval) { | 1145 | while (offset >= tk->cycle_interval) { |
| 1052 | offset = logarithmic_accumulation(offset, shift); | 1146 | offset = logarithmic_accumulation(tk, offset, shift); |
| 1053 | if(offset < timekeeper.cycle_interval<<shift) | 1147 | if (offset < tk->cycle_interval<<shift) |
| 1054 | shift--; | 1148 | shift--; |
| 1055 | } | 1149 | } |
| 1056 | 1150 | ||
| 1057 | /* correct the clock when NTP error is too big */ | 1151 | /* correct the clock when NTP error is too big */ |
| 1058 | timekeeping_adjust(offset); | 1152 | timekeeping_adjust(tk, offset); |
| 1059 | |||
| 1060 | /* | ||
| 1061 | * Since in the loop above, we accumulate any amount of time | ||
| 1062 | * in xtime_nsec over a second into xtime.tv_sec, its possible for | ||
| 1063 | * xtime_nsec to be fairly small after the loop. Further, if we're | ||
| 1064 | * slightly speeding the clocksource up in timekeeping_adjust(), | ||
| 1065 | * its possible the required corrective factor to xtime_nsec could | ||
| 1066 | * cause it to underflow. | ||
| 1067 | * | ||
| 1068 | * Now, we cannot simply roll the accumulated second back, since | ||
| 1069 | * the NTP subsystem has been notified via second_overflow. So | ||
| 1070 | * instead we push xtime_nsec forward by the amount we underflowed, | ||
| 1071 | * and add that amount into the error. | ||
| 1072 | * | ||
| 1073 | * We'll correct this error next time through this function, when | ||
| 1074 | * xtime_nsec is not as small. | ||
| 1075 | */ | ||
| 1076 | if (unlikely((s64)timekeeper.xtime_nsec < 0)) { | ||
| 1077 | s64 neg = -(s64)timekeeper.xtime_nsec; | ||
| 1078 | timekeeper.xtime_nsec = 0; | ||
| 1079 | timekeeper.ntp_error += neg << timekeeper.ntp_error_shift; | ||
| 1080 | } | ||
| 1081 | 1153 | ||
| 1082 | 1154 | ||
| 1083 | /* | 1155 | /* |
| 1084 | * Store full nanoseconds into xtime after rounding it up and | 1156 | * Store only full nanoseconds into xtime_nsec after rounding |
| 1085 | * add the remainder to the error difference. | 1157 | * it up and add the remainder to the error difference. |
| 1086 | */ | 1158 | * XXX - This is necessary to avoid small 1ns inconsistnecies caused |
| 1087 | timekeeper.xtime.tv_nsec = ((s64)timekeeper.xtime_nsec >> | 1159 | * by truncating the remainder in vsyscalls. However, it causes |
| 1088 | timekeeper.shift) + 1; | 1160 | * additional work to be done in timekeeping_adjust(). Once |
| 1089 | timekeeper.xtime_nsec -= (s64)timekeeper.xtime.tv_nsec << | 1161 | * the vsyscall implementations are converted to use xtime_nsec |
| 1090 | timekeeper.shift; | 1162 | * (shifted nanoseconds), this can be killed. |
| 1091 | timekeeper.ntp_error += timekeeper.xtime_nsec << | 1163 | */ |
| 1092 | timekeeper.ntp_error_shift; | 1164 | remainder = tk->xtime_nsec & ((1 << tk->shift) - 1); |
| 1165 | tk->xtime_nsec -= remainder; | ||
| 1166 | tk->xtime_nsec += 1 << tk->shift; | ||
| 1167 | tk->ntp_error += remainder << tk->ntp_error_shift; | ||
| 1093 | 1168 | ||
| 1094 | /* | 1169 | /* |
| 1095 | * Finally, make sure that after the rounding | 1170 | * Finally, make sure that after the rounding |
| 1096 | * xtime.tv_nsec isn't larger than NSEC_PER_SEC | 1171 | * xtime_nsec isn't larger than NSEC_PER_SEC |
| 1097 | */ | 1172 | */ |
| 1098 | if (unlikely(timekeeper.xtime.tv_nsec >= NSEC_PER_SEC)) { | 1173 | accumulate_nsecs_to_secs(tk); |
| 1099 | int leap; | ||
| 1100 | timekeeper.xtime.tv_nsec -= NSEC_PER_SEC; | ||
| 1101 | timekeeper.xtime.tv_sec++; | ||
| 1102 | leap = second_overflow(timekeeper.xtime.tv_sec); | ||
| 1103 | timekeeper.xtime.tv_sec += leap; | ||
| 1104 | timekeeper.wall_to_monotonic.tv_sec -= leap; | ||
| 1105 | if (leap) | ||
| 1106 | clock_was_set_delayed(); | ||
| 1107 | } | ||
| 1108 | 1174 | ||
| 1109 | timekeeping_update(false); | 1175 | timekeeping_update(tk, false); |
| 1110 | 1176 | ||
| 1111 | out: | 1177 | out: |
| 1112 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 1178 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 1113 | 1179 | ||
| 1114 | } | 1180 | } |
| 1115 | 1181 | ||
| @@ -1126,18 +1192,18 @@ out: | |||
| 1126 | */ | 1192 | */ |
| 1127 | void getboottime(struct timespec *ts) | 1193 | void getboottime(struct timespec *ts) |
| 1128 | { | 1194 | { |
| 1195 | struct timekeeper *tk = &timekeeper; | ||
| 1129 | struct timespec boottime = { | 1196 | struct timespec boottime = { |
| 1130 | .tv_sec = timekeeper.wall_to_monotonic.tv_sec + | 1197 | .tv_sec = tk->wall_to_monotonic.tv_sec + |
| 1131 | timekeeper.total_sleep_time.tv_sec, | 1198 | tk->total_sleep_time.tv_sec, |
| 1132 | .tv_nsec = timekeeper.wall_to_monotonic.tv_nsec + | 1199 | .tv_nsec = tk->wall_to_monotonic.tv_nsec + |
| 1133 | timekeeper.total_sleep_time.tv_nsec | 1200 | tk->total_sleep_time.tv_nsec |
| 1134 | }; | 1201 | }; |
| 1135 | 1202 | ||
| 1136 | set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); | 1203 | set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); |
| 1137 | } | 1204 | } |
| 1138 | EXPORT_SYMBOL_GPL(getboottime); | 1205 | EXPORT_SYMBOL_GPL(getboottime); |
| 1139 | 1206 | ||
| 1140 | |||
| 1141 | /** | 1207 | /** |
| 1142 | * get_monotonic_boottime - Returns monotonic time since boot | 1208 | * get_monotonic_boottime - Returns monotonic time since boot |
| 1143 | * @ts: pointer to the timespec to be set | 1209 | * @ts: pointer to the timespec to be set |
| @@ -1149,23 +1215,23 @@ EXPORT_SYMBOL_GPL(getboottime); | |||
| 1149 | */ | 1215 | */ |
| 1150 | void get_monotonic_boottime(struct timespec *ts) | 1216 | void get_monotonic_boottime(struct timespec *ts) |
| 1151 | { | 1217 | { |
| 1218 | struct timekeeper *tk = &timekeeper; | ||
| 1152 | struct timespec tomono, sleep; | 1219 | struct timespec tomono, sleep; |
| 1153 | unsigned int seq; | 1220 | unsigned int seq; |
| 1154 | s64 nsecs; | ||
| 1155 | 1221 | ||
| 1156 | WARN_ON(timekeeping_suspended); | 1222 | WARN_ON(timekeeping_suspended); |
| 1157 | 1223 | ||
| 1158 | do { | 1224 | do { |
| 1159 | seq = read_seqbegin(&timekeeper.lock); | 1225 | seq = read_seqbegin(&tk->lock); |
| 1160 | *ts = timekeeper.xtime; | 1226 | ts->tv_sec = tk->xtime_sec; |
| 1161 | tomono = timekeeper.wall_to_monotonic; | 1227 | ts->tv_nsec = timekeeping_get_ns(tk); |
| 1162 | sleep = timekeeper.total_sleep_time; | 1228 | tomono = tk->wall_to_monotonic; |
| 1163 | nsecs = timekeeping_get_ns(); | 1229 | sleep = tk->total_sleep_time; |
| 1164 | 1230 | ||
| 1165 | } while (read_seqretry(&timekeeper.lock, seq)); | 1231 | } while (read_seqretry(&tk->lock, seq)); |
| 1166 | 1232 | ||
| 1167 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, | 1233 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, |
| 1168 | ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs); | 1234 | ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec); |
| 1169 | } | 1235 | } |
| 1170 | EXPORT_SYMBOL_GPL(get_monotonic_boottime); | 1236 | EXPORT_SYMBOL_GPL(get_monotonic_boottime); |
| 1171 | 1237 | ||
| @@ -1192,31 +1258,38 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime); | |||
| 1192 | */ | 1258 | */ |
| 1193 | void monotonic_to_bootbased(struct timespec *ts) | 1259 | void monotonic_to_bootbased(struct timespec *ts) |
| 1194 | { | 1260 | { |
| 1195 | *ts = timespec_add(*ts, timekeeper.total_sleep_time); | 1261 | struct timekeeper *tk = &timekeeper; |
| 1262 | |||
| 1263 | *ts = timespec_add(*ts, tk->total_sleep_time); | ||
| 1196 | } | 1264 | } |
| 1197 | EXPORT_SYMBOL_GPL(monotonic_to_bootbased); | 1265 | EXPORT_SYMBOL_GPL(monotonic_to_bootbased); |
| 1198 | 1266 | ||
| 1199 | unsigned long get_seconds(void) | 1267 | unsigned long get_seconds(void) |
| 1200 | { | 1268 | { |
| 1201 | return timekeeper.xtime.tv_sec; | 1269 | struct timekeeper *tk = &timekeeper; |
| 1270 | |||
| 1271 | return tk->xtime_sec; | ||
| 1202 | } | 1272 | } |
| 1203 | EXPORT_SYMBOL(get_seconds); | 1273 | EXPORT_SYMBOL(get_seconds); |
| 1204 | 1274 | ||
| 1205 | struct timespec __current_kernel_time(void) | 1275 | struct timespec __current_kernel_time(void) |
| 1206 | { | 1276 | { |
| 1207 | return timekeeper.xtime; | 1277 | struct timekeeper *tk = &timekeeper; |
| 1278 | |||
| 1279 | return tk_xtime(tk); | ||
| 1208 | } | 1280 | } |
| 1209 | 1281 | ||
| 1210 | struct timespec current_kernel_time(void) | 1282 | struct timespec current_kernel_time(void) |
| 1211 | { | 1283 | { |
| 1284 | struct timekeeper *tk = &timekeeper; | ||
| 1212 | struct timespec now; | 1285 | struct timespec now; |
| 1213 | unsigned long seq; | 1286 | unsigned long seq; |
| 1214 | 1287 | ||
| 1215 | do { | 1288 | do { |
| 1216 | seq = read_seqbegin(&timekeeper.lock); | 1289 | seq = read_seqbegin(&tk->lock); |
| 1217 | 1290 | ||
| 1218 | now = timekeeper.xtime; | 1291 | now = tk_xtime(tk); |
| 1219 | } while (read_seqretry(&timekeeper.lock, seq)); | 1292 | } while (read_seqretry(&tk->lock, seq)); |
| 1220 | 1293 | ||
| 1221 | return now; | 1294 | return now; |
| 1222 | } | 1295 | } |
| @@ -1224,15 +1297,16 @@ EXPORT_SYMBOL(current_kernel_time); | |||
| 1224 | 1297 | ||
| 1225 | struct timespec get_monotonic_coarse(void) | 1298 | struct timespec get_monotonic_coarse(void) |
| 1226 | { | 1299 | { |
| 1300 | struct timekeeper *tk = &timekeeper; | ||
| 1227 | struct timespec now, mono; | 1301 | struct timespec now, mono; |
| 1228 | unsigned long seq; | 1302 | unsigned long seq; |
| 1229 | 1303 | ||
| 1230 | do { | 1304 | do { |
| 1231 | seq = read_seqbegin(&timekeeper.lock); | 1305 | seq = read_seqbegin(&tk->lock); |
| 1232 | 1306 | ||
| 1233 | now = timekeeper.xtime; | 1307 | now = tk_xtime(tk); |
| 1234 | mono = timekeeper.wall_to_monotonic; | 1308 | mono = tk->wall_to_monotonic; |
| 1235 | } while (read_seqretry(&timekeeper.lock, seq)); | 1309 | } while (read_seqretry(&tk->lock, seq)); |
| 1236 | 1310 | ||
| 1237 | set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, | 1311 | set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, |
| 1238 | now.tv_nsec + mono.tv_nsec); | 1312 | now.tv_nsec + mono.tv_nsec); |
| @@ -1261,14 +1335,15 @@ void do_timer(unsigned long ticks) | |||
| 1261 | void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, | 1335 | void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, |
| 1262 | struct timespec *wtom, struct timespec *sleep) | 1336 | struct timespec *wtom, struct timespec *sleep) |
| 1263 | { | 1337 | { |
| 1338 | struct timekeeper *tk = &timekeeper; | ||
| 1264 | unsigned long seq; | 1339 | unsigned long seq; |
| 1265 | 1340 | ||
| 1266 | do { | 1341 | do { |
| 1267 | seq = read_seqbegin(&timekeeper.lock); | 1342 | seq = read_seqbegin(&tk->lock); |
| 1268 | *xtim = timekeeper.xtime; | 1343 | *xtim = tk_xtime(tk); |
| 1269 | *wtom = timekeeper.wall_to_monotonic; | 1344 | *wtom = tk->wall_to_monotonic; |
| 1270 | *sleep = timekeeper.total_sleep_time; | 1345 | *sleep = tk->total_sleep_time; |
| 1271 | } while (read_seqretry(&timekeeper.lock, seq)); | 1346 | } while (read_seqretry(&tk->lock, seq)); |
| 1272 | } | 1347 | } |
| 1273 | 1348 | ||
| 1274 | #ifdef CONFIG_HIGH_RES_TIMERS | 1349 | #ifdef CONFIG_HIGH_RES_TIMERS |
| @@ -1282,22 +1357,20 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, | |||
| 1282 | */ | 1357 | */ |
| 1283 | ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) | 1358 | ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) |
| 1284 | { | 1359 | { |
| 1360 | struct timekeeper *tk = &timekeeper; | ||
| 1285 | ktime_t now; | 1361 | ktime_t now; |
| 1286 | unsigned int seq; | 1362 | unsigned int seq; |
| 1287 | u64 secs, nsecs; | 1363 | u64 secs, nsecs; |
| 1288 | 1364 | ||
| 1289 | do { | 1365 | do { |
| 1290 | seq = read_seqbegin(&timekeeper.lock); | 1366 | seq = read_seqbegin(&tk->lock); |
| 1291 | 1367 | ||
| 1292 | secs = timekeeper.xtime.tv_sec; | 1368 | secs = tk->xtime_sec; |
| 1293 | nsecs = timekeeper.xtime.tv_nsec; | 1369 | nsecs = timekeeping_get_ns(tk); |
| 1294 | nsecs += timekeeping_get_ns(); | ||
| 1295 | /* If arch requires, add in gettimeoffset() */ | ||
| 1296 | nsecs += arch_gettimeoffset(); | ||
| 1297 | 1370 | ||
| 1298 | *offs_real = timekeeper.offs_real; | 1371 | *offs_real = tk->offs_real; |
| 1299 | *offs_boot = timekeeper.offs_boot; | 1372 | *offs_boot = tk->offs_boot; |
| 1300 | } while (read_seqretry(&timekeeper.lock, seq)); | 1373 | } while (read_seqretry(&tk->lock, seq)); |
| 1301 | 1374 | ||
| 1302 | now = ktime_add_ns(ktime_set(secs, 0), nsecs); | 1375 | now = ktime_add_ns(ktime_set(secs, 0), nsecs); |
| 1303 | now = ktime_sub(now, *offs_real); | 1376 | now = ktime_sub(now, *offs_real); |
| @@ -1310,19 +1383,19 @@ ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) | |||
| 1310 | */ | 1383 | */ |
| 1311 | ktime_t ktime_get_monotonic_offset(void) | 1384 | ktime_t ktime_get_monotonic_offset(void) |
| 1312 | { | 1385 | { |
| 1386 | struct timekeeper *tk = &timekeeper; | ||
| 1313 | unsigned long seq; | 1387 | unsigned long seq; |
| 1314 | struct timespec wtom; | 1388 | struct timespec wtom; |
| 1315 | 1389 | ||
| 1316 | do { | 1390 | do { |
| 1317 | seq = read_seqbegin(&timekeeper.lock); | 1391 | seq = read_seqbegin(&tk->lock); |
| 1318 | wtom = timekeeper.wall_to_monotonic; | 1392 | wtom = tk->wall_to_monotonic; |
| 1319 | } while (read_seqretry(&timekeeper.lock, seq)); | 1393 | } while (read_seqretry(&tk->lock, seq)); |
| 1320 | 1394 | ||
| 1321 | return timespec_to_ktime(wtom); | 1395 | return timespec_to_ktime(wtom); |
| 1322 | } | 1396 | } |
| 1323 | EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); | 1397 | EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); |
| 1324 | 1398 | ||
| 1325 | |||
| 1326 | /** | 1399 | /** |
| 1327 | * xtime_update() - advances the timekeeping infrastructure | 1400 | * xtime_update() - advances the timekeeping infrastructure |
| 1328 | * @ticks: number of ticks, that have elapsed since the last call. | 1401 | * @ticks: number of ticks, that have elapsed since the last call. |
