diff options
Diffstat (limited to 'kernel/time/timekeeping.c')
| -rw-r--r-- | kernel/time/timekeeping.c | 117 |
1 files changed, 34 insertions, 83 deletions
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 5ce06a3fa91e..e424970bb562 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
| @@ -8,6 +8,7 @@ | |||
| 8 | * | 8 | * |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | #include <linux/timekeeper_internal.h> | ||
| 11 | #include <linux/module.h> | 12 | #include <linux/module.h> |
| 12 | #include <linux/interrupt.h> | 13 | #include <linux/interrupt.h> |
| 13 | #include <linux/percpu.h> | 14 | #include <linux/percpu.h> |
| @@ -21,61 +22,6 @@ | |||
| 21 | #include <linux/tick.h> | 22 | #include <linux/tick.h> |
| 22 | #include <linux/stop_machine.h> | 23 | #include <linux/stop_machine.h> |
| 23 | 24 | ||
| 24 | /* Structure holding internal timekeeping values. */ | ||
| 25 | struct timekeeper { | ||
| 26 | /* Current clocksource used for timekeeping. */ | ||
| 27 | struct clocksource *clock; | ||
| 28 | /* NTP adjusted clock multiplier */ | ||
| 29 | u32 mult; | ||
| 30 | /* The shift value of the current clocksource. */ | ||
| 31 | u32 shift; | ||
| 32 | /* Number of clock cycles in one NTP interval. */ | ||
| 33 | cycle_t cycle_interval; | ||
| 34 | /* Number of clock shifted nano seconds in one NTP interval. */ | ||
| 35 | u64 xtime_interval; | ||
| 36 | /* shifted nano seconds left over when rounding cycle_interval */ | ||
| 37 | s64 xtime_remainder; | ||
| 38 | /* Raw nano seconds accumulated per NTP interval. */ | ||
| 39 | u32 raw_interval; | ||
| 40 | |||
| 41 | /* Current CLOCK_REALTIME time in seconds */ | ||
| 42 | u64 xtime_sec; | ||
| 43 | /* Clock shifted nano seconds */ | ||
| 44 | u64 xtime_nsec; | ||
| 45 | |||
| 46 | /* Difference between accumulated time and NTP time in ntp | ||
| 47 | * shifted nano seconds. */ | ||
| 48 | s64 ntp_error; | ||
| 49 | /* Shift conversion between clock shifted nano seconds and | ||
| 50 | * ntp shifted nano seconds. */ | ||
| 51 | u32 ntp_error_shift; | ||
| 52 | |||
| 53 | /* | ||
| 54 | * wall_to_monotonic is what we need to add to xtime (or xtime corrected | ||
| 55 | * for sub jiffie times) to get to monotonic time. Monotonic is pegged | ||
| 56 | * at zero at system boot time, so wall_to_monotonic will be negative, | ||
| 57 | * however, we will ALWAYS keep the tv_nsec part positive so we can use | ||
| 58 | * the usual normalization. | ||
| 59 | * | ||
| 60 | * wall_to_monotonic is moved after resume from suspend for the | ||
| 61 | * monotonic time not to jump. We need to add total_sleep_time to | ||
| 62 | * wall_to_monotonic to get the real boot based time offset. | ||
| 63 | * | ||
| 64 | * - wall_to_monotonic is no longer the boot time, getboottime must be | ||
| 65 | * used instead. | ||
| 66 | */ | ||
| 67 | struct timespec wall_to_monotonic; | ||
| 68 | /* Offset clock monotonic -> clock realtime */ | ||
| 69 | ktime_t offs_real; | ||
| 70 | /* time spent in suspend */ | ||
| 71 | struct timespec total_sleep_time; | ||
| 72 | /* Offset clock monotonic -> clock boottime */ | ||
| 73 | ktime_t offs_boot; | ||
| 74 | /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ | ||
| 75 | struct timespec raw_time; | ||
| 76 | /* Seqlock for all timekeeper values */ | ||
| 77 | seqlock_t lock; | ||
| 78 | }; | ||
| 79 | 25 | ||
| 80 | static struct timekeeper timekeeper; | 26 | static struct timekeeper timekeeper; |
| 81 | 27 | ||
| @@ -96,15 +42,6 @@ static inline void tk_normalize_xtime(struct timekeeper *tk) | |||
| 96 | } | 42 | } |
| 97 | } | 43 | } |
| 98 | 44 | ||
| 99 | static struct timespec tk_xtime(struct timekeeper *tk) | ||
| 100 | { | ||
| 101 | struct timespec ts; | ||
| 102 | |||
| 103 | ts.tv_sec = tk->xtime_sec; | ||
| 104 | ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift); | ||
| 105 | return ts; | ||
| 106 | } | ||
| 107 | |||
| 108 | static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) | 45 | static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) |
| 109 | { | 46 | { |
| 110 | tk->xtime_sec = ts->tv_sec; | 47 | tk->xtime_sec = ts->tv_sec; |
| @@ -246,14 +183,11 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) | |||
| 246 | /* must hold write on timekeeper.lock */ | 183 | /* must hold write on timekeeper.lock */ |
| 247 | static void timekeeping_update(struct timekeeper *tk, bool clearntp) | 184 | static void timekeeping_update(struct timekeeper *tk, bool clearntp) |
| 248 | { | 185 | { |
| 249 | struct timespec xt; | ||
| 250 | |||
| 251 | if (clearntp) { | 186 | if (clearntp) { |
| 252 | tk->ntp_error = 0; | 187 | tk->ntp_error = 0; |
| 253 | ntp_clear(); | 188 | ntp_clear(); |
| 254 | } | 189 | } |
| 255 | xt = tk_xtime(tk); | 190 | update_vsyscall(tk); |
| 256 | update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); | ||
| 257 | } | 191 | } |
| 258 | 192 | ||
| 259 | /** | 193 | /** |
| @@ -1113,7 +1047,7 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, | |||
| 1113 | accumulate_nsecs_to_secs(tk); | 1047 | accumulate_nsecs_to_secs(tk); |
| 1114 | 1048 | ||
| 1115 | /* Accumulate raw time */ | 1049 | /* Accumulate raw time */ |
| 1116 | raw_nsecs = tk->raw_interval << shift; | 1050 | raw_nsecs = (u64)tk->raw_interval << shift; |
| 1117 | raw_nsecs += tk->raw_time.tv_nsec; | 1051 | raw_nsecs += tk->raw_time.tv_nsec; |
| 1118 | if (raw_nsecs >= NSEC_PER_SEC) { | 1052 | if (raw_nsecs >= NSEC_PER_SEC) { |
| 1119 | u64 raw_secs = raw_nsecs; | 1053 | u64 raw_secs = raw_nsecs; |
| @@ -1130,6 +1064,33 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, | |||
| 1130 | return offset; | 1064 | return offset; |
| 1131 | } | 1065 | } |
| 1132 | 1066 | ||
| 1067 | #ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD | ||
| 1068 | static inline void old_vsyscall_fixup(struct timekeeper *tk) | ||
| 1069 | { | ||
| 1070 | s64 remainder; | ||
| 1071 | |||
| 1072 | /* | ||
| 1073 | * Store only full nanoseconds into xtime_nsec after rounding | ||
| 1074 | * it up and add the remainder to the error difference. | ||
| 1075 | * XXX - This is necessary to avoid small 1ns inconsistnecies caused | ||
| 1076 | * by truncating the remainder in vsyscalls. However, it causes | ||
| 1077 | * additional work to be done in timekeeping_adjust(). Once | ||
| 1078 | * the vsyscall implementations are converted to use xtime_nsec | ||
| 1079 | * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD | ||
| 1080 | * users are removed, this can be killed. | ||
| 1081 | */ | ||
| 1082 | remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1); | ||
| 1083 | tk->xtime_nsec -= remainder; | ||
| 1084 | tk->xtime_nsec += 1ULL << tk->shift; | ||
| 1085 | tk->ntp_error += remainder << tk->ntp_error_shift; | ||
| 1086 | |||
| 1087 | } | ||
| 1088 | #else | ||
| 1089 | #define old_vsyscall_fixup(tk) | ||
| 1090 | #endif | ||
| 1091 | |||
| 1092 | |||
| 1093 | |||
| 1133 | /** | 1094 | /** |
| 1134 | * update_wall_time - Uses the current clocksource to increment the wall time | 1095 | * update_wall_time - Uses the current clocksource to increment the wall time |
| 1135 | * | 1096 | * |
| @@ -1141,7 +1102,6 @@ static void update_wall_time(void) | |||
| 1141 | cycle_t offset; | 1102 | cycle_t offset; |
| 1142 | int shift = 0, maxshift; | 1103 | int shift = 0, maxshift; |
| 1143 | unsigned long flags; | 1104 | unsigned long flags; |
| 1144 | s64 remainder; | ||
| 1145 | 1105 | ||
| 1146 | write_seqlock_irqsave(&tk->lock, flags); | 1106 | write_seqlock_irqsave(&tk->lock, flags); |
| 1147 | 1107 | ||
| @@ -1183,20 +1143,11 @@ static void update_wall_time(void) | |||
| 1183 | /* correct the clock when NTP error is too big */ | 1143 | /* correct the clock when NTP error is too big */ |
| 1184 | timekeeping_adjust(tk, offset); | 1144 | timekeeping_adjust(tk, offset); |
| 1185 | 1145 | ||
| 1186 | |||
| 1187 | /* | 1146 | /* |
| 1188 | * Store only full nanoseconds into xtime_nsec after rounding | 1147 | * XXX This can be killed once everyone converts |
| 1189 | * it up and add the remainder to the error difference. | 1148 | * to the new update_vsyscall. |
| 1190 | * XXX - This is necessary to avoid small 1ns inconsistnecies caused | 1149 | */ |
| 1191 | * by truncating the remainder in vsyscalls. However, it causes | 1150 | old_vsyscall_fixup(tk); |
| 1192 | * additional work to be done in timekeeping_adjust(). Once | ||
| 1193 | * the vsyscall implementations are converted to use xtime_nsec | ||
| 1194 | * (shifted nanoseconds), this can be killed. | ||
| 1195 | */ | ||
| 1196 | remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1); | ||
| 1197 | tk->xtime_nsec -= remainder; | ||
| 1198 | tk->xtime_nsec += 1ULL << tk->shift; | ||
| 1199 | tk->ntp_error += remainder << tk->ntp_error_shift; | ||
| 1200 | 1151 | ||
| 1201 | /* | 1152 | /* |
| 1202 | * Finally, make sure that after the rounding | 1153 | * Finally, make sure that after the rounding |
