diff options
Diffstat (limited to 'kernel/sched_clock.c')
| -rw-r--r-- | kernel/sched_clock.c | 109 |
1 files changed, 13 insertions, 96 deletions
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c index 5a2dc7d8fd98..9a7844158ae8 100644 --- a/kernel/sched_clock.c +++ b/kernel/sched_clock.c | |||
| @@ -44,11 +44,6 @@ unsigned long long __attribute__((weak)) sched_clock(void) | |||
| 44 | 44 | ||
| 45 | #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK | 45 | #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK |
| 46 | 46 | ||
| 47 | #define MULTI_SHIFT 15 | ||
| 48 | /* Max is double, Min is 1/2 */ | ||
| 49 | #define MAX_MULTI (2LL << MULTI_SHIFT) | ||
| 50 | #define MIN_MULTI (1LL << (MULTI_SHIFT-1)) | ||
| 51 | |||
| 52 | struct sched_clock_data { | 47 | struct sched_clock_data { |
| 53 | /* | 48 | /* |
| 54 | * Raw spinlock - this is a special case: this might be called | 49 | * Raw spinlock - this is a special case: this might be called |
| @@ -62,10 +57,6 @@ struct sched_clock_data { | |||
| 62 | u64 tick_raw; | 57 | u64 tick_raw; |
| 63 | u64 tick_gtod; | 58 | u64 tick_gtod; |
| 64 | u64 clock; | 59 | u64 clock; |
| 65 | s64 multi; | ||
| 66 | #ifdef CONFIG_NO_HZ | ||
| 67 | int check_max; | ||
| 68 | #endif | ||
| 69 | }; | 60 | }; |
| 70 | 61 | ||
| 71 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); | 62 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); |
| @@ -97,53 +88,18 @@ void sched_clock_init(void) | |||
| 97 | scd->tick_raw = 0; | 88 | scd->tick_raw = 0; |
| 98 | scd->tick_gtod = ktime_now; | 89 | scd->tick_gtod = ktime_now; |
| 99 | scd->clock = ktime_now; | 90 | scd->clock = ktime_now; |
| 100 | scd->multi = 1 << MULTI_SHIFT; | ||
| 101 | #ifdef CONFIG_NO_HZ | ||
| 102 | scd->check_max = 1; | ||
| 103 | #endif | ||
| 104 | } | 91 | } |
| 105 | 92 | ||
| 106 | sched_clock_running = 1; | 93 | sched_clock_running = 1; |
| 107 | } | 94 | } |
| 108 | 95 | ||
| 109 | #ifdef CONFIG_NO_HZ | ||
| 110 | /* | ||
| 111 | * The dynamic ticks makes the delta jiffies inaccurate. This | ||
| 112 | * prevents us from checking the maximum time update. | ||
| 113 | * Disable the maximum check during stopped ticks. | ||
| 114 | */ | ||
| 115 | void sched_clock_tick_stop(int cpu) | ||
| 116 | { | ||
| 117 | struct sched_clock_data *scd = cpu_sdc(cpu); | ||
| 118 | |||
| 119 | scd->check_max = 0; | ||
| 120 | } | ||
| 121 | |||
| 122 | void sched_clock_tick_start(int cpu) | ||
| 123 | { | ||
| 124 | struct sched_clock_data *scd = cpu_sdc(cpu); | ||
| 125 | |||
| 126 | scd->check_max = 1; | ||
| 127 | } | ||
| 128 | |||
| 129 | static int check_max(struct sched_clock_data *scd) | ||
| 130 | { | ||
| 131 | return scd->check_max; | ||
| 132 | } | ||
| 133 | #else | ||
| 134 | static int check_max(struct sched_clock_data *scd) | ||
| 135 | { | ||
| 136 | return 1; | ||
| 137 | } | ||
| 138 | #endif /* CONFIG_NO_HZ */ | ||
| 139 | |||
| 140 | /* | 96 | /* |
| 141 | * update the percpu scd from the raw @now value | 97 | * update the percpu scd from the raw @now value |
| 142 | * | 98 | * |
| 143 | * - filter out backward motion | 99 | * - filter out backward motion |
| 144 | * - use jiffies to generate a min,max window to clip the raw values | 100 | * - use jiffies to generate a min,max window to clip the raw values |
| 145 | */ | 101 | */ |
| 146 | static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time) | 102 | static void __update_sched_clock(struct sched_clock_data *scd, u64 now) |
| 147 | { | 103 | { |
| 148 | unsigned long now_jiffies = jiffies; | 104 | unsigned long now_jiffies = jiffies; |
| 149 | long delta_jiffies = now_jiffies - scd->tick_jiffies; | 105 | long delta_jiffies = now_jiffies - scd->tick_jiffies; |
| @@ -152,31 +108,16 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim | |||
| 152 | s64 delta = now - scd->prev_raw; | 108 | s64 delta = now - scd->prev_raw; |
| 153 | 109 | ||
| 154 | WARN_ON_ONCE(!irqs_disabled()); | 110 | WARN_ON_ONCE(!irqs_disabled()); |
| 155 | 111 | min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC; | |
| 156 | /* | ||
| 157 | * At schedule tick the clock can be just under the gtod. We don't | ||
| 158 | * want to push it too prematurely. | ||
| 159 | */ | ||
| 160 | min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC); | ||
| 161 | if (min_clock > TICK_NSEC) | ||
| 162 | min_clock -= TICK_NSEC / 2; | ||
| 163 | 112 | ||
| 164 | if (unlikely(delta < 0)) { | 113 | if (unlikely(delta < 0)) { |
| 165 | clock++; | 114 | clock++; |
| 166 | goto out; | 115 | goto out; |
| 167 | } | 116 | } |
| 168 | 117 | ||
| 169 | /* | 118 | max_clock = min_clock + TICK_NSEC; |
| 170 | * The clock must stay within a jiffie of the gtod. | ||
| 171 | * But since we may be at the start of a jiffy or the end of one | ||
| 172 | * we add another jiffy buffer. | ||
| 173 | */ | ||
| 174 | max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC; | ||
| 175 | |||
| 176 | delta *= scd->multi; | ||
| 177 | delta >>= MULTI_SHIFT; | ||
| 178 | 119 | ||
| 179 | if (unlikely(clock + delta > max_clock) && check_max(scd)) { | 120 | if (unlikely(clock + delta > max_clock)) { |
| 180 | if (clock < max_clock) | 121 | if (clock < max_clock) |
| 181 | clock = max_clock; | 122 | clock = max_clock; |
| 182 | else | 123 | else |
| @@ -189,12 +130,9 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim | |||
| 189 | if (unlikely(clock < min_clock)) | 130 | if (unlikely(clock < min_clock)) |
| 190 | clock = min_clock; | 131 | clock = min_clock; |
| 191 | 132 | ||
| 192 | if (time) | 133 | scd->prev_raw = now; |
| 193 | *time = clock; | 134 | scd->tick_jiffies = now_jiffies; |
| 194 | else { | 135 | scd->clock = clock; |
| 195 | scd->prev_raw = now; | ||
| 196 | scd->clock = clock; | ||
| 197 | } | ||
| 198 | } | 136 | } |
| 199 | 137 | ||
| 200 | static void lock_double_clock(struct sched_clock_data *data1, | 138 | static void lock_double_clock(struct sched_clock_data *data1, |
| @@ -238,26 +176,21 @@ u64 sched_clock_cpu(int cpu) | |||
| 238 | now -= scd->tick_gtod; | 176 | now -= scd->tick_gtod; |
| 239 | 177 | ||
| 240 | __raw_spin_unlock(&my_scd->lock); | 178 | __raw_spin_unlock(&my_scd->lock); |
| 241 | |||
| 242 | __update_sched_clock(scd, now, &clock); | ||
| 243 | |||
| 244 | __raw_spin_unlock(&scd->lock); | ||
| 245 | |||
| 246 | } else { | 179 | } else { |
| 247 | __raw_spin_lock(&scd->lock); | 180 | __raw_spin_lock(&scd->lock); |
| 248 | __update_sched_clock(scd, now, NULL); | ||
| 249 | clock = scd->clock; | ||
| 250 | __raw_spin_unlock(&scd->lock); | ||
| 251 | } | 181 | } |
| 252 | 182 | ||
| 183 | __update_sched_clock(scd, now); | ||
| 184 | clock = scd->clock; | ||
| 185 | |||
| 186 | __raw_spin_unlock(&scd->lock); | ||
| 187 | |||
| 253 | return clock; | 188 | return clock; |
| 254 | } | 189 | } |
| 255 | 190 | ||
| 256 | void sched_clock_tick(void) | 191 | void sched_clock_tick(void) |
| 257 | { | 192 | { |
| 258 | struct sched_clock_data *scd = this_scd(); | 193 | struct sched_clock_data *scd = this_scd(); |
| 259 | unsigned long now_jiffies = jiffies; | ||
| 260 | s64 mult, delta_gtod, delta_raw; | ||
| 261 | u64 now, now_gtod; | 194 | u64 now, now_gtod; |
| 262 | 195 | ||
| 263 | if (unlikely(!sched_clock_running)) | 196 | if (unlikely(!sched_clock_running)) |
| @@ -269,29 +202,14 @@ void sched_clock_tick(void) | |||
| 269 | now = sched_clock(); | 202 | now = sched_clock(); |
| 270 | 203 | ||
| 271 | __raw_spin_lock(&scd->lock); | 204 | __raw_spin_lock(&scd->lock); |
| 272 | __update_sched_clock(scd, now, NULL); | 205 | __update_sched_clock(scd, now); |
| 273 | /* | 206 | /* |
| 274 | * update tick_gtod after __update_sched_clock() because that will | 207 | * update tick_gtod after __update_sched_clock() because that will |
| 275 | * already observe 1 new jiffy; adding a new tick_gtod to that would | 208 | * already observe 1 new jiffy; adding a new tick_gtod to that would |
| 276 | * increase the clock 2 jiffies. | 209 | * increase the clock 2 jiffies. |
| 277 | */ | 210 | */ |
| 278 | delta_gtod = now_gtod - scd->tick_gtod; | ||
| 279 | delta_raw = now - scd->tick_raw; | ||
| 280 | |||
| 281 | if ((long)delta_raw > 0) { | ||
| 282 | mult = delta_gtod << MULTI_SHIFT; | ||
| 283 | do_div(mult, delta_raw); | ||
| 284 | scd->multi = mult; | ||
| 285 | if (scd->multi > MAX_MULTI) | ||
| 286 | scd->multi = MAX_MULTI; | ||
| 287 | else if (scd->multi < MIN_MULTI) | ||
| 288 | scd->multi = MIN_MULTI; | ||
| 289 | } else | ||
| 290 | scd->multi = 1 << MULTI_SHIFT; | ||
| 291 | |||
| 292 | scd->tick_raw = now; | 211 | scd->tick_raw = now; |
| 293 | scd->tick_gtod = now_gtod; | 212 | scd->tick_gtod = now_gtod; |
| 294 | scd->tick_jiffies = now_jiffies; | ||
| 295 | __raw_spin_unlock(&scd->lock); | 213 | __raw_spin_unlock(&scd->lock); |
| 296 | } | 214 | } |
| 297 | 215 | ||
| @@ -321,7 +239,6 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) | |||
| 321 | __raw_spin_lock(&scd->lock); | 239 | __raw_spin_lock(&scd->lock); |
| 322 | scd->prev_raw = now; | 240 | scd->prev_raw = now; |
| 323 | scd->clock += delta_ns; | 241 | scd->clock += delta_ns; |
| 324 | scd->multi = 1 << MULTI_SHIFT; | ||
| 325 | __raw_spin_unlock(&scd->lock); | 242 | __raw_spin_unlock(&scd->lock); |
| 326 | 243 | ||
| 327 | touch_softlockup_watchdog(); | 244 | touch_softlockup_watchdog(); |
