diff options
Diffstat (limited to 'arch/x86/kernel/tsc.c')
-rw-r--r-- | arch/x86/kernel/tsc.c | 290 |
1 files changed, 232 insertions, 58 deletions
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 8f98e9de1b8..161bb850fc4 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c | |||
@@ -104,7 +104,7 @@ __setup("notsc", notsc_setup); | |||
104 | /* | 104 | /* |
105 | * Read TSC and the reference counters. Take care of SMI disturbance | 105 | * Read TSC and the reference counters. Take care of SMI disturbance |
106 | */ | 106 | */ |
107 | static u64 tsc_read_refs(u64 *pm, u64 *hpet) | 107 | static u64 tsc_read_refs(u64 *p, int hpet) |
108 | { | 108 | { |
109 | u64 t1, t2; | 109 | u64 t1, t2; |
110 | int i; | 110 | int i; |
@@ -112,9 +112,9 @@ static u64 tsc_read_refs(u64 *pm, u64 *hpet) | |||
112 | for (i = 0; i < MAX_RETRIES; i++) { | 112 | for (i = 0; i < MAX_RETRIES; i++) { |
113 | t1 = get_cycles(); | 113 | t1 = get_cycles(); |
114 | if (hpet) | 114 | if (hpet) |
115 | *hpet = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF; | 115 | *p = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF; |
116 | else | 116 | else |
117 | *pm = acpi_pm_read_early(); | 117 | *p = acpi_pm_read_early(); |
118 | t2 = get_cycles(); | 118 | t2 = get_cycles(); |
119 | if ((t2 - t1) < SMI_TRESHOLD) | 119 | if ((t2 - t1) < SMI_TRESHOLD) |
120 | return t2; | 120 | return t2; |
@@ -123,13 +123,59 @@ static u64 tsc_read_refs(u64 *pm, u64 *hpet) | |||
123 | } | 123 | } |
124 | 124 | ||
125 | /* | 125 | /* |
126 | * Calculate the TSC frequency from HPET reference | ||
127 | */ | ||
128 | static unsigned long calc_hpet_ref(u64 deltatsc, u64 hpet1, u64 hpet2) | ||
129 | { | ||
130 | u64 tmp; | ||
131 | |||
132 | if (hpet2 < hpet1) | ||
133 | hpet2 += 0x100000000ULL; | ||
134 | hpet2 -= hpet1; | ||
135 | tmp = ((u64)hpet2 * hpet_readl(HPET_PERIOD)); | ||
136 | do_div(tmp, 1000000); | ||
137 | do_div(deltatsc, tmp); | ||
138 | |||
139 | return (unsigned long) deltatsc; | ||
140 | } | ||
141 | |||
142 | /* | ||
143 | * Calculate the TSC frequency from PMTimer reference | ||
144 | */ | ||
145 | static unsigned long calc_pmtimer_ref(u64 deltatsc, u64 pm1, u64 pm2) | ||
146 | { | ||
147 | u64 tmp; | ||
148 | |||
149 | if (!pm1 && !pm2) | ||
150 | return ULONG_MAX; | ||
151 | |||
152 | if (pm2 < pm1) | ||
153 | pm2 += (u64)ACPI_PM_OVRRUN; | ||
154 | pm2 -= pm1; | ||
155 | tmp = pm2 * 1000000000LL; | ||
156 | do_div(tmp, PMTMR_TICKS_PER_SEC); | ||
157 | do_div(deltatsc, tmp); | ||
158 | |||
159 | return (unsigned long) deltatsc; | ||
160 | } | ||
161 | |||
162 | #define CAL_MS 10 | ||
163 | #define CAL_LATCH (CLOCK_TICK_RATE / (1000 / CAL_MS)) | ||
164 | #define CAL_PIT_LOOPS 1000 | ||
165 | |||
166 | #define CAL2_MS 50 | ||
167 | #define CAL2_LATCH (CLOCK_TICK_RATE / (1000 / CAL2_MS)) | ||
168 | #define CAL2_PIT_LOOPS 5000 | ||
169 | |||
170 | |||
171 | /* | ||
126 | * Try to calibrate the TSC against the Programmable | 172 | * Try to calibrate the TSC against the Programmable |
127 | * Interrupt Timer and return the frequency of the TSC | 173 | * Interrupt Timer and return the frequency of the TSC |
128 | * in kHz. | 174 | * in kHz. |
129 | * | 175 | * |
130 | * Return ULONG_MAX on failure to calibrate. | 176 | * Return ULONG_MAX on failure to calibrate. |
131 | */ | 177 | */ |
132 | static unsigned long pit_calibrate_tsc(void) | 178 | static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin) |
133 | { | 179 | { |
134 | u64 tsc, t1, t2, delta; | 180 | u64 tsc, t1, t2, delta; |
135 | unsigned long tscmin, tscmax; | 181 | unsigned long tscmin, tscmax; |
@@ -144,8 +190,8 @@ static unsigned long pit_calibrate_tsc(void) | |||
144 | * (LSB then MSB) to begin countdown. | 190 | * (LSB then MSB) to begin countdown. |
145 | */ | 191 | */ |
146 | outb(0xb0, 0x43); | 192 | outb(0xb0, 0x43); |
147 | outb((CLOCK_TICK_RATE / (1000 / 50)) & 0xff, 0x42); | 193 | outb(latch & 0xff, 0x42); |
148 | outb((CLOCK_TICK_RATE / (1000 / 50)) >> 8, 0x42); | 194 | outb(latch >> 8, 0x42); |
149 | 195 | ||
150 | tsc = t1 = t2 = get_cycles(); | 196 | tsc = t1 = t2 = get_cycles(); |
151 | 197 | ||
@@ -166,31 +212,154 @@ static unsigned long pit_calibrate_tsc(void) | |||
166 | /* | 212 | /* |
167 | * Sanity checks: | 213 | * Sanity checks: |
168 | * | 214 | * |
169 | * If we were not able to read the PIT more than 5000 | 215 | * If we were not able to read the PIT more than loopmin |
170 | * times, then we have been hit by a massive SMI | 216 | * times, then we have been hit by a massive SMI |
171 | * | 217 | * |
172 | * If the maximum is 10 times larger than the minimum, | 218 | * If the maximum is 10 times larger than the minimum, |
173 | * then we got hit by an SMI as well. | 219 | * then we got hit by an SMI as well. |
174 | */ | 220 | */ |
175 | if (pitcnt < 5000 || tscmax > 10 * tscmin) | 221 | if (pitcnt < loopmin || tscmax > 10 * tscmin) |
176 | return ULONG_MAX; | 222 | return ULONG_MAX; |
177 | 223 | ||
178 | /* Calculate the PIT value */ | 224 | /* Calculate the PIT value */ |
179 | delta = t2 - t1; | 225 | delta = t2 - t1; |
180 | do_div(delta, 50); | 226 | do_div(delta, ms); |
181 | return delta; | 227 | return delta; |
182 | } | 228 | } |
183 | 229 | ||
230 | /* | ||
231 | * This reads the current MSB of the PIT counter, and | ||
232 | * checks if we are running on sufficiently fast and | ||
233 | * non-virtualized hardware. | ||
234 | * | ||
235 | * Our expectations are: | ||
236 | * | ||
237 | * - the PIT is running at roughly 1.19MHz | ||
238 | * | ||
239 | * - each IO is going to take about 1us on real hardware, | ||
240 | * but we allow it to be much faster (by a factor of 10) or | ||
241 | * _slightly_ slower (ie we allow up to a 2us read+counter | ||
242 | * update - anything else implies a unacceptably slow CPU | ||
243 | * or PIT for the fast calibration to work. | ||
244 | * | ||
245 | * - with 256 PIT ticks to read the value, we have 214us to | ||
246 | * see the same MSB (and overhead like doing a single TSC | ||
247 | * read per MSB value etc). | ||
248 | * | ||
249 | * - We're doing 2 reads per loop (LSB, MSB), and we expect | ||
250 | * them each to take about a microsecond on real hardware. | ||
251 | * So we expect a count value of around 100. But we'll be | ||
252 | * generous, and accept anything over 50. | ||
253 | * | ||
254 | * - if the PIT is stuck, and we see *many* more reads, we | ||
255 | * return early (and the next caller of pit_expect_msb() | ||
256 | * then consider it a failure when they don't see the | ||
257 | * next expected value). | ||
258 | * | ||
259 | * These expectations mean that we know that we have seen the | ||
260 | * transition from one expected value to another with a fairly | ||
261 | * high accuracy, and we didn't miss any events. We can thus | ||
262 | * use the TSC value at the transitions to calculate a pretty | ||
263 | * good value for the TSC frequencty. | ||
264 | */ | ||
265 | static inline int pit_expect_msb(unsigned char val) | ||
266 | { | ||
267 | int count = 0; | ||
268 | |||
269 | for (count = 0; count < 50000; count++) { | ||
270 | /* Ignore LSB */ | ||
271 | inb(0x42); | ||
272 | if (inb(0x42) != val) | ||
273 | break; | ||
274 | } | ||
275 | return count > 50; | ||
276 | } | ||
277 | |||
278 | /* | ||
279 | * How many MSB values do we want to see? We aim for a | ||
280 | * 15ms calibration, which assuming a 2us counter read | ||
281 | * error should give us roughly 150 ppm precision for | ||
282 | * the calibration. | ||
283 | */ | ||
284 | #define QUICK_PIT_MS 15 | ||
285 | #define QUICK_PIT_ITERATIONS (QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256) | ||
286 | |||
287 | static unsigned long quick_pit_calibrate(void) | ||
288 | { | ||
289 | /* Set the Gate high, disable speaker */ | ||
290 | outb((inb(0x61) & ~0x02) | 0x01, 0x61); | ||
291 | |||
292 | /* | ||
293 | * Counter 2, mode 0 (one-shot), binary count | ||
294 | * | ||
295 | * NOTE! Mode 2 decrements by two (and then the | ||
296 | * output is flipped each time, giving the same | ||
297 | * final output frequency as a decrement-by-one), | ||
298 | * so mode 0 is much better when looking at the | ||
299 | * individual counts. | ||
300 | */ | ||
301 | outb(0xb0, 0x43); | ||
302 | |||
303 | /* Start at 0xffff */ | ||
304 | outb(0xff, 0x42); | ||
305 | outb(0xff, 0x42); | ||
306 | |||
307 | if (pit_expect_msb(0xff)) { | ||
308 | int i; | ||
309 | u64 t1, t2, delta; | ||
310 | unsigned char expect = 0xfe; | ||
311 | |||
312 | t1 = get_cycles(); | ||
313 | for (i = 0; i < QUICK_PIT_ITERATIONS; i++, expect--) { | ||
314 | if (!pit_expect_msb(expect)) | ||
315 | goto failed; | ||
316 | } | ||
317 | t2 = get_cycles(); | ||
318 | |||
319 | /* | ||
320 | * Make sure we can rely on the second TSC timestamp: | ||
321 | */ | ||
322 | if (!pit_expect_msb(expect)) | ||
323 | goto failed; | ||
324 | |||
325 | /* | ||
326 | * Ok, if we get here, then we've seen the | ||
327 | * MSB of the PIT decrement QUICK_PIT_ITERATIONS | ||
328 | * times, and each MSB had many hits, so we never | ||
329 | * had any sudden jumps. | ||
330 | * | ||
331 | * As a result, we can depend on there not being | ||
332 | * any odd delays anywhere, and the TSC reads are | ||
333 | * reliable. | ||
334 | * | ||
335 | * kHz = ticks / time-in-seconds / 1000; | ||
336 | * kHz = (t2 - t1) / (QPI * 256 / PIT_TICK_RATE) / 1000 | ||
337 | * kHz = ((t2 - t1) * PIT_TICK_RATE) / (QPI * 256 * 1000) | ||
338 | */ | ||
339 | delta = (t2 - t1)*PIT_TICK_RATE; | ||
340 | do_div(delta, QUICK_PIT_ITERATIONS*256*1000); | ||
341 | printk("Fast TSC calibration using PIT\n"); | ||
342 | return delta; | ||
343 | } | ||
344 | failed: | ||
345 | return 0; | ||
346 | } | ||
184 | 347 | ||
185 | /** | 348 | /** |
186 | * native_calibrate_tsc - calibrate the tsc on boot | 349 | * native_calibrate_tsc - calibrate the tsc on boot |
187 | */ | 350 | */ |
188 | unsigned long native_calibrate_tsc(void) | 351 | unsigned long native_calibrate_tsc(void) |
189 | { | 352 | { |
190 | u64 tsc1, tsc2, delta, pm1, pm2, hpet1, hpet2; | 353 | u64 tsc1, tsc2, delta, ref1, ref2; |
191 | unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX; | 354 | unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX; |
192 | unsigned long flags; | 355 | unsigned long flags, latch, ms, fast_calibrate; |
193 | int hpet = is_hpet_enabled(), i; | 356 | int hpet = is_hpet_enabled(), i, loopmin; |
357 | |||
358 | local_irq_save(flags); | ||
359 | fast_calibrate = quick_pit_calibrate(); | ||
360 | local_irq_restore(flags); | ||
361 | if (fast_calibrate) | ||
362 | return fast_calibrate; | ||
194 | 363 | ||
195 | /* | 364 | /* |
196 | * Run 5 calibration loops to get the lowest frequency value | 365 | * Run 5 calibration loops to get the lowest frequency value |
@@ -216,7 +385,13 @@ unsigned long native_calibrate_tsc(void) | |||
216 | * calibration delay loop as we have to wait for a certain | 385 | * calibration delay loop as we have to wait for a certain |
217 | * amount of time anyway. | 386 | * amount of time anyway. |
218 | */ | 387 | */ |
219 | for (i = 0; i < 5; i++) { | 388 | |
389 | /* Preset PIT loop values */ | ||
390 | latch = CAL_LATCH; | ||
391 | ms = CAL_MS; | ||
392 | loopmin = CAL_PIT_LOOPS; | ||
393 | |||
394 | for (i = 0; i < 3; i++) { | ||
220 | unsigned long tsc_pit_khz; | 395 | unsigned long tsc_pit_khz; |
221 | 396 | ||
222 | /* | 397 | /* |
@@ -226,16 +401,16 @@ unsigned long native_calibrate_tsc(void) | |||
226 | * read the end value. | 401 | * read the end value. |
227 | */ | 402 | */ |
228 | local_irq_save(flags); | 403 | local_irq_save(flags); |
229 | tsc1 = tsc_read_refs(&pm1, hpet ? &hpet1 : NULL); | 404 | tsc1 = tsc_read_refs(&ref1, hpet); |
230 | tsc_pit_khz = pit_calibrate_tsc(); | 405 | tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin); |
231 | tsc2 = tsc_read_refs(&pm2, hpet ? &hpet2 : NULL); | 406 | tsc2 = tsc_read_refs(&ref2, hpet); |
232 | local_irq_restore(flags); | 407 | local_irq_restore(flags); |
233 | 408 | ||
234 | /* Pick the lowest PIT TSC calibration so far */ | 409 | /* Pick the lowest PIT TSC calibration so far */ |
235 | tsc_pit_min = min(tsc_pit_min, tsc_pit_khz); | 410 | tsc_pit_min = min(tsc_pit_min, tsc_pit_khz); |
236 | 411 | ||
237 | /* hpet or pmtimer available ? */ | 412 | /* hpet or pmtimer available ? */ |
238 | if (!hpet && !pm1 && !pm2) | 413 | if (!hpet && !ref1 && !ref2) |
239 | continue; | 414 | continue; |
240 | 415 | ||
241 | /* Check, whether the sampling was disturbed by an SMI */ | 416 | /* Check, whether the sampling was disturbed by an SMI */ |
@@ -243,23 +418,41 @@ unsigned long native_calibrate_tsc(void) | |||
243 | continue; | 418 | continue; |
244 | 419 | ||
245 | tsc2 = (tsc2 - tsc1) * 1000000LL; | 420 | tsc2 = (tsc2 - tsc1) * 1000000LL; |
421 | if (hpet) | ||
422 | tsc2 = calc_hpet_ref(tsc2, ref1, ref2); | ||
423 | else | ||
424 | tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2); | ||
246 | 425 | ||
247 | if (hpet) { | 426 | tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2); |
248 | if (hpet2 < hpet1) | 427 | |
249 | hpet2 += 0x100000000ULL; | 428 | /* Check the reference deviation */ |
250 | hpet2 -= hpet1; | 429 | delta = ((u64) tsc_pit_min) * 100; |
251 | tsc1 = ((u64)hpet2 * hpet_readl(HPET_PERIOD)); | 430 | do_div(delta, tsc_ref_min); |
252 | do_div(tsc1, 1000000); | 431 | |
253 | } else { | 432 | /* |
254 | if (pm2 < pm1) | 433 | * If both calibration results are inside a 10% window |
255 | pm2 += (u64)ACPI_PM_OVRRUN; | 434 | * then we can be sure, that the calibration |
256 | pm2 -= pm1; | 435 | * succeeded. We break out of the loop right away. We |
257 | tsc1 = pm2 * 1000000000LL; | 436 | * use the reference value, as it is more precise. |
258 | do_div(tsc1, PMTMR_TICKS_PER_SEC); | 437 | */ |
438 | if (delta >= 90 && delta <= 110) { | ||
439 | printk(KERN_INFO | ||
440 | "TSC: PIT calibration matches %s. %d loops\n", | ||
441 | hpet ? "HPET" : "PMTIMER", i + 1); | ||
442 | return tsc_ref_min; | ||
259 | } | 443 | } |
260 | 444 | ||
261 | do_div(tsc2, tsc1); | 445 | /* |
262 | tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2); | 446 | * Check whether PIT failed more than once. This |
447 | * happens in virtualized environments. We need to | ||
448 | * give the virtual PC a slightly longer timeframe for | ||
449 | * the HPET/PMTIMER to make the result precise. | ||
450 | */ | ||
451 | if (i == 1 && tsc_pit_min == ULONG_MAX) { | ||
452 | latch = CAL2_LATCH; | ||
453 | ms = CAL2_MS; | ||
454 | loopmin = CAL2_PIT_LOOPS; | ||
455 | } | ||
263 | } | 456 | } |
264 | 457 | ||
265 | /* | 458 | /* |
@@ -270,7 +463,7 @@ unsigned long native_calibrate_tsc(void) | |||
270 | printk(KERN_WARNING "TSC: Unable to calibrate against PIT\n"); | 463 | printk(KERN_WARNING "TSC: Unable to calibrate against PIT\n"); |
271 | 464 | ||
272 | /* We don't have an alternative source, disable TSC */ | 465 | /* We don't have an alternative source, disable TSC */ |
273 | if (!hpet && !pm1 && !pm2) { | 466 | if (!hpet && !ref1 && !ref2) { |
274 | printk("TSC: No reference (HPET/PMTIMER) available\n"); | 467 | printk("TSC: No reference (HPET/PMTIMER) available\n"); |
275 | return 0; | 468 | return 0; |
276 | } | 469 | } |
@@ -278,7 +471,7 @@ unsigned long native_calibrate_tsc(void) | |||
278 | /* The alternative source failed as well, disable TSC */ | 471 | /* The alternative source failed as well, disable TSC */ |
279 | if (tsc_ref_min == ULONG_MAX) { | 472 | if (tsc_ref_min == ULONG_MAX) { |
280 | printk(KERN_WARNING "TSC: HPET/PMTIMER calibration " | 473 | printk(KERN_WARNING "TSC: HPET/PMTIMER calibration " |
281 | "failed due to SMI disturbance.\n"); | 474 | "failed.\n"); |
282 | return 0; | 475 | return 0; |
283 | } | 476 | } |
284 | 477 | ||
@@ -290,44 +483,25 @@ unsigned long native_calibrate_tsc(void) | |||
290 | } | 483 | } |
291 | 484 | ||
292 | /* We don't have an alternative source, use the PIT calibration value */ | 485 | /* We don't have an alternative source, use the PIT calibration value */ |
293 | if (!hpet && !pm1 && !pm2) { | 486 | if (!hpet && !ref1 && !ref2) { |
294 | printk(KERN_INFO "TSC: Using PIT calibration value\n"); | 487 | printk(KERN_INFO "TSC: Using PIT calibration value\n"); |
295 | return tsc_pit_min; | 488 | return tsc_pit_min; |
296 | } | 489 | } |
297 | 490 | ||
298 | /* The alternative source failed, use the PIT calibration value */ | 491 | /* The alternative source failed, use the PIT calibration value */ |
299 | if (tsc_ref_min == ULONG_MAX) { | 492 | if (tsc_ref_min == ULONG_MAX) { |
300 | printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed due " | 493 | printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed. " |
301 | "to SMI disturbance. Using PIT calibration\n"); | 494 | "Using PIT calibration\n"); |
302 | return tsc_pit_min; | 495 | return tsc_pit_min; |
303 | } | 496 | } |
304 | 497 | ||
305 | /* Check the reference deviation */ | ||
306 | delta = ((u64) tsc_pit_min) * 100; | ||
307 | do_div(delta, tsc_ref_min); | ||
308 | |||
309 | /* | ||
310 | * If both calibration results are inside a 5% window, the we | ||
311 | * use the lower frequency of those as it is probably the | ||
312 | * closest estimate. | ||
313 | */ | ||
314 | if (delta >= 95 && delta <= 105) { | ||
315 | printk(KERN_INFO "TSC: PIT calibration confirmed by %s.\n", | ||
316 | hpet ? "HPET" : "PMTIMER"); | ||
317 | printk(KERN_INFO "TSC: using %s calibration value\n", | ||
318 | tsc_pit_min <= tsc_ref_min ? "PIT" : | ||
319 | hpet ? "HPET" : "PMTIMER"); | ||
320 | return tsc_pit_min <= tsc_ref_min ? tsc_pit_min : tsc_ref_min; | ||
321 | } | ||
322 | |||
323 | printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n", | ||
324 | hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min); | ||
325 | |||
326 | /* | 498 | /* |
327 | * The calibration values differ too much. In doubt, we use | 499 | * The calibration values differ too much. In doubt, we use |
328 | * the PIT value as we know that there are PMTIMERs around | 500 | * the PIT value as we know that there are PMTIMERs around |
329 | * running at double speed. | 501 | * running at double speed. At least we let the user know: |
330 | */ | 502 | */ |
503 | printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n", | ||
504 | hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min); | ||
331 | printk(KERN_INFO "TSC: Using PIT calibration value\n"); | 505 | printk(KERN_INFO "TSC: Using PIT calibration value\n"); |
332 | return tsc_pit_min; | 506 | return tsc_pit_min; |
333 | } | 507 | } |