1 files changed, 232 insertions, 58 deletions

diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 8f98e9de1b82..161bb850fc47 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -104,7 +104,7 @@ __setup("notsc", notsc_setup);
 /*
  * Read TSC and the reference counters. Take care of SMI disturbance
  */
-static u64 tsc_read_refs(u64 *pm, u64 *hpet)
+static u64 tsc_read_refs(u64 *p, int hpet)
 {
         u64 t1, t2;
         int i;
@@ -112,9 +112,9 @@ static u64 tsc_read_refs(u64 *pm, u64 *hpet)
         for (i = 0; i < MAX_RETRIES; i++) {
                 t1 = get_cycles();
                 if (hpet)
-                        *hpet = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
+                        *p = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
                 else
-                        *pm = acpi_pm_read_early();
+                        *p = acpi_pm_read_early();
                 t2 = get_cycles();
                 if ((t2 - t1) < SMI_TRESHOLD)
                         return t2;
@@ -123,13 +123,59 @@ static u64 tsc_read_refs(u64 *pm, u64 *hpet)
 }
 
 /*
+ * Calculate the TSC frequency from HPET reference
+ */
+static unsigned long calc_hpet_ref(u64 deltatsc, u64 hpet1, u64 hpet2)
+{
+        u64 tmp;
+
+        if (hpet2 < hpet1)
+                hpet2 += 0x100000000ULL;
+        hpet2 -= hpet1;
+        tmp = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
+        do_div(tmp, 1000000);
+        do_div(deltatsc, tmp);
+
+        return (unsigned long) deltatsc;
+}
+
+/*
+ * Calculate the TSC frequency from PMTimer reference
+ */
+static unsigned long calc_pmtimer_ref(u64 deltatsc, u64 pm1, u64 pm2)
+{
+        u64 tmp;
+
+        if (!pm1 && !pm2)
+                return ULONG_MAX;
+
+        if (pm2 < pm1)
+                pm2 += (u64)ACPI_PM_OVRRUN;
+        pm2 -= pm1;
+        tmp = pm2 * 1000000000LL;
+        do_div(tmp, PMTMR_TICKS_PER_SEC);
+        do_div(deltatsc, tmp);
+
+        return (unsigned long) deltatsc;
+}
+
+#define CAL_MS          10
+#define CAL_LATCH       (CLOCK_TICK_RATE / (1000 / CAL_MS))
+#define CAL_PIT_LOOPS   1000
+
+#define CAL2_MS         50
+#define CAL2_LATCH      (CLOCK_TICK_RATE / (1000 / CAL2_MS))
+#define CAL2_PIT_LOOPS  5000
+
+
+/*
  * Try to calibrate the TSC against the Programmable
  * Interrupt Timer and return the frequency of the TSC
  * in kHz.
  *
  * Return ULONG_MAX on failure to calibrate.
  */
-static unsigned long pit_calibrate_tsc(void)
+static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin)
 {
         u64 tsc, t1, t2, delta;
         unsigned long tscmin, tscmax;
@@ -144,8 +190,8 @@ static unsigned long pit_calibrate_tsc(void)
          * (LSB then MSB) to begin countdown.
          */
         outb(0xb0, 0x43);
-        outb((CLOCK_TICK_RATE / (1000 / 50)) & 0xff, 0x42);
-        outb((CLOCK_TICK_RATE / (1000 / 50)) >> 8, 0x42);
+        outb(latch & 0xff, 0x42);
+        outb(latch >> 8, 0x42);
 
         tsc = t1 = t2 = get_cycles();
 
@@ -166,31 +212,154 @@ static unsigned long pit_calibrate_tsc(void)
         /*
          * Sanity checks:
          *
-         * If we were not able to read the PIT more than 5000
+         * If we were not able to read the PIT more than loopmin
          * times, then we have been hit by a massive SMI
          *
          * If the maximum is 10 times larger than the minimum,
          * then we got hit by an SMI as well.
          */
-        if (pitcnt < 5000 || tscmax > 10 * tscmin)
+        if (pitcnt < loopmin || tscmax > 10 * tscmin)
                 return ULONG_MAX;
 
         /* Calculate the PIT value */
         delta = t2 - t1;
-        do_div(delta, 50);
+        do_div(delta, ms);
         return delta;
 }
 
+/*
+ * This reads the current MSB of the PIT counter, and
+ * checks if we are running on sufficiently fast and
+ * non-virtualized hardware.
+ *
+ * Our expectations are:
+ *
+ *  - the PIT is running at roughly 1.19MHz
+ *
+ *  - each IO is going to take about 1us on real hardware,
+ *    but we allow it to be much faster (by a factor of 10) or
+ *    _slightly_ slower (ie we allow up to a 2us read+counter
+ *    update - anything else implies a unacceptably slow CPU
+ *    or PIT for the fast calibration to work.
+ *
+ *  - with 256 PIT ticks to read the value, we have 214us to
+ *    see the same MSB (and overhead like doing a single TSC
+ *    read per MSB value etc).
+ *
+ *  - We're doing 2 reads per loop (LSB, MSB), and we expect
+ *    them each to take about a microsecond on real hardware.
+ *    So we expect a count value of around 100. But we'll be
+ *    generous, and accept anything over 50.
+ *
+ *  - if the PIT is stuck, and we see *many* more reads, we
+ *    return early (and the next caller of pit_expect_msb()
+ *    then consider it a failure when they don't see the
+ *    next expected value).
+ *
+ * These expectations mean that we know that we have seen the
+ * transition from one expected value to another with a fairly
+ * high accuracy, and we didn't miss any events. We can thus
+ * use the TSC value at the transitions to calculate a pretty
+ * good value for the TSC frequencty.
+ */
+static inline int pit_expect_msb(unsigned char val)
+{
+        int count = 0;
+
+        for (count = 0; count < 50000; count++) {
+                /* Ignore LSB */
+                inb(0x42);
+                if (inb(0x42) != val)
+                        break;
+        }
+        return count > 50;
+}
+
+/*
+ * How many MSB values do we want to see? We aim for a
+ * 15ms calibration, which assuming a 2us counter read
+ * error should give us roughly 150 ppm precision for
+ * the calibration.
+ */
+#define QUICK_PIT_MS 15
+#define QUICK_PIT_ITERATIONS (QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
+
+static unsigned long quick_pit_calibrate(void)
+{
+        /* Set the Gate high, disable speaker */
+        outb((inb(0x61) & ~0x02) | 0x01, 0x61);
+
+        /*
+         * Counter 2, mode 0 (one-shot), binary count
+         *
+         * NOTE! Mode 2 decrements by two (and then the
+         * output is flipped each time, giving the same
+         * final output frequency as a decrement-by-one),
+         * so mode 0 is much better when looking at the
+         * individual counts.
+         */
+        outb(0xb0, 0x43);
+
+        /* Start at 0xffff */
+        outb(0xff, 0x42);
+        outb(0xff, 0x42);
+
+        if (pit_expect_msb(0xff)) {
+                int i;
+                u64 t1, t2, delta;
+                unsigned char expect = 0xfe;
+
+                t1 = get_cycles();
+                for (i = 0; i < QUICK_PIT_ITERATIONS; i++, expect--) {
+                        if (!pit_expect_msb(expect))
+                                goto failed;
+                }
+                t2 = get_cycles();
+
+                /*
+                 * Make sure we can rely on the second TSC timestamp:
+                 */
+                if (!pit_expect_msb(expect))
+                        goto failed;
+
+                /*
+                 * Ok, if we get here, then we've seen the
+                 * MSB of the PIT decrement QUICK_PIT_ITERATIONS
+                 * times, and each MSB had many hits, so we never
+                 * had any sudden jumps.
+                 *
+                 * As a result, we can depend on there not being
+                 * any odd delays anywhere, and the TSC reads are
+                 * reliable.
+                 *
+                 * kHz = ticks / time-in-seconds / 1000;
+                 * kHz = (t2 - t1) / (QPI * 256 / PIT_TICK_RATE) / 1000
+                 * kHz = ((t2 - t1) * PIT_TICK_RATE) / (QPI * 256 * 1000)
+                 */
+                delta = (t2 - t1)*PIT_TICK_RATE;
+                do_div(delta, QUICK_PIT_ITERATIONS*256*1000);
+                printk("Fast TSC calibration using PIT\n");
+                return delta;
+        }
+failed:
+        return 0;
+}
 
 /**
  * native_calibrate_tsc - calibrate the tsc on boot
  */
 unsigned long native_calibrate_tsc(void)
 {
-        u64 tsc1, tsc2, delta, pm1, pm2, hpet1, hpet2;
+        u64 tsc1, tsc2, delta, ref1, ref2;
         unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
-        unsigned long flags;
-        int hpet = is_hpet_enabled(), i;
+        unsigned long flags, latch, ms, fast_calibrate;
+        int hpet = is_hpet_enabled(), i, loopmin;
+
+        local_irq_save(flags);
+        fast_calibrate = quick_pit_calibrate();
+        local_irq_restore(flags);
+        if (fast_calibrate)
+                return fast_calibrate;
 
         /*
          * Run 5 calibration loops to get the lowest frequency value
@@ -216,7 +385,13 @@ unsigned long native_calibrate_tsc(void)
          * calibration delay loop as we have to wait for a certain
          * amount of time anyway.
          */
-        for (i = 0; i < 5; i++) {
+
+        /* Preset PIT loop values */
+        latch = CAL_LATCH;
+        ms = CAL_MS;
+        loopmin = CAL_PIT_LOOPS;
+
+        for (i = 0; i < 3; i++) {
                 unsigned long tsc_pit_khz;
 
                 /*
@@ -226,16 +401,16 @@ unsigned long native_calibrate_tsc(void)
                  * read the end value.
                  */
                 local_irq_save(flags);
-                tsc1 = tsc_read_refs(&pm1, hpet ? &hpet1 : NULL);
-                tsc_pit_khz = pit_calibrate_tsc();
-                tsc2 = tsc_read_refs(&pm2, hpet ? &hpet2 : NULL);
+                tsc1 = tsc_read_refs(&ref1, hpet);
+                tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin);
+                tsc2 = tsc_read_refs(&ref2, hpet);
                 local_irq_restore(flags);
 
                 /* Pick the lowest PIT TSC calibration so far */
                 tsc_pit_min = min(tsc_pit_min, tsc_pit_khz);
 
                 /* hpet or pmtimer available ? */
-                if (!hpet && !pm1 && !pm2)
+                if (!hpet && !ref1 && !ref2)
                         continue;
 
                 /* Check, whether the sampling was disturbed by an SMI */
@@ -243,23 +418,41 @@ unsigned long native_calibrate_tsc(void)
                         continue;
 
                 tsc2 = (tsc2 - tsc1) * 1000000LL;
+                if (hpet)
+                        tsc2 = calc_hpet_ref(tsc2, ref1, ref2);
+                else
+                        tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2);
 
-                if (hpet) {
-                        if (hpet2 < hpet1)
-                                hpet2 += 0x100000000ULL;
-                        hpet2 -= hpet1;
-                        tsc1 = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
-                        do_div(tsc1, 1000000);
-                } else {
-                        if (pm2 < pm1)
-                                pm2 += (u64)ACPI_PM_OVRRUN;
-                        pm2 -= pm1;
-                        tsc1 = pm2 * 1000000000LL;
-                        do_div(tsc1, PMTMR_TICKS_PER_SEC);
+                tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2);
+
+                /* Check the reference deviation */
+                delta = ((u64) tsc_pit_min) * 100;
+                do_div(delta, tsc_ref_min);
+
+                /*
+                 * If both calibration results are inside a 10% window
+                 * then we can be sure, that the calibration
+                 * succeeded. We break out of the loop right away. We
+                 * use the reference value, as it is more precise.
+                 */
+                if (delta >= 90 && delta <= 110) {
+                        printk(KERN_INFO
+                               "TSC: PIT calibration matches %s. %d loops\n",
+                               hpet ? "HPET" : "PMTIMER", i + 1);
+                        return tsc_ref_min;
                 }
 
-                do_div(tsc2, tsc1);
-                tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2);
+                /*
+                 * Check whether PIT failed more than once. This
+                 * happens in virtualized environments. We need to
+                 * give the virtual PC a slightly longer timeframe for
+                 * the HPET/PMTIMER to make the result precise.
+                 */
+                if (i == 1 && tsc_pit_min == ULONG_MAX) {
+                        latch = CAL2_LATCH;
+                        ms = CAL2_MS;
+                        loopmin = CAL2_PIT_LOOPS;
+                }
         }
 
         /*
@@ -270,7 +463,7 @@ unsigned long native_calibrate_tsc(void)
                 printk(KERN_WARNING "TSC: Unable to calibrate against PIT\n");
 
                 /* We don't have an alternative source, disable TSC */
-                if (!hpet && !pm1 && !pm2) {
+                if (!hpet && !ref1 && !ref2) {
                         printk("TSC: No reference (HPET/PMTIMER) available\n");
                         return 0;
                 }
@@ -278,7 +471,7 @@ unsigned long native_calibrate_tsc(void)
                 /* The alternative source failed as well, disable TSC */
                 if (tsc_ref_min == ULONG_MAX) {
                         printk(KERN_WARNING "TSC: HPET/PMTIMER calibration "
-                               "failed due to SMI disturbance.\n");
+                               "failed.\n");
                         return 0;
                 }
 
@@ -290,44 +483,25 @@ unsigned long native_calibrate_tsc(void)
         }
 
         /* We don't have an alternative source, use the PIT calibration value */
-        if (!hpet && !pm1 && !pm2) {
+        if (!hpet && !ref1 && !ref2) {
                 printk(KERN_INFO "TSC: Using PIT calibration value\n");
                 return tsc_pit_min;
         }
 
         /* The alternative source failed, use the PIT calibration value */
         if (tsc_ref_min == ULONG_MAX) {
-                printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed due "
-                       "to SMI disturbance. Using PIT calibration\n");
+                printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed. "
+                       "Using PIT calibration\n");
                 return tsc_pit_min;
         }
 
-        /* Check the reference deviation */
-        delta = ((u64) tsc_pit_min) * 100;
-        do_div(delta, tsc_ref_min);
-
-        /*
-         * If both calibration results are inside a 5% window, the we
-         * use the lower frequency of those as it is probably the
-         * closest estimate.
-         */
-        if (delta >= 95 && delta <= 105) {
-                printk(KERN_INFO "TSC: PIT calibration confirmed by %s.\n",
-                       hpet ? "HPET" : "PMTIMER");
-                printk(KERN_INFO "TSC: using %s calibration value\n",
-                       tsc_pit_min <= tsc_ref_min ? "PIT" :
-                       hpet ? "HPET" : "PMTIMER");
-                return tsc_pit_min <= tsc_ref_min ? tsc_pit_min : tsc_ref_min;
-        }
-
-        printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n",
-               hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min);
-
         /*
          * The calibration values differ too much. In doubt, we use
          * the PIT value as we know that there are PMTIMERs around
-         * running at double speed.
+         * running at double speed. At least we let the user know:
          */
+        printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n",
+               hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min);
         printk(KERN_INFO "TSC: Using PIT calibration value\n");
         return tsc_pit_min;
 }