[PATCH] x86: rewrite SMP TSC sync code

make the TSC synchronization code more robust, and unify it between x86_64 and
i386.

The biggest change is the removal of the 'fix up TSCs' code on x86_64 and
i386, in some rare cases it was /causing/ time-warps on SMP systems.

The new code only checks for TSC asynchronity - and if it can prove a
time-warp (if it can observe the TSC going backwards when going from one CPU
to another within a critical section), then the TSC clock-source is turned
off.

The TSC synchronization-checking code also got moved into a separate file.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

4 files changed, 210 insertions, 220 deletions

diff --git a/arch/x86_64/kernel/Makefile b/arch/x86_64/kernel/Makefile
index ae399458024b..6465eee6d920 100644
--- a/arch/x86_64/kernel/Makefile
+++ b/arch/x86_64/kernel/Makefile
@@ -19,7 +19,7 @@ obj-$(CONFIG_ACPI)		+= acpi/
 obj-$(CONFIG_X86_MSR)           += msr.o
 obj-$(CONFIG_MICROCODE)         += microcode.o
 obj-$(CONFIG_X86_CPUID)         += cpuid.o
-obj-$(CONFIG_SMP)               += smp.o smpboot.o trampoline.o
+obj-$(CONFIG_SMP)               += smp.o smpboot.o trampoline.o tsc_sync.o
 obj-y                           += apic.o  nmi.o
 obj-y                           += io_apic.o mpparse.o \
                 genapic.o genapic_cluster.o genapic_flat.o
diff --git a/arch/x86_64/kernel/smpboot.c b/arch/x86_64/kernel/smpboot.c
index daf19332f0dd..62d828433c30 100644
--- a/arch/x86_64/kernel/smpboot.c
+++ b/arch/x86_64/kernel/smpboot.c
@@ -148,217 +148,6 @@ static void __cpuinit smp_store_cpu_info(int id)
         print_cpu_info(c);
 }
 
-/*
- * New Funky TSC sync algorithm borrowed from IA64.
- * Main advantage is that it doesn't reset the TSCs fully and
- * in general looks more robust and it works better than my earlier
- * attempts. I believe it was written by David Mosberger. Some minor
- * adjustments for x86-64 by me -AK
- *
- * Original comment reproduced below.
- *
- * Synchronize TSC of the current (slave) CPU with the TSC of the
- * MASTER CPU (normally the time-keeper CPU).  We use a closed loop to
- * eliminate the possibility of unaccounted-for errors (such as
- * getting a machine check in the middle of a calibration step).  The
- * basic idea is for the slave to ask the master what itc value it has
- * and to read its own itc before and after the master responds.  Each
- * iteration gives us three timestamps:
- *
- *      slave           master
- *
- *      t0 ---\
- *             ---\
- *                 --->
- *                      tm
- *                 /---
- *             /---
- *      t1 <---
- *
- *
- * The goal is to adjust the slave's TSC such that tm falls exactly
- * half-way between t0 and t1.  If we achieve this, the clocks are
- * synchronized provided the interconnect between the slave and the
- * master is symmetric.  Even if the interconnect were asymmetric, we
- * would still know that the synchronization error is smaller than the
- * roundtrip latency (t0 - t1).
- *
- * When the interconnect is quiet and symmetric, this lets us
- * synchronize the TSC to within one or two cycles.  However, we can
- * only *guarantee* that the synchronization is accurate to within a
- * round-trip time, which is typically in the range of several hundred
- * cycles (e.g., ~500 cycles).  In practice, this means that the TSCs
- * are usually almost perfectly synchronized, but we shouldn't assume
- * that the accuracy is much better than half a micro second or so.
- *
- * [there are other errors like the latency of RDTSC and of the
- * WRMSR. These can also account to hundreds of cycles. So it's
- * probably worse. It claims 153 cycles error on a dual Opteron,
- * but I suspect the numbers are actually somewhat worse -AK]
- */
-
-#define MASTER  0
-#define SLAVE   (SMP_CACHE_BYTES/8)
-
-/* Intentionally don't use cpu_relax() while TSC synchronization
-   because we don't want to go into funky power save modi or cause
-   hypervisors to schedule us away.  Going to sleep would likely affect
-   latency and low latency is the primary objective here. -AK */
-#define no_cpu_relax() barrier()
-
-static __cpuinitdata DEFINE_SPINLOCK(tsc_sync_lock);
-static volatile __cpuinitdata unsigned long go[SLAVE + 1];
-static int notscsync __cpuinitdata;
-
-#undef DEBUG_TSC_SYNC
-
-#define NUM_ROUNDS      64      /* magic value */
-#define NUM_ITERS       5       /* likewise */
-
-/* Callback on boot CPU */
-static __cpuinit void sync_master(void *arg)
-{
-        unsigned long flags, i;
-
-        go[MASTER] = 0;
-
-        local_irq_save(flags);
-        {
-                for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
-                        while (!go[MASTER])
-                                no_cpu_relax();
-                        go[MASTER] = 0;
-                        rdtscll(go[SLAVE]);
-                }
-        }
-        local_irq_restore(flags);
-}
-
-/*
- * Return the number of cycles by which our tsc differs from the tsc
- * on the master (time-keeper) CPU.  A positive number indicates our
- * tsc is ahead of the master, negative that it is behind.
- */
-static inline long
-get_delta(long *rt, long *master)
-{
-        unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
-        unsigned long tcenter, t0, t1, tm;
-        int i;
-
-        for (i = 0; i < NUM_ITERS; ++i) {
-                rdtscll(t0);
-                go[MASTER] = 1;
-                while (!(tm = go[SLAVE]))
-                        no_cpu_relax();
-                go[SLAVE] = 0;
-                rdtscll(t1);
-
-                if (t1 - t0 < best_t1 - best_t0)
-                        best_t0 = t0, best_t1 = t1, best_tm = tm;
-        }
-
-        *rt = best_t1 - best_t0;
-        *master = best_tm - best_t0;
-
-        /* average best_t0 and best_t1 without overflow: */
-        tcenter = (best_t0/2 + best_t1/2);
-        if (best_t0 % 2 + best_t1 % 2 == 2)
-                ++tcenter;
-        return tcenter - best_tm;
-}
-
-static __cpuinit void sync_tsc(unsigned int master)
-{
-        int i, done = 0;
-        long delta, adj, adjust_latency = 0;
-        unsigned long flags, rt, master_time_stamp, bound;
-#ifdef DEBUG_TSC_SYNC
-        static struct syncdebug {
-                long rt;        /* roundtrip time */
-                long master;    /* master's timestamp */
-                long diff;      /* difference between midpoint and master's timestamp */
-                long lat;       /* estimate of tsc adjustment latency */
-        } t[NUM_ROUNDS] __cpuinitdata;
-#endif
-
-        printk(KERN_INFO "CPU %d: Syncing TSC to CPU %u.\n",
-                smp_processor_id(), master);
-
-        go[MASTER] = 1;
-
-        /* It is dangerous to broadcast IPI as cpus are coming up,
-         * as they may not be ready to accept them.  So since
-         * we only need to send the ipi to the boot cpu direct
-         * the message, and avoid the race.
-         */
-        smp_call_function_single(master, sync_master, NULL, 1, 0);
-
-        while (go[MASTER])      /* wait for master to be ready */
-                no_cpu_relax();
-
-        spin_lock_irqsave(&tsc_sync_lock, flags);
-        {
-                for (i = 0; i < NUM_ROUNDS; ++i) {
-                        delta = get_delta(&rt, &master_time_stamp);
-                        if (delta == 0) {
-                                done = 1;       /* let's lock on to this... */
-                                bound = rt;
-                        }
-
-                        if (!done) {
-                                unsigned long t;
-                                if (i > 0) {
-                                        adjust_latency += -delta;
-                                        adj = -delta + adjust_latency/4;
-                                } else
-                                        adj = -delta;
-
-                                rdtscll(t);
-                                wrmsrl(MSR_IA32_TSC, t + adj);
-                        }
-#ifdef DEBUG_TSC_SYNC
-                        t[i].rt = rt;
-                        t[i].master = master_time_stamp;
-                        t[i].diff = delta;
-                        t[i].lat = adjust_latency/4;
-#endif
-                }
-        }
-        spin_unlock_irqrestore(&tsc_sync_lock, flags);
-
-#ifdef DEBUG_TSC_SYNC
-        for (i = 0; i < NUM_ROUNDS; ++i)
-                printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
-                       t[i].rt, t[i].master, t[i].diff, t[i].lat);
-#endif
-
-        printk(KERN_INFO
-               "CPU %d: synchronized TSC with CPU %u (last diff %ld cycles, "
-               "maxerr %lu cycles)\n",
-               smp_processor_id(), master, delta, rt);
-}
-
-static void __cpuinit tsc_sync_wait(void)
-{
-        /*
-         * When the CPU has synchronized TSCs assume the BIOS
-         * or the hardware already synced.  Otherwise we could
-         * mess up a possible perfect synchronization with a
-         * not-quite-perfect algorithm.
-         */
-        if (notscsync || !cpu_has_tsc || !unsynchronized_tsc())
-                return;
-        sync_tsc(0);
-}
-
-static __init int notscsync_setup(char *s)
-{
-        notscsync = 1;
-        return 1;
-}
-__setup("notscsync", notscsync_setup);
-
 static atomic_t init_deasserted __cpuinitdata;
 
 /*
@@ -546,6 +335,11 @@ void __cpuinit start_secondary(void)
         /* otherwise gcc will move up the smp_processor_id before the cpu_init */
         barrier();
 
+        /*
+         * Check TSC sync first:
+         */
+        check_tsc_sync_target();
+
         Dprintk("cpu %d: setting up apic clock\n", smp_processor_id());         
         setup_secondary_APIC_clock();
 
@@ -565,14 +359,6 @@ void __cpuinit start_secondary(void)
          */
         set_cpu_sibling_map(smp_processor_id());
 
-        /* 
-         * Wait for TSC sync to not schedule things before.
-         * We still process interrupts, which could see an inconsistent
-         * time in that window unfortunately. 
-         * Do this here because TSC sync has global unprotected state.
-         */
-        tsc_sync_wait();
-
         /*
          * We need to hold call_lock, so there is no inconsistency
          * between the time smp_call_function() determines number of
@@ -592,6 +378,7 @@ void __cpuinit start_secondary(void)
         cpu_set(smp_processor_id(), cpu_online_map);
         per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
         spin_unlock(&vector_lock);
+
         unlock_ipi_call_lock();
 
         cpu_idle();
@@ -1168,6 +955,11 @@ int __cpuinit __cpu_up(unsigned int cpu)
         /* Unleash the CPU! */
         Dprintk("waiting for cpu %d\n", cpu);
 
+        /*
+         * Make sure and check TSC sync:
+         */
+        check_tsc_sync_source(cpu);
+
         while (!cpu_isset(cpu, cpu_online_map))
                 cpu_relax();
 
diff --git a/arch/x86_64/kernel/time.c b/arch/x86_64/kernel/time.c
index 3cc6886f1fb7..8cb2b2d35f5d 100644
--- a/arch/x86_64/kernel/time.c
+++ b/arch/x86_64/kernel/time.c
@@ -944,12 +944,23 @@ void __init time_init(void)
 #endif
 }
 
+static int tsc_unstable = 0;
+
+void mark_tsc_unstable(void)
+{
+        tsc_unstable = 1;
+}
+EXPORT_SYMBOL_GPL(mark_tsc_unstable);
+
 /*
  * Make an educated guess if the TSC is trustworthy and synchronized
  * over all CPUs.
  */
 __cpuinit int unsynchronized_tsc(void)
 {
+        if (tsc_unstable)
+                return 1;
+
 #ifdef CONFIG_SMP
         if (apic_is_clustered_box())
                 return 1;
diff --git a/arch/x86_64/kernel/tsc_sync.c b/arch/x86_64/kernel/tsc_sync.c
new file mode 100644
index 000000000000..014f0db45dfa
--- /dev/null
+++ b/arch/x86_64/kernel/tsc_sync.c
@@ -0,0 +1,187 @@
+/*
+ * arch/x86_64/kernel/tsc_sync.c: check TSC synchronization.
+ *
+ * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar
+ *
+ * We check whether all boot CPUs have their TSC's synchronized,
+ * print a warning if not and turn off the TSC clock-source.
+ *
+ * The warp-check is point-to-point between two CPUs, the CPU
+ * initiating the bootup is the 'source CPU', the freshly booting
+ * CPU is the 'target CPU'.
+ *
+ * Only two CPUs may participate - they can enter in any order.
+ * ( The serial nature of the boot logic and the CPU hotplug lock
+ *   protects against more than 2 CPUs entering this code. )
+ */
+#include <linux/spinlock.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/nmi.h>
+#include <asm/tsc.h>
+
+/*
+ * Entry/exit counters that make sure that both CPUs
+ * run the measurement code at once:
+ */
+static __cpuinitdata atomic_t start_count;
+static __cpuinitdata atomic_t stop_count;
+
+/*
+ * We use a raw spinlock in this exceptional case, because
+ * we want to have the fastest, inlined, non-debug version
+ * of a critical section, to be able to prove TSC time-warps:
+ */
+static __cpuinitdata raw_spinlock_t sync_lock = __RAW_SPIN_LOCK_UNLOCKED;
+static __cpuinitdata cycles_t last_tsc;
+static __cpuinitdata cycles_t max_warp;
+static __cpuinitdata int nr_warps;
+
+/*
+ * TSC-warp measurement loop running on both CPUs:
+ */
+static __cpuinit void check_tsc_warp(void)
+{
+        cycles_t start, now, prev, end;
+        int i;
+
+        start = get_cycles_sync();
+        /*
+         * The measurement runs for 20 msecs:
+         */
+        end = start + cpu_khz * 20ULL;
+        now = start;
+
+        for (i = 0; ; i++) {
+                /*
+                 * We take the global lock, measure TSC, save the
+                 * previous TSC that was measured (possibly on
+                 * another CPU) and update the previous TSC timestamp.
+                 */
+                __raw_spin_lock(&sync_lock);
+                prev = last_tsc;
+                now = get_cycles_sync();
+                last_tsc = now;
+                __raw_spin_unlock(&sync_lock);
+
+                /*
+                 * Be nice every now and then (and also check whether
+                 * measurement is done [we also insert a 100 million
+                 * loops safety exit, so we dont lock up in case the
+                 * TSC readout is totally broken]):
+                 */
+                if (unlikely(!(i & 7))) {
+                        if (now > end || i > 100000000)
+                                break;
+                        cpu_relax();
+                        touch_nmi_watchdog();
+                }
+                /*
+                 * Outside the critical section we can now see whether
+                 * we saw a time-warp of the TSC going backwards:
+                 */
+                if (unlikely(prev > now)) {
+                        __raw_spin_lock(&sync_lock);
+                        max_warp = max(max_warp, prev - now);
+                        nr_warps++;
+                        __raw_spin_unlock(&sync_lock);
+                }
+
+        }
+}
+
+/*
+ * Source CPU calls into this - it waits for the freshly booted
+ * target CPU to arrive and then starts the measurement:
+ */
+void __cpuinit check_tsc_sync_source(int cpu)
+{
+        int cpus = 2;
+
+        /*
+         * No need to check if we already know that the TSC is not
+         * synchronized:
+         */
+        if (unsynchronized_tsc())
+                return;
+
+        printk(KERN_INFO "checking TSC synchronization [CPU#%d -> CPU#%d]:",
+                          smp_processor_id(), cpu);
+
+        /*
+         * Reset it - in case this is a second bootup:
+         */
+        atomic_set(&stop_count, 0);
+
+        /*
+         * Wait for the target to arrive:
+         */
+        while (atomic_read(&start_count) != cpus-1)
+                cpu_relax();
+        /*
+         * Trigger the target to continue into the measurement too:
+         */
+        atomic_inc(&start_count);
+
+        check_tsc_warp();
+
+        while (atomic_read(&stop_count) != cpus-1)
+                cpu_relax();
+
+        /*
+         * Reset it - just in case we boot another CPU later:
+         */
+        atomic_set(&start_count, 0);
+
+        if (nr_warps) {
+                printk("\n");
+                printk(KERN_WARNING "Measured %Ld cycles TSC warp between CPUs,"
+                                    " turning off TSC clock.\n", max_warp);
+                mark_tsc_unstable();
+                nr_warps = 0;
+                max_warp = 0;
+                last_tsc = 0;
+        } else {
+                printk(" passed.\n");
+        }
+
+        /*
+         * Let the target continue with the bootup:
+         */
+        atomic_inc(&stop_count);
+}
+
+/*
+ * Freshly booted CPUs call into this:
+ */
+void __cpuinit check_tsc_sync_target(void)
+{
+        int cpus = 2;
+
+        if (unsynchronized_tsc())
+                return;
+
+        /*
+         * Register this CPU's participation and wait for the
+         * source CPU to start the measurement:
+         */
+        atomic_inc(&start_count);
+        while (atomic_read(&start_count) != cpus)
+                cpu_relax();
+
+        check_tsc_warp();
+
+        /*
+         * Ok, we are done:
+         */
+        atomic_inc(&stop_count);
+
+        /*
+         * Wait for the source CPU to print stuff:
+         */
+        while (atomic_read(&stop_count) != cpus)
+                cpu_relax();
+}
+#undef NR_LOOPS
+