3 files changed, 199 insertions, 65 deletions

diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c
index a998e1a7d46f..48179928ff38 100644
--- a/arch/x86/kernel/cpu/microcode/amd.c
+++ b/arch/x86/kernel/cpu/microcode/amd.c
@@ -339,7 +339,7 @@ int __init save_microcode_in_initrd_amd(unsigned int cpuid_1_eax)
                 return -EINVAL;
 
         ret = load_microcode_amd(true, x86_family(cpuid_1_eax), desc.data, desc.size);
-        if (ret != UCODE_OK)
+        if (ret > UCODE_UPDATED)
                 return -EINVAL;
 
         return 0;
@@ -683,27 +683,35 @@ static enum ucode_state __load_microcode_amd(u8 family, const u8 *data,
 static enum ucode_state
 load_microcode_amd(bool save, u8 family, const u8 *data, size_t size)
 {
+        struct ucode_patch *p;
         enum ucode_state ret;
 
         /* free old equiv table */
         free_equiv_cpu_table();
 
         ret = __load_microcode_amd(family, data, size);
-
-        if (ret != UCODE_OK)
+        if (ret != UCODE_OK) {
                 cleanup();
+                return ret;
+        }
 
-#ifdef CONFIG_X86_32
-        /* save BSP's matching patch for early load */
-        if (save) {
-                struct ucode_patch *p = find_patch(0);
-                if (p) {
-                        memset(amd_ucode_patch, 0, PATCH_MAX_SIZE);
-                        memcpy(amd_ucode_patch, p->data, min_t(u32, ksize(p->data),
-                                                               PATCH_MAX_SIZE));
-                }
+        p = find_patch(0);
+        if (!p) {
+                return ret;
+        } else {
+                if (boot_cpu_data.microcode == p->patch_id)
+                        return ret;
+
+                ret = UCODE_NEW;
         }
-#endif
+
+        /* save BSP's matching patch for early load */
+        if (!save)
+                return ret;
+
+        memset(amd_ucode_patch, 0, PATCH_MAX_SIZE);
+        memcpy(amd_ucode_patch, p->data, min_t(u32, ksize(p->data), PATCH_MAX_SIZE));
+
         return ret;
 }
 
diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
index aa1b9a422f2b..10c4fc2c91f8 100644
--- a/arch/x86/kernel/cpu/microcode/core.c
+++ b/arch/x86/kernel/cpu/microcode/core.c
@@ -22,13 +22,16 @@
 #define pr_fmt(fmt) "microcode: " fmt
 
 #include <linux/platform_device.h>
+#include <linux/stop_machine.h>
 #include <linux/syscore_ops.h>
 #include <linux/miscdevice.h>
 #include <linux/capability.h>
 #include <linux/firmware.h>
 #include <linux/kernel.h>
+#include <linux/delay.h>
 #include <linux/mutex.h>
 #include <linux/cpu.h>
+#include <linux/nmi.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
 
@@ -64,6 +67,11 @@ LIST_HEAD(microcode_cache);
  */
 static DEFINE_MUTEX(microcode_mutex);
 
+/*
+ * Serialize late loading so that CPUs get updated one-by-one.
+ */
+static DEFINE_SPINLOCK(update_lock);
+
 struct ucode_cpu_info           ucode_cpu_info[NR_CPUS];
 
 struct cpu_info_ctx {
@@ -373,26 +381,23 @@ static int collect_cpu_info(int cpu)
         return ret;
 }
 
-struct apply_microcode_ctx {
-        enum ucode_state err;
-};
-
 static void apply_microcode_local(void *arg)
 {
-        struct apply_microcode_ctx *ctx = arg;
+        enum ucode_state *err = arg;
 
-        ctx->err = microcode_ops->apply_microcode(smp_processor_id());
+        *err = microcode_ops->apply_microcode(smp_processor_id());
 }
 
 static int apply_microcode_on_target(int cpu)
 {
-        struct apply_microcode_ctx ctx = { .err = 0 };
+        enum ucode_state err;
         int ret;
 
-        ret = smp_call_function_single(cpu, apply_microcode_local, &ctx, 1);
-        if (!ret)
-                ret = ctx.err;
-
+        ret = smp_call_function_single(cpu, apply_microcode_local, &err, 1);
+        if (!ret) {
+                if (err == UCODE_ERROR)
+                        ret = 1;
+        }
         return ret;
 }
 
@@ -489,19 +494,114 @@ static void __exit microcode_dev_exit(void)
 /* fake device for request_firmware */
 static struct platform_device   *microcode_pdev;
 
-static enum ucode_state reload_for_cpu(int cpu)
+/*
+ * Late loading dance. Why the heavy-handed stomp_machine effort?
+ *
+ * - HT siblings must be idle and not execute other code while the other sibling
+ *   is loading microcode in order to avoid any negative interactions caused by
+ *   the loading.
+ *
+ * - In addition, microcode update on the cores must be serialized until this
+ *   requirement can be relaxed in the future. Right now, this is conservative
+ *   and good.
+ */
+#define SPINUNIT 100 /* 100 nsec */
+
+static int check_online_cpus(void)
 {
-        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
-        enum ucode_state ustate;
+        if (num_online_cpus() == num_present_cpus())
+                return 0;
 
-        if (!uci->valid)
-                return UCODE_OK;
+        pr_err("Not all CPUs online, aborting microcode update.\n");
+
+        return -EINVAL;
+}
+
+static atomic_t late_cpus_in;
+static atomic_t late_cpus_out;
+
+static int __wait_for_cpus(atomic_t *t, long long timeout)
+{
+        int all_cpus = num_online_cpus();
+
+        atomic_inc(t);
 
-        ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev, true);
-        if (ustate != UCODE_OK)
-                return ustate;
+        while (atomic_read(t) < all_cpus) {
+                if (timeout < SPINUNIT) {
+                        pr_err("Timeout while waiting for CPUs rendezvous, remaining: %d\n",
+                                all_cpus - atomic_read(t));
+                        return 1;
+                }
 
-        return apply_microcode_on_target(cpu);
+                ndelay(SPINUNIT);
+                timeout -= SPINUNIT;
+
+                touch_nmi_watchdog();
+        }
+        return 0;
+}
+
+/*
+ * Returns:
+ * < 0 - on error
+ *   0 - no update done
+ *   1 - microcode was updated
+ */
+static int __reload_late(void *info)
+{
+        int cpu = smp_processor_id();
+        enum ucode_state err;
+        int ret = 0;
+
+        /*
+         * Wait for all CPUs to arrive. A load will not be attempted unless all
+         * CPUs show up.
+         * */
+        if (__wait_for_cpus(&late_cpus_in, NSEC_PER_SEC))
+                return -1;
+
+        spin_lock(&update_lock);
+        apply_microcode_local(&err);
+        spin_unlock(&update_lock);
+
+        if (err > UCODE_NFOUND) {
+                pr_warn("Error reloading microcode on CPU %d\n", cpu);
+                return -1;
+        /* siblings return UCODE_OK because their engine got updated already */
+        } else if (err == UCODE_UPDATED || err == UCODE_OK) {
+                ret = 1;
+        } else {
+                return ret;
+        }
+
+        /*
+         * Increase the wait timeout to a safe value here since we're
+         * serializing the microcode update and that could take a while on a
+         * large number of CPUs. And that is fine as the *actual* timeout will
+         * be determined by the last CPU finished updating and thus cut short.
+         */
+        if (__wait_for_cpus(&late_cpus_out, NSEC_PER_SEC * num_online_cpus()))
+                panic("Timeout during microcode update!\n");
+
+        return ret;
+}
+
+/*
+ * Reload microcode late on all CPUs. Wait for a sec until they
+ * all gather together.
+ */
+static int microcode_reload_late(void)
+{
+        int ret;
+
+        atomic_set(&late_cpus_in,  0);
+        atomic_set(&late_cpus_out, 0);
+
+        ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask);
+        if (ret > 0)
+                microcode_check();
+
+        return ret;
 }
 
 static ssize_t reload_store(struct device *dev,
@@ -509,10 +609,9 @@ static ssize_t reload_store(struct device *dev,
                             const char *buf, size_t size)
 {
         enum ucode_state tmp_ret = UCODE_OK;
-        bool do_callback = false;
+        int bsp = boot_cpu_data.cpu_index;
         unsigned long val;
         ssize_t ret = 0;
-        int cpu;
 
         ret = kstrtoul(buf, 0, &val);
         if (ret)
@@ -521,29 +620,24 @@ static ssize_t reload_store(struct device *dev,
         if (val != 1)
                 return size;
 
-        get_online_cpus();
-        mutex_lock(&microcode_mutex);
-        for_each_online_cpu(cpu) {
-                tmp_ret = reload_for_cpu(cpu);
-                if (tmp_ret > UCODE_NFOUND) {
-                        pr_warn("Error reloading microcode on CPU %d\n", cpu);
-
-                        /* set retval for the first encountered reload error */
-                        if (!ret)
-                                ret = -EINVAL;
-                }
+        tmp_ret = microcode_ops->request_microcode_fw(bsp, &microcode_pdev->dev, true);
+        if (tmp_ret != UCODE_NEW)
+                return size;
 
-                if (tmp_ret == UCODE_UPDATED)
-                        do_callback = true;
-        }
+        get_online_cpus();
 
-        if (!ret && do_callback)
-                microcode_check();
+        ret = check_online_cpus();
+        if (ret)
+                goto put;
 
+        mutex_lock(&microcode_mutex);
+        ret = microcode_reload_late();
         mutex_unlock(&microcode_mutex);
+
+put:
         put_online_cpus();
 
-        if (!ret)
+        if (ret >= 0)
                 ret = size;
 
         return ret;
@@ -611,10 +705,8 @@ static enum ucode_state microcode_init_cpu(int cpu, bool refresh_fw)
         if (system_state != SYSTEM_RUNNING)
                 return UCODE_NFOUND;
 
-        ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev,
-                                                     refresh_fw);
-
-        if (ustate == UCODE_OK) {
+        ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev, refresh_fw);
+        if (ustate == UCODE_NEW) {
                 pr_debug("CPU%d updated upon init\n", cpu);
                 apply_microcode_on_target(cpu);
         }
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 923054a6b760..32b8e5724f96 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -589,6 +589,23 @@ static int apply_microcode_early(struct ucode_cpu_info *uci, bool early)
         if (!mc)
                 return 0;
 
+        /*
+         * Save us the MSR write below - which is a particular expensive
+         * operation - when the other hyperthread has updated the microcode
+         * already.
+         */
+        rev = intel_get_microcode_revision();
+        if (rev >= mc->hdr.rev) {
+                uci->cpu_sig.rev = rev;
+                return UCODE_OK;
+        }
+
+        /*
+         * Writeback and invalidate caches before updating microcode to avoid
+         * internal issues depending on what the microcode is updating.
+         */
+        native_wbinvd();
+
         /* write microcode via MSR 0x79 */
         native_wrmsrl(MSR_IA32_UCODE_WRITE, (unsigned long)mc->bits);
 
@@ -774,9 +791,9 @@ static int collect_cpu_info(int cpu_num, struct cpu_signature *csig)
 
 static enum ucode_state apply_microcode_intel(int cpu)
 {
+        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
+        struct cpuinfo_x86 *c = &cpu_data(cpu);
         struct microcode_intel *mc;
-        struct ucode_cpu_info *uci;
-        struct cpuinfo_x86 *c;
         static int prev_rev;
         u32 rev;
 
@@ -784,15 +801,32 @@ static enum ucode_state apply_microcode_intel(int cpu)
         if (WARN_ON(raw_smp_processor_id() != cpu))
                 return UCODE_ERROR;
 
-        uci = ucode_cpu_info + cpu;
-        mc = uci->mc;
+        /* Look for a newer patch in our cache: */
+        mc = find_patch(uci);
         if (!mc) {
-                /* Look for a newer patch in our cache: */
-                mc = find_patch(uci);
+                mc = uci->mc;
                 if (!mc)
                         return UCODE_NFOUND;
         }
 
+        /*
+         * Save us the MSR write below - which is a particular expensive
+         * operation - when the other hyperthread has updated the microcode
+         * already.
+         */
+        rev = intel_get_microcode_revision();
+        if (rev >= mc->hdr.rev) {
+                uci->cpu_sig.rev = rev;
+                c->microcode = rev;
+                return UCODE_OK;
+        }
+
+        /*
+         * Writeback and invalidate caches before updating microcode to avoid
+         * internal issues depending on what the microcode is updating.
+         */
+        native_wbinvd();
+
         /* write microcode via MSR 0x79 */
         wrmsrl(MSR_IA32_UCODE_WRITE, (unsigned long)mc->bits);
 
@@ -813,8 +847,6 @@ static enum ucode_state apply_microcode_intel(int cpu)
                 prev_rev = rev;
         }
 
-        c = &cpu_data(cpu);
-
         uci->cpu_sig.rev = rev;
         c->microcode = rev;
 
@@ -830,6 +862,7 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
         unsigned int leftover = size;
         unsigned int curr_mc_size = 0, new_mc_size = 0;
         unsigned int csig, cpf;
+        enum ucode_state ret = UCODE_OK;
 
         while (leftover) {
                 struct microcode_header_intel mc_header;
@@ -871,6 +904,7 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
                         new_mc  = mc;
                         new_mc_size = mc_size;
                         mc = NULL;      /* trigger new vmalloc */
+                        ret = UCODE_NEW;
                 }
 
                 ucode_ptr += mc_size;
@@ -900,7 +934,7 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
         pr_debug("CPU%d found a matching microcode update with version 0x%x (current=0x%x)\n",
                  cpu, new_rev, uci->cpu_sig.rev);
 
-        return UCODE_OK;
+        return ret;
 }
 
 static int get_ucode_fw(void *to, const void *from, size_t n)