x86, mce: Support specifying context for software mce injection

The cpu context is specified via the new mce.inject_flags fields.
This allows more realistic machine check testing in different
situations. "RANDOM" context is implemented via NMI broadcasting to
add randomization to testing.

AK: Fix NMI broadcasting check. Fix 32-bit building. Some race
fixes. Move to module. Various changes

ChangeLog:

v3:

- Re-based on latest x86-tip.git/mce4

- Fix 32-bit building

v2:

- Re-base on latest x86-tip.git/mce3

Signed-off-by: Huang Ying <ying.huang@intel.com>
Signed-off-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>

2 files changed, 135 insertions, 32 deletions

diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index ad7535372918..8945be9ad2b1 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -38,6 +38,13 @@
 #define MCM_ADDR_MEM     3      /* memory address */
 #define MCM_ADDR_GENERIC 7      /* generic */
 
+#define MCJ_CTX_MASK            3
+#define MCJ_CTX(flags)          ((flags) & MCJ_CTX_MASK)
+#define MCJ_CTX_RANDOM          0    /* inject context: random */
+#define MCJ_CTX_PROCESS         1    /* inject context: process */
+#define MCJ_CTX_IRQ             2    /* inject context: IRQ */
+#define MCJ_NMI_BROADCAST       4    /* do NMI broadcasting */
+
 /* Fields are zero when not available */
 struct mce {
         __u64 status;
@@ -48,8 +55,8 @@ struct mce {
         __u64 tsc;      /* cpu time stamp counter */
         __u64 time;     /* wall time_t when error was detected */
         __u8  cpuvendor;        /* cpu vendor as encoded in system.h */
-        __u8  pad1;
-        __u16 pad2;
+        __u8  inject_flags;     /* software inject flags */
+        __u16  pad;
         __u32 cpuid;    /* CPUID 1 EAX */
         __u8  cs;               /* code segment */
         __u8  bank;     /* machine check bank */
diff --git a/arch/x86/kernel/cpu/mcheck/mce-inject.c b/arch/x86/kernel/cpu/mcheck/mce-inject.c
index a3a235a53f09..ad5d92790ebc 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-inject.c
+++ b/arch/x86/kernel/cpu/mcheck/mce-inject.c
@@ -18,7 +18,12 @@
 #include <linux/string.h>
 #include <linux/fs.h>
 #include <linux/smp.h>
+#include <linux/notifier.h>
+#include <linux/kdebug.h>
+#include <linux/cpu.h>
+#include <linux/sched.h>
 #include <asm/mce.h>
+#include <asm/apic.h>
 
 /* Update fake mce registers on current CPU. */
 static void inject_mce(struct mce *m)
@@ -39,44 +44,141 @@ static void inject_mce(struct mce *m)
         i->finished = 1;
 }
 
-struct delayed_mce {
-        struct timer_list timer;
-        struct mce m;
+static void raise_corrected(struct mce *m)
+{
+        unsigned long flags;
+        mce_banks_t b;
+
+        memset(&b, 0xff, sizeof(mce_banks_t));
+        local_irq_save(flags);
+        machine_check_poll(0, &b);
+        local_irq_restore(flags);
+        m->finished = 0;
+}
+
+static void raise_uncorrected(struct mce *m, struct pt_regs *pregs)
+{
+        struct pt_regs regs;
+        unsigned long flags;
+
+        if (!pregs) {
+                memset(&regs, 0, sizeof(struct pt_regs));
+                regs.ip = m->ip;
+                regs.cs = m->cs;
+                pregs = &regs;
+        }
+        /* in mcheck exeception handler, irq will be disabled */
+        local_irq_save(flags);
+        do_machine_check(pregs, 0);
+        local_irq_restore(flags);
+        m->finished = 0;
+}
+
+static cpumask_t mce_inject_cpumask;
+
+static int mce_raise_notify(struct notifier_block *self,
+                            unsigned long val, void *data)
+{
+        struct die_args *args = (struct die_args *)data;
+        int cpu = smp_processor_id();
+        struct mce *m = &__get_cpu_var(injectm);
+        if (val != DIE_NMI_IPI || !cpu_isset(cpu, mce_inject_cpumask))
+                return NOTIFY_DONE;
+        cpu_clear(cpu, mce_inject_cpumask);
+        if (m->status & MCI_STATUS_UC)
+                raise_uncorrected(m, args->regs);
+        else if (m->status)
+                raise_corrected(m);
+        return NOTIFY_STOP;
+}
+
+static struct notifier_block mce_raise_nb = {
+        .notifier_call = mce_raise_notify,
+        .priority = 1000,
 };
 
 /* Inject mce on current CPU */
-static void raise_mce(unsigned long data)
+static int raise_local(struct mce *m)
 {
-        struct delayed_mce *dm = (struct delayed_mce *)data;
-        struct mce *m = &dm->m;
+        int context = MCJ_CTX(m->inject_flags);
+        int ret = 0;
         int cpu = m->extcpu;
 
-        inject_mce(m);
         if (m->status & MCI_STATUS_UC) {
-                struct pt_regs regs;
-                memset(&regs, 0, sizeof(struct pt_regs));
-                regs.ip = m->ip;
-                regs.cs = m->cs;
                 printk(KERN_INFO "Triggering MCE exception on CPU %d\n", cpu);
-                do_machine_check(&regs, 0);
+                switch (context) {
+                case MCJ_CTX_IRQ:
+                        /*
+                         * Could do more to fake interrupts like
+                         * calling irq_enter, but the necessary
+                         * machinery isn't exported currently.
+                         */
+                        /*FALL THROUGH*/
+                case MCJ_CTX_PROCESS:
+                        raise_uncorrected(m, NULL);
+                        break;
+                default:
+                        printk(KERN_INFO "Invalid MCE context\n");
+                        ret = -EINVAL;
+                }
                 printk(KERN_INFO "MCE exception done on CPU %d\n", cpu);
-        } else {
-                mce_banks_t b;
-                memset(&b, 0xff, sizeof(mce_banks_t));
+        } else if (m->status) {
                 printk(KERN_INFO "Starting machine check poll CPU %d\n", cpu);
-                machine_check_poll(0, &b);
+                raise_corrected(m);
                 mce_notify_irq();
-                printk(KERN_INFO "Finished machine check poll on CPU %d\n",
-                       cpu);
-        }
-        kfree(dm);
+                printk(KERN_INFO "Machine check poll done on CPU %d\n", cpu);
+        } else
+                m->finished = 0;
+
+        return ret;
+}
+
+static void raise_mce(struct mce *m)
+{
+        int context = MCJ_CTX(m->inject_flags);
+
+        inject_mce(m);
+
+        if (context == MCJ_CTX_RANDOM)
+                return;
+
+#ifdef CONFIG_X86_LOCAL_APIC
+        if (m->inject_flags & MCJ_NMI_BROADCAST) {
+                unsigned long start;
+                int cpu;
+                get_online_cpus();
+                mce_inject_cpumask = cpu_online_map;
+                cpu_clear(get_cpu(), mce_inject_cpumask);
+                for_each_online_cpu(cpu) {
+                        struct mce *mcpu = &per_cpu(injectm, cpu);
+                        if (!mcpu->finished ||
+                            MCJ_CTX(mcpu->inject_flags) != MCJ_CTX_RANDOM)
+                                cpu_clear(cpu, mce_inject_cpumask);
+                }
+                if (!cpus_empty(mce_inject_cpumask))
+                        apic->send_IPI_mask(&mce_inject_cpumask, NMI_VECTOR);
+                start = jiffies;
+                while (!cpus_empty(mce_inject_cpumask)) {
+                        if (!time_before(jiffies, start + 2*HZ)) {
+                                printk(KERN_ERR
+                                "Timeout waiting for mce inject NMI %lx\n",
+                                        *cpus_addr(mce_inject_cpumask));
+                                break;
+                        }
+                        cpu_relax();
+                }
+                raise_local(m);
+                put_cpu();
+                put_online_cpus();
+        } else
+#endif
+                raise_local(m);
 }
 
 /* Error injection interface */
 static ssize_t mce_write(struct file *filp, const char __user *ubuf,
                          size_t usize, loff_t *off)
 {
-        struct delayed_mce *dm;
         struct mce m;
 
         if (!capable(CAP_SYS_ADMIN))
@@ -96,19 +198,12 @@ static ssize_t mce_write(struct file *filp, const char __user *ubuf,
         if (m.extcpu >= num_possible_cpus() || !cpu_online(m.extcpu))
                 return -EINVAL;
 
-        dm = kmalloc(sizeof(struct delayed_mce), GFP_KERNEL);
-        if (!dm)
-                return -ENOMEM;
-
         /*
          * Need to give user space some time to set everything up,
          * so do it a jiffie or two later everywhere.
-         * Should we use a hrtimer here for better synchronization?
          */
-        memcpy(&dm->m, &m, sizeof(struct mce));
-        setup_timer(&dm->timer, raise_mce, (unsigned long)dm);
-        dm->timer.expires = jiffies + 2;
-        add_timer_on(&dm->timer, m.extcpu);
+        schedule_timeout(2);
+        raise_mce(&m);
         return usize;
 }
 
@@ -116,6 +211,7 @@ static int inject_init(void)
 {
         printk(KERN_INFO "Machine check injector initialized\n");
         mce_chrdev_ops.write = mce_write;
+        register_die_notifier(&mce_raise_nb);
         return 0;
 }