Merge tag 'v3.3-rc7' into x86/mce

Merge reason: Update from an ancient -rc1 base to an almost-final stable kernel.

Signed-off-by: Ingo Molnar <mingo@elte.hu>

16 files changed, 174 insertions, 117 deletions

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index d43cad74f166..c0f7d68d318f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1044,6 +1044,9 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) =
 
 DEFINE_PER_CPU(unsigned int, irq_count) = -1;
 
+DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
+EXPORT_PER_CPU_SYMBOL(fpu_owner_task);
+
 /*
  * Special IST stacks which the CPU switches to when it calls
  * an IST-marked descriptor entry. Up to 7 stacks (hardware
@@ -1111,6 +1114,8 @@ void debug_stack_reset(void)
 
 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
+DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
+EXPORT_PER_CPU_SYMBOL(fpu_owner_task);
 
 #ifdef CONFIG_CC_STACKPROTECTOR
 DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 6b45e5e7a901..73d08ed98a64 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -326,8 +326,7 @@ static void __cpuinit amd_calc_l3_indices(struct amd_northbridge *nb)
         l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1;
 }
 
-static void __cpuinit amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf,
-                                        int index)
+static void __cpuinit amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index)
 {
         int node;
 
@@ -725,14 +724,16 @@ static DEFINE_PER_CPU(struct _cpuid4_info *, ici_cpuid4_info);
 #define CPUID4_INFO_IDX(x, y)   (&((per_cpu(ici_cpuid4_info, x))[y]))
 
 #ifdef CONFIG_SMP
-static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
+
+static int __cpuinit cache_shared_amd_cpu_map_setup(unsigned int cpu, int index)
 {
-        struct _cpuid4_info     *this_leaf, *sibling_leaf;
-        unsigned long num_threads_sharing;
-        int index_msb, i, sibling;
+        struct _cpuid4_info *this_leaf;
+        int ret, i, sibling;
         struct cpuinfo_x86 *c = &cpu_data(cpu);
 
-        if ((index == 3) && (c->x86_vendor == X86_VENDOR_AMD)) {
+        ret = 0;
+        if (index == 3) {
+                ret = 1;
                 for_each_cpu(i, cpu_llc_shared_mask(cpu)) {
                         if (!per_cpu(ici_cpuid4_info, i))
                                 continue;
@@ -743,8 +744,35 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
                                 set_bit(sibling, this_leaf->shared_cpu_map);
                         }
                 }
-                return;
+        } else if ((c->x86 == 0x15) && ((index == 1) || (index == 2))) {
+                ret = 1;
+                for_each_cpu(i, cpu_sibling_mask(cpu)) {
+                        if (!per_cpu(ici_cpuid4_info, i))
+                                continue;
+                        this_leaf = CPUID4_INFO_IDX(i, index);
+                        for_each_cpu(sibling, cpu_sibling_mask(cpu)) {
+                                if (!cpu_online(sibling))
+                                        continue;
+                                set_bit(sibling, this_leaf->shared_cpu_map);
+                        }
+                }
         }
+
+        return ret;
+}
+
+static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
+{
+        struct _cpuid4_info *this_leaf, *sibling_leaf;
+        unsigned long num_threads_sharing;
+        int index_msb, i;
+        struct cpuinfo_x86 *c = &cpu_data(cpu);
+
+        if (c->x86_vendor == X86_VENDOR_AMD) {
+                if (cache_shared_amd_cpu_map_setup(cpu, index))
+                        return;
+        }
+
         this_leaf = CPUID4_INFO_IDX(cpu, index);
         num_threads_sharing = 1 + this_leaf->base.eax.split.num_threads_sharing;
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 786e76a86322..e4eeaaf58a47 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -528,6 +528,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 
         sprintf(name, "threshold_bank%i", bank);
 
+#ifdef CONFIG_SMP
         if (cpu_data(cpu).cpu_core_id && shared_bank[bank]) {   /* symlink */
                 i = cpumask_first(cpu_llc_shared_mask(cpu));
 
@@ -553,6 +554,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 
                 goto out;
         }
+#endif
 
         b = kzalloc(sizeof(struct threshold_bank), GFP_KERNEL);
         if (!b) {
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index 8944062f46e2..c30c807ddc72 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -147,7 +147,9 @@ struct cpu_hw_events {
         /*
          * AMD specific bits
          */
-        struct amd_nb           *amd_nb;
+        struct amd_nb                   *amd_nb;
+        /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
+        u64                             perf_ctr_virt_mask;
 
         void                            *kfree_on_online;
 };
@@ -417,9 +419,11 @@ void x86_pmu_disable_all(void);
 static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
                                           u64 enable_mask)
 {
+        u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
+
         if (hwc->extra_reg.reg)
                 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
-        wrmsrl(hwc->config_base, hwc->config | enable_mask);
+        wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
 }
 
 void x86_pmu_enable_all(int added);
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index 0397b23be8e9..67250a52430b 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -1,4 +1,5 @@
 #include <linux/perf_event.h>
+#include <linux/export.h>
 #include <linux/types.h>
 #include <linux/init.h>
 #include <linux/slab.h>
@@ -357,7 +358,9 @@ static void amd_pmu_cpu_starting(int cpu)
         struct amd_nb *nb;
         int i, nb_id;
 
-        if (boot_cpu_data.x86_max_cores < 2)
+        cpuc->perf_ctr_virt_mask = AMD_PERFMON_EVENTSEL_HOSTONLY;
+
+        if (boot_cpu_data.x86_max_cores < 2 || boot_cpu_data.x86 == 0x15)
                 return;
 
         nb_id = amd_get_nb_id(cpu);
@@ -587,9 +590,9 @@ static __initconst const struct x86_pmu amd_pmu_f15h = {
         .put_event_constraints  = amd_put_event_constraints,
 
         .cpu_prepare            = amd_pmu_cpu_prepare,
-        .cpu_starting           = amd_pmu_cpu_starting,
         .cpu_dead               = amd_pmu_cpu_dead,
 #endif
+        .cpu_starting           = amd_pmu_cpu_starting,
 };
 
 __init int amd_pmu_init(void)
@@ -621,3 +624,33 @@ __init int amd_pmu_init(void)
 
         return 0;
 }
+
+void amd_pmu_enable_virt(void)
+{
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+        cpuc->perf_ctr_virt_mask = 0;
+
+        /* Reload all events */
+        x86_pmu_disable_all();
+        x86_pmu_enable_all(0);
+}
+EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
+
+void amd_pmu_disable_virt(void)
+{
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+        /*
+         * We only mask out the Host-only bit so that host-only counting works
+         * when SVM is disabled. If someone sets up a guest-only counter when
+         * SVM is disabled the Guest-only bits still gets set and the counter
+         * will not count anything.
+         */
+        cpuc->perf_ctr_virt_mask = AMD_PERFMON_EVENTSEL_HOSTONLY;
+
+        /* Reload all events */
+        x86_pmu_disable_all();
+        x86_pmu_enable_all(0);
+}
+EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 73da6b64f5b7..d6bd49faa40c 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -439,7 +439,6 @@ void intel_pmu_pebs_enable(struct perf_event *event)
         hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
 
         cpuc->pebs_enabled |= 1ULL << hwc->idx;
-        WARN_ON_ONCE(cpuc->enabled);
 
         if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
                 intel_pmu_lbr_enable(event);
diff --git a/arch/x86/kernel/cpu/perf_event_intel_lbr.c b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
index 3fab3de3ce96..47a7e63bfe54 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_lbr.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
@@ -72,8 +72,6 @@ void intel_pmu_lbr_enable(struct perf_event *event)
         if (!x86_pmu.lbr_nr)
                 return;
 
-        WARN_ON_ONCE(cpuc->enabled);
-
         /*
          * Reset the LBR stack if we changed task context to
          * avoid data leaks.
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index 1aae78f775fc..4025fe4f928f 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -252,7 +252,8 @@ int __kprobes __die(const char *str, struct pt_regs *regs, long err)
         unsigned short ss;
         unsigned long sp;
 #endif
-        printk(KERN_EMERG "%s: %04lx [#%d] ", str, err & 0xffff, ++die_counter);
+        printk(KERN_DEFAULT
+               "%s: %04lx [#%d] ", str, err & 0xffff, ++die_counter);
 #ifdef CONFIG_PREEMPT
         printk("PREEMPT ");
 #endif
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 6d728d9284bd..17107bd6e1f0 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -129,7 +129,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
         if (!stack) {
                 if (regs)
                         stack = (unsigned long *)regs->sp;
-                else if (task && task != current)
+                else if (task != current)
                         stack = (unsigned long *)task->thread.sp;
                 else
                         stack = &dummy;
@@ -269,11 +269,11 @@ void show_registers(struct pt_regs *regs)
                 unsigned char c;
                 u8 *ip;
 
-                printk(KERN_EMERG "Stack:\n");
+                printk(KERN_DEFAULT "Stack:\n");
                 show_stack_log_lvl(NULL, regs, (unsigned long *)sp,
-                                   0, KERN_EMERG);
+                                   0, KERN_DEFAULT);
 
-                printk(KERN_EMERG "Code: ");
+                printk(KERN_DEFAULT "Code: ");
 
                 ip = (u8 *)regs->ip - code_prologue;
                 if (ip < (u8 *)PAGE_OFFSET || probe_kernel_address(ip, c)) {
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 3fe8239fd8fb..1333d9851778 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1532,10 +1532,17 @@ ENTRY(nmi)
         pushq_cfi %rdx
 
         /*
+         * If %cs was not the kernel segment, then the NMI triggered in user
+         * space, which means it is definitely not nested.
+         */
+        cmpl $__KERNEL_CS, 16(%rsp)
+        jne first_nmi
+
+        /*
          * Check the special variable on the stack to see if NMIs are
          * executing.
          */
-        cmp $1, -8(%rsp)
+        cmpl $1, -8(%rsp)
         je nested_nmi
 
         /*
diff --git a/arch/x86/kernel/microcode_amd.c b/arch/x86/kernel/microcode_amd.c
index fe86493f3ed1..73465aab28f8 100644
--- a/arch/x86/kernel/microcode_amd.c
+++ b/arch/x86/kernel/microcode_amd.c
@@ -311,13 +311,33 @@ out:
         return state;
 }
 
+/*
+ * AMD microcode firmware naming convention, up to family 15h they are in
+ * the legacy file:
+ *
+ *    amd-ucode/microcode_amd.bin
+ *
+ * This legacy file is always smaller than 2K in size.
+ *
+ * Starting at family 15h they are in family specific firmware files:
+ *
+ *    amd-ucode/microcode_amd_fam15h.bin
+ *    amd-ucode/microcode_amd_fam16h.bin
+ *    ...
+ *
+ * These might be larger than 2K.
+ */
 static enum ucode_state request_microcode_amd(int cpu, struct device *device)
 {
-        const char *fw_name = "amd-ucode/microcode_amd.bin";
+        char fw_name[36] = "amd-ucode/microcode_amd.bin";
         const struct firmware *fw;
         enum ucode_state ret = UCODE_NFOUND;
+        struct cpuinfo_x86 *c = &cpu_data(cpu);
+
+        if (c->x86 >= 0x15)
+                snprintf(fw_name, sizeof(fw_name), "amd-ucode/microcode_amd_fam%.2xh.bin", c->x86);
 
-        if (request_firmware(&fw, fw_name, device)) {
+        if (request_firmware(&fw, (const char *)fw_name, device)) {
                 pr_err("failed to load file %s\n", fw_name);
                 goto out;
         }
@@ -340,7 +360,6 @@ out:
 static enum ucode_state
 request_microcode_user(int cpu, const void __user *buf, size_t size)
 {
-        pr_info("AMD microcode update via /dev/cpu/microcode not supported\n");
         return UCODE_ERROR;
 }
 
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 485204f58cda..c08d1ff12b7c 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -214,6 +214,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
 
         task_user_gs(p) = get_user_gs(regs);
 
+        p->fpu_counter = 0;
         p->thread.io_bitmap_ptr = NULL;
         tsk = current;
         err = -ENOMEM;
@@ -299,22 +300,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                                  *next = &next_p->thread;
         int cpu = smp_processor_id();
         struct tss_struct *tss = &per_cpu(init_tss, cpu);
-        bool preload_fpu;
+        fpu_switch_t fpu;
 
         /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
 
-        /*
-         * If the task has used fpu the last 5 timeslices, just do a full
-         * restore of the math state immediately to avoid the trap; the
-         * chances of needing FPU soon are obviously high now
-         */
-        preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
-
-        __unlazy_fpu(prev_p);
-
-        /* we're going to use this soon, after a few expensive things */
-        if (preload_fpu)
-                prefetch(next->fpu.state);
+        fpu = switch_fpu_prepare(prev_p, next_p, cpu);
 
         /*
          * Reload esp0.
@@ -354,11 +344,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                      task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
                 __switch_to_xtra(prev_p, next_p, tss);
 
-        /* If we're going to preload the fpu context, make sure clts
-           is run while we're batching the cpu state updates. */
-        if (preload_fpu)
-                clts();
-
         /*
          * Leave lazy mode, flushing any hypercalls made here.
          * This must be done before restoring TLS segments so
@@ -368,15 +353,14 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
          */
         arch_end_context_switch(next_p);
 
-        if (preload_fpu)
-                __math_state_restore();
-
         /*
          * Restore %gs if needed (which is common)
          */
         if (prev->gs | next->gs)
                 lazy_load_gs(next->gs);
 
+        switch_fpu_finish(next_p, fpu);
+
         percpu_write(current_task, next_p);
 
         return prev_p;
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 9b9fe4a85c87..cfa5c90c01db 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -286,6 +286,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
 
         set_tsk_thread_flag(p, TIF_FORK);
 
+        p->fpu_counter = 0;
         p->thread.io_bitmap_ptr = NULL;
 
         savesegment(gs, p->thread.gsindex);
@@ -386,18 +387,9 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         int cpu = smp_processor_id();
         struct tss_struct *tss = &per_cpu(init_tss, cpu);
         unsigned fsindex, gsindex;
-        bool preload_fpu;
+        fpu_switch_t fpu;
 
-        /*
-         * If the task has used fpu the last 5 timeslices, just do a full
-         * restore of the math state immediately to avoid the trap; the
-         * chances of needing FPU soon are obviously high now
-         */
-        preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
-
-        /* we're going to use this soon, after a few expensive things */
-        if (preload_fpu)
-                prefetch(next->fpu.state);
+        fpu = switch_fpu_prepare(prev_p, next_p, cpu);
 
         /*
          * Reload esp0, LDT and the page table pointer:
@@ -427,13 +419,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 
         load_TLS(next, cpu);
 
-        /* Must be after DS reload */
-        __unlazy_fpu(prev_p);
-
-        /* Make sure cpu is ready for new context */
-        if (preload_fpu)
-                clts();
-
         /*
          * Leave lazy mode, flushing any hypercalls made here.
          * This must be done before restoring TLS segments so
@@ -474,6 +459,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
         prev->gsindex = gsindex;
 
+        switch_fpu_finish(next_p, fpu);
+
         /*
          * Switch the PDA and FPU contexts.
          */
@@ -492,13 +479,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                      task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
                 __switch_to_xtra(prev_p, next_p, tss);
 
-        /*
-         * Preload the FPU context, now that we've determined that the
-         * task is likely to be using it. 
-         */
-        if (preload_fpu)
-                __math_state_restore();
-
         return prev_p;
 }
 
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index 37a458b521a6..d840e69a853c 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -39,6 +39,14 @@ static int reboot_mode;
 enum reboot_type reboot_type = BOOT_ACPI;
 int reboot_force;
 
+/* This variable is used privately to keep track of whether or not
+ * reboot_type is still set to its default value (i.e., reboot= hasn't
+ * been set on the command line).  This is needed so that we can
+ * suppress DMI scanning for reboot quirks.  Without it, it's
+ * impossible to override a faulty reboot quirk without recompiling.
+ */
+static int reboot_default = 1;
+
 #if defined(CONFIG_X86_32) && defined(CONFIG_SMP)
 static int reboot_cpu = -1;
 #endif
@@ -67,6 +75,12 @@ bool port_cf9_safe = false;
 static int __init reboot_setup(char *str)
 {
         for (;;) {
+                /* Having anything passed on the command line via
+                 * reboot= will cause us to disable DMI checking
+                 * below.
+                 */
+                reboot_default = 0;
+
                 switch (*str) {
                 case 'w':
                         reboot_mode = 0x1234;
@@ -295,14 +309,6 @@ static struct dmi_system_id __initdata reboot_dmi_table[] = {
                         DMI_MATCH(DMI_BOARD_NAME, "P4S800"),
                 },
         },
-        {       /* Handle problems with rebooting on VersaLogic Menlow boards */
-                .callback = set_bios_reboot,
-                .ident = "VersaLogic Menlow based board",
-                .matches = {
-                        DMI_MATCH(DMI_BOARD_VENDOR, "VersaLogic Corporation"),
-                        DMI_MATCH(DMI_BOARD_NAME, "VersaLogic Menlow board"),
-                },
-        },
         { /* Handle reboot issue on Acer Aspire one */
                 .callback = set_kbd_reboot,
                 .ident = "Acer Aspire One A110",
@@ -316,7 +322,12 @@ static struct dmi_system_id __initdata reboot_dmi_table[] = {
 
 static int __init reboot_init(void)
 {
-        dmi_check_system(reboot_dmi_table);
+        /* Only do the DMI check if reboot_type hasn't been overridden
+         * on the command line
+         */
+        if (reboot_default) {
+                dmi_check_system(reboot_dmi_table);
+        }
         return 0;
 }
 core_initcall(reboot_init);
@@ -465,7 +476,12 @@ static struct dmi_system_id __initdata pci_reboot_dmi_table[] = {
 
 static int __init pci_reboot_init(void)
 {
-        dmi_check_system(pci_reboot_dmi_table);
+        /* Only do the DMI check if reboot_type hasn't been overridden
+         * on the command line
+         */
+        if (reboot_default) {
+                dmi_check_system(pci_reboot_dmi_table);
+        }
         return 0;
 }
 core_initcall(pci_reboot_init);
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 482ec3af2067..4bbe04d96744 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -571,41 +571,18 @@ asmlinkage void __attribute__((weak)) smp_threshold_interrupt(void)
 }
 
 /*
- * __math_state_restore assumes that cr0.TS is already clear and the
- * fpu state is all ready for use.  Used during context switch.
- */
-void __math_state_restore(void)
-{
-        struct thread_info *thread = current_thread_info();
-        struct task_struct *tsk = thread->task;
-
-        /*
-         * Paranoid restore. send a SIGSEGV if we fail to restore the state.
-         */
-        if (unlikely(restore_fpu_checking(tsk))) {
-                stts();
-                force_sig(SIGSEGV, tsk);
-                return;
-        }
-
-        thread->status |= TS_USEDFPU;   /* So we fnsave on switch_to() */
-        tsk->fpu_counter++;
-}
-
-/*
  * 'math_state_restore()' saves the current math information in the
  * old math state array, and gets the new ones from the current task
  *
  * Careful.. There are problems with IBM-designed IRQ13 behaviour.
  * Don't touch unless you *really* know how it works.
  *
- * Must be called with kernel preemption disabled (in this case,
- * local interrupts are disabled at the call-site in entry.S).
+ * Must be called with kernel preemption disabled (eg with local
+ * local interrupts as in the case of do_device_not_available).
  */
-asmlinkage void math_state_restore(void)
+void math_state_restore(void)
 {
-        struct thread_info *thread = current_thread_info();
-        struct task_struct *tsk = thread->task;
+        struct task_struct *tsk = current;
 
         if (!tsk_used_math(tsk)) {
                 local_irq_enable();
@@ -622,9 +599,17 @@ asmlinkage void math_state_restore(void)
                 local_irq_disable();
         }
 
-        clts();                         /* Allow maths ops (or we recurse) */
+        __thread_fpu_begin(tsk);
+        /*
+         * Paranoid restore. send a SIGSEGV if we fail to restore the state.
+         */
+        if (unlikely(restore_fpu_checking(tsk))) {
+                __thread_fpu_end(tsk);
+                force_sig(SIGSEGV, tsk);
+                return;
+        }
 
-        __math_state_restore();
+        tsk->fpu_counter++;
 }
 EXPORT_SYMBOL_GPL(math_state_restore);
 
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index a3911343976b..711091114119 100644
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -47,7 +47,7 @@ void __sanitize_i387_state(struct task_struct *tsk)
         if (!fx)
                 return;
 
-        BUG_ON(task_thread_info(tsk)->status & TS_USEDFPU);
+        BUG_ON(__thread_has_fpu(tsk));
 
         xstate_bv = tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv;
 
@@ -168,7 +168,7 @@ int save_i387_xstate(void __user *buf)
         if (!used_math())
                 return 0;
 
-        if (task_thread_info(tsk)->status & TS_USEDFPU) {
+        if (user_has_fpu()) {
                 if (use_xsave())
                         err = xsave_user(buf);
                 else
@@ -176,8 +176,7 @@ int save_i387_xstate(void __user *buf)
 
                 if (err)
                         return err;
-                task_thread_info(tsk)->status &= ~TS_USEDFPU;
-                stts();
+                user_fpu_end();
         } else {
                 sanitize_i387_state(tsk);
                 if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave,
@@ -292,10 +291,7 @@ int restore_i387_xstate(void __user *buf)
                         return err;
         }
 
-        if (!(task_thread_info(current)->status & TS_USEDFPU)) {
-                clts();
-                task_thread_info(current)->status |= TS_USEDFPU;
-        }
+        user_fpu_begin();
         if (use_xsave())
                 err = restore_user_xstate(buf);
         else