25 files changed, 493 insertions, 226 deletions

diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 6919e936345b..247904945d3f 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -29,10 +29,11 @@ extern unsigned int sig_xstate_size;
 extern void fpu_init(void);
 extern void mxcsr_feature_mask_init(void);
 extern int init_fpu(struct task_struct *child);
-extern asmlinkage void math_state_restore(void);
-extern void __math_state_restore(void);
+extern void math_state_restore(void);
 extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
 
+DECLARE_PER_CPU(struct task_struct *, fpu_owner_task);
+
 extern user_regset_active_fn fpregs_active, xfpregs_active;
 extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
                                 xstateregs_get;
@@ -212,19 +213,11 @@ static inline void fpu_fxsave(struct fpu *fpu)
 
 #endif  /* CONFIG_X86_64 */
 
-/* We need a safe address that is cheap to find and that is already
-   in L1 during context switch. The best choices are unfortunately
-   different for UP and SMP */
-#ifdef CONFIG_SMP
-#define safe_address (__per_cpu_offset[0])
-#else
-#define safe_address (__get_cpu_var(kernel_cpustat).cpustat[CPUTIME_USER])
-#endif
-
 /*
- * These must be called with preempt disabled
+ * These must be called with preempt disabled. Returns
+ * 'true' if the FPU state is still intact.
  */
-static inline void fpu_save_init(struct fpu *fpu)
+static inline int fpu_save_init(struct fpu *fpu)
 {
         if (use_xsave()) {
                 fpu_xsave(fpu);
@@ -233,33 +226,33 @@ static inline void fpu_save_init(struct fpu *fpu)
                  * xsave header may indicate the init state of the FP.
                  */
                 if (!(fpu->state->xsave.xsave_hdr.xstate_bv & XSTATE_FP))
-                        return;
+                        return 1;
         } else if (use_fxsr()) {
                 fpu_fxsave(fpu);
         } else {
                 asm volatile("fnsave %[fx]; fwait"
                              : [fx] "=m" (fpu->state->fsave));
-                return;
+                return 0;
         }
 
-        if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES))
+        /*
+         * If exceptions are pending, we need to clear them so
+         * that we don't randomly get exceptions later.
+         *
+         * FIXME! Is this perhaps only true for the old-style
+         * irq13 case? Maybe we could leave the x87 state
+         * intact otherwise?
+         */
+        if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES)) {
                 asm volatile("fnclex");
-
-        /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
-           is pending.  Clear the x87 state here by setting it to fixed
-           values. safe_address is a random variable that should be in L1 */
-        alternative_input(
-                ASM_NOP8 ASM_NOP2,
-                "emms\n\t"              /* clear stack tags */
-                "fildl %P[addr]",       /* set F?P to defined value */
-                X86_FEATURE_FXSAVE_LEAK,
-                [addr] "m" (safe_address));
+                return 0;
+        }
+        return 1;
 }
 
-static inline void __save_init_fpu(struct task_struct *tsk)
+static inline int __save_init_fpu(struct task_struct *tsk)
 {
-        fpu_save_init(&tsk->thread.fpu);
-        task_thread_info(tsk)->status &= ~TS_USEDFPU;
+        return fpu_save_init(&tsk->thread.fpu);
 }
 
 static inline int fpu_fxrstor_checking(struct fpu *fpu)
@@ -277,44 +270,212 @@ static inline int fpu_restore_checking(struct fpu *fpu)
 
 static inline int restore_fpu_checking(struct task_struct *tsk)
 {
+        /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
+           is pending.  Clear the x87 state here by setting it to fixed
+           values. "m" is a random variable that should be in L1 */
+        alternative_input(
+                ASM_NOP8 ASM_NOP2,
+                "emms\n\t"              /* clear stack tags */
+                "fildl %P[addr]",       /* set F?P to defined value */
+                X86_FEATURE_FXSAVE_LEAK,
+                [addr] "m" (tsk->thread.fpu.has_fpu));
+
         return fpu_restore_checking(&tsk->thread.fpu);
 }
 
 /*
- * Signal frame handlers...
+ * Software FPU state helpers. Careful: these need to
+ * be preemption protection *and* they need to be
+ * properly paired with the CR0.TS changes!
  */
-extern int save_i387_xstate(void __user *buf);
-extern int restore_i387_xstate(void __user *buf);
+static inline int __thread_has_fpu(struct task_struct *tsk)
+{
+        return tsk->thread.fpu.has_fpu;
+}
 
-static inline void __unlazy_fpu(struct task_struct *tsk)
+/* Must be paired with an 'stts' after! */
+static inline void __thread_clear_has_fpu(struct task_struct *tsk)
 {
-        if (task_thread_info(tsk)->status & TS_USEDFPU) {
-                __save_init_fpu(tsk);
-                stts();
-        } else
-                tsk->fpu_counter = 0;
+        tsk->thread.fpu.has_fpu = 0;
+        percpu_write(fpu_owner_task, NULL);
+}
+
+/* Must be paired with a 'clts' before! */
+static inline void __thread_set_has_fpu(struct task_struct *tsk)
+{
+        tsk->thread.fpu.has_fpu = 1;
+        percpu_write(fpu_owner_task, tsk);
+}
+
+/*
+ * Encapsulate the CR0.TS handling together with the
+ * software flag.
+ *
+ * These generally need preemption protection to work,
+ * do try to avoid using these on their own.
+ */
+static inline void __thread_fpu_end(struct task_struct *tsk)
+{
+        __thread_clear_has_fpu(tsk);
+        stts();
+}
+
+static inline void __thread_fpu_begin(struct task_struct *tsk)
+{
+        clts();
+        __thread_set_has_fpu(tsk);
+}
+
+/*
+ * FPU state switching for scheduling.
+ *
+ * This is a two-stage process:
+ *
+ *  - switch_fpu_prepare() saves the old state and
+ *    sets the new state of the CR0.TS bit. This is
+ *    done within the context of the old process.
+ *
+ *  - switch_fpu_finish() restores the new state as
+ *    necessary.
+ */
+typedef struct { int preload; } fpu_switch_t;
+
+/*
+ * FIXME! We could do a totally lazy restore, but we need to
+ * add a per-cpu "this was the task that last touched the FPU
+ * on this CPU" variable, and the task needs to have a "I last
+ * touched the FPU on this CPU" and check them.
+ *
+ * We don't do that yet, so "fpu_lazy_restore()" always returns
+ * false, but some day..
+ */
+static inline int fpu_lazy_restore(struct task_struct *new, unsigned int cpu)
+{
+        return new == percpu_read_stable(fpu_owner_task) &&
+                cpu == new->thread.fpu.last_cpu;
+}
+
+static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct task_struct *new, int cpu)
+{
+        fpu_switch_t fpu;
+
+        fpu.preload = tsk_used_math(new) && new->fpu_counter > 5;
+        if (__thread_has_fpu(old)) {
+                if (!__save_init_fpu(old))
+                        cpu = ~0;
+                old->thread.fpu.last_cpu = cpu;
+                old->thread.fpu.has_fpu = 0;    /* But leave fpu_owner_task! */
+
+                /* Don't change CR0.TS if we just switch! */
+                if (fpu.preload) {
+                        new->fpu_counter++;
+                        __thread_set_has_fpu(new);
+                        prefetch(new->thread.fpu.state);
+                } else
+                        stts();
+        } else {
+                old->fpu_counter = 0;
+                old->thread.fpu.last_cpu = ~0;
+                if (fpu.preload) {
+                        new->fpu_counter++;
+                        if (fpu_lazy_restore(new, cpu))
+                                fpu.preload = 0;
+                        else
+                                prefetch(new->thread.fpu.state);
+                        __thread_fpu_begin(new);
+                }
+        }
+        return fpu;
+}
+
+/*
+ * By the time this gets called, we've already cleared CR0.TS and
+ * given the process the FPU if we are going to preload the FPU
+ * state - all we need to do is to conditionally restore the register
+ * state itself.
+ */
+static inline void switch_fpu_finish(struct task_struct *new, fpu_switch_t fpu)
+{
+        if (fpu.preload) {
+                if (unlikely(restore_fpu_checking(new)))
+                        __thread_fpu_end(new);
+        }
 }
 
+/*
+ * Signal frame handlers...
+ */
+extern int save_i387_xstate(void __user *buf);
+extern int restore_i387_xstate(void __user *buf);
+
 static inline void __clear_fpu(struct task_struct *tsk)
 {
-        if (task_thread_info(tsk)->status & TS_USEDFPU) {
+        if (__thread_has_fpu(tsk)) {
                 /* Ignore delayed exceptions from user space */
                 asm volatile("1: fwait\n"
                              "2:\n"
                              _ASM_EXTABLE(1b, 2b));
-                task_thread_info(tsk)->status &= ~TS_USEDFPU;
-                stts();
+                __thread_fpu_end(tsk);
         }
 }
 
+/*
+ * Were we in an interrupt that interrupted kernel mode?
+ *
+ * We can do a kernel_fpu_begin/end() pair *ONLY* if that
+ * pair does nothing at all: the thread must not have fpu (so
+ * that we don't try to save the FPU state), and TS must
+ * be set (so that the clts/stts pair does nothing that is
+ * visible in the interrupted kernel thread).
+ */
+static inline bool interrupted_kernel_fpu_idle(void)
+{
+        return !__thread_has_fpu(current) &&
+                (read_cr0() & X86_CR0_TS);
+}
+
+/*
+ * Were we in user mode (or vm86 mode) when we were
+ * interrupted?
+ *
+ * Doing kernel_fpu_begin/end() is ok if we are running
+ * in an interrupt context from user mode - we'll just
+ * save the FPU state as required.
+ */
+static inline bool interrupted_user_mode(void)
+{
+        struct pt_regs *regs = get_irq_regs();
+        return regs && user_mode_vm(regs);
+}
+
+/*
+ * Can we use the FPU in kernel mode with the
+ * whole "kernel_fpu_begin/end()" sequence?
+ *
+ * It's always ok in process context (ie "not interrupt")
+ * but it is sometimes ok even from an irq.
+ */
+static inline bool irq_fpu_usable(void)
+{
+        return !in_interrupt() ||
+                interrupted_user_mode() ||
+                interrupted_kernel_fpu_idle();
+}
+
 static inline void kernel_fpu_begin(void)
 {
-        struct thread_info *me = current_thread_info();
+        struct task_struct *me = current;
+
+        WARN_ON_ONCE(!irq_fpu_usable());
         preempt_disable();
-        if (me->status & TS_USEDFPU)
-                __save_init_fpu(me->task);
-        else
+        if (__thread_has_fpu(me)) {
+                __save_init_fpu(me);
+                __thread_clear_has_fpu(me);
+                /* We do 'stts()' in kernel_fpu_end() */
+        } else {
+                percpu_write(fpu_owner_task, NULL);
                 clts();
+        }
 }
 
 static inline void kernel_fpu_end(void)
@@ -323,14 +484,6 @@ static inline void kernel_fpu_end(void)
         preempt_enable();
 }
 
-static inline bool irq_fpu_usable(void)
-{
-        struct pt_regs *regs;
-
-        return !in_interrupt() || !(regs = get_irq_regs()) || \
-                user_mode(regs) || (read_cr0() & X86_CR0_TS);
-}
-
 /*
  * Some instructions like VIA's padlock instructions generate a spurious
  * DNA fault but don't modify SSE registers. And these instructions
@@ -363,20 +516,64 @@ static inline void irq_ts_restore(int TS_state)
 }
 
 /*
+ * The question "does this thread have fpu access?"
+ * is slightly racy, since preemption could come in
+ * and revoke it immediately after the test.
+ *
+ * However, even in that very unlikely scenario,
+ * we can just assume we have FPU access - typically
+ * to save the FP state - we'll just take a #NM
+ * fault and get the FPU access back.
+ *
+ * The actual user_fpu_begin/end() functions
+ * need to be preemption-safe, though.
+ *
+ * NOTE! user_fpu_end() must be used only after you
+ * have saved the FP state, and user_fpu_begin() must
+ * be used only immediately before restoring it.
+ * These functions do not do any save/restore on
+ * their own.
+ */
+static inline int user_has_fpu(void)
+{
+        return __thread_has_fpu(current);
+}
+
+static inline void user_fpu_end(void)
+{
+        preempt_disable();
+        __thread_fpu_end(current);
+        preempt_enable();
+}
+
+static inline void user_fpu_begin(void)
+{
+        preempt_disable();
+        if (!user_has_fpu())
+                __thread_fpu_begin(current);
+        preempt_enable();
+}
+
+/*
  * These disable preemption on their own and are safe
  */
 static inline void save_init_fpu(struct task_struct *tsk)
 {
+        WARN_ON_ONCE(!__thread_has_fpu(tsk));
         preempt_disable();
         __save_init_fpu(tsk);
-        stts();
+        __thread_fpu_end(tsk);
         preempt_enable();
 }
 
 static inline void unlazy_fpu(struct task_struct *tsk)
 {
         preempt_disable();
-        __unlazy_fpu(tsk);
+        if (__thread_has_fpu(tsk)) {
+                __save_init_fpu(tsk);
+                __thread_fpu_end(tsk);
+        } else
+                tsk->fpu_counter = 0;
         preempt_enable();
 }
 
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 096c975e099f..461ce432b1c2 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -242,4 +242,12 @@ static inline void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
 static inline void perf_events_lapic_init(void) { }
 #endif
 
+#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD)
+ extern void amd_pmu_enable_virt(void);
+ extern void amd_pmu_disable_virt(void);
+#else
+ static inline void amd_pmu_enable_virt(void) { }
+ static inline void amd_pmu_disable_virt(void) { }
+#endif
+
 #endif /* _ASM_X86_PERF_EVENT_H */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index e34f95129f16..02ce0b379647 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -374,6 +374,8 @@ union thread_xstate {
 };
 
 struct fpu {
+        unsigned int last_cpu;
+        unsigned int has_fpu;
         union thread_xstate *state;
 };
 
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 912e93511466..af1db7e722f4 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -251,8 +251,6 @@ static inline struct thread_info *current_thread_info(void)
  * ever touches our thread-synchronous status, so we don't
  * have to worry about atomic accesses.
  */
-#define TS_USEDFPU              0x0001  /* FPU was used by this task
-                                           this quantum (SMP) */
 #define TS_COMPAT               0x0002  /* 32bit syscall active (64BIT)*/
 #define TS_POLLING              0x0004  /* idle task polling need_resched,
                                            skip sending interrupt */
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 0012d0902c5f..88eae2aec619 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -89,4 +89,29 @@ asmlinkage void smp_thermal_interrupt(void);
 asmlinkage void mce_threshold_interrupt(void);
 #endif
 
+/* Interrupts/Exceptions */
+enum {
+        X86_TRAP_DE = 0,        /*  0, Divide-by-zero */
+        X86_TRAP_DB,            /*  1, Debug */
+        X86_TRAP_NMI,           /*  2, Non-maskable Interrupt */
+        X86_TRAP_BP,            /*  3, Breakpoint */
+        X86_TRAP_OF,            /*  4, Overflow */
+        X86_TRAP_BR,            /*  5, Bound Range Exceeded */
+        X86_TRAP_UD,            /*  6, Invalid Opcode */
+        X86_TRAP_NM,            /*  7, Device Not Available */
+        X86_TRAP_DF,            /*  8, Double Fault */
+        X86_TRAP_OLD_MF,        /*  9, Coprocessor Segment Overrun */
+        X86_TRAP_TS,            /* 10, Invalid TSS */
+        X86_TRAP_NP,            /* 11, Segment Not Present */
+        X86_TRAP_SS,            /* 12, Stack Segment Fault */
+        X86_TRAP_GP,            /* 13, General Protection Fault */
+        X86_TRAP_PF,            /* 14, Page Fault */
+        X86_TRAP_SPURIOUS,      /* 15, Spurious Interrupt */
+        X86_TRAP_MF,            /* 16, x87 Floating-Point Exception */
+        X86_TRAP_AC,            /* 17, Alignment Check */
+        X86_TRAP_MC,            /* 18, Machine Check */
+        X86_TRAP_XF,            /* 19, SIMD Floating-Point Exception */
+        X86_TRAP_IRET = 32,     /* 32, IRET Exception */
+};
+
 #endif /* _ASM_X86_TRAPS_H */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index d43cad74f166..c0f7d68d318f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1044,6 +1044,9 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) =
 
 DEFINE_PER_CPU(unsigned int, irq_count) = -1;
 
+DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
+EXPORT_PER_CPU_SYMBOL(fpu_owner_task);
+
 /*
  * Special IST stacks which the CPU switches to when it calls
  * an IST-marked descriptor entry. Up to 7 stacks (hardware
@@ -1111,6 +1114,8 @@ void debug_stack_reset(void)
 
 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
+DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
+EXPORT_PER_CPU_SYMBOL(fpu_owner_task);
 
 #ifdef CONFIG_CC_STACKPROTECTOR
 DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 6b45e5e7a901..73d08ed98a64 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -326,8 +326,7 @@ static void __cpuinit amd_calc_l3_indices(struct amd_northbridge *nb)
         l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1;
 }
 
-static void __cpuinit amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf,
-                                        int index)
+static void __cpuinit amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index)
 {
         int node;
 
@@ -725,14 +724,16 @@ static DEFINE_PER_CPU(struct _cpuid4_info *, ici_cpuid4_info);
 #define CPUID4_INFO_IDX(x, y)   (&((per_cpu(ici_cpuid4_info, x))[y]))
 
 #ifdef CONFIG_SMP
-static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
+
+static int __cpuinit cache_shared_amd_cpu_map_setup(unsigned int cpu, int index)
 {
-        struct _cpuid4_info     *this_leaf, *sibling_leaf;
-        unsigned long num_threads_sharing;
-        int index_msb, i, sibling;
+        struct _cpuid4_info *this_leaf;
+        int ret, i, sibling;
         struct cpuinfo_x86 *c = &cpu_data(cpu);
 
-        if ((index == 3) && (c->x86_vendor == X86_VENDOR_AMD)) {
+        ret = 0;
+        if (index == 3) {
+                ret = 1;
                 for_each_cpu(i, cpu_llc_shared_mask(cpu)) {
                         if (!per_cpu(ici_cpuid4_info, i))
                                 continue;
@@ -743,8 +744,35 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
                                 set_bit(sibling, this_leaf->shared_cpu_map);
                         }
                 }
-                return;
+        } else if ((c->x86 == 0x15) && ((index == 1) || (index == 2))) {
+                ret = 1;
+                for_each_cpu(i, cpu_sibling_mask(cpu)) {
+                        if (!per_cpu(ici_cpuid4_info, i))
+                                continue;
+                        this_leaf = CPUID4_INFO_IDX(i, index);
+                        for_each_cpu(sibling, cpu_sibling_mask(cpu)) {
+                                if (!cpu_online(sibling))
+                                        continue;
+                                set_bit(sibling, this_leaf->shared_cpu_map);
+                        }
+                }
         }
+
+        return ret;
+}
+
+static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
+{
+        struct _cpuid4_info *this_leaf, *sibling_leaf;
+        unsigned long num_threads_sharing;
+        int index_msb, i;
+        struct cpuinfo_x86 *c = &cpu_data(cpu);
+
+        if (c->x86_vendor == X86_VENDOR_AMD) {
+                if (cache_shared_amd_cpu_map_setup(cpu, index))
+                        return;
+        }
+
         this_leaf = CPUID4_INFO_IDX(cpu, index);
         num_threads_sharing = 1 + this_leaf->base.eax.split.num_threads_sharing;
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 786e76a86322..e4eeaaf58a47 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -528,6 +528,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 
         sprintf(name, "threshold_bank%i", bank);
 
+#ifdef CONFIG_SMP
         if (cpu_data(cpu).cpu_core_id && shared_bank[bank]) {   /* symlink */
                 i = cpumask_first(cpu_llc_shared_mask(cpu));
 
@@ -553,6 +554,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 
                 goto out;
         }
+#endif
 
         b = kzalloc(sizeof(struct threshold_bank), GFP_KERNEL);
         if (!b) {
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index 8944062f46e2..c30c807ddc72 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -147,7 +147,9 @@ struct cpu_hw_events {
         /*
          * AMD specific bits
          */
-        struct amd_nb           *amd_nb;
+        struct amd_nb                   *amd_nb;
+        /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
+        u64                             perf_ctr_virt_mask;
 
         void                            *kfree_on_online;
 };
@@ -417,9 +419,11 @@ void x86_pmu_disable_all(void);
 static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
                                           u64 enable_mask)
 {
+        u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
+
         if (hwc->extra_reg.reg)
                 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
-        wrmsrl(hwc->config_base, hwc->config | enable_mask);
+        wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
 }
 
 void x86_pmu_enable_all(int added);
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index 0397b23be8e9..67250a52430b 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -1,4 +1,5 @@
 #include <linux/perf_event.h>
+#include <linux/export.h>
 #include <linux/types.h>
 #include <linux/init.h>
 #include <linux/slab.h>
@@ -357,7 +358,9 @@ static void amd_pmu_cpu_starting(int cpu)
         struct amd_nb *nb;
         int i, nb_id;
 
-        if (boot_cpu_data.x86_max_cores < 2)
+        cpuc->perf_ctr_virt_mask = AMD_PERFMON_EVENTSEL_HOSTONLY;
+
+        if (boot_cpu_data.x86_max_cores < 2 || boot_cpu_data.x86 == 0x15)
                 return;
 
         nb_id = amd_get_nb_id(cpu);
@@ -587,9 +590,9 @@ static __initconst const struct x86_pmu amd_pmu_f15h = {
         .put_event_constraints  = amd_put_event_constraints,
 
         .cpu_prepare            = amd_pmu_cpu_prepare,
-        .cpu_starting           = amd_pmu_cpu_starting,
         .cpu_dead               = amd_pmu_cpu_dead,
 #endif
+        .cpu_starting           = amd_pmu_cpu_starting,
 };
 
 __init int amd_pmu_init(void)
@@ -621,3 +624,33 @@ __init int amd_pmu_init(void)
 
         return 0;
 }
+
+void amd_pmu_enable_virt(void)
+{
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+        cpuc->perf_ctr_virt_mask = 0;
+
+        /* Reload all events */
+        x86_pmu_disable_all();
+        x86_pmu_enable_all(0);
+}
+EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
+
+void amd_pmu_disable_virt(void)
+{
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+        /*
+         * We only mask out the Host-only bit so that host-only counting works
+         * when SVM is disabled. If someone sets up a guest-only counter when
+         * SVM is disabled the Guest-only bits still gets set and the counter
+         * will not count anything.
+         */
+        cpuc->perf_ctr_virt_mask = AMD_PERFMON_EVENTSEL_HOSTONLY;
+
+        /* Reload all events */
+        x86_pmu_disable_all();
+        x86_pmu_enable_all(0);
+}
+EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 73da6b64f5b7..d6bd49faa40c 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -439,7 +439,6 @@ void intel_pmu_pebs_enable(struct perf_event *event)
         hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
 
         cpuc->pebs_enabled |= 1ULL << hwc->idx;
-        WARN_ON_ONCE(cpuc->enabled);
 
         if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
                 intel_pmu_lbr_enable(event);
diff --git a/arch/x86/kernel/cpu/perf_event_intel_lbr.c b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
index 3fab3de3ce96..47a7e63bfe54 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_lbr.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
@@ -72,8 +72,6 @@ void intel_pmu_lbr_enable(struct perf_event *event)
         if (!x86_pmu.lbr_nr)
                 return;
 
-        WARN_ON_ONCE(cpuc->enabled);
-
         /*
          * Reset the LBR stack if we changed task context to
          * avoid data leaks.
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 9e036f0ce5e0..2925e14fb1d9 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1576,10 +1576,17 @@ ENTRY(nmi)
         pushq_cfi %rdx
 
         /*
+         * If %cs was not the kernel segment, then the NMI triggered in user
+         * space, which means it is definitely not nested.
+         */
+        cmpl $__KERNEL_CS, 16(%rsp)
+        jne first_nmi
+
+        /*
          * Check the special variable on the stack to see if NMIs are
          * executing.
          */
-        cmp $1, -8(%rsp)
+        cmpl $1, -8(%rsp)
         je nested_nmi
 
         /*
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 313fb5cddbce..7b77062dea11 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -61,7 +61,7 @@ static irqreturn_t math_error_irq(int cpl, void *dev_id)
         outb(0, 0xF0);
         if (ignore_fpu_irq || !boot_cpu_data.hard_math)
                 return IRQ_NONE;
-        math_error(get_irq_regs(), 0, 16);
+        math_error(get_irq_regs(), 0, X86_TRAP_MF);
         return IRQ_HANDLED;
 }
 
diff --git a/arch/x86/kernel/microcode_amd.c b/arch/x86/kernel/microcode_amd.c
index ac0417be9131..73465aab28f8 100644
--- a/arch/x86/kernel/microcode_amd.c
+++ b/arch/x86/kernel/microcode_amd.c
@@ -360,7 +360,6 @@ out:
 static enum ucode_state
 request_microcode_user(int cpu, const void __user *buf, size_t size)
 {
-        pr_info("AMD microcode update via /dev/cpu/microcode not supported\n");
         return UCODE_ERROR;
 }
 
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 485204f58cda..c08d1ff12b7c 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -214,6 +214,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
 
         task_user_gs(p) = get_user_gs(regs);
 
+        p->fpu_counter = 0;
         p->thread.io_bitmap_ptr = NULL;
         tsk = current;
         err = -ENOMEM;
@@ -299,22 +300,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                                  *next = &next_p->thread;
         int cpu = smp_processor_id();
         struct tss_struct *tss = &per_cpu(init_tss, cpu);
-        bool preload_fpu;
+        fpu_switch_t fpu;
 
         /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
 
-        /*
-         * If the task has used fpu the last 5 timeslices, just do a full
-         * restore of the math state immediately to avoid the trap; the
-         * chances of needing FPU soon are obviously high now
-         */
-        preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
-
-        __unlazy_fpu(prev_p);
-
-        /* we're going to use this soon, after a few expensive things */
-        if (preload_fpu)
-                prefetch(next->fpu.state);
+        fpu = switch_fpu_prepare(prev_p, next_p, cpu);
 
         /*
          * Reload esp0.
@@ -354,11 +344,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                      task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
                 __switch_to_xtra(prev_p, next_p, tss);
 
-        /* If we're going to preload the fpu context, make sure clts
-           is run while we're batching the cpu state updates. */
-        if (preload_fpu)
-                clts();
-
         /*
          * Leave lazy mode, flushing any hypercalls made here.
          * This must be done before restoring TLS segments so
@@ -368,15 +353,14 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
          */
         arch_end_context_switch(next_p);
 
-        if (preload_fpu)
-                __math_state_restore();
-
         /*
          * Restore %gs if needed (which is common)
          */
         if (prev->gs | next->gs)
                 lazy_load_gs(next->gs);
 
+        switch_fpu_finish(next_p, fpu);
+
         percpu_write(current_task, next_p);
 
         return prev_p;
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index a4659739e202..550e77b1b948 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -286,6 +286,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
 
         set_tsk_thread_flag(p, TIF_FORK);
 
+        p->fpu_counter = 0;
         p->thread.io_bitmap_ptr = NULL;
 
         savesegment(gs, p->thread.gsindex);
@@ -388,18 +389,9 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         int cpu = smp_processor_id();
         struct tss_struct *tss = &per_cpu(init_tss, cpu);
         unsigned fsindex, gsindex;
-        bool preload_fpu;
+        fpu_switch_t fpu;
 
-        /*
-         * If the task has used fpu the last 5 timeslices, just do a full
-         * restore of the math state immediately to avoid the trap; the
-         * chances of needing FPU soon are obviously high now
-         */
-        preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
-
-        /* we're going to use this soon, after a few expensive things */
-        if (preload_fpu)
-                prefetch(next->fpu.state);
+        fpu = switch_fpu_prepare(prev_p, next_p, cpu);
 
         /*
          * Reload esp0, LDT and the page table pointer:
@@ -429,13 +421,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 
         load_TLS(next, cpu);
 
-        /* Must be after DS reload */
-        __unlazy_fpu(prev_p);
-
-        /* Make sure cpu is ready for new context */
-        if (preload_fpu)
-                clts();
-
         /*
          * Leave lazy mode, flushing any hypercalls made here.
          * This must be done before restoring TLS segments so
@@ -476,6 +461,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
         prev->gsindex = gsindex;
 
+        switch_fpu_finish(next_p, fpu);
+
         /*
          * Switch the PDA and FPU contexts.
          */
@@ -494,13 +481,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                      task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
                 __switch_to_xtra(prev_p, next_p, tss);
 
-        /*
-         * Preload the FPU context, now that we've determined that the
-         * task is likely to be using it. 
-         */
-        if (preload_fpu)
-                __math_state_restore();
-
         return prev_p;
 }
 
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 482ec3af2067..037fc2bc5316 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -119,7 +119,7 @@ do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
                  * traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
                  * On nmi (interrupt 2), do_trap should not be called.
                  */
-                if (trapnr < 6)
+                if (trapnr < X86_TRAP_UD)
                         goto vm86_trap;
                 goto trap_signal;
         }
@@ -203,27 +203,31 @@ dotraplinkage void do_##name(struct pt_regs *regs, long error_code)	\
         do_trap(trapnr, signr, str, regs, error_code, &info);           \
 }
 
-DO_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip)
-DO_ERROR(4, SIGSEGV, "overflow", overflow)
-DO_ERROR(5, SIGSEGV, "bounds", bounds)
-DO_ERROR_INFO(6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip)
-DO_ERROR(9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun)
-DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS)
-DO_ERROR(11, SIGBUS, "segment not present", segment_not_present)
+DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV,
+                regs->ip)
+DO_ERROR(X86_TRAP_OF, SIGSEGV, "overflow", overflow)
+DO_ERROR(X86_TRAP_BR, SIGSEGV, "bounds", bounds)
+DO_ERROR_INFO(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN,
+                regs->ip)
+DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, "coprocessor segment overrun",
+                coprocessor_segment_overrun)
+DO_ERROR(X86_TRAP_TS, SIGSEGV, "invalid TSS", invalid_TSS)
+DO_ERROR(X86_TRAP_NP, SIGBUS, "segment not present", segment_not_present)
 #ifdef CONFIG_X86_32
-DO_ERROR(12, SIGBUS, "stack segment", stack_segment)
+DO_ERROR(X86_TRAP_SS, SIGBUS, "stack segment", stack_segment)
 #endif
-DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0)
+DO_ERROR_INFO(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check,
+                BUS_ADRALN, 0)
 
 #ifdef CONFIG_X86_64
 /* Runs on IST stack */
 dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code)
 {
         if (notify_die(DIE_TRAP, "stack segment", regs, error_code,
-                        12, SIGBUS) == NOTIFY_STOP)
+                        X86_TRAP_SS, SIGBUS) == NOTIFY_STOP)
                 return;
         preempt_conditional_sti(regs);
-        do_trap(12, SIGBUS, "stack segment", regs, error_code, NULL);
+        do_trap(X86_TRAP_SS, SIGBUS, "stack segment", regs, error_code, NULL);
         preempt_conditional_cli(regs);
 }
 
@@ -233,10 +237,10 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
         struct task_struct *tsk = current;
 
         /* Return not checked because double check cannot be ignored */
-        notify_die(DIE_TRAP, str, regs, error_code, 8, SIGSEGV);
+        notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
 
         tsk->thread.error_code = error_code;
-        tsk->thread.trap_no = 8;
+        tsk->thread.trap_no = X86_TRAP_DF;
 
         /*
          * This is always a kernel trap and never fixable (and thus must
@@ -264,7 +268,7 @@ do_general_protection(struct pt_regs *regs, long error_code)
                 goto gp_in_kernel;
 
         tsk->thread.error_code = error_code;
-        tsk->thread.trap_no = 13;
+        tsk->thread.trap_no = X86_TRAP_GP;
 
         if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
                         printk_ratelimit()) {
@@ -291,9 +295,9 @@ gp_in_kernel:
                 return;
 
         tsk->thread.error_code = error_code;
-        tsk->thread.trap_no = 13;
-        if (notify_die(DIE_GPF, "general protection fault", regs,
-                                error_code, 13, SIGSEGV) == NOTIFY_STOP)
+        tsk->thread.trap_no = X86_TRAP_GP;
+        if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
+                        X86_TRAP_GP, SIGSEGV) == NOTIFY_STOP)
                 return;
         die("general protection fault", regs, error_code);
 }
@@ -302,13 +306,13 @@ gp_in_kernel:
 dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code)
 {
 #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
-        if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP)
-                        == NOTIFY_STOP)
+        if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
+                                SIGTRAP) == NOTIFY_STOP)
                 return;
 #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
 
-        if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP)
-                        == NOTIFY_STOP)
+        if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
+                        SIGTRAP) == NOTIFY_STOP)
                 return;
 
         /*
@@ -317,7 +321,7 @@ dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code)
          */
         debug_stack_usage_inc();
         preempt_conditional_sti(regs);
-        do_trap(3, SIGTRAP, "int3", regs, error_code, NULL);
+        do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
         preempt_conditional_cli(regs);
         debug_stack_usage_dec();
 }
@@ -422,8 +426,8 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
         preempt_conditional_sti(regs);
 
         if (regs->flags & X86_VM_MASK) {
-                handle_vm86_trap((struct kernel_vm86_regs *) regs,
-                                error_code, 1);
+                handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,
+                                        X86_TRAP_DB);
                 preempt_conditional_cli(regs);
                 debug_stack_usage_dec();
                 return;
@@ -460,7 +464,8 @@ void math_error(struct pt_regs *regs, int error_code, int trapnr)
         struct task_struct *task = current;
         siginfo_t info;
         unsigned short err;
-        char *str = (trapnr == 16) ? "fpu exception" : "simd exception";
+        char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" :
+                                                "simd exception";
 
         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, SIGFPE) == NOTIFY_STOP)
                 return;
@@ -485,7 +490,7 @@ void math_error(struct pt_regs *regs, int error_code, int trapnr)
         info.si_signo = SIGFPE;
         info.si_errno = 0;
         info.si_addr = (void __user *)regs->ip;
-        if (trapnr == 16) {
+        if (trapnr == X86_TRAP_MF) {
                 unsigned short cwd, swd;
                 /*
                  * (~cwd & swd) will mask out exceptions that are not set to unmasked
@@ -529,10 +534,11 @@ void math_error(struct pt_regs *regs, int error_code, int trapnr)
                 info.si_code = FPE_FLTRES;
         } else {
                 /*
-                 * If we're using IRQ 13, or supposedly even some trap 16
-                 * implementations, it's possible we get a spurious trap...
+                 * If we're using IRQ 13, or supposedly even some trap
+                 * X86_TRAP_MF implementations, it's possible
+                 * we get a spurious trap, which is not an error.
                  */
-                return;         /* Spurious trap, no error */
+                return;
         }
         force_sig_info(SIGFPE, &info, task);
 }
@@ -543,13 +549,13 @@ dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
         ignore_fpu_irq = 1;
 #endif
 
-        math_error(regs, error_code, 16);
+        math_error(regs, error_code, X86_TRAP_MF);
 }
 
 dotraplinkage void
 do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
 {
-        math_error(regs, error_code, 19);
+        math_error(regs, error_code, X86_TRAP_XF);
 }
 
 dotraplinkage void
@@ -571,41 +577,18 @@ asmlinkage void __attribute__((weak)) smp_threshold_interrupt(void)
 }
 
 /*
- * __math_state_restore assumes that cr0.TS is already clear and the
- * fpu state is all ready for use.  Used during context switch.
- */
-void __math_state_restore(void)
-{
-        struct thread_info *thread = current_thread_info();
-        struct task_struct *tsk = thread->task;
-
-        /*
-         * Paranoid restore. send a SIGSEGV if we fail to restore the state.
-         */
-        if (unlikely(restore_fpu_checking(tsk))) {
-                stts();
-                force_sig(SIGSEGV, tsk);
-                return;
-        }
-
-        thread->status |= TS_USEDFPU;   /* So we fnsave on switch_to() */
-        tsk->fpu_counter++;
-}
-
-/*
  * 'math_state_restore()' saves the current math information in the
  * old math state array, and gets the new ones from the current task
  *
  * Careful.. There are problems with IBM-designed IRQ13 behaviour.
  * Don't touch unless you *really* know how it works.
  *
- * Must be called with kernel preemption disabled (in this case,
- * local interrupts are disabled at the call-site in entry.S).
+ * Must be called with kernel preemption disabled (eg with local
+ * local interrupts as in the case of do_device_not_available).
  */
-asmlinkage void math_state_restore(void)
+void math_state_restore(void)
 {
-        struct thread_info *thread = current_thread_info();
-        struct task_struct *tsk = thread->task;
+        struct task_struct *tsk = current;
 
         if (!tsk_used_math(tsk)) {
                 local_irq_enable();
@@ -622,9 +605,17 @@ asmlinkage void math_state_restore(void)
                 local_irq_disable();
         }
 
-        clts();                         /* Allow maths ops (or we recurse) */
+        __thread_fpu_begin(tsk);
+        /*
+         * Paranoid restore. send a SIGSEGV if we fail to restore the state.
+         */
+        if (unlikely(restore_fpu_checking(tsk))) {
+                __thread_fpu_end(tsk);
+                force_sig(SIGSEGV, tsk);
+                return;
+        }
 
-        __math_state_restore();
+        tsk->fpu_counter++;
 }
 EXPORT_SYMBOL_GPL(math_state_restore);
 
@@ -658,20 +649,21 @@ dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
         info.si_errno = 0;
         info.si_code = ILL_BADSTK;
         info.si_addr = NULL;
-        if (notify_die(DIE_TRAP, "iret exception",
-                        regs, error_code, 32, SIGILL) == NOTIFY_STOP)
+        if (notify_die(DIE_TRAP, "iret exception", regs, error_code,
+                        X86_TRAP_IRET, SIGILL) == NOTIFY_STOP)
                 return;
-        do_trap(32, SIGILL, "iret exception", regs, error_code, &info);
+        do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,
+                &info);
 }
 #endif
 
 /* Set of traps needed for early debugging. */
 void __init early_trap_init(void)
 {
-        set_intr_gate_ist(1, &debug, DEBUG_STACK);
+        set_intr_gate_ist(X86_TRAP_DB, &debug, DEBUG_STACK);
         /* int3 can be called from all */
-        set_system_intr_gate_ist(3, &int3, DEBUG_STACK);
-        set_intr_gate(14, &page_fault);
+        set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
+        set_intr_gate(X86_TRAP_PF, &page_fault);
         load_idt(&idt_descr);
 }
 
@@ -687,30 +679,30 @@ void __init trap_init(void)
         early_iounmap(p, 4);
 #endif
 
-        set_intr_gate(0, &divide_error);
-        set_intr_gate_ist(2, &nmi, NMI_STACK);
+        set_intr_gate(X86_TRAP_DE, &divide_error);
+        set_intr_gate_ist(X86_TRAP_NMI, &nmi, NMI_STACK);
         /* int4 can be called from all */
-        set_system_intr_gate(4, &overflow);
-        set_intr_gate(5, &bounds);
-        set_intr_gate(6, &invalid_op);
-        set_intr_gate(7, &device_not_available);
+        set_system_intr_gate(X86_TRAP_OF, &overflow);
+        set_intr_gate(X86_TRAP_BR, &bounds);
+        set_intr_gate(X86_TRAP_UD, &invalid_op);
+        set_intr_gate(X86_TRAP_NM, &device_not_available);
 #ifdef CONFIG_X86_32
-        set_task_gate(8, GDT_ENTRY_DOUBLEFAULT_TSS);
+        set_task_gate(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS);
 #else
-        set_intr_gate_ist(8, &double_fault, DOUBLEFAULT_STACK);
+        set_intr_gate_ist(X86_TRAP_DF, &double_fault, DOUBLEFAULT_STACK);
 #endif
-        set_intr_gate(9, &coprocessor_segment_overrun);
-        set_intr_gate(10, &invalid_TSS);
-        set_intr_gate(11, &segment_not_present);
-        set_intr_gate_ist(12, &stack_segment, STACKFAULT_STACK);
-        set_intr_gate(13, &general_protection);
-        set_intr_gate(15, &spurious_interrupt_bug);
-        set_intr_gate(16, &coprocessor_error);
-        set_intr_gate(17, &alignment_check);
+        set_intr_gate(X86_TRAP_OLD_MF, &coprocessor_segment_overrun);
+        set_intr_gate(X86_TRAP_TS, &invalid_TSS);
+        set_intr_gate(X86_TRAP_NP, &segment_not_present);
+        set_intr_gate_ist(X86_TRAP_SS, &stack_segment, STACKFAULT_STACK);
+        set_intr_gate(X86_TRAP_GP, &general_protection);
+        set_intr_gate(X86_TRAP_SPURIOUS, &spurious_interrupt_bug);
+        set_intr_gate(X86_TRAP_MF, &coprocessor_error);
+        set_intr_gate(X86_TRAP_AC, &alignment_check);
 #ifdef CONFIG_X86_MCE
-        set_intr_gate_ist(18, &machine_check, MCE_STACK);
+        set_intr_gate_ist(X86_TRAP_MC, &machine_check, MCE_STACK);
 #endif
-        set_intr_gate(19, &simd_coprocessor_error);
+        set_intr_gate(X86_TRAP_XF, &simd_coprocessor_error);
 
         /* Reserve all the builtin and the syscall vector: */
         for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
@@ -735,7 +727,7 @@ void __init trap_init(void)
 
 #ifdef CONFIG_X86_64
         memcpy(&nmi_idt_table, &idt_table, IDT_ENTRIES * 16);
-        set_nmi_gate(1, &debug);
-        set_nmi_gate(3, &int3);
+        set_nmi_gate(X86_TRAP_DB, &debug);
+        set_nmi_gate(X86_TRAP_BP, &int3);
 #endif
 }
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index a3911343976b..711091114119 100644
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -47,7 +47,7 @@ void __sanitize_i387_state(struct task_struct *tsk)
         if (!fx)
                 return;
 
-        BUG_ON(task_thread_info(tsk)->status & TS_USEDFPU);
+        BUG_ON(__thread_has_fpu(tsk));
 
         xstate_bv = tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv;
 
@@ -168,7 +168,7 @@ int save_i387_xstate(void __user *buf)
         if (!used_math())
                 return 0;
 
-        if (task_thread_info(tsk)->status & TS_USEDFPU) {
+        if (user_has_fpu()) {
                 if (use_xsave())
                         err = xsave_user(buf);
                 else
@@ -176,8 +176,7 @@ int save_i387_xstate(void __user *buf)
 
                 if (err)
                         return err;
-                task_thread_info(tsk)->status &= ~TS_USEDFPU;
-                stts();
+                user_fpu_end();
         } else {
                 sanitize_i387_state(tsk);
                 if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave,
@@ -292,10 +291,7 @@ int restore_i387_xstate(void __user *buf)
                         return err;
         }
 
-        if (!(task_thread_info(current)->status & TS_USEDFPU)) {
-                clts();
-                task_thread_info(current)->status |= TS_USEDFPU;
-        }
+        user_fpu_begin();
         if (use_xsave())
                 err = restore_user_xstate(buf);
         else
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 5fa553babe56..e385214711cb 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -29,6 +29,7 @@
 #include <linux/ftrace_event.h>
 #include <linux/slab.h>
 
+#include <asm/perf_event.h>
 #include <asm/tlbflush.h>
 #include <asm/desc.h>
 #include <asm/kvm_para.h>
@@ -575,6 +576,8 @@ static void svm_hardware_disable(void *garbage)
                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 
         cpu_svm_disable();
+
+        amd_pmu_disable_virt();
 }
 
 static int svm_hardware_enable(void *garbage)
@@ -622,6 +625,8 @@ static int svm_hardware_enable(void *garbage)
 
         svm_init_erratum_383();
 
+        amd_pmu_enable_virt();
+
         return 0;
 }
 
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index d29216c462b3..3b4c8d8ad906 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1457,7 +1457,7 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)
 #ifdef CONFIG_X86_64
         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
 #endif
-        if (current_thread_info()->status & TS_USEDFPU)
+        if (__thread_has_fpu(current))
                 clts();
         load_gdt(&__get_cpu_var(host_gdt));
 }
diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c
index 492ade8c978e..d99346ea8fdb 100644
--- a/arch/x86/pci/xen.c
+++ b/arch/x86/pci/xen.c
@@ -374,7 +374,7 @@ int __init pci_xen_init(void)
 
 int __init pci_xen_hvm_init(void)
 {
-        if (!xen_feature(XENFEAT_hvm_pirqs))
+        if (!xen_have_vector_callback || !xen_feature(XENFEAT_hvm_pirqs))
                 return 0;
 
 #ifdef CONFIG_ACPI
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 12eb07bfb267..4172af8ceeb3 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1141,7 +1141,9 @@ asmlinkage void __init xen_start_kernel(void)
 
         /* Prevent unwanted bits from being set in PTEs. */
         __supported_pte_mask &= ~_PAGE_GLOBAL;
+#if 0
         if (!xen_initial_domain())
+#endif
                 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
 
         __supported_pte_mask |= _PAGE_IOMAP;
@@ -1204,10 +1206,6 @@ asmlinkage void __init xen_start_kernel(void)
 
         pgd = (pgd_t *)xen_start_info->pt_base;
 
-        if (!xen_initial_domain())
-                __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
-
-        __supported_pte_mask |= _PAGE_IOMAP;
         /* Don't do the full vcpu_info placement stuff until we have a
            possible map and a non-dummy shared_info. */
         per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 58a0e46c404d..95c1cf60c669 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -415,13 +415,13 @@ static pteval_t iomap_pte(pteval_t val)
 static pteval_t xen_pte_val(pte_t pte)
 {
         pteval_t pteval = pte.pte;
-
+#if 0
         /* If this is a WC pte, convert back from Xen WC to Linux WC */
         if ((pteval & (_PAGE_PAT | _PAGE_PCD | _PAGE_PWT)) == _PAGE_PAT) {
                 WARN_ON(!pat_enabled);
                 pteval = (pteval & ~_PAGE_PAT) | _PAGE_PWT;
         }
-
+#endif
         if (xen_initial_domain() && (pteval & _PAGE_IOMAP))
                 return pteval;
 
@@ -463,7 +463,7 @@ void xen_set_pat(u64 pat)
 static pte_t xen_make_pte(pteval_t pte)
 {
         phys_addr_t addr = (pte & PTE_PFN_MASK);
-
+#if 0
         /* If Linux is trying to set a WC pte, then map to the Xen WC.
          * If _PAGE_PAT is set, then it probably means it is really
          * _PAGE_PSE, so avoid fiddling with the PAT mapping and hope
@@ -476,7 +476,7 @@ static pte_t xen_make_pte(pteval_t pte)
                 if ((pte & (_PAGE_PCD | _PAGE_PWT)) == _PAGE_PWT)
                         pte = (pte & ~(_PAGE_PCD | _PAGE_PWT)) | _PAGE_PAT;
         }
-
+#endif
         /*
          * Unprivileged domains are allowed to do IOMAPpings for
          * PCI passthrough, but not map ISA space.  The ISA
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe9dfe4..501d4e0244ba 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -409,6 +409,13 @@ static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */
         play_dead_common();
         HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
         cpu_bringup();
+        /*
+         * Balance out the preempt calls - as we are running in cpu_idle
+         * loop which has been called at bootup from cpu_bringup_and_idle.
+         * The cpucpu_bringup_and_idle called cpu_bringup which made a
+         * preempt_disable() So this preempt_enable will balance it out.
+         */
+        preempt_enable();
 }
 
 #else /* !CONFIG_HOTPLUG_CPU */