3 files changed, 139 insertions, 40 deletions

diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 01b115d86770..247904945d3f 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -32,6 +32,8 @@ extern int init_fpu(struct task_struct *child);
 extern void math_state_restore(void);
 extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
 
+DECLARE_PER_CPU(struct task_struct *, fpu_owner_task);
+
 extern user_regset_active_fn fpregs_active, xfpregs_active;
 extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
                                 xstateregs_get;
@@ -212,9 +214,10 @@ static inline void fpu_fxsave(struct fpu *fpu)
 #endif  /* CONFIG_X86_64 */
 
 /*
- * These must be called with preempt disabled
+ * These must be called with preempt disabled. Returns
+ * 'true' if the FPU state is still intact.
  */
-static inline void fpu_save_init(struct fpu *fpu)
+static inline int fpu_save_init(struct fpu *fpu)
 {
         if (use_xsave()) {
                 fpu_xsave(fpu);
@@ -223,22 +226,33 @@ static inline void fpu_save_init(struct fpu *fpu)
                  * xsave header may indicate the init state of the FP.
                  */
                 if (!(fpu->state->xsave.xsave_hdr.xstate_bv & XSTATE_FP))
-                        return;
+                        return 1;
         } else if (use_fxsr()) {
                 fpu_fxsave(fpu);
         } else {
                 asm volatile("fnsave %[fx]; fwait"
                              : [fx] "=m" (fpu->state->fsave));
-                return;
+                return 0;
         }
 
-        if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES))
+        /*
+         * If exceptions are pending, we need to clear them so
+         * that we don't randomly get exceptions later.
+         *
+         * FIXME! Is this perhaps only true for the old-style
+         * irq13 case? Maybe we could leave the x87 state
+         * intact otherwise?
+         */
+        if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES)) {
                 asm volatile("fnclex");
+                return 0;
+        }
+        return 1;
 }
 
-static inline void __save_init_fpu(struct task_struct *tsk)
+static inline int __save_init_fpu(struct task_struct *tsk)
 {
-        fpu_save_init(&tsk->thread.fpu);
+        return fpu_save_init(&tsk->thread.fpu);
 }
 
 static inline int fpu_fxrstor_checking(struct fpu *fpu)
@@ -256,6 +270,16 @@ static inline int fpu_restore_checking(struct fpu *fpu)
 
 static inline int restore_fpu_checking(struct task_struct *tsk)
 {
+        /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
+           is pending.  Clear the x87 state here by setting it to fixed
+           values. "m" is a random variable that should be in L1 */
+        alternative_input(
+                ASM_NOP8 ASM_NOP2,
+                "emms\n\t"              /* clear stack tags */
+                "fildl %P[addr]",       /* set F?P to defined value */
+                X86_FEATURE_FXSAVE_LEAK,
+                [addr] "m" (tsk->thread.fpu.has_fpu));
+
         return fpu_restore_checking(&tsk->thread.fpu);
 }
 
@@ -264,21 +288,23 @@ static inline int restore_fpu_checking(struct task_struct *tsk)
  * be preemption protection *and* they need to be
  * properly paired with the CR0.TS changes!
  */
-static inline int __thread_has_fpu(struct thread_info *ti)
+static inline int __thread_has_fpu(struct task_struct *tsk)
 {
-        return ti->status & TS_USEDFPU;
+        return tsk->thread.fpu.has_fpu;
 }
 
 /* Must be paired with an 'stts' after! */
-static inline void __thread_clear_has_fpu(struct thread_info *ti)
+static inline void __thread_clear_has_fpu(struct task_struct *tsk)
 {
-        ti->status &= ~TS_USEDFPU;
+        tsk->thread.fpu.has_fpu = 0;
+        percpu_write(fpu_owner_task, NULL);
 }
 
 /* Must be paired with a 'clts' before! */
-static inline void __thread_set_has_fpu(struct thread_info *ti)
+static inline void __thread_set_has_fpu(struct task_struct *tsk)
 {
-        ti->status |= TS_USEDFPU;
+        tsk->thread.fpu.has_fpu = 1;
+        percpu_write(fpu_owner_task, tsk);
 }
 
 /*
@@ -288,41 +314,108 @@ static inline void __thread_set_has_fpu(struct thread_info *ti)
  * These generally need preemption protection to work,
  * do try to avoid using these on their own.
  */
-static inline void __thread_fpu_end(struct thread_info *ti)
+static inline void __thread_fpu_end(struct task_struct *tsk)
 {
-        __thread_clear_has_fpu(ti);
+        __thread_clear_has_fpu(tsk);
         stts();
 }
 
-static inline void __thread_fpu_begin(struct thread_info *ti)
+static inline void __thread_fpu_begin(struct task_struct *tsk)
 {
         clts();
-        __thread_set_has_fpu(ti);
+        __thread_set_has_fpu(tsk);
 }
 
 /*
- * Signal frame handlers...
+ * FPU state switching for scheduling.
+ *
+ * This is a two-stage process:
+ *
+ *  - switch_fpu_prepare() saves the old state and
+ *    sets the new state of the CR0.TS bit. This is
+ *    done within the context of the old process.
+ *
+ *  - switch_fpu_finish() restores the new state as
+ *    necessary.
  */
-extern int save_i387_xstate(void __user *buf);
-extern int restore_i387_xstate(void __user *buf);
+typedef struct { int preload; } fpu_switch_t;
 
-static inline void __unlazy_fpu(struct task_struct *tsk)
+/*
+ * FIXME! We could do a totally lazy restore, but we need to
+ * add a per-cpu "this was the task that last touched the FPU
+ * on this CPU" variable, and the task needs to have a "I last
+ * touched the FPU on this CPU" and check them.
+ *
+ * We don't do that yet, so "fpu_lazy_restore()" always returns
+ * false, but some day..
+ */
+static inline int fpu_lazy_restore(struct task_struct *new, unsigned int cpu)
 {
-        if (__thread_has_fpu(task_thread_info(tsk))) {
-                __save_init_fpu(tsk);
-                __thread_fpu_end(task_thread_info(tsk));
-        } else
-                tsk->fpu_counter = 0;
+        return new == percpu_read_stable(fpu_owner_task) &&
+                cpu == new->thread.fpu.last_cpu;
+}
+
+static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct task_struct *new, int cpu)
+{
+        fpu_switch_t fpu;
+
+        fpu.preload = tsk_used_math(new) && new->fpu_counter > 5;
+        if (__thread_has_fpu(old)) {
+                if (!__save_init_fpu(old))
+                        cpu = ~0;
+                old->thread.fpu.last_cpu = cpu;
+                old->thread.fpu.has_fpu = 0;    /* But leave fpu_owner_task! */
+
+                /* Don't change CR0.TS if we just switch! */
+                if (fpu.preload) {
+                        new->fpu_counter++;
+                        __thread_set_has_fpu(new);
+                        prefetch(new->thread.fpu.state);
+                } else
+                        stts();
+        } else {
+                old->fpu_counter = 0;
+                old->thread.fpu.last_cpu = ~0;
+                if (fpu.preload) {
+                        new->fpu_counter++;
+                        if (fpu_lazy_restore(new, cpu))
+                                fpu.preload = 0;
+                        else
+                                prefetch(new->thread.fpu.state);
+                        __thread_fpu_begin(new);
+                }
+        }
+        return fpu;
+}
+
+/*
+ * By the time this gets called, we've already cleared CR0.TS and
+ * given the process the FPU if we are going to preload the FPU
+ * state - all we need to do is to conditionally restore the register
+ * state itself.
+ */
+static inline void switch_fpu_finish(struct task_struct *new, fpu_switch_t fpu)
+{
+        if (fpu.preload) {
+                if (unlikely(restore_fpu_checking(new)))
+                        __thread_fpu_end(new);
+        }
 }
 
+/*
+ * Signal frame handlers...
+ */
+extern int save_i387_xstate(void __user *buf);
+extern int restore_i387_xstate(void __user *buf);
+
 static inline void __clear_fpu(struct task_struct *tsk)
 {
-        if (__thread_has_fpu(task_thread_info(tsk))) {
+        if (__thread_has_fpu(tsk)) {
                 /* Ignore delayed exceptions from user space */
                 asm volatile("1: fwait\n"
                              "2:\n"
                              _ASM_EXTABLE(1b, 2b));
-                __thread_fpu_end(task_thread_info(tsk));
+                __thread_fpu_end(tsk);
         }
 }
 
@@ -337,7 +430,7 @@ static inline void __clear_fpu(struct task_struct *tsk)
  */
 static inline bool interrupted_kernel_fpu_idle(void)
 {
-        return !__thread_has_fpu(current_thread_info()) &&
+        return !__thread_has_fpu(current) &&
                 (read_cr0() & X86_CR0_TS);
 }
 
@@ -371,16 +464,18 @@ static inline bool irq_fpu_usable(void)
 
 static inline void kernel_fpu_begin(void)
 {
-        struct thread_info *me = current_thread_info();
+        struct task_struct *me = current;
 
         WARN_ON_ONCE(!irq_fpu_usable());
         preempt_disable();
         if (__thread_has_fpu(me)) {
-                __save_init_fpu(me->task);
+                __save_init_fpu(me);
                 __thread_clear_has_fpu(me);
                 /* We do 'stts()' in kernel_fpu_end() */
-        } else
+        } else {
+                percpu_write(fpu_owner_task, NULL);
                 clts();
+        }
 }
 
 static inline void kernel_fpu_end(void)
@@ -441,13 +536,13 @@ static inline void irq_ts_restore(int TS_state)
  */
 static inline int user_has_fpu(void)
 {
-        return __thread_has_fpu(current_thread_info());
+        return __thread_has_fpu(current);
 }
 
 static inline void user_fpu_end(void)
 {
         preempt_disable();
-        __thread_fpu_end(current_thread_info());
+        __thread_fpu_end(current);
         preempt_enable();
 }
 
@@ -455,7 +550,7 @@ static inline void user_fpu_begin(void)
 {
         preempt_disable();
         if (!user_has_fpu())
-                __thread_fpu_begin(current_thread_info());
+                __thread_fpu_begin(current);
         preempt_enable();
 }
 
@@ -464,17 +559,21 @@ static inline void user_fpu_begin(void)
  */
 static inline void save_init_fpu(struct task_struct *tsk)
 {
-        WARN_ON_ONCE(!__thread_has_fpu(task_thread_info(tsk)));
+        WARN_ON_ONCE(!__thread_has_fpu(tsk));
         preempt_disable();
         __save_init_fpu(tsk);
-        __thread_fpu_end(task_thread_info(tsk));
+        __thread_fpu_end(tsk);
         preempt_enable();
 }
 
 static inline void unlazy_fpu(struct task_struct *tsk)
 {
         preempt_disable();
-        __unlazy_fpu(tsk);
+        if (__thread_has_fpu(tsk)) {
+                __save_init_fpu(tsk);
+                __thread_fpu_end(tsk);
+        } else
+                tsk->fpu_counter = 0;
         preempt_enable();
 }
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index aa9088c26931..58545c97d071 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -374,6 +374,8 @@ union thread_xstate {
 };
 
 struct fpu {
+        unsigned int last_cpu;
+        unsigned int has_fpu;
         union thread_xstate *state;
 };
 
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index bc817cd8b443..cfd8144d5527 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -247,8 +247,6 @@ static inline struct thread_info *current_thread_info(void)
  * ever touches our thread-synchronous status, so we don't
  * have to worry about atomic accesses.
  */
-#define TS_USEDFPU              0x0001  /* FPU was used by this task
-                                           this quantum (SMP) */
 #define TS_COMPAT               0x0002  /* 32bit syscall active (64BIT)*/
 #define TS_POLLING              0x0004  /* idle task polling need_resched,
                                            skip sending interrupt */