Merge branch 'x86/xsave' into x86/core

9 files changed, 507 insertions, 139 deletions

diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index dac24c4390dd..c9be69fedb70 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -38,7 +38,7 @@ obj-y			+= tsc.o io_delay.o rtc.o
 
 obj-$(CONFIG_X86_TRAMPOLINE)    += trampoline.o
 obj-y                           += process.o
-obj-y                           += i387.o
+obj-y                           += i387.o xsave.o
 obj-y                           += ptrace.o
 obj-y                           += ds.o
 obj-$(CONFIG_X86_32)            += tls.o
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 0785b3c8d043..c63ec65f484c 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -739,9 +739,20 @@ void __cpuinit cpu_init(void)
         /*
          * Force FPU initialization:
          */
-        current_thread_info()->status = 0;
+        if (cpu_has_xsave)
+                current_thread_info()->status = TS_XSAVE;
+        else
+                current_thread_info()->status = 0;
         clear_used_math();
         mxcsr_feature_mask_init();
+
+        /*
+         * Boot processor to setup the FP and extended state context info.
+         */
+        if (!smp_processor_id())
+                init_thread_xstate();
+
+        xsave_init();
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index eb9ddd8efb82..45723f1fe198 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -21,9 +21,12 @@
 # include <asm/sigcontext32.h>
 # include <asm/user32.h>
 #else
-# define save_i387_ia32         save_i387
-# define restore_i387_ia32      restore_i387
+# define save_i387_xstate_ia32          save_i387_xstate
+# define restore_i387_xstate_ia32       restore_i387_xstate
 # define _fpstate_ia32          _fpstate
+# define _xstate_ia32           _xstate
+# define sig_xstate_ia32_size   sig_xstate_size
+# define fx_sw_reserved_ia32    fx_sw_reserved
 # define user_i387_ia32_struct  user_i387_struct
 # define user32_fxsr_struct     user_fxsr_struct
 #endif
@@ -36,6 +39,7 @@
 
 static unsigned int             mxcsr_feature_mask __read_mostly = 0xffffffffu;
 unsigned int xstate_size;
+unsigned int sig_xstate_ia32_size = sizeof(struct _fpstate_ia32);
 static struct i387_fxsave_struct fx_scratch __cpuinitdata;
 
 void __cpuinit mxcsr_feature_mask_init(void)
@@ -61,6 +65,11 @@ void __init init_thread_xstate(void)
                 return;
         }
 
+        if (cpu_has_xsave) {
+                xsave_cntxt_init();
+                return;
+        }
+
         if (cpu_has_fxsr)
                 xstate_size = sizeof(struct i387_fxsave_struct);
 #ifdef CONFIG_X86_32
@@ -83,9 +92,19 @@ void __cpuinit fpu_init(void)
 
         write_cr0(oldcr0 & ~(X86_CR0_TS|X86_CR0_EM)); /* clear TS and EM */
 
+        /*
+         * Boot processor to setup the FP and extended state context info.
+         */
+        if (!smp_processor_id())
+                init_thread_xstate();
+        xsave_init();
+
         mxcsr_feature_mask_init();
         /* clean state in init */
-        current_thread_info()->status = 0;
+        if (cpu_has_xsave)
+                current_thread_info()->status = TS_XSAVE;
+        else
+                current_thread_info()->status = 0;
         clear_used_math();
 }
 #endif  /* CONFIG_X86_64 */
@@ -195,6 +214,13 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
          */
         target->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask;
 
+        /*
+         * update the header bits in the xsave header, indicating the
+         * presence of FP and SSE state.
+         */
+        if (cpu_has_xsave)
+                target->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
+
         return ret;
 }
 
@@ -395,6 +421,12 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
         if (!ret)
                 convert_to_fxsr(target, &env);
 
+        /*
+         * update the header bit in the xsave header, indicating the
+         * presence of FP.
+         */
+        if (cpu_has_xsave)
+                target->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
         return ret;
 }
 
@@ -407,7 +439,6 @@ static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
         struct task_struct *tsk = current;
         struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave;
 
-        unlazy_fpu(tsk);
         fp->status = fp->swd;
         if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct)))
                 return -1;
@@ -421,8 +452,6 @@ static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
         struct user_i387_ia32_struct env;
         int err = 0;
 
-        unlazy_fpu(tsk);
-
         convert_from_fxsr(&env, tsk);
         if (__copy_to_user(buf, &env, sizeof(env)))
                 return -1;
@@ -432,16 +461,40 @@ static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
         if (err)
                 return -1;
 
-        if (__copy_to_user(&buf->_fxsr_env[0], fx,
-                           sizeof(struct i387_fxsave_struct)))
+        if (__copy_to_user(&buf->_fxsr_env[0], fx, xstate_size))
                 return -1;
         return 1;
 }
 
-int save_i387_ia32(struct _fpstate_ia32 __user *buf)
+static int save_i387_xsave(void __user *buf)
+{
+        struct _fpstate_ia32 __user *fx = buf;
+        int err = 0;
+
+        if (save_i387_fxsave(fx) < 0)
+                return -1;
+
+        err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved_ia32,
+                             sizeof(struct _fpx_sw_bytes));
+        err |= __put_user(FP_XSTATE_MAGIC2,
+                          (__u32 __user *) (buf + sig_xstate_ia32_size
+                                            - FP_XSTATE_MAGIC2_SIZE));
+        if (err)
+                return -1;
+
+        return 1;
+}
+
+int save_i387_xstate_ia32(void __user *buf)
 {
+        struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
+        struct task_struct *tsk = current;
+
         if (!used_math())
                 return 0;
+
+        if (!access_ok(VERIFY_WRITE, buf, sig_xstate_ia32_size))
+                return -EACCES;
         /*
          * This will cause a "finit" to be triggered by the next
          * attempted FPU operation by the 'current' process.
@@ -451,13 +504,17 @@ int save_i387_ia32(struct _fpstate_ia32 __user *buf)
         if (!HAVE_HWFP) {
                 return fpregs_soft_get(current, NULL,
                                        0, sizeof(struct user_i387_ia32_struct),
-                                       NULL, buf) ? -1 : 1;
+                                       NULL, fp) ? -1 : 1;
         }
 
+        unlazy_fpu(tsk);
+
+        if (cpu_has_xsave)
+                return save_i387_xsave(fp);
         if (cpu_has_fxsr)
-                return save_i387_fxsave(buf);
+                return save_i387_fxsave(fp);
         else
-                return save_i387_fsave(buf);
+                return save_i387_fsave(fp);
 }
 
 static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
@@ -468,14 +525,15 @@ static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
                                 sizeof(struct i387_fsave_struct));
 }
 
-static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf)
+static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf,
+                               unsigned int size)
 {
         struct task_struct *tsk = current;
         struct user_i387_ia32_struct env;
         int err;
 
         err = __copy_from_user(&tsk->thread.xstate->fxsave, &buf->_fxsr_env[0],
-                               sizeof(struct i387_fxsave_struct));
+                               size);
         /* mxcsr reserved bits must be masked to zero for security reasons */
         tsk->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask;
         if (err || __copy_from_user(&env, buf, sizeof(env)))
@@ -485,14 +543,69 @@ static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf)
         return 0;
 }
 
-int restore_i387_ia32(struct _fpstate_ia32 __user *buf)
+static int restore_i387_xsave(void __user *buf)
+{
+        struct _fpx_sw_bytes fx_sw_user;
+        struct _fpstate_ia32 __user *fx_user =
+                        ((struct _fpstate_ia32 __user *) buf);
+        struct i387_fxsave_struct __user *fx =
+                (struct i387_fxsave_struct __user *) &fx_user->_fxsr_env[0];
+        struct xsave_hdr_struct *xsave_hdr =
+                                &current->thread.xstate->xsave.xsave_hdr;
+        u64 mask;
+        int err;
+
+        if (check_for_xstate(fx, buf, &fx_sw_user))
+                goto fx_only;
+
+        mask = fx_sw_user.xstate_bv;
+
+        err = restore_i387_fxsave(buf, fx_sw_user.xstate_size);
+
+        xsave_hdr->xstate_bv &= pcntxt_mask;
+        /*
+         * These bits must be zero.
+         */
+        xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
+
+        /*
+         * Init the state that is not present in the memory layout
+         * and enabled by the OS.
+         */
+        mask = ~(pcntxt_mask & ~mask);
+        xsave_hdr->xstate_bv &= mask;
+
+        return err;
+fx_only:
+        /*
+         * Couldn't find the extended state information in the memory
+         * layout. Restore the FP/SSE and init the other extended state
+         * enabled by the OS.
+         */
+        xsave_hdr->xstate_bv = XSTATE_FPSSE;
+        return restore_i387_fxsave(buf, sizeof(struct i387_fxsave_struct));
+}
+
+int restore_i387_xstate_ia32(void __user *buf)
 {
         int err;
         struct task_struct *tsk = current;
+        struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
 
         if (HAVE_HWFP)
                 clear_fpu(tsk);
 
+        if (!buf) {
+                if (used_math()) {
+                        clear_fpu(tsk);
+                        clear_used_math();
+                }
+
+                return 0;
+        } else
+                if (!access_ok(VERIFY_READ, buf, sig_xstate_ia32_size))
+                        return -EACCES;
+
         if (!used_math()) {
                 err = init_fpu(tsk);
                 if (err)
@@ -500,14 +613,17 @@ int restore_i387_ia32(struct _fpstate_ia32 __user *buf)
         }
 
         if (HAVE_HWFP) {
-                if (cpu_has_fxsr)
-                        err = restore_i387_fxsave(buf);
+                if (cpu_has_xsave)
+                        err = restore_i387_xsave(buf);
+                else if (cpu_has_fxsr)
+                        err = restore_i387_fxsave(fp, sizeof(struct
+                                                           i387_fxsave_struct));
                 else
-                        err = restore_i387_fsave(buf);
+                        err = restore_i387_fsave(fp);
         } else {
                 err = fpregs_soft_set(current, NULL,
                                       0, sizeof(struct user_i387_ia32_struct),
-                                      NULL, buf) != 0;
+                                      NULL, fp) != 0;
         }
         set_used_math();
 
diff --git a/arch/x86/kernel/sigframe.h b/arch/x86/kernel/sigframe.h
index 72bbb519d2dc..6dd7e2b70a4b 100644
--- a/arch/x86/kernel/sigframe.h
+++ b/arch/x86/kernel/sigframe.h
@@ -3,9 +3,18 @@ struct sigframe {
         char __user *pretcode;
         int sig;
         struct sigcontext sc;
-        struct _fpstate fpstate;
+        /*
+         * fpstate is unused. fpstate is moved/allocated after
+         * retcode[] below. This movement allows to have the FP state and the
+         * future state extensions (xsave) stay together.
+         * And at the same time retaining the unused fpstate, prevents changing
+         * the offset of extramask[] in the sigframe and thus prevent any
+         * legacy application accessing/modifying it.
+         */
+        struct _fpstate fpstate_unused;
         unsigned long extramask[_NSIG_WORDS-1];
         char retcode[8];
+        /* fp state follows here */
 };
 
 struct rt_sigframe {
@@ -15,13 +24,14 @@ struct rt_sigframe {
         void __user *puc;
         struct siginfo info;
         struct ucontext uc;
-        struct _fpstate fpstate;
         char retcode[8];
+        /* fp state follows here */
 };
 #else
 struct rt_sigframe {
         char __user *pretcode;
         struct ucontext uc;
         struct siginfo info;
+        /* fp state follows here */
 };
 #endif
diff --git a/arch/x86/kernel/signal_32.c b/arch/x86/kernel/signal_32.c
index 0c727f64e79b..8d380b699c0c 100644
--- a/arch/x86/kernel/signal_32.c
+++ b/arch/x86/kernel/signal_32.c
@@ -160,28 +160,14 @@ restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
         }
 
         {
-                struct _fpstate __user *buf;
+                void __user *buf;
 
                 err |= __get_user(buf, &sc->fpstate);
-                if (buf) {
-                        if (!access_ok(VERIFY_READ, buf, sizeof(*buf)))
-                                goto badframe;
-                        err |= restore_i387(buf);
-                } else {
-                        struct task_struct *me = current;
-
-                        if (used_math()) {
-                                clear_fpu(me);
-                                clear_used_math();
-                        }
-                }
+                err |= restore_i387_xstate(buf);
         }
 
         err |= __get_user(*pax, &sc->ax);
         return err;
-
-badframe:
-        return 1;
 }
 
 asmlinkage unsigned long sys_sigreturn(unsigned long __unused)
@@ -263,7 +249,7 @@ badframe:
  * Set up a signal frame.
  */
 static int
-setup_sigcontext(struct sigcontext __user *sc, struct _fpstate __user *fpstate,
+setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate,
                  struct pt_regs *regs, unsigned long mask)
 {
         int tmp, err = 0;
@@ -290,7 +276,7 @@ setup_sigcontext(struct sigcontext __user *sc, struct _fpstate __user *fpstate,
         err |= __put_user(regs->sp, &sc->sp_at_signal);
         err |= __put_user(regs->ss, (unsigned int __user *)&sc->ss);
 
-        tmp = save_i387(fpstate);
+        tmp = save_i387_xstate(fpstate);
         if (tmp < 0)
                 err = 1;
         else
@@ -307,7 +293,8 @@ setup_sigcontext(struct sigcontext __user *sc, struct _fpstate __user *fpstate,
  * Determine which stack to use..
  */
 static inline void __user *
-get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size)
+get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
+             void **fpstate)
 {
         unsigned long sp;
 
@@ -333,6 +320,11 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size)
                         sp = (unsigned long) ka->sa.sa_restorer;
         }
 
+        if (used_math()) {
+                sp = sp - sig_xstate_size;
+                *fpstate = (struct _fpstate *) sp;
+        }
+
         sp -= frame_size;
         /*
          * Align the stack pointer according to the i386 ABI,
@@ -351,8 +343,9 @@ setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
         void __user *restorer;
         int err = 0;
         int usig;
+        void __user *fpstate = NULL;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 goto give_sigsegv;
@@ -367,7 +360,7 @@ setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
         if (err)
                 goto give_sigsegv;
 
-        err = setup_sigcontext(&frame->sc, &frame->fpstate, regs, set->sig[0]);
+        err = setup_sigcontext(&frame->sc, fpstate, regs, set->sig[0]);
         if (err)
                 goto give_sigsegv;
 
@@ -428,8 +421,9 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         void __user *restorer;
         int err = 0;
         int usig;
+        void __user *fpstate = NULL;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 goto give_sigsegv;
@@ -448,13 +442,16 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 goto give_sigsegv;
 
         /* Create the ucontext.  */
-        err |= __put_user(0, &frame->uc.uc_flags);
+        if (cpu_has_xsave)
+                err |= __put_user(UC_FP_XSTATE, &frame->uc.uc_flags);
+        else
+                err |= __put_user(0, &frame->uc.uc_flags);
         err |= __put_user(0, &frame->uc.uc_link);
         err |= __put_user(current->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
         err |= __put_user(sas_ss_flags(regs->sp),
                           &frame->uc.uc_stack.ss_flags);
         err |= __put_user(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
-        err |= setup_sigcontext(&frame->uc.uc_mcontext, &frame->fpstate,
+        err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
                                 regs, set->sig[0]);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
         if (err)
diff --git a/arch/x86/kernel/signal_64.c b/arch/x86/kernel/signal_64.c
index 2f1464050059..4665b598a376 100644
--- a/arch/x86/kernel/signal_64.c
+++ b/arch/x86/kernel/signal_64.c
@@ -55,69 +55,6 @@ sys_sigaltstack(const stack_t __user *uss, stack_t __user *uoss,
 }
 
 /*
- * Signal frame handlers.
- */
-
-static inline int save_i387(struct _fpstate __user *buf)
-{
-        struct task_struct *tsk = current;
-        int err = 0;
-
-        BUILD_BUG_ON(sizeof(struct user_i387_struct) !=
-                        sizeof(tsk->thread.xstate->fxsave));
-
-        if ((unsigned long)buf % 16)
-                printk("save_i387: bad fpstate %p\n", buf);
-
-        if (!used_math())
-                return 0;
-        clear_used_math(); /* trigger finit */
-        if (task_thread_info(tsk)->status & TS_USEDFPU) {
-                err = save_i387_checking((struct i387_fxsave_struct __user *)
-                                         buf);
-                if (err)
-                        return err;
-                task_thread_info(tsk)->status &= ~TS_USEDFPU;
-                stts();
-        } else {
-                if (__copy_to_user(buf, &tsk->thread.xstate->fxsave,
-                                   sizeof(struct i387_fxsave_struct)))
-                        return -1;
-        }
-        return 1;
-}
-
-/*
- * This restores directly out of user space. Exceptions are handled.
- */
-static inline int restore_i387(struct _fpstate __user *buf)
-{
-        struct task_struct *tsk = current;
-        int err;
-
-        if (!used_math()) {
-                err = init_fpu(tsk);
-                if (err)
-                        return err;
-        }
-
-        if (!(task_thread_info(current)->status & TS_USEDFPU)) {
-                clts();
-                task_thread_info(current)->status |= TS_USEDFPU;
-        }
-        err = restore_fpu_checking((__force struct i387_fxsave_struct *)buf);
-        if (unlikely(err)) {
-                /*
-                 * Encountered an error while doing the restore from the
-                 * user buffer, clear the fpu state.
-                 */
-                clear_fpu(tsk);
-                clear_used_math();
-        }
-        return err;
-}
-
-/*
  * Do a signal return; undo the signal stack.
  */
 static int
@@ -161,25 +98,11 @@ restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
         {
                 struct _fpstate __user * buf;
                 err |= __get_user(buf, &sc->fpstate);
-
-                if (buf) {
-                        if (!access_ok(VERIFY_READ, buf, sizeof(*buf)))
-                                goto badframe;
-                        err |= restore_i387(buf);
-                } else {
-                        struct task_struct *me = current;
-                        if (used_math()) {
-                                clear_fpu(me);
-                                clear_used_math();
-                        }
-                }
+                err |= restore_i387_xstate(buf);
         }
 
         err |= __get_user(*pax, &sc->ax);
         return err;
-
-badframe:
-        return 1;
 }
 
 asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
@@ -270,26 +193,23 @@ get_stack(struct k_sigaction *ka, struct pt_regs *regs, unsigned long size)
                         sp = current->sas_ss_sp + current->sas_ss_size;
         }
 
-        return (void __user *)round_down(sp - size, 16);
+        return (void __user *)round_down(sp - size, 64);
 }
 
 static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                            sigset_t *set, struct pt_regs * regs)
 {
         struct rt_sigframe __user *frame;
-        struct _fpstate __user *fp = NULL; 
+        void __user *fp = NULL;
         int err = 0;
         struct task_struct *me = current;
 
         if (used_math()) {
-                fp = get_stack(ka, regs, sizeof(struct _fpstate)); 
+                fp = get_stack(ka, regs, sig_xstate_size);
                 frame = (void __user *)round_down(
                         (unsigned long)fp - sizeof(struct rt_sigframe), 16) - 8;
 
-                if (!access_ok(VERIFY_WRITE, fp, sizeof(struct _fpstate)))
-                        goto give_sigsegv;
-
-                if (save_i387(fp) < 0) 
+                if (save_i387_xstate(fp) < 0)
                         err |= -1; 
         } else
                 frame = get_stack(ka, regs, sizeof(struct rt_sigframe)) - 8;
@@ -304,7 +224,10 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         }
                 
         /* Create the ucontext.  */
-        err |= __put_user(0, &frame->uc.uc_flags);
+        if (cpu_has_xsave)
+                err |= __put_user(UC_FP_XSTATE, &frame->uc.uc_flags);
+        else
+                err |= __put_user(0, &frame->uc.uc_flags);
         err |= __put_user(0, &frame->uc.uc_link);
         err |= __put_user(me->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
         err |= __put_user(sas_ss_flags(regs->sp),
diff --git a/arch/x86/kernel/traps_32.c b/arch/x86/kernel/traps_32.c
index 03df8e45e5a1..da5a5964fccb 100644
--- a/arch/x86/kernel/traps_32.c
+++ b/arch/x86/kernel/traps_32.c
@@ -1228,7 +1228,6 @@ void __init trap_init(void)
 
         set_bit(SYSCALL_VECTOR, used_vectors);
 
-        init_thread_xstate();
         /*
          * Should be a barrier for any external CPU state:
          */
diff --git a/arch/x86/kernel/traps_64.c b/arch/x86/kernel/traps_64.c
index 513caaca7115..38eb76156a47 100644
--- a/arch/x86/kernel/traps_64.c
+++ b/arch/x86/kernel/traps_64.c
@@ -1134,7 +1134,7 @@ asmlinkage void math_state_restore(void)
         /*
          * Paranoid restore. send a SIGSEGV if we fail to restore the state.
          */
-        if (unlikely(restore_fpu_checking(&me->thread.xstate->fxsave))) {
+        if (unlikely(restore_fpu_checking(me))) {
                 stts();
                 force_sig(SIGSEGV, me);
                 return;
@@ -1173,10 +1173,6 @@ void __init trap_init(void)
         set_system_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
 #endif
         /*
-         * initialize the per thread extended state:
-         */
-        init_thread_xstate();
-        /*
          * Should be a barrier for any external CPU state:
          */
         cpu_init();
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
new file mode 100644
index 000000000000..07713d64debe
--- /dev/null
+++ b/arch/x86/kernel/xsave.c
@@ -0,0 +1,316 @@
+/*
+ * xsave/xrstor support.
+ *
+ * Author: Suresh Siddha <suresh.b.siddha@intel.com>
+ */
+#include <linux/bootmem.h>
+#include <linux/compat.h>
+#include <asm/i387.h>
+#ifdef CONFIG_IA32_EMULATION
+#include <asm/sigcontext32.h>
+#endif
+#include <asm/xcr.h>
+
+/*
+ * Supported feature mask by the CPU and the kernel.
+ */
+u64 pcntxt_mask;
+
+struct _fpx_sw_bytes fx_sw_reserved;
+#ifdef CONFIG_IA32_EMULATION
+struct _fpx_sw_bytes fx_sw_reserved_ia32;
+#endif
+
+/*
+ * Check for the presence of extended state information in the
+ * user fpstate pointer in the sigcontext.
+ */
+int check_for_xstate(struct i387_fxsave_struct __user *buf,
+                     void __user *fpstate,
+                     struct _fpx_sw_bytes *fx_sw_user)
+{
+        int min_xstate_size = sizeof(struct i387_fxsave_struct) +
+                              sizeof(struct xsave_hdr_struct);
+        unsigned int magic2;
+        int err;
+
+        err = __copy_from_user(fx_sw_user, &buf->sw_reserved[0],
+                               sizeof(struct _fpx_sw_bytes));
+
+        if (err)
+                return err;
+
+        /*
+         * First Magic check failed.
+         */
+        if (fx_sw_user->magic1 != FP_XSTATE_MAGIC1)
+                return -1;
+
+        /*
+         * Check for error scenarios.
+         */
+        if (fx_sw_user->xstate_size < min_xstate_size ||
+            fx_sw_user->xstate_size > xstate_size ||
+            fx_sw_user->xstate_size > fx_sw_user->extended_size)
+                return -1;
+
+        err = __get_user(magic2, (__u32 *) (((void *)fpstate) +
+                                            fx_sw_user->extended_size -
+                                            FP_XSTATE_MAGIC2_SIZE));
+        /*
+         * Check for the presence of second magic word at the end of memory
+         * layout. This detects the case where the user just copied the legacy
+         * fpstate layout with out copying the extended state information
+         * in the memory layout.
+         */
+        if (err || magic2 != FP_XSTATE_MAGIC2)
+                return -1;
+
+        return 0;
+}
+
+#ifdef CONFIG_X86_64
+/*
+ * Signal frame handlers.
+ */
+
+int save_i387_xstate(void __user *buf)
+{
+        struct task_struct *tsk = current;
+        int err = 0;
+
+        if (!access_ok(VERIFY_WRITE, buf, sig_xstate_size))
+                return -EACCES;
+
+        BUG_ON(sig_xstate_size < xstate_size);
+
+        if ((unsigned long)buf % 64)
+                printk("save_i387_xstate: bad fpstate %p\n", buf);
+
+        if (!used_math())
+                return 0;
+        clear_used_math(); /* trigger finit */
+        if (task_thread_info(tsk)->status & TS_USEDFPU) {
+                /*
+                 * Start with clearing the user buffer. This will present a
+                 * clean context for the bytes not touched by the fxsave/xsave.
+                 */
+                __clear_user(buf, sig_xstate_size);
+
+                if (task_thread_info(tsk)->status & TS_XSAVE)
+                        err = xsave_user(buf);
+                else
+                        err = fxsave_user(buf);
+
+                if (err)
+                        return err;
+                task_thread_info(tsk)->status &= ~TS_USEDFPU;
+                stts();
+        } else {
+                if (__copy_to_user(buf, &tsk->thread.xstate->fxsave,
+                                   xstate_size))
+                        return -1;
+        }
+
+        if (task_thread_info(tsk)->status & TS_XSAVE) {
+                struct _fpstate __user *fx = buf;
+
+                err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved,
+                                     sizeof(struct _fpx_sw_bytes));
+
+                err |= __put_user(FP_XSTATE_MAGIC2,
+                                  (__u32 __user *) (buf + sig_xstate_size
+                                                    - FP_XSTATE_MAGIC2_SIZE));
+        }
+
+        return 1;
+}
+
+/*
+ * Restore the extended state if present. Otherwise, restore the FP/SSE
+ * state.
+ */
+int restore_user_xstate(void __user *buf)
+{
+        struct _fpx_sw_bytes fx_sw_user;
+        u64 mask;
+        int err;
+
+        if (((unsigned long)buf % 64) ||
+             check_for_xstate(buf, buf, &fx_sw_user))
+                goto fx_only;
+
+        mask = fx_sw_user.xstate_bv;
+
+        /*
+         * restore the state passed by the user.
+         */
+        err = xrestore_user(buf, mask);
+        if (err)
+                return err;
+
+        /*
+         * init the state skipped by the user.
+         */
+        mask = pcntxt_mask & ~mask;
+
+        xrstor_state(init_xstate_buf, mask);
+
+        return 0;
+
+fx_only:
+        /*
+         * couldn't find the extended state information in the
+         * memory layout. Restore just the FP/SSE and init all
+         * the other extended state.
+         */
+        xrstor_state(init_xstate_buf, pcntxt_mask & ~XSTATE_FPSSE);
+        return fxrstor_checking((__force struct i387_fxsave_struct *)buf);
+}
+
+/*
+ * This restores directly out of user space. Exceptions are handled.
+ */
+int restore_i387_xstate(void __user *buf)
+{
+        struct task_struct *tsk = current;
+        int err = 0;
+
+        if (!buf) {
+                if (used_math())
+                        goto clear;
+                return 0;
+        } else
+                if (!access_ok(VERIFY_READ, buf, sig_xstate_size))
+                        return -EACCES;
+
+        if (!used_math()) {
+                err = init_fpu(tsk);
+                if (err)
+                        return err;
+        }
+
+        if (!(task_thread_info(current)->status & TS_USEDFPU)) {
+                clts();
+                task_thread_info(current)->status |= TS_USEDFPU;
+        }
+        if (task_thread_info(tsk)->status & TS_XSAVE)
+                err = restore_user_xstate(buf);
+        else
+                err = fxrstor_checking((__force struct i387_fxsave_struct *)
+                                       buf);
+        if (unlikely(err)) {
+                /*
+                 * Encountered an error while doing the restore from the
+                 * user buffer, clear the fpu state.
+                 */
+clear:
+                clear_fpu(tsk);
+                clear_used_math();
+        }
+        return err;
+}
+#endif
+
+/*
+ * Prepare the SW reserved portion of the fxsave memory layout, indicating
+ * the presence of the extended state information in the memory layout
+ * pointed by the fpstate pointer in the sigcontext.
+ * This will be saved when ever the FP and extended state context is
+ * saved on the user stack during the signal handler delivery to the user.
+ */
+void prepare_fx_sw_frame(void)
+{
+        int size_extended = (xstate_size - sizeof(struct i387_fxsave_struct)) +
+                             FP_XSTATE_MAGIC2_SIZE;
+
+        sig_xstate_size = sizeof(struct _fpstate) + size_extended;
+
+#ifdef CONFIG_IA32_EMULATION
+        sig_xstate_ia32_size = sizeof(struct _fpstate_ia32) + size_extended;
+#endif
+
+        memset(&fx_sw_reserved, 0, sizeof(fx_sw_reserved));
+
+        fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
+        fx_sw_reserved.extended_size = sig_xstate_size;
+        fx_sw_reserved.xstate_bv = pcntxt_mask;
+        fx_sw_reserved.xstate_size = xstate_size;
+#ifdef CONFIG_IA32_EMULATION
+        memcpy(&fx_sw_reserved_ia32, &fx_sw_reserved,
+               sizeof(struct _fpx_sw_bytes));
+        fx_sw_reserved_ia32.extended_size = sig_xstate_ia32_size;
+#endif
+}
+
+/*
+ * Represents init state for the supported extended state.
+ */
+struct xsave_struct *init_xstate_buf;
+
+#ifdef CONFIG_X86_64
+unsigned int sig_xstate_size = sizeof(struct _fpstate);
+#endif
+
+/*
+ * Enable the extended processor state save/restore feature
+ */
+void __cpuinit xsave_init(void)
+{
+        if (!cpu_has_xsave)
+                return;
+
+        set_in_cr4(X86_CR4_OSXSAVE);
+
+        /*
+         * Enable all the features that the HW is capable of
+         * and the Linux kernel is aware of.
+         */
+        xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
+}
+
+/*
+ * setup the xstate image representing the init state
+ */
+void setup_xstate_init(void)
+{
+        init_xstate_buf = alloc_bootmem(xstate_size);
+        init_xstate_buf->i387.mxcsr = MXCSR_DEFAULT;
+}
+
+/*
+ * Enable and initialize the xsave feature.
+ */
+void __init xsave_cntxt_init(void)
+{
+        unsigned int eax, ebx, ecx, edx;
+
+        cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx);
+        pcntxt_mask = eax + ((u64)edx << 32);
+
+        if ((pcntxt_mask & XSTATE_FPSSE) != XSTATE_FPSSE) {
+                printk(KERN_ERR "FP/SSE not shown under xsave features 0x%llx\n",
+                       pcntxt_mask);
+                BUG();
+        }
+
+        /*
+         * for now OS knows only about FP/SSE
+         */
+        pcntxt_mask = pcntxt_mask & XCNTXT_MASK;
+        xsave_init();
+
+        /*
+         * Recompute the context size for enabled features
+         */
+        cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx);
+        xstate_size = ebx;
+
+        prepare_fx_sw_frame();
+
+        setup_xstate_init();
+
+        printk(KERN_INFO "xsave/xrstor: enabled xstate_bv 0x%llx, "
+               "cntxt size 0x%x\n",
+               pcntxt_mask, xstate_size);
+}