aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorH. Peter Anvin <hpa@linux.intel.com>2012-09-21 20:18:44 -0400
committerH. Peter Anvin <hpa@linux.intel.com>2012-09-21 20:18:44 -0400
commit49b8c695e331c9685e6ffdbf34872509d77c8459 (patch)
treed4afdfae0115b2ab56687d23d6329d6ad934788f /arch
parente59d1b0a24199db01978e6c1e89859eda93ce683 (diff)
parentb1a74bf8212367be2b1d6685c11a84e056eaaaf1 (diff)
Merge branch 'x86/fpu' into x86/smap
Reason for merge: x86/fpu changed the structure of some of the code that x86/smap changes; mostly fpu-internal.h but also minor changes to the signal code. Signed-off-by: H. Peter Anvin <hpa@linux.intel.com> Resolved Conflicts: arch/x86/ia32/ia32_signal.c arch/x86/include/asm/fpu-internal.h arch/x86/kernel/signal.c
Diffstat (limited to 'arch')
-rw-r--r--arch/x86/ia32/ia32_signal.c21
-rw-r--r--arch/x86/ia32/sys_ia32.c2
-rw-r--r--arch/x86/include/asm/cpufeature.h3
-rw-r--r--arch/x86/include/asm/fpu-internal.h411
-rw-r--r--arch/x86/include/asm/i387.h29
-rw-r--r--arch/x86/include/asm/iommu_table.h6
-rw-r--r--arch/x86/include/asm/signal.h4
-rw-r--r--arch/x86/include/asm/sys_ia32.h2
-rw-r--r--arch/x86/include/asm/vdso.h3
-rw-r--r--arch/x86/include/asm/xor_32.h56
-rw-r--r--arch/x86/include/asm/xor_64.h61
-rw-r--r--arch/x86/include/asm/xor_avx.h54
-rw-r--r--arch/x86/include/asm/xsave.h13
-rw-r--r--arch/x86/kernel/cpu/bugs.c7
-rw-r--r--arch/x86/kernel/cpu/common.c2
-rw-r--r--arch/x86/kernel/i387.c292
-rw-r--r--arch/x86/kernel/probe_roms.c2
-rw-r--r--arch/x86/kernel/process.c22
-rw-r--r--arch/x86/kernel/process_32.c4
-rw-r--r--arch/x86/kernel/process_64.c4
-rw-r--r--arch/x86/kernel/ptrace.c3
-rw-r--r--arch/x86/kernel/signal.c215
-rw-r--r--arch/x86/kernel/traps.c5
-rw-r--r--arch/x86/kernel/xsave.c517
-rw-r--r--arch/x86/kvm/vmx.c10
-rw-r--r--arch/x86/kvm/x86.c3
26 files changed, 813 insertions, 938 deletions
diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c
index 05e62a312bd9..efc6a958b71d 100644
--- a/arch/x86/ia32/ia32_signal.c
+++ b/arch/x86/ia32/ia32_signal.c
@@ -32,6 +32,7 @@
32#include <asm/sigframe.h> 32#include <asm/sigframe.h>
33#include <asm/sighandling.h> 33#include <asm/sighandling.h>
34#include <asm/sys_ia32.h> 34#include <asm/sys_ia32.h>
35#include <asm/smap.h>
35 36
36#define FIX_EFLAGS __FIX_EFLAGS 37#define FIX_EFLAGS __FIX_EFLAGS
37 38
@@ -162,7 +163,8 @@ asmlinkage long sys32_sigaltstack(const stack_ia32_t __user *uss_ptr,
162 } 163 }
163 seg = get_fs(); 164 seg = get_fs();
164 set_fs(KERNEL_DS); 165 set_fs(KERNEL_DS);
165 ret = do_sigaltstack(uss_ptr ? &uss : NULL, &uoss, regs->sp); 166 ret = do_sigaltstack((stack_t __force __user *) (uss_ptr ? &uss : NULL),
167 (stack_t __force __user *) &uoss, regs->sp);
166 set_fs(seg); 168 set_fs(seg);
167 if (ret >= 0 && uoss_ptr) { 169 if (ret >= 0 && uoss_ptr) {
168 if (!access_ok(VERIFY_WRITE, uoss_ptr, sizeof(stack_ia32_t))) 170 if (!access_ok(VERIFY_WRITE, uoss_ptr, sizeof(stack_ia32_t)))
@@ -254,7 +256,7 @@ static int ia32_restore_sigcontext(struct pt_regs *regs,
254 get_user_ex(*pax, &sc->ax); 256 get_user_ex(*pax, &sc->ax);
255 } get_user_catch(err); 257 } get_user_catch(err);
256 258
257 err |= restore_i387_xstate_ia32(buf); 259 err |= restore_xstate_sig(buf, 1);
258 260
259 return err; 261 return err;
260} 262}
@@ -362,7 +364,7 @@ static int ia32_setup_sigcontext(struct sigcontext_ia32 __user *sc,
362 */ 364 */
363static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, 365static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
364 size_t frame_size, 366 size_t frame_size,
365 void **fpstate) 367 void __user **fpstate)
366{ 368{
367 unsigned long sp; 369 unsigned long sp;
368 370
@@ -382,9 +384,12 @@ static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
382 sp = (unsigned long) ka->sa.sa_restorer; 384 sp = (unsigned long) ka->sa.sa_restorer;
383 385
384 if (used_math()) { 386 if (used_math()) {
385 sp = sp - sig_xstate_ia32_size; 387 unsigned long fx_aligned, math_size;
386 *fpstate = (struct _fpstate_ia32 *) sp; 388
387 if (save_i387_xstate_ia32(*fpstate) < 0) 389 sp = alloc_mathframe(sp, 1, &fx_aligned, &math_size);
390 *fpstate = (struct _fpstate_ia32 __user *) sp;
391 if (save_xstate_sig(*fpstate, (void __user *)fx_aligned,
392 math_size) < 0)
388 return (void __user *) -1L; 393 return (void __user *) -1L;
389 } 394 }
390 395
@@ -449,7 +454,7 @@ int ia32_setup_frame(int sig, struct k_sigaction *ka,
449 * These are actually not used anymore, but left because some 454 * These are actually not used anymore, but left because some
450 * gdb versions depend on them as a marker. 455 * gdb versions depend on them as a marker.
451 */ 456 */
452 put_user_ex(*((u64 *)&code), (u64 *)frame->retcode); 457 put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode);
453 } put_user_catch(err); 458 } put_user_catch(err);
454 459
455 if (err) 460 if (err)
@@ -526,7 +531,7 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
526 * Not actually used anymore, but left because some gdb 531 * Not actually used anymore, but left because some gdb
527 * versions need it. 532 * versions need it.
528 */ 533 */
529 put_user_ex(*((u64 *)&code), (u64 *)frame->retcode); 534 put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode);
530 } put_user_catch(err); 535 } put_user_catch(err);
531 536
532 err |= copy_siginfo_to_user32(&frame->info, info); 537 err |= copy_siginfo_to_user32(&frame->info, info);
diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/ia32/sys_ia32.c
index 4540bece0946..c5b938d92eab 100644
--- a/arch/x86/ia32/sys_ia32.c
+++ b/arch/x86/ia32/sys_ia32.c
@@ -287,7 +287,7 @@ asmlinkage long sys32_sigaction(int sig, struct old_sigaction32 __user *act,
287 return ret; 287 return ret;
288} 288}
289 289
290asmlinkage long sys32_waitpid(compat_pid_t pid, unsigned int *stat_addr, 290asmlinkage long sys32_waitpid(compat_pid_t pid, unsigned int __user *stat_addr,
291 int options) 291 int options)
292{ 292{
293 return compat_sys_wait4(pid, stat_addr, options, NULL); 293 return compat_sys_wait4(pid, stat_addr, options, NULL);
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 633b6176cf60..16cae425d1f8 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -97,6 +97,7 @@
97#define X86_FEATURE_EXTD_APICID (3*32+26) /* has extended APICID (8 bits) */ 97#define X86_FEATURE_EXTD_APICID (3*32+26) /* has extended APICID (8 bits) */
98#define X86_FEATURE_AMD_DCM (3*32+27) /* multi-node processor */ 98#define X86_FEATURE_AMD_DCM (3*32+27) /* multi-node processor */
99#define X86_FEATURE_APERFMPERF (3*32+28) /* APERFMPERF */ 99#define X86_FEATURE_APERFMPERF (3*32+28) /* APERFMPERF */
100#define X86_FEATURE_EAGER_FPU (3*32+29) /* "eagerfpu" Non lazy FPU restore */
100 101
101/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ 102/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
102#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */ 103#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */
@@ -300,12 +301,14 @@ extern const char * const x86_power_flags[32];
300#define cpu_has_xmm4_2 boot_cpu_has(X86_FEATURE_XMM4_2) 301#define cpu_has_xmm4_2 boot_cpu_has(X86_FEATURE_XMM4_2)
301#define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC) 302#define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC)
302#define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE) 303#define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE)
304#define cpu_has_xsaveopt boot_cpu_has(X86_FEATURE_XSAVEOPT)
303#define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE) 305#define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE)
304#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR) 306#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR)
305#define cpu_has_pclmulqdq boot_cpu_has(X86_FEATURE_PCLMULQDQ) 307#define cpu_has_pclmulqdq boot_cpu_has(X86_FEATURE_PCLMULQDQ)
306#define cpu_has_perfctr_core boot_cpu_has(X86_FEATURE_PERFCTR_CORE) 308#define cpu_has_perfctr_core boot_cpu_has(X86_FEATURE_PERFCTR_CORE)
307#define cpu_has_cx8 boot_cpu_has(X86_FEATURE_CX8) 309#define cpu_has_cx8 boot_cpu_has(X86_FEATURE_CX8)
308#define cpu_has_cx16 boot_cpu_has(X86_FEATURE_CX16) 310#define cpu_has_cx16 boot_cpu_has(X86_FEATURE_CX16)
311#define cpu_has_eager_fpu boot_cpu_has(X86_FEATURE_EAGER_FPU)
309 312
310#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64) 313#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)
311# define cpu_has_invlpg 1 314# define cpu_has_invlpg 1
diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h
index 0fe13583a028..409b9ccf5518 100644
--- a/arch/x86/include/asm/fpu-internal.h
+++ b/arch/x86/include/asm/fpu-internal.h
@@ -12,6 +12,7 @@
12 12
13#include <linux/kernel_stat.h> 13#include <linux/kernel_stat.h>
14#include <linux/regset.h> 14#include <linux/regset.h>
15#include <linux/compat.h>
15#include <linux/slab.h> 16#include <linux/slab.h>
16#include <asm/asm.h> 17#include <asm/asm.h>
17#include <asm/cpufeature.h> 18#include <asm/cpufeature.h>
@@ -20,43 +21,76 @@
20#include <asm/user.h> 21#include <asm/user.h>
21#include <asm/uaccess.h> 22#include <asm/uaccess.h>
22#include <asm/xsave.h> 23#include <asm/xsave.h>
24#include <asm/smap.h>
23 25
24extern unsigned int sig_xstate_size; 26#ifdef CONFIG_X86_64
27# include <asm/sigcontext32.h>
28# include <asm/user32.h>
29int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
30 compat_sigset_t *set, struct pt_regs *regs);
31int ia32_setup_frame(int sig, struct k_sigaction *ka,
32 compat_sigset_t *set, struct pt_regs *regs);
33#else
34# define user_i387_ia32_struct user_i387_struct
35# define user32_fxsr_struct user_fxsr_struct
36# define ia32_setup_frame __setup_frame
37# define ia32_setup_rt_frame __setup_rt_frame
38#endif
39
40extern unsigned int mxcsr_feature_mask;
25extern void fpu_init(void); 41extern void fpu_init(void);
42extern void eager_fpu_init(void);
26 43
27DECLARE_PER_CPU(struct task_struct *, fpu_owner_task); 44DECLARE_PER_CPU(struct task_struct *, fpu_owner_task);
28 45
46extern void convert_from_fxsr(struct user_i387_ia32_struct *env,
47 struct task_struct *tsk);
48extern void convert_to_fxsr(struct task_struct *tsk,
49 const struct user_i387_ia32_struct *env);
50
29extern user_regset_active_fn fpregs_active, xfpregs_active; 51extern user_regset_active_fn fpregs_active, xfpregs_active;
30extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, 52extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
31 xstateregs_get; 53 xstateregs_get;
32extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, 54extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
33 xstateregs_set; 55 xstateregs_set;
34 56
35
36/* 57/*
37 * xstateregs_active == fpregs_active. Please refer to the comment 58 * xstateregs_active == fpregs_active. Please refer to the comment
38 * at the definition of fpregs_active. 59 * at the definition of fpregs_active.
39 */ 60 */
40#define xstateregs_active fpregs_active 61#define xstateregs_active fpregs_active
41 62
42extern struct _fpx_sw_bytes fx_sw_reserved;
43#ifdef CONFIG_IA32_EMULATION
44extern unsigned int sig_xstate_ia32_size;
45extern struct _fpx_sw_bytes fx_sw_reserved_ia32;
46struct _fpstate_ia32;
47struct _xstate_ia32;
48extern int save_i387_xstate_ia32(void __user *buf);
49extern int restore_i387_xstate_ia32(void __user *buf);
50#endif
51
52#ifdef CONFIG_MATH_EMULATION 63#ifdef CONFIG_MATH_EMULATION
64# define HAVE_HWFP (boot_cpu_data.hard_math)
53extern void finit_soft_fpu(struct i387_soft_struct *soft); 65extern void finit_soft_fpu(struct i387_soft_struct *soft);
54#else 66#else
67# define HAVE_HWFP 1
55static inline void finit_soft_fpu(struct i387_soft_struct *soft) {} 68static inline void finit_soft_fpu(struct i387_soft_struct *soft) {}
56#endif 69#endif
57 70
71static inline int is_ia32_compat_frame(void)
72{
73 return config_enabled(CONFIG_IA32_EMULATION) &&
74 test_thread_flag(TIF_IA32);
75}
76
77static inline int is_ia32_frame(void)
78{
79 return config_enabled(CONFIG_X86_32) || is_ia32_compat_frame();
80}
81
82static inline int is_x32_frame(void)
83{
84 return config_enabled(CONFIG_X86_X32_ABI) && test_thread_flag(TIF_X32);
85}
86
58#define X87_FSW_ES (1 << 7) /* Exception Summary */ 87#define X87_FSW_ES (1 << 7) /* Exception Summary */
59 88
89static __always_inline __pure bool use_eager_fpu(void)
90{
91 return static_cpu_has(X86_FEATURE_EAGER_FPU);
92}
93
60static __always_inline __pure bool use_xsaveopt(void) 94static __always_inline __pure bool use_xsaveopt(void)
61{ 95{
62 return static_cpu_has(X86_FEATURE_XSAVEOPT); 96 return static_cpu_has(X86_FEATURE_XSAVEOPT);
@@ -72,6 +106,13 @@ static __always_inline __pure bool use_fxsr(void)
72 return static_cpu_has(X86_FEATURE_FXSR); 106 return static_cpu_has(X86_FEATURE_FXSR);
73} 107}
74 108
109static inline void fx_finit(struct i387_fxsave_struct *fx)
110{
111 memset(fx, 0, xstate_size);
112 fx->cwd = 0x37f;
113 fx->mxcsr = MXCSR_DEFAULT;
114}
115
75extern void __sanitize_i387_state(struct task_struct *); 116extern void __sanitize_i387_state(struct task_struct *);
76 117
77static inline void sanitize_i387_state(struct task_struct *tsk) 118static inline void sanitize_i387_state(struct task_struct *tsk)
@@ -81,133 +122,104 @@ static inline void sanitize_i387_state(struct task_struct *tsk)
81 __sanitize_i387_state(tsk); 122 __sanitize_i387_state(tsk);
82} 123}
83 124
84#ifdef CONFIG_X86_64 125#define user_insn(insn, output, input...) \
85static inline int fxrstor_checking(struct i387_fxsave_struct *fx) 126({ \
86{ 127 int err; \
87 int err; 128 asm volatile(ASM_STAC "\n" \
88 129 "1:" #insn "\n\t" \
89 /* See comment in fxsave() below. */ 130 "2: " ASM_CLAC "\n" \
90#ifdef CONFIG_AS_FXSAVEQ 131 ".section .fixup,\"ax\"\n" \
91 asm volatile("1: fxrstorq %[fx]\n\t" 132 "3: movl $-1,%[err]\n" \
92 "2:\n" 133 " jmp 2b\n" \
93 ".section .fixup,\"ax\"\n" 134 ".previous\n" \
94 "3: movl $-1,%[err]\n" 135 _ASM_EXTABLE(1b, 3b) \
95 " jmp 2b\n" 136 : [err] "=r" (err), output \
96 ".previous\n" 137 : "0"(0), input); \
97 _ASM_EXTABLE(1b, 3b) 138 err; \
98 : [err] "=r" (err) 139})
99 : [fx] "m" (*fx), "0" (0)); 140
100#else 141#define check_insn(insn, output, input...) \
101 asm volatile("1: rex64/fxrstor (%[fx])\n\t" 142({ \
102 "2:\n" 143 int err; \
103 ".section .fixup,\"ax\"\n" 144 asm volatile("1:" #insn "\n\t" \
104 "3: movl $-1,%[err]\n" 145 "2:\n" \
105 " jmp 2b\n" 146 ".section .fixup,\"ax\"\n" \
106 ".previous\n" 147 "3: movl $-1,%[err]\n" \
107 _ASM_EXTABLE(1b, 3b) 148 " jmp 2b\n" \
108 : [err] "=r" (err) 149 ".previous\n" \
109 : [fx] "R" (fx), "m" (*fx), "0" (0)); 150 _ASM_EXTABLE(1b, 3b) \
110#endif 151 : [err] "=r" (err), output \
111 return err; 152 : "0"(0), input); \
153 err; \
154})
155
156static inline int fsave_user(struct i387_fsave_struct __user *fx)
157{
158 return user_insn(fnsave %[fx]; fwait, [fx] "=m" (*fx), "m" (*fx));
112} 159}
113 160
114static inline int fxsave_user(struct i387_fxsave_struct __user *fx) 161static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
115{ 162{
116 int err; 163 if (config_enabled(CONFIG_X86_32))
164 return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx));
165 else if (config_enabled(CONFIG_AS_FXSAVEQ))
166 return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx));
117 167
118 /* 168 /* See comment in fpu_fxsave() below. */
119 * Clear the bytes not touched by the fxsave and reserved 169 return user_insn(rex64/fxsave (%[fx]), "=m" (*fx), [fx] "R" (fx));
120 * for the SW usage.
121 */
122 err = __clear_user(&fx->sw_reserved,
123 sizeof(struct _fpx_sw_bytes));
124 if (unlikely(err))
125 return -EFAULT;
126
127 /* See comment in fxsave() below. */
128#ifdef CONFIG_AS_FXSAVEQ
129 asm volatile(ASM_STAC "\n"
130 "1: fxsaveq %[fx]\n\t"
131 "2: " ASM_CLAC "\n"
132 ".section .fixup,\"ax\"\n"
133 "3: movl $-1,%[err]\n"
134 " jmp 2b\n"
135 ".previous\n"
136 _ASM_EXTABLE(1b, 3b)
137 : [err] "=r" (err), [fx] "=m" (*fx)
138 : "0" (0));
139#else
140 asm volatile(ASM_STAC "\n"
141 "1: rex64/fxsave (%[fx])\n\t"
142 "2: " ASM_CLAC "\n"
143 ".section .fixup,\"ax\"\n"
144 "3: movl $-1,%[err]\n"
145 " jmp 2b\n"
146 ".previous\n"
147 _ASM_EXTABLE(1b, 3b)
148 : [err] "=r" (err), "=m" (*fx)
149 : [fx] "R" (fx), "0" (0));
150#endif
151 if (unlikely(err) &&
152 __clear_user(fx, sizeof(struct i387_fxsave_struct)))
153 err = -EFAULT;
154 /* No need to clear here because the caller clears USED_MATH */
155 return err;
156} 170}
157 171
158static inline void fpu_fxsave(struct fpu *fpu) 172static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
159{ 173{
160 /* Using "rex64; fxsave %0" is broken because, if the memory operand 174 if (config_enabled(CONFIG_X86_32))
161 uses any extended registers for addressing, a second REX prefix 175 return check_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
162 will be generated (to the assembler, rex64 followed by semicolon 176 else if (config_enabled(CONFIG_AS_FXSAVEQ))
163 is a separate instruction), and hence the 64-bitness is lost. */ 177 return check_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
164 178
165#ifdef CONFIG_AS_FXSAVEQ 179 /* See comment in fpu_fxsave() below. */
166 /* Using "fxsaveq %0" would be the ideal choice, but is only supported 180 return check_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx),
167 starting with gas 2.16. */ 181 "m" (*fx));
168 __asm__ __volatile__("fxsaveq %0"
169 : "=m" (fpu->state->fxsave));
170#else
171 /* Using, as a workaround, the properly prefixed form below isn't
172 accepted by any binutils version so far released, complaining that
173 the same type of prefix is used twice if an extended register is
174 needed for addressing (fix submitted to mainline 2005-11-21).
175 asm volatile("rex64/fxsave %0"
176 : "=m" (fpu->state->fxsave));
177 This, however, we can work around by forcing the compiler to select
178 an addressing mode that doesn't require extended registers. */
179 asm volatile("rex64/fxsave (%[fx])"
180 : "=m" (fpu->state->fxsave)
181 : [fx] "R" (&fpu->state->fxsave));
182#endif
183} 182}
184 183
185#else /* CONFIG_X86_32 */ 184static inline int frstor_checking(struct i387_fsave_struct *fx)
186
187/* perform fxrstor iff the processor has extended states, otherwise frstor */
188static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
189{ 185{
190 /* 186 return check_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
191 * The "nop" is needed to make the instructions the same
192 * length.
193 */
194 alternative_input(
195 "nop ; frstor %1",
196 "fxrstor %1",
197 X86_FEATURE_FXSR,
198 "m" (*fx));
199
200 return 0;
201} 187}
202 188
203static inline void fpu_fxsave(struct fpu *fpu) 189static inline void fpu_fxsave(struct fpu *fpu)
204{ 190{
205 asm volatile("fxsave %[fx]" 191 if (config_enabled(CONFIG_X86_32))
206 : [fx] "=m" (fpu->state->fxsave)); 192 asm volatile( "fxsave %[fx]" : [fx] "=m" (fpu->state->fxsave));
193 else if (config_enabled(CONFIG_AS_FXSAVEQ))
194 asm volatile("fxsaveq %0" : "=m" (fpu->state->fxsave));
195 else {
196 /* Using "rex64; fxsave %0" is broken because, if the memory
197 * operand uses any extended registers for addressing, a second
198 * REX prefix will be generated (to the assembler, rex64
199 * followed by semicolon is a separate instruction), and hence
200 * the 64-bitness is lost.
201 *
202 * Using "fxsaveq %0" would be the ideal choice, but is only
203 * supported starting with gas 2.16.
204 *
205 * Using, as a workaround, the properly prefixed form below
206 * isn't accepted by any binutils version so far released,
207 * complaining that the same type of prefix is used twice if
208 * an extended register is needed for addressing (fix submitted
209 * to mainline 2005-11-21).
210 *
211 * asm volatile("rex64/fxsave %0" : "=m" (fpu->state->fxsave));
212 *
213 * This, however, we can work around by forcing the compiler to
214 * select an addressing mode that doesn't require extended
215 * registers.
216 */
217 asm volatile( "rex64/fxsave (%[fx])"
218 : "=m" (fpu->state->fxsave)
219 : [fx] "R" (&fpu->state->fxsave));
220 }
207} 221}
208 222
209#endif /* CONFIG_X86_64 */
210
211/* 223/*
212 * These must be called with preempt disabled. Returns 224 * These must be called with preempt disabled. Returns
213 * 'true' if the FPU state is still intact. 225 * 'true' if the FPU state is still intact.
@@ -250,17 +262,14 @@ static inline int __save_init_fpu(struct task_struct *tsk)
250 return fpu_save_init(&tsk->thread.fpu); 262 return fpu_save_init(&tsk->thread.fpu);
251} 263}
252 264
253static inline int fpu_fxrstor_checking(struct fpu *fpu)
254{
255 return fxrstor_checking(&fpu->state->fxsave);
256}
257
258static inline int fpu_restore_checking(struct fpu *fpu) 265static inline int fpu_restore_checking(struct fpu *fpu)
259{ 266{
260 if (use_xsave()) 267 if (use_xsave())
261 return fpu_xrstor_checking(fpu); 268 return fpu_xrstor_checking(&fpu->state->xsave);
269 else if (use_fxsr())
270 return fxrstor_checking(&fpu->state->fxsave);
262 else 271 else
263 return fpu_fxrstor_checking(fpu); 272 return frstor_checking(&fpu->state->fsave);
264} 273}
265 274
266static inline int restore_fpu_checking(struct task_struct *tsk) 275static inline int restore_fpu_checking(struct task_struct *tsk)
@@ -312,15 +321,52 @@ static inline void __thread_set_has_fpu(struct task_struct *tsk)
312static inline void __thread_fpu_end(struct task_struct *tsk) 321static inline void __thread_fpu_end(struct task_struct *tsk)
313{ 322{
314 __thread_clear_has_fpu(tsk); 323 __thread_clear_has_fpu(tsk);
315 stts(); 324 if (!use_eager_fpu())
325 stts();
316} 326}
317 327
318static inline void __thread_fpu_begin(struct task_struct *tsk) 328static inline void __thread_fpu_begin(struct task_struct *tsk)
319{ 329{
320 clts(); 330 if (!use_eager_fpu())
331 clts();
321 __thread_set_has_fpu(tsk); 332 __thread_set_has_fpu(tsk);
322} 333}
323 334
335static inline void __drop_fpu(struct task_struct *tsk)
336{
337 if (__thread_has_fpu(tsk)) {
338 /* Ignore delayed exceptions from user space */
339 asm volatile("1: fwait\n"
340 "2:\n"
341 _ASM_EXTABLE(1b, 2b));
342 __thread_fpu_end(tsk);
343 }
344}
345
346static inline void drop_fpu(struct task_struct *tsk)
347{
348 /*
349 * Forget coprocessor state..
350 */
351 preempt_disable();
352 tsk->fpu_counter = 0;
353 __drop_fpu(tsk);
354 clear_used_math();
355 preempt_enable();
356}
357
358static inline void drop_init_fpu(struct task_struct *tsk)
359{
360 if (!use_eager_fpu())
361 drop_fpu(tsk);
362 else {
363 if (use_xsave())
364 xrstor_state(init_xstate_buf, -1);
365 else
366 fxrstor_checking(&init_xstate_buf->i387);
367 }
368}
369
324/* 370/*
325 * FPU state switching for scheduling. 371 * FPU state switching for scheduling.
326 * 372 *
@@ -354,7 +400,12 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta
354{ 400{
355 fpu_switch_t fpu; 401 fpu_switch_t fpu;
356 402
357 fpu.preload = tsk_used_math(new) && new->fpu_counter > 5; 403 /*
404 * If the task has used the math, pre-load the FPU on xsave processors
405 * or if the past 5 consecutive context-switches used math.
406 */
407 fpu.preload = tsk_used_math(new) && (use_eager_fpu() ||
408 new->fpu_counter > 5);
358 if (__thread_has_fpu(old)) { 409 if (__thread_has_fpu(old)) {
359 if (!__save_init_fpu(old)) 410 if (!__save_init_fpu(old))
360 cpu = ~0; 411 cpu = ~0;
@@ -366,14 +417,14 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta
366 new->fpu_counter++; 417 new->fpu_counter++;
367 __thread_set_has_fpu(new); 418 __thread_set_has_fpu(new);
368 prefetch(new->thread.fpu.state); 419 prefetch(new->thread.fpu.state);
369 } else 420 } else if (!use_eager_fpu())
370 stts(); 421 stts();
371 } else { 422 } else {
372 old->fpu_counter = 0; 423 old->fpu_counter = 0;
373 old->thread.fpu.last_cpu = ~0; 424 old->thread.fpu.last_cpu = ~0;
374 if (fpu.preload) { 425 if (fpu.preload) {
375 new->fpu_counter++; 426 new->fpu_counter++;
376 if (fpu_lazy_restore(new, cpu)) 427 if (!use_eager_fpu() && fpu_lazy_restore(new, cpu))
377 fpu.preload = 0; 428 fpu.preload = 0;
378 else 429 else
379 prefetch(new->thread.fpu.state); 430 prefetch(new->thread.fpu.state);
@@ -393,44 +444,40 @@ static inline void switch_fpu_finish(struct task_struct *new, fpu_switch_t fpu)
393{ 444{
394 if (fpu.preload) { 445 if (fpu.preload) {
395 if (unlikely(restore_fpu_checking(new))) 446 if (unlikely(restore_fpu_checking(new)))
396 __thread_fpu_end(new); 447 drop_init_fpu(new);
397 } 448 }
398} 449}
399 450
400/* 451/*
401 * Signal frame handlers... 452 * Signal frame handlers...
402 */ 453 */
403extern int save_i387_xstate(void __user *buf); 454extern int save_xstate_sig(void __user *buf, void __user *fx, int size);
404extern int restore_i387_xstate(void __user *buf); 455extern int __restore_xstate_sig(void __user *buf, void __user *fx, int size);
405 456
406static inline void __clear_fpu(struct task_struct *tsk) 457static inline int xstate_sigframe_size(void)
407{ 458{
408 if (__thread_has_fpu(tsk)) { 459 return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size;
409 /* Ignore delayed exceptions from user space */ 460}
410 asm volatile("1: fwait\n" 461
411 "2:\n" 462static inline int restore_xstate_sig(void __user *buf, int ia32_frame)
412 _ASM_EXTABLE(1b, 2b)); 463{
413 __thread_fpu_end(tsk); 464 void __user *buf_fx = buf;
465 int size = xstate_sigframe_size();
466
467 if (ia32_frame && use_fxsr()) {
468 buf_fx = buf + sizeof(struct i387_fsave_struct);
469 size += sizeof(struct i387_fsave_struct);
414 } 470 }
471
472 return __restore_xstate_sig(buf, buf_fx, size);
415} 473}
416 474
417/* 475/*
418 * The actual user_fpu_begin/end() functions 476 * Need to be preemption-safe.
419 * need to be preemption-safe.
420 * 477 *
421 * NOTE! user_fpu_end() must be used only after you 478 * NOTE! user_fpu_begin() must be used only immediately before restoring
422 * have saved the FP state, and user_fpu_begin() must 479 * it. This function does not do any save/restore on their own.
423 * be used only immediately before restoring it.
424 * These functions do not do any save/restore on
425 * their own.
426 */ 480 */
427static inline void user_fpu_end(void)
428{
429 preempt_disable();
430 __thread_fpu_end(current);
431 preempt_enable();
432}
433
434static inline void user_fpu_begin(void) 481static inline void user_fpu_begin(void)
435{ 482{
436 preempt_disable(); 483 preempt_disable();
@@ -439,25 +486,32 @@ static inline void user_fpu_begin(void)
439 preempt_enable(); 486 preempt_enable();
440} 487}
441 488
489static inline void __save_fpu(struct task_struct *tsk)
490{
491 if (use_xsave())
492 xsave_state(&tsk->thread.fpu.state->xsave, -1);
493 else
494 fpu_fxsave(&tsk->thread.fpu);
495}
496
442/* 497/*
443 * These disable preemption on their own and are safe 498 * These disable preemption on their own and are safe
444 */ 499 */
445static inline void save_init_fpu(struct task_struct *tsk) 500static inline void save_init_fpu(struct task_struct *tsk)
446{ 501{
447 WARN_ON_ONCE(!__thread_has_fpu(tsk)); 502 WARN_ON_ONCE(!__thread_has_fpu(tsk));
503
504 if (use_eager_fpu()) {
505 __save_fpu(tsk);
506 return;
507 }
508
448 preempt_disable(); 509 preempt_disable();
449 __save_init_fpu(tsk); 510 __save_init_fpu(tsk);
450 __thread_fpu_end(tsk); 511 __thread_fpu_end(tsk);
451 preempt_enable(); 512 preempt_enable();
452} 513}
453 514
454static inline void clear_fpu(struct task_struct *tsk)
455{
456 preempt_disable();
457 __clear_fpu(tsk);
458 preempt_enable();
459}
460
461/* 515/*
462 * i387 state interaction 516 * i387 state interaction
463 */ 517 */
@@ -512,11 +566,34 @@ static inline void fpu_free(struct fpu *fpu)
512 } 566 }
513} 567}
514 568
515static inline void fpu_copy(struct fpu *dst, struct fpu *src) 569static inline void fpu_copy(struct task_struct *dst, struct task_struct *src)
516{ 570{
517 memcpy(dst->state, src->state, xstate_size); 571 if (use_eager_fpu()) {
572 memset(&dst->thread.fpu.state->xsave, 0, xstate_size);
573 __save_fpu(dst);
574 } else {
575 struct fpu *dfpu = &dst->thread.fpu;
576 struct fpu *sfpu = &src->thread.fpu;
577
578 unlazy_fpu(src);
579 memcpy(dfpu->state, sfpu->state, xstate_size);
580 }
518} 581}
519 582
520extern void fpu_finit(struct fpu *fpu); 583static inline unsigned long
584alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx,
585 unsigned long *size)
586{
587 unsigned long frame_size = xstate_sigframe_size();
588
589 *buf_fx = sp = round_down(sp - frame_size, 64);
590 if (ia32_frame && use_fxsr()) {
591 frame_size += sizeof(struct i387_fsave_struct);
592 sp -= sizeof(struct i387_fsave_struct);
593 }
594
595 *size = frame_size;
596 return sp;
597}
521 598
522#endif 599#endif
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 257d9cca214f..ed8089d69094 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -19,12 +19,37 @@ struct pt_regs;
19struct user_i387_struct; 19struct user_i387_struct;
20 20
21extern int init_fpu(struct task_struct *child); 21extern int init_fpu(struct task_struct *child);
22extern void fpu_finit(struct fpu *fpu);
22extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); 23extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
23extern void math_state_restore(void); 24extern void math_state_restore(void);
24 25
25extern bool irq_fpu_usable(void); 26extern bool irq_fpu_usable(void);
26extern void kernel_fpu_begin(void); 27
27extern void kernel_fpu_end(void); 28/*
29 * Careful: __kernel_fpu_begin/end() must be called with preempt disabled
30 * and they don't touch the preempt state on their own.
31 * If you enable preemption after __kernel_fpu_begin(), preempt notifier
32 * should call the __kernel_fpu_end() to prevent the kernel/user FPU
33 * state from getting corrupted. KVM for example uses this model.
34 *
35 * All other cases use kernel_fpu_begin/end() which disable preemption
36 * during kernel FPU usage.
37 */
38extern void __kernel_fpu_begin(void);
39extern void __kernel_fpu_end(void);
40
41static inline void kernel_fpu_begin(void)
42{
43 WARN_ON_ONCE(!irq_fpu_usable());
44 preempt_disable();
45 __kernel_fpu_begin();
46}
47
48static inline void kernel_fpu_end(void)
49{
50 __kernel_fpu_end();
51 preempt_enable();
52}
28 53
29/* 54/*
30 * Some instructions like VIA's padlock instructions generate a spurious 55 * Some instructions like VIA's padlock instructions generate a spurious
diff --git a/arch/x86/include/asm/iommu_table.h b/arch/x86/include/asm/iommu_table.h
index f229b13a5f30..f42a04735a0a 100644
--- a/arch/x86/include/asm/iommu_table.h
+++ b/arch/x86/include/asm/iommu_table.h
@@ -48,7 +48,7 @@ struct iommu_table_entry {
48 48
49 49
50#define __IOMMU_INIT(_detect, _depend, _early_init, _late_init, _finish)\ 50#define __IOMMU_INIT(_detect, _depend, _early_init, _late_init, _finish)\
51 static const struct iommu_table_entry const \ 51 static const struct iommu_table_entry \
52 __iommu_entry_##_detect __used \ 52 __iommu_entry_##_detect __used \
53 __attribute__ ((unused, __section__(".iommu_table"), \ 53 __attribute__ ((unused, __section__(".iommu_table"), \
54 aligned((sizeof(void *))))) \ 54 aligned((sizeof(void *))))) \
@@ -63,10 +63,10 @@ struct iommu_table_entry {
63 * to stop detecting the other IOMMUs after yours has been detected. 63 * to stop detecting the other IOMMUs after yours has been detected.
64 */ 64 */
65#define IOMMU_INIT_POST(_detect) \ 65#define IOMMU_INIT_POST(_detect) \
66 __IOMMU_INIT(_detect, pci_swiotlb_detect_4gb, 0, 0, 0) 66 __IOMMU_INIT(_detect, pci_swiotlb_detect_4gb, NULL, NULL, 0)
67 67
68#define IOMMU_INIT_POST_FINISH(detect) \ 68#define IOMMU_INIT_POST_FINISH(detect) \
69 __IOMMU_INIT(_detect, pci_swiotlb_detect_4gb, 0, 0, 1) 69 __IOMMU_INIT(_detect, pci_swiotlb_detect_4gb, NULL, NULL, 1)
70 70
71/* 71/*
72 * A more sophisticated version of IOMMU_INIT. This variant requires: 72 * A more sophisticated version of IOMMU_INIT. This variant requires:
diff --git a/arch/x86/include/asm/signal.h b/arch/x86/include/asm/signal.h
index 598457cbd0f8..323973f4abf1 100644
--- a/arch/x86/include/asm/signal.h
+++ b/arch/x86/include/asm/signal.h
@@ -31,6 +31,10 @@ typedef struct {
31 unsigned long sig[_NSIG_WORDS]; 31 unsigned long sig[_NSIG_WORDS];
32} sigset_t; 32} sigset_t;
33 33
34#ifndef CONFIG_COMPAT
35typedef sigset_t compat_sigset_t;
36#endif
37
34#else 38#else
35/* Here we must cater to libcs that poke about in kernel headers. */ 39/* Here we must cater to libcs that poke about in kernel headers. */
36 40
diff --git a/arch/x86/include/asm/sys_ia32.h b/arch/x86/include/asm/sys_ia32.h
index 3fda9db48819..4ca1c611b552 100644
--- a/arch/x86/include/asm/sys_ia32.h
+++ b/arch/x86/include/asm/sys_ia32.h
@@ -40,7 +40,7 @@ asmlinkage long sys32_sigaction(int, struct old_sigaction32 __user *,
40 struct old_sigaction32 __user *); 40 struct old_sigaction32 __user *);
41asmlinkage long sys32_alarm(unsigned int); 41asmlinkage long sys32_alarm(unsigned int);
42 42
43asmlinkage long sys32_waitpid(compat_pid_t, unsigned int *, int); 43asmlinkage long sys32_waitpid(compat_pid_t, unsigned int __user *, int);
44asmlinkage long sys32_sysfs(int, u32, u32); 44asmlinkage long sys32_sysfs(int, u32, u32);
45 45
46asmlinkage long sys32_sched_rr_get_interval(compat_pid_t, 46asmlinkage long sys32_sched_rr_get_interval(compat_pid_t,
diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h
index bb0522850b74..fddb53d63915 100644
--- a/arch/x86/include/asm/vdso.h
+++ b/arch/x86/include/asm/vdso.h
@@ -11,7 +11,8 @@ extern const char VDSO32_PRELINK[];
11#define VDSO32_SYMBOL(base, name) \ 11#define VDSO32_SYMBOL(base, name) \
12({ \ 12({ \
13 extern const char VDSO32_##name[]; \ 13 extern const char VDSO32_##name[]; \
14 (void *)(VDSO32_##name - VDSO32_PRELINK + (unsigned long)(base)); \ 14 (void __user *)(VDSO32_##name - VDSO32_PRELINK + \
15 (unsigned long)(base)); \
15}) 16})
16#endif 17#endif
17 18
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h
index 454570891bdc..aabd5850bdb9 100644
--- a/arch/x86/include/asm/xor_32.h
+++ b/arch/x86/include/asm/xor_32.h
@@ -534,38 +534,6 @@ static struct xor_block_template xor_block_p5_mmx = {
534 * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo) 534 * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
535 */ 535 */
536 536
537#define XMMS_SAVE \
538do { \
539 preempt_disable(); \
540 cr0 = read_cr0(); \
541 clts(); \
542 asm volatile( \
543 "movups %%xmm0,(%0) ;\n\t" \
544 "movups %%xmm1,0x10(%0) ;\n\t" \
545 "movups %%xmm2,0x20(%0) ;\n\t" \
546 "movups %%xmm3,0x30(%0) ;\n\t" \
547 : \
548 : "r" (xmm_save) \
549 : "memory"); \
550} while (0)
551
552#define XMMS_RESTORE \
553do { \
554 asm volatile( \
555 "sfence ;\n\t" \
556 "movups (%0),%%xmm0 ;\n\t" \
557 "movups 0x10(%0),%%xmm1 ;\n\t" \
558 "movups 0x20(%0),%%xmm2 ;\n\t" \
559 "movups 0x30(%0),%%xmm3 ;\n\t" \
560 : \
561 : "r" (xmm_save) \
562 : "memory"); \
563 write_cr0(cr0); \
564 preempt_enable(); \
565} while (0)
566
567#define ALIGN16 __attribute__((aligned(16)))
568
569#define OFFS(x) "16*("#x")" 537#define OFFS(x) "16*("#x")"
570#define PF_OFFS(x) "256+16*("#x")" 538#define PF_OFFS(x) "256+16*("#x")"
571#define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n" 539#define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n"
@@ -587,10 +555,8 @@ static void
587xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) 555xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
588{ 556{
589 unsigned long lines = bytes >> 8; 557 unsigned long lines = bytes >> 8;
590 char xmm_save[16*4] ALIGN16;
591 int cr0;
592 558
593 XMMS_SAVE; 559 kernel_fpu_begin();
594 560
595 asm volatile( 561 asm volatile(
596#undef BLOCK 562#undef BLOCK
@@ -633,7 +599,7 @@ xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
633 : 599 :
634 : "memory"); 600 : "memory");
635 601
636 XMMS_RESTORE; 602 kernel_fpu_end();
637} 603}
638 604
639static void 605static void
@@ -641,10 +607,8 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
641 unsigned long *p3) 607 unsigned long *p3)
642{ 608{
643 unsigned long lines = bytes >> 8; 609 unsigned long lines = bytes >> 8;
644 char xmm_save[16*4] ALIGN16;
645 int cr0;
646 610
647 XMMS_SAVE; 611 kernel_fpu_begin();
648 612
649 asm volatile( 613 asm volatile(
650#undef BLOCK 614#undef BLOCK
@@ -694,7 +658,7 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
694 : 658 :
695 : "memory" ); 659 : "memory" );
696 660
697 XMMS_RESTORE; 661 kernel_fpu_end();
698} 662}
699 663
700static void 664static void
@@ -702,10 +666,8 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
702 unsigned long *p3, unsigned long *p4) 666 unsigned long *p3, unsigned long *p4)
703{ 667{
704 unsigned long lines = bytes >> 8; 668 unsigned long lines = bytes >> 8;
705 char xmm_save[16*4] ALIGN16;
706 int cr0;
707 669
708 XMMS_SAVE; 670 kernel_fpu_begin();
709 671
710 asm volatile( 672 asm volatile(
711#undef BLOCK 673#undef BLOCK
@@ -762,7 +724,7 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
762 : 724 :
763 : "memory" ); 725 : "memory" );
764 726
765 XMMS_RESTORE; 727 kernel_fpu_end();
766} 728}
767 729
768static void 730static void
@@ -770,10 +732,8 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
770 unsigned long *p3, unsigned long *p4, unsigned long *p5) 732 unsigned long *p3, unsigned long *p4, unsigned long *p5)
771{ 733{
772 unsigned long lines = bytes >> 8; 734 unsigned long lines = bytes >> 8;
773 char xmm_save[16*4] ALIGN16;
774 int cr0;
775 735
776 XMMS_SAVE; 736 kernel_fpu_begin();
777 737
778 /* Make sure GCC forgets anything it knows about p4 or p5, 738 /* Make sure GCC forgets anything it knows about p4 or p5,
779 such that it won't pass to the asm volatile below a 739 such that it won't pass to the asm volatile below a
@@ -850,7 +810,7 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
850 like assuming they have some legal value. */ 810 like assuming they have some legal value. */
851 asm("" : "=r" (p4), "=r" (p5)); 811 asm("" : "=r" (p4), "=r" (p5));
852 812
853 XMMS_RESTORE; 813 kernel_fpu_end();
854} 814}
855 815
856static struct xor_block_template xor_block_pIII_sse = { 816static struct xor_block_template xor_block_pIII_sse = {
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h
index b9b2323e90fe..5fc06d0b7eb5 100644
--- a/arch/x86/include/asm/xor_64.h
+++ b/arch/x86/include/asm/xor_64.h
@@ -34,41 +34,7 @@
34 * no advantages to be gotten from x86-64 here anyways. 34 * no advantages to be gotten from x86-64 here anyways.
35 */ 35 */
36 36
37typedef struct { 37#include <asm/i387.h>
38 unsigned long a, b;
39} __attribute__((aligned(16))) xmm_store_t;
40
41/* Doesn't use gcc to save the XMM registers, because there is no easy way to
42 tell it to do a clts before the register saving. */
43#define XMMS_SAVE \
44do { \
45 preempt_disable(); \
46 asm volatile( \
47 "movq %%cr0,%0 ;\n\t" \
48 "clts ;\n\t" \
49 "movups %%xmm0,(%1) ;\n\t" \
50 "movups %%xmm1,0x10(%1) ;\n\t" \
51 "movups %%xmm2,0x20(%1) ;\n\t" \
52 "movups %%xmm3,0x30(%1) ;\n\t" \
53 : "=&r" (cr0) \
54 : "r" (xmm_save) \
55 : "memory"); \
56} while (0)
57
58#define XMMS_RESTORE \
59do { \
60 asm volatile( \
61 "sfence ;\n\t" \
62 "movups (%1),%%xmm0 ;\n\t" \
63 "movups 0x10(%1),%%xmm1 ;\n\t" \
64 "movups 0x20(%1),%%xmm2 ;\n\t" \
65 "movups 0x30(%1),%%xmm3 ;\n\t" \
66 "movq %0,%%cr0 ;\n\t" \
67 : \
68 : "r" (cr0), "r" (xmm_save) \
69 : "memory"); \
70 preempt_enable(); \
71} while (0)
72 38
73#define OFFS(x) "16*("#x")" 39#define OFFS(x) "16*("#x")"
74#define PF_OFFS(x) "256+16*("#x")" 40#define PF_OFFS(x) "256+16*("#x")"
@@ -91,10 +57,8 @@ static void
91xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) 57xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
92{ 58{
93 unsigned int lines = bytes >> 8; 59 unsigned int lines = bytes >> 8;
94 unsigned long cr0;
95 xmm_store_t xmm_save[4];
96 60
97 XMMS_SAVE; 61 kernel_fpu_begin();
98 62
99 asm volatile( 63 asm volatile(
100#undef BLOCK 64#undef BLOCK
@@ -135,7 +99,7 @@ xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
135 : [inc] "r" (256UL) 99 : [inc] "r" (256UL)
136 : "memory"); 100 : "memory");
137 101
138 XMMS_RESTORE; 102 kernel_fpu_end();
139} 103}
140 104
141static void 105static void
@@ -143,11 +107,8 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
143 unsigned long *p3) 107 unsigned long *p3)
144{ 108{
145 unsigned int lines = bytes >> 8; 109 unsigned int lines = bytes >> 8;
146 xmm_store_t xmm_save[4];
147 unsigned long cr0;
148
149 XMMS_SAVE;
150 110
111 kernel_fpu_begin();
151 asm volatile( 112 asm volatile(
152#undef BLOCK 113#undef BLOCK
153#define BLOCK(i) \ 114#define BLOCK(i) \
@@ -194,7 +155,7 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
194 [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3) 155 [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
195 : [inc] "r" (256UL) 156 : [inc] "r" (256UL)
196 : "memory"); 157 : "memory");
197 XMMS_RESTORE; 158 kernel_fpu_end();
198} 159}
199 160
200static void 161static void
@@ -202,10 +163,8 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
202 unsigned long *p3, unsigned long *p4) 163 unsigned long *p3, unsigned long *p4)
203{ 164{
204 unsigned int lines = bytes >> 8; 165 unsigned int lines = bytes >> 8;
205 xmm_store_t xmm_save[4];
206 unsigned long cr0;
207 166
208 XMMS_SAVE; 167 kernel_fpu_begin();
209 168
210 asm volatile( 169 asm volatile(
211#undef BLOCK 170#undef BLOCK
@@ -261,7 +220,7 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
261 : [inc] "r" (256UL) 220 : [inc] "r" (256UL)
262 : "memory" ); 221 : "memory" );
263 222
264 XMMS_RESTORE; 223 kernel_fpu_end();
265} 224}
266 225
267static void 226static void
@@ -269,10 +228,8 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
269 unsigned long *p3, unsigned long *p4, unsigned long *p5) 228 unsigned long *p3, unsigned long *p4, unsigned long *p5)
270{ 229{
271 unsigned int lines = bytes >> 8; 230 unsigned int lines = bytes >> 8;
272 xmm_store_t xmm_save[4];
273 unsigned long cr0;
274 231
275 XMMS_SAVE; 232 kernel_fpu_begin();
276 233
277 asm volatile( 234 asm volatile(
278#undef BLOCK 235#undef BLOCK
@@ -336,7 +293,7 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
336 : [inc] "r" (256UL) 293 : [inc] "r" (256UL)
337 : "memory"); 294 : "memory");
338 295
339 XMMS_RESTORE; 296 kernel_fpu_end();
340} 297}
341 298
342static struct xor_block_template xor_block_sse = { 299static struct xor_block_template xor_block_sse = {
diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h
index 2510d35f480e..7ea79c5fa1f2 100644
--- a/arch/x86/include/asm/xor_avx.h
+++ b/arch/x86/include/asm/xor_avx.h
@@ -20,32 +20,6 @@
20#include <linux/compiler.h> 20#include <linux/compiler.h>
21#include <asm/i387.h> 21#include <asm/i387.h>
22 22
23#define ALIGN32 __aligned(32)
24
25#define YMM_SAVED_REGS 4
26
27#define YMMS_SAVE \
28do { \
29 preempt_disable(); \
30 cr0 = read_cr0(); \
31 clts(); \
32 asm volatile("vmovaps %%ymm0, %0" : "=m" (ymm_save[0]) : : "memory"); \
33 asm volatile("vmovaps %%ymm1, %0" : "=m" (ymm_save[32]) : : "memory"); \
34 asm volatile("vmovaps %%ymm2, %0" : "=m" (ymm_save[64]) : : "memory"); \
35 asm volatile("vmovaps %%ymm3, %0" : "=m" (ymm_save[96]) : : "memory"); \
36} while (0);
37
38#define YMMS_RESTORE \
39do { \
40 asm volatile("sfence" : : : "memory"); \
41 asm volatile("vmovaps %0, %%ymm3" : : "m" (ymm_save[96])); \
42 asm volatile("vmovaps %0, %%ymm2" : : "m" (ymm_save[64])); \
43 asm volatile("vmovaps %0, %%ymm1" : : "m" (ymm_save[32])); \
44 asm volatile("vmovaps %0, %%ymm0" : : "m" (ymm_save[0])); \
45 write_cr0(cr0); \
46 preempt_enable(); \
47} while (0);
48
49#define BLOCK4(i) \ 23#define BLOCK4(i) \
50 BLOCK(32 * i, 0) \ 24 BLOCK(32 * i, 0) \
51 BLOCK(32 * (i + 1), 1) \ 25 BLOCK(32 * (i + 1), 1) \
@@ -60,10 +34,9 @@ do { \
60 34
61static void xor_avx_2(unsigned long bytes, unsigned long *p0, unsigned long *p1) 35static void xor_avx_2(unsigned long bytes, unsigned long *p0, unsigned long *p1)
62{ 36{
63 unsigned long cr0, lines = bytes >> 9; 37 unsigned long lines = bytes >> 9;
64 char ymm_save[32 * YMM_SAVED_REGS] ALIGN32;
65 38
66 YMMS_SAVE 39 kernel_fpu_begin();
67 40
68 while (lines--) { 41 while (lines--) {
69#undef BLOCK 42#undef BLOCK
@@ -82,16 +55,15 @@ do { \
82 p1 = (unsigned long *)((uintptr_t)p1 + 512); 55 p1 = (unsigned long *)((uintptr_t)p1 + 512);
83 } 56 }
84 57
85 YMMS_RESTORE 58 kernel_fpu_end();
86} 59}
87 60
88static void xor_avx_3(unsigned long bytes, unsigned long *p0, unsigned long *p1, 61static void xor_avx_3(unsigned long bytes, unsigned long *p0, unsigned long *p1,
89 unsigned long *p2) 62 unsigned long *p2)
90{ 63{
91 unsigned long cr0, lines = bytes >> 9; 64 unsigned long lines = bytes >> 9;
92 char ymm_save[32 * YMM_SAVED_REGS] ALIGN32;
93 65
94 YMMS_SAVE 66 kernel_fpu_begin();
95 67
96 while (lines--) { 68 while (lines--) {
97#undef BLOCK 69#undef BLOCK
@@ -113,16 +85,15 @@ do { \
113 p2 = (unsigned long *)((uintptr_t)p2 + 512); 85 p2 = (unsigned long *)((uintptr_t)p2 + 512);
114 } 86 }
115 87
116 YMMS_RESTORE 88 kernel_fpu_end();
117} 89}
118 90
119static void xor_avx_4(unsigned long bytes, unsigned long *p0, unsigned long *p1, 91static void xor_avx_4(unsigned long bytes, unsigned long *p0, unsigned long *p1,
120 unsigned long *p2, unsigned long *p3) 92 unsigned long *p2, unsigned long *p3)
121{ 93{
122 unsigned long cr0, lines = bytes >> 9; 94 unsigned long lines = bytes >> 9;
123 char ymm_save[32 * YMM_SAVED_REGS] ALIGN32;
124 95
125 YMMS_SAVE 96 kernel_fpu_begin();
126 97
127 while (lines--) { 98 while (lines--) {
128#undef BLOCK 99#undef BLOCK
@@ -147,16 +118,15 @@ do { \
147 p3 = (unsigned long *)((uintptr_t)p3 + 512); 118 p3 = (unsigned long *)((uintptr_t)p3 + 512);
148 } 119 }
149 120
150 YMMS_RESTORE 121 kernel_fpu_end();
151} 122}
152 123
153static void xor_avx_5(unsigned long bytes, unsigned long *p0, unsigned long *p1, 124static void xor_avx_5(unsigned long bytes, unsigned long *p0, unsigned long *p1,
154 unsigned long *p2, unsigned long *p3, unsigned long *p4) 125 unsigned long *p2, unsigned long *p3, unsigned long *p4)
155{ 126{
156 unsigned long cr0, lines = bytes >> 9; 127 unsigned long lines = bytes >> 9;
157 char ymm_save[32 * YMM_SAVED_REGS] ALIGN32;
158 128
159 YMMS_SAVE 129 kernel_fpu_begin();
160 130
161 while (lines--) { 131 while (lines--) {
162#undef BLOCK 132#undef BLOCK
@@ -184,7 +154,7 @@ do { \
184 p4 = (unsigned long *)((uintptr_t)p4 + 512); 154 p4 = (unsigned long *)((uintptr_t)p4 + 512);
185 } 155 }
186 156
187 YMMS_RESTORE 157 kernel_fpu_end();
188} 158}
189 159
190static struct xor_block_template xor_block_avx = { 160static struct xor_block_template xor_block_avx = {
diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h
index 2a923bd54341..0415cdabb5a6 100644
--- a/arch/x86/include/asm/xsave.h
+++ b/arch/x86/include/asm/xsave.h
@@ -34,17 +34,14 @@
34extern unsigned int xstate_size; 34extern unsigned int xstate_size;
35extern u64 pcntxt_mask; 35extern u64 pcntxt_mask;
36extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS]; 36extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
37extern struct xsave_struct *init_xstate_buf;
37 38
38extern void xsave_init(void); 39extern void xsave_init(void);
39extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); 40extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
40extern int init_fpu(struct task_struct *child); 41extern int init_fpu(struct task_struct *child);
41extern int check_for_xstate(struct i387_fxsave_struct __user *buf,
42 void __user *fpstate,
43 struct _fpx_sw_bytes *sw);
44 42
45static inline int fpu_xrstor_checking(struct fpu *fpu) 43static inline int fpu_xrstor_checking(struct xsave_struct *fx)
46{ 44{
47 struct xsave_struct *fx = &fpu->state->xsave;
48 int err; 45 int err;
49 46
50 asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t" 47 asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t"
@@ -69,8 +66,7 @@ static inline int xsave_user(struct xsave_struct __user *buf)
69 * Clear the xsave header first, so that reserved fields are 66 * Clear the xsave header first, so that reserved fields are
70 * initialized to zero. 67 * initialized to zero.
71 */ 68 */
72 err = __clear_user(&buf->xsave_hdr, 69 err = __clear_user(&buf->xsave_hdr, sizeof(buf->xsave_hdr));
73 sizeof(struct xsave_hdr_struct));
74 if (unlikely(err)) 70 if (unlikely(err))
75 return -EFAULT; 71 return -EFAULT;
76 72
@@ -85,9 +81,6 @@ static inline int xsave_user(struct xsave_struct __user *buf)
85 : [err] "=r" (err) 81 : [err] "=r" (err)
86 : "D" (buf), "a" (-1), "d" (-1), "0" (0) 82 : "D" (buf), "a" (-1), "d" (-1), "0" (0)
87 : "memory"); 83 : "memory");
88 if (unlikely(err) && __clear_user(buf, xstate_size))
89 err = -EFAULT;
90 /* No need to clear here because the caller clears USED_MATH */
91 return err; 84 return err;
92} 85}
93 86
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index c97bb7b5a9f8..d0e910da16c5 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -165,10 +165,15 @@ void __init check_bugs(void)
165 print_cpu_info(&boot_cpu_data); 165 print_cpu_info(&boot_cpu_data);
166#endif 166#endif
167 check_config(); 167 check_config();
168 check_fpu();
169 check_hlt(); 168 check_hlt();
170 check_popad(); 169 check_popad();
171 init_utsname()->machine[1] = 170 init_utsname()->machine[1] =
172 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 171 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
173 alternative_instructions(); 172 alternative_instructions();
173
174 /*
175 * kernel_fpu_begin/end() in check_fpu() relies on the patched
176 * alternative instructions.
177 */
178 check_fpu();
174} 179}
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 7d35d6594118..44aec5d4dfaf 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1324,7 +1324,6 @@ void __cpuinit cpu_init(void)
1324 dbg_restore_debug_regs(); 1324 dbg_restore_debug_regs();
1325 1325
1326 fpu_init(); 1326 fpu_init();
1327 xsave_init();
1328 1327
1329 raw_local_save_flags(kernel_eflags); 1328 raw_local_save_flags(kernel_eflags);
1330 1329
@@ -1379,6 +1378,5 @@ void __cpuinit cpu_init(void)
1379 dbg_restore_debug_regs(); 1378 dbg_restore_debug_regs();
1380 1379
1381 fpu_init(); 1380 fpu_init();
1382 xsave_init();
1383} 1381}
1384#endif 1382#endif
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index f250431fb505..675a05012449 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -19,24 +19,17 @@
19#include <asm/fpu-internal.h> 19#include <asm/fpu-internal.h>
20#include <asm/user.h> 20#include <asm/user.h>
21 21
22#ifdef CONFIG_X86_64
23# include <asm/sigcontext32.h>
24# include <asm/user32.h>
25#else
26# define save_i387_xstate_ia32 save_i387_xstate
27# define restore_i387_xstate_ia32 restore_i387_xstate
28# define _fpstate_ia32 _fpstate
29# define _xstate_ia32 _xstate
30# define sig_xstate_ia32_size sig_xstate_size
31# define fx_sw_reserved_ia32 fx_sw_reserved
32# define user_i387_ia32_struct user_i387_struct
33# define user32_fxsr_struct user_fxsr_struct
34#endif
35
36/* 22/*
37 * Were we in an interrupt that interrupted kernel mode? 23 * Were we in an interrupt that interrupted kernel mode?
38 * 24 *
39 * We can do a kernel_fpu_begin/end() pair *ONLY* if that 25 * For now, with eagerfpu we will return interrupted kernel FPU
26 * state as not-idle. TBD: Ideally we can change the return value
27 * to something like __thread_has_fpu(current). But we need to
28 * be careful of doing __thread_clear_has_fpu() before saving
29 * the FPU etc for supporting nested uses etc. For now, take
30 * the simple route!
31 *
32 * On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
40 * pair does nothing at all: the thread must not have fpu (so 33 * pair does nothing at all: the thread must not have fpu (so
41 * that we don't try to save the FPU state), and TS must 34 * that we don't try to save the FPU state), and TS must
42 * be set (so that the clts/stts pair does nothing that is 35 * be set (so that the clts/stts pair does nothing that is
@@ -44,6 +37,9 @@
44 */ 37 */
45static inline bool interrupted_kernel_fpu_idle(void) 38static inline bool interrupted_kernel_fpu_idle(void)
46{ 39{
40 if (use_eager_fpu())
41 return 0;
42
47 return !__thread_has_fpu(current) && 43 return !__thread_has_fpu(current) &&
48 (read_cr0() & X86_CR0_TS); 44 (read_cr0() & X86_CR0_TS);
49} 45}
@@ -77,29 +73,29 @@ bool irq_fpu_usable(void)
77} 73}
78EXPORT_SYMBOL(irq_fpu_usable); 74EXPORT_SYMBOL(irq_fpu_usable);
79 75
80void kernel_fpu_begin(void) 76void __kernel_fpu_begin(void)
81{ 77{
82 struct task_struct *me = current; 78 struct task_struct *me = current;
83 79
84 WARN_ON_ONCE(!irq_fpu_usable());
85 preempt_disable();
86 if (__thread_has_fpu(me)) { 80 if (__thread_has_fpu(me)) {
87 __save_init_fpu(me); 81 __save_init_fpu(me);
88 __thread_clear_has_fpu(me); 82 __thread_clear_has_fpu(me);
89 /* We do 'stts()' in kernel_fpu_end() */ 83 /* We do 'stts()' in __kernel_fpu_end() */
90 } else { 84 } else if (!use_eager_fpu()) {
91 this_cpu_write(fpu_owner_task, NULL); 85 this_cpu_write(fpu_owner_task, NULL);
92 clts(); 86 clts();
93 } 87 }
94} 88}
95EXPORT_SYMBOL(kernel_fpu_begin); 89EXPORT_SYMBOL(__kernel_fpu_begin);
96 90
97void kernel_fpu_end(void) 91void __kernel_fpu_end(void)
98{ 92{
99 stts(); 93 if (use_eager_fpu())
100 preempt_enable(); 94 math_state_restore();
95 else
96 stts();
101} 97}
102EXPORT_SYMBOL(kernel_fpu_end); 98EXPORT_SYMBOL(__kernel_fpu_end);
103 99
104void unlazy_fpu(struct task_struct *tsk) 100void unlazy_fpu(struct task_struct *tsk)
105{ 101{
@@ -113,23 +109,15 @@ void unlazy_fpu(struct task_struct *tsk)
113} 109}
114EXPORT_SYMBOL(unlazy_fpu); 110EXPORT_SYMBOL(unlazy_fpu);
115 111
116#ifdef CONFIG_MATH_EMULATION 112unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
117# define HAVE_HWFP (boot_cpu_data.hard_math)
118#else
119# define HAVE_HWFP 1
120#endif
121
122static unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
123unsigned int xstate_size; 113unsigned int xstate_size;
124EXPORT_SYMBOL_GPL(xstate_size); 114EXPORT_SYMBOL_GPL(xstate_size);
125unsigned int sig_xstate_ia32_size = sizeof(struct _fpstate_ia32);
126static struct i387_fxsave_struct fx_scratch __cpuinitdata; 115static struct i387_fxsave_struct fx_scratch __cpuinitdata;
127 116
128static void __cpuinit mxcsr_feature_mask_init(void) 117static void __cpuinit mxcsr_feature_mask_init(void)
129{ 118{
130 unsigned long mask = 0; 119 unsigned long mask = 0;
131 120
132 clts();
133 if (cpu_has_fxsr) { 121 if (cpu_has_fxsr) {
134 memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct)); 122 memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
135 asm volatile("fxsave %0" : : "m" (fx_scratch)); 123 asm volatile("fxsave %0" : : "m" (fx_scratch));
@@ -138,7 +126,6 @@ static void __cpuinit mxcsr_feature_mask_init(void)
138 mask = 0x0000ffbf; 126 mask = 0x0000ffbf;
139 } 127 }
140 mxcsr_feature_mask &= mask; 128 mxcsr_feature_mask &= mask;
141 stts();
142} 129}
143 130
144static void __cpuinit init_thread_xstate(void) 131static void __cpuinit init_thread_xstate(void)
@@ -192,9 +179,8 @@ void __cpuinit fpu_init(void)
192 init_thread_xstate(); 179 init_thread_xstate();
193 180
194 mxcsr_feature_mask_init(); 181 mxcsr_feature_mask_init();
195 /* clean state in init */ 182 xsave_init();
196 current_thread_info()->status = 0; 183 eager_fpu_init();
197 clear_used_math();
198} 184}
199 185
200void fpu_finit(struct fpu *fpu) 186void fpu_finit(struct fpu *fpu)
@@ -205,12 +191,7 @@ void fpu_finit(struct fpu *fpu)
205 } 191 }
206 192
207 if (cpu_has_fxsr) { 193 if (cpu_has_fxsr) {
208 struct i387_fxsave_struct *fx = &fpu->state->fxsave; 194 fx_finit(&fpu->state->fxsave);
209
210 memset(fx, 0, xstate_size);
211 fx->cwd = 0x37f;
212 if (cpu_has_xmm)
213 fx->mxcsr = MXCSR_DEFAULT;
214 } else { 195 } else {
215 struct i387_fsave_struct *fp = &fpu->state->fsave; 196 struct i387_fsave_struct *fp = &fpu->state->fsave;
216 memset(fp, 0, xstate_size); 197 memset(fp, 0, xstate_size);
@@ -454,7 +435,7 @@ static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
454 * FXSR floating point environment conversions. 435 * FXSR floating point environment conversions.
455 */ 436 */
456 437
457static void 438void
458convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) 439convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
459{ 440{
460 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave; 441 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
@@ -491,8 +472,8 @@ convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
491 memcpy(&to[i], &from[i], sizeof(to[0])); 472 memcpy(&to[i], &from[i], sizeof(to[0]));
492} 473}
493 474
494static void convert_to_fxsr(struct task_struct *tsk, 475void convert_to_fxsr(struct task_struct *tsk,
495 const struct user_i387_ia32_struct *env) 476 const struct user_i387_ia32_struct *env)
496 477
497{ 478{
498 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave; 479 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
@@ -589,223 +570,6 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
589} 570}
590 571
591/* 572/*
592 * Signal frame handlers.
593 */
594
595static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
596{
597 struct task_struct *tsk = current;
598 struct i387_fsave_struct *fp = &tsk->thread.fpu.state->fsave;
599
600 fp->status = fp->swd;
601 if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct)))
602 return -1;
603 return 1;
604}
605
606static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
607{
608 struct task_struct *tsk = current;
609 struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave;
610 struct user_i387_ia32_struct env;
611 int err = 0;
612
613 convert_from_fxsr(&env, tsk);
614 if (__copy_to_user(buf, &env, sizeof(env)))
615 return -1;
616
617 err |= __put_user(fx->swd, &buf->status);
618 err |= __put_user(X86_FXSR_MAGIC, &buf->magic);
619 if (err)
620 return -1;
621
622 if (__copy_to_user(&buf->_fxsr_env[0], fx, xstate_size))
623 return -1;
624 return 1;
625}
626
627static int save_i387_xsave(void __user *buf)
628{
629 struct task_struct *tsk = current;
630 struct _fpstate_ia32 __user *fx = buf;
631 int err = 0;
632
633
634 sanitize_i387_state(tsk);
635
636 /*
637 * For legacy compatible, we always set FP/SSE bits in the bit
638 * vector while saving the state to the user context.
639 * This will enable us capturing any changes(during sigreturn) to
640 * the FP/SSE bits by the legacy applications which don't touch
641 * xstate_bv in the xsave header.
642 *
643 * xsave aware applications can change the xstate_bv in the xsave
644 * header as well as change any contents in the memory layout.
645 * xrestore as part of sigreturn will capture all the changes.
646 */
647 tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
648
649 if (save_i387_fxsave(fx) < 0)
650 return -1;
651
652 err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved_ia32,
653 sizeof(struct _fpx_sw_bytes));
654 err |= __put_user(FP_XSTATE_MAGIC2,
655 (__u32 __user *) (buf + sig_xstate_ia32_size
656 - FP_XSTATE_MAGIC2_SIZE));
657 if (err)
658 return -1;
659
660 return 1;
661}
662
663int save_i387_xstate_ia32(void __user *buf)
664{
665 struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
666 struct task_struct *tsk = current;
667
668 if (!used_math())
669 return 0;
670
671 if (!access_ok(VERIFY_WRITE, buf, sig_xstate_ia32_size))
672 return -EACCES;
673 /*
674 * This will cause a "finit" to be triggered by the next
675 * attempted FPU operation by the 'current' process.
676 */
677 clear_used_math();
678
679 if (!HAVE_HWFP) {
680 return fpregs_soft_get(current, NULL,
681 0, sizeof(struct user_i387_ia32_struct),
682 NULL, fp) ? -1 : 1;
683 }
684
685 unlazy_fpu(tsk);
686
687 if (cpu_has_xsave)
688 return save_i387_xsave(fp);
689 if (cpu_has_fxsr)
690 return save_i387_fxsave(fp);
691 else
692 return save_i387_fsave(fp);
693}
694
695static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
696{
697 struct task_struct *tsk = current;
698
699 return __copy_from_user(&tsk->thread.fpu.state->fsave, buf,
700 sizeof(struct i387_fsave_struct));
701}
702
703static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf,
704 unsigned int size)
705{
706 struct task_struct *tsk = current;
707 struct user_i387_ia32_struct env;
708 int err;
709
710 err = __copy_from_user(&tsk->thread.fpu.state->fxsave, &buf->_fxsr_env[0],
711 size);
712 /* mxcsr reserved bits must be masked to zero for security reasons */
713 tsk->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
714 if (err || __copy_from_user(&env, buf, sizeof(env)))
715 return 1;
716 convert_to_fxsr(tsk, &env);
717
718 return 0;
719}
720
721static int restore_i387_xsave(void __user *buf)
722{
723 struct _fpx_sw_bytes fx_sw_user;
724 struct _fpstate_ia32 __user *fx_user =
725 ((struct _fpstate_ia32 __user *) buf);
726 struct i387_fxsave_struct __user *fx =
727 (struct i387_fxsave_struct __user *) &fx_user->_fxsr_env[0];
728 struct xsave_hdr_struct *xsave_hdr =
729 &current->thread.fpu.state->xsave.xsave_hdr;
730 u64 mask;
731 int err;
732
733 if (check_for_xstate(fx, buf, &fx_sw_user))
734 goto fx_only;
735
736 mask = fx_sw_user.xstate_bv;
737
738 err = restore_i387_fxsave(buf, fx_sw_user.xstate_size);
739
740 xsave_hdr->xstate_bv &= pcntxt_mask;
741 /*
742 * These bits must be zero.
743 */
744 xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
745
746 /*
747 * Init the state that is not present in the memory layout
748 * and enabled by the OS.
749 */
750 mask = ~(pcntxt_mask & ~mask);
751 xsave_hdr->xstate_bv &= mask;
752
753 return err;
754fx_only:
755 /*
756 * Couldn't find the extended state information in the memory
757 * layout. Restore the FP/SSE and init the other extended state
758 * enabled by the OS.
759 */
760 xsave_hdr->xstate_bv = XSTATE_FPSSE;
761 return restore_i387_fxsave(buf, sizeof(struct i387_fxsave_struct));
762}
763
764int restore_i387_xstate_ia32(void __user *buf)
765{
766 int err;
767 struct task_struct *tsk = current;
768 struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
769
770 if (HAVE_HWFP)
771 clear_fpu(tsk);
772
773 if (!buf) {
774 if (used_math()) {
775 clear_fpu(tsk);
776 clear_used_math();
777 }
778
779 return 0;
780 } else
781 if (!access_ok(VERIFY_READ, buf, sig_xstate_ia32_size))
782 return -EACCES;
783
784 if (!used_math()) {
785 err = init_fpu(tsk);
786 if (err)
787 return err;
788 }
789
790 if (HAVE_HWFP) {
791 if (cpu_has_xsave)
792 err = restore_i387_xsave(buf);
793 else if (cpu_has_fxsr)
794 err = restore_i387_fxsave(fp, sizeof(struct
795 i387_fxsave_struct));
796 else
797 err = restore_i387_fsave(fp);
798 } else {
799 err = fpregs_soft_set(current, NULL,
800 0, sizeof(struct user_i387_ia32_struct),
801 NULL, fp) != 0;
802 }
803 set_used_math();
804
805 return err;
806}
807
808/*
809 * FPU state for core dumps. 573 * FPU state for core dumps.
810 * This is only used for a.out dumps now. 574 * This is only used for a.out dumps now.
811 * It is declared generically using elf_fpregset_t (which is 575 * It is declared generically using elf_fpregset_t (which is
diff --git a/arch/x86/kernel/probe_roms.c b/arch/x86/kernel/probe_roms.c
index 0bc72e2069e3..d5f15c3f7b25 100644
--- a/arch/x86/kernel/probe_roms.c
+++ b/arch/x86/kernel/probe_roms.c
@@ -150,7 +150,7 @@ static struct resource *find_oprom(struct pci_dev *pdev)
150 return oprom; 150 return oprom;
151} 151}
152 152
153void *pci_map_biosrom(struct pci_dev *pdev) 153void __iomem *pci_map_biosrom(struct pci_dev *pdev)
154{ 154{
155 struct resource *oprom = find_oprom(pdev); 155 struct resource *oprom = find_oprom(pdev);
156 156
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index ef6a8456f719..dc3567e083f9 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -66,15 +66,13 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
66{ 66{
67 int ret; 67 int ret;
68 68
69 unlazy_fpu(src);
70
71 *dst = *src; 69 *dst = *src;
72 if (fpu_allocated(&src->thread.fpu)) { 70 if (fpu_allocated(&src->thread.fpu)) {
73 memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu)); 71 memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu));
74 ret = fpu_alloc(&dst->thread.fpu); 72 ret = fpu_alloc(&dst->thread.fpu);
75 if (ret) 73 if (ret)
76 return ret; 74 return ret;
77 fpu_copy(&dst->thread.fpu, &src->thread.fpu); 75 fpu_copy(dst, src);
78 } 76 }
79 return 0; 77 return 0;
80} 78}
@@ -97,16 +95,6 @@ void arch_task_cache_init(void)
97 SLAB_PANIC | SLAB_NOTRACK, NULL); 95 SLAB_PANIC | SLAB_NOTRACK, NULL);
98} 96}
99 97
100static inline void drop_fpu(struct task_struct *tsk)
101{
102 /*
103 * Forget coprocessor state..
104 */
105 tsk->fpu_counter = 0;
106 clear_fpu(tsk);
107 clear_used_math();
108}
109
110/* 98/*
111 * Free current thread data structures etc.. 99 * Free current thread data structures etc..
112 */ 100 */
@@ -163,7 +151,13 @@ void flush_thread(void)
163 151
164 flush_ptrace_hw_breakpoint(tsk); 152 flush_ptrace_hw_breakpoint(tsk);
165 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); 153 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
166 drop_fpu(tsk); 154 drop_init_fpu(tsk);
155 /*
156 * Free the FPU state for non xsave platforms. They get reallocated
157 * lazily at the first use.
158 */
159 if (!use_eager_fpu())
160 free_thread_xstate(tsk);
167} 161}
168 162
169static void hard_disable_TSC(void) 163static void hard_disable_TSC(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 516fa186121b..b9ff83c7135b 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -190,10 +190,6 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
190 regs->cs = __USER_CS; 190 regs->cs = __USER_CS;
191 regs->ip = new_ip; 191 regs->ip = new_ip;
192 regs->sp = new_sp; 192 regs->sp = new_sp;
193 /*
194 * Free the old FP and other extended state
195 */
196 free_thread_xstate(current);
197} 193}
198EXPORT_SYMBOL_GPL(start_thread); 194EXPORT_SYMBOL_GPL(start_thread);
199 195
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 0a980c9d7cb8..8a6d20ce1978 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -232,10 +232,6 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
232 regs->cs = _cs; 232 regs->cs = _cs;
233 regs->ss = _ss; 233 regs->ss = _ss;
234 regs->flags = X86_EFLAGS_IF; 234 regs->flags = X86_EFLAGS_IF;
235 /*
236 * Free the old FP and other extended state
237 */
238 free_thread_xstate(current);
239} 235}
240 236
241void 237void
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index c4c6a5c2bf0f..861a9d1a463d 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -1332,9 +1332,6 @@ static const struct user_regset_view user_x86_64_view = {
1332#define genregs32_get genregs_get 1332#define genregs32_get genregs_get
1333#define genregs32_set genregs_set 1333#define genregs32_set genregs_set
1334 1334
1335#define user_i387_ia32_struct user_i387_struct
1336#define user32_fxsr_struct user_fxsr_struct
1337
1338#endif /* CONFIG_X86_64 */ 1335#endif /* CONFIG_X86_64 */
1339 1336
1340#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION 1337#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index 932612887e92..036bddb46236 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -118,7 +118,7 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
118 get_user_ex(*pax, &sc->ax); 118 get_user_ex(*pax, &sc->ax);
119 } get_user_catch(err); 119 } get_user_catch(err);
120 120
121 err |= restore_i387_xstate(buf); 121 err |= restore_xstate_sig(buf, config_enabled(CONFIG_X86_32));
122 122
123 return err; 123 return err;
124} 124}
@@ -207,35 +207,32 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
207 void __user **fpstate) 207 void __user **fpstate)
208{ 208{
209 /* Default to using normal stack */ 209 /* Default to using normal stack */
210 unsigned long math_size = 0;
210 unsigned long sp = regs->sp; 211 unsigned long sp = regs->sp;
212 unsigned long buf_fx = 0;
211 int onsigstack = on_sig_stack(sp); 213 int onsigstack = on_sig_stack(sp);
212 214
213#ifdef CONFIG_X86_64
214 /* redzone */ 215 /* redzone */
215 sp -= 128; 216 if (config_enabled(CONFIG_X86_64))
216#endif /* CONFIG_X86_64 */ 217 sp -= 128;
217 218
218 if (!onsigstack) { 219 if (!onsigstack) {
219 /* This is the X/Open sanctioned signal stack switching. */ 220 /* This is the X/Open sanctioned signal stack switching. */
220 if (ka->sa.sa_flags & SA_ONSTACK) { 221 if (ka->sa.sa_flags & SA_ONSTACK) {
221 if (current->sas_ss_size) 222 if (current->sas_ss_size)
222 sp = current->sas_ss_sp + current->sas_ss_size; 223 sp = current->sas_ss_sp + current->sas_ss_size;
223 } else { 224 } else if (config_enabled(CONFIG_X86_32) &&
224#ifdef CONFIG_X86_32 225 (regs->ss & 0xffff) != __USER_DS &&
225 /* This is the legacy signal stack switching. */ 226 !(ka->sa.sa_flags & SA_RESTORER) &&
226 if ((regs->ss & 0xffff) != __USER_DS && 227 ka->sa.sa_restorer) {
227 !(ka->sa.sa_flags & SA_RESTORER) && 228 /* This is the legacy signal stack switching. */
228 ka->sa.sa_restorer)
229 sp = (unsigned long) ka->sa.sa_restorer; 229 sp = (unsigned long) ka->sa.sa_restorer;
230#endif /* CONFIG_X86_32 */
231 } 230 }
232 } 231 }
233 232
234 if (used_math()) { 233 if (used_math()) {
235 sp -= sig_xstate_size; 234 sp = alloc_mathframe(sp, config_enabled(CONFIG_X86_32),
236#ifdef CONFIG_X86_64 235 &buf_fx, &math_size);
237 sp = round_down(sp, 64);
238#endif /* CONFIG_X86_64 */
239 *fpstate = (void __user *)sp; 236 *fpstate = (void __user *)sp;
240 } 237 }
241 238
@@ -248,8 +245,9 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
248 if (onsigstack && !likely(on_sig_stack(sp))) 245 if (onsigstack && !likely(on_sig_stack(sp)))
249 return (void __user *)-1L; 246 return (void __user *)-1L;
250 247
251 /* save i387 state */ 248 /* save i387 and extended state */
252 if (used_math() && save_i387_xstate(*fpstate) < 0) 249 if (used_math() &&
250 save_xstate_sig(*fpstate, (void __user *)buf_fx, math_size) < 0)
253 return (void __user *)-1L; 251 return (void __user *)-1L;
254 252
255 return (void __user *)sp; 253 return (void __user *)sp;
@@ -385,7 +383,7 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
385 */ 383 */
386 put_user_ex(*((u64 *)&rt_retcode), (u64 *)frame->retcode); 384 put_user_ex(*((u64 *)&rt_retcode), (u64 *)frame->retcode);
387 } put_user_catch(err); 385 } put_user_catch(err);
388 386
389 err |= copy_siginfo_to_user(&frame->info, info); 387 err |= copy_siginfo_to_user(&frame->info, info);
390 err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate, 388 err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
391 regs, set->sig[0]); 389 regs, set->sig[0]);
@@ -477,6 +475,75 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
477} 475}
478#endif /* CONFIG_X86_32 */ 476#endif /* CONFIG_X86_32 */
479 477
478static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
479 siginfo_t *info, compat_sigset_t *set,
480 struct pt_regs *regs)
481{
482#ifdef CONFIG_X86_X32_ABI
483 struct rt_sigframe_x32 __user *frame;
484 void __user *restorer;
485 int err = 0;
486 void __user *fpstate = NULL;
487
488 frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
489
490 if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
491 return -EFAULT;
492
493 if (ka->sa.sa_flags & SA_SIGINFO) {
494 if (copy_siginfo_to_user32(&frame->info, info))
495 return -EFAULT;
496 }
497
498 put_user_try {
499 /* Create the ucontext. */
500 if (cpu_has_xsave)
501 put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
502 else
503 put_user_ex(0, &frame->uc.uc_flags);
504 put_user_ex(0, &frame->uc.uc_link);
505 put_user_ex(current->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
506 put_user_ex(sas_ss_flags(regs->sp),
507 &frame->uc.uc_stack.ss_flags);
508 put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
509 put_user_ex(0, &frame->uc.uc__pad0);
510
511 if (ka->sa.sa_flags & SA_RESTORER) {
512 restorer = ka->sa.sa_restorer;
513 } else {
514 /* could use a vstub here */
515 restorer = NULL;
516 err |= -EFAULT;
517 }
518 put_user_ex(restorer, &frame->pretcode);
519 } put_user_catch(err);
520
521 err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
522 regs, set->sig[0]);
523 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
524
525 if (err)
526 return -EFAULT;
527
528 /* Set up registers for signal handler */
529 regs->sp = (unsigned long) frame;
530 regs->ip = (unsigned long) ka->sa.sa_handler;
531
532 /* We use the x32 calling convention here... */
533 regs->di = sig;
534 regs->si = (unsigned long) &frame->info;
535 regs->dx = (unsigned long) &frame->uc;
536
537 loadsegment(ds, __USER_DS);
538 loadsegment(es, __USER_DS);
539
540 regs->cs = __USER_CS;
541 regs->ss = __USER_DS;
542#endif /* CONFIG_X86_X32_ABI */
543
544 return 0;
545}
546
480#ifdef CONFIG_X86_32 547#ifdef CONFIG_X86_32
481/* 548/*
482 * Atomically swap in the new signal mask, and wait for a signal. 549 * Atomically swap in the new signal mask, and wait for a signal.
@@ -615,55 +682,22 @@ static int signr_convert(int sig)
615 return sig; 682 return sig;
616} 683}
617 684
618#ifdef CONFIG_X86_32
619
620#define is_ia32 1
621#define ia32_setup_frame __setup_frame
622#define ia32_setup_rt_frame __setup_rt_frame
623
624#else /* !CONFIG_X86_32 */
625
626#ifdef CONFIG_IA32_EMULATION
627#define is_ia32 test_thread_flag(TIF_IA32)
628#else /* !CONFIG_IA32_EMULATION */
629#define is_ia32 0
630#endif /* CONFIG_IA32_EMULATION */
631
632#ifdef CONFIG_X86_X32_ABI
633#define is_x32 test_thread_flag(TIF_X32)
634
635static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
636 siginfo_t *info, compat_sigset_t *set,
637 struct pt_regs *regs);
638#else /* !CONFIG_X86_X32_ABI */
639#define is_x32 0
640#endif /* CONFIG_X86_X32_ABI */
641
642int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
643 sigset_t *set, struct pt_regs *regs);
644int ia32_setup_frame(int sig, struct k_sigaction *ka,
645 sigset_t *set, struct pt_regs *regs);
646
647#endif /* CONFIG_X86_32 */
648
649static int 685static int
650setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info, 686setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
651 struct pt_regs *regs) 687 struct pt_regs *regs)
652{ 688{
653 int usig = signr_convert(sig); 689 int usig = signr_convert(sig);
654 sigset_t *set = sigmask_to_save(); 690 sigset_t *set = sigmask_to_save();
691 compat_sigset_t *cset = (compat_sigset_t *) set;
655 692
656 /* Set up the stack frame */ 693 /* Set up the stack frame */
657 if (is_ia32) { 694 if (is_ia32_frame()) {
658 if (ka->sa.sa_flags & SA_SIGINFO) 695 if (ka->sa.sa_flags & SA_SIGINFO)
659 return ia32_setup_rt_frame(usig, ka, info, set, regs); 696 return ia32_setup_rt_frame(usig, ka, info, cset, regs);
660 else 697 else
661 return ia32_setup_frame(usig, ka, set, regs); 698 return ia32_setup_frame(usig, ka, cset, regs);
662#ifdef CONFIG_X86_X32_ABI 699 } else if (is_x32_frame()) {
663 } else if (is_x32) { 700 return x32_setup_rt_frame(usig, ka, info, cset, regs);
664 return x32_setup_rt_frame(usig, ka, info,
665 (compat_sigset_t *)set, regs);
666#endif
667 } else { 701 } else {
668 return __setup_rt_frame(sig, ka, info, set, regs); 702 return __setup_rt_frame(sig, ka, info, set, regs);
669 } 703 }
@@ -827,73 +861,6 @@ void signal_fault(struct pt_regs *regs, void __user *frame, char *where)
827} 861}
828 862
829#ifdef CONFIG_X86_X32_ABI 863#ifdef CONFIG_X86_X32_ABI
830static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
831 siginfo_t *info, compat_sigset_t *set,
832 struct pt_regs *regs)
833{
834 struct rt_sigframe_x32 __user *frame;
835 void __user *restorer;
836 int err = 0;
837 void __user *fpstate = NULL;
838
839 frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
840
841 if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
842 return -EFAULT;
843
844 if (ka->sa.sa_flags & SA_SIGINFO) {
845 if (copy_siginfo_to_user32(&frame->info, info))
846 return -EFAULT;
847 }
848
849 put_user_try {
850 /* Create the ucontext. */
851 if (cpu_has_xsave)
852 put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
853 else
854 put_user_ex(0, &frame->uc.uc_flags);
855 put_user_ex(0, &frame->uc.uc_link);
856 put_user_ex(current->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
857 put_user_ex(sas_ss_flags(regs->sp),
858 &frame->uc.uc_stack.ss_flags);
859 put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
860 put_user_ex(0, &frame->uc.uc__pad0);
861
862 if (ka->sa.sa_flags & SA_RESTORER) {
863 restorer = ka->sa.sa_restorer;
864 } else {
865 /* could use a vstub here */
866 restorer = NULL;
867 err |= -EFAULT;
868 }
869 put_user_ex(restorer, &frame->pretcode);
870 } put_user_catch(err);
871
872 err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
873 regs, set->sig[0]);
874 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
875
876 if (err)
877 return -EFAULT;
878
879 /* Set up registers for signal handler */
880 regs->sp = (unsigned long) frame;
881 regs->ip = (unsigned long) ka->sa.sa_handler;
882
883 /* We use the x32 calling convention here... */
884 regs->di = sig;
885 regs->si = (unsigned long) &frame->info;
886 regs->dx = (unsigned long) &frame->uc;
887
888 loadsegment(ds, __USER_DS);
889 loadsegment(es, __USER_DS);
890
891 regs->cs = __USER_CS;
892 regs->ss = __USER_DS;
893
894 return 0;
895}
896
897asmlinkage long sys32_x32_rt_sigreturn(struct pt_regs *regs) 864asmlinkage long sys32_x32_rt_sigreturn(struct pt_regs *regs)
898{ 865{
899 struct rt_sigframe_x32 __user *frame; 866 struct rt_sigframe_x32 __user *frame;
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index b481341c9369..4f4aba0551b0 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -613,11 +613,12 @@ void math_state_restore(void)
613 } 613 }
614 614
615 __thread_fpu_begin(tsk); 615 __thread_fpu_begin(tsk);
616
616 /* 617 /*
617 * Paranoid restore. send a SIGSEGV if we fail to restore the state. 618 * Paranoid restore. send a SIGSEGV if we fail to restore the state.
618 */ 619 */
619 if (unlikely(restore_fpu_checking(tsk))) { 620 if (unlikely(restore_fpu_checking(tsk))) {
620 __thread_fpu_end(tsk); 621 drop_init_fpu(tsk);
621 force_sig(SIGSEGV, tsk); 622 force_sig(SIGSEGV, tsk);
622 return; 623 return;
623 } 624 }
@@ -629,6 +630,8 @@ EXPORT_SYMBOL_GPL(math_state_restore);
629dotraplinkage void __kprobes 630dotraplinkage void __kprobes
630do_device_not_available(struct pt_regs *regs, long error_code) 631do_device_not_available(struct pt_regs *regs, long error_code)
631{ 632{
633 BUG_ON(use_eager_fpu());
634
632#ifdef CONFIG_MATH_EMULATION 635#ifdef CONFIG_MATH_EMULATION
633 if (read_cr0() & X86_CR0_EM) { 636 if (read_cr0() & X86_CR0_EM) {
634 struct math_emu_info info = { }; 637 struct math_emu_info info = { };
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index 3d3e20709119..4e89b3dd408d 100644
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -10,9 +10,7 @@
10#include <linux/compat.h> 10#include <linux/compat.h>
11#include <asm/i387.h> 11#include <asm/i387.h>
12#include <asm/fpu-internal.h> 12#include <asm/fpu-internal.h>
13#ifdef CONFIG_IA32_EMULATION 13#include <asm/sigframe.h>
14#include <asm/sigcontext32.h>
15#endif
16#include <asm/xcr.h> 14#include <asm/xcr.h>
17 15
18/* 16/*
@@ -23,13 +21,9 @@ u64 pcntxt_mask;
23/* 21/*
24 * Represents init state for the supported extended state. 22 * Represents init state for the supported extended state.
25 */ 23 */
26static struct xsave_struct *init_xstate_buf; 24struct xsave_struct *init_xstate_buf;
27
28struct _fpx_sw_bytes fx_sw_reserved;
29#ifdef CONFIG_IA32_EMULATION
30struct _fpx_sw_bytes fx_sw_reserved_ia32;
31#endif
32 25
26static struct _fpx_sw_bytes fx_sw_reserved, fx_sw_reserved_ia32;
33static unsigned int *xstate_offsets, *xstate_sizes, xstate_features; 27static unsigned int *xstate_offsets, *xstate_sizes, xstate_features;
34 28
35/* 29/*
@@ -44,9 +38,9 @@ static unsigned int *xstate_offsets, *xstate_sizes, xstate_features;
44 */ 38 */
45void __sanitize_i387_state(struct task_struct *tsk) 39void __sanitize_i387_state(struct task_struct *tsk)
46{ 40{
47 u64 xstate_bv;
48 int feature_bit = 0x2;
49 struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave; 41 struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave;
42 int feature_bit = 0x2;
43 u64 xstate_bv;
50 44
51 if (!fx) 45 if (!fx)
52 return; 46 return;
@@ -104,213 +98,326 @@ void __sanitize_i387_state(struct task_struct *tsk)
104 * Check for the presence of extended state information in the 98 * Check for the presence of extended state information in the
105 * user fpstate pointer in the sigcontext. 99 * user fpstate pointer in the sigcontext.
106 */ 100 */
107int check_for_xstate(struct i387_fxsave_struct __user *buf, 101static inline int check_for_xstate(struct i387_fxsave_struct __user *buf,
108 void __user *fpstate, 102 void __user *fpstate,
109 struct _fpx_sw_bytes *fx_sw_user) 103 struct _fpx_sw_bytes *fx_sw)
110{ 104{
111 int min_xstate_size = sizeof(struct i387_fxsave_struct) + 105 int min_xstate_size = sizeof(struct i387_fxsave_struct) +
112 sizeof(struct xsave_hdr_struct); 106 sizeof(struct xsave_hdr_struct);
113 unsigned int magic2; 107 unsigned int magic2;
114 int err;
115 108
116 err = __copy_from_user(fx_sw_user, &buf->sw_reserved[0], 109 if (__copy_from_user(fx_sw, &buf->sw_reserved[0], sizeof(*fx_sw)))
117 sizeof(struct _fpx_sw_bytes)); 110 return -1;
118 if (err)
119 return -EFAULT;
120 111
121 /* 112 /* Check for the first magic field and other error scenarios. */
122 * First Magic check failed. 113 if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
123 */ 114 fx_sw->xstate_size < min_xstate_size ||
124 if (fx_sw_user->magic1 != FP_XSTATE_MAGIC1) 115 fx_sw->xstate_size > xstate_size ||
125 return -EINVAL; 116 fx_sw->xstate_size > fx_sw->extended_size)
117 return -1;
126 118
127 /* 119 /*
128 * Check for error scenarios.
129 */
130 if (fx_sw_user->xstate_size < min_xstate_size ||
131 fx_sw_user->xstate_size > xstate_size ||
132 fx_sw_user->xstate_size > fx_sw_user->extended_size)
133 return -EINVAL;
134
135 err = __get_user(magic2, (__u32 *) (((void *)fpstate) +
136 fx_sw_user->extended_size -
137 FP_XSTATE_MAGIC2_SIZE));
138 if (err)
139 return err;
140 /*
141 * Check for the presence of second magic word at the end of memory 120 * Check for the presence of second magic word at the end of memory
142 * layout. This detects the case where the user just copied the legacy 121 * layout. This detects the case where the user just copied the legacy
143 * fpstate layout with out copying the extended state information 122 * fpstate layout with out copying the extended state information
144 * in the memory layout. 123 * in the memory layout.
145 */ 124 */
146 if (magic2 != FP_XSTATE_MAGIC2) 125 if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size))
147 return -EFAULT; 126 || magic2 != FP_XSTATE_MAGIC2)
127 return -1;
148 128
149 return 0; 129 return 0;
150} 130}
151 131
152#ifdef CONFIG_X86_64
153/* 132/*
154 * Signal frame handlers. 133 * Signal frame handlers.
155 */ 134 */
156 135static inline int save_fsave_header(struct task_struct *tsk, void __user *buf)
157int save_i387_xstate(void __user *buf)
158{ 136{
159 struct task_struct *tsk = current; 137 if (use_fxsr()) {
160 int err = 0; 138 struct xsave_struct *xsave = &tsk->thread.fpu.state->xsave;
161 139 struct user_i387_ia32_struct env;
162 if (!access_ok(VERIFY_WRITE, buf, sig_xstate_size)) 140 struct _fpstate_ia32 __user *fp = buf;
163 return -EACCES;
164 141
165 BUG_ON(sig_xstate_size < xstate_size); 142 convert_from_fxsr(&env, tsk);
166 143
167 if ((unsigned long)buf % 64) 144 if (__copy_to_user(buf, &env, sizeof(env)) ||
168 pr_err("%s: bad fpstate %p\n", __func__, buf); 145 __put_user(xsave->i387.swd, &fp->status) ||
169 146 __put_user(X86_FXSR_MAGIC, &fp->magic))
170 if (!used_math()) 147 return -1;
171 return 0;
172
173 if (user_has_fpu()) {
174 if (use_xsave())
175 err = xsave_user(buf);
176 else
177 err = fxsave_user(buf);
178
179 if (err)
180 return err;
181 user_fpu_end();
182 } else { 148 } else {
183 sanitize_i387_state(tsk); 149 struct i387_fsave_struct __user *fp = buf;
184 if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave, 150 u32 swd;
185 xstate_size)) 151 if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status))
186 return -1; 152 return -1;
187 } 153 }
188 154
189 clear_used_math(); /* trigger finit */ 155 return 0;
156}
190 157
191 if (use_xsave()) { 158static inline int save_xstate_epilog(void __user *buf, int ia32_frame)
192 struct _fpstate __user *fx = buf; 159{
193 struct _xstate __user *x = buf; 160 struct xsave_struct __user *x = buf;
194 u64 xstate_bv; 161 struct _fpx_sw_bytes *sw_bytes;
162 u32 xstate_bv;
163 int err;
195 164
196 err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved, 165 /* Setup the bytes not touched by the [f]xsave and reserved for SW. */
197 sizeof(struct _fpx_sw_bytes)); 166 sw_bytes = ia32_frame ? &fx_sw_reserved_ia32 : &fx_sw_reserved;
167 err = __copy_to_user(&x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes));
198 168
199 err |= __put_user(FP_XSTATE_MAGIC2, 169 if (!use_xsave())
200 (__u32 __user *) (buf + sig_xstate_size 170 return err;
201 - FP_XSTATE_MAGIC2_SIZE));
202 171
203 /* 172 err |= __put_user(FP_XSTATE_MAGIC2, (__u32 *)(buf + xstate_size));
204 * Read the xstate_bv which we copied (directly from the cpu or
205 * from the state in task struct) to the user buffers and
206 * set the FP/SSE bits.
207 */
208 err |= __get_user(xstate_bv, &x->xstate_hdr.xstate_bv);
209 173
210 /* 174 /*
211 * For legacy compatible, we always set FP/SSE bits in the bit 175 * Read the xstate_bv which we copied (directly from the cpu or
212 * vector while saving the state to the user context. This will 176 * from the state in task struct) to the user buffers.
213 * enable us capturing any changes(during sigreturn) to 177 */
214 * the FP/SSE bits by the legacy applications which don't touch 178 err |= __get_user(xstate_bv, (__u32 *)&x->xsave_hdr.xstate_bv);
215 * xstate_bv in the xsave header.
216 *
217 * xsave aware apps can change the xstate_bv in the xsave
218 * header as well as change any contents in the memory layout.
219 * xrestore as part of sigreturn will capture all the changes.
220 */
221 xstate_bv |= XSTATE_FPSSE;
222 179
223 err |= __put_user(xstate_bv, &x->xstate_hdr.xstate_bv); 180 /*
181 * For legacy compatible, we always set FP/SSE bits in the bit
182 * vector while saving the state to the user context. This will
183 * enable us capturing any changes(during sigreturn) to
184 * the FP/SSE bits by the legacy applications which don't touch
185 * xstate_bv in the xsave header.
186 *
187 * xsave aware apps can change the xstate_bv in the xsave
188 * header as well as change any contents in the memory layout.
189 * xrestore as part of sigreturn will capture all the changes.
190 */
191 xstate_bv |= XSTATE_FPSSE;
224 192
225 if (err) 193 err |= __put_user(xstate_bv, (__u32 *)&x->xsave_hdr.xstate_bv);
226 return err;
227 }
228 194
229 return 1; 195 return err;
196}
197
198static inline int save_user_xstate(struct xsave_struct __user *buf)
199{
200 int err;
201
202 if (use_xsave())
203 err = xsave_user(buf);
204 else if (use_fxsr())
205 err = fxsave_user((struct i387_fxsave_struct __user *) buf);
206 else
207 err = fsave_user((struct i387_fsave_struct __user *) buf);
208
209 if (unlikely(err) && __clear_user(buf, xstate_size))
210 err = -EFAULT;
211 return err;
230} 212}
231 213
232/* 214/*
233 * Restore the extended state if present. Otherwise, restore the FP/SSE 215 * Save the fpu, extended register state to the user signal frame.
234 * state. 216 *
217 * 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save
218 * state is copied.
219 * 'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'.
220 *
221 * buf == buf_fx for 64-bit frames and 32-bit fsave frame.
222 * buf != buf_fx for 32-bit frames with fxstate.
223 *
224 * If the fpu, extended register state is live, save the state directly
225 * to the user frame pointed by the aligned pointer 'buf_fx'. Otherwise,
226 * copy the thread's fpu state to the user frame starting at 'buf_fx'.
227 *
228 * If this is a 32-bit frame with fxstate, put a fsave header before
229 * the aligned state at 'buf_fx'.
230 *
231 * For [f]xsave state, update the SW reserved fields in the [f]xsave frame
232 * indicating the absence/presence of the extended state to the user.
235 */ 233 */
236static int restore_user_xstate(void __user *buf) 234int save_xstate_sig(void __user *buf, void __user *buf_fx, int size)
237{ 235{
238 struct _fpx_sw_bytes fx_sw_user; 236 struct xsave_struct *xsave = &current->thread.fpu.state->xsave;
239 u64 mask; 237 struct task_struct *tsk = current;
240 int err; 238 int ia32_fxstate = (buf != buf_fx);
241 239
242 if (((unsigned long)buf % 64) || 240 ia32_fxstate &= (config_enabled(CONFIG_X86_32) ||
243 check_for_xstate(buf, buf, &fx_sw_user)) 241 config_enabled(CONFIG_IA32_EMULATION));
244 goto fx_only;
245 242
246 mask = fx_sw_user.xstate_bv; 243 if (!access_ok(VERIFY_WRITE, buf, size))
244 return -EACCES;
247 245
248 /* 246 if (!HAVE_HWFP)
249 * restore the state passed by the user. 247 return fpregs_soft_get(current, NULL, 0,
250 */ 248 sizeof(struct user_i387_ia32_struct), NULL,
251 err = xrestore_user(buf, mask); 249 (struct _fpstate_ia32 __user *) buf) ? -1 : 1;
252 if (err)
253 return err;
254 250
255 /* 251 if (user_has_fpu()) {
256 * init the state skipped by the user. 252 /* Save the live register state to the user directly. */
257 */ 253 if (save_user_xstate(buf_fx))
258 mask = pcntxt_mask & ~mask; 254 return -1;
259 if (unlikely(mask)) 255 /* Update the thread's fxstate to save the fsave header. */
260 xrstor_state(init_xstate_buf, mask); 256 if (ia32_fxstate)
257 fpu_fxsave(&tsk->thread.fpu);
258 } else {
259 sanitize_i387_state(tsk);
260 if (__copy_to_user(buf_fx, xsave, xstate_size))
261 return -1;
262 }
263
264 /* Save the fsave header for the 32-bit frames. */
265 if ((ia32_fxstate || !use_fxsr()) && save_fsave_header(tsk, buf))
266 return -1;
267
268 if (use_fxsr() && save_xstate_epilog(buf_fx, ia32_fxstate))
269 return -1;
270
271 drop_init_fpu(tsk); /* trigger finit */
261 272
262 return 0; 273 return 0;
274}
263 275
264fx_only: 276static inline void
265 /* 277sanitize_restored_xstate(struct task_struct *tsk,
266 * couldn't find the extended state information in the 278 struct user_i387_ia32_struct *ia32_env,
267 * memory layout. Restore just the FP/SSE and init all 279 u64 xstate_bv, int fx_only)
268 * the other extended state. 280{
269 */ 281 struct xsave_struct *xsave = &tsk->thread.fpu.state->xsave;
270 xrstor_state(init_xstate_buf, pcntxt_mask & ~XSTATE_FPSSE); 282 struct xsave_hdr_struct *xsave_hdr = &xsave->xsave_hdr;
271 return fxrstor_checking((__force struct i387_fxsave_struct *)buf); 283
284 if (use_xsave()) {
285 /* These bits must be zero. */
286 xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
287
288 /*
289 * Init the state that is not present in the memory
290 * layout and not enabled by the OS.
291 */
292 if (fx_only)
293 xsave_hdr->xstate_bv = XSTATE_FPSSE;
294 else
295 xsave_hdr->xstate_bv &= (pcntxt_mask & xstate_bv);
296 }
297
298 if (use_fxsr()) {
299 /*
300 * mscsr reserved bits must be masked to zero for security
301 * reasons.
302 */
303 xsave->i387.mxcsr &= mxcsr_feature_mask;
304
305 convert_to_fxsr(tsk, ia32_env);
306 }
272} 307}
273 308
274/* 309/*
275 * This restores directly out of user space. Exceptions are handled. 310 * Restore the extended state if present. Otherwise, restore the FP/SSE state.
276 */ 311 */
277int restore_i387_xstate(void __user *buf) 312static inline int restore_user_xstate(void __user *buf, u64 xbv, int fx_only)
278{ 313{
314 if (use_xsave()) {
315 if ((unsigned long)buf % 64 || fx_only) {
316 u64 init_bv = pcntxt_mask & ~XSTATE_FPSSE;
317 xrstor_state(init_xstate_buf, init_bv);
318 return fxrstor_checking((__force void *) buf);
319 } else {
320 u64 init_bv = pcntxt_mask & ~xbv;
321 if (unlikely(init_bv))
322 xrstor_state(init_xstate_buf, init_bv);
323 return xrestore_user(buf, xbv);
324 }
325 } else if (use_fxsr()) {
326 return fxrstor_checking((__force void *) buf);
327 } else
328 return frstor_checking((__force void *) buf);
329}
330
331int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size)
332{
333 int ia32_fxstate = (buf != buf_fx);
279 struct task_struct *tsk = current; 334 struct task_struct *tsk = current;
280 int err = 0; 335 int state_size = xstate_size;
336 u64 xstate_bv = 0;
337 int fx_only = 0;
338
339 ia32_fxstate &= (config_enabled(CONFIG_X86_32) ||
340 config_enabled(CONFIG_IA32_EMULATION));
281 341
282 if (!buf) { 342 if (!buf) {
283 if (used_math()) 343 drop_init_fpu(tsk);
284 goto clear;
285 return 0; 344 return 0;
286 } else 345 }
287 if (!access_ok(VERIFY_READ, buf, sig_xstate_size))
288 return -EACCES;
289 346
290 if (!used_math()) { 347 if (!access_ok(VERIFY_READ, buf, size))
291 err = init_fpu(tsk); 348 return -EACCES;
292 if (err) 349
293 return err; 350 if (!used_math() && init_fpu(tsk))
351 return -1;
352
353 if (!HAVE_HWFP) {
354 return fpregs_soft_set(current, NULL,
355 0, sizeof(struct user_i387_ia32_struct),
356 NULL, buf) != 0;
294 } 357 }
295 358
296 user_fpu_begin(); 359 if (use_xsave()) {
297 if (use_xsave()) 360 struct _fpx_sw_bytes fx_sw_user;
298 err = restore_user_xstate(buf); 361 if (unlikely(check_for_xstate(buf_fx, buf_fx, &fx_sw_user))) {
299 else 362 /*
300 err = fxrstor_checking((__force struct i387_fxsave_struct *) 363 * Couldn't find the extended state information in the
301 buf); 364 * memory layout. Restore just the FP/SSE and init all
302 if (unlikely(err)) { 365 * the other extended state.
366 */
367 state_size = sizeof(struct i387_fxsave_struct);
368 fx_only = 1;
369 } else {
370 state_size = fx_sw_user.xstate_size;
371 xstate_bv = fx_sw_user.xstate_bv;
372 }
373 }
374
375 if (ia32_fxstate) {
376 /*
377 * For 32-bit frames with fxstate, copy the user state to the
378 * thread's fpu state, reconstruct fxstate from the fsave
379 * header. Sanitize the copied state etc.
380 */
381 struct xsave_struct *xsave = &tsk->thread.fpu.state->xsave;
382 struct user_i387_ia32_struct env;
383 int err = 0;
384
385 /*
386 * Drop the current fpu which clears used_math(). This ensures
387 * that any context-switch during the copy of the new state,
388 * avoids the intermediate state from getting restored/saved.
389 * Thus avoiding the new restored state from getting corrupted.
390 * We will be ready to restore/save the state only after
391 * set_used_math() is again set.
392 */
393 drop_fpu(tsk);
394
395 if (__copy_from_user(xsave, buf_fx, state_size) ||
396 __copy_from_user(&env, buf, sizeof(env))) {
397 err = -1;
398 } else {
399 sanitize_restored_xstate(tsk, &env, xstate_bv, fx_only);
400 set_used_math();
401 }
402
403 if (use_eager_fpu())
404 math_state_restore();
405
406 return err;
407 } else {
303 /* 408 /*
304 * Encountered an error while doing the restore from the 409 * For 64-bit frames and 32-bit fsave frames, restore the user
305 * user buffer, clear the fpu state. 410 * state to the registers directly (with exceptions handled).
306 */ 411 */
307clear: 412 user_fpu_begin();
308 clear_fpu(tsk); 413 if (restore_user_xstate(buf_fx, xstate_bv, fx_only)) {
309 clear_used_math(); 414 drop_init_fpu(tsk);
415 return -1;
416 }
310 } 417 }
311 return err; 418
419 return 0;
312} 420}
313#endif
314 421
315/* 422/*
316 * Prepare the SW reserved portion of the fxsave memory layout, indicating 423 * Prepare the SW reserved portion of the fxsave memory layout, indicating
@@ -321,31 +428,22 @@ clear:
321 */ 428 */
322static void prepare_fx_sw_frame(void) 429static void prepare_fx_sw_frame(void)
323{ 430{
324 int size_extended = (xstate_size - sizeof(struct i387_fxsave_struct)) + 431 int fsave_header_size = sizeof(struct i387_fsave_struct);
325 FP_XSTATE_MAGIC2_SIZE; 432 int size = xstate_size + FP_XSTATE_MAGIC2_SIZE;
326 433
327 sig_xstate_size = sizeof(struct _fpstate) + size_extended; 434 if (config_enabled(CONFIG_X86_32))
328 435 size += fsave_header_size;
329#ifdef CONFIG_IA32_EMULATION
330 sig_xstate_ia32_size = sizeof(struct _fpstate_ia32) + size_extended;
331#endif
332
333 memset(&fx_sw_reserved, 0, sizeof(fx_sw_reserved));
334 436
335 fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1; 437 fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
336 fx_sw_reserved.extended_size = sig_xstate_size; 438 fx_sw_reserved.extended_size = size;
337 fx_sw_reserved.xstate_bv = pcntxt_mask; 439 fx_sw_reserved.xstate_bv = pcntxt_mask;
338 fx_sw_reserved.xstate_size = xstate_size; 440 fx_sw_reserved.xstate_size = xstate_size;
339#ifdef CONFIG_IA32_EMULATION
340 memcpy(&fx_sw_reserved_ia32, &fx_sw_reserved,
341 sizeof(struct _fpx_sw_bytes));
342 fx_sw_reserved_ia32.extended_size = sig_xstate_ia32_size;
343#endif
344}
345 441
346#ifdef CONFIG_X86_64 442 if (config_enabled(CONFIG_IA32_EMULATION)) {
347unsigned int sig_xstate_size = sizeof(struct _fpstate); 443 fx_sw_reserved_ia32 = fx_sw_reserved;
348#endif 444 fx_sw_reserved_ia32.extended_size += fsave_header_size;
445 }
446}
349 447
350/* 448/*
351 * Enable the extended processor state save/restore feature 449 * Enable the extended processor state save/restore feature
@@ -384,19 +482,21 @@ static void __init setup_xstate_features(void)
384/* 482/*
385 * setup the xstate image representing the init state 483 * setup the xstate image representing the init state
386 */ 484 */
387static void __init setup_xstate_init(void) 485static void __init setup_init_fpu_buf(void)
388{ 486{
389 setup_xstate_features();
390
391 /* 487 /*
392 * Setup init_xstate_buf to represent the init state of 488 * Setup init_xstate_buf to represent the init state of
393 * all the features managed by the xsave 489 * all the features managed by the xsave
394 */ 490 */
395 init_xstate_buf = alloc_bootmem_align(xstate_size, 491 init_xstate_buf = alloc_bootmem_align(xstate_size,
396 __alignof__(struct xsave_struct)); 492 __alignof__(struct xsave_struct));
397 init_xstate_buf->i387.mxcsr = MXCSR_DEFAULT; 493 fx_finit(&init_xstate_buf->i387);
494
495 if (!cpu_has_xsave)
496 return;
497
498 setup_xstate_features();
398 499
399 clts();
400 /* 500 /*
401 * Init all the features state with header_bv being 0x0 501 * Init all the features state with header_bv being 0x0
402 */ 502 */
@@ -406,9 +506,21 @@ static void __init setup_xstate_init(void)
406 * of any feature which is not represented by all zero's. 506 * of any feature which is not represented by all zero's.
407 */ 507 */
408 xsave_state(init_xstate_buf, -1); 508 xsave_state(init_xstate_buf, -1);
409 stts();
410} 509}
411 510
511static enum { AUTO, ENABLE, DISABLE } eagerfpu = AUTO;
512static int __init eager_fpu_setup(char *s)
513{
514 if (!strcmp(s, "on"))
515 eagerfpu = ENABLE;
516 else if (!strcmp(s, "off"))
517 eagerfpu = DISABLE;
518 else if (!strcmp(s, "auto"))
519 eagerfpu = AUTO;
520 return 1;
521}
522__setup("eagerfpu=", eager_fpu_setup);
523
412/* 524/*
413 * Enable and initialize the xsave feature. 525 * Enable and initialize the xsave feature.
414 */ 526 */
@@ -445,8 +557,11 @@ static void __init xstate_enable_boot_cpu(void)
445 557
446 update_regset_xstate_info(xstate_size, pcntxt_mask); 558 update_regset_xstate_info(xstate_size, pcntxt_mask);
447 prepare_fx_sw_frame(); 559 prepare_fx_sw_frame();
560 setup_init_fpu_buf();
448 561
449 setup_xstate_init(); 562 /* Auto enable eagerfpu for xsaveopt */
563 if (cpu_has_xsaveopt && eagerfpu != DISABLE)
564 eagerfpu = ENABLE;
450 565
451 pr_info("enabled xstate_bv 0x%llx, cntxt size 0x%x\n", 566 pr_info("enabled xstate_bv 0x%llx, cntxt size 0x%x\n",
452 pcntxt_mask, xstate_size); 567 pcntxt_mask, xstate_size);
@@ -471,3 +586,43 @@ void __cpuinit xsave_init(void)
471 next_func = xstate_enable; 586 next_func = xstate_enable;
472 this_func(); 587 this_func();
473} 588}
589
590static inline void __init eager_fpu_init_bp(void)
591{
592 current->thread.fpu.state =
593 alloc_bootmem_align(xstate_size, __alignof__(struct xsave_struct));
594 if (!init_xstate_buf)
595 setup_init_fpu_buf();
596}
597
598void __cpuinit eager_fpu_init(void)
599{
600 static __refdata void (*boot_func)(void) = eager_fpu_init_bp;
601
602 clear_used_math();
603 current_thread_info()->status = 0;
604
605 if (eagerfpu == ENABLE)
606 setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
607
608 if (!cpu_has_eager_fpu) {
609 stts();
610 return;
611 }
612
613 if (boot_func) {
614 boot_func();
615 boot_func = NULL;
616 }
617
618 /*
619 * This is same as math_state_restore(). But use_xsave() is
620 * not yet patched to use math_state_restore().
621 */
622 init_fpu(current);
623 __thread_fpu_begin(current);
624 if (cpu_has_xsave)
625 xrstor_state(init_xstate_buf, -1);
626 else
627 fxrstor_checking(&init_xstate_buf->i387);
628}
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index c00f03de1b79..70dfcec3c463 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1493,8 +1493,12 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)
1493#ifdef CONFIG_X86_64 1493#ifdef CONFIG_X86_64
1494 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); 1494 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1495#endif 1495#endif
1496 if (user_has_fpu()) 1496 /*
1497 clts(); 1497 * If the FPU is not active (through the host task or
1498 * the guest vcpu), then restore the cr0.TS bit.
1499 */
1500 if (!user_has_fpu() && !vmx->vcpu.guest_fpu_loaded)
1501 stts();
1498 load_gdt(&__get_cpu_var(host_gdt)); 1502 load_gdt(&__get_cpu_var(host_gdt));
1499} 1503}
1500 1504
@@ -3730,7 +3734,7 @@ static void vmx_set_constant_host_state(void)
3730 unsigned long tmpl; 3734 unsigned long tmpl;
3731 struct desc_ptr dt; 3735 struct desc_ptr dt;
3732 3736
3733 vmcs_writel(HOST_CR0, read_cr0() | X86_CR0_TS); /* 22.2.3 */ 3737 vmcs_writel(HOST_CR0, read_cr0() & ~X86_CR0_TS); /* 22.2.3 */
3734 vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */ 3738 vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */
3735 vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */ 3739 vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
3736 3740
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 148ed666e311..02b2cd520693 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5972,7 +5972,7 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
5972 */ 5972 */
5973 kvm_put_guest_xcr0(vcpu); 5973 kvm_put_guest_xcr0(vcpu);
5974 vcpu->guest_fpu_loaded = 1; 5974 vcpu->guest_fpu_loaded = 1;
5975 unlazy_fpu(current); 5975 __kernel_fpu_begin();
5976 fpu_restore_checking(&vcpu->arch.guest_fpu); 5976 fpu_restore_checking(&vcpu->arch.guest_fpu);
5977 trace_kvm_fpu(1); 5977 trace_kvm_fpu(1);
5978} 5978}
@@ -5986,6 +5986,7 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
5986 5986
5987 vcpu->guest_fpu_loaded = 0; 5987 vcpu->guest_fpu_loaded = 0;
5988 fpu_save_init(&vcpu->arch.guest_fpu); 5988 fpu_save_init(&vcpu->arch.guest_fpu);
5989 __kernel_fpu_end();
5989 ++vcpu->stat.fpu_reload; 5990 ++vcpu->stat.fpu_reload;
5990 kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu); 5991 kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);
5991 trace_kvm_fpu(0); 5992 trace_kvm_fpu(0);
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-02 19:36:34 -0500