From 200b08a970b2ae764b670a326088ab8bc0a989cc Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:34 -0700 Subject: x86/xsaves: Define macros for xsave instructions Define macros for xsave, xsaveopt, xsaves, xrstor, and xrstors inline instructions. The instructions will be used for saving and restoring xstate. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-7-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/xsave.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index d949ef28c48b..71bdde45b519 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -52,6 +52,12 @@ extern void xsave_init(void); extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); extern int init_fpu(struct task_struct *child); +#define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27" +#define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37" +#define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f" +#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f" +#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" + static inline int fpu_xrstor_checking(struct xsave_struct *fx) { int err; -- cgit v1.2.2 From b84e70552e5aad71a1c14536e6ffcfe7934b73e4 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:35 -0700 Subject: x86/xsaves: Define a macro for handling xsave/xrstor instruction fault Define a macro to handle fault generated by xsave, xsaveopt, xsaves, xrstor, and xrstors instructions. It is used in functions like xsave_state() etc. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-8-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/xsave.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 71bdde45b519..76c2459188c8 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -58,6 +58,13 @@ extern int init_fpu(struct task_struct *child); #define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f" #define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" +#define xstate_fault ".section .fixup,\"ax\"\n" \ + "3: movl $-1,%[err]\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : [err] "=r" (err) + static inline int fpu_xrstor_checking(struct xsave_struct *fx) { int err; -- cgit v1.2.2 From f31a9f7c71691569359fa7fb8b0acaa44bce0324 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:36 -0700 Subject: x86/xsaves: Use xsaves/xrstors to save and restore xsave area If xsaves is eanbled, use xsaves/xrstors instrucitons to save and restore xstate. xsaves and xrstors support compacted format, init optimization, modified optimization, and supervisor states. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-9-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/xsave.h | 84 +++++++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 20 deletions(-) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 76c2459188c8..f9177a2a97e9 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -65,6 +65,70 @@ extern int init_fpu(struct task_struct *child); _ASM_EXTABLE(1b, 3b) \ : [err] "=r" (err) +/* + * Save processor xstate to xsave area. + */ +static inline int xsave_state(struct xsave_struct *fx, u64 mask) +{ + u32 lmask = mask; + u32 hmask = mask >> 32; + int err = 0; + + /* + * If xsaves is enabled, xsaves replaces xsaveopt because + * it supports compact format and supervisor states in addition to + * modified optimization in xsaveopt. + * + * Otherwise, if xsaveopt is enabled, xsaveopt replaces xsave + * because xsaveopt supports modified optimization which is not + * supported by xsave. + * + * If none of xsaves and xsaveopt is enabled, use xsave. + */ + alternative_input_2( + "1:"XSAVE, + "1:"XSAVEOPT, + X86_FEATURE_XSAVEOPT, + "1:"XSAVES, + X86_FEATURE_XSAVES, + [fx] "D" (fx), "a" (lmask), "d" (hmask) : + "memory"); + asm volatile("2:\n\t" + xstate_fault + : "0" (0) + : "memory"); + + return err; +} + +/* + * Restore processor xstate from xsave area. + */ +static inline int xrstor_state(struct xsave_struct *fx, u64 mask) +{ + int err = 0; + u32 lmask = mask; + u32 hmask = mask >> 32; + + /* + * Use xrstors to restore context if it is enabled. xrstors supports + * compacted format of xsave area which is not supported by xrstor. + */ + alternative_input( + "1: " XRSTOR, + "1: " XRSTORS, + X86_FEATURE_XSAVES, + "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + + asm volatile("2:\n" + xstate_fault + : "0" (0) + : "memory"); + + return err; +} + static inline int fpu_xrstor_checking(struct xsave_struct *fx) { int err; @@ -130,26 +194,6 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask) return err; } -static inline void xrstor_state(struct xsave_struct *fx, u64 mask) -{ - u32 lmask = mask; - u32 hmask = mask >> 32; - - asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x2f\n\t" - : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) - : "memory"); -} - -static inline void xsave_state(struct xsave_struct *fx, u64 mask) -{ - u32 lmask = mask; - u32 hmask = mask >> 32; - - asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x27\n\t" - : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) - : "memory"); -} - static inline void fpu_xsave(struct fpu *fpu) { /* This, however, we can work around by forcing the compiler to select -- cgit v1.2.2 From f9de314b340f4816671f037e79ed01f685ac9787 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:37 -0700 Subject: x86/xsaves: Use xsaves/xrstors for context switch If xsaves is eanbled, use xsaves/xrstors for context switch to support compacted format xsave area to occupy less memory and modified optimization to improve saving performance. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-10-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/xsave.h | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index f9177a2a97e9..8b75824e41dd 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -129,22 +129,20 @@ static inline int xrstor_state(struct xsave_struct *fx, u64 mask) return err; } -static inline int fpu_xrstor_checking(struct xsave_struct *fx) +/* + * Save xstate context for old process during context switch. + */ +static inline void fpu_xsave(struct fpu *fpu) { - int err; - - asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t" - "2:\n" - ".section .fixup,\"ax\"\n" - "3: movl $-1,%[err]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) - : [err] "=r" (err) - : "D" (fx), "m" (*fx), "a" (-1), "d" (-1), "0" (0) - : "memory"); + xsave_state(&fpu->state->xsave, -1); +} - return err; +/* + * Restore xstate context for new process during context switch. + */ +static inline int fpu_xrstor_checking(struct xsave_struct *fx) +{ + return xrstor_state(fx, -1); } static inline int xsave_user(struct xsave_struct __user *buf) @@ -194,15 +192,4 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask) return err; } -static inline void fpu_xsave(struct fpu *fpu) -{ - /* This, however, we can work around by forcing the compiler to select - an addressing mode that doesn't require extended registers. */ - alternative_input( - ".byte " REX_PREFIX "0x0f,0xae,0x27", - ".byte " REX_PREFIX "0x0f,0xae,0x37", - X86_FEATURE_XSAVEOPT, - [fx] "D" (&fpu->state->xsave), "a" (-1), "d" (-1) : - "memory"); -} #endif -- cgit v1.2.2 From facbf4d91ae64f84ef93a00e4037135cd9f4b2ab Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:38 -0700 Subject: x86/xsaves: Use xsave/xrstor for saving and restoring user space context We use legacy xsave/xrstor to save and restore standard form of xsave area in user space context. No xsaveopt or xsaves is used here for two reasons. First, we don't want to use modified optimization which is implemented in xsaveopt and xsaves because xrstor/xrstors might track a wrong user space application. Secondly, we don't use compacted format of xsave area for backward compatibility because legacy user space applications only don't understand the compacted format of the xsave area. Using standard form of the xsave area may allocate more memory for user context than compacted form, but preserves compatibility with legacy applications. Furthermore, even with holes, the relevant cache lines don't get touched and thus the performance impact is limited. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-11-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/xsave.h | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 8b75824e41dd..0d1523146545 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -145,6 +145,16 @@ static inline int fpu_xrstor_checking(struct xsave_struct *fx) return xrstor_state(fx, -1); } +/* + * Save xstate to user space xsave area. + * + * We don't use modified optimization because xrstor/xrstors might track + * a different application. + * + * We don't use compacted format xsave area for + * backward compatibility for old applications which don't understand + * compacted format of xsave area. + */ static inline int xsave_user(struct xsave_struct __user *buf) { int err; @@ -158,35 +168,28 @@ static inline int xsave_user(struct xsave_struct __user *buf) return -EFAULT; __asm__ __volatile__(ASM_STAC "\n" - "1: .byte " REX_PREFIX "0x0f,0xae,0x27\n" + "1:"XSAVE"\n" "2: " ASM_CLAC "\n" - ".section .fixup,\"ax\"\n" - "3: movl $-1,%[err]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b,3b) - : [err] "=r" (err) + xstate_fault : "D" (buf), "a" (-1), "d" (-1), "0" (0) : "memory"); return err; } +/* + * Restore xstate from user space xsave area. + */ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask) { - int err; + int err = 0; struct xsave_struct *xstate = ((__force struct xsave_struct *)buf); u32 lmask = mask; u32 hmask = mask >> 32; __asm__ __volatile__(ASM_STAC "\n" - "1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n" + "1:"XRSTOR"\n" "2: " ASM_CLAC "\n" - ".section .fixup,\"ax\"\n" - "3: movl $-1,%[err]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b,3b) - : [err] "=r" (err) + xstate_fault : "D" (xstate), "a" (lmask), "d" (hmask), "0" (0) : "memory"); /* memory required? */ return err; -- cgit v1.2.2 From adb9d526e98268b647a74726346e1c40e6a37d2e Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:40 -0700 Subject: x86/xsaves: Add xsaves and xrstors support for booting time Since boot_cpu_data and cpu capabilities are not enabled yet during early booting time, alternative can not be used in some functions to access xsave area. Therefore, we define two new functions xrstor_state_booting() and xsave_state_booting() to access xsave area just during early booting time. xrstor_state_booting restores xstate from xsave area during early booting time. xsave_state_booting saves xstate to xsave area during early booting time. The two functions are similar to xrstor_state and xsave_state respectively. But the two functions don't use alternatives because alternatives are not enabled when they are called in such early booting time. xrstor_state_booting is called only by functions defined as __init. So it's defined as __init and will be removed from memory after booting time. There is no extra memory cost caused by this function during running time. But because xsave_state_booting can be called by run-time function __save_fpu(), it's not defined as __init and will stay in memory during running time although it will not be called anymore during running time. It is not ideal to have this function stay in memory during running time. But it's a pretty small function and the memory cost will be small. By doing in this way, we can avoid to change a lot of code to just remove this small function and save a bit memory for running time. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-13-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/xsave.h | 60 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 0d1523146545..aa3ff0cca9a1 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -65,6 +65,66 @@ extern int init_fpu(struct task_struct *child); _ASM_EXTABLE(1b, 3b) \ : [err] "=r" (err) +/* + * This function is called only during boot time when x86 caps are not set + * up and alternative can not be used yet. + */ +static int xsave_state_booting(struct xsave_struct *fx, u64 mask) +{ + u32 lmask = mask; + u32 hmask = mask >> 32; + int err = 0; + + WARN_ON(system_state != SYSTEM_BOOTING); + + if (boot_cpu_has(X86_FEATURE_XSAVES)) + asm volatile("1:"XSAVES"\n\t" + "2:\n\t" + : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + else + asm volatile("1:"XSAVE"\n\t" + "2:\n\t" + : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + + asm volatile(xstate_fault + : "0" (0) + : "memory"); + + return err; +} + +/* + * This function is called only during boot time when x86 caps are not set + * up and alternative can not be used yet. + */ +static inline int xrstor_state_booting(struct xsave_struct *fx, u64 mask) +{ + u32 lmask = mask; + u32 hmask = mask >> 32; + int err = 0; + + WARN_ON(system_state != SYSTEM_BOOTING); + + if (boot_cpu_has(X86_FEATURE_XSAVES)) + asm volatile("1:"XRSTORS"\n\t" + "2:\n\t" + : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + else + asm volatile("1:"XRSTOR"\n\t" + "2:\n\t" + : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + + asm volatile(xstate_fault + : "0" (0) + : "memory"); + + return err; +} + /* * Save processor xstate to xsave area. */ -- cgit v1.2.2 From 7496d6458fe3219d63848ce4a9afbd86245cab22 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:44 -0700 Subject: Define kernel API to get address of each state in xsave area In standard form, each state is saved in the xsave area in fixed offset. But in compacted form, offset of each saved state only can be calculated during run time because some xstates may not be enabled and saved. We define kernel API get_xsave_addr() returns address of a given state saved in a xsave area. It can be called in kernel to get address of each xstate in xsave area in either standard format or compacted format. It's useful when kernel wants to directly access each state in xsave area. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-17-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/xsave.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index aa3ff0cca9a1..1ba577c670ad 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -255,4 +255,7 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask) return err; } +void *get_xsave_addr(struct xsave_struct *xsave, int xstate); +void setup_xstate_comp(void); + #endif -- cgit v1.2.2 From c9e5a5a7034146493386d985ff432aed8059929a Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Fri, 30 May 2014 08:19:21 -0700 Subject: x86/xsave: Make it clear that the XSAVE macros use (%edi)/(%rdi) The XSAVE instruction family takes a memory argment. The macros use (%edi)/(%rdi) as that memory argument - make that clear to the reader. Signed-off-by: H. Peter Anvin Cc: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-7-git-send-email-fenghua.yu@intel.com --- arch/x86/include/asm/xsave.h | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 1ba577c670ad..bbebd6e0a9ce 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -52,6 +52,7 @@ extern void xsave_init(void); extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); extern int init_fpu(struct task_struct *child); +/* These macros all use (%edi)/(%rdi) as the single memory argument. */ #define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27" #define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37" #define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f" -- cgit v1.2.2 From d0f2dd186133a0241a2ccefb188a0e49e8187859 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Tue, 17 Jun 2014 12:21:08 +0200 Subject: x86, xsave: Add forgotten inline annotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a missing inline annotation on a static function, in order to shut up a bunch of warnings like: In file included from arch/x86/crypto/camellia_aesni_avx_glue.c:23:0: ./arch/x86/include/asm/xsave.h:73:12: warning: ‘xsave_state_booting’ defined but not used [-Wunused-function] static int xsave_state_booting(struct xsave_struct *fx, u64 mask) ^ In file included from arch/x86/crypto/camellia_aesni_avx2_glue.c:23:0: ./arch/x86/include/asm/xsave.h:73:12: warning: ‘xsave_state_booting’ defined but not used [-Wunused-function] static int xsave_state_booting(struct xsave_struct *fx, u64 mask) ^ ... Cc: Fenghua Yu Signed-off-by: Borislav Petkov Link: http://lkml.kernel.org/r/1403000468-30094-1-git-send-email-bp@alien8.de Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/xsave.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/include/asm/xsave.h') diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index bbebd6e0a9ce..7e7a79ada658 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -70,7 +70,7 @@ extern int init_fpu(struct task_struct *child); * This function is called only during boot time when x86 caps are not set * up and alternative can not be used yet. */ -static int xsave_state_booting(struct xsave_struct *fx, u64 mask) +static inline int xsave_state_booting(struct xsave_struct *fx, u64 mask) { u32 lmask = mask; u32 hmask = mask >> 32; -- cgit v1.2.2