Merge branch 'x86-xsave-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86/xsave changes from Peter Anvin:
 "This is a patchset to support the XSAVES instruction required to
  support context switch of supervisor-only features in upcoming
  silicon.

  This patchset missed the 3.16 merge window, which is why it is based
  on 3.15-rc7"

* 'x86-xsave-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86, xsave: Add forgotten inline annotation
  x86/xsaves: Clean up code in xstate offsets computation in xsave area
  x86/xsave: Make it clear that the XSAVE macros use (%edi)/(%rdi)
  Define kernel API to get address of each state in xsave area
  x86/xsaves: Enable xsaves/xrstors
  x86/xsaves: Call booting time xsaves and xrstors in setup_init_fpu_buf
  x86/xsaves: Save xstate to task's xsave area in __save_fpu during booting time
  x86/xsaves: Add xsaves and xrstors support for booting time
  x86/xsaves: Clear reserved bits in xsave header
  x86/xsaves: Use xsave/xrstor for saving and restoring user space context
  x86/xsaves: Use xsaves/xrstors for context switch
  x86/xsaves: Use xsaves/xrstors to save and restore xsave area
  x86/xsaves: Define a macro for handling xsave/xrstor instruction fault
  x86/xsaves: Define macros for xsave instructions
  x86/xsaves: Change compacted format xsave area header
  x86/alternative: Add alternative_input_2 to support alternative with two features and input
  x86/xsaves: Add a kernel parameter noxsaves to disable xsaves/xrstors

4 files changed, 189 insertions, 61 deletions

diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 0a3f9c9f98d5..473bdbee378a 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -161,6 +161,20 @@ static inline int alternatives_text_reserved(void *start, void *end)
         asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)          \
                 : : "i" (0), ## input)
 
+/*
+ * This is similar to alternative_input. But it has two features and
+ * respective instructions.
+ *
+ * If CPU has feature2, newinstr2 is used.
+ * Otherwise, if CPU has feature1, newinstr1 is used.
+ * Otherwise, oldinstr is used.
+ */
+#define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2,        \
+                           feature2, input...)                               \
+        asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1,            \
+                newinstr2, feature2)                                         \
+                : : "i" (0), ## input)
+
 /* Like alternative_input, but with a single output argument */
 #define alternative_io(oldinstr, newinstr, feature, output, input...)   \
         asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)          \
diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h
index e3b85422cf12..412ececa00b9 100644
--- a/arch/x86/include/asm/fpu-internal.h
+++ b/arch/x86/include/asm/fpu-internal.h
@@ -508,9 +508,12 @@ static inline void user_fpu_begin(void)
 
 static inline void __save_fpu(struct task_struct *tsk)
 {
-        if (use_xsave())
-                xsave_state(&tsk->thread.fpu.state->xsave, -1);
-        else
+        if (use_xsave()) {
+                if (unlikely(system_state == SYSTEM_BOOTING))
+                        xsave_state_booting(&tsk->thread.fpu.state->xsave, -1);
+                else
+                        xsave_state(&tsk->thread.fpu.state->xsave, -1);
+        } else
                 fpu_fxsave(&tsk->thread.fpu);
 }
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index ee30b9f0b91c..eb71ec794732 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -385,8 +385,8 @@ struct bndcsr_struct {
 
 struct xsave_hdr_struct {
         u64 xstate_bv;
-        u64 reserved1[2];
-        u64 reserved2[5];
+        u64 xcomp_bv;
+        u64 reserved[6];
 } __attribute__((packed));
 
 struct xsave_struct {
diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h
index d949ef28c48b..7e7a79ada658 100644
--- a/arch/x86/include/asm/xsave.h
+++ b/arch/x86/include/asm/xsave.h
@@ -52,24 +52,170 @@ extern void xsave_init(void);
 extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
 extern int init_fpu(struct task_struct *child);
 
-static inline int fpu_xrstor_checking(struct xsave_struct *fx)
+/* These macros all use (%edi)/(%rdi) as the single memory argument. */
+#define XSAVE           ".byte " REX_PREFIX "0x0f,0xae,0x27"
+#define XSAVEOPT        ".byte " REX_PREFIX "0x0f,0xae,0x37"
+#define XSAVES          ".byte " REX_PREFIX "0x0f,0xc7,0x2f"
+#define XRSTOR          ".byte " REX_PREFIX "0x0f,0xae,0x2f"
+#define XRSTORS         ".byte " REX_PREFIX "0x0f,0xc7,0x1f"
+
+#define xstate_fault    ".section .fixup,\"ax\"\n"      \
+                        "3:  movl $-1,%[err]\n"         \
+                        "    jmp  2b\n"                 \
+                        ".previous\n"                   \
+                        _ASM_EXTABLE(1b, 3b)            \
+                        : [err] "=r" (err)
+
+/*
+ * This function is called only during boot time when x86 caps are not set
+ * up and alternative can not be used yet.
+ */
+static inline int xsave_state_booting(struct xsave_struct *fx, u64 mask)
 {
-        int err;
+        u32 lmask = mask;
+        u32 hmask = mask >> 32;
+        int err = 0;
+
+        WARN_ON(system_state != SYSTEM_BOOTING);
+
+        if (boot_cpu_has(X86_FEATURE_XSAVES))
+                asm volatile("1:"XSAVES"\n\t"
+                        "2:\n\t"
+                        : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                        :   "memory");
+        else
+                asm volatile("1:"XSAVE"\n\t"
+                        "2:\n\t"
+                        : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                        :   "memory");
+
+        asm volatile(xstate_fault
+                     : "0" (0)
+                     : "memory");
+
+        return err;
+}
+
+/*
+ * This function is called only during boot time when x86 caps are not set
+ * up and alternative can not be used yet.
+ */
+static inline int xrstor_state_booting(struct xsave_struct *fx, u64 mask)
+{
+        u32 lmask = mask;
+        u32 hmask = mask >> 32;
+        int err = 0;
+
+        WARN_ON(system_state != SYSTEM_BOOTING);
 
-        asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t"
-                     "2:\n"
-                     ".section .fixup,\"ax\"\n"
-                     "3:  movl $-1,%[err]\n"
-                     "    jmp  2b\n"
-                     ".previous\n"
-                     _ASM_EXTABLE(1b, 3b)
-                     : [err] "=r" (err)
-                     : "D" (fx), "m" (*fx), "a" (-1), "d" (-1), "0" (0)
+        if (boot_cpu_has(X86_FEATURE_XSAVES))
+                asm volatile("1:"XRSTORS"\n\t"
+                        "2:\n\t"
+                        : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                        :   "memory");
+        else
+                asm volatile("1:"XRSTOR"\n\t"
+                        "2:\n\t"
+                        : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                        :   "memory");
+
+        asm volatile(xstate_fault
+                     : "0" (0)
+                     : "memory");
+
+        return err;
+}
+
+/*
+ * Save processor xstate to xsave area.
+ */
+static inline int xsave_state(struct xsave_struct *fx, u64 mask)
+{
+        u32 lmask = mask;
+        u32 hmask = mask >> 32;
+        int err = 0;
+
+        /*
+         * If xsaves is enabled, xsaves replaces xsaveopt because
+         * it supports compact format and supervisor states in addition to
+         * modified optimization in xsaveopt.
+         *
+         * Otherwise, if xsaveopt is enabled, xsaveopt replaces xsave
+         * because xsaveopt supports modified optimization which is not
+         * supported by xsave.
+         *
+         * If none of xsaves and xsaveopt is enabled, use xsave.
+         */
+        alternative_input_2(
+                "1:"XSAVE,
+                "1:"XSAVEOPT,
+                X86_FEATURE_XSAVEOPT,
+                "1:"XSAVES,
+                X86_FEATURE_XSAVES,
+                [fx] "D" (fx), "a" (lmask), "d" (hmask) :
+                "memory");
+        asm volatile("2:\n\t"
+                     xstate_fault
+                     : "0" (0)
                      : "memory");
 
         return err;
 }
 
+/*
+ * Restore processor xstate from xsave area.
+ */
+static inline int xrstor_state(struct xsave_struct *fx, u64 mask)
+{
+        int err = 0;
+        u32 lmask = mask;
+        u32 hmask = mask >> 32;
+
+        /*
+         * Use xrstors to restore context if it is enabled. xrstors supports
+         * compacted format of xsave area which is not supported by xrstor.
+         */
+        alternative_input(
+                "1: " XRSTOR,
+                "1: " XRSTORS,
+                X86_FEATURE_XSAVES,
+                "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                : "memory");
+
+        asm volatile("2:\n"
+                     xstate_fault
+                     : "0" (0)
+                     : "memory");
+
+        return err;
+}
+
+/*
+ * Save xstate context for old process during context switch.
+ */
+static inline void fpu_xsave(struct fpu *fpu)
+{
+        xsave_state(&fpu->state->xsave, -1);
+}
+
+/*
+ * Restore xstate context for new process during context switch.
+ */
+static inline int fpu_xrstor_checking(struct xsave_struct *fx)
+{
+        return xrstor_state(fx, -1);
+}
+
+/*
+ * Save xstate to user space xsave area.
+ *
+ * We don't use modified optimization because xrstor/xrstors might track
+ * a different application.
+ *
+ * We don't use compacted format xsave area for
+ * backward compatibility for old applications which don't understand
+ * compacted format of xsave area.
+ */
 static inline int xsave_user(struct xsave_struct __user *buf)
 {
         int err;
@@ -83,69 +229,34 @@ static inline int xsave_user(struct xsave_struct __user *buf)
                 return -EFAULT;
 
         __asm__ __volatile__(ASM_STAC "\n"
-                             "1: .byte " REX_PREFIX "0x0f,0xae,0x27\n"
+                             "1:"XSAVE"\n"
                              "2: " ASM_CLAC "\n"
-                             ".section .fixup,\"ax\"\n"
-                             "3:  movl $-1,%[err]\n"
-                             "    jmp  2b\n"
-                             ".previous\n"
-                             _ASM_EXTABLE(1b,3b)
-                             : [err] "=r" (err)
+                             xstate_fault
                              : "D" (buf), "a" (-1), "d" (-1), "0" (0)
                              : "memory");
         return err;
 }
 
+/*
+ * Restore xstate from user space xsave area.
+ */
 static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask)
 {
-        int err;
+        int err = 0;
         struct xsave_struct *xstate = ((__force struct xsave_struct *)buf);
         u32 lmask = mask;
         u32 hmask = mask >> 32;
 
         __asm__ __volatile__(ASM_STAC "\n"
-                             "1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n"
+                             "1:"XRSTOR"\n"
                              "2: " ASM_CLAC "\n"
-                             ".section .fixup,\"ax\"\n"
-                             "3:  movl $-1,%[err]\n"
-                             "    jmp  2b\n"
-                             ".previous\n"
-                             _ASM_EXTABLE(1b,3b)
-                             : [err] "=r" (err)
+                             xstate_fault
                              : "D" (xstate), "a" (lmask), "d" (hmask), "0" (0)
                              : "memory");       /* memory required? */
         return err;
 }
 
-static inline void xrstor_state(struct xsave_struct *fx, u64 mask)
-{
-        u32 lmask = mask;
-        u32 hmask = mask >> 32;
-
-        asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x2f\n\t"
-                     : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
-                     :   "memory");
-}
-
-static inline void xsave_state(struct xsave_struct *fx, u64 mask)
-{
-        u32 lmask = mask;
-        u32 hmask = mask >> 32;
+void *get_xsave_addr(struct xsave_struct *xsave, int xstate);
+void setup_xstate_comp(void);
 
-        asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x27\n\t"
-                     : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
-                     :   "memory");
-}
-
-static inline void fpu_xsave(struct fpu *fpu)
-{
-        /* This, however, we can work around by forcing the compiler to select
-           an addressing mode that doesn't require extended registers. */
-        alternative_input(
-                ".byte " REX_PREFIX "0x0f,0xae,0x27",
-                ".byte " REX_PREFIX "0x0f,0xae,0x37",
-                X86_FEATURE_XSAVEOPT,
-                [fx] "D" (&fpu->state->xsave), "a" (-1), "d" (-1) :
-                "memory");
-}
 #endif