1 files changed, 122 insertions, 21 deletions

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 81a1be326571..f928ad9b143f 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -8,6 +8,50 @@
 #include <asm/cpufeatures.h>
 #include <asm/msr-index.h>
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; lfence; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS         32      /* To forcibly overwrite all entries */
+#define RSB_FILL_LOOPS          16      /* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp)       \
+        mov     $(nr/2), reg;                   \
+771:                                            \
+        call    772f;                           \
+773:    /* speculation trap */                  \
+        pause;                                  \
+        lfence;                                 \
+        jmp     773b;                           \
+772:                                            \
+        call    774f;                           \
+775:    /* speculation trap */                  \
+        pause;                                  \
+        lfence;                                 \
+        jmp     775b;                           \
+774:                                            \
+        dec     reg;                            \
+        jnz     771b;                           \
+        add     $(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -24,6 +68,18 @@
 .endm
 
 /*
+ * This should be used immediately before an indirect jump/call. It tells
+ * objtool the subsequent indirect jump/call is vouched safe for retpoline
+ * builds.
+ */
+.macro ANNOTATE_RETPOLINE_SAFE
+        .Lannotate_\@:
+        .pushsection .discard.retpoline_safe
+        _ASM_PTR .Lannotate_\@
+        .popsection
+.endm
+
+/*
  * These are the bare retpoline primitives for indirect jmp and call.
  * Do not use these directly; they only exist to make the ALTERNATIVE
  * invocation below less ugly.
@@ -59,9 +115,9 @@
 .macro JMP_NOSPEC reg:req
 #ifdef CONFIG_RETPOLINE
         ANNOTATE_NOSPEC_ALTERNATIVE
-        ALTERNATIVE_2 __stringify(jmp *\reg),                           \
+        ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg),  \
                 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
-                __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+                __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
 #else
         jmp     *\reg
 #endif
@@ -70,18 +126,25 @@
 .macro CALL_NOSPEC reg:req
 #ifdef CONFIG_RETPOLINE
         ANNOTATE_NOSPEC_ALTERNATIVE
-        ALTERNATIVE_2 __stringify(call *\reg),                          \
+        ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \
                 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
-                __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+                __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD
 #else
         call    *\reg
 #endif
 .endm
 
-/* This clobbers the BX register */
-.macro FILL_RETURN_BUFFER nr:req ftr:req
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
 #ifdef CONFIG_RETPOLINE
-        ALTERNATIVE "", "call __clear_rsb", \ftr
+        ANNOTATE_NOSPEC_ALTERNATIVE
+        ALTERNATIVE "jmp .Lskip_rsb_\@",                                \
+                __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))    \
+                \ftr
+.Lskip_rsb_\@:
 #endif
 .endm
 
@@ -93,6 +156,12 @@
         ".long 999b - .\n\t"                                    \
         ".popsection\n\t"
 
+#define ANNOTATE_RETPOLINE_SAFE                                 \
+        "999:\n\t"                                              \
+        ".pushsection .discard.retpoline_safe\n\t"              \
+        _ASM_PTR " 999b\n\t"                                    \
+        ".popsection\n\t"
+
 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
 
 /*
@@ -102,6 +171,7 @@
 # define CALL_NOSPEC                                            \
         ANNOTATE_NOSPEC_ALTERNATIVE                             \
         ALTERNATIVE(                                            \
+        ANNOTATE_RETPOLINE_SAFE                                 \
         "call *%[thunk_target]\n",                              \
         "call __x86_indirect_thunk_%V[thunk_target]\n",         \
         X86_FEATURE_RETPOLINE)
@@ -113,7 +183,10 @@
  * otherwise we'll run out of registers. We don't care about CET
  * here, anyway.
  */
-# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",     \
+# define CALL_NOSPEC                                            \
+        ALTERNATIVE(                                            \
+        ANNOTATE_RETPOLINE_SAFE                                 \
+        "call *%[thunk_target]\n",                              \
         "       jmp    904f;\n"                                 \
         "       .align 16\n"                                    \
         "901:   call   903f;\n"                                 \
@@ -156,26 +229,54 @@ extern char __indirect_thunk_end[];
 static inline void vmexit_fill_RSB(void)
 {
 #ifdef CONFIG_RETPOLINE
-        alternative_input("",
-                          "call __fill_rsb",
-                          X86_FEATURE_RETPOLINE,
-                          ASM_NO_INPUT_CLOBBER(_ASM_BX, "memory"));
+        unsigned long loops;
+
+        asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+                      ALTERNATIVE("jmp 910f",
+                                  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
+                                  X86_FEATURE_RETPOLINE)
+                      "910:"
+                      : "=r" (loops), ASM_CALL_CONSTRAINT
+                      : : "memory" );
 #endif
 }
 
+#define alternative_msr_write(_msr, _val, _feature)             \
+        asm volatile(ALTERNATIVE("",                            \
+                                 "movl %[msr], %%ecx\n\t"       \
+                                 "movl %[val], %%eax\n\t"       \
+                                 "movl $0, %%edx\n\t"           \
+                                 "wrmsr",                       \
+                                 _feature)                      \
+                     : : [msr] "i" (_msr), [val] "i" (_val)     \
+                     : "eax", "ecx", "edx", "memory")
+
 static inline void indirect_branch_prediction_barrier(void)
 {
-        asm volatile(ALTERNATIVE("",
-                                 "movl %[msr], %%ecx\n\t"
-                                 "movl %[val], %%eax\n\t"
-                                 "movl $0, %%edx\n\t"
-                                 "wrmsr",
-                                 X86_FEATURE_USE_IBPB)
-                     : : [msr] "i" (MSR_IA32_PRED_CMD),
-                         [val] "i" (PRED_CMD_IBPB)
-                     : "eax", "ecx", "edx", "memory");
+        alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB,
+                              X86_FEATURE_USE_IBPB);
 }
 
+/*
+ * With retpoline, we must use IBRS to restrict branch prediction
+ * before calling into firmware.
+ *
+ * (Implemented as CPP macros due to header hell.)
+ */
+#define firmware_restrict_branch_speculation_start()                    \
+do {                                                                    \
+        preempt_disable();                                              \
+        alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS,       \
+                              X86_FEATURE_USE_IBRS_FW);                 \
+} while (0)
+
+#define firmware_restrict_branch_speculation_end()                      \
+do {                                                                    \
+        alternative_msr_write(MSR_IA32_SPEC_CTRL, 0,                    \
+                              X86_FEATURE_USE_IBRS_FW);                 \
+        preempt_enable();                                               \
+} while (0)
+
 #endif /* __ASSEMBLY__ */
 
 /*