3 files changed, 37 insertions, 3 deletions

diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 9f1f9e3b8230..7ce7ac9d9d3f 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -314,6 +314,23 @@ For 32-bit we have the following conventions - kernel is built with
 
 #endif
 
+/*
+ * Mitigate Spectre v1 for conditional swapgs code paths.
+ *
+ * FENCE_SWAPGS_USER_ENTRY is used in the user entry swapgs code path, to
+ * prevent a speculative swapgs when coming from kernel space.
+ *
+ * FENCE_SWAPGS_KERNEL_ENTRY is used in the kernel entry non-swapgs code path,
+ * to prevent the swapgs from getting speculatively skipped when coming from
+ * user space.
+ */
+.macro FENCE_SWAPGS_USER_ENTRY
+        ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_USER
+.endm
+.macro FENCE_SWAPGS_KERNEL_ENTRY
+        ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_KERNEL
+.endm
+
 .macro STACKLEAK_ERASE_NOCLOBBER
 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
         PUSH_AND_CLEAR_REGS
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index a829dd3117d0..57a0d96d6beb 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -519,7 +519,7 @@ ENTRY(interrupt_entry)
         testb   $3, CS-ORIG_RAX+8(%rsp)
         jz      1f
         SWAPGS
-
+        FENCE_SWAPGS_USER_ENTRY
         /*
          * Switch to the thread stack. The IRET frame and orig_ax are
          * on the stack, as well as the return address. RDI..R12 are
@@ -549,8 +549,10 @@ ENTRY(interrupt_entry)
         UNWIND_HINT_FUNC
 
         movq    (%rdi), %rdi
+        jmpq    2f
 1:
-
+        FENCE_SWAPGS_KERNEL_ENTRY
+2:
         PUSH_AND_CLEAR_REGS save_ret=1
         ENCODE_FRAME_POINTER 8
 
@@ -1221,6 +1223,13 @@ ENTRY(paranoid_entry)
          */
         SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
 
+        /*
+         * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
+         * unconditional CR3 write, even in the PTI case.  So do an lfence
+         * to prevent GS speculation, regardless of whether PTI is enabled.
+         */
+        FENCE_SWAPGS_KERNEL_ENTRY
+
         ret
 END(paranoid_entry)
 
@@ -1271,6 +1280,7 @@ ENTRY(error_entry)
          * from user mode due to an IRET fault.
          */
         SWAPGS
+        FENCE_SWAPGS_USER_ENTRY
         /* We have user CR3.  Change to kernel CR3. */
         SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 
@@ -1292,6 +1302,8 @@ ENTRY(error_entry)
         CALL_enter_from_user_mode
         ret
 
+.Lerror_entry_done_lfence:
+        FENCE_SWAPGS_KERNEL_ENTRY
 .Lerror_entry_done:
         TRACE_IRQS_OFF
         ret
@@ -1310,7 +1322,7 @@ ENTRY(error_entry)
         cmpq    %rax, RIP+8(%rsp)
         je      .Lbstep_iret
         cmpq    $.Lgs_change, RIP+8(%rsp)
-        jne     .Lerror_entry_done
+        jne     .Lerror_entry_done_lfence
 
         /*
          * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
@@ -1318,6 +1330,7 @@ ENTRY(error_entry)
          * .Lgs_change's error handler with kernel gsbase.
          */
         SWAPGS
+        FENCE_SWAPGS_USER_ENTRY
         SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
         jmp .Lerror_entry_done
 
@@ -1332,6 +1345,7 @@ ENTRY(error_entry)
          * gsbase and CR3.  Switch to kernel gsbase and CR3:
          */
         SWAPGS
+        FENCE_SWAPGS_USER_ENTRY
         SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 
         /*
@@ -1423,6 +1437,7 @@ ENTRY(nmi)
 
         swapgs
         cld
+        FENCE_SWAPGS_USER_ENTRY
         SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
         movq    %rsp, %rdx
         movq    PER_CPU_VAR(cpu_current_top_of_stack), %rsp
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 998c2cc08363..4393278666d9 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -281,6 +281,8 @@
 #define X86_FEATURE_CQM_OCCUP_LLC       (11*32+ 1) /* LLC occupancy monitoring */
 #define X86_FEATURE_CQM_MBM_TOTAL       (11*32+ 2) /* LLC Total MBM monitoring */
 #define X86_FEATURE_CQM_MBM_LOCAL       (11*32+ 3) /* LLC Local MBM monitoring */
+#define X86_FEATURE_FENCE_SWAPGS_USER   (11*32+ 4) /* "" LFENCE in user entry SWAPGS path */
+#define X86_FEATURE_FENCE_SWAPGS_KERNEL (11*32+ 5) /* "" LFENCE in kernel entry SWAPGS path */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
 #define X86_FEATURE_AVX512_BF16         (12*32+ 5) /* AVX512 BFLOAT16 instructions */