1 files changed, 82 insertions, 19 deletions

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index c93fcfdf1673..7d31192296a8 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -40,6 +40,8 @@
 #include <asm/prctl.h>
 #include <asm/spec-ctrl.h>
 
+#include "process.h"
+
 /*
  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
  * no more per-task TSS's. The TSS size is kept cacheline-aligned
@@ -252,11 +254,12 @@ void arch_setup_new_exec(void)
                 enable_cpuid();
 }
 
-static inline void switch_to_bitmap(struct tss_struct *tss,
-                                    struct thread_struct *prev,
+static inline void switch_to_bitmap(struct thread_struct *prev,
                                     struct thread_struct *next,
                                     unsigned long tifp, unsigned long tifn)
 {
+        struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
+
         if (tifn & _TIF_IO_BITMAP) {
                 /*
                  * Copy the relevant range of the IO bitmap.
@@ -395,32 +398,85 @@ static __always_inline void amd_set_ssb_virt_state(unsigned long tifn)
         wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn));
 }
 
-static __always_inline void intel_set_ssb_state(unsigned long tifn)
+/*
+ * Update the MSRs managing speculation control, during context switch.
+ *
+ * tifp: Previous task's thread flags
+ * tifn: Next task's thread flags
+ */
+static __always_inline void __speculation_ctrl_update(unsigned long tifp,
+                                                      unsigned long tifn)
 {
-        u64 msr = x86_spec_ctrl_base | ssbd_tif_to_spec_ctrl(tifn);
+        unsigned long tif_diff = tifp ^ tifn;
+        u64 msr = x86_spec_ctrl_base;
+        bool updmsr = false;
+
+        /*
+         * If TIF_SSBD is different, select the proper mitigation
+         * method. Note that if SSBD mitigation is disabled or permanentely
+         * enabled this branch can't be taken because nothing can set
+         * TIF_SSBD.
+         */
+        if (tif_diff & _TIF_SSBD) {
+                if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) {
+                        amd_set_ssb_virt_state(tifn);
+                } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) {
+                        amd_set_core_ssb_state(tifn);
+                } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
+                           static_cpu_has(X86_FEATURE_AMD_SSBD)) {
+                        msr |= ssbd_tif_to_spec_ctrl(tifn);
+                        updmsr  = true;
+                }
+        }
+
+        /*
+         * Only evaluate TIF_SPEC_IB if conditional STIBP is enabled,
+         * otherwise avoid the MSR write.
+         */
+        if (IS_ENABLED(CONFIG_SMP) &&
+            static_branch_unlikely(&switch_to_cond_stibp)) {
+                updmsr |= !!(tif_diff & _TIF_SPEC_IB);
+                msr |= stibp_tif_to_spec_ctrl(tifn);
+        }
 
-        wrmsrl(MSR_IA32_SPEC_CTRL, msr);
+        if (updmsr)
+                wrmsrl(MSR_IA32_SPEC_CTRL, msr);
 }
 
-static __always_inline void __speculative_store_bypass_update(unsigned long tifn)
+static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk)
 {
-        if (static_cpu_has(X86_FEATURE_VIRT_SSBD))
-                amd_set_ssb_virt_state(tifn);
-        else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD))
-                amd_set_core_ssb_state(tifn);
-        else
-                intel_set_ssb_state(tifn);
+        if (test_and_clear_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE)) {
+                if (task_spec_ssb_disable(tsk))
+                        set_tsk_thread_flag(tsk, TIF_SSBD);
+                else
+                        clear_tsk_thread_flag(tsk, TIF_SSBD);
+
+                if (task_spec_ib_disable(tsk))
+                        set_tsk_thread_flag(tsk, TIF_SPEC_IB);
+                else
+                        clear_tsk_thread_flag(tsk, TIF_SPEC_IB);
+        }
+        /* Return the updated threadinfo flags*/
+        return task_thread_info(tsk)->flags;
 }
 
-void speculative_store_bypass_update(unsigned long tif)
+void speculation_ctrl_update(unsigned long tif)
 {
+        /* Forced update. Make sure all relevant TIF flags are different */
         preempt_disable();
-        __speculative_store_bypass_update(tif);
+        __speculation_ctrl_update(~tif, tif);
         preempt_enable();
 }
 
-void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
-                      struct tss_struct *tss)
+/* Called from seccomp/prctl update */
+void speculation_ctrl_update_current(void)
+{
+        preempt_disable();
+        speculation_ctrl_update(speculation_ctrl_update_tif(current));
+        preempt_enable();
+}
+
+void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p)
 {
         struct thread_struct *prev, *next;
         unsigned long tifp, tifn;
@@ -430,7 +486,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
 
         tifn = READ_ONCE(task_thread_info(next_p)->flags);
         tifp = READ_ONCE(task_thread_info(prev_p)->flags);
-        switch_to_bitmap(tss, prev, next, tifp, tifn);
+        switch_to_bitmap(prev, next, tifp, tifn);
 
         propagate_user_return_notify(prev_p, next_p);
 
@@ -451,8 +507,15 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
         if ((tifp ^ tifn) & _TIF_NOCPUID)
                 set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
 
-        if ((tifp ^ tifn) & _TIF_SSBD)
-                __speculative_store_bypass_update(tifn);
+        if (likely(!((tifp | tifn) & _TIF_SPEC_FORCE_UPDATE))) {
+                __speculation_ctrl_update(tifp, tifn);
+        } else {
+                speculation_ctrl_update_tif(prev_p);
+                tifn = speculation_ctrl_update_tif(next_p);
+
+                /* Enforce MSR update to ensure consistent state */
+                __speculation_ctrl_update(~tifn, tifn);
+        }
 }
 
 /*