1 files changed, 88 insertions, 0 deletions

diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 254eefb3b57f..4e3c79530526 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -184,6 +184,8 @@ struct vcpu_svm {
                 u64 gs_base;
         } host;
 
+        u64 spec_ctrl;
+
         u32 *msrpm;
 
         ulong nmi_iret_rip;
@@ -249,6 +251,7 @@ static const struct svm_direct_access_msrs {
         { .index = MSR_CSTAR,                           .always = true  },
         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 #endif
+        { .index = MSR_IA32_SPEC_CTRL,                  .always = false },
         { .index = MSR_IA32_PRED_CMD,                   .always = false },
         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
@@ -882,6 +885,25 @@ static bool valid_msr_intercept(u32 index)
         return false;
 }
 
+static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
+{
+        u8 bit_write;
+        unsigned long tmp;
+        u32 offset;
+        u32 *msrpm;
+
+        msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
+                                      to_svm(vcpu)->msrpm;
+
+        offset    = svm_msrpm_offset(msr);
+        bit_write = 2 * (msr & 0x0f) + 1;
+        tmp       = msrpm[offset];
+
+        BUG_ON(offset == MSR_INVALID);
+
+        return !!test_bit(bit_write,  &tmp);
+}
+
 static void set_msr_interception(u32 *msrpm, unsigned msr,
                                  int read, int write)
 {
@@ -1584,6 +1606,8 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
         u32 dummy;
         u32 eax = 1;
 
+        svm->spec_ctrl = 0;
+
         if (!init_event) {
                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
                                            MSR_IA32_APICBASE_ENABLE;
@@ -3605,6 +3629,13 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
         case MSR_VM_CR:
                 msr_info->data = svm->nested.vm_cr_msr;
                 break;
+        case MSR_IA32_SPEC_CTRL:
+                if (!msr_info->host_initiated &&
+                    !guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
+                        return 1;
+
+                msr_info->data = svm->spec_ctrl;
+                break;
         case MSR_IA32_UCODE_REV:
                 msr_info->data = 0x01000065;
                 break;
@@ -3696,6 +3727,33 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
         case MSR_IA32_TSC:
                 kvm_write_tsc(vcpu, msr);
                 break;
+        case MSR_IA32_SPEC_CTRL:
+                if (!msr->host_initiated &&
+                    !guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
+                        return 1;
+
+                /* The STIBP bit doesn't fault even if it's not advertised */
+                if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP))
+                        return 1;
+
+                svm->spec_ctrl = data;
+
+                if (!data)
+                        break;
+
+                /*
+                 * For non-nested:
+                 * When it's written (to non-zero) for the first time, pass
+                 * it through.
+                 *
+                 * For nested:
+                 * The handling of the MSR bitmap for L2 guests is done in
+                 * nested_svm_vmrun_msrpm.
+                 * We update the L1 MSR bit as well since it will end up
+                 * touching the MSR anyway now.
+                 */
+                set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
+                break;
         case MSR_IA32_PRED_CMD:
                 if (!msr->host_initiated &&
                     !guest_cpuid_has(vcpu, X86_FEATURE_IBPB))
@@ -4964,6 +5022,15 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 
         local_irq_enable();
 
+        /*
+         * If this vCPU has touched SPEC_CTRL, restore the guest's value if
+         * it's non-zero. Since vmentry is serialising on affected CPUs, there
+         * is no need to worry about the conditional branch over the wrmsr
+         * being speculatively taken.
+         */
+        if (svm->spec_ctrl)
+                wrmsrl(MSR_IA32_SPEC_CTRL, svm->spec_ctrl);
+
         asm volatile (
                 "push %%" _ASM_BP "; \n\t"
                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
@@ -5056,6 +5123,27 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
                 );
 
+        /*
+         * We do not use IBRS in the kernel. If this vCPU has used the
+         * SPEC_CTRL MSR it may have left it on; save the value and
+         * turn it off. This is much more efficient than blindly adding
+         * it to the atomic save/restore list. Especially as the former
+         * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
+         *
+         * For non-nested case:
+         * If the L01 MSR bitmap does not intercept the MSR, then we need to
+         * save it.
+         *
+         * For nested case:
+         * If the L02 MSR bitmap does not intercept the MSR, then we need to
+         * save it.
+         */
+        if (!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL))
+                rdmsrl(MSR_IA32_SPEC_CTRL, svm->spec_ctrl);
+
+        if (svm->spec_ctrl)
+                wrmsrl(MSR_IA32_SPEC_CTRL, 0);
+
         /* Eliminate branch target predictions from guest mode */
         vmexit_fill_RSB();