diff options
author | Paolo Bonzini <pbonzini@redhat.com> | 2018-01-16 10:51:18 -0500 |
---|---|---|
committer | Paolo Bonzini <pbonzini@redhat.com> | 2018-01-31 12:40:45 -0500 |
commit | 904e14fb7cb96401a7dc803ca2863fd5ba32ffe6 (patch) | |
tree | f2b192c2d4783bd1a882f2d4534b54f538c4333d | |
parent | f21f165ef922c2146cc5bdc620f542953c41714b (diff) |
KVM: VMX: make MSR bitmaps per-VCPU
Place the MSR bitmap in struct loaded_vmcs, and update it in place
every time the x2apic or APICv state can change. This is rare and
the loop can handle 64 MSRs per iteration, in a similar fashion as
nested_vmx_prepare_msr_bitmap.
This prepares for choosing, on a per-VM basis, whether to intercept
the SPEC_CTRL and PRED_CMD MSRs.
Cc: stable@vger.kernel.org # prereq for Spectre mitigation
Suggested-by: Jim Mattson <jmattson@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
-rw-r--r-- | arch/x86/kvm/vmx.c | 270 |
1 files changed, 147 insertions, 123 deletions
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index ab4b9bc99a52..896af99a8606 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c | |||
@@ -111,6 +111,14 @@ static u64 __read_mostly host_xss; | |||
111 | static bool __read_mostly enable_pml = 1; | 111 | static bool __read_mostly enable_pml = 1; |
112 | module_param_named(pml, enable_pml, bool, S_IRUGO); | 112 | module_param_named(pml, enable_pml, bool, S_IRUGO); |
113 | 113 | ||
114 | #define MSR_TYPE_R 1 | ||
115 | #define MSR_TYPE_W 2 | ||
116 | #define MSR_TYPE_RW 3 | ||
117 | |||
118 | #define MSR_BITMAP_MODE_X2APIC 1 | ||
119 | #define MSR_BITMAP_MODE_X2APIC_APICV 2 | ||
120 | #define MSR_BITMAP_MODE_LM 4 | ||
121 | |||
114 | #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL | 122 | #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL |
115 | 123 | ||
116 | /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ | 124 | /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ |
@@ -209,6 +217,7 @@ struct loaded_vmcs { | |||
209 | int soft_vnmi_blocked; | 217 | int soft_vnmi_blocked; |
210 | ktime_t entry_time; | 218 | ktime_t entry_time; |
211 | s64 vnmi_blocked_time; | 219 | s64 vnmi_blocked_time; |
220 | unsigned long *msr_bitmap; | ||
212 | struct list_head loaded_vmcss_on_cpu_link; | 221 | struct list_head loaded_vmcss_on_cpu_link; |
213 | }; | 222 | }; |
214 | 223 | ||
@@ -449,8 +458,6 @@ struct nested_vmx { | |||
449 | bool pi_pending; | 458 | bool pi_pending; |
450 | u16 posted_intr_nv; | 459 | u16 posted_intr_nv; |
451 | 460 | ||
452 | unsigned long *msr_bitmap; | ||
453 | |||
454 | struct hrtimer preemption_timer; | 461 | struct hrtimer preemption_timer; |
455 | bool preemption_timer_expired; | 462 | bool preemption_timer_expired; |
456 | 463 | ||
@@ -573,6 +580,7 @@ struct vcpu_vmx { | |||
573 | struct kvm_vcpu vcpu; | 580 | struct kvm_vcpu vcpu; |
574 | unsigned long host_rsp; | 581 | unsigned long host_rsp; |
575 | u8 fail; | 582 | u8 fail; |
583 | u8 msr_bitmap_mode; | ||
576 | u32 exit_intr_info; | 584 | u32 exit_intr_info; |
577 | u32 idt_vectoring_info; | 585 | u32 idt_vectoring_info; |
578 | ulong rflags; | 586 | ulong rflags; |
@@ -927,6 +935,7 @@ static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu); | |||
927 | static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked); | 935 | static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked); |
928 | static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, | 936 | static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, |
929 | u16 error_code); | 937 | u16 error_code); |
938 | static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu); | ||
930 | 939 | ||
931 | static DEFINE_PER_CPU(struct vmcs *, vmxarea); | 940 | static DEFINE_PER_CPU(struct vmcs *, vmxarea); |
932 | static DEFINE_PER_CPU(struct vmcs *, current_vmcs); | 941 | static DEFINE_PER_CPU(struct vmcs *, current_vmcs); |
@@ -946,12 +955,6 @@ static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock); | |||
946 | enum { | 955 | enum { |
947 | VMX_IO_BITMAP_A, | 956 | VMX_IO_BITMAP_A, |
948 | VMX_IO_BITMAP_B, | 957 | VMX_IO_BITMAP_B, |
949 | VMX_MSR_BITMAP_LEGACY, | ||
950 | VMX_MSR_BITMAP_LONGMODE, | ||
951 | VMX_MSR_BITMAP_LEGACY_X2APIC_APICV, | ||
952 | VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV, | ||
953 | VMX_MSR_BITMAP_LEGACY_X2APIC, | ||
954 | VMX_MSR_BITMAP_LONGMODE_X2APIC, | ||
955 | VMX_VMREAD_BITMAP, | 958 | VMX_VMREAD_BITMAP, |
956 | VMX_VMWRITE_BITMAP, | 959 | VMX_VMWRITE_BITMAP, |
957 | VMX_BITMAP_NR | 960 | VMX_BITMAP_NR |
@@ -961,12 +964,6 @@ static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; | |||
961 | 964 | ||
962 | #define vmx_io_bitmap_a (vmx_bitmap[VMX_IO_BITMAP_A]) | 965 | #define vmx_io_bitmap_a (vmx_bitmap[VMX_IO_BITMAP_A]) |
963 | #define vmx_io_bitmap_b (vmx_bitmap[VMX_IO_BITMAP_B]) | 966 | #define vmx_io_bitmap_b (vmx_bitmap[VMX_IO_BITMAP_B]) |
964 | #define vmx_msr_bitmap_legacy (vmx_bitmap[VMX_MSR_BITMAP_LEGACY]) | ||
965 | #define vmx_msr_bitmap_longmode (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE]) | ||
966 | #define vmx_msr_bitmap_legacy_x2apic_apicv (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC_APICV]) | ||
967 | #define vmx_msr_bitmap_longmode_x2apic_apicv (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV]) | ||
968 | #define vmx_msr_bitmap_legacy_x2apic (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC]) | ||
969 | #define vmx_msr_bitmap_longmode_x2apic (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC]) | ||
970 | #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) | 967 | #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) |
971 | #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) | 968 | #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) |
972 | 969 | ||
@@ -2564,36 +2561,6 @@ static void move_msr_up(struct vcpu_vmx *vmx, int from, int to) | |||
2564 | vmx->guest_msrs[from] = tmp; | 2561 | vmx->guest_msrs[from] = tmp; |
2565 | } | 2562 | } |
2566 | 2563 | ||
2567 | static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu) | ||
2568 | { | ||
2569 | unsigned long *msr_bitmap; | ||
2570 | |||
2571 | if (is_guest_mode(vcpu)) | ||
2572 | msr_bitmap = to_vmx(vcpu)->nested.msr_bitmap; | ||
2573 | else if (cpu_has_secondary_exec_ctrls() && | ||
2574 | (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) & | ||
2575 | SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { | ||
2576 | if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) { | ||
2577 | if (is_long_mode(vcpu)) | ||
2578 | msr_bitmap = vmx_msr_bitmap_longmode_x2apic_apicv; | ||
2579 | else | ||
2580 | msr_bitmap = vmx_msr_bitmap_legacy_x2apic_apicv; | ||
2581 | } else { | ||
2582 | if (is_long_mode(vcpu)) | ||
2583 | msr_bitmap = vmx_msr_bitmap_longmode_x2apic; | ||
2584 | else | ||
2585 | msr_bitmap = vmx_msr_bitmap_legacy_x2apic; | ||
2586 | } | ||
2587 | } else { | ||
2588 | if (is_long_mode(vcpu)) | ||
2589 | msr_bitmap = vmx_msr_bitmap_longmode; | ||
2590 | else | ||
2591 | msr_bitmap = vmx_msr_bitmap_legacy; | ||
2592 | } | ||
2593 | |||
2594 | vmcs_write64(MSR_BITMAP, __pa(msr_bitmap)); | ||
2595 | } | ||
2596 | |||
2597 | /* | 2564 | /* |
2598 | * Set up the vmcs to automatically save and restore system | 2565 | * Set up the vmcs to automatically save and restore system |
2599 | * msrs. Don't touch the 64-bit msrs if the guest is in legacy | 2566 | * msrs. Don't touch the 64-bit msrs if the guest is in legacy |
@@ -2634,7 +2601,7 @@ static void setup_msrs(struct vcpu_vmx *vmx) | |||
2634 | vmx->save_nmsrs = save_nmsrs; | 2601 | vmx->save_nmsrs = save_nmsrs; |
2635 | 2602 | ||
2636 | if (cpu_has_vmx_msr_bitmap()) | 2603 | if (cpu_has_vmx_msr_bitmap()) |
2637 | vmx_set_msr_bitmap(&vmx->vcpu); | 2604 | vmx_update_msr_bitmap(&vmx->vcpu); |
2638 | } | 2605 | } |
2639 | 2606 | ||
2640 | /* | 2607 | /* |
@@ -3844,6 +3811,8 @@ static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) | |||
3844 | loaded_vmcs_clear(loaded_vmcs); | 3811 | loaded_vmcs_clear(loaded_vmcs); |
3845 | free_vmcs(loaded_vmcs->vmcs); | 3812 | free_vmcs(loaded_vmcs->vmcs); |
3846 | loaded_vmcs->vmcs = NULL; | 3813 | loaded_vmcs->vmcs = NULL; |
3814 | if (loaded_vmcs->msr_bitmap) | ||
3815 | free_page((unsigned long)loaded_vmcs->msr_bitmap); | ||
3847 | WARN_ON(loaded_vmcs->shadow_vmcs != NULL); | 3816 | WARN_ON(loaded_vmcs->shadow_vmcs != NULL); |
3848 | } | 3817 | } |
3849 | 3818 | ||
@@ -3860,7 +3829,18 @@ static int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) | |||
3860 | 3829 | ||
3861 | loaded_vmcs->shadow_vmcs = NULL; | 3830 | loaded_vmcs->shadow_vmcs = NULL; |
3862 | loaded_vmcs_init(loaded_vmcs); | 3831 | loaded_vmcs_init(loaded_vmcs); |
3832 | |||
3833 | if (cpu_has_vmx_msr_bitmap()) { | ||
3834 | loaded_vmcs->msr_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL); | ||
3835 | if (!loaded_vmcs->msr_bitmap) | ||
3836 | goto out_vmcs; | ||
3837 | memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); | ||
3838 | } | ||
3863 | return 0; | 3839 | return 0; |
3840 | |||
3841 | out_vmcs: | ||
3842 | free_loaded_vmcs(loaded_vmcs); | ||
3843 | return -ENOMEM; | ||
3864 | } | 3844 | } |
3865 | 3845 | ||
3866 | static void free_kvm_area(void) | 3846 | static void free_kvm_area(void) |
@@ -4921,10 +4901,8 @@ static void free_vpid(int vpid) | |||
4921 | spin_unlock(&vmx_vpid_lock); | 4901 | spin_unlock(&vmx_vpid_lock); |
4922 | } | 4902 | } |
4923 | 4903 | ||
4924 | #define MSR_TYPE_R 1 | 4904 | static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, |
4925 | #define MSR_TYPE_W 2 | 4905 | u32 msr, int type) |
4926 | static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, | ||
4927 | u32 msr, int type) | ||
4928 | { | 4906 | { |
4929 | int f = sizeof(unsigned long); | 4907 | int f = sizeof(unsigned long); |
4930 | 4908 | ||
@@ -4958,6 +4936,50 @@ static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, | |||
4958 | } | 4936 | } |
4959 | } | 4937 | } |
4960 | 4938 | ||
4939 | static void __always_inline vmx_enable_intercept_for_msr(unsigned long *msr_bitmap, | ||
4940 | u32 msr, int type) | ||
4941 | { | ||
4942 | int f = sizeof(unsigned long); | ||
4943 | |||
4944 | if (!cpu_has_vmx_msr_bitmap()) | ||
4945 | return; | ||
4946 | |||
4947 | /* | ||
4948 | * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals | ||
4949 | * have the write-low and read-high bitmap offsets the wrong way round. | ||
4950 | * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. | ||
4951 | */ | ||
4952 | if (msr <= 0x1fff) { | ||
4953 | if (type & MSR_TYPE_R) | ||
4954 | /* read-low */ | ||
4955 | __set_bit(msr, msr_bitmap + 0x000 / f); | ||
4956 | |||
4957 | if (type & MSR_TYPE_W) | ||
4958 | /* write-low */ | ||
4959 | __set_bit(msr, msr_bitmap + 0x800 / f); | ||
4960 | |||
4961 | } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { | ||
4962 | msr &= 0x1fff; | ||
4963 | if (type & MSR_TYPE_R) | ||
4964 | /* read-high */ | ||
4965 | __set_bit(msr, msr_bitmap + 0x400 / f); | ||
4966 | |||
4967 | if (type & MSR_TYPE_W) | ||
4968 | /* write-high */ | ||
4969 | __set_bit(msr, msr_bitmap + 0xc00 / f); | ||
4970 | |||
4971 | } | ||
4972 | } | ||
4973 | |||
4974 | static void __always_inline vmx_set_intercept_for_msr(unsigned long *msr_bitmap, | ||
4975 | u32 msr, int type, bool value) | ||
4976 | { | ||
4977 | if (value) | ||
4978 | vmx_enable_intercept_for_msr(msr_bitmap, msr, type); | ||
4979 | else | ||
4980 | vmx_disable_intercept_for_msr(msr_bitmap, msr, type); | ||
4981 | } | ||
4982 | |||
4961 | /* | 4983 | /* |
4962 | * If a msr is allowed by L0, we should check whether it is allowed by L1. | 4984 | * If a msr is allowed by L0, we should check whether it is allowed by L1. |
4963 | * The corresponding bit will be cleared unless both of L0 and L1 allow it. | 4985 | * The corresponding bit will be cleared unless both of L0 and L1 allow it. |
@@ -5004,30 +5026,70 @@ static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1, | |||
5004 | } | 5026 | } |
5005 | } | 5027 | } |
5006 | 5028 | ||
5007 | static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only) | 5029 | static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu) |
5008 | { | 5030 | { |
5009 | if (!longmode_only) | 5031 | u8 mode = 0; |
5010 | __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, | 5032 | |
5011 | msr, MSR_TYPE_R | MSR_TYPE_W); | 5033 | if (cpu_has_secondary_exec_ctrls() && |
5012 | __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, | 5034 | (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) & |
5013 | msr, MSR_TYPE_R | MSR_TYPE_W); | 5035 | SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { |
5036 | mode |= MSR_BITMAP_MODE_X2APIC; | ||
5037 | if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) | ||
5038 | mode |= MSR_BITMAP_MODE_X2APIC_APICV; | ||
5039 | } | ||
5040 | |||
5041 | if (is_long_mode(vcpu)) | ||
5042 | mode |= MSR_BITMAP_MODE_LM; | ||
5043 | |||
5044 | return mode; | ||
5014 | } | 5045 | } |
5015 | 5046 | ||
5016 | static void vmx_disable_intercept_msr_x2apic(u32 msr, int type, bool apicv_active) | 5047 | #define X2APIC_MSR(r) (APIC_BASE_MSR + ((r) >> 4)) |
5048 | |||
5049 | static void vmx_update_msr_bitmap_x2apic(unsigned long *msr_bitmap, | ||
5050 | u8 mode) | ||
5017 | { | 5051 | { |
5018 | if (apicv_active) { | 5052 | int msr; |
5019 | __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv, | 5053 | |
5020 | msr, type); | 5054 | for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { |
5021 | __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv, | 5055 | unsigned word = msr / BITS_PER_LONG; |
5022 | msr, type); | 5056 | msr_bitmap[word] = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0; |
5023 | } else { | 5057 | msr_bitmap[word + (0x800 / sizeof(long))] = ~0; |
5024 | __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic, | 5058 | } |
5025 | msr, type); | 5059 | |
5026 | __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic, | 5060 | if (mode & MSR_BITMAP_MODE_X2APIC) { |
5027 | msr, type); | 5061 | /* |
5062 | * TPR reads and writes can be virtualized even if virtual interrupt | ||
5063 | * delivery is not in use. | ||
5064 | */ | ||
5065 | vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW); | ||
5066 | if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { | ||
5067 | vmx_enable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_R); | ||
5068 | vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); | ||
5069 | vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); | ||
5070 | } | ||
5028 | } | 5071 | } |
5029 | } | 5072 | } |
5030 | 5073 | ||
5074 | static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu) | ||
5075 | { | ||
5076 | struct vcpu_vmx *vmx = to_vmx(vcpu); | ||
5077 | unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; | ||
5078 | u8 mode = vmx_msr_bitmap_mode(vcpu); | ||
5079 | u8 changed = mode ^ vmx->msr_bitmap_mode; | ||
5080 | |||
5081 | if (!changed) | ||
5082 | return; | ||
5083 | |||
5084 | vmx_set_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW, | ||
5085 | !(mode & MSR_BITMAP_MODE_LM)); | ||
5086 | |||
5087 | if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV)) | ||
5088 | vmx_update_msr_bitmap_x2apic(msr_bitmap, mode); | ||
5089 | |||
5090 | vmx->msr_bitmap_mode = mode; | ||
5091 | } | ||
5092 | |||
5031 | static bool vmx_get_enable_apicv(struct kvm_vcpu *vcpu) | 5093 | static bool vmx_get_enable_apicv(struct kvm_vcpu *vcpu) |
5032 | { | 5094 | { |
5033 | return enable_apicv; | 5095 | return enable_apicv; |
@@ -5277,7 +5339,7 @@ static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) | |||
5277 | } | 5339 | } |
5278 | 5340 | ||
5279 | if (cpu_has_vmx_msr_bitmap()) | 5341 | if (cpu_has_vmx_msr_bitmap()) |
5280 | vmx_set_msr_bitmap(vcpu); | 5342 | vmx_update_msr_bitmap(vcpu); |
5281 | } | 5343 | } |
5282 | 5344 | ||
5283 | static u32 vmx_exec_control(struct vcpu_vmx *vmx) | 5345 | static u32 vmx_exec_control(struct vcpu_vmx *vmx) |
@@ -5464,7 +5526,7 @@ static void vmx_vcpu_setup(struct vcpu_vmx *vmx) | |||
5464 | vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); | 5526 | vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); |
5465 | } | 5527 | } |
5466 | if (cpu_has_vmx_msr_bitmap()) | 5528 | if (cpu_has_vmx_msr_bitmap()) |
5467 | vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy)); | 5529 | vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); |
5468 | 5530 | ||
5469 | vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */ | 5531 | vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */ |
5470 | 5532 | ||
@@ -6747,7 +6809,7 @@ void vmx_enable_tdp(void) | |||
6747 | 6809 | ||
6748 | static __init int hardware_setup(void) | 6810 | static __init int hardware_setup(void) |
6749 | { | 6811 | { |
6750 | int r = -ENOMEM, i, msr; | 6812 | int r = -ENOMEM, i; |
6751 | 6813 | ||
6752 | rdmsrl_safe(MSR_EFER, &host_efer); | 6814 | rdmsrl_safe(MSR_EFER, &host_efer); |
6753 | 6815 | ||
@@ -6767,9 +6829,6 @@ static __init int hardware_setup(void) | |||
6767 | 6829 | ||
6768 | memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE); | 6830 | memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE); |
6769 | 6831 | ||
6770 | memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE); | ||
6771 | memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE); | ||
6772 | |||
6773 | if (setup_vmcs_config(&vmcs_config) < 0) { | 6832 | if (setup_vmcs_config(&vmcs_config) < 0) { |
6774 | r = -EIO; | 6833 | r = -EIO; |
6775 | goto out; | 6834 | goto out; |
@@ -6838,42 +6897,8 @@ static __init int hardware_setup(void) | |||
6838 | kvm_tsc_scaling_ratio_frac_bits = 48; | 6897 | kvm_tsc_scaling_ratio_frac_bits = 48; |
6839 | } | 6898 | } |
6840 | 6899 | ||
6841 | vmx_disable_intercept_for_msr(MSR_FS_BASE, false); | ||
6842 | vmx_disable_intercept_for_msr(MSR_GS_BASE, false); | ||
6843 | vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true); | ||
6844 | vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false); | ||
6845 | vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false); | ||
6846 | vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false); | ||
6847 | |||
6848 | memcpy(vmx_msr_bitmap_legacy_x2apic_apicv, | ||
6849 | vmx_msr_bitmap_legacy, PAGE_SIZE); | ||
6850 | memcpy(vmx_msr_bitmap_longmode_x2apic_apicv, | ||
6851 | vmx_msr_bitmap_longmode, PAGE_SIZE); | ||
6852 | memcpy(vmx_msr_bitmap_legacy_x2apic, | ||
6853 | vmx_msr_bitmap_legacy, PAGE_SIZE); | ||
6854 | memcpy(vmx_msr_bitmap_longmode_x2apic, | ||
6855 | vmx_msr_bitmap_longmode, PAGE_SIZE); | ||
6856 | |||
6857 | set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ | 6900 | set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ |
6858 | 6901 | ||
6859 | for (msr = 0x800; msr <= 0x8ff; msr++) { | ||
6860 | if (msr == 0x839 /* TMCCT */) | ||
6861 | continue; | ||
6862 | vmx_disable_intercept_msr_x2apic(msr, MSR_TYPE_R, true); | ||
6863 | } | ||
6864 | |||
6865 | /* | ||
6866 | * TPR reads and writes can be virtualized even if virtual interrupt | ||
6867 | * delivery is not in use. | ||
6868 | */ | ||
6869 | vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_W, true); | ||
6870 | vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_R | MSR_TYPE_W, false); | ||
6871 | |||
6872 | /* EOI */ | ||
6873 | vmx_disable_intercept_msr_x2apic(0x80b, MSR_TYPE_W, true); | ||
6874 | /* SELF-IPI */ | ||
6875 | vmx_disable_intercept_msr_x2apic(0x83f, MSR_TYPE_W, true); | ||
6876 | |||
6877 | if (enable_ept) | 6902 | if (enable_ept) |
6878 | vmx_enable_tdp(); | 6903 | vmx_enable_tdp(); |
6879 | else | 6904 | else |
@@ -7162,13 +7187,6 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) | |||
7162 | if (r < 0) | 7187 | if (r < 0) |
7163 | goto out_vmcs02; | 7188 | goto out_vmcs02; |
7164 | 7189 | ||
7165 | if (cpu_has_vmx_msr_bitmap()) { | ||
7166 | vmx->nested.msr_bitmap = | ||
7167 | (unsigned long *)__get_free_page(GFP_KERNEL); | ||
7168 | if (!vmx->nested.msr_bitmap) | ||
7169 | goto out_msr_bitmap; | ||
7170 | } | ||
7171 | |||
7172 | vmx->nested.cached_vmcs12 = kmalloc(VMCS12_SIZE, GFP_KERNEL); | 7190 | vmx->nested.cached_vmcs12 = kmalloc(VMCS12_SIZE, GFP_KERNEL); |
7173 | if (!vmx->nested.cached_vmcs12) | 7191 | if (!vmx->nested.cached_vmcs12) |
7174 | goto out_cached_vmcs12; | 7192 | goto out_cached_vmcs12; |
@@ -7195,9 +7213,6 @@ out_shadow_vmcs: | |||
7195 | kfree(vmx->nested.cached_vmcs12); | 7213 | kfree(vmx->nested.cached_vmcs12); |
7196 | 7214 | ||
7197 | out_cached_vmcs12: | 7215 | out_cached_vmcs12: |
7198 | free_page((unsigned long)vmx->nested.msr_bitmap); | ||
7199 | |||
7200 | out_msr_bitmap: | ||
7201 | free_loaded_vmcs(&vmx->nested.vmcs02); | 7216 | free_loaded_vmcs(&vmx->nested.vmcs02); |
7202 | 7217 | ||
7203 | out_vmcs02: | 7218 | out_vmcs02: |
@@ -7343,10 +7358,6 @@ static void free_nested(struct vcpu_vmx *vmx) | |||
7343 | free_vpid(vmx->nested.vpid02); | 7358 | free_vpid(vmx->nested.vpid02); |
7344 | vmx->nested.posted_intr_nv = -1; | 7359 | vmx->nested.posted_intr_nv = -1; |
7345 | vmx->nested.current_vmptr = -1ull; | 7360 | vmx->nested.current_vmptr = -1ull; |
7346 | if (vmx->nested.msr_bitmap) { | ||
7347 | free_page((unsigned long)vmx->nested.msr_bitmap); | ||
7348 | vmx->nested.msr_bitmap = NULL; | ||
7349 | } | ||
7350 | if (enable_shadow_vmcs) { | 7361 | if (enable_shadow_vmcs) { |
7351 | vmx_disable_shadow_vmcs(vmx); | 7362 | vmx_disable_shadow_vmcs(vmx); |
7352 | vmcs_clear(vmx->vmcs01.shadow_vmcs); | 7363 | vmcs_clear(vmx->vmcs01.shadow_vmcs); |
@@ -8862,7 +8873,7 @@ static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set) | |||
8862 | } | 8873 | } |
8863 | vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control); | 8874 | vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control); |
8864 | 8875 | ||
8865 | vmx_set_msr_bitmap(vcpu); | 8876 | vmx_update_msr_bitmap(vcpu); |
8866 | } | 8877 | } |
8867 | 8878 | ||
8868 | static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa) | 8879 | static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa) |
@@ -9523,6 +9534,7 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) | |||
9523 | { | 9534 | { |
9524 | int err; | 9535 | int err; |
9525 | struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); | 9536 | struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); |
9537 | unsigned long *msr_bitmap; | ||
9526 | int cpu; | 9538 | int cpu; |
9527 | 9539 | ||
9528 | if (!vmx) | 9540 | if (!vmx) |
@@ -9559,6 +9571,15 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) | |||
9559 | if (err < 0) | 9571 | if (err < 0) |
9560 | goto free_msrs; | 9572 | goto free_msrs; |
9561 | 9573 | ||
9574 | msr_bitmap = vmx->vmcs01.msr_bitmap; | ||
9575 | vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE, MSR_TYPE_RW); | ||
9576 | vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE, MSR_TYPE_RW); | ||
9577 | vmx_disable_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); | ||
9578 | vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); | ||
9579 | vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); | ||
9580 | vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); | ||
9581 | vmx->msr_bitmap_mode = 0; | ||
9582 | |||
9562 | vmx->loaded_vmcs = &vmx->vmcs01; | 9583 | vmx->loaded_vmcs = &vmx->vmcs01; |
9563 | cpu = get_cpu(); | 9584 | cpu = get_cpu(); |
9564 | vmx_vcpu_load(&vmx->vcpu, cpu); | 9585 | vmx_vcpu_load(&vmx->vcpu, cpu); |
@@ -10022,7 +10043,7 @@ static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu, | |||
10022 | int msr; | 10043 | int msr; |
10023 | struct page *page; | 10044 | struct page *page; |
10024 | unsigned long *msr_bitmap_l1; | 10045 | unsigned long *msr_bitmap_l1; |
10025 | unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.msr_bitmap; | 10046 | unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap; |
10026 | 10047 | ||
10027 | /* This shortcut is ok because we support only x2APIC MSRs so far. */ | 10048 | /* This shortcut is ok because we support only x2APIC MSRs so far. */ |
10028 | if (!nested_cpu_has_virt_x2apic_mode(vmcs12)) | 10049 | if (!nested_cpu_has_virt_x2apic_mode(vmcs12)) |
@@ -10599,6 +10620,9 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, | |||
10599 | if (kvm_has_tsc_control) | 10620 | if (kvm_has_tsc_control) |
10600 | decache_tsc_multiplier(vmx); | 10621 | decache_tsc_multiplier(vmx); |
10601 | 10622 | ||
10623 | if (cpu_has_vmx_msr_bitmap()) | ||
10624 | vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); | ||
10625 | |||
10602 | if (enable_vpid) { | 10626 | if (enable_vpid) { |
10603 | /* | 10627 | /* |
10604 | * There is no direct mapping between vpid02 and vpid12, the | 10628 | * There is no direct mapping between vpid02 and vpid12, the |
@@ -11397,7 +11421,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, | |||
11397 | vmcs_write64(GUEST_IA32_DEBUGCTL, 0); | 11421 | vmcs_write64(GUEST_IA32_DEBUGCTL, 0); |
11398 | 11422 | ||
11399 | if (cpu_has_vmx_msr_bitmap()) | 11423 | if (cpu_has_vmx_msr_bitmap()) |
11400 | vmx_set_msr_bitmap(vcpu); | 11424 | vmx_update_msr_bitmap(vcpu); |
11401 | 11425 | ||
11402 | if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, | 11426 | if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, |
11403 | vmcs12->vm_exit_msr_load_count)) | 11427 | vmcs12->vm_exit_msr_load_count)) |