diff options
author | Ingo Molnar <mingo@elte.hu> | 2008-10-28 11:26:12 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2008-10-28 11:26:12 -0400 |
commit | 7a9787e1eba95a166265e6a260cf30af04ef0a99 (patch) | |
tree | e730a4565e0318140d2fbd2f0415d18a339d7336 /arch/x86/kvm/x86.c | |
parent | 41b9eb264c8407655db57b60b4457fe1b2ec9977 (diff) | |
parent | 0173a3265b228da319ceb9c1ec6a5682fd1b2d92 (diff) |
Merge commit 'v2.6.28-rc2' into x86/pci-ioapic-boot-irq-quirks
Diffstat (limited to 'arch/x86/kvm/x86.c')
-rw-r--r-- | arch/x86/kvm/x86.c | 984 |
1 files changed, 544 insertions, 440 deletions
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0faa2546b1cd..4f0677d1eae8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c | |||
@@ -4,10 +4,14 @@ | |||
4 | * derived from drivers/kvm/kvm_main.c | 4 | * derived from drivers/kvm/kvm_main.c |
5 | * | 5 | * |
6 | * Copyright (C) 2006 Qumranet, Inc. | 6 | * Copyright (C) 2006 Qumranet, Inc. |
7 | * Copyright (C) 2008 Qumranet, Inc. | ||
8 | * Copyright IBM Corporation, 2008 | ||
7 | * | 9 | * |
8 | * Authors: | 10 | * Authors: |
9 | * Avi Kivity <avi@qumranet.com> | 11 | * Avi Kivity <avi@qumranet.com> |
10 | * Yaniv Kamay <yaniv@qumranet.com> | 12 | * Yaniv Kamay <yaniv@qumranet.com> |
13 | * Amit Shah <amit.shah@qumranet.com> | ||
14 | * Ben-Ami Yassour <benami@il.ibm.com> | ||
11 | * | 15 | * |
12 | * This work is licensed under the terms of the GNU GPL, version 2. See | 16 | * This work is licensed under the terms of the GNU GPL, version 2. See |
13 | * the COPYING file in the top-level directory. | 17 | * the COPYING file in the top-level directory. |
@@ -19,14 +23,18 @@ | |||
19 | #include "mmu.h" | 23 | #include "mmu.h" |
20 | #include "i8254.h" | 24 | #include "i8254.h" |
21 | #include "tss.h" | 25 | #include "tss.h" |
26 | #include "kvm_cache_regs.h" | ||
27 | #include "x86.h" | ||
22 | 28 | ||
23 | #include <linux/clocksource.h> | 29 | #include <linux/clocksource.h> |
30 | #include <linux/interrupt.h> | ||
24 | #include <linux/kvm.h> | 31 | #include <linux/kvm.h> |
25 | #include <linux/fs.h> | 32 | #include <linux/fs.h> |
26 | #include <linux/vmalloc.h> | 33 | #include <linux/vmalloc.h> |
27 | #include <linux/module.h> | 34 | #include <linux/module.h> |
28 | #include <linux/mman.h> | 35 | #include <linux/mman.h> |
29 | #include <linux/highmem.h> | 36 | #include <linux/highmem.h> |
37 | #include <linux/intel-iommu.h> | ||
30 | 38 | ||
31 | #include <asm/uaccess.h> | 39 | #include <asm/uaccess.h> |
32 | #include <asm/msr.h> | 40 | #include <asm/msr.h> |
@@ -61,6 +69,7 @@ static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, | |||
61 | struct kvm_cpuid_entry2 __user *entries); | 69 | struct kvm_cpuid_entry2 __user *entries); |
62 | 70 | ||
63 | struct kvm_x86_ops *kvm_x86_ops; | 71 | struct kvm_x86_ops *kvm_x86_ops; |
72 | EXPORT_SYMBOL_GPL(kvm_x86_ops); | ||
64 | 73 | ||
65 | struct kvm_stats_debugfs_item debugfs_entries[] = { | 74 | struct kvm_stats_debugfs_item debugfs_entries[] = { |
66 | { "pf_fixed", VCPU_STAT(pf_fixed) }, | 75 | { "pf_fixed", VCPU_STAT(pf_fixed) }, |
@@ -72,6 +81,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { | |||
72 | { "mmio_exits", VCPU_STAT(mmio_exits) }, | 81 | { "mmio_exits", VCPU_STAT(mmio_exits) }, |
73 | { "signal_exits", VCPU_STAT(signal_exits) }, | 82 | { "signal_exits", VCPU_STAT(signal_exits) }, |
74 | { "irq_window", VCPU_STAT(irq_window_exits) }, | 83 | { "irq_window", VCPU_STAT(irq_window_exits) }, |
84 | { "nmi_window", VCPU_STAT(nmi_window_exits) }, | ||
75 | { "halt_exits", VCPU_STAT(halt_exits) }, | 85 | { "halt_exits", VCPU_STAT(halt_exits) }, |
76 | { "halt_wakeup", VCPU_STAT(halt_wakeup) }, | 86 | { "halt_wakeup", VCPU_STAT(halt_wakeup) }, |
77 | { "hypercalls", VCPU_STAT(hypercalls) }, | 87 | { "hypercalls", VCPU_STAT(hypercalls) }, |
@@ -82,6 +92,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { | |||
82 | { "fpu_reload", VCPU_STAT(fpu_reload) }, | 92 | { "fpu_reload", VCPU_STAT(fpu_reload) }, |
83 | { "insn_emulation", VCPU_STAT(insn_emulation) }, | 93 | { "insn_emulation", VCPU_STAT(insn_emulation) }, |
84 | { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, | 94 | { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, |
95 | { "irq_injections", VCPU_STAT(irq_injections) }, | ||
85 | { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, | 96 | { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, |
86 | { "mmu_pte_write", VM_STAT(mmu_pte_write) }, | 97 | { "mmu_pte_write", VM_STAT(mmu_pte_write) }, |
87 | { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, | 98 | { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, |
@@ -89,12 +100,12 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { | |||
89 | { "mmu_flooded", VM_STAT(mmu_flooded) }, | 100 | { "mmu_flooded", VM_STAT(mmu_flooded) }, |
90 | { "mmu_recycled", VM_STAT(mmu_recycled) }, | 101 | { "mmu_recycled", VM_STAT(mmu_recycled) }, |
91 | { "mmu_cache_miss", VM_STAT(mmu_cache_miss) }, | 102 | { "mmu_cache_miss", VM_STAT(mmu_cache_miss) }, |
103 | { "mmu_unsync", VM_STAT(mmu_unsync) }, | ||
92 | { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, | 104 | { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, |
93 | { "largepages", VM_STAT(lpages) }, | 105 | { "largepages", VM_STAT(lpages) }, |
94 | { NULL } | 106 | { NULL } |
95 | }; | 107 | }; |
96 | 108 | ||
97 | |||
98 | unsigned long segment_base(u16 selector) | 109 | unsigned long segment_base(u16 selector) |
99 | { | 110 | { |
100 | struct descriptor_table gdt; | 111 | struct descriptor_table gdt; |
@@ -173,6 +184,12 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, | |||
173 | kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); | 184 | kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); |
174 | } | 185 | } |
175 | 186 | ||
187 | void kvm_inject_nmi(struct kvm_vcpu *vcpu) | ||
188 | { | ||
189 | vcpu->arch.nmi_pending = 1; | ||
190 | } | ||
191 | EXPORT_SYMBOL_GPL(kvm_inject_nmi); | ||
192 | |||
176 | void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) | 193 | void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) |
177 | { | 194 | { |
178 | WARN_ON(vcpu->arch.exception.pending); | 195 | WARN_ON(vcpu->arch.exception.pending); |
@@ -345,6 +362,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cr4); | |||
345 | void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) | 362 | void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) |
346 | { | 363 | { |
347 | if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) { | 364 | if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) { |
365 | kvm_mmu_sync_roots(vcpu); | ||
348 | kvm_mmu_flush_tlb(vcpu); | 366 | kvm_mmu_flush_tlb(vcpu); |
349 | return; | 367 | return; |
350 | } | 368 | } |
@@ -557,7 +575,7 @@ static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info * | |||
557 | hv_clock->tsc_to_system_mul = div_frac(nsecs, tps32); | 575 | hv_clock->tsc_to_system_mul = div_frac(nsecs, tps32); |
558 | 576 | ||
559 | pr_debug("%s: tsc_khz %u, tsc_shift %d, tsc_mul %u\n", | 577 | pr_debug("%s: tsc_khz %u, tsc_shift %d, tsc_mul %u\n", |
560 | __FUNCTION__, tsc_khz, hv_clock->tsc_shift, | 578 | __func__, tsc_khz, hv_clock->tsc_shift, |
561 | hv_clock->tsc_to_system_mul); | 579 | hv_clock->tsc_to_system_mul); |
562 | } | 580 | } |
563 | 581 | ||
@@ -604,6 +622,38 @@ static void kvm_write_guest_time(struct kvm_vcpu *v) | |||
604 | mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT); | 622 | mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT); |
605 | } | 623 | } |
606 | 624 | ||
625 | static bool msr_mtrr_valid(unsigned msr) | ||
626 | { | ||
627 | switch (msr) { | ||
628 | case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1: | ||
629 | case MSR_MTRRfix64K_00000: | ||
630 | case MSR_MTRRfix16K_80000: | ||
631 | case MSR_MTRRfix16K_A0000: | ||
632 | case MSR_MTRRfix4K_C0000: | ||
633 | case MSR_MTRRfix4K_C8000: | ||
634 | case MSR_MTRRfix4K_D0000: | ||
635 | case MSR_MTRRfix4K_D8000: | ||
636 | case MSR_MTRRfix4K_E0000: | ||
637 | case MSR_MTRRfix4K_E8000: | ||
638 | case MSR_MTRRfix4K_F0000: | ||
639 | case MSR_MTRRfix4K_F8000: | ||
640 | case MSR_MTRRdefType: | ||
641 | case MSR_IA32_CR_PAT: | ||
642 | return true; | ||
643 | case 0x2f8: | ||
644 | return true; | ||
645 | } | ||
646 | return false; | ||
647 | } | ||
648 | |||
649 | static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data) | ||
650 | { | ||
651 | if (!msr_mtrr_valid(msr)) | ||
652 | return 1; | ||
653 | |||
654 | vcpu->arch.mtrr[msr - 0x200] = data; | ||
655 | return 0; | ||
656 | } | ||
607 | 657 | ||
608 | int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | 658 | int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) |
609 | { | 659 | { |
@@ -623,10 +673,23 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | |||
623 | pr_unimpl(vcpu, "%s: MSR_IA32_MCG_CTL 0x%llx, nop\n", | 673 | pr_unimpl(vcpu, "%s: MSR_IA32_MCG_CTL 0x%llx, nop\n", |
624 | __func__, data); | 674 | __func__, data); |
625 | break; | 675 | break; |
676 | case MSR_IA32_DEBUGCTLMSR: | ||
677 | if (!data) { | ||
678 | /* We support the non-activated case already */ | ||
679 | break; | ||
680 | } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) { | ||
681 | /* Values other than LBR and BTF are vendor-specific, | ||
682 | thus reserved and should throw a #GP */ | ||
683 | return 1; | ||
684 | } | ||
685 | pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n", | ||
686 | __func__, data); | ||
687 | break; | ||
626 | case MSR_IA32_UCODE_REV: | 688 | case MSR_IA32_UCODE_REV: |
627 | case MSR_IA32_UCODE_WRITE: | 689 | case MSR_IA32_UCODE_WRITE: |
628 | case 0x200 ... 0x2ff: /* MTRRs */ | ||
629 | break; | 690 | break; |
691 | case 0x200 ... 0x2ff: | ||
692 | return set_msr_mtrr(vcpu, msr, data); | ||
630 | case MSR_IA32_APICBASE: | 693 | case MSR_IA32_APICBASE: |
631 | kvm_set_apic_base(vcpu, data); | 694 | kvm_set_apic_base(vcpu, data); |
632 | break; | 695 | break; |
@@ -652,10 +715,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | |||
652 | /* ...but clean it before doing the actual write */ | 715 | /* ...but clean it before doing the actual write */ |
653 | vcpu->arch.time_offset = data & ~(PAGE_MASK | 1); | 716 | vcpu->arch.time_offset = data & ~(PAGE_MASK | 1); |
654 | 717 | ||
655 | down_read(¤t->mm->mmap_sem); | ||
656 | vcpu->arch.time_page = | 718 | vcpu->arch.time_page = |
657 | gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT); | 719 | gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT); |
658 | up_read(¤t->mm->mmap_sem); | ||
659 | 720 | ||
660 | if (is_error_page(vcpu->arch.time_page)) { | 721 | if (is_error_page(vcpu->arch.time_page)) { |
661 | kvm_release_page_clean(vcpu->arch.time_page); | 722 | kvm_release_page_clean(vcpu->arch.time_page); |
@@ -684,6 +745,15 @@ int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) | |||
684 | return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); | 745 | return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); |
685 | } | 746 | } |
686 | 747 | ||
748 | static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) | ||
749 | { | ||
750 | if (!msr_mtrr_valid(msr)) | ||
751 | return 1; | ||
752 | |||
753 | *pdata = vcpu->arch.mtrr[msr - 0x200]; | ||
754 | return 0; | ||
755 | } | ||
756 | |||
687 | int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) | 757 | int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) |
688 | { | 758 | { |
689 | u64 data; | 759 | u64 data; |
@@ -703,13 +773,21 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) | |||
703 | case MSR_IA32_MC0_MISC+8: | 773 | case MSR_IA32_MC0_MISC+8: |
704 | case MSR_IA32_MC0_MISC+12: | 774 | case MSR_IA32_MC0_MISC+12: |
705 | case MSR_IA32_MC0_MISC+16: | 775 | case MSR_IA32_MC0_MISC+16: |
776 | case MSR_IA32_MC0_MISC+20: | ||
706 | case MSR_IA32_UCODE_REV: | 777 | case MSR_IA32_UCODE_REV: |
707 | case MSR_IA32_EBL_CR_POWERON: | 778 | case MSR_IA32_EBL_CR_POWERON: |
708 | /* MTRR registers */ | 779 | case MSR_IA32_DEBUGCTLMSR: |
709 | case 0xfe: | 780 | case MSR_IA32_LASTBRANCHFROMIP: |
710 | case 0x200 ... 0x2ff: | 781 | case MSR_IA32_LASTBRANCHTOIP: |
782 | case MSR_IA32_LASTINTFROMIP: | ||
783 | case MSR_IA32_LASTINTTOIP: | ||
711 | data = 0; | 784 | data = 0; |
712 | break; | 785 | break; |
786 | case MSR_MTRRcap: | ||
787 | data = 0x500 | KVM_NR_VAR_MTRR; | ||
788 | break; | ||
789 | case 0x200 ... 0x2ff: | ||
790 | return get_msr_mtrr(vcpu, msr, pdata); | ||
713 | case 0xcd: /* fsb frequency */ | 791 | case 0xcd: /* fsb frequency */ |
714 | data = 3; | 792 | data = 3; |
715 | break; | 793 | break; |
@@ -817,41 +895,6 @@ out: | |||
817 | return r; | 895 | return r; |
818 | } | 896 | } |
819 | 897 | ||
820 | /* | ||
821 | * Make sure that a cpu that is being hot-unplugged does not have any vcpus | ||
822 | * cached on it. | ||
823 | */ | ||
824 | void decache_vcpus_on_cpu(int cpu) | ||
825 | { | ||
826 | struct kvm *vm; | ||
827 | struct kvm_vcpu *vcpu; | ||
828 | int i; | ||
829 | |||
830 | spin_lock(&kvm_lock); | ||
831 | list_for_each_entry(vm, &vm_list, vm_list) | ||
832 | for (i = 0; i < KVM_MAX_VCPUS; ++i) { | ||
833 | vcpu = vm->vcpus[i]; | ||
834 | if (!vcpu) | ||
835 | continue; | ||
836 | /* | ||
837 | * If the vcpu is locked, then it is running on some | ||
838 | * other cpu and therefore it is not cached on the | ||
839 | * cpu in question. | ||
840 | * | ||
841 | * If it's not locked, check the last cpu it executed | ||
842 | * on. | ||
843 | */ | ||
844 | if (mutex_trylock(&vcpu->mutex)) { | ||
845 | if (vcpu->cpu == cpu) { | ||
846 | kvm_x86_ops->vcpu_decache(vcpu); | ||
847 | vcpu->cpu = -1; | ||
848 | } | ||
849 | mutex_unlock(&vcpu->mutex); | ||
850 | } | ||
851 | } | ||
852 | spin_unlock(&kvm_lock); | ||
853 | } | ||
854 | |||
855 | int kvm_dev_ioctl_check_extension(long ext) | 898 | int kvm_dev_ioctl_check_extension(long ext) |
856 | { | 899 | { |
857 | int r; | 900 | int r; |
@@ -867,8 +910,12 @@ int kvm_dev_ioctl_check_extension(long ext) | |||
867 | case KVM_CAP_PIT: | 910 | case KVM_CAP_PIT: |
868 | case KVM_CAP_NOP_IO_DELAY: | 911 | case KVM_CAP_NOP_IO_DELAY: |
869 | case KVM_CAP_MP_STATE: | 912 | case KVM_CAP_MP_STATE: |
913 | case KVM_CAP_SYNC_MMU: | ||
870 | r = 1; | 914 | r = 1; |
871 | break; | 915 | break; |
916 | case KVM_CAP_COALESCED_MMIO: | ||
917 | r = KVM_COALESCED_MMIO_PAGE_OFFSET; | ||
918 | break; | ||
872 | case KVM_CAP_VAPIC: | 919 | case KVM_CAP_VAPIC: |
873 | r = !kvm_x86_ops->cpu_has_accelerated_tpr(); | 920 | r = !kvm_x86_ops->cpu_has_accelerated_tpr(); |
874 | break; | 921 | break; |
@@ -881,6 +928,9 @@ int kvm_dev_ioctl_check_extension(long ext) | |||
881 | case KVM_CAP_PV_MMU: | 928 | case KVM_CAP_PV_MMU: |
882 | r = !tdp_enabled; | 929 | r = !tdp_enabled; |
883 | break; | 930 | break; |
931 | case KVM_CAP_IOMMU: | ||
932 | r = intel_iommu_found(); | ||
933 | break; | ||
884 | default: | 934 | default: |
885 | r = 0; | 935 | r = 0; |
886 | break; | 936 | break; |
@@ -1283,28 +1333,33 @@ long kvm_arch_vcpu_ioctl(struct file *filp, | |||
1283 | struct kvm_vcpu *vcpu = filp->private_data; | 1333 | struct kvm_vcpu *vcpu = filp->private_data; |
1284 | void __user *argp = (void __user *)arg; | 1334 | void __user *argp = (void __user *)arg; |
1285 | int r; | 1335 | int r; |
1336 | struct kvm_lapic_state *lapic = NULL; | ||
1286 | 1337 | ||
1287 | switch (ioctl) { | 1338 | switch (ioctl) { |
1288 | case KVM_GET_LAPIC: { | 1339 | case KVM_GET_LAPIC: { |
1289 | struct kvm_lapic_state lapic; | 1340 | lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); |
1290 | 1341 | ||
1291 | memset(&lapic, 0, sizeof lapic); | 1342 | r = -ENOMEM; |
1292 | r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic); | 1343 | if (!lapic) |
1344 | goto out; | ||
1345 | r = kvm_vcpu_ioctl_get_lapic(vcpu, lapic); | ||
1293 | if (r) | 1346 | if (r) |
1294 | goto out; | 1347 | goto out; |
1295 | r = -EFAULT; | 1348 | r = -EFAULT; |
1296 | if (copy_to_user(argp, &lapic, sizeof lapic)) | 1349 | if (copy_to_user(argp, lapic, sizeof(struct kvm_lapic_state))) |
1297 | goto out; | 1350 | goto out; |
1298 | r = 0; | 1351 | r = 0; |
1299 | break; | 1352 | break; |
1300 | } | 1353 | } |
1301 | case KVM_SET_LAPIC: { | 1354 | case KVM_SET_LAPIC: { |
1302 | struct kvm_lapic_state lapic; | 1355 | lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); |
1303 | 1356 | r = -ENOMEM; | |
1357 | if (!lapic) | ||
1358 | goto out; | ||
1304 | r = -EFAULT; | 1359 | r = -EFAULT; |
1305 | if (copy_from_user(&lapic, argp, sizeof lapic)) | 1360 | if (copy_from_user(lapic, argp, sizeof(struct kvm_lapic_state))) |
1306 | goto out; | 1361 | goto out; |
1307 | r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);; | 1362 | r = kvm_vcpu_ioctl_set_lapic(vcpu, lapic); |
1308 | if (r) | 1363 | if (r) |
1309 | goto out; | 1364 | goto out; |
1310 | r = 0; | 1365 | r = 0; |
@@ -1402,6 +1457,8 @@ long kvm_arch_vcpu_ioctl(struct file *filp, | |||
1402 | r = -EINVAL; | 1457 | r = -EINVAL; |
1403 | } | 1458 | } |
1404 | out: | 1459 | out: |
1460 | if (lapic) | ||
1461 | kfree(lapic); | ||
1405 | return r; | 1462 | return r; |
1406 | } | 1463 | } |
1407 | 1464 | ||
@@ -1476,6 +1533,7 @@ static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, | |||
1476 | goto out; | 1533 | goto out; |
1477 | 1534 | ||
1478 | down_write(&kvm->slots_lock); | 1535 | down_write(&kvm->slots_lock); |
1536 | spin_lock(&kvm->mmu_lock); | ||
1479 | 1537 | ||
1480 | p = &kvm->arch.aliases[alias->slot]; | 1538 | p = &kvm->arch.aliases[alias->slot]; |
1481 | p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; | 1539 | p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; |
@@ -1487,6 +1545,7 @@ static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, | |||
1487 | break; | 1545 | break; |
1488 | kvm->arch.naliases = n; | 1546 | kvm->arch.naliases = n; |
1489 | 1547 | ||
1548 | spin_unlock(&kvm->mmu_lock); | ||
1490 | kvm_mmu_zap_all(kvm); | 1549 | kvm_mmu_zap_all(kvm); |
1491 | 1550 | ||
1492 | up_write(&kvm->slots_lock); | 1551 | up_write(&kvm->slots_lock); |
@@ -1608,6 +1667,15 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
1608 | struct kvm *kvm = filp->private_data; | 1667 | struct kvm *kvm = filp->private_data; |
1609 | void __user *argp = (void __user *)arg; | 1668 | void __user *argp = (void __user *)arg; |
1610 | int r = -EINVAL; | 1669 | int r = -EINVAL; |
1670 | /* | ||
1671 | * This union makes it completely explicit to gcc-3.x | ||
1672 | * that these two variables' stack usage should be | ||
1673 | * combined, not added together. | ||
1674 | */ | ||
1675 | union { | ||
1676 | struct kvm_pit_state ps; | ||
1677 | struct kvm_memory_alias alias; | ||
1678 | } u; | ||
1611 | 1679 | ||
1612 | switch (ioctl) { | 1680 | switch (ioctl) { |
1613 | case KVM_SET_TSS_ADDR: | 1681 | case KVM_SET_TSS_ADDR: |
@@ -1639,17 +1707,14 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
1639 | case KVM_GET_NR_MMU_PAGES: | 1707 | case KVM_GET_NR_MMU_PAGES: |
1640 | r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); | 1708 | r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); |
1641 | break; | 1709 | break; |
1642 | case KVM_SET_MEMORY_ALIAS: { | 1710 | case KVM_SET_MEMORY_ALIAS: |
1643 | struct kvm_memory_alias alias; | ||
1644 | |||
1645 | r = -EFAULT; | 1711 | r = -EFAULT; |
1646 | if (copy_from_user(&alias, argp, sizeof alias)) | 1712 | if (copy_from_user(&u.alias, argp, sizeof(struct kvm_memory_alias))) |
1647 | goto out; | 1713 | goto out; |
1648 | r = kvm_vm_ioctl_set_memory_alias(kvm, &alias); | 1714 | r = kvm_vm_ioctl_set_memory_alias(kvm, &u.alias); |
1649 | if (r) | 1715 | if (r) |
1650 | goto out; | 1716 | goto out; |
1651 | break; | 1717 | break; |
1652 | } | ||
1653 | case KVM_CREATE_IRQCHIP: | 1718 | case KVM_CREATE_IRQCHIP: |
1654 | r = -ENOMEM; | 1719 | r = -ENOMEM; |
1655 | kvm->arch.vpic = kvm_create_pic(kvm); | 1720 | kvm->arch.vpic = kvm_create_pic(kvm); |
@@ -1677,13 +1742,7 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
1677 | goto out; | 1742 | goto out; |
1678 | if (irqchip_in_kernel(kvm)) { | 1743 | if (irqchip_in_kernel(kvm)) { |
1679 | mutex_lock(&kvm->lock); | 1744 | mutex_lock(&kvm->lock); |
1680 | if (irq_event.irq < 16) | 1745 | kvm_set_irq(kvm, irq_event.irq, irq_event.level); |
1681 | kvm_pic_set_irq(pic_irqchip(kvm), | ||
1682 | irq_event.irq, | ||
1683 | irq_event.level); | ||
1684 | kvm_ioapic_set_irq(kvm->arch.vioapic, | ||
1685 | irq_event.irq, | ||
1686 | irq_event.level); | ||
1687 | mutex_unlock(&kvm->lock); | 1746 | mutex_unlock(&kvm->lock); |
1688 | r = 0; | 1747 | r = 0; |
1689 | } | 1748 | } |
@@ -1691,65 +1750,77 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
1691 | } | 1750 | } |
1692 | case KVM_GET_IRQCHIP: { | 1751 | case KVM_GET_IRQCHIP: { |
1693 | /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ | 1752 | /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ |
1694 | struct kvm_irqchip chip; | 1753 | struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL); |
1695 | 1754 | ||
1696 | r = -EFAULT; | 1755 | r = -ENOMEM; |
1697 | if (copy_from_user(&chip, argp, sizeof chip)) | 1756 | if (!chip) |
1698 | goto out; | 1757 | goto out; |
1758 | r = -EFAULT; | ||
1759 | if (copy_from_user(chip, argp, sizeof *chip)) | ||
1760 | goto get_irqchip_out; | ||
1699 | r = -ENXIO; | 1761 | r = -ENXIO; |
1700 | if (!irqchip_in_kernel(kvm)) | 1762 | if (!irqchip_in_kernel(kvm)) |
1701 | goto out; | 1763 | goto get_irqchip_out; |
1702 | r = kvm_vm_ioctl_get_irqchip(kvm, &chip); | 1764 | r = kvm_vm_ioctl_get_irqchip(kvm, chip); |
1703 | if (r) | 1765 | if (r) |
1704 | goto out; | 1766 | goto get_irqchip_out; |
1705 | r = -EFAULT; | 1767 | r = -EFAULT; |
1706 | if (copy_to_user(argp, &chip, sizeof chip)) | 1768 | if (copy_to_user(argp, chip, sizeof *chip)) |
1707 | goto out; | 1769 | goto get_irqchip_out; |
1708 | r = 0; | 1770 | r = 0; |
1771 | get_irqchip_out: | ||
1772 | kfree(chip); | ||
1773 | if (r) | ||
1774 | goto out; | ||
1709 | break; | 1775 | break; |
1710 | } | 1776 | } |
1711 | case KVM_SET_IRQCHIP: { | 1777 | case KVM_SET_IRQCHIP: { |
1712 | /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ | 1778 | /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ |
1713 | struct kvm_irqchip chip; | 1779 | struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL); |
1714 | 1780 | ||
1715 | r = -EFAULT; | 1781 | r = -ENOMEM; |
1716 | if (copy_from_user(&chip, argp, sizeof chip)) | 1782 | if (!chip) |
1717 | goto out; | 1783 | goto out; |
1784 | r = -EFAULT; | ||
1785 | if (copy_from_user(chip, argp, sizeof *chip)) | ||
1786 | goto set_irqchip_out; | ||
1718 | r = -ENXIO; | 1787 | r = -ENXIO; |
1719 | if (!irqchip_in_kernel(kvm)) | 1788 | if (!irqchip_in_kernel(kvm)) |
1720 | goto out; | 1789 | goto set_irqchip_out; |
1721 | r = kvm_vm_ioctl_set_irqchip(kvm, &chip); | 1790 | r = kvm_vm_ioctl_set_irqchip(kvm, chip); |
1722 | if (r) | 1791 | if (r) |
1723 | goto out; | 1792 | goto set_irqchip_out; |
1724 | r = 0; | 1793 | r = 0; |
1794 | set_irqchip_out: | ||
1795 | kfree(chip); | ||
1796 | if (r) | ||
1797 | goto out; | ||
1725 | break; | 1798 | break; |
1726 | } | 1799 | } |
1727 | case KVM_GET_PIT: { | 1800 | case KVM_GET_PIT: { |
1728 | struct kvm_pit_state ps; | ||
1729 | r = -EFAULT; | 1801 | r = -EFAULT; |
1730 | if (copy_from_user(&ps, argp, sizeof ps)) | 1802 | if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state))) |
1731 | goto out; | 1803 | goto out; |
1732 | r = -ENXIO; | 1804 | r = -ENXIO; |
1733 | if (!kvm->arch.vpit) | 1805 | if (!kvm->arch.vpit) |
1734 | goto out; | 1806 | goto out; |
1735 | r = kvm_vm_ioctl_get_pit(kvm, &ps); | 1807 | r = kvm_vm_ioctl_get_pit(kvm, &u.ps); |
1736 | if (r) | 1808 | if (r) |
1737 | goto out; | 1809 | goto out; |
1738 | r = -EFAULT; | 1810 | r = -EFAULT; |
1739 | if (copy_to_user(argp, &ps, sizeof ps)) | 1811 | if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state))) |
1740 | goto out; | 1812 | goto out; |
1741 | r = 0; | 1813 | r = 0; |
1742 | break; | 1814 | break; |
1743 | } | 1815 | } |
1744 | case KVM_SET_PIT: { | 1816 | case KVM_SET_PIT: { |
1745 | struct kvm_pit_state ps; | ||
1746 | r = -EFAULT; | 1817 | r = -EFAULT; |
1747 | if (copy_from_user(&ps, argp, sizeof ps)) | 1818 | if (copy_from_user(&u.ps, argp, sizeof u.ps)) |
1748 | goto out; | 1819 | goto out; |
1749 | r = -ENXIO; | 1820 | r = -ENXIO; |
1750 | if (!kvm->arch.vpit) | 1821 | if (!kvm->arch.vpit) |
1751 | goto out; | 1822 | goto out; |
1752 | r = kvm_vm_ioctl_set_pit(kvm, &ps); | 1823 | r = kvm_vm_ioctl_set_pit(kvm, &u.ps); |
1753 | if (r) | 1824 | if (r) |
1754 | goto out; | 1825 | goto out; |
1755 | r = 0; | 1826 | r = 0; |
@@ -1781,13 +1852,14 @@ static void kvm_init_msr_list(void) | |||
1781 | * Only apic need an MMIO device hook, so shortcut now.. | 1852 | * Only apic need an MMIO device hook, so shortcut now.. |
1782 | */ | 1853 | */ |
1783 | static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, | 1854 | static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, |
1784 | gpa_t addr) | 1855 | gpa_t addr, int len, |
1856 | int is_write) | ||
1785 | { | 1857 | { |
1786 | struct kvm_io_device *dev; | 1858 | struct kvm_io_device *dev; |
1787 | 1859 | ||
1788 | if (vcpu->arch.apic) { | 1860 | if (vcpu->arch.apic) { |
1789 | dev = &vcpu->arch.apic->dev; | 1861 | dev = &vcpu->arch.apic->dev; |
1790 | if (dev->in_range(dev, addr)) | 1862 | if (dev->in_range(dev, addr, len, is_write)) |
1791 | return dev; | 1863 | return dev; |
1792 | } | 1864 | } |
1793 | return NULL; | 1865 | return NULL; |
@@ -1795,13 +1867,15 @@ static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, | |||
1795 | 1867 | ||
1796 | 1868 | ||
1797 | static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, | 1869 | static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, |
1798 | gpa_t addr) | 1870 | gpa_t addr, int len, |
1871 | int is_write) | ||
1799 | { | 1872 | { |
1800 | struct kvm_io_device *dev; | 1873 | struct kvm_io_device *dev; |
1801 | 1874 | ||
1802 | dev = vcpu_find_pervcpu_dev(vcpu, addr); | 1875 | dev = vcpu_find_pervcpu_dev(vcpu, addr, len, is_write); |
1803 | if (dev == NULL) | 1876 | if (dev == NULL) |
1804 | dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr); | 1877 | dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr, len, |
1878 | is_write); | ||
1805 | return dev; | 1879 | return dev; |
1806 | } | 1880 | } |
1807 | 1881 | ||
@@ -1869,7 +1943,7 @@ mmio: | |||
1869 | * Is this MMIO handled locally? | 1943 | * Is this MMIO handled locally? |
1870 | */ | 1944 | */ |
1871 | mutex_lock(&vcpu->kvm->lock); | 1945 | mutex_lock(&vcpu->kvm->lock); |
1872 | mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); | 1946 | mmio_dev = vcpu_find_mmio_dev(vcpu, gpa, bytes, 0); |
1873 | if (mmio_dev) { | 1947 | if (mmio_dev) { |
1874 | kvm_iodevice_read(mmio_dev, gpa, bytes, val); | 1948 | kvm_iodevice_read(mmio_dev, gpa, bytes, val); |
1875 | mutex_unlock(&vcpu->kvm->lock); | 1949 | mutex_unlock(&vcpu->kvm->lock); |
@@ -1924,7 +1998,7 @@ mmio: | |||
1924 | * Is this MMIO handled locally? | 1998 | * Is this MMIO handled locally? |
1925 | */ | 1999 | */ |
1926 | mutex_lock(&vcpu->kvm->lock); | 2000 | mutex_lock(&vcpu->kvm->lock); |
1927 | mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); | 2001 | mmio_dev = vcpu_find_mmio_dev(vcpu, gpa, bytes, 1); |
1928 | if (mmio_dev) { | 2002 | if (mmio_dev) { |
1929 | kvm_iodevice_write(mmio_dev, gpa, bytes, val); | 2003 | kvm_iodevice_write(mmio_dev, gpa, bytes, val); |
1930 | mutex_unlock(&vcpu->kvm->lock); | 2004 | mutex_unlock(&vcpu->kvm->lock); |
@@ -1993,9 +2067,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr, | |||
1993 | 2067 | ||
1994 | val = *(u64 *)new; | 2068 | val = *(u64 *)new; |
1995 | 2069 | ||
1996 | down_read(¤t->mm->mmap_sem); | ||
1997 | page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); | 2070 | page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); |
1998 | up_read(¤t->mm->mmap_sem); | ||
1999 | 2071 | ||
2000 | kaddr = kmap_atomic(page, KM_USER0); | 2072 | kaddr = kmap_atomic(page, KM_USER0); |
2001 | set_64bit((u64 *)(kaddr + offset_in_page(gpa)), val); | 2073 | set_64bit((u64 *)(kaddr + offset_in_page(gpa)), val); |
@@ -2015,11 +2087,13 @@ static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) | |||
2015 | 2087 | ||
2016 | int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) | 2088 | int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) |
2017 | { | 2089 | { |
2090 | kvm_mmu_invlpg(vcpu, address); | ||
2018 | return X86EMUL_CONTINUE; | 2091 | return X86EMUL_CONTINUE; |
2019 | } | 2092 | } |
2020 | 2093 | ||
2021 | int emulate_clts(struct kvm_vcpu *vcpu) | 2094 | int emulate_clts(struct kvm_vcpu *vcpu) |
2022 | { | 2095 | { |
2096 | KVMTRACE_0D(CLTS, vcpu, handler); | ||
2023 | kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS); | 2097 | kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS); |
2024 | return X86EMUL_CONTINUE; | 2098 | return X86EMUL_CONTINUE; |
2025 | } | 2099 | } |
@@ -2053,21 +2127,19 @@ int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) | |||
2053 | 2127 | ||
2054 | void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) | 2128 | void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) |
2055 | { | 2129 | { |
2056 | static int reported; | ||
2057 | u8 opcodes[4]; | 2130 | u8 opcodes[4]; |
2058 | unsigned long rip = vcpu->arch.rip; | 2131 | unsigned long rip = kvm_rip_read(vcpu); |
2059 | unsigned long rip_linear; | 2132 | unsigned long rip_linear; |
2060 | 2133 | ||
2061 | rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); | 2134 | if (!printk_ratelimit()) |
2062 | |||
2063 | if (reported) | ||
2064 | return; | 2135 | return; |
2065 | 2136 | ||
2137 | rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); | ||
2138 | |||
2066 | emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); | 2139 | emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); |
2067 | 2140 | ||
2068 | printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", | 2141 | printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", |
2069 | context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); | 2142 | context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); |
2070 | reported = 1; | ||
2071 | } | 2143 | } |
2072 | EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); | 2144 | EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); |
2073 | 2145 | ||
@@ -2078,6 +2150,14 @@ static struct x86_emulate_ops emulate_ops = { | |||
2078 | .cmpxchg_emulated = emulator_cmpxchg_emulated, | 2150 | .cmpxchg_emulated = emulator_cmpxchg_emulated, |
2079 | }; | 2151 | }; |
2080 | 2152 | ||
2153 | static void cache_all_regs(struct kvm_vcpu *vcpu) | ||
2154 | { | ||
2155 | kvm_register_read(vcpu, VCPU_REGS_RAX); | ||
2156 | kvm_register_read(vcpu, VCPU_REGS_RSP); | ||
2157 | kvm_register_read(vcpu, VCPU_REGS_RIP); | ||
2158 | vcpu->arch.regs_dirty = ~0; | ||
2159 | } | ||
2160 | |||
2081 | int emulate_instruction(struct kvm_vcpu *vcpu, | 2161 | int emulate_instruction(struct kvm_vcpu *vcpu, |
2082 | struct kvm_run *run, | 2162 | struct kvm_run *run, |
2083 | unsigned long cr2, | 2163 | unsigned long cr2, |
@@ -2087,8 +2167,15 @@ int emulate_instruction(struct kvm_vcpu *vcpu, | |||
2087 | int r; | 2167 | int r; |
2088 | struct decode_cache *c; | 2168 | struct decode_cache *c; |
2089 | 2169 | ||
2170 | kvm_clear_exception_queue(vcpu); | ||
2090 | vcpu->arch.mmio_fault_cr2 = cr2; | 2171 | vcpu->arch.mmio_fault_cr2 = cr2; |
2091 | kvm_x86_ops->cache_regs(vcpu); | 2172 | /* |
2173 | * TODO: fix x86_emulate.c to use guest_read/write_register | ||
2174 | * instead of direct ->regs accesses, can save hundred cycles | ||
2175 | * on Intel for instructions that don't read/change RSP, for | ||
2176 | * for example. | ||
2177 | */ | ||
2178 | cache_all_regs(vcpu); | ||
2092 | 2179 | ||
2093 | vcpu->mmio_is_write = 0; | 2180 | vcpu->mmio_is_write = 0; |
2094 | vcpu->arch.pio.string = 0; | 2181 | vcpu->arch.pio.string = 0; |
@@ -2105,27 +2192,6 @@ int emulate_instruction(struct kvm_vcpu *vcpu, | |||
2105 | ? X86EMUL_MODE_PROT64 : cs_db | 2192 | ? X86EMUL_MODE_PROT64 : cs_db |
2106 | ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; | 2193 | ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; |
2107 | 2194 | ||
2108 | if (vcpu->arch.emulate_ctxt.mode == X86EMUL_MODE_PROT64) { | ||
2109 | vcpu->arch.emulate_ctxt.cs_base = 0; | ||
2110 | vcpu->arch.emulate_ctxt.ds_base = 0; | ||
2111 | vcpu->arch.emulate_ctxt.es_base = 0; | ||
2112 | vcpu->arch.emulate_ctxt.ss_base = 0; | ||
2113 | } else { | ||
2114 | vcpu->arch.emulate_ctxt.cs_base = | ||
2115 | get_segment_base(vcpu, VCPU_SREG_CS); | ||
2116 | vcpu->arch.emulate_ctxt.ds_base = | ||
2117 | get_segment_base(vcpu, VCPU_SREG_DS); | ||
2118 | vcpu->arch.emulate_ctxt.es_base = | ||
2119 | get_segment_base(vcpu, VCPU_SREG_ES); | ||
2120 | vcpu->arch.emulate_ctxt.ss_base = | ||
2121 | get_segment_base(vcpu, VCPU_SREG_SS); | ||
2122 | } | ||
2123 | |||
2124 | vcpu->arch.emulate_ctxt.gs_base = | ||
2125 | get_segment_base(vcpu, VCPU_SREG_GS); | ||
2126 | vcpu->arch.emulate_ctxt.fs_base = | ||
2127 | get_segment_base(vcpu, VCPU_SREG_FS); | ||
2128 | |||
2129 | r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); | 2195 | r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); |
2130 | 2196 | ||
2131 | /* Reject the instructions other than VMCALL/VMMCALL when | 2197 | /* Reject the instructions other than VMCALL/VMMCALL when |
@@ -2169,7 +2235,6 @@ int emulate_instruction(struct kvm_vcpu *vcpu, | |||
2169 | return EMULATE_DO_MMIO; | 2235 | return EMULATE_DO_MMIO; |
2170 | } | 2236 | } |
2171 | 2237 | ||
2172 | kvm_x86_ops->decache_regs(vcpu); | ||
2173 | kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags); | 2238 | kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags); |
2174 | 2239 | ||
2175 | if (vcpu->mmio_is_write) { | 2240 | if (vcpu->mmio_is_write) { |
@@ -2222,20 +2287,19 @@ int complete_pio(struct kvm_vcpu *vcpu) | |||
2222 | struct kvm_pio_request *io = &vcpu->arch.pio; | 2287 | struct kvm_pio_request *io = &vcpu->arch.pio; |
2223 | long delta; | 2288 | long delta; |
2224 | int r; | 2289 | int r; |
2225 | 2290 | unsigned long val; | |
2226 | kvm_x86_ops->cache_regs(vcpu); | ||
2227 | 2291 | ||
2228 | if (!io->string) { | 2292 | if (!io->string) { |
2229 | if (io->in) | 2293 | if (io->in) { |
2230 | memcpy(&vcpu->arch.regs[VCPU_REGS_RAX], vcpu->arch.pio_data, | 2294 | val = kvm_register_read(vcpu, VCPU_REGS_RAX); |
2231 | io->size); | 2295 | memcpy(&val, vcpu->arch.pio_data, io->size); |
2296 | kvm_register_write(vcpu, VCPU_REGS_RAX, val); | ||
2297 | } | ||
2232 | } else { | 2298 | } else { |
2233 | if (io->in) { | 2299 | if (io->in) { |
2234 | r = pio_copy_data(vcpu); | 2300 | r = pio_copy_data(vcpu); |
2235 | if (r) { | 2301 | if (r) |
2236 | kvm_x86_ops->cache_regs(vcpu); | ||
2237 | return r; | 2302 | return r; |
2238 | } | ||
2239 | } | 2303 | } |
2240 | 2304 | ||
2241 | delta = 1; | 2305 | delta = 1; |
@@ -2245,19 +2309,24 @@ int complete_pio(struct kvm_vcpu *vcpu) | |||
2245 | * The size of the register should really depend on | 2309 | * The size of the register should really depend on |
2246 | * current address size. | 2310 | * current address size. |
2247 | */ | 2311 | */ |
2248 | vcpu->arch.regs[VCPU_REGS_RCX] -= delta; | 2312 | val = kvm_register_read(vcpu, VCPU_REGS_RCX); |
2313 | val -= delta; | ||
2314 | kvm_register_write(vcpu, VCPU_REGS_RCX, val); | ||
2249 | } | 2315 | } |
2250 | if (io->down) | 2316 | if (io->down) |
2251 | delta = -delta; | 2317 | delta = -delta; |
2252 | delta *= io->size; | 2318 | delta *= io->size; |
2253 | if (io->in) | 2319 | if (io->in) { |
2254 | vcpu->arch.regs[VCPU_REGS_RDI] += delta; | 2320 | val = kvm_register_read(vcpu, VCPU_REGS_RDI); |
2255 | else | 2321 | val += delta; |
2256 | vcpu->arch.regs[VCPU_REGS_RSI] += delta; | 2322 | kvm_register_write(vcpu, VCPU_REGS_RDI, val); |
2323 | } else { | ||
2324 | val = kvm_register_read(vcpu, VCPU_REGS_RSI); | ||
2325 | val += delta; | ||
2326 | kvm_register_write(vcpu, VCPU_REGS_RSI, val); | ||
2327 | } | ||
2257 | } | 2328 | } |
2258 | 2329 | ||
2259 | kvm_x86_ops->decache_regs(vcpu); | ||
2260 | |||
2261 | io->count -= io->cur_count; | 2330 | io->count -= io->cur_count; |
2262 | io->cur_count = 0; | 2331 | io->cur_count = 0; |
2263 | 2332 | ||
@@ -2300,15 +2369,17 @@ static void pio_string_write(struct kvm_io_device *pio_dev, | |||
2300 | } | 2369 | } |
2301 | 2370 | ||
2302 | static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, | 2371 | static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, |
2303 | gpa_t addr) | 2372 | gpa_t addr, int len, |
2373 | int is_write) | ||
2304 | { | 2374 | { |
2305 | return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); | 2375 | return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr, len, is_write); |
2306 | } | 2376 | } |
2307 | 2377 | ||
2308 | int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | 2378 | int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, |
2309 | int size, unsigned port) | 2379 | int size, unsigned port) |
2310 | { | 2380 | { |
2311 | struct kvm_io_device *pio_dev; | 2381 | struct kvm_io_device *pio_dev; |
2382 | unsigned long val; | ||
2312 | 2383 | ||
2313 | vcpu->run->exit_reason = KVM_EXIT_IO; | 2384 | vcpu->run->exit_reason = KVM_EXIT_IO; |
2314 | vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; | 2385 | vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; |
@@ -2329,13 +2400,12 @@ int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2329 | KVMTRACE_2D(IO_WRITE, vcpu, vcpu->run->io.port, (u32)size, | 2400 | KVMTRACE_2D(IO_WRITE, vcpu, vcpu->run->io.port, (u32)size, |
2330 | handler); | 2401 | handler); |
2331 | 2402 | ||
2332 | kvm_x86_ops->cache_regs(vcpu); | 2403 | val = kvm_register_read(vcpu, VCPU_REGS_RAX); |
2333 | memcpy(vcpu->arch.pio_data, &vcpu->arch.regs[VCPU_REGS_RAX], 4); | 2404 | memcpy(vcpu->arch.pio_data, &val, 4); |
2334 | kvm_x86_ops->decache_regs(vcpu); | ||
2335 | 2405 | ||
2336 | kvm_x86_ops->skip_emulated_instruction(vcpu); | 2406 | kvm_x86_ops->skip_emulated_instruction(vcpu); |
2337 | 2407 | ||
2338 | pio_dev = vcpu_find_pio_dev(vcpu, port); | 2408 | pio_dev = vcpu_find_pio_dev(vcpu, port, size, !in); |
2339 | if (pio_dev) { | 2409 | if (pio_dev) { |
2340 | kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data); | 2410 | kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data); |
2341 | complete_pio(vcpu); | 2411 | complete_pio(vcpu); |
@@ -2417,7 +2487,9 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2417 | } | 2487 | } |
2418 | } | 2488 | } |
2419 | 2489 | ||
2420 | pio_dev = vcpu_find_pio_dev(vcpu, port); | 2490 | pio_dev = vcpu_find_pio_dev(vcpu, port, |
2491 | vcpu->arch.pio.cur_count, | ||
2492 | !vcpu->arch.pio.in); | ||
2421 | if (!vcpu->arch.pio.in) { | 2493 | if (!vcpu->arch.pio.in) { |
2422 | /* string PIO write */ | 2494 | /* string PIO write */ |
2423 | ret = pio_copy_data(vcpu); | 2495 | ret = pio_copy_data(vcpu); |
@@ -2487,11 +2559,6 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu) | |||
2487 | KVMTRACE_0D(HLT, vcpu, handler); | 2559 | KVMTRACE_0D(HLT, vcpu, handler); |
2488 | if (irqchip_in_kernel(vcpu->kvm)) { | 2560 | if (irqchip_in_kernel(vcpu->kvm)) { |
2489 | vcpu->arch.mp_state = KVM_MP_STATE_HALTED; | 2561 | vcpu->arch.mp_state = KVM_MP_STATE_HALTED; |
2490 | up_read(&vcpu->kvm->slots_lock); | ||
2491 | kvm_vcpu_block(vcpu); | ||
2492 | down_read(&vcpu->kvm->slots_lock); | ||
2493 | if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE) | ||
2494 | return -EINTR; | ||
2495 | return 1; | 2562 | return 1; |
2496 | } else { | 2563 | } else { |
2497 | vcpu->run->exit_reason = KVM_EXIT_HLT; | 2564 | vcpu->run->exit_reason = KVM_EXIT_HLT; |
@@ -2514,13 +2581,11 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) | |||
2514 | unsigned long nr, a0, a1, a2, a3, ret; | 2581 | unsigned long nr, a0, a1, a2, a3, ret; |
2515 | int r = 1; | 2582 | int r = 1; |
2516 | 2583 | ||
2517 | kvm_x86_ops->cache_regs(vcpu); | 2584 | nr = kvm_register_read(vcpu, VCPU_REGS_RAX); |
2518 | 2585 | a0 = kvm_register_read(vcpu, VCPU_REGS_RBX); | |
2519 | nr = vcpu->arch.regs[VCPU_REGS_RAX]; | 2586 | a1 = kvm_register_read(vcpu, VCPU_REGS_RCX); |
2520 | a0 = vcpu->arch.regs[VCPU_REGS_RBX]; | 2587 | a2 = kvm_register_read(vcpu, VCPU_REGS_RDX); |
2521 | a1 = vcpu->arch.regs[VCPU_REGS_RCX]; | 2588 | a3 = kvm_register_read(vcpu, VCPU_REGS_RSI); |
2522 | a2 = vcpu->arch.regs[VCPU_REGS_RDX]; | ||
2523 | a3 = vcpu->arch.regs[VCPU_REGS_RSI]; | ||
2524 | 2589 | ||
2525 | KVMTRACE_1D(VMMCALL, vcpu, (u32)nr, handler); | 2590 | KVMTRACE_1D(VMMCALL, vcpu, (u32)nr, handler); |
2526 | 2591 | ||
@@ -2543,8 +2608,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) | |||
2543 | ret = -KVM_ENOSYS; | 2608 | ret = -KVM_ENOSYS; |
2544 | break; | 2609 | break; |
2545 | } | 2610 | } |
2546 | vcpu->arch.regs[VCPU_REGS_RAX] = ret; | 2611 | kvm_register_write(vcpu, VCPU_REGS_RAX, ret); |
2547 | kvm_x86_ops->decache_regs(vcpu); | ||
2548 | ++vcpu->stat.hypercalls; | 2612 | ++vcpu->stat.hypercalls; |
2549 | return r; | 2613 | return r; |
2550 | } | 2614 | } |
@@ -2554,6 +2618,7 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu) | |||
2554 | { | 2618 | { |
2555 | char instruction[3]; | 2619 | char instruction[3]; |
2556 | int ret = 0; | 2620 | int ret = 0; |
2621 | unsigned long rip = kvm_rip_read(vcpu); | ||
2557 | 2622 | ||
2558 | 2623 | ||
2559 | /* | 2624 | /* |
@@ -2563,9 +2628,8 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu) | |||
2563 | */ | 2628 | */ |
2564 | kvm_mmu_zap_all(vcpu->kvm); | 2629 | kvm_mmu_zap_all(vcpu->kvm); |
2565 | 2630 | ||
2566 | kvm_x86_ops->cache_regs(vcpu); | ||
2567 | kvm_x86_ops->patch_hypercall(vcpu, instruction); | 2631 | kvm_x86_ops->patch_hypercall(vcpu, instruction); |
2568 | if (emulator_write_emulated(vcpu->arch.rip, instruction, 3, vcpu) | 2632 | if (emulator_write_emulated(rip, instruction, 3, vcpu) |
2569 | != X86EMUL_CONTINUE) | 2633 | != X86EMUL_CONTINUE) |
2570 | ret = -EFAULT; | 2634 | ret = -EFAULT; |
2571 | 2635 | ||
@@ -2600,27 +2664,41 @@ void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, | |||
2600 | 2664 | ||
2601 | unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) | 2665 | unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) |
2602 | { | 2666 | { |
2667 | unsigned long value; | ||
2668 | |||
2603 | kvm_x86_ops->decache_cr4_guest_bits(vcpu); | 2669 | kvm_x86_ops->decache_cr4_guest_bits(vcpu); |
2604 | switch (cr) { | 2670 | switch (cr) { |
2605 | case 0: | 2671 | case 0: |
2606 | return vcpu->arch.cr0; | 2672 | value = vcpu->arch.cr0; |
2673 | break; | ||
2607 | case 2: | 2674 | case 2: |
2608 | return vcpu->arch.cr2; | 2675 | value = vcpu->arch.cr2; |
2676 | break; | ||
2609 | case 3: | 2677 | case 3: |
2610 | return vcpu->arch.cr3; | 2678 | value = vcpu->arch.cr3; |
2679 | break; | ||
2611 | case 4: | 2680 | case 4: |
2612 | return vcpu->arch.cr4; | 2681 | value = vcpu->arch.cr4; |
2682 | break; | ||
2613 | case 8: | 2683 | case 8: |
2614 | return kvm_get_cr8(vcpu); | 2684 | value = kvm_get_cr8(vcpu); |
2685 | break; | ||
2615 | default: | 2686 | default: |
2616 | vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr); | 2687 | vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr); |
2617 | return 0; | 2688 | return 0; |
2618 | } | 2689 | } |
2690 | KVMTRACE_3D(CR_READ, vcpu, (u32)cr, (u32)value, | ||
2691 | (u32)((u64)value >> 32), handler); | ||
2692 | |||
2693 | return value; | ||
2619 | } | 2694 | } |
2620 | 2695 | ||
2621 | void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, | 2696 | void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, |
2622 | unsigned long *rflags) | 2697 | unsigned long *rflags) |
2623 | { | 2698 | { |
2699 | KVMTRACE_3D(CR_WRITE, vcpu, (u32)cr, (u32)val, | ||
2700 | (u32)((u64)val >> 32), handler); | ||
2701 | |||
2624 | switch (cr) { | 2702 | switch (cr) { |
2625 | case 0: | 2703 | case 0: |
2626 | kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val)); | 2704 | kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val)); |
@@ -2681,13 +2759,12 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) | |||
2681 | u32 function, index; | 2759 | u32 function, index; |
2682 | struct kvm_cpuid_entry2 *e, *best; | 2760 | struct kvm_cpuid_entry2 *e, *best; |
2683 | 2761 | ||
2684 | kvm_x86_ops->cache_regs(vcpu); | 2762 | function = kvm_register_read(vcpu, VCPU_REGS_RAX); |
2685 | function = vcpu->arch.regs[VCPU_REGS_RAX]; | 2763 | index = kvm_register_read(vcpu, VCPU_REGS_RCX); |
2686 | index = vcpu->arch.regs[VCPU_REGS_RCX]; | 2764 | kvm_register_write(vcpu, VCPU_REGS_RAX, 0); |
2687 | vcpu->arch.regs[VCPU_REGS_RAX] = 0; | 2765 | kvm_register_write(vcpu, VCPU_REGS_RBX, 0); |
2688 | vcpu->arch.regs[VCPU_REGS_RBX] = 0; | 2766 | kvm_register_write(vcpu, VCPU_REGS_RCX, 0); |
2689 | vcpu->arch.regs[VCPU_REGS_RCX] = 0; | 2767 | kvm_register_write(vcpu, VCPU_REGS_RDX, 0); |
2690 | vcpu->arch.regs[VCPU_REGS_RDX] = 0; | ||
2691 | best = NULL; | 2768 | best = NULL; |
2692 | for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { | 2769 | for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { |
2693 | e = &vcpu->arch.cpuid_entries[i]; | 2770 | e = &vcpu->arch.cpuid_entries[i]; |
@@ -2705,18 +2782,17 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) | |||
2705 | best = e; | 2782 | best = e; |
2706 | } | 2783 | } |
2707 | if (best) { | 2784 | if (best) { |
2708 | vcpu->arch.regs[VCPU_REGS_RAX] = best->eax; | 2785 | kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax); |
2709 | vcpu->arch.regs[VCPU_REGS_RBX] = best->ebx; | 2786 | kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx); |
2710 | vcpu->arch.regs[VCPU_REGS_RCX] = best->ecx; | 2787 | kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx); |
2711 | vcpu->arch.regs[VCPU_REGS_RDX] = best->edx; | 2788 | kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx); |
2712 | } | 2789 | } |
2713 | kvm_x86_ops->decache_regs(vcpu); | ||
2714 | kvm_x86_ops->skip_emulated_instruction(vcpu); | 2790 | kvm_x86_ops->skip_emulated_instruction(vcpu); |
2715 | KVMTRACE_5D(CPUID, vcpu, function, | 2791 | KVMTRACE_5D(CPUID, vcpu, function, |
2716 | (u32)vcpu->arch.regs[VCPU_REGS_RAX], | 2792 | (u32)kvm_register_read(vcpu, VCPU_REGS_RAX), |
2717 | (u32)vcpu->arch.regs[VCPU_REGS_RBX], | 2793 | (u32)kvm_register_read(vcpu, VCPU_REGS_RBX), |
2718 | (u32)vcpu->arch.regs[VCPU_REGS_RCX], | 2794 | (u32)kvm_register_read(vcpu, VCPU_REGS_RCX), |
2719 | (u32)vcpu->arch.regs[VCPU_REGS_RDX], handler); | 2795 | (u32)kvm_register_read(vcpu, VCPU_REGS_RDX), handler); |
2720 | } | 2796 | } |
2721 | EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); | 2797 | EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); |
2722 | 2798 | ||
@@ -2757,9 +2833,7 @@ static void vapic_enter(struct kvm_vcpu *vcpu) | |||
2757 | if (!apic || !apic->vapic_addr) | 2833 | if (!apic || !apic->vapic_addr) |
2758 | return; | 2834 | return; |
2759 | 2835 | ||
2760 | down_read(¤t->mm->mmap_sem); | ||
2761 | page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); | 2836 | page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); |
2762 | up_read(¤t->mm->mmap_sem); | ||
2763 | 2837 | ||
2764 | vcpu->arch.apic->vapic_page = page; | 2838 | vcpu->arch.apic->vapic_page = page; |
2765 | } | 2839 | } |
@@ -2771,32 +2845,16 @@ static void vapic_exit(struct kvm_vcpu *vcpu) | |||
2771 | if (!apic || !apic->vapic_addr) | 2845 | if (!apic || !apic->vapic_addr) |
2772 | return; | 2846 | return; |
2773 | 2847 | ||
2848 | down_read(&vcpu->kvm->slots_lock); | ||
2774 | kvm_release_page_dirty(apic->vapic_page); | 2849 | kvm_release_page_dirty(apic->vapic_page); |
2775 | mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); | 2850 | mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); |
2851 | up_read(&vcpu->kvm->slots_lock); | ||
2776 | } | 2852 | } |
2777 | 2853 | ||
2778 | static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2854 | static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) |
2779 | { | 2855 | { |
2780 | int r; | 2856 | int r; |
2781 | 2857 | ||
2782 | if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) { | ||
2783 | pr_debug("vcpu %d received sipi with vector # %x\n", | ||
2784 | vcpu->vcpu_id, vcpu->arch.sipi_vector); | ||
2785 | kvm_lapic_reset(vcpu); | ||
2786 | r = kvm_x86_ops->vcpu_reset(vcpu); | ||
2787 | if (r) | ||
2788 | return r; | ||
2789 | vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; | ||
2790 | } | ||
2791 | |||
2792 | down_read(&vcpu->kvm->slots_lock); | ||
2793 | vapic_enter(vcpu); | ||
2794 | |||
2795 | preempted: | ||
2796 | if (vcpu->guest_debug.enabled) | ||
2797 | kvm_x86_ops->guest_debug_pre(vcpu); | ||
2798 | |||
2799 | again: | ||
2800 | if (vcpu->requests) | 2858 | if (vcpu->requests) |
2801 | if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests)) | 2859 | if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests)) |
2802 | kvm_mmu_unload(vcpu); | 2860 | kvm_mmu_unload(vcpu); |
@@ -2808,6 +2866,8 @@ again: | |||
2808 | if (vcpu->requests) { | 2866 | if (vcpu->requests) { |
2809 | if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests)) | 2867 | if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests)) |
2810 | __kvm_migrate_timers(vcpu); | 2868 | __kvm_migrate_timers(vcpu); |
2869 | if (test_and_clear_bit(KVM_REQ_MMU_SYNC, &vcpu->requests)) | ||
2870 | kvm_mmu_sync_roots(vcpu); | ||
2811 | if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) | 2871 | if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) |
2812 | kvm_x86_ops->tlb_flush(vcpu); | 2872 | kvm_x86_ops->tlb_flush(vcpu); |
2813 | if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS, | 2873 | if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS, |
@@ -2833,21 +2893,15 @@ again: | |||
2833 | 2893 | ||
2834 | local_irq_disable(); | 2894 | local_irq_disable(); |
2835 | 2895 | ||
2836 | if (vcpu->requests || need_resched()) { | 2896 | if (vcpu->requests || need_resched() || signal_pending(current)) { |
2837 | local_irq_enable(); | 2897 | local_irq_enable(); |
2838 | preempt_enable(); | 2898 | preempt_enable(); |
2839 | r = 1; | 2899 | r = 1; |
2840 | goto out; | 2900 | goto out; |
2841 | } | 2901 | } |
2842 | 2902 | ||
2843 | if (signal_pending(current)) { | 2903 | if (vcpu->guest_debug.enabled) |
2844 | local_irq_enable(); | 2904 | kvm_x86_ops->guest_debug_pre(vcpu); |
2845 | preempt_enable(); | ||
2846 | r = -EINTR; | ||
2847 | kvm_run->exit_reason = KVM_EXIT_INTR; | ||
2848 | ++vcpu->stat.signal_exits; | ||
2849 | goto out; | ||
2850 | } | ||
2851 | 2905 | ||
2852 | vcpu->guest_mode = 1; | 2906 | vcpu->guest_mode = 1; |
2853 | /* | 2907 | /* |
@@ -2896,8 +2950,8 @@ again: | |||
2896 | * Profile KVM exit RIPs: | 2950 | * Profile KVM exit RIPs: |
2897 | */ | 2951 | */ |
2898 | if (unlikely(prof_on == KVM_PROFILING)) { | 2952 | if (unlikely(prof_on == KVM_PROFILING)) { |
2899 | kvm_x86_ops->cache_regs(vcpu); | 2953 | unsigned long rip = kvm_rip_read(vcpu); |
2900 | profile_hit(KVM_PROFILING, (void *)vcpu->arch.rip); | 2954 | profile_hit(KVM_PROFILING, (void *)rip); |
2901 | } | 2955 | } |
2902 | 2956 | ||
2903 | if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu)) | 2957 | if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu)) |
@@ -2906,31 +2960,66 @@ again: | |||
2906 | kvm_lapic_sync_from_vapic(vcpu); | 2960 | kvm_lapic_sync_from_vapic(vcpu); |
2907 | 2961 | ||
2908 | r = kvm_x86_ops->handle_exit(kvm_run, vcpu); | 2962 | r = kvm_x86_ops->handle_exit(kvm_run, vcpu); |
2963 | out: | ||
2964 | return r; | ||
2965 | } | ||
2909 | 2966 | ||
2910 | if (r > 0) { | 2967 | static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) |
2911 | if (dm_request_for_irq_injection(vcpu, kvm_run)) { | 2968 | { |
2912 | r = -EINTR; | 2969 | int r; |
2913 | kvm_run->exit_reason = KVM_EXIT_INTR; | 2970 | |
2914 | ++vcpu->stat.request_irq_exits; | 2971 | if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) { |
2915 | goto out; | 2972 | pr_debug("vcpu %d received sipi with vector # %x\n", |
2916 | } | 2973 | vcpu->vcpu_id, vcpu->arch.sipi_vector); |
2917 | if (!need_resched()) | 2974 | kvm_lapic_reset(vcpu); |
2918 | goto again; | 2975 | r = kvm_x86_ops->vcpu_reset(vcpu); |
2976 | if (r) | ||
2977 | return r; | ||
2978 | vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; | ||
2919 | } | 2979 | } |
2920 | 2980 | ||
2921 | out: | 2981 | down_read(&vcpu->kvm->slots_lock); |
2922 | up_read(&vcpu->kvm->slots_lock); | 2982 | vapic_enter(vcpu); |
2923 | if (r > 0) { | 2983 | |
2924 | kvm_resched(vcpu); | 2984 | r = 1; |
2925 | down_read(&vcpu->kvm->slots_lock); | 2985 | while (r > 0) { |
2926 | goto preempted; | 2986 | if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE) |
2987 | r = vcpu_enter_guest(vcpu, kvm_run); | ||
2988 | else { | ||
2989 | up_read(&vcpu->kvm->slots_lock); | ||
2990 | kvm_vcpu_block(vcpu); | ||
2991 | down_read(&vcpu->kvm->slots_lock); | ||
2992 | if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests)) | ||
2993 | if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) | ||
2994 | vcpu->arch.mp_state = | ||
2995 | KVM_MP_STATE_RUNNABLE; | ||
2996 | if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE) | ||
2997 | r = -EINTR; | ||
2998 | } | ||
2999 | |||
3000 | if (r > 0) { | ||
3001 | if (dm_request_for_irq_injection(vcpu, kvm_run)) { | ||
3002 | r = -EINTR; | ||
3003 | kvm_run->exit_reason = KVM_EXIT_INTR; | ||
3004 | ++vcpu->stat.request_irq_exits; | ||
3005 | } | ||
3006 | if (signal_pending(current)) { | ||
3007 | r = -EINTR; | ||
3008 | kvm_run->exit_reason = KVM_EXIT_INTR; | ||
3009 | ++vcpu->stat.signal_exits; | ||
3010 | } | ||
3011 | if (need_resched()) { | ||
3012 | up_read(&vcpu->kvm->slots_lock); | ||
3013 | kvm_resched(vcpu); | ||
3014 | down_read(&vcpu->kvm->slots_lock); | ||
3015 | } | ||
3016 | } | ||
2927 | } | 3017 | } |
2928 | 3018 | ||
3019 | up_read(&vcpu->kvm->slots_lock); | ||
2929 | post_kvm_run_save(vcpu, kvm_run); | 3020 | post_kvm_run_save(vcpu, kvm_run); |
2930 | 3021 | ||
2931 | down_read(&vcpu->kvm->slots_lock); | ||
2932 | vapic_exit(vcpu); | 3022 | vapic_exit(vcpu); |
2933 | up_read(&vcpu->kvm->slots_lock); | ||
2934 | 3023 | ||
2935 | return r; | 3024 | return r; |
2936 | } | 3025 | } |
@@ -2942,15 +3031,16 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2942 | 3031 | ||
2943 | vcpu_load(vcpu); | 3032 | vcpu_load(vcpu); |
2944 | 3033 | ||
3034 | if (vcpu->sigset_active) | ||
3035 | sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); | ||
3036 | |||
2945 | if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { | 3037 | if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { |
2946 | kvm_vcpu_block(vcpu); | 3038 | kvm_vcpu_block(vcpu); |
2947 | vcpu_put(vcpu); | 3039 | clear_bit(KVM_REQ_UNHALT, &vcpu->requests); |
2948 | return -EAGAIN; | 3040 | r = -EAGAIN; |
3041 | goto out; | ||
2949 | } | 3042 | } |
2950 | 3043 | ||
2951 | if (vcpu->sigset_active) | ||
2952 | sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); | ||
2953 | |||
2954 | /* re-sync apic's tpr */ | 3044 | /* re-sync apic's tpr */ |
2955 | if (!irqchip_in_kernel(vcpu->kvm)) | 3045 | if (!irqchip_in_kernel(vcpu->kvm)) |
2956 | kvm_set_cr8(vcpu, kvm_run->cr8); | 3046 | kvm_set_cr8(vcpu, kvm_run->cr8); |
@@ -2980,11 +3070,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2980 | } | 3070 | } |
2981 | } | 3071 | } |
2982 | #endif | 3072 | #endif |
2983 | if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { | 3073 | if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) |
2984 | kvm_x86_ops->cache_regs(vcpu); | 3074 | kvm_register_write(vcpu, VCPU_REGS_RAX, |
2985 | vcpu->arch.regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; | 3075 | kvm_run->hypercall.ret); |
2986 | kvm_x86_ops->decache_regs(vcpu); | ||
2987 | } | ||
2988 | 3076 | ||
2989 | r = __vcpu_run(vcpu, kvm_run); | 3077 | r = __vcpu_run(vcpu, kvm_run); |
2990 | 3078 | ||
@@ -3000,28 +3088,26 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |||
3000 | { | 3088 | { |
3001 | vcpu_load(vcpu); | 3089 | vcpu_load(vcpu); |
3002 | 3090 | ||
3003 | kvm_x86_ops->cache_regs(vcpu); | 3091 | regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX); |
3004 | 3092 | regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX); | |
3005 | regs->rax = vcpu->arch.regs[VCPU_REGS_RAX]; | 3093 | regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX); |
3006 | regs->rbx = vcpu->arch.regs[VCPU_REGS_RBX]; | 3094 | regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX); |
3007 | regs->rcx = vcpu->arch.regs[VCPU_REGS_RCX]; | 3095 | regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI); |
3008 | regs->rdx = vcpu->arch.regs[VCPU_REGS_RDX]; | 3096 | regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI); |
3009 | regs->rsi = vcpu->arch.regs[VCPU_REGS_RSI]; | 3097 | regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP); |
3010 | regs->rdi = vcpu->arch.regs[VCPU_REGS_RDI]; | 3098 | regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP); |
3011 | regs->rsp = vcpu->arch.regs[VCPU_REGS_RSP]; | ||
3012 | regs->rbp = vcpu->arch.regs[VCPU_REGS_RBP]; | ||
3013 | #ifdef CONFIG_X86_64 | 3099 | #ifdef CONFIG_X86_64 |
3014 | regs->r8 = vcpu->arch.regs[VCPU_REGS_R8]; | 3100 | regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8); |
3015 | regs->r9 = vcpu->arch.regs[VCPU_REGS_R9]; | 3101 | regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9); |
3016 | regs->r10 = vcpu->arch.regs[VCPU_REGS_R10]; | 3102 | regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10); |
3017 | regs->r11 = vcpu->arch.regs[VCPU_REGS_R11]; | 3103 | regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11); |
3018 | regs->r12 = vcpu->arch.regs[VCPU_REGS_R12]; | 3104 | regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12); |
3019 | regs->r13 = vcpu->arch.regs[VCPU_REGS_R13]; | 3105 | regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13); |
3020 | regs->r14 = vcpu->arch.regs[VCPU_REGS_R14]; | 3106 | regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14); |
3021 | regs->r15 = vcpu->arch.regs[VCPU_REGS_R15]; | 3107 | regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15); |
3022 | #endif | 3108 | #endif |
3023 | 3109 | ||
3024 | regs->rip = vcpu->arch.rip; | 3110 | regs->rip = kvm_rip_read(vcpu); |
3025 | regs->rflags = kvm_x86_ops->get_rflags(vcpu); | 3111 | regs->rflags = kvm_x86_ops->get_rflags(vcpu); |
3026 | 3112 | ||
3027 | /* | 3113 | /* |
@@ -3039,29 +3125,29 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |||
3039 | { | 3125 | { |
3040 | vcpu_load(vcpu); | 3126 | vcpu_load(vcpu); |
3041 | 3127 | ||
3042 | vcpu->arch.regs[VCPU_REGS_RAX] = regs->rax; | 3128 | kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax); |
3043 | vcpu->arch.regs[VCPU_REGS_RBX] = regs->rbx; | 3129 | kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx); |
3044 | vcpu->arch.regs[VCPU_REGS_RCX] = regs->rcx; | 3130 | kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx); |
3045 | vcpu->arch.regs[VCPU_REGS_RDX] = regs->rdx; | 3131 | kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx); |
3046 | vcpu->arch.regs[VCPU_REGS_RSI] = regs->rsi; | 3132 | kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi); |
3047 | vcpu->arch.regs[VCPU_REGS_RDI] = regs->rdi; | 3133 | kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi); |
3048 | vcpu->arch.regs[VCPU_REGS_RSP] = regs->rsp; | 3134 | kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp); |
3049 | vcpu->arch.regs[VCPU_REGS_RBP] = regs->rbp; | 3135 | kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp); |
3050 | #ifdef CONFIG_X86_64 | 3136 | #ifdef CONFIG_X86_64 |
3051 | vcpu->arch.regs[VCPU_REGS_R8] = regs->r8; | 3137 | kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8); |
3052 | vcpu->arch.regs[VCPU_REGS_R9] = regs->r9; | 3138 | kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9); |
3053 | vcpu->arch.regs[VCPU_REGS_R10] = regs->r10; | 3139 | kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10); |
3054 | vcpu->arch.regs[VCPU_REGS_R11] = regs->r11; | 3140 | kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11); |
3055 | vcpu->arch.regs[VCPU_REGS_R12] = regs->r12; | 3141 | kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12); |
3056 | vcpu->arch.regs[VCPU_REGS_R13] = regs->r13; | 3142 | kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13); |
3057 | vcpu->arch.regs[VCPU_REGS_R14] = regs->r14; | 3143 | kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14); |
3058 | vcpu->arch.regs[VCPU_REGS_R15] = regs->r15; | 3144 | kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15); |
3145 | |||
3059 | #endif | 3146 | #endif |
3060 | 3147 | ||
3061 | vcpu->arch.rip = regs->rip; | 3148 | kvm_rip_write(vcpu, regs->rip); |
3062 | kvm_x86_ops->set_rflags(vcpu, regs->rflags); | 3149 | kvm_x86_ops->set_rflags(vcpu, regs->rflags); |
3063 | 3150 | ||
3064 | kvm_x86_ops->decache_regs(vcpu); | ||
3065 | 3151 | ||
3066 | vcpu->arch.exception.pending = false; | 3152 | vcpu->arch.exception.pending = false; |
3067 | 3153 | ||
@@ -3070,8 +3156,8 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |||
3070 | return 0; | 3156 | return 0; |
3071 | } | 3157 | } |
3072 | 3158 | ||
3073 | static void get_segment(struct kvm_vcpu *vcpu, | 3159 | void kvm_get_segment(struct kvm_vcpu *vcpu, |
3074 | struct kvm_segment *var, int seg) | 3160 | struct kvm_segment *var, int seg) |
3075 | { | 3161 | { |
3076 | kvm_x86_ops->get_segment(vcpu, var, seg); | 3162 | kvm_x86_ops->get_segment(vcpu, var, seg); |
3077 | } | 3163 | } |
@@ -3080,7 +3166,7 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) | |||
3080 | { | 3166 | { |
3081 | struct kvm_segment cs; | 3167 | struct kvm_segment cs; |
3082 | 3168 | ||
3083 | get_segment(vcpu, &cs, VCPU_SREG_CS); | 3169 | kvm_get_segment(vcpu, &cs, VCPU_SREG_CS); |
3084 | *db = cs.db; | 3170 | *db = cs.db; |
3085 | *l = cs.l; | 3171 | *l = cs.l; |
3086 | } | 3172 | } |
@@ -3094,15 +3180,15 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, | |||
3094 | 3180 | ||
3095 | vcpu_load(vcpu); | 3181 | vcpu_load(vcpu); |
3096 | 3182 | ||
3097 | get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); | 3183 | kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); |
3098 | get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); | 3184 | kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); |
3099 | get_segment(vcpu, &sregs->es, VCPU_SREG_ES); | 3185 | kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); |
3100 | get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); | 3186 | kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); |
3101 | get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); | 3187 | kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); |
3102 | get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); | 3188 | kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); |
3103 | 3189 | ||
3104 | get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); | 3190 | kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); |
3105 | get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); | 3191 | kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); |
3106 | 3192 | ||
3107 | kvm_x86_ops->get_idt(vcpu, &dt); | 3193 | kvm_x86_ops->get_idt(vcpu, &dt); |
3108 | sregs->idt.limit = dt.limit; | 3194 | sregs->idt.limit = dt.limit; |
@@ -3154,7 +3240,7 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, | |||
3154 | return 0; | 3240 | return 0; |
3155 | } | 3241 | } |
3156 | 3242 | ||
3157 | static void set_segment(struct kvm_vcpu *vcpu, | 3243 | static void kvm_set_segment(struct kvm_vcpu *vcpu, |
3158 | struct kvm_segment *var, int seg) | 3244 | struct kvm_segment *var, int seg) |
3159 | { | 3245 | { |
3160 | kvm_x86_ops->set_segment(vcpu, var, seg); | 3246 | kvm_x86_ops->set_segment(vcpu, var, seg); |
@@ -3168,6 +3254,10 @@ static void seg_desct_to_kvm_desct(struct desc_struct *seg_desc, u16 selector, | |||
3168 | kvm_desct->base |= seg_desc->base2 << 24; | 3254 | kvm_desct->base |= seg_desc->base2 << 24; |
3169 | kvm_desct->limit = seg_desc->limit0; | 3255 | kvm_desct->limit = seg_desc->limit0; |
3170 | kvm_desct->limit |= seg_desc->limit << 16; | 3256 | kvm_desct->limit |= seg_desc->limit << 16; |
3257 | if (seg_desc->g) { | ||
3258 | kvm_desct->limit <<= 12; | ||
3259 | kvm_desct->limit |= 0xfff; | ||
3260 | } | ||
3171 | kvm_desct->selector = selector; | 3261 | kvm_desct->selector = selector; |
3172 | kvm_desct->type = seg_desc->type; | 3262 | kvm_desct->type = seg_desc->type; |
3173 | kvm_desct->present = seg_desc->p; | 3263 | kvm_desct->present = seg_desc->p; |
@@ -3191,7 +3281,7 @@ static void get_segment_descritptor_dtable(struct kvm_vcpu *vcpu, | |||
3191 | if (selector & 1 << 2) { | 3281 | if (selector & 1 << 2) { |
3192 | struct kvm_segment kvm_seg; | 3282 | struct kvm_segment kvm_seg; |
3193 | 3283 | ||
3194 | get_segment(vcpu, &kvm_seg, VCPU_SREG_LDTR); | 3284 | kvm_get_segment(vcpu, &kvm_seg, VCPU_SREG_LDTR); |
3195 | 3285 | ||
3196 | if (kvm_seg.unusable) | 3286 | if (kvm_seg.unusable) |
3197 | dtable->limit = 0; | 3287 | dtable->limit = 0; |
@@ -3207,6 +3297,7 @@ static void get_segment_descritptor_dtable(struct kvm_vcpu *vcpu, | |||
3207 | static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, | 3297 | static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, |
3208 | struct desc_struct *seg_desc) | 3298 | struct desc_struct *seg_desc) |
3209 | { | 3299 | { |
3300 | gpa_t gpa; | ||
3210 | struct descriptor_table dtable; | 3301 | struct descriptor_table dtable; |
3211 | u16 index = selector >> 3; | 3302 | u16 index = selector >> 3; |
3212 | 3303 | ||
@@ -3216,13 +3307,16 @@ static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, | |||
3216 | kvm_queue_exception_e(vcpu, GP_VECTOR, selector & 0xfffc); | 3307 | kvm_queue_exception_e(vcpu, GP_VECTOR, selector & 0xfffc); |
3217 | return 1; | 3308 | return 1; |
3218 | } | 3309 | } |
3219 | return kvm_read_guest(vcpu->kvm, dtable.base + index * 8, seg_desc, 8); | 3310 | gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, dtable.base); |
3311 | gpa += index * 8; | ||
3312 | return kvm_read_guest(vcpu->kvm, gpa, seg_desc, 8); | ||
3220 | } | 3313 | } |
3221 | 3314 | ||
3222 | /* allowed just for 8 bytes segments */ | 3315 | /* allowed just for 8 bytes segments */ |
3223 | static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, | 3316 | static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, |
3224 | struct desc_struct *seg_desc) | 3317 | struct desc_struct *seg_desc) |
3225 | { | 3318 | { |
3319 | gpa_t gpa; | ||
3226 | struct descriptor_table dtable; | 3320 | struct descriptor_table dtable; |
3227 | u16 index = selector >> 3; | 3321 | u16 index = selector >> 3; |
3228 | 3322 | ||
@@ -3230,7 +3324,9 @@ static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, | |||
3230 | 3324 | ||
3231 | if (dtable.limit < index * 8 + 7) | 3325 | if (dtable.limit < index * 8 + 7) |
3232 | return 1; | 3326 | return 1; |
3233 | return kvm_write_guest(vcpu->kvm, dtable.base + index * 8, seg_desc, 8); | 3327 | gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, dtable.base); |
3328 | gpa += index * 8; | ||
3329 | return kvm_write_guest(vcpu->kvm, gpa, seg_desc, 8); | ||
3234 | } | 3330 | } |
3235 | 3331 | ||
3236 | static u32 get_tss_base_addr(struct kvm_vcpu *vcpu, | 3332 | static u32 get_tss_base_addr(struct kvm_vcpu *vcpu, |
@@ -3242,62 +3338,14 @@ static u32 get_tss_base_addr(struct kvm_vcpu *vcpu, | |||
3242 | base_addr |= (seg_desc->base1 << 16); | 3338 | base_addr |= (seg_desc->base1 << 16); |
3243 | base_addr |= (seg_desc->base2 << 24); | 3339 | base_addr |= (seg_desc->base2 << 24); |
3244 | 3340 | ||
3245 | return base_addr; | 3341 | return vcpu->arch.mmu.gva_to_gpa(vcpu, base_addr); |
3246 | } | ||
3247 | |||
3248 | static int load_tss_segment32(struct kvm_vcpu *vcpu, | ||
3249 | struct desc_struct *seg_desc, | ||
3250 | struct tss_segment_32 *tss) | ||
3251 | { | ||
3252 | u32 base_addr; | ||
3253 | |||
3254 | base_addr = get_tss_base_addr(vcpu, seg_desc); | ||
3255 | |||
3256 | return kvm_read_guest(vcpu->kvm, base_addr, tss, | ||
3257 | sizeof(struct tss_segment_32)); | ||
3258 | } | ||
3259 | |||
3260 | static int save_tss_segment32(struct kvm_vcpu *vcpu, | ||
3261 | struct desc_struct *seg_desc, | ||
3262 | struct tss_segment_32 *tss) | ||
3263 | { | ||
3264 | u32 base_addr; | ||
3265 | |||
3266 | base_addr = get_tss_base_addr(vcpu, seg_desc); | ||
3267 | |||
3268 | return kvm_write_guest(vcpu->kvm, base_addr, tss, | ||
3269 | sizeof(struct tss_segment_32)); | ||
3270 | } | ||
3271 | |||
3272 | static int load_tss_segment16(struct kvm_vcpu *vcpu, | ||
3273 | struct desc_struct *seg_desc, | ||
3274 | struct tss_segment_16 *tss) | ||
3275 | { | ||
3276 | u32 base_addr; | ||
3277 | |||
3278 | base_addr = get_tss_base_addr(vcpu, seg_desc); | ||
3279 | |||
3280 | return kvm_read_guest(vcpu->kvm, base_addr, tss, | ||
3281 | sizeof(struct tss_segment_16)); | ||
3282 | } | ||
3283 | |||
3284 | static int save_tss_segment16(struct kvm_vcpu *vcpu, | ||
3285 | struct desc_struct *seg_desc, | ||
3286 | struct tss_segment_16 *tss) | ||
3287 | { | ||
3288 | u32 base_addr; | ||
3289 | |||
3290 | base_addr = get_tss_base_addr(vcpu, seg_desc); | ||
3291 | |||
3292 | return kvm_write_guest(vcpu->kvm, base_addr, tss, | ||
3293 | sizeof(struct tss_segment_16)); | ||
3294 | } | 3342 | } |
3295 | 3343 | ||
3296 | static u16 get_segment_selector(struct kvm_vcpu *vcpu, int seg) | 3344 | static u16 get_segment_selector(struct kvm_vcpu *vcpu, int seg) |
3297 | { | 3345 | { |
3298 | struct kvm_segment kvm_seg; | 3346 | struct kvm_segment kvm_seg; |
3299 | 3347 | ||
3300 | get_segment(vcpu, &kvm_seg, seg); | 3348 | kvm_get_segment(vcpu, &kvm_seg, seg); |
3301 | return kvm_seg.selector; | 3349 | return kvm_seg.selector; |
3302 | } | 3350 | } |
3303 | 3351 | ||
@@ -3313,11 +3361,33 @@ static int load_segment_descriptor_to_kvm_desct(struct kvm_vcpu *vcpu, | |||
3313 | return 0; | 3361 | return 0; |
3314 | } | 3362 | } |
3315 | 3363 | ||
3316 | static int load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, | 3364 | static int kvm_load_realmode_segment(struct kvm_vcpu *vcpu, u16 selector, int seg) |
3317 | int type_bits, int seg) | 3365 | { |
3366 | struct kvm_segment segvar = { | ||
3367 | .base = selector << 4, | ||
3368 | .limit = 0xffff, | ||
3369 | .selector = selector, | ||
3370 | .type = 3, | ||
3371 | .present = 1, | ||
3372 | .dpl = 3, | ||
3373 | .db = 0, | ||
3374 | .s = 1, | ||
3375 | .l = 0, | ||
3376 | .g = 0, | ||
3377 | .avl = 0, | ||
3378 | .unusable = 0, | ||
3379 | }; | ||
3380 | kvm_x86_ops->set_segment(vcpu, &segvar, seg); | ||
3381 | return 0; | ||
3382 | } | ||
3383 | |||
3384 | int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, | ||
3385 | int type_bits, int seg) | ||
3318 | { | 3386 | { |
3319 | struct kvm_segment kvm_seg; | 3387 | struct kvm_segment kvm_seg; |
3320 | 3388 | ||
3389 | if (!(vcpu->arch.cr0 & X86_CR0_PE)) | ||
3390 | return kvm_load_realmode_segment(vcpu, selector, seg); | ||
3321 | if (load_segment_descriptor_to_kvm_desct(vcpu, selector, &kvm_seg)) | 3391 | if (load_segment_descriptor_to_kvm_desct(vcpu, selector, &kvm_seg)) |
3322 | return 1; | 3392 | return 1; |
3323 | kvm_seg.type |= type_bits; | 3393 | kvm_seg.type |= type_bits; |
@@ -3327,7 +3397,7 @@ static int load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, | |||
3327 | if (!kvm_seg.s) | 3397 | if (!kvm_seg.s) |
3328 | kvm_seg.unusable = 1; | 3398 | kvm_seg.unusable = 1; |
3329 | 3399 | ||
3330 | set_segment(vcpu, &kvm_seg, seg); | 3400 | kvm_set_segment(vcpu, &kvm_seg, seg); |
3331 | return 0; | 3401 | return 0; |
3332 | } | 3402 | } |
3333 | 3403 | ||
@@ -3335,17 +3405,16 @@ static void save_state_to_tss32(struct kvm_vcpu *vcpu, | |||
3335 | struct tss_segment_32 *tss) | 3405 | struct tss_segment_32 *tss) |
3336 | { | 3406 | { |
3337 | tss->cr3 = vcpu->arch.cr3; | 3407 | tss->cr3 = vcpu->arch.cr3; |
3338 | tss->eip = vcpu->arch.rip; | 3408 | tss->eip = kvm_rip_read(vcpu); |
3339 | tss->eflags = kvm_x86_ops->get_rflags(vcpu); | 3409 | tss->eflags = kvm_x86_ops->get_rflags(vcpu); |
3340 | tss->eax = vcpu->arch.regs[VCPU_REGS_RAX]; | 3410 | tss->eax = kvm_register_read(vcpu, VCPU_REGS_RAX); |
3341 | tss->ecx = vcpu->arch.regs[VCPU_REGS_RCX]; | 3411 | tss->ecx = kvm_register_read(vcpu, VCPU_REGS_RCX); |
3342 | tss->edx = vcpu->arch.regs[VCPU_REGS_RDX]; | 3412 | tss->edx = kvm_register_read(vcpu, VCPU_REGS_RDX); |
3343 | tss->ebx = vcpu->arch.regs[VCPU_REGS_RBX]; | 3413 | tss->ebx = kvm_register_read(vcpu, VCPU_REGS_RBX); |
3344 | tss->esp = vcpu->arch.regs[VCPU_REGS_RSP]; | 3414 | tss->esp = kvm_register_read(vcpu, VCPU_REGS_RSP); |
3345 | tss->ebp = vcpu->arch.regs[VCPU_REGS_RBP]; | 3415 | tss->ebp = kvm_register_read(vcpu, VCPU_REGS_RBP); |
3346 | tss->esi = vcpu->arch.regs[VCPU_REGS_RSI]; | 3416 | tss->esi = kvm_register_read(vcpu, VCPU_REGS_RSI); |
3347 | tss->edi = vcpu->arch.regs[VCPU_REGS_RDI]; | 3417 | tss->edi = kvm_register_read(vcpu, VCPU_REGS_RDI); |
3348 | |||
3349 | tss->es = get_segment_selector(vcpu, VCPU_SREG_ES); | 3418 | tss->es = get_segment_selector(vcpu, VCPU_SREG_ES); |
3350 | tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS); | 3419 | tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS); |
3351 | tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS); | 3420 | tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS); |
@@ -3361,37 +3430,37 @@ static int load_state_from_tss32(struct kvm_vcpu *vcpu, | |||
3361 | { | 3430 | { |
3362 | kvm_set_cr3(vcpu, tss->cr3); | 3431 | kvm_set_cr3(vcpu, tss->cr3); |
3363 | 3432 | ||
3364 | vcpu->arch.rip = tss->eip; | 3433 | kvm_rip_write(vcpu, tss->eip); |
3365 | kvm_x86_ops->set_rflags(vcpu, tss->eflags | 2); | 3434 | kvm_x86_ops->set_rflags(vcpu, tss->eflags | 2); |
3366 | 3435 | ||
3367 | vcpu->arch.regs[VCPU_REGS_RAX] = tss->eax; | 3436 | kvm_register_write(vcpu, VCPU_REGS_RAX, tss->eax); |
3368 | vcpu->arch.regs[VCPU_REGS_RCX] = tss->ecx; | 3437 | kvm_register_write(vcpu, VCPU_REGS_RCX, tss->ecx); |
3369 | vcpu->arch.regs[VCPU_REGS_RDX] = tss->edx; | 3438 | kvm_register_write(vcpu, VCPU_REGS_RDX, tss->edx); |
3370 | vcpu->arch.regs[VCPU_REGS_RBX] = tss->ebx; | 3439 | kvm_register_write(vcpu, VCPU_REGS_RBX, tss->ebx); |
3371 | vcpu->arch.regs[VCPU_REGS_RSP] = tss->esp; | 3440 | kvm_register_write(vcpu, VCPU_REGS_RSP, tss->esp); |
3372 | vcpu->arch.regs[VCPU_REGS_RBP] = tss->ebp; | 3441 | kvm_register_write(vcpu, VCPU_REGS_RBP, tss->ebp); |
3373 | vcpu->arch.regs[VCPU_REGS_RSI] = tss->esi; | 3442 | kvm_register_write(vcpu, VCPU_REGS_RSI, tss->esi); |
3374 | vcpu->arch.regs[VCPU_REGS_RDI] = tss->edi; | 3443 | kvm_register_write(vcpu, VCPU_REGS_RDI, tss->edi); |
3375 | 3444 | ||
3376 | if (load_segment_descriptor(vcpu, tss->ldt_selector, 0, VCPU_SREG_LDTR)) | 3445 | if (kvm_load_segment_descriptor(vcpu, tss->ldt_selector, 0, VCPU_SREG_LDTR)) |
3377 | return 1; | 3446 | return 1; |
3378 | 3447 | ||
3379 | if (load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES)) | 3448 | if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES)) |
3380 | return 1; | 3449 | return 1; |
3381 | 3450 | ||
3382 | if (load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS)) | 3451 | if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS)) |
3383 | return 1; | 3452 | return 1; |
3384 | 3453 | ||
3385 | if (load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS)) | 3454 | if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS)) |
3386 | return 1; | 3455 | return 1; |
3387 | 3456 | ||
3388 | if (load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS)) | 3457 | if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS)) |
3389 | return 1; | 3458 | return 1; |
3390 | 3459 | ||
3391 | if (load_segment_descriptor(vcpu, tss->fs, 1, VCPU_SREG_FS)) | 3460 | if (kvm_load_segment_descriptor(vcpu, tss->fs, 1, VCPU_SREG_FS)) |
3392 | return 1; | 3461 | return 1; |
3393 | 3462 | ||
3394 | if (load_segment_descriptor(vcpu, tss->gs, 1, VCPU_SREG_GS)) | 3463 | if (kvm_load_segment_descriptor(vcpu, tss->gs, 1, VCPU_SREG_GS)) |
3395 | return 1; | 3464 | return 1; |
3396 | return 0; | 3465 | return 0; |
3397 | } | 3466 | } |
@@ -3399,16 +3468,16 @@ static int load_state_from_tss32(struct kvm_vcpu *vcpu, | |||
3399 | static void save_state_to_tss16(struct kvm_vcpu *vcpu, | 3468 | static void save_state_to_tss16(struct kvm_vcpu *vcpu, |
3400 | struct tss_segment_16 *tss) | 3469 | struct tss_segment_16 *tss) |
3401 | { | 3470 | { |
3402 | tss->ip = vcpu->arch.rip; | 3471 | tss->ip = kvm_rip_read(vcpu); |
3403 | tss->flag = kvm_x86_ops->get_rflags(vcpu); | 3472 | tss->flag = kvm_x86_ops->get_rflags(vcpu); |
3404 | tss->ax = vcpu->arch.regs[VCPU_REGS_RAX]; | 3473 | tss->ax = kvm_register_read(vcpu, VCPU_REGS_RAX); |
3405 | tss->cx = vcpu->arch.regs[VCPU_REGS_RCX]; | 3474 | tss->cx = kvm_register_read(vcpu, VCPU_REGS_RCX); |
3406 | tss->dx = vcpu->arch.regs[VCPU_REGS_RDX]; | 3475 | tss->dx = kvm_register_read(vcpu, VCPU_REGS_RDX); |
3407 | tss->bx = vcpu->arch.regs[VCPU_REGS_RBX]; | 3476 | tss->bx = kvm_register_read(vcpu, VCPU_REGS_RBX); |
3408 | tss->sp = vcpu->arch.regs[VCPU_REGS_RSP]; | 3477 | tss->sp = kvm_register_read(vcpu, VCPU_REGS_RSP); |
3409 | tss->bp = vcpu->arch.regs[VCPU_REGS_RBP]; | 3478 | tss->bp = kvm_register_read(vcpu, VCPU_REGS_RBP); |
3410 | tss->si = vcpu->arch.regs[VCPU_REGS_RSI]; | 3479 | tss->si = kvm_register_read(vcpu, VCPU_REGS_RSI); |
3411 | tss->di = vcpu->arch.regs[VCPU_REGS_RDI]; | 3480 | tss->di = kvm_register_read(vcpu, VCPU_REGS_RDI); |
3412 | 3481 | ||
3413 | tss->es = get_segment_selector(vcpu, VCPU_SREG_ES); | 3482 | tss->es = get_segment_selector(vcpu, VCPU_SREG_ES); |
3414 | tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS); | 3483 | tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS); |
@@ -3421,49 +3490,55 @@ static void save_state_to_tss16(struct kvm_vcpu *vcpu, | |||
3421 | static int load_state_from_tss16(struct kvm_vcpu *vcpu, | 3490 | static int load_state_from_tss16(struct kvm_vcpu *vcpu, |
3422 | struct tss_segment_16 *tss) | 3491 | struct tss_segment_16 *tss) |
3423 | { | 3492 | { |
3424 | vcpu->arch.rip = tss->ip; | 3493 | kvm_rip_write(vcpu, tss->ip); |
3425 | kvm_x86_ops->set_rflags(vcpu, tss->flag | 2); | 3494 | kvm_x86_ops->set_rflags(vcpu, tss->flag | 2); |
3426 | vcpu->arch.regs[VCPU_REGS_RAX] = tss->ax; | 3495 | kvm_register_write(vcpu, VCPU_REGS_RAX, tss->ax); |
3427 | vcpu->arch.regs[VCPU_REGS_RCX] = tss->cx; | 3496 | kvm_register_write(vcpu, VCPU_REGS_RCX, tss->cx); |
3428 | vcpu->arch.regs[VCPU_REGS_RDX] = tss->dx; | 3497 | kvm_register_write(vcpu, VCPU_REGS_RDX, tss->dx); |
3429 | vcpu->arch.regs[VCPU_REGS_RBX] = tss->bx; | 3498 | kvm_register_write(vcpu, VCPU_REGS_RBX, tss->bx); |
3430 | vcpu->arch.regs[VCPU_REGS_RSP] = tss->sp; | 3499 | kvm_register_write(vcpu, VCPU_REGS_RSP, tss->sp); |
3431 | vcpu->arch.regs[VCPU_REGS_RBP] = tss->bp; | 3500 | kvm_register_write(vcpu, VCPU_REGS_RBP, tss->bp); |
3432 | vcpu->arch.regs[VCPU_REGS_RSI] = tss->si; | 3501 | kvm_register_write(vcpu, VCPU_REGS_RSI, tss->si); |
3433 | vcpu->arch.regs[VCPU_REGS_RDI] = tss->di; | 3502 | kvm_register_write(vcpu, VCPU_REGS_RDI, tss->di); |
3434 | 3503 | ||
3435 | if (load_segment_descriptor(vcpu, tss->ldt, 0, VCPU_SREG_LDTR)) | 3504 | if (kvm_load_segment_descriptor(vcpu, tss->ldt, 0, VCPU_SREG_LDTR)) |
3436 | return 1; | 3505 | return 1; |
3437 | 3506 | ||
3438 | if (load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES)) | 3507 | if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES)) |
3439 | return 1; | 3508 | return 1; |
3440 | 3509 | ||
3441 | if (load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS)) | 3510 | if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS)) |
3442 | return 1; | 3511 | return 1; |
3443 | 3512 | ||
3444 | if (load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS)) | 3513 | if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS)) |
3445 | return 1; | 3514 | return 1; |
3446 | 3515 | ||
3447 | if (load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS)) | 3516 | if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS)) |
3448 | return 1; | 3517 | return 1; |
3449 | return 0; | 3518 | return 0; |
3450 | } | 3519 | } |
3451 | 3520 | ||
3452 | int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector, | 3521 | static int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector, |
3453 | struct desc_struct *cseg_desc, | 3522 | u32 old_tss_base, |
3454 | struct desc_struct *nseg_desc) | 3523 | struct desc_struct *nseg_desc) |
3455 | { | 3524 | { |
3456 | struct tss_segment_16 tss_segment_16; | 3525 | struct tss_segment_16 tss_segment_16; |
3457 | int ret = 0; | 3526 | int ret = 0; |
3458 | 3527 | ||
3459 | if (load_tss_segment16(vcpu, cseg_desc, &tss_segment_16)) | 3528 | if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_16, |
3529 | sizeof tss_segment_16)) | ||
3460 | goto out; | 3530 | goto out; |
3461 | 3531 | ||
3462 | save_state_to_tss16(vcpu, &tss_segment_16); | 3532 | save_state_to_tss16(vcpu, &tss_segment_16); |
3463 | save_tss_segment16(vcpu, cseg_desc, &tss_segment_16); | ||
3464 | 3533 | ||
3465 | if (load_tss_segment16(vcpu, nseg_desc, &tss_segment_16)) | 3534 | if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_16, |
3535 | sizeof tss_segment_16)) | ||
3466 | goto out; | 3536 | goto out; |
3537 | |||
3538 | if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc), | ||
3539 | &tss_segment_16, sizeof tss_segment_16)) | ||
3540 | goto out; | ||
3541 | |||
3467 | if (load_state_from_tss16(vcpu, &tss_segment_16)) | 3542 | if (load_state_from_tss16(vcpu, &tss_segment_16)) |
3468 | goto out; | 3543 | goto out; |
3469 | 3544 | ||
@@ -3472,21 +3547,27 @@ out: | |||
3472 | return ret; | 3547 | return ret; |
3473 | } | 3548 | } |
3474 | 3549 | ||
3475 | int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector, | 3550 | static int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector, |
3476 | struct desc_struct *cseg_desc, | 3551 | u32 old_tss_base, |
3477 | struct desc_struct *nseg_desc) | 3552 | struct desc_struct *nseg_desc) |
3478 | { | 3553 | { |
3479 | struct tss_segment_32 tss_segment_32; | 3554 | struct tss_segment_32 tss_segment_32; |
3480 | int ret = 0; | 3555 | int ret = 0; |
3481 | 3556 | ||
3482 | if (load_tss_segment32(vcpu, cseg_desc, &tss_segment_32)) | 3557 | if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_32, |
3558 | sizeof tss_segment_32)) | ||
3483 | goto out; | 3559 | goto out; |
3484 | 3560 | ||
3485 | save_state_to_tss32(vcpu, &tss_segment_32); | 3561 | save_state_to_tss32(vcpu, &tss_segment_32); |
3486 | save_tss_segment32(vcpu, cseg_desc, &tss_segment_32); | ||
3487 | 3562 | ||
3488 | if (load_tss_segment32(vcpu, nseg_desc, &tss_segment_32)) | 3563 | if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_32, |
3564 | sizeof tss_segment_32)) | ||
3489 | goto out; | 3565 | goto out; |
3566 | |||
3567 | if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc), | ||
3568 | &tss_segment_32, sizeof tss_segment_32)) | ||
3569 | goto out; | ||
3570 | |||
3490 | if (load_state_from_tss32(vcpu, &tss_segment_32)) | 3571 | if (load_state_from_tss32(vcpu, &tss_segment_32)) |
3491 | goto out; | 3572 | goto out; |
3492 | 3573 | ||
@@ -3501,16 +3582,20 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason) | |||
3501 | struct desc_struct cseg_desc; | 3582 | struct desc_struct cseg_desc; |
3502 | struct desc_struct nseg_desc; | 3583 | struct desc_struct nseg_desc; |
3503 | int ret = 0; | 3584 | int ret = 0; |
3585 | u32 old_tss_base = get_segment_base(vcpu, VCPU_SREG_TR); | ||
3586 | u16 old_tss_sel = get_segment_selector(vcpu, VCPU_SREG_TR); | ||
3504 | 3587 | ||
3505 | get_segment(vcpu, &tr_seg, VCPU_SREG_TR); | 3588 | old_tss_base = vcpu->arch.mmu.gva_to_gpa(vcpu, old_tss_base); |
3506 | 3589 | ||
3590 | /* FIXME: Handle errors. Failure to read either TSS or their | ||
3591 | * descriptors should generate a pagefault. | ||
3592 | */ | ||
3507 | if (load_guest_segment_descriptor(vcpu, tss_selector, &nseg_desc)) | 3593 | if (load_guest_segment_descriptor(vcpu, tss_selector, &nseg_desc)) |
3508 | goto out; | 3594 | goto out; |
3509 | 3595 | ||
3510 | if (load_guest_segment_descriptor(vcpu, tr_seg.selector, &cseg_desc)) | 3596 | if (load_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc)) |
3511 | goto out; | 3597 | goto out; |
3512 | 3598 | ||
3513 | |||
3514 | if (reason != TASK_SWITCH_IRET) { | 3599 | if (reason != TASK_SWITCH_IRET) { |
3515 | int cpl; | 3600 | int cpl; |
3516 | 3601 | ||
@@ -3528,8 +3613,7 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason) | |||
3528 | 3613 | ||
3529 | if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) { | 3614 | if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) { |
3530 | cseg_desc.type &= ~(1 << 1); //clear the B flag | 3615 | cseg_desc.type &= ~(1 << 1); //clear the B flag |
3531 | save_guest_segment_descriptor(vcpu, tr_seg.selector, | 3616 | save_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc); |
3532 | &cseg_desc); | ||
3533 | } | 3617 | } |
3534 | 3618 | ||
3535 | if (reason == TASK_SWITCH_IRET) { | 3619 | if (reason == TASK_SWITCH_IRET) { |
@@ -3538,13 +3622,12 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason) | |||
3538 | } | 3622 | } |
3539 | 3623 | ||
3540 | kvm_x86_ops->skip_emulated_instruction(vcpu); | 3624 | kvm_x86_ops->skip_emulated_instruction(vcpu); |
3541 | kvm_x86_ops->cache_regs(vcpu); | ||
3542 | 3625 | ||
3543 | if (nseg_desc.type & 8) | 3626 | if (nseg_desc.type & 8) |
3544 | ret = kvm_task_switch_32(vcpu, tss_selector, &cseg_desc, | 3627 | ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_base, |
3545 | &nseg_desc); | 3628 | &nseg_desc); |
3546 | else | 3629 | else |
3547 | ret = kvm_task_switch_16(vcpu, tss_selector, &cseg_desc, | 3630 | ret = kvm_task_switch_16(vcpu, tss_selector, old_tss_base, |
3548 | &nseg_desc); | 3631 | &nseg_desc); |
3549 | 3632 | ||
3550 | if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) { | 3633 | if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) { |
@@ -3561,9 +3644,8 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason) | |||
3561 | kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 | X86_CR0_TS); | 3644 | kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 | X86_CR0_TS); |
3562 | seg_desct_to_kvm_desct(&nseg_desc, tss_selector, &tr_seg); | 3645 | seg_desct_to_kvm_desct(&nseg_desc, tss_selector, &tr_seg); |
3563 | tr_seg.type = 11; | 3646 | tr_seg.type = 11; |
3564 | set_segment(vcpu, &tr_seg, VCPU_SREG_TR); | 3647 | kvm_set_segment(vcpu, &tr_seg, VCPU_SREG_TR); |
3565 | out: | 3648 | out: |
3566 | kvm_x86_ops->decache_regs(vcpu); | ||
3567 | return ret; | 3649 | return ret; |
3568 | } | 3650 | } |
3569 | EXPORT_SYMBOL_GPL(kvm_task_switch); | 3651 | EXPORT_SYMBOL_GPL(kvm_task_switch); |
@@ -3626,17 +3708,24 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, | |||
3626 | pr_debug("Set back pending irq %d\n", | 3708 | pr_debug("Set back pending irq %d\n", |
3627 | pending_vec); | 3709 | pending_vec); |
3628 | } | 3710 | } |
3711 | kvm_pic_clear_isr_ack(vcpu->kvm); | ||
3629 | } | 3712 | } |
3630 | 3713 | ||
3631 | set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); | 3714 | kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); |
3632 | set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); | 3715 | kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); |
3633 | set_segment(vcpu, &sregs->es, VCPU_SREG_ES); | 3716 | kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); |
3634 | set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); | 3717 | kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); |
3635 | set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); | 3718 | kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); |
3636 | set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); | 3719 | kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); |
3720 | |||
3721 | kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); | ||
3722 | kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); | ||
3637 | 3723 | ||
3638 | set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); | 3724 | /* Older userspace won't unhalt the vcpu on reset. */ |
3639 | set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); | 3725 | if (vcpu->vcpu_id == 0 && kvm_rip_read(vcpu) == 0xfff0 && |
3726 | sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && | ||
3727 | !(vcpu->arch.cr0 & X86_CR0_PE)) | ||
3728 | vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; | ||
3640 | 3729 | ||
3641 | vcpu_put(vcpu); | 3730 | vcpu_put(vcpu); |
3642 | 3731 | ||
@@ -3751,14 +3840,14 @@ void fx_init(struct kvm_vcpu *vcpu) | |||
3751 | * allocate ram with GFP_KERNEL. | 3840 | * allocate ram with GFP_KERNEL. |
3752 | */ | 3841 | */ |
3753 | if (!used_math()) | 3842 | if (!used_math()) |
3754 | fx_save(&vcpu->arch.host_fx_image); | 3843 | kvm_fx_save(&vcpu->arch.host_fx_image); |
3755 | 3844 | ||
3756 | /* Initialize guest FPU by resetting ours and saving into guest's */ | 3845 | /* Initialize guest FPU by resetting ours and saving into guest's */ |
3757 | preempt_disable(); | 3846 | preempt_disable(); |
3758 | fx_save(&vcpu->arch.host_fx_image); | 3847 | kvm_fx_save(&vcpu->arch.host_fx_image); |
3759 | fx_finit(); | 3848 | kvm_fx_finit(); |
3760 | fx_save(&vcpu->arch.guest_fx_image); | 3849 | kvm_fx_save(&vcpu->arch.guest_fx_image); |
3761 | fx_restore(&vcpu->arch.host_fx_image); | 3850 | kvm_fx_restore(&vcpu->arch.host_fx_image); |
3762 | preempt_enable(); | 3851 | preempt_enable(); |
3763 | 3852 | ||
3764 | vcpu->arch.cr0 |= X86_CR0_ET; | 3853 | vcpu->arch.cr0 |= X86_CR0_ET; |
@@ -3775,8 +3864,8 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) | |||
3775 | return; | 3864 | return; |
3776 | 3865 | ||
3777 | vcpu->guest_fpu_loaded = 1; | 3866 | vcpu->guest_fpu_loaded = 1; |
3778 | fx_save(&vcpu->arch.host_fx_image); | 3867 | kvm_fx_save(&vcpu->arch.host_fx_image); |
3779 | fx_restore(&vcpu->arch.guest_fx_image); | 3868 | kvm_fx_restore(&vcpu->arch.guest_fx_image); |
3780 | } | 3869 | } |
3781 | EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); | 3870 | EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); |
3782 | 3871 | ||
@@ -3786,8 +3875,8 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) | |||
3786 | return; | 3875 | return; |
3787 | 3876 | ||
3788 | vcpu->guest_fpu_loaded = 0; | 3877 | vcpu->guest_fpu_loaded = 0; |
3789 | fx_save(&vcpu->arch.guest_fx_image); | 3878 | kvm_fx_save(&vcpu->arch.guest_fx_image); |
3790 | fx_restore(&vcpu->arch.host_fx_image); | 3879 | kvm_fx_restore(&vcpu->arch.host_fx_image); |
3791 | ++vcpu->stat.fpu_reload; | 3880 | ++vcpu->stat.fpu_reload; |
3792 | } | 3881 | } |
3793 | EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); | 3882 | EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); |
@@ -3922,6 +4011,7 @@ struct kvm *kvm_arch_create_vm(void) | |||
3922 | return ERR_PTR(-ENOMEM); | 4011 | return ERR_PTR(-ENOMEM); |
3923 | 4012 | ||
3924 | INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); | 4013 | INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); |
4014 | INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); | ||
3925 | 4015 | ||
3926 | return kvm; | 4016 | return kvm; |
3927 | } | 4017 | } |
@@ -3954,6 +4044,8 @@ static void kvm_free_vcpus(struct kvm *kvm) | |||
3954 | 4044 | ||
3955 | void kvm_arch_destroy_vm(struct kvm *kvm) | 4045 | void kvm_arch_destroy_vm(struct kvm *kvm) |
3956 | { | 4046 | { |
4047 | kvm_iommu_unmap_guest(kvm); | ||
4048 | kvm_free_all_assigned_devices(kvm); | ||
3957 | kvm_free_pit(kvm); | 4049 | kvm_free_pit(kvm); |
3958 | kfree(kvm->arch.vpic); | 4050 | kfree(kvm->arch.vpic); |
3959 | kfree(kvm->arch.vioapic); | 4051 | kfree(kvm->arch.vioapic); |
@@ -3979,16 +4071,23 @@ int kvm_arch_set_memory_region(struct kvm *kvm, | |||
3979 | */ | 4071 | */ |
3980 | if (!user_alloc) { | 4072 | if (!user_alloc) { |
3981 | if (npages && !old.rmap) { | 4073 | if (npages && !old.rmap) { |
4074 | unsigned long userspace_addr; | ||
4075 | |||
3982 | down_write(¤t->mm->mmap_sem); | 4076 | down_write(¤t->mm->mmap_sem); |
3983 | memslot->userspace_addr = do_mmap(NULL, 0, | 4077 | userspace_addr = do_mmap(NULL, 0, |
3984 | npages * PAGE_SIZE, | 4078 | npages * PAGE_SIZE, |
3985 | PROT_READ | PROT_WRITE, | 4079 | PROT_READ | PROT_WRITE, |
3986 | MAP_SHARED | MAP_ANONYMOUS, | 4080 | MAP_PRIVATE | MAP_ANONYMOUS, |
3987 | 0); | 4081 | 0); |
3988 | up_write(¤t->mm->mmap_sem); | 4082 | up_write(¤t->mm->mmap_sem); |
3989 | 4083 | ||
3990 | if (IS_ERR((void *)memslot->userspace_addr)) | 4084 | if (IS_ERR((void *)userspace_addr)) |
3991 | return PTR_ERR((void *)memslot->userspace_addr); | 4085 | return PTR_ERR((void *)userspace_addr); |
4086 | |||
4087 | /* set userspace_addr atomically for kvm_hva_to_rmapp */ | ||
4088 | spin_lock(&kvm->mmu_lock); | ||
4089 | memslot->userspace_addr = userspace_addr; | ||
4090 | spin_unlock(&kvm->mmu_lock); | ||
3992 | } else { | 4091 | } else { |
3993 | if (!old.user_alloc && old.rmap) { | 4092 | if (!old.user_alloc && old.rmap) { |
3994 | int ret; | 4093 | int ret; |
@@ -4016,6 +4115,11 @@ int kvm_arch_set_memory_region(struct kvm *kvm, | |||
4016 | return 0; | 4115 | return 0; |
4017 | } | 4116 | } |
4018 | 4117 | ||
4118 | void kvm_arch_flush_shadow(struct kvm *kvm) | ||
4119 | { | ||
4120 | kvm_mmu_zap_all(kvm); | ||
4121 | } | ||
4122 | |||
4019 | int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) | 4123 | int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) |
4020 | { | 4124 | { |
4021 | return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE | 4125 | return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE |