Merge branch 'kvm-updates/2.6.32' of git://git.kernel.org/pub/scm/virt/kvm/kvm

* 'kvm-updates/2.6.32' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (202 commits)
  MAINTAINERS: update KVM entry
  KVM: correct error-handling code
  KVM: fix compile warnings on s390
  KVM: VMX: Check cpl before emulating debug register access
  KVM: fix misreporting of coalesced interrupts by kvm tracer
  KVM: x86: drop duplicate kvm_flush_remote_tlb calls
  KVM: VMX: call vmx_load_host_state() only if msr is cached
  KVM: VMX: Conditionally reload debug register 6
  KVM: Use thread debug register storage instead of kvm specific data
  KVM guest: do not batch pte updates from interrupt context
  KVM: Fix coalesced interrupt reporting in IOAPIC
  KVM guest: fix bogus wallclock physical address calculation
  KVM: VMX: Fix cr8 exiting control clobbering by EPT
  KVM: Optimize kvm_mmu_unprotect_page_virt() for tdp
  KVM: Document KVM_CAP_IRQCHIP
  KVM: Protect update_cr8_intercept() when running without an apic
  KVM: VMX: Fix EPT with WP bit change during paging
  KVM: Use kvm_{read,write}_guest_virt() to read and write segment descriptors
  KVM: x86 emulator: Add adc and sbb missing decoder flags
  KVM: Add missing #include
  ...

1 files changed, 523 insertions, 366 deletions

diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index b1f658ad2f06..944cc9c04b3c 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -15,7 +15,6 @@
  */
 #include <linux/kvm_host.h>
 
-#include "kvm_svm.h"
 #include "irq.h"
 #include "mmu.h"
 #include "kvm_cache_regs.h"
@@ -26,10 +25,12 @@
 #include <linux/vmalloc.h>
 #include <linux/highmem.h>
 #include <linux/sched.h>
+#include <linux/ftrace_event.h>
 
 #include <asm/desc.h>
 
 #include <asm/virtext.h>
+#include "trace.h"
 
 #define __ex(x) __kvm_handle_fault_on_reboot(x)
 
@@ -46,6 +47,10 @@ MODULE_LICENSE("GPL");
 #define SVM_FEATURE_LBRV (1 << 1)
 #define SVM_FEATURE_SVML (1 << 2)
 
+#define NESTED_EXIT_HOST        0       /* Exit handled on host level */
+#define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
+#define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
+
 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
 
 /* Turn on to get debugging output*/
@@ -57,6 +62,58 @@ MODULE_LICENSE("GPL");
 #define nsvm_printk(fmt, args...) do {} while(0)
 #endif
 
+static const u32 host_save_user_msrs[] = {
+#ifdef CONFIG_X86_64
+        MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
+        MSR_FS_BASE,
+#endif
+        MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
+};
+
+#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
+
+struct kvm_vcpu;
+
+struct nested_state {
+        struct vmcb *hsave;
+        u64 hsave_msr;
+        u64 vmcb;
+
+        /* These are the merged vectors */
+        u32 *msrpm;
+
+        /* gpa pointers to the real vectors */
+        u64 vmcb_msrpm;
+
+        /* cache for intercepts of the guest */
+        u16 intercept_cr_read;
+        u16 intercept_cr_write;
+        u16 intercept_dr_read;
+        u16 intercept_dr_write;
+        u32 intercept_exceptions;
+        u64 intercept;
+
+};
+
+struct vcpu_svm {
+        struct kvm_vcpu vcpu;
+        struct vmcb *vmcb;
+        unsigned long vmcb_pa;
+        struct svm_cpu_data *svm_data;
+        uint64_t asid_generation;
+        uint64_t sysenter_esp;
+        uint64_t sysenter_eip;
+
+        u64 next_rip;
+
+        u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
+        u64 host_gs_base;
+
+        u32 *msrpm;
+
+        struct nested_state nested;
+};
+
 /* enable NPT for AMD64 and X86 with PAE */
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 static bool npt_enabled = true;
@@ -67,15 +124,14 @@ static int npt = 1;
 
 module_param(npt, int, S_IRUGO);
 
-static int nested = 0;
+static int nested = 1;
 module_param(nested, int, S_IRUGO);
 
 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
+static void svm_complete_interrupts(struct vcpu_svm *svm);
 
-static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override);
+static int nested_svm_exit_handled(struct vcpu_svm *svm);
 static int nested_svm_vmexit(struct vcpu_svm *svm);
-static int nested_svm_vmsave(struct vcpu_svm *svm, void *nested_vmcb,
-                             void *arg2, void *opaque);
 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
                                       bool has_error_code, u32 error_code);
 
@@ -86,7 +142,22 @@ static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 
 static inline bool is_nested(struct vcpu_svm *svm)
 {
-        return svm->nested_vmcb;
+        return svm->nested.vmcb;
+}
+
+static inline void enable_gif(struct vcpu_svm *svm)
+{
+        svm->vcpu.arch.hflags |= HF_GIF_MASK;
+}
+
+static inline void disable_gif(struct vcpu_svm *svm)
+{
+        svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
+}
+
+static inline bool gif_set(struct vcpu_svm *svm)
+{
+        return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 }
 
 static unsigned long iopm_base;
@@ -147,19 +218,6 @@ static inline void invlpga(unsigned long addr, u32 asid)
         asm volatile (__ex(SVM_INVLPGA) :: "a"(addr), "c"(asid));
 }
 
-static inline unsigned long kvm_read_cr2(void)
-{
-        unsigned long cr2;
-
-        asm volatile ("mov %%cr2, %0" : "=r" (cr2));
-        return cr2;
-}
-
-static inline void kvm_write_cr2(unsigned long val)
-{
-        asm volatile ("mov %0, %%cr2" :: "r" (val));
-}
-
 static inline void force_new_asid(struct kvm_vcpu *vcpu)
 {
         to_svm(vcpu)->asid_generation--;
@@ -263,7 +321,7 @@ static void svm_hardware_enable(void *garbage)
 
         struct svm_cpu_data *svm_data;
         uint64_t efer;
-        struct desc_ptr gdt_descr;
+        struct descriptor_table gdt_descr;
         struct desc_struct *gdt;
         int me = raw_smp_processor_id();
 
@@ -283,8 +341,8 @@ static void svm_hardware_enable(void *garbage)
         svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
         svm_data->next_asid = svm_data->max_asid + 1;
 
-        asm volatile ("sgdt %0" : "=m"(gdt_descr));
-        gdt = (struct desc_struct *)gdt_descr.address;
+        kvm_get_gdt(&gdt_descr);
+        gdt = (struct desc_struct *)gdt_descr.base;
         svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 
         rdmsrl(MSR_EFER, efer);
@@ -367,8 +425,6 @@ static void svm_vcpu_init_msrpm(u32 *msrpm)
 #endif
         set_msr_interception(msrpm, MSR_K6_STAR, 1, 1);
         set_msr_interception(msrpm, MSR_IA32_SYSENTER_CS, 1, 1);
-        set_msr_interception(msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
-        set_msr_interception(msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
 }
 
 static void svm_enable_lbrv(struct vcpu_svm *svm)
@@ -595,8 +651,10 @@ static void init_vmcb(struct vcpu_svm *svm)
         }
         force_new_asid(&svm->vcpu);
 
-        svm->nested_vmcb = 0;
-        svm->vcpu.arch.hflags = HF_GIF_MASK;
+        svm->nested.vmcb = 0;
+        svm->vcpu.arch.hflags = 0;
+
+        enable_gif(svm);
 }
 
 static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
@@ -605,7 +663,7 @@ static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
 
         init_vmcb(svm);
 
-        if (vcpu->vcpu_id != 0) {
+        if (!kvm_vcpu_is_bsp(vcpu)) {
                 kvm_rip_write(vcpu, 0);
                 svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
                 svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
@@ -656,9 +714,9 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
         hsave_page = alloc_page(GFP_KERNEL);
         if (!hsave_page)
                 goto uninit;
-        svm->hsave = page_address(hsave_page);
+        svm->nested.hsave = page_address(hsave_page);
 
-        svm->nested_msrpm = page_address(nested_msrpm_pages);
+        svm->nested.msrpm = page_address(nested_msrpm_pages);
 
         svm->vmcb = page_address(page);
         clear_page(svm->vmcb);
@@ -669,7 +727,7 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
         fx_init(&svm->vcpu);
         svm->vcpu.fpu_active = 1;
         svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
-        if (svm->vcpu.vcpu_id == 0)
+        if (kvm_vcpu_is_bsp(&svm->vcpu))
                 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
 
         return &svm->vcpu;
@@ -688,8 +746,8 @@ static void svm_free_vcpu(struct kvm_vcpu *vcpu)
 
         __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
-        __free_page(virt_to_page(svm->hsave));
-        __free_pages(virt_to_page(svm->nested_msrpm), MSRPM_ALLOC_ORDER);
+        __free_page(virt_to_page(svm->nested.hsave));
+        __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
         kvm_vcpu_uninit(vcpu);
         kmem_cache_free(kvm_vcpu_cache, svm);
 }
@@ -740,6 +798,18 @@ static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
         to_svm(vcpu)->vmcb->save.rflags = rflags;
 }
 
+static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
+{
+        switch (reg) {
+        case VCPU_EXREG_PDPTR:
+                BUG_ON(!npt_enabled);
+                load_pdptrs(vcpu, vcpu->arch.cr3);
+                break;
+        default:
+                BUG();
+        }
+}
+
 static void svm_set_vintr(struct vcpu_svm *svm)
 {
         svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
@@ -1061,7 +1131,6 @@ static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr)
                 val = 0;
         }
 
-        KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler);
         return val;
 }
 
@@ -1070,8 +1139,6 @@ static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        KVMTRACE_2D(DR_WRITE, vcpu, (u32)dr, (u32)value, handler);
-
         *exception = 0;
 
         switch (dr) {
@@ -1119,25 +1186,9 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         fault_address  = svm->vmcb->control.exit_info_2;
         error_code = svm->vmcb->control.exit_info_1;
 
-        if (!npt_enabled)
-                KVMTRACE_3D(PAGE_FAULT, &svm->vcpu, error_code,
-                            (u32)fault_address, (u32)(fault_address >> 32),
-                            handler);
-        else
-                KVMTRACE_3D(TDP_FAULT, &svm->vcpu, error_code,
-                            (u32)fault_address, (u32)(fault_address >> 32),
-                            handler);
-        /*
-         * FIXME: Tis shouldn't be necessary here, but there is a flush
-         * missing in the MMU code. Until we find this bug, flush the
-         * complete TLB here on an NPF
-         */
-        if (npt_enabled)
-                svm_flush_tlb(&svm->vcpu);
-        else {
-                if (kvm_event_needs_reinjection(&svm->vcpu))
-                        kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
-        }
+        trace_kvm_page_fault(fault_address, error_code);
+        if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
+                kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
 }
 
@@ -1253,14 +1304,12 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 
 static int nmi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
-        KVMTRACE_0D(NMI, &svm->vcpu, handler);
         return 1;
 }
 
 static int intr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
         ++svm->vcpu.stat.irq_exits;
-        KVMTRACE_0D(INTR, &svm->vcpu, handler);
         return 1;
 }
 
@@ -1303,44 +1352,39 @@ static int nested_svm_check_permissions(struct vcpu_svm *svm)
 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
                                       bool has_error_code, u32 error_code)
 {
-        if (is_nested(svm)) {
-                svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
-                svm->vmcb->control.exit_code_hi = 0;
-                svm->vmcb->control.exit_info_1 = error_code;
-                svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
-                if (nested_svm_exit_handled(svm, false)) {
-                        nsvm_printk("VMexit -> EXCP 0x%x\n", nr);
-
-                        nested_svm_vmexit(svm);
-                        return 1;
-                }
-        }
+        if (!is_nested(svm))
+                return 0;
 
-        return 0;
+        svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
+        svm->vmcb->control.exit_code_hi = 0;
+        svm->vmcb->control.exit_info_1 = error_code;
+        svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
+
+        return nested_svm_exit_handled(svm);
 }
 
 static inline int nested_svm_intr(struct vcpu_svm *svm)
 {
-        if (is_nested(svm)) {
-                if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
-                        return 0;
+        if (!is_nested(svm))
+                return 0;
 
-                if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
-                        return 0;
+        if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
+                return 0;
 
-                svm->vmcb->control.exit_code = SVM_EXIT_INTR;
+        if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
+                return 0;
 
-                if (nested_svm_exit_handled(svm, false)) {
-                        nsvm_printk("VMexit -> INTR\n");
-                        nested_svm_vmexit(svm);
-                        return 1;
-                }
+        svm->vmcb->control.exit_code = SVM_EXIT_INTR;
+
+        if (nested_svm_exit_handled(svm)) {
+                nsvm_printk("VMexit -> INTR\n");
+                return 1;
         }
 
         return 0;
 }
 
-static struct page *nested_svm_get_page(struct vcpu_svm *svm, u64 gpa)
+static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, enum km_type idx)
 {
         struct page *page;
 
@@ -1348,236 +1392,246 @@ static struct page *nested_svm_get_page(struct vcpu_svm *svm, u64 gpa)
         page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
         up_read(&current->mm->mmap_sem);
 
-        if (is_error_page(page)) {
-                printk(KERN_INFO "%s: could not find page at 0x%llx\n",
-                       __func__, gpa);
-                kvm_release_page_clean(page);
-                kvm_inject_gp(&svm->vcpu, 0);
-                return NULL;
-        }
-        return page;
+        if (is_error_page(page))
+                goto error;
+
+        return kmap_atomic(page, idx);
+
+error:
+        kvm_release_page_clean(page);
+        kvm_inject_gp(&svm->vcpu, 0);
+
+        return NULL;
 }
 
-static int nested_svm_do(struct vcpu_svm *svm,
-                         u64 arg1_gpa, u64 arg2_gpa, void *opaque,
-                         int (*handler)(struct vcpu_svm *svm,
-                                        void *arg1,
-                                        void *arg2,
-                                        void *opaque))
+static void nested_svm_unmap(void *addr, enum km_type idx)
 {
-        struct page *arg1_page;
-        struct page *arg2_page = NULL;
-        void *arg1;
-        void *arg2 = NULL;
-        int retval;
+        struct page *page;
 
-        arg1_page = nested_svm_get_page(svm, arg1_gpa);
-        if(arg1_page == NULL)
-                return 1;
+        if (!addr)
+                return;
 
-        if (arg2_gpa) {
-                arg2_page = nested_svm_get_page(svm, arg2_gpa);
-                if(arg2_page == NULL) {
-                        kvm_release_page_clean(arg1_page);
-                        return 1;
-                }
-        }
+        page = kmap_atomic_to_page(addr);
+
+        kunmap_atomic(addr, idx);
+        kvm_release_page_dirty(page);
+}
+
+static bool nested_svm_exit_handled_msr(struct vcpu_svm *svm)
+{
+        u32 param = svm->vmcb->control.exit_info_1 & 1;
+        u32 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
+        bool ret = false;
+        u32 t0, t1;
+        u8 *msrpm;
 
-        arg1 = kmap_atomic(arg1_page, KM_USER0);
-        if (arg2_gpa)
-                arg2 = kmap_atomic(arg2_page, KM_USER1);
+        if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
+                return false;
 
-        retval = handler(svm, arg1, arg2, opaque);
+        msrpm = nested_svm_map(svm, svm->nested.vmcb_msrpm, KM_USER0);
+
+        if (!msrpm)
+                goto out;
+
+        switch (msr) {
+        case 0 ... 0x1fff:
+                t0 = (msr * 2) % 8;
+                t1 = msr / 8;
+                break;
+        case 0xc0000000 ... 0xc0001fff:
+                t0 = (8192 + msr - 0xc0000000) * 2;
+                t1 = (t0 / 8);
+                t0 %= 8;
+                break;
+        case 0xc0010000 ... 0xc0011fff:
+                t0 = (16384 + msr - 0xc0010000) * 2;
+                t1 = (t0 / 8);
+                t0 %= 8;
+                break;
+        default:
+                ret = true;
+                goto out;
+        }
 
-        kunmap_atomic(arg1, KM_USER0);
-        if (arg2_gpa)
-                kunmap_atomic(arg2, KM_USER1);
+        ret = msrpm[t1] & ((1 << param) << t0);
 
-        kvm_release_page_dirty(arg1_page);
-        if (arg2_gpa)
-                kvm_release_page_dirty(arg2_page);
+out:
+        nested_svm_unmap(msrpm, KM_USER0);
 
-        return retval;
+        return ret;
 }
 
-static int nested_svm_exit_handled_real(struct vcpu_svm *svm,
-                                        void *arg1,
-                                        void *arg2,
-                                        void *opaque)
+static int nested_svm_exit_special(struct vcpu_svm *svm)
 {
-        struct vmcb *nested_vmcb = (struct vmcb *)arg1;
-        bool kvm_overrides = *(bool *)opaque;
         u32 exit_code = svm->vmcb->control.exit_code;
 
-        if (kvm_overrides) {
-                switch (exit_code) {
-                case SVM_EXIT_INTR:
-                case SVM_EXIT_NMI:
-                        return 0;
+        switch (exit_code) {
+        case SVM_EXIT_INTR:
+        case SVM_EXIT_NMI:
+                return NESTED_EXIT_HOST;
                 /* For now we are always handling NPFs when using them */
-                case SVM_EXIT_NPF:
-                        if (npt_enabled)
-                                return 0;
-                        break;
-                /* When we're shadowing, trap PFs */
-                case SVM_EXIT_EXCP_BASE + PF_VECTOR:
-                        if (!npt_enabled)
-                                return 0;
-                        break;
-                default:
-                        break;
-                }
+        case SVM_EXIT_NPF:
+                if (npt_enabled)
+                        return NESTED_EXIT_HOST;
+                break;
+        /* When we're shadowing, trap PFs */
+        case SVM_EXIT_EXCP_BASE + PF_VECTOR:
+                if (!npt_enabled)
+                        return NESTED_EXIT_HOST;
+                break;
+        default:
+                break;
         }
 
+        return NESTED_EXIT_CONTINUE;
+}
+
+/*
+ * If this function returns true, this #vmexit was already handled
+ */
+static int nested_svm_exit_handled(struct vcpu_svm *svm)
+{
+        u32 exit_code = svm->vmcb->control.exit_code;
+        int vmexit = NESTED_EXIT_HOST;
+
         switch (exit_code) {
+        case SVM_EXIT_MSR:
+                vmexit = nested_svm_exit_handled_msr(svm);
+                break;
         case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
                 u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
-                if (nested_vmcb->control.intercept_cr_read & cr_bits)
-                        return 1;
+                if (svm->nested.intercept_cr_read & cr_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
                 u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
-                if (nested_vmcb->control.intercept_cr_write & cr_bits)
-                        return 1;
+                if (svm->nested.intercept_cr_write & cr_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
                 u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
-                if (nested_vmcb->control.intercept_dr_read & dr_bits)
-                        return 1;
+                if (svm->nested.intercept_dr_read & dr_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
                 u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
-                if (nested_vmcb->control.intercept_dr_write & dr_bits)
-                        return 1;
+                if (svm->nested.intercept_dr_write & dr_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
-                if (nested_vmcb->control.intercept_exceptions & excp_bits)
-                        return 1;
+                if (svm->nested.intercept_exceptions & excp_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         default: {
                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
                 nsvm_printk("exit code: 0x%x\n", exit_code);
-                if (nested_vmcb->control.intercept & exit_bits)
-                        return 1;
+                if (svm->nested.intercept & exit_bits)
+                        vmexit = NESTED_EXIT_DONE;
         }
         }
 
-        return 0;
-}
-
-static int nested_svm_exit_handled_msr(struct vcpu_svm *svm,
-                                       void *arg1, void *arg2,
-                                       void *opaque)
-{
-        struct vmcb *nested_vmcb = (struct vmcb *)arg1;
-        u8 *msrpm = (u8 *)arg2;
-        u32 t0, t1;
-        u32 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
-        u32 param = svm->vmcb->control.exit_info_1 & 1;
-
-        if (!(nested_vmcb->control.intercept & (1ULL << INTERCEPT_MSR_PROT)))
-                return 0;
-
-        switch(msr) {
-        case 0 ... 0x1fff:
-                t0 = (msr * 2) % 8;
-                t1 = msr / 8;
-                break;
-        case 0xc0000000 ... 0xc0001fff:
-                t0 = (8192 + msr - 0xc0000000) * 2;
-                t1 = (t0 / 8);
-                t0 %= 8;
-                break;
-        case 0xc0010000 ... 0xc0011fff:
-                t0 = (16384 + msr - 0xc0010000) * 2;
-                t1 = (t0 / 8);
-                t0 %= 8;
-                break;
-        default:
-                return 1;
-                break;
+        if (vmexit == NESTED_EXIT_DONE) {
+                nsvm_printk("#VMEXIT reason=%04x\n", exit_code);
+                nested_svm_vmexit(svm);
         }
-        if (msrpm[t1] & ((1 << param) << t0))
-                return 1;
 
-        return 0;
+        return vmexit;
+}
+
+static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
+{
+        struct vmcb_control_area *dst  = &dst_vmcb->control;
+        struct vmcb_control_area *from = &from_vmcb->control;
+
+        dst->intercept_cr_read    = from->intercept_cr_read;
+        dst->intercept_cr_write   = from->intercept_cr_write;
+        dst->intercept_dr_read    = from->intercept_dr_read;
+        dst->intercept_dr_write   = from->intercept_dr_write;
+        dst->intercept_exceptions = from->intercept_exceptions;
+        dst->intercept            = from->intercept;
+        dst->iopm_base_pa         = from->iopm_base_pa;
+        dst->msrpm_base_pa        = from->msrpm_base_pa;
+        dst->tsc_offset           = from->tsc_offset;
+        dst->asid                 = from->asid;
+        dst->tlb_ctl              = from->tlb_ctl;
+        dst->int_ctl              = from->int_ctl;
+        dst->int_vector           = from->int_vector;
+        dst->int_state            = from->int_state;
+        dst->exit_code            = from->exit_code;
+        dst->exit_code_hi         = from->exit_code_hi;
+        dst->exit_info_1          = from->exit_info_1;
+        dst->exit_info_2          = from->exit_info_2;
+        dst->exit_int_info        = from->exit_int_info;
+        dst->exit_int_info_err    = from->exit_int_info_err;
+        dst->nested_ctl           = from->nested_ctl;
+        dst->event_inj            = from->event_inj;
+        dst->event_inj_err        = from->event_inj_err;
+        dst->nested_cr3           = from->nested_cr3;
+        dst->lbr_ctl              = from->lbr_ctl;
 }
 
-static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override)
+static int nested_svm_vmexit(struct vcpu_svm *svm)
 {
-        bool k = kvm_override;
-
-        switch (svm->vmcb->control.exit_code) {
-        case SVM_EXIT_MSR:
-                return nested_svm_do(svm, svm->nested_vmcb,
-                                     svm->nested_vmcb_msrpm, NULL,
-                                     nested_svm_exit_handled_msr);
-        default: break;
-        }
+        struct vmcb *nested_vmcb;
+        struct vmcb *hsave = svm->nested.hsave;
+        struct vmcb *vmcb = svm->vmcb;
 
-        return nested_svm_do(svm, svm->nested_vmcb, 0, &k,
-                             nested_svm_exit_handled_real);
-}
-
-static int nested_svm_vmexit_real(struct vcpu_svm *svm, void *arg1,
-                                  void *arg2, void *opaque)
-{
-        struct vmcb *nested_vmcb = (struct vmcb *)arg1;
-        struct vmcb *hsave = svm->hsave;
-        u64 nested_save[] = { nested_vmcb->save.cr0,
-                              nested_vmcb->save.cr3,
-                              nested_vmcb->save.cr4,
-                              nested_vmcb->save.efer,
-                              nested_vmcb->control.intercept_cr_read,
-                              nested_vmcb->control.intercept_cr_write,
-                              nested_vmcb->control.intercept_dr_read,
-                              nested_vmcb->control.intercept_dr_write,
-                              nested_vmcb->control.intercept_exceptions,
-                              nested_vmcb->control.intercept,
-                              nested_vmcb->control.msrpm_base_pa,
-                              nested_vmcb->control.iopm_base_pa,
-                              nested_vmcb->control.tsc_offset };
+        nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, KM_USER0);
+        if (!nested_vmcb)
+                return 1;
 
         /* Give the current vmcb to the guest */
-        memcpy(nested_vmcb, svm->vmcb, sizeof(struct vmcb));
-        nested_vmcb->save.cr0 = nested_save[0];
-        if (!npt_enabled)
-                nested_vmcb->save.cr3 = nested_save[1];
-        nested_vmcb->save.cr4 = nested_save[2];
-        nested_vmcb->save.efer = nested_save[3];
-        nested_vmcb->control.intercept_cr_read = nested_save[4];
-        nested_vmcb->control.intercept_cr_write = nested_save[5];
-        nested_vmcb->control.intercept_dr_read = nested_save[6];
-        nested_vmcb->control.intercept_dr_write = nested_save[7];
-        nested_vmcb->control.intercept_exceptions = nested_save[8];
-        nested_vmcb->control.intercept = nested_save[9];
-        nested_vmcb->control.msrpm_base_pa = nested_save[10];
-        nested_vmcb->control.iopm_base_pa = nested_save[11];
-        nested_vmcb->control.tsc_offset = nested_save[12];
+        disable_gif(svm);
+
+        nested_vmcb->save.es     = vmcb->save.es;
+        nested_vmcb->save.cs     = vmcb->save.cs;
+        nested_vmcb->save.ss     = vmcb->save.ss;
+        nested_vmcb->save.ds     = vmcb->save.ds;
+        nested_vmcb->save.gdtr   = vmcb->save.gdtr;
+        nested_vmcb->save.idtr   = vmcb->save.idtr;
+        if (npt_enabled)
+                nested_vmcb->save.cr3    = vmcb->save.cr3;
+        nested_vmcb->save.cr2    = vmcb->save.cr2;
+        nested_vmcb->save.rflags = vmcb->save.rflags;
+        nested_vmcb->save.rip    = vmcb->save.rip;
+        nested_vmcb->save.rsp    = vmcb->save.rsp;
+        nested_vmcb->save.rax    = vmcb->save.rax;
+        nested_vmcb->save.dr7    = vmcb->save.dr7;
+        nested_vmcb->save.dr6    = vmcb->save.dr6;
+        nested_vmcb->save.cpl    = vmcb->save.cpl;
+
+        nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
+        nested_vmcb->control.int_vector        = vmcb->control.int_vector;
+        nested_vmcb->control.int_state         = vmcb->control.int_state;
+        nested_vmcb->control.exit_code         = vmcb->control.exit_code;
+        nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
+        nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
+        nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
+        nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
+        nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
+        nested_vmcb->control.tlb_ctl           = 0;
+        nested_vmcb->control.event_inj         = 0;
+        nested_vmcb->control.event_inj_err     = 0;
 
         /* We always set V_INTR_MASKING and remember the old value in hflags */
         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
 
-        if ((nested_vmcb->control.int_ctl & V_IRQ_MASK) &&
-            (nested_vmcb->control.int_vector)) {
-                nsvm_printk("WARNING: IRQ 0x%x still enabled on #VMEXIT\n",
-                                nested_vmcb->control.int_vector);
-        }
-
         /* Restore the original control entries */
-        svm->vmcb->control = hsave->control;
+        copy_vmcb_control_area(vmcb, hsave);
 
         /* Kill any pending exceptions */
         if (svm->vcpu.arch.exception.pending == true)
                 nsvm_printk("WARNING: Pending Exception\n");
-        svm->vcpu.arch.exception.pending = false;
+
+        kvm_clear_exception_queue(&svm->vcpu);
+        kvm_clear_interrupt_queue(&svm->vcpu);
 
         /* Restore selected save entries */
         svm->vmcb->save.es = hsave->save.es;
@@ -1603,19 +1657,10 @@ static int nested_svm_vmexit_real(struct vcpu_svm *svm, void *arg1,
         svm->vmcb->save.cpl = 0;
         svm->vmcb->control.exit_int_info = 0;
 
-        svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
         /* Exit nested SVM mode */
-        svm->nested_vmcb = 0;
+        svm->nested.vmcb = 0;
 
-        return 0;
-}
-
-static int nested_svm_vmexit(struct vcpu_svm *svm)
-{
-        nsvm_printk("VMexit\n");
-        if (nested_svm_do(svm, svm->nested_vmcb, 0,
-                          NULL, nested_svm_vmexit_real))
-                return 1;
+        nested_svm_unmap(nested_vmcb, KM_USER0);
 
         kvm_mmu_reset_context(&svm->vcpu);
         kvm_mmu_load(&svm->vcpu);
@@ -1623,38 +1668,63 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         return 0;
 }
 
-static int nested_svm_vmrun_msrpm(struct vcpu_svm *svm, void *arg1,
-                                  void *arg2, void *opaque)
+static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
 {
+        u32 *nested_msrpm;
         int i;
-        u32 *nested_msrpm = (u32*)arg1;
+
+        nested_msrpm = nested_svm_map(svm, svm->nested.vmcb_msrpm, KM_USER0);
+        if (!nested_msrpm)
+                return false;
+
         for (i=0; i< PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER) / 4; i++)
-                svm->nested_msrpm[i] = svm->msrpm[i] | nested_msrpm[i];
-        svm->vmcb->control.msrpm_base_pa = __pa(svm->nested_msrpm);
+                svm->nested.msrpm[i] = svm->msrpm[i] | nested_msrpm[i];
 
-        return 0;
+        svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
+
+        nested_svm_unmap(nested_msrpm, KM_USER0);
+
+        return true;
 }
 
-static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
-                            void *arg2, void *opaque)
+static bool nested_svm_vmrun(struct vcpu_svm *svm)
 {
-        struct vmcb *nested_vmcb = (struct vmcb *)arg1;
-        struct vmcb *hsave = svm->hsave;
+        struct vmcb *nested_vmcb;
+        struct vmcb *hsave = svm->nested.hsave;
+        struct vmcb *vmcb = svm->vmcb;
+
+        nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, KM_USER0);
+        if (!nested_vmcb)
+                return false;
 
         /* nested_vmcb is our indicator if nested SVM is activated */
-        svm->nested_vmcb = svm->vmcb->save.rax;
+        svm->nested.vmcb = svm->vmcb->save.rax;
 
         /* Clear internal status */
-        svm->vcpu.arch.exception.pending = false;
+        kvm_clear_exception_queue(&svm->vcpu);
+        kvm_clear_interrupt_queue(&svm->vcpu);
 
         /* Save the old vmcb, so we don't need to pick what we save, but
            can restore everything when a VMEXIT occurs */
-        memcpy(hsave, svm->vmcb, sizeof(struct vmcb));
-        /* We need to remember the original CR3 in the SPT case */
-        if (!npt_enabled)
-                hsave->save.cr3 = svm->vcpu.arch.cr3;
-        hsave->save.cr4 = svm->vcpu.arch.cr4;
-        hsave->save.rip = svm->next_rip;
+        hsave->save.es     = vmcb->save.es;
+        hsave->save.cs     = vmcb->save.cs;
+        hsave->save.ss     = vmcb->save.ss;
+        hsave->save.ds     = vmcb->save.ds;
+        hsave->save.gdtr   = vmcb->save.gdtr;
+        hsave->save.idtr   = vmcb->save.idtr;
+        hsave->save.efer   = svm->vcpu.arch.shadow_efer;
+        hsave->save.cr0    = svm->vcpu.arch.cr0;
+        hsave->save.cr4    = svm->vcpu.arch.cr4;
+        hsave->save.rflags = vmcb->save.rflags;
+        hsave->save.rip    = svm->next_rip;
+        hsave->save.rsp    = vmcb->save.rsp;
+        hsave->save.rax    = vmcb->save.rax;
+        if (npt_enabled)
+                hsave->save.cr3    = vmcb->save.cr3;
+        else
+                hsave->save.cr3    = svm->vcpu.arch.cr3;
+
+        copy_vmcb_control_area(hsave, vmcb);
 
         if (svm->vmcb->save.rflags & X86_EFLAGS_IF)
                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
@@ -1679,7 +1749,7 @@ static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
                 kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
                 kvm_mmu_reset_context(&svm->vcpu);
         }
-        svm->vmcb->save.cr2 = nested_vmcb->save.cr2;
+        svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
@@ -1706,7 +1776,15 @@ static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
 
         svm->vmcb->control.intercept |= nested_vmcb->control.intercept;
 
-        svm->nested_vmcb_msrpm = nested_vmcb->control.msrpm_base_pa;
+        svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa;
+
+        /* cache intercepts */
+        svm->nested.intercept_cr_read    = nested_vmcb->control.intercept_cr_read;
+        svm->nested.intercept_cr_write   = nested_vmcb->control.intercept_cr_write;
+        svm->nested.intercept_dr_read    = nested_vmcb->control.intercept_dr_read;
+        svm->nested.intercept_dr_write   = nested_vmcb->control.intercept_dr_write;
+        svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
+        svm->nested.intercept            = nested_vmcb->control.intercept;
 
         force_new_asid(&svm->vcpu);
         svm->vmcb->control.exit_int_info = nested_vmcb->control.exit_int_info;
@@ -1734,12 +1812,14 @@ static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
 
-        svm->vcpu.arch.hflags |= HF_GIF_MASK;
+        nested_svm_unmap(nested_vmcb, KM_USER0);
 
-        return 0;
+        enable_gif(svm);
+
+        return true;
 }
 
-static int nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
+static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
 {
         to_vmcb->save.fs = from_vmcb->save.fs;
         to_vmcb->save.gs = from_vmcb->save.gs;
@@ -1753,44 +1833,44 @@ static int nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
-
-        return 1;
-}
-
-static int nested_svm_vmload(struct vcpu_svm *svm, void *nested_vmcb,
-                             void *arg2, void *opaque)
-{
-        return nested_svm_vmloadsave((struct vmcb *)nested_vmcb, svm->vmcb);
-}
-
-static int nested_svm_vmsave(struct vcpu_svm *svm, void *nested_vmcb,
-                             void *arg2, void *opaque)
-{
-        return nested_svm_vmloadsave(svm->vmcb, (struct vmcb *)nested_vmcb);
 }
 
 static int vmload_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
+        struct vmcb *nested_vmcb;
+
         if (nested_svm_check_permissions(svm))
                 return 1;
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        nested_svm_do(svm, svm->vmcb->save.rax, 0, NULL, nested_svm_vmload);
+        nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, KM_USER0);
+        if (!nested_vmcb)
+                return 1;
+
+        nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
+        nested_svm_unmap(nested_vmcb, KM_USER0);
 
         return 1;
 }
 
 static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
+        struct vmcb *nested_vmcb;
+
         if (nested_svm_check_permissions(svm))
                 return 1;
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        nested_svm_do(svm, svm->vmcb->save.rax, 0, NULL, nested_svm_vmsave);
+        nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, KM_USER0);
+        if (!nested_vmcb)
+                return 1;
+
+        nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
+        nested_svm_unmap(nested_vmcb, KM_USER0);
 
         return 1;
 }
@@ -1798,19 +1878,29 @@ static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 static int vmrun_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
         nsvm_printk("VMrun\n");
+
         if (nested_svm_check_permissions(svm))
                 return 1;
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        if (nested_svm_do(svm, svm->vmcb->save.rax, 0,
-                          NULL, nested_svm_vmrun))
+        if (!nested_svm_vmrun(svm))
                 return 1;
 
-        if (nested_svm_do(svm, svm->nested_vmcb_msrpm, 0,
-                      NULL, nested_svm_vmrun_msrpm))
-                return 1;
+        if (!nested_svm_vmrun_msrpm(svm))
+                goto failed;
+
+        return 1;
+
+failed:
+
+        svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
+        svm->vmcb->control.exit_code_hi = 0;
+        svm->vmcb->control.exit_info_1  = 0;
+        svm->vmcb->control.exit_info_2  = 0;
+
+        nested_svm_vmexit(svm);
 
         return 1;
 }
@@ -1823,7 +1913,7 @@ static int stgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        svm->vcpu.arch.hflags |= HF_GIF_MASK;
+        enable_gif(svm);
 
         return 1;
 }
@@ -1836,7 +1926,7 @@ static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
+        disable_gif(svm);
 
         /* After a CLGI no interrupts should come */
         svm_clear_vintr(svm);
@@ -1845,6 +1935,19 @@ static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
+static int invlpga_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+{
+        struct kvm_vcpu *vcpu = &svm->vcpu;
+        nsvm_printk("INVLPGA\n");
+
+        /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
+        kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);
+
+        svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
+        skip_emulated_instruction(&svm->vcpu);
+        return 1;
+}
+
 static int invalid_op_interception(struct vcpu_svm *svm,
                                    struct kvm_run *kvm_run)
 {
@@ -1953,7 +2056,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         switch (ecx) {
-        case MSR_IA32_TIME_STAMP_COUNTER: {
+        case MSR_IA32_TSC: {
                 u64 tsc;
 
                 rdtscll(tsc);
@@ -1981,10 +2084,10 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
                 *data = svm->vmcb->save.sysenter_cs;
                 break;
         case MSR_IA32_SYSENTER_EIP:
-                *data = svm->vmcb->save.sysenter_eip;
+                *data = svm->sysenter_eip;
                 break;
         case MSR_IA32_SYSENTER_ESP:
-                *data = svm->vmcb->save.sysenter_esp;
+                *data = svm->sysenter_esp;
                 break;
         /* Nobody will change the following 5 values in the VMCB so
            we can safely return them on rdmsr. They will always be 0
@@ -2005,7 +2108,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
                 *data = svm->vmcb->save.last_excp_to;
                 break;
         case MSR_VM_HSAVE_PA:
-                *data = svm->hsave_msr;
+                *data = svm->nested.hsave_msr;
                 break;
         case MSR_VM_CR:
                 *data = 0;
@@ -2027,8 +2130,7 @@ static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         if (svm_get_msr(&svm->vcpu, ecx, &data))
                 kvm_inject_gp(&svm->vcpu, 0);
         else {
-                KVMTRACE_3D(MSR_READ, &svm->vcpu, ecx, (u32)data,
-                            (u32)(data >> 32), handler);
+                trace_kvm_msr_read(ecx, data);
 
                 svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
                 svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
@@ -2043,7 +2145,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         switch (ecx) {
-        case MSR_IA32_TIME_STAMP_COUNTER: {
+        case MSR_IA32_TSC: {
                 u64 tsc;
 
                 rdtscll(tsc);
@@ -2071,9 +2173,11 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
                 svm->vmcb->save.sysenter_cs = data;
                 break;
         case MSR_IA32_SYSENTER_EIP:
+                svm->sysenter_eip = data;
                 svm->vmcb->save.sysenter_eip = data;
                 break;
         case MSR_IA32_SYSENTER_ESP:
+                svm->sysenter_esp = data;
                 svm->vmcb->save.sysenter_esp = data;
                 break;
         case MSR_IA32_DEBUGCTLMSR:
@@ -2091,24 +2195,12 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
                 else
                         svm_disable_lbrv(svm);
                 break;
-        case MSR_K7_EVNTSEL0:
-        case MSR_K7_EVNTSEL1:
-        case MSR_K7_EVNTSEL2:
-        case MSR_K7_EVNTSEL3:
-        case MSR_K7_PERFCTR0:
-        case MSR_K7_PERFCTR1:
-        case MSR_K7_PERFCTR2:
-        case MSR_K7_PERFCTR3:
-                /*
-                 * Just discard all writes to the performance counters; this
-                 * should keep both older linux and windows 64-bit guests
-                 * happy
-                 */
-                pr_unimpl(vcpu, "unimplemented perfctr wrmsr: 0x%x data 0x%llx\n", ecx, data);
-
-                break;
         case MSR_VM_HSAVE_PA:
-                svm->hsave_msr = data;
+                svm->nested.hsave_msr = data;
+                break;
+        case MSR_VM_CR:
+        case MSR_VM_IGNNE:
+                pr_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
                 break;
         default:
                 return kvm_set_msr_common(vcpu, ecx, data);
@@ -2122,8 +2214,7 @@ static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
                 | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
 
-        KVMTRACE_3D(MSR_WRITE, &svm->vcpu, ecx, (u32)data, (u32)(data >> 32),
-                    handler);
+        trace_kvm_msr_write(ecx, data);
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
         if (svm_set_msr(&svm->vcpu, ecx, data))
@@ -2144,8 +2235,6 @@ static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 static int interrupt_window_interception(struct vcpu_svm *svm,
                                    struct kvm_run *kvm_run)
 {
-        KVMTRACE_0D(PEND_INTR, &svm->vcpu, handler);
-
         svm_clear_vintr(svm);
         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
         /*
@@ -2201,7 +2290,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
         [SVM_EXIT_INVD]                         = emulate_on_interception,
         [SVM_EXIT_HLT]                          = halt_interception,
         [SVM_EXIT_INVLPG]                       = invlpg_interception,
-        [SVM_EXIT_INVLPGA]                      = invalid_op_interception,
+        [SVM_EXIT_INVLPGA]                      = invlpga_interception,
         [SVM_EXIT_IOIO]                         = io_interception,
         [SVM_EXIT_MSR]                          = msr_interception,
         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
@@ -2224,20 +2313,26 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
         struct vcpu_svm *svm = to_svm(vcpu);
         u32 exit_code = svm->vmcb->control.exit_code;
 
-        KVMTRACE_3D(VMEXIT, vcpu, exit_code, (u32)svm->vmcb->save.rip,
-                    (u32)((u64)svm->vmcb->save.rip >> 32), entryexit);
+        trace_kvm_exit(exit_code, svm->vmcb->save.rip);
 
         if (is_nested(svm)) {
+                int vmexit;
+
                 nsvm_printk("nested handle_exit: 0x%x | 0x%lx | 0x%lx | 0x%lx\n",
                             exit_code, svm->vmcb->control.exit_info_1,
                             svm->vmcb->control.exit_info_2, svm->vmcb->save.rip);
-                if (nested_svm_exit_handled(svm, true)) {
-                        nested_svm_vmexit(svm);
-                        nsvm_printk("-> #VMEXIT\n");
+
+                vmexit = nested_svm_exit_special(svm);
+
+                if (vmexit == NESTED_EXIT_CONTINUE)
+                        vmexit = nested_svm_exit_handled(svm);
+
+                if (vmexit == NESTED_EXIT_DONE)
                         return 1;
-                }
         }
 
+        svm_complete_interrupts(svm);
+
         if (npt_enabled) {
                 int mmu_reload = 0;
                 if ((vcpu->arch.cr0 ^ svm->vmcb->save.cr0) & X86_CR0_PG) {
@@ -2246,12 +2341,6 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
                 }
                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
-                if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
-                        if (!load_pdptrs(vcpu, vcpu->arch.cr3)) {
-                                kvm_inject_gp(vcpu, 0);
-                                return 1;
-                        }
-                }
                 if (mmu_reload) {
                         kvm_mmu_reset_context(vcpu);
                         kvm_mmu_load(vcpu);
@@ -2319,7 +2408,7 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
 {
         struct vmcb_control_area *control;
 
-        KVMTRACE_1D(INJ_VIRQ, &svm->vcpu, (u32)irq, handler);
+        trace_kvm_inj_virq(irq);
 
         ++svm->vcpu.stat.irq_injections;
         control = &svm->vmcb->control;
@@ -2329,21 +2418,14 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
 }
 
-static void svm_queue_irq(struct kvm_vcpu *vcpu, unsigned nr)
-{
-        struct vcpu_svm *svm = to_svm(vcpu);
-
-        svm->vmcb->control.event_inj = nr |
-                SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
-}
-
 static void svm_set_irq(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        nested_svm_intr(svm);
+        BUG_ON(!(gif_set(svm)));
 
-        svm_queue_irq(vcpu, vcpu->arch.interrupt.nr);
+        svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
+                SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
 }
 
 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
@@ -2371,13 +2453,25 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
         struct vmcb *vmcb = svm->vmcb;
         return (vmcb->save.rflags & X86_EFLAGS_IF) &&
                 !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
-                (svm->vcpu.arch.hflags & HF_GIF_MASK);
+                gif_set(svm) &&
+                !(is_nested(svm) && (svm->vcpu.arch.hflags & HF_VINTR_MASK));
 }
 
 static void enable_irq_window(struct kvm_vcpu *vcpu)
 {
-        svm_set_vintr(to_svm(vcpu));
-        svm_inject_irq(to_svm(vcpu), 0x0);
+        struct vcpu_svm *svm = to_svm(vcpu);
+        nsvm_printk("Trying to open IRQ window\n");
+
+        nested_svm_intr(svm);
+
+        /* In case GIF=0 we can't rely on the CPU to tell us when
+         * GIF becomes 1, because that's a separate STGI/VMRUN intercept.
+         * The next time we get that intercept, this function will be
+         * called again though and we'll get the vintr intercept. */
+        if (gif_set(svm)) {
+                svm_set_vintr(svm);
+                svm_inject_irq(svm, 0x0);
+        }
 }
 
 static void enable_nmi_window(struct kvm_vcpu *vcpu)
@@ -2456,6 +2550,8 @@ static void svm_complete_interrupts(struct vcpu_svm *svm)
         case SVM_EXITINTINFO_TYPE_EXEPT:
                 /* In case of software exception do not reinject an exception
                    vector, but re-execute and instruction instead */
+                if (is_nested(svm))
+                        break;
                 if (kvm_exception_is_soft(vector))
                         break;
                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
@@ -2498,9 +2594,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         fs_selector = kvm_read_fs();
         gs_selector = kvm_read_gs();
         ldt_selector = kvm_read_ldt();
-        svm->host_cr2 = kvm_read_cr2();
-        if (!is_nested(svm))
-                svm->vmcb->save.cr2 = vcpu->arch.cr2;
+        svm->vmcb->save.cr2 = vcpu->arch.cr2;
         /* required for live migration with NPT */
         if (npt_enabled)
                 svm->vmcb->save.cr3 = vcpu->arch.cr3;
@@ -2585,8 +2679,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
 
-        kvm_write_cr2(svm->host_cr2);
-
         kvm_load_fs(fs_selector);
         kvm_load_gs(gs_selector);
         kvm_load_ldt(ldt_selector);
@@ -2602,7 +2694,10 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
         svm->next_rip = 0;
 
-        svm_complete_interrupts(svm);
+        if (npt_enabled) {
+                vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
+                vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
+        }
 }
 
 #undef R
@@ -2673,6 +2768,64 @@ static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
         return 0;
 }
 
+static const struct trace_print_flags svm_exit_reasons_str[] = {
+        { SVM_EXIT_READ_CR0,                    "read_cr0" },
+        { SVM_EXIT_READ_CR3,                    "read_cr3" },
+        { SVM_EXIT_READ_CR4,                    "read_cr4" },
+        { SVM_EXIT_READ_CR8,                    "read_cr8" },
+        { SVM_EXIT_WRITE_CR0,                   "write_cr0" },
+        { SVM_EXIT_WRITE_CR3,                   "write_cr3" },
+        { SVM_EXIT_WRITE_CR4,                   "write_cr4" },
+        { SVM_EXIT_WRITE_CR8,                   "write_cr8" },
+        { SVM_EXIT_READ_DR0,                    "read_dr0" },
+        { SVM_EXIT_READ_DR1,                    "read_dr1" },
+        { SVM_EXIT_READ_DR2,                    "read_dr2" },
+        { SVM_EXIT_READ_DR3,                    "read_dr3" },
+        { SVM_EXIT_WRITE_DR0,                   "write_dr0" },
+        { SVM_EXIT_WRITE_DR1,                   "write_dr1" },
+        { SVM_EXIT_WRITE_DR2,                   "write_dr2" },
+        { SVM_EXIT_WRITE_DR3,                   "write_dr3" },
+        { SVM_EXIT_WRITE_DR5,                   "write_dr5" },
+        { SVM_EXIT_WRITE_DR7,                   "write_dr7" },
+        { SVM_EXIT_EXCP_BASE + DB_VECTOR,       "DB excp" },
+        { SVM_EXIT_EXCP_BASE + BP_VECTOR,       "BP excp" },
+        { SVM_EXIT_EXCP_BASE + UD_VECTOR,       "UD excp" },
+        { SVM_EXIT_EXCP_BASE + PF_VECTOR,       "PF excp" },
+        { SVM_EXIT_EXCP_BASE + NM_VECTOR,       "NM excp" },
+        { SVM_EXIT_EXCP_BASE + MC_VECTOR,       "MC excp" },
+        { SVM_EXIT_INTR,                        "interrupt" },
+        { SVM_EXIT_NMI,                         "nmi" },
+        { SVM_EXIT_SMI,                         "smi" },
+        { SVM_EXIT_INIT,                        "init" },
+        { SVM_EXIT_VINTR,                       "vintr" },
+        { SVM_EXIT_CPUID,                       "cpuid" },
+        { SVM_EXIT_INVD,                        "invd" },
+        { SVM_EXIT_HLT,                         "hlt" },
+        { SVM_EXIT_INVLPG,                      "invlpg" },
+        { SVM_EXIT_INVLPGA,                     "invlpga" },
+        { SVM_EXIT_IOIO,                        "io" },
+        { SVM_EXIT_MSR,                         "msr" },
+        { SVM_EXIT_TASK_SWITCH,                 "task_switch" },
+        { SVM_EXIT_SHUTDOWN,                    "shutdown" },
+        { SVM_EXIT_VMRUN,                       "vmrun" },
+        { SVM_EXIT_VMMCALL,                     "hypercall" },
+        { SVM_EXIT_VMLOAD,                      "vmload" },
+        { SVM_EXIT_VMSAVE,                      "vmsave" },
+        { SVM_EXIT_STGI,                        "stgi" },
+        { SVM_EXIT_CLGI,                        "clgi" },
+        { SVM_EXIT_SKINIT,                      "skinit" },
+        { SVM_EXIT_WBINVD,                      "wbinvd" },
+        { SVM_EXIT_MONITOR,                     "monitor" },
+        { SVM_EXIT_MWAIT,                       "mwait" },
+        { SVM_EXIT_NPF,                         "npf" },
+        { -1, NULL }
+};
+
+static bool svm_gb_page_enable(void)
+{
+        return true;
+}
+
 static struct kvm_x86_ops svm_x86_ops = {
         .cpu_has_kvm_support = has_svm,
         .disabled_by_bios = is_disabled,
@@ -2710,6 +2863,7 @@ static struct kvm_x86_ops svm_x86_ops = {
         .set_gdt = svm_set_gdt,
         .get_dr = svm_get_dr,
         .set_dr = svm_set_dr,
+        .cache_reg = svm_cache_reg,
         .get_rflags = svm_get_rflags,
         .set_rflags = svm_set_rflags,
 
@@ -2733,6 +2887,9 @@ static struct kvm_x86_ops svm_x86_ops = {
         .set_tss_addr = svm_set_tss_addr,
         .get_tdp_level = get_npt_level,
         .get_mt_mask = svm_get_mt_mask,
+
+        .exit_reasons_str = svm_exit_reasons_str,
+        .gb_page_enable = svm_gb_page_enable,
 };
 
 static int __init svm_init(void)