1 files changed, 390 insertions, 270 deletions

diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index c17404add91f..737361fcd503 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -26,6 +26,7 @@
 #include <linux/highmem.h>
 #include <linux/sched.h>
 #include <linux/ftrace_event.h>
+#include <linux/slab.h>
 
 #include <asm/desc.h>
 
@@ -46,6 +47,7 @@ MODULE_LICENSE("GPL");
 #define SVM_FEATURE_NPT  (1 << 0)
 #define SVM_FEATURE_LBRV (1 << 1)
 #define SVM_FEATURE_SVML (1 << 2)
+#define SVM_FEATURE_PAUSE_FILTER (1 << 10)
 
 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
@@ -53,15 +55,6 @@ MODULE_LICENSE("GPL");
 
 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
 
-/* Turn on to get debugging output*/
-/* #define NESTED_DEBUG */
-
-#ifdef NESTED_DEBUG
-#define nsvm_printk(fmt, args...) printk(KERN_INFO fmt, ## args)
-#else
-#define nsvm_printk(fmt, args...) do {} while(0)
-#endif
-
 static const u32 host_save_user_msrs[] = {
 #ifdef CONFIG_X86_64
         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
@@ -85,6 +78,9 @@ struct nested_state {
         /* gpa pointers to the real vectors */
         u64 vmcb_msrpm;
 
+        /* A VMEXIT is required but not yet emulated */
+        bool exit_required;
+
         /* cache for intercepts of the guest */
         u16 intercept_cr_read;
         u16 intercept_cr_write;
@@ -112,6 +108,8 @@ struct vcpu_svm {
         u32 *msrpm;
 
         struct nested_state nested;
+
+        bool nmi_singlestep;
 };
 
 /* enable NPT for AMD64 and X86 with PAE */
@@ -234,7 +232,7 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
                 efer &= ~EFER_LME;
 
         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
-        vcpu->arch.shadow_efer = efer;
+        vcpu->arch.efer = efer;
 }
 
 static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
@@ -286,7 +284,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         if (!svm->next_rip) {
-                if (emulate_instruction(vcpu, vcpu->run, 0, 0, EMULTYPE_SKIP) !=
+                if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) !=
                                 EMULATE_DONE)
                         printk(KERN_DEBUG "%s: NOP\n", __func__);
                 return;
@@ -316,75 +314,79 @@ static void svm_hardware_disable(void *garbage)
         cpu_svm_disable();
 }
 
-static void svm_hardware_enable(void *garbage)
+static int svm_hardware_enable(void *garbage)
 {
 
-        struct svm_cpu_data *svm_data;
+        struct svm_cpu_data *sd;
         uint64_t efer;
         struct descriptor_table gdt_descr;
         struct desc_struct *gdt;
         int me = raw_smp_processor_id();
 
+        rdmsrl(MSR_EFER, efer);
+        if (efer & EFER_SVME)
+                return -EBUSY;
+
         if (!has_svm()) {
-                printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me);
-                return;
+                printk(KERN_ERR "svm_hardware_enable: err EOPNOTSUPP on %d\n",
+                       me);
+                return -EINVAL;
         }
-        svm_data = per_cpu(svm_data, me);
+        sd = per_cpu(svm_data, me);
 
-        if (!svm_data) {
-                printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n",
+        if (!sd) {
+                printk(KERN_ERR "svm_hardware_enable: svm_data is NULL on %d\n",
                        me);
-                return;
+                return -EINVAL;
         }
 
-        svm_data->asid_generation = 1;
-        svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
-        svm_data->next_asid = svm_data->max_asid + 1;
+        sd->asid_generation = 1;
+        sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
+        sd->next_asid = sd->max_asid + 1;
 
         kvm_get_gdt(&gdt_descr);
         gdt = (struct desc_struct *)gdt_descr.base;
-        svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
+        sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 
-        rdmsrl(MSR_EFER, efer);
         wrmsrl(MSR_EFER, efer | EFER_SVME);
 
-        wrmsrl(MSR_VM_HSAVE_PA,
-               page_to_pfn(svm_data->save_area) << PAGE_SHIFT);
+        wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
+
+        return 0;
 }
 
 static void svm_cpu_uninit(int cpu)
 {
-        struct svm_cpu_data *svm_data
-                = per_cpu(svm_data, raw_smp_processor_id());
+        struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 
-        if (!svm_data)
+        if (!sd)
                 return;
 
         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
-        __free_page(svm_data->save_area);
-        kfree(svm_data);
+        __free_page(sd->save_area);
+        kfree(sd);
 }
 
 static int svm_cpu_init(int cpu)
 {
-        struct svm_cpu_data *svm_data;
+        struct svm_cpu_data *sd;
         int r;
 
-        svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
-        if (!svm_data)
+        sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
+        if (!sd)
                 return -ENOMEM;
-        svm_data->cpu = cpu;
-        svm_data->save_area = alloc_page(GFP_KERNEL);
+        sd->cpu = cpu;
+        sd->save_area = alloc_page(GFP_KERNEL);
         r = -ENOMEM;
-        if (!svm_data->save_area)
+        if (!sd->save_area)
                 goto err_1;
 
-        per_cpu(svm_data, cpu) = svm_data;
+        per_cpu(svm_data, cpu) = sd;
 
         return 0;
 
 err_1:
-        kfree(svm_data);
+        kfree(sd);
         return r;
 
 }
@@ -476,7 +478,7 @@ static __init int svm_hardware_setup(void)
                 kvm_enable_efer_bits(EFER_SVME);
         }
 
-        for_each_online_cpu(cpu) {
+        for_each_possible_cpu(cpu) {
                 r = svm_cpu_init(cpu);
                 if (r)
                         goto err;
@@ -510,7 +512,7 @@ static __exit void svm_hardware_unsetup(void)
 {
         int cpu;
 
-        for_each_online_cpu(cpu)
+        for_each_possible_cpu(cpu)
                 svm_cpu_uninit(cpu);
 
         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
@@ -539,6 +541,8 @@ static void init_vmcb(struct vcpu_svm *svm)
         struct vmcb_control_area *control = &svm->vmcb->control;
         struct vmcb_save_area *save = &svm->vmcb->save;
 
+        svm->vcpu.fpu_active = 1;
+
         control->intercept_cr_read =    INTERCEPT_CR0_MASK |
                                         INTERCEPT_CR3_MASK |
                                         INTERCEPT_CR4_MASK;
@@ -551,13 +555,19 @@ static void init_vmcb(struct vcpu_svm *svm)
         control->intercept_dr_read =    INTERCEPT_DR0_MASK |
                                         INTERCEPT_DR1_MASK |
                                         INTERCEPT_DR2_MASK |
-                                        INTERCEPT_DR3_MASK;
+                                        INTERCEPT_DR3_MASK |
+                                        INTERCEPT_DR4_MASK |
+                                        INTERCEPT_DR5_MASK |
+                                        INTERCEPT_DR6_MASK |
+                                        INTERCEPT_DR7_MASK;
 
         control->intercept_dr_write =   INTERCEPT_DR0_MASK |
                                         INTERCEPT_DR1_MASK |
                                         INTERCEPT_DR2_MASK |
                                         INTERCEPT_DR3_MASK |
+                                        INTERCEPT_DR4_MASK |
                                         INTERCEPT_DR5_MASK |
+                                        INTERCEPT_DR6_MASK |
                                         INTERCEPT_DR7_MASK;
 
         control->intercept_exceptions = (1 << PF_VECTOR) |
@@ -568,6 +578,7 @@ static void init_vmcb(struct vcpu_svm *svm)
         control->intercept =    (1ULL << INTERCEPT_INTR) |
                                 (1ULL << INTERCEPT_NMI) |
                                 (1ULL << INTERCEPT_SMI) |
+                                (1ULL << INTERCEPT_SELECTIVE_CR0) |
                                 (1ULL << INTERCEPT_CPUID) |
                                 (1ULL << INTERCEPT_INVD) |
                                 (1ULL << INTERCEPT_HLT) |
@@ -625,11 +636,12 @@ static void init_vmcb(struct vcpu_svm *svm)
         save->rip = 0x0000fff0;
         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
 
-        /*
-         * cr0 val on cpu init should be 0x60000010, we enable cpu
-         * cache by default. the orderly way is to enable cache in bios.
+        /* This is the guest-visible cr0 value.
+         * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
          */
-        save->cr0 = 0x00000010 | X86_CR0_PG | X86_CR0_WP;
+        svm->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
+        kvm_set_cr0(&svm->vcpu, svm->vcpu.arch.cr0);
+
         save->cr4 = X86_CR4_PAE;
         /* rdx = ?? */
 
@@ -639,13 +651,9 @@ static void init_vmcb(struct vcpu_svm *svm)
                 control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
                                         (1ULL << INTERCEPT_INVLPG));
                 control->intercept_exceptions &= ~(1 << PF_VECTOR);
-                control->intercept_cr_read &= ~(INTERCEPT_CR0_MASK|
-                                                INTERCEPT_CR3_MASK);
-                control->intercept_cr_write &= ~(INTERCEPT_CR0_MASK|
-                                                 INTERCEPT_CR3_MASK);
+                control->intercept_cr_read &= ~INTERCEPT_CR3_MASK;
+                control->intercept_cr_write &= ~INTERCEPT_CR3_MASK;
                 save->g_pat = 0x0007040600070406ULL;
-                /* enable caching because the QEMU Bios doesn't enable it */
-                save->cr0 = X86_CR0_ET;
                 save->cr3 = 0;
                 save->cr4 = 0;
         }
@@ -654,6 +662,11 @@ static void init_vmcb(struct vcpu_svm *svm)
         svm->nested.vmcb = 0;
         svm->vcpu.arch.hflags = 0;
 
+        if (svm_has(SVM_FEATURE_PAUSE_FILTER)) {
+                control->pause_filter_count = 3000;
+                control->intercept |= (1ULL << INTERCEPT_PAUSE);
+        }
+
         enable_gif(svm);
 }
 
@@ -693,29 +706,28 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
         if (err)
                 goto free_svm;
 
+        err = -ENOMEM;
         page = alloc_page(GFP_KERNEL);
-        if (!page) {
-                err = -ENOMEM;
+        if (!page)
                 goto uninit;
-        }
 
-        err = -ENOMEM;
         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
         if (!msrpm_pages)
-                goto uninit;
+                goto free_page1;
 
         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
         if (!nested_msrpm_pages)
-                goto uninit;
-
-        svm->msrpm = page_address(msrpm_pages);
-        svm_vcpu_init_msrpm(svm->msrpm);
+                goto free_page2;
 
         hsave_page = alloc_page(GFP_KERNEL);
         if (!hsave_page)
-                goto uninit;
+                goto free_page3;
+
         svm->nested.hsave = page_address(hsave_page);
 
+        svm->msrpm = page_address(msrpm_pages);
+        svm_vcpu_init_msrpm(svm->msrpm);
+
         svm->nested.msrpm = page_address(nested_msrpm_pages);
 
         svm->vmcb = page_address(page);
@@ -725,13 +737,18 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
         init_vmcb(svm);
 
         fx_init(&svm->vcpu);
-        svm->vcpu.fpu_active = 1;
         svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
         if (kvm_vcpu_is_bsp(&svm->vcpu))
                 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
 
         return &svm->vcpu;
 
+free_page3:
+        __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
+free_page2:
+        __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
+free_page1:
+        __free_page(page);
 uninit:
         kvm_vcpu_uninit(&svm->vcpu);
 free_svm:
@@ -758,17 +775,18 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
         int i;
 
         if (unlikely(cpu != vcpu->cpu)) {
-                u64 tsc_this, delta;
-
-                /*
-                 * Make sure that the guest sees a monotonically
-                 * increasing TSC.
-                 */
-                rdtscll(tsc_this);
-                delta = vcpu->arch.host_tsc - tsc_this;
-                svm->vmcb->control.tsc_offset += delta;
-                if (is_nested(svm))
-                        svm->nested.hsave->control.tsc_offset += delta;
+                u64 delta;
+
+                if (check_tsc_unstable()) {
+                        /*
+                         * Make sure that the guest sees a monotonically
+                         * increasing TSC.
+                         */
+                        delta = vcpu->arch.host_tsc - native_read_tsc();
+                        svm->vmcb->control.tsc_offset += delta;
+                        if (is_nested(svm))
+                                svm->nested.hsave->control.tsc_offset += delta;
+                }
                 vcpu->cpu = cpu;
                 kvm_migrate_timers(vcpu);
                 svm->asid_generation = 0;
@@ -787,7 +805,7 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu)
         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
 
-        rdtscll(vcpu->arch.host_tsc);
+        vcpu->arch.host_tsc = native_read_tsc();
 }
 
 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
@@ -950,42 +968,59 @@ static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
         svm->vmcb->save.gdtr.base = dt->base ;
 }
 
+static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
+{
+}
+
 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
 {
 }
 
+static void update_cr0_intercept(struct vcpu_svm *svm)
+{
+        ulong gcr0 = svm->vcpu.arch.cr0;
+        u64 *hcr0 = &svm->vmcb->save.cr0;
+
+        if (!svm->vcpu.fpu_active)
+                *hcr0 |= SVM_CR0_SELECTIVE_MASK;
+        else
+                *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
+                        | (gcr0 & SVM_CR0_SELECTIVE_MASK);
+
+
+        if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
+                svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
+                svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
+        } else {
+                svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
+                svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
+        }
+}
+
 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
 #ifdef CONFIG_X86_64
-        if (vcpu->arch.shadow_efer & EFER_LME) {
+        if (vcpu->arch.efer & EFER_LME) {
                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
-                        vcpu->arch.shadow_efer |= EFER_LMA;
+                        vcpu->arch.efer |= EFER_LMA;
                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
                 }
 
                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
-                        vcpu->arch.shadow_efer &= ~EFER_LMA;
+                        vcpu->arch.efer &= ~EFER_LMA;
                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
                 }
         }
 #endif
-        if (npt_enabled)
-                goto set;
+        vcpu->arch.cr0 = cr0;
 
-        if ((vcpu->arch.cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) {
-                svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
-                vcpu->fpu_active = 1;
-        }
+        if (!npt_enabled)
+                cr0 |= X86_CR0_PG | X86_CR0_WP;
 
-        vcpu->arch.cr0 = cr0;
-        cr0 |= X86_CR0_PG | X86_CR0_WP;
-        if (!vcpu->fpu_active) {
-                svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
+        if (!vcpu->fpu_active)
                 cr0 |= X86_CR0_TS;
-        }
-set:
         /*
          * re-enable caching here because the QEMU bios
          * does not do it - this results in some delay at
@@ -993,6 +1028,7 @@ set:
          */
         cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
         svm->vmcb->save.cr0 = cr0;
+        update_cr0_intercept(svm);
 }
 
 static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
@@ -1045,7 +1081,7 @@ static void update_db_intercept(struct kvm_vcpu *vcpu)
         svm->vmcb->control.intercept_exceptions &=
                 ~((1 << DB_VECTOR) | (1 << BP_VECTOR));
 
-        if (vcpu->arch.singlestep)
+        if (svm->nmi_singlestep)
                 svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);
 
         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
@@ -1060,26 +1096,16 @@ static void update_db_intercept(struct kvm_vcpu *vcpu)
                 vcpu->guest_debug = 0;
 }
 
-static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
+static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
 {
-        int old_debug = vcpu->guest_debug;
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        vcpu->guest_debug = dbg->control;
-
-        update_db_intercept(vcpu);
-
         if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
                 svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
         else
                 svm->vmcb->save.dr7 = vcpu->arch.dr7;
 
-        if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
-                svm->vmcb->save.rflags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
-        else if (old_debug & KVM_GUESTDBG_SINGLESTEP)
-                svm->vmcb->save.rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
-
-        return 0;
+        update_db_intercept(vcpu);
 }
 
 static void load_host_msrs(struct kvm_vcpu *vcpu)
@@ -1096,91 +1122,85 @@ static void save_host_msrs(struct kvm_vcpu *vcpu)
 #endif
 }
 
-static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data)
+static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
 {
-        if (svm_data->next_asid > svm_data->max_asid) {
-                ++svm_data->asid_generation;
-                svm_data->next_asid = 1;
+        if (sd->next_asid > sd->max_asid) {
+                ++sd->asid_generation;
+                sd->next_asid = 1;
                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
         }
 
-        svm->asid_generation = svm_data->asid_generation;
-        svm->vmcb->control.asid = svm_data->next_asid++;
+        svm->asid_generation = sd->asid_generation;
+        svm->vmcb->control.asid = sd->next_asid++;
 }
 
-static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr)
+static int svm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *dest)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
-        unsigned long val;
 
         switch (dr) {
         case 0 ... 3:
-                val = vcpu->arch.db[dr];
+                *dest = vcpu->arch.db[dr];
                 break;
+        case 4:
+                if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
+                        return EMULATE_FAIL; /* will re-inject UD */
+                /* fall through */
         case 6:
                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
-                        val = vcpu->arch.dr6;
+                        *dest = vcpu->arch.dr6;
                 else
-                        val = svm->vmcb->save.dr6;
+                        *dest = svm->vmcb->save.dr6;
                 break;
+        case 5:
+                if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
+                        return EMULATE_FAIL; /* will re-inject UD */
+                /* fall through */
         case 7:
                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
-                        val = vcpu->arch.dr7;
+                        *dest = vcpu->arch.dr7;
                 else
-                        val = svm->vmcb->save.dr7;
+                        *dest = svm->vmcb->save.dr7;
                 break;
-        default:
-                val = 0;
         }
 
-        return val;
+        return EMULATE_DONE;
 }
 
-static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
-                       int *exception)
+static int svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        *exception = 0;
-
         switch (dr) {
         case 0 ... 3:
                 vcpu->arch.db[dr] = value;
                 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
                         vcpu->arch.eff_db[dr] = value;
-                return;
-        case 4 ... 5:
-                if (vcpu->arch.cr4 & X86_CR4_DE)
-                        *exception = UD_VECTOR;
-                return;
+                break;
+        case 4:
+                if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
+                        return EMULATE_FAIL; /* will re-inject UD */
+                /* fall through */
         case 6:
-                if (value & 0xffffffff00000000ULL) {
-                        *exception = GP_VECTOR;
-                        return;
-                }
                 vcpu->arch.dr6 = (value & DR6_VOLATILE) | DR6_FIXED_1;
-                return;
+                break;
+        case 5:
+                if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
+                        return EMULATE_FAIL; /* will re-inject UD */
+                /* fall through */
         case 7:
-                if (value & 0xffffffff00000000ULL) {
-                        *exception = GP_VECTOR;
-                        return;
-                }
                 vcpu->arch.dr7 = (value & DR7_VOLATILE) | DR7_FIXED_1;
                 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
                         svm->vmcb->save.dr7 = vcpu->arch.dr7;
                         vcpu->arch.switch_db_regs = (value & DR7_BP_EN_MASK);
                 }
-                return;
-        default:
-                /* FIXME: Possible case? */
-                printk(KERN_DEBUG "%s: unexpected dr %u\n",
-                       __func__, dr);
-                *exception = UD_VECTOR;
-                return;
+                break;
         }
+
+        return EMULATE_DONE;
 }
 
-static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int pf_interception(struct vcpu_svm *svm)
 {
         u64 fault_address;
         u32 error_code;
@@ -1194,17 +1214,19 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
 }
 
-static int db_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int db_interception(struct vcpu_svm *svm)
 {
+        struct kvm_run *kvm_run = svm->vcpu.run;
+
         if (!(svm->vcpu.guest_debug &
               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
-                !svm->vcpu.arch.singlestep) {
+                !svm->nmi_singlestep) {
                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
                 return 1;
         }
 
-        if (svm->vcpu.arch.singlestep) {
-                svm->vcpu.arch.singlestep = false;
+        if (svm->nmi_singlestep) {
+                svm->nmi_singlestep = false;
                 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
                         svm->vmcb->save.rflags &=
                                 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
@@ -1223,35 +1245,41 @@ static int db_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
-static int bp_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int bp_interception(struct vcpu_svm *svm)
 {
+        struct kvm_run *kvm_run = svm->vcpu.run;
+
         kvm_run->exit_reason = KVM_EXIT_DEBUG;
         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
         kvm_run->debug.arch.exception = BP_VECTOR;
         return 0;
 }
 
-static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int ud_interception(struct vcpu_svm *svm)
 {
         int er;
 
-        er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD);
+        er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD);
         if (er != EMULATE_DONE)
                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
         return 1;
 }
 
-static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static void svm_fpu_activate(struct kvm_vcpu *vcpu)
 {
+        struct vcpu_svm *svm = to_svm(vcpu);
         svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
-        if (!(svm->vcpu.arch.cr0 & X86_CR0_TS))
-                svm->vmcb->save.cr0 &= ~X86_CR0_TS;
         svm->vcpu.fpu_active = 1;
+        update_cr0_intercept(svm);
+}
 
+static int nm_interception(struct vcpu_svm *svm)
+{
+        svm_fpu_activate(&svm->vcpu);
         return 1;
 }
 
-static int mc_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int mc_interception(struct vcpu_svm *svm)
 {
         /*
          * On an #MC intercept the MCE handler is not called automatically in
@@ -1264,8 +1292,10 @@ static int mc_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
-static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int shutdown_interception(struct vcpu_svm *svm)
 {
+        struct kvm_run *kvm_run = svm->vcpu.run;
+
         /*
          * VMCB is undefined after a SHUTDOWN intercept
          * so reinitialize it.
@@ -1277,7 +1307,7 @@ static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 0;
 }
 
-static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int io_interception(struct vcpu_svm *svm)
 {
         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
         int size, in, string;
@@ -1291,7 +1321,7 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 
         if (string) {
                 if (emulate_instruction(&svm->vcpu,
-                                        kvm_run, 0, 0, 0) == EMULATE_DO_MMIO)
+                                        0, 0, 0) == EMULATE_DO_MMIO)
                         return 0;
                 return 1;
         }
@@ -1301,33 +1331,33 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
 
         skip_emulated_instruction(&svm->vcpu);
-        return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port);
+        return kvm_emulate_pio(&svm->vcpu, in, size, port);
 }
 
-static int nmi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int nmi_interception(struct vcpu_svm *svm)
 {
         return 1;
 }
 
-static int intr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int intr_interception(struct vcpu_svm *svm)
 {
         ++svm->vcpu.stat.irq_exits;
         return 1;
 }
 
-static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int nop_on_interception(struct vcpu_svm *svm)
 {
         return 1;
 }
 
-static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int halt_interception(struct vcpu_svm *svm)
 {
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
         skip_emulated_instruction(&svm->vcpu);
         return kvm_emulate_halt(&svm->vcpu);
 }
 
-static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int vmmcall_interception(struct vcpu_svm *svm)
 {
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
@@ -1337,7 +1367,7 @@ static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 
 static int nested_svm_check_permissions(struct vcpu_svm *svm)
 {
-        if (!(svm->vcpu.arch.shadow_efer & EFER_SVME)
+        if (!(svm->vcpu.arch.efer & EFER_SVME)
             || !is_paging(&svm->vcpu)) {
                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
                 return 1;
@@ -1378,8 +1408,15 @@ static inline int nested_svm_intr(struct vcpu_svm *svm)
 
         svm->vmcb->control.exit_code = SVM_EXIT_INTR;
 
-        if (nested_svm_exit_handled(svm)) {
-                nsvm_printk("VMexit -> INTR\n");
+        if (svm->nested.intercept & 1ULL) {
+                /*
+                 * The #vmexit can't be emulated here directly because this
+                 * code path runs with irqs and preemtion disabled. A
+                 * #vmexit emulation might sleep. Only signal request for
+                 * the #vmexit here.
+                 */
+                svm->nested.exit_required = true;
+                trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
                 return 1;
         }
 
@@ -1390,10 +1427,7 @@ static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, enum km_type idx)
 {
         struct page *page;
 
-        down_read(&current->mm->mmap_sem);
         page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
-        up_read(&current->mm->mmap_sem);
-
         if (is_error_page(page))
                 goto error;
 
@@ -1532,14 +1566,12 @@ static int nested_svm_exit_handled(struct vcpu_svm *svm)
         }
         default: {
                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
-                nsvm_printk("exit code: 0x%x\n", exit_code);
                 if (svm->nested.intercept & exit_bits)
                         vmexit = NESTED_EXIT_DONE;
         }
         }
 
         if (vmexit == NESTED_EXIT_DONE) {
-                nsvm_printk("#VMEXIT reason=%04x\n", exit_code);
                 nested_svm_vmexit(svm);
         }
 
@@ -1584,6 +1616,12 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         struct vmcb *hsave = svm->nested.hsave;
         struct vmcb *vmcb = svm->vmcb;
 
+        trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
+                                       vmcb->control.exit_info_1,
+                                       vmcb->control.exit_info_2,
+                                       vmcb->control.exit_int_info,
+                                       vmcb->control.exit_int_info_err);
+
         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, KM_USER0);
         if (!nested_vmcb)
                 return 1;
@@ -1617,6 +1655,22 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
+
+        /*
+         * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
+         * to make sure that we do not lose injected events. So check event_inj
+         * here and copy it to exit_int_info if it is valid.
+         * Exit_int_info and event_inj can't be both valid because the case
+         * below only happens on a VMRUN instruction intercept which has
+         * no valid exit_int_info set.
+         */
+        if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
+                struct vmcb_control_area *nc = &nested_vmcb->control;
+
+                nc->exit_int_info     = vmcb->control.event_inj;
+                nc->exit_int_info_err = vmcb->control.event_inj_err;
+        }
+
         nested_vmcb->control.tlb_ctl           = 0;
         nested_vmcb->control.event_inj         = 0;
         nested_vmcb->control.event_inj_err     = 0;
@@ -1628,10 +1682,6 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         /* Restore the original control entries */
         copy_vmcb_control_area(vmcb, hsave);
 
-        /* Kill any pending exceptions */
-        if (svm->vcpu.arch.exception.pending == true)
-                nsvm_printk("WARNING: Pending Exception\n");
-
         kvm_clear_exception_queue(&svm->vcpu);
         kvm_clear_interrupt_queue(&svm->vcpu);
 
@@ -1702,6 +1752,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
         /* nested_vmcb is our indicator if nested SVM is activated */
         svm->nested.vmcb = svm->vmcb->save.rax;
 
+        trace_kvm_nested_vmrun(svm->vmcb->save.rip - 3, svm->nested.vmcb,
+                               nested_vmcb->save.rip,
+                               nested_vmcb->control.int_ctl,
+                               nested_vmcb->control.event_inj,
+                               nested_vmcb->control.nested_ctl);
+
         /* Clear internal status */
         kvm_clear_exception_queue(&svm->vcpu);
         kvm_clear_interrupt_queue(&svm->vcpu);
@@ -1714,8 +1770,8 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
         hsave->save.ds     = vmcb->save.ds;
         hsave->save.gdtr   = vmcb->save.gdtr;
         hsave->save.idtr   = vmcb->save.idtr;
-        hsave->save.efer   = svm->vcpu.arch.shadow_efer;
-        hsave->save.cr0    = svm->vcpu.arch.cr0;
+        hsave->save.efer   = svm->vcpu.arch.efer;
+        hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
         hsave->save.cr4    = svm->vcpu.arch.cr4;
         hsave->save.rflags = vmcb->save.rflags;
         hsave->save.rip    = svm->next_rip;
@@ -1789,28 +1845,15 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
         svm->nested.intercept            = nested_vmcb->control.intercept;
 
         force_new_asid(&svm->vcpu);
-        svm->vmcb->control.exit_int_info = nested_vmcb->control.exit_int_info;
-        svm->vmcb->control.exit_int_info_err = nested_vmcb->control.exit_int_info_err;
         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
-        if (nested_vmcb->control.int_ctl & V_IRQ_MASK) {
-                nsvm_printk("nSVM Injecting Interrupt: 0x%x\n",
-                                nested_vmcb->control.int_ctl);
-        }
         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
         else
                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
 
-        nsvm_printk("nSVM exit_int_info: 0x%x | int_state: 0x%x\n",
-                        nested_vmcb->control.exit_int_info,
-                        nested_vmcb->control.int_state);
-
         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
-        if (nested_vmcb->control.event_inj & SVM_EVTINJ_VALID)
-                nsvm_printk("Injecting Event: 0x%x\n",
-                                nested_vmcb->control.event_inj);
         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
 
@@ -1837,7 +1880,7 @@ static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
 }
 
-static int vmload_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int vmload_interception(struct vcpu_svm *svm)
 {
         struct vmcb *nested_vmcb;
 
@@ -1857,7 +1900,7 @@ static int vmload_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
-static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int vmsave_interception(struct vcpu_svm *svm)
 {
         struct vmcb *nested_vmcb;
 
@@ -1877,10 +1920,8 @@ static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
-static int vmrun_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int vmrun_interception(struct vcpu_svm *svm)
 {
-        nsvm_printk("VMrun\n");
-
         if (nested_svm_check_permissions(svm))
                 return 1;
 
@@ -1907,7 +1948,7 @@ failed:
         return 1;
 }
 
-static int stgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int stgi_interception(struct vcpu_svm *svm)
 {
         if (nested_svm_check_permissions(svm))
                 return 1;
@@ -1920,7 +1961,7 @@ static int stgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
-static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int clgi_interception(struct vcpu_svm *svm)
 {
         if (nested_svm_check_permissions(svm))
                 return 1;
@@ -1937,10 +1978,12 @@ static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
-static int invlpga_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int invlpga_interception(struct vcpu_svm *svm)
 {
         struct kvm_vcpu *vcpu = &svm->vcpu;
-        nsvm_printk("INVLPGA\n");
+
+        trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX],
+                          vcpu->arch.regs[VCPU_REGS_RAX]);
 
         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
         kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);
@@ -1950,15 +1993,21 @@ static int invlpga_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
-static int invalid_op_interception(struct vcpu_svm *svm,
-                                   struct kvm_run *kvm_run)
+static int skinit_interception(struct vcpu_svm *svm)
+{
+        trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]);
+
+        kvm_queue_exception(&svm->vcpu, UD_VECTOR);
+        return 1;
+}
+
+static int invalid_op_interception(struct vcpu_svm *svm)
 {
         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
         return 1;
 }
 
-static int task_switch_interception(struct vcpu_svm *svm,
-                                    struct kvm_run *kvm_run)
+static int task_switch_interception(struct vcpu_svm *svm)
 {
         u16 tss_selector;
         int reason;
@@ -2008,41 +2057,42 @@ static int task_switch_interception(struct vcpu_svm *svm,
         return kvm_task_switch(&svm->vcpu, tss_selector, reason);
 }
 
-static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int cpuid_interception(struct vcpu_svm *svm)
 {
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
         kvm_emulate_cpuid(&svm->vcpu);
         return 1;
 }
 
-static int iret_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int iret_interception(struct vcpu_svm *svm)
 {
         ++svm->vcpu.stat.nmi_window_exits;
-        svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET);
+        svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
         svm->vcpu.arch.hflags |= HF_IRET_MASK;
         return 1;
 }
 
-static int invlpg_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int invlpg_interception(struct vcpu_svm *svm)
 {
-        if (emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0) != EMULATE_DONE)
+        if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE)
                 pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
         return 1;
 }
 
-static int emulate_on_interception(struct vcpu_svm *svm,
-                                   struct kvm_run *kvm_run)
+static int emulate_on_interception(struct vcpu_svm *svm)
 {
-        if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE)
+        if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE)
                 pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
         return 1;
 }
 
-static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int cr8_write_interception(struct vcpu_svm *svm)
 {
+        struct kvm_run *kvm_run = svm->vcpu.run;
+
         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
         /* instruction emulation calls kvm_set_cr8() */
-        emulate_instruction(&svm->vcpu, NULL, 0, 0, 0);
+        emulate_instruction(&svm->vcpu, 0, 0, 0);
         if (irqchip_in_kernel(svm->vcpu.kvm)) {
                 svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
                 return 1;
@@ -2128,14 +2178,15 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
         return 0;
 }
 
-static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int rdmsr_interception(struct vcpu_svm *svm)
 {
         u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
         u64 data;
 
-        if (svm_get_msr(&svm->vcpu, ecx, &data))
+        if (svm_get_msr(&svm->vcpu, ecx, &data)) {
+                trace_kvm_msr_read_ex(ecx);
                 kvm_inject_gp(&svm->vcpu, 0);
-        else {
+        } else {
                 trace_kvm_msr_read(ecx, data);
 
                 svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
@@ -2221,33 +2272,36 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
         return 0;
 }
 
-static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int wrmsr_interception(struct vcpu_svm *svm)
 {
         u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
         u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
                 | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
 
-        trace_kvm_msr_write(ecx, data);
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
-        if (svm_set_msr(&svm->vcpu, ecx, data))
+        if (svm_set_msr(&svm->vcpu, ecx, data)) {
+                trace_kvm_msr_write_ex(ecx, data);
                 kvm_inject_gp(&svm->vcpu, 0);
-        else
+        } else {
+                trace_kvm_msr_write(ecx, data);
                 skip_emulated_instruction(&svm->vcpu);
+        }
         return 1;
 }
 
-static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+static int msr_interception(struct vcpu_svm *svm)
 {
         if (svm->vmcb->control.exit_info_1)
-                return wrmsr_interception(svm, kvm_run);
+                return wrmsr_interception(svm);
         else
-                return rdmsr_interception(svm, kvm_run);
+                return rdmsr_interception(svm);
 }
 
-static int interrupt_window_interception(struct vcpu_svm *svm,
-                                   struct kvm_run *kvm_run)
+static int interrupt_window_interception(struct vcpu_svm *svm)
 {
+        struct kvm_run *kvm_run = svm->vcpu.run;
+
         svm_clear_vintr(svm);
         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
         /*
@@ -2265,13 +2319,18 @@ static int interrupt_window_interception(struct vcpu_svm *svm,
         return 1;
 }
 
-static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
-                                      struct kvm_run *kvm_run) = {
+static int pause_interception(struct vcpu_svm *svm)
+{
+        kvm_vcpu_on_spin(&(svm->vcpu));
+        return 1;
+}
+
+static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
         [SVM_EXIT_READ_CR0]                     = emulate_on_interception,
         [SVM_EXIT_READ_CR3]                     = emulate_on_interception,
         [SVM_EXIT_READ_CR4]                     = emulate_on_interception,
         [SVM_EXIT_READ_CR8]                     = emulate_on_interception,
-        /* for now: */
+        [SVM_EXIT_CR0_SEL_WRITE]                = emulate_on_interception,
         [SVM_EXIT_WRITE_CR0]                    = emulate_on_interception,
         [SVM_EXIT_WRITE_CR3]                    = emulate_on_interception,
         [SVM_EXIT_WRITE_CR4]                    = emulate_on_interception,
@@ -2280,11 +2339,17 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
         [SVM_EXIT_READ_DR1]                     = emulate_on_interception,
         [SVM_EXIT_READ_DR2]                     = emulate_on_interception,
         [SVM_EXIT_READ_DR3]                     = emulate_on_interception,
+        [SVM_EXIT_READ_DR4]                     = emulate_on_interception,
+        [SVM_EXIT_READ_DR5]                     = emulate_on_interception,
+        [SVM_EXIT_READ_DR6]                     = emulate_on_interception,
+        [SVM_EXIT_READ_DR7]                     = emulate_on_interception,
         [SVM_EXIT_WRITE_DR0]                    = emulate_on_interception,
         [SVM_EXIT_WRITE_DR1]                    = emulate_on_interception,
         [SVM_EXIT_WRITE_DR2]                    = emulate_on_interception,
         [SVM_EXIT_WRITE_DR3]                    = emulate_on_interception,
+        [SVM_EXIT_WRITE_DR4]                    = emulate_on_interception,
         [SVM_EXIT_WRITE_DR5]                    = emulate_on_interception,
+        [SVM_EXIT_WRITE_DR6]                    = emulate_on_interception,
         [SVM_EXIT_WRITE_DR7]                    = emulate_on_interception,
         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
@@ -2301,6 +2366,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
         [SVM_EXIT_CPUID]                        = cpuid_interception,
         [SVM_EXIT_IRET]                         = iret_interception,
         [SVM_EXIT_INVD]                         = emulate_on_interception,
+        [SVM_EXIT_PAUSE]                        = pause_interception,
         [SVM_EXIT_HLT]                          = halt_interception,
         [SVM_EXIT_INVLPG]                       = invlpg_interception,
         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
@@ -2314,26 +2380,36 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
         [SVM_EXIT_STGI]                         = stgi_interception,
         [SVM_EXIT_CLGI]                         = clgi_interception,
-        [SVM_EXIT_SKINIT]                       = invalid_op_interception,
+        [SVM_EXIT_SKINIT]                       = skinit_interception,
         [SVM_EXIT_WBINVD]                       = emulate_on_interception,
         [SVM_EXIT_MONITOR]                      = invalid_op_interception,
         [SVM_EXIT_MWAIT]                        = invalid_op_interception,
         [SVM_EXIT_NPF]                          = pf_interception,
 };
 
-static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
+static int handle_exit(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
+        struct kvm_run *kvm_run = vcpu->run;
         u32 exit_code = svm->vmcb->control.exit_code;
 
         trace_kvm_exit(exit_code, svm->vmcb->save.rip);
 
+        if (unlikely(svm->nested.exit_required)) {
+                nested_svm_vmexit(svm);
+                svm->nested.exit_required = false;
+
+                return 1;
+        }
+
         if (is_nested(svm)) {
                 int vmexit;
 
-                nsvm_printk("nested handle_exit: 0x%x | 0x%lx | 0x%lx | 0x%lx\n",
-                            exit_code, svm->vmcb->control.exit_info_1,
-                            svm->vmcb->control.exit_info_2, svm->vmcb->save.rip);
+                trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
+                                        svm->vmcb->control.exit_info_1,
+                                        svm->vmcb->control.exit_info_2,
+                                        svm->vmcb->control.exit_int_info,
+                                        svm->vmcb->control.exit_int_info_err);
 
                 vmexit = nested_svm_exit_special(svm);
 
@@ -2346,20 +2422,10 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 
         svm_complete_interrupts(svm);
 
-        if (npt_enabled) {
-                int mmu_reload = 0;
-                if ((vcpu->arch.cr0 ^ svm->vmcb->save.cr0) & X86_CR0_PG) {
-                        svm_set_cr0(vcpu, svm->vmcb->save.cr0);
-                        mmu_reload = 1;
-                }
+        if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK))
                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
+        if (npt_enabled)
                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
-                if (mmu_reload) {
-                        kvm_mmu_reset_context(vcpu);
-                        kvm_mmu_load(vcpu);
-                }
-        }
-
 
         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
@@ -2383,15 +2449,15 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
                 return 0;
         }
 
-        return svm_exit_handlers[exit_code](svm, kvm_run);
+        return svm_exit_handlers[exit_code](svm);
 }
 
 static void reload_tss(struct kvm_vcpu *vcpu)
 {
         int cpu = raw_smp_processor_id();
 
-        struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
-        svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */
+        struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
+        sd->tss_desc->type = 9; /* available 32/64-bit TSS */
         load_TR_desc();
 }
 
@@ -2399,12 +2465,12 @@ static void pre_svm_run(struct vcpu_svm *svm)
 {
         int cpu = raw_smp_processor_id();
 
-        struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
+        struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
 
         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
         /* FIXME: handle wraparound of asid_generation */
-        if (svm->asid_generation != svm_data->asid_generation)
-                new_asid(svm, svm_data);
+        if (svm->asid_generation != sd->asid_generation)
+                new_asid(svm, sd);
 }
 
 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
@@ -2413,7 +2479,7 @@ static void svm_inject_nmi(struct kvm_vcpu *vcpu)
 
         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
         vcpu->arch.hflags |= HF_NMI_MASK;
-        svm->vmcb->control.intercept |= (1UL << INTERCEPT_IRET);
+        svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
         ++vcpu->stat.nmi_injections;
 }
 
@@ -2460,20 +2526,47 @@ static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
                 !(svm->vcpu.arch.hflags & HF_NMI_MASK);
 }
 
+static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
+{
+        struct vcpu_svm *svm = to_svm(vcpu);
+
+        return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
+}
+
+static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
+{
+        struct vcpu_svm *svm = to_svm(vcpu);
+
+        if (masked) {
+                svm->vcpu.arch.hflags |= HF_NMI_MASK;
+                svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
+        } else {
+                svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
+                svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
+        }
+}
+
 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
         struct vmcb *vmcb = svm->vmcb;
-        return (vmcb->save.rflags & X86_EFLAGS_IF) &&
-                !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
-                gif_set(svm) &&
-                !(is_nested(svm) && (svm->vcpu.arch.hflags & HF_VINTR_MASK));
+        int ret;
+
+        if (!gif_set(svm) ||
+             (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
+                return 0;
+
+        ret = !!(vmcb->save.rflags & X86_EFLAGS_IF);
+
+        if (is_nested(svm))
+                return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
+
+        return ret;
 }
 
 static void enable_irq_window(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
-        nsvm_printk("Trying to open IRQ window\n");
 
         nested_svm_intr(svm);
 
@@ -2498,7 +2591,7 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu)
         /* Something prevents NMI from been injected. Single step over
            possible problem (IRET or exception injection or interrupt
            shadow) */
-        vcpu->arch.singlestep = true;
+        svm->nmi_singlestep = true;
         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
         update_db_intercept(vcpu);
 }
@@ -2588,13 +2681,20 @@ static void svm_complete_interrupts(struct vcpu_svm *svm)
 #define R "e"
 #endif
 
-static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
+static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
         u16 fs_selector;
         u16 gs_selector;
         u16 ldt_selector;
 
+        /*
+         * A vmexit emulation is required before the vcpu can be executed
+         * again.
+         */
+        if (unlikely(svm->nested.exit_required))
+                return;
+
         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
@@ -2727,12 +2827,6 @@ static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
 
         svm->vmcb->save.cr3 = root;
         force_new_asid(vcpu);
-
-        if (vcpu->fpu_active) {
-                svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
-                svm->vmcb->save.cr0 |= X86_CR0_TS;
-                vcpu->fpu_active = 0;
-        }
 }
 
 static int is_disabled(void)
@@ -2781,6 +2875,10 @@ static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
         return 0;
 }
 
+static void svm_cpuid_update(struct kvm_vcpu *vcpu)
+{
+}
+
 static const struct trace_print_flags svm_exit_reasons_str[] = {
         { SVM_EXIT_READ_CR0,                    "read_cr0" },
         { SVM_EXIT_READ_CR3,                    "read_cr3" },
@@ -2834,9 +2932,22 @@ static const struct trace_print_flags svm_exit_reasons_str[] = {
         { -1, NULL }
 };
 
-static bool svm_gb_page_enable(void)
+static int svm_get_lpage_level(void)
 {
-        return true;
+        return PT_PDPE_LEVEL;
+}
+
+static bool svm_rdtscp_supported(void)
+{
+        return false;
+}
+
+static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
+{
+        struct vcpu_svm *svm = to_svm(vcpu);
+
+        update_cr0_intercept(svm);
+        svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR;
 }
 
 static struct kvm_x86_ops svm_x86_ops = {
@@ -2865,6 +2976,7 @@ static struct kvm_x86_ops svm_x86_ops = {
         .set_segment = svm_set_segment,
         .get_cpl = svm_get_cpl,
         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
+        .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
         .set_cr0 = svm_set_cr0,
         .set_cr3 = svm_set_cr3,
@@ -2879,6 +2991,8 @@ static struct kvm_x86_ops svm_x86_ops = {
         .cache_reg = svm_cache_reg,
         .get_rflags = svm_get_rflags,
         .set_rflags = svm_set_rflags,
+        .fpu_activate = svm_fpu_activate,
+        .fpu_deactivate = svm_fpu_deactivate,
 
         .tlb_flush = svm_flush_tlb,
 
@@ -2893,6 +3007,8 @@ static struct kvm_x86_ops svm_x86_ops = {
         .queue_exception = svm_queue_exception,
         .interrupt_allowed = svm_interrupt_allowed,
         .nmi_allowed = svm_nmi_allowed,
+        .get_nmi_mask = svm_get_nmi_mask,
+        .set_nmi_mask = svm_set_nmi_mask,
         .enable_nmi_window = enable_nmi_window,
         .enable_irq_window = enable_irq_window,
         .update_cr8_intercept = update_cr8_intercept,
@@ -2902,7 +3018,11 @@ static struct kvm_x86_ops svm_x86_ops = {
         .get_mt_mask = svm_get_mt_mask,
 
         .exit_reasons_str = svm_exit_reasons_str,
-        .gb_page_enable = svm_gb_page_enable,
+        .get_lpage_level = svm_get_lpage_level,
+
+        .cpuid_update = svm_cpuid_update,
+
+        .rdtscp_supported = svm_rdtscp_supported,
 };
 
 static int __init svm_init(void)