1 files changed, 124 insertions, 31 deletions

diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index ce7fe640f18d..67c1154960f0 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -58,6 +58,9 @@ static struct kvm_stats_debugfs_item {
         { "io_exits", &kvm_stat.io_exits },
         { "mmio_exits", &kvm_stat.mmio_exits },
         { "signal_exits", &kvm_stat.signal_exits },
+        { "irq_window", &kvm_stat.irq_window_exits },
+        { "halt_exits", &kvm_stat.halt_exits },
+        { "request_irq", &kvm_stat.request_irq_exits },
         { "irq_exits", &kvm_stat.irq_exits },
         { 0, 0 }
 };
@@ -227,6 +230,7 @@ static int kvm_dev_open(struct inode *inode, struct file *filp)
                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
 
                 mutex_init(&vcpu->mutex);
+                vcpu->kvm = kvm;
                 vcpu->mmu.root_hpa = INVALID_PAGE;
                 INIT_LIST_HEAD(&vcpu->free_pages);
         }
@@ -268,8 +272,8 @@ static void kvm_free_physmem(struct kvm *kvm)
 
 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 {
-        kvm_arch_ops->vcpu_free(vcpu);
         kvm_mmu_destroy(vcpu);
+        kvm_arch_ops->vcpu_free(vcpu);
 }
 
 static void kvm_free_vcpus(struct kvm *kvm)
@@ -295,14 +299,17 @@ static void inject_gp(struct kvm_vcpu *vcpu)
         kvm_arch_ops->inject_gp(vcpu, 0);
 }
 
-static int pdptrs_have_reserved_bits_set(struct kvm_vcpu *vcpu,
-                                         unsigned long cr3)
+/*
+ * Load the pae pdptrs.  Return true is they are all valid.
+ */
+static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 {
         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
-        unsigned offset = (cr3 & (PAGE_SIZE-1)) >> 5;
+        unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
         int i;
         u64 pdpte;
         u64 *pdpt;
+        int ret;
         struct kvm_memory_slot *memslot;
 
         spin_lock(&vcpu->kvm->lock);
@@ -310,16 +317,23 @@ static int pdptrs_have_reserved_bits_set(struct kvm_vcpu *vcpu,
         /* FIXME: !memslot - emulate? 0xff? */
         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
 
+        ret = 1;
         for (i = 0; i < 4; ++i) {
                 pdpte = pdpt[offset + i];
-                if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull))
-                        break;
+                if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
+                        ret = 0;
+                        goto out;
+                }
         }
 
+        for (i = 0; i < 4; ++i)
+                vcpu->pdptrs[i] = pdpt[offset + i];
+
+out:
         kunmap_atomic(pdpt, KM_USER0);
         spin_unlock(&vcpu->kvm->lock);
 
-        return i != 4;
+        return ret;
 }
 
 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
@@ -365,8 +379,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
                         }
                 } else
 #endif
-                if (is_pae(vcpu) &&
-                            pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) {
+                if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
                                "reserved bits\n");
                         inject_gp(vcpu);
@@ -387,6 +400,7 @@ EXPORT_SYMBOL_GPL(set_cr0);
 
 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 {
+        kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 }
 EXPORT_SYMBOL_GPL(lmsw);
@@ -407,7 +421,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
                         return;
                 }
         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
-                   && pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) {
+                   && !load_pdptrs(vcpu, vcpu->cr3)) {
                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
                 inject_gp(vcpu);
         }
@@ -439,7 +453,7 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
                         return;
                 }
                 if (is_paging(vcpu) && is_pae(vcpu) &&
-                    pdptrs_have_reserved_bits_set(vcpu, cr3)) {
+                    !load_pdptrs(vcpu, cr3)) {
                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
                                "reserved bits\n");
                         inject_gp(vcpu);
@@ -449,7 +463,19 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 
         vcpu->cr3 = cr3;
         spin_lock(&vcpu->kvm->lock);
-        vcpu->mmu.new_cr3(vcpu);
+        /*
+         * Does the new cr3 value map to physical memory? (Note, we
+         * catch an invalid cr3 even in real-mode, because it would
+         * cause trouble later on when we turn on paging anyway.)
+         *
+         * A real CPU would silently accept an invalid cr3 and would
+         * attempt to use it - with largely undefined (and often hard
+         * to debug) behavior on the guest side.
+         */
+        if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
+                inject_gp(vcpu);
+        else
+                vcpu->mmu.new_cr3(vcpu);
         spin_unlock(&vcpu->kvm->lock);
 }
 EXPORT_SYMBOL_GPL(set_cr3);
@@ -517,7 +543,6 @@ static int kvm_dev_ioctl_create_vcpu(struct kvm *kvm, int n)
         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
 
         vcpu->cpu = -1;  /* First load will set up TR */
-        vcpu->kvm = kvm;
         r = kvm_arch_ops->vcpu_create(vcpu);
         if (r < 0)
                 goto out_free_vcpus;
@@ -634,6 +659,7 @@ raced:
                                                      | __GFP_ZERO);
                         if (!new.phys_mem[i])
                                 goto out_free;
+                        new.phys_mem[i]->private = 0;
                 }
         }
 
@@ -688,6 +714,13 @@ out:
         return r;
 }
 
+static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
+{
+        spin_lock(&vcpu->kvm->lock);
+        kvm_mmu_slot_remove_write_access(vcpu, slot);
+        spin_unlock(&vcpu->kvm->lock);
+}
+
 /*
  * Get (and clear) the dirty memory log for a memory slot.
  */
@@ -697,6 +730,7 @@ static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm,
         struct kvm_memory_slot *memslot;
         int r, i;
         int n;
+        int cleared;
         unsigned long any = 0;
 
         spin_lock(&kvm->lock);
@@ -727,15 +761,17 @@ static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm,
 
 
         if (any) {
-                spin_lock(&kvm->lock);
-                kvm_mmu_slot_remove_write_access(kvm, log->slot);
-                spin_unlock(&kvm->lock);
-                memset(memslot->dirty_bitmap, 0, n);
+                cleared = 0;
                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
                         struct kvm_vcpu *vcpu = vcpu_load(kvm, i);
 
                         if (!vcpu)
                                 continue;
+                        if (!cleared) {
+                                do_remove_write_access(vcpu, log->slot);
+                                memset(memslot->dirty_bitmap, 0, n);
+                                cleared = 1;
+                        }
                         kvm_arch_ops->tlb_flush(vcpu);
                         vcpu_put(vcpu);
                 }
@@ -863,6 +899,27 @@ static int emulator_read_emulated(unsigned long addr,
         }
 }
 
+static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
+                               unsigned long val, int bytes)
+{
+        struct kvm_memory_slot *m;
+        struct page *page;
+        void *virt;
+
+        if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
+                return 0;
+        m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
+        if (!m)
+                return 0;
+        page = gfn_to_page(m, gpa >> PAGE_SHIFT);
+        kvm_mmu_pre_write(vcpu, gpa, bytes);
+        virt = kmap_atomic(page, KM_USER0);
+        memcpy(virt + offset_in_page(gpa), &val, bytes);
+        kunmap_atomic(virt, KM_USER0);
+        kvm_mmu_post_write(vcpu, gpa, bytes);
+        return 1;
+}
+
 static int emulator_write_emulated(unsigned long addr,
                                    unsigned long val,
                                    unsigned int bytes,
@@ -874,6 +931,9 @@ static int emulator_write_emulated(unsigned long addr,
         if (gpa == UNMAPPED_GVA)
                 return X86EMUL_PROPAGATE_FAULT;
 
+        if (emulator_write_phys(vcpu, gpa, val, bytes))
+                return X86EMUL_CONTINUE;
+
         vcpu->mmio_needed = 1;
         vcpu->mmio_phys_addr = gpa;
         vcpu->mmio_size = bytes;
@@ -898,6 +958,30 @@ static int emulator_cmpxchg_emulated(unsigned long addr,
         return emulator_write_emulated(addr, new, bytes, ctxt);
 }
 
+#ifdef CONFIG_X86_32
+
+static int emulator_cmpxchg8b_emulated(unsigned long addr,
+                                       unsigned long old_lo,
+                                       unsigned long old_hi,
+                                       unsigned long new_lo,
+                                       unsigned long new_hi,
+                                       struct x86_emulate_ctxt *ctxt)
+{
+        static int reported;
+        int r;
+
+        if (!reported) {
+                reported = 1;
+                printk(KERN_WARNING "kvm: emulating exchange8b as write\n");
+        }
+        r = emulator_write_emulated(addr, new_lo, 4, ctxt);
+        if (r != X86EMUL_CONTINUE)
+                return r;
+        return emulator_write_emulated(addr+4, new_hi, 4, ctxt);
+}
+
+#endif
+
 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
 {
         return kvm_arch_ops->get_segment_base(vcpu, seg);
@@ -905,18 +989,15 @@ static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
 
 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
 {
-        spin_lock(&vcpu->kvm->lock);
-        vcpu->mmu.inval_page(vcpu, address);
-        spin_unlock(&vcpu->kvm->lock);
-        kvm_arch_ops->invlpg(vcpu, address);
         return X86EMUL_CONTINUE;
 }
 
 int emulate_clts(struct kvm_vcpu *vcpu)
 {
-        unsigned long cr0 = vcpu->cr0;
+        unsigned long cr0;
 
-        cr0 &= ~CR0_TS_MASK;
+        kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
+        cr0 = vcpu->cr0 & ~CR0_TS_MASK;
         kvm_arch_ops->set_cr0(vcpu, cr0);
         return X86EMUL_CONTINUE;
 }
@@ -975,6 +1056,9 @@ struct x86_emulate_ops emulate_ops = {
         .read_emulated       = emulator_read_emulated,
         .write_emulated      = emulator_write_emulated,
         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
+#ifdef CONFIG_X86_32
+        .cmpxchg8b_emulated  = emulator_cmpxchg8b_emulated,
+#endif
 };
 
 int emulate_instruction(struct kvm_vcpu *vcpu,
@@ -1024,6 +1108,8 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
         }
 
         if (r) {
+                if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
+                        return EMULATE_DONE;
                 if (!vcpu->mmio_needed) {
                         report_emulation_failure(&emulate_ctxt);
                         return EMULATE_FAIL;
@@ -1069,6 +1155,7 @@ void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
 
 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
 {
+        kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
         switch (cr) {
         case 0:
                 return vcpu->cr0;
@@ -1403,6 +1490,7 @@ static int kvm_dev_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
         sregs->gdt.limit = dt.limit;
         sregs->gdt.base = dt.base;
 
+        kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
         sregs->cr0 = vcpu->cr0;
         sregs->cr2 = vcpu->cr2;
         sregs->cr3 = vcpu->cr3;
@@ -1467,11 +1555,15 @@ static int kvm_dev_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
 #endif
         vcpu->apic_base = sregs->apic_base;
 
+        kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
+
         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
 
         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
+        if (!is_long_mode(vcpu) && is_pae(vcpu))
+                load_pdptrs(vcpu, vcpu->cr3);
 
         if (mmu_reset_needed)
                 kvm_mmu_reset_context(vcpu);
@@ -1693,12 +1785,12 @@ static long kvm_dev_ioctl(struct file *filp,
                 if (copy_from_user(&kvm_run, (void *)arg, sizeof kvm_run))
                         goto out;
                 r = kvm_dev_ioctl_run(kvm, &kvm_run);
-                if (r < 0)
+                if (r < 0 &&  r != -EINTR)
                         goto out;
-                r = -EFAULT;
-                if (copy_to_user((void *)arg, &kvm_run, sizeof kvm_run))
+                if (copy_to_user((void *)arg, &kvm_run, sizeof kvm_run)) {
+                        r = -EFAULT;
                         goto out;
-                r = 0;
+                }
                 break;
         }
         case KVM_GET_REGS: {
@@ -1842,6 +1934,7 @@ static long kvm_dev_ioctl(struct file *filp,
                                  num_msrs_to_save * sizeof(u32)))
                         goto out;
                 r = 0;
+                break;
         }
         default:
                 ;
@@ -1944,17 +2037,17 @@ int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
                 return -EEXIST;
         }
 
-        kvm_arch_ops = ops;
-
-        if (!kvm_arch_ops->cpu_has_kvm_support()) {
+        if (!ops->cpu_has_kvm_support()) {
                 printk(KERN_ERR "kvm: no hardware support\n");
                 return -EOPNOTSUPP;
         }
-        if (kvm_arch_ops->disabled_by_bios()) {
+        if (ops->disabled_by_bios()) {
                 printk(KERN_ERR "kvm: disabled by bios\n");
                 return -EOPNOTSUPP;
         }
 
+        kvm_arch_ops = ops;
+
         r = kvm_arch_ops->hardware_setup();
         if (r < 0)
             return r;