1 files changed, 133 insertions, 159 deletions

diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c
index e8e228118de9..b297a6b111ac 100644
--- a/drivers/kvm/mmu.c
+++ b/drivers/kvm/mmu.c
@@ -16,15 +16,18 @@
  * the COPYING file in the top-level directory.
  *
  */
+
+#include "vmx.h"
+#include "kvm.h"
+
 #include <linux/types.h>
 #include <linux/string.h>
-#include <asm/page.h>
 #include <linux/mm.h>
 #include <linux/highmem.h>
 #include <linux/module.h>
 
-#include "vmx.h"
-#include "kvm.h"
+#include <asm/page.h>
+#include <asm/cmpxchg.h>
 
 #undef MMU_DEBUG
 
@@ -90,25 +93,11 @@ static int dbg = 1;
 #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
 
 
-#define PT32_PTE_COPY_MASK \
-        (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
-
-#define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
-
 #define PT_FIRST_AVAIL_BITS_SHIFT 9
 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
 
-#define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
 
-#define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
-#define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
-
-#define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
-#define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
-
-#define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
-
 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
 
 #define PT64_LEVEL_BITS 9
@@ -165,6 +154,8 @@ struct kvm_rmap_desc {
 
 static struct kmem_cache *pte_chain_cache;
 static struct kmem_cache *rmap_desc_cache;
+static struct kmem_cache *mmu_page_cache;
+static struct kmem_cache *mmu_page_header_cache;
 
 static int is_write_protection(struct kvm_vcpu *vcpu)
 {
@@ -202,6 +193,15 @@ static int is_rmap_pte(u64 pte)
                 == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
 }
 
+static void set_shadow_pte(u64 *sptep, u64 spte)
+{
+#ifdef CONFIG_X86_64
+        set_64bit((unsigned long *)sptep, spte);
+#else
+        set_64bit((unsigned long long *)sptep, spte);
+#endif
+}
+
 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
                                   struct kmem_cache *base_cache, int min,
                                   gfp_t gfp_flags)
@@ -235,6 +235,14 @@ static int __mmu_topup_memory_caches(struct kvm_vcpu *vcpu, gfp_t gfp_flags)
                 goto out;
         r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
                                    rmap_desc_cache, 1, gfp_flags);
+        if (r)
+                goto out;
+        r = mmu_topup_memory_cache(&vcpu->mmu_page_cache,
+                                   mmu_page_cache, 4, gfp_flags);
+        if (r)
+                goto out;
+        r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
+                                   mmu_page_header_cache, 4, gfp_flags);
 out:
         return r;
 }
@@ -258,6 +266,8 @@ static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 {
         mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
         mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
+        mmu_free_memory_cache(&vcpu->mmu_page_cache);
+        mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
 }
 
 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
@@ -433,19 +443,18 @@ static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
                 BUG_ON(!(*spte & PT_WRITABLE_MASK));
                 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
                 rmap_remove(vcpu, spte);
-                kvm_arch_ops->tlb_flush(vcpu);
-                *spte &= ~(u64)PT_WRITABLE_MASK;
+                set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
+                kvm_flush_remote_tlbs(vcpu->kvm);
         }
 }
 
 #ifdef MMU_DEBUG
-static int is_empty_shadow_page(hpa_t page_hpa)
+static int is_empty_shadow_page(u64 *spt)
 {
         u64 *pos;
         u64 *end;
 
-        for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u64);
-                      pos != end; pos++)
+        for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
                 if (*pos != 0) {
                         printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
                                pos, *pos);
@@ -455,13 +464,13 @@ static int is_empty_shadow_page(hpa_t page_hpa)
 }
 #endif
 
-static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
+static void kvm_mmu_free_page(struct kvm_vcpu *vcpu,
+                              struct kvm_mmu_page *page_head)
 {
-        struct kvm_mmu_page *page_head = page_header(page_hpa);
-
-        ASSERT(is_empty_shadow_page(page_hpa));
-        page_head->page_hpa = page_hpa;
-        list_move(&page_head->link, &vcpu->free_pages);
+        ASSERT(is_empty_shadow_page(page_head->spt));
+        list_del(&page_head->link);
+        mmu_memory_cache_free(&vcpu->mmu_page_cache, page_head->spt);
+        mmu_memory_cache_free(&vcpu->mmu_page_header_cache, page_head);
         ++vcpu->kvm->n_free_mmu_pages;
 }
 
@@ -475,12 +484,15 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
 {
         struct kvm_mmu_page *page;
 
-        if (list_empty(&vcpu->free_pages))
+        if (!vcpu->kvm->n_free_mmu_pages)
                 return NULL;
 
-        page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
-        list_move(&page->link, &vcpu->kvm->active_mmu_pages);
-        ASSERT(is_empty_shadow_page(page->page_hpa));
+        page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
+                                      sizeof *page);
+        page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
+        set_page_private(virt_to_page(page->spt), (unsigned long)page);
+        list_add(&page->link, &vcpu->kvm->active_mmu_pages);
+        ASSERT(is_empty_shadow_page(page->spt));
         page->slot_bitmap = 0;
         page->multimapped = 0;
         page->parent_pte = parent_pte;
@@ -638,7 +650,7 @@ static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
         u64 *pt;
         u64 ent;
 
-        pt = __va(page->page_hpa);
+        pt = page->spt;
 
         if (page->role.level == PT_PAGE_TABLE_LEVEL) {
                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
@@ -646,7 +658,7 @@ static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
                                 rmap_remove(vcpu, &pt[i]);
                         pt[i] = 0;
                 }
-                kvm_arch_ops->tlb_flush(vcpu);
+                kvm_flush_remote_tlbs(vcpu->kvm);
                 return;
         }
 
@@ -659,6 +671,7 @@ static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
                 ent &= PT64_BASE_ADDR_MASK;
                 mmu_page_remove_parent_pte(vcpu, page_header(ent), &pt[i]);
         }
+        kvm_flush_remote_tlbs(vcpu->kvm);
 }
 
 static void kvm_mmu_put_page(struct kvm_vcpu *vcpu,
@@ -685,12 +698,12 @@ static void kvm_mmu_zap_page(struct kvm_vcpu *vcpu,
                 }
                 BUG_ON(!parent_pte);
                 kvm_mmu_put_page(vcpu, page, parent_pte);
-                *parent_pte = 0;
+                set_shadow_pte(parent_pte, 0);
         }
         kvm_mmu_page_unlink_children(vcpu, page);
         if (!page->root_count) {
                 hlist_del(&page->hash_link);
-                kvm_mmu_free_page(vcpu, page->page_hpa);
+                kvm_mmu_free_page(vcpu, page);
         } else
                 list_move(&page->link, &vcpu->kvm->active_mmu_pages);
 }
@@ -717,6 +730,17 @@ static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
         return r;
 }
 
+static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        struct kvm_mmu_page *page;
+
+        while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) {
+                pgprintk("%s: zap %lx %x\n",
+                         __FUNCTION__, gfn, page->role.word);
+                kvm_mmu_zap_page(vcpu, page);
+        }
+}
+
 static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
 {
         int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
@@ -805,7 +829,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
                                 return -ENOMEM;
                         }
 
-                        table[index] = new_table->page_hpa | PT_PRESENT_MASK
+                        table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
                                 | PT_WRITABLE_MASK | PT_USER_MASK;
                 }
                 table_addr = table[index] & PT64_BASE_ADDR_MASK;
@@ -817,11 +841,12 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu)
         int i;
         struct kvm_mmu_page *page;
 
+        if (!VALID_PAGE(vcpu->mmu.root_hpa))
+                return;
 #ifdef CONFIG_X86_64
         if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
                 hpa_t root = vcpu->mmu.root_hpa;
 
-                ASSERT(VALID_PAGE(root));
                 page = page_header(root);
                 --page->root_count;
                 vcpu->mmu.root_hpa = INVALID_PAGE;
@@ -832,7 +857,6 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu)
                 hpa_t root = vcpu->mmu.pae_root[i];
 
                 if (root) {
-                        ASSERT(VALID_PAGE(root));
                         root &= PT64_BASE_ADDR_MASK;
                         page = page_header(root);
                         --page->root_count;
@@ -857,7 +881,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
                 ASSERT(!VALID_PAGE(root));
                 page = kvm_mmu_get_page(vcpu, root_gfn, 0,
                                         PT64_ROOT_LEVEL, 0, 0, NULL);
-                root = page->page_hpa;
+                root = __pa(page->spt);
                 ++page->root_count;
                 vcpu->mmu.root_hpa = root;
                 return;
@@ -878,7 +902,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
                 page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
                                         PT32_ROOT_LEVEL, !is_paging(vcpu),
                                         0, NULL);
-                root = page->page_hpa;
+                root = __pa(page->spt);
                 ++page->root_count;
                 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
         }
@@ -928,9 +952,7 @@ static int nonpaging_init_context(struct kvm_vcpu *vcpu)
         context->free = nonpaging_free;
         context->root_level = 0;
         context->shadow_root_level = PT32E_ROOT_LEVEL;
-        mmu_alloc_roots(vcpu);
-        ASSERT(VALID_PAGE(context->root_hpa));
-        kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
+        context->root_hpa = INVALID_PAGE;
         return 0;
 }
 
@@ -944,59 +966,6 @@ static void paging_new_cr3(struct kvm_vcpu *vcpu)
 {
         pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
         mmu_free_roots(vcpu);
-        if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES))
-                kvm_mmu_free_some_pages(vcpu);
-        mmu_alloc_roots(vcpu);
-        kvm_mmu_flush_tlb(vcpu);
-        kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
-}
-
-static inline void set_pte_common(struct kvm_vcpu *vcpu,
-                             u64 *shadow_pte,
-                             gpa_t gaddr,
-                             int dirty,
-                             u64 access_bits,
-                             gfn_t gfn)
-{
-        hpa_t paddr;
-
-        *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
-        if (!dirty)
-                access_bits &= ~PT_WRITABLE_MASK;
-
-        paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
-
-        *shadow_pte |= access_bits;
-
-        if (is_error_hpa(paddr)) {
-                *shadow_pte |= gaddr;
-                *shadow_pte |= PT_SHADOW_IO_MARK;
-                *shadow_pte &= ~PT_PRESENT_MASK;
-                return;
-        }
-
-        *shadow_pte |= paddr;
-
-        if (access_bits & PT_WRITABLE_MASK) {
-                struct kvm_mmu_page *shadow;
-
-                shadow = kvm_mmu_lookup_page(vcpu, gfn);
-                if (shadow) {
-                        pgprintk("%s: found shadow page for %lx, marking ro\n",
-                                 __FUNCTION__, gfn);
-                        access_bits &= ~PT_WRITABLE_MASK;
-                        if (is_writeble_pte(*shadow_pte)) {
-                                    *shadow_pte &= ~PT_WRITABLE_MASK;
-                                    kvm_arch_ops->tlb_flush(vcpu);
-                        }
-                }
-        }
-
-        if (access_bits & PT_WRITABLE_MASK)
-                mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
-
-        page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
-        rmap_add(vcpu, shadow_pte);
 }
 
 static void inject_page_fault(struct kvm_vcpu *vcpu,
@@ -1006,23 +975,6 @@ static void inject_page_fault(struct kvm_vcpu *vcpu,
         kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
 }
 
-static inline int fix_read_pf(u64 *shadow_ent)
-{
-        if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
-            !(*shadow_ent & PT_USER_MASK)) {
-                /*
-                 * If supervisor write protect is disabled, we shadow kernel
-                 * pages as user pages so we can trap the write access.
-                 */
-                *shadow_ent |= PT_USER_MASK;
-                *shadow_ent &= ~PT_WRITABLE_MASK;
-
-                return 1;
-
-        }
-        return 0;
-}
-
 static void paging_free(struct kvm_vcpu *vcpu)
 {
         nonpaging_free(vcpu);
@@ -1047,10 +999,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
         context->free = paging_free;
         context->root_level = level;
         context->shadow_root_level = level;
-        mmu_alloc_roots(vcpu);
-        ASSERT(VALID_PAGE(context->root_hpa));
-        kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
-                    (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
+        context->root_hpa = INVALID_PAGE;
         return 0;
 }
 
@@ -1069,10 +1018,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu)
         context->free = paging_free;
         context->root_level = PT32_ROOT_LEVEL;
         context->shadow_root_level = PT32E_ROOT_LEVEL;
-        mmu_alloc_roots(vcpu);
-        ASSERT(VALID_PAGE(context->root_hpa));
-        kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
-                    (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
+        context->root_hpa = INVALID_PAGE;
         return 0;
 }
 
@@ -1107,18 +1053,33 @@ static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
 
 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
 {
+        destroy_kvm_mmu(vcpu);
+        return init_kvm_mmu(vcpu);
+}
+
+int kvm_mmu_load(struct kvm_vcpu *vcpu)
+{
         int r;
 
-        destroy_kvm_mmu(vcpu);
-        r = init_kvm_mmu(vcpu);
-        if (r < 0)
-                goto out;
+        spin_lock(&vcpu->kvm->lock);
         r = mmu_topup_memory_caches(vcpu);
+        if (r)
+                goto out;
+        mmu_alloc_roots(vcpu);
+        kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
+        kvm_mmu_flush_tlb(vcpu);
 out:
+        spin_unlock(&vcpu->kvm->lock);
         return r;
 }
+EXPORT_SYMBOL_GPL(kvm_mmu_load);
+
+void kvm_mmu_unload(struct kvm_vcpu *vcpu)
+{
+        mmu_free_roots(vcpu);
+}
 
-static void mmu_pre_write_zap_pte(struct kvm_vcpu *vcpu,
+static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
                                   struct kvm_mmu_page *page,
                                   u64 *spte)
 {
@@ -1135,9 +1096,25 @@ static void mmu_pre_write_zap_pte(struct kvm_vcpu *vcpu,
                 }
         }
         *spte = 0;
+        kvm_flush_remote_tlbs(vcpu->kvm);
+}
+
+static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
+                                  struct kvm_mmu_page *page,
+                                  u64 *spte,
+                                  const void *new, int bytes)
+{
+        if (page->role.level != PT_PAGE_TABLE_LEVEL)
+                return;
+
+        if (page->role.glevels == PT32_ROOT_LEVEL)
+                paging32_update_pte(vcpu, page, spte, new, bytes);
+        else
+                paging64_update_pte(vcpu, page, spte, new, bytes);
 }
 
-void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
+void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
+                       const u8 *old, const u8 *new, int bytes)
 {
         gfn_t gfn = gpa >> PAGE_SHIFT;
         struct kvm_mmu_page *page;
@@ -1149,6 +1126,7 @@ void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
         unsigned pte_size;
         unsigned page_offset;
         unsigned misaligned;
+        unsigned quadrant;
         int level;
         int flooded = 0;
         int npte;
@@ -1169,6 +1147,7 @@ void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
                         continue;
                 pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
                 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
+                misaligned |= bytes < 4;
                 if (misaligned || flooded) {
                         /*
                          * Misaligned accesses are too much trouble to fix
@@ -1200,21 +1179,20 @@ void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
                                 page_offset <<= 1;
                                 npte = 2;
                         }
+                        quadrant = page_offset >> PAGE_SHIFT;
                         page_offset &= ~PAGE_MASK;
+                        if (quadrant != page->role.quadrant)
+                                continue;
                 }
-                spte = __va(page->page_hpa);
-                spte += page_offset / sizeof(*spte);
+                spte = &page->spt[page_offset / sizeof(*spte)];
                 while (npte--) {
-                        mmu_pre_write_zap_pte(vcpu, page, spte);
+                        mmu_pte_write_zap_pte(vcpu, page, spte);
+                        mmu_pte_write_new_pte(vcpu, page, spte, new, bytes);
                         ++spte;
                 }
         }
 }
 
-void kvm_mmu_post_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
-{
-}
-
 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
 {
         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
@@ -1243,13 +1221,6 @@ static void free_mmu_pages(struct kvm_vcpu *vcpu)
                                     struct kvm_mmu_page, link);
                 kvm_mmu_zap_page(vcpu, page);
         }
-        while (!list_empty(&vcpu->free_pages)) {
-                page = list_entry(vcpu->free_pages.next,
-                                  struct kvm_mmu_page, link);
-                list_del(&page->link);
-                __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
-                page->page_hpa = INVALID_PAGE;
-        }
         free_page((unsigned long)vcpu->mmu.pae_root);
 }
 
@@ -1260,18 +1231,7 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
 
         ASSERT(vcpu);
 
-        for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
-                struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
-
-                INIT_LIST_HEAD(&page_header->link);
-                if ((page = alloc_page(GFP_KERNEL)) == NULL)
-                        goto error_1;
-                set_page_private(page, (unsigned long)page_header);
-                page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
-                memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
-                list_add(&page_header->link, &vcpu->free_pages);
-                ++vcpu->kvm->n_free_mmu_pages;
-        }
+        vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES;
 
         /*
          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
@@ -1296,7 +1256,6 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
 {
         ASSERT(vcpu);
         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
-        ASSERT(list_empty(&vcpu->free_pages));
 
         return alloc_mmu_pages(vcpu);
 }
@@ -1305,7 +1264,6 @@ int kvm_mmu_setup(struct kvm_vcpu *vcpu)
 {
         ASSERT(vcpu);
         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
-        ASSERT(!list_empty(&vcpu->free_pages));
 
         return init_kvm_mmu(vcpu);
 }
@@ -1331,7 +1289,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot)
                 if (!test_bit(slot, &page->slot_bitmap))
                         continue;
 
-                pt = __va(page->page_hpa);
+                pt = page->spt;
                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
                         /* avoid RMW */
                         if (pt[i] & PT_WRITABLE_MASK) {
@@ -1354,7 +1312,7 @@ void kvm_mmu_zap_all(struct kvm_vcpu *vcpu)
         }
 
         mmu_free_memory_caches(vcpu);
-        kvm_arch_ops->tlb_flush(vcpu);
+        kvm_flush_remote_tlbs(vcpu->kvm);
         init_kvm_mmu(vcpu);
 }
 
@@ -1364,6 +1322,10 @@ void kvm_mmu_module_exit(void)
                 kmem_cache_destroy(pte_chain_cache);
         if (rmap_desc_cache)
                 kmem_cache_destroy(rmap_desc_cache);
+        if (mmu_page_cache)
+                kmem_cache_destroy(mmu_page_cache);
+        if (mmu_page_header_cache)
+                kmem_cache_destroy(mmu_page_header_cache);
 }
 
 int kvm_mmu_module_init(void)
@@ -1379,6 +1341,18 @@ int kvm_mmu_module_init(void)
         if (!rmap_desc_cache)
                 goto nomem;
 
+        mmu_page_cache = kmem_cache_create("kvm_mmu_page",
+                                           PAGE_SIZE,
+                                           PAGE_SIZE, 0, NULL, NULL);
+        if (!mmu_page_cache)
+                goto nomem;
+
+        mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
+                                                  sizeof(struct kvm_mmu_page),
+                                                  0, 0, NULL, NULL);
+        if (!mmu_page_header_cache)
+                goto nomem;
+
         return 0;
 
 nomem:
@@ -1482,7 +1456,7 @@ static int count_writable_mappings(struct kvm_vcpu *vcpu)
         int i;
 
         list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
-                u64 *pt = __va(page->page_hpa);
+                u64 *pt = page->spt;
 
                 if (page->role.level != PT_PAGE_TABLE_LEVEL)
                         continue;