Merge branch 'linus' into sfi-release

Conflicts:
	arch/x86/kernel/setup.c
	drivers/acpi/power.c
	init/main.c

Signed-off-by: Len Brown <len.brown@intel.com>

1 files changed, 384 insertions, 203 deletions

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 0ef5bb2b4043..eca41ae9f453 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -18,6 +18,7 @@
  */
 
 #include "mmu.h"
+#include "kvm_cache_regs.h"
 
 #include <linux/kvm_host.h>
 #include <linux/types.h>
@@ -107,6 +108,9 @@ module_param(oos_shadow, bool, 0644);
 
 #define PT32_LEVEL_MASK(level) \
                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
+#define PT32_LVL_OFFSET_MASK(level) \
+        (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
+                                                * PT32_LEVEL_BITS))) - 1))
 
 #define PT32_INDEX(address, level)\
         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
@@ -115,10 +119,19 @@ module_param(oos_shadow, bool, 0644);
 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
 #define PT64_DIR_BASE_ADDR_MASK \
         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
+#define PT64_LVL_ADDR_MASK(level) \
+        (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
+                                                * PT64_LEVEL_BITS))) - 1))
+#define PT64_LVL_OFFSET_MASK(level) \
+        (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
+                                                * PT64_LEVEL_BITS))) - 1))
 
 #define PT32_BASE_ADDR_MASK PAGE_MASK
 #define PT32_DIR_BASE_ADDR_MASK \
         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
+#define PT32_LVL_ADDR_MASK(level) \
+        (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
+                                            * PT32_LEVEL_BITS))) - 1))
 
 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
                         | PT64_NX_MASK)
@@ -129,6 +142,7 @@ module_param(oos_shadow, bool, 0644);
 #define PFERR_RSVD_MASK (1U << 3)
 #define PFERR_FETCH_MASK (1U << 4)
 
+#define PT_PDPE_LEVEL 3
 #define PT_DIRECTORY_LEVEL 2
 #define PT_PAGE_TABLE_LEVEL 1
 
@@ -139,10 +153,13 @@ module_param(oos_shadow, bool, 0644);
 #define ACC_USER_MASK    PT_USER_MASK
 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
 
+#define CREATE_TRACE_POINTS
+#include "mmutrace.h"
+
 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
 
 struct kvm_rmap_desc {
-        u64 *shadow_ptes[RMAP_EXT];
+        u64 *sptes[RMAP_EXT];
         struct kvm_rmap_desc *more;
 };
 
@@ -239,16 +256,25 @@ static int is_writeble_pte(unsigned long pte)
         return pte & PT_WRITABLE_MASK;
 }
 
-static int is_dirty_pte(unsigned long pte)
+static int is_dirty_gpte(unsigned long pte)
 {
-        return pte & shadow_dirty_mask;
+        return pte & PT_DIRTY_MASK;
 }
 
-static int is_rmap_pte(u64 pte)
+static int is_rmap_spte(u64 pte)
 {
         return is_shadow_present_pte(pte);
 }
 
+static int is_last_spte(u64 pte, int level)
+{
+        if (level == PT_PAGE_TABLE_LEVEL)
+                return 1;
+        if (is_large_pte(pte))
+                return 1;
+        return 0;
+}
+
 static pfn_t spte_to_pfn(u64 pte)
 {
         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
@@ -261,7 +287,7 @@ static gfn_t pse36_gfn_delta(u32 gpte)
         return (gpte & PT32_DIR_PSE36_MASK) << shift;
 }
 
-static void set_shadow_pte(u64 *sptep, u64 spte)
+static void __set_spte(u64 *sptep, u64 spte)
 {
 #ifdef CONFIG_X86_64
         set_64bit((unsigned long *)sptep, spte);
@@ -380,37 +406,52 @@ static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
  * Return the pointer to the largepage write count for a given
  * gfn, handling slots that are not large page aligned.
  */
-static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
+static int *slot_largepage_idx(gfn_t gfn,
+                               struct kvm_memory_slot *slot,
+                               int level)
 {
         unsigned long idx;
 
-        idx = (gfn / KVM_PAGES_PER_HPAGE) -
-              (slot->base_gfn / KVM_PAGES_PER_HPAGE);
-        return &slot->lpage_info[idx].write_count;
+        idx = (gfn / KVM_PAGES_PER_HPAGE(level)) -
+              (slot->base_gfn / KVM_PAGES_PER_HPAGE(level));
+        return &slot->lpage_info[level - 2][idx].write_count;
 }
 
 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
 {
+        struct kvm_memory_slot *slot;
         int *write_count;
+        int i;
 
         gfn = unalias_gfn(kvm, gfn);
-        write_count = slot_largepage_idx(gfn,
-                                         gfn_to_memslot_unaliased(kvm, gfn));
-        *write_count += 1;
+
+        slot = gfn_to_memslot_unaliased(kvm, gfn);
+        for (i = PT_DIRECTORY_LEVEL;
+             i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
+                write_count   = slot_largepage_idx(gfn, slot, i);
+                *write_count += 1;
+        }
 }
 
 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
 {
+        struct kvm_memory_slot *slot;
         int *write_count;
+        int i;
 
         gfn = unalias_gfn(kvm, gfn);
-        write_count = slot_largepage_idx(gfn,
-                                         gfn_to_memslot_unaliased(kvm, gfn));
-        *write_count -= 1;
-        WARN_ON(*write_count < 0);
+        for (i = PT_DIRECTORY_LEVEL;
+             i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
+                slot          = gfn_to_memslot_unaliased(kvm, gfn);
+                write_count   = slot_largepage_idx(gfn, slot, i);
+                *write_count -= 1;
+                WARN_ON(*write_count < 0);
+        }
 }
 
-static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
+static int has_wrprotected_page(struct kvm *kvm,
+                                gfn_t gfn,
+                                int level)
 {
         struct kvm_memory_slot *slot;
         int *largepage_idx;
@@ -418,47 +459,67 @@ static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
         gfn = unalias_gfn(kvm, gfn);
         slot = gfn_to_memslot_unaliased(kvm, gfn);
         if (slot) {
-                largepage_idx = slot_largepage_idx(gfn, slot);
+                largepage_idx = slot_largepage_idx(gfn, slot, level);
                 return *largepage_idx;
         }
 
         return 1;
 }
 
-static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
+static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
 {
+        unsigned long page_size = PAGE_SIZE;
         struct vm_area_struct *vma;
         unsigned long addr;
-        int ret = 0;
+        int i, ret = 0;
 
         addr = gfn_to_hva(kvm, gfn);
         if (kvm_is_error_hva(addr))
-                return ret;
+                return page_size;
 
         down_read(&current->mm->mmap_sem);
         vma = find_vma(current->mm, addr);
-        if (vma && is_vm_hugetlb_page(vma))
-                ret = 1;
+        if (!vma)
+                goto out;
+
+        page_size = vma_kernel_pagesize(vma);
+
+out:
         up_read(&current->mm->mmap_sem);
 
+        for (i = PT_PAGE_TABLE_LEVEL;
+             i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
+                if (page_size >= KVM_HPAGE_SIZE(i))
+                        ret = i;
+                else
+                        break;
+        }
+
         return ret;
 }
 
-static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
+static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
 {
         struct kvm_memory_slot *slot;
-
-        if (has_wrprotected_page(vcpu->kvm, large_gfn))
-                return 0;
-
-        if (!host_largepage_backed(vcpu->kvm, large_gfn))
-                return 0;
+        int host_level;
+        int level = PT_PAGE_TABLE_LEVEL;
 
         slot = gfn_to_memslot(vcpu->kvm, large_gfn);
         if (slot && slot->dirty_bitmap)
-                return 0;
+                return PT_PAGE_TABLE_LEVEL;
 
-        return 1;
+        host_level = host_mapping_level(vcpu->kvm, large_gfn);
+
+        if (host_level == PT_PAGE_TABLE_LEVEL)
+                return host_level;
+
+        for (level = PT_DIRECTORY_LEVEL; level <= host_level; ++level) {
+
+                if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
+                        break;
+        }
+
+        return level - 1;
 }
 
 /*
@@ -466,19 +527,19 @@ static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  * Note: gfn must be unaliased before this function get called
  */
 
-static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
+static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
 {
         struct kvm_memory_slot *slot;
         unsigned long idx;
 
         slot = gfn_to_memslot(kvm, gfn);
-        if (!lpage)
+        if (likely(level == PT_PAGE_TABLE_LEVEL))
                 return &slot->rmap[gfn - slot->base_gfn];
 
-        idx = (gfn / KVM_PAGES_PER_HPAGE) -
-              (slot->base_gfn / KVM_PAGES_PER_HPAGE);
+        idx = (gfn / KVM_PAGES_PER_HPAGE(level)) -
+                (slot->base_gfn / KVM_PAGES_PER_HPAGE(level));
 
-        return &slot->lpage_info[idx].rmap_pde;
+        return &slot->lpage_info[level - 2][idx].rmap_pde;
 }
 
 /*
@@ -494,42 +555,42 @@ static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
  * the spte was not added.
  *
  */
-static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
+static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 {
         struct kvm_mmu_page *sp;
         struct kvm_rmap_desc *desc;
         unsigned long *rmapp;
         int i, count = 0;
 
-        if (!is_rmap_pte(*spte))
+        if (!is_rmap_spte(*spte))
                 return count;
         gfn = unalias_gfn(vcpu->kvm, gfn);
         sp = page_header(__pa(spte));
         sp->gfns[spte - sp->spt] = gfn;
-        rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
+        rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
         if (!*rmapp) {
                 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
                 *rmapp = (unsigned long)spte;
         } else if (!(*rmapp & 1)) {
                 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
                 desc = mmu_alloc_rmap_desc(vcpu);
-                desc->shadow_ptes[0] = (u64 *)*rmapp;
-                desc->shadow_ptes[1] = spte;
+                desc->sptes[0] = (u64 *)*rmapp;
+                desc->sptes[1] = spte;
                 *rmapp = (unsigned long)desc | 1;
         } else {
                 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
-                while (desc->shadow_ptes[RMAP_EXT-1] && desc->more) {
+                while (desc->sptes[RMAP_EXT-1] && desc->more) {
                         desc = desc->more;
                         count += RMAP_EXT;
                 }
-                if (desc->shadow_ptes[RMAP_EXT-1]) {
+                if (desc->sptes[RMAP_EXT-1]) {
                         desc->more = mmu_alloc_rmap_desc(vcpu);
                         desc = desc->more;
                 }
-                for (i = 0; desc->shadow_ptes[i]; ++i)
+                for (i = 0; desc->sptes[i]; ++i)
                         ;
-                desc->shadow_ptes[i] = spte;
+                desc->sptes[i] = spte;
         }
         return count;
 }
@@ -541,14 +602,14 @@ static void rmap_desc_remove_entry(unsigned long *rmapp,
 {
         int j;
 
-        for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
+        for (j = RMAP_EXT - 1; !desc->sptes[j] && j > i; --j)
                 ;
-        desc->shadow_ptes[i] = desc->shadow_ptes[j];
-        desc->shadow_ptes[j] = NULL;
+        desc->sptes[i] = desc->sptes[j];
+        desc->sptes[j] = NULL;
         if (j != 0)
                 return;
         if (!prev_desc && !desc->more)
-                *rmapp = (unsigned long)desc->shadow_ptes[0];
+                *rmapp = (unsigned long)desc->sptes[0];
         else
                 if (prev_desc)
                         prev_desc->more = desc->more;
@@ -566,7 +627,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
         unsigned long *rmapp;
         int i;
 
-        if (!is_rmap_pte(*spte))
+        if (!is_rmap_spte(*spte))
                 return;
         sp = page_header(__pa(spte));
         pfn = spte_to_pfn(*spte);
@@ -576,7 +637,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
                 kvm_release_pfn_dirty(pfn);
         else
                 kvm_release_pfn_clean(pfn);
-        rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
+        rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], sp->role.level);
         if (!*rmapp) {
                 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
                 BUG();
@@ -593,8 +654,8 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
                 prev_desc = NULL;
                 while (desc) {
-                        for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
-                                if (desc->shadow_ptes[i] == spte) {
+                        for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i)
+                                if (desc->sptes[i] == spte) {
                                         rmap_desc_remove_entry(rmapp,
                                                                desc, i,
                                                                prev_desc);
@@ -625,10 +686,10 @@ static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
         prev_desc = NULL;
         prev_spte = NULL;
         while (desc) {
-                for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
+                for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i) {
                         if (prev_spte == spte)
-                                return desc->shadow_ptes[i];
-                        prev_spte = desc->shadow_ptes[i];
+                                return desc->sptes[i];
+                        prev_spte = desc->sptes[i];
                 }
                 desc = desc->more;
         }
@@ -639,10 +700,10 @@ static int rmap_write_protect(struct kvm *kvm, u64 gfn)
 {
         unsigned long *rmapp;
         u64 *spte;
-        int write_protected = 0;
+        int i, write_protected = 0;
 
         gfn = unalias_gfn(kvm, gfn);
-        rmapp = gfn_to_rmap(kvm, gfn, 0);
+        rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
 
         spte = rmap_next(kvm, rmapp, NULL);
         while (spte) {
@@ -650,7 +711,7 @@ static int rmap_write_protect(struct kvm *kvm, u64 gfn)
                 BUG_ON(!(*spte & PT_PRESENT_MASK));
                 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
                 if (is_writeble_pte(*spte)) {
-                        set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
+                        __set_spte(spte, *spte & ~PT_WRITABLE_MASK);
                         write_protected = 1;
                 }
                 spte = rmap_next(kvm, rmapp, spte);
@@ -664,21 +725,24 @@ static int rmap_write_protect(struct kvm *kvm, u64 gfn)
         }
 
         /* check for huge page mappings */
-        rmapp = gfn_to_rmap(kvm, gfn, 1);
-        spte = rmap_next(kvm, rmapp, NULL);
-        while (spte) {
-                BUG_ON(!spte);
-                BUG_ON(!(*spte & PT_PRESENT_MASK));
-                BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
-                pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
-                if (is_writeble_pte(*spte)) {
-                        rmap_remove(kvm, spte);
-                        --kvm->stat.lpages;
-                        set_shadow_pte(spte, shadow_trap_nonpresent_pte);
-                        spte = NULL;
-                        write_protected = 1;
+        for (i = PT_DIRECTORY_LEVEL;
+             i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
+                rmapp = gfn_to_rmap(kvm, gfn, i);
+                spte = rmap_next(kvm, rmapp, NULL);
+                while (spte) {
+                        BUG_ON(!spte);
+                        BUG_ON(!(*spte & PT_PRESENT_MASK));
+                        BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
+                        pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
+                        if (is_writeble_pte(*spte)) {
+                                rmap_remove(kvm, spte);
+                                --kvm->stat.lpages;
+                                __set_spte(spte, shadow_trap_nonpresent_pte);
+                                spte = NULL;
+                                write_protected = 1;
+                        }
+                        spte = rmap_next(kvm, rmapp, spte);
                 }
-                spte = rmap_next(kvm, rmapp, spte);
         }
 
         return write_protected;
@@ -693,7 +757,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
                 BUG_ON(!(*spte & PT_PRESENT_MASK));
                 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
                 rmap_remove(kvm, spte);
-                set_shadow_pte(spte, shadow_trap_nonpresent_pte);
+                __set_spte(spte, shadow_trap_nonpresent_pte);
                 need_tlb_flush = 1;
         }
         return need_tlb_flush;
@@ -702,7 +766,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
                           int (*handler)(struct kvm *kvm, unsigned long *rmapp))
 {
-        int i;
+        int i, j;
         int retval = 0;
 
         /*
@@ -721,11 +785,15 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
                 end = start + (memslot->npages << PAGE_SHIFT);
                 if (hva >= start && hva < end) {
                         gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
+
                         retval |= handler(kvm, &memslot->rmap[gfn_offset]);
-                        retval |= handler(kvm,
-                                          &memslot->lpage_info[
-                                                  gfn_offset /
-                                                  KVM_PAGES_PER_HPAGE].rmap_pde);
+
+                        for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
+                                int idx = gfn_offset;
+                                idx /= KVM_PAGES_PER_HPAGE(PT_DIRECTORY_LEVEL + j);
+                                retval |= handler(kvm,
+                                        &memslot->lpage_info[j][idx].rmap_pde);
+                        }
                 }
         }
 
@@ -763,12 +831,15 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
 
 #define RMAP_RECYCLE_THRESHOLD 1000
 
-static void rmap_recycle(struct kvm_vcpu *vcpu, gfn_t gfn, int lpage)
+static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 {
         unsigned long *rmapp;
+        struct kvm_mmu_page *sp;
+
+        sp = page_header(__pa(spte));
 
         gfn = unalias_gfn(vcpu->kvm, gfn);
-        rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
+        rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
 
         kvm_unmap_rmapp(vcpu->kvm, rmapp);
         kvm_flush_remote_tlbs(vcpu->kvm);
@@ -1109,6 +1180,7 @@ static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
                 return 1;
         }
 
+        trace_kvm_mmu_sync_page(sp);
         if (rmap_write_protect(vcpu->kvm, sp->gfn))
                 kvm_flush_remote_tlbs(vcpu->kvm);
         kvm_unlink_unsync_page(vcpu->kvm, sp);
@@ -1231,8 +1303,6 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
                 role.quadrant = quadrant;
         }
-        pgprintk("%s: looking gfn %lx role %x\n", __func__,
-                 gfn, role.word);
         index = kvm_page_table_hashfn(gfn);
         bucket = &vcpu->kvm->arch.mmu_page_hash[index];
         hlist_for_each_entry_safe(sp, node, tmp, bucket, hash_link)
@@ -1249,14 +1319,13 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
                                 set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
                                 kvm_mmu_mark_parents_unsync(vcpu, sp);
                         }
-                        pgprintk("%s: found\n", __func__);
+                        trace_kvm_mmu_get_page(sp, false);
                         return sp;
                 }
         ++vcpu->kvm->stat.mmu_cache_miss;
         sp = kvm_mmu_alloc_page(vcpu, parent_pte);
         if (!sp)
                 return sp;
-        pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
         sp->gfn = gfn;
         sp->role = role;
         hlist_add_head(&sp->hash_link, bucket);
@@ -1269,6 +1338,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
                 vcpu->arch.mmu.prefetch_page(vcpu, sp);
         else
                 nonpaging_prefetch_page(vcpu, sp);
+        trace_kvm_mmu_get_page(sp, true);
         return sp;
 }
 
@@ -1292,6 +1362,11 @@ static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
 {
         if (iterator->level < PT_PAGE_TABLE_LEVEL)
                 return false;
+
+        if (iterator->level == PT_PAGE_TABLE_LEVEL)
+                if (is_large_pte(*iterator->sptep))
+                        return false;
+
         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
         return true;
@@ -1312,25 +1387,17 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm,
 
         pt = sp->spt;
 
-        if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
-                for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
-                        if (is_shadow_present_pte(pt[i]))
-                                rmap_remove(kvm, &pt[i]);
-                        pt[i] = shadow_trap_nonpresent_pte;
-                }
-                return;
-        }
-
         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
                 ent = pt[i];
 
                 if (is_shadow_present_pte(ent)) {
-                        if (!is_large_pte(ent)) {
+                        if (!is_last_spte(ent, sp->role.level)) {
                                 ent &= PT64_BASE_ADDR_MASK;
                                 mmu_page_remove_parent_pte(page_header(ent),
                                                            &pt[i]);
                         } else {
-                                --kvm->stat.lpages;
+                                if (is_large_pte(ent))
+                                        --kvm->stat.lpages;
                                 rmap_remove(kvm, &pt[i]);
                         }
                 }
@@ -1346,10 +1413,10 @@ static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
 static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
 {
         int i;
+        struct kvm_vcpu *vcpu;
 
-        for (i = 0; i < KVM_MAX_VCPUS; ++i)
-                if (kvm->vcpus[i])
-                        kvm->vcpus[i]->arch.last_pte_updated = NULL;
+        kvm_for_each_vcpu(i, vcpu, kvm)
+                vcpu->arch.last_pte_updated = NULL;
 }
 
 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
@@ -1368,7 +1435,7 @@ static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
                 }
                 BUG_ON(!parent_pte);
                 kvm_mmu_put_page(sp, parent_pte);
-                set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
+                __set_spte(parent_pte, shadow_trap_nonpresent_pte);
         }
 }
 
@@ -1400,6 +1467,8 @@ static int mmu_zap_unsync_children(struct kvm *kvm,
 static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 {
         int ret;
+
+        trace_kvm_mmu_zap_page(sp);
         ++kvm->stat.mmu_shadow_zapped;
         ret = mmu_zap_unsync_children(kvm, sp);
         kvm_mmu_page_unlink_children(kvm, sp);
@@ -1516,7 +1585,7 @@ static void mmu_convert_notrap(struct kvm_mmu_page *sp)
 
         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
                 if (pt[i] == shadow_notrap_nonpresent_pte)
-                        set_shadow_pte(&pt[i], shadow_trap_nonpresent_pte);
+                        __set_spte(&pt[i], shadow_trap_nonpresent_pte);
         }
 }
 
@@ -1646,6 +1715,7 @@ static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
         struct kvm_mmu_page *s;
         struct hlist_node *node, *n;
 
+        trace_kvm_mmu_unsync_page(sp);
         index = kvm_page_table_hashfn(sp->gfn);
         bucket = &vcpu->kvm->arch.mmu_page_hash[index];
         /* don't unsync if pagetable is shadowed with multiple roles */
@@ -1682,9 +1752,9 @@ static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
         return 0;
 }
 
-static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
+static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                     unsigned pte_access, int user_fault,
-                    int write_fault, int dirty, int largepage,
+                    int write_fault, int dirty, int level,
                     gfn_t gfn, pfn_t pfn, bool speculative,
                     bool can_unsync)
 {
@@ -1707,7 +1777,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                 spte |= shadow_nx_mask;
         if (pte_access & ACC_USER_MASK)
                 spte |= shadow_user_mask;
-        if (largepage)
+        if (level > PT_PAGE_TABLE_LEVEL)
                 spte |= PT_PAGE_SIZE_MASK;
         if (tdp_enabled)
                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
@@ -1718,7 +1788,8 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
         if ((pte_access & ACC_WRITE_MASK)
             || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
 
-                if (largepage && has_wrprotected_page(vcpu->kvm, gfn)) {
+                if (level > PT_PAGE_TABLE_LEVEL &&
+                    has_wrprotected_page(vcpu->kvm, gfn, level)) {
                         ret = 1;
                         spte = shadow_trap_nonpresent_pte;
                         goto set_pte;
@@ -1732,7 +1803,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                  * is responsibility of mmu_get_page / kvm_sync_page.
                  * Same reasoning can be applied to dirty page accounting.
                  */
-                if (!can_unsync && is_writeble_pte(*shadow_pte))
+                if (!can_unsync && is_writeble_pte(*sptep))
                         goto set_pte;
 
                 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
@@ -1749,65 +1820,67 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                 mark_page_dirty(vcpu->kvm, gfn);
 
 set_pte:
-        set_shadow_pte(shadow_pte, spte);
+        __set_spte(sptep, spte);
         return ret;
 }
 
-static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
+static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                          unsigned pt_access, unsigned pte_access,
                          int user_fault, int write_fault, int dirty,
-                         int *ptwrite, int largepage, gfn_t gfn,
+                         int *ptwrite, int level, gfn_t gfn,
                          pfn_t pfn, bool speculative)
 {
         int was_rmapped = 0;
-        int was_writeble = is_writeble_pte(*shadow_pte);
+        int was_writeble = is_writeble_pte(*sptep);
         int rmap_count;
 
         pgprintk("%s: spte %llx access %x write_fault %d"
                  " user_fault %d gfn %lx\n",
-                 __func__, *shadow_pte, pt_access,
+                 __func__, *sptep, pt_access,
                  write_fault, user_fault, gfn);
 
-        if (is_rmap_pte(*shadow_pte)) {
+        if (is_rmap_spte(*sptep)) {
                 /*
                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
                  * the parent of the now unreachable PTE.
                  */
-                if (largepage && !is_large_pte(*shadow_pte)) {
+                if (level > PT_PAGE_TABLE_LEVEL &&
+                    !is_large_pte(*sptep)) {
                         struct kvm_mmu_page *child;
-                        u64 pte = *shadow_pte;
+                        u64 pte = *sptep;
 
                         child = page_header(pte & PT64_BASE_ADDR_MASK);
-                        mmu_page_remove_parent_pte(child, shadow_pte);
-                } else if (pfn != spte_to_pfn(*shadow_pte)) {
+                        mmu_page_remove_parent_pte(child, sptep);
+                } else if (pfn != spte_to_pfn(*sptep)) {
                         pgprintk("hfn old %lx new %lx\n",
-                                 spte_to_pfn(*shadow_pte), pfn);
-                        rmap_remove(vcpu->kvm, shadow_pte);
+                                 spte_to_pfn(*sptep), pfn);
+                        rmap_remove(vcpu->kvm, sptep);
                 } else
                         was_rmapped = 1;
         }
-        if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault,
-                      dirty, largepage, gfn, pfn, speculative, true)) {
+
+        if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
+                      dirty, level, gfn, pfn, speculative, true)) {
                 if (write_fault)
                         *ptwrite = 1;
                 kvm_x86_ops->tlb_flush(vcpu);
         }
 
-        pgprintk("%s: setting spte %llx\n", __func__, *shadow_pte);
+        pgprintk("%s: setting spte %llx\n", __func__, *sptep);
         pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
-                 is_large_pte(*shadow_pte)? "2MB" : "4kB",
-                 is_present_pte(*shadow_pte)?"RW":"R", gfn,
-                 *shadow_pte, shadow_pte);
-        if (!was_rmapped && is_large_pte(*shadow_pte))
+                 is_large_pte(*sptep)? "2MB" : "4kB",
+                 *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
+                 *sptep, sptep);
+        if (!was_rmapped && is_large_pte(*sptep))
                 ++vcpu->kvm->stat.lpages;
 
-        page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
+        page_header_update_slot(vcpu->kvm, sptep, gfn);
         if (!was_rmapped) {
-                rmap_count = rmap_add(vcpu, shadow_pte, gfn, largepage);
-                if (!is_rmap_pte(*shadow_pte))
+                rmap_count = rmap_add(vcpu, sptep, gfn);
+                if (!is_rmap_spte(*sptep))
                         kvm_release_pfn_clean(pfn);
                 if (rmap_count > RMAP_RECYCLE_THRESHOLD)
-                        rmap_recycle(vcpu, gfn, largepage);
+                        rmap_recycle(vcpu, sptep, gfn);
         } else {
                 if (was_writeble)
                         kvm_release_pfn_dirty(pfn);
@@ -1815,7 +1888,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                         kvm_release_pfn_clean(pfn);
         }
         if (speculative) {
-                vcpu->arch.last_pte_updated = shadow_pte;
+                vcpu->arch.last_pte_updated = sptep;
                 vcpu->arch.last_pte_gfn = gfn;
         }
 }
@@ -1825,7 +1898,7 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
 }
 
 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
-                        int largepage, gfn_t gfn, pfn_t pfn)
+                        int level, gfn_t gfn, pfn_t pfn)
 {
         struct kvm_shadow_walk_iterator iterator;
         struct kvm_mmu_page *sp;
@@ -1833,11 +1906,10 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
         gfn_t pseudo_gfn;
 
         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
-                if (iterator.level == PT_PAGE_TABLE_LEVEL
-                    || (largepage && iterator.level == PT_DIRECTORY_LEVEL)) {
+                if (iterator.level == level) {
                         mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
                                      0, write, 1, &pt_write,
-                                     largepage, gfn, pfn, false);
+                                     level, gfn, pfn, false);
                         ++vcpu->stat.pf_fixed;
                         break;
                 }
@@ -1853,10 +1925,10 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
                                 return -ENOMEM;
                         }
 
-                        set_shadow_pte(iterator.sptep,
-                                       __pa(sp->spt)
-                                       | PT_PRESENT_MASK | PT_WRITABLE_MASK
-                                       | shadow_user_mask | shadow_x_mask);
+                        __set_spte(iterator.sptep,
+                                   __pa(sp->spt)
+                                   | PT_PRESENT_MASK | PT_WRITABLE_MASK
+                                   | shadow_user_mask | shadow_x_mask);
                 }
         }
         return pt_write;
@@ -1865,14 +1937,20 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
 {
         int r;
-        int largepage = 0;
+        int level;
         pfn_t pfn;
         unsigned long mmu_seq;
 
-        if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
-                gfn &= ~(KVM_PAGES_PER_HPAGE-1);
-                largepage = 1;
-        }
+        level = mapping_level(vcpu, gfn);
+
+        /*
+         * This path builds a PAE pagetable - so we can map 2mb pages at
+         * maximum. Therefore check if the level is larger than that.
+         */
+        if (level > PT_DIRECTORY_LEVEL)
+                level = PT_DIRECTORY_LEVEL;
+
+        gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
@@ -1888,7 +1966,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
         if (mmu_notifier_retry(vcpu, mmu_seq))
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
-        r = __direct_map(vcpu, v, write, largepage, gfn, pfn);
+        r = __direct_map(vcpu, v, write, level, gfn, pfn);
         spin_unlock(&vcpu->kvm->mmu_lock);
 
 
@@ -1954,6 +2032,7 @@ static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
         gfn_t root_gfn;
         struct kvm_mmu_page *sp;
         int direct = 0;
+        u64 pdptr;
 
         root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
 
@@ -1981,11 +2060,12 @@ static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
 
                 ASSERT(!VALID_PAGE(root));
                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
-                        if (!is_present_pte(vcpu->arch.pdptrs[i])) {
+                        pdptr = kvm_pdptr_read(vcpu, i);
+                        if (!is_present_gpte(pdptr)) {
                                 vcpu->arch.mmu.pae_root[i] = 0;
                                 continue;
                         }
-                        root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
+                        root_gfn = pdptr >> PAGE_SHIFT;
                 } else if (vcpu->arch.mmu.root_level == 0)
                         root_gfn = 0;
                 if (mmu_check_root(vcpu, root_gfn))
@@ -2062,7 +2142,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
 {
         pfn_t pfn;
         int r;
-        int largepage = 0;
+        int level;
         gfn_t gfn = gpa >> PAGE_SHIFT;
         unsigned long mmu_seq;
 
@@ -2073,10 +2153,10 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
         if (r)
                 return r;
 
-        if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
-                gfn &= ~(KVM_PAGES_PER_HPAGE-1);
-                largepage = 1;
-        }
+        level = mapping_level(vcpu, gfn);
+
+        gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
+
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
         pfn = gfn_to_pfn(vcpu->kvm, gfn);
@@ -2089,7 +2169,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
         r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
-                         largepage, gfn, pfn);
+                         level, gfn, pfn);
         spin_unlock(&vcpu->kvm->mmu_lock);
 
         return r;
@@ -2206,7 +2286,9 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, int level)
                 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
                         rsvd_bits(maxphyaddr, 51);
                 context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
-                context->rsvd_bits_mask[1][2] = context->rsvd_bits_mask[0][2];
+                context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 51) |
+                        rsvd_bits(13, 29);
                 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
                         rsvd_bits(maxphyaddr, 51) |
                         rsvd_bits(13, 20);              /* large page */
@@ -2357,8 +2439,8 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
         spin_unlock(&vcpu->kvm->mmu_lock);
         if (r)
                 goto out;
+        /* set_cr3() should ensure TLB has been flushed */
         kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
-        kvm_mmu_flush_tlb(vcpu);
 out:
         return r;
 }
@@ -2378,15 +2460,14 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
 
         pte = *spte;
         if (is_shadow_present_pte(pte)) {
-                if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
-                    is_large_pte(pte))
+                if (is_last_spte(pte, sp->role.level))
                         rmap_remove(vcpu->kvm, spte);
                 else {
                         child = page_header(pte & PT64_BASE_ADDR_MASK);
                         mmu_page_remove_parent_pte(child, spte);
                 }
         }
-        set_shadow_pte(spte, shadow_trap_nonpresent_pte);
+        __set_spte(spte, shadow_trap_nonpresent_pte);
         if (is_large_pte(pte))
                 --vcpu->kvm->stat.lpages;
 }
@@ -2397,11 +2478,8 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
                                   const void *new)
 {
         if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
-                if (!vcpu->arch.update_pte.largepage ||
-                    sp->role.glevels == PT32_ROOT_LEVEL) {
-                        ++vcpu->kvm->stat.mmu_pde_zapped;
-                        return;
-                }
+                ++vcpu->kvm->stat.mmu_pde_zapped;
+                return;
         }
 
         ++vcpu->kvm->stat.mmu_pte_updated;
@@ -2447,8 +2525,6 @@ static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
         u64 gpte = 0;
         pfn_t pfn;
 
-        vcpu->arch.update_pte.largepage = 0;
-
         if (bytes != 4 && bytes != 8)
                 return;
 
@@ -2472,14 +2548,10 @@ static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
                 if ((bytes == 4) && (gpa % 4 == 0))
                         memcpy((void *)&gpte, new, 4);
         }
-        if (!is_present_pte(gpte))
+        if (!is_present_gpte(gpte))
                 return;
         gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
 
-        if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
-                gfn &= ~(KVM_PAGES_PER_HPAGE-1);
-                vcpu->arch.update_pte.largepage = 1;
-        }
         vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
         pfn = gfn_to_pfn(vcpu->kvm, gfn);
@@ -2622,6 +2694,9 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
         gpa_t gpa;
         int r;
 
+        if (tdp_enabled)
+                return 0;
+
         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
 
         spin_lock(&vcpu->kvm->mmu_lock);
@@ -2633,7 +2708,8 @@ EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
 
 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
 {
-        while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
+        while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES &&
+               !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
                 struct kvm_mmu_page *sp;
 
                 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
@@ -2670,8 +2746,9 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
                 ++vcpu->stat.mmio_exits;
                 return 0;
         case EMULATE_FAIL:
-                kvm_report_emulation_failure(vcpu, "pagetable");
-                return 1;
+                vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+                vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
+                return 0;
         default:
                 BUG();
         }
@@ -2712,12 +2789,6 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
 
         ASSERT(vcpu);
 
-        if (vcpu->kvm->arch.n_requested_mmu_pages)
-                vcpu->kvm->arch.n_free_mmu_pages =
-                                        vcpu->kvm->arch.n_requested_mmu_pages;
-        else
-                vcpu->kvm->arch.n_free_mmu_pages =
-                                        vcpu->kvm->arch.n_alloc_mmu_pages;
         /*
          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
          * Therefore we need to allocate shadow page tables in the first
@@ -3029,6 +3100,24 @@ out:
         return r;
 }
 
+int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
+{
+        struct kvm_shadow_walk_iterator iterator;
+        int nr_sptes = 0;
+
+        spin_lock(&vcpu->kvm->mmu_lock);
+        for_each_shadow_entry(vcpu, addr, iterator) {
+                sptes[iterator.level-1] = *iterator.sptep;
+                nr_sptes++;
+                if (!is_shadow_present_pte(*iterator.sptep))
+                        break;
+        }
+        spin_unlock(&vcpu->kvm->mmu_lock);
+
+        return nr_sptes;
+}
+EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
+
 #ifdef AUDIT
 
 static const char *audit_msg;
@@ -3041,6 +3130,54 @@ static gva_t canonicalize(gva_t gva)
         return gva;
 }
 
+
+typedef void (*inspect_spte_fn) (struct kvm *kvm, struct kvm_mmu_page *sp,
+                                 u64 *sptep);
+
+static void __mmu_spte_walk(struct kvm *kvm, struct kvm_mmu_page *sp,
+                            inspect_spte_fn fn)
+{
+        int i;
+
+        for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
+                u64 ent = sp->spt[i];
+
+                if (is_shadow_present_pte(ent)) {
+                        if (!is_last_spte(ent, sp->role.level)) {
+                                struct kvm_mmu_page *child;
+                                child = page_header(ent & PT64_BASE_ADDR_MASK);
+                                __mmu_spte_walk(kvm, child, fn);
+                        } else
+                                fn(kvm, sp, &sp->spt[i]);
+                }
+        }
+}
+
+static void mmu_spte_walk(struct kvm_vcpu *vcpu, inspect_spte_fn fn)
+{
+        int i;
+        struct kvm_mmu_page *sp;
+
+        if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
+                return;
+        if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
+                hpa_t root = vcpu->arch.mmu.root_hpa;
+                sp = page_header(root);
+                __mmu_spte_walk(vcpu->kvm, sp, fn);
+                return;
+        }
+        for (i = 0; i < 4; ++i) {
+                hpa_t root = vcpu->arch.mmu.pae_root[i];
+
+                if (root && VALID_PAGE(root)) {
+                        root &= PT64_BASE_ADDR_MASK;
+                        sp = page_header(root);
+                        __mmu_spte_walk(vcpu->kvm, sp, fn);
+                }
+        }
+        return;
+}
+
 static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
                                 gva_t va, int level)
 {
@@ -3055,20 +3192,19 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
                         continue;
 
                 va = canonicalize(va);
-                if (level > 1) {
-                        if (ent == shadow_notrap_nonpresent_pte)
-                                printk(KERN_ERR "audit: (%s) nontrapping pte"
-                                       " in nonleaf level: levels %d gva %lx"
-                                       " level %d pte %llx\n", audit_msg,
-                                       vcpu->arch.mmu.root_level, va, level, ent);
-                        else
-                                audit_mappings_page(vcpu, ent, va, level - 1);
-                } else {
+                if (is_shadow_present_pte(ent) && !is_last_spte(ent, level))
+                        audit_mappings_page(vcpu, ent, va, level - 1);
+                else {
                         gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
                         gfn_t gfn = gpa >> PAGE_SHIFT;
                         pfn_t pfn = gfn_to_pfn(vcpu->kvm, gfn);
                         hpa_t hpa = (hpa_t)pfn << PAGE_SHIFT;
 
+                        if (is_error_pfn(pfn)) {
+                                kvm_release_pfn_clean(pfn);
+                                continue;
+                        }
+
                         if (is_shadow_present_pte(ent)
                             && (ent & PT64_BASE_ADDR_MASK) != hpa)
                                 printk(KERN_ERR "xx audit error: (%s) levels %d"
@@ -3122,7 +3258,7 @@ static int count_rmaps(struct kvm_vcpu *vcpu)
                         d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
                         while (d) {
                                 for (k = 0; k < RMAP_EXT; ++k)
-                                        if (d->shadow_ptes[k])
+                                        if (d->sptes[k])
                                                 ++nmaps;
                                         else
                                                 break;
@@ -3133,9 +3269,48 @@ static int count_rmaps(struct kvm_vcpu *vcpu)
         return nmaps;
 }
 
-static int count_writable_mappings(struct kvm_vcpu *vcpu)
+void inspect_spte_has_rmap(struct kvm *kvm, struct kvm_mmu_page *sp, u64 *sptep)
+{
+        unsigned long *rmapp;
+        struct kvm_mmu_page *rev_sp;
+        gfn_t gfn;
+
+        if (*sptep & PT_WRITABLE_MASK) {
+                rev_sp = page_header(__pa(sptep));
+                gfn = rev_sp->gfns[sptep - rev_sp->spt];
+
+                if (!gfn_to_memslot(kvm, gfn)) {
+                        if (!printk_ratelimit())
+                                return;
+                        printk(KERN_ERR "%s: no memslot for gfn %ld\n",
+                                         audit_msg, gfn);
+                        printk(KERN_ERR "%s: index %ld of sp (gfn=%lx)\n",
+                                        audit_msg, sptep - rev_sp->spt,
+                                        rev_sp->gfn);
+                        dump_stack();
+                        return;
+                }
+
+                rmapp = gfn_to_rmap(kvm, rev_sp->gfns[sptep - rev_sp->spt],
+                                    is_large_pte(*sptep));
+                if (!*rmapp) {
+                        if (!printk_ratelimit())
+                                return;
+                        printk(KERN_ERR "%s: no rmap for writable spte %llx\n",
+                                         audit_msg, *sptep);
+                        dump_stack();
+                }
+        }
+
+}
+
+void audit_writable_sptes_have_rmaps(struct kvm_vcpu *vcpu)
+{
+        mmu_spte_walk(vcpu, inspect_spte_has_rmap);
+}
+
+static void check_writable_mappings_rmap(struct kvm_vcpu *vcpu)
 {
-        int nmaps = 0;
         struct kvm_mmu_page *sp;
         int i;
 
@@ -3152,20 +3327,16 @@ static int count_writable_mappings(struct kvm_vcpu *vcpu)
                                 continue;
                         if (!(ent & PT_WRITABLE_MASK))
                                 continue;
-                        ++nmaps;
+                        inspect_spte_has_rmap(vcpu->kvm, sp, &pt[i]);
                 }
         }
-        return nmaps;
+        return;
 }
 
 static void audit_rmap(struct kvm_vcpu *vcpu)
 {
-        int n_rmap = count_rmaps(vcpu);
-        int n_actual = count_writable_mappings(vcpu);
-
-        if (n_rmap != n_actual)
-                printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
-                       __func__, audit_msg, n_rmap, n_actual);
+        check_writable_mappings_rmap(vcpu);
+        count_rmaps(vcpu);
 }
 
 static void audit_write_protection(struct kvm_vcpu *vcpu)
@@ -3173,20 +3344,28 @@ static void audit_write_protection(struct kvm_vcpu *vcpu)
         struct kvm_mmu_page *sp;
         struct kvm_memory_slot *slot;
         unsigned long *rmapp;
+        u64 *spte;
         gfn_t gfn;
 
         list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
                 if (sp->role.direct)
                         continue;
+                if (sp->unsync)
+                        continue;
 
                 gfn = unalias_gfn(vcpu->kvm, sp->gfn);
                 slot = gfn_to_memslot_unaliased(vcpu->kvm, sp->gfn);
                 rmapp = &slot->rmap[gfn - slot->base_gfn];
-                if (*rmapp)
-                        printk(KERN_ERR "%s: (%s) shadow page has writable"
-                               " mappings: gfn %lx role %x\n",
+
+                spte = rmap_next(vcpu->kvm, rmapp, NULL);
+                while (spte) {
+                        if (*spte & PT_WRITABLE_MASK)
+                                printk(KERN_ERR "%s: (%s) shadow page has "
+                                "writable mappings: gfn %lx role %x\n",
                                __func__, audit_msg, sp->gfn,
                                sp->role.word);
+                        spte = rmap_next(vcpu->kvm, rmapp, spte);
+                }
         }
 }
 
@@ -3198,7 +3377,9 @@ static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
         audit_msg = msg;
         audit_rmap(vcpu);
         audit_write_protection(vcpu);
-        audit_mappings(vcpu);
+        if (strcmp("pre pte write", audit_msg) != 0)
+                audit_mappings(vcpu);
+        audit_writable_sptes_have_rmaps(vcpu);
         dbg = olddbg;
 }