1 files changed, 54 insertions, 61 deletions

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index a119b361b8b7..e5e66e5c6640 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -150,6 +150,20 @@ module_param(dbg, bool, 0644);
 /* make pte_list_desc fit well in cache line */
 #define PTE_LIST_EXT 3
 
+/*
+ * Return values of handle_mmio_page_fault and mmu.page_fault:
+ * RET_PF_RETRY: let CPU fault again on the address.
+ * RET_PF_EMULATE: mmio page fault, emulate the instruction directly.
+ *
+ * For handle_mmio_page_fault only:
+ * RET_PF_INVALID: the spte is invalid, let the real page fault path update it.
+ */
+enum {
+        RET_PF_RETRY = 0,
+        RET_PF_EMULATE = 1,
+        RET_PF_INVALID = 2,
+};
+
 struct pte_list_desc {
         u64 *sptes[PTE_LIST_EXT];
         struct pte_list_desc *more;
@@ -2424,7 +2438,7 @@ static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
 
 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
 {
-        return __shadow_walk_next(iterator, *iterator->sptep);
+        __shadow_walk_next(iterator, *iterator->sptep);
 }
 
 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
@@ -2794,13 +2808,13 @@ done:
         return ret;
 }
 
-static bool mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
-                         int write_fault, int level, gfn_t gfn, kvm_pfn_t pfn,
-                         bool speculative, bool host_writable)
+static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
+                        int write_fault, int level, gfn_t gfn, kvm_pfn_t pfn,
+                        bool speculative, bool host_writable)
 {
         int was_rmapped = 0;
         int rmap_count;
-        bool emulate = false;
+        int ret = RET_PF_RETRY;
 
         pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
                  *sptep, write_fault, gfn);
@@ -2830,12 +2844,12 @@ static bool mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
         if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
               true, host_writable)) {
                 if (write_fault)
-                        emulate = true;
+                        ret = RET_PF_EMULATE;
                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
         }
 
         if (unlikely(is_mmio_spte(*sptep)))
-                emulate = true;
+                ret = RET_PF_EMULATE;
 
         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
         pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
@@ -2855,7 +2869,7 @@ static bool mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
 
         kvm_release_pfn_clean(pfn);
 
-        return emulate;
+        return ret;
 }
 
 static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
@@ -2994,14 +3008,13 @@ static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
          * Do not cache the mmio info caused by writing the readonly gfn
          * into the spte otherwise read access on readonly gfn also can
          * caused mmio page fault and treat it as mmio access.
-         * Return 1 to tell kvm to emulate it.
          */
         if (pfn == KVM_PFN_ERR_RO_FAULT)
-                return 1;
+                return RET_PF_EMULATE;
 
         if (pfn == KVM_PFN_ERR_HWPOISON) {
                 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
-                return 0;
+                return RET_PF_RETRY;
         }
 
         return -EFAULT;
@@ -3286,13 +3299,13 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
         }
 
         if (fast_page_fault(vcpu, v, level, error_code))
-                return 0;
+                return RET_PF_RETRY;
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
 
         if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
-                return 0;
+                return RET_PF_RETRY;
 
         if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
                 return r;
@@ -3312,7 +3325,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
 out_unlock:
         spin_unlock(&vcpu->kvm->mmu_lock);
         kvm_release_pfn_clean(pfn);
-        return 0;
+        return RET_PF_RETRY;
 }
 
 
@@ -3659,54 +3672,38 @@ exit:
         return reserved;
 }
 
-/*
- * Return values of handle_mmio_page_fault:
- * RET_MMIO_PF_EMULATE: it is a real mmio page fault, emulate the instruction
- *                      directly.
- * RET_MMIO_PF_INVALID: invalid spte is detected then let the real page
- *                      fault path update the mmio spte.
- * RET_MMIO_PF_RETRY: let CPU fault again on the address.
- * RET_MMIO_PF_BUG: a bug was detected (and a WARN was printed).
- */
-enum {
-        RET_MMIO_PF_EMULATE = 1,
-        RET_MMIO_PF_INVALID = 2,
-        RET_MMIO_PF_RETRY = 0,
-        RET_MMIO_PF_BUG = -1
-};
-
 static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
 {
         u64 spte;
         bool reserved;
 
         if (mmio_info_in_cache(vcpu, addr, direct))
-                return RET_MMIO_PF_EMULATE;
+                return RET_PF_EMULATE;
 
         reserved = walk_shadow_page_get_mmio_spte(vcpu, addr, &spte);
         if (WARN_ON(reserved))
-                return RET_MMIO_PF_BUG;
+                return -EINVAL;
 
         if (is_mmio_spte(spte)) {
                 gfn_t gfn = get_mmio_spte_gfn(spte);
                 unsigned access = get_mmio_spte_access(spte);
 
                 if (!check_mmio_spte(vcpu, spte))
-                        return RET_MMIO_PF_INVALID;
+                        return RET_PF_INVALID;
 
                 if (direct)
                         addr = 0;
 
                 trace_handle_mmio_page_fault(addr, gfn, access);
                 vcpu_cache_mmio_info(vcpu, addr, gfn, access);
-                return RET_MMIO_PF_EMULATE;
+                return RET_PF_EMULATE;
         }
 
         /*
          * If the page table is zapped by other cpus, let CPU fault again on
          * the address.
          */
-        return RET_MMIO_PF_RETRY;
+        return RET_PF_RETRY;
 }
 EXPORT_SYMBOL_GPL(handle_mmio_page_fault);
 
@@ -3756,7 +3753,7 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
         pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
 
         if (page_fault_handle_page_track(vcpu, error_code, gfn))
-                return 1;
+                return RET_PF_EMULATE;
 
         r = mmu_topup_memory_caches(vcpu);
         if (r)
@@ -3820,8 +3817,7 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
 }
 
 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
-                                u64 fault_address, char *insn, int insn_len,
-                                bool need_unprotect)
+                                u64 fault_address, char *insn, int insn_len)
 {
         int r = 1;
 
@@ -3829,7 +3825,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
         default:
                 trace_kvm_page_fault(fault_address, error_code);
 
-                if (need_unprotect && kvm_event_needs_reinjection(vcpu))
+                if (kvm_event_needs_reinjection(vcpu))
                         kvm_mmu_unprotect_page_virt(vcpu, fault_address);
                 r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
                                 insn_len);
@@ -3876,7 +3872,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
         MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
 
         if (page_fault_handle_page_track(vcpu, error_code, gfn))
-                return 1;
+                return RET_PF_EMULATE;
 
         r = mmu_topup_memory_caches(vcpu);
         if (r)
@@ -3893,13 +3889,13 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
         }
 
         if (fast_page_fault(vcpu, gpa, level, error_code))
-                return 0;
+                return RET_PF_RETRY;
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
 
         if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
-                return 0;
+                return RET_PF_RETRY;
 
         if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
                 return r;
@@ -3919,7 +3915,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
 out_unlock:
         spin_unlock(&vcpu->kvm->mmu_lock);
         kvm_release_pfn_clean(pfn);
-        return 0;
+        return RET_PF_RETRY;
 }
 
 static void nonpaging_init_context(struct kvm_vcpu *vcpu,
@@ -4918,25 +4914,25 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
                 vcpu->arch.gpa_val = cr2;
         }
 
+        r = RET_PF_INVALID;
         if (unlikely(error_code & PFERR_RSVD_MASK)) {
                 r = handle_mmio_page_fault(vcpu, cr2, direct);
-                if (r == RET_MMIO_PF_EMULATE) {
+                if (r == RET_PF_EMULATE) {
                         emulation_type = 0;
                         goto emulate;
                 }
-                if (r == RET_MMIO_PF_RETRY)
-                        return 1;
-                if (r < 0)
-                        return r;
-                /* Must be RET_MMIO_PF_INVALID.  */
         }
 
-        r = vcpu->arch.mmu.page_fault(vcpu, cr2, lower_32_bits(error_code),
-                                      false);
+        if (r == RET_PF_INVALID) {
+                r = vcpu->arch.mmu.page_fault(vcpu, cr2, lower_32_bits(error_code),
+                                              false);
+                WARN_ON(r == RET_PF_INVALID);
+        }
+
+        if (r == RET_PF_RETRY)
+                return 1;
         if (r < 0)
                 return r;
-        if (!r)
-                return 1;
 
         /*
          * Before emulating the instruction, check if the error code
@@ -4993,8 +4989,7 @@ EXPORT_SYMBOL_GPL(kvm_disable_tdp);
 static void free_mmu_pages(struct kvm_vcpu *vcpu)
 {
         free_page((unsigned long)vcpu->arch.mmu.pae_root);
-        if (vcpu->arch.mmu.lm_root != NULL)
-                free_page((unsigned long)vcpu->arch.mmu.lm_root);
+        free_page((unsigned long)vcpu->arch.mmu.lm_root);
 }
 
 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
@@ -5464,10 +5459,8 @@ static struct shrinker mmu_shrinker = {
 
 static void mmu_destroy_caches(void)
 {
-        if (pte_list_desc_cache)
-                kmem_cache_destroy(pte_list_desc_cache);
-        if (mmu_page_header_cache)
-                kmem_cache_destroy(mmu_page_header_cache);
+        kmem_cache_destroy(pte_list_desc_cache);
+        kmem_cache_destroy(mmu_page_header_cache);
 }
 
 int kvm_mmu_module_init(void)
@@ -5476,13 +5469,13 @@ int kvm_mmu_module_init(void)
 
         pte_list_desc_cache = kmem_cache_create("pte_list_desc",
                                             sizeof(struct pte_list_desc),
-                                            0, 0, NULL);
+                                            0, SLAB_ACCOUNT, NULL);
         if (!pte_list_desc_cache)
                 goto nomem;
 
         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
                                                   sizeof(struct kvm_mmu_page),
-                                                  0, 0, NULL);
+                                                  0, SLAB_ACCOUNT, NULL);
         if (!mmu_page_header_cache)
                 goto nomem;