1 files changed, 92 insertions, 7 deletions

diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 57a403a5c22b..3756dd3e85c2 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -611,6 +611,71 @@ static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
         unmap_range(kvm, kvm->arch.pgd, start, size);
 }
 
+static void stage2_unmap_memslot(struct kvm *kvm,
+                                 struct kvm_memory_slot *memslot)
+{
+        hva_t hva = memslot->userspace_addr;
+        phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
+        phys_addr_t size = PAGE_SIZE * memslot->npages;
+        hva_t reg_end = hva + size;
+
+        /*
+         * A memory region could potentially cover multiple VMAs, and any holes
+         * between them, so iterate over all of them to find out if we should
+         * unmap any of them.
+         *
+         *     +--------------------------------------------+
+         * +---------------+----------------+   +----------------+
+         * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
+         * +---------------+----------------+   +----------------+
+         *     |               memory region                |
+         *     +--------------------------------------------+
+         */
+        do {
+                struct vm_area_struct *vma = find_vma(current->mm, hva);
+                hva_t vm_start, vm_end;
+
+                if (!vma || vma->vm_start >= reg_end)
+                        break;
+
+                /*
+                 * Take the intersection of this VMA with the memory region
+                 */
+                vm_start = max(hva, vma->vm_start);
+                vm_end = min(reg_end, vma->vm_end);
+
+                if (!(vma->vm_flags & VM_PFNMAP)) {
+                        gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
+                        unmap_stage2_range(kvm, gpa, vm_end - vm_start);
+                }
+                hva = vm_end;
+        } while (hva < reg_end);
+}
+
+/**
+ * stage2_unmap_vm - Unmap Stage-2 RAM mappings
+ * @kvm: The struct kvm pointer
+ *
+ * Go through the memregions and unmap any reguler RAM
+ * backing memory already mapped to the VM.
+ */
+void stage2_unmap_vm(struct kvm *kvm)
+{
+        struct kvm_memslots *slots;
+        struct kvm_memory_slot *memslot;
+        int idx;
+
+        idx = srcu_read_lock(&kvm->srcu);
+        spin_lock(&kvm->mmu_lock);
+
+        slots = kvm_memslots(kvm);
+        kvm_for_each_memslot(memslot, slots)
+                stage2_unmap_memslot(kvm, memslot);
+
+        spin_unlock(&kvm->mmu_lock);
+        srcu_read_unlock(&kvm->srcu, idx);
+}
+
 /**
  * kvm_free_stage2_pgd - free all stage-2 tables
  * @kvm:        The KVM struct pointer for the VM.
@@ -834,6 +899,11 @@ static bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
         return kvm_vcpu_dabt_iswrite(vcpu);
 }
 
+static bool kvm_is_device_pfn(unsigned long pfn)
+{
+        return !pfn_valid(pfn);
+}
+
 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                           struct kvm_memory_slot *memslot, unsigned long hva,
                           unsigned long fault_status)
@@ -847,6 +917,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
         struct vm_area_struct *vma;
         pfn_t pfn;
         pgprot_t mem_type = PAGE_S2;
+        bool fault_ipa_uncached;
 
         write_fault = kvm_is_write_fault(vcpu);
         if (fault_status == FSC_PERM && !write_fault) {
@@ -904,7 +975,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
         if (is_error_pfn(pfn))
                 return -EFAULT;
 
-        if (kvm_is_mmio_pfn(pfn))
+        if (kvm_is_device_pfn(pfn))
                 mem_type = PAGE_S2_DEVICE;
 
         spin_lock(&kvm->mmu_lock);
@@ -913,6 +984,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
         if (!hugetlb && !force_pte)
                 hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa);
 
+        fault_ipa_uncached = memslot->flags & KVM_MEMSLOT_INCOHERENT;
+
         if (hugetlb) {
                 pmd_t new_pmd = pfn_pmd(pfn, mem_type);
                 new_pmd = pmd_mkhuge(new_pmd);
@@ -920,7 +993,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                         kvm_set_s2pmd_writable(&new_pmd);
                         kvm_set_pfn_dirty(pfn);
                 }
-                coherent_cache_guest_page(vcpu, hva & PMD_MASK, PMD_SIZE);
+                coherent_cache_guest_page(vcpu, hva & PMD_MASK, PMD_SIZE,
+                                          fault_ipa_uncached);
                 ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
         } else {
                 pte_t new_pte = pfn_pte(pfn, mem_type);
@@ -928,7 +1002,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                         kvm_set_s2pte_writable(&new_pte);
                         kvm_set_pfn_dirty(pfn);
                 }
-                coherent_cache_guest_page(vcpu, hva, PAGE_SIZE);
+                coherent_cache_guest_page(vcpu, hva, PAGE_SIZE,
+                                          fault_ipa_uncached);
                 ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte,
                         pgprot_val(mem_type) == pgprot_val(PAGE_S2_DEVICE));
         }
@@ -1288,11 +1363,12 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
                 hva = vm_end;
         } while (hva < reg_end);
 
-        if (ret) {
-                spin_lock(&kvm->mmu_lock);
+        spin_lock(&kvm->mmu_lock);
+        if (ret)
                 unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size);
-                spin_unlock(&kvm->mmu_lock);
-        }
+        else
+                stage2_flush_memslot(kvm, memslot);
+        spin_unlock(&kvm->mmu_lock);
         return ret;
 }
 
@@ -1304,6 +1380,15 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
                             unsigned long npages)
 {
+        /*
+         * Readonly memslots are not incoherent with the caches by definition,
+         * but in practice, they are used mostly to emulate ROMs or NOR flashes
+         * that the guest may consider devices and hence map as uncached.
+         * To prevent incoherency issues in these cases, tag all readonly
+         * regions as incoherent.
+         */
+        if (slot->flags & KVM_MEM_READONLY)
+                slot->flags |= KVM_MEMSLOT_INCOHERENT;
         return 0;
 }