15 files changed, 330 insertions, 178 deletions

diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 66ce17655bb9..7b0152321b20 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -38,6 +38,16 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
         vcpu->arch.hcr = HCR_GUEST_MASK;
 }
 
+static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.hcr;
+}
+
+static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr)
+{
+        vcpu->arch.hcr = hcr;
+}
+
 static inline bool vcpu_mode_is_32bit(struct kvm_vcpu *vcpu)
 {
         return 1;
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 254e0650e48b..04b4ea0b550a 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -125,9 +125,6 @@ struct kvm_vcpu_arch {
          * Anything that is not used directly from assembly code goes
          * here.
          */
-        /* dcache set/way operation pending */
-        int last_pcpu;
-        cpumask_t require_dcache_flush;
 
         /* Don't run the guest on this vcpu */
         bool pause;
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 63e0ecc04901..1bca8f8af442 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -44,6 +44,7 @@
 
 #ifndef __ASSEMBLY__
 
+#include <linux/highmem.h>
 #include <asm/cacheflush.h>
 #include <asm/pgalloc.h>
 
@@ -161,13 +162,10 @@ static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
         return (vcpu->arch.cp15[c1_SCTLR] & 0b101) == 0b101;
 }
 
-static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
-                                             unsigned long size,
-                                             bool ipa_uncached)
+static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu, pfn_t pfn,
+                                               unsigned long size,
+                                               bool ipa_uncached)
 {
-        if (!vcpu_has_cache_enabled(vcpu) || ipa_uncached)
-                kvm_flush_dcache_to_poc((void *)hva, size);
-        
         /*
          * If we are going to insert an instruction page and the icache is
          * either VIPT or PIPT, there is a potential problem where the host
@@ -179,18 +177,77 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
          *
          * VIVT caches are tagged using both the ASID and the VMID and doesn't
          * need any kind of flushing (DDI 0406C.b - Page B3-1392).
+         *
+         * We need to do this through a kernel mapping (using the
+         * user-space mapping has proved to be the wrong
+         * solution). For that, we need to kmap one page at a time,
+         * and iterate over the range.
          */
-        if (icache_is_pipt()) {
-                __cpuc_coherent_user_range(hva, hva + size);
-        } else if (!icache_is_vivt_asid_tagged()) {
+
+        bool need_flush = !vcpu_has_cache_enabled(vcpu) || ipa_uncached;
+
+        VM_BUG_ON(size & PAGE_MASK);
+
+        if (!need_flush && !icache_is_pipt())
+                goto vipt_cache;
+
+        while (size) {
+                void *va = kmap_atomic_pfn(pfn);
+
+                if (need_flush)
+                        kvm_flush_dcache_to_poc(va, PAGE_SIZE);
+
+                if (icache_is_pipt())
+                        __cpuc_coherent_user_range((unsigned long)va,
+                                                   (unsigned long)va + PAGE_SIZE);
+
+                size -= PAGE_SIZE;
+                pfn++;
+
+                kunmap_atomic(va);
+        }
+
+vipt_cache:
+        if (!icache_is_pipt() && !icache_is_vivt_asid_tagged()) {
                 /* any kind of VIPT cache */
                 __flush_icache_all();
         }
 }
 
+static inline void __kvm_flush_dcache_pte(pte_t pte)
+{
+        void *va = kmap_atomic(pte_page(pte));
+
+        kvm_flush_dcache_to_poc(va, PAGE_SIZE);
+
+        kunmap_atomic(va);
+}
+
+static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
+{
+        unsigned long size = PMD_SIZE;
+        pfn_t pfn = pmd_pfn(pmd);
+
+        while (size) {
+                void *va = kmap_atomic_pfn(pfn);
+
+                kvm_flush_dcache_to_poc(va, PAGE_SIZE);
+
+                pfn++;
+                size -= PAGE_SIZE;
+
+                kunmap_atomic(va);
+        }
+}
+
+static inline void __kvm_flush_dcache_pud(pud_t pud)
+{
+}
+
 #define kvm_virt_to_phys(x)             virt_to_idmap((unsigned long)(x))
 
-void stage2_flush_vm(struct kvm *kvm);
+void kvm_set_way_flush(struct kvm_vcpu *vcpu);
+void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
 
 #endif  /* !__ASSEMBLY__ */
 
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 2d6d91001062..0b0d58a905c4 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -281,15 +281,6 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
         vcpu->cpu = cpu;
         vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
 
-        /*
-         * Check whether this vcpu requires the cache to be flushed on
-         * this physical CPU. This is a consequence of doing dcache
-         * operations by set/way on this vcpu. We do it here to be in
-         * a non-preemptible section.
-         */
-        if (cpumask_test_and_clear_cpu(cpu, &vcpu->arch.require_dcache_flush))
-                flush_cache_all(); /* We'd really want v7_flush_dcache_all() */
-
         kvm_arm_set_running_vcpu(vcpu);
 }
 
@@ -541,7 +532,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
                 ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);