12 files changed, 112 insertions, 23 deletions

diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index d7ea6bcb29bf..8ef05381984b 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -66,6 +66,7 @@ extern char __kvm_hyp_vector[];
 extern void __kvm_flush_vm_context(void);
 extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
 extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
+extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
 
 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 2d19e02d03fd..d5423ab15ed5 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -57,6 +57,9 @@ struct kvm_arch {
         /* VTTBR value associated with below pgd and vmid */
         u64    vttbr;
 
+        /* The last vcpu id that ran on each physical CPU */
+        int __percpu *last_vcpu_ran;
+
         /* Timer */
         struct arch_timer_kvm   timer;
 
diff --git a/arch/arm/include/asm/kvm_hyp.h b/arch/arm/include/asm/kvm_hyp.h
index 343135ede5fa..58508900c4bb 100644
--- a/arch/arm/include/asm/kvm_hyp.h
+++ b/arch/arm/include/asm/kvm_hyp.h
@@ -71,6 +71,7 @@
 #define ICIALLUIS       __ACCESS_CP15(c7, 0, c1, 0)
 #define ATS1CPR         __ACCESS_CP15(c7, 0, c8, 0)
 #define TLBIALLIS       __ACCESS_CP15(c8, 0, c3, 0)
+#define TLBIALL         __ACCESS_CP15(c8, 0, c7, 0)
 #define TLBIALLNSNHIS   __ACCESS_CP15(c8, 4, c3, 4)
 #define PRRR            __ACCESS_CP15(c10, 0, c2, 0)
 #define NMRR            __ACCESS_CP15(c10, 0, c2, 1)
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 08bb84f2ad58..19b5f5c1c0ff 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -114,11 +114,18 @@ void kvm_arch_check_processor_compat(void *rtn)
  */
 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 {
-        int ret = 0;
+        int ret, cpu;
 
         if (type)
                 return -EINVAL;
 
+        kvm->arch.last_vcpu_ran = alloc_percpu(typeof(*kvm->arch.last_vcpu_ran));
+        if (!kvm->arch.last_vcpu_ran)
+                return -ENOMEM;
+
+        for_each_possible_cpu(cpu)
+                *per_cpu_ptr(kvm->arch.last_vcpu_ran, cpu) = -1;
+
         ret = kvm_alloc_stage2_pgd(kvm);
         if (ret)
                 goto out_fail_alloc;
@@ -141,6 +148,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 out_free_stage2_pgd:
         kvm_free_stage2_pgd(kvm);
 out_fail_alloc:
+        free_percpu(kvm->arch.last_vcpu_ran);
+        kvm->arch.last_vcpu_ran = NULL;
         return ret;
 }
 
@@ -168,6 +177,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 {
         int i;
 
+        free_percpu(kvm->arch.last_vcpu_ran);
+        kvm->arch.last_vcpu_ran = NULL;
+
         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
                 if (kvm->vcpus[i]) {
                         kvm_arch_vcpu_free(kvm->vcpus[i]);
@@ -312,6 +324,19 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
+        int *last_ran;
+
+        last_ran = this_cpu_ptr(vcpu->kvm->arch.last_vcpu_ran);
+
+        /*
+         * We might get preempted before the vCPU actually runs, but
+         * over-invalidation doesn't affect correctness.
+         */
+        if (*last_ran != vcpu->vcpu_id) {
+                kvm_call_hyp(__kvm_tlb_flush_local_vmid, vcpu);
+                *last_ran = vcpu->vcpu_id;
+        }
+
         vcpu->cpu = cpu;
         vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
 
diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c
index 729652854f90..6d810af2d9fd 100644
--- a/arch/arm/kvm/hyp/tlb.c
+++ b/arch/arm/kvm/hyp/tlb.c
@@ -55,6 +55,21 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
         __kvm_tlb_flush_vmid(kvm);
 }
 
+void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
+{
+        struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
+
+        /* Switch to requested VMID */
+        write_sysreg(kvm->arch.vttbr, VTTBR);
+        isb();
+
+        write_sysreg(0, TLBIALL);
+        dsb(nsh);
+        isb();
+
+        write_sysreg(0, VTTBR);
+}
+
 void __hyp_text __kvm_flush_vm_context(void)
 {
         write_sysreg(0, TLBIALLNSNHIS);
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 18f746551bf6..ec3553eb9349 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -54,6 +54,7 @@ extern char __kvm_hyp_vector[];
 extern void __kvm_flush_vm_context(void);
 extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
 extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
+extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
 
 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index bd94e6766759..e5050388e062 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -62,6 +62,9 @@ struct kvm_arch {
         /* VTTBR value associated with above pgd and vmid */
         u64    vttbr;
 
+        /* The last vcpu id that ran on each physical CPU */
+        int __percpu *last_vcpu_ran;
+
         /* The maximum number of vCPUs depends on the used GIC model */
         int max_vcpus;
 
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index a79b969c26fc..6f72fe8b0e3e 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -128,7 +128,7 @@ static inline unsigned long __kern_hyp_va(unsigned long v)
         return v;
 }
 
-#define kern_hyp_va(v)  (typeof(v))(__kern_hyp_va((unsigned long)(v)))
+#define kern_hyp_va(v)  ((typeof(v))(__kern_hyp_va((unsigned long)(v))))
 
 /*
  * We currently only support a 40bit IPA.
diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c
index 9cc0ea784ae6..88e2f2b938f0 100644
--- a/arch/arm64/kvm/hyp/tlb.c
+++ b/arch/arm64/kvm/hyp/tlb.c
@@ -64,6 +64,21 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
         write_sysreg(0, vttbr_el2);
 }
 
+void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
+{
+        struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
+
+        /* Switch to requested VMID */
+        write_sysreg(kvm->arch.vttbr, vttbr_el2);
+        isb();
+
+        asm volatile("tlbi vmalle1" : : );
+        dsb(nsh);
+        isb();
+
+        write_sysreg(0, vttbr_el2);
+}
+
 void __hyp_text __kvm_flush_vm_context(void)
 {
         dsb(ishst);
diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c
index e18b30ddcdce..ebe1b9fa3c4d 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.c
+++ b/virt/kvm/arm/vgic/vgic-mmio.c
@@ -453,17 +453,33 @@ struct vgic_io_device *kvm_to_vgic_iodev(const struct kvm_io_device *dev)
         return container_of(dev, struct vgic_io_device, dev);
 }
 
-static bool check_region(const struct vgic_register_region *region,
+static bool check_region(const struct kvm *kvm,
+                         const struct vgic_register_region *region,
                          gpa_t addr, int len)
 {
-        if ((region->access_flags & VGIC_ACCESS_8bit) && len == 1)
-                return true;
-        if ((region->access_flags & VGIC_ACCESS_32bit) &&
-            len == sizeof(u32) && !(addr & 3))
-                return true;
-        if ((region->access_flags & VGIC_ACCESS_64bit) &&
-            len == sizeof(u64) && !(addr & 7))
-                return true;
+        int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
+
+        switch (len) {
+        case sizeof(u8):
+                flags = VGIC_ACCESS_8bit;
+                break;
+        case sizeof(u32):
+                flags = VGIC_ACCESS_32bit;
+                break;
+        case sizeof(u64):
+                flags = VGIC_ACCESS_64bit;
+                break;
+        default:
+                return false;
+        }
+
+        if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) {
+                if (!region->bits_per_irq)
+                        return true;
+
+                /* Do we access a non-allocated IRQ? */
+                return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs;
+        }
 
         return false;
 }
@@ -477,7 +493,7 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 
         region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
                                        addr - iodev->base_addr);
-        if (!region || !check_region(region, addr, len)) {
+        if (!region || !check_region(vcpu->kvm, region, addr, len)) {
                 memset(val, 0, len);
                 return 0;
         }
@@ -510,10 +526,7 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 
         region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
                                        addr - iodev->base_addr);
-        if (!region)
-                return 0;
-
-        if (!check_region(region, addr, len))
+        if (!region || !check_region(vcpu->kvm, region, addr, len))
                 return 0;
 
         switch (iodev->iodev_type) {
diff --git a/virt/kvm/arm/vgic/vgic-mmio.h b/virt/kvm/arm/vgic/vgic-mmio.h
index 4c34d39d44a0..84961b4e4422 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.h
+++ b/virt/kvm/arm/vgic/vgic-mmio.h
@@ -50,15 +50,15 @@ extern struct kvm_io_device_ops kvm_io_gic_ops;
 #define VGIC_ADDR_IRQ_MASK(bits) (((bits) * 1024 / 8) - 1)
 
 /*
- * (addr & mask) gives us the byte offset for the INT ID, so we want to
- * divide this with 'bytes per irq' to get the INT ID, which is given
- * by '(bits) / 8'.  But we do this with fixed-point-arithmetic and
- * take advantage of the fact that division by a fraction equals
- * multiplication with the inverted fraction, and scale up both the
- * numerator and denominator with 8 to support at most 64 bits per IRQ:
+ * (addr & mask) gives us the _byte_ offset for the INT ID.
+ * We multiply this by 8 the get the _bit_ offset, then divide this by
+ * the number of bits to learn the actual INT ID.
+ * But instead of a division (which requires a "long long div" implementation),
+ * we shift by the binary logarithm of <bits>.
+ * This assumes that <bits> is a power of two.
  */
 #define VGIC_ADDR_TO_INTID(addr, bits)  (((addr) & VGIC_ADDR_IRQ_MASK(bits)) * \
-                                        64 / (bits) / 8)
+                                        8 >> ilog2(bits))
 
 /*
  * Some VGIC registers store per-IRQ information, with a different number
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index 2893d5ba523a..6440b56ec90e 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -273,6 +273,18 @@ retry:
                  * no more work for us to do.
                  */
                 spin_unlock(&irq->irq_lock);
+
+                /*
+                 * We have to kick the VCPU here, because we could be
+                 * queueing an edge-triggered interrupt for which we
+                 * get no EOI maintenance interrupt. In that case,
+                 * while the IRQ is already on the VCPU's AP list, the
+                 * VCPU could have EOI'ed the original interrupt and
+                 * won't see this one until it exits for some other
+                 * reason.
+                 */
+                if (vcpu)
+                        kvm_vcpu_kick(vcpu);
                 return false;
         }