13 files changed, 109 insertions, 34 deletions

diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index d7e3299a7734..959e50d2588c 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -363,8 +363,6 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 {
         int ret = 0;
 
-        vcpu_load(vcpu);
-
         trace_kvm_set_guest_debug(vcpu, dbg->control);
 
         if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) {
@@ -386,7 +384,6 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
         }
 
 out:
-        vcpu_put(vcpu);
         return ret;
 }
 
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index cdbd142ca7f2..02924ae2527e 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -360,6 +360,7 @@ void kvm_vgic_put(struct kvm_vcpu *vcpu);
 bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
+void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid);
 
 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg);
 
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index c00c4c33e432..b26eccc78fb1 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -503,6 +503,7 @@
 
 #define ICH_HCR_EN                      (1 << 0)
 #define ICH_HCR_UIE                     (1 << 1)
+#define ICH_HCR_NPIE                    (1 << 3)
 #define ICH_HCR_TC                      (1 << 10)
 #define ICH_HCR_TALL0                   (1 << 11)
 #define ICH_HCR_TALL1                   (1 << 12)
diff --git a/include/linux/irqchip/arm-gic.h b/include/linux/irqchip/arm-gic.h
index d3453ee072fc..68d8b1f73682 100644
--- a/include/linux/irqchip/arm-gic.h
+++ b/include/linux/irqchip/arm-gic.h
@@ -84,6 +84,7 @@
 
 #define GICH_HCR_EN                     (1 << 0)
 #define GICH_HCR_UIE                    (1 << 1)
+#define GICH_HCR_NPIE                   (1 << 3)
 
 #define GICH_LR_VIRTUALID               (0x3ff << 0)
 #define GICH_LR_PHYSID_CPUID_SHIFT      (10)
diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c
index 70f4c30918eb..282389eb204f 100644
--- a/virt/kvm/arm/arch_timer.c
+++ b/virt/kvm/arm/arch_timer.c
@@ -581,6 +581,7 @@ void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 
 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
 {
+        struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 
@@ -594,6 +595,9 @@ int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
         ptimer->cnt_ctl = 0;
         kvm_timer_update_state(vcpu);
 
+        if (timer->enabled && irqchip_in_kernel(vcpu->kvm))
+                kvm_vgic_reset_mapped_irq(vcpu, vtimer->irq.irq);
+
         return 0;
 }
 
@@ -767,7 +771,7 @@ int kvm_timer_hyp_init(bool has_gic)
                 static_branch_enable(&has_gic_active_state);
         }
 
-        kvm_info("virtual timer IRQ%d\n", host_vtimer_irq);
+        kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
 
         cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
                           "kvm/arm/timer:starting", kvm_timer_starting_cpu,
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 86941f6181bb..53572304843b 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -384,14 +384,11 @@ static void vcpu_power_off(struct kvm_vcpu *vcpu)
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
                                     struct kvm_mp_state *mp_state)
 {
-        vcpu_load(vcpu);
-
         if (vcpu->arch.power_off)
                 mp_state->mp_state = KVM_MP_STATE_STOPPED;
         else
                 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
 
-        vcpu_put(vcpu);
         return 0;
 }
 
@@ -400,8 +397,6 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 {
         int ret = 0;
 
-        vcpu_load(vcpu);
-
         switch (mp_state->mp_state) {
         case KVM_MP_STATE_RUNNABLE:
                 vcpu->arch.power_off = false;
@@ -413,7 +408,6 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
                 ret = -EINVAL;
         }
 
-        vcpu_put(vcpu);
         return ret;
 }
 
@@ -1036,8 +1030,6 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
         struct kvm_device_attr attr;
         long r;
 
-        vcpu_load(vcpu);
-
         switch (ioctl) {
         case KVM_ARM_VCPU_INIT: {
                 struct kvm_vcpu_init init;
@@ -1114,7 +1106,6 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
                 r = -EINVAL;
         }
 
-        vcpu_put(vcpu);
         return r;
 }
 
diff --git a/virt/kvm/arm/hyp/vgic-v3-sr.c b/virt/kvm/arm/hyp/vgic-v3-sr.c
index f5c3d6d7019e..b89ce5432214 100644
--- a/virt/kvm/arm/hyp/vgic-v3-sr.c
+++ b/virt/kvm/arm/hyp/vgic-v3-sr.c
@@ -215,7 +215,8 @@ void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu)
          * are now visible to the system register interface.
          */
         if (!cpu_if->vgic_sre) {
-                dsb(st);
+                dsb(sy);
+                isb();
                 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
         }
 
diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c
index ec62d1cccab7..b960acdd0c05 100644
--- a/virt/kvm/arm/mmu.c
+++ b/virt/kvm/arm/mmu.c
@@ -1810,9 +1810,9 @@ int kvm_mmu_init(void)
          */
         BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
 
-        kvm_info("IDMAP page: %lx\n", hyp_idmap_start);
-        kvm_info("HYP VA range: %lx:%lx\n",
-                 kern_hyp_va(PAGE_OFFSET), kern_hyp_va(~0UL));
+        kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);
+        kvm_debug("HYP VA range: %lx:%lx\n",
+                  kern_hyp_va(PAGE_OFFSET), kern_hyp_va(~0UL));
 
         if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) &&
             hyp_idmap_start <  kern_hyp_va(~0UL) &&
diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c
index 83d82bd7dc4e..dbe99d635c80 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.c
+++ b/virt/kvm/arm/vgic/vgic-mmio.c
@@ -113,9 +113,12 @@ unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
         /* Loop over all IRQs affected by this read */
         for (i = 0; i < len * 8; i++) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
+                unsigned long flags;
 
+                spin_lock_irqsave(&irq->irq_lock, flags);
                 if (irq_is_pending(irq))
                         value |= (1U << i);
+                spin_unlock_irqrestore(&irq->irq_lock, flags);
 
                 vgic_put_irq(vcpu->kvm, irq);
         }
diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c
index c32d7b93ffd1..29556f71b691 100644
--- a/virt/kvm/arm/vgic/vgic-v2.c
+++ b/virt/kvm/arm/vgic/vgic-v2.c
@@ -37,6 +37,13 @@ void vgic_v2_init_lrs(void)
                 vgic_v2_write_lr(i, 0);
 }
 
+void vgic_v2_set_npie(struct kvm_vcpu *vcpu)
+{
+        struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
+
+        cpuif->vgic_hcr |= GICH_HCR_NPIE;
+}
+
 void vgic_v2_set_underflow(struct kvm_vcpu *vcpu)
 {
         struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
@@ -64,7 +71,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
         int lr;
         unsigned long flags;
 
-        cpuif->vgic_hcr &= ~GICH_HCR_UIE;
+        cpuif->vgic_hcr &= ~(GICH_HCR_UIE | GICH_HCR_NPIE);
 
         for (lr = 0; lr < vgic_cpu->used_lrs; lr++) {
                 u32 val = cpuif->vgic_lr[lr];
@@ -410,7 +417,7 @@ int vgic_v2_probe(const struct gic_kvm_info *info)
         kvm_vgic_global_state.type = VGIC_V2;
         kvm_vgic_global_state.max_gic_vcpus = VGIC_V2_MAX_CPUS;
 
-        kvm_info("vgic-v2@%llx\n", info->vctrl.start);
+        kvm_debug("vgic-v2@%llx\n", info->vctrl.start);
 
         return 0;
 out:
diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c
index 6b329414e57a..0ff2006f3781 100644
--- a/virt/kvm/arm/vgic/vgic-v3.c
+++ b/virt/kvm/arm/vgic/vgic-v3.c
@@ -26,6 +26,13 @@ static bool group1_trap;
 static bool common_trap;
 static bool gicv4_enable;
 
+void vgic_v3_set_npie(struct kvm_vcpu *vcpu)
+{
+        struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;
+
+        cpuif->vgic_hcr |= ICH_HCR_NPIE;
+}
+
 void vgic_v3_set_underflow(struct kvm_vcpu *vcpu)
 {
         struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;
@@ -47,7 +54,7 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
         int lr;
         unsigned long flags;
 
-        cpuif->vgic_hcr &= ~ICH_HCR_UIE;
+        cpuif->vgic_hcr &= ~(ICH_HCR_UIE | ICH_HCR_NPIE);
 
         for (lr = 0; lr < vgic_cpu->used_lrs; lr++) {
                 u64 val = cpuif->vgic_lr[lr];
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index c7c5ef190afa..8201899126f6 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -495,6 +495,32 @@ int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
         return ret;
 }
 
+/**
+ * kvm_vgic_reset_mapped_irq - Reset a mapped IRQ
+ * @vcpu: The VCPU pointer
+ * @vintid: The INTID of the interrupt
+ *
+ * Reset the active and pending states of a mapped interrupt.  Kernel
+ * subsystems injecting mapped interrupts should reset their interrupt lines
+ * when we are doing a reset of the VM.
+ */
+void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid)
+{
+        struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
+        unsigned long flags;
+
+        if (!irq->hw)
+                goto out;
+
+        spin_lock_irqsave(&irq->irq_lock, flags);
+        irq->active = false;
+        irq->pending_latch = false;
+        irq->line_level = false;
+        spin_unlock_irqrestore(&irq->irq_lock, flags);
+out:
+        vgic_put_irq(vcpu->kvm, irq);
+}
+
 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid)
 {
         struct vgic_irq *irq;
@@ -684,22 +710,37 @@ static inline void vgic_set_underflow(struct kvm_vcpu *vcpu)
                 vgic_v3_set_underflow(vcpu);
 }
 
+static inline void vgic_set_npie(struct kvm_vcpu *vcpu)
+{
+        if (kvm_vgic_global_state.type == VGIC_V2)
+                vgic_v2_set_npie(vcpu);
+        else
+                vgic_v3_set_npie(vcpu);
+}
+
 /* Requires the ap_list_lock to be held. */
-static int compute_ap_list_depth(struct kvm_vcpu *vcpu)
+static int compute_ap_list_depth(struct kvm_vcpu *vcpu,
+                                 bool *multi_sgi)
 {
         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
         struct vgic_irq *irq;
         int count = 0;
 
+        *multi_sgi = false;
+
         DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock));
 
         list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
                 spin_lock(&irq->irq_lock);
                 /* GICv2 SGIs can count for more than one... */
-                if (vgic_irq_is_sgi(irq->intid) && irq->source)
-                        count += hweight8(irq->source);
-                else
+                if (vgic_irq_is_sgi(irq->intid) && irq->source) {
+                        int w = hweight8(irq->source);
+
+                        count += w;
+                        *multi_sgi |= (w > 1);
+                } else {
                         count++;
+                }
                 spin_unlock(&irq->irq_lock);
         }
         return count;
@@ -710,28 +751,43 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
 {
         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
         struct vgic_irq *irq;
-        int count = 0;
+        int count;
+        bool npie = false;
+        bool multi_sgi;
+        u8 prio = 0xff;
 
         DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock));
 
-        if (compute_ap_list_depth(vcpu) > kvm_vgic_global_state.nr_lr)
+        count = compute_ap_list_depth(vcpu, &multi_sgi);
+        if (count > kvm_vgic_global_state.nr_lr || multi_sgi)
                 vgic_sort_ap_list(vcpu);
 
+        count = 0;
+
         list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
                 spin_lock(&irq->irq_lock);
 
-                if (unlikely(vgic_target_oracle(irq) != vcpu))
-                        goto next;
-
                 /*
-                 * If we get an SGI with multiple sources, try to get
-                 * them in all at once.
+                 * If we have multi-SGIs in the pipeline, we need to
+                 * guarantee that they are all seen before any IRQ of
+                 * lower priority. In that case, we need to filter out
+                 * these interrupts by exiting early. This is easy as
+                 * the AP list has been sorted already.
                  */
-                do {
+                if (multi_sgi && irq->priority > prio) {
+                        spin_unlock(&irq->irq_lock);
+                        break;
+                }
+
+                if (likely(vgic_target_oracle(irq) == vcpu)) {
                         vgic_populate_lr(vcpu, irq, count++);
-                } while (irq->source && count < kvm_vgic_global_state.nr_lr);
 
-next:
+                        if (irq->source) {
+                                npie = true;
+                                prio = irq->priority;
+                        }
+                }
+
                 spin_unlock(&irq->irq_lock);
 
                 if (count == kvm_vgic_global_state.nr_lr) {
@@ -742,6 +798,9 @@ next:
                 }
         }
 
+        if (npie)
+                vgic_set_npie(vcpu);
+
         vcpu->arch.vgic_cpu.used_lrs = count;
 
         /* Nuke remaining LRs */
diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h
index 12c37b89f7a3..f5b8519e5546 100644
--- a/virt/kvm/arm/vgic/vgic.h
+++ b/virt/kvm/arm/vgic/vgic.h
@@ -96,6 +96,7 @@
 /* we only support 64 kB translation table page size */
 #define KVM_ITS_L1E_ADDR_MASK           GENMASK_ULL(51, 16)
 
+/* Requires the irq_lock to be held by the caller. */
 static inline bool irq_is_pending(struct vgic_irq *irq)
 {
         if (irq->config == VGIC_CONFIG_EDGE)
@@ -159,6 +160,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu);
 void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
 void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr);
 void vgic_v2_set_underflow(struct kvm_vcpu *vcpu);
+void vgic_v2_set_npie(struct kvm_vcpu *vcpu);
 int vgic_v2_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr);
 int vgic_v2_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
                          int offset, u32 *val);
@@ -188,6 +190,7 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu);
 void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
 void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr);
 void vgic_v3_set_underflow(struct kvm_vcpu *vcpu);
+void vgic_v3_set_npie(struct kvm_vcpu *vcpu);
 void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
 void vgic_v3_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
 void vgic_v3_enable(struct kvm_vcpu *vcpu);