Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull KVM updates from Paolo Bonzini:
 "s390:
   - ioctl hardening
   - selftests

  ARM:
   - ITS translation cache
   - support for 512 vCPUs
   - various cleanups and bugfixes

  PPC:
   - various minor fixes and preparation

  x86:
   - bugfixes all over the place (posted interrupts, SVM, emulation
     corner cases, blocked INIT)
   - some IPI optimizations"

* tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (75 commits)
  KVM: X86: Use IPI shorthands in kvm guest when support
  KVM: x86: Fix INIT signal handling in various CPU states
  KVM: VMX: Introduce exit reason for receiving INIT signal on guest-mode
  KVM: VMX: Stop the preemption timer during vCPU reset
  KVM: LAPIC: Micro optimize IPI latency
  kvm: Nested KVM MMUs need PAE root too
  KVM: x86: set ctxt->have_exception in x86_decode_insn()
  KVM: x86: always stop emulation on page fault
  KVM: nVMX: trace nested VM-Enter failures detected by H/W
  KVM: nVMX: add tracepoint for failed nested VM-Enter
  x86: KVM: svm: Fix a check in nested_svm_vmrun()
  KVM: x86: Return to userspace with internal error on unexpected exit reason
  KVM: x86: Add kvm_emulate_{rd,wr}msr() to consolidate VXM/SVM code
  KVM: x86: Refactor up kvm_{g,s}et_msr() to simplify callers
  doc: kvm: Fix return description of KVM_SET_MSRS
  KVM: X86: Tune PLE Window tracepoint
  KVM: VMX: Change ple_window type to unsigned int
  KVM: X86: Remove tailing newline for tracepoints
  KVM: X86: Trace vcpu_id for vmexit
  KVM: x86: Manually calculate reserved bits when loading PDPTRS
  ...

9 files changed, 302 insertions, 81 deletions

diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 35a069815baf..86c6aa1cb58e 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -196,6 +196,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
         case KVM_CAP_MP_STATE:
         case KVM_CAP_IMMEDIATE_EXIT:
         case KVM_CAP_VCPU_EVENTS:
+        case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
                 r = 1;
                 break;
         case KVM_CAP_ARM_SET_DEVICE_ADDR:
@@ -888,6 +889,7 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
 
         irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
         vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
+        vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
         irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
 
         trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c
index e621b5d45b27..6f50c429196d 100644
--- a/virt/kvm/arm/vgic/vgic-init.c
+++ b/virt/kvm/arm/vgic/vgic-init.c
@@ -54,6 +54,7 @@ void kvm_vgic_early_init(struct kvm *kvm)
         struct vgic_dist *dist = &kvm->arch.vgic;
 
         INIT_LIST_HEAD(&dist->lpi_list_head);
+        INIT_LIST_HEAD(&dist->lpi_translation_cache);
         raw_spin_lock_init(&dist->lpi_list_lock);
 }
 
@@ -199,7 +200,6 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
         int i;
 
         vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
-        vgic_cpu->sgi_iodev.base_addr = VGIC_ADDR_UNDEF;
 
         INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
         raw_spin_lock_init(&vgic_cpu->ap_list_lock);
@@ -304,6 +304,7 @@ int vgic_init(struct kvm *kvm)
         }
 
         if (vgic_has_its(kvm)) {
+                vgic_lpi_translation_cache_init(kvm);
                 ret = vgic_v4_init(kvm);
                 if (ret)
                         goto out;
@@ -345,6 +346,9 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)
                 INIT_LIST_HEAD(&dist->rd_regions);
         }
 
+        if (vgic_has_its(kvm))
+                vgic_lpi_translation_cache_destroy(kvm);
+
         if (vgic_supports_direct_msis(kvm))
                 vgic_v4_teardown(kvm);
 }
@@ -515,7 +519,7 @@ int kvm_vgic_hyp_init(void)
                 break;
         default:
                 ret = -ENODEV;
-        };
+        }
 
         if (ret)
                 return ret;
diff --git a/virt/kvm/arm/vgic/vgic-irqfd.c b/virt/kvm/arm/vgic/vgic-irqfd.c
index c9304b88e720..d8cdfea5cc96 100644
--- a/virt/kvm/arm/vgic/vgic-irqfd.c
+++ b/virt/kvm/arm/vgic/vgic-irqfd.c
@@ -66,6 +66,15 @@ out:
         return r;
 }
 
+static void kvm_populate_msi(struct kvm_kernel_irq_routing_entry *e,
+                             struct kvm_msi *msi)
+{
+        msi->address_lo = e->msi.address_lo;
+        msi->address_hi = e->msi.address_hi;
+        msi->data = e->msi.data;
+        msi->flags = e->msi.flags;
+        msi->devid = e->msi.devid;
+}
 /**
  * kvm_set_msi: inject the MSI corresponding to the
  * MSI routing entry
@@ -79,21 +88,36 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
 {
         struct kvm_msi msi;
 
-        msi.address_lo = e->msi.address_lo;
-        msi.address_hi = e->msi.address_hi;
-        msi.data = e->msi.data;
-        msi.flags = e->msi.flags;
-        msi.devid = e->msi.devid;
-
         if (!vgic_has_its(kvm))
                 return -ENODEV;
 
         if (!level)
                 return -1;
 
+        kvm_populate_msi(e, &msi);
         return vgic_its_inject_msi(kvm, &msi);
 }
 
+/**
+ * kvm_arch_set_irq_inatomic: fast-path for irqfd injection
+ *
+ * Currently only direct MSI injection is supported.
+ */
+int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
+                              struct kvm *kvm, int irq_source_id, int level,
+                              bool line_status)
+{
+        if (e->type == KVM_IRQ_ROUTING_MSI && vgic_has_its(kvm) && level) {
+                struct kvm_msi msi;
+
+                kvm_populate_msi(e, &msi);
+                if (!vgic_its_inject_cached_translation(kvm, &msi))
+                        return 0;
+        }
+
+        return -EWOULDBLOCK;
+}
+
 int kvm_vgic_setup_default_irq_routing(struct kvm *kvm)
 {
         struct kvm_irq_routing_entry *entries;
diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c
index 482036612adf..2be6b66b3856 100644
--- a/virt/kvm/arm/vgic/vgic-its.c
+++ b/virt/kvm/arm/vgic/vgic-its.c
@@ -138,6 +138,14 @@ struct its_ite {
         u32 event_id;
 };
 
+struct vgic_translation_cache_entry {
+        struct list_head        entry;
+        phys_addr_t             db;
+        u32                     devid;
+        u32                     eventid;
+        struct vgic_irq         *irq;
+};
+
 /**
  * struct vgic_its_abi - ITS abi ops and settings
  * @cte_esz: collection table entry size
@@ -527,6 +535,127 @@ static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
         return 0;
 }
 
+static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist,
+                                               phys_addr_t db,
+                                               u32 devid, u32 eventid)
+{
+        struct vgic_translation_cache_entry *cte;
+
+        list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
+                /*
+                 * If we hit a NULL entry, there is nothing after this
+                 * point.
+                 */
+                if (!cte->irq)
+                        break;
+
+                if (cte->db != db || cte->devid != devid ||
+                    cte->eventid != eventid)
+                        continue;
+
+                /*
+                 * Move this entry to the head, as it is the most
+                 * recently used.
+                 */
+                if (!list_is_first(&cte->entry, &dist->lpi_translation_cache))
+                        list_move(&cte->entry, &dist->lpi_translation_cache);
+
+                return cte->irq;
+        }
+
+        return NULL;
+}
+
+static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db,
+                                             u32 devid, u32 eventid)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        struct vgic_irq *irq;
+        unsigned long flags;
+
+        raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
+        irq = __vgic_its_check_cache(dist, db, devid, eventid);
+        raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+
+        return irq;
+}
+
+static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
+                                       u32 devid, u32 eventid,
+                                       struct vgic_irq *irq)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        struct vgic_translation_cache_entry *cte;
+        unsigned long flags;
+        phys_addr_t db;
+
+        /* Do not cache a directly injected interrupt */
+        if (irq->hw)
+                return;
+
+        raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
+
+        if (unlikely(list_empty(&dist->lpi_translation_cache)))
+                goto out;
+
+        /*
+         * We could have raced with another CPU caching the same
+         * translation behind our back, so let's check it is not in
+         * already
+         */
+        db = its->vgic_its_base + GITS_TRANSLATER;
+        if (__vgic_its_check_cache(dist, db, devid, eventid))
+                goto out;
+
+        /* Always reuse the last entry (LRU policy) */
+        cte = list_last_entry(&dist->lpi_translation_cache,
+                              typeof(*cte), entry);
+
+        /*
+         * Caching the translation implies having an extra reference
+         * to the interrupt, so drop the potential reference on what
+         * was in the cache, and increment it on the new interrupt.
+         */
+        if (cte->irq)
+                __vgic_put_lpi_locked(kvm, cte->irq);
+
+        vgic_get_irq_kref(irq);
+
+        cte->db         = db;
+        cte->devid      = devid;
+        cte->eventid    = eventid;
+        cte->irq        = irq;
+
+        /* Move the new translation to the head of the list */
+        list_move(&cte->entry, &dist->lpi_translation_cache);
+
+out:
+        raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+}
+
+void vgic_its_invalidate_cache(struct kvm *kvm)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        struct vgic_translation_cache_entry *cte;
+        unsigned long flags;
+
+        raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
+
+        list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
+                /*
+                 * If we hit a NULL entry, there is nothing after this
+                 * point.
+                 */
+                if (!cte->irq)
+                        break;
+
+                __vgic_put_lpi_locked(kvm, cte->irq);
+                cte->irq = NULL;
+        }
+
+        raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+}
+
 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
                          u32 devid, u32 eventid, struct vgic_irq **irq)
 {
@@ -547,6 +676,8 @@ int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
         if (!vcpu->arch.vgic_cpu.lpis_enabled)
                 return -EBUSY;
 
+        vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
+
         *irq = ite->irq;
         return 0;
 }
@@ -608,6 +739,25 @@ static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
         return 0;
 }
 
+int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi)
+{
+        struct vgic_irq *irq;
+        unsigned long flags;
+        phys_addr_t db;
+
+        db = (u64)msi->address_hi << 32 | msi->address_lo;
+        irq = vgic_its_check_cache(kvm, db, msi->devid, msi->data);
+
+        if (!irq)
+                return -1;
+
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
+        irq->pending_latch = true;
+        vgic_queue_irq_unlock(kvm, irq, flags);
+
+        return 0;
+}
+
 /*
  * Queries the KVM IO bus framework to get the ITS pointer from the given
  * doorbell address.
@@ -619,6 +769,9 @@ int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
         struct vgic_its *its;
         int ret;
 
+        if (!vgic_its_inject_cached_translation(kvm, msi))
+                return 1;
+
         its = vgic_msi_to_its(kvm, msi);
         if (IS_ERR(its))
                 return PTR_ERR(its);
@@ -691,6 +844,8 @@ static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
                  * don't bother here since we clear the ITTE anyway and the
                  * pending state is a property of the ITTE struct.
                  */
+                vgic_its_invalidate_cache(kvm);
+
                 its_free_ite(kvm, ite);
                 return 0;
         }
@@ -726,6 +881,8 @@ static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
         ite->collection = collection;
         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 
+        vgic_its_invalidate_cache(kvm);
+
         return update_affinity(ite->irq, vcpu);
 }
 
@@ -954,6 +1111,8 @@ static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)
         list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
                 its_free_ite(kvm, ite);
 
+        vgic_its_invalidate_cache(kvm);
+
         list_del(&device->dev_list);
         kfree(device);
 }
@@ -1059,6 +1218,7 @@ static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
 
         if (!valid) {
                 vgic_its_free_collection(its, coll_id);
+                vgic_its_invalidate_cache(kvm);
         } else {
                 collection = find_collection(its, coll_id);
 
@@ -1207,6 +1367,8 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
                 vgic_put_irq(kvm, irq);
         }
 
+        vgic_its_invalidate_cache(kvm);
+
         kfree(intids);
         return 0;
 }
@@ -1557,6 +1719,8 @@ static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
                 goto out;
 
         its->enabled = !!(val & GITS_CTLR_ENABLE);
+        if (!its->enabled)
+                vgic_its_invalidate_cache(kvm);
 
         /*
          * Try to process any pending commands. This function bails out early
@@ -1657,6 +1821,47 @@ out:
         return ret;
 }
 
+/* Default is 16 cached LPIs per vcpu */
+#define LPI_DEFAULT_PCPU_CACHE_SIZE     16
+
+void vgic_lpi_translation_cache_init(struct kvm *kvm)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        unsigned int sz;
+        int i;
+
+        if (!list_empty(&dist->lpi_translation_cache))
+                return;
+
+        sz = atomic_read(&kvm->online_vcpus) * LPI_DEFAULT_PCPU_CACHE_SIZE;
+
+        for (i = 0; i < sz; i++) {
+                struct vgic_translation_cache_entry *cte;
+
+                /* An allocation failure is not fatal */
+                cte = kzalloc(sizeof(*cte), GFP_KERNEL);
+                if (WARN_ON(!cte))
+                        break;
+
+                INIT_LIST_HEAD(&cte->entry);
+                list_add(&cte->entry, &dist->lpi_translation_cache);
+        }
+}
+
+void vgic_lpi_translation_cache_destroy(struct kvm *kvm)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        struct vgic_translation_cache_entry *cte, *tmp;
+
+        vgic_its_invalidate_cache(kvm);
+
+        list_for_each_entry_safe(cte, tmp,
+                                 &dist->lpi_translation_cache, entry) {
+                list_del(&cte->entry);
+                kfree(cte);
+        }
+}
+
 #define INITIAL_BASER_VALUE                                               \
         (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)                | \
          GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)         | \
@@ -1685,6 +1890,8 @@ static int vgic_its_create(struct kvm_device *dev, u32 type)
                         kfree(its);
                         return ret;
                 }
+
+                vgic_lpi_translation_cache_init(dev->kvm);
         }
 
         mutex_init(&its->its_lock);
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c
index c45e2d7e942f..7dfd15dbb308 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v3.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c
@@ -192,8 +192,10 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
 
         vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS;
 
-        if (was_enabled && !vgic_cpu->lpis_enabled)
+        if (was_enabled && !vgic_cpu->lpis_enabled) {
                 vgic_flush_pending_lpis(vcpu);
+                vgic_its_invalidate_cache(vcpu->kvm);
+        }
 
         if (!was_enabled && vgic_cpu->lpis_enabled)
                 vgic_enable_lpis(vcpu);
@@ -515,7 +517,8 @@ static const struct vgic_register_region vgic_v3_dist_registers[] = {
                 VGIC_ACCESS_32bit),
 };
 
-static const struct vgic_register_region vgic_v3_rdbase_registers[] = {
+static const struct vgic_register_region vgic_v3_rd_registers[] = {
+        /* RD_base registers */
         REGISTER_DESC_WITH_LENGTH(GICR_CTLR,
                 vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,
                 VGIC_ACCESS_32bit),
@@ -540,44 +543,42 @@ static const struct vgic_register_region vgic_v3_rdbase_registers[] = {
         REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
                 vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
                 VGIC_ACCESS_32bit),
-};
-
-static const struct vgic_register_region vgic_v3_sgibase_registers[] = {
-        REGISTER_DESC_WITH_LENGTH(GICR_IGROUPR0,
+        /* SGI_base registers */
+        REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0,
                 vgic_mmio_read_group, vgic_mmio_write_group, 4,
                 VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH(GICR_ISENABLER0,
+        REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ISENABLER0,
                 vgic_mmio_read_enable, vgic_mmio_write_senable, 4,
                 VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH(GICR_ICENABLER0,
+        REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICENABLER0,
                 vgic_mmio_read_enable, vgic_mmio_write_cenable, 4,
                 VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ISPENDR0,
+        REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0,
                 vgic_mmio_read_pending, vgic_mmio_write_spending,
                 vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4,
                 VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ICPENDR0,
+        REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0,
                 vgic_mmio_read_pending, vgic_mmio_write_cpending,
                 vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4,
                 VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ISACTIVER0,
+        REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0,
                 vgic_mmio_read_active, vgic_mmio_write_sactive,
                 NULL, vgic_mmio_uaccess_write_sactive,
                 4, VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ICACTIVER0,
+        REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0,
                 vgic_mmio_read_active, vgic_mmio_write_cactive,
                 NULL, vgic_mmio_uaccess_write_cactive,
                 4, VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH(GICR_IPRIORITYR0,
+        REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0,
                 vgic_mmio_read_priority, vgic_mmio_write_priority, 32,
                 VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
-        REGISTER_DESC_WITH_LENGTH(GICR_ICFGR0,
+        REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0,
                 vgic_mmio_read_config, vgic_mmio_write_config, 8,
                 VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH(GICR_IGRPMODR0,
+        REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0,
                 vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
                 VGIC_ACCESS_32bit),
-        REGISTER_DESC_WITH_LENGTH(GICR_NSACR,
+        REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR,
                 vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
                 VGIC_ACCESS_32bit),
 };
@@ -607,9 +608,8 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
         struct vgic_dist *vgic = &kvm->arch.vgic;
         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
         struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
-        struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev;
         struct vgic_redist_region *rdreg;
-        gpa_t rd_base, sgi_base;
+        gpa_t rd_base;
         int ret;
 
         if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))
@@ -631,52 +631,31 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
         vgic_cpu->rdreg = rdreg;
 
         rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;
-        sgi_base = rd_base + SZ_64K;
 
         kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
         rd_dev->base_addr = rd_base;
         rd_dev->iodev_type = IODEV_REDIST;
-        rd_dev->regions = vgic_v3_rdbase_registers;
-        rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers);
+        rd_dev->regions = vgic_v3_rd_registers;
+        rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
         rd_dev->redist_vcpu = vcpu;
 
         mutex_lock(&kvm->slots_lock);
         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
-                                      SZ_64K, &rd_dev->dev);
+                                      2 * SZ_64K, &rd_dev->dev);
         mutex_unlock(&kvm->slots_lock);
 
         if (ret)
                 return ret;
 
-        kvm_iodevice_init(&sgi_dev->dev, &kvm_io_gic_ops);
-        sgi_dev->base_addr = sgi_base;
-        sgi_dev->iodev_type = IODEV_REDIST;
-        sgi_dev->regions = vgic_v3_sgibase_registers;
-        sgi_dev->nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers);
-        sgi_dev->redist_vcpu = vcpu;
-
-        mutex_lock(&kvm->slots_lock);
-        ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, sgi_base,
-                                      SZ_64K, &sgi_dev->dev);
-        if (ret) {
-                kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
-                                          &rd_dev->dev);
-                goto out;
-        }
-
         rdreg->free_index++;
-out:
-        mutex_unlock(&kvm->slots_lock);
-        return ret;
+        return 0;
 }
 
 static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)
 {
         struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
-        struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev;
 
         kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
-        kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &sgi_dev->dev);
 }
 
 static int vgic_register_all_redist_iodevs(struct kvm *kvm)
@@ -826,8 +805,8 @@ int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
                 iodev.base_addr = 0;
                 break;
         case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{
-                iodev.regions = vgic_v3_rdbase_registers;
-                iodev.nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers);
+                iodev.regions = vgic_v3_rd_registers;
+                iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
                 iodev.base_addr = 0;
                 break;
         }
@@ -985,21 +964,11 @@ int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
                            int offset, u32 *val)
 {
         struct vgic_io_device rd_dev = {
-                .regions = vgic_v3_rdbase_registers,
-                .nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers),
+                .regions = vgic_v3_rd_registers,
+                .nr_regions = ARRAY_SIZE(vgic_v3_rd_registers),
         };
 
-        struct vgic_io_device sgi_dev = {
-                .regions = vgic_v3_sgibase_registers,
-                .nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers),
-        };
-
-        /* SGI_base is the next 64K frame after RD_base */
-        if (offset >= SZ_64K)
-                return vgic_uaccess(vcpu, &sgi_dev, is_write, offset - SZ_64K,
-                                    val);
-        else
-                return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
+        return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
 }
 
 int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c
index b00aa304c260..621cc168fe3f 100644
--- a/virt/kvm/arm/vgic/vgic-v2.c
+++ b/virt/kvm/arm/vgic/vgic-v2.c
@@ -357,10 +357,11 @@ out:
 DEFINE_STATIC_KEY_FALSE(vgic_v2_cpuif_trap);
 
 /**
- * vgic_v2_probe - probe for a GICv2 compatible interrupt controller in DT
- * @node:       pointer to the DT node
+ * vgic_v2_probe - probe for a VGICv2 compatible interrupt controller
+ * @info:       pointer to the GIC description
  *
- * Returns 0 if a GICv2 has been found, returns an error code otherwise
+ * Returns 0 if the VGICv2 has been probed successfully, returns an error code
+ * otherwise
  */
 int vgic_v2_probe(const struct gic_kvm_info *info)
 {
diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c
index a4ad431c92a9..8d69f007dd0c 100644
--- a/virt/kvm/arm/vgic/vgic-v3.c
+++ b/virt/kvm/arm/vgic/vgic-v3.c
@@ -573,10 +573,11 @@ static int __init early_gicv4_enable(char *buf)
 early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable);
 
 /**
- * vgic_v3_probe - probe for a GICv3 compatible interrupt controller in DT
- * @node:       pointer to the DT node
+ * vgic_v3_probe - probe for a VGICv3 compatible interrupt controller
+ * @info:       pointer to the GIC description
  *
- * Returns 0 if a GICv3 has been found, returns an error code otherwise
+ * Returns 0 if the VGICv3 has been probed successfully, returns an error code
+ * otherwise
  */
 int vgic_v3_probe(const struct gic_kvm_info *info)
 {
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index e7bde65ba67c..45a870cb63f5 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -119,6 +119,22 @@ static void vgic_irq_release(struct kref *ref)
 {
 }
 
+/*
+ * Drop the refcount on the LPI. Must be called with lpi_list_lock held.
+ */
+void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+
+        if (!kref_put(&irq->refcount, vgic_irq_release))
+                return;
+
+        list_del(&irq->lpi_list);
+        dist->lpi_list_count--;
+
+        kfree(irq);
+}
+
 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
 {
         struct vgic_dist *dist = &kvm->arch.vgic;
@@ -128,16 +144,8 @@ void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
                 return;
 
         raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
-        if (!kref_put(&irq->refcount, vgic_irq_release)) {
-                raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
-                return;
-        };
-
-        list_del(&irq->lpi_list);
-        dist->lpi_list_count--;
+        __vgic_put_lpi_locked(kvm, irq);
         raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
-
-        kfree(irq);
 }
 
 void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu)
diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h
index 797e05004d80..83066a81b16a 100644
--- a/virt/kvm/arm/vgic/vgic.h
+++ b/virt/kvm/arm/vgic/vgic.h
@@ -161,6 +161,7 @@ vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
                      gpa_t addr, int len);
 struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
                               u32 intid);
+void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq);
 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq);
 bool vgic_get_phys_line_level(struct vgic_irq *irq);
 void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending);
@@ -307,6 +308,10 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr);
 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
                          u32 devid, u32 eventid, struct vgic_irq **irq);
 struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi);
+int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi);
+void vgic_lpi_translation_cache_init(struct kvm *kvm);
+void vgic_lpi_translation_cache_destroy(struct kvm *kvm);
+void vgic_its_invalidate_cache(struct kvm *kvm);
 
 bool vgic_supports_direct_msis(struct kvm *kvm);
 int vgic_v4_init(struct kvm *kvm);