Merge tag 'kvm-arm-for-3.20' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into kvm-next

KVM/ARM changes for v3.20 including GICv3 emulation, dirty page logging, added
trace symbols, and adding an explicit VGIC init device control IOCTL.

Conflicts:
	arch/arm64/include/asm/kvm_arm.h
	arch/arm64/kvm/handle_exit.c

50 files changed, 3152 insertions, 996 deletions

diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index 0007fef4ed81..f4b19d78782b 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -612,11 +612,14 @@ Type: vm ioctl
 Parameters: none
 Returns: 0 on success, -1 on error
 
-Creates an interrupt controller model in the kernel.  On x86, creates a virtual
-ioapic, a virtual PIC (two PICs, nested), and sets up future vcpus to have a
-local APIC.  IRQ routing for GSIs 0-15 is set to both PIC and IOAPIC; GSI 16-23
-only go to the IOAPIC.  On ARM/arm64, a GIC is
-created. On s390, a dummy irq routing table is created.
+Creates an interrupt controller model in the kernel.
+On x86, creates a virtual ioapic, a virtual PIC (two PICs, nested), and sets up
+future vcpus to have a local APIC.  IRQ routing for GSIs 0-15 is set to both
+PIC and IOAPIC; GSI 16-23 only go to the IOAPIC.
+On ARM/arm64, a GICv2 is created. Any other GIC versions require the usage of
+KVM_CREATE_DEVICE, which also supports creating a GICv2.  Using
+KVM_CREATE_DEVICE is preferred over KVM_CREATE_IRQCHIP for GICv2.
+On s390, a dummy irq routing table is created.
 
 Note that on s390 the KVM_CAP_S390_IRQCHIP vm capability needs to be enabled
 before KVM_CREATE_IRQCHIP can be used.
diff --git a/Documentation/virtual/kvm/devices/arm-vgic.txt b/Documentation/virtual/kvm/devices/arm-vgic.txt
index df8b0c7540b6..3fb905429e8a 100644
--- a/Documentation/virtual/kvm/devices/arm-vgic.txt
+++ b/Documentation/virtual/kvm/devices/arm-vgic.txt
@@ -3,22 +3,42 @@ ARM Virtual Generic Interrupt Controller (VGIC)
 
 Device types supported:
   KVM_DEV_TYPE_ARM_VGIC_V2     ARM Generic Interrupt Controller v2.0
+  KVM_DEV_TYPE_ARM_VGIC_V3     ARM Generic Interrupt Controller v3.0
 
 Only one VGIC instance may be instantiated through either this API or the
 legacy KVM_CREATE_IRQCHIP api.  The created VGIC will act as the VM interrupt
 controller, requiring emulated user-space devices to inject interrupts to the
 VGIC instead of directly to CPUs.
 
+Creating a guest GICv3 device requires a host GICv3 as well.
+GICv3 implementations with hardware compatibility support allow a guest GICv2
+as well.
+
 Groups:
   KVM_DEV_ARM_VGIC_GRP_ADDR
   Attributes:
     KVM_VGIC_V2_ADDR_TYPE_DIST (rw, 64-bit)
       Base address in the guest physical address space of the GIC distributor
-      register mappings.
+      register mappings. Only valid for KVM_DEV_TYPE_ARM_VGIC_V2.
+      This address needs to be 4K aligned and the region covers 4 KByte.
 
     KVM_VGIC_V2_ADDR_TYPE_CPU (rw, 64-bit)
       Base address in the guest physical address space of the GIC virtual cpu
-      interface register mappings.
+      interface register mappings. Only valid for KVM_DEV_TYPE_ARM_VGIC_V2.
+      This address needs to be 4K aligned and the region covers 4 KByte.
+
+    KVM_VGIC_V3_ADDR_TYPE_DIST (rw, 64-bit)
+      Base address in the guest physical address space of the GICv3 distributor
+      register mappings. Only valid for KVM_DEV_TYPE_ARM_VGIC_V3.
+      This address needs to be 64K aligned and the region covers 64 KByte.
+
+    KVM_VGIC_V3_ADDR_TYPE_REDIST (rw, 64-bit)
+      Base address in the guest physical address space of the GICv3
+      redistributor register mappings. There are two 64K pages for each
+      VCPU and all of the redistributor pages are contiguous.
+      Only valid for KVM_DEV_TYPE_ARM_VGIC_V3.
+      This address needs to be 64K aligned.
+
 
   KVM_DEV_ARM_VGIC_GRP_DIST_REGS
   Attributes:
@@ -36,6 +56,7 @@ Groups:
     the register.
   Limitations:
     - Priorities are not implemented, and registers are RAZ/WI
+    - Currently only implemented for KVM_DEV_TYPE_ARM_VGIC_V2.
   Errors:
     -ENODEV: Getting or setting this register is not yet supported
     -EBUSY: One or more VCPUs are running
@@ -68,6 +89,7 @@ Groups:
 
   Limitations:
     - Priorities are not implemented, and registers are RAZ/WI
+    - Currently only implemented for KVM_DEV_TYPE_ARM_VGIC_V2.
   Errors:
     -ENODEV: Getting or setting this register is not yet supported
     -EBUSY: One or more VCPUs are running
@@ -81,3 +103,14 @@ Groups:
     -EINVAL: Value set is out of the expected range
     -EBUSY: Value has already be set, or GIC has already been initialized
             with default values.
+
+  KVM_DEV_ARM_VGIC_GRP_CTRL
+  Attributes:
+    KVM_DEV_ARM_VGIC_CTRL_INIT
+      request the initialization of the VGIC, no additional parameter in
+      kvm_device_attr.addr.
+  Errors:
+    -ENXIO: VGIC not properly configured as required prior to calling
+     this attribute
+    -ENODEV: no online VCPU
+    -ENOMEM: memory shortage when allocating vgic internal data
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 3a67bec72d0c..25410b2d8bc1 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -96,6 +96,7 @@ extern char __kvm_hyp_code_end[];
 
 extern void __kvm_flush_vm_context(void);
 extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
 
 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 #endif
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 66ce17655bb9..c52861577567 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -23,6 +23,7 @@
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmio.h>
 #include <asm/kvm_arm.h>
+#include <asm/cputype.h>
 
 unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num);
 unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu);
@@ -167,9 +168,9 @@ static inline u32 kvm_vcpu_hvc_get_imm(struct kvm_vcpu *vcpu)
         return kvm_vcpu_get_hsr(vcpu) & HSR_HVC_IMM_MASK;
 }
 
-static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu)
+static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
 {
-        return vcpu->arch.cp15[c0_MPIDR];
+        return vcpu->arch.cp15[c0_MPIDR] & MPIDR_HWID_BITMASK;
 }
 
 static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 254e0650e48b..bde494654bcc 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -68,6 +68,7 @@ struct kvm_arch {
 
         /* Interrupt controller */
         struct vgic_dist        vgic;
+        int max_vcpus;
 };
 
 #define KVM_NR_MEM_OBJS     40
@@ -234,6 +235,10 @@ static inline void vgic_arch_setup(const struct vgic_params *vgic)
 int kvm_perf_init(void);
 int kvm_perf_teardown(void);
 
+void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
+
+struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
+
 static inline void kvm_arch_hardware_disable(void) {}
 static inline void kvm_arch_hardware_unsetup(void) {}
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
diff --git a/arch/arm/include/asm/kvm_mmio.h b/arch/arm/include/asm/kvm_mmio.h
index adcc0d7d3175..3f83db2f6cf0 100644
--- a/arch/arm/include/asm/kvm_mmio.h
+++ b/arch/arm/include/asm/kvm_mmio.h
@@ -37,6 +37,7 @@ struct kvm_exit_mmio {
         u8              data[8];
         u32             len;
         bool            is_write;
+        void            *private;
 };
 
 static inline void kvm_prepare_mmio(struct kvm_run *run,
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 63e0ecc04901..2672cf84afd1 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -114,6 +114,27 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
         pmd_val(*pmd) |= L_PMD_S2_RDWR;
 }
 
+static inline void kvm_set_s2pte_readonly(pte_t *pte)
+{
+        pte_val(*pte) = (pte_val(*pte) & ~L_PTE_S2_RDWR) | L_PTE_S2_RDONLY;
+}
+
+static inline bool kvm_s2pte_readonly(pte_t *pte)
+{
+        return (pte_val(*pte) & L_PTE_S2_RDWR) == L_PTE_S2_RDONLY;
+}
+
+static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
+{
+        pmd_val(*pmd) = (pmd_val(*pmd) & ~L_PMD_S2_RDWR) | L_PMD_S2_RDONLY;
+}
+
+static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
+{
+        return (pmd_val(*pmd) & L_PMD_S2_RDWR) == L_PMD_S2_RDONLY;
+}
+
+
 /* Open coded p*d_addr_end that can deal with 64bit addresses */
 #define kvm_pgd_addr_end(addr, end)                                     \
 ({      u64 __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK;            \
diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
index a31ecdad4b59..ae1d30a1aaae 100644
--- a/arch/arm/include/asm/pgtable-3level.h
+++ b/arch/arm/include/asm/pgtable-3level.h
@@ -130,6 +130,7 @@
 #define L_PTE_S2_RDONLY                 (_AT(pteval_t, 1) << 6)   /* HAP[1]   */
 #define L_PTE_S2_RDWR                   (_AT(pteval_t, 3) << 6)   /* HAP[2:1] */
 
+#define L_PMD_S2_RDONLY                 (_AT(pmdval_t, 1) << 6)   /* HAP[1]   */
 #define L_PMD_S2_RDWR                   (_AT(pmdval_t, 3) << 6)   /* HAP[2:1] */
 
 /*
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index 09ee408c1a67..0db25bc32864 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -175,6 +175,8 @@ struct kvm_arch_memory_slot {
 #define   KVM_DEV_ARM_VGIC_OFFSET_SHIFT 0
 #define   KVM_DEV_ARM_VGIC_OFFSET_MASK  (0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT)
 #define KVM_DEV_ARM_VGIC_GRP_NR_IRQS    3
+#define KVM_DEV_ARM_VGIC_GRP_CTRL       4
+#define   KVM_DEV_ARM_VGIC_CTRL_INIT    0
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_TYPE_SHIFT          24
diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig
index 466bd299b1a8..a8d1ace3ea51 100644
--- a/arch/arm/kvm/Kconfig
+++ b/arch/arm/kvm/Kconfig
@@ -21,8 +21,10 @@ config KVM
         select PREEMPT_NOTIFIERS
         select ANON_INODES
         select HAVE_KVM_CPU_RELAX_INTERCEPT
+        select HAVE_KVM_ARCH_TLB_FLUSH_ALL
         select KVM_MMIO
         select KVM_ARM_HOST
+        select KVM_GENERIC_DIRTYLOG_READ_PROTECT
         depends on ARM_VIRT_EXT && ARM_LPAE
         ---help---
           Support hosting virtualized guest machines. You will also
diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile
index f7057ed045b6..443b8bea43e9 100644
--- a/arch/arm/kvm/Makefile
+++ b/arch/arm/kvm/Makefile
@@ -22,4 +22,5 @@ obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o
 obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o
 obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
 obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
+obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o
 obj-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 2d6d91001062..6fbfa5fff05d 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -132,6 +132,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
         /* Mark the initial VMID generation invalid */
         kvm->arch.vmid_gen = 0;
 
+        /* The maximum number of VCPUs is limited by the host's GIC model */
+        kvm->arch.max_vcpus = kvm_vgic_get_max_vcpus();
+
         return ret;
 out_free_stage2_pgd:
         kvm_free_stage2_pgd(kvm);
@@ -218,6 +221,11 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
                 goto out;
         }
 
+        if (id >= kvm->arch.max_vcpus) {
+                err = -EINVAL;
+                goto out;
+        }
+
         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
         if (!vcpu) {
                 err = -ENOMEM;
@@ -787,9 +795,39 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
         }
 }
 
+/**
+ * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
+ * @kvm: kvm instance
+ * @log: slot id and address to which we copy the log
+ *
+ * Steps 1-4 below provide general overview of dirty page logging. See
+ * kvm_get_dirty_log_protect() function description for additional details.
+ *
+ * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
+ * always flush the TLB (step 4) even if previous step failed  and the dirty
+ * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
+ * does not preclude user space subsequent dirty log read. Flushing TLB ensures
+ * writes will be marked dirty for next log read.
+ *
+ *   1. Take a snapshot of the bit and clear it if needed.
+ *   2. Write protect the corresponding page.
+ *   3. Copy the snapshot to the userspace.
+ *   4. Flush TLB's if needed.
+ */
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
 {
-        return -EINVAL;
+        bool is_dirty = false;
+        int r;
+
+        mutex_lock(&kvm->slots_lock);
+
+        r = kvm_get_dirty_log_protect(kvm, log, &is_dirty);
+
+        if (is_dirty)
+                kvm_flush_remote_tlbs(kvm);
+
+        mutex_unlock(&kvm->slots_lock);
+        return r;
 }
 
 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
@@ -821,7 +859,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
         switch (ioctl) {
         case KVM_CREATE_IRQCHIP: {
                 if (vgic_present)
-                        return kvm_vgic_create(kvm);
+                        return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
                 else
                         return -ENXIO;
         }
@@ -1045,6 +1083,19 @@ static void check_kvm_target_cpu(void *ret)
         *(int *)ret = kvm_target_cpu();
 }
 
+struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
+{
+        struct kvm_vcpu *vcpu;
+        int i;
+
+        mpidr &= MPIDR_HWID_BITMASK;
+        kvm_for_each_vcpu(i, vcpu, kvm) {
+                if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
+                        return vcpu;
+        }
+        return NULL;
+}
+
 /**
  * Initialize Hyp-mode and memory mappings on all CPUs.
  */
diff --git a/arch/arm/kvm/handle_exit.c b/arch/arm/kvm/handle_exit.c
index a96a8043277c..95f12b2ccdcb 100644
--- a/arch/arm/kvm/handle_exit.c
+++ b/arch/arm/kvm/handle_exit.c
@@ -87,11 +87,13 @@ static int handle_dabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run)
  */
 static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
-        trace_kvm_wfi(*vcpu_pc(vcpu));
-        if (kvm_vcpu_get_hsr(vcpu) & HSR_WFI_IS_WFE)
+        if (kvm_vcpu_get_hsr(vcpu) & HSR_WFI_IS_WFE) {
+                trace_kvm_wfx(*vcpu_pc(vcpu), true);
                 kvm_vcpu_on_spin(vcpu);
-        else
+        } else {
+                trace_kvm_wfx(*vcpu_pc(vcpu), false);
                 kvm_vcpu_block(vcpu);
+        }
 
         kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 
diff --git a/arch/arm/kvm/interrupts.S b/arch/arm/kvm/interrupts.S
index 01dcb0e752d9..79caf79b304a 100644
--- a/arch/arm/kvm/interrupts.S
+++ b/arch/arm/kvm/interrupts.S
@@ -66,6 +66,17 @@ ENTRY(__kvm_tlb_flush_vmid_ipa)
         bx      lr
 ENDPROC(__kvm_tlb_flush_vmid_ipa)
 
+/**
+ * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
+ *
+ * Reuses __kvm_tlb_flush_vmid_ipa() for ARMv7, without passing address
+ * parameter
+ */
+
+ENTRY(__kvm_tlb_flush_vmid)
+        b       __kvm_tlb_flush_vmid_ipa
+ENDPROC(__kvm_tlb_flush_vmid)
+
 /********************************************************************
  * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
  * domain, for all VMIDs
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 1dc9778a00af..74aeabaa3c4d 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -45,6 +45,26 @@ static phys_addr_t hyp_idmap_vector;
 #define hyp_pgd_order get_order(PTRS_PER_PGD * sizeof(pgd_t))
 
 #define kvm_pmd_huge(_x)        (pmd_huge(_x) || pmd_trans_huge(_x))
+#define kvm_pud_huge(_x)        pud_huge(_x)
+
+#define KVM_S2PTE_FLAG_IS_IOMAP         (1UL << 0)
+#define KVM_S2_FLAG_LOGGING_ACTIVE      (1UL << 1)
+
+static bool memslot_is_logging(struct kvm_memory_slot *memslot)
+{
+        return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY);
+}
+
+/**
+ * kvm_flush_remote_tlbs() - flush all VM TLB entries for v7/8
+ * @kvm:        pointer to kvm structure.
+ *
+ * Interface to HYP function to flush all VM TLB entries
+ */
+void kvm_flush_remote_tlbs(struct kvm *kvm)
+{
+        kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
+}
 
 static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
 {
@@ -58,6 +78,25 @@ static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
                 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);
 }
 
+/**
+ * stage2_dissolve_pmd() - clear and flush huge PMD entry
+ * @kvm:        pointer to kvm structure.
+ * @addr:       IPA
+ * @pmd:        pmd pointer for IPA
+ *
+ * Function clears a PMD entry, flushes addr 1st and 2nd stage TLBs. Marks all
+ * pages in the range dirty.
+ */
+static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd)
+{
+        if (!kvm_pmd_huge(*pmd))
+                return;
+
+        pmd_clear(pmd);
+        kvm_tlb_flush_vmid_ipa(kvm, addr);
+        put_page(virt_to_page(pmd));
+}
+
 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
                                   int min, int max)
 {
@@ -767,10 +806,15 @@ static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
 }
 
 static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
-                          phys_addr_t addr, const pte_t *new_pte, bool iomap)
+                          phys_addr_t addr, const pte_t *new_pte,
+                          unsigned long flags)
 {
         pmd_t *pmd;
         pte_t *pte, old_pte;
+        bool iomap = flags & KVM_S2PTE_FLAG_IS_IOMAP;
+        bool logging_active = flags & KVM_S2_FLAG_LOGGING_ACTIVE;
+
+        VM_BUG_ON(logging_active && !cache);
 
         /* Create stage-2 page table mapping - Levels 0 and 1 */
         pmd = stage2_get_pmd(kvm, cache, addr);
@@ -782,6 +826,13 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
                 return 0;
         }
 
+        /*
+         * While dirty page logging - dissolve huge PMD, then continue on to
+         * allocate page.
+         */
+        if (logging_active)
+                stage2_dissolve_pmd(kvm, addr, pmd);
+
         /* Create stage-2 page mappings - Level 2 */
         if (pmd_none(*pmd)) {
                 if (!cache)
@@ -838,7 +889,8 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
                 if (ret)
                         goto out;
                 spin_lock(&kvm->mmu_lock);
-                ret = stage2_set_pte(kvm, &cache, addr, &pte, true);
+                ret = stage2_set_pte(kvm, &cache, addr, &pte,
+                                                KVM_S2PTE_FLAG_IS_IOMAP);
                 spin_unlock(&kvm->mmu_lock);
                 if (ret)
                         goto out;
@@ -905,6 +957,151 @@ static bool kvm_is_device_pfn(unsigned long pfn)
         return !pfn_valid(pfn);
 }
 
+/**
+ * stage2_wp_ptes - write protect PMD range
+ * @pmd:        pointer to pmd entry
+ * @addr:       range start address
+ * @end:        range end address
+ */
+static void stage2_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end)
+{
+        pte_t *pte;
+
+        pte = pte_offset_kernel(pmd, addr);
+        do {
+                if (!pte_none(*pte)) {
+                        if (!kvm_s2pte_readonly(pte))
+                                kvm_set_s2pte_readonly(pte);
+                }
+        } while (pte++, addr += PAGE_SIZE, addr != end);
+}
+
+/**
+ * stage2_wp_pmds - write protect PUD range
+ * @pud:        pointer to pud entry
+ * @addr:       range start address
+ * @end:        range end address
+ */
+static void stage2_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end)
+{
+        pmd_t *pmd;
+        phys_addr_t next;
+
+        pmd = pmd_offset(pud, addr);
+
+        do {
+                next = kvm_pmd_addr_end(addr, end);
+                if (!pmd_none(*pmd)) {
+                        if (kvm_pmd_huge(*pmd)) {
+                                if (!kvm_s2pmd_readonly(pmd))
+                                        kvm_set_s2pmd_readonly(pmd);
+                        } else {
+                                stage2_wp_ptes(pmd, addr, next);
+                        }
+                }
+        } while (pmd++, addr = next, addr != end);
+}
+
+/**
+  * stage2_wp_puds - write protect PGD range
+  * @pgd:       pointer to pgd entry
+  * @addr:      range start address
+  * @end:       range end address
+  *
+  * Process PUD entries, for a huge PUD we cause a panic.
+  */
+static void  stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
+{
+        pud_t *pud;
+        phys_addr_t next;
+
+        pud = pud_offset(pgd, addr);
+        do {
+                next = kvm_pud_addr_end(addr, end);
+                if (!pud_none(*pud)) {
+                        /* TODO:PUD not supported, revisit later if supported */
+                        BUG_ON(kvm_pud_huge(*pud));
+                        stage2_wp_pmds(pud, addr, next);
+                }
+        } while (pud++, addr = next, addr != end);
+}
+
+/**
+ * stage2_wp_range() - write protect stage2 memory region range
+ * @kvm:        The KVM pointer
+ * @addr:       Start address of range
+ * @end:        End address of range
+ */
+static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
+{
+        pgd_t *pgd;
+        phys_addr_t next;
+
+        pgd = kvm->arch.pgd + pgd_index(addr);
+        do {
+                /*
+                 * Release kvm_mmu_lock periodically if the memory region is
+                 * large. Otherwise, we may see kernel panics with
+                 * CONFIG_DETECT_HUNG_TASK, CONFIG_LOCKUP_DETECTOR,
+                 * CONFIG_LOCKDEP. Additionally, holding the lock too long
+                 * will also starve other vCPUs.
+                 */
+                if (need_resched() || spin_needbreak(&kvm->mmu_lock))
+                        cond_resched_lock(&kvm->mmu_lock);
+
+                next = kvm_pgd_addr_end(addr, end);
+                if (pgd_present(*pgd))
+                        stage2_wp_puds(pgd, addr, next);
+        } while (pgd++, addr = next, addr != end);
+}
+
+/**
+ * kvm_mmu_wp_memory_region() - write protect stage 2 entries for memory slot
+ * @kvm:        The KVM pointer
+ * @slot:       The memory slot to write protect
+ *
+ * Called to start logging dirty pages after memory region
+ * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns
+ * all present PMD and PTEs are write protected in the memory region.
+ * Afterwards read of dirty page log can be called.
+ *
+ * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired,
+ * serializing operations for VM memory regions.
+ */
+void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
+{
+        struct kvm_memory_slot *memslot = id_to_memslot(kvm->memslots, slot);
+        phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
+        phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
+
+        spin_lock(&kvm->mmu_lock);
+        stage2_wp_range(kvm, start, end);
+        spin_unlock(&kvm->mmu_lock);
+        kvm_flush_remote_tlbs(kvm);
+}
+
+/**
+ * kvm_arch_mmu_write_protect_pt_masked() - write protect dirty pages
+ * @kvm:        The KVM pointer
+ * @slot:       The memory slot associated with mask
+ * @gfn_offset: The gfn offset in memory slot
+ * @mask:       The mask of dirty pages at offset 'gfn_offset' in this memory
+ *              slot to be write protected
+ *
+ * Walks bits set in mask write protects the associated pte's. Caller must
+ * acquire kvm_mmu_lock.
+ */
+void kvm_arch_mmu_write_protect_pt_masked(struct kvm *kvm,
+                struct kvm_memory_slot *slot,
+                gfn_t gfn_offset, unsigned long mask)
+{
+        phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
+        phys_addr_t start = (base_gfn +  __ffs(mask)) << PAGE_SHIFT;
+        phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
+
+        stage2_wp_range(kvm, start, end);
+}
+
 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                           struct kvm_memory_slot *memslot, unsigned long hva,
                           unsigned long fault_status)
@@ -919,6 +1116,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
         pfn_t pfn;
         pgprot_t mem_type = PAGE_S2;
         bool fault_ipa_uncached;
+        bool logging_active = memslot_is_logging(memslot);
+        unsigned long flags = 0;
 
         write_fault = kvm_is_write_fault(vcpu);
         if (fault_status == FSC_PERM && !write_fault) {
@@ -935,7 +1134,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                 return -EFAULT;
         }
 
-        if (is_vm_hugetlb_page(vma)) {
+        if (is_vm_hugetlb_page(vma) && !logging_active) {
                 hugetlb = true;
                 gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
         } else {
@@ -976,12 +1175,30 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
         if (is_error_pfn(pfn))
                 return -EFAULT;
 
-        if (kvm_is_device_pfn(pfn))
+        if (kvm_is_device_pfn(pfn)) {
                 mem_type = PAGE_S2_DEVICE;
+                flags |= KVM_S2PTE_FLAG_IS_IOMAP;
+        } else if (logging_active) {
+                /*
+                 * Faults on pages in a memslot with logging enabled
+                 * should not be mapped with huge pages (it introduces churn
+                 * and performance degradation), so force a pte mapping.
+                 */
+                force_pte = true;
+                flags |= KVM_S2_FLAG_LOGGING_ACTIVE;
+
+                /*
+                 * Only actually map the page as writable if this was a write
+                 * fault.
+                 */
+                if (!write_fault)
+                        writable = false;
+        }
 
         spin_lock(&kvm->mmu_lock);
         if (mmu_notifier_retry(kvm, mmu_seq))
                 goto out_unlock;
+
         if (!hugetlb && !force_pte)
                 hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa);
 
@@ -999,17 +1216,17 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                 ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
         } else {
                 pte_t new_pte = pfn_pte(pfn, mem_type);
+
                 if (writable) {
                         kvm_set_s2pte_writable(&new_pte);
                         kvm_set_pfn_dirty(pfn);
+                        mark_page_dirty(kvm, gfn);
                 }
                 coherent_cache_guest_page(vcpu, hva, PAGE_SIZE,
                                           fault_ipa_uncached);
-                ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte,
-                        pgprot_val(mem_type) == pgprot_val(PAGE_S2_DEVICE));
+                ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags);
         }
 
-
 out_unlock:
         spin_unlock(&kvm->mmu_lock);
         kvm_release_pfn_clean(pfn);
@@ -1159,7 +1376,14 @@ static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
 {
         pte_t *pte = (pte_t *)data;
 
-        stage2_set_pte(kvm, NULL, gpa, pte, false);
+        /*
+         * We can always call stage2_set_pte with KVM_S2PTE_FLAG_LOGGING_ACTIVE
+         * flag clear because MMU notifiers will have unmapped a huge PMD before
+         * calling ->change_pte() (which in turn calls kvm_set_spte_hva()) and
+         * therefore stage2_set_pte() never needs to clear out a huge PMD
+         * through this calling path.
+         */
+        stage2_set_pte(kvm, NULL, gpa, pte, 0);
 }
 
 
@@ -1292,6 +1516,13 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
                                    const struct kvm_memory_slot *old,
                                    enum kvm_mr_change change)
 {
+        /*
+         * At this point memslot has been committed and there is an
+         * allocated dirty_bitmap[], dirty pages will be be tracked while the
+         * memory slot is write protected.
+         */
+        if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES)
+                kvm_mmu_wp_memory_region(kvm, mem->slot);
 }
 
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
@@ -1304,7 +1535,8 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
         bool writable = !(mem->flags & KVM_MEM_READONLY);
         int ret = 0;
 
-        if (change != KVM_MR_CREATE && change != KVM_MR_MOVE)
+        if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
+                        change != KVM_MR_FLAGS_ONLY)
                 return 0;
 
         /*
@@ -1355,6 +1587,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
                         phys_addr_t pa = (vma->vm_pgoff << PAGE_SHIFT) +
                                          vm_start - vma->vm_start;
 
+                        /* IO region dirty page logging not allowed */
+                        if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES)
+                                return -EINVAL;
+
                         ret = kvm_phys_addr_ioremap(kvm, gpa, pa,
                                                     vm_end - vm_start,
                                                     writable);
@@ -1364,6 +1600,9 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
                 hva = vm_end;
         } while (hva < reg_end);
 
+        if (change == KVM_MR_FLAGS_ONLY)
+                return ret;
+
         spin_lock(&kvm->mmu_lock);
         if (ret)
                 unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size);
diff --git a/arch/arm/kvm/psci.c b/arch/arm/kvm/psci.c
index 58cb3248d277..02fa8eff6ae1 100644
--- a/arch/arm/kvm/psci.c
+++ b/arch/arm/kvm/psci.c
@@ -22,6 +22,7 @@
 #include <asm/cputype.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_psci.h>
+#include <asm/kvm_host.h>
 
 /*
  * This is an implementation of the Power State Coordination Interface
@@ -66,25 +67,17 @@ static void kvm_psci_vcpu_off(struct kvm_vcpu *vcpu)
 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
 {
         struct kvm *kvm = source_vcpu->kvm;
-        struct kvm_vcpu *vcpu = NULL, *tmp;
+        struct kvm_vcpu *vcpu = NULL;
         wait_queue_head_t *wq;
         unsigned long cpu_id;
         unsigned long context_id;
-        unsigned long mpidr;
         phys_addr_t target_pc;
-        int i;
 
-        cpu_id = *vcpu_reg(source_vcpu, 1);
+        cpu_id = *vcpu_reg(source_vcpu, 1) & MPIDR_HWID_BITMASK;
         if (vcpu_mode_is_32bit(source_vcpu))
                 cpu_id &= ~((u32) 0);
 
-        kvm_for_each_vcpu(i, tmp, kvm) {
-                mpidr = kvm_vcpu_get_mpidr(tmp);
-                if ((mpidr & MPIDR_HWID_BITMASK) == (cpu_id & MPIDR_HWID_BITMASK)) {
-                        vcpu = tmp;
-                        break;
-                }
-        }
+        vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id);
 
         /*
          * Make sure the caller requested a valid CPU and that the CPU is
@@ -155,7 +148,7 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)
          * then ON else OFF
          */
         kvm_for_each_vcpu(i, tmp, kvm) {
-                mpidr = kvm_vcpu_get_mpidr(tmp);
+                mpidr = kvm_vcpu_get_mpidr_aff(tmp);
                 if (((mpidr & target_affinity_mask) == target_affinity) &&
                     !tmp->arch.pause) {
                         return PSCI_0_2_AFFINITY_LEVEL_ON;
diff --git a/arch/arm/kvm/trace.h b/arch/arm/kvm/trace.h
index b1d640f78623..f741449121f3 100644
--- a/arch/arm/kvm/trace.h
+++ b/arch/arm/kvm/trace.h
@@ -140,19 +140,22 @@ TRACE_EVENT(kvm_emulate_cp15_imp,
                         __entry->CRm, __entry->Op2)
 );
 
-TRACE_EVENT(kvm_wfi,
-        TP_PROTO(unsigned long vcpu_pc),
-        TP_ARGS(vcpu_pc),
+TRACE_EVENT(kvm_wfx,
+        TP_PROTO(unsigned long vcpu_pc, bool is_wfe),
+        TP_ARGS(vcpu_pc, is_wfe),
 
         TP_STRUCT__entry(
                 __field(        unsigned long,  vcpu_pc         )
+                __field(                 bool,  is_wfe          )
         ),
 
         TP_fast_assign(
                 __entry->vcpu_pc                = vcpu_pc;
+                __entry->is_wfe                 = is_wfe;
         ),
 
-        TP_printk("guest executed wfi at: 0x%08lx", __entry->vcpu_pc)
+        TP_printk("guest executed wf%c at: 0x%08lx",
+                __entry->is_wfe ? 'e' : 'i', __entry->vcpu_pc)
 );
 
 TRACE_EVENT(kvm_unmap_hva,
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index 62167090937d..92bbae381598 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -96,6 +96,7 @@
 #define ESR_ELx_COND_SHIFT      (20)
 #define ESR_ELx_COND_MASK       (UL(0xF) << ESR_ELx_COND_SHIFT)
 #define ESR_ELx_WFx_ISS_WFE     (UL(1) << 0)
+#define ESR_ELx_xVC_IMM_MASK    ((1UL << 16) - 1)
 
 #ifndef __ASSEMBLY__
 #include <asm/types.h>
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 483842180f8f..4f7310fa77f0 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -126,6 +126,7 @@ extern char __kvm_hyp_vector[];
 
 extern void __kvm_flush_vm_context(void);
 extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
+extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
 
 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 5c56c0d2cef1..c3baa971edab 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -29,6 +29,7 @@
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmio.h>
 #include <asm/ptrace.h>
+#include <asm/cputype.h>
 
 unsigned long *vcpu_reg32(const struct kvm_vcpu *vcpu, u8 reg_num);
 unsigned long *vcpu_spsr32(const struct kvm_vcpu *vcpu);
@@ -128,6 +129,11 @@ static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
         return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
 }
 
+static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
+{
+        return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_xVC_IMM_MASK;
+}
+
 static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
 {
         return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_ISV);
@@ -189,9 +195,9 @@ static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
         return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE;
 }
 
-static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu)
+static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
 {
-        return vcpu_sys_reg(vcpu, MPIDR_EL1);
+        return vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
 }
 
 static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 0b7dfdb931df..2c49aa4ac818 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -59,6 +59,9 @@ struct kvm_arch {
         /* VTTBR value associated with above pgd and vmid */
         u64    vttbr;
 
+        /* The maximum number of vCPUs depends on the used GIC model */
+        int max_vcpus;
+
         /* Interrupt controller */
         struct vgic_dist        vgic;
 
@@ -199,6 +202,7 @@ struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
 
 u64 kvm_call_hyp(void *hypfn, ...);
 void force_vm_exit(const cpumask_t *mask);
+void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
 
 int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
                 int exception_index);
@@ -206,6 +210,8 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 int kvm_perf_init(void);
 int kvm_perf_teardown(void);
 
+struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
+
 static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
                                        phys_addr_t pgd_ptr,
                                        unsigned long hyp_stack_ptr,
diff --git a/arch/arm64/include/asm/kvm_mmio.h b/arch/arm64/include/asm/kvm_mmio.h
index fc2f689c0694..9f52beb7cb13 100644
--- a/arch/arm64/include/asm/kvm_mmio.h
+++ b/arch/arm64/include/asm/kvm_mmio.h
@@ -40,6 +40,7 @@ struct kvm_exit_mmio {
         u8              data[8];
         u32             len;
         bool            is_write;
+        void            *private;
 };
 
 static inline void kvm_prepare_mmio(struct kvm_run *run,
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 14a74f136272..66577581ce68 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -118,6 +118,27 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
         pmd_val(*pmd) |= PMD_S2_RDWR;
 }
 
+static inline void kvm_set_s2pte_readonly(pte_t *pte)
+{
+        pte_val(*pte) = (pte_val(*pte) & ~PTE_S2_RDWR) | PTE_S2_RDONLY;
+}
+
+static inline bool kvm_s2pte_readonly(pte_t *pte)
+{
+        return (pte_val(*pte) & PTE_S2_RDWR) == PTE_S2_RDONLY;
+}
+
+static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
+{
+        pmd_val(*pmd) = (pmd_val(*pmd) & ~PMD_S2_RDWR) | PMD_S2_RDONLY;
+}
+
+static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
+{
+        return (pmd_val(*pmd) & PMD_S2_RDWR) == PMD_S2_RDONLY;
+}
+
+
 #define kvm_pgd_addr_end(addr, end)     pgd_addr_end(addr, end)
 #define kvm_pud_addr_end(addr, end)     pud_addr_end(addr, end)
 #define kvm_pmd_addr_end(addr, end)     pmd_addr_end(addr, end)
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index 88174e0bfafe..5f930cc9ea83 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -119,6 +119,7 @@
 #define PTE_S2_RDONLY           (_AT(pteval_t, 1) << 6)   /* HAP[2:1] */
 #define PTE_S2_RDWR             (_AT(pteval_t, 3) << 6)   /* HAP[2:1] */
 
+#define PMD_S2_RDONLY           (_AT(pmdval_t, 1) << 6)   /* HAP[2:1] */
 #define PMD_S2_RDWR             (_AT(pmdval_t, 3) << 6)   /* HAP[2:1] */
 
 /*
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 8e38878c87c6..3ef77a466018 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -78,6 +78,13 @@ struct kvm_regs {
 #define KVM_VGIC_V2_DIST_SIZE           0x1000
 #define KVM_VGIC_V2_CPU_SIZE            0x2000
 
+/* Supported VGICv3 address types  */
+#define KVM_VGIC_V3_ADDR_TYPE_DIST      2
+#define KVM_VGIC_V3_ADDR_TYPE_REDIST    3
+
+#define KVM_VGIC_V3_DIST_SIZE           SZ_64K
+#define KVM_VGIC_V3_REDIST_SIZE         (2 * SZ_64K)
+
 #define KVM_ARM_VCPU_POWER_OFF          0 /* CPU is started in OFF state */
 #define KVM_ARM_VCPU_EL1_32BIT          1 /* CPU running a 32bit VM */
 #define KVM_ARM_VCPU_PSCI_0_2           2 /* CPU uses PSCI v0.2 */
@@ -161,6 +168,8 @@ struct kvm_arch_memory_slot {
 #define   KVM_DEV_ARM_VGIC_OFFSET_SHIFT 0
 #define   KVM_DEV_ARM_VGIC_OFFSET_MASK  (0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT)
 #define KVM_DEV_ARM_VGIC_GRP_NR_IRQS    3
+#define KVM_DEV_ARM_VGIC_GRP_CTRL       4
+#define   KVM_DEV_ARM_VGIC_CTRL_INIT    0
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_TYPE_SHIFT          24
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 9a9fce090d58..9d34486985fd 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -140,6 +140,7 @@ int main(void)
   DEFINE(VGIC_V2_CPU_ELRSR,     offsetof(struct vgic_cpu, vgic_v2.vgic_elrsr));
   DEFINE(VGIC_V2_CPU_APR,       offsetof(struct vgic_cpu, vgic_v2.vgic_apr));
   DEFINE(VGIC_V2_CPU_LR,        offsetof(struct vgic_cpu, vgic_v2.vgic_lr));
+  DEFINE(VGIC_V3_CPU_SRE,       offsetof(struct vgic_cpu, vgic_v3.vgic_sre));
   DEFINE(VGIC_V3_CPU_HCR,       offsetof(struct vgic_cpu, vgic_v3.vgic_hcr));
   DEFINE(VGIC_V3_CPU_VMCR,      offsetof(struct vgic_cpu, vgic_v3.vgic_vmcr));
   DEFINE(VGIC_V3_CPU_MISR,      offsetof(struct vgic_cpu, vgic_v3.vgic_misr));
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 8ba85e9ea388..3ce389b3c21c 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -22,10 +22,12 @@ config KVM
         select PREEMPT_NOTIFIERS
         select ANON_INODES
         select HAVE_KVM_CPU_RELAX_INTERCEPT
+        select HAVE_KVM_ARCH_TLB_FLUSH_ALL
         select KVM_MMIO
         select KVM_ARM_HOST
         select KVM_ARM_VGIC
         select KVM_ARM_TIMER
+        select KVM_GENERIC_DIRTYLOG_READ_PROTECT
         ---help---
           Support hosting virtualized guest machines.
 
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 32a096174b94..4e6e09ee4033 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -21,7 +21,9 @@ kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
 
 kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
 kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
+kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o
 kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v2-switch.o
 kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3.o
+kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3-emul.o
 kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v3-switch.o
 kvm-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 29b184a8f3f8..524fa25671fc 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -28,12 +28,18 @@
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_psci.h>
 
+#define CREATE_TRACE_POINTS
+#include "trace.h"
+
 typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *);
 
 static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
         int ret;
 
+        trace_kvm_hvc_arm64(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
+                            kvm_vcpu_hvc_get_imm(vcpu));
+
         ret = kvm_psci_call(vcpu);
         if (ret < 0) {
                 kvm_inject_undefined(vcpu);
@@ -63,10 +69,13 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
  */
 static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
-        if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE)
+        if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
+                trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
                 kvm_vcpu_on_spin(vcpu);
-        else
+        } else {
+                trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false);
                 kvm_vcpu_block(vcpu);
+        }
 
         kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 
diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
index c0d820280a5e..31b4911b8522 100644
--- a/arch/arm64/kvm/hyp.S
+++ b/arch/arm64/kvm/hyp.S
@@ -1031,6 +1031,28 @@ ENTRY(__kvm_tlb_flush_vmid_ipa)
         ret
 ENDPROC(__kvm_tlb_flush_vmid_ipa)
 
+/**
+ * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
+ * @struct kvm *kvm - pointer to kvm structure
+ *
+ * Invalidates all Stage 1 and 2 TLB entries for current VMID.
+ */
+ENTRY(__kvm_tlb_flush_vmid)
+        dsb     ishst
+
+        kern_hyp_va     x0
+        ldr     x2, [x0, #KVM_VTTBR]
+        msr     vttbr_el2, x2
+        isb
+
+        tlbi    vmalls12e1is
+        dsb     ish
+        isb
+
+        msr     vttbr_el2, xzr
+        ret
+ENDPROC(__kvm_tlb_flush_vmid)
+
 ENTRY(__kvm_flush_vm_context)
         dsb     ishst
         tlbi    alle1is
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 6b859d7a48e7..7ad7af51856f 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -168,6 +168,27 @@ static bool access_sctlr(struct kvm_vcpu *vcpu,
         return true;
 }
 
+/*
+ * Trap handler for the GICv3 SGI generation system register.
+ * Forward the request to the VGIC emulation.
+ * The cp15_64 code makes sure this automatically works
+ * for both AArch64 and AArch32 accesses.
+ */
+static bool access_gic_sgi(struct kvm_vcpu *vcpu,
+                           const struct sys_reg_params *p,
+                           const struct sys_reg_desc *r)
+{
+        u64 val;
+
+        if (!p->is_write)
+                return read_from_write_only(vcpu, p);
+
+        val = *vcpu_reg(vcpu, p->Rt);
+        vgic_v3_dispatch_sgi(vcpu, val);
+
+        return true;
+}
+
 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
                         const struct sys_reg_params *p,
                         const struct sys_reg_desc *r)
@@ -255,10 +276,19 @@ static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 
 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 {
+        u64 mpidr;
+
         /*
-         * Simply map the vcpu_id into the Aff0 field of the MPIDR.
+         * Map the vcpu_id into the first three affinity level fields of
+         * the MPIDR. We limit the number of VCPUs in level 0 due to a
+         * limitation to 16 CPUs in that level in the ICC_SGIxR registers
+         * of the GICv3 to be able to address each CPU directly when
+         * sending IPIs.
          */
-        vcpu_sys_reg(vcpu, MPIDR_EL1) = (1UL << 31) | (vcpu->vcpu_id & 0xff);
+        mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
+        mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
+        mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
+        vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
 }
 
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
@@ -428,6 +458,9 @@ static const struct sys_reg_desc sys_reg_descs[] = {
         { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
           NULL, reset_val, VBAR_EL1, 0 },
 
+        /* ICC_SGI1R_EL1 */
+        { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1011), Op2(0b101),
+          access_gic_sgi },
         /* ICC_SRE_EL1 */
         { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101),
           trap_raz_wi },
@@ -660,6 +693,8 @@ static const struct sys_reg_desc cp14_64_regs[] = {
  * register).
  */
 static const struct sys_reg_desc cp15_regs[] = {
+        { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
+
         { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_sctlr, NULL, c1_SCTLR },
         { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
         { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
@@ -707,6 +742,7 @@ static const struct sys_reg_desc cp15_regs[] = {
 
 static const struct sys_reg_desc cp15_64_regs[] = {
         { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
+        { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
         { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
 };
 
diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h
new file mode 100644
index 000000000000..157416e963f2
--- /dev/null
+++ b/arch/arm64/kvm/trace.h
@@ -0,0 +1,55 @@
+#if !defined(_TRACE_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_ARM64_KVM_H
+
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM kvm
+
+TRACE_EVENT(kvm_wfx_arm64,
+        TP_PROTO(unsigned long vcpu_pc, bool is_wfe),
+        TP_ARGS(vcpu_pc, is_wfe),
+
+        TP_STRUCT__entry(
+                __field(unsigned long,  vcpu_pc)
+                __field(bool,           is_wfe)
+        ),
+
+        TP_fast_assign(
+                __entry->vcpu_pc = vcpu_pc;
+                __entry->is_wfe  = is_wfe;
+        ),
+
+        TP_printk("guest executed wf%c at: 0x%08lx",
+                  __entry->is_wfe ? 'e' : 'i', __entry->vcpu_pc)
+);
+
+TRACE_EVENT(kvm_hvc_arm64,
+        TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
+        TP_ARGS(vcpu_pc, r0, imm),
+
+        TP_STRUCT__entry(
+                __field(unsigned long, vcpu_pc)
+                __field(unsigned long, r0)
+                __field(unsigned long, imm)
+        ),
+
+        TP_fast_assign(
+                __entry->vcpu_pc = vcpu_pc;
+                __entry->r0 = r0;
+                __entry->imm = imm;
+        ),
+
+        TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx)",
+                  __entry->vcpu_pc, __entry->r0, __entry->imm)
+);
+
+#endif /* _TRACE_ARM64_KVM_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/arch/arm64/kvm/vgic-v3-switch.S b/arch/arm64/kvm/vgic-v3-switch.S
index d16046999e06..617a012a0107 100644
--- a/arch/arm64/kvm/vgic-v3-switch.S
+++ b/arch/arm64/kvm/vgic-v3-switch.S
@@ -148,17 +148,18 @@
  * x0: Register pointing to VCPU struct
  */
 .macro  restore_vgic_v3_state
-        // Disable SRE_EL1 access. Necessary, otherwise
-        // ICH_VMCR_EL2.VFIQEn becomes one, and FIQ happens...
-        msr_s   ICC_SRE_EL1, xzr
-        isb
-
         // Compute the address of struct vgic_cpu
         add     x3, x0, #VCPU_VGIC_CPU
 
         // Restore all interesting registers
         ldr     w4, [x3, #VGIC_V3_CPU_HCR]
         ldr     w5, [x3, #VGIC_V3_CPU_VMCR]
+        ldr     w25, [x3, #VGIC_V3_CPU_SRE]
+
+        msr_s   ICC_SRE_EL1, x25
+
+        // make sure SRE is valid before writing the other registers
+        isb
 
         msr_s   ICH_HCR_EL2, x4
         msr_s   ICH_VMCR_EL2, x5
@@ -244,9 +245,12 @@
         dsb     sy
 
         // Prevent the guest from touching the GIC system registers
+        // if SRE isn't enabled for GICv3 emulation
+        cbnz    x25, 1f
         mrs_s   x5, ICC_SRE_EL2
         and     x5, x5, #~ICC_SRE_EL2_ENABLE
         msr_s   ICC_SRE_EL2, x5
+1:
 .endm
 
 ENTRY(__save_vgic_v3_state)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 4327af53e544..843bea0e70fd 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -835,9 +835,6 @@ void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
 
 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot);
-void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
-                                     struct kvm_memory_slot *slot,
-                                     gfn_t gfn_offset, unsigned long mask);
 void kvm_mmu_zap_all(struct kvm *kvm);
 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm);
 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm);
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index f9d16ff56c6b..d07359466d5d 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -39,6 +39,7 @@ config KVM
         select PERF_EVENTS
         select HAVE_KVM_MSI
         select HAVE_KVM_CPU_RELAX_INTERCEPT
+        select KVM_GENERIC_DIRTYLOG_READ_PROTECT
         select KVM_VFIO
         ---help---
           Support hosting fully virtualized guest machines using hardware
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 97898abe8386..0ed9f795e4f0 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -1216,7 +1216,7 @@ static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
 }
 
 /**
- * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
+ * kvm_arch_mmu_write_protect_pt_masked - write protect selected PT level pages
  * @kvm: kvm instance
  * @slot: slot to protect
  * @gfn_offset: start of the BITS_PER_LONG pages we care about
@@ -1225,7 +1225,7 @@ static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
  * Used when we do not need to care about huge page mappings: e.g. during dirty
  * logging we do not have any such mappings.
  */
-void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
+void kvm_arch_mmu_write_protect_pt_masked(struct kvm *kvm,
                                      struct kvm_memory_slot *slot,
                                      gfn_t gfn_offset, unsigned long mask)
 {
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 917672f8034a..d2bbb2d86610 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3759,83 +3759,37 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm,
  * @kvm: kvm instance
  * @log: slot id and address to which we copy the log
  *
- * We need to keep it in mind that VCPU threads can write to the bitmap
- * concurrently.  So, to avoid losing data, we keep the following order for
- * each bit:
+ * Steps 1-4 below provide general overview of dirty page logging. See
+ * kvm_get_dirty_log_protect() function description for additional details.
+ *
+ * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
+ * always flush the TLB (step 4) even if previous step failed  and the dirty
+ * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
+ * does not preclude user space subsequent dirty log read. Flushing TLB ensures
+ * writes will be marked dirty for next log read.
  *
  *   1. Take a snapshot of the bit and clear it if needed.
  *   2. Write protect the corresponding page.
- *   3. Flush TLB's if needed.
- *   4. Copy the snapshot to the userspace.
- *
- * Between 2 and 3, the guest may write to the page using the remaining TLB
- * entry.  This is not a problem because the page will be reported dirty at
- * step 4 using the snapshot taken before and step 3 ensures that successive
- * writes will be logged for the next call.
+ *   3. Copy the snapshot to the userspace.
+ *   4. Flush TLB's if needed.
  */
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
 {
-        int r;
-        struct kvm_memory_slot *memslot;
-        unsigned long n, i;
-        unsigned long *dirty_bitmap;
-        unsigned long *dirty_bitmap_buffer;
         bool is_dirty = false;
+        int r;
 
         mutex_lock(&kvm->slots_lock);
 
-        r = -EINVAL;
-        if (log->slot >= KVM_USER_MEM_SLOTS)
-                goto out;
-
-        memslot = id_to_memslot(kvm->memslots, log->slot);
-
-        dirty_bitmap = memslot->dirty_bitmap;
-        r = -ENOENT;
-        if (!dirty_bitmap)
-                goto out;
-
-        n = kvm_dirty_bitmap_bytes(memslot);
-
-        dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
-        memset(dirty_bitmap_buffer, 0, n);
-
-        spin_lock(&kvm->mmu_lock);
-
-        for (i = 0; i < n / sizeof(long); i++) {
-                unsigned long mask;
-                gfn_t offset;
-
-                if (!dirty_bitmap[i])
-                        continue;
-
-                is_dirty = true;
-
-                mask = xchg(&dirty_bitmap[i], 0);
-                dirty_bitmap_buffer[i] = mask;
-
-                offset = i * BITS_PER_LONG;
-                kvm_mmu_write_protect_pt_masked(kvm, memslot, offset, mask);
-        }
-
-        spin_unlock(&kvm->mmu_lock);
-
-        /* See the comments in kvm_mmu_slot_remove_write_access(). */
-        lockdep_assert_held(&kvm->slots_lock);
+        r = kvm_get_dirty_log_protect(kvm, log, &is_dirty);
 
         /*
          * All the TLBs can be flushed out of mmu lock, see the comments in
          * kvm_mmu_slot_remove_write_access().
          */
+        lockdep_assert_held(&kvm->slots_lock);
         if (is_dirty)
                 kvm_flush_remote_tlbs(kvm);
 
-        r = -EFAULT;
-        if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
-                goto out;
-
-        r = 0;
-out:
         mutex_unlock(&kvm->slots_lock);
         return r;
 }
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 1a146ccee701..2ab290bec655 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -481,15 +481,19 @@ out:
         return tlist;
 }
 
+#define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
+        (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
+                << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
+
 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
 {
         u64 val;
 
-        val = (MPIDR_AFFINITY_LEVEL(cluster_id, 3) << 48        |
-               MPIDR_AFFINITY_LEVEL(cluster_id, 2) << 32        |
-               irq << 24                                        |
-               MPIDR_AFFINITY_LEVEL(cluster_id, 1) << 16        |
-               tlist);
+        val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)     |
+               MPIDR_TO_SGI_AFFINITY(cluster_id, 2)     |
+               irq << ICC_SGI1R_SGI_ID_SHIFT            |
+               MPIDR_TO_SGI_AFFINITY(cluster_id, 1)     |
+               tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
 
         pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
         gic_write_sgi1r(val);
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index ac4888dc86bc..7c55dd5dd2c9 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -33,10 +33,11 @@
 #define VGIC_V2_MAX_LRS         (1 << 6)
 #define VGIC_V3_MAX_LRS         16
 #define VGIC_MAX_IRQS           1024
+#define VGIC_V2_MAX_CPUS        8
 
 /* Sanity checks... */
-#if (KVM_MAX_VCPUS > 8)
-#error  Invalid number of CPU interfaces
+#if (KVM_MAX_VCPUS > 255)
+#error Too many KVM VCPUs, the VGIC only supports up to 255 VCPUs for now
 #endif
 
 #if (VGIC_NR_IRQS_LEGACY & 31)
@@ -132,6 +133,18 @@ struct vgic_params {
         unsigned int    maint_irq;
         /* Virtual control interface base address */
         void __iomem    *vctrl_base;
+        int             max_gic_vcpus;
+        /* Only needed for the legacy KVM_CREATE_IRQCHIP */
+        bool            can_emulate_gicv2;
+};
+
+struct vgic_vm_ops {
+        bool    (*handle_mmio)(struct kvm_vcpu *, struct kvm_run *,
+                               struct kvm_exit_mmio *);
+        bool    (*queue_sgi)(struct kvm_vcpu *, int irq);
+        void    (*add_sgi_source)(struct kvm_vcpu *, int irq, int source);
+        int     (*init_model)(struct kvm *);
+        int     (*map_resources)(struct kvm *, const struct vgic_params *);
 };
 
 struct vgic_dist {
@@ -140,6 +153,9 @@ struct vgic_dist {
         bool                    in_kernel;
         bool                    ready;
 
+        /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */
+        u32                     vgic_model;
+
         int                     nr_cpus;
         int                     nr_irqs;
 
@@ -148,7 +164,11 @@ struct vgic_dist {
 
         /* Distributor and vcpu interface mapping in the guest */
         phys_addr_t             vgic_dist_base;
-        phys_addr_t             vgic_cpu_base;
+        /* GICv2 and GICv3 use different mapped register blocks */
+        union {
+                phys_addr_t             vgic_cpu_base;
+                phys_addr_t             vgic_redist_base;
+        };
 
         /* Distributor enabled */
         u32                     enabled;
@@ -210,8 +230,13 @@ struct vgic_dist {
          */
         struct vgic_bitmap      *irq_spi_target;
 
+        /* Target MPIDR for each IRQ (needed for GICv3 IROUTERn) only */
+        u32                     *irq_spi_mpidr;
+
         /* Bitmap indicating which CPU has something pending */
         unsigned long           *irq_pending_on_cpu;
+
+        struct vgic_vm_ops      vm_ops;
 #endif
 };
 
@@ -229,6 +254,7 @@ struct vgic_v3_cpu_if {
 #ifdef CONFIG_ARM_GIC_V3
         u32             vgic_hcr;
         u32             vgic_vmcr;
+        u32             vgic_sre;       /* Restored only, change ignored */
         u32             vgic_misr;      /* Saved only */
         u32             vgic_eisr;      /* Saved only */
         u32             vgic_elrsr;     /* Saved only */
@@ -275,13 +301,15 @@ struct kvm_exit_mmio;
 int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write);
 int kvm_vgic_hyp_init(void);
 int kvm_vgic_map_resources(struct kvm *kvm);
-int kvm_vgic_create(struct kvm *kvm);
+int kvm_vgic_get_max_vcpus(void);
+int kvm_vgic_create(struct kvm *kvm, u32 type);
 void kvm_vgic_destroy(struct kvm *kvm);
 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
 int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
                         bool level);
+void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg);
 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
 bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
                       struct kvm_exit_mmio *mmio);
@@ -327,7 +355,7 @@ static inline int kvm_vgic_map_resources(struct kvm *kvm)
         return 0;
 }
 
-static inline int kvm_vgic_create(struct kvm *kvm)
+static inline int kvm_vgic_create(struct kvm *kvm, u32 type)
 {
         return 0;
 }
@@ -379,6 +407,11 @@ static inline bool vgic_ready(struct kvm *kvm)
 {
         return true;
 }
+
+static inline int kvm_vgic_get_max_vcpus(void)
+{
+        return KVM_MAX_VCPUS;
+}
 #endif
 
 #endif
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index 1e8b0cf30792..800544bc7bfd 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -33,6 +33,7 @@
 #define GICD_SETSPI_SR                  0x0050
 #define GICD_CLRSPI_SR                  0x0058
 #define GICD_SEIR                       0x0068
+#define GICD_IGROUPR                    0x0080
 #define GICD_ISENABLER                  0x0100
 #define GICD_ICENABLER                  0x0180
 #define GICD_ISPENDR                    0x0200
@@ -41,14 +42,37 @@
 #define GICD_ICACTIVER                  0x0380
 #define GICD_IPRIORITYR                 0x0400
 #define GICD_ICFGR                      0x0C00
+#define GICD_IGRPMODR                   0x0D00
+#define GICD_NSACR                      0x0E00
 #define GICD_IROUTER                    0x6000
+#define GICD_IDREGS                     0xFFD0
 #define GICD_PIDR2                      0xFFE8
 
+/*
+ * Those registers are actually from GICv2, but the spec demands that they
+ * are implemented as RES0 if ARE is 1 (which we do in KVM's emulated GICv3).
+ */
+#define GICD_ITARGETSR                  0x0800
+#define GICD_SGIR                       0x0F00
+#define GICD_CPENDSGIR                  0x0F10
+#define GICD_SPENDSGIR                  0x0F20
+
 #define GICD_CTLR_RWP                   (1U << 31)
+#define GICD_CTLR_DS                    (1U << 6)
 #define GICD_CTLR_ARE_NS                (1U << 4)
 #define GICD_CTLR_ENABLE_G1A            (1U << 1)
 #define GICD_CTLR_ENABLE_G1             (1U << 0)
 
+/*
+ * In systems with a single security state (what we emulate in KVM)
+ * the meaning of the interrupt group enable bits is slightly different
+ */
+#define GICD_CTLR_ENABLE_SS_G1          (1U << 1)
+#define GICD_CTLR_ENABLE_SS_G0          (1U << 0)
+
+#define GICD_TYPER_LPIS                 (1U << 17)
+#define GICD_TYPER_MBIS                 (1U << 16)
+
 #define GICD_TYPER_ID_BITS(typer)       ((((typer) >> 19) & 0x1f) + 1)
 #define GICD_TYPER_IRQS(typer)          ((((typer) & 0x1f) + 1) * 32)
 #define GICD_TYPER_LPIS                 (1U << 17)
@@ -60,6 +84,8 @@
 #define GIC_PIDR2_ARCH_GICv3            0x30
 #define GIC_PIDR2_ARCH_GICv4            0x40
 
+#define GIC_V3_DIST_SIZE                0x10000
+
 /*
  * Re-Distributor registers, offsets from RD_base
  */
@@ -78,6 +104,7 @@
 #define GICR_SYNCR                      0x00C0
 #define GICR_MOVLPIR                    0x0100
 #define GICR_MOVALLR                    0x0110
+#define GICR_IDREGS                     GICD_IDREGS
 #define GICR_PIDR2                      GICD_PIDR2
 
 #define GICR_CTLR_ENABLE_LPIS           (1UL << 0)
@@ -104,6 +131,7 @@
 /*
  * Re-Distributor registers, offsets from SGI_base
  */
+#define GICR_IGROUPR0                   GICD_IGROUPR
 #define GICR_ISENABLER0                 GICD_ISENABLER
 #define GICR_ICENABLER0                 GICD_ICENABLER
 #define GICR_ISPENDR0                   GICD_ISPENDR
@@ -112,11 +140,15 @@
 #define GICR_ICACTIVER0                 GICD_ICACTIVER
 #define GICR_IPRIORITYR0                GICD_IPRIORITYR
 #define GICR_ICFGR0                     GICD_ICFGR
+#define GICR_IGRPMODR0                  GICD_IGRPMODR
+#define GICR_NSACR                      GICD_NSACR
 
 #define GICR_TYPER_PLPIS                (1U << 0)
 #define GICR_TYPER_VLPIS                (1U << 1)
 #define GICR_TYPER_LAST                 (1U << 4)
 
+#define GIC_V3_REDIST_SIZE              0x20000
+
 #define LPI_PROP_GROUP1                 (1 << 1)
 #define LPI_PROP_ENABLED                (1 << 0)
 
@@ -248,6 +280,18 @@
 #define ICC_SRE_EL2_SRE                 (1 << 0)
 #define ICC_SRE_EL2_ENABLE              (1 << 3)
 
+#define ICC_SGI1R_TARGET_LIST_SHIFT     0
+#define ICC_SGI1R_TARGET_LIST_MASK      (0xffff << ICC_SGI1R_TARGET_LIST_SHIFT)
+#define ICC_SGI1R_AFFINITY_1_SHIFT      16
+#define ICC_SGI1R_AFFINITY_1_MASK       (0xff << ICC_SGI1R_AFFINITY_1_SHIFT)
+#define ICC_SGI1R_SGI_ID_SHIFT          24
+#define ICC_SGI1R_SGI_ID_MASK           (0xff << ICC_SGI1R_SGI_ID_SHIFT)
+#define ICC_SGI1R_AFFINITY_2_SHIFT      32
+#define ICC_SGI1R_AFFINITY_2_MASK       (0xffULL << ICC_SGI1R_AFFINITY_1_SHIFT)
+#define ICC_SGI1R_IRQ_ROUTING_MODE_BIT  40
+#define ICC_SGI1R_AFFINITY_3_SHIFT      48
+#define ICC_SGI1R_AFFINITY_3_MASK       (0xffULL << ICC_SGI1R_AFFINITY_1_SHIFT)
+
 /*
  * System register definitions
  */
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 26f106022c88..0ef2daa199d8 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -611,6 +611,15 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
 
 int kvm_get_dirty_log(struct kvm *kvm,
                         struct kvm_dirty_log *log, int *is_dirty);
+
+int kvm_get_dirty_log_protect(struct kvm *kvm,
+                        struct kvm_dirty_log *log, bool *is_dirty);
+
+void kvm_arch_mmu_write_protect_pt_masked(struct kvm *kvm,
+                                        struct kvm_memory_slot *slot,
+                                        gfn_t gfn_offset,
+                                        unsigned long mask);
+
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
                                 struct kvm_dirty_log *log);
 
@@ -1042,6 +1051,8 @@ void kvm_unregister_device_ops(u32 type);
 
 extern struct kvm_device_ops kvm_mpic_ops;
 extern struct kvm_device_ops kvm_xics_ops;
+extern struct kvm_device_ops kvm_arm_vgic_v2_ops;
+extern struct kvm_device_ops kvm_arm_vgic_v3_ops;
 
 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
 
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index a37fd1224f36..b4e6f1e70f03 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -952,6 +952,8 @@ enum kvm_device_type {
 #define KVM_DEV_TYPE_ARM_VGIC_V2        KVM_DEV_TYPE_ARM_VGIC_V2
         KVM_DEV_TYPE_FLIC,
 #define KVM_DEV_TYPE_FLIC               KVM_DEV_TYPE_FLIC
+        KVM_DEV_TYPE_ARM_VGIC_V3,
+#define KVM_DEV_TYPE_ARM_VGIC_V3        KVM_DEV_TYPE_ARM_VGIC_V3
         KVM_DEV_TYPE_MAX,
 };
 
diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index fc0c5e603eb4..50d110654b42 100644
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -37,3 +37,9 @@ config HAVE_KVM_CPU_RELAX_INTERCEPT
 
 config KVM_VFIO
        bool
+
+config HAVE_KVM_ARCH_TLB_FLUSH_ALL
+       bool
+
+config KVM_GENERIC_DIRTYLOG_READ_PROTECT
+       bool
diff --git a/virt/kvm/arm/vgic-v2-emul.c b/virt/kvm/arm/vgic-v2-emul.c
new file mode 100644
index 000000000000..19c6210f02cf
--- /dev/null
+++ b/virt/kvm/arm/vgic-v2-emul.c
@@ -0,0 +1,847 @@
+/*
+ * Contains GICv2 specific emulation code, was in vgic.c before.
+ *
+ * Copyright (C) 2012 ARM Ltd.
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/cpu.h>
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/uaccess.h>
+
+#include <linux/irqchip/arm-gic.h>
+
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_arm.h>
+#include <asm/kvm_mmu.h>
+
+#include "vgic.h"
+
+#define GICC_ARCH_VERSION_V2            0x2
+
+static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg);
+static u8 *vgic_get_sgi_sources(struct vgic_dist *dist, int vcpu_id, int sgi)
+{
+        return dist->irq_sgi_sources + vcpu_id * VGIC_NR_SGIS + sgi;
+}
+
+static bool handle_mmio_misc(struct kvm_vcpu *vcpu,
+                             struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        u32 reg;
+        u32 word_offset = offset & 3;
+
+        switch (offset & ~3) {
+        case 0:                 /* GICD_CTLR */
+                reg = vcpu->kvm->arch.vgic.enabled;
+                vgic_reg_access(mmio, &reg, word_offset,
+                                ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
+                if (mmio->is_write) {
+                        vcpu->kvm->arch.vgic.enabled = reg & 1;
+                        vgic_update_state(vcpu->kvm);
+                        return true;
+                }
+                break;
+
+        case 4:                 /* GICD_TYPER */
+                reg  = (atomic_read(&vcpu->kvm->online_vcpus) - 1) << 5;
+                reg |= (vcpu->kvm->arch.vgic.nr_irqs >> 5) - 1;
+                vgic_reg_access(mmio, &reg, word_offset,
+                                ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+                break;
+
+        case 8:                 /* GICD_IIDR */
+                reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
+                vgic_reg_access(mmio, &reg, word_offset,
+                                ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+                break;
+        }
+
+        return false;
+}
+
+static bool handle_mmio_set_enable_reg(struct kvm_vcpu *vcpu,
+                                       struct kvm_exit_mmio *mmio,
+                                       phys_addr_t offset)
+{
+        return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
+                                      vcpu->vcpu_id, ACCESS_WRITE_SETBIT);
+}
+
+static bool handle_mmio_clear_enable_reg(struct kvm_vcpu *vcpu,
+                                         struct kvm_exit_mmio *mmio,
+                                         phys_addr_t offset)
+{
+        return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
+                                      vcpu->vcpu_id, ACCESS_WRITE_CLEARBIT);
+}
+
+static bool handle_mmio_set_pending_reg(struct kvm_vcpu *vcpu,
+                                        struct kvm_exit_mmio *mmio,
+                                        phys_addr_t offset)
+{
+        return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
+                                           vcpu->vcpu_id);
+}
+
+static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu,
+                                          struct kvm_exit_mmio *mmio,
+                                          phys_addr_t offset)
+{
+        return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
+                                             vcpu->vcpu_id);
+}
+
+static bool handle_mmio_priority_reg(struct kvm_vcpu *vcpu,
+                                     struct kvm_exit_mmio *mmio,
+                                     phys_addr_t offset)
+{
+        u32 *reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
+                                        vcpu->vcpu_id, offset);
+        vgic_reg_access(mmio, reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
+        return false;
+}
+
+#define GICD_ITARGETSR_SIZE     32
+#define GICD_CPUTARGETS_BITS    8
+#define GICD_IRQS_PER_ITARGETSR (GICD_ITARGETSR_SIZE / GICD_CPUTARGETS_BITS)
+static u32 vgic_get_target_reg(struct kvm *kvm, int irq)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        int i;
+        u32 val = 0;
+
+        irq -= VGIC_NR_PRIVATE_IRQS;
+
+        for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++)
+                val |= 1 << (dist->irq_spi_cpu[irq + i] + i * 8);
+
+        return val;
+}
+
+static void vgic_set_target_reg(struct kvm *kvm, u32 val, int irq)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        struct kvm_vcpu *vcpu;
+        int i, c;
+        unsigned long *bmap;
+        u32 target;
+
+        irq -= VGIC_NR_PRIVATE_IRQS;
+
+        /*
+         * Pick the LSB in each byte. This ensures we target exactly
+         * one vcpu per IRQ. If the byte is null, assume we target
+         * CPU0.
+         */
+        for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++) {
+                int shift = i * GICD_CPUTARGETS_BITS;
+
+                target = ffs((val >> shift) & 0xffU);
+                target = target ? (target - 1) : 0;
+                dist->irq_spi_cpu[irq + i] = target;
+                kvm_for_each_vcpu(c, vcpu, kvm) {
+                        bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[c]);
+                        if (c == target)
+                                set_bit(irq + i, bmap);
+                        else
+                                clear_bit(irq + i, bmap);
+                }
+        }
+}
+
+static bool handle_mmio_target_reg(struct kvm_vcpu *vcpu,
+                                   struct kvm_exit_mmio *mmio,
+                                   phys_addr_t offset)
+{
+        u32 reg;
+
+        /* We treat the banked interrupts targets as read-only */
+        if (offset < 32) {
+                u32 roreg;
+
+                roreg = 1 << vcpu->vcpu_id;
+                roreg |= roreg << 8;
+                roreg |= roreg << 16;
+
+                vgic_reg_access(mmio, &roreg, offset,
+                                ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+                return false;
+        }
+
+        reg = vgic_get_target_reg(vcpu->kvm, offset & ~3U);
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
+        if (mmio->is_write) {
+                vgic_set_target_reg(vcpu->kvm, reg, offset & ~3U);
+                vgic_update_state(vcpu->kvm);
+                return true;
+        }
+
+        return false;
+}
+
+static bool handle_mmio_cfg_reg(struct kvm_vcpu *vcpu,
+                                struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        u32 *reg;
+
+        reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
+                                  vcpu->vcpu_id, offset >> 1);
+
+        return vgic_handle_cfg_reg(reg, mmio, offset);
+}
+
+static bool handle_mmio_sgi_reg(struct kvm_vcpu *vcpu,
+                                struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        u32 reg;
+
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_RAZ | ACCESS_WRITE_VALUE);
+        if (mmio->is_write) {
+                vgic_dispatch_sgi(vcpu, reg);
+                vgic_update_state(vcpu->kvm);
+                return true;
+        }
+
+        return false;
+}
+
+/* Handle reads of GICD_CPENDSGIRn and GICD_SPENDSGIRn */
+static bool read_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
+                                        struct kvm_exit_mmio *mmio,
+                                        phys_addr_t offset)
+{
+        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+        int sgi;
+        int min_sgi = (offset & ~0x3);
+        int max_sgi = min_sgi + 3;
+        int vcpu_id = vcpu->vcpu_id;
+        u32 reg = 0;
+
+        /* Copy source SGIs from distributor side */
+        for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
+                u8 sources = *vgic_get_sgi_sources(dist, vcpu_id, sgi);
+
+                reg |= ((u32)sources) << (8 * (sgi - min_sgi));
+        }
+
+        mmio_data_write(mmio, ~0, reg);
+        return false;
+}
+
+static bool write_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
+                                         struct kvm_exit_mmio *mmio,
+                                         phys_addr_t offset, bool set)
+{
+        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+        int sgi;
+        int min_sgi = (offset & ~0x3);
+        int max_sgi = min_sgi + 3;
+        int vcpu_id = vcpu->vcpu_id;
+        u32 reg;
+        bool updated = false;
+
+        reg = mmio_data_read(mmio, ~0);
+
+        /* Clear pending SGIs on the distributor */
+        for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
+                u8 mask = reg >> (8 * (sgi - min_sgi));
+                u8 *src = vgic_get_sgi_sources(dist, vcpu_id, sgi);
+
+                if (set) {
+                        if ((*src & mask) != mask)
+                                updated = true;
+                        *src |= mask;
+                } else {
+                        if (*src & mask)
+                                updated = true;
+                        *src &= ~mask;
+                }
+        }
+
+        if (updated)
+                vgic_update_state(vcpu->kvm);
+
+        return updated;
+}
+
+static bool handle_mmio_sgi_set(struct kvm_vcpu *vcpu,
+                                struct kvm_exit_mmio *mmio,
+                                phys_addr_t offset)
+{
+        if (!mmio->is_write)
+                return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
+        else
+                return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, true);
+}
+
+static bool handle_mmio_sgi_clear(struct kvm_vcpu *vcpu,
+                                  struct kvm_exit_mmio *mmio,
+                                  phys_addr_t offset)
+{
+        if (!mmio->is_write)
+                return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
+        else
+                return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, false);
+}
+
+static const struct kvm_mmio_range vgic_dist_ranges[] = {
+        {
+                .base           = GIC_DIST_CTRL,
+                .len            = 12,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_misc,
+        },
+        {
+                .base           = GIC_DIST_IGROUP,
+                .len            = VGIC_MAX_IRQS / 8,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GIC_DIST_ENABLE_SET,
+                .len            = VGIC_MAX_IRQS / 8,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_set_enable_reg,
+        },
+        {
+                .base           = GIC_DIST_ENABLE_CLEAR,
+                .len            = VGIC_MAX_IRQS / 8,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_clear_enable_reg,
+        },
+        {
+                .base           = GIC_DIST_PENDING_SET,
+                .len            = VGIC_MAX_IRQS / 8,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_set_pending_reg,
+        },
+        {
+                .base           = GIC_DIST_PENDING_CLEAR,
+                .len            = VGIC_MAX_IRQS / 8,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_clear_pending_reg,
+        },
+        {
+                .base           = GIC_DIST_ACTIVE_SET,
+                .len            = VGIC_MAX_IRQS / 8,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GIC_DIST_ACTIVE_CLEAR,
+                .len            = VGIC_MAX_IRQS / 8,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GIC_DIST_PRI,
+                .len            = VGIC_MAX_IRQS,
+                .bits_per_irq   = 8,
+                .handle_mmio    = handle_mmio_priority_reg,
+        },
+        {
+                .base           = GIC_DIST_TARGET,
+                .len            = VGIC_MAX_IRQS,
+                .bits_per_irq   = 8,
+                .handle_mmio    = handle_mmio_target_reg,
+        },
+        {
+                .base           = GIC_DIST_CONFIG,
+                .len            = VGIC_MAX_IRQS / 4,
+                .bits_per_irq   = 2,
+                .handle_mmio    = handle_mmio_cfg_reg,
+        },
+        {
+                .base           = GIC_DIST_SOFTINT,
+                .len            = 4,
+                .handle_mmio    = handle_mmio_sgi_reg,
+        },
+        {
+                .base           = GIC_DIST_SGI_PENDING_CLEAR,
+                .len            = VGIC_NR_SGIS,
+                .handle_mmio    = handle_mmio_sgi_clear,
+        },
+        {
+                .base           = GIC_DIST_SGI_PENDING_SET,
+                .len            = VGIC_NR_SGIS,
+                .handle_mmio    = handle_mmio_sgi_set,
+        },
+        {}
+};
+
+static bool vgic_v2_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
+                                struct kvm_exit_mmio *mmio)
+{
+        unsigned long base = vcpu->kvm->arch.vgic.vgic_dist_base;
+
+        if (!is_in_range(mmio->phys_addr, mmio->len, base,
+                         KVM_VGIC_V2_DIST_SIZE))
+                return false;
+
+        /* GICv2 does not support accesses wider than 32 bits */
+        if (mmio->len > 4) {
+                kvm_inject_dabt(vcpu, mmio->phys_addr);
+                return true;
+        }
+
+        return vgic_handle_mmio_range(vcpu, run, mmio, vgic_dist_ranges, base);
+}
+
+static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg)
+{
+        struct kvm *kvm = vcpu->kvm;
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        int nrcpus = atomic_read(&kvm->online_vcpus);
+        u8 target_cpus;
+        int sgi, mode, c, vcpu_id;
+
+        vcpu_id = vcpu->vcpu_id;
+
+        sgi = reg & 0xf;
+        target_cpus = (reg >> 16) & 0xff;
+        mode = (reg >> 24) & 3;
+
+        switch (mode) {
+        case 0:
+                if (!target_cpus)
+                        return;
+                break;
+
+        case 1:
+                target_cpus = ((1 << nrcpus) - 1) & ~(1 << vcpu_id) & 0xff;
+                break;
+
+        case 2:
+                target_cpus = 1 << vcpu_id;
+                break;
+        }
+
+        kvm_for_each_vcpu(c, vcpu, kvm) {
+                if (target_cpus & 1) {
+                        /* Flag the SGI as pending */
+                        vgic_dist_irq_set_pending(vcpu, sgi);
+                        *vgic_get_sgi_sources(dist, c, sgi) |= 1 << vcpu_id;
+                        kvm_debug("SGI%d from CPU%d to CPU%d\n",
+                                  sgi, vcpu_id, c);
+                }
+
+                target_cpus >>= 1;
+        }
+}
+
+static bool vgic_v2_queue_sgi(struct kvm_vcpu *vcpu, int irq)
+{
+        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+        unsigned long sources;
+        int vcpu_id = vcpu->vcpu_id;
+        int c;
+
+        sources = *vgic_get_sgi_sources(dist, vcpu_id, irq);
+
+        for_each_set_bit(c, &sources, dist->nr_cpus) {
+                if (vgic_queue_irq(vcpu, c, irq))
+                        clear_bit(c, &sources);
+        }
+
+        *vgic_get_sgi_sources(dist, vcpu_id, irq) = sources;
+
+        /*
+         * If the sources bitmap has been cleared it means that we
+         * could queue all the SGIs onto link registers (see the
+         * clear_bit above), and therefore we are done with them in
+         * our emulated gic and can get rid of them.
+         */
+        if (!sources) {
+                vgic_dist_irq_clear_pending(vcpu, irq);
+                vgic_cpu_irq_clear(vcpu, irq);
+                return true;
+        }
+
+        return false;
+}
+
+/**
+ * kvm_vgic_map_resources - Configure global VGIC state before running any VCPUs
+ * @kvm: pointer to the kvm struct
+ *
+ * Map the virtual CPU interface into the VM before running any VCPUs.  We
+ * can't do this at creation time, because user space must first set the
+ * virtual CPU interface address in the guest physical address space.
+ */
+static int vgic_v2_map_resources(struct kvm *kvm,
+                                 const struct vgic_params *params)
+{
+        int ret = 0;
+
+        if (!irqchip_in_kernel(kvm))
+                return 0;
+
+        mutex_lock(&kvm->lock);
+
+        if (vgic_ready(kvm))
+                goto out;
+
+        if (IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_dist_base) ||
+            IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_cpu_base)) {
+                kvm_err("Need to set vgic cpu and dist addresses first\n");
+                ret = -ENXIO;
+                goto out;
+        }
+
+        /*
+         * Initialize the vgic if this hasn't already been done on demand by
+         * accessing the vgic state from userspace.
+         */
+        ret = vgic_init(kvm);
+        if (ret) {
+                kvm_err("Unable to allocate maps\n");
+                goto out;
+        }
+
+        ret = kvm_phys_addr_ioremap(kvm, kvm->arch.vgic.vgic_cpu_base,
+                                    params->vcpu_base, KVM_VGIC_V2_CPU_SIZE,
+                                    true);
+        if (ret) {
+                kvm_err("Unable to remap VGIC CPU to VCPU\n");
+                goto out;
+        }
+
+        kvm->arch.vgic.ready = true;
+out:
+        if (ret)
+                kvm_vgic_destroy(kvm);
+        mutex_unlock(&kvm->lock);
+        return ret;
+}
+
+static void vgic_v2_add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
+{
+        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+
+        *vgic_get_sgi_sources(dist, vcpu->vcpu_id, irq) |= 1 << source;
+}
+
+static int vgic_v2_init_model(struct kvm *kvm)
+{
+        int i;
+
+        for (i = VGIC_NR_PRIVATE_IRQS; i < kvm->arch.vgic.nr_irqs; i += 4)
+                vgic_set_target_reg(kvm, 0, i);
+
+        return 0;
+}
+
+void vgic_v2_init_emulation(struct kvm *kvm)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+
+        dist->vm_ops.handle_mmio = vgic_v2_handle_mmio;
+        dist->vm_ops.queue_sgi = vgic_v2_queue_sgi;
+        dist->vm_ops.add_sgi_source = vgic_v2_add_sgi_source;
+        dist->vm_ops.init_model = vgic_v2_init_model;
+        dist->vm_ops.map_resources = vgic_v2_map_resources;
+
+        kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS;
+}
+
+static bool handle_cpu_mmio_misc(struct kvm_vcpu *vcpu,
+                                 struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        bool updated = false;
+        struct vgic_vmcr vmcr;
+        u32 *vmcr_field;
+        u32 reg;
+
+        vgic_get_vmcr(vcpu, &vmcr);
+
+        switch (offset & ~0x3) {
+        case GIC_CPU_CTRL:
+                vmcr_field = &vmcr.ctlr;
+                break;
+        case GIC_CPU_PRIMASK:
+                vmcr_field = &vmcr.pmr;
+                break;
+        case GIC_CPU_BINPOINT:
+                vmcr_field = &vmcr.bpr;
+                break;
+        case GIC_CPU_ALIAS_BINPOINT:
+                vmcr_field = &vmcr.abpr;
+                break;
+        default:
+                BUG();
+        }
+
+        if (!mmio->is_write) {
+                reg = *vmcr_field;
+                mmio_data_write(mmio, ~0, reg);
+        } else {
+                reg = mmio_data_read(mmio, ~0);
+                if (reg != *vmcr_field) {
+                        *vmcr_field = reg;
+                        vgic_set_vmcr(vcpu, &vmcr);
+                        updated = true;
+                }
+        }
+        return updated;
+}
+
+static bool handle_mmio_abpr(struct kvm_vcpu *vcpu,
+                             struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        return handle_cpu_mmio_misc(vcpu, mmio, GIC_CPU_ALIAS_BINPOINT);
+}
+
+static bool handle_cpu_mmio_ident(struct kvm_vcpu *vcpu,
+                                  struct kvm_exit_mmio *mmio,
+                                  phys_addr_t offset)
+{
+        u32 reg;
+
+        if (mmio->is_write)
+                return false;
+
+        /* GICC_IIDR */
+        reg = (PRODUCT_ID_KVM << 20) |
+              (GICC_ARCH_VERSION_V2 << 16) |
+              (IMPLEMENTER_ARM << 0);
+        mmio_data_write(mmio, ~0, reg);
+        return false;
+}
+
+/*
+ * CPU Interface Register accesses - these are not accessed by the VM, but by
+ * user space for saving and restoring VGIC state.
+ */
+static const struct kvm_mmio_range vgic_cpu_ranges[] = {
+        {
+                .base           = GIC_CPU_CTRL,
+                .len            = 12,
+                .handle_mmio    = handle_cpu_mmio_misc,
+        },
+        {
+                .base           = GIC_CPU_ALIAS_BINPOINT,
+                .len            = 4,
+                .handle_mmio    = handle_mmio_abpr,
+        },
+        {
+                .base           = GIC_CPU_ACTIVEPRIO,
+                .len            = 16,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GIC_CPU_IDENT,
+                .len            = 4,
+                .handle_mmio    = handle_cpu_mmio_ident,
+        },
+};
+
+static int vgic_attr_regs_access(struct kvm_device *dev,
+                                 struct kvm_device_attr *attr,
+                                 u32 *reg, bool is_write)
+{
+        const struct kvm_mmio_range *r = NULL, *ranges;
+        phys_addr_t offset;
+        int ret, cpuid, c;
+        struct kvm_vcpu *vcpu, *tmp_vcpu;
+        struct vgic_dist *vgic;
+        struct kvm_exit_mmio mmio;
+
+        offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
+        cpuid = (attr->attr & KVM_DEV_ARM_VGIC_CPUID_MASK) >>
+                KVM_DEV_ARM_VGIC_CPUID_SHIFT;
+
+        mutex_lock(&dev->kvm->lock);
+
+        ret = vgic_init(dev->kvm);
+        if (ret)
+                goto out;
+
+        if (cpuid >= atomic_read(&dev->kvm->online_vcpus)) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        vcpu = kvm_get_vcpu(dev->kvm, cpuid);
+        vgic = &dev->kvm->arch.vgic;
+
+        mmio.len = 4;
+        mmio.is_write = is_write;
+        if (is_write)
+                mmio_data_write(&mmio, ~0, *reg);
+        switch (attr->group) {
+        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
+                mmio.phys_addr = vgic->vgic_dist_base + offset;
+                ranges = vgic_dist_ranges;
+                break;
+        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
+                mmio.phys_addr = vgic->vgic_cpu_base + offset;
+                ranges = vgic_cpu_ranges;
+                break;
+        default:
+                BUG();
+        }
+        r = vgic_find_range(ranges, &mmio, offset);
+
+        if (unlikely(!r || !r->handle_mmio)) {
+                ret = -ENXIO;
+                goto out;
+        }
+
+
+        spin_lock(&vgic->lock);
+
+        /*
+         * Ensure that no other VCPU is running by checking the vcpu->cpu
+         * field.  If no other VPCUs are running we can safely access the VGIC
+         * state, because even if another VPU is run after this point, that
+         * VCPU will not touch the vgic state, because it will block on
+         * getting the vgic->lock in kvm_vgic_sync_hwstate().
+         */
+        kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm) {
+                if (unlikely(tmp_vcpu->cpu != -1)) {
+                        ret = -EBUSY;
+                        goto out_vgic_unlock;
+                }
+        }
+
+        /*
+         * Move all pending IRQs from the LRs on all VCPUs so the pending
+         * state can be properly represented in the register state accessible
+         * through this API.
+         */
+        kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm)
+                vgic_unqueue_irqs(tmp_vcpu);
+
+        offset -= r->base;
+        r->handle_mmio(vcpu, &mmio, offset);
+
+        if (!is_write)
+                *reg = mmio_data_read(&mmio, ~0);
+
+        ret = 0;
+out_vgic_unlock:
+        spin_unlock(&vgic->lock);
+out:
+        mutex_unlock(&dev->kvm->lock);
+        return ret;
+}
+
+static int vgic_v2_create(struct kvm_device *dev, u32 type)
+{
+        return kvm_vgic_create(dev->kvm, type);
+}
+
+static void vgic_v2_destroy(struct kvm_device *dev)
+{
+        kfree(dev);
+}
+
+static int vgic_v2_set_attr(struct kvm_device *dev,
+                            struct kvm_device_attr *attr)
+{
+        int ret;
+
+        ret = vgic_set_common_attr(dev, attr);
+        if (ret != -ENXIO)
+                return ret;
+
+        switch (attr->group) {
+        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
+        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
+                u32 __user *uaddr = (u32 __user *)(long)attr->addr;
+                u32 reg;
+
+                if (get_user(reg, uaddr))
+                        return -EFAULT;
+
+                return vgic_attr_regs_access(dev, attr, &reg, true);
+        }
+
+        }
+
+        return -ENXIO;
+}
+
+static int vgic_v2_get_attr(struct kvm_device *dev,
+                            struct kvm_device_attr *attr)
+{
+        int ret;
+
+        ret = vgic_get_common_attr(dev, attr);
+        if (ret != -ENXIO)
+                return ret;
+
+        switch (attr->group) {
+        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
+        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
+                u32 __user *uaddr = (u32 __user *)(long)attr->addr;
+                u32 reg = 0;
+
+                ret = vgic_attr_regs_access(dev, attr, &reg, false);
+                if (ret)
+                        return ret;
+                return put_user(reg, uaddr);
+        }
+
+        }
+
+        return -ENXIO;
+}
+
+static int vgic_v2_has_attr(struct kvm_device *dev,
+                            struct kvm_device_attr *attr)
+{
+        phys_addr_t offset;
+
+        switch (attr->group) {
+        case KVM_DEV_ARM_VGIC_GRP_ADDR:
+                switch (attr->attr) {
+                case KVM_VGIC_V2_ADDR_TYPE_DIST:
+                case KVM_VGIC_V2_ADDR_TYPE_CPU:
+                        return 0;
+                }
+                break;
+        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
+                offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
+                return vgic_has_attr_regs(vgic_dist_ranges, offset);
+        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
+                offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
+                return vgic_has_attr_regs(vgic_cpu_ranges, offset);
+        case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
+                return 0;
+        case KVM_DEV_ARM_VGIC_GRP_CTRL:
+                switch (attr->attr) {
+                case KVM_DEV_ARM_VGIC_CTRL_INIT:
+                        return 0;
+                }
+        }
+        return -ENXIO;
+}
+
+struct kvm_device_ops kvm_arm_vgic_v2_ops = {
+        .name = "kvm-arm-vgic-v2",
+        .create = vgic_v2_create,
+        .destroy = vgic_v2_destroy,
+        .set_attr = vgic_v2_set_attr,
+        .get_attr = vgic_v2_get_attr,
+        .has_attr = vgic_v2_has_attr,
+};
diff --git a/virt/kvm/arm/vgic-v2.c b/virt/kvm/arm/vgic-v2.c
index 2935405ad22f..a0a7b5d1a070 100644
--- a/virt/kvm/arm/vgic-v2.c
+++ b/virt/kvm/arm/vgic-v2.c
@@ -229,12 +229,16 @@ int vgic_v2_probe(struct device_node *vgic_node,
                 goto out_unmap;
         }
 
+        vgic->can_emulate_gicv2 = true;
+        kvm_register_device_ops(&kvm_arm_vgic_v2_ops, KVM_DEV_TYPE_ARM_VGIC_V2);
+
         vgic->vcpu_base = vcpu_res.start;
 
         kvm_info("%s@%llx IRQ%d\n", vgic_node->name,
                  vctrl_res.start, vgic->maint_irq);
 
         vgic->type = VGIC_V2;
+        vgic->max_gic_vcpus = VGIC_V2_MAX_CPUS;
         *ops = &vgic_v2_ops;
         *params = vgic;
         goto out;
diff --git a/virt/kvm/arm/vgic-v3-emul.c b/virt/kvm/arm/vgic-v3-emul.c
new file mode 100644
index 000000000000..b3f154631515
--- /dev/null
+++ b/virt/kvm/arm/vgic-v3-emul.c
@@ -0,0 +1,1036 @@
+/*
+ * GICv3 distributor and redistributor emulation
+ *
+ * GICv3 emulation is currently only supported on a GICv3 host (because
+ * we rely on the hardware's CPU interface virtualization support), but
+ * supports both hardware with or without the optional GICv2 backwards
+ * compatibility features.
+ *
+ * Limitations of the emulation:
+ * (RAZ/WI: read as zero, write ignore, RAO/WI: read as one, write ignore)
+ * - We do not support LPIs (yet). TYPER.LPIS is reported as 0 and is RAZ/WI.
+ * - We do not support the message based interrupts (MBIs) triggered by
+ *   writes to the GICD_{SET,CLR}SPI_* registers. TYPER.MBIS is reported as 0.
+ * - We do not support the (optional) backwards compatibility feature.
+ *   GICD_CTLR.ARE resets to 1 and is RAO/WI. If the _host_ GIC supports
+ *   the compatiblity feature, you can use a GICv2 in the guest, though.
+ * - We only support a single security state. GICD_CTLR.DS is 1 and is RAO/WI.
+ * - Priorities are not emulated (same as the GICv2 emulation). Linux
+ *   as a guest is fine with this, because it does not use priorities.
+ * - We only support Group1 interrupts. Again Linux uses only those.
+ *
+ * Copyright (C) 2014 ARM Ltd.
+ * Author: Andre Przywara <andre.przywara@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/cpu.h>
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <linux/interrupt.h>
+
+#include <linux/irqchip/arm-gic-v3.h>
+#include <kvm/arm_vgic.h>
+
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_arm.h>
+#include <asm/kvm_mmu.h>
+
+#include "vgic.h"
+
+static bool handle_mmio_rao_wi(struct kvm_vcpu *vcpu,
+                               struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        u32 reg = 0xffffffff;
+
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+
+        return false;
+}
+
+static bool handle_mmio_ctlr(struct kvm_vcpu *vcpu,
+                             struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        u32 reg = 0;
+
+        /*
+         * Force ARE and DS to 1, the guest cannot change this.
+         * For the time being we only support Group1 interrupts.
+         */
+        if (vcpu->kvm->arch.vgic.enabled)
+                reg = GICD_CTLR_ENABLE_SS_G1;
+        reg |= GICD_CTLR_ARE_NS | GICD_CTLR_DS;
+
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
+        if (mmio->is_write) {
+                if (reg & GICD_CTLR_ENABLE_SS_G0)
+                        kvm_info("guest tried to enable unsupported Group0 interrupts\n");
+                vcpu->kvm->arch.vgic.enabled = !!(reg & GICD_CTLR_ENABLE_SS_G1);
+                vgic_update_state(vcpu->kvm);
+                return true;
+        }
+        return false;
+}
+
+/*
+ * As this implementation does not provide compatibility
+ * with GICv2 (ARE==1), we report zero CPUs in bits [5..7].
+ * Also LPIs and MBIs are not supported, so we set the respective bits to 0.
+ * Also we report at most 2**10=1024 interrupt IDs (to match 1024 SPIs).
+ */
+#define INTERRUPT_ID_BITS 10
+static bool handle_mmio_typer(struct kvm_vcpu *vcpu,
+                              struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        u32 reg;
+
+        reg = (min(vcpu->kvm->arch.vgic.nr_irqs, 1024) >> 5) - 1;
+
+        reg |= (INTERRUPT_ID_BITS - 1) << 19;
+
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+
+        return false;
+}
+
+static bool handle_mmio_iidr(struct kvm_vcpu *vcpu,
+                             struct kvm_exit_mmio *mmio, phys_addr_t offset)
+{
+        u32 reg;
+
+        reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+
+        return false;
+}
+
+static bool handle_mmio_set_enable_reg_dist(struct kvm_vcpu *vcpu,
+                                            struct kvm_exit_mmio *mmio,
+                                            phys_addr_t offset)
+{
+        if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
+                return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
+                                              vcpu->vcpu_id,
+                                              ACCESS_WRITE_SETBIT);
+
+        vgic_reg_access(mmio, NULL, offset,
+                        ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+        return false;
+}
+
+static bool handle_mmio_clear_enable_reg_dist(struct kvm_vcpu *vcpu,
+                                              struct kvm_exit_mmio *mmio,
+                                              phys_addr_t offset)
+{
+        if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
+                return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
+                                              vcpu->vcpu_id,
+                                              ACCESS_WRITE_CLEARBIT);
+
+        vgic_reg_access(mmio, NULL, offset,
+                        ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+        return false;
+}
+
+static bool handle_mmio_set_pending_reg_dist(struct kvm_vcpu *vcpu,
+                                             struct kvm_exit_mmio *mmio,
+                                             phys_addr_t offset)
+{
+        if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
+                return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
+                                                   vcpu->vcpu_id);
+
+        vgic_reg_access(mmio, NULL, offset,
+                        ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+        return false;
+}
+
+static bool handle_mmio_clear_pending_reg_dist(struct kvm_vcpu *vcpu,
+                                               struct kvm_exit_mmio *mmio,
+                                               phys_addr_t offset)
+{
+        if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
+                return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
+                                                     vcpu->vcpu_id);
+
+        vgic_reg_access(mmio, NULL, offset,
+                        ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+        return false;
+}
+
+static bool handle_mmio_priority_reg_dist(struct kvm_vcpu *vcpu,
+                                          struct kvm_exit_mmio *mmio,
+                                          phys_addr_t offset)
+{
+        u32 *reg;
+
+        if (unlikely(offset < VGIC_NR_PRIVATE_IRQS)) {
+                vgic_reg_access(mmio, NULL, offset,
+                                ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+                return false;
+        }
+
+        reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
+                                   vcpu->vcpu_id, offset);
+        vgic_reg_access(mmio, reg, offset,
+                ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
+        return false;
+}
+
+static bool handle_mmio_cfg_reg_dist(struct kvm_vcpu *vcpu,
+                                     struct kvm_exit_mmio *mmio,
+                                     phys_addr_t offset)
+{
+        u32 *reg;
+
+        if (unlikely(offset < VGIC_NR_PRIVATE_IRQS / 4)) {
+                vgic_reg_access(mmio, NULL, offset,
+                                ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+                return false;
+        }
+
+        reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
+                                  vcpu->vcpu_id, offset >> 1);
+
+        return vgic_handle_cfg_reg(reg, mmio, offset);
+}
+
+/*
+ * We use a compressed version of the MPIDR (all 32 bits in one 32-bit word)
+ * when we store the target MPIDR written by the guest.
+ */
+static u32 compress_mpidr(unsigned long mpidr)
+{
+        u32 ret;
+
+        ret = MPIDR_AFFINITY_LEVEL(mpidr, 0);
+        ret |= MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8;
+        ret |= MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16;
+        ret |= MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24;
+
+        return ret;
+}
+
+static unsigned long uncompress_mpidr(u32 value)
+{
+        unsigned long mpidr;
+
+        mpidr  = ((value >>  0) & 0xFF) << MPIDR_LEVEL_SHIFT(0);
+        mpidr |= ((value >>  8) & 0xFF) << MPIDR_LEVEL_SHIFT(1);
+        mpidr |= ((value >> 16) & 0xFF) << MPIDR_LEVEL_SHIFT(2);
+        mpidr |= (u64)((value >> 24) & 0xFF) << MPIDR_LEVEL_SHIFT(3);
+
+        return mpidr;
+}
+
+/*
+ * Lookup the given MPIDR value to get the vcpu_id (if there is one)
+ * and store that in the irq_spi_cpu[] array.
+ * This limits the number of VCPUs to 255 for now, extending the data
+ * type (or storing kvm_vcpu pointers) should lift the limit.
+ * Store the original MPIDR value in an extra array to support read-as-written.
+ * Unallocated MPIDRs are translated to a special value and caught
+ * before any array accesses.
+ */
+static bool handle_mmio_route_reg(struct kvm_vcpu *vcpu,
+                                  struct kvm_exit_mmio *mmio,
+                                  phys_addr_t offset)
+{
+        struct kvm *kvm = vcpu->kvm;
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        int spi;
+        u32 reg;
+        int vcpu_id;
+        unsigned long *bmap, mpidr;
+
+        /*
+         * The upper 32 bits of each 64 bit register are zero,
+         * as we don't support Aff3.
+         */
+        if ((offset & 4)) {
+                vgic_reg_access(mmio, NULL, offset,
+                                ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+                return false;
+        }
+
+        /* This region only covers SPIs, so no handling of private IRQs here. */
+        spi = offset / 8;
+
+        /* get the stored MPIDR for this IRQ */
+        mpidr = uncompress_mpidr(dist->irq_spi_mpidr[spi]);
+        reg = mpidr;
+
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
+
+        if (!mmio->is_write)
+                return false;
+
+        /*
+         * Now clear the currently assigned vCPU from the map, making room
+         * for the new one to be written below
+         */
+        vcpu = kvm_mpidr_to_vcpu(kvm, mpidr);
+        if (likely(vcpu)) {
+                vcpu_id = vcpu->vcpu_id;
+                bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]);
+                __clear_bit(spi, bmap);
+        }
+
+        dist->irq_spi_mpidr[spi] = compress_mpidr(reg);
+        vcpu = kvm_mpidr_to_vcpu(kvm, reg & MPIDR_HWID_BITMASK);
+
+        /*
+         * The spec says that non-existent MPIDR values should not be
+         * forwarded to any existent (v)CPU, but should be able to become
+         * pending anyway. We simply keep the irq_spi_target[] array empty, so
+         * the interrupt will never be injected.
+         * irq_spi_cpu[irq] gets a magic value in this case.
+         */
+        if (likely(vcpu)) {
+                vcpu_id = vcpu->vcpu_id;
+                dist->irq_spi_cpu[spi] = vcpu_id;
+                bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]);
+                __set_bit(spi, bmap);
+        } else {
+                dist->irq_spi_cpu[spi] = VCPU_NOT_ALLOCATED;
+        }
+
+        vgic_update_state(kvm);
+
+        return true;
+}
+
+/*
+ * We should be careful about promising too much when a guest reads
+ * this register. Don't claim to be like any hardware implementation,
+ * but just report the GIC as version 3 - which is what a Linux guest
+ * would check.
+ */
+static bool handle_mmio_idregs(struct kvm_vcpu *vcpu,
+                               struct kvm_exit_mmio *mmio,
+                               phys_addr_t offset)
+{
+        u32 reg = 0;
+
+        switch (offset + GICD_IDREGS) {
+        case GICD_PIDR2:
+                reg = 0x3b;
+                break;
+        }
+
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+
+        return false;
+}
+
+static const struct kvm_mmio_range vgic_v3_dist_ranges[] = {
+        {
+                .base           = GICD_CTLR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_ctlr,
+        },
+        {
+                .base           = GICD_TYPER,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_typer,
+        },
+        {
+                .base           = GICD_IIDR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_iidr,
+        },
+        {
+                /* this register is optional, it is RAZ/WI if not implemented */
+                .base           = GICD_STATUSR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                /* this write only register is WI when TYPER.MBIS=0 */
+                .base           = GICD_SETSPI_NSR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                /* this write only register is WI when TYPER.MBIS=0 */
+                .base           = GICD_CLRSPI_NSR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                /* this is RAZ/WI when DS=1 */
+                .base           = GICD_SETSPI_SR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                /* this is RAZ/WI when DS=1 */
+                .base           = GICD_CLRSPI_SR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICD_IGROUPR,
+                .len            = 0x80,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_rao_wi,
+        },
+        {
+                .base           = GICD_ISENABLER,
+                .len            = 0x80,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_set_enable_reg_dist,
+        },
+        {
+                .base           = GICD_ICENABLER,
+                .len            = 0x80,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_clear_enable_reg_dist,
+        },
+        {
+                .base           = GICD_ISPENDR,
+                .len            = 0x80,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_set_pending_reg_dist,
+        },
+        {
+                .base           = GICD_ICPENDR,
+                .len            = 0x80,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_clear_pending_reg_dist,
+        },
+        {
+                .base           = GICD_ISACTIVER,
+                .len            = 0x80,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICD_ICACTIVER,
+                .len            = 0x80,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICD_IPRIORITYR,
+                .len            = 0x400,
+                .bits_per_irq   = 8,
+                .handle_mmio    = handle_mmio_priority_reg_dist,
+        },
+        {
+                /* TARGETSRn is RES0 when ARE=1 */
+                .base           = GICD_ITARGETSR,
+                .len            = 0x400,
+                .bits_per_irq   = 8,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICD_ICFGR,
+                .len            = 0x100,
+                .bits_per_irq   = 2,
+                .handle_mmio    = handle_mmio_cfg_reg_dist,
+        },
+        {
+                /* this is RAZ/WI when DS=1 */
+                .base           = GICD_IGRPMODR,
+                .len            = 0x80,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                /* this is RAZ/WI when DS=1 */
+                .base           = GICD_NSACR,
+                .len            = 0x100,
+                .bits_per_irq   = 2,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                /* this is RAZ/WI when ARE=1 */
+                .base           = GICD_SGIR,
+                .len            = 0x04,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                /* this is RAZ/WI when ARE=1 */
+                .base           = GICD_CPENDSGIR,
+                .len            = 0x10,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                /* this is RAZ/WI when ARE=1 */
+                .base           = GICD_SPENDSGIR,
+                .len            = 0x10,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICD_IROUTER + 0x100,
+                .len            = 0x1ee0,
+                .bits_per_irq   = 64,
+                .handle_mmio    = handle_mmio_route_reg,
+        },
+        {
+                .base           = GICD_IDREGS,
+                .len            = 0x30,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_idregs,
+        },
+        {},
+};
+
+static bool handle_mmio_set_enable_reg_redist(struct kvm_vcpu *vcpu,
+                                              struct kvm_exit_mmio *mmio,
+                                              phys_addr_t offset)
+{
+        struct kvm_vcpu *redist_vcpu = mmio->private;
+
+        return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
+                                      redist_vcpu->vcpu_id,
+                                      ACCESS_WRITE_SETBIT);
+}
+
+static bool handle_mmio_clear_enable_reg_redist(struct kvm_vcpu *vcpu,
+                                                struct kvm_exit_mmio *mmio,
+                                                phys_addr_t offset)
+{
+        struct kvm_vcpu *redist_vcpu = mmio->private;
+
+        return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
+                                      redist_vcpu->vcpu_id,
+                                      ACCESS_WRITE_CLEARBIT);
+}
+
+static bool handle_mmio_set_pending_reg_redist(struct kvm_vcpu *vcpu,
+                                               struct kvm_exit_mmio *mmio,
+                                               phys_addr_t offset)
+{
+        struct kvm_vcpu *redist_vcpu = mmio->private;
+
+        return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
+                                           redist_vcpu->vcpu_id);
+}
+
+static bool handle_mmio_clear_pending_reg_redist(struct kvm_vcpu *vcpu,
+                                                 struct kvm_exit_mmio *mmio,
+                                                 phys_addr_t offset)
+{
+        struct kvm_vcpu *redist_vcpu = mmio->private;
+
+        return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
+                                             redist_vcpu->vcpu_id);
+}
+
+static bool handle_mmio_priority_reg_redist(struct kvm_vcpu *vcpu,
+                                            struct kvm_exit_mmio *mmio,
+                                            phys_addr_t offset)
+{
+        struct kvm_vcpu *redist_vcpu = mmio->private;
+        u32 *reg;
+
+        reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
+                                   redist_vcpu->vcpu_id, offset);
+        vgic_reg_access(mmio, reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
+        return false;
+}
+
+static bool handle_mmio_cfg_reg_redist(struct kvm_vcpu *vcpu,
+                                       struct kvm_exit_mmio *mmio,
+                                       phys_addr_t offset)
+{
+        struct kvm_vcpu *redist_vcpu = mmio->private;
+
+        u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
+                                       redist_vcpu->vcpu_id, offset >> 1);
+
+        return vgic_handle_cfg_reg(reg, mmio, offset);
+}
+
+static const struct kvm_mmio_range vgic_redist_sgi_ranges[] = {
+        {
+                .base           = GICR_IGROUPR0,
+                .len            = 0x04,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_rao_wi,
+        },
+        {
+                .base           = GICR_ISENABLER0,
+                .len            = 0x04,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_set_enable_reg_redist,
+        },
+        {
+                .base           = GICR_ICENABLER0,
+                .len            = 0x04,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_clear_enable_reg_redist,
+        },
+        {
+                .base           = GICR_ISPENDR0,
+                .len            = 0x04,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_set_pending_reg_redist,
+        },
+        {
+                .base           = GICR_ICPENDR0,
+                .len            = 0x04,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_clear_pending_reg_redist,
+        },
+        {
+                .base           = GICR_ISACTIVER0,
+                .len            = 0x04,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICR_ICACTIVER0,
+                .len            = 0x04,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICR_IPRIORITYR0,
+                .len            = 0x20,
+                .bits_per_irq   = 8,
+                .handle_mmio    = handle_mmio_priority_reg_redist,
+        },
+        {
+                .base           = GICR_ICFGR0,
+                .len            = 0x08,
+                .bits_per_irq   = 2,
+                .handle_mmio    = handle_mmio_cfg_reg_redist,
+        },
+        {
+                .base           = GICR_IGRPMODR0,
+                .len            = 0x04,
+                .bits_per_irq   = 1,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICR_NSACR,
+                .len            = 0x04,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {},
+};
+
+static bool handle_mmio_ctlr_redist(struct kvm_vcpu *vcpu,
+                                    struct kvm_exit_mmio *mmio,
+                                    phys_addr_t offset)
+{
+        /* since we don't support LPIs, this register is zero for now */
+        vgic_reg_access(mmio, NULL, offset,
+                        ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+        return false;
+}
+
+static bool handle_mmio_typer_redist(struct kvm_vcpu *vcpu,
+                                     struct kvm_exit_mmio *mmio,
+                                     phys_addr_t offset)
+{
+        u32 reg;
+        u64 mpidr;
+        struct kvm_vcpu *redist_vcpu = mmio->private;
+        int target_vcpu_id = redist_vcpu->vcpu_id;
+
+        /* the upper 32 bits contain the affinity value */
+        if ((offset & ~3) == 4) {
+                mpidr = kvm_vcpu_get_mpidr_aff(redist_vcpu);
+                reg = compress_mpidr(mpidr);
+
+                vgic_reg_access(mmio, &reg, offset,
+                                ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+                return false;
+        }
+
+        reg = redist_vcpu->vcpu_id << 8;
+        if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1)
+                reg |= GICR_TYPER_LAST;
+        vgic_reg_access(mmio, &reg, offset,
+                        ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
+        return false;
+}
+
+static const struct kvm_mmio_range vgic_redist_ranges[] = {
+        {
+                .base           = GICR_CTLR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_ctlr_redist,
+        },
+        {
+                .base           = GICR_TYPER,
+                .len            = 0x08,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_typer_redist,
+        },
+        {
+                .base           = GICR_IIDR,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_iidr,
+        },
+        {
+                .base           = GICR_WAKER,
+                .len            = 0x04,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_raz_wi,
+        },
+        {
+                .base           = GICR_IDREGS,
+                .len            = 0x30,
+                .bits_per_irq   = 0,
+                .handle_mmio    = handle_mmio_idregs,
+        },
+        {},
+};
+
+/*
+ * This function splits accesses between the distributor and the two
+ * redistributor parts (private/SPI). As each redistributor is accessible
+ * from any CPU, we have to determine the affected VCPU by taking the faulting
+ * address into account. We then pass this VCPU to the handler function via
+ * the private parameter.
+ */
+#define SGI_BASE_OFFSET SZ_64K
+static bool vgic_v3_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
+                                struct kvm_exit_mmio *mmio)
+{
+        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+        unsigned long dbase = dist->vgic_dist_base;
+        unsigned long rdbase = dist->vgic_redist_base;
+        int nrcpus = atomic_read(&vcpu->kvm->online_vcpus);
+        int vcpu_id;
+        const struct kvm_mmio_range *mmio_range;
+
+        if (is_in_range(mmio->phys_addr, mmio->len, dbase, GIC_V3_DIST_SIZE)) {
+                return vgic_handle_mmio_range(vcpu, run, mmio,
+                                              vgic_v3_dist_ranges, dbase);
+        }
+
+        if (!is_in_range(mmio->phys_addr, mmio->len, rdbase,
+            GIC_V3_REDIST_SIZE * nrcpus))
+                return false;
+
+        vcpu_id = (mmio->phys_addr - rdbase) / GIC_V3_REDIST_SIZE;
+        rdbase += (vcpu_id * GIC_V3_REDIST_SIZE);
+        mmio->private = kvm_get_vcpu(vcpu->kvm, vcpu_id);
+
+        if (mmio->phys_addr >= rdbase + SGI_BASE_OFFSET) {
+                rdbase += SGI_BASE_OFFSET;
+                mmio_range = vgic_redist_sgi_ranges;
+        } else {
+                mmio_range = vgic_redist_ranges;
+        }
+        return vgic_handle_mmio_range(vcpu, run, mmio, mmio_range, rdbase);
+}
+
+static bool vgic_v3_queue_sgi(struct kvm_vcpu *vcpu, int irq)
+{
+        if (vgic_queue_irq(vcpu, 0, irq)) {
+                vgic_dist_irq_clear_pending(vcpu, irq);
+                vgic_cpu_irq_clear(vcpu, irq);
+                return true;
+        }
+
+        return false;
+}
+
+static int vgic_v3_map_resources(struct kvm *kvm,
+                                 const struct vgic_params *params)
+{
+        int ret = 0;
+        struct vgic_dist *dist = &kvm->arch.vgic;
+
+        if (!irqchip_in_kernel(kvm))
+                return 0;
+
+        mutex_lock(&kvm->lock);
+
+        if (vgic_ready(kvm))
+                goto out;
+
+        if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
+            IS_VGIC_ADDR_UNDEF(dist->vgic_redist_base)) {
+                kvm_err("Need to set vgic distributor addresses first\n");
+                ret = -ENXIO;
+                goto out;
+        }
+
+        /*
+         * For a VGICv3 we require the userland to explicitly initialize
+         * the VGIC before we need to use it.
+         */
+        if (!vgic_initialized(kvm)) {
+                ret = -EBUSY;
+                goto out;
+        }
+
+        kvm->arch.vgic.ready = true;
+out:
+        if (ret)
+                kvm_vgic_destroy(kvm);
+        mutex_unlock(&kvm->lock);
+        return ret;
+}
+
+static int vgic_v3_init_model(struct kvm *kvm)
+{
+        int i;
+        u32 mpidr;
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        int nr_spis = dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
+
+        dist->irq_spi_mpidr = kcalloc(nr_spis, sizeof(dist->irq_spi_mpidr[0]),
+                                      GFP_KERNEL);
+
+        if (!dist->irq_spi_mpidr)
+                return -ENOMEM;
+
+        /* Initialize the target VCPUs for each IRQ to VCPU 0 */
+        mpidr = compress_mpidr(kvm_vcpu_get_mpidr_aff(kvm_get_vcpu(kvm, 0)));
+        for (i = VGIC_NR_PRIVATE_IRQS; i < dist->nr_irqs; i++) {
+                dist->irq_spi_cpu[i - VGIC_NR_PRIVATE_IRQS] = 0;
+                dist->irq_spi_mpidr[i - VGIC_NR_PRIVATE_IRQS] = mpidr;
+                vgic_bitmap_set_irq_val(dist->irq_spi_target, 0, i, 1);
+        }
+
+        return 0;
+}
+
+/* GICv3 does not keep track of SGI sources anymore. */
+static void vgic_v3_add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
+{
+}
+
+void vgic_v3_init_emulation(struct kvm *kvm)
+{
+        struct vgic_dist *dist = &kvm->arch.vgic;
+
+        dist->vm_ops.handle_mmio = vgic_v3_handle_mmio;
+        dist->vm_ops.queue_sgi = vgic_v3_queue_sgi;
+        dist->vm_ops.add_sgi_source = vgic_v3_add_sgi_source;
+        dist->vm_ops.init_model = vgic_v3_init_model;
+        dist->vm_ops.map_resources = vgic_v3_map_resources;
+
+        kvm->arch.max_vcpus = KVM_MAX_VCPUS;
+}
+
+/*
+ * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI
+ * generation register ICC_SGI1R_EL1) with a given VCPU.
+ * If the VCPU's MPIDR matches, return the level0 affinity, otherwise
+ * return -1.
+ */
+static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu)
+{
+        unsigned long affinity;
+        int level0;
+
+        /*
+         * Split the current VCPU's MPIDR into affinity level 0 and the
+         * rest as this is what we have to compare against.
+         */
+        affinity = kvm_vcpu_get_mpidr_aff(vcpu);
+        level0 = MPIDR_AFFINITY_LEVEL(affinity, 0);
+        affinity &= ~MPIDR_LEVEL_MASK;
+
+        /* bail out if the upper three levels don't match */
+        if (sgi_aff != affinity)
+                return -1;
+
+        /* Is this VCPU's bit set in the mask ? */
+        if (!(sgi_cpu_mask & BIT(level0)))
+                return -1;
+
+        return level0;
+}
+
+#define SGI_AFFINITY_LEVEL(reg, level) \
+        ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \
+        >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
+
+/**
+ * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs
+ * @vcpu: The VCPU requesting a SGI
+ * @reg: The value written into the ICC_SGI1R_EL1 register by that VCPU
+ *
+ * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register.
+ * This will trap in sys_regs.c and call this function.
+ * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the
+ * target processors as well as a bitmask of 16 Aff0 CPUs.
+ * If the interrupt routing mode bit is not set, we iterate over all VCPUs to
+ * check for matching ones. If this bit is set, we signal all, but not the
+ * calling VCPU.
+ */
+void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg)
+{
+        struct kvm *kvm = vcpu->kvm;
+        struct kvm_vcpu *c_vcpu;
+        struct vgic_dist *dist = &kvm->arch.vgic;
+        u16 target_cpus;
+        u64 mpidr;
+        int sgi, c;
+        int vcpu_id = vcpu->vcpu_id;
+        bool broadcast;
+        int updated = 0;
+
+        sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT;
+        broadcast = reg & BIT(ICC_SGI1R_IRQ_ROUTING_MODE_BIT);
+        target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT;
+        mpidr = SGI_AFFINITY_LEVEL(reg, 3);
+        mpidr |= SGI_AFFINITY_LEVEL(reg, 2);
+        mpidr |= SGI_AFFINITY_LEVEL(reg, 1);
+
+        /*
+         * We take the dist lock here, because we come from the sysregs
+         * code path and not from the MMIO one (which already takes the lock).
+         */
+        spin_lock(&dist->lock);
+
+        /*
+         * We iterate over all VCPUs to find the MPIDRs matching the request.
+         * If we have handled one CPU, we clear it's bit to detect early
+         * if we are already finished. This avoids iterating through all
+         * VCPUs when most of the times we just signal a single VCPU.
+         */
+        kvm_for_each_vcpu(c, c_vcpu, kvm) {
+
+                /* Exit early if we have dealt with all requested CPUs */
+                if (!broadcast && target_cpus == 0)
+                        break;
+
+                 /* Don't signal the calling VCPU */
+                if (broadcast && c == vcpu_id)
+                        continue;
+
+                if (!broadcast) {
+                        int level0;
+
+                        level0 = match_mpidr(mpidr, target_cpus, c_vcpu);
+                        if (level0 == -1)
+                                continue;
+
+                        /* remove this matching VCPU from the mask */
+                        target_cpus &= ~BIT(level0);
+                }
+
+                /* Flag the SGI as pending */
+                vgic_dist_irq_set_pending(c_vcpu, sgi);
+                updated = 1;
+                kvm_debug("SGI%d from CPU%d to CPU%d\n", sgi, vcpu_id, c);
+        }
+        if (updated)
+                vgic_update_state(vcpu->kvm);
+        spin_unlock(&dist->lock);
+        if (updated)
+                vgic_kick_vcpus(vcpu->kvm);
+}
+
+static int vgic_v3_create(struct kvm_device *dev, u32 type)
+{
+        return kvm_vgic_create(dev->kvm, type);
+}
+
+static void vgic_v3_destroy(struct kvm_device *dev)
+{
+        kfree(dev);
+}
+
+static int vgic_v3_set_attr(struct kvm_device *dev,
+                            struct kvm_device_attr *attr)
+{
+        int ret;
+
+        ret = vgic_set_common_attr(dev, attr);
+        if (ret != -ENXIO)
+                return ret;
+
+        switch (attr->group) {
+        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
+        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
+                return -ENXIO;
+        }
+
+        return -ENXIO;
+}
+
+static int vgic_v3_get_attr(struct kvm_device *dev,
+                            struct kvm_device_attr *attr)
+{
+        int ret;
+
+        ret = vgic_get_common_attr(dev, attr);
+        if (ret != -ENXIO)
+                return ret;
+
+        switch (attr->group) {
+        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
+        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
+                return -ENXIO;
+        }
+
+        return -ENXIO;
+}
+
+static int vgic_v3_has_attr(struct kvm_device *dev,
+                            struct kvm_device_attr *attr)
+{
+        switch (attr->group) {
+        case KVM_DEV_ARM_VGIC_GRP_ADDR:
+                switch (attr->attr) {
+                case KVM_VGIC_V2_ADDR_TYPE_DIST:
+                case KVM_VGIC_V2_ADDR_TYPE_CPU:
+                        return -ENXIO;
+                case KVM_VGIC_V3_ADDR_TYPE_DIST:
+                case KVM_VGIC_V3_ADDR_TYPE_REDIST:
+                        return 0;
+                }
+                break;
+        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
+        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
+                return -ENXIO;
+        case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
+                return 0;
+        case KVM_DEV_ARM_VGIC_GRP_CTRL:
+                switch (attr->attr) {
+                case KVM_DEV_ARM_VGIC_CTRL_INIT:
+                        return 0;
+                }
+        }
+        return -ENXIO;
+}
+
+struct kvm_device_ops kvm_arm_vgic_v3_ops = {
+        .name = "kvm-arm-vgic-v3",
+        .create = vgic_v3_create,
+        .destroy = vgic_v3_destroy,
+        .set_attr = vgic_v3_set_attr,
+        .get_attr = vgic_v3_get_attr,
+        .has_attr = vgic_v3_has_attr,
+};
diff --git a/virt/kvm/arm/vgic-v3.c b/virt/kvm/arm/vgic-v3.c
index 1c2c8eef0599..3a62d8a9a2c6 100644
--- a/virt/kvm/arm/vgic-v3.c
+++ b/virt/kvm/arm/vgic-v3.c
@@ -34,6 +34,7 @@
 #define GICH_LR_VIRTUALID               (0x3ffUL << 0)
 #define GICH_LR_PHYSID_CPUID_SHIFT      (10)
 #define GICH_LR_PHYSID_CPUID            (7UL << GICH_LR_PHYSID_CPUID_SHIFT)
+#define ICH_LR_VIRTUALID_MASK           (BIT_ULL(32) - 1)
 
 /*
  * LRs are stored in reverse order in memory. make sure we index them
@@ -48,12 +49,17 @@ static struct vgic_lr vgic_v3_get_lr(const struct kvm_vcpu *vcpu, int lr)
         struct vgic_lr lr_desc;
         u64 val = vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[LR_INDEX(lr)];
 
-        lr_desc.irq     = val & GICH_LR_VIRTUALID;
-        if (lr_desc.irq <= 15)
-                lr_desc.source  = (val >> GICH_LR_PHYSID_CPUID_SHIFT) & 0x7;
+        if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
+                lr_desc.irq = val & ICH_LR_VIRTUALID_MASK;
         else
-                lr_desc.source = 0;
-        lr_desc.state   = 0;
+                lr_desc.irq = val & GICH_LR_VIRTUALID;
+
+        lr_desc.source = 0;
+        if (lr_desc.irq <= 15 &&
+            vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
+                lr_desc.source = (val >> GICH_LR_PHYSID_CPUID_SHIFT) & 0x7;
+
+        lr_desc.state = 0;
 
         if (val & ICH_LR_PENDING_BIT)
                 lr_desc.state |= LR_STATE_PENDING;
@@ -68,8 +74,20 @@ static struct vgic_lr vgic_v3_get_lr(const struct kvm_vcpu *vcpu, int lr)
 static void vgic_v3_set_lr(struct kvm_vcpu *vcpu, int lr,
                            struct vgic_lr lr_desc)
 {
-        u64 lr_val = (((u32)lr_desc.source << GICH_LR_PHYSID_CPUID_SHIFT) |
-                      lr_desc.irq);
+        u64 lr_val;
+
+        lr_val = lr_desc.irq;
+
+        /*
+         * Currently all guest IRQs are Group1, as Group0 would result
+         * in a FIQ in the guest, which it wouldn't expect.
+         * Eventually we want to make this configurable, so we may revisit
+         * this in the future.
+         */
+        if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
+                lr_val |= ICH_LR_GROUP;
+        else
+                lr_val |= (u32)lr_desc.source << GICH_LR_PHYSID_CPUID_SHIFT;
 
         if (lr_desc.state & LR_STATE_PENDING)
                 lr_val |= ICH_LR_PENDING_BIT;
@@ -145,15 +163,27 @@ static void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
 
 static void vgic_v3_enable(struct kvm_vcpu *vcpu)
 {
+        struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;
+
         /*
          * By forcing VMCR to zero, the GIC will restore the binary
          * points to their reset values. Anything else resets to zero
          * anyway.
          */
-        vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr = 0;
+        vgic_v3->vgic_vmcr = 0;
+
+        /*
+         * If we are emulating a GICv3, we do it in an non-GICv2-compatible
+         * way, so we force SRE to 1 to demonstrate this to the guest.
+         * This goes with the spec allowing the value to be RAO/WI.
+         */
+        if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
+                vgic_v3->vgic_sre = ICC_SRE_EL1_SRE;
+        else
+                vgic_v3->vgic_sre = 0;
 
         /* Get the show on the road... */
-        vcpu->arch.vgic_cpu.vgic_v3.vgic_hcr = ICH_HCR_EN;
+        vgic_v3->vgic_hcr = ICH_HCR_EN;
 }
 
 static const struct vgic_ops vgic_v3_ops = {
@@ -205,35 +235,37 @@ int vgic_v3_probe(struct device_node *vgic_node,
          * maximum of 16 list registers. Just ignore bit 4...
          */
         vgic->nr_lr = (ich_vtr_el2 & 0xf) + 1;
+        vgic->can_emulate_gicv2 = false;
 
         if (of_property_read_u32(vgic_node, "#redistributor-regions", &gicv_idx))
                 gicv_idx = 1;
 
         gicv_idx += 3; /* Also skip GICD, GICC, GICH */
         if (of_address_to_resource(vgic_node, gicv_idx, &vcpu_res)) {
-                kvm_err("Cannot obtain GICV region\n");
-                ret = -ENXIO;
-                goto out;
-        }
-
-        if (!PAGE_ALIGNED(vcpu_res.start)) {
-                kvm_err("GICV physical address 0x%llx not page aligned\n",
+                kvm_info("GICv3: no GICV resource entry\n");
+                vgic->vcpu_base = 0;
+        } else if (!PAGE_ALIGNED(vcpu_res.start)) {
+                pr_warn("GICV physical address 0x%llx not page aligned\n",
                         (unsigned long long)vcpu_res.start);
-                ret = -ENXIO;
-                goto out;
-        }
-
-        if (!PAGE_ALIGNED(resource_size(&vcpu_res))) {
-                kvm_err("GICV size 0x%llx not a multiple of page size 0x%lx\n",
+                vgic->vcpu_base = 0;
+        } else if (!PAGE_ALIGNED(resource_size(&vcpu_res))) {
+                pr_warn("GICV size 0x%llx not a multiple of page size 0x%lx\n",
                         (unsigned long long)resource_size(&vcpu_res),
                         PAGE_SIZE);
-                ret = -ENXIO;
-                goto out;
+                vgic->vcpu_base = 0;
+        } else {
+                vgic->vcpu_base = vcpu_res.start;
+                vgic->can_emulate_gicv2 = true;
+                kvm_register_device_ops(&kvm_arm_vgic_v2_ops,
+                                        KVM_DEV_TYPE_ARM_VGIC_V2);
         }
+        if (vgic->vcpu_base == 0)
+                kvm_info("disabling GICv2 emulation\n");
+        kvm_register_device_ops(&kvm_arm_vgic_v3_ops, KVM_DEV_TYPE_ARM_VGIC_V3);
 
-        vgic->vcpu_base = vcpu_res.start;
         vgic->vctrl_base = NULL;
         vgic->type = VGIC_V3;
+        vgic->max_gic_vcpus = KVM_MAX_VCPUS;
 
         kvm_info("%s@%llx IRQ%d\n", vgic_node->name,
                  vcpu_res.start, vgic->maint_irq);
diff --git a/virt/kvm/arm/vgic.c b/virt/kvm/arm/vgic.c
index 03affc7bf453..0cc6ab6005a0 100644
--- a/virt/kvm/arm/vgic.c
+++ b/virt/kvm/arm/vgic.c
@@ -75,37 +75,31 @@
  *   inactive as long as the external input line is held high.
  */
 
-#define VGIC_ADDR_UNDEF         (-1)
-#define IS_VGIC_ADDR_UNDEF(_x)  ((_x) == VGIC_ADDR_UNDEF)
-
-#define PRODUCT_ID_KVM          0x4b    /* ASCII code K */
-#define IMPLEMENTER_ARM         0x43b
-#define GICC_ARCH_VERSION_V2    0x2
-
-#define ACCESS_READ_VALUE       (1 << 0)
-#define ACCESS_READ_RAZ         (0 << 0)
-#define ACCESS_READ_MASK(x)     ((x) & (1 << 0))
-#define ACCESS_WRITE_IGNORED    (0 << 1)
-#define ACCESS_WRITE_SETBIT     (1 << 1)
-#define ACCESS_WRITE_CLEARBIT   (2 << 1)
-#define ACCESS_WRITE_VALUE      (3 << 1)
-#define ACCESS_WRITE_MASK(x)    ((x) & (3 << 1))
-
-static int vgic_init(struct kvm *kvm);
+#include "vgic.h"
+
 static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu);
 static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu);
-static void vgic_update_state(struct kvm *kvm);
-static void vgic_kick_vcpus(struct kvm *kvm);
-static u8 *vgic_get_sgi_sources(struct vgic_dist *dist, int vcpu_id, int sgi);
-static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg);
 static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr);
 static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr, struct vgic_lr lr_desc);
-static void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
-static void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
 
 static const struct vgic_ops *vgic_ops;
 static const struct vgic_params *vgic;
 
+static void add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
+{
+        vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);
+}
+
+static bool queue_sgi(struct kvm_vcpu *vcpu, int irq)
+{
+        return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);
+}
+
+int kvm_vgic_map_resources(struct kvm *kvm)
+{
+        return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);
+}
+
 /*
  * struct vgic_bitmap contains a bitmap made of unsigned longs, but
  * extracts u32s out of them.
@@ -160,8 +154,7 @@ static unsigned long *u64_to_bitmask(u64 *val)
         return (unsigned long *)val;
 }
 
-static u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x,
-                                int cpuid, u32 offset)
+u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset)
 {
         offset >>= 2;
         if (!offset)
@@ -179,8 +172,8 @@ static int vgic_bitmap_get_irq_val(struct vgic_bitmap *x,
         return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared);
 }
 
-static void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
-                                    int irq, int val)
+void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
+                             int irq, int val)
 {
         unsigned long *reg;
 
@@ -202,7 +195,7 @@ static unsigned long *vgic_bitmap_get_cpu_map(struct vgic_bitmap *x, int cpuid)
         return x->private + cpuid;
 }
 
-static unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
+unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
 {
         return x->shared;
 }
@@ -229,7 +222,7 @@ static void vgic_free_bytemap(struct vgic_bytemap *b)
         b->shared = NULL;
 }
 
-static u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
+u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
 {
         u32 *reg;
 
@@ -326,14 +319,14 @@ static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)
         return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq);
 }
 
-static void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)
+void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)
 {
         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 
         vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1);
 }
 
-static void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)
+void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)
 {
         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 
@@ -349,7 +342,7 @@ static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)
                         vcpu->arch.vgic_cpu.pending_shared);
 }
 
-static void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
+void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
 {
         if (irq < VGIC_NR_PRIVATE_IRQS)
                 clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
@@ -363,16 +356,6 @@ static bool vgic_can_sample_irq(struct kvm_vcpu *vcpu, int irq)
         return vgic_irq_is_edge(vcpu, irq) || !vgic_irq_is_queued(vcpu, irq);
 }
 
-static u32 mmio_data_read(struct kvm_exit_mmio *mmio, u32 mask)
-{
-        return le32_to_cpu(*((u32 *)mmio->data)) & mask;
-}
-
-static void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
-{
-        *((u32 *)mmio->data) = cpu_to_le32(value) & mask;
-}
-
 /**
  * vgic_reg_access - access vgic register
  * @mmio:   pointer to the data describing the mmio access
@@ -384,8 +367,8 @@ static void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
  * modes defined for vgic register access
  * (read,raz,write-ignored,setbit,clearbit,write)
  */
-static void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
-                            phys_addr_t offset, int mode)
+void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
+                     phys_addr_t offset, int mode)
 {
         int word_offset = (offset & 3) * 8;
         u32 mask = (1UL << (mmio->len * 8)) - 1;
@@ -434,107 +417,58 @@ static void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
         }
 }
 
-static bool handle_mmio_misc(struct kvm_vcpu *vcpu,
-                             struct kvm_exit_mmio *mmio, phys_addr_t offset)
-{
-        u32 reg;
-        u32 word_offset = offset & 3;
-
-        switch (offset & ~3) {
-        case 0:                 /* GICD_CTLR */
-                reg = vcpu->kvm->arch.vgic.enabled;
-                vgic_reg_access(mmio, &reg, word_offset,
-                                ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
-                if (mmio->is_write) {
-                        vcpu->kvm->arch.vgic.enabled = reg & 1;
-                        vgic_update_state(vcpu->kvm);
-                        return true;
-                }
-                break;
-
-        case 4:                 /* GICD_TYPER */
-                reg  = (atomic_read(&vcpu->kvm->online_vcpus) - 1) << 5;
-                reg |= (vcpu->kvm->arch.vgic.nr_irqs >> 5) - 1;
-                vgic_reg_access(mmio, &reg, word_offset,
-                                ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
-                break;
-
-        case 8:                 /* GICD_IIDR */
-                reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
-                vgic_reg_access(mmio, &reg, word_offset,
-                                ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
-                break;
-        }
-
-        return false;
-}
-
-static bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu,
-                               struct kvm_exit_mmio *mmio, phys_addr_t offset)
+bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
+                        phys_addr_t offset)
 {
         vgic_reg_access(mmio, NULL, offset,
                         ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
         return false;
 }
 
-static bool handle_mmio_set_enable_reg(struct kvm_vcpu *vcpu,
-                                       struct kvm_exit_mmio *mmio,
-                                       phys_addr_t offset)
+bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
+                            phys_addr_t offset, int vcpu_id, int access)
 {
-        u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_enabled,
-                                       vcpu->vcpu_id, offset);
-        vgic_reg_access(mmio, reg, offset,
-                        ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
-        if (mmio->is_write) {
-                vgic_update_state(vcpu->kvm);
-                return true;
-        }
-
-        return false;
-}
+        u32 *reg;
+        int mode = ACCESS_READ_VALUE | access;
+        struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);
 
-static bool handle_mmio_clear_enable_reg(struct kvm_vcpu *vcpu,
-                                         struct kvm_exit_mmio *mmio,
-                                         phys_addr_t offset)
-{
-        u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_enabled,
-                                       vcpu->vcpu_id, offset);
-        vgic_reg_access(mmio, reg, offset,
-                        ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
+        reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);
+        vgic_reg_access(mmio, reg, offset, mode);
         if (mmio->is_write) {
-                if (offset < 4) /* Force SGI enabled */
-                        *reg |= 0xffff;
-                vgic_retire_disabled_irqs(vcpu);
-                vgic_update_state(vcpu->kvm);
+                if (access & ACCESS_WRITE_CLEARBIT) {
+                        if (offset < 4) /* Force SGI enabled */
+                                *reg |= 0xffff;
+                        vgic_retire_disabled_irqs(target_vcpu);
+                }
+                vgic_update_state(kvm);
                 return true;
         }
 
         return false;
 }
 
-static bool handle_mmio_set_pending_reg(struct kvm_vcpu *vcpu,
-                                        struct kvm_exit_mmio *mmio,
-                                        phys_addr_t offset)
+bool vgic_handle_set_pending_reg(struct kvm *kvm,
+                                 struct kvm_exit_mmio *mmio,
+                                 phys_addr_t offset, int vcpu_id)
 {
         u32 *reg, orig;
         u32 level_mask;
-        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+        int mode = ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT;
+        struct vgic_dist *dist = &kvm->arch.vgic;
 
-        reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu->vcpu_id, offset);
+        reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu_id, offset);
         level_mask = (~(*reg));
 
         /* Mark both level and edge triggered irqs as pending */
-        reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu->vcpu_id, offset);
+        reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
         orig = *reg;
-        vgic_reg_access(mmio, reg, offset,
-                        ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
+        vgic_reg_access(mmio, reg, offset, mode);
 
         if (mmio->is_write) {
                 /* Set the soft-pending flag only for level-triggered irqs */
                 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
-                                          vcpu->vcpu_id, offset);
-                vgic_reg_access(mmio, reg, offset,
-                                ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
+                                          vcpu_id, offset);
+                vgic_reg_access(mmio, reg, offset, mode);
                 *reg &= level_mask;
 
                 /* Ignore writes to SGIs */
@@ -543,31 +477,30 @@ static bool handle_mmio_set_pending_reg(struct kvm_vcpu *vcpu,
                         *reg |= orig & 0xffff;
                 }
 
-                vgic_update_state(vcpu->kvm);
+                vgic_update_state(kvm);
                 return true;
         }
 
         return false;
 }
 
-static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu,
-                                          struct kvm_exit_mmio *mmio,
-                                          phys_addr_t offset)
+bool vgic_handle_clear_pending_reg(struct kvm *kvm,
+                                   struct kvm_exit_mmio *mmio,
+                                   phys_addr_t offset, int vcpu_id)
 {
         u32 *level_active;
         u32 *reg, orig;
-        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+        int mode = ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT;
+        struct vgic_dist *dist = &kvm->arch.vgic;
 
-        reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu->vcpu_id, offset);
+        reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
         orig = *reg;
-        vgic_reg_access(mmio, reg, offset,
-                        ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
+        vgic_reg_access(mmio, reg, offset, mode);
         if (mmio->is_write) {
                 /* Re-set level triggered level-active interrupts */
                 level_active = vgic_bitmap_get_reg(&dist->irq_level,
-                                          vcpu->vcpu_id, offset);
-                reg = vgic_bitmap_get_reg(&dist->irq_pending,
-                                          vcpu->vcpu_id, offset);
+                                          vcpu_id, offset);
+                reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
                 *reg |= *level_active;
 
                 /* Ignore writes to SGIs */
@@ -578,101 +511,12 @@ static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu,
 
                 /* Clear soft-pending flags */
                 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
-                                          vcpu->vcpu_id, offset);
-                vgic_reg_access(mmio, reg, offset,
-                                ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
+                                          vcpu_id, offset);
+                vgic_reg_access(mmio, reg, offset, mode);
 
-                vgic_update_state(vcpu->kvm);
+                vgic_update_state(kvm);
                 return true;
         }
-
-        return false;
-}
-
-static bool handle_mmio_priority_reg(struct kvm_vcpu *vcpu,
-                                     struct kvm_exit_mmio *mmio,
-                                     phys_addr_t offset)
-{
-        u32 *reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
-                                        vcpu->vcpu_id, offset);
-        vgic_reg_access(mmio, reg, offset,
-                        ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
-        return false;
-}
-
-#define GICD_ITARGETSR_SIZE     32
-#define GICD_CPUTARGETS_BITS    8
-#define GICD_IRQS_PER_ITARGETSR (GICD_ITARGETSR_SIZE / GICD_CPUTARGETS_BITS)
-static u32 vgic_get_target_reg(struct kvm *kvm, int irq)
-{
-        struct vgic_dist *dist = &kvm->arch.vgic;
-        int i;
-        u32 val = 0;
-
-        irq -= VGIC_NR_PRIVATE_IRQS;
-
-        for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++)
-                val |= 1 << (dist->irq_spi_cpu[irq + i] + i * 8);
-
-        return val;
-}
-
-static void vgic_set_target_reg(struct kvm *kvm, u32 val, int irq)
-{
-        struct vgic_dist *dist = &kvm->arch.vgic;
-        struct kvm_vcpu *vcpu;
-        int i, c;
-        unsigned long *bmap;
-        u32 target;
-
-        irq -= VGIC_NR_PRIVATE_IRQS;
-
-        /*
-         * Pick the LSB in each byte. This ensures we target exactly
-         * one vcpu per IRQ. If the byte is null, assume we target
-         * CPU0.
-         */
-        for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++) {
-                int shift = i * GICD_CPUTARGETS_BITS;
-                target = ffs((val >> shift) & 0xffU);
-                target = target ? (target - 1) : 0;
-                dist->irq_spi_cpu[irq + i] = target;
-                kvm_for_each_vcpu(c, vcpu, kvm) {
-                        bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[c]);
-                        if (c == target)
-                                set_bit(irq + i, bmap);
-                        else
-                                clear_bit(irq + i, bmap);
-                }
-        }
-}
-
-static bool handle_mmio_target_reg(struct kvm_vcpu *vcpu,
-                                   struct kvm_exit_mmio *mmio,
-                                   phys_addr_t offset)
-{
-        u32 reg;
-
-        /* We treat the banked interrupts targets as read-only */
-        if (offset < 32) {
-                u32 roreg = 1 << vcpu->vcpu_id;
-                roreg |= roreg << 8;
-                roreg |= roreg << 16;
-
-                vgic_reg_access(mmio, &roreg, offset,
-                                ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
-                return false;
-        }
-
-        reg = vgic_get_target_reg(vcpu->kvm, offset & ~3U);
-        vgic_reg_access(mmio, &reg, offset,
-                        ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
-        if (mmio->is_write) {
-                vgic_set_target_reg(vcpu->kvm, reg, offset & ~3U);
-                vgic_update_state(vcpu->kvm);
-                return true;
-        }
-
         return false;
 }
 
@@ -711,14 +555,10 @@ static u16 vgic_cfg_compress(u32 val)
  * LSB is always 0. As such, we only keep the upper bit, and use the
  * two above functions to compress/expand the bits
  */
-static bool handle_mmio_cfg_reg(struct kvm_vcpu *vcpu,
-                                struct kvm_exit_mmio *mmio, phys_addr_t offset)
+bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
+                         phys_addr_t offset)
 {
         u32 val;
-        u32 *reg;
-
-        reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
-                                  vcpu->vcpu_id, offset >> 1);
 
         if (offset & 4)
                 val = *reg >> 16;
@@ -747,21 +587,6 @@ static bool handle_mmio_cfg_reg(struct kvm_vcpu *vcpu,
         return false;
 }
 
-static bool handle_mmio_sgi_reg(struct kvm_vcpu *vcpu,
-                                struct kvm_exit_mmio *mmio, phys_addr_t offset)
-{
-        u32 reg;
-        vgic_reg_access(mmio, &reg, offset,
-                        ACCESS_READ_RAZ | ACCESS_WRITE_VALUE);
-        if (mmio->is_write) {
-                vgic_dispatch_sgi(vcpu, reg);
-                vgic_update_state(vcpu->kvm);
-                return true;
-        }
-
-        return false;
-}
-
 /**
  * vgic_unqueue_irqs - move pending IRQs from LRs to the distributor
  * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs
@@ -774,11 +599,9 @@ static bool handle_mmio_sgi_reg(struct kvm_vcpu *vcpu,
  * to the distributor but the active state stays in the LRs, because we don't
  * track the active state on the distributor side.
  */
-static void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
+void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
 {
-        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
-        int vcpu_id = vcpu->vcpu_id;
         int i;
 
         for_each_set_bit(i, vgic_cpu->lr_used, vgic_cpu->nr_lr) {
@@ -805,7 +628,7 @@ static void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
                  */
                 vgic_dist_irq_set_pending(vcpu, lr.irq);
                 if (lr.irq < VGIC_NR_SGIS)
-                        *vgic_get_sgi_sources(dist, vcpu_id, lr.irq) |= 1 << lr.source;
+                        add_sgi_source(vcpu, lr.irq, lr.source);
                 lr.state &= ~LR_STATE_PENDING;
                 vgic_set_lr(vcpu, i, lr);
 
@@ -824,188 +647,12 @@ static void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
         }
 }
 
-/* Handle reads of GICD_CPENDSGIRn and GICD_SPENDSGIRn */
-static bool read_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
-                                        struct kvm_exit_mmio *mmio,
-                                        phys_addr_t offset)
-{
-        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
-        int sgi;
-        int min_sgi = (offset & ~0x3);
-        int max_sgi = min_sgi + 3;
-        int vcpu_id = vcpu->vcpu_id;
-        u32 reg = 0;
-
-        /* Copy source SGIs from distributor side */
-        for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
-                int shift = 8 * (sgi - min_sgi);
-                reg |= ((u32)*vgic_get_sgi_sources(dist, vcpu_id, sgi)) << shift;
-        }
-
-        mmio_data_write(mmio, ~0, reg);
-        return false;
-}
-
-static bool write_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
-                                         struct kvm_exit_mmio *mmio,
-                                         phys_addr_t offset, bool set)
-{
-        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
-        int sgi;
-        int min_sgi = (offset & ~0x3);
-        int max_sgi = min_sgi + 3;
-        int vcpu_id = vcpu->vcpu_id;
-        u32 reg;
-        bool updated = false;
-
-        reg = mmio_data_read(mmio, ~0);
-
-        /* Clear pending SGIs on the distributor */
-        for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
-                u8 mask = reg >> (8 * (sgi - min_sgi));
-                u8 *src = vgic_get_sgi_sources(dist, vcpu_id, sgi);
-                if (set) {
-                        if ((*src & mask) != mask)
-                                updated = true;
-                        *src |= mask;
-                } else {
-                        if (*src & mask)
-                                updated = true;
-                        *src &= ~mask;
-                }
-        }
-
-        if (updated)
-                vgic_update_state(vcpu->kvm);
-
-        return updated;
-}
-
-static bool handle_mmio_sgi_set(struct kvm_vcpu *vcpu,
-                                struct kvm_exit_mmio *mmio,
-                                phys_addr_t offset)
-{
-        if (!mmio->is_write)
-                return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
-        else
-                return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, true);
-}
-
-static bool handle_mmio_sgi_clear(struct kvm_vcpu *vcpu,
-                                  struct kvm_exit_mmio *mmio,
-                                  phys_addr_t offset)
-{
-        if (!mmio->is_write)
-                return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
-        else
-                return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, false);
-}
-
-/*
- * I would have liked to use the kvm_bus_io_*() API instead, but it
- * cannot cope with banked registers (only the VM pointer is passed
- * around, and we need the vcpu). One of these days, someone please
- * fix it!
- */
-struct mmio_range {
-        phys_addr_t base;
-        unsigned long len;
-        int bits_per_irq;
-        bool (*handle_mmio)(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
-                            phys_addr_t offset);
-};
-
-static const struct mmio_range vgic_dist_ranges[] = {
-        {
-                .base           = GIC_DIST_CTRL,
-                .len            = 12,
-                .bits_per_irq   = 0,
-                .handle_mmio    = handle_mmio_misc,
-        },
-        {
-                .base           = GIC_DIST_IGROUP,
-                .len            = VGIC_MAX_IRQS / 8,
-                .bits_per_irq   = 1,
-                .handle_mmio    = handle_mmio_raz_wi,
-        },
-        {
-                .base           = GIC_DIST_ENABLE_SET,
-                .len            = VGIC_MAX_IRQS / 8,
-                .bits_per_irq   = 1,
-                .handle_mmio    = handle_mmio_set_enable_reg,
-        },
-        {
-                .base           = GIC_DIST_ENABLE_CLEAR,
-                .len            = VGIC_MAX_IRQS / 8,
-                .bits_per_irq   = 1,
-                .handle_mmio    = handle_mmio_clear_enable_reg,
-        },
-        {
-                .base           = GIC_DIST_PENDING_SET,
-                .len            = VGIC_MAX_IRQS / 8,
-                .bits_per_irq   = 1,
-                .handle_mmio    = handle_mmio_set_pending_reg,
-        },
-        {
-                .base           = GIC_DIST_PENDING_CLEAR,
-                .len            = VGIC_MAX_IRQS / 8,
-                .bits_per_irq   = 1,
-                .handle_mmio    = handle_mmio_clear_pending_reg,
-        },
-        {
-                .base           = GIC_DIST_ACTIVE_SET,
-                .len            = VGIC_MAX_IRQS / 8,
-                .bits_per_irq   = 1,
-                .handle_mmio    = handle_mmio_raz_wi,
-        },
-        {
-                .base           = GIC_DIST_ACTIVE_CLEAR,
-                .len            = VGIC_MAX_IRQS / 8,
-                .bits_per_irq   = 1,
-                .handle_mmio    = handle_mmio_raz_wi,
-        },
-        {
-                .base           = GIC_DIST_PRI,
-                .len            = VGIC_MAX_IRQS,
-                .bits_per_irq   = 8,
-                .handle_mmio    = handle_mmio_priority_reg,
-        },
-        {
-                .base           = GIC_DIST_TARGET,
-                .len            = VGIC_MAX_IRQS,
-                .bits_per_irq   = 8,
-                .handle_mmio    = handle_mmio_target_reg,
-        },
-        {
-                .base           = GIC_DIST_CONFIG,
-                .len            = VGIC_MAX_IRQS / 4,
-                .bits_per_irq   = 2,
-                .handle_mmio    = handle_mmio_cfg_reg,
-        },
-        {
-                .base           = GIC_DIST_SOFTINT,
-                .len            = 4,
-                .handle_mmio    = handle_mmio_sgi_reg,
-        },
-        {
-                .base           = GIC_DIST_SGI_PENDING_CLEAR,
-                .len            = VGIC_NR_SGIS,
-                .handle_mmio    = handle_mmio_sgi_clear,
-        },
-        {
-                .base           = GIC_DIST_SGI_PENDING_SET,
-                .len            = VGIC_NR_SGIS,
-                .handle_mmio    = handle_mmio_sgi_set,
-        },
-        {}
-};
-
-static const
-struct mmio_range *find_matching_range(const struct mmio_range *ranges,
+const
+struct kvm_mmio_range *vgic_find_range(const struct kvm_mmio_range *ranges,
                                        struct kvm_exit_mmio *mmio,
                                        phys_addr_t offset)
 {
-        const struct mmio_range *r = ranges;
+        const struct kvm_mmio_range *r = ranges;
 
         while (r->len) {
                 if (offset >= r->base &&
@@ -1018,7 +665,7 @@ struct mmio_range *find_matching_range(const struct mmio_range *ranges,
 }
 
 static bool vgic_validate_access(const struct vgic_dist *dist,
-                                 const struct mmio_range *range,
+                                 const struct kvm_mmio_range *range,
                                  unsigned long offset)
 {
         int irq;
@@ -1033,37 +680,76 @@ static bool vgic_validate_access(const struct vgic_dist *dist,
         return true;
 }
 
+/*
+ * Call the respective handler function for the given range.
+ * We split up any 64 bit accesses into two consecutive 32 bit
+ * handler calls and merge the result afterwards.
+ * We do this in a little endian fashion regardless of the host's
+ * or guest's endianness, because the GIC is always LE and the rest of
+ * the code (vgic_reg_access) also puts it in a LE fashion already.
+ * At this point we have already identified the handle function, so
+ * range points to that one entry and offset is relative to this.
+ */
+static bool call_range_handler(struct kvm_vcpu *vcpu,
+                               struct kvm_exit_mmio *mmio,
+                               unsigned long offset,
+                               const struct kvm_mmio_range *range)
+{
+        u32 *data32 = (void *)mmio->data;
+        struct kvm_exit_mmio mmio32;
+        bool ret;
+
+        if (likely(mmio->len <= 4))
+                return range->handle_mmio(vcpu, mmio, offset);
+
+        /*
+         * Any access bigger than 4 bytes (that we currently handle in KVM)
+         * is actually 8 bytes long, caused by a 64-bit access
+         */
+
+        mmio32.len = 4;
+        mmio32.is_write = mmio->is_write;
+        mmio32.private = mmio->private;
+
+        mmio32.phys_addr = mmio->phys_addr + 4;
+        if (mmio->is_write)
+                *(u32 *)mmio32.data = data32[1];
+        ret = range->handle_mmio(vcpu, &mmio32, offset + 4);
+        if (!mmio->is_write)
+                data32[1] = *(u32 *)mmio32.data;
+
+        mmio32.phys_addr = mmio->phys_addr;
+        if (mmio->is_write)
+                *(u32 *)mmio32.data = data32[0];
+        ret |= range->handle_mmio(vcpu, &mmio32, offset);
+        if (!mmio->is_write)
+                data32[0] = *(u32 *)mmio32.data;
+
+        return ret;
+}
+
 /**
- * vgic_handle_mmio - handle an in-kernel MMIO access
+ * vgic_handle_mmio_range - handle an in-kernel MMIO access
  * @vcpu:       pointer to the vcpu performing the access
  * @run:        pointer to the kvm_run structure
  * @mmio:       pointer to the data describing the access
+ * @ranges:     array of MMIO ranges in a given region
+ * @mmio_base:  base address of that region
  *
- * returns true if the MMIO access has been performed in kernel space,
- * and false if it needs to be emulated in user space.
+ * returns true if the MMIO access could be performed
  */
-bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
-                      struct kvm_exit_mmio *mmio)
+bool vgic_handle_mmio_range(struct kvm_vcpu *vcpu, struct kvm_run *run,
+                            struct kvm_exit_mmio *mmio,
+                            const struct kvm_mmio_range *ranges,
+                            unsigned long mmio_base)
 {
-        const struct mmio_range *range;
+        const struct kvm_mmio_range *range;
         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
-        unsigned long base = dist->vgic_dist_base;
         bool updated_state;
         unsigned long offset;
 
-        if (!irqchip_in_kernel(vcpu->kvm) ||
-            mmio->phys_addr < base ||
-            (mmio->phys_addr + mmio->len) > (base + KVM_VGIC_V2_DIST_SIZE))
-                return false;
-
-        /* We don't support ldrd / strd or ldm / stm to the emulated vgic */
-        if (mmio->len > 4) {
-                kvm_inject_dabt(vcpu, mmio->phys_addr);
-                return true;
-        }
-
-        offset = mmio->phys_addr - base;
-        range = find_matching_range(vgic_dist_ranges, mmio, offset);
+        offset = mmio->phys_addr - mmio_base;
+        range = vgic_find_range(ranges, mmio, offset);
         if (unlikely(!range || !range->handle_mmio)) {
                 pr_warn("Unhandled access %d %08llx %d\n",
                         mmio->is_write, mmio->phys_addr, mmio->len);
@@ -1071,12 +757,12 @@ bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
         }
 
         spin_lock(&vcpu->kvm->arch.vgic.lock);
-        offset = mmio->phys_addr - range->base - base;
+        offset -= range->base;
         if (vgic_validate_access(dist, range, offset)) {
-                updated_state = range->handle_mmio(vcpu, mmio, offset);
+                updated_state = call_range_handler(vcpu, mmio, offset, range);
         } else {
-                vgic_reg_access(mmio, NULL, offset,
-                                ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
+                if (!mmio->is_write)
+                        memset(mmio->data, 0, mmio->len);
                 updated_state = false;
         }
         spin_unlock(&vcpu->kvm->arch.vgic.lock);
@@ -1089,50 +775,28 @@ bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
         return true;
 }
 
-static u8 *vgic_get_sgi_sources(struct vgic_dist *dist, int vcpu_id, int sgi)
-{
-        return dist->irq_sgi_sources + vcpu_id * VGIC_NR_SGIS + sgi;
-}
-
-static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg)
+/**
+ * vgic_handle_mmio - handle an in-kernel MMIO access for the GIC emulation
+ * @vcpu:      pointer to the vcpu performing the access
+ * @run:       pointer to the kvm_run structure
+ * @mmio:      pointer to the data describing the access
+ *
+ * returns true if the MMIO access has been performed in kernel space,
+ * and false if it needs to be emulated in user space.
+ * Calls the actual handling routine for the selected VGIC model.
+ */
+bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
+                      struct kvm_exit_mmio *mmio)
 {
-        struct kvm *kvm = vcpu->kvm;
-        struct vgic_dist *dist = &kvm->arch.vgic;
-        int nrcpus = atomic_read(&kvm->online_vcpus);
-        u8 target_cpus;
-        int sgi, mode, c, vcpu_id;
-
-        vcpu_id = vcpu->vcpu_id;
-
-        sgi = reg & 0xf;
-        target_cpus = (reg >> 16) & 0xff;
-        mode = (reg >> 24) & 3;
-
-        switch (mode) {
-        case 0:
-                if (!target_cpus)
-                        return;
-                break;
-
-        case 1:
-                target_cpus = ((1 << nrcpus) - 1) & ~(1 << vcpu_id) & 0xff;
-                break;
-
-        case 2:
-                target_cpus = 1 << vcpu_id;
-                break;
-        }
-
-        kvm_for_each_vcpu(c, vcpu, kvm) {
-                if (target_cpus & 1) {
-                        /* Flag the SGI as pending */
-                        vgic_dist_irq_set_pending(vcpu, sgi);
-                        *vgic_get_sgi_sources(dist, c, sgi) |= 1 << vcpu_id;
-                        kvm_debug("SGI%d from CPU%d to CPU%d\n", sgi, vcpu_id, c);
-                }
+        if (!irqchip_in_kernel(vcpu->kvm))
+                return false;
 
-                target_cpus >>= 1;
-        }
+        /*
+         * This will currently call either vgic_v2_handle_mmio() or
+         * vgic_v3_handle_mmio(), which in turn will call
+         * vgic_handle_mmio_range() defined above.
+         */
+        return vcpu->kvm->arch.vgic.vm_ops.handle_mmio(vcpu, run, mmio);
 }
 
 static int vgic_nr_shared_irqs(struct vgic_dist *dist)
@@ -1173,7 +837,7 @@ static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
  * Update the interrupt state and determine which CPUs have pending
  * interrupts. Must be called with distributor lock held.
  */
-static void vgic_update_state(struct kvm *kvm)
+void vgic_update_state(struct kvm *kvm)
 {
         struct vgic_dist *dist = &kvm->arch.vgic;
         struct kvm_vcpu *vcpu;
@@ -1234,12 +898,12 @@ static inline void vgic_disable_underflow(struct kvm_vcpu *vcpu)
         vgic_ops->disable_underflow(vcpu);
 }
 
-static inline void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
+void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 {
         vgic_ops->get_vmcr(vcpu, vmcr);
 }
 
-static void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
+void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 {
         vgic_ops->set_vmcr(vcpu, vmcr);
 }
@@ -1288,8 +952,9 @@ static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
 /*
  * Queue an interrupt to a CPU virtual interface. Return true on success,
  * or false if it wasn't possible to queue it.
+ * sgi_source must be zero for any non-SGI interrupts.
  */
-static bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
+bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
 {
         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
@@ -1338,37 +1003,6 @@ static bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
         return true;
 }
 
-static bool vgic_queue_sgi(struct kvm_vcpu *vcpu, int irq)
-{
-        struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
-        unsigned long sources;
-        int vcpu_id = vcpu->vcpu_id;
-        int c;
-
-        sources = *vgic_get_sgi_sources(dist, vcpu_id, irq);
-
-        for_each_set_bit(c, &sources, dist->nr_cpus) {
-                if (vgic_queue_irq(vcpu, c, irq))
-                        clear_bit(c, &sources);
-        }
-
-        *vgic_get_sgi_sources(dist, vcpu_id, irq) = sources;
-
-        /*
-         * If the sources bitmap has been cleared it means that we
-         * could queue all the SGIs onto link registers (see the
-         * clear_bit above), and therefore we are done with them in
-         * our emulated gic and can get rid of them.
-         */
-        if (!sources) {
-                vgic_dist_irq_clear_pending(vcpu, irq);
-                vgic_cpu_irq_clear(vcpu, irq);
-                return true;
-        }
-
-        return false;
-}
-
 static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)
 {
         if (!vgic_can_sample_irq(vcpu, irq))
@@ -1413,7 +1047,7 @@ static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 
         /* SGIs */
         for_each_set_bit(i, vgic_cpu->pending_percpu, VGIC_NR_SGIS) {
-                if (!vgic_queue_sgi(vcpu, i))
+                if (!queue_sgi(vcpu, i))
                         overflow = 1;
         }
 
@@ -1575,7 +1209,7 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
         return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
 }
 
-static void vgic_kick_vcpus(struct kvm *kvm)
+void vgic_kick_vcpus(struct kvm *kvm)
 {
         struct kvm_vcpu *vcpu;
         int c;
@@ -1615,7 +1249,7 @@ static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
         struct kvm_vcpu *vcpu;
         int edge_triggered, level_triggered;
         int enabled;
-        bool ret = true;
+        bool ret = true, can_inject = true;
 
         spin_lock(&dist->lock);
 
@@ -1630,6 +1264,11 @@ static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
 
         if (irq_num >= VGIC_NR_PRIVATE_IRQS) {
                 cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS];
+                if (cpuid == VCPU_NOT_ALLOCATED) {
+                        /* Pretend we use CPU0, and prevent injection */
+                        cpuid = 0;
+                        can_inject = false;
+                }
                 vcpu = kvm_get_vcpu(kvm, cpuid);
         }
 
@@ -1652,7 +1291,7 @@ static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
 
         enabled = vgic_irq_is_enabled(vcpu, irq_num);
 
-        if (!enabled) {
+        if (!enabled || !can_inject) {
                 ret = false;
                 goto out;
         }
@@ -1698,6 +1337,16 @@ int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
         int vcpu_id;
 
         if (unlikely(!vgic_initialized(kvm))) {
+                /*
+                 * We only provide the automatic initialization of the VGIC
+                 * for the legacy case of a GICv2. Any other type must
+                 * be explicitly initialized once setup with the respective
+                 * KVM device call.
+                 */
+                if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2) {
+                        ret = -EBUSY;
+                        goto out;
+                }
                 mutex_lock(&kvm->lock);
                 ret = vgic_init(kvm);
                 mutex_unlock(&kvm->lock);
@@ -1762,6 +1411,17 @@ static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs)
         return 0;
 }
 
+/**
+ * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
+ *
+ * The host's GIC naturally limits the maximum amount of VCPUs a guest
+ * can use.
+ */
+int kvm_vgic_get_max_vcpus(void)
+{
+        return vgic->max_gic_vcpus;
+}
+
 void kvm_vgic_destroy(struct kvm *kvm)
 {
         struct vgic_dist *dist = &kvm->arch.vgic;
@@ -1784,6 +1444,7 @@ void kvm_vgic_destroy(struct kvm *kvm)
         }
         kfree(dist->irq_sgi_sources);
         kfree(dist->irq_spi_cpu);
+        kfree(dist->irq_spi_mpidr);
         kfree(dist->irq_spi_target);
         kfree(dist->irq_pending_on_cpu);
         dist->irq_sgi_sources = NULL;
@@ -1797,7 +1458,7 @@ void kvm_vgic_destroy(struct kvm *kvm)
  * Allocate and initialize the various data structures. Must be called
  * with kvm->lock held!
  */
-static int vgic_init(struct kvm *kvm)
+int vgic_init(struct kvm *kvm)
 {
         struct vgic_dist *dist = &kvm->arch.vgic;
         struct kvm_vcpu *vcpu;
@@ -1809,7 +1470,7 @@ static int vgic_init(struct kvm *kvm)
 
         nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);
         if (!nr_cpus)           /* No vcpus? Can't be good... */
-                return -EINVAL;
+                return -ENODEV;
 
         /*
          * If nobody configured the number of interrupts, use the
@@ -1852,8 +1513,9 @@ static int vgic_init(struct kvm *kvm)
         if (ret)
                 goto out;
 
-        for (i = VGIC_NR_PRIVATE_IRQS; i < dist->nr_irqs; i += 4)
-                vgic_set_target_reg(kvm, 0, i);
+        ret = kvm->arch.vgic.vm_ops.init_model(kvm);
+        if (ret)
+                goto out;
 
         kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {
                 ret = vgic_vcpu_init_maps(vcpu, nr_irqs);
@@ -1882,72 +1544,49 @@ out:
         return ret;
 }
 
-/**
- * kvm_vgic_map_resources - Configure global VGIC state before running any VCPUs
- * @kvm: pointer to the kvm struct
- *
- * Map the virtual CPU interface into the VM before running any VCPUs.  We
- * can't do this at creation time, because user space must first set the
- * virtual CPU interface address in the guest physical address space.
- */
-int kvm_vgic_map_resources(struct kvm *kvm)
+static int init_vgic_model(struct kvm *kvm, int type)
 {
-        int ret = 0;
-
-        if (!irqchip_in_kernel(kvm))
-                return 0;
-
-        mutex_lock(&kvm->lock);
-
-        if (vgic_ready(kvm))
-                goto out;
-
-        if (IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_dist_base) ||
-            IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_cpu_base)) {
-                kvm_err("Need to set vgic cpu and dist addresses first\n");
-                ret = -ENXIO;
-                goto out;
-        }
-
-        /*
-         * Initialize the vgic if this hasn't already been done on demand by
-         * accessing the vgic state from userspace.
-         */
-        ret = vgic_init(kvm);
-        if (ret) {
-                kvm_err("Unable to allocate maps\n");
-                goto out;
+        switch (type) {
+        case KVM_DEV_TYPE_ARM_VGIC_V2:
+                vgic_v2_init_emulation(kvm);
+                break;
+#ifdef CONFIG_ARM_GIC_V3
+        case KVM_DEV_TYPE_ARM_VGIC_V3:
+                vgic_v3_init_emulation(kvm);
+                break;
+#endif
+        default:
+                return -ENODEV;
         }
 
-        ret = kvm_phys_addr_ioremap(kvm, kvm->arch.vgic.vgic_cpu_base,
-                                    vgic->vcpu_base, KVM_VGIC_V2_CPU_SIZE,
-                                    true);
-        if (ret) {
-                kvm_err("Unable to remap VGIC CPU to VCPU\n");
-                goto out;
-        }
+        if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)
+                return -E2BIG;
 
-        kvm->arch.vgic.ready = true;
-out:
-        if (ret)
-                kvm_vgic_destroy(kvm);
-        mutex_unlock(&kvm->lock);
-        return ret;
+        return 0;
 }
 
-int kvm_vgic_create(struct kvm *kvm)
+int kvm_vgic_create(struct kvm *kvm, u32 type)
 {
         int i, vcpu_lock_idx = -1, ret;
         struct kvm_vcpu *vcpu;
 
         mutex_lock(&kvm->lock);
 
-        if (kvm->arch.vgic.vctrl_base) {
+        if (irqchip_in_kernel(kvm)) {
                 ret = -EEXIST;
                 goto out;
         }
 
         /*
+         * This function is also called by the KVM_CREATE_IRQCHIP handler,
+         * which had no chance yet to check the availability of the GICv2
+         * emulation. So check this here again. KVM_CREATE_DEVICE does
+         * the proper checks already.
+         */
+        if (type == KVM_DEV_TYPE_ARM_VGIC_V2 && !vgic->can_emulate_gicv2)
+                return -ENODEV;
+
+        /*
          * Any time a vcpu is run, vcpu_load is called which tries to grab the
          * vcpu->mutex.  By grabbing the vcpu->mutex of all VCPUs we ensure
          * that no other VCPUs are run while we create the vgic.
@@ -1965,11 +1604,17 @@ int kvm_vgic_create(struct kvm *kvm)
         }
         ret = 0;
 
+        ret = init_vgic_model(kvm, type);
+        if (ret)
+                goto out_unlock;
+
         spin_lock_init(&kvm->arch.vgic.lock);
         kvm->arch.vgic.in_kernel = true;
+        kvm->arch.vgic.vgic_model = type;
         kvm->arch.vgic.vctrl_base = vgic->vctrl_base;
         kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
         kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
+        kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
 
 out_unlock:
         for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
@@ -2022,7 +1667,7 @@ static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,
 /**
  * kvm_vgic_addr - set or get vgic VM base addresses
  * @kvm:   pointer to the vm struct
- * @type:  the VGIC addr type, one of KVM_VGIC_V2_ADDR_TYPE_XXX
+ * @type:  the VGIC addr type, one of KVM_VGIC_V[23]_ADDR_TYPE_XXX
  * @addr:  pointer to address value
  * @write: if true set the address in the VM address space, if false read the
  *          address
@@ -2036,216 +1681,64 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
 {
         int r = 0;
         struct vgic_dist *vgic = &kvm->arch.vgic;
+        int type_needed;
+        phys_addr_t *addr_ptr, block_size;
+        phys_addr_t alignment;
 
         mutex_lock(&kvm->lock);
         switch (type) {
         case KVM_VGIC_V2_ADDR_TYPE_DIST:
-                if (write) {
-                        r = vgic_ioaddr_assign(kvm, &vgic->vgic_dist_base,
-                                               *addr, KVM_VGIC_V2_DIST_SIZE);
-                } else {
-                        *addr = vgic->vgic_dist_base;
-                }
+                type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
+                addr_ptr = &vgic->vgic_dist_base;
+                block_size = KVM_VGIC_V2_DIST_SIZE;
+                alignment = SZ_4K;
                 break;
         case KVM_VGIC_V2_ADDR_TYPE_CPU:
-                if (write) {
-                        r = vgic_ioaddr_assign(kvm, &vgic->vgic_cpu_base,
-                                               *addr, KVM_VGIC_V2_CPU_SIZE);
-                } else {
-                        *addr = vgic->vgic_cpu_base;
-                }
+                type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
+                addr_ptr = &vgic->vgic_cpu_base;
+                block_size = KVM_VGIC_V2_CPU_SIZE;
+                alignment = SZ_4K;
                 break;
-        default:
-                r = -ENODEV;
-        }
-
-        mutex_unlock(&kvm->lock);
-        return r;
-}
-
-static bool handle_cpu_mmio_misc(struct kvm_vcpu *vcpu,
-                                 struct kvm_exit_mmio *mmio, phys_addr_t offset)
-{
-        bool updated = false;
-        struct vgic_vmcr vmcr;
-        u32 *vmcr_field;
-        u32 reg;
-
-        vgic_get_vmcr(vcpu, &vmcr);
-
-        switch (offset & ~0x3) {
-        case GIC_CPU_CTRL:
-                vmcr_field = &vmcr.ctlr;
-                break;
-        case GIC_CPU_PRIMASK:
-                vmcr_field = &vmcr.pmr;
+#ifdef CONFIG_ARM_GIC_V3
+        case KVM_VGIC_V3_ADDR_TYPE_DIST:
+                type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
+                addr_ptr = &vgic->vgic_dist_base;
+                block_size = KVM_VGIC_V3_DIST_SIZE;
+                alignment = SZ_64K;
                 break;
-        case GIC_CPU_BINPOINT:
-                vmcr_field = &vmcr.bpr;
-                break;
-        case GIC_CPU_ALIAS_BINPOINT:
-                vmcr_field = &vmcr.abpr;
+        case KVM_VGIC_V3_ADDR_TYPE_REDIST:
+                type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
+                addr_ptr = &vgic->vgic_redist_base;
+                block_size = KVM_VGIC_V3_REDIST_SIZE;
+                alignment = SZ_64K;
                 break;
+#endif
         default:
-                BUG();
-        }
-
-        if (!mmio->is_write) {
-                reg = *vmcr_field;
-                mmio_data_write(mmio, ~0, reg);
-        } else {
-                reg = mmio_data_read(mmio, ~0);
-                if (reg != *vmcr_field) {
-                        *vmcr_field = reg;
-                        vgic_set_vmcr(vcpu, &vmcr);
-                        updated = true;
-                }
-        }
-        return updated;
-}
-
-static bool handle_mmio_abpr(struct kvm_vcpu *vcpu,
-                             struct kvm_exit_mmio *mmio, phys_addr_t offset)
-{
-        return handle_cpu_mmio_misc(vcpu, mmio, GIC_CPU_ALIAS_BINPOINT);
-}
-
-static bool handle_cpu_mmio_ident(struct kvm_vcpu *vcpu,
-                                  struct kvm_exit_mmio *mmio,
-                                  phys_addr_t offset)
-{
-        u32 reg;
-
-        if (mmio->is_write)
-                return false;
-
-        /* GICC_IIDR */
-        reg = (PRODUCT_ID_KVM << 20) |
-              (GICC_ARCH_VERSION_V2 << 16) |
-              (IMPLEMENTER_ARM << 0);
-        mmio_data_write(mmio, ~0, reg);
-        return false;
-}
-
-/*
- * CPU Interface Register accesses - these are not accessed by the VM, but by
- * user space for saving and restoring VGIC state.
- */
-static const struct mmio_range vgic_cpu_ranges[] = {
-        {
-                .base           = GIC_CPU_CTRL,
-                .len            = 12,
-                .handle_mmio    = handle_cpu_mmio_misc,
-        },
-        {
-                .base           = GIC_CPU_ALIAS_BINPOINT,
-                .len            = 4,
-                .handle_mmio    = handle_mmio_abpr,
-        },
-        {
-                .base           = GIC_CPU_ACTIVEPRIO,
-                .len            = 16,
-                .handle_mmio    = handle_mmio_raz_wi,
-        },
-        {
-                .base           = GIC_CPU_IDENT,
-                .len            = 4,
-                .handle_mmio    = handle_cpu_mmio_ident,
-        },
-};
-
-static int vgic_attr_regs_access(struct kvm_device *dev,
-                                 struct kvm_device_attr *attr,
-                                 u32 *reg, bool is_write)
-{
-        const struct mmio_range *r = NULL, *ranges;
-        phys_addr_t offset;
-        int ret, cpuid, c;
-        struct kvm_vcpu *vcpu, *tmp_vcpu;
-        struct vgic_dist *vgic;
-        struct kvm_exit_mmio mmio;
-
-        offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
-        cpuid = (attr->attr & KVM_DEV_ARM_VGIC_CPUID_MASK) >>
-                KVM_DEV_ARM_VGIC_CPUID_SHIFT;
-
-        mutex_lock(&dev->kvm->lock);
-
-        ret = vgic_init(dev->kvm);
-        if (ret)
-                goto out;
-
-        if (cpuid >= atomic_read(&dev->kvm->online_vcpus)) {
-                ret = -EINVAL;
+                r = -ENODEV;
                 goto out;
         }
 
-        vcpu = kvm_get_vcpu(dev->kvm, cpuid);
-        vgic = &dev->kvm->arch.vgic;
-
-        mmio.len = 4;
-        mmio.is_write = is_write;
-        if (is_write)
-                mmio_data_write(&mmio, ~0, *reg);
-        switch (attr->group) {
-        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
-                mmio.phys_addr = vgic->vgic_dist_base + offset;
-                ranges = vgic_dist_ranges;
-                break;
-        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
-                mmio.phys_addr = vgic->vgic_cpu_base + offset;
-                ranges = vgic_cpu_ranges;
-                break;
-        default:
-                BUG();
-        }
-        r = find_matching_range(ranges, &mmio, offset);
-
-        if (unlikely(!r || !r->handle_mmio)) {
-                ret = -ENXIO;
+        if (vgic->vgic_model != type_needed) {
+                r = -ENODEV;
                 goto out;
         }
 
-
-        spin_lock(&vgic->lock);
-
-        /*
-         * Ensure that no other VCPU is running by checking the vcpu->cpu
-         * field.  If no other VPCUs are running we can safely access the VGIC
-         * state, because even if another VPU is run after this point, that
-         * VCPU will not touch the vgic state, because it will block on
-         * getting the vgic->lock in kvm_vgic_sync_hwstate().
-         */
-        kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm) {
-                if (unlikely(tmp_vcpu->cpu != -1)) {
-                        ret = -EBUSY;
-                        goto out_vgic_unlock;
-                }
+        if (write) {
+                if (!IS_ALIGNED(*addr, alignment))
+                        r = -EINVAL;
+                else
+                        r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,
+                                               block_size);
+        } else {
+                *addr = *addr_ptr;
         }
 
-        /*
-         * Move all pending IRQs from the LRs on all VCPUs so the pending
-         * state can be properly represented in the register state accessible
-         * through this API.
-         */
-        kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm)
-                vgic_unqueue_irqs(tmp_vcpu);
-
-        offset -= r->base;
-        r->handle_mmio(vcpu, &mmio, offset);
-
-        if (!is_write)
-                *reg = mmio_data_read(&mmio, ~0);
-
-        ret = 0;
-out_vgic_unlock:
-        spin_unlock(&vgic->lock);
 out:
-        mutex_unlock(&dev->kvm->lock);
-        return ret;
+        mutex_unlock(&kvm->lock);
+        return r;
 }
 
-static int vgic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
+int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
 {
         int r;
 
@@ -2261,17 +1754,6 @@ static int vgic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
                 r = kvm_vgic_addr(dev->kvm, type, &addr, true);
                 return (r == -ENODEV) ? -ENXIO : r;
         }
-
-        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
-        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
-                u32 __user *uaddr = (u32 __user *)(long)attr->addr;
-                u32 reg;
-
-                if (get_user(reg, uaddr))
-                        return -EFAULT;
-
-                return vgic_attr_regs_access(dev, attr, &reg, true);
-        }
         case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
                 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
                 u32 val;
@@ -2302,13 +1784,20 @@ static int vgic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
 
                 return ret;
         }
-
+        case KVM_DEV_ARM_VGIC_GRP_CTRL: {
+                switch (attr->attr) {
+                case KVM_DEV_ARM_VGIC_CTRL_INIT:
+                        r = vgic_init(dev->kvm);
+                        return r;
+                }
+                break;
+        }
         }
 
         return -ENXIO;
 }
 
-static int vgic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
+int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
 {
         int r = -ENXIO;
 
@@ -2326,20 +1815,9 @@ static int vgic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
                         return -EFAULT;
                 break;
         }
-
-        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
-        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
-                u32 __user *uaddr = (u32 __user *)(long)attr->addr;
-                u32 reg = 0;
-
-                r = vgic_attr_regs_access(dev, attr, &reg, false);
-                if (r)
-                        return r;
-                r = put_user(reg, uaddr);
-                break;
-        }
         case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
                 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
+
                 r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);
                 break;
         }
@@ -2349,61 +1827,17 @@ static int vgic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
         return r;
 }
 
-static int vgic_has_attr_regs(const struct mmio_range *ranges,
-                              phys_addr_t offset)
+int vgic_has_attr_regs(const struct kvm_mmio_range *ranges, phys_addr_t offset)
 {
         struct kvm_exit_mmio dev_attr_mmio;
 
         dev_attr_mmio.len = 4;
-        if (find_matching_range(ranges, &dev_attr_mmio, offset))
+        if (vgic_find_range(ranges, &dev_attr_mmio, offset))
                 return 0;
         else
                 return -ENXIO;
 }
 
-static int vgic_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
-{
-        phys_addr_t offset;
-
-        switch (attr->group) {
-        case KVM_DEV_ARM_VGIC_GRP_ADDR:
-                switch (attr->attr) {
-                case KVM_VGIC_V2_ADDR_TYPE_DIST:
-                case KVM_VGIC_V2_ADDR_TYPE_CPU:
-                        return 0;
-                }
-                break;
-        case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
-                offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
-                return vgic_has_attr_regs(vgic_dist_ranges, offset);
-        case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
-                offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
-                return vgic_has_attr_regs(vgic_cpu_ranges, offset);
-        case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
-                return 0;
-        }
-        return -ENXIO;
-}
-
-static void vgic_destroy(struct kvm_device *dev)
-{
-        kfree(dev);
-}
-
-static int vgic_create(struct kvm_device *dev, u32 type)
-{
-        return kvm_vgic_create(dev->kvm);
-}
-
-static struct kvm_device_ops kvm_arm_vgic_v2_ops = {
-        .name = "kvm-arm-vgic",
-        .create = vgic_create,
-        .destroy = vgic_destroy,
-        .set_attr = vgic_set_attr,
-        .get_attr = vgic_get_attr,
-        .has_attr = vgic_has_attr,
-};
-
 static void vgic_init_maintenance_interrupt(void *info)
 {
         enable_percpu_irq(vgic->maint_irq, 0);
@@ -2474,8 +1908,7 @@ int kvm_vgic_hyp_init(void)
 
         on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);
 
-        return kvm_register_device_ops(&kvm_arm_vgic_v2_ops,
-                                       KVM_DEV_TYPE_ARM_VGIC_V2);
+        return 0;
 
 out_free_irq:
         free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
diff --git a/virt/kvm/arm/vgic.h b/virt/kvm/arm/vgic.h
new file mode 100644
index 000000000000..1e83bdf5f499
--- /dev/null
+++ b/virt/kvm/arm/vgic.h
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2012-2014 ARM Ltd.
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ *
+ * Derived from virt/kvm/arm/vgic.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __KVM_VGIC_H__
+#define __KVM_VGIC_H__
+
+#define VGIC_ADDR_UNDEF         (-1)
+#define IS_VGIC_ADDR_UNDEF(_x)  ((_x) == VGIC_ADDR_UNDEF)
+
+#define PRODUCT_ID_KVM          0x4b    /* ASCII code K */
+#define IMPLEMENTER_ARM         0x43b
+
+#define ACCESS_READ_VALUE       (1 << 0)
+#define ACCESS_READ_RAZ         (0 << 0)
+#define ACCESS_READ_MASK(x)     ((x) & (1 << 0))
+#define ACCESS_WRITE_IGNORED    (0 << 1)
+#define ACCESS_WRITE_SETBIT     (1 << 1)
+#define ACCESS_WRITE_CLEARBIT   (2 << 1)
+#define ACCESS_WRITE_VALUE      (3 << 1)
+#define ACCESS_WRITE_MASK(x)    ((x) & (3 << 1))
+
+#define VCPU_NOT_ALLOCATED      ((u8)-1)
+
+unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x);
+
+void vgic_update_state(struct kvm *kvm);
+int vgic_init_common_maps(struct kvm *kvm);
+
+u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset);
+u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset);
+
+void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq);
+void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq);
+void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq);
+void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
+                             int irq, int val);
+
+void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
+void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
+
+bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq);
+void vgic_unqueue_irqs(struct kvm_vcpu *vcpu);
+
+void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
+                     phys_addr_t offset, int mode);
+bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
+                        phys_addr_t offset);
+
+static inline
+u32 mmio_data_read(struct kvm_exit_mmio *mmio, u32 mask)
+{
+        return le32_to_cpu(*((u32 *)mmio->data)) & mask;
+}
+
+static inline
+void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
+{
+        *((u32 *)mmio->data) = cpu_to_le32(value) & mask;
+}
+
+struct kvm_mmio_range {
+        phys_addr_t base;
+        unsigned long len;
+        int bits_per_irq;
+        bool (*handle_mmio)(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
+                            phys_addr_t offset);
+};
+
+static inline bool is_in_range(phys_addr_t addr, unsigned long len,
+                               phys_addr_t baseaddr, unsigned long size)
+{
+        return (addr >= baseaddr) && (addr + len <= baseaddr + size);
+}
+
+const
+struct kvm_mmio_range *vgic_find_range(const struct kvm_mmio_range *ranges,
+                                       struct kvm_exit_mmio *mmio,
+                                       phys_addr_t offset);
+
+bool vgic_handle_mmio_range(struct kvm_vcpu *vcpu, struct kvm_run *run,
+                            struct kvm_exit_mmio *mmio,
+                            const struct kvm_mmio_range *ranges,
+                            unsigned long mmio_base);
+
+bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
+                            phys_addr_t offset, int vcpu_id, int access);
+
+bool vgic_handle_set_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
+                                 phys_addr_t offset, int vcpu_id);
+
+bool vgic_handle_clear_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
+                                   phys_addr_t offset, int vcpu_id);
+
+bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
+                         phys_addr_t offset);
+
+void vgic_kick_vcpus(struct kvm *kvm);
+
+int vgic_has_attr_regs(const struct kvm_mmio_range *ranges, phys_addr_t offset);
+int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr);
+int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr);
+
+int vgic_init(struct kvm *kvm);
+void vgic_v2_init_emulation(struct kvm *kvm);
+void vgic_v3_init_emulation(struct kvm *kvm);
+
+#endif
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 167e8c14b143..246cf291c6fd 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -176,6 +176,7 @@ bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
         return called;
 }
 
+#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
 void kvm_flush_remote_tlbs(struct kvm *kvm)
 {
         long dirty_count = kvm->tlbs_dirty;
@@ -186,6 +187,7 @@ void kvm_flush_remote_tlbs(struct kvm *kvm)
         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
+#endif
 
 void kvm_reload_remote_mmus(struct kvm *kvm)
 {
@@ -993,6 +995,86 @@ out:
 }
 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
 
+#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
+/**
+ * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
+ *      are dirty write protect them for next write.
+ * @kvm:        pointer to kvm instance
+ * @log:        slot id and address to which we copy the log
+ * @is_dirty:   flag set if any page is dirty
+ *
+ * We need to keep it in mind that VCPU threads can write to the bitmap
+ * concurrently. So, to avoid losing track of dirty pages we keep the
+ * following order:
+ *
+ *    1. Take a snapshot of the bit and clear it if needed.
+ *    2. Write protect the corresponding page.
+ *    3. Copy the snapshot to the userspace.
+ *    4. Upon return caller flushes TLB's if needed.
+ *
+ * Between 2 and 4, the guest may write to the page using the remaining TLB
+ * entry.  This is not a problem because the page is reported dirty using
+ * the snapshot taken before and step 4 ensures that writes done after
+ * exiting to userspace will be logged for the next call.
+ *
+ */
+int kvm_get_dirty_log_protect(struct kvm *kvm,
+                        struct kvm_dirty_log *log, bool *is_dirty)
+{
+        struct kvm_memory_slot *memslot;
+        int r, i;
+        unsigned long n;
+        unsigned long *dirty_bitmap;
+        unsigned long *dirty_bitmap_buffer;
+
+        r = -EINVAL;
+        if (log->slot >= KVM_USER_MEM_SLOTS)
+                goto out;
+
+        memslot = id_to_memslot(kvm->memslots, log->slot);
+
+        dirty_bitmap = memslot->dirty_bitmap;
+        r = -ENOENT;
+        if (!dirty_bitmap)
+                goto out;
+
+        n = kvm_dirty_bitmap_bytes(memslot);
+
+        dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
+        memset(dirty_bitmap_buffer, 0, n);
+
+        spin_lock(&kvm->mmu_lock);
+        *is_dirty = false;
+        for (i = 0; i < n / sizeof(long); i++) {
+                unsigned long mask;
+                gfn_t offset;
+
+                if (!dirty_bitmap[i])
+                        continue;
+
+                *is_dirty = true;
+
+                mask = xchg(&dirty_bitmap[i], 0);
+                dirty_bitmap_buffer[i] = mask;
+
+                offset = i * BITS_PER_LONG;
+                kvm_arch_mmu_write_protect_pt_masked(kvm, memslot, offset,
+                                                                mask);
+        }
+
+        spin_unlock(&kvm->mmu_lock);
+
+        r = -EFAULT;
+        if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
+                goto out;
+
+        r = 0;
+out:
+        return r;
+}
+EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
+#endif
+
 bool kvm_largepages_enabled(void)
 {
         return largepages_enabled;