aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRadim Krčmář <rkrcmar@redhat.com>2016-07-22 14:27:26 -0400
committerRadim Krčmář <rkrcmar@redhat.com>2016-07-22 14:27:26 -0400
commit912902ce78b0d48f717f9128e61fb9bffbd65f86 (patch)
tree5a97dd19149ba1e0386df9bb467f8c5fb7ba2393
parent61f5dea179653558562ba9a5dd71eb29d91a383e (diff)
parent3a88bded203591d4683aacdbb65cd0f549bc58cb (diff)
Merge tag 'kvm-arm-for-4.8' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into next
KVM/ARM changes for Linux 4.8 - GICv3 ITS emulation - Simpler idmap management that fixes potential TLB conflicts - Honor the kernel protection in HYP mode - Removal of the old vgic implementation
Diffstat
-rw-r--r--Documentation/virtual/kvm/api.txt14
-rw-r--r--Documentation/virtual/kvm/devices/arm-vgic.txt25
-rw-r--r--arch/arm/include/asm/kvm_asm.h2
-rw-r--r--arch/arm/include/asm/kvm_host.h27
-rw-r--r--arch/arm/include/asm/kvm_hyp.h3
-rw-r--r--arch/arm/include/asm/kvm_mmu.h15
-rw-r--r--arch/arm/include/asm/pgtable.h4
-rw-r--r--arch/arm/include/asm/virt.h4
-rw-r--r--arch/arm/kvm/Kconfig7
-rw-r--r--arch/arm/kvm/Makefile6
-rw-r--r--arch/arm/kvm/arm.c36
-rw-r--r--arch/arm/kvm/init.S56
-rw-r--r--arch/arm/kvm/mmu.c142
-rw-r--r--arch/arm64/include/asm/cpufeature.h3
-rw-r--r--arch/arm64/include/asm/kvm_host.h19
-rw-r--r--arch/arm64/include/asm/kvm_hyp.h23
-rw-r--r--arch/arm64/include/asm/kvm_mmu.h96
-rw-r--r--arch/arm64/include/asm/pgtable-hwdef.h1
-rw-r--r--arch/arm64/include/asm/pgtable-prot.h4
-rw-r--r--arch/arm64/include/asm/virt.h4
-rw-r--r--arch/arm64/include/uapi/asm/kvm.h2
-rw-r--r--arch/arm64/kernel/cpufeature.c19
-rw-r--r--arch/arm64/kvm/Kconfig8
-rw-r--r--arch/arm64/kvm/Makefile9
-rw-r--r--arch/arm64/kvm/hyp-init.S61
-rw-r--r--arch/arm64/kvm/hyp/entry.S19
-rw-r--r--arch/arm64/kvm/hyp/hyp-entry.S15
-rw-r--r--arch/arm64/kvm/hyp/switch.c11
-rw-r--r--arch/arm64/kvm/reset.c36
-rw-r--r--arch/arm64/kvm/sys_regs.c4
-rw-r--r--include/kvm/arm_vgic.h438
-rw-r--r--include/kvm/vgic/vgic.h246
-rw-r--r--include/linux/irqchip/arm-gic-v3.h212
-rw-r--r--include/linux/kvm_host.h2
-rw-r--r--include/uapi/linux/kvm.h7
-rw-r--r--virt/kvm/arm/hyp/vgic-v2-sr.c15
-rw-r--r--virt/kvm/arm/vgic-v2-emul.c856
-rw-r--r--virt/kvm/arm/vgic-v2.c274
-rw-r--r--virt/kvm/arm/vgic-v3-emul.c1074
-rw-r--r--virt/kvm/arm/vgic-v3.c279
-rw-r--r--virt/kvm/arm/vgic.c2440
-rw-r--r--virt/kvm/arm/vgic.h140
-rw-r--r--virt/kvm/arm/vgic/vgic-init.c9
-rw-r--r--virt/kvm/arm/vgic/vgic-its.c1500
-rw-r--r--virt/kvm/arm/vgic/vgic-kvm-device.c22
-rw-r--r--virt/kvm/arm/vgic/vgic-mmio-v2.c10
-rw-r--r--virt/kvm/arm/vgic/vgic-mmio-v3.c247
-rw-r--r--virt/kvm/arm/vgic/vgic-mmio.c64
-rw-r--r--virt/kvm/arm/vgic/vgic-mmio.h31
-rw-r--r--virt/kvm/arm/vgic/vgic-v2.c12
-rw-r--r--virt/kvm/arm/vgic/vgic-v3.c29
-rw-r--r--virt/kvm/arm/vgic/vgic.c119
-rw-r--r--virt/kvm/arm/vgic/vgic.h38
-rw-r--r--virt/kvm/kvm_main.c24
54 files changed, 2698 insertions, 6065 deletions
diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index 299306db5d84..5237e1b2fd66 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -2182,7 +2182,7 @@ after pausing the vcpu, but before it is resumed.
21824.71 KVM_SIGNAL_MSI 21824.71 KVM_SIGNAL_MSI
2183 2183
2184Capability: KVM_CAP_SIGNAL_MSI 2184Capability: KVM_CAP_SIGNAL_MSI
2185Architectures: x86 2185Architectures: x86 arm64
2186Type: vm ioctl 2186Type: vm ioctl
2187Parameters: struct kvm_msi (in) 2187Parameters: struct kvm_msi (in)
2188Returns: >0 on delivery, 0 if guest blocked the MSI, and -1 on error 2188Returns: >0 on delivery, 0 if guest blocked the MSI, and -1 on error
@@ -2195,10 +2195,18 @@ struct kvm_msi {
2195 __u32 address_hi; 2195 __u32 address_hi;
2196 __u32 data; 2196 __u32 data;
2197 __u32 flags; 2197 __u32 flags;
2198 __u8 pad[16]; 2198 __u32 devid;
2199 __u8 pad[12];
2199}; 2200};
2200 2201
2201No flags are defined so far. The corresponding field must be 0. 2202flags: KVM_MSI_VALID_DEVID: devid contains a valid value
2203devid: If KVM_MSI_VALID_DEVID is set, contains a unique device identifier
2204 for the device that wrote the MSI message.
2205 For PCI, this is usually a BFD identifier in the lower 16 bits.
2206
2207The per-VM KVM_CAP_MSI_DEVID capability advertises the need to provide
2208the device ID. If this capability is not set, userland cannot rely on
2209the kernel to allow the KVM_MSI_VALID_DEVID flag being set.
2202 2210
2203On x86, address_hi is ignored unless the KVM_CAP_X2APIC_API capability is 2211On x86, address_hi is ignored unless the KVM_CAP_X2APIC_API capability is
2204enabled. If it is enabled, address_hi bits 31-8 provide bits 31-8 of the 2212enabled. If it is enabled, address_hi bits 31-8 provide bits 31-8 of the
diff --git a/Documentation/virtual/kvm/devices/arm-vgic.txt b/Documentation/virtual/kvm/devices/arm-vgic.txt
index 59541d49e15c..89182f80cc7f 100644
--- a/Documentation/virtual/kvm/devices/arm-vgic.txt
+++ b/Documentation/virtual/kvm/devices/arm-vgic.txt
@@ -4,16 +4,22 @@ ARM Virtual Generic Interrupt Controller (VGIC)
4Device types supported: 4Device types supported:
5 KVM_DEV_TYPE_ARM_VGIC_V2 ARM Generic Interrupt Controller v2.0 5 KVM_DEV_TYPE_ARM_VGIC_V2 ARM Generic Interrupt Controller v2.0
6 KVM_DEV_TYPE_ARM_VGIC_V3 ARM Generic Interrupt Controller v3.0 6 KVM_DEV_TYPE_ARM_VGIC_V3 ARM Generic Interrupt Controller v3.0
7 KVM_DEV_TYPE_ARM_VGIC_ITS ARM Interrupt Translation Service Controller
7 8
8Only one VGIC instance may be instantiated through either this API or the 9Only one VGIC instance of the V2/V3 types above may be instantiated through
9legacy KVM_CREATE_IRQCHIP api. The created VGIC will act as the VM interrupt 10either this API or the legacy KVM_CREATE_IRQCHIP api. The created VGIC will
10controller, requiring emulated user-space devices to inject interrupts to the 11act as the VM interrupt controller, requiring emulated user-space devices to
11VGIC instead of directly to CPUs. 12inject interrupts to the VGIC instead of directly to CPUs.
12 13
13Creating a guest GICv3 device requires a host GICv3 as well. 14Creating a guest GICv3 device requires a host GICv3 as well.
14GICv3 implementations with hardware compatibility support allow a guest GICv2 15GICv3 implementations with hardware compatibility support allow a guest GICv2
15as well. 16as well.
16 17
18Creating a virtual ITS controller requires a host GICv3 (but does not depend
19on having physical ITS controllers).
20There can be multiple ITS controllers per guest, each of them has to have
21a separate, non-overlapping MMIO region.
22
17Groups: 23Groups:
18 KVM_DEV_ARM_VGIC_GRP_ADDR 24 KVM_DEV_ARM_VGIC_GRP_ADDR
19 Attributes: 25 Attributes:
@@ -39,6 +45,13 @@ Groups:
39 Only valid for KVM_DEV_TYPE_ARM_VGIC_V3. 45 Only valid for KVM_DEV_TYPE_ARM_VGIC_V3.
40 This address needs to be 64K aligned. 46 This address needs to be 64K aligned.
41 47
48 KVM_VGIC_V3_ADDR_TYPE_ITS (rw, 64-bit)
49 Base address in the guest physical address space of the GICv3 ITS
50 control register frame. The ITS allows MSI(-X) interrupts to be
51 injected into guests. This extension is optional. If the kernel
52 does not support the ITS, the call returns -ENODEV.
53 Only valid for KVM_DEV_TYPE_ARM_VGIC_ITS.
54 This address needs to be 64K aligned and the region covers 128K.
42 55
43 KVM_DEV_ARM_VGIC_GRP_DIST_REGS 56 KVM_DEV_ARM_VGIC_GRP_DIST_REGS
44 Attributes: 57 Attributes:
@@ -109,8 +122,8 @@ Groups:
109 KVM_DEV_ARM_VGIC_GRP_CTRL 122 KVM_DEV_ARM_VGIC_GRP_CTRL
110 Attributes: 123 Attributes:
111 KVM_DEV_ARM_VGIC_CTRL_INIT 124 KVM_DEV_ARM_VGIC_CTRL_INIT
112 request the initialization of the VGIC, no additional parameter in 125 request the initialization of the VGIC or ITS, no additional parameter
113 kvm_device_attr.addr. 126 in kvm_device_attr.addr.
114 Errors: 127 Errors:
115 -ENXIO: VGIC not properly configured as required prior to calling 128 -ENXIO: VGIC not properly configured as required prior to calling
116 this attribute 129 this attribute
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 3d5a5cd071bd..58faff5f1eb2 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -66,6 +66,8 @@ extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
66extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); 66extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
67 67
68extern void __init_stage2_translation(void); 68extern void __init_stage2_translation(void);
69
70extern void __kvm_hyp_reset(unsigned long);
69#endif 71#endif
70 72
71#endif /* __ARM_KVM_ASM_H__ */ 73#endif /* __ARM_KVM_ASM_H__ */
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 96387d477e91..de338d93d11b 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -241,8 +241,7 @@ int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *);
241int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, 241int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
242 int exception_index); 242 int exception_index);
243 243
244static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr, 244static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
245 phys_addr_t pgd_ptr,
246 unsigned long hyp_stack_ptr, 245 unsigned long hyp_stack_ptr,
247 unsigned long vector_ptr) 246 unsigned long vector_ptr)
248{ 247{
@@ -251,18 +250,13 @@ static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
251 * code. The init code doesn't need to preserve these 250 * code. The init code doesn't need to preserve these
252 * registers as r0-r3 are already callee saved according to 251 * registers as r0-r3 are already callee saved according to
253 * the AAPCS. 252 * the AAPCS.
254 * Note that we slightly misuse the prototype by casing the 253 * Note that we slightly misuse the prototype by casting the
255 * stack pointer to a void *. 254 * stack pointer to a void *.
256 *
257 * We don't have enough registers to perform the full init in
258 * one go. Install the boot PGD first, and then install the
259 * runtime PGD, stack pointer and vectors. The PGDs are always
260 * passed as the third argument, in order to be passed into
261 * r2-r3 to the init code (yes, this is compliant with the
262 * PCS!).
263 */
264 255
265 kvm_call_hyp(NULL, 0, boot_pgd_ptr); 256 * The PGDs are always passed as the third argument, in order
257 * to be passed into r2-r3 to the init code (yes, this is
258 * compliant with the PCS!).
259 */
266 260
267 kvm_call_hyp((void*)hyp_stack_ptr, vector_ptr, pgd_ptr); 261 kvm_call_hyp((void*)hyp_stack_ptr, vector_ptr, pgd_ptr);
268} 262}
@@ -272,16 +266,13 @@ static inline void __cpu_init_stage2(void)
272 kvm_call_hyp(__init_stage2_translation); 266 kvm_call_hyp(__init_stage2_translation);
273} 267}
274 268
275static inline void __cpu_reset_hyp_mode(phys_addr_t boot_pgd_ptr, 269static inline void __cpu_reset_hyp_mode(unsigned long vector_ptr,
276 phys_addr_t phys_idmap_start) 270 phys_addr_t phys_idmap_start)
277{ 271{
278 /* 272 kvm_call_hyp((void *)virt_to_idmap(__kvm_hyp_reset), vector_ptr);
279 * TODO
280 * kvm_call_reset(boot_pgd_ptr, phys_idmap_start);
281 */
282} 273}
283 274
284static inline int kvm_arch_dev_ioctl_check_extension(long ext) 275static inline int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext)
285{ 276{
286 return 0; 277 return 0;
287} 278}
diff --git a/arch/arm/include/asm/kvm_hyp.h b/arch/arm/include/asm/kvm_hyp.h
index f0e860761380..6eaff28f2ff3 100644
--- a/arch/arm/include/asm/kvm_hyp.h
+++ b/arch/arm/include/asm/kvm_hyp.h
@@ -25,9 +25,6 @@
25 25
26#define __hyp_text __section(.hyp.text) notrace 26#define __hyp_text __section(.hyp.text) notrace
27 27
28#define kern_hyp_va(v) (v)
29#define hyp_kern_va(v) (v)
30
31#define __ACCESS_CP15(CRn, Op1, CRm, Op2) \ 28#define __ACCESS_CP15(CRn, Op1, CRm, Op2) \
32 "mrc", "mcr", __stringify(p15, Op1, %0, CRn, CRm, Op2), u32 29 "mrc", "mcr", __stringify(p15, Op1, %0, CRn, CRm, Op2), u32
33#define __ACCESS_CP15_64(Op1, CRm) \ 30#define __ACCESS_CP15_64(Op1, CRm) \
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index f9a65061130b..3bb803d6814b 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -26,16 +26,7 @@
26 * We directly use the kernel VA for the HYP, as we can directly share 26 * We directly use the kernel VA for the HYP, as we can directly share
27 * the mapping (HTTBR "covers" TTBR1). 27 * the mapping (HTTBR "covers" TTBR1).
28 */ 28 */
29#define HYP_PAGE_OFFSET_MASK UL(~0) 29#define kern_hyp_va(kva) (kva)
30#define HYP_PAGE_OFFSET PAGE_OFFSET
31#define KERN_TO_HYP(kva) (kva)
32
33/*
34 * Our virtual mapping for the boot-time MMU-enable code. Must be
35 * shared across all the page-tables. Conveniently, we use the vectors
36 * page, where no kernel data will ever be shared with HYP.
37 */
38#define TRAMPOLINE_VA UL(CONFIG_VECTORS_BASE)
39 30
40/* 31/*
41 * KVM_MMU_CACHE_MIN_PAGES is the number of stage2 page table translation levels. 32 * KVM_MMU_CACHE_MIN_PAGES is the number of stage2 page table translation levels.
@@ -49,9 +40,8 @@
49#include <asm/pgalloc.h> 40#include <asm/pgalloc.h>
50#include <asm/stage2_pgtable.h> 41#include <asm/stage2_pgtable.h>
51 42
52int create_hyp_mappings(void *from, void *to); 43int create_hyp_mappings(void *from, void *to, pgprot_t prot);
53int create_hyp_io_mappings(void *from, void *to, phys_addr_t); 44int create_hyp_io_mappings(void *from, void *to, phys_addr_t);
54void free_boot_hyp_pgd(void);
55void free_hyp_pgds(void); 45void free_hyp_pgds(void);
56 46
57void stage2_unmap_vm(struct kvm *kvm); 47void stage2_unmap_vm(struct kvm *kvm);
@@ -65,7 +55,6 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
65void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); 55void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
66 56
67phys_addr_t kvm_mmu_get_httbr(void); 57phys_addr_t kvm_mmu_get_httbr(void);
68phys_addr_t kvm_mmu_get_boot_httbr(void);
69phys_addr_t kvm_get_idmap_vector(void); 58phys_addr_t kvm_get_idmap_vector(void);
70phys_addr_t kvm_get_idmap_start(void); 59phys_addr_t kvm_get_idmap_start(void);
71int kvm_mmu_init(void); 60int kvm_mmu_init(void);
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index 348caabb7625..e0d76ba24b30 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -97,7 +97,9 @@ extern pgprot_t pgprot_s2_device;
97#define PAGE_READONLY_EXEC _MOD_PROT(pgprot_user, L_PTE_USER | L_PTE_RDONLY) 97#define PAGE_READONLY_EXEC _MOD_PROT(pgprot_user, L_PTE_USER | L_PTE_RDONLY)
98#define PAGE_KERNEL _MOD_PROT(pgprot_kernel, L_PTE_XN) 98#define PAGE_KERNEL _MOD_PROT(pgprot_kernel, L_PTE_XN)
99#define PAGE_KERNEL_EXEC pgprot_kernel 99#define PAGE_KERNEL_EXEC pgprot_kernel
100#define PAGE_HYP _MOD_PROT(pgprot_kernel, L_PTE_HYP) 100#define PAGE_HYP _MOD_PROT(pgprot_kernel, L_PTE_HYP | L_PTE_XN)
101#define PAGE_HYP_EXEC _MOD_PROT(pgprot_kernel, L_PTE_HYP | L_PTE_RDONLY)
102#define PAGE_HYP_RO _MOD_PROT(pgprot_kernel, L_PTE_HYP | L_PTE_RDONLY | L_PTE_XN)
101#define PAGE_HYP_DEVICE _MOD_PROT(pgprot_hyp_device, L_PTE_HYP) 103#define PAGE_HYP_DEVICE _MOD_PROT(pgprot_hyp_device, L_PTE_HYP)
102#define PAGE_S2 _MOD_PROT(pgprot_s2, L_PTE_S2_RDONLY) 104#define PAGE_S2 _MOD_PROT(pgprot_s2, L_PTE_S2_RDONLY)
103#define PAGE_S2_DEVICE _MOD_PROT(pgprot_s2_device, L_PTE_S2_RDONLY) 105#define PAGE_S2_DEVICE _MOD_PROT(pgprot_s2_device, L_PTE_S2_RDONLY)
diff --git a/arch/arm/include/asm/virt.h b/arch/arm/include/asm/virt.h
index d4ceaf5f299b..a2e75b84e2ae 100644
--- a/arch/arm/include/asm/virt.h
+++ b/arch/arm/include/asm/virt.h
@@ -80,6 +80,10 @@ static inline bool is_kernel_in_hyp_mode(void)
80 return false; 80 return false;
81} 81}
82 82
83/* The section containing the hypervisor idmap text */
84extern char __hyp_idmap_text_start[];
85extern char __hyp_idmap_text_end[];
86
83/* The section containing the hypervisor text */ 87/* The section containing the hypervisor text */
84extern char __hyp_text_start[]; 88extern char __hyp_text_start[];
85extern char __hyp_text_end[]; 89extern char __hyp_text_end[];
diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig
index 02abfff68ee5..95a000515e43 100644
--- a/arch/arm/kvm/Kconfig
+++ b/arch/arm/kvm/Kconfig
@@ -46,13 +46,6 @@ config KVM_ARM_HOST
46 ---help--- 46 ---help---
47 Provides host support for ARM processors. 47 Provides host support for ARM processors.
48 48
49config KVM_NEW_VGIC
50 bool "New VGIC implementation"
51 depends on KVM
52 default y
53 ---help---
54 uses the new VGIC implementation
55
56source drivers/vhost/Kconfig 49source drivers/vhost/Kconfig
57 50
58endif # VIRTUALIZATION 51endif # VIRTUALIZATION
diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile
index a596b58f6d37..5e28df80dca7 100644
--- a/arch/arm/kvm/Makefile
+++ b/arch/arm/kvm/Makefile
@@ -22,7 +22,6 @@ obj-y += kvm-arm.o init.o interrupts.o
22obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o 22obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o
23obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o 23obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o
24 24
25ifeq ($(CONFIG_KVM_NEW_VGIC),y)
26obj-y += $(KVM)/arm/vgic/vgic.o 25obj-y += $(KVM)/arm/vgic/vgic.o
27obj-y += $(KVM)/arm/vgic/vgic-init.o 26obj-y += $(KVM)/arm/vgic/vgic-init.o
28obj-y += $(KVM)/arm/vgic/vgic-irqfd.o 27obj-y += $(KVM)/arm/vgic/vgic-irqfd.o
@@ -30,9 +29,4 @@ obj-y += $(KVM)/arm/vgic/vgic-v2.o
30obj-y += $(KVM)/arm/vgic/vgic-mmio.o 29obj-y += $(KVM)/arm/vgic/vgic-mmio.o
31obj-y += $(KVM)/arm/vgic/vgic-mmio-v2.o 30obj-y += $(KVM)/arm/vgic/vgic-mmio-v2.o
32obj-y += $(KVM)/arm/vgic/vgic-kvm-device.o 31obj-y += $(KVM)/arm/vgic/vgic-kvm-device.o
33else
34obj-y += $(KVM)/arm/vgic.o
35obj-y += $(KVM)/arm/vgic-v2.o
36obj-y += $(KVM)/arm/vgic-v2-emul.o
37endif
38obj-y += $(KVM)/arm/arch_timer.o 32obj-y += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 9ac4970882fe..abdb2ea19bf0 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -20,6 +20,7 @@
20#include <linux/errno.h> 20#include <linux/errno.h>
21#include <linux/err.h> 21#include <linux/err.h>
22#include <linux/kvm_host.h> 22#include <linux/kvm_host.h>
23#include <linux/list.h>
23#include <linux/module.h> 24#include <linux/module.h>
24#include <linux/vmalloc.h> 25#include <linux/vmalloc.h>
25#include <linux/fs.h> 26#include <linux/fs.h>
@@ -122,7 +123,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
122 if (ret) 123 if (ret)
123 goto out_fail_alloc; 124 goto out_fail_alloc;
124 125
125 ret = create_hyp_mappings(kvm, kvm + 1); 126 ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP);
126 if (ret) 127 if (ret)
127 goto out_free_stage2_pgd; 128 goto out_free_stage2_pgd;
128 129
@@ -201,7 +202,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
201 r = KVM_MAX_VCPUS; 202 r = KVM_MAX_VCPUS;
202 break; 203 break;
203 default: 204 default:
204 r = kvm_arch_dev_ioctl_check_extension(ext); 205 r = kvm_arch_dev_ioctl_check_extension(kvm, ext);
205 break; 206 break;
206 } 207 }
207 return r; 208 return r;
@@ -239,7 +240,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
239 if (err) 240 if (err)
240 goto free_vcpu; 241 goto free_vcpu;
241 242
242 err = create_hyp_mappings(vcpu, vcpu + 1); 243 err = create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP);
243 if (err) 244 if (err)
244 goto vcpu_uninit; 245 goto vcpu_uninit;
245 246
@@ -1038,7 +1039,6 @@ long kvm_arch_vm_ioctl(struct file *filp,
1038 1039
1039static void cpu_init_hyp_mode(void *dummy) 1040static void cpu_init_hyp_mode(void *dummy)
1040{ 1041{
1041 phys_addr_t boot_pgd_ptr;
1042 phys_addr_t pgd_ptr; 1042 phys_addr_t pgd_ptr;
1043 unsigned long hyp_stack_ptr; 1043 unsigned long hyp_stack_ptr;
1044 unsigned long stack_page; 1044 unsigned long stack_page;
@@ -1047,13 +1047,12 @@ static void cpu_init_hyp_mode(void *dummy)
1047 /* Switch from the HYP stub to our own HYP init vector */ 1047 /* Switch from the HYP stub to our own HYP init vector */
1048 __hyp_set_vectors(kvm_get_idmap_vector()); 1048 __hyp_set_vectors(kvm_get_idmap_vector());
1049 1049
1050 boot_pgd_ptr = kvm_mmu_get_boot_httbr();
1051 pgd_ptr = kvm_mmu_get_httbr(); 1050 pgd_ptr = kvm_mmu_get_httbr();
1052 stack_page = __this_cpu_read(kvm_arm_hyp_stack_page); 1051 stack_page = __this_cpu_read(kvm_arm_hyp_stack_page);
1053 hyp_stack_ptr = stack_page + PAGE_SIZE; 1052 hyp_stack_ptr = stack_page + PAGE_SIZE;
1054 vector_ptr = (unsigned long)kvm_ksym_ref(__kvm_hyp_vector); 1053 vector_ptr = (unsigned long)kvm_ksym_ref(__kvm_hyp_vector);
1055 1054
1056 __cpu_init_hyp_mode(boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr); 1055 __cpu_init_hyp_mode(pgd_ptr, hyp_stack_ptr, vector_ptr);
1057 __cpu_init_stage2(); 1056 __cpu_init_stage2();
1058 1057
1059 kvm_arm_init_debug(); 1058 kvm_arm_init_debug();
@@ -1075,15 +1074,9 @@ static void cpu_hyp_reinit(void)
1075 1074
1076static void cpu_hyp_reset(void) 1075static void cpu_hyp_reset(void)
1077{ 1076{
1078 phys_addr_t boot_pgd_ptr; 1077 if (!is_kernel_in_hyp_mode())
1079 phys_addr_t phys_idmap_start; 1078 __cpu_reset_hyp_mode(hyp_default_vectors,
1080 1079 kvm_get_idmap_start());
1081 if (!is_kernel_in_hyp_mode()) {
1082 boot_pgd_ptr = kvm_mmu_get_boot_httbr();
1083 phys_idmap_start = kvm_get_idmap_start();
1084
1085 __cpu_reset_hyp_mode(boot_pgd_ptr, phys_idmap_start);
1086 }
1087} 1080}
1088 1081
1089static void _kvm_arch_hardware_enable(void *discard) 1082static void _kvm_arch_hardware_enable(void *discard)
@@ -1293,14 +1286,14 @@ static int init_hyp_mode(void)
1293 * Map the Hyp-code called directly from the host 1286 * Map the Hyp-code called directly from the host
1294 */ 1287 */
1295 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start), 1288 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
1296 kvm_ksym_ref(__hyp_text_end)); 1289 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
1297 if (err) { 1290 if (err) {
1298 kvm_err("Cannot map world-switch code\n"); 1291 kvm_err("Cannot map world-switch code\n");
1299 goto out_err; 1292 goto out_err;
1300 } 1293 }
1301 1294
1302 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata), 1295 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
1303 kvm_ksym_ref(__end_rodata)); 1296 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
1304 if (err) { 1297 if (err) {
1305 kvm_err("Cannot map rodata section\n"); 1298 kvm_err("Cannot map rodata section\n");
1306 goto out_err; 1299 goto out_err;
@@ -1311,7 +1304,8 @@ static int init_hyp_mode(void)
1311 */ 1304 */
1312 for_each_possible_cpu(cpu) { 1305 for_each_possible_cpu(cpu) {
1313 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu); 1306 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
1314 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE); 1307 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE,
1308 PAGE_HYP);
1315 1309
1316 if (err) { 1310 if (err) {
1317 kvm_err("Cannot map hyp stack\n"); 1311 kvm_err("Cannot map hyp stack\n");
@@ -1323,7 +1317,7 @@ static int init_hyp_mode(void)
1323 kvm_cpu_context_t *cpu_ctxt; 1317 kvm_cpu_context_t *cpu_ctxt;
1324 1318
1325 cpu_ctxt = per_cpu_ptr(kvm_host_cpu_state, cpu); 1319 cpu_ctxt = per_cpu_ptr(kvm_host_cpu_state, cpu);
1326 err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1); 1320 err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1, PAGE_HYP);
1327 1321
1328 if (err) { 1322 if (err) {
1329 kvm_err("Cannot map host CPU state: %d\n", err); 1323 kvm_err("Cannot map host CPU state: %d\n", err);
@@ -1331,10 +1325,6 @@ static int init_hyp_mode(void)
1331 } 1325 }
1332 } 1326 }
1333 1327
1334#ifndef CONFIG_HOTPLUG_CPU
1335 free_boot_hyp_pgd();
1336#endif
1337
1338 /* set size of VMID supported by CPU */ 1328 /* set size of VMID supported by CPU */
1339 kvm_vmid_bits = kvm_get_vmid_bits(); 1329 kvm_vmid_bits = kvm_get_vmid_bits();
1340 kvm_info("%d-bit VMID\n", kvm_vmid_bits); 1330 kvm_info("%d-bit VMID\n", kvm_vmid_bits);
diff --git a/arch/arm/kvm/init.S b/arch/arm/kvm/init.S
index 1f9ae17476f9..bf89c919efc1 100644
--- a/arch/arm/kvm/init.S
+++ b/arch/arm/kvm/init.S
@@ -32,23 +32,13 @@
32 * r2,r3 = Hypervisor pgd pointer 32 * r2,r3 = Hypervisor pgd pointer
33 * 33 *
34 * The init scenario is: 34 * The init scenario is:
35 * - We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd, 35 * - We jump in HYP with 3 parameters: runtime HYP pgd, runtime stack,
36 * runtime stack, runtime vectors 36 * runtime vectors
37 * - Enable the MMU with the boot pgd
38 * - Jump to a target into the trampoline page (remember, this is the same
39 * physical page!)
40 * - Now switch to the runtime pgd (same VA, and still the same physical
41 * page!)
42 * - Invalidate TLBs 37 * - Invalidate TLBs
43 * - Set stack and vectors 38 * - Set stack and vectors
39 * - Setup the page tables
40 * - Enable the MMU
44 * - Profit! (or eret, if you only care about the code). 41 * - Profit! (or eret, if you only care about the code).
45 *
46 * As we only have four registers available to pass parameters (and we
47 * need six), we split the init in two phases:
48 * - Phase 1: r0 = 0, r1 = 0, r2,r3 contain the boot PGD.
49 * Provides the basic HYP init, and enable the MMU.
50 * - Phase 2: r0 = ToS, r1 = vectors, r2,r3 contain the runtime PGD.
51 * Switches to the runtime PGD, set stack and vectors.
52 */ 42 */
53 43
54 .text 44 .text
@@ -68,8 +58,11 @@ __kvm_hyp_init:
68 W(b) . 58 W(b) .
69 59
70__do_hyp_init: 60__do_hyp_init:
71 cmp r0, #0 @ We have a SP? 61 @ Set stack pointer
72 bne phase2 @ Yes, second stage init 62 mov sp, r0
63
64 @ Set HVBAR to point to the HYP vectors
65 mcr p15, 4, r1, c12, c0, 0 @ HVBAR
73 66
74 @ Set the HTTBR to point to the hypervisor PGD pointer passed 67 @ Set the HTTBR to point to the hypervisor PGD pointer passed
75 mcrr p15, 4, rr_lo_hi(r2, r3), c2 68 mcrr p15, 4, rr_lo_hi(r2, r3), c2
@@ -114,34 +107,25 @@ __do_hyp_init:
114 THUMB( ldr r2, =(HSCTLR_M | HSCTLR_A | HSCTLR_TE) ) 107 THUMB( ldr r2, =(HSCTLR_M | HSCTLR_A | HSCTLR_TE) )
115 orr r1, r1, r2 108 orr r1, r1, r2
116 orr r0, r0, r1 109 orr r0, r0, r1
117 isb
118 mcr p15, 4, r0, c1, c0, 0 @ HSCR 110 mcr p15, 4, r0, c1, c0, 0 @ HSCR
111 isb
119 112
120 @ End of init phase-1
121 eret 113 eret
122 114
123phase2: 115 @ r0 : stub vectors address
124 @ Set stack pointer 116ENTRY(__kvm_hyp_reset)
125 mov sp, r0 117 /* We're now in idmap, disable MMU */
126 118 mrc p15, 4, r1, c1, c0, 0 @ HSCTLR
127 @ Set HVBAR to point to the HYP vectors 119 ldr r2, =(HSCTLR_M | HSCTLR_A | HSCTLR_C | HSCTLR_I)
128 mcr p15, 4, r1, c12, c0, 0 @ HVBAR 120 bic r1, r1, r2
129 121 mcr p15, 4, r1, c1, c0, 0 @ HSCTLR
130 @ Jump to the trampoline page
131 ldr r0, =TRAMPOLINE_VA
132 adr r1, target
133 bfi r0, r1, #0, #PAGE_SHIFT
134 ret r0
135 122
136target: @ We're now in the trampoline code, switch page tables 123 /* Install stub vectors */
137 mcrr p15, 4, rr_lo_hi(r2, r3), c2 124 mcr p15, 4, r0, c12, c0, 0 @ HVBAR
138 isb 125 isb
139 126
140 @ Invalidate the old TLBs
141 mcr p15, 4, r0, c8, c7, 0 @ TLBIALLH
142 dsb ish
143
144 eret 127 eret
128ENDPROC(__kvm_hyp_reset)
145 129
146 .ltorg 130 .ltorg
147 131
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 45c43aecb8f2..bda27b6b1aa2 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -32,8 +32,6 @@
32 32
33#include "trace.h" 33#include "trace.h"
34 34
35extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
36
37static pgd_t *boot_hyp_pgd; 35static pgd_t *boot_hyp_pgd;
38static pgd_t *hyp_pgd; 36static pgd_t *hyp_pgd;
39static pgd_t *merged_hyp_pgd; 37static pgd_t *merged_hyp_pgd;
@@ -484,28 +482,6 @@ static void unmap_hyp_range(pgd_t *pgdp, phys_addr_t start, u64 size)
484} 482}
485 483
486/** 484/**
487 * free_boot_hyp_pgd - free HYP boot page tables
488 *
489 * Free the HYP boot page tables. The bounce page is also freed.
490 */
491void free_boot_hyp_pgd(void)
492{
493 mutex_lock(&kvm_hyp_pgd_mutex);
494
495 if (boot_hyp_pgd) {
496 unmap_hyp_range(boot_hyp_pgd, hyp_idmap_start, PAGE_SIZE);
497 unmap_hyp_range(boot_hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE);
498 free_pages((unsigned long)boot_hyp_pgd, hyp_pgd_order);
499 boot_hyp_pgd = NULL;
500 }
501
502 if (hyp_pgd)
503 unmap_hyp_range(hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE);
504
505 mutex_unlock(&kvm_hyp_pgd_mutex);
506}
507
508/**
509 * free_hyp_pgds - free Hyp-mode page tables 485 * free_hyp_pgds - free Hyp-mode page tables
510 * 486 *
511 * Assumes hyp_pgd is a page table used strictly in Hyp-mode and 487 * Assumes hyp_pgd is a page table used strictly in Hyp-mode and
@@ -519,15 +495,20 @@ void free_hyp_pgds(void)
519{ 495{
520 unsigned long addr; 496 unsigned long addr;
521 497
522 free_boot_hyp_pgd();
523
524 mutex_lock(&kvm_hyp_pgd_mutex); 498 mutex_lock(&kvm_hyp_pgd_mutex);
525 499
500 if (boot_hyp_pgd) {
501 unmap_hyp_range(boot_hyp_pgd, hyp_idmap_start, PAGE_SIZE);
502 free_pages((unsigned long)boot_hyp_pgd, hyp_pgd_order);
503 boot_hyp_pgd = NULL;
504 }
505
526 if (hyp_pgd) { 506 if (hyp_pgd) {
507 unmap_hyp_range(hyp_pgd, hyp_idmap_start, PAGE_SIZE);
527 for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE) 508 for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE)
528 unmap_hyp_range(hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE); 509 unmap_hyp_range(hyp_pgd, kern_hyp_va(addr), PGDIR_SIZE);
529 for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE) 510 for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE)
530 unmap_hyp_range(hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE); 511 unmap_hyp_range(hyp_pgd, kern_hyp_va(addr), PGDIR_SIZE);
531 512
532 free_pages((unsigned long)hyp_pgd, hyp_pgd_order); 513 free_pages((unsigned long)hyp_pgd, hyp_pgd_order);
533 hyp_pgd = NULL; 514 hyp_pgd = NULL;
@@ -679,17 +660,18 @@ static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
679 * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode 660 * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
680 * @from: The virtual kernel start address of the range 661 * @from: The virtual kernel start address of the range
681 * @to: The virtual kernel end address of the range (exclusive) 662 * @to: The virtual kernel end address of the range (exclusive)
663 * @prot: The protection to be applied to this range
682 * 664 *
683 * The same virtual address as the kernel virtual address is also used 665 * The same virtual address as the kernel virtual address is also used
684 * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying 666 * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
685 * physical pages. 667 * physical pages.
686 */ 668 */
687int create_hyp_mappings(void *from, void *to) 669int create_hyp_mappings(void *from, void *to, pgprot_t prot)
688{ 670{
689 phys_addr_t phys_addr; 671 phys_addr_t phys_addr;
690 unsigned long virt_addr; 672 unsigned long virt_addr;
691 unsigned long start = KERN_TO_HYP((unsigned long)from); 673 unsigned long start = kern_hyp_va((unsigned long)from);
692 unsigned long end = KERN_TO_HYP((unsigned long)to); 674 unsigned long end = kern_hyp_va((unsigned long)to);
693 675
694 if (is_kernel_in_hyp_mode()) 676 if (is_kernel_in_hyp_mode())
695 return 0; 677 return 0;
@@ -704,7 +686,7 @@ int create_hyp_mappings(void *from, void *to)
704 err = __create_hyp_mappings(hyp_pgd, virt_addr, 686 err = __create_hyp_mappings(hyp_pgd, virt_addr,
705 virt_addr + PAGE_SIZE, 687 virt_addr + PAGE_SIZE,
706 __phys_to_pfn(phys_addr), 688 __phys_to_pfn(phys_addr),
707 PAGE_HYP); 689 prot);
708 if (err) 690 if (err)
709 return err; 691 return err;
710 } 692 }
@@ -723,8 +705,8 @@ int create_hyp_mappings(void *from, void *to)
723 */ 705 */
724int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr) 706int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
725{ 707{
726 unsigned long start = KERN_TO_HYP((unsigned long)from); 708 unsigned long start = kern_hyp_va((unsigned long)from);
727 unsigned long end = KERN_TO_HYP((unsigned long)to); 709 unsigned long end = kern_hyp_va((unsigned long)to);
728 710
729 if (is_kernel_in_hyp_mode()) 711 if (is_kernel_in_hyp_mode())
730 return 0; 712 return 0;
@@ -1687,14 +1669,6 @@ phys_addr_t kvm_mmu_get_httbr(void)
1687 return virt_to_phys(hyp_pgd); 1669 return virt_to_phys(hyp_pgd);
1688} 1670}
1689 1671
1690phys_addr_t kvm_mmu_get_boot_httbr(void)
1691{
1692 if (__kvm_cpu_uses_extended_idmap())
1693 return virt_to_phys(merged_hyp_pgd);
1694 else
1695 return virt_to_phys(boot_hyp_pgd);
1696}
1697
1698phys_addr_t kvm_get_idmap_vector(void) 1672phys_addr_t kvm_get_idmap_vector(void)
1699{ 1673{
1700 return hyp_idmap_vector; 1674 return hyp_idmap_vector;
@@ -1705,6 +1679,22 @@ phys_addr_t kvm_get_idmap_start(void)
1705 return hyp_idmap_start; 1679 return hyp_idmap_start;
1706} 1680}
1707 1681
1682static int kvm_map_idmap_text(pgd_t *pgd)
1683{
1684 int err;
1685
1686 /* Create the idmap in the boot page tables */
1687 err = __create_hyp_mappings(pgd,
1688 hyp_idmap_start, hyp_idmap_end,
1689 __phys_to_pfn(hyp_idmap_start),
1690 PAGE_HYP_EXEC);
1691 if (err)
1692 kvm_err("Failed to idmap %lx-%lx\n",
1693 hyp_idmap_start, hyp_idmap_end);
1694
1695 return err;
1696}
1697
1708int kvm_mmu_init(void) 1698int kvm_mmu_init(void)
1709{ 1699{
1710 int err; 1700 int err;
@@ -1719,28 +1709,41 @@ int kvm_mmu_init(void)
1719 */ 1709 */
1720 BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK); 1710 BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
1721 1711
1722 hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order); 1712 kvm_info("IDMAP page: %lx\n", hyp_idmap_start);
1723 boot_hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order); 1713 kvm_info("HYP VA range: %lx:%lx\n",
1714 kern_hyp_va(PAGE_OFFSET), kern_hyp_va(~0UL));
1724 1715
1725 if (!hyp_pgd || !boot_hyp_pgd) { 1716 if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) &&
1726 kvm_err("Hyp mode PGD not allocated\n"); 1717 hyp_idmap_start < kern_hyp_va(~0UL)) {
1727 err = -ENOMEM; 1718 /*
1719 * The idmap page is intersecting with the VA space,
1720 * it is not safe to continue further.
1721 */
1722 kvm_err("IDMAP intersecting with HYP VA, unable to continue\n");
1723 err = -EINVAL;
1728 goto out; 1724 goto out;
1729 } 1725 }
1730 1726
1731 /* Create the idmap in the boot page tables */ 1727 hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order);
1732 err = __create_hyp_mappings(boot_hyp_pgd, 1728 if (!hyp_pgd) {
1733 hyp_idmap_start, hyp_idmap_end, 1729 kvm_err("Hyp mode PGD not allocated\n");
1734 __phys_to_pfn(hyp_idmap_start), 1730 err = -ENOMEM;
1735 PAGE_HYP);
1736
1737 if (err) {
1738 kvm_err("Failed to idmap %lx-%lx\n",
1739 hyp_idmap_start, hyp_idmap_end);
1740 goto out; 1731 goto out;
1741 } 1732 }
1742 1733
1743 if (__kvm_cpu_uses_extended_idmap()) { 1734 if (__kvm_cpu_uses_extended_idmap()) {
1735 boot_hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1736 hyp_pgd_order);
1737 if (!boot_hyp_pgd) {
1738 kvm_err("Hyp boot PGD not allocated\n");
1739 err = -ENOMEM;
1740 goto out;
1741 }
1742
1743 err = kvm_map_idmap_text(boot_hyp_pgd);
1744 if (err)
1745 goto out;
1746
1744 merged_hyp_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 1747 merged_hyp_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
1745 if (!merged_hyp_pgd) { 1748 if (!merged_hyp_pgd) {
1746 kvm_err("Failed to allocate extra HYP pgd\n"); 1749 kvm_err("Failed to allocate extra HYP pgd\n");
@@ -1748,29 +1751,10 @@ int kvm_mmu_init(void)
1748 } 1751 }
1749 __kvm_extend_hypmap(boot_hyp_pgd, hyp_pgd, merged_hyp_pgd, 1752 __kvm_extend_hypmap(boot_hyp_pgd, hyp_pgd, merged_hyp_pgd,
1750 hyp_idmap_start); 1753 hyp_idmap_start);
1751 return 0; 1754 } else {
1752 } 1755 err = kvm_map_idmap_text(hyp_pgd);
1753 1756 if (err)
1754 /* Map the very same page at the trampoline VA */ 1757 goto out;
1755 err = __create_hyp_mappings(boot_hyp_pgd,
1756 TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE,
1757 __phys_to_pfn(hyp_idmap_start),
1758 PAGE_HYP);
1759 if (err) {
1760 kvm_err("Failed to map trampoline @%lx into boot HYP pgd\n",
1761 TRAMPOLINE_VA);
1762 goto out;
1763 }
1764
1765 /* Map the same page again into the runtime page tables */
1766 err = __create_hyp_mappings(hyp_pgd,
1767 TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE,
1768 __phys_to_pfn(hyp_idmap_start),
1769 PAGE_HYP);
1770 if (err) {
1771 kvm_err("Failed to map trampoline @%lx into runtime HYP pgd\n",
1772 TRAMPOLINE_VA);
1773 goto out;
1774 } 1758 }
1775 1759
1776 return 0; 1760 return 0;
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 224efe730e46..d40edbb6ef23 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -36,8 +36,9 @@
36#define ARM64_HAS_VIRT_HOST_EXTN 11 36#define ARM64_HAS_VIRT_HOST_EXTN 11
37#define ARM64_WORKAROUND_CAVIUM_27456 12 37#define ARM64_WORKAROUND_CAVIUM_27456 12
38#define ARM64_HAS_32BIT_EL0 13 38#define ARM64_HAS_32BIT_EL0 13
39#define ARM64_HYP_OFFSET_LOW 14
39 40
40#define ARM64_NCAPS 14 41#define ARM64_NCAPS 15
41 42
42#ifndef __ASSEMBLY__ 43#ifndef __ASSEMBLY__
43 44
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 49095fc4b482..3eda975837d0 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -47,8 +47,7 @@
47 47
48int __attribute_const__ kvm_target_cpu(void); 48int __attribute_const__ kvm_target_cpu(void);
49int kvm_reset_vcpu(struct kvm_vcpu *vcpu); 49int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
50int kvm_arch_dev_ioctl_check_extension(long ext); 50int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext);
51unsigned long kvm_hyp_reset_entry(void);
52void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start); 51void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
53 52
54struct kvm_arch { 53struct kvm_arch {
@@ -348,8 +347,7 @@ int kvm_perf_teardown(void);
348 347
349struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr); 348struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
350 349
351static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr, 350static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
352 phys_addr_t pgd_ptr,
353 unsigned long hyp_stack_ptr, 351 unsigned long hyp_stack_ptr,
354 unsigned long vector_ptr) 352 unsigned long vector_ptr)
355{ 353{
@@ -357,19 +355,14 @@ static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
357 * Call initialization code, and switch to the full blown 355 * Call initialization code, and switch to the full blown
358 * HYP code. 356 * HYP code.
359 */ 357 */
360 __kvm_call_hyp((void *)boot_pgd_ptr, pgd_ptr, 358 __kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);
361 hyp_stack_ptr, vector_ptr);
362} 359}
363 360
364static inline void __cpu_reset_hyp_mode(phys_addr_t boot_pgd_ptr, 361void __kvm_hyp_teardown(void);
362static inline void __cpu_reset_hyp_mode(unsigned long vector_ptr,
365 phys_addr_t phys_idmap_start) 363 phys_addr_t phys_idmap_start)
366{ 364{
367 /* 365 kvm_call_hyp(__kvm_hyp_teardown, phys_idmap_start);
368 * Call reset code, and switch back to stub hyp vectors.
369 * Uses __kvm_call_hyp() to avoid kaslr's kvm_ksym_ref() translation.
370 */
371 __kvm_call_hyp((void *)kvm_hyp_reset_entry(),
372 boot_pgd_ptr, phys_idmap_start);
373} 366}
374 367
375static inline void kvm_arch_hardware_unsetup(void) {} 368static inline void kvm_arch_hardware_unsetup(void) {}
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 44eaff70da6a..cff510574fae 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -25,29 +25,6 @@
25 25
26#define __hyp_text __section(.hyp.text) notrace 26#define __hyp_text __section(.hyp.text) notrace
27 27
28static inline unsigned long __kern_hyp_va(unsigned long v)
29{
30 asm volatile(ALTERNATIVE("and %0, %0, %1",
31 "nop",
32 ARM64_HAS_VIRT_HOST_EXTN)
33 : "+r" (v) : "i" (HYP_PAGE_OFFSET_MASK));
34 return v;
35}
36
37#define kern_hyp_va(v) (typeof(v))(__kern_hyp_va((unsigned long)(v)))
38
39static inline unsigned long __hyp_kern_va(unsigned long v)
40{
41 u64 offset = PAGE_OFFSET - HYP_PAGE_OFFSET;
42 asm volatile(ALTERNATIVE("add %0, %0, %1",
43 "nop",
44 ARM64_HAS_VIRT_HOST_EXTN)
45 : "+r" (v) : "r" (offset));
46 return v;
47}
48
49#define hyp_kern_va(v) (typeof(v))(__hyp_kern_va((unsigned long)(v)))
50
51#define read_sysreg_elx(r,nvh,vh) \ 28#define read_sysreg_elx(r,nvh,vh) \
52 ({ \ 29 ({ \
53 u64 reg; \ 30 u64 reg; \
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index f05ac27d033e..b6bb83400cd8 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -29,21 +29,48 @@
29 * 29 *
30 * Instead, give the HYP mode its own VA region at a fixed offset from 30 * Instead, give the HYP mode its own VA region at a fixed offset from
31 * the kernel by just masking the top bits (which are all ones for a 31 * the kernel by just masking the top bits (which are all ones for a
32 * kernel address). 32 * kernel address). We need to find out how many bits to mask.
33 * 33 *
34 * ARMv8.1 (using VHE) does have a TTBR1_EL2, and doesn't use these 34 * We want to build a set of page tables that cover both parts of the
35 * macros (the entire kernel runs at EL2). 35 * idmap (the trampoline page used to initialize EL2), and our normal
36 * runtime VA space, at the same time.
37 *
38 * Given that the kernel uses VA_BITS for its entire address space,
39 * and that half of that space (VA_BITS - 1) is used for the linear
40 * mapping, we can also limit the EL2 space to (VA_BITS - 1).
41 *
42 * The main question is "Within the VA_BITS space, does EL2 use the
43 * top or the bottom half of that space to shadow the kernel's linear
44 * mapping?". As we need to idmap the trampoline page, this is
45 * determined by the range in which this page lives.
46 *
47 * If the page is in the bottom half, we have to use the top half. If
48 * the page is in the top half, we have to use the bottom half:
49 *
50 * T = __virt_to_phys(__hyp_idmap_text_start)
51 * if (T & BIT(VA_BITS - 1))
52 * HYP_VA_MIN = 0 //idmap in upper half
53 * else
54 * HYP_VA_MIN = 1 << (VA_BITS - 1)
55 * HYP_VA_MAX = HYP_VA_MIN + (1 << (VA_BITS - 1)) - 1
56 *
57 * This of course assumes that the trampoline page exists within the
58 * VA_BITS range. If it doesn't, then it means we're in the odd case
59 * where the kernel idmap (as well as HYP) uses more levels than the
60 * kernel runtime page tables (as seen when the kernel is configured
61 * for 4k pages, 39bits VA, and yet memory lives just above that
62 * limit, forcing the idmap to use 4 levels of page tables while the
63 * kernel itself only uses 3). In this particular case, it doesn't
64 * matter which side of VA_BITS we use, as we're guaranteed not to
65 * conflict with anything.
66 *
67 * When using VHE, there are no separate hyp mappings and all KVM
68 * functionality is already mapped as part of the main kernel
69 * mappings, and none of this applies in that case.
36 */ 70 */
37#define HYP_PAGE_OFFSET_SHIFT VA_BITS
38#define HYP_PAGE_OFFSET_MASK ((UL(1) << HYP_PAGE_OFFSET_SHIFT) - 1)
39#define HYP_PAGE_OFFSET (PAGE_OFFSET & HYP_PAGE_OFFSET_MASK)
40 71
41/* 72#define HYP_PAGE_OFFSET_HIGH_MASK ((UL(1) << VA_BITS) - 1)
42 * Our virtual mapping for the idmap-ed MMU-enable code. Must be 73#define HYP_PAGE_OFFSET_LOW_MASK ((UL(1) << (VA_BITS - 1)) - 1)
43 * shared across all the page-tables. Conveniently, we use the last
44 * possible page, where no kernel mapping will ever exist.
45 */
46#define TRAMPOLINE_VA (HYP_PAGE_OFFSET_MASK & PAGE_MASK)
47 74
48#ifdef __ASSEMBLY__ 75#ifdef __ASSEMBLY__
49 76
@@ -53,13 +80,33 @@
53/* 80/*
54 * Convert a kernel VA into a HYP VA. 81 * Convert a kernel VA into a HYP VA.
55 * reg: VA to be converted. 82 * reg: VA to be converted.
83 *
84 * This generates the following sequences:
85 * - High mask:
86 * and x0, x0, #HYP_PAGE_OFFSET_HIGH_MASK
87 * nop
88 * - Low mask:
89 * and x0, x0, #HYP_PAGE_OFFSET_HIGH_MASK
90 * and x0, x0, #HYP_PAGE_OFFSET_LOW_MASK
91 * - VHE:
92 * nop
93 * nop
94 *
95 * The "low mask" version works because the mask is a strict subset of
96 * the "high mask", hence performing the first mask for nothing.
97 * Should be completely invisible on any viable CPU.
56 */ 98 */
57.macro kern_hyp_va reg 99.macro kern_hyp_va reg
58alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 100alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
59 and \reg, \reg, #HYP_PAGE_OFFSET_MASK 101 and \reg, \reg, #HYP_PAGE_OFFSET_HIGH_MASK
60alternative_else 102alternative_else
61 nop 103 nop
62alternative_endif 104alternative_endif
105alternative_if_not ARM64_HYP_OFFSET_LOW
106 nop
107alternative_else
108 and \reg, \reg, #HYP_PAGE_OFFSET_LOW_MASK
109alternative_endif
63.endm 110.endm
64 111
65#else 112#else
@@ -70,7 +117,22 @@ alternative_endif
70#include <asm/mmu_context.h> 117#include <asm/mmu_context.h>
71#include <asm/pgtable.h> 118#include <asm/pgtable.h>
72 119
73#define KERN_TO_HYP(kva) ((unsigned long)kva - PAGE_OFFSET + HYP_PAGE_OFFSET) 120static inline unsigned long __kern_hyp_va(unsigned long v)
121{
122 asm volatile(ALTERNATIVE("and %0, %0, %1",
123 "nop",
124 ARM64_HAS_VIRT_HOST_EXTN)
125 : "+r" (v)
126 : "i" (HYP_PAGE_OFFSET_HIGH_MASK));
127 asm volatile(ALTERNATIVE("nop",
128 "and %0, %0, %1",
129 ARM64_HYP_OFFSET_LOW)
130 : "+r" (v)
131 : "i" (HYP_PAGE_OFFSET_LOW_MASK));
132 return v;
133}
134
135#define kern_hyp_va(v) (typeof(v))(__kern_hyp_va((unsigned long)(v)))
74 136
75/* 137/*
76 * We currently only support a 40bit IPA. 138 * We currently only support a 40bit IPA.
@@ -81,9 +143,8 @@ alternative_endif
81 143
82#include <asm/stage2_pgtable.h> 144#include <asm/stage2_pgtable.h>
83 145
84int create_hyp_mappings(void *from, void *to); 146int create_hyp_mappings(void *from, void *to, pgprot_t prot);
85int create_hyp_io_mappings(void *from, void *to, phys_addr_t); 147int create_hyp_io_mappings(void *from, void *to, phys_addr_t);
86void free_boot_hyp_pgd(void);
87void free_hyp_pgds(void); 148void free_hyp_pgds(void);
88 149
89void stage2_unmap_vm(struct kvm *kvm); 150void stage2_unmap_vm(struct kvm *kvm);
@@ -97,7 +158,6 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
97void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); 158void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
98 159
99phys_addr_t kvm_mmu_get_httbr(void); 160phys_addr_t kvm_mmu_get_httbr(void);
100phys_addr_t kvm_mmu_get_boot_httbr(void);
101phys_addr_t kvm_get_idmap_vector(void); 161phys_addr_t kvm_get_idmap_vector(void);
102phys_addr_t kvm_get_idmap_start(void); 162phys_addr_t kvm_get_idmap_start(void);
103int kvm_mmu_init(void); 163int kvm_mmu_init(void);
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index 2813748e2f24..c3ae239db3ee 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -164,6 +164,7 @@
164#define PTE_CONT (_AT(pteval_t, 1) << 52) /* Contiguous range */ 164#define PTE_CONT (_AT(pteval_t, 1) << 52) /* Contiguous range */
165#define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */ 165#define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */
166#define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */ 166#define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */
167#define PTE_HYP_XN (_AT(pteval_t, 1) << 54) /* HYP XN */
167 168
168/* 169/*
169 * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers). 170 * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h
index 29fcb33ab401..39f5252673f7 100644
--- a/arch/arm64/include/asm/pgtable-prot.h
+++ b/arch/arm64/include/asm/pgtable-prot.h
@@ -55,7 +55,9 @@
55#define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE) 55#define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE)
56#define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT) 56#define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT)
57 57
58#define PAGE_HYP __pgprot(_PAGE_DEFAULT | PTE_HYP) 58#define PAGE_HYP __pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_HYP_XN)
59#define PAGE_HYP_EXEC __pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY)
60#define PAGE_HYP_RO __pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY | PTE_HYP_XN)
59#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP) 61#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP)
60 62
61#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY) 63#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h
index dcbcf8dcbefb..88aa8ec784f6 100644
--- a/arch/arm64/include/asm/virt.h
+++ b/arch/arm64/include/asm/virt.h
@@ -82,6 +82,10 @@ extern void verify_cpu_run_el(void);
82static inline void verify_cpu_run_el(void) {} 82static inline void verify_cpu_run_el(void) {}
83#endif 83#endif
84 84
85/* The section containing the hypervisor idmap text */
86extern char __hyp_idmap_text_start[];
87extern char __hyp_idmap_text_end[];
88
85/* The section containing the hypervisor text */ 89/* The section containing the hypervisor text */
86extern char __hyp_text_start[]; 90extern char __hyp_text_start[];
87extern char __hyp_text_end[]; 91extern char __hyp_text_end[];
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index f209ea151dca..3051f86a9b5f 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -87,9 +87,11 @@ struct kvm_regs {
87/* Supported VGICv3 address types */ 87/* Supported VGICv3 address types */
88#define KVM_VGIC_V3_ADDR_TYPE_DIST 2 88#define KVM_VGIC_V3_ADDR_TYPE_DIST 2
89#define KVM_VGIC_V3_ADDR_TYPE_REDIST 3 89#define KVM_VGIC_V3_ADDR_TYPE_REDIST 3
90#define KVM_VGIC_ITS_ADDR_TYPE 4
90 91
91#define KVM_VGIC_V3_DIST_SIZE SZ_64K 92#define KVM_VGIC_V3_DIST_SIZE SZ_64K
92#define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) 93#define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K)
94#define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K)
93 95
94#define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ 96#define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */
95#define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ 97#define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 811773d1c1d0..ffb3e14dda60 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -726,6 +726,19 @@ static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused
726 return is_kernel_in_hyp_mode(); 726 return is_kernel_in_hyp_mode();
727} 727}
728 728
729static bool hyp_offset_low(const struct arm64_cpu_capabilities *entry,
730 int __unused)
731{
732 phys_addr_t idmap_addr = virt_to_phys(__hyp_idmap_text_start);
733
734 /*
735 * Activate the lower HYP offset only if:
736 * - the idmap doesn't clash with it,
737 * - the kernel is not running at EL2.
738 */
739 return idmap_addr > GENMASK(VA_BITS - 2, 0) && !is_kernel_in_hyp_mode();
740}
741
729static const struct arm64_cpu_capabilities arm64_features[] = { 742static const struct arm64_cpu_capabilities arm64_features[] = {
730 { 743 {
731 .desc = "GIC system register CPU interface", 744 .desc = "GIC system register CPU interface",
@@ -803,6 +816,12 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
803 .field_pos = ID_AA64PFR0_EL0_SHIFT, 816 .field_pos = ID_AA64PFR0_EL0_SHIFT,
804 .min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT, 817 .min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT,
805 }, 818 },
819 {
820 .desc = "Reduced HYP mapping offset",
821 .capability = ARM64_HYP_OFFSET_LOW,
822 .def_scope = SCOPE_SYSTEM,
823 .matches = hyp_offset_low,
824 },
806 {}, 825 {},
807}; 826};
808 827
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index c4f26ef91e77..9d2eff0b3ad3 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -36,6 +36,7 @@ config KVM
36 select HAVE_KVM_IRQFD 36 select HAVE_KVM_IRQFD
37 select KVM_ARM_VGIC_V3 37 select KVM_ARM_VGIC_V3
38 select KVM_ARM_PMU if HW_PERF_EVENTS 38 select KVM_ARM_PMU if HW_PERF_EVENTS
39 select HAVE_KVM_MSI
39 ---help--- 40 ---help---
40 Support hosting virtualized guest machines. 41 Support hosting virtualized guest machines.
41 We don't support KVM with 16K page tables yet, due to the multiple 42 We don't support KVM with 16K page tables yet, due to the multiple
@@ -54,13 +55,6 @@ config KVM_ARM_PMU
54 Adds support for a virtual Performance Monitoring Unit (PMU) in 55 Adds support for a virtual Performance Monitoring Unit (PMU) in
55 virtual machines. 56 virtual machines.
56 57
57config KVM_NEW_VGIC
58 bool "New VGIC implementation"
59 depends on KVM
60 default y
61 ---help---
62 uses the new VGIC implementation
63
64source drivers/vhost/Kconfig 58source drivers/vhost/Kconfig
65 59
66endif # VIRTUALIZATION 60endif # VIRTUALIZATION
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index a7a958ca29d5..a5b96642a9cb 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -20,7 +20,6 @@ kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o
20kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o 20kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
21kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o 21kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o
22 22
23ifeq ($(CONFIG_KVM_NEW_VGIC),y)
24kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic.o 23kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic.o
25kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-init.o 24kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-init.o
26kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-irqfd.o 25kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-irqfd.o
@@ -30,12 +29,6 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-mmio.o
30kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-mmio-v2.o 29kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-mmio-v2.o
31kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-mmio-v3.o 30kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-mmio-v3.o
32kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-kvm-device.o 31kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-kvm-device.o
33else 32kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-its.o
34kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o
35kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o
36kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2-emul.o
37kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
38kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
39endif
40kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o 33kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
41kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o 34kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o
diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S
index a873a6d8be90..6b29d3d9e1f2 100644
--- a/arch/arm64/kvm/hyp-init.S
+++ b/arch/arm64/kvm/hyp-init.S
@@ -53,10 +53,9 @@ __invalid:
53 b . 53 b .
54 54
55 /* 55 /*
56 * x0: HYP boot pgd 56 * x0: HYP pgd
57 * x1: HYP pgd 57 * x1: HYP stack
58 * x2: HYP stack 58 * x2: HYP vectors
59 * x3: HYP vectors
60 */ 59 */
61__do_hyp_init: 60__do_hyp_init:
62 61
@@ -110,71 +109,27 @@ __do_hyp_init:
110 msr sctlr_el2, x4 109 msr sctlr_el2, x4
111 isb 110 isb
112 111
113 /* Skip the trampoline dance if we merged the boot and runtime PGDs */
114 cmp x0, x1
115 b.eq merged
116
117 /* MMU is now enabled. Get ready for the trampoline dance */
118 ldr x4, =TRAMPOLINE_VA
119 adr x5, target
120 bfi x4, x5, #0, #PAGE_SHIFT
121 br x4
122
123target: /* We're now in the trampoline code, switch page tables */
124 msr ttbr0_el2, x1
125 isb
126
127 /* Invalidate the old TLBs */
128 tlbi alle2
129 dsb sy
130
131merged:
132 /* Set the stack and new vectors */ 112 /* Set the stack and new vectors */
113 kern_hyp_va x1
114 mov sp, x1
133 kern_hyp_va x2 115 kern_hyp_va x2
134 mov sp, x2 116 msr vbar_el2, x2
135 kern_hyp_va x3
136 msr vbar_el2, x3
137 117
138 /* Hello, World! */ 118 /* Hello, World! */
139 eret 119 eret
140ENDPROC(__kvm_hyp_init) 120ENDPROC(__kvm_hyp_init)
141 121
142 /* 122 /*
143 * Reset kvm back to the hyp stub. This is the trampoline dance in 123 * Reset kvm back to the hyp stub.
144 * reverse. If kvm used an extended idmap, __extended_idmap_trampoline
145 * calls this code directly in the idmap. In this case switching to the
146 * boot tables is a no-op.
147 *
148 * x0: HYP boot pgd
149 * x1: HYP phys_idmap_start
150 */ 124 */
151ENTRY(__kvm_hyp_reset) 125ENTRY(__kvm_hyp_reset)
152 /* We're in trampoline code in VA, switch back to boot page tables */
153 msr ttbr0_el2, x0
154 isb
155
156 /* Ensure the PA branch doesn't find a stale tlb entry or stale code. */
157 ic iallu
158 tlbi alle2
159 dsb sy
160 isb
161
162 /* Branch into PA space */
163 adr x0, 1f
164 bfi x1, x0, #0, #PAGE_SHIFT
165 br x1
166
167 /* We're now in idmap, disable MMU */ 126 /* We're now in idmap, disable MMU */
1681: mrs x0, sctlr_el2 127 mrs x0, sctlr_el2
169 ldr x1, =SCTLR_ELx_FLAGS 128 ldr x1, =SCTLR_ELx_FLAGS
170 bic x0, x0, x1 // Clear SCTL_M and etc 129 bic x0, x0, x1 // Clear SCTL_M and etc
171 msr sctlr_el2, x0 130 msr sctlr_el2, x0
172 isb 131 isb
173 132
174 /* Invalidate the old TLBs */
175 tlbi alle2
176 dsb sy
177
178 /* Install stub vectors */ 133 /* Install stub vectors */
179 adr_l x0, __hyp_stub_vectors 134 adr_l x0, __hyp_stub_vectors
180 msr vbar_el2, x0 135 msr vbar_el2, x0
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index 70254a65bd5b..ce9e5e5f28cf 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -164,22 +164,3 @@ alternative_endif
164 164
165 eret 165 eret
166ENDPROC(__fpsimd_guest_restore) 166ENDPROC(__fpsimd_guest_restore)
167
168/*
169 * When using the extended idmap, we don't have a trampoline page we can use
170 * while we switch pages tables during __kvm_hyp_reset. Accessing the idmap
171 * directly would be ideal, but if we're using the extended idmap then the
172 * idmap is located above HYP_PAGE_OFFSET, and the address will be masked by
173 * kvm_call_hyp using kern_hyp_va.
174 *
175 * x0: HYP boot pgd
176 * x1: HYP phys_idmap_start
177 */
178ENTRY(__extended_idmap_trampoline)
179 mov x4, x1
180 adr_l x3, __kvm_hyp_reset
181
182 /* insert __kvm_hyp_reset()s offset into phys_idmap_start */
183 bfi x4, x3, #0, #PAGE_SHIFT
184 br x4
185ENDPROC(__extended_idmap_trampoline)
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 2d87f36d5cb4..f6d9694ae3b1 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -62,6 +62,21 @@ ENTRY(__vhe_hyp_call)
62 isb 62 isb
63 ret 63 ret
64ENDPROC(__vhe_hyp_call) 64ENDPROC(__vhe_hyp_call)
65
66/*
67 * Compute the idmap address of __kvm_hyp_reset based on the idmap
68 * start passed as a parameter, and jump there.
69 *
70 * x0: HYP phys_idmap_start
71 */
72ENTRY(__kvm_hyp_teardown)
73 mov x4, x0
74 adr_l x3, __kvm_hyp_reset
75
76 /* insert __kvm_hyp_reset()s offset into phys_idmap_start */
77 bfi x4, x3, #0, #PAGE_SHIFT
78 br x4
79ENDPROC(__kvm_hyp_teardown)
65 80
66el1_sync: // Guest trapped into EL2 81el1_sync: // Guest trapped into EL2
67 save_x0_to_x3 82 save_x0_to_x3
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 437cfad5e3d8..81f21a2ab968 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -299,9 +299,16 @@ static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%
299 299
300static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par) 300static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par)
301{ 301{
302 unsigned long str_va = (unsigned long)__hyp_panic_string; 302 unsigned long str_va;
303 303
304 __hyp_do_panic(hyp_kern_va(str_va), 304 /*
305 * Force the panic string to be loaded from the literal pool,
306 * making sure it is a kernel address and not a PC-relative
307 * reference.
308 */
309 asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va));
310
311 __hyp_do_panic(str_va,
305 spsr, elr, 312 spsr, elr,
306 read_sysreg(esr_el2), read_sysreg_el2(far), 313 read_sysreg(esr_el2), read_sysreg_el2(far),
307 read_sysreg(hpfar_el2), par, 314 read_sysreg(hpfar_el2), par,
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index 7be24f2b18db..5bc460884639 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -65,7 +65,7 @@ static bool cpu_has_32bit_el1(void)
65 * We currently assume that the number of HW registers is uniform 65 * We currently assume that the number of HW registers is uniform
66 * across all CPUs (see cpuinfo_sanity_check). 66 * across all CPUs (see cpuinfo_sanity_check).
67 */ 67 */
68int kvm_arch_dev_ioctl_check_extension(long ext) 68int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext)
69{ 69{
70 int r; 70 int r;
71 71
@@ -86,6 +86,12 @@ int kvm_arch_dev_ioctl_check_extension(long ext)
86 case KVM_CAP_VCPU_ATTRIBUTES: 86 case KVM_CAP_VCPU_ATTRIBUTES:
87 r = 1; 87 r = 1;
88 break; 88 break;
89 case KVM_CAP_MSI_DEVID:
90 if (!kvm)
91 r = -EINVAL;
92 else
93 r = kvm->arch.vgic.msis_require_devid;
94 break;
89 default: 95 default:
90 r = 0; 96 r = 0;
91 } 97 }
@@ -132,31 +138,3 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
132 /* Reset timer */ 138 /* Reset timer */
133 return kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq); 139 return kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
134} 140}
135
136extern char __hyp_idmap_text_start[];
137
138unsigned long kvm_hyp_reset_entry(void)
139{
140 if (!__kvm_cpu_uses_extended_idmap()) {
141 unsigned long offset;
142
143 /*
144 * Find the address of __kvm_hyp_reset() in the trampoline page.
145 * This is present in the running page tables, and the boot page
146 * tables, so we call the code here to start the trampoline
147 * dance in reverse.
148 */
149 offset = (unsigned long)__kvm_hyp_reset
150 - ((unsigned long)__hyp_idmap_text_start & PAGE_MASK);
151
152 return TRAMPOLINE_VA + offset;
153 } else {
154 /*
155 * KVM is running with merged page tables, which don't have the
156 * trampoline page mapped. We know the idmap is still mapped,
157 * but can't be called into directly. Use
158 * __extended_idmap_trampoline to do the call.
159 */
160 return (unsigned long)kvm_ksym_ref(__extended_idmap_trampoline);
161 }
162}
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index a57d650f552c..b0b225ceca18 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1546,7 +1546,7 @@ static void unhandled_cp_access(struct kvm_vcpu *vcpu,
1546 struct sys_reg_params *params) 1546 struct sys_reg_params *params)
1547{ 1547{
1548 u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu); 1548 u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
1549 int cp; 1549 int cp = -1;
1550 1550
1551 switch(hsr_ec) { 1551 switch(hsr_ec) {
1552 case ESR_ELx_EC_CP15_32: 1552 case ESR_ELx_EC_CP15_32:
@@ -1558,7 +1558,7 @@ static void unhandled_cp_access(struct kvm_vcpu *vcpu,
1558 cp = 14; 1558 cp = 14;
1559 break; 1559 break;
1560 default: 1560 default:
1561 WARN_ON((cp = -1)); 1561 WARN_ON(1);
1562 } 1562 }
1563 1563
1564 kvm_err("Unsupported guest CP%d access at: %08lx\n", 1564 kvm_err("Unsupported guest CP%d access at: %08lx\n",
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index da0a524802cb..540da5149ba7 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -1,6 +1,5 @@
1/* 1/*
2 * Copyright (C) 2012 ARM Ltd. 2 * Copyright (C) 2015, 2016 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 * 3 *
5 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -12,16 +11,10 @@
12 * GNU General Public License for more details. 11 * GNU General Public License for more details.
13 * 12 *
14 * You should have received a copy of the GNU General Public License 13 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */ 15 */
18 16#ifndef __KVM_ARM_VGIC_H
19#ifndef __ASM_ARM_KVM_VGIC_H 17#define __KVM_ARM_VGIC_H
20#define __ASM_ARM_KVM_VGIC_H
21
22#ifdef CONFIG_KVM_NEW_VGIC
23#include <kvm/vgic/vgic.h>
24#else
25 18
26#include <linux/kernel.h> 19#include <linux/kernel.h>
27#include <linux/kvm.h> 20#include <linux/kvm.h>
@@ -29,248 +22,187 @@
29#include <linux/spinlock.h> 22#include <linux/spinlock.h>
30#include <linux/types.h> 23#include <linux/types.h>
31#include <kvm/iodev.h> 24#include <kvm/iodev.h>
32#include <linux/irqchip/arm-gic-common.h> 25#include <linux/list.h>
33 26
34#define VGIC_NR_IRQS_LEGACY 256 27#define VGIC_V3_MAX_CPUS 255
28#define VGIC_V2_MAX_CPUS 8
29#define VGIC_NR_IRQS_LEGACY 256
35#define VGIC_NR_SGIS 16 30#define VGIC_NR_SGIS 16
36#define VGIC_NR_PPIS 16 31#define VGIC_NR_PPIS 16
37#define VGIC_NR_PRIVATE_IRQS (VGIC_NR_SGIS + VGIC_NR_PPIS) 32#define VGIC_NR_PRIVATE_IRQS (VGIC_NR_SGIS + VGIC_NR_PPIS)
33#define VGIC_MAX_PRIVATE (VGIC_NR_PRIVATE_IRQS - 1)
34#define VGIC_MAX_SPI 1019
35#define VGIC_MAX_RESERVED 1023
36#define VGIC_MIN_LPI 8192
38 37
39#define VGIC_V2_MAX_LRS (1 << 6) 38enum vgic_type {
40#define VGIC_V3_MAX_LRS 16 39 VGIC_V2, /* Good ol' GICv2 */
41#define VGIC_MAX_IRQS 1024 40 VGIC_V3, /* New fancy GICv3 */
42#define VGIC_V2_MAX_CPUS 8 41};
43#define VGIC_V3_MAX_CPUS 255
44 42
45#if (VGIC_NR_IRQS_LEGACY & 31) 43/* same for all guests, as depending only on the _host's_ GIC model */
46#error "VGIC_NR_IRQS must be a multiple of 32" 44struct vgic_global {
47#endif 45 /* type of the host GIC */
46 enum vgic_type type;
48 47
49#if (VGIC_NR_IRQS_LEGACY > VGIC_MAX_IRQS) 48 /* Physical address of vgic virtual cpu interface */
50#error "VGIC_NR_IRQS must be <= 1024" 49 phys_addr_t vcpu_base;
51#endif
52 50
53/* 51 /* virtual control interface mapping */
54 * The GIC distributor registers describing interrupts have two parts: 52 void __iomem *vctrl_base;
55 * - 32 per-CPU interrupts (SGI + PPI)
56 * - a bunch of shared interrupts (SPI)
57 */
58struct vgic_bitmap {
59 /*
60 * - One UL per VCPU for private interrupts (assumes UL is at
61 * least 32 bits)
62 * - As many UL as necessary for shared interrupts.
63 *
64 * The private interrupts are accessed via the "private"
65 * field, one UL per vcpu (the state for vcpu n is in
66 * private[n]). The shared interrupts are accessed via the
67 * "shared" pointer (IRQn state is at bit n-32 in the bitmap).
68 */
69 unsigned long *private;
70 unsigned long *shared;
71};
72 53
73struct vgic_bytemap { 54 /* Number of implemented list registers */
74 /* 55 int nr_lr;
75 * - 8 u32 per VCPU for private interrupts
76 * - As many u32 as necessary for shared interrupts.
77 *
78 * The private interrupts are accessed via the "private"
79 * field, (the state for vcpu n is in private[n*8] to
80 * private[n*8 + 7]). The shared interrupts are accessed via
81 * the "shared" pointer (IRQn state is at byte (n-32)%4 of the
82 * shared[(n-32)/4] word).
83 */
84 u32 *private;
85 u32 *shared;
86};
87 56
88struct kvm_vcpu; 57 /* Maintenance IRQ number */
58 unsigned int maint_irq;
89 59
90enum vgic_type { 60 /* maximum number of VCPUs allowed (GICv2 limits us to 8) */
91 VGIC_V2, /* Good ol' GICv2 */ 61 int max_gic_vcpus;
92 VGIC_V3, /* New fancy GICv3 */ 62
63 /* Only needed for the legacy KVM_CREATE_IRQCHIP */
64 bool can_emulate_gicv2;
93}; 65};
94 66
95#define LR_STATE_PENDING (1 << 0) 67extern struct vgic_global kvm_vgic_global_state;
96#define LR_STATE_ACTIVE (1 << 1)
97#define LR_STATE_MASK (3 << 0)
98#define LR_EOI_INT (1 << 2)
99#define LR_HW (1 << 3)
100 68
101struct vgic_lr { 69#define VGIC_V2_MAX_LRS (1 << 6)
102 unsigned irq:10; 70#define VGIC_V3_MAX_LRS 16
103 union { 71#define VGIC_V3_LR_INDEX(lr) (VGIC_V3_MAX_LRS - 1 - lr)
104 unsigned hwirq:10;
105 unsigned source:3;
106 };
107 unsigned state:4;
108};
109 72
110struct vgic_vmcr { 73enum vgic_irq_config {
111 u32 ctlr; 74 VGIC_CONFIG_EDGE = 0,
112 u32 abpr; 75 VGIC_CONFIG_LEVEL
113 u32 bpr;
114 u32 pmr;
115}; 76};
116 77
117struct vgic_ops { 78struct vgic_irq {
118 struct vgic_lr (*get_lr)(const struct kvm_vcpu *, int); 79 spinlock_t irq_lock; /* Protects the content of the struct */
119 void (*set_lr)(struct kvm_vcpu *, int, struct vgic_lr); 80 struct list_head lpi_list; /* Used to link all LPIs together */
120 u64 (*get_elrsr)(const struct kvm_vcpu *vcpu); 81 struct list_head ap_list;
121 u64 (*get_eisr)(const struct kvm_vcpu *vcpu); 82
122 void (*clear_eisr)(struct kvm_vcpu *vcpu); 83 struct kvm_vcpu *vcpu; /* SGIs and PPIs: The VCPU
123 u32 (*get_interrupt_status)(const struct kvm_vcpu *vcpu); 84 * SPIs and LPIs: The VCPU whose ap_list
124 void (*enable_underflow)(struct kvm_vcpu *vcpu); 85 * this is queued on.
125 void (*disable_underflow)(struct kvm_vcpu *vcpu); 86 */
126 void (*get_vmcr)(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); 87
127 void (*set_vmcr)(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); 88 struct kvm_vcpu *target_vcpu; /* The VCPU that this interrupt should
128 void (*enable)(struct kvm_vcpu *vcpu); 89 * be sent to, as a result of the
90 * targets reg (v2) or the
91 * affinity reg (v3).
92 */
93
94 u32 intid; /* Guest visible INTID */
95 bool pending;
96 bool line_level; /* Level only */
97 bool soft_pending; /* Level only */
98 bool active; /* not used for LPIs */
99 bool enabled;
100 bool hw; /* Tied to HW IRQ */
101 struct kref refcount; /* Used for LPIs */
102 u32 hwintid; /* HW INTID number */
103 union {
104 u8 targets; /* GICv2 target VCPUs mask */
105 u32 mpidr; /* GICv3 target VCPU */
106 };
107 u8 source; /* GICv2 SGIs only */
108 u8 priority;
109 enum vgic_irq_config config; /* Level or edge */
129}; 110};
130 111
131struct vgic_params { 112struct vgic_register_region;
132 /* vgic type */ 113struct vgic_its;
133 enum vgic_type type;
134 /* Physical address of vgic virtual cpu interface */
135 phys_addr_t vcpu_base;
136 /* Number of list registers */
137 u32 nr_lr;
138 /* Interrupt number */
139 unsigned int maint_irq;
140 /* Virtual control interface base address */
141 void __iomem *vctrl_base;
142 int max_gic_vcpus;
143 /* Only needed for the legacy KVM_CREATE_IRQCHIP */
144 bool can_emulate_gicv2;
145};
146 114
147struct vgic_vm_ops { 115enum iodev_type {
148 bool (*queue_sgi)(struct kvm_vcpu *, int irq); 116 IODEV_CPUIF,
149 void (*add_sgi_source)(struct kvm_vcpu *, int irq, int source); 117 IODEV_DIST,
150 int (*init_model)(struct kvm *); 118 IODEV_REDIST,
151 int (*map_resources)(struct kvm *, const struct vgic_params *); 119 IODEV_ITS
152}; 120};
153 121
154struct vgic_io_device { 122struct vgic_io_device {
155 gpa_t addr; 123 gpa_t base_addr;
156 int len; 124 union {
157 const struct vgic_io_range *reg_ranges; 125 struct kvm_vcpu *redist_vcpu;
158 struct kvm_vcpu *redist_vcpu; 126 struct vgic_its *its;
127 };
128 const struct vgic_register_region *regions;
129 enum iodev_type iodev_type;
130 int nr_regions;
159 struct kvm_io_device dev; 131 struct kvm_io_device dev;
160}; 132};
161 133
162struct irq_phys_map { 134struct vgic_its {
163 u32 virt_irq; 135 /* The base address of the ITS control register frame */
164 u32 phys_irq; 136 gpa_t vgic_its_base;
165}; 137
166 138 bool enabled;
167struct irq_phys_map_entry { 139 bool initialized;
168 struct list_head entry; 140 struct vgic_io_device iodev;
169 struct rcu_head rcu; 141 struct kvm_device *dev;
170 struct irq_phys_map map; 142
143 /* These registers correspond to GITS_BASER{0,1} */
144 u64 baser_device_table;
145 u64 baser_coll_table;
146
147 /* Protects the command queue */
148 struct mutex cmd_lock;
149 u64 cbaser;
150 u32 creadr;
151 u32 cwriter;
152
153 /* Protects the device and collection lists */
154 struct mutex its_lock;
155 struct list_head device_list;
156 struct list_head collection_list;
171}; 157};
172 158
173struct vgic_dist { 159struct vgic_dist {
174 spinlock_t lock;
175 bool in_kernel; 160 bool in_kernel;
176 bool ready; 161 bool ready;
162 bool initialized;
177 163
178 /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */ 164 /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */
179 u32 vgic_model; 165 u32 vgic_model;
180 166
181 int nr_cpus; 167 /* Do injected MSIs require an additional device ID? */
182 int nr_irqs; 168 bool msis_require_devid;
169
170 int nr_spis;
183 171
172 /* TODO: Consider moving to global state */
184 /* Virtual control interface mapping */ 173 /* Virtual control interface mapping */
185 void __iomem *vctrl_base; 174 void __iomem *vctrl_base;
186 175
187 /* Distributor and vcpu interface mapping in the guest */ 176 /* base addresses in guest physical address space: */
188 phys_addr_t vgic_dist_base; 177 gpa_t vgic_dist_base; /* distributor */
189 /* GICv2 and GICv3 use different mapped register blocks */
190 union { 178 union {
191 phys_addr_t vgic_cpu_base; 179 /* either a GICv2 CPU interface */
192 phys_addr_t vgic_redist_base; 180 gpa_t vgic_cpu_base;
181 /* or a number of GICv3 redistributor regions */
182 gpa_t vgic_redist_base;
193 }; 183 };
194 184
195 /* Distributor enabled */ 185 /* distributor enabled */
196 u32 enabled; 186 bool enabled;
197
198 /* Interrupt enabled (one bit per IRQ) */
199 struct vgic_bitmap irq_enabled;
200
201 /* Level-triggered interrupt external input is asserted */
202 struct vgic_bitmap irq_level;
203
204 /*
205 * Interrupt state is pending on the distributor
206 */
207 struct vgic_bitmap irq_pending;
208
209 /*
210 * Tracks writes to GICD_ISPENDRn and GICD_ICPENDRn for level-triggered
211 * interrupts. Essentially holds the state of the flip-flop in
212 * Figure 4-10 on page 4-101 in ARM IHI 0048B.b.
213 * Once set, it is only cleared for level-triggered interrupts on
214 * guest ACKs (when we queue it) or writes to GICD_ICPENDRn.
215 */
216 struct vgic_bitmap irq_soft_pend;
217
218 /* Level-triggered interrupt queued on VCPU interface */
219 struct vgic_bitmap irq_queued;
220
221 /* Interrupt was active when unqueue from VCPU interface */
222 struct vgic_bitmap irq_active;
223
224 /* Interrupt priority. Not used yet. */
225 struct vgic_bytemap irq_priority;
226 187
227 /* Level/edge triggered */ 188 struct vgic_irq *spis;
228 struct vgic_bitmap irq_cfg;
229 189
230 /* 190 struct vgic_io_device dist_iodev;
231 * Source CPU per SGI and target CPU:
232 *
233 * Each byte represent a SGI observable on a VCPU, each bit of
234 * this byte indicating if the corresponding VCPU has
235 * generated this interrupt. This is a GICv2 feature only.
236 *
237 * For VCPUn (n < 8), irq_sgi_sources[n*16] to [n*16 + 15] are
238 * the SGIs observable on VCPUn.
239 */
240 u8 *irq_sgi_sources;
241 191
242 /* 192 bool has_its;
243 * Target CPU for each SPI:
244 *
245 * Array of available SPI, each byte indicating the target
246 * VCPU for SPI. IRQn (n >=32) is at irq_spi_cpu[n-32].
247 */
248 u8 *irq_spi_cpu;
249 193
250 /* 194 /*
251 * Reverse lookup of irq_spi_cpu for faster compute pending: 195 * Contains the attributes and gpa of the LPI configuration table.
252 * 196 * Since we report GICR_TYPER.CommonLPIAff as 0b00, we can share
253 * Array of bitmaps, one per VCPU, describing if IRQn is 197 * one address across all redistributors.
254 * routed to a particular VCPU. 198 * GICv3 spec: 6.1.2 "LPI Configuration tables"
255 */ 199 */
256 struct vgic_bitmap *irq_spi_target; 200 u64 propbaser;
257
258 /* Target MPIDR for each IRQ (needed for GICv3 IROUTERn) only */
259 u32 *irq_spi_mpidr;
260 201
261 /* Bitmap indicating which CPU has something pending */ 202 /* Protects the lpi_list and the count value below. */
262 unsigned long *irq_pending_on_cpu; 203 spinlock_t lpi_list_lock;
263 204 struct list_head lpi_list_head;
264 /* Bitmap indicating which CPU has active IRQs */ 205 int lpi_list_count;
265 unsigned long *irq_active_on_cpu;
266
267 struct vgic_vm_ops vm_ops;
268 struct vgic_io_device dist_iodev;
269 struct vgic_io_device *redist_iodevs;
270
271 /* Virtual irq to hwirq mapping */
272 spinlock_t irq_phys_map_lock;
273 struct list_head irq_phys_map_list;
274}; 206};
275 207
276struct vgic_v2_cpu_if { 208struct vgic_v2_cpu_if {
@@ -298,78 +230,88 @@ struct vgic_v3_cpu_if {
298}; 230};
299 231
300struct vgic_cpu { 232struct vgic_cpu {
301 /* Pending/active/both interrupts on this VCPU */
302 DECLARE_BITMAP(pending_percpu, VGIC_NR_PRIVATE_IRQS);
303 DECLARE_BITMAP(active_percpu, VGIC_NR_PRIVATE_IRQS);
304 DECLARE_BITMAP(pend_act_percpu, VGIC_NR_PRIVATE_IRQS);
305
306 /* Pending/active/both shared interrupts, dynamically sized */
307 unsigned long *pending_shared;
308 unsigned long *active_shared;
309 unsigned long *pend_act_shared;
310
311 /* CPU vif control registers for world switch */ 233 /* CPU vif control registers for world switch */
312 union { 234 union {
313 struct vgic_v2_cpu_if vgic_v2; 235 struct vgic_v2_cpu_if vgic_v2;
314 struct vgic_v3_cpu_if vgic_v3; 236 struct vgic_v3_cpu_if vgic_v3;
315 }; 237 };
316 238
317 /* Protected by the distributor's irq_phys_map_lock */ 239 unsigned int used_lrs;
318 struct list_head irq_phys_map_list; 240 struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS];
319 241
320 u64 live_lrs; 242 spinlock_t ap_list_lock; /* Protects the ap_list */
321}; 243
244 /*
245 * List of IRQs that this VCPU should consider because they are either
246 * Active or Pending (hence the name; AP list), or because they recently
247 * were one of the two and need to be migrated off this list to another
248 * VCPU.
249 */
250 struct list_head ap_list_head;
322 251
323#define LR_EMPTY 0xff 252 u64 live_lrs;
324 253
325#define INT_STATUS_EOI (1 << 0) 254 /*
326#define INT_STATUS_UNDERFLOW (1 << 1) 255 * Members below are used with GICv3 emulation only and represent
256 * parts of the redistributor.
257 */
258 struct vgic_io_device rd_iodev;
259 struct vgic_io_device sgi_iodev;
327 260
328struct kvm; 261 /* Contains the attributes and gpa of the LPI pending tables. */
329struct kvm_vcpu; 262 u64 pendbaser;
263
264 bool lpis_enabled;
265};
330 266
331int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write); 267int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write);
332int kvm_vgic_hyp_init(void);
333int kvm_vgic_map_resources(struct kvm *kvm);
334int kvm_vgic_get_max_vcpus(void);
335void kvm_vgic_early_init(struct kvm *kvm); 268void kvm_vgic_early_init(struct kvm *kvm);
336int kvm_vgic_create(struct kvm *kvm, u32 type); 269int kvm_vgic_create(struct kvm *kvm, u32 type);
337void kvm_vgic_destroy(struct kvm *kvm); 270void kvm_vgic_destroy(struct kvm *kvm);
338void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu); 271void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu);
339void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu); 272void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
340void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu); 273int kvm_vgic_map_resources(struct kvm *kvm);
341void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu); 274int kvm_vgic_hyp_init(void);
342int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num, 275
276int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid,
343 bool level); 277 bool level);
344int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, 278int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, unsigned int intid,
345 unsigned int virt_irq, bool level); 279 bool level);
346void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg); 280int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, u32 virt_irq, u32 phys_irq);
347int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
348int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, int virt_irq, int phys_irq);
349int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq); 281int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq);
350bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq); 282bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq);
351 283
284int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
285
352#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel)) 286#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel))
353#define vgic_initialized(k) (!!((k)->arch.vgic.nr_cpus)) 287#define vgic_initialized(k) ((k)->arch.vgic.initialized)
354#define vgic_ready(k) ((k)->arch.vgic.ready) 288#define vgic_ready(k) ((k)->arch.vgic.ready)
355#define vgic_valid_spi(k, i) (((i) >= VGIC_NR_PRIVATE_IRQS) && \ 289#define vgic_valid_spi(k, i) (((i) >= VGIC_NR_PRIVATE_IRQS) && \
356 ((i) < (k)->arch.vgic.nr_irqs)) 290 ((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
291
292bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
293void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
294void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
357 295
358int vgic_v2_probe(const struct gic_kvm_info *gic_kvm_info,
359 const struct vgic_ops **ops,
360 const struct vgic_params **params);
361#ifdef CONFIG_KVM_ARM_VGIC_V3 296#ifdef CONFIG_KVM_ARM_VGIC_V3
362int vgic_v3_probe(const struct gic_kvm_info *gic_kvm_info, 297void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg);
363 const struct vgic_ops **ops,
364 const struct vgic_params **params);
365#else 298#else
366static inline int vgic_v3_probe(const struct gic_kvm_info *gic_kvm_info, 299static inline void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg)
367 const struct vgic_ops **ops,
368 const struct vgic_params **params)
369{ 300{
370 return -ENODEV;
371} 301}
372#endif 302#endif
373 303
374#endif /* old VGIC include */ 304/**
375#endif 305 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
306 *
307 * The host's GIC naturally limits the maximum amount of VCPUs a guest
308 * can use.
309 */
310static inline int kvm_vgic_get_max_vcpus(void)
311{
312 return kvm_vgic_global_state.max_gic_vcpus;
313}
314
315int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
316
317#endif /* __KVM_ARM_VGIC_H */
diff --git a/include/kvm/vgic/vgic.h b/include/kvm/vgic/vgic.h
deleted file mode 100644
index 3fbd175265ae..000000000000
--- a/include/kvm/vgic/vgic.h
+++ /dev/null
@@ -1,246 +0,0 @@
1/*
2 * Copyright (C) 2015, 2016 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_ARM_KVM_VGIC_VGIC_H
17#define __ASM_ARM_KVM_VGIC_VGIC_H
18
19#include <linux/kernel.h>
20#include <linux/kvm.h>
21#include <linux/irqreturn.h>
22#include <linux/spinlock.h>
23#include <linux/types.h>
24#include <kvm/iodev.h>
25
26#define VGIC_V3_MAX_CPUS 255
27#define VGIC_V2_MAX_CPUS 8
28#define VGIC_NR_IRQS_LEGACY 256
29#define VGIC_NR_SGIS 16
30#define VGIC_NR_PPIS 16
31#define VGIC_NR_PRIVATE_IRQS (VGIC_NR_SGIS + VGIC_NR_PPIS)
32#define VGIC_MAX_PRIVATE (VGIC_NR_PRIVATE_IRQS - 1)
33#define VGIC_MAX_SPI 1019
34#define VGIC_MAX_RESERVED 1023
35#define VGIC_MIN_LPI 8192
36
37enum vgic_type {
38 VGIC_V2, /* Good ol' GICv2 */
39 VGIC_V3, /* New fancy GICv3 */
40};
41
42/* same for all guests, as depending only on the _host's_ GIC model */
43struct vgic_global {
44 /* type of the host GIC */
45 enum vgic_type type;
46
47 /* Physical address of vgic virtual cpu interface */
48 phys_addr_t vcpu_base;
49
50 /* virtual control interface mapping */
51 void __iomem *vctrl_base;
52
53 /* Number of implemented list registers */
54 int nr_lr;
55
56 /* Maintenance IRQ number */
57 unsigned int maint_irq;
58
59 /* maximum number of VCPUs allowed (GICv2 limits us to 8) */
60 int max_gic_vcpus;
61
62 /* Only needed for the legacy KVM_CREATE_IRQCHIP */
63 bool can_emulate_gicv2;
64};
65
66extern struct vgic_global kvm_vgic_global_state;
67
68#define VGIC_V2_MAX_LRS (1 << 6)
69#define VGIC_V3_MAX_LRS 16
70#define VGIC_V3_LR_INDEX(lr) (VGIC_V3_MAX_LRS - 1 - lr)
71
72enum vgic_irq_config {
73 VGIC_CONFIG_EDGE = 0,
74 VGIC_CONFIG_LEVEL
75};
76
77struct vgic_irq {
78 spinlock_t irq_lock; /* Protects the content of the struct */
79 struct list_head ap_list;
80
81 struct kvm_vcpu *vcpu; /* SGIs and PPIs: The VCPU
82 * SPIs and LPIs: The VCPU whose ap_list
83 * this is queued on.
84 */
85
86 struct kvm_vcpu *target_vcpu; /* The VCPU that this interrupt should
87 * be sent to, as a result of the
88 * targets reg (v2) or the
89 * affinity reg (v3).
90 */
91
92 u32 intid; /* Guest visible INTID */
93 bool pending;
94 bool line_level; /* Level only */
95 bool soft_pending; /* Level only */
96 bool active; /* not used for LPIs */
97 bool enabled;
98 bool hw; /* Tied to HW IRQ */
99 u32 hwintid; /* HW INTID number */
100 union {
101 u8 targets; /* GICv2 target VCPUs mask */
102 u32 mpidr; /* GICv3 target VCPU */
103 };
104 u8 source; /* GICv2 SGIs only */
105 u8 priority;
106 enum vgic_irq_config config; /* Level or edge */
107};
108
109struct vgic_register_region;
110
111struct vgic_io_device {
112 gpa_t base_addr;
113 struct kvm_vcpu *redist_vcpu;
114 const struct vgic_register_region *regions;
115 int nr_regions;
116 struct kvm_io_device dev;
117};
118
119struct vgic_dist {
120 bool in_kernel;
121 bool ready;
122 bool initialized;
123
124 /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */
125 u32 vgic_model;
126
127 int nr_spis;
128
129 /* TODO: Consider moving to global state */
130 /* Virtual control interface mapping */
131 void __iomem *vctrl_base;
132
133 /* base addresses in guest physical address space: */
134 gpa_t vgic_dist_base; /* distributor */
135 union {
136 /* either a GICv2 CPU interface */
137 gpa_t vgic_cpu_base;
138 /* or a number of GICv3 redistributor regions */
139 gpa_t vgic_redist_base;
140 };
141
142 /* distributor enabled */
143 bool enabled;
144
145 struct vgic_irq *spis;
146
147 struct vgic_io_device dist_iodev;
148 struct vgic_io_device *redist_iodevs;
149};
150
151struct vgic_v2_cpu_if {
152 u32 vgic_hcr;
153 u32 vgic_vmcr;
154 u32 vgic_misr; /* Saved only */
155 u64 vgic_eisr; /* Saved only */
156 u64 vgic_elrsr; /* Saved only */
157 u32 vgic_apr;
158 u32 vgic_lr[VGIC_V2_MAX_LRS];
159};
160
161struct vgic_v3_cpu_if {
162#ifdef CONFIG_KVM_ARM_VGIC_V3
163 u32 vgic_hcr;
164 u32 vgic_vmcr;
165 u32 vgic_sre; /* Restored only, change ignored */
166 u32 vgic_misr; /* Saved only */
167 u32 vgic_eisr; /* Saved only */
168 u32 vgic_elrsr; /* Saved only */
169 u32 vgic_ap0r[4];
170 u32 vgic_ap1r[4];
171 u64 vgic_lr[VGIC_V3_MAX_LRS];
172#endif
173};
174
175struct vgic_cpu {
176 /* CPU vif control registers for world switch */
177 union {
178 struct vgic_v2_cpu_if vgic_v2;
179 struct vgic_v3_cpu_if vgic_v3;
180 };
181
182 unsigned int used_lrs;
183 struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS];
184
185 spinlock_t ap_list_lock; /* Protects the ap_list */
186
187 /*
188 * List of IRQs that this VCPU should consider because they are either
189 * Active or Pending (hence the name; AP list), or because they recently
190 * were one of the two and need to be migrated off this list to another
191 * VCPU.
192 */
193 struct list_head ap_list_head;
194
195 u64 live_lrs;
196};
197
198int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write);
199void kvm_vgic_early_init(struct kvm *kvm);
200int kvm_vgic_create(struct kvm *kvm, u32 type);
201void kvm_vgic_destroy(struct kvm *kvm);
202void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu);
203void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
204int kvm_vgic_map_resources(struct kvm *kvm);
205int kvm_vgic_hyp_init(void);
206
207int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid,
208 bool level);
209int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, unsigned int intid,
210 bool level);
211int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, u32 virt_irq, u32 phys_irq);
212int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq);
213bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq);
214
215int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
216
217#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel))
218#define vgic_initialized(k) ((k)->arch.vgic.initialized)
219#define vgic_ready(k) ((k)->arch.vgic.ready)
220#define vgic_valid_spi(k, i) (((i) >= VGIC_NR_PRIVATE_IRQS) && \
221 ((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
222
223bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
224void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
225void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
226
227#ifdef CONFIG_KVM_ARM_VGIC_V3
228void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg);
229#else
230static inline void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg)
231{
232}
233#endif
234
235/**
236 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
237 *
238 * The host's GIC naturally limits the maximum amount of VCPUs a guest
239 * can use.
240 */
241static inline int kvm_vgic_get_max_vcpus(void)
242{
243 return kvm_vgic_global_state.max_gic_vcpus;
244}
245
246#endif /* __ASM_ARM_KVM_VGIC_VGIC_H */
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index bfbd707de390..700b4216c87a 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -112,34 +112,76 @@
112#define GICR_WAKER_ProcessorSleep (1U << 1) 112#define GICR_WAKER_ProcessorSleep (1U << 1)
113#define GICR_WAKER_ChildrenAsleep (1U << 2) 113#define GICR_WAKER_ChildrenAsleep (1U << 2)
114 114
115#define GICR_PROPBASER_NonShareable (0U << 10) 115#define GIC_BASER_CACHE_nCnB 0ULL
116#define GICR_PROPBASER_InnerShareable (1U << 10) 116#define GIC_BASER_CACHE_SameAsInner 0ULL
117#define GICR_PROPBASER_OuterShareable (2U << 10) 117#define GIC_BASER_CACHE_nC 1ULL
118#define GICR_PROPBASER_SHAREABILITY_MASK (3UL << 10) 118#define GIC_BASER_CACHE_RaWt 2ULL
119#define GICR_PROPBASER_nCnB (0U << 7) 119#define GIC_BASER_CACHE_RaWb 3ULL
120#define GICR_PROPBASER_nC (1U << 7) 120#define GIC_BASER_CACHE_WaWt 4ULL
121#define GICR_PROPBASER_RaWt (2U << 7) 121#define GIC_BASER_CACHE_WaWb 5ULL
122#define GICR_PROPBASER_RaWb (3U << 7) 122#define GIC_BASER_CACHE_RaWaWt 6ULL
123#define GICR_PROPBASER_WaWt (4U << 7) 123#define GIC_BASER_CACHE_RaWaWb 7ULL
124#define GICR_PROPBASER_WaWb (5U << 7) 124#define GIC_BASER_CACHE_MASK 7ULL
125#define GICR_PROPBASER_RaWaWt (6U << 7) 125#define GIC_BASER_NonShareable 0ULL
126#define GICR_PROPBASER_RaWaWb (7U << 7) 126#define GIC_BASER_InnerShareable 1ULL
127#define GICR_PROPBASER_CACHEABILITY_MASK (7U << 7) 127#define GIC_BASER_OuterShareable 2ULL
128#define GICR_PROPBASER_IDBITS_MASK (0x1f) 128#define GIC_BASER_SHAREABILITY_MASK 3ULL
129 129
130#define GICR_PENDBASER_NonShareable (0U << 10) 130#define GIC_BASER_CACHEABILITY(reg, inner_outer, type) \
131#define GICR_PENDBASER_InnerShareable (1U << 10) 131 (GIC_BASER_CACHE_##type << reg##_##inner_outer##_CACHEABILITY_SHIFT)
132#define GICR_PENDBASER_OuterShareable (2U << 10) 132
133#define GICR_PENDBASER_SHAREABILITY_MASK (3UL << 10) 133#define GIC_BASER_SHAREABILITY(reg, type) \
134#define GICR_PENDBASER_nCnB (0U << 7) 134 (GIC_BASER_##type << reg##_SHAREABILITY_SHIFT)
135#define GICR_PENDBASER_nC (1U << 7) 135
136#define GICR_PENDBASER_RaWt (2U << 7) 136#define GICR_PROPBASER_SHAREABILITY_SHIFT (10)
137#define GICR_PENDBASER_RaWb (3U << 7) 137#define GICR_PROPBASER_INNER_CACHEABILITY_SHIFT (7)
138#define GICR_PENDBASER_WaWt (4U << 7) 138#define GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT (56)
139#define GICR_PENDBASER_WaWb (5U << 7) 139#define GICR_PROPBASER_SHAREABILITY_MASK \
140#define GICR_PENDBASER_RaWaWt (6U << 7) 140 GIC_BASER_SHAREABILITY(GICR_PROPBASER, SHAREABILITY_MASK)
141#define GICR_PENDBASER_RaWaWb (7U << 7) 141#define GICR_PROPBASER_INNER_CACHEABILITY_MASK \
142#define GICR_PENDBASER_CACHEABILITY_MASK (7U << 7) 142 GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, MASK)
143#define GICR_PROPBASER_OUTER_CACHEABILITY_MASK \
144 GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, MASK)
145#define GICR_PROPBASER_CACHEABILITY_MASK GICR_PROPBASER_INNER_CACHEABILITY_MASK
146
147#define GICR_PROPBASER_InnerShareable \
148 GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable)
149
150#define GICR_PROPBASER_nCnB GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, nCnB)
151#define GICR_PROPBASER_nC GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, nC)
152#define GICR_PROPBASER_RaWt GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWt)
153#define GICR_PROPBASER_RaWb GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWt)
154#define GICR_PROPBASER_WaWt GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, WaWt)
155#define GICR_PROPBASER_WaWb GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, WaWb)
156#define GICR_PROPBASER_RaWaWt GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWaWt)
157#define GICR_PROPBASER_RaWaWb GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWaWb)
158
159#define GICR_PROPBASER_IDBITS_MASK (0x1f)
160
161#define GICR_PENDBASER_SHAREABILITY_SHIFT (10)
162#define GICR_PENDBASER_INNER_CACHEABILITY_SHIFT (7)
163#define GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT (56)
164#define GICR_PENDBASER_SHAREABILITY_MASK \
165 GIC_BASER_SHAREABILITY(GICR_PENDBASER, SHAREABILITY_MASK)
166#define GICR_PENDBASER_INNER_CACHEABILITY_MASK \
167 GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, MASK)
168#define GICR_PENDBASER_OUTER_CACHEABILITY_MASK \
169 GIC_BASER_CACHEABILITY(GICR_PENDBASER, OUTER, MASK)
170#define GICR_PENDBASER_CACHEABILITY_MASK GICR_PENDBASER_INNER_CACHEABILITY_MASK
171
172#define GICR_PENDBASER_InnerShareable \
173 GIC_BASER_SHAREABILITY(GICR_PENDBASER, InnerShareable)
174
175#define GICR_PENDBASER_nCnB GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, nCnB)
176#define GICR_PENDBASER_nC GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, nC)
177#define GICR_PENDBASER_RaWt GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, RaWt)
178#define GICR_PENDBASER_RaWb GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, RaWt)
179#define GICR_PENDBASER_WaWt GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, WaWt)
180#define GICR_PENDBASER_WaWb GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, WaWb)
181#define GICR_PENDBASER_RaWaWt GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, RaWaWt)
182#define GICR_PENDBASER_RaWaWb GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, RaWaWb)
183
184#define GICR_PENDBASER_PTZ BIT_ULL(62)
143 185
144/* 186/*
145 * Re-Distributor registers, offsets from SGI_base 187 * Re-Distributor registers, offsets from SGI_base
@@ -175,59 +217,90 @@
175#define GITS_CWRITER 0x0088 217#define GITS_CWRITER 0x0088
176#define GITS_CREADR 0x0090 218#define GITS_CREADR 0x0090
177#define GITS_BASER 0x0100 219#define GITS_BASER 0x0100
220#define GITS_IDREGS_BASE 0xffd0
221#define GITS_PIDR0 0xffe0
222#define GITS_PIDR1 0xffe4
178#define GITS_PIDR2 GICR_PIDR2 223#define GITS_PIDR2 GICR_PIDR2
224#define GITS_PIDR4 0xffd0
225#define GITS_CIDR0 0xfff0
226#define GITS_CIDR1 0xfff4
227#define GITS_CIDR2 0xfff8
228#define GITS_CIDR3 0xfffc
179 229
180#define GITS_TRANSLATER 0x10040 230#define GITS_TRANSLATER 0x10040
181 231
182#define GITS_CTLR_ENABLE (1U << 0) 232#define GITS_CTLR_ENABLE (1U << 0)
183#define GITS_CTLR_QUIESCENT (1U << 31) 233#define GITS_CTLR_QUIESCENT (1U << 31)
184 234
235#define GITS_TYPER_PLPIS (1UL << 0)
236#define GITS_TYPER_IDBITS_SHIFT 8
185#define GITS_TYPER_DEVBITS_SHIFT 13 237#define GITS_TYPER_DEVBITS_SHIFT 13
186#define GITS_TYPER_DEVBITS(r) ((((r) >> GITS_TYPER_DEVBITS_SHIFT) & 0x1f) + 1) 238#define GITS_TYPER_DEVBITS(r) ((((r) >> GITS_TYPER_DEVBITS_SHIFT) & 0x1f) + 1)
187#define GITS_TYPER_PTA (1UL << 19) 239#define GITS_TYPER_PTA (1UL << 19)
188 240#define GITS_TYPER_HWCOLLCNT_SHIFT 24
189#define GITS_CBASER_VALID (1UL << 63) 241
190#define GITS_CBASER_nCnB (0UL << 59) 242#define GITS_CBASER_VALID (1UL << 63)
191#define GITS_CBASER_nC (1UL << 59) 243#define GITS_CBASER_SHAREABILITY_SHIFT (10)
192#define GITS_CBASER_RaWt (2UL << 59) 244#define GITS_CBASER_INNER_CACHEABILITY_SHIFT (59)
193#define GITS_CBASER_RaWb (3UL << 59) 245#define GITS_CBASER_OUTER_CACHEABILITY_SHIFT (53)
194#define GITS_CBASER_WaWt (4UL << 59) 246#define GITS_CBASER_SHAREABILITY_MASK \
195#define GITS_CBASER_WaWb (5UL << 59) 247 GIC_BASER_SHAREABILITY(GITS_CBASER, SHAREABILITY_MASK)
196#define GITS_CBASER_RaWaWt (6UL << 59) 248#define GITS_CBASER_INNER_CACHEABILITY_MASK \
197#define GITS_CBASER_RaWaWb (7UL << 59) 249 GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, MASK)
198#define GITS_CBASER_CACHEABILITY_MASK (7UL << 59) 250#define GITS_CBASER_OUTER_CACHEABILITY_MASK \
199#define GITS_CBASER_NonShareable (0UL << 10) 251 GIC_BASER_CACHEABILITY(GITS_CBASER, OUTER, MASK)
200#define GITS_CBASER_InnerShareable (1UL << 10) 252#define GITS_CBASER_CACHEABILITY_MASK GITS_CBASER_INNER_CACHEABILITY_MASK
201#define GITS_CBASER_OuterShareable (2UL << 10) 253
202#define GITS_CBASER_SHAREABILITY_MASK (3UL << 10) 254#define GITS_CBASER_InnerShareable \
255 GIC_BASER_SHAREABILITY(GITS_CBASER, InnerShareable)
256
257#define GITS_CBASER_nCnB GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, nCnB)
258#define GITS_CBASER_nC GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, nC)
259#define GITS_CBASER_RaWt GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, RaWt)
260#define GITS_CBASER_RaWb GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, RaWt)
261#define GITS_CBASER_WaWt GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, WaWt)
262#define GITS_CBASER_WaWb GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, WaWb)
263#define GITS_CBASER_RaWaWt GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, RaWaWt)
264#define GITS_CBASER_RaWaWb GIC_BASER_CACHEABILITY(GITS_CBASER, INNER, RaWaWb)
203 265
204#define GITS_BASER_NR_REGS 8 266#define GITS_BASER_NR_REGS 8
205 267
206#define GITS_BASER_VALID (1UL << 63) 268#define GITS_BASER_VALID (1UL << 63)
207#define GITS_BASER_nCnB (0UL << 59) 269#define GITS_BASER_INDIRECT (1ULL << 62)
208#define GITS_BASER_nC (1UL << 59) 270
209#define GITS_BASER_RaWt (2UL << 59) 271#define GITS_BASER_INNER_CACHEABILITY_SHIFT (59)
210#define GITS_BASER_RaWb (3UL << 59) 272#define GITS_BASER_OUTER_CACHEABILITY_SHIFT (53)
211#define GITS_BASER_WaWt (4UL << 59) 273#define GITS_BASER_INNER_CACHEABILITY_MASK \
212#define GITS_BASER_WaWb (5UL << 59) 274 GIC_BASER_CACHEABILITY(GITS_BASER, INNER, MASK)
213#define GITS_BASER_RaWaWt (6UL << 59) 275#define GITS_BASER_CACHEABILITY_MASK GITS_BASER_INNER_CACHEABILITY_MASK
214#define GITS_BASER_RaWaWb (7UL << 59) 276#define GITS_BASER_OUTER_CACHEABILITY_MASK \
215#define GITS_BASER_CACHEABILITY_MASK (7UL << 59) 277 GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, MASK)
216#define GITS_BASER_TYPE_SHIFT (56) 278#define GITS_BASER_SHAREABILITY_MASK \
279 GIC_BASER_SHAREABILITY(GITS_BASER, SHAREABILITY_MASK)
280
281#define GITS_BASER_nCnB GIC_BASER_CACHEABILITY(GITS_BASER, INNER, nCnB)
282#define GITS_BASER_nC GIC_BASER_CACHEABILITY(GITS_BASER, INNER, nC)
283#define GITS_BASER_RaWt GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWt)
284#define GITS_BASER_RaWb GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWt)
285#define GITS_BASER_WaWt GIC_BASER_CACHEABILITY(GITS_BASER, INNER, WaWt)
286#define GITS_BASER_WaWb GIC_BASER_CACHEABILITY(GITS_BASER, INNER, WaWb)
287#define GITS_BASER_RaWaWt GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWaWt)
288#define GITS_BASER_RaWaWb GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWaWb)
289
290#define GITS_BASER_TYPE_SHIFT (56)
217#define GITS_BASER_TYPE(r) (((r) >> GITS_BASER_TYPE_SHIFT) & 7) 291#define GITS_BASER_TYPE(r) (((r) >> GITS_BASER_TYPE_SHIFT) & 7)
218#define GITS_BASER_ENTRY_SIZE_SHIFT (48) 292#define GITS_BASER_ENTRY_SIZE_SHIFT (48)
219#define GITS_BASER_ENTRY_SIZE(r) ((((r) >> GITS_BASER_ENTRY_SIZE_SHIFT) & 0xff) + 1) 293#define GITS_BASER_ENTRY_SIZE(r) ((((r) >> GITS_BASER_ENTRY_SIZE_SHIFT) & 0xff) + 1)
220#define GITS_BASER_NonShareable (0UL << 10)
221#define GITS_BASER_InnerShareable (1UL << 10)
222#define GITS_BASER_OuterShareable (2UL << 10)
223#define GITS_BASER_SHAREABILITY_SHIFT (10) 294#define GITS_BASER_SHAREABILITY_SHIFT (10)
224#define GITS_BASER_SHAREABILITY_MASK (3UL << GITS_BASER_SHAREABILITY_SHIFT) 295#define GITS_BASER_InnerShareable \
296 GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)
225#define GITS_BASER_PAGE_SIZE_SHIFT (8) 297#define GITS_BASER_PAGE_SIZE_SHIFT (8)
226#define GITS_BASER_PAGE_SIZE_4K (0UL << GITS_BASER_PAGE_SIZE_SHIFT) 298#define GITS_BASER_PAGE_SIZE_4K (0UL << GITS_BASER_PAGE_SIZE_SHIFT)
227#define GITS_BASER_PAGE_SIZE_16K (1UL << GITS_BASER_PAGE_SIZE_SHIFT) 299#define GITS_BASER_PAGE_SIZE_16K (1UL << GITS_BASER_PAGE_SIZE_SHIFT)
228#define GITS_BASER_PAGE_SIZE_64K (2UL << GITS_BASER_PAGE_SIZE_SHIFT) 300#define GITS_BASER_PAGE_SIZE_64K (2UL << GITS_BASER_PAGE_SIZE_SHIFT)
229#define GITS_BASER_PAGE_SIZE_MASK (3UL << GITS_BASER_PAGE_SIZE_SHIFT) 301#define GITS_BASER_PAGE_SIZE_MASK (3UL << GITS_BASER_PAGE_SIZE_SHIFT)
230#define GITS_BASER_PAGES_MAX 256 302#define GITS_BASER_PAGES_MAX 256
303#define GITS_BASER_NR_PAGES(r) (((r) & 0xff) + 1)
231 304
232#define GITS_BASER_TYPE_NONE 0 305#define GITS_BASER_TYPE_NONE 0
233#define GITS_BASER_TYPE_DEVICE 1 306#define GITS_BASER_TYPE_DEVICE 1
@@ -243,7 +316,10 @@
243 */ 316 */
244#define GITS_CMD_MAPD 0x08 317#define GITS_CMD_MAPD 0x08
245#define GITS_CMD_MAPC 0x09 318#define GITS_CMD_MAPC 0x09
246#define GITS_CMD_MAPVI 0x0a 319#define GITS_CMD_MAPTI 0x0a
320/* older GIC documentation used MAPVI for this command */
321#define GITS_CMD_MAPVI GITS_CMD_MAPTI
322#define GITS_CMD_MAPI 0x0b
247#define GITS_CMD_MOVI 0x01 323#define GITS_CMD_MOVI 0x01
248#define GITS_CMD_DISCARD 0x0f 324#define GITS_CMD_DISCARD 0x0f
249#define GITS_CMD_INV 0x0c 325#define GITS_CMD_INV 0x0c
@@ -254,6 +330,22 @@
254#define GITS_CMD_SYNC 0x05 330#define GITS_CMD_SYNC 0x05
255 331
256/* 332/*
333 * ITS error numbers
334 */
335#define E_ITS_MOVI_UNMAPPED_INTERRUPT 0x010107
336#define E_ITS_MOVI_UNMAPPED_COLLECTION 0x010109
337#define E_ITS_CLEAR_UNMAPPED_INTERRUPT 0x010507
338#define E_ITS_MAPD_DEVICE_OOR 0x010801
339#define E_ITS_MAPC_PROCNUM_OOR 0x010902
340#define E_ITS_MAPC_COLLECTION_OOR 0x010903
341#define E_ITS_MAPTI_UNMAPPED_DEVICE 0x010a04
342#define E_ITS_MAPTI_PHYSICALID_OOR 0x010a06
343#define E_ITS_INV_UNMAPPED_INTERRUPT 0x010c07
344#define E_ITS_INVALL_UNMAPPED_COLLECTION 0x010d09
345#define E_ITS_MOVALL_PROCNUM_OOR 0x010e01
346#define E_ITS_DISCARD_UNMAPPED_INTERRUPT 0x010f07
347
348/*
257 * CPU interface registers 349 * CPU interface registers
258 */ 350 */
259#define ICC_CTLR_EL1_EOImode_drop_dir (0U << 1) 351#define ICC_CTLR_EL1_EOImode_drop_dir (0U << 1)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 60d339faa94c..aafd702f3e21 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -164,6 +164,8 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
164 int len, struct kvm_io_device *dev); 164 int len, struct kvm_io_device *dev);
165int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 165int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
166 struct kvm_io_device *dev); 166 struct kvm_io_device *dev);
167struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
168 gpa_t addr);
167 169
168#ifdef CONFIG_KVM_ASYNC_PF 170#ifdef CONFIG_KVM_ASYNC_PF
169struct kvm_async_pf { 171struct kvm_async_pf {
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 70941f4ab6d8..8f2756c263d4 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -868,6 +868,7 @@ struct kvm_ppc_smmu_info {
868#define KVM_CAP_MAX_VCPU_ID 128 868#define KVM_CAP_MAX_VCPU_ID 128
869#define KVM_CAP_X2APIC_API 129 869#define KVM_CAP_X2APIC_API 129
870#define KVM_CAP_S390_USER_INSTR0 130 870#define KVM_CAP_S390_USER_INSTR0 130
871#define KVM_CAP_MSI_DEVID 131
871 872
872#ifdef KVM_CAP_IRQ_ROUTING 873#ifdef KVM_CAP_IRQ_ROUTING
873 874
@@ -1026,12 +1027,14 @@ struct kvm_one_reg {
1026 __u64 addr; 1027 __u64 addr;
1027}; 1028};
1028 1029
1030#define KVM_MSI_VALID_DEVID (1U << 0)
1029struct kvm_msi { 1031struct kvm_msi {
1030 __u32 address_lo; 1032 __u32 address_lo;
1031 __u32 address_hi; 1033 __u32 address_hi;
1032 __u32 data; 1034 __u32 data;
1033 __u32 flags; 1035 __u32 flags;
1034 __u8 pad[16]; 1036 __u32 devid;
1037 __u8 pad[12];
1035}; 1038};
1036 1039
1037struct kvm_arm_device_addr { 1040struct kvm_arm_device_addr {
@@ -1076,6 +1079,8 @@ enum kvm_device_type {
1076#define KVM_DEV_TYPE_FLIC KVM_DEV_TYPE_FLIC 1079#define KVM_DEV_TYPE_FLIC KVM_DEV_TYPE_FLIC
1077 KVM_DEV_TYPE_ARM_VGIC_V3, 1080 KVM_DEV_TYPE_ARM_VGIC_V3,
1078#define KVM_DEV_TYPE_ARM_VGIC_V3 KVM_DEV_TYPE_ARM_VGIC_V3 1081#define KVM_DEV_TYPE_ARM_VGIC_V3 KVM_DEV_TYPE_ARM_VGIC_V3
1082 KVM_DEV_TYPE_ARM_VGIC_ITS,
1083#define KVM_DEV_TYPE_ARM_VGIC_ITS KVM_DEV_TYPE_ARM_VGIC_ITS
1079 KVM_DEV_TYPE_MAX, 1084 KVM_DEV_TYPE_MAX,
1080}; 1085};
1081 1086
diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c
index 3a3a699b7489..7cffd9338c49 100644
--- a/virt/kvm/arm/hyp/vgic-v2-sr.c
+++ b/virt/kvm/arm/hyp/vgic-v2-sr.c
@@ -21,18 +21,11 @@
21 21
22#include <asm/kvm_hyp.h> 22#include <asm/kvm_hyp.h>
23 23
24#ifdef CONFIG_KVM_NEW_VGIC
25extern struct vgic_global kvm_vgic_global_state;
26#define vgic_v2_params kvm_vgic_global_state
27#else
28extern struct vgic_params vgic_v2_params;
29#endif
30
31static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu, 24static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu,
32 void __iomem *base) 25 void __iomem *base)
33{ 26{
34 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 27 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
35 int nr_lr = (kern_hyp_va(&vgic_v2_params))->nr_lr; 28 int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
36 u32 eisr0, eisr1; 29 u32 eisr0, eisr1;
37 int i; 30 int i;
38 bool expect_mi; 31 bool expect_mi;
@@ -74,7 +67,7 @@ static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu,
74static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base) 67static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base)
75{ 68{
76 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 69 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
77 int nr_lr = (kern_hyp_va(&vgic_v2_params))->nr_lr; 70 int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
78 u32 elrsr0, elrsr1; 71 u32 elrsr0, elrsr1;
79 72
80 elrsr0 = readl_relaxed(base + GICH_ELRSR0); 73 elrsr0 = readl_relaxed(base + GICH_ELRSR0);
@@ -93,7 +86,7 @@ static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base)
93static void __hyp_text save_lrs(struct kvm_vcpu *vcpu, void __iomem *base) 86static void __hyp_text save_lrs(struct kvm_vcpu *vcpu, void __iomem *base)
94{ 87{
95 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 88 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
96 int nr_lr = (kern_hyp_va(&vgic_v2_params))->nr_lr; 89 int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
97 int i; 90 int i;
98 91
99 for (i = 0; i < nr_lr; i++) { 92 for (i = 0; i < nr_lr; i++) {
@@ -147,7 +140,7 @@ void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu)
147 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 140 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
148 struct vgic_dist *vgic = &kvm->arch.vgic; 141 struct vgic_dist *vgic = &kvm->arch.vgic;
149 void __iomem *base = kern_hyp_va(vgic->vctrl_base); 142 void __iomem *base = kern_hyp_va(vgic->vctrl_base);
150 int nr_lr = (kern_hyp_va(&vgic_v2_params))->nr_lr; 143 int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
151 int i; 144 int i;
152 u64 live_lrs = 0; 145 u64 live_lrs = 0;
153 146
diff --git a/virt/kvm/arm/vgic-v2-emul.c b/virt/kvm/arm/vgic-v2-emul.c
deleted file mode 100644
index 1b0bee095427..000000000000
--- a/virt/kvm/arm/vgic-v2-emul.c
+++ /dev/null
@@ -1,856 +0,0 @@
1/*
2 * Contains GICv2 specific emulation code, was in vgic.c before.
3 *
4 * Copyright (C) 2012 ARM Ltd.
5 * Author: Marc Zyngier <marc.zyngier@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/cpu.h>
21#include <linux/kvm.h>
22#include <linux/kvm_host.h>
23#include <linux/interrupt.h>
24#include <linux/io.h>
25#include <linux/uaccess.h>
26
27#include <linux/irqchip/arm-gic.h>
28
29#include <asm/kvm_emulate.h>
30#include <asm/kvm_arm.h>
31#include <asm/kvm_mmu.h>
32
33#include "vgic.h"
34
35#define GICC_ARCH_VERSION_V2 0x2
36
37static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg);
38static u8 *vgic_get_sgi_sources(struct vgic_dist *dist, int vcpu_id, int sgi)
39{
40 return dist->irq_sgi_sources + vcpu_id * VGIC_NR_SGIS + sgi;
41}
42
43static bool handle_mmio_misc(struct kvm_vcpu *vcpu,
44 struct kvm_exit_mmio *mmio, phys_addr_t offset)
45{
46 u32 reg;
47 u32 word_offset = offset & 3;
48
49 switch (offset & ~3) {
50 case 0: /* GICD_CTLR */
51 reg = vcpu->kvm->arch.vgic.enabled;
52 vgic_reg_access(mmio, &reg, word_offset,
53 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
54 if (mmio->is_write) {
55 vcpu->kvm->arch.vgic.enabled = reg & 1;
56 vgic_update_state(vcpu->kvm);
57 return true;
58 }
59 break;
60
61 case 4: /* GICD_TYPER */
62 reg = (atomic_read(&vcpu->kvm->online_vcpus) - 1) << 5;
63 reg |= (vcpu->kvm->arch.vgic.nr_irqs >> 5) - 1;
64 vgic_reg_access(mmio, &reg, word_offset,
65 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
66 break;
67
68 case 8: /* GICD_IIDR */
69 reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
70 vgic_reg_access(mmio, &reg, word_offset,
71 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
72 break;
73 }
74
75 return false;
76}
77
78static bool handle_mmio_set_enable_reg(struct kvm_vcpu *vcpu,
79 struct kvm_exit_mmio *mmio,
80 phys_addr_t offset)
81{
82 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
83 vcpu->vcpu_id, ACCESS_WRITE_SETBIT);
84}
85
86static bool handle_mmio_clear_enable_reg(struct kvm_vcpu *vcpu,
87 struct kvm_exit_mmio *mmio,
88 phys_addr_t offset)
89{
90 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
91 vcpu->vcpu_id, ACCESS_WRITE_CLEARBIT);
92}
93
94static bool handle_mmio_set_pending_reg(struct kvm_vcpu *vcpu,
95 struct kvm_exit_mmio *mmio,
96 phys_addr_t offset)
97{
98 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
99 vcpu->vcpu_id);
100}
101
102static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu,
103 struct kvm_exit_mmio *mmio,
104 phys_addr_t offset)
105{
106 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
107 vcpu->vcpu_id);
108}
109
110static bool handle_mmio_set_active_reg(struct kvm_vcpu *vcpu,
111 struct kvm_exit_mmio *mmio,
112 phys_addr_t offset)
113{
114 return vgic_handle_set_active_reg(vcpu->kvm, mmio, offset,
115 vcpu->vcpu_id);
116}
117
118static bool handle_mmio_clear_active_reg(struct kvm_vcpu *vcpu,
119 struct kvm_exit_mmio *mmio,
120 phys_addr_t offset)
121{
122 return vgic_handle_clear_active_reg(vcpu->kvm, mmio, offset,
123 vcpu->vcpu_id);
124}
125
126static bool handle_mmio_priority_reg(struct kvm_vcpu *vcpu,
127 struct kvm_exit_mmio *mmio,
128 phys_addr_t offset)
129{
130 u32 *reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
131 vcpu->vcpu_id, offset);
132 vgic_reg_access(mmio, reg, offset,
133 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
134 return false;
135}
136
137#define GICD_ITARGETSR_SIZE 32
138#define GICD_CPUTARGETS_BITS 8
139#define GICD_IRQS_PER_ITARGETSR (GICD_ITARGETSR_SIZE / GICD_CPUTARGETS_BITS)
140static u32 vgic_get_target_reg(struct kvm *kvm, int irq)
141{
142 struct vgic_dist *dist = &kvm->arch.vgic;
143 int i;
144 u32 val = 0;
145
146 irq -= VGIC_NR_PRIVATE_IRQS;
147
148 for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++)
149 val |= 1 << (dist->irq_spi_cpu[irq + i] + i * 8);
150
151 return val;
152}
153
154static void vgic_set_target_reg(struct kvm *kvm, u32 val, int irq)
155{
156 struct vgic_dist *dist = &kvm->arch.vgic;
157 struct kvm_vcpu *vcpu;
158 int i, c;
159 unsigned long *bmap;
160 u32 target;
161
162 irq -= VGIC_NR_PRIVATE_IRQS;
163
164 /*
165 * Pick the LSB in each byte. This ensures we target exactly
166 * one vcpu per IRQ. If the byte is null, assume we target
167 * CPU0.
168 */
169 for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++) {
170 int shift = i * GICD_CPUTARGETS_BITS;
171
172 target = ffs((val >> shift) & 0xffU);
173 target = target ? (target - 1) : 0;
174 dist->irq_spi_cpu[irq + i] = target;
175 kvm_for_each_vcpu(c, vcpu, kvm) {
176 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[c]);
177 if (c == target)
178 set_bit(irq + i, bmap);
179 else
180 clear_bit(irq + i, bmap);
181 }
182 }
183}
184
185static bool handle_mmio_target_reg(struct kvm_vcpu *vcpu,
186 struct kvm_exit_mmio *mmio,
187 phys_addr_t offset)
188{
189 u32 reg;
190
191 /* We treat the banked interrupts targets as read-only */
192 if (offset < 32) {
193 u32 roreg;
194
195 roreg = 1 << vcpu->vcpu_id;
196 roreg |= roreg << 8;
197 roreg |= roreg << 16;
198
199 vgic_reg_access(mmio, &roreg, offset,
200 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
201 return false;
202 }
203
204 reg = vgic_get_target_reg(vcpu->kvm, offset & ~3U);
205 vgic_reg_access(mmio, &reg, offset,
206 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
207 if (mmio->is_write) {
208 vgic_set_target_reg(vcpu->kvm, reg, offset & ~3U);
209 vgic_update_state(vcpu->kvm);
210 return true;
211 }
212
213 return false;
214}
215
216static bool handle_mmio_cfg_reg(struct kvm_vcpu *vcpu,
217 struct kvm_exit_mmio *mmio, phys_addr_t offset)
218{
219 u32 *reg;
220
221 reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
222 vcpu->vcpu_id, offset >> 1);
223
224 return vgic_handle_cfg_reg(reg, mmio, offset);
225}
226
227static bool handle_mmio_sgi_reg(struct kvm_vcpu *vcpu,
228 struct kvm_exit_mmio *mmio, phys_addr_t offset)
229{
230 u32 reg;
231
232 vgic_reg_access(mmio, &reg, offset,
233 ACCESS_READ_RAZ | ACCESS_WRITE_VALUE);
234 if (mmio->is_write) {
235 vgic_dispatch_sgi(vcpu, reg);
236 vgic_update_state(vcpu->kvm);
237 return true;
238 }
239
240 return false;
241}
242
243/* Handle reads of GICD_CPENDSGIRn and GICD_SPENDSGIRn */
244static bool read_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
245 struct kvm_exit_mmio *mmio,
246 phys_addr_t offset)
247{
248 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
249 int sgi;
250 int min_sgi = (offset & ~0x3);
251 int max_sgi = min_sgi + 3;
252 int vcpu_id = vcpu->vcpu_id;
253 u32 reg = 0;
254
255 /* Copy source SGIs from distributor side */
256 for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
257 u8 sources = *vgic_get_sgi_sources(dist, vcpu_id, sgi);
258
259 reg |= ((u32)sources) << (8 * (sgi - min_sgi));
260 }
261
262 mmio_data_write(mmio, ~0, reg);
263 return false;
264}
265
266static bool write_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
267 struct kvm_exit_mmio *mmio,
268 phys_addr_t offset, bool set)
269{
270 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
271 int sgi;
272 int min_sgi = (offset & ~0x3);
273 int max_sgi = min_sgi + 3;
274 int vcpu_id = vcpu->vcpu_id;
275 u32 reg;
276 bool updated = false;
277
278 reg = mmio_data_read(mmio, ~0);
279
280 /* Clear pending SGIs on the distributor */
281 for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
282 u8 mask = reg >> (8 * (sgi - min_sgi));
283 u8 *src = vgic_get_sgi_sources(dist, vcpu_id, sgi);
284
285 if (set) {
286 if ((*src & mask) != mask)
287 updated = true;
288 *src |= mask;
289 } else {
290 if (*src & mask)
291 updated = true;
292 *src &= ~mask;
293 }
294 }
295
296 if (updated)
297 vgic_update_state(vcpu->kvm);
298
299 return updated;
300}
301
302static bool handle_mmio_sgi_set(struct kvm_vcpu *vcpu,
303 struct kvm_exit_mmio *mmio,
304 phys_addr_t offset)
305{
306 if (!mmio->is_write)
307 return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
308 else
309 return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, true);
310}
311
312static bool handle_mmio_sgi_clear(struct kvm_vcpu *vcpu,
313 struct kvm_exit_mmio *mmio,
314 phys_addr_t offset)
315{
316 if (!mmio->is_write)
317 return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
318 else
319 return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, false);
320}
321
322static const struct vgic_io_range vgic_dist_ranges[] = {
323 {
324 .base = GIC_DIST_SOFTINT,
325 .len = 4,
326 .handle_mmio = handle_mmio_sgi_reg,
327 },
328 {
329 .base = GIC_DIST_CTRL,
330 .len = 12,
331 .bits_per_irq = 0,
332 .handle_mmio = handle_mmio_misc,
333 },
334 {
335 .base = GIC_DIST_IGROUP,
336 .len = VGIC_MAX_IRQS / 8,
337 .bits_per_irq = 1,
338 .handle_mmio = handle_mmio_raz_wi,
339 },
340 {
341 .base = GIC_DIST_ENABLE_SET,
342 .len = VGIC_MAX_IRQS / 8,
343 .bits_per_irq = 1,
344 .handle_mmio = handle_mmio_set_enable_reg,
345 },
346 {
347 .base = GIC_DIST_ENABLE_CLEAR,
348 .len = VGIC_MAX_IRQS / 8,
349 .bits_per_irq = 1,
350 .handle_mmio = handle_mmio_clear_enable_reg,
351 },
352 {
353 .base = GIC_DIST_PENDING_SET,
354 .len = VGIC_MAX_IRQS / 8,
355 .bits_per_irq = 1,
356 .handle_mmio = handle_mmio_set_pending_reg,
357 },
358 {
359 .base = GIC_DIST_PENDING_CLEAR,
360 .len = VGIC_MAX_IRQS / 8,
361 .bits_per_irq = 1,
362 .handle_mmio = handle_mmio_clear_pending_reg,
363 },
364 {
365 .base = GIC_DIST_ACTIVE_SET,
366 .len = VGIC_MAX_IRQS / 8,
367 .bits_per_irq = 1,
368 .handle_mmio = handle_mmio_set_active_reg,
369 },
370 {
371 .base = GIC_DIST_ACTIVE_CLEAR,
372 .len = VGIC_MAX_IRQS / 8,
373 .bits_per_irq = 1,
374 .handle_mmio = handle_mmio_clear_active_reg,
375 },
376 {
377 .base = GIC_DIST_PRI,
378 .len = VGIC_MAX_IRQS,
379 .bits_per_irq = 8,
380 .handle_mmio = handle_mmio_priority_reg,
381 },
382 {
383 .base = GIC_DIST_TARGET,
384 .len = VGIC_MAX_IRQS,
385 .bits_per_irq = 8,
386 .handle_mmio = handle_mmio_target_reg,
387 },
388 {
389 .base = GIC_DIST_CONFIG,
390 .len = VGIC_MAX_IRQS / 4,
391 .bits_per_irq = 2,
392 .handle_mmio = handle_mmio_cfg_reg,
393 },
394 {
395 .base = GIC_DIST_SGI_PENDING_CLEAR,
396 .len = VGIC_NR_SGIS,
397 .handle_mmio = handle_mmio_sgi_clear,
398 },
399 {
400 .base = GIC_DIST_SGI_PENDING_SET,
401 .len = VGIC_NR_SGIS,
402 .handle_mmio = handle_mmio_sgi_set,
403 },
404 {}
405};
406
407static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg)
408{
409 struct kvm *kvm = vcpu->kvm;
410 struct vgic_dist *dist = &kvm->arch.vgic;
411 int nrcpus = atomic_read(&kvm->online_vcpus);
412 u8 target_cpus;
413 int sgi, mode, c, vcpu_id;
414
415 vcpu_id = vcpu->vcpu_id;
416
417 sgi = reg & 0xf;
418 target_cpus = (reg >> 16) & 0xff;
419 mode = (reg >> 24) & 3;
420
421 switch (mode) {
422 case 0:
423 if (!target_cpus)
424 return;
425 break;
426
427 case 1:
428 target_cpus = ((1 << nrcpus) - 1) & ~(1 << vcpu_id) & 0xff;
429 break;
430
431 case 2:
432 target_cpus = 1 << vcpu_id;
433 break;
434 }
435
436 kvm_for_each_vcpu(c, vcpu, kvm) {
437 if (target_cpus & 1) {
438 /* Flag the SGI as pending */
439 vgic_dist_irq_set_pending(vcpu, sgi);
440 *vgic_get_sgi_sources(dist, c, sgi) |= 1 << vcpu_id;
441 kvm_debug("SGI%d from CPU%d to CPU%d\n",
442 sgi, vcpu_id, c);
443 }
444
445 target_cpus >>= 1;
446 }
447}
448
449static bool vgic_v2_queue_sgi(struct kvm_vcpu *vcpu, int irq)
450{
451 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
452 unsigned long sources;
453 int vcpu_id = vcpu->vcpu_id;
454 int c;
455
456 sources = *vgic_get_sgi_sources(dist, vcpu_id, irq);
457
458 for_each_set_bit(c, &sources, dist->nr_cpus) {
459 if (vgic_queue_irq(vcpu, c, irq))
460 clear_bit(c, &sources);
461 }
462
463 *vgic_get_sgi_sources(dist, vcpu_id, irq) = sources;
464
465 /*
466 * If the sources bitmap has been cleared it means that we
467 * could queue all the SGIs onto link registers (see the
468 * clear_bit above), and therefore we are done with them in
469 * our emulated gic and can get rid of them.
470 */
471 if (!sources) {
472 vgic_dist_irq_clear_pending(vcpu, irq);
473 vgic_cpu_irq_clear(vcpu, irq);
474 return true;
475 }
476
477 return false;
478}
479
480/**
481 * kvm_vgic_map_resources - Configure global VGIC state before running any VCPUs
482 * @kvm: pointer to the kvm struct
483 *
484 * Map the virtual CPU interface into the VM before running any VCPUs. We
485 * can't do this at creation time, because user space must first set the
486 * virtual CPU interface address in the guest physical address space.
487 */
488static int vgic_v2_map_resources(struct kvm *kvm,
489 const struct vgic_params *params)
490{
491 struct vgic_dist *dist = &kvm->arch.vgic;
492 int ret = 0;
493
494 if (!irqchip_in_kernel(kvm))
495 return 0;
496
497 mutex_lock(&kvm->lock);
498
499 if (vgic_ready(kvm))
500 goto out;
501
502 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
503 IS_VGIC_ADDR_UNDEF(dist->vgic_cpu_base)) {
504 kvm_err("Need to set vgic cpu and dist addresses first\n");
505 ret = -ENXIO;
506 goto out;
507 }
508
509 vgic_register_kvm_io_dev(kvm, dist->vgic_dist_base,
510 KVM_VGIC_V2_DIST_SIZE,
511 vgic_dist_ranges, -1, &dist->dist_iodev);
512
513 /*
514 * Initialize the vgic if this hasn't already been done on demand by
515 * accessing the vgic state from userspace.
516 */
517 ret = vgic_init(kvm);
518 if (ret) {
519 kvm_err("Unable to allocate maps\n");
520 goto out_unregister;
521 }
522
523 ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,
524 params->vcpu_base, KVM_VGIC_V2_CPU_SIZE,
525 true);
526 if (ret) {
527 kvm_err("Unable to remap VGIC CPU to VCPU\n");
528 goto out_unregister;
529 }
530
531 dist->ready = true;
532 goto out;
533
534out_unregister:
535 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &dist->dist_iodev.dev);
536
537out:
538 if (ret)
539 kvm_vgic_destroy(kvm);
540 mutex_unlock(&kvm->lock);
541 return ret;
542}
543
544static void vgic_v2_add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
545{
546 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
547
548 *vgic_get_sgi_sources(dist, vcpu->vcpu_id, irq) |= 1 << source;
549}
550
551static int vgic_v2_init_model(struct kvm *kvm)
552{
553 int i;
554
555 for (i = VGIC_NR_PRIVATE_IRQS; i < kvm->arch.vgic.nr_irqs; i += 4)
556 vgic_set_target_reg(kvm, 0, i);
557
558 return 0;
559}
560
561void vgic_v2_init_emulation(struct kvm *kvm)
562{
563 struct vgic_dist *dist = &kvm->arch.vgic;
564
565 dist->vm_ops.queue_sgi = vgic_v2_queue_sgi;
566 dist->vm_ops.add_sgi_source = vgic_v2_add_sgi_source;
567 dist->vm_ops.init_model = vgic_v2_init_model;
568 dist->vm_ops.map_resources = vgic_v2_map_resources;
569
570 kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS;
571}
572
573static bool handle_cpu_mmio_misc(struct kvm_vcpu *vcpu,
574 struct kvm_exit_mmio *mmio, phys_addr_t offset)
575{
576 bool updated = false;
577 struct vgic_vmcr vmcr;
578 u32 *vmcr_field;
579 u32 reg;
580
581 vgic_get_vmcr(vcpu, &vmcr);
582
583 switch (offset & ~0x3) {
584 case GIC_CPU_CTRL:
585 vmcr_field = &vmcr.ctlr;
586 break;
587 case GIC_CPU_PRIMASK:
588 vmcr_field = &vmcr.pmr;
589 break;
590 case GIC_CPU_BINPOINT:
591 vmcr_field = &vmcr.bpr;
592 break;
593 case GIC_CPU_ALIAS_BINPOINT:
594 vmcr_field = &vmcr.abpr;
595 break;
596 default:
597 BUG();
598 }
599
600 if (!mmio->is_write) {
601 reg = *vmcr_field;
602 mmio_data_write(mmio, ~0, reg);
603 } else {
604 reg = mmio_data_read(mmio, ~0);
605 if (reg != *vmcr_field) {
606 *vmcr_field = reg;
607 vgic_set_vmcr(vcpu, &vmcr);
608 updated = true;
609 }
610 }
611 return updated;
612}
613
614static bool handle_mmio_abpr(struct kvm_vcpu *vcpu,
615 struct kvm_exit_mmio *mmio, phys_addr_t offset)
616{
617 return handle_cpu_mmio_misc(vcpu, mmio, GIC_CPU_ALIAS_BINPOINT);
618}
619
620static bool handle_cpu_mmio_ident(struct kvm_vcpu *vcpu,
621 struct kvm_exit_mmio *mmio,
622 phys_addr_t offset)
623{
624 u32 reg;
625
626 if (mmio->is_write)
627 return false;
628
629 /* GICC_IIDR */
630 reg = (PRODUCT_ID_KVM << 20) |
631 (GICC_ARCH_VERSION_V2 << 16) |
632 (IMPLEMENTER_ARM << 0);
633 mmio_data_write(mmio, ~0, reg);
634 return false;
635}
636
637/*
638 * CPU Interface Register accesses - these are not accessed by the VM, but by
639 * user space for saving and restoring VGIC state.
640 */
641static const struct vgic_io_range vgic_cpu_ranges[] = {
642 {
643 .base = GIC_CPU_CTRL,
644 .len = 12,
645 .handle_mmio = handle_cpu_mmio_misc,
646 },
647 {
648 .base = GIC_CPU_ALIAS_BINPOINT,
649 .len = 4,
650 .handle_mmio = handle_mmio_abpr,
651 },
652 {
653 .base = GIC_CPU_ACTIVEPRIO,
654 .len = 16,
655 .handle_mmio = handle_mmio_raz_wi,
656 },
657 {
658 .base = GIC_CPU_IDENT,
659 .len = 4,
660 .handle_mmio = handle_cpu_mmio_ident,
661 },
662};
663
664static int vgic_attr_regs_access(struct kvm_device *dev,
665 struct kvm_device_attr *attr,
666 u32 *reg, bool is_write)
667{
668 const struct vgic_io_range *r = NULL, *ranges;
669 phys_addr_t offset;
670 int ret, cpuid, c;
671 struct kvm_vcpu *vcpu, *tmp_vcpu;
672 struct vgic_dist *vgic;
673 struct kvm_exit_mmio mmio;
674 u32 data;
675
676 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
677 cpuid = (attr->attr & KVM_DEV_ARM_VGIC_CPUID_MASK) >>
678 KVM_DEV_ARM_VGIC_CPUID_SHIFT;
679
680 mutex_lock(&dev->kvm->lock);
681
682 ret = vgic_init(dev->kvm);
683 if (ret)
684 goto out;
685
686 if (cpuid >= atomic_read(&dev->kvm->online_vcpus)) {
687 ret = -EINVAL;
688 goto out;
689 }
690
691 vcpu = kvm_get_vcpu(dev->kvm, cpuid);
692 vgic = &dev->kvm->arch.vgic;
693
694 mmio.len = 4;
695 mmio.is_write = is_write;
696 mmio.data = &data;
697 if (is_write)
698 mmio_data_write(&mmio, ~0, *reg);
699 switch (attr->group) {
700 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
701 mmio.phys_addr = vgic->vgic_dist_base + offset;
702 ranges = vgic_dist_ranges;
703 break;
704 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
705 mmio.phys_addr = vgic->vgic_cpu_base + offset;
706 ranges = vgic_cpu_ranges;
707 break;
708 default:
709 BUG();
710 }
711 r = vgic_find_range(ranges, 4, offset);
712
713 if (unlikely(!r || !r->handle_mmio)) {
714 ret = -ENXIO;
715 goto out;
716 }
717
718
719 spin_lock(&vgic->lock);
720
721 /*
722 * Ensure that no other VCPU is running by checking the vcpu->cpu
723 * field. If no other VPCUs are running we can safely access the VGIC
724 * state, because even if another VPU is run after this point, that
725 * VCPU will not touch the vgic state, because it will block on
726 * getting the vgic->lock in kvm_vgic_sync_hwstate().
727 */
728 kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm) {
729 if (unlikely(tmp_vcpu->cpu != -1)) {
730 ret = -EBUSY;
731 goto out_vgic_unlock;
732 }
733 }
734
735 /*
736 * Move all pending IRQs from the LRs on all VCPUs so the pending
737 * state can be properly represented in the register state accessible
738 * through this API.
739 */
740 kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm)
741 vgic_unqueue_irqs(tmp_vcpu);
742
743 offset -= r->base;
744 r->handle_mmio(vcpu, &mmio, offset);
745
746 if (!is_write)
747 *reg = mmio_data_read(&mmio, ~0);
748
749 ret = 0;
750out_vgic_unlock:
751 spin_unlock(&vgic->lock);
752out:
753 mutex_unlock(&dev->kvm->lock);
754 return ret;
755}
756
757static int vgic_v2_create(struct kvm_device *dev, u32 type)
758{
759 return kvm_vgic_create(dev->kvm, type);
760}
761
762static void vgic_v2_destroy(struct kvm_device *dev)
763{
764 kfree(dev);
765}
766
767static int vgic_v2_set_attr(struct kvm_device *dev,
768 struct kvm_device_attr *attr)
769{
770 int ret;
771
772 ret = vgic_set_common_attr(dev, attr);
773 if (ret != -ENXIO)
774 return ret;
775
776 switch (attr->group) {
777 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
778 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
779 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
780 u32 reg;
781
782 if (get_user(reg, uaddr))
783 return -EFAULT;
784
785 return vgic_attr_regs_access(dev, attr, &reg, true);
786 }
787
788 }
789
790 return -ENXIO;
791}
792
793static int vgic_v2_get_attr(struct kvm_device *dev,
794 struct kvm_device_attr *attr)
795{
796 int ret;
797
798 ret = vgic_get_common_attr(dev, attr);
799 if (ret != -ENXIO)
800 return ret;
801
802 switch (attr->group) {
803 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
804 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
805 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
806 u32 reg = 0;
807
808 ret = vgic_attr_regs_access(dev, attr, &reg, false);
809 if (ret)
810 return ret;
811 return put_user(reg, uaddr);
812 }
813
814 }
815
816 return -ENXIO;
817}
818
819static int vgic_v2_has_attr(struct kvm_device *dev,
820 struct kvm_device_attr *attr)
821{
822 phys_addr_t offset;
823
824 switch (attr->group) {
825 case KVM_DEV_ARM_VGIC_GRP_ADDR:
826 switch (attr->attr) {
827 case KVM_VGIC_V2_ADDR_TYPE_DIST:
828 case KVM_VGIC_V2_ADDR_TYPE_CPU:
829 return 0;
830 }
831 break;
832 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
833 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
834 return vgic_has_attr_regs(vgic_dist_ranges, offset);
835 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
836 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
837 return vgic_has_attr_regs(vgic_cpu_ranges, offset);
838 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
839 return 0;
840 case KVM_DEV_ARM_VGIC_GRP_CTRL:
841 switch (attr->attr) {
842 case KVM_DEV_ARM_VGIC_CTRL_INIT:
843 return 0;
844 }
845 }
846 return -ENXIO;
847}
848
849struct kvm_device_ops kvm_arm_vgic_v2_ops = {
850 .name = "kvm-arm-vgic-v2",
851 .create = vgic_v2_create,
852 .destroy = vgic_v2_destroy,
853 .set_attr = vgic_v2_set_attr,
854 .get_attr = vgic_v2_get_attr,
855 .has_attr = vgic_v2_has_attr,
856};
diff --git a/virt/kvm/arm/vgic-v2.c b/virt/kvm/arm/vgic-v2.c
deleted file mode 100644
index 334cd7a89106..000000000000
--- a/virt/kvm/arm/vgic-v2.c
+++ /dev/null
@@ -1,274 +0,0 @@
1/*
2 * Copyright (C) 2012,2013 ARM Limited, All Rights Reserved.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/cpu.h>
19#include <linux/kvm.h>
20#include <linux/kvm_host.h>
21#include <linux/interrupt.h>
22#include <linux/io.h>
23
24#include <linux/irqchip/arm-gic.h>
25
26#include <asm/kvm_emulate.h>
27#include <asm/kvm_arm.h>
28#include <asm/kvm_mmu.h>
29
30static struct vgic_lr vgic_v2_get_lr(const struct kvm_vcpu *vcpu, int lr)
31{
32 struct vgic_lr lr_desc;
33 u32 val = vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr];
34
35 lr_desc.irq = val & GICH_LR_VIRTUALID;
36 if (lr_desc.irq <= 15)
37 lr_desc.source = (val >> GICH_LR_PHYSID_CPUID_SHIFT) & 0x7;
38 else
39 lr_desc.source = 0;
40 lr_desc.state = 0;
41
42 if (val & GICH_LR_PENDING_BIT)
43 lr_desc.state |= LR_STATE_PENDING;
44 if (val & GICH_LR_ACTIVE_BIT)
45 lr_desc.state |= LR_STATE_ACTIVE;
46 if (val & GICH_LR_EOI)
47 lr_desc.state |= LR_EOI_INT;
48 if (val & GICH_LR_HW) {
49 lr_desc.state |= LR_HW;
50 lr_desc.hwirq = (val & GICH_LR_PHYSID_CPUID) >> GICH_LR_PHYSID_CPUID_SHIFT;
51 }
52
53 return lr_desc;
54}
55
56static void vgic_v2_set_lr(struct kvm_vcpu *vcpu, int lr,
57 struct vgic_lr lr_desc)
58{
59 u32 lr_val;
60
61 lr_val = lr_desc.irq;
62
63 if (lr_desc.state & LR_STATE_PENDING)
64 lr_val |= GICH_LR_PENDING_BIT;
65 if (lr_desc.state & LR_STATE_ACTIVE)
66 lr_val |= GICH_LR_ACTIVE_BIT;
67 if (lr_desc.state & LR_EOI_INT)
68 lr_val |= GICH_LR_EOI;
69
70 if (lr_desc.state & LR_HW) {
71 lr_val |= GICH_LR_HW;
72 lr_val |= (u32)lr_desc.hwirq << GICH_LR_PHYSID_CPUID_SHIFT;
73 }
74
75 if (lr_desc.irq < VGIC_NR_SGIS)
76 lr_val |= (lr_desc.source << GICH_LR_PHYSID_CPUID_SHIFT);
77
78 vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = lr_val;
79
80 if (!(lr_desc.state & LR_STATE_MASK))
81 vcpu->arch.vgic_cpu.vgic_v2.vgic_elrsr |= (1ULL << lr);
82 else
83 vcpu->arch.vgic_cpu.vgic_v2.vgic_elrsr &= ~(1ULL << lr);
84}
85
86static u64 vgic_v2_get_elrsr(const struct kvm_vcpu *vcpu)
87{
88 return vcpu->arch.vgic_cpu.vgic_v2.vgic_elrsr;
89}
90
91static u64 vgic_v2_get_eisr(const struct kvm_vcpu *vcpu)
92{
93 return vcpu->arch.vgic_cpu.vgic_v2.vgic_eisr;
94}
95
96static void vgic_v2_clear_eisr(struct kvm_vcpu *vcpu)
97{
98 vcpu->arch.vgic_cpu.vgic_v2.vgic_eisr = 0;
99}
100
101static u32 vgic_v2_get_interrupt_status(const struct kvm_vcpu *vcpu)
102{
103 u32 misr = vcpu->arch.vgic_cpu.vgic_v2.vgic_misr;
104 u32 ret = 0;
105
106 if (misr & GICH_MISR_EOI)
107 ret |= INT_STATUS_EOI;
108 if (misr & GICH_MISR_U)
109 ret |= INT_STATUS_UNDERFLOW;
110
111 return ret;
112}
113
114static void vgic_v2_enable_underflow(struct kvm_vcpu *vcpu)
115{
116 vcpu->arch.vgic_cpu.vgic_v2.vgic_hcr |= GICH_HCR_UIE;
117}
118
119static void vgic_v2_disable_underflow(struct kvm_vcpu *vcpu)
120{
121 vcpu->arch.vgic_cpu.vgic_v2.vgic_hcr &= ~GICH_HCR_UIE;
122}
123
124static void vgic_v2_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
125{
126 u32 vmcr = vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr;
127
128 vmcrp->ctlr = (vmcr & GICH_VMCR_CTRL_MASK) >> GICH_VMCR_CTRL_SHIFT;
129 vmcrp->abpr = (vmcr & GICH_VMCR_ALIAS_BINPOINT_MASK) >> GICH_VMCR_ALIAS_BINPOINT_SHIFT;
130 vmcrp->bpr = (vmcr & GICH_VMCR_BINPOINT_MASK) >> GICH_VMCR_BINPOINT_SHIFT;
131 vmcrp->pmr = (vmcr & GICH_VMCR_PRIMASK_MASK) >> GICH_VMCR_PRIMASK_SHIFT;
132}
133
134static void vgic_v2_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
135{
136 u32 vmcr;
137
138 vmcr = (vmcrp->ctlr << GICH_VMCR_CTRL_SHIFT) & GICH_VMCR_CTRL_MASK;
139 vmcr |= (vmcrp->abpr << GICH_VMCR_ALIAS_BINPOINT_SHIFT) & GICH_VMCR_ALIAS_BINPOINT_MASK;
140 vmcr |= (vmcrp->bpr << GICH_VMCR_BINPOINT_SHIFT) & GICH_VMCR_BINPOINT_MASK;
141 vmcr |= (vmcrp->pmr << GICH_VMCR_PRIMASK_SHIFT) & GICH_VMCR_PRIMASK_MASK;
142
143 vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = vmcr;
144}
145
146static void vgic_v2_enable(struct kvm_vcpu *vcpu)
147{
148 /*
149 * By forcing VMCR to zero, the GIC will restore the binary
150 * points to their reset values. Anything else resets to zero
151 * anyway.
152 */
153 vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = 0;
154 vcpu->arch.vgic_cpu.vgic_v2.vgic_elrsr = ~0;
155
156 /* Get the show on the road... */
157 vcpu->arch.vgic_cpu.vgic_v2.vgic_hcr = GICH_HCR_EN;
158}
159
160static const struct vgic_ops vgic_v2_ops = {
161 .get_lr = vgic_v2_get_lr,
162 .set_lr = vgic_v2_set_lr,
163 .get_elrsr = vgic_v2_get_elrsr,
164 .get_eisr = vgic_v2_get_eisr,
165 .clear_eisr = vgic_v2_clear_eisr,
166 .get_interrupt_status = vgic_v2_get_interrupt_status,
167 .enable_underflow = vgic_v2_enable_underflow,
168 .disable_underflow = vgic_v2_disable_underflow,
169 .get_vmcr = vgic_v2_get_vmcr,
170 .set_vmcr = vgic_v2_set_vmcr,
171 .enable = vgic_v2_enable,
172};
173
174struct vgic_params __section(.hyp.text) vgic_v2_params;
175
176static void vgic_cpu_init_lrs(void *params)
177{
178 struct vgic_params *vgic = params;
179 int i;
180
181 for (i = 0; i < vgic->nr_lr; i++)
182 writel_relaxed(0, vgic->vctrl_base + GICH_LR0 + (i * 4));
183}
184
185/**
186 * vgic_v2_probe - probe for a GICv2 compatible interrupt controller
187 * @gic_kvm_info: pointer to the GIC description
188 * @ops: address of a pointer to the GICv2 operations
189 * @params: address of a pointer to HW-specific parameters
190 *
191 * Returns 0 if a GICv2 has been found, with the low level operations
192 * in *ops and the HW parameters in *params. Returns an error code
193 * otherwise.
194 */
195int vgic_v2_probe(const struct gic_kvm_info *gic_kvm_info,
196 const struct vgic_ops **ops,
197 const struct vgic_params **params)
198{
199 int ret;
200 struct vgic_params *vgic = &vgic_v2_params;
201 const struct resource *vctrl_res = &gic_kvm_info->vctrl;
202 const struct resource *vcpu_res = &gic_kvm_info->vcpu;
203
204 memset(vgic, 0, sizeof(*vgic));
205
206 if (!gic_kvm_info->maint_irq) {
207 kvm_err("error getting vgic maintenance irq\n");
208 ret = -ENXIO;
209 goto out;
210 }
211 vgic->maint_irq = gic_kvm_info->maint_irq;
212
213 if (!gic_kvm_info->vctrl.start) {
214 kvm_err("GICH not present in the firmware table\n");
215 ret = -ENXIO;
216 goto out;
217 }
218
219 vgic->vctrl_base = ioremap(gic_kvm_info->vctrl.start,
220 resource_size(&gic_kvm_info->vctrl));
221 if (!vgic->vctrl_base) {
222 kvm_err("Cannot ioremap GICH\n");
223 ret = -ENOMEM;
224 goto out;
225 }
226
227 vgic->nr_lr = readl_relaxed(vgic->vctrl_base + GICH_VTR);
228 vgic->nr_lr = (vgic->nr_lr & 0x3f) + 1;
229
230 ret = create_hyp_io_mappings(vgic->vctrl_base,
231 vgic->vctrl_base + resource_size(vctrl_res),
232 vctrl_res->start);
233 if (ret) {
234 kvm_err("Cannot map VCTRL into hyp\n");
235 goto out_unmap;
236 }
237
238 if (!PAGE_ALIGNED(vcpu_res->start)) {
239 kvm_err("GICV physical address 0x%llx not page aligned\n",
240 (unsigned long long)vcpu_res->start);
241 ret = -ENXIO;
242 goto out_unmap;
243 }
244
245 if (!PAGE_ALIGNED(resource_size(vcpu_res))) {
246 kvm_err("GICV size 0x%llx not a multiple of page size 0x%lx\n",
247 (unsigned long long)resource_size(vcpu_res),
248 PAGE_SIZE);
249 ret = -ENXIO;
250 goto out_unmap;
251 }
252
253 vgic->can_emulate_gicv2 = true;
254 kvm_register_device_ops(&kvm_arm_vgic_v2_ops, KVM_DEV_TYPE_ARM_VGIC_V2);
255
256 vgic->vcpu_base = vcpu_res->start;
257
258 kvm_info("GICH base=0x%llx, GICV base=0x%llx, IRQ=%d\n",
259 gic_kvm_info->vctrl.start, vgic->vcpu_base, vgic->maint_irq);
260
261 vgic->type = VGIC_V2;
262 vgic->max_gic_vcpus = VGIC_V2_MAX_CPUS;
263
264 on_each_cpu(vgic_cpu_init_lrs, vgic, 1);
265
266 *ops = &vgic_v2_ops;
267 *params = vgic;
268 goto out;
269
270out_unmap:
271 iounmap(vgic->vctrl_base);
272out:
273 return ret;
274}
diff --git a/virt/kvm/arm/vgic-v3-emul.c b/virt/kvm/arm/vgic-v3-emul.c
deleted file mode 100644
index e661e7fb9d91..000000000000
--- a/virt/kvm/arm/vgic-v3-emul.c
+++ /dev/null
@@ -1,1074 +0,0 @@
1/*
2 * GICv3 distributor and redistributor emulation
3 *
4 * GICv3 emulation is currently only supported on a GICv3 host (because
5 * we rely on the hardware's CPU interface virtualization support), but
6 * supports both hardware with or without the optional GICv2 backwards
7 * compatibility features.
8 *
9 * Limitations of the emulation:
10 * (RAZ/WI: read as zero, write ignore, RAO/WI: read as one, write ignore)
11 * - We do not support LPIs (yet). TYPER.LPIS is reported as 0 and is RAZ/WI.
12 * - We do not support the message based interrupts (MBIs) triggered by
13 * writes to the GICD_{SET,CLR}SPI_* registers. TYPER.MBIS is reported as 0.
14 * - We do not support the (optional) backwards compatibility feature.
15 * GICD_CTLR.ARE resets to 1 and is RAO/WI. If the _host_ GIC supports
16 * the compatiblity feature, you can use a GICv2 in the guest, though.
17 * - We only support a single security state. GICD_CTLR.DS is 1 and is RAO/WI.
18 * - Priorities are not emulated (same as the GICv2 emulation). Linux
19 * as a guest is fine with this, because it does not use priorities.
20 * - We only support Group1 interrupts. Again Linux uses only those.
21 *
22 * Copyright (C) 2014 ARM Ltd.
23 * Author: Andre Przywara <andre.przywara@arm.com>
24 *
25 * This program is free software; you can redistribute it and/or modify
26 * it under the terms of the GNU General Public License version 2 as
27 * published by the Free Software Foundation.
28 *
29 * This program is distributed in the hope that it will be useful,
30 * but WITHOUT ANY WARRANTY; without even the implied warranty of
31 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32 * GNU General Public License for more details.
33 *
34 * You should have received a copy of the GNU General Public License
35 * along with this program. If not, see <http://www.gnu.org/licenses/>.
36 */
37
38#include <linux/cpu.h>
39#include <linux/kvm.h>
40#include <linux/kvm_host.h>
41#include <linux/interrupt.h>
42
43#include <linux/irqchip/arm-gic-v3.h>
44#include <kvm/arm_vgic.h>
45
46#include <asm/kvm_emulate.h>
47#include <asm/kvm_arm.h>
48#include <asm/kvm_mmu.h>
49
50#include "vgic.h"
51
52static bool handle_mmio_rao_wi(struct kvm_vcpu *vcpu,
53 struct kvm_exit_mmio *mmio, phys_addr_t offset)
54{
55 u32 reg = 0xffffffff;
56
57 vgic_reg_access(mmio, &reg, offset,
58 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
59
60 return false;
61}
62
63static bool handle_mmio_ctlr(struct kvm_vcpu *vcpu,
64 struct kvm_exit_mmio *mmio, phys_addr_t offset)
65{
66 u32 reg = 0;
67
68 /*
69 * Force ARE and DS to 1, the guest cannot change this.
70 * For the time being we only support Group1 interrupts.
71 */
72 if (vcpu->kvm->arch.vgic.enabled)
73 reg = GICD_CTLR_ENABLE_SS_G1;
74 reg |= GICD_CTLR_ARE_NS | GICD_CTLR_DS;
75
76 vgic_reg_access(mmio, &reg, offset,
77 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
78 if (mmio->is_write) {
79 vcpu->kvm->arch.vgic.enabled = !!(reg & GICD_CTLR_ENABLE_SS_G1);
80 vgic_update_state(vcpu->kvm);
81 return true;
82 }
83 return false;
84}
85
86/*
87 * As this implementation does not provide compatibility
88 * with GICv2 (ARE==1), we report zero CPUs in bits [5..7].
89 * Also LPIs and MBIs are not supported, so we set the respective bits to 0.
90 * Also we report at most 2**10=1024 interrupt IDs (to match 1024 SPIs).
91 */
92#define INTERRUPT_ID_BITS 10
93static bool handle_mmio_typer(struct kvm_vcpu *vcpu,
94 struct kvm_exit_mmio *mmio, phys_addr_t offset)
95{
96 u32 reg;
97
98 reg = (min(vcpu->kvm->arch.vgic.nr_irqs, 1024) >> 5) - 1;
99
100 reg |= (INTERRUPT_ID_BITS - 1) << 19;
101
102 vgic_reg_access(mmio, &reg, offset,
103 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
104
105 return false;
106}
107
108static bool handle_mmio_iidr(struct kvm_vcpu *vcpu,
109 struct kvm_exit_mmio *mmio, phys_addr_t offset)
110{
111 u32 reg;
112
113 reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
114 vgic_reg_access(mmio, &reg, offset,
115 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
116
117 return false;
118}
119
120static bool handle_mmio_set_enable_reg_dist(struct kvm_vcpu *vcpu,
121 struct kvm_exit_mmio *mmio,
122 phys_addr_t offset)
123{
124 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
125 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
126 vcpu->vcpu_id,
127 ACCESS_WRITE_SETBIT);
128
129 vgic_reg_access(mmio, NULL, offset,
130 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
131 return false;
132}
133
134static bool handle_mmio_clear_enable_reg_dist(struct kvm_vcpu *vcpu,
135 struct kvm_exit_mmio *mmio,
136 phys_addr_t offset)
137{
138 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
139 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
140 vcpu->vcpu_id,
141 ACCESS_WRITE_CLEARBIT);
142
143 vgic_reg_access(mmio, NULL, offset,
144 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
145 return false;
146}
147
148static bool handle_mmio_set_pending_reg_dist(struct kvm_vcpu *vcpu,
149 struct kvm_exit_mmio *mmio,
150 phys_addr_t offset)
151{
152 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
153 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
154 vcpu->vcpu_id);
155
156 vgic_reg_access(mmio, NULL, offset,
157 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
158 return false;
159}
160
161static bool handle_mmio_clear_pending_reg_dist(struct kvm_vcpu *vcpu,
162 struct kvm_exit_mmio *mmio,
163 phys_addr_t offset)
164{
165 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
166 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
167 vcpu->vcpu_id);
168
169 vgic_reg_access(mmio, NULL, offset,
170 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
171 return false;
172}
173
174static bool handle_mmio_set_active_reg_dist(struct kvm_vcpu *vcpu,
175 struct kvm_exit_mmio *mmio,
176 phys_addr_t offset)
177{
178 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
179 return vgic_handle_set_active_reg(vcpu->kvm, mmio, offset,
180 vcpu->vcpu_id);
181
182 vgic_reg_access(mmio, NULL, offset,
183 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
184 return false;
185}
186
187static bool handle_mmio_clear_active_reg_dist(struct kvm_vcpu *vcpu,
188 struct kvm_exit_mmio *mmio,
189 phys_addr_t offset)
190{
191 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
192 return vgic_handle_clear_active_reg(vcpu->kvm, mmio, offset,
193 vcpu->vcpu_id);
194
195 vgic_reg_access(mmio, NULL, offset,
196 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
197 return false;
198}
199
200static bool handle_mmio_priority_reg_dist(struct kvm_vcpu *vcpu,
201 struct kvm_exit_mmio *mmio,
202 phys_addr_t offset)
203{
204 u32 *reg;
205
206 if (unlikely(offset < VGIC_NR_PRIVATE_IRQS)) {
207 vgic_reg_access(mmio, NULL, offset,
208 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
209 return false;
210 }
211
212 reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
213 vcpu->vcpu_id, offset);
214 vgic_reg_access(mmio, reg, offset,
215 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
216 return false;
217}
218
219static bool handle_mmio_cfg_reg_dist(struct kvm_vcpu *vcpu,
220 struct kvm_exit_mmio *mmio,
221 phys_addr_t offset)
222{
223 u32 *reg;
224
225 if (unlikely(offset < VGIC_NR_PRIVATE_IRQS / 4)) {
226 vgic_reg_access(mmio, NULL, offset,
227 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
228 return false;
229 }
230
231 reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
232 vcpu->vcpu_id, offset >> 1);
233
234 return vgic_handle_cfg_reg(reg, mmio, offset);
235}
236
237/*
238 * We use a compressed version of the MPIDR (all 32 bits in one 32-bit word)
239 * when we store the target MPIDR written by the guest.
240 */
241static u32 compress_mpidr(unsigned long mpidr)
242{
243 u32 ret;
244
245 ret = MPIDR_AFFINITY_LEVEL(mpidr, 0);
246 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8;
247 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16;
248 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24;
249
250 return ret;
251}
252
253static unsigned long uncompress_mpidr(u32 value)
254{
255 unsigned long mpidr;
256
257 mpidr = ((value >> 0) & 0xFF) << MPIDR_LEVEL_SHIFT(0);
258 mpidr |= ((value >> 8) & 0xFF) << MPIDR_LEVEL_SHIFT(1);
259 mpidr |= ((value >> 16) & 0xFF) << MPIDR_LEVEL_SHIFT(2);
260 mpidr |= (u64)((value >> 24) & 0xFF) << MPIDR_LEVEL_SHIFT(3);
261
262 return mpidr;
263}
264
265/*
266 * Lookup the given MPIDR value to get the vcpu_id (if there is one)
267 * and store that in the irq_spi_cpu[] array.
268 * This limits the number of VCPUs to 255 for now, extending the data
269 * type (or storing kvm_vcpu pointers) should lift the limit.
270 * Store the original MPIDR value in an extra array to support read-as-written.
271 * Unallocated MPIDRs are translated to a special value and caught
272 * before any array accesses.
273 */
274static bool handle_mmio_route_reg(struct kvm_vcpu *vcpu,
275 struct kvm_exit_mmio *mmio,
276 phys_addr_t offset)
277{
278 struct kvm *kvm = vcpu->kvm;
279 struct vgic_dist *dist = &kvm->arch.vgic;
280 int spi;
281 u32 reg;
282 int vcpu_id;
283 unsigned long *bmap, mpidr;
284
285 /*
286 * The upper 32 bits of each 64 bit register are zero,
287 * as we don't support Aff3.
288 */
289 if ((offset & 4)) {
290 vgic_reg_access(mmio, NULL, offset,
291 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
292 return false;
293 }
294
295 /* This region only covers SPIs, so no handling of private IRQs here. */
296 spi = offset / 8;
297
298 /* get the stored MPIDR for this IRQ */
299 mpidr = uncompress_mpidr(dist->irq_spi_mpidr[spi]);
300 reg = mpidr;
301
302 vgic_reg_access(mmio, &reg, offset,
303 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
304
305 if (!mmio->is_write)
306 return false;
307
308 /*
309 * Now clear the currently assigned vCPU from the map, making room
310 * for the new one to be written below
311 */
312 vcpu = kvm_mpidr_to_vcpu(kvm, mpidr);
313 if (likely(vcpu)) {
314 vcpu_id = vcpu->vcpu_id;
315 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]);
316 __clear_bit(spi, bmap);
317 }
318
319 dist->irq_spi_mpidr[spi] = compress_mpidr(reg);
320 vcpu = kvm_mpidr_to_vcpu(kvm, reg & MPIDR_HWID_BITMASK);
321
322 /*
323 * The spec says that non-existent MPIDR values should not be
324 * forwarded to any existent (v)CPU, but should be able to become
325 * pending anyway. We simply keep the irq_spi_target[] array empty, so
326 * the interrupt will never be injected.
327 * irq_spi_cpu[irq] gets a magic value in this case.
328 */
329 if (likely(vcpu)) {
330 vcpu_id = vcpu->vcpu_id;
331 dist->irq_spi_cpu[spi] = vcpu_id;
332 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]);
333 __set_bit(spi, bmap);
334 } else {
335 dist->irq_spi_cpu[spi] = VCPU_NOT_ALLOCATED;
336 }
337
338 vgic_update_state(kvm);
339
340 return true;
341}
342
343/*
344 * We should be careful about promising too much when a guest reads
345 * this register. Don't claim to be like any hardware implementation,
346 * but just report the GIC as version 3 - which is what a Linux guest
347 * would check.
348 */
349static bool handle_mmio_idregs(struct kvm_vcpu *vcpu,
350 struct kvm_exit_mmio *mmio,
351 phys_addr_t offset)
352{
353 u32 reg = 0;
354
355 switch (offset + GICD_IDREGS) {
356 case GICD_PIDR2:
357 reg = 0x3b;
358 break;
359 }
360
361 vgic_reg_access(mmio, &reg, offset,
362 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
363
364 return false;
365}
366
367static const struct vgic_io_range vgic_v3_dist_ranges[] = {
368 {
369 .base = GICD_CTLR,
370 .len = 0x04,
371 .bits_per_irq = 0,
372 .handle_mmio = handle_mmio_ctlr,
373 },
374 {
375 .base = GICD_TYPER,
376 .len = 0x04,
377 .bits_per_irq = 0,
378 .handle_mmio = handle_mmio_typer,
379 },
380 {
381 .base = GICD_IIDR,
382 .len = 0x04,
383 .bits_per_irq = 0,
384 .handle_mmio = handle_mmio_iidr,
385 },
386 {
387 /* this register is optional, it is RAZ/WI if not implemented */
388 .base = GICD_STATUSR,
389 .len = 0x04,
390 .bits_per_irq = 0,
391 .handle_mmio = handle_mmio_raz_wi,
392 },
393 {
394 /* this write only register is WI when TYPER.MBIS=0 */
395 .base = GICD_SETSPI_NSR,
396 .len = 0x04,
397 .bits_per_irq = 0,
398 .handle_mmio = handle_mmio_raz_wi,
399 },
400 {
401 /* this write only register is WI when TYPER.MBIS=0 */
402 .base = GICD_CLRSPI_NSR,
403 .len = 0x04,
404 .bits_per_irq = 0,
405 .handle_mmio = handle_mmio_raz_wi,
406 },
407 {
408 /* this is RAZ/WI when DS=1 */
409 .base = GICD_SETSPI_SR,
410 .len = 0x04,
411 .bits_per_irq = 0,
412 .handle_mmio = handle_mmio_raz_wi,
413 },
414 {
415 /* this is RAZ/WI when DS=1 */
416 .base = GICD_CLRSPI_SR,
417 .len = 0x04,
418 .bits_per_irq = 0,
419 .handle_mmio = handle_mmio_raz_wi,
420 },
421 {
422 .base = GICD_IGROUPR,
423 .len = 0x80,
424 .bits_per_irq = 1,
425 .handle_mmio = handle_mmio_rao_wi,
426 },
427 {
428 .base = GICD_ISENABLER,
429 .len = 0x80,
430 .bits_per_irq = 1,
431 .handle_mmio = handle_mmio_set_enable_reg_dist,
432 },
433 {
434 .base = GICD_ICENABLER,
435 .len = 0x80,
436 .bits_per_irq = 1,
437 .handle_mmio = handle_mmio_clear_enable_reg_dist,
438 },
439 {
440 .base = GICD_ISPENDR,
441 .len = 0x80,
442 .bits_per_irq = 1,
443 .handle_mmio = handle_mmio_set_pending_reg_dist,
444 },
445 {
446 .base = GICD_ICPENDR,
447 .len = 0x80,
448 .bits_per_irq = 1,
449 .handle_mmio = handle_mmio_clear_pending_reg_dist,
450 },
451 {
452 .base = GICD_ISACTIVER,
453 .len = 0x80,
454 .bits_per_irq = 1,
455 .handle_mmio = handle_mmio_set_active_reg_dist,
456 },
457 {
458 .base = GICD_ICACTIVER,
459 .len = 0x80,
460 .bits_per_irq = 1,
461 .handle_mmio = handle_mmio_clear_active_reg_dist,
462 },
463 {
464 .base = GICD_IPRIORITYR,
465 .len = 0x400,
466 .bits_per_irq = 8,
467 .handle_mmio = handle_mmio_priority_reg_dist,
468 },
469 {
470 /* TARGETSRn is RES0 when ARE=1 */
471 .base = GICD_ITARGETSR,
472 .len = 0x400,
473 .bits_per_irq = 8,
474 .handle_mmio = handle_mmio_raz_wi,
475 },
476 {
477 .base = GICD_ICFGR,
478 .len = 0x100,
479 .bits_per_irq = 2,
480 .handle_mmio = handle_mmio_cfg_reg_dist,
481 },
482 {
483 /* this is RAZ/WI when DS=1 */
484 .base = GICD_IGRPMODR,
485 .len = 0x80,
486 .bits_per_irq = 1,
487 .handle_mmio = handle_mmio_raz_wi,
488 },
489 {
490 /* this is RAZ/WI when DS=1 */
491 .base = GICD_NSACR,
492 .len = 0x100,
493 .bits_per_irq = 2,
494 .handle_mmio = handle_mmio_raz_wi,
495 },
496 {
497 /* this is RAZ/WI when ARE=1 */
498 .base = GICD_SGIR,
499 .len = 0x04,
500 .handle_mmio = handle_mmio_raz_wi,
501 },
502 {
503 /* this is RAZ/WI when ARE=1 */
504 .base = GICD_CPENDSGIR,
505 .len = 0x10,
506 .handle_mmio = handle_mmio_raz_wi,
507 },
508 {
509 /* this is RAZ/WI when ARE=1 */
510 .base = GICD_SPENDSGIR,
511 .len = 0x10,
512 .handle_mmio = handle_mmio_raz_wi,
513 },
514 {
515 .base = GICD_IROUTER + 0x100,
516 .len = 0x1ee0,
517 .bits_per_irq = 64,
518 .handle_mmio = handle_mmio_route_reg,
519 },
520 {
521 .base = GICD_IDREGS,
522 .len = 0x30,
523 .bits_per_irq = 0,
524 .handle_mmio = handle_mmio_idregs,
525 },
526 {},
527};
528
529static bool handle_mmio_ctlr_redist(struct kvm_vcpu *vcpu,
530 struct kvm_exit_mmio *mmio,
531 phys_addr_t offset)
532{
533 /* since we don't support LPIs, this register is zero for now */
534 vgic_reg_access(mmio, NULL, offset,
535 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
536 return false;
537}
538
539static bool handle_mmio_typer_redist(struct kvm_vcpu *vcpu,
540 struct kvm_exit_mmio *mmio,
541 phys_addr_t offset)
542{
543 u32 reg;
544 u64 mpidr;
545 struct kvm_vcpu *redist_vcpu = mmio->private;
546 int target_vcpu_id = redist_vcpu->vcpu_id;
547
548 /* the upper 32 bits contain the affinity value */
549 if ((offset & ~3) == 4) {
550 mpidr = kvm_vcpu_get_mpidr_aff(redist_vcpu);
551 reg = compress_mpidr(mpidr);
552
553 vgic_reg_access(mmio, &reg, offset,
554 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
555 return false;
556 }
557
558 reg = redist_vcpu->vcpu_id << 8;
559 if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1)
560 reg |= GICR_TYPER_LAST;
561 vgic_reg_access(mmio, &reg, offset,
562 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
563 return false;
564}
565
566static bool handle_mmio_set_enable_reg_redist(struct kvm_vcpu *vcpu,
567 struct kvm_exit_mmio *mmio,
568 phys_addr_t offset)
569{
570 struct kvm_vcpu *redist_vcpu = mmio->private;
571
572 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
573 redist_vcpu->vcpu_id,
574 ACCESS_WRITE_SETBIT);
575}
576
577static bool handle_mmio_clear_enable_reg_redist(struct kvm_vcpu *vcpu,
578 struct kvm_exit_mmio *mmio,
579 phys_addr_t offset)
580{
581 struct kvm_vcpu *redist_vcpu = mmio->private;
582
583 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
584 redist_vcpu->vcpu_id,
585 ACCESS_WRITE_CLEARBIT);
586}
587
588static bool handle_mmio_set_active_reg_redist(struct kvm_vcpu *vcpu,
589 struct kvm_exit_mmio *mmio,
590 phys_addr_t offset)
591{
592 struct kvm_vcpu *redist_vcpu = mmio->private;
593
594 return vgic_handle_set_active_reg(vcpu->kvm, mmio, offset,
595 redist_vcpu->vcpu_id);
596}
597
598static bool handle_mmio_clear_active_reg_redist(struct kvm_vcpu *vcpu,
599 struct kvm_exit_mmio *mmio,
600 phys_addr_t offset)
601{
602 struct kvm_vcpu *redist_vcpu = mmio->private;
603
604 return vgic_handle_clear_active_reg(vcpu->kvm, mmio, offset,
605 redist_vcpu->vcpu_id);
606}
607
608static bool handle_mmio_set_pending_reg_redist(struct kvm_vcpu *vcpu,
609 struct kvm_exit_mmio *mmio,
610 phys_addr_t offset)
611{
612 struct kvm_vcpu *redist_vcpu = mmio->private;
613
614 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
615 redist_vcpu->vcpu_id);
616}
617
618static bool handle_mmio_clear_pending_reg_redist(struct kvm_vcpu *vcpu,
619 struct kvm_exit_mmio *mmio,
620 phys_addr_t offset)
621{
622 struct kvm_vcpu *redist_vcpu = mmio->private;
623
624 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
625 redist_vcpu->vcpu_id);
626}
627
628static bool handle_mmio_priority_reg_redist(struct kvm_vcpu *vcpu,
629 struct kvm_exit_mmio *mmio,
630 phys_addr_t offset)
631{
632 struct kvm_vcpu *redist_vcpu = mmio->private;
633 u32 *reg;
634
635 reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
636 redist_vcpu->vcpu_id, offset);
637 vgic_reg_access(mmio, reg, offset,
638 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
639 return false;
640}
641
642static bool handle_mmio_cfg_reg_redist(struct kvm_vcpu *vcpu,
643 struct kvm_exit_mmio *mmio,
644 phys_addr_t offset)
645{
646 struct kvm_vcpu *redist_vcpu = mmio->private;
647
648 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
649 redist_vcpu->vcpu_id, offset >> 1);
650
651 return vgic_handle_cfg_reg(reg, mmio, offset);
652}
653
654#define SGI_base(x) ((x) + SZ_64K)
655
656static const struct vgic_io_range vgic_redist_ranges[] = {
657 {
658 .base = GICR_CTLR,
659 .len = 0x04,
660 .bits_per_irq = 0,
661 .handle_mmio = handle_mmio_ctlr_redist,
662 },
663 {
664 .base = GICR_TYPER,
665 .len = 0x08,
666 .bits_per_irq = 0,
667 .handle_mmio = handle_mmio_typer_redist,
668 },
669 {
670 .base = GICR_IIDR,
671 .len = 0x04,
672 .bits_per_irq = 0,
673 .handle_mmio = handle_mmio_iidr,
674 },
675 {
676 .base = GICR_WAKER,
677 .len = 0x04,
678 .bits_per_irq = 0,
679 .handle_mmio = handle_mmio_raz_wi,
680 },
681 {
682 .base = GICR_IDREGS,
683 .len = 0x30,
684 .bits_per_irq = 0,
685 .handle_mmio = handle_mmio_idregs,
686 },
687 {
688 .base = SGI_base(GICR_IGROUPR0),
689 .len = 0x04,
690 .bits_per_irq = 1,
691 .handle_mmio = handle_mmio_rao_wi,
692 },
693 {
694 .base = SGI_base(GICR_ISENABLER0),
695 .len = 0x04,
696 .bits_per_irq = 1,
697 .handle_mmio = handle_mmio_set_enable_reg_redist,
698 },
699 {
700 .base = SGI_base(GICR_ICENABLER0),
701 .len = 0x04,
702 .bits_per_irq = 1,
703 .handle_mmio = handle_mmio_clear_enable_reg_redist,
704 },
705 {
706 .base = SGI_base(GICR_ISPENDR0),
707 .len = 0x04,
708 .bits_per_irq = 1,
709 .handle_mmio = handle_mmio_set_pending_reg_redist,
710 },
711 {
712 .base = SGI_base(GICR_ICPENDR0),
713 .len = 0x04,
714 .bits_per_irq = 1,
715 .handle_mmio = handle_mmio_clear_pending_reg_redist,
716 },
717 {
718 .base = SGI_base(GICR_ISACTIVER0),
719 .len = 0x04,
720 .bits_per_irq = 1,
721 .handle_mmio = handle_mmio_set_active_reg_redist,
722 },
723 {
724 .base = SGI_base(GICR_ICACTIVER0),
725 .len = 0x04,
726 .bits_per_irq = 1,
727 .handle_mmio = handle_mmio_clear_active_reg_redist,
728 },
729 {
730 .base = SGI_base(GICR_IPRIORITYR0),
731 .len = 0x20,
732 .bits_per_irq = 8,
733 .handle_mmio = handle_mmio_priority_reg_redist,
734 },
735 {
736 .base = SGI_base(GICR_ICFGR0),
737 .len = 0x08,
738 .bits_per_irq = 2,
739 .handle_mmio = handle_mmio_cfg_reg_redist,
740 },
741 {
742 .base = SGI_base(GICR_IGRPMODR0),
743 .len = 0x04,
744 .bits_per_irq = 1,
745 .handle_mmio = handle_mmio_raz_wi,
746 },
747 {
748 .base = SGI_base(GICR_NSACR),
749 .len = 0x04,
750 .handle_mmio = handle_mmio_raz_wi,
751 },
752 {},
753};
754
755static bool vgic_v3_queue_sgi(struct kvm_vcpu *vcpu, int irq)
756{
757 if (vgic_queue_irq(vcpu, 0, irq)) {
758 vgic_dist_irq_clear_pending(vcpu, irq);
759 vgic_cpu_irq_clear(vcpu, irq);
760 return true;
761 }
762
763 return false;
764}
765
766static int vgic_v3_map_resources(struct kvm *kvm,
767 const struct vgic_params *params)
768{
769 int ret = 0;
770 struct vgic_dist *dist = &kvm->arch.vgic;
771 gpa_t rdbase = dist->vgic_redist_base;
772 struct vgic_io_device *iodevs = NULL;
773 int i;
774
775 if (!irqchip_in_kernel(kvm))
776 return 0;
777
778 mutex_lock(&kvm->lock);
779
780 if (vgic_ready(kvm))
781 goto out;
782
783 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
784 IS_VGIC_ADDR_UNDEF(dist->vgic_redist_base)) {
785 kvm_err("Need to set vgic distributor addresses first\n");
786 ret = -ENXIO;
787 goto out;
788 }
789
790 /*
791 * For a VGICv3 we require the userland to explicitly initialize
792 * the VGIC before we need to use it.
793 */
794 if (!vgic_initialized(kvm)) {
795 ret = -EBUSY;
796 goto out;
797 }
798
799 ret = vgic_register_kvm_io_dev(kvm, dist->vgic_dist_base,
800 GIC_V3_DIST_SIZE, vgic_v3_dist_ranges,
801 -1, &dist->dist_iodev);
802 if (ret)
803 goto out;
804
805 iodevs = kcalloc(dist->nr_cpus, sizeof(iodevs[0]), GFP_KERNEL);
806 if (!iodevs) {
807 ret = -ENOMEM;
808 goto out_unregister;
809 }
810
811 for (i = 0; i < dist->nr_cpus; i++) {
812 ret = vgic_register_kvm_io_dev(kvm, rdbase,
813 SZ_128K, vgic_redist_ranges,
814 i, &iodevs[i]);
815 if (ret)
816 goto out_unregister;
817 rdbase += GIC_V3_REDIST_SIZE;
818 }
819
820 dist->redist_iodevs = iodevs;
821 dist->ready = true;
822 goto out;
823
824out_unregister:
825 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &dist->dist_iodev.dev);
826 if (iodevs) {
827 for (i = 0; i < dist->nr_cpus; i++) {
828 if (iodevs[i].dev.ops)
829 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
830 &iodevs[i].dev);
831 }
832 }
833
834out:
835 if (ret)
836 kvm_vgic_destroy(kvm);
837 mutex_unlock(&kvm->lock);
838 return ret;
839}
840
841static int vgic_v3_init_model(struct kvm *kvm)
842{
843 int i;
844 u32 mpidr;
845 struct vgic_dist *dist = &kvm->arch.vgic;
846 int nr_spis = dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
847
848 dist->irq_spi_mpidr = kcalloc(nr_spis, sizeof(dist->irq_spi_mpidr[0]),
849 GFP_KERNEL);
850
851 if (!dist->irq_spi_mpidr)
852 return -ENOMEM;
853
854 /* Initialize the target VCPUs for each IRQ to VCPU 0 */
855 mpidr = compress_mpidr(kvm_vcpu_get_mpidr_aff(kvm_get_vcpu(kvm, 0)));
856 for (i = VGIC_NR_PRIVATE_IRQS; i < dist->nr_irqs; i++) {
857 dist->irq_spi_cpu[i - VGIC_NR_PRIVATE_IRQS] = 0;
858 dist->irq_spi_mpidr[i - VGIC_NR_PRIVATE_IRQS] = mpidr;
859 vgic_bitmap_set_irq_val(dist->irq_spi_target, 0, i, 1);
860 }
861
862 return 0;
863}
864
865/* GICv3 does not keep track of SGI sources anymore. */
866static void vgic_v3_add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
867{
868}
869
870void vgic_v3_init_emulation(struct kvm *kvm)
871{
872 struct vgic_dist *dist = &kvm->arch.vgic;
873
874 dist->vm_ops.queue_sgi = vgic_v3_queue_sgi;
875 dist->vm_ops.add_sgi_source = vgic_v3_add_sgi_source;
876 dist->vm_ops.init_model = vgic_v3_init_model;
877 dist->vm_ops.map_resources = vgic_v3_map_resources;
878
879 kvm->arch.max_vcpus = KVM_MAX_VCPUS;
880}
881
882/*
883 * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI
884 * generation register ICC_SGI1R_EL1) with a given VCPU.
885 * If the VCPU's MPIDR matches, return the level0 affinity, otherwise
886 * return -1.
887 */
888static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu)
889{
890 unsigned long affinity;
891 int level0;
892
893 /*
894 * Split the current VCPU's MPIDR into affinity level 0 and the
895 * rest as this is what we have to compare against.
896 */
897 affinity = kvm_vcpu_get_mpidr_aff(vcpu);
898 level0 = MPIDR_AFFINITY_LEVEL(affinity, 0);
899 affinity &= ~MPIDR_LEVEL_MASK;
900
901 /* bail out if the upper three levels don't match */
902 if (sgi_aff != affinity)
903 return -1;
904
905 /* Is this VCPU's bit set in the mask ? */
906 if (!(sgi_cpu_mask & BIT(level0)))
907 return -1;
908
909 return level0;
910}
911
912#define SGI_AFFINITY_LEVEL(reg, level) \
913 ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \
914 >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
915
916/**
917 * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs
918 * @vcpu: The VCPU requesting a SGI
919 * @reg: The value written into the ICC_SGI1R_EL1 register by that VCPU
920 *
921 * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register.
922 * This will trap in sys_regs.c and call this function.
923 * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the
924 * target processors as well as a bitmask of 16 Aff0 CPUs.
925 * If the interrupt routing mode bit is not set, we iterate over all VCPUs to
926 * check for matching ones. If this bit is set, we signal all, but not the
927 * calling VCPU.
928 */
929void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg)
930{
931 struct kvm *kvm = vcpu->kvm;
932 struct kvm_vcpu *c_vcpu;
933 struct vgic_dist *dist = &kvm->arch.vgic;
934 u16 target_cpus;
935 u64 mpidr;
936 int sgi, c;
937 int vcpu_id = vcpu->vcpu_id;
938 bool broadcast;
939 int updated = 0;
940
941 sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT;
942 broadcast = reg & BIT(ICC_SGI1R_IRQ_ROUTING_MODE_BIT);
943 target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT;
944 mpidr = SGI_AFFINITY_LEVEL(reg, 3);
945 mpidr |= SGI_AFFINITY_LEVEL(reg, 2);
946 mpidr |= SGI_AFFINITY_LEVEL(reg, 1);
947
948 /*
949 * We take the dist lock here, because we come from the sysregs
950 * code path and not from the MMIO one (which already takes the lock).
951 */
952 spin_lock(&dist->lock);
953
954 /*
955 * We iterate over all VCPUs to find the MPIDRs matching the request.
956 * If we have handled one CPU, we clear it's bit to detect early
957 * if we are already finished. This avoids iterating through all
958 * VCPUs when most of the times we just signal a single VCPU.
959 */
960 kvm_for_each_vcpu(c, c_vcpu, kvm) {
961
962 /* Exit early if we have dealt with all requested CPUs */
963 if (!broadcast && target_cpus == 0)
964 break;
965
966 /* Don't signal the calling VCPU */
967 if (broadcast && c == vcpu_id)
968 continue;
969
970 if (!broadcast) {
971 int level0;
972
973 level0 = match_mpidr(mpidr, target_cpus, c_vcpu);
974 if (level0 == -1)
975 continue;
976
977 /* remove this matching VCPU from the mask */
978 target_cpus &= ~BIT(level0);
979 }
980
981 /* Flag the SGI as pending */
982 vgic_dist_irq_set_pending(c_vcpu, sgi);
983 updated = 1;
984 kvm_debug("SGI%d from CPU%d to CPU%d\n", sgi, vcpu_id, c);
985 }
986 if (updated)
987 vgic_update_state(vcpu->kvm);
988 spin_unlock(&dist->lock);
989 if (updated)
990 vgic_kick_vcpus(vcpu->kvm);
991}
992
993static int vgic_v3_create(struct kvm_device *dev, u32 type)
994{
995 return kvm_vgic_create(dev->kvm, type);
996}
997
998static void vgic_v3_destroy(struct kvm_device *dev)
999{
1000 kfree(dev);
1001}
1002
1003static int vgic_v3_set_attr(struct kvm_device *dev,
1004 struct kvm_device_attr *attr)
1005{
1006 int ret;
1007
1008 ret = vgic_set_common_attr(dev, attr);
1009 if (ret != -ENXIO)
1010 return ret;
1011
1012 switch (attr->group) {
1013 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
1014 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
1015 return -ENXIO;
1016 }
1017
1018 return -ENXIO;
1019}
1020
1021static int vgic_v3_get_attr(struct kvm_device *dev,
1022 struct kvm_device_attr *attr)
1023{
1024 int ret;
1025
1026 ret = vgic_get_common_attr(dev, attr);
1027 if (ret != -ENXIO)
1028 return ret;
1029
1030 switch (attr->group) {
1031 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
1032 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
1033 return -ENXIO;
1034 }
1035
1036 return -ENXIO;
1037}
1038
1039static int vgic_v3_has_attr(struct kvm_device *dev,
1040 struct kvm_device_attr *attr)
1041{
1042 switch (attr->group) {
1043 case KVM_DEV_ARM_VGIC_GRP_ADDR:
1044 switch (attr->attr) {
1045 case KVM_VGIC_V2_ADDR_TYPE_DIST:
1046 case KVM_VGIC_V2_ADDR_TYPE_CPU:
1047 return -ENXIO;
1048 case KVM_VGIC_V3_ADDR_TYPE_DIST:
1049 case KVM_VGIC_V3_ADDR_TYPE_REDIST:
1050 return 0;
1051 }
1052 break;
1053 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
1054 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
1055 return -ENXIO;
1056 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
1057 return 0;
1058 case KVM_DEV_ARM_VGIC_GRP_CTRL:
1059 switch (attr->attr) {
1060 case KVM_DEV_ARM_VGIC_CTRL_INIT:
1061 return 0;
1062 }
1063 }
1064 return -ENXIO;
1065}
1066
1067struct kvm_device_ops kvm_arm_vgic_v3_ops = {
1068 .name = "kvm-arm-vgic-v3",
1069 .create = vgic_v3_create,
1070 .destroy = vgic_v3_destroy,
1071 .set_attr = vgic_v3_set_attr,
1072 .get_attr = vgic_v3_get_attr,
1073 .has_attr = vgic_v3_has_attr,
1074};
diff --git a/virt/kvm/arm/vgic-v3.c b/virt/kvm/arm/vgic-v3.c
deleted file mode 100644
index 75b02fa86436..000000000000
--- a/virt/kvm/arm/vgic-v3.c
+++ /dev/null
@@ -1,279 +0,0 @@
1/*
2 * Copyright (C) 2013 ARM Limited, All Rights Reserved.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/cpu.h>
19#include <linux/kvm.h>
20#include <linux/kvm_host.h>
21#include <linux/interrupt.h>
22#include <linux/io.h>
23
24#include <linux/irqchip/arm-gic-v3.h>
25#include <linux/irqchip/arm-gic-common.h>
26
27#include <asm/kvm_emulate.h>
28#include <asm/kvm_arm.h>
29#include <asm/kvm_asm.h>
30#include <asm/kvm_mmu.h>
31
32static u32 ich_vtr_el2;
33
34static struct vgic_lr vgic_v3_get_lr(const struct kvm_vcpu *vcpu, int lr)
35{
36 struct vgic_lr lr_desc;
37 u64 val = vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr];
38
39 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
40 lr_desc.irq = val & ICH_LR_VIRTUAL_ID_MASK;
41 else
42 lr_desc.irq = val & GICH_LR_VIRTUALID;
43
44 lr_desc.source = 0;
45 if (lr_desc.irq <= 15 &&
46 vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
47 lr_desc.source = (val >> GICH_LR_PHYSID_CPUID_SHIFT) & 0x7;
48
49 lr_desc.state = 0;
50
51 if (val & ICH_LR_PENDING_BIT)
52 lr_desc.state |= LR_STATE_PENDING;
53 if (val & ICH_LR_ACTIVE_BIT)
54 lr_desc.state |= LR_STATE_ACTIVE;
55 if (val & ICH_LR_EOI)
56 lr_desc.state |= LR_EOI_INT;
57 if (val & ICH_LR_HW) {
58 lr_desc.state |= LR_HW;
59 lr_desc.hwirq = (val >> ICH_LR_PHYS_ID_SHIFT) & GENMASK(9, 0);
60 }
61
62 return lr_desc;
63}
64
65static void vgic_v3_set_lr(struct kvm_vcpu *vcpu, int lr,
66 struct vgic_lr lr_desc)
67{
68 u64 lr_val;
69
70 lr_val = lr_desc.irq;
71
72 /*
73 * Currently all guest IRQs are Group1, as Group0 would result
74 * in a FIQ in the guest, which it wouldn't expect.
75 * Eventually we want to make this configurable, so we may revisit
76 * this in the future.
77 */
78 switch (vcpu->kvm->arch.vgic.vgic_model) {
79 case KVM_DEV_TYPE_ARM_VGIC_V3:
80 lr_val |= ICH_LR_GROUP;
81 break;
82 case KVM_DEV_TYPE_ARM_VGIC_V2:
83 if (lr_desc.irq < VGIC_NR_SGIS)
84 lr_val |= (u32)lr_desc.source << GICH_LR_PHYSID_CPUID_SHIFT;
85 break;
86 default:
87 BUG();
88 }
89
90 if (lr_desc.state & LR_STATE_PENDING)
91 lr_val |= ICH_LR_PENDING_BIT;
92 if (lr_desc.state & LR_STATE_ACTIVE)
93 lr_val |= ICH_LR_ACTIVE_BIT;
94 if (lr_desc.state & LR_EOI_INT)
95 lr_val |= ICH_LR_EOI;
96 if (lr_desc.state & LR_HW) {
97 lr_val |= ICH_LR_HW;
98 lr_val |= ((u64)lr_desc.hwirq) << ICH_LR_PHYS_ID_SHIFT;
99 }
100
101 vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = lr_val;
102
103 if (!(lr_desc.state & LR_STATE_MASK))
104 vcpu->arch.vgic_cpu.vgic_v3.vgic_elrsr |= (1U << lr);
105 else
106 vcpu->arch.vgic_cpu.vgic_v3.vgic_elrsr &= ~(1U << lr);
107}
108
109static u64 vgic_v3_get_elrsr(const struct kvm_vcpu *vcpu)
110{
111 return vcpu->arch.vgic_cpu.vgic_v3.vgic_elrsr;
112}
113
114static u64 vgic_v3_get_eisr(const struct kvm_vcpu *vcpu)
115{
116 return vcpu->arch.vgic_cpu.vgic_v3.vgic_eisr;
117}
118
119static void vgic_v3_clear_eisr(struct kvm_vcpu *vcpu)
120{
121 vcpu->arch.vgic_cpu.vgic_v3.vgic_eisr = 0;
122}
123
124static u32 vgic_v3_get_interrupt_status(const struct kvm_vcpu *vcpu)
125{
126 u32 misr = vcpu->arch.vgic_cpu.vgic_v3.vgic_misr;
127 u32 ret = 0;
128
129 if (misr & ICH_MISR_EOI)
130 ret |= INT_STATUS_EOI;
131 if (misr & ICH_MISR_U)
132 ret |= INT_STATUS_UNDERFLOW;
133
134 return ret;
135}
136
137static void vgic_v3_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
138{
139 u32 vmcr = vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr;
140
141 vmcrp->ctlr = (vmcr & ICH_VMCR_CTLR_MASK) >> ICH_VMCR_CTLR_SHIFT;
142 vmcrp->abpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
143 vmcrp->bpr = (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
144 vmcrp->pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
145}
146
147static void vgic_v3_enable_underflow(struct kvm_vcpu *vcpu)
148{
149 vcpu->arch.vgic_cpu.vgic_v3.vgic_hcr |= ICH_HCR_UIE;
150}
151
152static void vgic_v3_disable_underflow(struct kvm_vcpu *vcpu)
153{
154 vcpu->arch.vgic_cpu.vgic_v3.vgic_hcr &= ~ICH_HCR_UIE;
155}
156
157static void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
158{
159 u32 vmcr;
160
161 vmcr = (vmcrp->ctlr << ICH_VMCR_CTLR_SHIFT) & ICH_VMCR_CTLR_MASK;
162 vmcr |= (vmcrp->abpr << ICH_VMCR_BPR1_SHIFT) & ICH_VMCR_BPR1_MASK;
163 vmcr |= (vmcrp->bpr << ICH_VMCR_BPR0_SHIFT) & ICH_VMCR_BPR0_MASK;
164 vmcr |= (vmcrp->pmr << ICH_VMCR_PMR_SHIFT) & ICH_VMCR_PMR_MASK;
165
166 vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr = vmcr;
167}
168
169static void vgic_v3_enable(struct kvm_vcpu *vcpu)
170{
171 struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;
172
173 /*
174 * By forcing VMCR to zero, the GIC will restore the binary
175 * points to their reset values. Anything else resets to zero
176 * anyway.
177 */
178 vgic_v3->vgic_vmcr = 0;
179 vgic_v3->vgic_elrsr = ~0;
180
181 /*
182 * If we are emulating a GICv3, we do it in an non-GICv2-compatible
183 * way, so we force SRE to 1 to demonstrate this to the guest.
184 * This goes with the spec allowing the value to be RAO/WI.
185 */
186 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
187 vgic_v3->vgic_sre = ICC_SRE_EL1_SRE;
188 else
189 vgic_v3->vgic_sre = 0;
190
191 /* Get the show on the road... */
192 vgic_v3->vgic_hcr = ICH_HCR_EN;
193}
194
195static const struct vgic_ops vgic_v3_ops = {
196 .get_lr = vgic_v3_get_lr,
197 .set_lr = vgic_v3_set_lr,
198 .get_elrsr = vgic_v3_get_elrsr,
199 .get_eisr = vgic_v3_get_eisr,
200 .clear_eisr = vgic_v3_clear_eisr,
201 .get_interrupt_status = vgic_v3_get_interrupt_status,
202 .enable_underflow = vgic_v3_enable_underflow,
203 .disable_underflow = vgic_v3_disable_underflow,
204 .get_vmcr = vgic_v3_get_vmcr,
205 .set_vmcr = vgic_v3_set_vmcr,
206 .enable = vgic_v3_enable,
207};
208
209static struct vgic_params vgic_v3_params;
210
211static void vgic_cpu_init_lrs(void *params)
212{
213 kvm_call_hyp(__vgic_v3_init_lrs);
214}
215
216/**
217 * vgic_v3_probe - probe for a GICv3 compatible interrupt controller
218 * @gic_kvm_info: pointer to the GIC description
219 * @ops: address of a pointer to the GICv3 operations
220 * @params: address of a pointer to HW-specific parameters
221 *
222 * Returns 0 if a GICv3 has been found, with the low level operations
223 * in *ops and the HW parameters in *params. Returns an error code
224 * otherwise.
225 */
226int vgic_v3_probe(const struct gic_kvm_info *gic_kvm_info,
227 const struct vgic_ops **ops,
228 const struct vgic_params **params)
229{
230 int ret = 0;
231 struct vgic_params *vgic = &vgic_v3_params;
232 const struct resource *vcpu_res = &gic_kvm_info->vcpu;
233
234 vgic->maint_irq = gic_kvm_info->maint_irq;
235
236 ich_vtr_el2 = kvm_call_hyp(__vgic_v3_get_ich_vtr_el2);
237
238 /*
239 * The ListRegs field is 5 bits, but there is a architectural
240 * maximum of 16 list registers. Just ignore bit 4...
241 */
242 vgic->nr_lr = (ich_vtr_el2 & 0xf) + 1;
243 vgic->can_emulate_gicv2 = false;
244
245 if (!vcpu_res->start) {
246 kvm_info("GICv3: no GICV resource entry\n");
247 vgic->vcpu_base = 0;
248 } else if (!PAGE_ALIGNED(vcpu_res->start)) {
249 pr_warn("GICV physical address 0x%llx not page aligned\n",
250 (unsigned long long)vcpu_res->start);
251 vgic->vcpu_base = 0;
252 } else if (!PAGE_ALIGNED(resource_size(vcpu_res))) {
253 pr_warn("GICV size 0x%llx not a multiple of page size 0x%lx\n",
254 (unsigned long long)resource_size(vcpu_res),
255 PAGE_SIZE);
256 } else {
257 vgic->vcpu_base = vcpu_res->start;
258 vgic->can_emulate_gicv2 = true;
259 kvm_register_device_ops(&kvm_arm_vgic_v2_ops,
260 KVM_DEV_TYPE_ARM_VGIC_V2);
261 }
262 if (vgic->vcpu_base == 0)
263 kvm_info("disabling GICv2 emulation\n");
264 kvm_register_device_ops(&kvm_arm_vgic_v3_ops, KVM_DEV_TYPE_ARM_VGIC_V3);
265
266 vgic->vctrl_base = NULL;
267 vgic->type = VGIC_V3;
268 vgic->max_gic_vcpus = VGIC_V3_MAX_CPUS;
269
270 kvm_info("GICV base=0x%llx, IRQ=%d\n",
271 vgic->vcpu_base, vgic->maint_irq);
272
273 on_each_cpu(vgic_cpu_init_lrs, vgic, 1);
274
275 *ops = &vgic_v3_ops;
276 *params = vgic;
277
278 return ret;
279}
diff --git a/virt/kvm/arm/vgic.c b/virt/kvm/arm/vgic.c
deleted file mode 100644
index c3bfbb981e73..000000000000
--- a/virt/kvm/arm/vgic.c
+++ /dev/null
@@ -1,2440 +0,0 @@
1/*
2 * Copyright (C) 2012 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */
18
19#include <linux/cpu.h>
20#include <linux/kvm.h>
21#include <linux/kvm_host.h>
22#include <linux/interrupt.h>
23#include <linux/io.h>
24#include <linux/irq.h>
25#include <linux/rculist.h>
26#include <linux/uaccess.h>
27
28#include <asm/kvm_emulate.h>
29#include <asm/kvm_arm.h>
30#include <asm/kvm_mmu.h>
31#include <trace/events/kvm.h>
32#include <asm/kvm.h>
33#include <kvm/iodev.h>
34#include <linux/irqchip/arm-gic-common.h>
35
36#define CREATE_TRACE_POINTS
37#include "trace.h"
38
39/*
40 * How the whole thing works (courtesy of Christoffer Dall):
41 *
42 * - At any time, the dist->irq_pending_on_cpu is the oracle that knows if
43 * something is pending on the CPU interface.
44 * - Interrupts that are pending on the distributor are stored on the
45 * vgic.irq_pending vgic bitmap (this bitmap is updated by both user land
46 * ioctls and guest mmio ops, and other in-kernel peripherals such as the
47 * arch. timers).
48 * - Every time the bitmap changes, the irq_pending_on_cpu oracle is
49 * recalculated
50 * - To calculate the oracle, we need info for each cpu from
51 * compute_pending_for_cpu, which considers:
52 * - PPI: dist->irq_pending & dist->irq_enable
53 * - SPI: dist->irq_pending & dist->irq_enable & dist->irq_spi_target
54 * - irq_spi_target is a 'formatted' version of the GICD_ITARGETSRn
55 * registers, stored on each vcpu. We only keep one bit of
56 * information per interrupt, making sure that only one vcpu can
57 * accept the interrupt.
58 * - If any of the above state changes, we must recalculate the oracle.
59 * - The same is true when injecting an interrupt, except that we only
60 * consider a single interrupt at a time. The irq_spi_cpu array
61 * contains the target CPU for each SPI.
62 *
63 * The handling of level interrupts adds some extra complexity. We
64 * need to track when the interrupt has been EOIed, so we can sample
65 * the 'line' again. This is achieved as such:
66 *
67 * - When a level interrupt is moved onto a vcpu, the corresponding
68 * bit in irq_queued is set. As long as this bit is set, the line
69 * will be ignored for further interrupts. The interrupt is injected
70 * into the vcpu with the GICH_LR_EOI bit set (generate a
71 * maintenance interrupt on EOI).
72 * - When the interrupt is EOIed, the maintenance interrupt fires,
73 * and clears the corresponding bit in irq_queued. This allows the
74 * interrupt line to be sampled again.
75 * - Note that level-triggered interrupts can also be set to pending from
76 * writes to GICD_ISPENDRn and lowering the external input line does not
77 * cause the interrupt to become inactive in such a situation.
78 * Conversely, writes to GICD_ICPENDRn do not cause the interrupt to become
79 * inactive as long as the external input line is held high.
80 *
81 *
82 * Initialization rules: there are multiple stages to the vgic
83 * initialization, both for the distributor and the CPU interfaces.
84 *
85 * Distributor:
86 *
87 * - kvm_vgic_early_init(): initialization of static data that doesn't
88 * depend on any sizing information or emulation type. No allocation
89 * is allowed there.
90 *
91 * - vgic_init(): allocation and initialization of the generic data
92 * structures that depend on sizing information (number of CPUs,
93 * number of interrupts). Also initializes the vcpu specific data
94 * structures. Can be executed lazily for GICv2.
95 * [to be renamed to kvm_vgic_init??]
96 *
97 * CPU Interface:
98 *
99 * - kvm_vgic_cpu_early_init(): initialization of static data that
100 * doesn't depend on any sizing information or emulation type. No
101 * allocation is allowed there.
102 */
103
104#include "vgic.h"
105
106static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu);
107static void vgic_retire_lr(int lr_nr, struct kvm_vcpu *vcpu);
108static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr);
109static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr, struct vgic_lr lr_desc);
110static u64 vgic_get_elrsr(struct kvm_vcpu *vcpu);
111static struct irq_phys_map *vgic_irq_map_search(struct kvm_vcpu *vcpu,
112 int virt_irq);
113static int compute_pending_for_cpu(struct kvm_vcpu *vcpu);
114
115static const struct vgic_ops *vgic_ops;
116static const struct vgic_params *vgic;
117
118static void add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
119{
120 vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);
121}
122
123static bool queue_sgi(struct kvm_vcpu *vcpu, int irq)
124{
125 return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);
126}
127
128int kvm_vgic_map_resources(struct kvm *kvm)
129{
130 return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);
131}
132
133/*
134 * struct vgic_bitmap contains a bitmap made of unsigned longs, but
135 * extracts u32s out of them.
136 *
137 * This does not work on 64-bit BE systems, because the bitmap access
138 * will store two consecutive 32-bit words with the higher-addressed
139 * register's bits at the lower index and the lower-addressed register's
140 * bits at the higher index.
141 *
142 * Therefore, swizzle the register index when accessing the 32-bit word
143 * registers to access the right register's value.
144 */
145#if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 64
146#define REG_OFFSET_SWIZZLE 1
147#else
148#define REG_OFFSET_SWIZZLE 0
149#endif
150
151static int vgic_init_bitmap(struct vgic_bitmap *b, int nr_cpus, int nr_irqs)
152{
153 int nr_longs;
154
155 nr_longs = nr_cpus + BITS_TO_LONGS(nr_irqs - VGIC_NR_PRIVATE_IRQS);
156
157 b->private = kzalloc(sizeof(unsigned long) * nr_longs, GFP_KERNEL);
158 if (!b->private)
159 return -ENOMEM;
160
161 b->shared = b->private + nr_cpus;
162
163 return 0;
164}
165
166static void vgic_free_bitmap(struct vgic_bitmap *b)
167{
168 kfree(b->private);
169 b->private = NULL;
170 b->shared = NULL;
171}
172
173/*
174 * Call this function to convert a u64 value to an unsigned long * bitmask
175 * in a way that works on both 32-bit and 64-bit LE and BE platforms.
176 *
177 * Warning: Calling this function may modify *val.
178 */
179static unsigned long *u64_to_bitmask(u64 *val)
180{
181#if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 32
182 *val = (*val >> 32) | (*val << 32);
183#endif
184 return (unsigned long *)val;
185}
186
187u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset)
188{
189 offset >>= 2;
190 if (!offset)
191 return (u32 *)(x->private + cpuid) + REG_OFFSET_SWIZZLE;
192 else
193 return (u32 *)(x->shared) + ((offset - 1) ^ REG_OFFSET_SWIZZLE);
194}
195
196static int vgic_bitmap_get_irq_val(struct vgic_bitmap *x,
197 int cpuid, int irq)
198{
199 if (irq < VGIC_NR_PRIVATE_IRQS)
200 return test_bit(irq, x->private + cpuid);
201
202 return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared);
203}
204
205void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
206 int irq, int val)
207{
208 unsigned long *reg;
209
210 if (irq < VGIC_NR_PRIVATE_IRQS) {
211 reg = x->private + cpuid;
212 } else {
213 reg = x->shared;
214 irq -= VGIC_NR_PRIVATE_IRQS;
215 }
216
217 if (val)
218 set_bit(irq, reg);
219 else
220 clear_bit(irq, reg);
221}
222
223static unsigned long *vgic_bitmap_get_cpu_map(struct vgic_bitmap *x, int cpuid)
224{
225 return x->private + cpuid;
226}
227
228unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
229{
230 return x->shared;
231}
232
233static int vgic_init_bytemap(struct vgic_bytemap *x, int nr_cpus, int nr_irqs)
234{
235 int size;
236
237 size = nr_cpus * VGIC_NR_PRIVATE_IRQS;
238 size += nr_irqs - VGIC_NR_PRIVATE_IRQS;
239
240 x->private = kzalloc(size, GFP_KERNEL);
241 if (!x->private)
242 return -ENOMEM;
243
244 x->shared = x->private + nr_cpus * VGIC_NR_PRIVATE_IRQS / sizeof(u32);
245 return 0;
246}
247
248static void vgic_free_bytemap(struct vgic_bytemap *b)
249{
250 kfree(b->private);
251 b->private = NULL;
252 b->shared = NULL;
253}
254
255u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
256{
257 u32 *reg;
258
259 if (offset < VGIC_NR_PRIVATE_IRQS) {
260 reg = x->private;
261 offset += cpuid * VGIC_NR_PRIVATE_IRQS;
262 } else {
263 reg = x->shared;
264 offset -= VGIC_NR_PRIVATE_IRQS;
265 }
266
267 return reg + (offset / sizeof(u32));
268}
269
270#define VGIC_CFG_LEVEL 0
271#define VGIC_CFG_EDGE 1
272
273static bool vgic_irq_is_edge(struct kvm_vcpu *vcpu, int irq)
274{
275 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
276 int irq_val;
277
278 irq_val = vgic_bitmap_get_irq_val(&dist->irq_cfg, vcpu->vcpu_id, irq);
279 return irq_val == VGIC_CFG_EDGE;
280}
281
282static int vgic_irq_is_enabled(struct kvm_vcpu *vcpu, int irq)
283{
284 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
285
286 return vgic_bitmap_get_irq_val(&dist->irq_enabled, vcpu->vcpu_id, irq);
287}
288
289static int vgic_irq_is_queued(struct kvm_vcpu *vcpu, int irq)
290{
291 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
292
293 return vgic_bitmap_get_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq);
294}
295
296static int vgic_irq_is_active(struct kvm_vcpu *vcpu, int irq)
297{
298 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
299
300 return vgic_bitmap_get_irq_val(&dist->irq_active, vcpu->vcpu_id, irq);
301}
302
303static void vgic_irq_set_queued(struct kvm_vcpu *vcpu, int irq)
304{
305 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
306
307 vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 1);
308}
309
310static void vgic_irq_clear_queued(struct kvm_vcpu *vcpu, int irq)
311{
312 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
313
314 vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 0);
315}
316
317static void vgic_irq_set_active(struct kvm_vcpu *vcpu, int irq)
318{
319 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
320
321 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 1);
322}
323
324static void vgic_irq_clear_active(struct kvm_vcpu *vcpu, int irq)
325{
326 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
327
328 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 0);
329}
330
331static int vgic_dist_irq_get_level(struct kvm_vcpu *vcpu, int irq)
332{
333 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
334
335 return vgic_bitmap_get_irq_val(&dist->irq_level, vcpu->vcpu_id, irq);
336}
337
338static void vgic_dist_irq_set_level(struct kvm_vcpu *vcpu, int irq)
339{
340 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
341
342 vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 1);
343}
344
345static void vgic_dist_irq_clear_level(struct kvm_vcpu *vcpu, int irq)
346{
347 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
348
349 vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 0);
350}
351
352static int vgic_dist_irq_soft_pend(struct kvm_vcpu *vcpu, int irq)
353{
354 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
355
356 return vgic_bitmap_get_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq);
357}
358
359static void vgic_dist_irq_clear_soft_pend(struct kvm_vcpu *vcpu, int irq)
360{
361 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
362
363 vgic_bitmap_set_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq, 0);
364 if (!vgic_dist_irq_get_level(vcpu, irq)) {
365 vgic_dist_irq_clear_pending(vcpu, irq);
366 if (!compute_pending_for_cpu(vcpu))
367 clear_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
368 }
369}
370
371static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)
372{
373 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
374
375 return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq);
376}
377
378void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)
379{
380 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
381
382 vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1);
383}
384
385void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)
386{
387 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
388
389 vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 0);
390}
391
392static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)
393{
394 if (irq < VGIC_NR_PRIVATE_IRQS)
395 set_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
396 else
397 set_bit(irq - VGIC_NR_PRIVATE_IRQS,
398 vcpu->arch.vgic_cpu.pending_shared);
399}
400
401void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
402{
403 if (irq < VGIC_NR_PRIVATE_IRQS)
404 clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
405 else
406 clear_bit(irq - VGIC_NR_PRIVATE_IRQS,
407 vcpu->arch.vgic_cpu.pending_shared);
408}
409
410static bool vgic_can_sample_irq(struct kvm_vcpu *vcpu, int irq)
411{
412 return !vgic_irq_is_queued(vcpu, irq);
413}
414
415/**
416 * vgic_reg_access - access vgic register
417 * @mmio: pointer to the data describing the mmio access
418 * @reg: pointer to the virtual backing of vgic distributor data
419 * @offset: least significant 2 bits used for word offset
420 * @mode: ACCESS_ mode (see defines above)
421 *
422 * Helper to make vgic register access easier using one of the access
423 * modes defined for vgic register access
424 * (read,raz,write-ignored,setbit,clearbit,write)
425 */
426void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
427 phys_addr_t offset, int mode)
428{
429 int word_offset = (offset & 3) * 8;
430 u32 mask = (1UL << (mmio->len * 8)) - 1;
431 u32 regval;
432
433 /*
434 * Any alignment fault should have been delivered to the guest
435 * directly (ARM ARM B3.12.7 "Prioritization of aborts").
436 */
437
438 if (reg) {
439 regval = *reg;
440 } else {
441 BUG_ON(mode != (ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED));
442 regval = 0;
443 }
444
445 if (mmio->is_write) {
446 u32 data = mmio_data_read(mmio, mask) << word_offset;
447 switch (ACCESS_WRITE_MASK(mode)) {
448 case ACCESS_WRITE_IGNORED:
449 return;
450
451 case ACCESS_WRITE_SETBIT:
452 regval |= data;
453 break;
454
455 case ACCESS_WRITE_CLEARBIT:
456 regval &= ~data;
457 break;
458
459 case ACCESS_WRITE_VALUE:
460 regval = (regval & ~(mask << word_offset)) | data;
461 break;
462 }
463 *reg = regval;
464 } else {
465 switch (ACCESS_READ_MASK(mode)) {
466 case ACCESS_READ_RAZ:
467 regval = 0;
468 /* fall through */
469
470 case ACCESS_READ_VALUE:
471 mmio_data_write(mmio, mask, regval >> word_offset);
472 }
473 }
474}
475
476bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
477 phys_addr_t offset)
478{
479 vgic_reg_access(mmio, NULL, offset,
480 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
481 return false;
482}
483
484bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
485 phys_addr_t offset, int vcpu_id, int access)
486{
487 u32 *reg;
488 int mode = ACCESS_READ_VALUE | access;
489 struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);
490
491 reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);
492 vgic_reg_access(mmio, reg, offset, mode);
493 if (mmio->is_write) {
494 if (access & ACCESS_WRITE_CLEARBIT) {
495 if (offset < 4) /* Force SGI enabled */
496 *reg |= 0xffff;
497 vgic_retire_disabled_irqs(target_vcpu);
498 }
499 vgic_update_state(kvm);
500 return true;
501 }
502
503 return false;
504}
505
506bool vgic_handle_set_pending_reg(struct kvm *kvm,
507 struct kvm_exit_mmio *mmio,
508 phys_addr_t offset, int vcpu_id)
509{
510 u32 *reg, orig;
511 u32 level_mask;
512 int mode = ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT;
513 struct vgic_dist *dist = &kvm->arch.vgic;
514
515 reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu_id, offset);
516 level_mask = (~(*reg));
517
518 /* Mark both level and edge triggered irqs as pending */
519 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
520 orig = *reg;
521 vgic_reg_access(mmio, reg, offset, mode);
522
523 if (mmio->is_write) {
524 /* Set the soft-pending flag only for level-triggered irqs */
525 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
526 vcpu_id, offset);
527 vgic_reg_access(mmio, reg, offset, mode);
528 *reg &= level_mask;
529
530 /* Ignore writes to SGIs */
531 if (offset < 2) {
532 *reg &= ~0xffff;
533 *reg |= orig & 0xffff;
534 }
535
536 vgic_update_state(kvm);
537 return true;
538 }
539
540 return false;
541}
542
543bool vgic_handle_clear_pending_reg(struct kvm *kvm,
544 struct kvm_exit_mmio *mmio,
545 phys_addr_t offset, int vcpu_id)
546{
547 u32 *level_active;
548 u32 *reg, orig;
549 int mode = ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT;
550 struct vgic_dist *dist = &kvm->arch.vgic;
551
552 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
553 orig = *reg;
554 vgic_reg_access(mmio, reg, offset, mode);
555 if (mmio->is_write) {
556 /* Re-set level triggered level-active interrupts */
557 level_active = vgic_bitmap_get_reg(&dist->irq_level,
558 vcpu_id, offset);
559 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
560 *reg |= *level_active;
561
562 /* Ignore writes to SGIs */
563 if (offset < 2) {
564 *reg &= ~0xffff;
565 *reg |= orig & 0xffff;
566 }
567
568 /* Clear soft-pending flags */
569 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
570 vcpu_id, offset);
571 vgic_reg_access(mmio, reg, offset, mode);
572
573 vgic_update_state(kvm);
574 return true;
575 }
576 return false;
577}
578
579bool vgic_handle_set_active_reg(struct kvm *kvm,
580 struct kvm_exit_mmio *mmio,
581 phys_addr_t offset, int vcpu_id)
582{
583 u32 *reg;
584 struct vgic_dist *dist = &kvm->arch.vgic;
585
586 reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
587 vgic_reg_access(mmio, reg, offset,
588 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
589
590 if (mmio->is_write) {
591 vgic_update_state(kvm);
592 return true;
593 }
594
595 return false;
596}
597
598bool vgic_handle_clear_active_reg(struct kvm *kvm,
599 struct kvm_exit_mmio *mmio,
600 phys_addr_t offset, int vcpu_id)
601{
602 u32 *reg;
603 struct vgic_dist *dist = &kvm->arch.vgic;
604
605 reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
606 vgic_reg_access(mmio, reg, offset,
607 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
608
609 if (mmio->is_write) {
610 vgic_update_state(kvm);
611 return true;
612 }
613
614 return false;
615}
616
617static u32 vgic_cfg_expand(u16 val)
618{
619 u32 res = 0;
620 int i;
621
622 /*
623 * Turn a 16bit value like abcd...mnop into a 32bit word
624 * a0b0c0d0...m0n0o0p0, which is what the HW cfg register is.
625 */
626 for (i = 0; i < 16; i++)
627 res |= ((val >> i) & VGIC_CFG_EDGE) << (2 * i + 1);
628
629 return res;
630}
631
632static u16 vgic_cfg_compress(u32 val)
633{
634 u16 res = 0;
635 int i;
636
637 /*
638 * Turn a 32bit word a0b0c0d0...m0n0o0p0 into 16bit value like
639 * abcd...mnop which is what we really care about.
640 */
641 for (i = 0; i < 16; i++)
642 res |= ((val >> (i * 2 + 1)) & VGIC_CFG_EDGE) << i;
643
644 return res;
645}
646
647/*
648 * The distributor uses 2 bits per IRQ for the CFG register, but the
649 * LSB is always 0. As such, we only keep the upper bit, and use the
650 * two above functions to compress/expand the bits
651 */
652bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
653 phys_addr_t offset)
654{
655 u32 val;
656
657 if (offset & 4)
658 val = *reg >> 16;
659 else
660 val = *reg & 0xffff;
661
662 val = vgic_cfg_expand(val);
663 vgic_reg_access(mmio, &val, offset,
664 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
665 if (mmio->is_write) {
666 /* Ignore writes to read-only SGI and PPI bits */
667 if (offset < 8)
668 return false;
669
670 val = vgic_cfg_compress(val);
671 if (offset & 4) {
672 *reg &= 0xffff;
673 *reg |= val << 16;
674 } else {
675 *reg &= 0xffff << 16;
676 *reg |= val;
677 }
678 }
679
680 return false;
681}
682
683/**
684 * vgic_unqueue_irqs - move pending/active IRQs from LRs to the distributor
685 * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs
686 *
687 * Move any IRQs that have already been assigned to LRs back to the
688 * emulated distributor state so that the complete emulated state can be read
689 * from the main emulation structures without investigating the LRs.
690 */
691void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
692{
693 u64 elrsr = vgic_get_elrsr(vcpu);
694 unsigned long *elrsr_ptr = u64_to_bitmask(&elrsr);
695 int i;
696
697 for_each_clear_bit(i, elrsr_ptr, vgic->nr_lr) {
698 struct vgic_lr lr = vgic_get_lr(vcpu, i);
699
700 /*
701 * There are three options for the state bits:
702 *
703 * 01: pending
704 * 10: active
705 * 11: pending and active
706 */
707 BUG_ON(!(lr.state & LR_STATE_MASK));
708
709 /* Reestablish SGI source for pending and active IRQs */
710 if (lr.irq < VGIC_NR_SGIS)
711 add_sgi_source(vcpu, lr.irq, lr.source);
712
713 /*
714 * If the LR holds an active (10) or a pending and active (11)
715 * interrupt then move the active state to the
716 * distributor tracking bit.
717 */
718 if (lr.state & LR_STATE_ACTIVE)
719 vgic_irq_set_active(vcpu, lr.irq);
720
721 /*
722 * Reestablish the pending state on the distributor and the
723 * CPU interface and mark the LR as free for other use.
724 */
725 vgic_retire_lr(i, vcpu);
726
727 /* Finally update the VGIC state. */
728 vgic_update_state(vcpu->kvm);
729 }
730}
731
732const
733struct vgic_io_range *vgic_find_range(const struct vgic_io_range *ranges,
734 int len, gpa_t offset)
735{
736 while (ranges->len) {
737 if (offset >= ranges->base &&
738 (offset + len) <= (ranges->base + ranges->len))
739 return ranges;
740 ranges++;
741 }
742
743 return NULL;
744}
745
746static bool vgic_validate_access(const struct vgic_dist *dist,
747 const struct vgic_io_range *range,
748 unsigned long offset)
749{
750 int irq;
751
752 if (!range->bits_per_irq)
753 return true; /* Not an irq-based access */
754
755 irq = offset * 8 / range->bits_per_irq;
756 if (irq >= dist->nr_irqs)
757 return false;
758
759 return true;
760}
761
762/*
763 * Call the respective handler function for the given range.
764 * We split up any 64 bit accesses into two consecutive 32 bit
765 * handler calls and merge the result afterwards.
766 * We do this in a little endian fashion regardless of the host's
767 * or guest's endianness, because the GIC is always LE and the rest of
768 * the code (vgic_reg_access) also puts it in a LE fashion already.
769 * At this point we have already identified the handle function, so
770 * range points to that one entry and offset is relative to this.
771 */
772static bool call_range_handler(struct kvm_vcpu *vcpu,
773 struct kvm_exit_mmio *mmio,
774 unsigned long offset,
775 const struct vgic_io_range *range)
776{
777 struct kvm_exit_mmio mmio32;
778 bool ret;
779
780 if (likely(mmio->len <= 4))
781 return range->handle_mmio(vcpu, mmio, offset);
782
783 /*
784 * Any access bigger than 4 bytes (that we currently handle in KVM)
785 * is actually 8 bytes long, caused by a 64-bit access
786 */
787
788 mmio32.len = 4;
789 mmio32.is_write = mmio->is_write;
790 mmio32.private = mmio->private;
791
792 mmio32.phys_addr = mmio->phys_addr + 4;
793 mmio32.data = &((u32 *)mmio->data)[1];
794 ret = range->handle_mmio(vcpu, &mmio32, offset + 4);
795
796 mmio32.phys_addr = mmio->phys_addr;
797 mmio32.data = &((u32 *)mmio->data)[0];
798 ret |= range->handle_mmio(vcpu, &mmio32, offset);
799
800 return ret;
801}
802
803/**
804 * vgic_handle_mmio_access - handle an in-kernel MMIO access
805 * This is called by the read/write KVM IO device wrappers below.
806 * @vcpu: pointer to the vcpu performing the access
807 * @this: pointer to the KVM IO device in charge
808 * @addr: guest physical address of the access
809 * @len: size of the access
810 * @val: pointer to the data region
811 * @is_write: read or write access
812 *
813 * returns true if the MMIO access could be performed
814 */
815static int vgic_handle_mmio_access(struct kvm_vcpu *vcpu,
816 struct kvm_io_device *this, gpa_t addr,
817 int len, void *val, bool is_write)
818{
819 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
820 struct vgic_io_device *iodev = container_of(this,
821 struct vgic_io_device, dev);
822 const struct vgic_io_range *range;
823 struct kvm_exit_mmio mmio;
824 bool updated_state;
825 gpa_t offset;
826
827 offset = addr - iodev->addr;
828 range = vgic_find_range(iodev->reg_ranges, len, offset);
829 if (unlikely(!range || !range->handle_mmio)) {
830 pr_warn("Unhandled access %d %08llx %d\n", is_write, addr, len);
831 return -ENXIO;
832 }
833
834 mmio.phys_addr = addr;
835 mmio.len = len;
836 mmio.is_write = is_write;
837 mmio.data = val;
838 mmio.private = iodev->redist_vcpu;
839
840 spin_lock(&dist->lock);
841 offset -= range->base;
842 if (vgic_validate_access(dist, range, offset)) {
843 updated_state = call_range_handler(vcpu, &mmio, offset, range);
844 } else {
845 if (!is_write)
846 memset(val, 0, len);
847 updated_state = false;
848 }
849 spin_unlock(&dist->lock);
850
851 if (updated_state)
852 vgic_kick_vcpus(vcpu->kvm);
853
854 return 0;
855}
856
857static int vgic_handle_mmio_read(struct kvm_vcpu *vcpu,
858 struct kvm_io_device *this,
859 gpa_t addr, int len, void *val)
860{
861 return vgic_handle_mmio_access(vcpu, this, addr, len, val, false);
862}
863
864static int vgic_handle_mmio_write(struct kvm_vcpu *vcpu,
865 struct kvm_io_device *this,
866 gpa_t addr, int len, const void *val)
867{
868 return vgic_handle_mmio_access(vcpu, this, addr, len, (void *)val,
869 true);
870}
871
872static struct kvm_io_device_ops vgic_io_ops = {
873 .read = vgic_handle_mmio_read,
874 .write = vgic_handle_mmio_write,
875};
876
877/**
878 * vgic_register_kvm_io_dev - register VGIC register frame on the KVM I/O bus
879 * @kvm: The VM structure pointer
880 * @base: The (guest) base address for the register frame
881 * @len: Length of the register frame window
882 * @ranges: Describing the handler functions for each register
883 * @redist_vcpu_id: The VCPU ID to pass on to the handlers on call
884 * @iodev: Points to memory to be passed on to the handler
885 *
886 * @iodev stores the parameters of this function to be usable by the handler
887 * respectively the dispatcher function (since the KVM I/O bus framework lacks
888 * an opaque parameter). Initialization is done in this function, but the
889 * reference should be valid and unique for the whole VGIC lifetime.
890 * If the register frame is not mapped for a specific VCPU, pass -1 to
891 * @redist_vcpu_id.
892 */
893int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,
894 const struct vgic_io_range *ranges,
895 int redist_vcpu_id,
896 struct vgic_io_device *iodev)
897{
898 struct kvm_vcpu *vcpu = NULL;
899 int ret;
900
901 if (redist_vcpu_id >= 0)
902 vcpu = kvm_get_vcpu(kvm, redist_vcpu_id);
903
904 iodev->addr = base;
905 iodev->len = len;
906 iodev->reg_ranges = ranges;
907 iodev->redist_vcpu = vcpu;
908
909 kvm_iodevice_init(&iodev->dev, &vgic_io_ops);
910
911 mutex_lock(&kvm->slots_lock);
912
913 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, base, len,
914 &iodev->dev);
915 mutex_unlock(&kvm->slots_lock);
916
917 /* Mark the iodev as invalid if registration fails. */
918 if (ret)
919 iodev->dev.ops = NULL;
920
921 return ret;
922}
923
924static int vgic_nr_shared_irqs(struct vgic_dist *dist)
925{
926 return dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
927}
928
929static int compute_active_for_cpu(struct kvm_vcpu *vcpu)
930{
931 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
932 unsigned long *active, *enabled, *act_percpu, *act_shared;
933 unsigned long active_private, active_shared;
934 int nr_shared = vgic_nr_shared_irqs(dist);
935 int vcpu_id;
936
937 vcpu_id = vcpu->vcpu_id;
938 act_percpu = vcpu->arch.vgic_cpu.active_percpu;
939 act_shared = vcpu->arch.vgic_cpu.active_shared;
940
941 active = vgic_bitmap_get_cpu_map(&dist->irq_active, vcpu_id);
942 enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
943 bitmap_and(act_percpu, active, enabled, VGIC_NR_PRIVATE_IRQS);
944
945 active = vgic_bitmap_get_shared_map(&dist->irq_active);
946 enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
947 bitmap_and(act_shared, active, enabled, nr_shared);
948 bitmap_and(act_shared, act_shared,
949 vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
950 nr_shared);
951
952 active_private = find_first_bit(act_percpu, VGIC_NR_PRIVATE_IRQS);
953 active_shared = find_first_bit(act_shared, nr_shared);
954
955 return (active_private < VGIC_NR_PRIVATE_IRQS ||
956 active_shared < nr_shared);
957}
958
959static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
960{
961 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
962 unsigned long *pending, *enabled, *pend_percpu, *pend_shared;
963 unsigned long pending_private, pending_shared;
964 int nr_shared = vgic_nr_shared_irqs(dist);
965 int vcpu_id;
966
967 vcpu_id = vcpu->vcpu_id;
968 pend_percpu = vcpu->arch.vgic_cpu.pending_percpu;
969 pend_shared = vcpu->arch.vgic_cpu.pending_shared;
970
971 if (!dist->enabled) {
972 bitmap_zero(pend_percpu, VGIC_NR_PRIVATE_IRQS);
973 bitmap_zero(pend_shared, nr_shared);
974 return 0;
975 }
976
977 pending = vgic_bitmap_get_cpu_map(&dist->irq_pending, vcpu_id);
978 enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
979 bitmap_and(pend_percpu, pending, enabled, VGIC_NR_PRIVATE_IRQS);
980
981 pending = vgic_bitmap_get_shared_map(&dist->irq_pending);
982 enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
983 bitmap_and(pend_shared, pending, enabled, nr_shared);
984 bitmap_and(pend_shared, pend_shared,
985 vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
986 nr_shared);
987
988 pending_private = find_first_bit(pend_percpu, VGIC_NR_PRIVATE_IRQS);
989 pending_shared = find_first_bit(pend_shared, nr_shared);
990 return (pending_private < VGIC_NR_PRIVATE_IRQS ||
991 pending_shared < vgic_nr_shared_irqs(dist));
992}
993
994/*
995 * Update the interrupt state and determine which CPUs have pending
996 * or active interrupts. Must be called with distributor lock held.
997 */
998void vgic_update_state(struct kvm *kvm)
999{
1000 struct vgic_dist *dist = &kvm->arch.vgic;
1001 struct kvm_vcpu *vcpu;
1002 int c;
1003
1004 kvm_for_each_vcpu(c, vcpu, kvm) {
1005 if (compute_pending_for_cpu(vcpu))
1006 set_bit(c, dist->irq_pending_on_cpu);
1007
1008 if (compute_active_for_cpu(vcpu))
1009 set_bit(c, dist->irq_active_on_cpu);
1010 else
1011 clear_bit(c, dist->irq_active_on_cpu);
1012 }
1013}
1014
1015static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr)
1016{
1017 return vgic_ops->get_lr(vcpu, lr);
1018}
1019
1020static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr,
1021 struct vgic_lr vlr)
1022{
1023 vgic_ops->set_lr(vcpu, lr, vlr);
1024}
1025
1026static inline u64 vgic_get_elrsr(struct kvm_vcpu *vcpu)
1027{
1028 return vgic_ops->get_elrsr(vcpu);
1029}
1030
1031static inline u64 vgic_get_eisr(struct kvm_vcpu *vcpu)
1032{
1033 return vgic_ops->get_eisr(vcpu);
1034}
1035
1036static inline void vgic_clear_eisr(struct kvm_vcpu *vcpu)
1037{
1038 vgic_ops->clear_eisr(vcpu);
1039}
1040
1041static inline u32 vgic_get_interrupt_status(struct kvm_vcpu *vcpu)
1042{
1043 return vgic_ops->get_interrupt_status(vcpu);
1044}
1045
1046static inline void vgic_enable_underflow(struct kvm_vcpu *vcpu)
1047{
1048 vgic_ops->enable_underflow(vcpu);
1049}
1050
1051static inline void vgic_disable_underflow(struct kvm_vcpu *vcpu)
1052{
1053 vgic_ops->disable_underflow(vcpu);
1054}
1055
1056void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1057{
1058 vgic_ops->get_vmcr(vcpu, vmcr);
1059}
1060
1061void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1062{
1063 vgic_ops->set_vmcr(vcpu, vmcr);
1064}
1065
1066static inline void vgic_enable(struct kvm_vcpu *vcpu)
1067{
1068 vgic_ops->enable(vcpu);
1069}
1070
1071static void vgic_retire_lr(int lr_nr, struct kvm_vcpu *vcpu)
1072{
1073 struct vgic_lr vlr = vgic_get_lr(vcpu, lr_nr);
1074
1075 vgic_irq_clear_queued(vcpu, vlr.irq);
1076
1077 /*
1078 * We must transfer the pending state back to the distributor before
1079 * retiring the LR, otherwise we may loose edge-triggered interrupts.
1080 */
1081 if (vlr.state & LR_STATE_PENDING) {
1082 vgic_dist_irq_set_pending(vcpu, vlr.irq);
1083 vlr.hwirq = 0;
1084 }
1085
1086 vlr.state = 0;
1087 vgic_set_lr(vcpu, lr_nr, vlr);
1088}
1089
1090static bool dist_active_irq(struct kvm_vcpu *vcpu)
1091{
1092 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1093
1094 return test_bit(vcpu->vcpu_id, dist->irq_active_on_cpu);
1095}
1096
1097bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq)
1098{
1099 int i;
1100
1101 for (i = 0; i < vgic->nr_lr; i++) {
1102 struct vgic_lr vlr = vgic_get_lr(vcpu, i);
1103
1104 if (vlr.irq == virt_irq && vlr.state & LR_STATE_ACTIVE)
1105 return true;
1106 }
1107
1108 return vgic_irq_is_active(vcpu, virt_irq);
1109}
1110
1111/*
1112 * An interrupt may have been disabled after being made pending on the
1113 * CPU interface (the classic case is a timer running while we're
1114 * rebooting the guest - the interrupt would kick as soon as the CPU
1115 * interface gets enabled, with deadly consequences).
1116 *
1117 * The solution is to examine already active LRs, and check the
1118 * interrupt is still enabled. If not, just retire it.
1119 */
1120static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
1121{
1122 u64 elrsr = vgic_get_elrsr(vcpu);
1123 unsigned long *elrsr_ptr = u64_to_bitmask(&elrsr);
1124 int lr;
1125
1126 for_each_clear_bit(lr, elrsr_ptr, vgic->nr_lr) {
1127 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1128
1129 if (!vgic_irq_is_enabled(vcpu, vlr.irq))
1130 vgic_retire_lr(lr, vcpu);
1131 }
1132}
1133
1134static void vgic_queue_irq_to_lr(struct kvm_vcpu *vcpu, int irq,
1135 int lr_nr, struct vgic_lr vlr)
1136{
1137 if (vgic_irq_is_active(vcpu, irq)) {
1138 vlr.state |= LR_STATE_ACTIVE;
1139 kvm_debug("Set active, clear distributor: 0x%x\n", vlr.state);
1140 vgic_irq_clear_active(vcpu, irq);
1141 vgic_update_state(vcpu->kvm);
1142 } else {
1143 WARN_ON(!vgic_dist_irq_is_pending(vcpu, irq));
1144 vlr.state |= LR_STATE_PENDING;
1145 kvm_debug("Set pending: 0x%x\n", vlr.state);
1146 }
1147
1148 if (!vgic_irq_is_edge(vcpu, irq))
1149 vlr.state |= LR_EOI_INT;
1150
1151 if (vlr.irq >= VGIC_NR_SGIS) {
1152 struct irq_phys_map *map;
1153 map = vgic_irq_map_search(vcpu, irq);
1154
1155 if (map) {
1156 vlr.hwirq = map->phys_irq;
1157 vlr.state |= LR_HW;
1158 vlr.state &= ~LR_EOI_INT;
1159
1160 /*
1161 * Make sure we're not going to sample this
1162 * again, as a HW-backed interrupt cannot be
1163 * in the PENDING_ACTIVE stage.
1164 */
1165 vgic_irq_set_queued(vcpu, irq);
1166 }
1167 }
1168
1169 vgic_set_lr(vcpu, lr_nr, vlr);
1170}
1171
1172/*
1173 * Queue an interrupt to a CPU virtual interface. Return true on success,
1174 * or false if it wasn't possible to queue it.
1175 * sgi_source must be zero for any non-SGI interrupts.
1176 */
1177bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
1178{
1179 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1180 u64 elrsr = vgic_get_elrsr(vcpu);
1181 unsigned long *elrsr_ptr = u64_to_bitmask(&elrsr);
1182 struct vgic_lr vlr;
1183 int lr;
1184
1185 /* Sanitize the input... */
1186 BUG_ON(sgi_source_id & ~7);
1187 BUG_ON(sgi_source_id && irq >= VGIC_NR_SGIS);
1188 BUG_ON(irq >= dist->nr_irqs);
1189
1190 kvm_debug("Queue IRQ%d\n", irq);
1191
1192 /* Do we have an active interrupt for the same CPUID? */
1193 for_each_clear_bit(lr, elrsr_ptr, vgic->nr_lr) {
1194 vlr = vgic_get_lr(vcpu, lr);
1195 if (vlr.irq == irq && vlr.source == sgi_source_id) {
1196 kvm_debug("LR%d piggyback for IRQ%d\n", lr, vlr.irq);
1197 vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1198 return true;
1199 }
1200 }
1201
1202 /* Try to use another LR for this interrupt */
1203 lr = find_first_bit(elrsr_ptr, vgic->nr_lr);
1204 if (lr >= vgic->nr_lr)
1205 return false;
1206
1207 kvm_debug("LR%d allocated for IRQ%d %x\n", lr, irq, sgi_source_id);
1208
1209 vlr.irq = irq;
1210 vlr.source = sgi_source_id;
1211 vlr.state = 0;
1212 vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1213
1214 return true;
1215}
1216
1217static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)
1218{
1219 if (!vgic_can_sample_irq(vcpu, irq))
1220 return true; /* level interrupt, already queued */
1221
1222 if (vgic_queue_irq(vcpu, 0, irq)) {
1223 if (vgic_irq_is_edge(vcpu, irq)) {
1224 vgic_dist_irq_clear_pending(vcpu, irq);
1225 vgic_cpu_irq_clear(vcpu, irq);
1226 } else {
1227 vgic_irq_set_queued(vcpu, irq);
1228 }
1229
1230 return true;
1231 }
1232
1233 return false;
1234}
1235
1236/*
1237 * Fill the list registers with pending interrupts before running the
1238 * guest.
1239 */
1240static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1241{
1242 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1243 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1244 unsigned long *pa_percpu, *pa_shared;
1245 int i, vcpu_id;
1246 int overflow = 0;
1247 int nr_shared = vgic_nr_shared_irqs(dist);
1248
1249 vcpu_id = vcpu->vcpu_id;
1250
1251 pa_percpu = vcpu->arch.vgic_cpu.pend_act_percpu;
1252 pa_shared = vcpu->arch.vgic_cpu.pend_act_shared;
1253
1254 bitmap_or(pa_percpu, vgic_cpu->pending_percpu, vgic_cpu->active_percpu,
1255 VGIC_NR_PRIVATE_IRQS);
1256 bitmap_or(pa_shared, vgic_cpu->pending_shared, vgic_cpu->active_shared,
1257 nr_shared);
1258 /*
1259 * We may not have any pending interrupt, or the interrupts
1260 * may have been serviced from another vcpu. In all cases,
1261 * move along.
1262 */
1263 if (!kvm_vgic_vcpu_pending_irq(vcpu) && !dist_active_irq(vcpu))
1264 goto epilog;
1265
1266 /* SGIs */
1267 for_each_set_bit(i, pa_percpu, VGIC_NR_SGIS) {
1268 if (!queue_sgi(vcpu, i))
1269 overflow = 1;
1270 }
1271
1272 /* PPIs */
1273 for_each_set_bit_from(i, pa_percpu, VGIC_NR_PRIVATE_IRQS) {
1274 if (!vgic_queue_hwirq(vcpu, i))
1275 overflow = 1;
1276 }
1277
1278 /* SPIs */
1279 for_each_set_bit(i, pa_shared, nr_shared) {
1280 if (!vgic_queue_hwirq(vcpu, i + VGIC_NR_PRIVATE_IRQS))
1281 overflow = 1;
1282 }
1283
1284
1285
1286
1287epilog:
1288 if (overflow) {
1289 vgic_enable_underflow(vcpu);
1290 } else {
1291 vgic_disable_underflow(vcpu);
1292 /*
1293 * We're about to run this VCPU, and we've consumed
1294 * everything the distributor had in store for
1295 * us. Claim we don't have anything pending. We'll
1296 * adjust that if needed while exiting.
1297 */
1298 clear_bit(vcpu_id, dist->irq_pending_on_cpu);
1299 }
1300}
1301
1302static int process_queued_irq(struct kvm_vcpu *vcpu,
1303 int lr, struct vgic_lr vlr)
1304{
1305 int pending = 0;
1306
1307 /*
1308 * If the IRQ was EOIed (called from vgic_process_maintenance) or it
1309 * went from active to non-active (called from vgic_sync_hwirq) it was
1310 * also ACKed and we we therefore assume we can clear the soft pending
1311 * state (should it had been set) for this interrupt.
1312 *
1313 * Note: if the IRQ soft pending state was set after the IRQ was
1314 * acked, it actually shouldn't be cleared, but we have no way of
1315 * knowing that unless we start trapping ACKs when the soft-pending
1316 * state is set.
1317 */
1318 vgic_dist_irq_clear_soft_pend(vcpu, vlr.irq);
1319
1320 /*
1321 * Tell the gic to start sampling this interrupt again.
1322 */
1323 vgic_irq_clear_queued(vcpu, vlr.irq);
1324
1325 /* Any additional pending interrupt? */
1326 if (vgic_irq_is_edge(vcpu, vlr.irq)) {
1327 BUG_ON(!(vlr.state & LR_HW));
1328 pending = vgic_dist_irq_is_pending(vcpu, vlr.irq);
1329 } else {
1330 if (vgic_dist_irq_get_level(vcpu, vlr.irq)) {
1331 vgic_cpu_irq_set(vcpu, vlr.irq);
1332 pending = 1;
1333 } else {
1334 vgic_dist_irq_clear_pending(vcpu, vlr.irq);
1335 vgic_cpu_irq_clear(vcpu, vlr.irq);
1336 }
1337 }
1338
1339 /*
1340 * Despite being EOIed, the LR may not have
1341 * been marked as empty.
1342 */
1343 vlr.state = 0;
1344 vlr.hwirq = 0;
1345 vgic_set_lr(vcpu, lr, vlr);
1346
1347 return pending;
1348}
1349
1350static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1351{
1352 u32 status = vgic_get_interrupt_status(vcpu);
1353 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1354 struct kvm *kvm = vcpu->kvm;
1355 int level_pending = 0;
1356
1357 kvm_debug("STATUS = %08x\n", status);
1358
1359 if (status & INT_STATUS_EOI) {
1360 /*
1361 * Some level interrupts have been EOIed. Clear their
1362 * active bit.
1363 */
1364 u64 eisr = vgic_get_eisr(vcpu);
1365 unsigned long *eisr_ptr = u64_to_bitmask(&eisr);
1366 int lr;
1367
1368 for_each_set_bit(lr, eisr_ptr, vgic->nr_lr) {
1369 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1370
1371 WARN_ON(vgic_irq_is_edge(vcpu, vlr.irq));
1372 WARN_ON(vlr.state & LR_STATE_MASK);
1373
1374
1375 /*
1376 * kvm_notify_acked_irq calls kvm_set_irq()
1377 * to reset the IRQ level, which grabs the dist->lock
1378 * so we call this before taking the dist->lock.
1379 */
1380 kvm_notify_acked_irq(kvm, 0,
1381 vlr.irq - VGIC_NR_PRIVATE_IRQS);
1382
1383 spin_lock(&dist->lock);
1384 level_pending |= process_queued_irq(vcpu, lr, vlr);
1385 spin_unlock(&dist->lock);
1386 }
1387 }
1388
1389 if (status & INT_STATUS_UNDERFLOW)
1390 vgic_disable_underflow(vcpu);
1391
1392 /*
1393 * In the next iterations of the vcpu loop, if we sync the vgic state
1394 * after flushing it, but before entering the guest (this happens for
1395 * pending signals and vmid rollovers), then make sure we don't pick
1396 * up any old maintenance interrupts here.
1397 */
1398 vgic_clear_eisr(vcpu);
1399
1400 return level_pending;
1401}
1402
1403/*
1404 * Save the physical active state, and reset it to inactive.
1405 *
1406 * Return true if there's a pending forwarded interrupt to queue.
1407 */
1408static bool vgic_sync_hwirq(struct kvm_vcpu *vcpu, int lr, struct vgic_lr vlr)
1409{
1410 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1411 bool level_pending;
1412
1413 if (!(vlr.state & LR_HW))
1414 return false;
1415
1416 if (vlr.state & LR_STATE_ACTIVE)
1417 return false;
1418
1419 spin_lock(&dist->lock);
1420 level_pending = process_queued_irq(vcpu, lr, vlr);
1421 spin_unlock(&dist->lock);
1422 return level_pending;
1423}
1424
1425/* Sync back the VGIC state after a guest run */
1426static void __kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1427{
1428 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1429 u64 elrsr;
1430 unsigned long *elrsr_ptr;
1431 int lr, pending;
1432 bool level_pending;
1433
1434 level_pending = vgic_process_maintenance(vcpu);
1435
1436 /* Deal with HW interrupts, and clear mappings for empty LRs */
1437 for (lr = 0; lr < vgic->nr_lr; lr++) {
1438 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1439
1440 level_pending |= vgic_sync_hwirq(vcpu, lr, vlr);
1441 BUG_ON(vlr.irq >= dist->nr_irqs);
1442 }
1443
1444 /* Check if we still have something up our sleeve... */
1445 elrsr = vgic_get_elrsr(vcpu);
1446 elrsr_ptr = u64_to_bitmask(&elrsr);
1447 pending = find_first_zero_bit(elrsr_ptr, vgic->nr_lr);
1448 if (level_pending || pending < vgic->nr_lr)
1449 set_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1450}
1451
1452void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1453{
1454 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1455
1456 if (!irqchip_in_kernel(vcpu->kvm))
1457 return;
1458
1459 spin_lock(&dist->lock);
1460 __kvm_vgic_flush_hwstate(vcpu);
1461 spin_unlock(&dist->lock);
1462}
1463
1464void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1465{
1466 if (!irqchip_in_kernel(vcpu->kvm))
1467 return;
1468
1469 __kvm_vgic_sync_hwstate(vcpu);
1470}
1471
1472int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
1473{
1474 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1475
1476 if (!irqchip_in_kernel(vcpu->kvm))
1477 return 0;
1478
1479 return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1480}
1481
1482void vgic_kick_vcpus(struct kvm *kvm)
1483{
1484 struct kvm_vcpu *vcpu;
1485 int c;
1486
1487 /*
1488 * We've injected an interrupt, time to find out who deserves
1489 * a good kick...
1490 */
1491 kvm_for_each_vcpu(c, vcpu, kvm) {
1492 if (kvm_vgic_vcpu_pending_irq(vcpu))
1493 kvm_vcpu_kick(vcpu);
1494 }
1495}
1496
1497static int vgic_validate_injection(struct kvm_vcpu *vcpu, int irq, int level)
1498{
1499 int edge_triggered = vgic_irq_is_edge(vcpu, irq);
1500
1501 /*
1502 * Only inject an interrupt if:
1503 * - edge triggered and we have a rising edge
1504 * - level triggered and we change level
1505 */
1506 if (edge_triggered) {
1507 int state = vgic_dist_irq_is_pending(vcpu, irq);
1508 return level > state;
1509 } else {
1510 int state = vgic_dist_irq_get_level(vcpu, irq);
1511 return level != state;
1512 }
1513}
1514
1515static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
1516 unsigned int irq_num, bool level)
1517{
1518 struct vgic_dist *dist = &kvm->arch.vgic;
1519 struct kvm_vcpu *vcpu;
1520 int edge_triggered, level_triggered;
1521 int enabled;
1522 bool ret = true, can_inject = true;
1523
1524 trace_vgic_update_irq_pending(cpuid, irq_num, level);
1525
1526 if (irq_num >= min(kvm->arch.vgic.nr_irqs, 1020))
1527 return -EINVAL;
1528
1529 spin_lock(&dist->lock);
1530
1531 vcpu = kvm_get_vcpu(kvm, cpuid);
1532 edge_triggered = vgic_irq_is_edge(vcpu, irq_num);
1533 level_triggered = !edge_triggered;
1534
1535 if (!vgic_validate_injection(vcpu, irq_num, level)) {
1536 ret = false;
1537 goto out;
1538 }
1539
1540 if (irq_num >= VGIC_NR_PRIVATE_IRQS) {
1541 cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS];
1542 if (cpuid == VCPU_NOT_ALLOCATED) {
1543 /* Pretend we use CPU0, and prevent injection */
1544 cpuid = 0;
1545 can_inject = false;
1546 }
1547 vcpu = kvm_get_vcpu(kvm, cpuid);
1548 }
1549
1550 kvm_debug("Inject IRQ%d level %d CPU%d\n", irq_num, level, cpuid);
1551
1552 if (level) {
1553 if (level_triggered)
1554 vgic_dist_irq_set_level(vcpu, irq_num);
1555 vgic_dist_irq_set_pending(vcpu, irq_num);
1556 } else {
1557 if (level_triggered) {
1558 vgic_dist_irq_clear_level(vcpu, irq_num);
1559 if (!vgic_dist_irq_soft_pend(vcpu, irq_num)) {
1560 vgic_dist_irq_clear_pending(vcpu, irq_num);
1561 vgic_cpu_irq_clear(vcpu, irq_num);
1562 if (!compute_pending_for_cpu(vcpu))
1563 clear_bit(cpuid, dist->irq_pending_on_cpu);
1564 }
1565 }
1566
1567 ret = false;
1568 goto out;
1569 }
1570
1571 enabled = vgic_irq_is_enabled(vcpu, irq_num);
1572
1573 if (!enabled || !can_inject) {
1574 ret = false;
1575 goto out;
1576 }
1577
1578 if (!vgic_can_sample_irq(vcpu, irq_num)) {
1579 /*
1580 * Level interrupt in progress, will be picked up
1581 * when EOId.
1582 */
1583 ret = false;
1584 goto out;
1585 }
1586
1587 if (level) {
1588 vgic_cpu_irq_set(vcpu, irq_num);
1589 set_bit(cpuid, dist->irq_pending_on_cpu);
1590 }
1591
1592out:
1593 spin_unlock(&dist->lock);
1594
1595 if (ret) {
1596 /* kick the specified vcpu */
1597 kvm_vcpu_kick(kvm_get_vcpu(kvm, cpuid));
1598 }
1599
1600 return 0;
1601}
1602
1603static int vgic_lazy_init(struct kvm *kvm)
1604{
1605 int ret = 0;
1606
1607 if (unlikely(!vgic_initialized(kvm))) {
1608 /*
1609 * We only provide the automatic initialization of the VGIC
1610 * for the legacy case of a GICv2. Any other type must
1611 * be explicitly initialized once setup with the respective
1612 * KVM device call.
1613 */
1614 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
1615 return -EBUSY;
1616
1617 mutex_lock(&kvm->lock);
1618 ret = vgic_init(kvm);
1619 mutex_unlock(&kvm->lock);
1620 }
1621
1622 return ret;
1623}
1624
1625/**
1626 * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
1627 * @kvm: The VM structure pointer
1628 * @cpuid: The CPU for PPIs
1629 * @irq_num: The IRQ number that is assigned to the device. This IRQ
1630 * must not be mapped to a HW interrupt.
1631 * @level: Edge-triggered: true: to trigger the interrupt
1632 * false: to ignore the call
1633 * Level-sensitive true: raise the input signal
1634 * false: lower the input signal
1635 *
1636 * The GIC is not concerned with devices being active-LOW or active-HIGH for
1637 * level-sensitive interrupts. You can think of the level parameter as 1
1638 * being HIGH and 0 being LOW and all devices being active-HIGH.
1639 */
1640int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
1641 bool level)
1642{
1643 struct irq_phys_map *map;
1644 int ret;
1645
1646 ret = vgic_lazy_init(kvm);
1647 if (ret)
1648 return ret;
1649
1650 map = vgic_irq_map_search(kvm_get_vcpu(kvm, cpuid), irq_num);
1651 if (map)
1652 return -EINVAL;
1653
1654 return vgic_update_irq_pending(kvm, cpuid, irq_num, level);
1655}
1656
1657/**
1658 * kvm_vgic_inject_mapped_irq - Inject a physically mapped IRQ to the vgic
1659 * @kvm: The VM structure pointer
1660 * @cpuid: The CPU for PPIs
1661 * @virt_irq: The virtual IRQ to be injected
1662 * @level: Edge-triggered: true: to trigger the interrupt
1663 * false: to ignore the call
1664 * Level-sensitive true: raise the input signal
1665 * false: lower the input signal
1666 *
1667 * The GIC is not concerned with devices being active-LOW or active-HIGH for
1668 * level-sensitive interrupts. You can think of the level parameter as 1
1669 * being HIGH and 0 being LOW and all devices being active-HIGH.
1670 */
1671int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid,
1672 unsigned int virt_irq, bool level)
1673{
1674 int ret;
1675
1676 ret = vgic_lazy_init(kvm);
1677 if (ret)
1678 return ret;
1679
1680 return vgic_update_irq_pending(kvm, cpuid, virt_irq, level);
1681}
1682
1683static irqreturn_t vgic_maintenance_handler(int irq, void *data)
1684{
1685 /*
1686 * We cannot rely on the vgic maintenance interrupt to be
1687 * delivered synchronously. This means we can only use it to
1688 * exit the VM, and we perform the handling of EOIed
1689 * interrupts on the exit path (see vgic_process_maintenance).
1690 */
1691 return IRQ_HANDLED;
1692}
1693
1694static struct list_head *vgic_get_irq_phys_map_list(struct kvm_vcpu *vcpu,
1695 int virt_irq)
1696{
1697 if (virt_irq < VGIC_NR_PRIVATE_IRQS)
1698 return &vcpu->arch.vgic_cpu.irq_phys_map_list;
1699 else
1700 return &vcpu->kvm->arch.vgic.irq_phys_map_list;
1701}
1702
1703/**
1704 * kvm_vgic_map_phys_irq - map a virtual IRQ to a physical IRQ
1705 * @vcpu: The VCPU pointer
1706 * @virt_irq: The virtual IRQ number for the guest
1707 * @phys_irq: The hardware IRQ number of the host
1708 *
1709 * Establish a mapping between a guest visible irq (@virt_irq) and a
1710 * hardware irq (@phys_irq). On injection, @virt_irq will be associated with
1711 * the physical interrupt represented by @phys_irq. This mapping can be
1712 * established multiple times as long as the parameters are the same.
1713 *
1714 * Returns 0 on success or an error value otherwise.
1715 */
1716int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, int virt_irq, int phys_irq)
1717{
1718 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1719 struct list_head *root = vgic_get_irq_phys_map_list(vcpu, virt_irq);
1720 struct irq_phys_map *map;
1721 struct irq_phys_map_entry *entry;
1722 int ret = 0;
1723
1724 /* Create a new mapping */
1725 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1726 if (!entry)
1727 return -ENOMEM;
1728
1729 spin_lock(&dist->irq_phys_map_lock);
1730
1731 /* Try to match an existing mapping */
1732 map = vgic_irq_map_search(vcpu, virt_irq);
1733 if (map) {
1734 /* Make sure this mapping matches */
1735 if (map->phys_irq != phys_irq)
1736 ret = -EINVAL;
1737
1738 /* Found an existing, valid mapping */
1739 goto out;
1740 }
1741
1742 map = &entry->map;
1743 map->virt_irq = virt_irq;
1744 map->phys_irq = phys_irq;
1745
1746 list_add_tail_rcu(&entry->entry, root);
1747
1748out:
1749 spin_unlock(&dist->irq_phys_map_lock);
1750 /* If we've found a hit in the existing list, free the useless
1751 * entry */
1752 if (ret || map != &entry->map)
1753 kfree(entry);
1754 return ret;
1755}
1756
1757static struct irq_phys_map *vgic_irq_map_search(struct kvm_vcpu *vcpu,
1758 int virt_irq)
1759{
1760 struct list_head *root = vgic_get_irq_phys_map_list(vcpu, virt_irq);
1761 struct irq_phys_map_entry *entry;
1762 struct irq_phys_map *map;
1763
1764 rcu_read_lock();
1765
1766 list_for_each_entry_rcu(entry, root, entry) {
1767 map = &entry->map;
1768 if (map->virt_irq == virt_irq) {
1769 rcu_read_unlock();
1770 return map;
1771 }
1772 }
1773
1774 rcu_read_unlock();
1775
1776 return NULL;
1777}
1778
1779static void vgic_free_phys_irq_map_rcu(struct rcu_head *rcu)
1780{
1781 struct irq_phys_map_entry *entry;
1782
1783 entry = container_of(rcu, struct irq_phys_map_entry, rcu);
1784 kfree(entry);
1785}
1786
1787/**
1788 * kvm_vgic_unmap_phys_irq - Remove a virtual to physical IRQ mapping
1789 * @vcpu: The VCPU pointer
1790 * @virt_irq: The virtual IRQ number to be unmapped
1791 *
1792 * Remove an existing mapping between virtual and physical interrupts.
1793 */
1794int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq)
1795{
1796 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1797 struct irq_phys_map_entry *entry;
1798 struct list_head *root;
1799
1800 root = vgic_get_irq_phys_map_list(vcpu, virt_irq);
1801
1802 spin_lock(&dist->irq_phys_map_lock);
1803
1804 list_for_each_entry(entry, root, entry) {
1805 if (entry->map.virt_irq == virt_irq) {
1806 list_del_rcu(&entry->entry);
1807 call_rcu(&entry->rcu, vgic_free_phys_irq_map_rcu);
1808 break;
1809 }
1810 }
1811
1812 spin_unlock(&dist->irq_phys_map_lock);
1813
1814 return 0;
1815}
1816
1817static void vgic_destroy_irq_phys_map(struct kvm *kvm, struct list_head *root)
1818{
1819 struct vgic_dist *dist = &kvm->arch.vgic;
1820 struct irq_phys_map_entry *entry;
1821
1822 spin_lock(&dist->irq_phys_map_lock);
1823
1824 list_for_each_entry(entry, root, entry) {
1825 list_del_rcu(&entry->entry);
1826 call_rcu(&entry->rcu, vgic_free_phys_irq_map_rcu);
1827 }
1828
1829 spin_unlock(&dist->irq_phys_map_lock);
1830}
1831
1832void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
1833{
1834 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1835
1836 kfree(vgic_cpu->pending_shared);
1837 kfree(vgic_cpu->active_shared);
1838 kfree(vgic_cpu->pend_act_shared);
1839 vgic_destroy_irq_phys_map(vcpu->kvm, &vgic_cpu->irq_phys_map_list);
1840 vgic_cpu->pending_shared = NULL;
1841 vgic_cpu->active_shared = NULL;
1842 vgic_cpu->pend_act_shared = NULL;
1843}
1844
1845static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs)
1846{
1847 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1848 int nr_longs = BITS_TO_LONGS(nr_irqs - VGIC_NR_PRIVATE_IRQS);
1849 int sz = nr_longs * sizeof(unsigned long);
1850 vgic_cpu->pending_shared = kzalloc(sz, GFP_KERNEL);
1851 vgic_cpu->active_shared = kzalloc(sz, GFP_KERNEL);
1852 vgic_cpu->pend_act_shared = kzalloc(sz, GFP_KERNEL);
1853
1854 if (!vgic_cpu->pending_shared
1855 || !vgic_cpu->active_shared
1856 || !vgic_cpu->pend_act_shared) {
1857 kvm_vgic_vcpu_destroy(vcpu);
1858 return -ENOMEM;
1859 }
1860
1861 return 0;
1862}
1863
1864/**
1865 * kvm_vgic_vcpu_early_init - Earliest possible per-vcpu vgic init stage
1866 *
1867 * No memory allocation should be performed here, only static init.
1868 */
1869void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu)
1870{
1871 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1872 INIT_LIST_HEAD(&vgic_cpu->irq_phys_map_list);
1873}
1874
1875/**
1876 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
1877 *
1878 * The host's GIC naturally limits the maximum amount of VCPUs a guest
1879 * can use.
1880 */
1881int kvm_vgic_get_max_vcpus(void)
1882{
1883 return vgic->max_gic_vcpus;
1884}
1885
1886void kvm_vgic_destroy(struct kvm *kvm)
1887{
1888 struct vgic_dist *dist = &kvm->arch.vgic;
1889 struct kvm_vcpu *vcpu;
1890 int i;
1891
1892 kvm_for_each_vcpu(i, vcpu, kvm)
1893 kvm_vgic_vcpu_destroy(vcpu);
1894
1895 vgic_free_bitmap(&dist->irq_enabled);
1896 vgic_free_bitmap(&dist->irq_level);
1897 vgic_free_bitmap(&dist->irq_pending);
1898 vgic_free_bitmap(&dist->irq_soft_pend);
1899 vgic_free_bitmap(&dist->irq_queued);
1900 vgic_free_bitmap(&dist->irq_cfg);
1901 vgic_free_bytemap(&dist->irq_priority);
1902 if (dist->irq_spi_target) {
1903 for (i = 0; i < dist->nr_cpus; i++)
1904 vgic_free_bitmap(&dist->irq_spi_target[i]);
1905 }
1906 kfree(dist->irq_sgi_sources);
1907 kfree(dist->irq_spi_cpu);
1908 kfree(dist->irq_spi_mpidr);
1909 kfree(dist->irq_spi_target);
1910 kfree(dist->irq_pending_on_cpu);
1911 kfree(dist->irq_active_on_cpu);
1912 vgic_destroy_irq_phys_map(kvm, &dist->irq_phys_map_list);
1913 dist->irq_sgi_sources = NULL;
1914 dist->irq_spi_cpu = NULL;
1915 dist->irq_spi_target = NULL;
1916 dist->irq_pending_on_cpu = NULL;
1917 dist->irq_active_on_cpu = NULL;
1918 dist->nr_cpus = 0;
1919}
1920
1921/*
1922 * Allocate and initialize the various data structures. Must be called
1923 * with kvm->lock held!
1924 */
1925int vgic_init(struct kvm *kvm)
1926{
1927 struct vgic_dist *dist = &kvm->arch.vgic;
1928 struct kvm_vcpu *vcpu;
1929 int nr_cpus, nr_irqs;
1930 int ret, i, vcpu_id;
1931
1932 if (vgic_initialized(kvm))
1933 return 0;
1934
1935 nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);
1936 if (!nr_cpus) /* No vcpus? Can't be good... */
1937 return -ENODEV;
1938
1939 /*
1940 * If nobody configured the number of interrupts, use the
1941 * legacy one.
1942 */
1943 if (!dist->nr_irqs)
1944 dist->nr_irqs = VGIC_NR_IRQS_LEGACY;
1945
1946 nr_irqs = dist->nr_irqs;
1947
1948 ret = vgic_init_bitmap(&dist->irq_enabled, nr_cpus, nr_irqs);
1949 ret |= vgic_init_bitmap(&dist->irq_level, nr_cpus, nr_irqs);
1950 ret |= vgic_init_bitmap(&dist->irq_pending, nr_cpus, nr_irqs);
1951 ret |= vgic_init_bitmap(&dist->irq_soft_pend, nr_cpus, nr_irqs);
1952 ret |= vgic_init_bitmap(&dist->irq_queued, nr_cpus, nr_irqs);
1953 ret |= vgic_init_bitmap(&dist->irq_active, nr_cpus, nr_irqs);
1954 ret |= vgic_init_bitmap(&dist->irq_cfg, nr_cpus, nr_irqs);
1955 ret |= vgic_init_bytemap(&dist->irq_priority, nr_cpus, nr_irqs);
1956
1957 if (ret)
1958 goto out;
1959
1960 dist->irq_sgi_sources = kzalloc(nr_cpus * VGIC_NR_SGIS, GFP_KERNEL);
1961 dist->irq_spi_cpu = kzalloc(nr_irqs - VGIC_NR_PRIVATE_IRQS, GFP_KERNEL);
1962 dist->irq_spi_target = kzalloc(sizeof(*dist->irq_spi_target) * nr_cpus,
1963 GFP_KERNEL);
1964 dist->irq_pending_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
1965 GFP_KERNEL);
1966 dist->irq_active_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
1967 GFP_KERNEL);
1968 if (!dist->irq_sgi_sources ||
1969 !dist->irq_spi_cpu ||
1970 !dist->irq_spi_target ||
1971 !dist->irq_pending_on_cpu ||
1972 !dist->irq_active_on_cpu) {
1973 ret = -ENOMEM;
1974 goto out;
1975 }
1976
1977 for (i = 0; i < nr_cpus; i++)
1978 ret |= vgic_init_bitmap(&dist->irq_spi_target[i],
1979 nr_cpus, nr_irqs);
1980
1981 if (ret)
1982 goto out;
1983
1984 ret = kvm->arch.vgic.vm_ops.init_model(kvm);
1985 if (ret)
1986 goto out;
1987
1988 kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {
1989 ret = vgic_vcpu_init_maps(vcpu, nr_irqs);
1990 if (ret) {
1991 kvm_err("VGIC: Failed to allocate vcpu memory\n");
1992 break;
1993 }
1994
1995 /*
1996 * Enable and configure all SGIs to be edge-triggere and
1997 * configure all PPIs as level-triggered.
1998 */
1999 for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
2000 if (i < VGIC_NR_SGIS) {
2001 /* SGIs */
2002 vgic_bitmap_set_irq_val(&dist->irq_enabled,
2003 vcpu->vcpu_id, i, 1);
2004 vgic_bitmap_set_irq_val(&dist->irq_cfg,
2005 vcpu->vcpu_id, i,
2006 VGIC_CFG_EDGE);
2007 } else if (i < VGIC_NR_PRIVATE_IRQS) {
2008 /* PPIs */
2009 vgic_bitmap_set_irq_val(&dist->irq_cfg,
2010 vcpu->vcpu_id, i,
2011 VGIC_CFG_LEVEL);
2012 }
2013 }
2014
2015 vgic_enable(vcpu);
2016 }
2017
2018out:
2019 if (ret)
2020 kvm_vgic_destroy(kvm);
2021
2022 return ret;
2023}
2024
2025static int init_vgic_model(struct kvm *kvm, int type)
2026{
2027 switch (type) {
2028 case KVM_DEV_TYPE_ARM_VGIC_V2:
2029 vgic_v2_init_emulation(kvm);
2030 break;
2031#ifdef CONFIG_KVM_ARM_VGIC_V3
2032 case KVM_DEV_TYPE_ARM_VGIC_V3:
2033 vgic_v3_init_emulation(kvm);
2034 break;
2035#endif
2036 default:
2037 return -ENODEV;
2038 }
2039
2040 if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)
2041 return -E2BIG;
2042
2043 return 0;
2044}
2045
2046/**
2047 * kvm_vgic_early_init - Earliest possible vgic initialization stage
2048 *
2049 * No memory allocation should be performed here, only static init.
2050 */
2051void kvm_vgic_early_init(struct kvm *kvm)
2052{
2053 spin_lock_init(&kvm->arch.vgic.lock);
2054 spin_lock_init(&kvm->arch.vgic.irq_phys_map_lock);
2055 INIT_LIST_HEAD(&kvm->arch.vgic.irq_phys_map_list);
2056}
2057
2058int kvm_vgic_create(struct kvm *kvm, u32 type)
2059{
2060 int i, vcpu_lock_idx = -1, ret;
2061 struct kvm_vcpu *vcpu;
2062
2063 mutex_lock(&kvm->lock);
2064
2065 if (irqchip_in_kernel(kvm)) {
2066 ret = -EEXIST;
2067 goto out;
2068 }
2069
2070 /*
2071 * This function is also called by the KVM_CREATE_IRQCHIP handler,
2072 * which had no chance yet to check the availability of the GICv2
2073 * emulation. So check this here again. KVM_CREATE_DEVICE does
2074 * the proper checks already.
2075 */
2076 if (type == KVM_DEV_TYPE_ARM_VGIC_V2 && !vgic->can_emulate_gicv2) {
2077 ret = -ENODEV;
2078 goto out;
2079 }
2080
2081 /*
2082 * Any time a vcpu is run, vcpu_load is called which tries to grab the
2083 * vcpu->mutex. By grabbing the vcpu->mutex of all VCPUs we ensure
2084 * that no other VCPUs are run while we create the vgic.
2085 */
2086 ret = -EBUSY;
2087 kvm_for_each_vcpu(i, vcpu, kvm) {
2088 if (!mutex_trylock(&vcpu->mutex))
2089 goto out_unlock;
2090 vcpu_lock_idx = i;
2091 }
2092
2093 kvm_for_each_vcpu(i, vcpu, kvm) {
2094 if (vcpu->arch.has_run_once)
2095 goto out_unlock;
2096 }
2097 ret = 0;
2098
2099 ret = init_vgic_model(kvm, type);
2100 if (ret)
2101 goto out_unlock;
2102
2103 kvm->arch.vgic.in_kernel = true;
2104 kvm->arch.vgic.vgic_model = type;
2105 kvm->arch.vgic.vctrl_base = vgic->vctrl_base;
2106 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
2107 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
2108 kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
2109
2110out_unlock:
2111 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
2112 vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
2113 mutex_unlock(&vcpu->mutex);
2114 }
2115
2116out:
2117 mutex_unlock(&kvm->lock);
2118 return ret;
2119}
2120
2121static int vgic_ioaddr_overlap(struct kvm *kvm)
2122{
2123 phys_addr_t dist = kvm->arch.vgic.vgic_dist_base;
2124 phys_addr_t cpu = kvm->arch.vgic.vgic_cpu_base;
2125
2126 if (IS_VGIC_ADDR_UNDEF(dist) || IS_VGIC_ADDR_UNDEF(cpu))
2127 return 0;
2128 if ((dist <= cpu && dist + KVM_VGIC_V2_DIST_SIZE > cpu) ||
2129 (cpu <= dist && cpu + KVM_VGIC_V2_CPU_SIZE > dist))
2130 return -EBUSY;
2131 return 0;
2132}
2133
2134static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,
2135 phys_addr_t addr, phys_addr_t size)
2136{
2137 int ret;
2138
2139 if (addr & ~KVM_PHYS_MASK)
2140 return -E2BIG;
2141
2142 if (addr & (SZ_4K - 1))
2143 return -EINVAL;
2144
2145 if (!IS_VGIC_ADDR_UNDEF(*ioaddr))
2146 return -EEXIST;
2147 if (addr + size < addr)
2148 return -EINVAL;
2149
2150 *ioaddr = addr;
2151 ret = vgic_ioaddr_overlap(kvm);
2152 if (ret)
2153 *ioaddr = VGIC_ADDR_UNDEF;
2154
2155 return ret;
2156}
2157
2158/**
2159 * kvm_vgic_addr - set or get vgic VM base addresses
2160 * @kvm: pointer to the vm struct
2161 * @type: the VGIC addr type, one of KVM_VGIC_V[23]_ADDR_TYPE_XXX
2162 * @addr: pointer to address value
2163 * @write: if true set the address in the VM address space, if false read the
2164 * address
2165 *
2166 * Set or get the vgic base addresses for the distributor and the virtual CPU
2167 * interface in the VM physical address space. These addresses are properties
2168 * of the emulated core/SoC and therefore user space initially knows this
2169 * information.
2170 */
2171int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
2172{
2173 int r = 0;
2174 struct vgic_dist *vgic = &kvm->arch.vgic;
2175 int type_needed;
2176 phys_addr_t *addr_ptr, block_size;
2177 phys_addr_t alignment;
2178
2179 mutex_lock(&kvm->lock);
2180 switch (type) {
2181 case KVM_VGIC_V2_ADDR_TYPE_DIST:
2182 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
2183 addr_ptr = &vgic->vgic_dist_base;
2184 block_size = KVM_VGIC_V2_DIST_SIZE;
2185 alignment = SZ_4K;
2186 break;
2187 case KVM_VGIC_V2_ADDR_TYPE_CPU:
2188 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
2189 addr_ptr = &vgic->vgic_cpu_base;
2190 block_size = KVM_VGIC_V2_CPU_SIZE;
2191 alignment = SZ_4K;
2192 break;
2193#ifdef CONFIG_KVM_ARM_VGIC_V3
2194 case KVM_VGIC_V3_ADDR_TYPE_DIST:
2195 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
2196 addr_ptr = &vgic->vgic_dist_base;
2197 block_size = KVM_VGIC_V3_DIST_SIZE;
2198 alignment = SZ_64K;
2199 break;
2200 case KVM_VGIC_V3_ADDR_TYPE_REDIST:
2201 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
2202 addr_ptr = &vgic->vgic_redist_base;
2203 block_size = KVM_VGIC_V3_REDIST_SIZE;
2204 alignment = SZ_64K;
2205 break;
2206#endif
2207 default:
2208 r = -ENODEV;
2209 goto out;
2210 }
2211
2212 if (vgic->vgic_model != type_needed) {
2213 r = -ENODEV;
2214 goto out;
2215 }
2216
2217 if (write) {
2218 if (!IS_ALIGNED(*addr, alignment))
2219 r = -EINVAL;
2220 else
2221 r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,
2222 block_size);
2223 } else {
2224 *addr = *addr_ptr;
2225 }
2226
2227out:
2228 mutex_unlock(&kvm->lock);
2229 return r;
2230}
2231
2232int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2233{
2234 int r;
2235
2236 switch (attr->group) {
2237 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2238 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2239 u64 addr;
2240 unsigned long type = (unsigned long)attr->attr;
2241
2242 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2243 return -EFAULT;
2244
2245 r = kvm_vgic_addr(dev->kvm, type, &addr, true);
2246 return (r == -ENODEV) ? -ENXIO : r;
2247 }
2248 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2249 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2250 u32 val;
2251 int ret = 0;
2252
2253 if (get_user(val, uaddr))
2254 return -EFAULT;
2255
2256 /*
2257 * We require:
2258 * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
2259 * - at most 1024 interrupts
2260 * - a multiple of 32 interrupts
2261 */
2262 if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
2263 val > VGIC_MAX_IRQS ||
2264 (val & 31))
2265 return -EINVAL;
2266
2267 mutex_lock(&dev->kvm->lock);
2268
2269 if (vgic_ready(dev->kvm) || dev->kvm->arch.vgic.nr_irqs)
2270 ret = -EBUSY;
2271 else
2272 dev->kvm->arch.vgic.nr_irqs = val;
2273
2274 mutex_unlock(&dev->kvm->lock);
2275
2276 return ret;
2277 }
2278 case KVM_DEV_ARM_VGIC_GRP_CTRL: {
2279 switch (attr->attr) {
2280 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2281 r = vgic_init(dev->kvm);
2282 return r;
2283 }
2284 break;
2285 }
2286 }
2287
2288 return -ENXIO;
2289}
2290
2291int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2292{
2293 int r = -ENXIO;
2294
2295 switch (attr->group) {
2296 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2297 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2298 u64 addr;
2299 unsigned long type = (unsigned long)attr->attr;
2300
2301 r = kvm_vgic_addr(dev->kvm, type, &addr, false);
2302 if (r)
2303 return (r == -ENODEV) ? -ENXIO : r;
2304
2305 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2306 return -EFAULT;
2307 break;
2308 }
2309 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2310 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2311
2312 r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);
2313 break;
2314 }
2315
2316 }
2317
2318 return r;
2319}
2320
2321int vgic_has_attr_regs(const struct vgic_io_range *ranges, phys_addr_t offset)
2322{
2323 if (vgic_find_range(ranges, 4, offset))
2324 return 0;
2325 else
2326 return -ENXIO;
2327}
2328
2329static void vgic_init_maintenance_interrupt(void *info)
2330{
2331 enable_percpu_irq(vgic->maint_irq, 0);
2332}
2333
2334static int vgic_cpu_notify(struct notifier_block *self,
2335 unsigned long action, void *cpu)
2336{
2337 switch (action) {
2338 case CPU_STARTING:
2339 case CPU_STARTING_FROZEN:
2340 vgic_init_maintenance_interrupt(NULL);
2341 break;
2342 case CPU_DYING:
2343 case CPU_DYING_FROZEN:
2344 disable_percpu_irq(vgic->maint_irq);
2345 break;
2346 }
2347
2348 return NOTIFY_OK;
2349}
2350
2351static struct notifier_block vgic_cpu_nb = {
2352 .notifier_call = vgic_cpu_notify,
2353};
2354
2355static int kvm_vgic_probe(void)
2356{
2357 const struct gic_kvm_info *gic_kvm_info;
2358 int ret;
2359
2360 gic_kvm_info = gic_get_kvm_info();
2361 if (!gic_kvm_info)
2362 return -ENODEV;
2363
2364 switch (gic_kvm_info->type) {
2365 case GIC_V2:
2366 ret = vgic_v2_probe(gic_kvm_info, &vgic_ops, &vgic);
2367 break;
2368 case GIC_V3:
2369 ret = vgic_v3_probe(gic_kvm_info, &vgic_ops, &vgic);
2370 break;
2371 default:
2372 ret = -ENODEV;
2373 }
2374
2375 return ret;
2376}
2377
2378int kvm_vgic_hyp_init(void)
2379{
2380 int ret;
2381
2382 ret = kvm_vgic_probe();
2383 if (ret) {
2384 kvm_err("error: KVM vGIC probing failed\n");
2385 return ret;
2386 }
2387
2388 ret = request_percpu_irq(vgic->maint_irq, vgic_maintenance_handler,
2389 "vgic", kvm_get_running_vcpus());
2390 if (ret) {
2391 kvm_err("Cannot register interrupt %d\n", vgic->maint_irq);
2392 return ret;
2393 }
2394
2395 ret = __register_cpu_notifier(&vgic_cpu_nb);
2396 if (ret) {
2397 kvm_err("Cannot register vgic CPU notifier\n");
2398 goto out_free_irq;
2399 }
2400
2401 on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);
2402
2403 return 0;
2404
2405out_free_irq:
2406 free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
2407 return ret;
2408}
2409
2410int kvm_irq_map_gsi(struct kvm *kvm,
2411 struct kvm_kernel_irq_routing_entry *entries,
2412 int gsi)
2413{
2414 return 0;
2415}
2416
2417int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
2418{
2419 return pin;
2420}
2421
2422int kvm_set_irq(struct kvm *kvm, int irq_source_id,
2423 u32 irq, int level, bool line_status)
2424{
2425 unsigned int spi = irq + VGIC_NR_PRIVATE_IRQS;
2426
2427 trace_kvm_set_irq(irq, level, irq_source_id);
2428
2429 BUG_ON(!vgic_initialized(kvm));
2430
2431 return kvm_vgic_inject_irq(kvm, 0, spi, level);
2432}
2433
2434/* MSI not implemented yet */
2435int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
2436 struct kvm *kvm, int irq_source_id,
2437 int level, bool line_status)
2438{
2439 return 0;
2440}
diff --git a/virt/kvm/arm/vgic.h b/virt/kvm/arm/vgic.h
deleted file mode 100644
index 0df74cbb6200..000000000000
--- a/virt/kvm/arm/vgic.h
+++ /dev/null
@@ -1,140 +0,0 @@
1/*
2 * Copyright (C) 2012-2014 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * Derived from virt/kvm/arm/vgic.c
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#ifndef __KVM_VGIC_H__
21#define __KVM_VGIC_H__
22
23#include <kvm/iodev.h>
24
25#define VGIC_ADDR_UNDEF (-1)
26#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF)
27
28#define PRODUCT_ID_KVM 0x4b /* ASCII code K */
29#define IMPLEMENTER_ARM 0x43b
30
31#define ACCESS_READ_VALUE (1 << 0)
32#define ACCESS_READ_RAZ (0 << 0)
33#define ACCESS_READ_MASK(x) ((x) & (1 << 0))
34#define ACCESS_WRITE_IGNORED (0 << 1)
35#define ACCESS_WRITE_SETBIT (1 << 1)
36#define ACCESS_WRITE_CLEARBIT (2 << 1)
37#define ACCESS_WRITE_VALUE (3 << 1)
38#define ACCESS_WRITE_MASK(x) ((x) & (3 << 1))
39
40#define VCPU_NOT_ALLOCATED ((u8)-1)
41
42unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x);
43
44void vgic_update_state(struct kvm *kvm);
45int vgic_init_common_maps(struct kvm *kvm);
46
47u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset);
48u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset);
49
50void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq);
51void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq);
52void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq);
53void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
54 int irq, int val);
55
56void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
57void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
58
59bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq);
60void vgic_unqueue_irqs(struct kvm_vcpu *vcpu);
61
62struct kvm_exit_mmio {
63 phys_addr_t phys_addr;
64 void *data;
65 u32 len;
66 bool is_write;
67 void *private;
68};
69
70void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
71 phys_addr_t offset, int mode);
72bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
73 phys_addr_t offset);
74
75static inline
76u32 mmio_data_read(struct kvm_exit_mmio *mmio, u32 mask)
77{
78 return le32_to_cpu(*((u32 *)mmio->data)) & mask;
79}
80
81static inline
82void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
83{
84 *((u32 *)mmio->data) = cpu_to_le32(value) & mask;
85}
86
87struct vgic_io_range {
88 phys_addr_t base;
89 unsigned long len;
90 int bits_per_irq;
91 bool (*handle_mmio)(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
92 phys_addr_t offset);
93};
94
95int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,
96 const struct vgic_io_range *ranges,
97 int redist_id,
98 struct vgic_io_device *iodev);
99
100static inline bool is_in_range(phys_addr_t addr, unsigned long len,
101 phys_addr_t baseaddr, unsigned long size)
102{
103 return (addr >= baseaddr) && (addr + len <= baseaddr + size);
104}
105
106const
107struct vgic_io_range *vgic_find_range(const struct vgic_io_range *ranges,
108 int len, gpa_t offset);
109
110bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
111 phys_addr_t offset, int vcpu_id, int access);
112
113bool vgic_handle_set_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
114 phys_addr_t offset, int vcpu_id);
115
116bool vgic_handle_clear_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
117 phys_addr_t offset, int vcpu_id);
118
119bool vgic_handle_set_active_reg(struct kvm *kvm,
120 struct kvm_exit_mmio *mmio,
121 phys_addr_t offset, int vcpu_id);
122
123bool vgic_handle_clear_active_reg(struct kvm *kvm,
124 struct kvm_exit_mmio *mmio,
125 phys_addr_t offset, int vcpu_id);
126
127bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
128 phys_addr_t offset);
129
130void vgic_kick_vcpus(struct kvm *kvm);
131
132int vgic_has_attr_regs(const struct vgic_io_range *ranges, phys_addr_t offset);
133int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr);
134int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr);
135
136int vgic_init(struct kvm *kvm);
137void vgic_v2_init_emulation(struct kvm *kvm);
138void vgic_v3_init_emulation(struct kvm *kvm);
139
140#endif
diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c
index a1442f7c9c4d..01a60dcd05d6 100644
--- a/virt/kvm/arm/vgic/vgic-init.c
+++ b/virt/kvm/arm/vgic/vgic-init.c
@@ -157,6 +157,9 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
157 struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); 157 struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
158 int i; 158 int i;
159 159
160 INIT_LIST_HEAD(&dist->lpi_list_head);
161 spin_lock_init(&dist->lpi_list_lock);
162
160 dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL); 163 dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL);
161 if (!dist->spis) 164 if (!dist->spis)
162 return -ENOMEM; 165 return -ENOMEM;
@@ -177,6 +180,7 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
177 spin_lock_init(&irq->irq_lock); 180 spin_lock_init(&irq->irq_lock);
178 irq->vcpu = NULL; 181 irq->vcpu = NULL;
179 irq->target_vcpu = vcpu0; 182 irq->target_vcpu = vcpu0;
183 kref_init(&irq->refcount);
180 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) 184 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
181 irq->targets = 0; 185 irq->targets = 0;
182 else 186 else
@@ -211,6 +215,7 @@ static void kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
211 irq->vcpu = NULL; 215 irq->vcpu = NULL;
212 irq->target_vcpu = vcpu; 216 irq->target_vcpu = vcpu;
213 irq->targets = 1U << vcpu->vcpu_id; 217 irq->targets = 1U << vcpu->vcpu_id;
218 kref_init(&irq->refcount);
214 if (vgic_irq_is_sgi(i)) { 219 if (vgic_irq_is_sgi(i)) {
215 /* SGIs */ 220 /* SGIs */
216 irq->enabled = 1; 221 irq->enabled = 1;
@@ -253,6 +258,9 @@ int vgic_init(struct kvm *kvm)
253 if (ret) 258 if (ret)
254 goto out; 259 goto out;
255 260
261 if (vgic_has_its(kvm))
262 dist->msis_require_devid = true;
263
256 kvm_for_each_vcpu(i, vcpu, kvm) 264 kvm_for_each_vcpu(i, vcpu, kvm)
257 kvm_vgic_vcpu_init(vcpu); 265 kvm_vgic_vcpu_init(vcpu);
258 266
@@ -271,7 +279,6 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)
271 dist->initialized = false; 279 dist->initialized = false;
272 280
273 kfree(dist->spis); 281 kfree(dist->spis);
274 kfree(dist->redist_iodevs);
275 dist->nr_spis = 0; 282 dist->nr_spis = 0;
276 283
277 mutex_unlock(&kvm->lock); 284 mutex_unlock(&kvm->lock);
diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c
new file mode 100644
index 000000000000..07411cf967b9
--- /dev/null
+++ b/virt/kvm/arm/vgic/vgic-its.c
@@ -0,0 +1,1500 @@
1/*
2 * GICv3 ITS emulation
3 *
4 * Copyright (C) 2015,2016 ARM Ltd.
5 * Author: Andre Przywara <andre.przywara@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/cpu.h>
21#include <linux/kvm.h>
22#include <linux/kvm_host.h>
23#include <linux/interrupt.h>
24#include <linux/list.h>
25#include <linux/uaccess.h>
26
27#include <linux/irqchip/arm-gic-v3.h>
28
29#include <asm/kvm_emulate.h>
30#include <asm/kvm_arm.h>
31#include <asm/kvm_mmu.h>
32
33#include "vgic.h"
34#include "vgic-mmio.h"
35
36/*
37 * Creates a new (reference to a) struct vgic_irq for a given LPI.
38 * If this LPI is already mapped on another ITS, we increase its refcount
39 * and return a pointer to the existing structure.
40 * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
41 * This function returns a pointer to the _unlocked_ structure.
42 */
43static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid)
44{
45 struct vgic_dist *dist = &kvm->arch.vgic;
46 struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
47
48 /* In this case there is no put, since we keep the reference. */
49 if (irq)
50 return irq;
51
52 irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL);
53 if (!irq)
54 return NULL;
55
56 INIT_LIST_HEAD(&irq->lpi_list);
57 INIT_LIST_HEAD(&irq->ap_list);
58 spin_lock_init(&irq->irq_lock);
59
60 irq->config = VGIC_CONFIG_EDGE;
61 kref_init(&irq->refcount);
62 irq->intid = intid;
63
64 spin_lock(&dist->lpi_list_lock);
65
66 /*
67 * There could be a race with another vgic_add_lpi(), so we need to
68 * check that we don't add a second list entry with the same LPI.
69 */
70 list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
71 if (oldirq->intid != intid)
72 continue;
73
74 /* Someone was faster with adding this LPI, lets use that. */
75 kfree(irq);
76 irq = oldirq;
77
78 /*
79 * This increases the refcount, the caller is expected to
80 * call vgic_put_irq() on the returned pointer once it's
81 * finished with the IRQ.
82 */
83 vgic_get_irq_kref(irq);
84
85 goto out_unlock;
86 }
87
88 list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
89 dist->lpi_list_count++;
90
91out_unlock:
92 spin_unlock(&dist->lpi_list_lock);
93
94 return irq;
95}
96
97struct its_device {
98 struct list_head dev_list;
99
100 /* the head for the list of ITTEs */
101 struct list_head itt_head;
102 u32 device_id;
103};
104
105#define COLLECTION_NOT_MAPPED ((u32)~0)
106
107struct its_collection {
108 struct list_head coll_list;
109
110 u32 collection_id;
111 u32 target_addr;
112};
113
114#define its_is_collection_mapped(coll) ((coll) && \
115 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
116
117struct its_itte {
118 struct list_head itte_list;
119
120 struct vgic_irq *irq;
121 struct its_collection *collection;
122 u32 lpi;
123 u32 event_id;
124};
125
126/*
127 * Find and returns a device in the device table for an ITS.
128 * Must be called with the its_lock mutex held.
129 */
130static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
131{
132 struct its_device *device;
133
134 list_for_each_entry(device, &its->device_list, dev_list)
135 if (device_id == device->device_id)
136 return device;
137
138 return NULL;
139}
140
141/*
142 * Find and returns an interrupt translation table entry (ITTE) for a given
143 * Device ID/Event ID pair on an ITS.
144 * Must be called with the its_lock mutex held.
145 */
146static struct its_itte *find_itte(struct vgic_its *its, u32 device_id,
147 u32 event_id)
148{
149 struct its_device *device;
150 struct its_itte *itte;
151
152 device = find_its_device(its, device_id);
153 if (device == NULL)
154 return NULL;
155
156 list_for_each_entry(itte, &device->itt_head, itte_list)
157 if (itte->event_id == event_id)
158 return itte;
159
160 return NULL;
161}
162
163/* To be used as an iterator this macro misses the enclosing parentheses */
164#define for_each_lpi_its(dev, itte, its) \
165 list_for_each_entry(dev, &(its)->device_list, dev_list) \
166 list_for_each_entry(itte, &(dev)->itt_head, itte_list)
167
168/*
169 * We only implement 48 bits of PA at the moment, although the ITS
170 * supports more. Let's be restrictive here.
171 */
172#define BASER_ADDRESS(x) ((x) & GENMASK_ULL(47, 16))
173#define CBASER_ADDRESS(x) ((x) & GENMASK_ULL(47, 12))
174#define PENDBASER_ADDRESS(x) ((x) & GENMASK_ULL(47, 16))
175#define PROPBASER_ADDRESS(x) ((x) & GENMASK_ULL(47, 12))
176
177#define GIC_LPI_OFFSET 8192
178
179/*
180 * Finds and returns a collection in the ITS collection table.
181 * Must be called with the its_lock mutex held.
182 */
183static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
184{
185 struct its_collection *collection;
186
187 list_for_each_entry(collection, &its->collection_list, coll_list) {
188 if (coll_id == collection->collection_id)
189 return collection;
190 }
191
192 return NULL;
193}
194
195#define LPI_PROP_ENABLE_BIT(p) ((p) & LPI_PROP_ENABLED)
196#define LPI_PROP_PRIORITY(p) ((p) & 0xfc)
197
198/*
199 * Reads the configuration data for a given LPI from guest memory and
200 * updates the fields in struct vgic_irq.
201 * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
202 * VCPU. Unconditionally applies if filter_vcpu is NULL.
203 */
204static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
205 struct kvm_vcpu *filter_vcpu)
206{
207 u64 propbase = PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
208 u8 prop;
209 int ret;
210
211 ret = kvm_read_guest(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
212 &prop, 1);
213
214 if (ret)
215 return ret;
216
217 spin_lock(&irq->irq_lock);
218
219 if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
220 irq->priority = LPI_PROP_PRIORITY(prop);
221 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
222
223 vgic_queue_irq_unlock(kvm, irq);
224 } else {
225 spin_unlock(&irq->irq_lock);
226 }
227
228 return 0;
229}
230
231/*
232 * Create a snapshot of the current LPI list, so that we can enumerate all
233 * LPIs without holding any lock.
234 * Returns the array length and puts the kmalloc'ed array into intid_ptr.
235 */
236static int vgic_copy_lpi_list(struct kvm *kvm, u32 **intid_ptr)
237{
238 struct vgic_dist *dist = &kvm->arch.vgic;
239 struct vgic_irq *irq;
240 u32 *intids;
241 int irq_count = dist->lpi_list_count, i = 0;
242
243 /*
244 * We use the current value of the list length, which may change
245 * after the kmalloc. We don't care, because the guest shouldn't
246 * change anything while the command handling is still running,
247 * and in the worst case we would miss a new IRQ, which one wouldn't
248 * expect to be covered by this command anyway.
249 */
250 intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL);
251 if (!intids)
252 return -ENOMEM;
253
254 spin_lock(&dist->lpi_list_lock);
255 list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
256 /* We don't need to "get" the IRQ, as we hold the list lock. */
257 intids[i] = irq->intid;
258 if (++i == irq_count)
259 break;
260 }
261 spin_unlock(&dist->lpi_list_lock);
262
263 *intid_ptr = intids;
264 return irq_count;
265}
266
267/*
268 * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
269 * is targeting) to the VGIC's view, which deals with target VCPUs.
270 * Needs to be called whenever either the collection for a LPIs has
271 * changed or the collection itself got retargeted.
272 */
273static void update_affinity_itte(struct kvm *kvm, struct its_itte *itte)
274{
275 struct kvm_vcpu *vcpu;
276
277 if (!its_is_collection_mapped(itte->collection))
278 return;
279
280 vcpu = kvm_get_vcpu(kvm, itte->collection->target_addr);
281
282 spin_lock(&itte->irq->irq_lock);
283 itte->irq->target_vcpu = vcpu;
284 spin_unlock(&itte->irq->irq_lock);
285}
286
287/*
288 * Updates the target VCPU for every LPI targeting this collection.
289 * Must be called with the its_lock mutex held.
290 */
291static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
292 struct its_collection *coll)
293{
294 struct its_device *device;
295 struct its_itte *itte;
296
297 for_each_lpi_its(device, itte, its) {
298 if (!itte->collection || coll != itte->collection)
299 continue;
300
301 update_affinity_itte(kvm, itte);
302 }
303}
304
305static u32 max_lpis_propbaser(u64 propbaser)
306{
307 int nr_idbits = (propbaser & 0x1f) + 1;
308
309 return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
310}
311
312/*
313 * Scan the whole LPI pending table and sync the pending bit in there
314 * with our own data structures. This relies on the LPI being
315 * mapped before.
316 */
317static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
318{
319 gpa_t pendbase = PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
320 struct vgic_irq *irq;
321 int last_byte_offset = -1;
322 int ret = 0;
323 u32 *intids;
324 int nr_irqs, i;
325
326 nr_irqs = vgic_copy_lpi_list(vcpu->kvm, &intids);
327 if (nr_irqs < 0)
328 return nr_irqs;
329
330 for (i = 0; i < nr_irqs; i++) {
331 int byte_offset, bit_nr;
332 u8 pendmask;
333
334 byte_offset = intids[i] / BITS_PER_BYTE;
335 bit_nr = intids[i] % BITS_PER_BYTE;
336
337 /*
338 * For contiguously allocated LPIs chances are we just read
339 * this very same byte in the last iteration. Reuse that.
340 */
341 if (byte_offset != last_byte_offset) {
342 ret = kvm_read_guest(vcpu->kvm, pendbase + byte_offset,
343 &pendmask, 1);
344 if (ret) {
345 kfree(intids);
346 return ret;
347 }
348 last_byte_offset = byte_offset;
349 }
350
351 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
352 spin_lock(&irq->irq_lock);
353 irq->pending = pendmask & (1U << bit_nr);
354 vgic_queue_irq_unlock(vcpu->kvm, irq);
355 vgic_put_irq(vcpu->kvm, irq);
356 }
357
358 kfree(intids);
359
360 return ret;
361}
362
363static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
364 struct vgic_its *its,
365 gpa_t addr, unsigned int len)
366{
367 u32 reg = 0;
368
369 mutex_lock(&its->cmd_lock);
370 if (its->creadr == its->cwriter)
371 reg |= GITS_CTLR_QUIESCENT;
372 if (its->enabled)
373 reg |= GITS_CTLR_ENABLE;
374 mutex_unlock(&its->cmd_lock);
375
376 return reg;
377}
378
379static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
380 gpa_t addr, unsigned int len,
381 unsigned long val)
382{
383 its->enabled = !!(val & GITS_CTLR_ENABLE);
384}
385
386static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
387 struct vgic_its *its,
388 gpa_t addr, unsigned int len)
389{
390 u64 reg = GITS_TYPER_PLPIS;
391
392 /*
393 * We use linear CPU numbers for redistributor addressing,
394 * so GITS_TYPER.PTA is 0.
395 * Also we force all PROPBASER registers to be the same, so
396 * CommonLPIAff is 0 as well.
397 * To avoid memory waste in the guest, we keep the number of IDBits and
398 * DevBits low - as least for the time being.
399 */
400 reg |= 0x0f << GITS_TYPER_DEVBITS_SHIFT;
401 reg |= 0x0f << GITS_TYPER_IDBITS_SHIFT;
402
403 return extract_bytes(reg, addr & 7, len);
404}
405
406static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
407 struct vgic_its *its,
408 gpa_t addr, unsigned int len)
409{
410 return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
411}
412
413static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
414 struct vgic_its *its,
415 gpa_t addr, unsigned int len)
416{
417 switch (addr & 0xffff) {
418 case GITS_PIDR0:
419 return 0x92; /* part number, bits[7:0] */
420 case GITS_PIDR1:
421 return 0xb4; /* part number, bits[11:8] */
422 case GITS_PIDR2:
423 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
424 case GITS_PIDR4:
425 return 0x40; /* This is a 64K software visible page */
426 /* The following are the ID registers for (any) GIC. */
427 case GITS_CIDR0:
428 return 0x0d;
429 case GITS_CIDR1:
430 return 0xf0;
431 case GITS_CIDR2:
432 return 0x05;
433 case GITS_CIDR3:
434 return 0xb1;
435 }
436
437 return 0;
438}
439
440/*
441 * Find the target VCPU and the LPI number for a given devid/eventid pair
442 * and make this IRQ pending, possibly injecting it.
443 * Must be called with the its_lock mutex held.
444 */
445static void vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
446 u32 devid, u32 eventid)
447{
448 struct its_itte *itte;
449
450 if (!its->enabled)
451 return;
452
453 itte = find_itte(its, devid, eventid);
454 /* Triggering an unmapped IRQ gets silently dropped. */
455 if (itte && its_is_collection_mapped(itte->collection)) {
456 struct kvm_vcpu *vcpu;
457
458 vcpu = kvm_get_vcpu(kvm, itte->collection->target_addr);
459 if (vcpu && vcpu->arch.vgic_cpu.lpis_enabled) {
460 spin_lock(&itte->irq->irq_lock);
461 itte->irq->pending = true;
462 vgic_queue_irq_unlock(kvm, itte->irq);
463 }
464 }
465}
466
467/*
468 * Queries the KVM IO bus framework to get the ITS pointer from the given
469 * doorbell address.
470 * We then call vgic_its_trigger_msi() with the decoded data.
471 */
472int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
473{
474 u64 address;
475 struct kvm_io_device *kvm_io_dev;
476 struct vgic_io_device *iodev;
477
478 if (!vgic_has_its(kvm))
479 return -ENODEV;
480
481 if (!(msi->flags & KVM_MSI_VALID_DEVID))
482 return -EINVAL;
483
484 address = (u64)msi->address_hi << 32 | msi->address_lo;
485
486 kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
487 if (!kvm_io_dev)
488 return -ENODEV;
489
490 iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);
491
492 mutex_lock(&iodev->its->its_lock);
493 vgic_its_trigger_msi(kvm, iodev->its, msi->devid, msi->data);
494 mutex_unlock(&iodev->its->its_lock);
495
496 return 0;
497}
498
499/* Requires the its_lock to be held. */
500static void its_free_itte(struct kvm *kvm, struct its_itte *itte)
501{
502 list_del(&itte->itte_list);
503
504 /* This put matches the get in vgic_add_lpi. */
505 vgic_put_irq(kvm, itte->irq);
506
507 kfree(itte);
508}
509
510static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
511{
512 return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
513}
514
515#define its_cmd_get_command(cmd) its_cmd_mask_field(cmd, 0, 0, 8)
516#define its_cmd_get_deviceid(cmd) its_cmd_mask_field(cmd, 0, 32, 32)
517#define its_cmd_get_id(cmd) its_cmd_mask_field(cmd, 1, 0, 32)
518#define its_cmd_get_physical_id(cmd) its_cmd_mask_field(cmd, 1, 32, 32)
519#define its_cmd_get_collection(cmd) its_cmd_mask_field(cmd, 2, 0, 16)
520#define its_cmd_get_target_addr(cmd) its_cmd_mask_field(cmd, 2, 16, 32)
521#define its_cmd_get_validbit(cmd) its_cmd_mask_field(cmd, 2, 63, 1)
522
523/*
524 * The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
525 * Must be called with the its_lock mutex held.
526 */
527static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
528 u64 *its_cmd)
529{
530 u32 device_id = its_cmd_get_deviceid(its_cmd);
531 u32 event_id = its_cmd_get_id(its_cmd);
532 struct its_itte *itte;
533
534
535 itte = find_itte(its, device_id, event_id);
536 if (itte && itte->collection) {
537 /*
538 * Though the spec talks about removing the pending state, we
539 * don't bother here since we clear the ITTE anyway and the
540 * pending state is a property of the ITTE struct.
541 */
542 its_free_itte(kvm, itte);
543 return 0;
544 }
545
546 return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
547}
548
549/*
550 * The MOVI command moves an ITTE to a different collection.
551 * Must be called with the its_lock mutex held.
552 */
553static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
554 u64 *its_cmd)
555{
556 u32 device_id = its_cmd_get_deviceid(its_cmd);
557 u32 event_id = its_cmd_get_id(its_cmd);
558 u32 coll_id = its_cmd_get_collection(its_cmd);
559 struct kvm_vcpu *vcpu;
560 struct its_itte *itte;
561 struct its_collection *collection;
562
563 itte = find_itte(its, device_id, event_id);
564 if (!itte)
565 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
566
567 if (!its_is_collection_mapped(itte->collection))
568 return E_ITS_MOVI_UNMAPPED_COLLECTION;
569
570 collection = find_collection(its, coll_id);
571 if (!its_is_collection_mapped(collection))
572 return E_ITS_MOVI_UNMAPPED_COLLECTION;
573
574 itte->collection = collection;
575 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
576
577 spin_lock(&itte->irq->irq_lock);
578 itte->irq->target_vcpu = vcpu;
579 spin_unlock(&itte->irq->irq_lock);
580
581 return 0;
582}
583
584/*
585 * Check whether an ID can be stored into the corresponding guest table.
586 * For a direct table this is pretty easy, but gets a bit nasty for
587 * indirect tables. We check whether the resulting guest physical address
588 * is actually valid (covered by a memslot and guest accessbible).
589 * For this we have to read the respective first level entry.
590 */
591static bool vgic_its_check_id(struct vgic_its *its, u64 baser, int id)
592{
593 int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
594 int index;
595 u64 indirect_ptr;
596 gfn_t gfn;
597
598 if (!(baser & GITS_BASER_INDIRECT)) {
599 phys_addr_t addr;
600
601 if (id >= (l1_tbl_size / GITS_BASER_ENTRY_SIZE(baser)))
602 return false;
603
604 addr = BASER_ADDRESS(baser) + id * GITS_BASER_ENTRY_SIZE(baser);
605 gfn = addr >> PAGE_SHIFT;
606
607 return kvm_is_visible_gfn(its->dev->kvm, gfn);
608 }
609
610 /* calculate and check the index into the 1st level */
611 index = id / (SZ_64K / GITS_BASER_ENTRY_SIZE(baser));
612 if (index >= (l1_tbl_size / sizeof(u64)))
613 return false;
614
615 /* Each 1st level entry is represented by a 64-bit value. */
616 if (kvm_read_guest(its->dev->kvm,
617 BASER_ADDRESS(baser) + index * sizeof(indirect_ptr),
618 &indirect_ptr, sizeof(indirect_ptr)))
619 return false;
620
621 indirect_ptr = le64_to_cpu(indirect_ptr);
622
623 /* check the valid bit of the first level entry */
624 if (!(indirect_ptr & BIT_ULL(63)))
625 return false;
626
627 /*
628 * Mask the guest physical address and calculate the frame number.
629 * Any address beyond our supported 48 bits of PA will be caught
630 * by the actual check in the final step.
631 */
632 indirect_ptr &= GENMASK_ULL(51, 16);
633
634 /* Find the address of the actual entry */
635 index = id % (SZ_64K / GITS_BASER_ENTRY_SIZE(baser));
636 indirect_ptr += index * GITS_BASER_ENTRY_SIZE(baser);
637 gfn = indirect_ptr >> PAGE_SHIFT;
638
639 return kvm_is_visible_gfn(its->dev->kvm, gfn);
640}
641
642static int vgic_its_alloc_collection(struct vgic_its *its,
643 struct its_collection **colp,
644 u32 coll_id)
645{
646 struct its_collection *collection;
647
648 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id))
649 return E_ITS_MAPC_COLLECTION_OOR;
650
651 collection = kzalloc(sizeof(*collection), GFP_KERNEL);
652
653 collection->collection_id = coll_id;
654 collection->target_addr = COLLECTION_NOT_MAPPED;
655
656 list_add_tail(&collection->coll_list, &its->collection_list);
657 *colp = collection;
658
659 return 0;
660}
661
662static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
663{
664 struct its_collection *collection;
665 struct its_device *device;
666 struct its_itte *itte;
667
668 /*
669 * Clearing the mapping for that collection ID removes the
670 * entry from the list. If there wasn't any before, we can
671 * go home early.
672 */
673 collection = find_collection(its, coll_id);
674 if (!collection)
675 return;
676
677 for_each_lpi_its(device, itte, its)
678 if (itte->collection &&
679 itte->collection->collection_id == coll_id)
680 itte->collection = NULL;
681
682 list_del(&collection->coll_list);
683 kfree(collection);
684}
685
686/*
687 * The MAPTI and MAPI commands map LPIs to ITTEs.
688 * Must be called with its_lock mutex held.
689 */
690static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
691 u64 *its_cmd)
692{
693 u32 device_id = its_cmd_get_deviceid(its_cmd);
694 u32 event_id = its_cmd_get_id(its_cmd);
695 u32 coll_id = its_cmd_get_collection(its_cmd);
696 struct its_itte *itte;
697 struct its_device *device;
698 struct its_collection *collection, *new_coll = NULL;
699 int lpi_nr;
700
701 device = find_its_device(its, device_id);
702 if (!device)
703 return E_ITS_MAPTI_UNMAPPED_DEVICE;
704
705 if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
706 lpi_nr = its_cmd_get_physical_id(its_cmd);
707 else
708 lpi_nr = event_id;
709 if (lpi_nr < GIC_LPI_OFFSET ||
710 lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
711 return E_ITS_MAPTI_PHYSICALID_OOR;
712
713 collection = find_collection(its, coll_id);
714 if (!collection) {
715 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
716 if (ret)
717 return ret;
718 new_coll = collection;
719 }
720
721 itte = find_itte(its, device_id, event_id);
722 if (!itte) {
723 itte = kzalloc(sizeof(struct its_itte), GFP_KERNEL);
724 if (!itte) {
725 if (new_coll)
726 vgic_its_free_collection(its, coll_id);
727 return -ENOMEM;
728 }
729
730 itte->event_id = event_id;
731 list_add_tail(&itte->itte_list, &device->itt_head);
732 }
733
734 itte->collection = collection;
735 itte->lpi = lpi_nr;
736 itte->irq = vgic_add_lpi(kvm, lpi_nr);
737 update_affinity_itte(kvm, itte);
738
739 /*
740 * We "cache" the configuration table entries in out struct vgic_irq's.
741 * However we only have those structs for mapped IRQs, so we read in
742 * the respective config data from memory here upon mapping the LPI.
743 */
744 update_lpi_config(kvm, itte->irq, NULL);
745
746 return 0;
747}
748
749/* Requires the its_lock to be held. */
750static void vgic_its_unmap_device(struct kvm *kvm, struct its_device *device)
751{
752 struct its_itte *itte, *temp;
753
754 /*
755 * The spec says that unmapping a device with still valid
756 * ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
757 * since we cannot leave the memory unreferenced.
758 */
759 list_for_each_entry_safe(itte, temp, &device->itt_head, itte_list)
760 its_free_itte(kvm, itte);
761
762 list_del(&device->dev_list);
763 kfree(device);
764}
765
766/*
767 * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
768 * Must be called with the its_lock mutex held.
769 */
770static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
771 u64 *its_cmd)
772{
773 u32 device_id = its_cmd_get_deviceid(its_cmd);
774 bool valid = its_cmd_get_validbit(its_cmd);
775 struct its_device *device;
776
777 if (!vgic_its_check_id(its, its->baser_device_table, device_id))
778 return E_ITS_MAPD_DEVICE_OOR;
779
780 device = find_its_device(its, device_id);
781
782 /*
783 * The spec says that calling MAPD on an already mapped device
784 * invalidates all cached data for this device. We implement this
785 * by removing the mapping and re-establishing it.
786 */
787 if (device)
788 vgic_its_unmap_device(kvm, device);
789
790 /*
791 * The spec does not say whether unmapping a not-mapped device
792 * is an error, so we are done in any case.
793 */
794 if (!valid)
795 return 0;
796
797 device = kzalloc(sizeof(struct its_device), GFP_KERNEL);
798 if (!device)
799 return -ENOMEM;
800
801 device->device_id = device_id;
802 INIT_LIST_HEAD(&device->itt_head);
803
804 list_add_tail(&device->dev_list, &its->device_list);
805
806 return 0;
807}
808
809/*
810 * The MAPC command maps collection IDs to redistributors.
811 * Must be called with the its_lock mutex held.
812 */
813static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
814 u64 *its_cmd)
815{
816 u16 coll_id;
817 u32 target_addr;
818 struct its_collection *collection;
819 bool valid;
820
821 valid = its_cmd_get_validbit(its_cmd);
822 coll_id = its_cmd_get_collection(its_cmd);
823 target_addr = its_cmd_get_target_addr(its_cmd);
824
825 if (target_addr >= atomic_read(&kvm->online_vcpus))
826 return E_ITS_MAPC_PROCNUM_OOR;
827
828 if (!valid) {
829 vgic_its_free_collection(its, coll_id);
830 } else {
831 collection = find_collection(its, coll_id);
832
833 if (!collection) {
834 int ret;
835
836 ret = vgic_its_alloc_collection(its, &collection,
837 coll_id);
838 if (ret)
839 return ret;
840 collection->target_addr = target_addr;
841 } else {
842 collection->target_addr = target_addr;
843 update_affinity_collection(kvm, its, collection);
844 }
845 }
846
847 return 0;
848}
849
850/*
851 * The CLEAR command removes the pending state for a particular LPI.
852 * Must be called with the its_lock mutex held.
853 */
854static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
855 u64 *its_cmd)
856{
857 u32 device_id = its_cmd_get_deviceid(its_cmd);
858 u32 event_id = its_cmd_get_id(its_cmd);
859 struct its_itte *itte;
860
861
862 itte = find_itte(its, device_id, event_id);
863 if (!itte)
864 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
865
866 itte->irq->pending = false;
867
868 return 0;
869}
870
871/*
872 * The INV command syncs the configuration bits from the memory table.
873 * Must be called with the its_lock mutex held.
874 */
875static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
876 u64 *its_cmd)
877{
878 u32 device_id = its_cmd_get_deviceid(its_cmd);
879 u32 event_id = its_cmd_get_id(its_cmd);
880 struct its_itte *itte;
881
882
883 itte = find_itte(its, device_id, event_id);
884 if (!itte)
885 return E_ITS_INV_UNMAPPED_INTERRUPT;
886
887 return update_lpi_config(kvm, itte->irq, NULL);
888}
889
890/*
891 * The INVALL command requests flushing of all IRQ data in this collection.
892 * Find the VCPU mapped to that collection, then iterate over the VM's list
893 * of mapped LPIs and update the configuration for each IRQ which targets
894 * the specified vcpu. The configuration will be read from the in-memory
895 * configuration table.
896 * Must be called with the its_lock mutex held.
897 */
898static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
899 u64 *its_cmd)
900{
901 u32 coll_id = its_cmd_get_collection(its_cmd);
902 struct its_collection *collection;
903 struct kvm_vcpu *vcpu;
904 struct vgic_irq *irq;
905 u32 *intids;
906 int irq_count, i;
907
908 collection = find_collection(its, coll_id);
909 if (!its_is_collection_mapped(collection))
910 return E_ITS_INVALL_UNMAPPED_COLLECTION;
911
912 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
913
914 irq_count = vgic_copy_lpi_list(kvm, &intids);
915 if (irq_count < 0)
916 return irq_count;
917
918 for (i = 0; i < irq_count; i++) {
919 irq = vgic_get_irq(kvm, NULL, intids[i]);
920 if (!irq)
921 continue;
922 update_lpi_config(kvm, irq, vcpu);
923 vgic_put_irq(kvm, irq);
924 }
925
926 kfree(intids);
927
928 return 0;
929}
930
931/*
932 * The MOVALL command moves the pending state of all IRQs targeting one
933 * redistributor to another. We don't hold the pending state in the VCPUs,
934 * but in the IRQs instead, so there is really not much to do for us here.
935 * However the spec says that no IRQ must target the old redistributor
936 * afterwards, so we make sure that no LPI is using the associated target_vcpu.
937 * This command affects all LPIs in the system that target that redistributor.
938 */
939static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
940 u64 *its_cmd)
941{
942 struct vgic_dist *dist = &kvm->arch.vgic;
943 u32 target1_addr = its_cmd_get_target_addr(its_cmd);
944 u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
945 struct kvm_vcpu *vcpu1, *vcpu2;
946 struct vgic_irq *irq;
947
948 if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
949 target2_addr >= atomic_read(&kvm->online_vcpus))
950 return E_ITS_MOVALL_PROCNUM_OOR;
951
952 if (target1_addr == target2_addr)
953 return 0;
954
955 vcpu1 = kvm_get_vcpu(kvm, target1_addr);
956 vcpu2 = kvm_get_vcpu(kvm, target2_addr);
957
958 spin_lock(&dist->lpi_list_lock);
959
960 list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
961 spin_lock(&irq->irq_lock);
962
963 if (irq->target_vcpu == vcpu1)
964 irq->target_vcpu = vcpu2;
965
966 spin_unlock(&irq->irq_lock);
967 }
968
969 spin_unlock(&dist->lpi_list_lock);
970
971 return 0;
972}
973
974/*
975 * The INT command injects the LPI associated with that DevID/EvID pair.
976 * Must be called with the its_lock mutex held.
977 */
978static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
979 u64 *its_cmd)
980{
981 u32 msi_data = its_cmd_get_id(its_cmd);
982 u64 msi_devid = its_cmd_get_deviceid(its_cmd);
983
984 vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
985
986 return 0;
987}
988
989/*
990 * This function is called with the its_cmd lock held, but the ITS data
991 * structure lock dropped.
992 */
993static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
994 u64 *its_cmd)
995{
996 int ret = -ENODEV;
997
998 mutex_lock(&its->its_lock);
999 switch (its_cmd_get_command(its_cmd)) {
1000 case GITS_CMD_MAPD:
1001 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1002 break;
1003 case GITS_CMD_MAPC:
1004 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1005 break;
1006 case GITS_CMD_MAPI:
1007 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1008 break;
1009 case GITS_CMD_MAPTI:
1010 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1011 break;
1012 case GITS_CMD_MOVI:
1013 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1014 break;
1015 case GITS_CMD_DISCARD:
1016 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1017 break;
1018 case GITS_CMD_CLEAR:
1019 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1020 break;
1021 case GITS_CMD_MOVALL:
1022 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1023 break;
1024 case GITS_CMD_INT:
1025 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1026 break;
1027 case GITS_CMD_INV:
1028 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1029 break;
1030 case GITS_CMD_INVALL:
1031 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1032 break;
1033 case GITS_CMD_SYNC:
1034 /* we ignore this command: we are in sync all of the time */
1035 ret = 0;
1036 break;
1037 }
1038 mutex_unlock(&its->its_lock);
1039
1040 return ret;
1041}
1042
1043static u64 vgic_sanitise_its_baser(u64 reg)
1044{
1045 reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1046 GITS_BASER_SHAREABILITY_SHIFT,
1047 vgic_sanitise_shareability);
1048 reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1049 GITS_BASER_INNER_CACHEABILITY_SHIFT,
1050 vgic_sanitise_inner_cacheability);
1051 reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1052 GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1053 vgic_sanitise_outer_cacheability);
1054
1055 /* Bits 15:12 contain bits 51:48 of the PA, which we don't support. */
1056 reg &= ~GENMASK_ULL(15, 12);
1057
1058 /* We support only one (ITS) page size: 64K */
1059 reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1060
1061 return reg;
1062}
1063
1064static u64 vgic_sanitise_its_cbaser(u64 reg)
1065{
1066 reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1067 GITS_CBASER_SHAREABILITY_SHIFT,
1068 vgic_sanitise_shareability);
1069 reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1070 GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1071 vgic_sanitise_inner_cacheability);
1072 reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1073 GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1074 vgic_sanitise_outer_cacheability);
1075
1076 /*
1077 * Sanitise the physical address to be 64k aligned.
1078 * Also limit the physical addresses to 48 bits.
1079 */
1080 reg &= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
1081
1082 return reg;
1083}
1084
1085static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1086 struct vgic_its *its,
1087 gpa_t addr, unsigned int len)
1088{
1089 return extract_bytes(its->cbaser, addr & 7, len);
1090}
1091
1092static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1093 gpa_t addr, unsigned int len,
1094 unsigned long val)
1095{
1096 /* When GITS_CTLR.Enable is 1, this register is RO. */
1097 if (its->enabled)
1098 return;
1099
1100 mutex_lock(&its->cmd_lock);
1101 its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1102 its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1103 its->creadr = 0;
1104 /*
1105 * CWRITER is architecturally UNKNOWN on reset, but we need to reset
1106 * it to CREADR to make sure we start with an empty command buffer.
1107 */
1108 its->cwriter = its->creadr;
1109 mutex_unlock(&its->cmd_lock);
1110}
1111
1112#define ITS_CMD_BUFFER_SIZE(baser) ((((baser) & 0xff) + 1) << 12)
1113#define ITS_CMD_SIZE 32
1114#define ITS_CMD_OFFSET(reg) ((reg) & GENMASK(19, 5))
1115
1116/*
1117 * By writing to CWRITER the guest announces new commands to be processed.
1118 * To avoid any races in the first place, we take the its_cmd lock, which
1119 * protects our ring buffer variables, so that there is only one user
1120 * per ITS handling commands at a given time.
1121 */
1122static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1123 gpa_t addr, unsigned int len,
1124 unsigned long val)
1125{
1126 gpa_t cbaser;
1127 u64 cmd_buf[4];
1128 u32 reg;
1129
1130 if (!its)
1131 return;
1132
1133 mutex_lock(&its->cmd_lock);
1134
1135 reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1136 reg = ITS_CMD_OFFSET(reg);
1137 if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1138 mutex_unlock(&its->cmd_lock);
1139 return;
1140 }
1141
1142 its->cwriter = reg;
1143 cbaser = CBASER_ADDRESS(its->cbaser);
1144
1145 while (its->cwriter != its->creadr) {
1146 int ret = kvm_read_guest(kvm, cbaser + its->creadr,
1147 cmd_buf, ITS_CMD_SIZE);
1148 /*
1149 * If kvm_read_guest() fails, this could be due to the guest
1150 * programming a bogus value in CBASER or something else going
1151 * wrong from which we cannot easily recover.
1152 * According to section 6.3.2 in the GICv3 spec we can just
1153 * ignore that command then.
1154 */
1155 if (!ret)
1156 vgic_its_handle_command(kvm, its, cmd_buf);
1157
1158 its->creadr += ITS_CMD_SIZE;
1159 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1160 its->creadr = 0;
1161 }
1162
1163 mutex_unlock(&its->cmd_lock);
1164}
1165
1166static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1167 struct vgic_its *its,
1168 gpa_t addr, unsigned int len)
1169{
1170 return extract_bytes(its->cwriter, addr & 0x7, len);
1171}
1172
1173static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1174 struct vgic_its *its,
1175 gpa_t addr, unsigned int len)
1176{
1177 return extract_bytes(its->creadr, addr & 0x7, len);
1178}
1179
1180#define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1181static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1182 struct vgic_its *its,
1183 gpa_t addr, unsigned int len)
1184{
1185 u64 reg;
1186
1187 switch (BASER_INDEX(addr)) {
1188 case 0:
1189 reg = its->baser_device_table;
1190 break;
1191 case 1:
1192 reg = its->baser_coll_table;
1193 break;
1194 default:
1195 reg = 0;
1196 break;
1197 }
1198
1199 return extract_bytes(reg, addr & 7, len);
1200}
1201
1202#define GITS_BASER_RO_MASK (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1203static void vgic_mmio_write_its_baser(struct kvm *kvm,
1204 struct vgic_its *its,
1205 gpa_t addr, unsigned int len,
1206 unsigned long val)
1207{
1208 u64 entry_size, device_type;
1209 u64 reg, *regptr, clearbits = 0;
1210
1211 /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1212 if (its->enabled)
1213 return;
1214
1215 switch (BASER_INDEX(addr)) {
1216 case 0:
1217 regptr = &its->baser_device_table;
1218 entry_size = 8;
1219 device_type = GITS_BASER_TYPE_DEVICE;
1220 break;
1221 case 1:
1222 regptr = &its->baser_coll_table;
1223 entry_size = 8;
1224 device_type = GITS_BASER_TYPE_COLLECTION;
1225 clearbits = GITS_BASER_INDIRECT;
1226 break;
1227 default:
1228 return;
1229 }
1230
1231 reg = update_64bit_reg(*regptr, addr & 7, len, val);
1232 reg &= ~GITS_BASER_RO_MASK;
1233 reg &= ~clearbits;
1234
1235 reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1236 reg |= device_type << GITS_BASER_TYPE_SHIFT;
1237 reg = vgic_sanitise_its_baser(reg);
1238
1239 *regptr = reg;
1240}
1241
1242#define REGISTER_ITS_DESC(off, rd, wr, length, acc) \
1243{ \
1244 .reg_offset = off, \
1245 .len = length, \
1246 .access_flags = acc, \
1247 .its_read = rd, \
1248 .its_write = wr, \
1249}
1250
1251static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1252 gpa_t addr, unsigned int len, unsigned long val)
1253{
1254 /* Ignore */
1255}
1256
1257static struct vgic_register_region its_registers[] = {
1258 REGISTER_ITS_DESC(GITS_CTLR,
1259 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1260 VGIC_ACCESS_32bit),
1261 REGISTER_ITS_DESC(GITS_IIDR,
1262 vgic_mmio_read_its_iidr, its_mmio_write_wi, 4,
1263 VGIC_ACCESS_32bit),
1264 REGISTER_ITS_DESC(GITS_TYPER,
1265 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1266 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1267 REGISTER_ITS_DESC(GITS_CBASER,
1268 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1269 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1270 REGISTER_ITS_DESC(GITS_CWRITER,
1271 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1272 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1273 REGISTER_ITS_DESC(GITS_CREADR,
1274 vgic_mmio_read_its_creadr, its_mmio_write_wi, 8,
1275 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1276 REGISTER_ITS_DESC(GITS_BASER,
1277 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1278 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1279 REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1280 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1281 VGIC_ACCESS_32bit),
1282};
1283
1284/* This is called on setting the LPI enable bit in the redistributor. */
1285void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1286{
1287 if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1288 its_sync_lpi_pending_table(vcpu);
1289}
1290
1291static int vgic_its_init_its(struct kvm *kvm, struct vgic_its *its)
1292{
1293 struct vgic_io_device *iodev = &its->iodev;
1294 int ret;
1295
1296 if (its->initialized)
1297 return 0;
1298
1299 if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base))
1300 return -ENXIO;
1301
1302 iodev->regions = its_registers;
1303 iodev->nr_regions = ARRAY_SIZE(its_registers);
1304 kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1305
1306 iodev->base_addr = its->vgic_its_base;
1307 iodev->iodev_type = IODEV_ITS;
1308 iodev->its = its;
1309 mutex_lock(&kvm->slots_lock);
1310 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1311 KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1312 mutex_unlock(&kvm->slots_lock);
1313
1314 if (!ret)
1315 its->initialized = true;
1316
1317 return ret;
1318}
1319
1320#define INITIAL_BASER_VALUE \
1321 (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) | \
1322 GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) | \
1323 GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) | \
1324 ((8ULL - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | \
1325 GITS_BASER_PAGE_SIZE_64K)
1326
1327#define INITIAL_PROPBASER_VALUE \
1328 (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb) | \
1329 GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner) | \
1330 GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1331
1332static int vgic_its_create(struct kvm_device *dev, u32 type)
1333{
1334 struct vgic_its *its;
1335
1336 if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1337 return -ENODEV;
1338
1339 its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL);
1340 if (!its)
1341 return -ENOMEM;
1342
1343 mutex_init(&its->its_lock);
1344 mutex_init(&its->cmd_lock);
1345
1346 its->vgic_its_base = VGIC_ADDR_UNDEF;
1347
1348 INIT_LIST_HEAD(&its->device_list);
1349 INIT_LIST_HEAD(&its->collection_list);
1350
1351 dev->kvm->arch.vgic.has_its = true;
1352 its->initialized = false;
1353 its->enabled = false;
1354 its->dev = dev;
1355
1356 its->baser_device_table = INITIAL_BASER_VALUE |
1357 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1358 its->baser_coll_table = INITIAL_BASER_VALUE |
1359 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1360 dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1361
1362 dev->private = its;
1363
1364 return 0;
1365}
1366
1367static void vgic_its_destroy(struct kvm_device *kvm_dev)
1368{
1369 struct kvm *kvm = kvm_dev->kvm;
1370 struct vgic_its *its = kvm_dev->private;
1371 struct its_device *dev;
1372 struct its_itte *itte;
1373 struct list_head *dev_cur, *dev_temp;
1374 struct list_head *cur, *temp;
1375
1376 /*
1377 * We may end up here without the lists ever having been initialized.
1378 * Check this and bail out early to avoid dereferencing a NULL pointer.
1379 */
1380 if (!its->device_list.next)
1381 return;
1382
1383 mutex_lock(&its->its_lock);
1384 list_for_each_safe(dev_cur, dev_temp, &its->device_list) {
1385 dev = container_of(dev_cur, struct its_device, dev_list);
1386 list_for_each_safe(cur, temp, &dev->itt_head) {
1387 itte = (container_of(cur, struct its_itte, itte_list));
1388 its_free_itte(kvm, itte);
1389 }
1390 list_del(dev_cur);
1391 kfree(dev);
1392 }
1393
1394 list_for_each_safe(cur, temp, &its->collection_list) {
1395 list_del(cur);
1396 kfree(container_of(cur, struct its_collection, coll_list));
1397 }
1398 mutex_unlock(&its->its_lock);
1399
1400 kfree(its);
1401}
1402
1403static int vgic_its_has_attr(struct kvm_device *dev,
1404 struct kvm_device_attr *attr)
1405{
1406 switch (attr->group) {
1407 case KVM_DEV_ARM_VGIC_GRP_ADDR:
1408 switch (attr->attr) {
1409 case KVM_VGIC_ITS_ADDR_TYPE:
1410 return 0;
1411 }
1412 break;
1413 case KVM_DEV_ARM_VGIC_GRP_CTRL:
1414 switch (attr->attr) {
1415 case KVM_DEV_ARM_VGIC_CTRL_INIT:
1416 return 0;
1417 }
1418 break;
1419 }
1420 return -ENXIO;
1421}
1422
1423static int vgic_its_set_attr(struct kvm_device *dev,
1424 struct kvm_device_attr *attr)
1425{
1426 struct vgic_its *its = dev->private;
1427 int ret;
1428
1429 switch (attr->group) {
1430 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
1431 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
1432 unsigned long type = (unsigned long)attr->attr;
1433 u64 addr;
1434
1435 if (type != KVM_VGIC_ITS_ADDR_TYPE)
1436 return -ENODEV;
1437
1438 if (its->initialized)
1439 return -EBUSY;
1440
1441 if (copy_from_user(&addr, uaddr, sizeof(addr)))
1442 return -EFAULT;
1443
1444 ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base,
1445 addr, SZ_64K);
1446 if (ret)
1447 return ret;
1448
1449 its->vgic_its_base = addr;
1450
1451 return 0;
1452 }
1453 case KVM_DEV_ARM_VGIC_GRP_CTRL:
1454 switch (attr->attr) {
1455 case KVM_DEV_ARM_VGIC_CTRL_INIT:
1456 return vgic_its_init_its(dev->kvm, its);
1457 }
1458 break;
1459 }
1460 return -ENXIO;
1461}
1462
1463static int vgic_its_get_attr(struct kvm_device *dev,
1464 struct kvm_device_attr *attr)
1465{
1466 switch (attr->group) {
1467 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
1468 struct vgic_its *its = dev->private;
1469 u64 addr = its->vgic_its_base;
1470 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
1471 unsigned long type = (unsigned long)attr->attr;
1472
1473 if (type != KVM_VGIC_ITS_ADDR_TYPE)
1474 return -ENODEV;
1475
1476 if (copy_to_user(uaddr, &addr, sizeof(addr)))
1477 return -EFAULT;
1478 break;
1479 default:
1480 return -ENXIO;
1481 }
1482 }
1483
1484 return 0;
1485}
1486
1487static struct kvm_device_ops kvm_arm_vgic_its_ops = {
1488 .name = "kvm-arm-vgic-its",
1489 .create = vgic_its_create,
1490 .destroy = vgic_its_destroy,
1491 .set_attr = vgic_its_set_attr,
1492 .get_attr = vgic_its_get_attr,
1493 .has_attr = vgic_its_has_attr,
1494};
1495
1496int kvm_vgic_register_its_device(void)
1497{
1498 return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
1499 KVM_DEV_TYPE_ARM_VGIC_ITS);
1500}
diff --git a/virt/kvm/arm/vgic/vgic-kvm-device.c b/virt/kvm/arm/vgic/vgic-kvm-device.c
index 0130c4b147b7..1813f93b5cde 100644
--- a/virt/kvm/arm/vgic/vgic-kvm-device.c
+++ b/virt/kvm/arm/vgic/vgic-kvm-device.c
@@ -21,8 +21,8 @@
21 21
22/* common helpers */ 22/* common helpers */
23 23
24static int vgic_check_ioaddr(struct kvm *kvm, phys_addr_t *ioaddr, 24int vgic_check_ioaddr(struct kvm *kvm, phys_addr_t *ioaddr,
25 phys_addr_t addr, phys_addr_t alignment) 25 phys_addr_t addr, phys_addr_t alignment)
26{ 26{
27 if (addr & ~KVM_PHYS_MASK) 27 if (addr & ~KVM_PHYS_MASK)
28 return -E2BIG; 28 return -E2BIG;
@@ -210,20 +210,27 @@ static void vgic_destroy(struct kvm_device *dev)
210 kfree(dev); 210 kfree(dev);
211} 211}
212 212
213void kvm_register_vgic_device(unsigned long type) 213int kvm_register_vgic_device(unsigned long type)
214{ 214{
215 int ret = -ENODEV;
216
215 switch (type) { 217 switch (type) {
216 case KVM_DEV_TYPE_ARM_VGIC_V2: 218 case KVM_DEV_TYPE_ARM_VGIC_V2:
217 kvm_register_device_ops(&kvm_arm_vgic_v2_ops, 219 ret = kvm_register_device_ops(&kvm_arm_vgic_v2_ops,
218 KVM_DEV_TYPE_ARM_VGIC_V2); 220 KVM_DEV_TYPE_ARM_VGIC_V2);
219 break; 221 break;
220#ifdef CONFIG_KVM_ARM_VGIC_V3 222#ifdef CONFIG_KVM_ARM_VGIC_V3
221 case KVM_DEV_TYPE_ARM_VGIC_V3: 223 case KVM_DEV_TYPE_ARM_VGIC_V3:
222 kvm_register_device_ops(&kvm_arm_vgic_v3_ops, 224 ret = kvm_register_device_ops(&kvm_arm_vgic_v3_ops,
223 KVM_DEV_TYPE_ARM_VGIC_V3); 225 KVM_DEV_TYPE_ARM_VGIC_V3);
226 if (ret)
227 break;
228 ret = kvm_vgic_register_its_device();
224 break; 229 break;
225#endif 230#endif
226 } 231 }
232
233 return ret;
227} 234}
228 235
229/** vgic_attr_regs_access: allows user space to read/write VGIC registers 236/** vgic_attr_regs_access: allows user space to read/write VGIC registers
@@ -428,4 +435,3 @@ struct kvm_device_ops kvm_arm_vgic_v3_ops = {
428}; 435};
429 436
430#endif /* CONFIG_KVM_ARM_VGIC_V3 */ 437#endif /* CONFIG_KVM_ARM_VGIC_V3 */
431
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v2.c b/virt/kvm/arm/vgic/vgic-mmio-v2.c
index a21393637e4b..b44b359cbbad 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v2.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v2.c
@@ -102,6 +102,7 @@ static void vgic_mmio_write_sgir(struct kvm_vcpu *source_vcpu,
102 irq->source |= 1U << source_vcpu->vcpu_id; 102 irq->source |= 1U << source_vcpu->vcpu_id;
103 103
104 vgic_queue_irq_unlock(source_vcpu->kvm, irq); 104 vgic_queue_irq_unlock(source_vcpu->kvm, irq);
105 vgic_put_irq(source_vcpu->kvm, irq);
105 } 106 }
106} 107}
107 108
@@ -116,6 +117,8 @@ static unsigned long vgic_mmio_read_target(struct kvm_vcpu *vcpu,
116 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 117 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
117 118
118 val |= (u64)irq->targets << (i * 8); 119 val |= (u64)irq->targets << (i * 8);
120
121 vgic_put_irq(vcpu->kvm, irq);
119 } 122 }
120 123
121 return val; 124 return val;
@@ -143,6 +146,7 @@ static void vgic_mmio_write_target(struct kvm_vcpu *vcpu,
143 irq->target_vcpu = kvm_get_vcpu(vcpu->kvm, target); 146 irq->target_vcpu = kvm_get_vcpu(vcpu->kvm, target);
144 147
145 spin_unlock(&irq->irq_lock); 148 spin_unlock(&irq->irq_lock);
149 vgic_put_irq(vcpu->kvm, irq);
146 } 150 }
147} 151}
148 152
@@ -157,6 +161,8 @@ static unsigned long vgic_mmio_read_sgipend(struct kvm_vcpu *vcpu,
157 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 161 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
158 162
159 val |= (u64)irq->source << (i * 8); 163 val |= (u64)irq->source << (i * 8);
164
165 vgic_put_irq(vcpu->kvm, irq);
160 } 166 }
161 return val; 167 return val;
162} 168}
@@ -178,6 +184,7 @@ static void vgic_mmio_write_sgipendc(struct kvm_vcpu *vcpu,
178 irq->pending = false; 184 irq->pending = false;
179 185
180 spin_unlock(&irq->irq_lock); 186 spin_unlock(&irq->irq_lock);
187 vgic_put_irq(vcpu->kvm, irq);
181 } 188 }
182} 189}
183 190
@@ -201,6 +208,7 @@ static void vgic_mmio_write_sgipends(struct kvm_vcpu *vcpu,
201 } else { 208 } else {
202 spin_unlock(&irq->irq_lock); 209 spin_unlock(&irq->irq_lock);
203 } 210 }
211 vgic_put_irq(vcpu->kvm, irq);
204 } 212 }
205} 213}
206 214
@@ -429,6 +437,7 @@ int vgic_v2_cpuif_uaccess(struct kvm_vcpu *vcpu, bool is_write,
429 struct vgic_io_device dev = { 437 struct vgic_io_device dev = {
430 .regions = vgic_v2_cpu_registers, 438 .regions = vgic_v2_cpu_registers,
431 .nr_regions = ARRAY_SIZE(vgic_v2_cpu_registers), 439 .nr_regions = ARRAY_SIZE(vgic_v2_cpu_registers),
440 .iodev_type = IODEV_CPUIF,
432 }; 441 };
433 442
434 return vgic_uaccess(vcpu, &dev, is_write, offset, val); 443 return vgic_uaccess(vcpu, &dev, is_write, offset, val);
@@ -440,6 +449,7 @@ int vgic_v2_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
440 struct vgic_io_device dev = { 449 struct vgic_io_device dev = {
441 .regions = vgic_v2_dist_registers, 450 .regions = vgic_v2_dist_registers,
442 .nr_regions = ARRAY_SIZE(vgic_v2_dist_registers), 451 .nr_regions = ARRAY_SIZE(vgic_v2_dist_registers),
452 .iodev_type = IODEV_DIST,
443 }; 453 };
444 454
445 return vgic_uaccess(vcpu, &dev, is_write, offset, val); 455 return vgic_uaccess(vcpu, &dev, is_write, offset, val);
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c
index a0c515a412a7..ff668e0dd586 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v3.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c
@@ -23,12 +23,35 @@
23#include "vgic-mmio.h" 23#include "vgic-mmio.h"
24 24
25/* extract @num bytes at @offset bytes offset in data */ 25/* extract @num bytes at @offset bytes offset in data */
26static unsigned long extract_bytes(unsigned long data, unsigned int offset, 26unsigned long extract_bytes(unsigned long data, unsigned int offset,
27 unsigned int num) 27 unsigned int num)
28{ 28{
29 return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0); 29 return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0);
30} 30}
31 31
32/* allows updates of any half of a 64-bit register (or the whole thing) */
33u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len,
34 unsigned long val)
35{
36 int lower = (offset & 4) * 8;
37 int upper = lower + 8 * len - 1;
38
39 reg &= ~GENMASK_ULL(upper, lower);
40 val &= GENMASK_ULL(len * 8 - 1, 0);
41
42 return reg | ((u64)val << lower);
43}
44
45bool vgic_has_its(struct kvm *kvm)
46{
47 struct vgic_dist *dist = &kvm->arch.vgic;
48
49 if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
50 return false;
51
52 return dist->has_its;
53}
54
32static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, 55static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
33 gpa_t addr, unsigned int len) 56 gpa_t addr, unsigned int len)
34{ 57{
@@ -43,7 +66,12 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
43 case GICD_TYPER: 66 case GICD_TYPER:
44 value = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; 67 value = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
45 value = (value >> 5) - 1; 68 value = (value >> 5) - 1;
46 value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19; 69 if (vgic_has_its(vcpu->kvm)) {
70 value |= (INTERRUPT_ID_BITS_ITS - 1) << 19;
71 value |= GICD_TYPER_LPIS;
72 } else {
73 value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19;
74 }
47 break; 75 break;
48 case GICD_IIDR: 76 case GICD_IIDR:
49 value = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0); 77 value = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
@@ -80,15 +108,17 @@ static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu,
80{ 108{
81 int intid = VGIC_ADDR_TO_INTID(addr, 64); 109 int intid = VGIC_ADDR_TO_INTID(addr, 64);
82 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid); 110 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid);
111 unsigned long ret = 0;
83 112
84 if (!irq) 113 if (!irq)
85 return 0; 114 return 0;
86 115
87 /* The upper word is RAZ for us. */ 116 /* The upper word is RAZ for us. */
88 if (addr & 4) 117 if (!(addr & 4))
89 return 0; 118 ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len);
90 119
91 return extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len); 120 vgic_put_irq(vcpu->kvm, irq);
121 return ret;
92} 122}
93 123
94static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu, 124static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
@@ -96,15 +126,17 @@ static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
96 unsigned long val) 126 unsigned long val)
97{ 127{
98 int intid = VGIC_ADDR_TO_INTID(addr, 64); 128 int intid = VGIC_ADDR_TO_INTID(addr, 64);
99 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid); 129 struct vgic_irq *irq;
100
101 if (!irq)
102 return;
103 130
104 /* The upper word is WI for us since we don't implement Aff3. */ 131 /* The upper word is WI for us since we don't implement Aff3. */
105 if (addr & 4) 132 if (addr & 4)
106 return; 133 return;
107 134
135 irq = vgic_get_irq(vcpu->kvm, NULL, intid);
136
137 if (!irq)
138 return;
139
108 spin_lock(&irq->irq_lock); 140 spin_lock(&irq->irq_lock);
109 141
110 /* We only care about and preserve Aff0, Aff1 and Aff2. */ 142 /* We only care about and preserve Aff0, Aff1 and Aff2. */
@@ -112,6 +144,32 @@ static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
112 irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr); 144 irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr);
113 145
114 spin_unlock(&irq->irq_lock); 146 spin_unlock(&irq->irq_lock);
147 vgic_put_irq(vcpu->kvm, irq);
148}
149
150static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu,
151 gpa_t addr, unsigned int len)
152{
153 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
154
155 return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0;
156}
157
158
159static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
160 gpa_t addr, unsigned int len,
161 unsigned long val)
162{
163 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
164 bool was_enabled = vgic_cpu->lpis_enabled;
165
166 if (!vgic_has_its(vcpu->kvm))
167 return;
168
169 vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS;
170
171 if (!was_enabled && vgic_cpu->lpis_enabled)
172 vgic_enable_lpis(vcpu);
115} 173}
116 174
117static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu, 175static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu,
@@ -125,6 +183,8 @@ static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu,
125 value |= ((target_vcpu_id & 0xffff) << 8); 183 value |= ((target_vcpu_id & 0xffff) << 8);
126 if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1) 184 if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1)
127 value |= GICR_TYPER_LAST; 185 value |= GICR_TYPER_LAST;
186 if (vgic_has_its(vcpu->kvm))
187 value |= GICR_TYPER_PLPIS;
128 188
129 return extract_bytes(value, addr & 7, len); 189 return extract_bytes(value, addr & 7, len);
130} 190}
@@ -147,6 +207,142 @@ static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu,
147 return 0; 207 return 0;
148} 208}
149 209
210/* We want to avoid outer shareable. */
211u64 vgic_sanitise_shareability(u64 field)
212{
213 switch (field) {
214 case GIC_BASER_OuterShareable:
215 return GIC_BASER_InnerShareable;
216 default:
217 return field;
218 }
219}
220
221/* Avoid any inner non-cacheable mapping. */
222u64 vgic_sanitise_inner_cacheability(u64 field)
223{
224 switch (field) {
225 case GIC_BASER_CACHE_nCnB:
226 case GIC_BASER_CACHE_nC:
227 return GIC_BASER_CACHE_RaWb;
228 default:
229 return field;
230 }
231}
232
233/* Non-cacheable or same-as-inner are OK. */
234u64 vgic_sanitise_outer_cacheability(u64 field)
235{
236 switch (field) {
237 case GIC_BASER_CACHE_SameAsInner:
238 case GIC_BASER_CACHE_nC:
239 return field;
240 default:
241 return GIC_BASER_CACHE_nC;
242 }
243}
244
245u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift,
246 u64 (*sanitise_fn)(u64))
247{
248 u64 field = (reg & field_mask) >> field_shift;
249
250 field = sanitise_fn(field) << field_shift;
251 return (reg & ~field_mask) | field;
252}
253
254#define PROPBASER_RES0_MASK \
255 (GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5))
256#define PENDBASER_RES0_MASK \
257 (BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) | \
258 GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0))
259
260static u64 vgic_sanitise_pendbaser(u64 reg)
261{
262 reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK,
263 GICR_PENDBASER_SHAREABILITY_SHIFT,
264 vgic_sanitise_shareability);
265 reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK,
266 GICR_PENDBASER_INNER_CACHEABILITY_SHIFT,
267 vgic_sanitise_inner_cacheability);
268 reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK,
269 GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT,
270 vgic_sanitise_outer_cacheability);
271
272 reg &= ~PENDBASER_RES0_MASK;
273 reg &= ~GENMASK_ULL(51, 48);
274
275 return reg;
276}
277
278static u64 vgic_sanitise_propbaser(u64 reg)
279{
280 reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK,
281 GICR_PROPBASER_SHAREABILITY_SHIFT,
282 vgic_sanitise_shareability);
283 reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK,
284 GICR_PROPBASER_INNER_CACHEABILITY_SHIFT,
285 vgic_sanitise_inner_cacheability);
286 reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK,
287 GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT,
288 vgic_sanitise_outer_cacheability);
289
290 reg &= ~PROPBASER_RES0_MASK;
291 reg &= ~GENMASK_ULL(51, 48);
292 return reg;
293}
294
295static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu,
296 gpa_t addr, unsigned int len)
297{
298 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
299
300 return extract_bytes(dist->propbaser, addr & 7, len);
301}
302
303static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu,
304 gpa_t addr, unsigned int len,
305 unsigned long val)
306{
307 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
308 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
309 u64 propbaser = dist->propbaser;
310
311 /* Storing a value with LPIs already enabled is undefined */
312 if (vgic_cpu->lpis_enabled)
313 return;
314
315 propbaser = update_64bit_reg(propbaser, addr & 4, len, val);
316 propbaser = vgic_sanitise_propbaser(propbaser);
317
318 dist->propbaser = propbaser;
319}
320
321static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu,
322 gpa_t addr, unsigned int len)
323{
324 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
325
326 return extract_bytes(vgic_cpu->pendbaser, addr & 7, len);
327}
328
329static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu,
330 gpa_t addr, unsigned int len,
331 unsigned long val)
332{
333 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
334 u64 pendbaser = vgic_cpu->pendbaser;
335
336 /* Storing a value with LPIs already enabled is undefined */
337 if (vgic_cpu->lpis_enabled)
338 return;
339
340 pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val);
341 pendbaser = vgic_sanitise_pendbaser(pendbaser);
342
343 vgic_cpu->pendbaser = pendbaser;
344}
345
150/* 346/*
151 * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the 347 * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the
152 * redistributors, while SPIs are covered by registers in the distributor 348 * redistributors, while SPIs are covered by registers in the distributor
@@ -218,7 +414,7 @@ static const struct vgic_register_region vgic_v3_dist_registers[] = {
218 414
219static const struct vgic_register_region vgic_v3_rdbase_registers[] = { 415static const struct vgic_register_region vgic_v3_rdbase_registers[] = {
220 REGISTER_DESC_WITH_LENGTH(GICR_CTLR, 416 REGISTER_DESC_WITH_LENGTH(GICR_CTLR,
221 vgic_mmio_read_raz, vgic_mmio_write_wi, 4, 417 vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,
222 VGIC_ACCESS_32bit), 418 VGIC_ACCESS_32bit),
223 REGISTER_DESC_WITH_LENGTH(GICR_IIDR, 419 REGISTER_DESC_WITH_LENGTH(GICR_IIDR,
224 vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4, 420 vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4,
@@ -227,10 +423,10 @@ static const struct vgic_register_region vgic_v3_rdbase_registers[] = {
227 vgic_mmio_read_v3r_typer, vgic_mmio_write_wi, 8, 423 vgic_mmio_read_v3r_typer, vgic_mmio_write_wi, 8,
228 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), 424 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
229 REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER, 425 REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER,
230 vgic_mmio_read_raz, vgic_mmio_write_wi, 8, 426 vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8,
231 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), 427 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
232 REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER, 428 REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER,
233 vgic_mmio_read_raz, vgic_mmio_write_wi, 8, 429 vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8,
234 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), 430 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
235 REGISTER_DESC_WITH_LENGTH(GICR_IDREGS, 431 REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
236 vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, 432 vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
@@ -285,24 +481,18 @@ unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev)
285 481
286int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t redist_base_address) 482int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t redist_base_address)
287{ 483{
288 int nr_vcpus = atomic_read(&kvm->online_vcpus);
289 struct kvm_vcpu *vcpu; 484 struct kvm_vcpu *vcpu;
290 struct vgic_io_device *devices;
291 int c, ret = 0; 485 int c, ret = 0;
292 486
293 devices = kmalloc(sizeof(struct vgic_io_device) * nr_vcpus * 2,
294 GFP_KERNEL);
295 if (!devices)
296 return -ENOMEM;
297
298 kvm_for_each_vcpu(c, vcpu, kvm) { 487 kvm_for_each_vcpu(c, vcpu, kvm) {
299 gpa_t rd_base = redist_base_address + c * SZ_64K * 2; 488 gpa_t rd_base = redist_base_address + c * SZ_64K * 2;
300 gpa_t sgi_base = rd_base + SZ_64K; 489 gpa_t sgi_base = rd_base + SZ_64K;
301 struct vgic_io_device *rd_dev = &devices[c * 2]; 490 struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
302 struct vgic_io_device *sgi_dev = &devices[c * 2 + 1]; 491 struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev;
303 492
304 kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops); 493 kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
305 rd_dev->base_addr = rd_base; 494 rd_dev->base_addr = rd_base;
495 rd_dev->iodev_type = IODEV_REDIST;
306 rd_dev->regions = vgic_v3_rdbase_registers; 496 rd_dev->regions = vgic_v3_rdbase_registers;
307 rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers); 497 rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers);
308 rd_dev->redist_vcpu = vcpu; 498 rd_dev->redist_vcpu = vcpu;
@@ -317,6 +507,7 @@ int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t redist_base_address)
317 507
318 kvm_iodevice_init(&sgi_dev->dev, &kvm_io_gic_ops); 508 kvm_iodevice_init(&sgi_dev->dev, &kvm_io_gic_ops);
319 sgi_dev->base_addr = sgi_base; 509 sgi_dev->base_addr = sgi_base;
510 sgi_dev->iodev_type = IODEV_REDIST;
320 sgi_dev->regions = vgic_v3_sgibase_registers; 511 sgi_dev->regions = vgic_v3_sgibase_registers;
321 sgi_dev->nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers); 512 sgi_dev->nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers);
322 sgi_dev->redist_vcpu = vcpu; 513 sgi_dev->redist_vcpu = vcpu;
@@ -335,14 +526,15 @@ int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t redist_base_address)
335 if (ret) { 526 if (ret) {
336 /* The current c failed, so we start with the previous one. */ 527 /* The current c failed, so we start with the previous one. */
337 for (c--; c >= 0; c--) { 528 for (c--; c >= 0; c--) {
529 struct vgic_cpu *vgic_cpu;
530
531 vcpu = kvm_get_vcpu(kvm, c);
532 vgic_cpu = &vcpu->arch.vgic_cpu;
338 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, 533 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
339 &devices[c * 2].dev); 534 &vgic_cpu->rd_iodev.dev);
340 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, 535 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
341 &devices[c * 2 + 1].dev); 536 &vgic_cpu->sgi_iodev.dev);
342 } 537 }
343 kfree(devices);
344 } else {
345 kvm->arch.vgic.redist_iodevs = devices;
346 } 538 }
347 539
348 return ret; 540 return ret;
@@ -451,5 +643,6 @@ void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg)
451 irq->pending = true; 643 irq->pending = true;
452 644
453 vgic_queue_irq_unlock(vcpu->kvm, irq); 645 vgic_queue_irq_unlock(vcpu->kvm, irq);
646 vgic_put_irq(vcpu->kvm, irq);
454 } 647 }
455} 648}
diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c
index 9f6fab74dce7..3bad3c5ed431 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.c
+++ b/virt/kvm/arm/vgic/vgic-mmio.c
@@ -56,6 +56,8 @@ unsigned long vgic_mmio_read_enable(struct kvm_vcpu *vcpu,
56 56
57 if (irq->enabled) 57 if (irq->enabled)
58 value |= (1U << i); 58 value |= (1U << i);
59
60 vgic_put_irq(vcpu->kvm, irq);
59 } 61 }
60 62
61 return value; 63 return value;
@@ -74,6 +76,8 @@ void vgic_mmio_write_senable(struct kvm_vcpu *vcpu,
74 spin_lock(&irq->irq_lock); 76 spin_lock(&irq->irq_lock);
75 irq->enabled = true; 77 irq->enabled = true;
76 vgic_queue_irq_unlock(vcpu->kvm, irq); 78 vgic_queue_irq_unlock(vcpu->kvm, irq);
79
80 vgic_put_irq(vcpu->kvm, irq);
77 } 81 }
78} 82}
79 83
@@ -92,6 +96,7 @@ void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu,
92 irq->enabled = false; 96 irq->enabled = false;
93 97
94 spin_unlock(&irq->irq_lock); 98 spin_unlock(&irq->irq_lock);
99 vgic_put_irq(vcpu->kvm, irq);
95 } 100 }
96} 101}
97 102
@@ -108,6 +113,8 @@ unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
108 113
109 if (irq->pending) 114 if (irq->pending)
110 value |= (1U << i); 115 value |= (1U << i);
116
117 vgic_put_irq(vcpu->kvm, irq);
111 } 118 }
112 119
113 return value; 120 return value;
@@ -129,6 +136,7 @@ void vgic_mmio_write_spending(struct kvm_vcpu *vcpu,
129 irq->soft_pending = true; 136 irq->soft_pending = true;
130 137
131 vgic_queue_irq_unlock(vcpu->kvm, irq); 138 vgic_queue_irq_unlock(vcpu->kvm, irq);
139 vgic_put_irq(vcpu->kvm, irq);
132 } 140 }
133} 141}
134 142
@@ -152,6 +160,7 @@ void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu,
152 } 160 }
153 161
154 spin_unlock(&irq->irq_lock); 162 spin_unlock(&irq->irq_lock);
163 vgic_put_irq(vcpu->kvm, irq);
155 } 164 }
156} 165}
157 166
@@ -168,6 +177,8 @@ unsigned long vgic_mmio_read_active(struct kvm_vcpu *vcpu,
168 177
169 if (irq->active) 178 if (irq->active)
170 value |= (1U << i); 179 value |= (1U << i);
180
181 vgic_put_irq(vcpu->kvm, irq);
171 } 182 }
172 183
173 return value; 184 return value;
@@ -242,6 +253,7 @@ void vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
242 for_each_set_bit(i, &val, len * 8) { 253 for_each_set_bit(i, &val, len * 8) {
243 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 254 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
244 vgic_mmio_change_active(vcpu, irq, false); 255 vgic_mmio_change_active(vcpu, irq, false);
256 vgic_put_irq(vcpu->kvm, irq);
245 } 257 }
246 vgic_change_active_finish(vcpu, intid); 258 vgic_change_active_finish(vcpu, intid);
247} 259}
@@ -257,6 +269,7 @@ void vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
257 for_each_set_bit(i, &val, len * 8) { 269 for_each_set_bit(i, &val, len * 8) {
258 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 270 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
259 vgic_mmio_change_active(vcpu, irq, true); 271 vgic_mmio_change_active(vcpu, irq, true);
272 vgic_put_irq(vcpu->kvm, irq);
260 } 273 }
261 vgic_change_active_finish(vcpu, intid); 274 vgic_change_active_finish(vcpu, intid);
262} 275}
@@ -272,6 +285,8 @@ unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu,
272 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 285 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
273 286
274 val |= (u64)irq->priority << (i * 8); 287 val |= (u64)irq->priority << (i * 8);
288
289 vgic_put_irq(vcpu->kvm, irq);
275 } 290 }
276 291
277 return val; 292 return val;
@@ -298,6 +313,8 @@ void vgic_mmio_write_priority(struct kvm_vcpu *vcpu,
298 /* Narrow the priority range to what we actually support */ 313 /* Narrow the priority range to what we actually support */
299 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS); 314 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);
300 spin_unlock(&irq->irq_lock); 315 spin_unlock(&irq->irq_lock);
316
317 vgic_put_irq(vcpu->kvm, irq);
301 } 318 }
302} 319}
303 320
@@ -313,6 +330,8 @@ unsigned long vgic_mmio_read_config(struct kvm_vcpu *vcpu,
313 330
314 if (irq->config == VGIC_CONFIG_EDGE) 331 if (irq->config == VGIC_CONFIG_EDGE)
315 value |= (2U << (i * 2)); 332 value |= (2U << (i * 2));
333
334 vgic_put_irq(vcpu->kvm, irq);
316 } 335 }
317 336
318 return value; 337 return value;
@@ -326,7 +345,7 @@ void vgic_mmio_write_config(struct kvm_vcpu *vcpu,
326 int i; 345 int i;
327 346
328 for (i = 0; i < len * 4; i++) { 347 for (i = 0; i < len * 4; i++) {
329 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 348 struct vgic_irq *irq;
330 349
331 /* 350 /*
332 * The configuration cannot be changed for SGIs in general, 351 * The configuration cannot be changed for SGIs in general,
@@ -337,14 +356,18 @@ void vgic_mmio_write_config(struct kvm_vcpu *vcpu,
337 if (intid + i < VGIC_NR_PRIVATE_IRQS) 356 if (intid + i < VGIC_NR_PRIVATE_IRQS)
338 continue; 357 continue;
339 358
359 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
340 spin_lock(&irq->irq_lock); 360 spin_lock(&irq->irq_lock);
361
341 if (test_bit(i * 2 + 1, &val)) { 362 if (test_bit(i * 2 + 1, &val)) {
342 irq->config = VGIC_CONFIG_EDGE; 363 irq->config = VGIC_CONFIG_EDGE;
343 } else { 364 } else {
344 irq->config = VGIC_CONFIG_LEVEL; 365 irq->config = VGIC_CONFIG_LEVEL;
345 irq->pending = irq->line_level | irq->soft_pending; 366 irq->pending = irq->line_level | irq->soft_pending;
346 } 367 }
368
347 spin_unlock(&irq->irq_lock); 369 spin_unlock(&irq->irq_lock);
370 vgic_put_irq(vcpu->kvm, irq);
348 } 371 }
349} 372}
350 373
@@ -450,8 +473,7 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
450{ 473{
451 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev); 474 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
452 const struct vgic_register_region *region; 475 const struct vgic_register_region *region;
453 struct kvm_vcpu *r_vcpu; 476 unsigned long data = 0;
454 unsigned long data;
455 477
456 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions, 478 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
457 addr - iodev->base_addr); 479 addr - iodev->base_addr);
@@ -460,8 +482,21 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
460 return 0; 482 return 0;
461 } 483 }
462 484
463 r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu; 485 switch (iodev->iodev_type) {
464 data = region->read(r_vcpu, addr, len); 486 case IODEV_CPUIF:
487 data = region->read(vcpu, addr, len);
488 break;
489 case IODEV_DIST:
490 data = region->read(vcpu, addr, len);
491 break;
492 case IODEV_REDIST:
493 data = region->read(iodev->redist_vcpu, addr, len);
494 break;
495 case IODEV_ITS:
496 data = region->its_read(vcpu->kvm, iodev->its, addr, len);
497 break;
498 }
499
465 vgic_data_host_to_mmio_bus(val, len, data); 500 vgic_data_host_to_mmio_bus(val, len, data);
466 return 0; 501 return 0;
467} 502}
@@ -471,7 +506,6 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
471{ 506{
472 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev); 507 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
473 const struct vgic_register_region *region; 508 const struct vgic_register_region *region;
474 struct kvm_vcpu *r_vcpu;
475 unsigned long data = vgic_data_mmio_bus_to_host(val, len); 509 unsigned long data = vgic_data_mmio_bus_to_host(val, len);
476 510
477 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions, 511 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
@@ -482,8 +516,21 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
482 if (!check_region(region, addr, len)) 516 if (!check_region(region, addr, len))
483 return 0; 517 return 0;
484 518
485 r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu; 519 switch (iodev->iodev_type) {
486 region->write(r_vcpu, addr, len, data); 520 case IODEV_CPUIF:
521 region->write(vcpu, addr, len, data);
522 break;
523 case IODEV_DIST:
524 region->write(vcpu, addr, len, data);
525 break;
526 case IODEV_REDIST:
527 region->write(iodev->redist_vcpu, addr, len, data);
528 break;
529 case IODEV_ITS:
530 region->its_write(vcpu->kvm, iodev->its, addr, len, data);
531 break;
532 }
533
487 return 0; 534 return 0;
488} 535}
489 536
@@ -513,6 +560,7 @@ int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
513 } 560 }
514 561
515 io_device->base_addr = dist_base_address; 562 io_device->base_addr = dist_base_address;
563 io_device->iodev_type = IODEV_DIST;
516 io_device->redist_vcpu = NULL; 564 io_device->redist_vcpu = NULL;
517 565
518 mutex_lock(&kvm->slots_lock); 566 mutex_lock(&kvm->slots_lock);
diff --git a/virt/kvm/arm/vgic/vgic-mmio.h b/virt/kvm/arm/vgic/vgic-mmio.h
index 850901482aec..0b3ecf9d100e 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.h
+++ b/virt/kvm/arm/vgic/vgic-mmio.h
@@ -21,10 +21,19 @@ struct vgic_register_region {
21 unsigned int len; 21 unsigned int len;
22 unsigned int bits_per_irq; 22 unsigned int bits_per_irq;
23 unsigned int access_flags; 23 unsigned int access_flags;
24 unsigned long (*read)(struct kvm_vcpu *vcpu, gpa_t addr, 24 union {
25 unsigned int len); 25 unsigned long (*read)(struct kvm_vcpu *vcpu, gpa_t addr,
26 void (*write)(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, 26 unsigned int len);
27 unsigned long val); 27 unsigned long (*its_read)(struct kvm *kvm, struct vgic_its *its,
28 gpa_t addr, unsigned int len);
29 };
30 union {
31 void (*write)(struct kvm_vcpu *vcpu, gpa_t addr,
32 unsigned int len, unsigned long val);
33 void (*its_write)(struct kvm *kvm, struct vgic_its *its,
34 gpa_t addr, unsigned int len,
35 unsigned long val);
36 };
28}; 37};
29 38
30extern struct kvm_io_device_ops kvm_io_gic_ops; 39extern struct kvm_io_device_ops kvm_io_gic_ops;
@@ -87,6 +96,12 @@ unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len);
87void vgic_data_host_to_mmio_bus(void *buf, unsigned int len, 96void vgic_data_host_to_mmio_bus(void *buf, unsigned int len,
88 unsigned long data); 97 unsigned long data);
89 98
99unsigned long extract_bytes(unsigned long data, unsigned int offset,
100 unsigned int num);
101
102u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len,
103 unsigned long val);
104
90unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu, 105unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu,
91 gpa_t addr, unsigned int len); 106 gpa_t addr, unsigned int len);
92 107
@@ -147,4 +162,12 @@ unsigned int vgic_v2_init_dist_iodev(struct vgic_io_device *dev);
147 162
148unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev); 163unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev);
149 164
165#ifdef CONFIG_KVM_ARM_VGIC_V3
166u64 vgic_sanitise_outer_cacheability(u64 reg);
167u64 vgic_sanitise_inner_cacheability(u64 reg);
168u64 vgic_sanitise_shareability(u64 reg);
169u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift,
170 u64 (*sanitise_fn)(u64));
171#endif
172
150#endif 173#endif
diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c
index e31405ee5515..0bf6709d1006 100644
--- a/virt/kvm/arm/vgic/vgic-v2.c
+++ b/virt/kvm/arm/vgic/vgic-v2.c
@@ -124,6 +124,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
124 } 124 }
125 125
126 spin_unlock(&irq->irq_lock); 126 spin_unlock(&irq->irq_lock);
127 vgic_put_irq(vcpu->kvm, irq);
127 } 128 }
128} 129}
129 130
@@ -332,20 +333,25 @@ int vgic_v2_probe(const struct gic_kvm_info *info)
332 vtr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VTR); 333 vtr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VTR);
333 kvm_vgic_global_state.nr_lr = (vtr & 0x3f) + 1; 334 kvm_vgic_global_state.nr_lr = (vtr & 0x3f) + 1;
334 335
336 ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2);
337 if (ret) {
338 kvm_err("Cannot register GICv2 KVM device\n");
339 iounmap(kvm_vgic_global_state.vctrl_base);
340 return ret;
341 }
342
335 ret = create_hyp_io_mappings(kvm_vgic_global_state.vctrl_base, 343 ret = create_hyp_io_mappings(kvm_vgic_global_state.vctrl_base,
336 kvm_vgic_global_state.vctrl_base + 344 kvm_vgic_global_state.vctrl_base +
337 resource_size(&info->vctrl), 345 resource_size(&info->vctrl),
338 info->vctrl.start); 346 info->vctrl.start);
339
340 if (ret) { 347 if (ret) {
341 kvm_err("Cannot map VCTRL into hyp\n"); 348 kvm_err("Cannot map VCTRL into hyp\n");
349 kvm_unregister_device_ops(KVM_DEV_TYPE_ARM_VGIC_V2);
342 iounmap(kvm_vgic_global_state.vctrl_base); 350 iounmap(kvm_vgic_global_state.vctrl_base);
343 return ret; 351 return ret;
344 } 352 }
345 353
346 kvm_vgic_global_state.can_emulate_gicv2 = true; 354 kvm_vgic_global_state.can_emulate_gicv2 = true;
347 kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2);
348
349 kvm_vgic_global_state.vcpu_base = info->vcpu.start; 355 kvm_vgic_global_state.vcpu_base = info->vcpu.start;
350 kvm_vgic_global_state.type = VGIC_V2; 356 kvm_vgic_global_state.type = VGIC_V2;
351 kvm_vgic_global_state.max_gic_vcpus = VGIC_V2_MAX_CPUS; 357 kvm_vgic_global_state.max_gic_vcpus = VGIC_V2_MAX_CPUS;
diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c
index 346b4ad12b49..0506543df38a 100644
--- a/virt/kvm/arm/vgic/vgic-v3.c
+++ b/virt/kvm/arm/vgic/vgic-v3.c
@@ -81,6 +81,8 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
81 else 81 else
82 intid = val & GICH_LR_VIRTUALID; 82 intid = val & GICH_LR_VIRTUALID;
83 irq = vgic_get_irq(vcpu->kvm, vcpu, intid); 83 irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
84 if (!irq) /* An LPI could have been unmapped. */
85 continue;
84 86
85 spin_lock(&irq->irq_lock); 87 spin_lock(&irq->irq_lock);
86 88
@@ -113,6 +115,7 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
113 } 115 }
114 116
115 spin_unlock(&irq->irq_lock); 117 spin_unlock(&irq->irq_lock);
118 vgic_put_irq(vcpu->kvm, irq);
116 } 119 }
117} 120}
118 121
@@ -190,6 +193,11 @@ void vgic_v3_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
190 vmcrp->pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT; 193 vmcrp->pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
191} 194}
192 195
196#define INITIAL_PENDBASER_VALUE \
197 (GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, RaWb) | \
198 GIC_BASER_CACHEABILITY(GICR_PENDBASER, OUTER, SameAsInner) | \
199 GIC_BASER_SHAREABILITY(GICR_PENDBASER, InnerShareable))
200
193void vgic_v3_enable(struct kvm_vcpu *vcpu) 201void vgic_v3_enable(struct kvm_vcpu *vcpu)
194{ 202{
195 struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3; 203 struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;
@@ -207,10 +215,12 @@ void vgic_v3_enable(struct kvm_vcpu *vcpu)
207 * way, so we force SRE to 1 to demonstrate this to the guest. 215 * way, so we force SRE to 1 to demonstrate this to the guest.
208 * This goes with the spec allowing the value to be RAO/WI. 216 * This goes with the spec allowing the value to be RAO/WI.
209 */ 217 */
210 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) 218 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
211 vgic_v3->vgic_sre = ICC_SRE_EL1_SRE; 219 vgic_v3->vgic_sre = ICC_SRE_EL1_SRE;
212 else 220 vcpu->arch.vgic_cpu.pendbaser = INITIAL_PENDBASER_VALUE;
221 } else {
213 vgic_v3->vgic_sre = 0; 222 vgic_v3->vgic_sre = 0;
223 }
214 224
215 /* Get the show on the road... */ 225 /* Get the show on the road... */
216 vgic_v3->vgic_hcr = ICH_HCR_EN; 226 vgic_v3->vgic_hcr = ICH_HCR_EN;
@@ -296,6 +306,7 @@ out:
296int vgic_v3_probe(const struct gic_kvm_info *info) 306int vgic_v3_probe(const struct gic_kvm_info *info)
297{ 307{
298 u32 ich_vtr_el2 = kvm_call_hyp(__vgic_v3_get_ich_vtr_el2); 308 u32 ich_vtr_el2 = kvm_call_hyp(__vgic_v3_get_ich_vtr_el2);
309 int ret;
299 310
300 /* 311 /*
301 * The ListRegs field is 5 bits, but there is a architectural 312 * The ListRegs field is 5 bits, but there is a architectural
@@ -319,12 +330,22 @@ int vgic_v3_probe(const struct gic_kvm_info *info)
319 } else { 330 } else {
320 kvm_vgic_global_state.vcpu_base = info->vcpu.start; 331 kvm_vgic_global_state.vcpu_base = info->vcpu.start;
321 kvm_vgic_global_state.can_emulate_gicv2 = true; 332 kvm_vgic_global_state.can_emulate_gicv2 = true;
322 kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2); 333 ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2);
334 if (ret) {
335 kvm_err("Cannot register GICv2 KVM device.\n");
336 return ret;
337 }
323 kvm_info("vgic-v2@%llx\n", info->vcpu.start); 338 kvm_info("vgic-v2@%llx\n", info->vcpu.start);
324 } 339 }
340 ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3);
341 if (ret) {
342 kvm_err("Cannot register GICv3 KVM device.\n");
343 kvm_unregister_device_ops(KVM_DEV_TYPE_ARM_VGIC_V2);
344 return ret;
345 }
346
325 if (kvm_vgic_global_state.vcpu_base == 0) 347 if (kvm_vgic_global_state.vcpu_base == 0)
326 kvm_info("disabling GICv2 emulation\n"); 348 kvm_info("disabling GICv2 emulation\n");
327 kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3);
328 349
329 kvm_vgic_global_state.vctrl_base = NULL; 350 kvm_vgic_global_state.vctrl_base = NULL;
330 kvm_vgic_global_state.type = VGIC_V3; 351 kvm_vgic_global_state.type = VGIC_V3;
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index 69b61abefa19..39f3358c6d91 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -33,10 +33,17 @@ struct vgic_global __section(.hyp.text) kvm_vgic_global_state;
33 33
34/* 34/*
35 * Locking order is always: 35 * Locking order is always:
36 * vgic_cpu->ap_list_lock 36 * its->cmd_lock (mutex)
37 * vgic_irq->irq_lock 37 * its->its_lock (mutex)
38 * vgic_cpu->ap_list_lock
39 * kvm->lpi_list_lock
40 * vgic_irq->irq_lock
38 * 41 *
39 * (that is, always take the ap_list_lock before the struct vgic_irq lock). 42 * If you need to take multiple locks, always take the upper lock first,
43 * then the lower ones, e.g. first take the its_lock, then the irq_lock.
44 * If you are already holding a lock and need to take a higher one, you
45 * have to drop the lower ranking lock first and re-aquire it after having
46 * taken the upper one.
40 * 47 *
41 * When taking more than one ap_list_lock at the same time, always take the 48 * When taking more than one ap_list_lock at the same time, always take the
42 * lowest numbered VCPU's ap_list_lock first, so: 49 * lowest numbered VCPU's ap_list_lock first, so:
@@ -45,6 +52,41 @@ struct vgic_global __section(.hyp.text) kvm_vgic_global_state;
45 * spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock); 52 * spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock);
46 */ 53 */
47 54
55/*
56 * Iterate over the VM's list of mapped LPIs to find the one with a
57 * matching interrupt ID and return a reference to the IRQ structure.
58 */
59static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
60{
61 struct vgic_dist *dist = &kvm->arch.vgic;
62 struct vgic_irq *irq = NULL;
63
64 spin_lock(&dist->lpi_list_lock);
65
66 list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
67 if (irq->intid != intid)
68 continue;
69
70 /*
71 * This increases the refcount, the caller is expected to
72 * call vgic_put_irq() later once it's finished with the IRQ.
73 */
74 vgic_get_irq_kref(irq);
75 goto out_unlock;
76 }
77 irq = NULL;
78
79out_unlock:
80 spin_unlock(&dist->lpi_list_lock);
81
82 return irq;
83}
84
85/*
86 * This looks up the virtual interrupt ID to get the corresponding
87 * struct vgic_irq. It also increases the refcount, so any caller is expected
88 * to call vgic_put_irq() once it's finished with this IRQ.
89 */
48struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, 90struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
49 u32 intid) 91 u32 intid)
50{ 92{
@@ -56,14 +98,43 @@ struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
56 if (intid <= VGIC_MAX_SPI) 98 if (intid <= VGIC_MAX_SPI)
57 return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS]; 99 return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS];
58 100
59 /* LPIs are not yet covered */ 101 /* LPIs */
60 if (intid >= VGIC_MIN_LPI) 102 if (intid >= VGIC_MIN_LPI)
61 return NULL; 103 return vgic_get_lpi(kvm, intid);
62 104
63 WARN(1, "Looking up struct vgic_irq for reserved INTID"); 105 WARN(1, "Looking up struct vgic_irq for reserved INTID");
64 return NULL; 106 return NULL;
65} 107}
66 108
109/*
110 * We can't do anything in here, because we lack the kvm pointer to
111 * lock and remove the item from the lpi_list. So we keep this function
112 * empty and use the return value of kref_put() to trigger the freeing.
113 */
114static void vgic_irq_release(struct kref *ref)
115{
116}
117
118void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
119{
120 struct vgic_dist *dist;
121
122 if (irq->intid < VGIC_MIN_LPI)
123 return;
124
125 if (!kref_put(&irq->refcount, vgic_irq_release))
126 return;
127
128 dist = &kvm->arch.vgic;
129
130 spin_lock(&dist->lpi_list_lock);
131 list_del(&irq->lpi_list);
132 dist->lpi_list_count--;
133 spin_unlock(&dist->lpi_list_lock);
134
135 kfree(irq);
136}
137
67/** 138/**
68 * kvm_vgic_target_oracle - compute the target vcpu for an irq 139 * kvm_vgic_target_oracle - compute the target vcpu for an irq
69 * 140 *
@@ -236,6 +307,11 @@ retry:
236 goto retry; 307 goto retry;
237 } 308 }
238 309
310 /*
311 * Grab a reference to the irq to reflect the fact that it is
312 * now in the ap_list.
313 */
314 vgic_get_irq_kref(irq);
239 list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head); 315 list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);
240 irq->vcpu = vcpu; 316 irq->vcpu = vcpu;
241 317
@@ -269,14 +345,17 @@ static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
269 if (!irq) 345 if (!irq)
270 return -EINVAL; 346 return -EINVAL;
271 347
272 if (irq->hw != mapped_irq) 348 if (irq->hw != mapped_irq) {
349 vgic_put_irq(kvm, irq);
273 return -EINVAL; 350 return -EINVAL;
351 }
274 352
275 spin_lock(&irq->irq_lock); 353 spin_lock(&irq->irq_lock);
276 354
277 if (!vgic_validate_injection(irq, level)) { 355 if (!vgic_validate_injection(irq, level)) {
278 /* Nothing to see here, move along... */ 356 /* Nothing to see here, move along... */
279 spin_unlock(&irq->irq_lock); 357 spin_unlock(&irq->irq_lock);
358 vgic_put_irq(kvm, irq);
280 return 0; 359 return 0;
281 } 360 }
282 361
@@ -288,6 +367,7 @@ static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
288 } 367 }
289 368
290 vgic_queue_irq_unlock(kvm, irq); 369 vgic_queue_irq_unlock(kvm, irq);
370 vgic_put_irq(kvm, irq);
291 371
292 return 0; 372 return 0;
293} 373}
@@ -330,25 +410,28 @@ int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, u32 virt_irq, u32 phys_irq)
330 irq->hwintid = phys_irq; 410 irq->hwintid = phys_irq;
331 411
332 spin_unlock(&irq->irq_lock); 412 spin_unlock(&irq->irq_lock);
413 vgic_put_irq(vcpu->kvm, irq);
333 414
334 return 0; 415 return 0;
335} 416}
336 417
337int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq) 418int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq)
338{ 419{
339 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq); 420 struct vgic_irq *irq;
340
341 BUG_ON(!irq);
342 421
343 if (!vgic_initialized(vcpu->kvm)) 422 if (!vgic_initialized(vcpu->kvm))
344 return -EAGAIN; 423 return -EAGAIN;
345 424
425 irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq);
426 BUG_ON(!irq);
427
346 spin_lock(&irq->irq_lock); 428 spin_lock(&irq->irq_lock);
347 429
348 irq->hw = false; 430 irq->hw = false;
349 irq->hwintid = 0; 431 irq->hwintid = 0;
350 432
351 spin_unlock(&irq->irq_lock); 433 spin_unlock(&irq->irq_lock);
434 vgic_put_irq(vcpu->kvm, irq);
352 435
353 return 0; 436 return 0;
354} 437}
@@ -386,6 +469,15 @@ retry:
386 list_del(&irq->ap_list); 469 list_del(&irq->ap_list);
387 irq->vcpu = NULL; 470 irq->vcpu = NULL;
388 spin_unlock(&irq->irq_lock); 471 spin_unlock(&irq->irq_lock);
472
473 /*
474 * This vgic_put_irq call matches the
475 * vgic_get_irq_kref in vgic_queue_irq_unlock,
476 * where we added the LPI to the ap_list. As
477 * we remove the irq from the list, we drop
478 * also drop the refcount.
479 */
480 vgic_put_irq(vcpu->kvm, irq);
389 continue; 481 continue;
390 } 482 }
391 483
@@ -614,6 +706,15 @@ bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq)
614 spin_lock(&irq->irq_lock); 706 spin_lock(&irq->irq_lock);
615 map_is_active = irq->hw && irq->active; 707 map_is_active = irq->hw && irq->active;
616 spin_unlock(&irq->irq_lock); 708 spin_unlock(&irq->irq_lock);
709 vgic_put_irq(vcpu->kvm, irq);
617 710
618 return map_is_active; 711 return map_is_active;
619} 712}
713
714int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi)
715{
716 if (vgic_has_its(kvm))
717 return vgic_its_inject_msi(kvm, msi);
718 else
719 return -ENODEV;
720}
diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h
index 7b300ca370b7..1d8e21d5c13f 100644
--- a/virt/kvm/arm/vgic/vgic.h
+++ b/virt/kvm/arm/vgic/vgic.h
@@ -25,6 +25,7 @@
25#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF) 25#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF)
26 26
27#define INTERRUPT_ID_BITS_SPIS 10 27#define INTERRUPT_ID_BITS_SPIS 10
28#define INTERRUPT_ID_BITS_ITS 16
28#define VGIC_PRI_BITS 5 29#define VGIC_PRI_BITS 5
29 30
30#define vgic_irq_is_sgi(intid) ((intid) < VGIC_NR_SGIS) 31#define vgic_irq_is_sgi(intid) ((intid) < VGIC_NR_SGIS)
@@ -38,9 +39,13 @@ struct vgic_vmcr {
38 39
39struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, 40struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
40 u32 intid); 41 u32 intid);
42void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq);
41bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq); 43bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq);
42void vgic_kick_vcpus(struct kvm *kvm); 44void vgic_kick_vcpus(struct kvm *kvm);
43 45
46int vgic_check_ioaddr(struct kvm *kvm, phys_addr_t *ioaddr,
47 phys_addr_t addr, phys_addr_t alignment);
48
44void vgic_v2_process_maintenance(struct kvm_vcpu *vcpu); 49void vgic_v2_process_maintenance(struct kvm_vcpu *vcpu);
45void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu); 50void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu);
46void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr); 51void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
@@ -59,6 +64,14 @@ int vgic_v2_map_resources(struct kvm *kvm);
59int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address, 64int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
60 enum vgic_type); 65 enum vgic_type);
61 66
67static inline void vgic_get_irq_kref(struct vgic_irq *irq)
68{
69 if (irq->intid < VGIC_MIN_LPI)
70 return;
71
72 kref_get(&irq->refcount);
73}
74
62#ifdef CONFIG_KVM_ARM_VGIC_V3 75#ifdef CONFIG_KVM_ARM_VGIC_V3
63void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu); 76void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu);
64void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu); 77void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu);
@@ -71,6 +84,10 @@ void vgic_v3_enable(struct kvm_vcpu *vcpu);
71int vgic_v3_probe(const struct gic_kvm_info *info); 84int vgic_v3_probe(const struct gic_kvm_info *info);
72int vgic_v3_map_resources(struct kvm *kvm); 85int vgic_v3_map_resources(struct kvm *kvm);
73int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t dist_base_address); 86int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t dist_base_address);
87bool vgic_has_its(struct kvm *kvm);
88int kvm_vgic_register_its_device(void);
89void vgic_enable_lpis(struct kvm_vcpu *vcpu);
90int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi);
74#else 91#else
75static inline void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu) 92static inline void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu)
76{ 93{
@@ -122,9 +139,28 @@ static inline int vgic_register_redist_iodevs(struct kvm *kvm,
122{ 139{
123 return -ENODEV; 140 return -ENODEV;
124} 141}
142
143static inline bool vgic_has_its(struct kvm *kvm)
144{
145 return false;
146}
147
148static inline int kvm_vgic_register_its_device(void)
149{
150 return -ENODEV;
151}
152
153static inline void vgic_enable_lpis(struct kvm_vcpu *vcpu)
154{
155}
156
157static inline int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
158{
159 return -ENODEV;
160}
125#endif 161#endif
126 162
127void kvm_register_vgic_device(unsigned long type); 163int kvm_register_vgic_device(unsigned long type);
128int vgic_lazy_init(struct kvm *kvm); 164int vgic_lazy_init(struct kvm *kvm);
129int vgic_init(struct kvm *kvm); 165int vgic_init(struct kvm *kvm);
130 166
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 154b9ab459b0..61b31a5f76c8 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -3545,6 +3545,30 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3545 return r; 3545 return r;
3546} 3546}
3547 3547
3548struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3549 gpa_t addr)
3550{
3551 struct kvm_io_bus *bus;
3552 int dev_idx, srcu_idx;
3553 struct kvm_io_device *iodev = NULL;
3554
3555 srcu_idx = srcu_read_lock(&kvm->srcu);
3556
3557 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
3558
3559 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
3560 if (dev_idx < 0)
3561 goto out_unlock;
3562
3563 iodev = bus->range[dev_idx].dev;
3564
3565out_unlock:
3566 srcu_read_unlock(&kvm->srcu, srcu_idx);
3567
3568 return iodev;
3569}
3570EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
3571
3548static struct notifier_block kvm_cpu_notifier = { 3572static struct notifier_block kvm_cpu_notifier = {
3549 .notifier_call = kvm_cpu_hotplug, 3573 .notifier_call = kvm_cpu_hotplug,
3550}; 3574};
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-24 11:35:25 -0400