aboutsummaryrefslogtreecommitdiffstats
path: root/virt/kvm
diff options
context:
space:
mode:
Diffstat (limited to 'virt/kvm')
-rw-r--r--virt/kvm/Kconfig10
-rw-r--r--virt/kvm/arm/arch_timer.c3
-rw-r--r--virt/kvm/arm/vgic-v2-emul.c847
-rw-r--r--virt/kvm/arm/vgic-v2.c12
-rw-r--r--virt/kvm/arm/vgic-v3-emul.c1036
-rw-r--r--virt/kvm/arm/vgic-v3.c90
-rw-r--r--virt/kvm/arm/vgic.c1145
-rw-r--r--virt/kvm/arm/vgic.h123
-rw-r--r--virt/kvm/async_pf.c2
-rw-r--r--virt/kvm/kvm_main.c235
10 files changed, 2557 insertions, 946 deletions
diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index fc0c5e603eb4..e2c876d5a03b 100644
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -37,3 +37,13 @@ config HAVE_KVM_CPU_RELAX_INTERCEPT
37 37
38config KVM_VFIO 38config KVM_VFIO
39 bool 39 bool
40
41config HAVE_KVM_ARCH_TLB_FLUSH_ALL
42 bool
43
44config KVM_GENERIC_DIRTYLOG_READ_PROTECT
45 bool
46
47config KVM_COMPAT
48 def_bool y
49 depends on COMPAT && !S390
diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c
index 1c0772b340d8..6e54f3542126 100644
--- a/virt/kvm/arm/arch_timer.c
+++ b/virt/kvm/arm/arch_timer.c
@@ -152,7 +152,8 @@ void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
152 return; 152 return;
153 } 153 }
154 154
155 ns = cyclecounter_cyc2ns(timecounter->cc, cval - now); 155 ns = cyclecounter_cyc2ns(timecounter->cc, cval - now, timecounter->mask,
156 &timecounter->frac);
156 timer_arm(timer, ns); 157 timer_arm(timer, ns);
157} 158}
158 159
diff --git a/virt/kvm/arm/vgic-v2-emul.c b/virt/kvm/arm/vgic-v2-emul.c
new file mode 100644
index 000000000000..19c6210f02cf
--- /dev/null
+++ b/virt/kvm/arm/vgic-v2-emul.c
@@ -0,0 +1,847 @@
1/*
2 * Contains GICv2 specific emulation code, was in vgic.c before.
3 *
4 * Copyright (C) 2012 ARM Ltd.
5 * Author: Marc Zyngier <marc.zyngier@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/cpu.h>
21#include <linux/kvm.h>
22#include <linux/kvm_host.h>
23#include <linux/interrupt.h>
24#include <linux/io.h>
25#include <linux/uaccess.h>
26
27#include <linux/irqchip/arm-gic.h>
28
29#include <asm/kvm_emulate.h>
30#include <asm/kvm_arm.h>
31#include <asm/kvm_mmu.h>
32
33#include "vgic.h"
34
35#define GICC_ARCH_VERSION_V2 0x2
36
37static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg);
38static u8 *vgic_get_sgi_sources(struct vgic_dist *dist, int vcpu_id, int sgi)
39{
40 return dist->irq_sgi_sources + vcpu_id * VGIC_NR_SGIS + sgi;
41}
42
43static bool handle_mmio_misc(struct kvm_vcpu *vcpu,
44 struct kvm_exit_mmio *mmio, phys_addr_t offset)
45{
46 u32 reg;
47 u32 word_offset = offset & 3;
48
49 switch (offset & ~3) {
50 case 0: /* GICD_CTLR */
51 reg = vcpu->kvm->arch.vgic.enabled;
52 vgic_reg_access(mmio, &reg, word_offset,
53 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
54 if (mmio->is_write) {
55 vcpu->kvm->arch.vgic.enabled = reg & 1;
56 vgic_update_state(vcpu->kvm);
57 return true;
58 }
59 break;
60
61 case 4: /* GICD_TYPER */
62 reg = (atomic_read(&vcpu->kvm->online_vcpus) - 1) << 5;
63 reg |= (vcpu->kvm->arch.vgic.nr_irqs >> 5) - 1;
64 vgic_reg_access(mmio, &reg, word_offset,
65 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
66 break;
67
68 case 8: /* GICD_IIDR */
69 reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
70 vgic_reg_access(mmio, &reg, word_offset,
71 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
72 break;
73 }
74
75 return false;
76}
77
78static bool handle_mmio_set_enable_reg(struct kvm_vcpu *vcpu,
79 struct kvm_exit_mmio *mmio,
80 phys_addr_t offset)
81{
82 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
83 vcpu->vcpu_id, ACCESS_WRITE_SETBIT);
84}
85
86static bool handle_mmio_clear_enable_reg(struct kvm_vcpu *vcpu,
87 struct kvm_exit_mmio *mmio,
88 phys_addr_t offset)
89{
90 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
91 vcpu->vcpu_id, ACCESS_WRITE_CLEARBIT);
92}
93
94static bool handle_mmio_set_pending_reg(struct kvm_vcpu *vcpu,
95 struct kvm_exit_mmio *mmio,
96 phys_addr_t offset)
97{
98 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
99 vcpu->vcpu_id);
100}
101
102static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu,
103 struct kvm_exit_mmio *mmio,
104 phys_addr_t offset)
105{
106 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
107 vcpu->vcpu_id);
108}
109
110static bool handle_mmio_priority_reg(struct kvm_vcpu *vcpu,
111 struct kvm_exit_mmio *mmio,
112 phys_addr_t offset)
113{
114 u32 *reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
115 vcpu->vcpu_id, offset);
116 vgic_reg_access(mmio, reg, offset,
117 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
118 return false;
119}
120
121#define GICD_ITARGETSR_SIZE 32
122#define GICD_CPUTARGETS_BITS 8
123#define GICD_IRQS_PER_ITARGETSR (GICD_ITARGETSR_SIZE / GICD_CPUTARGETS_BITS)
124static u32 vgic_get_target_reg(struct kvm *kvm, int irq)
125{
126 struct vgic_dist *dist = &kvm->arch.vgic;
127 int i;
128 u32 val = 0;
129
130 irq -= VGIC_NR_PRIVATE_IRQS;
131
132 for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++)
133 val |= 1 << (dist->irq_spi_cpu[irq + i] + i * 8);
134
135 return val;
136}
137
138static void vgic_set_target_reg(struct kvm *kvm, u32 val, int irq)
139{
140 struct vgic_dist *dist = &kvm->arch.vgic;
141 struct kvm_vcpu *vcpu;
142 int i, c;
143 unsigned long *bmap;
144 u32 target;
145
146 irq -= VGIC_NR_PRIVATE_IRQS;
147
148 /*
149 * Pick the LSB in each byte. This ensures we target exactly
150 * one vcpu per IRQ. If the byte is null, assume we target
151 * CPU0.
152 */
153 for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++) {
154 int shift = i * GICD_CPUTARGETS_BITS;
155
156 target = ffs((val >> shift) & 0xffU);
157 target = target ? (target - 1) : 0;
158 dist->irq_spi_cpu[irq + i] = target;
159 kvm_for_each_vcpu(c, vcpu, kvm) {
160 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[c]);
161 if (c == target)
162 set_bit(irq + i, bmap);
163 else
164 clear_bit(irq + i, bmap);
165 }
166 }
167}
168
169static bool handle_mmio_target_reg(struct kvm_vcpu *vcpu,
170 struct kvm_exit_mmio *mmio,
171 phys_addr_t offset)
172{
173 u32 reg;
174
175 /* We treat the banked interrupts targets as read-only */
176 if (offset < 32) {
177 u32 roreg;
178
179 roreg = 1 << vcpu->vcpu_id;
180 roreg |= roreg << 8;
181 roreg |= roreg << 16;
182
183 vgic_reg_access(mmio, &roreg, offset,
184 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
185 return false;
186 }
187
188 reg = vgic_get_target_reg(vcpu->kvm, offset & ~3U);
189 vgic_reg_access(mmio, &reg, offset,
190 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
191 if (mmio->is_write) {
192 vgic_set_target_reg(vcpu->kvm, reg, offset & ~3U);
193 vgic_update_state(vcpu->kvm);
194 return true;
195 }
196
197 return false;
198}
199
200static bool handle_mmio_cfg_reg(struct kvm_vcpu *vcpu,
201 struct kvm_exit_mmio *mmio, phys_addr_t offset)
202{
203 u32 *reg;
204
205 reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
206 vcpu->vcpu_id, offset >> 1);
207
208 return vgic_handle_cfg_reg(reg, mmio, offset);
209}
210
211static bool handle_mmio_sgi_reg(struct kvm_vcpu *vcpu,
212 struct kvm_exit_mmio *mmio, phys_addr_t offset)
213{
214 u32 reg;
215
216 vgic_reg_access(mmio, &reg, offset,
217 ACCESS_READ_RAZ | ACCESS_WRITE_VALUE);
218 if (mmio->is_write) {
219 vgic_dispatch_sgi(vcpu, reg);
220 vgic_update_state(vcpu->kvm);
221 return true;
222 }
223
224 return false;
225}
226
227/* Handle reads of GICD_CPENDSGIRn and GICD_SPENDSGIRn */
228static bool read_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
229 struct kvm_exit_mmio *mmio,
230 phys_addr_t offset)
231{
232 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
233 int sgi;
234 int min_sgi = (offset & ~0x3);
235 int max_sgi = min_sgi + 3;
236 int vcpu_id = vcpu->vcpu_id;
237 u32 reg = 0;
238
239 /* Copy source SGIs from distributor side */
240 for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
241 u8 sources = *vgic_get_sgi_sources(dist, vcpu_id, sgi);
242
243 reg |= ((u32)sources) << (8 * (sgi - min_sgi));
244 }
245
246 mmio_data_write(mmio, ~0, reg);
247 return false;
248}
249
250static bool write_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
251 struct kvm_exit_mmio *mmio,
252 phys_addr_t offset, bool set)
253{
254 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
255 int sgi;
256 int min_sgi = (offset & ~0x3);
257 int max_sgi = min_sgi + 3;
258 int vcpu_id = vcpu->vcpu_id;
259 u32 reg;
260 bool updated = false;
261
262 reg = mmio_data_read(mmio, ~0);
263
264 /* Clear pending SGIs on the distributor */
265 for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
266 u8 mask = reg >> (8 * (sgi - min_sgi));
267 u8 *src = vgic_get_sgi_sources(dist, vcpu_id, sgi);
268
269 if (set) {
270 if ((*src & mask) != mask)
271 updated = true;
272 *src |= mask;
273 } else {
274 if (*src & mask)
275 updated = true;
276 *src &= ~mask;
277 }
278 }
279
280 if (updated)
281 vgic_update_state(vcpu->kvm);
282
283 return updated;
284}
285
286static bool handle_mmio_sgi_set(struct kvm_vcpu *vcpu,
287 struct kvm_exit_mmio *mmio,
288 phys_addr_t offset)
289{
290 if (!mmio->is_write)
291 return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
292 else
293 return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, true);
294}
295
296static bool handle_mmio_sgi_clear(struct kvm_vcpu *vcpu,
297 struct kvm_exit_mmio *mmio,
298 phys_addr_t offset)
299{
300 if (!mmio->is_write)
301 return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
302 else
303 return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, false);
304}
305
306static const struct kvm_mmio_range vgic_dist_ranges[] = {
307 {
308 .base = GIC_DIST_CTRL,
309 .len = 12,
310 .bits_per_irq = 0,
311 .handle_mmio = handle_mmio_misc,
312 },
313 {
314 .base = GIC_DIST_IGROUP,
315 .len = VGIC_MAX_IRQS / 8,
316 .bits_per_irq = 1,
317 .handle_mmio = handle_mmio_raz_wi,
318 },
319 {
320 .base = GIC_DIST_ENABLE_SET,
321 .len = VGIC_MAX_IRQS / 8,
322 .bits_per_irq = 1,
323 .handle_mmio = handle_mmio_set_enable_reg,
324 },
325 {
326 .base = GIC_DIST_ENABLE_CLEAR,
327 .len = VGIC_MAX_IRQS / 8,
328 .bits_per_irq = 1,
329 .handle_mmio = handle_mmio_clear_enable_reg,
330 },
331 {
332 .base = GIC_DIST_PENDING_SET,
333 .len = VGIC_MAX_IRQS / 8,
334 .bits_per_irq = 1,
335 .handle_mmio = handle_mmio_set_pending_reg,
336 },
337 {
338 .base = GIC_DIST_PENDING_CLEAR,
339 .len = VGIC_MAX_IRQS / 8,
340 .bits_per_irq = 1,
341 .handle_mmio = handle_mmio_clear_pending_reg,
342 },
343 {
344 .base = GIC_DIST_ACTIVE_SET,
345 .len = VGIC_MAX_IRQS / 8,
346 .bits_per_irq = 1,
347 .handle_mmio = handle_mmio_raz_wi,
348 },
349 {
350 .base = GIC_DIST_ACTIVE_CLEAR,
351 .len = VGIC_MAX_IRQS / 8,
352 .bits_per_irq = 1,
353 .handle_mmio = handle_mmio_raz_wi,
354 },
355 {
356 .base = GIC_DIST_PRI,
357 .len = VGIC_MAX_IRQS,
358 .bits_per_irq = 8,
359 .handle_mmio = handle_mmio_priority_reg,
360 },
361 {
362 .base = GIC_DIST_TARGET,
363 .len = VGIC_MAX_IRQS,
364 .bits_per_irq = 8,
365 .handle_mmio = handle_mmio_target_reg,
366 },
367 {
368 .base = GIC_DIST_CONFIG,
369 .len = VGIC_MAX_IRQS / 4,
370 .bits_per_irq = 2,
371 .handle_mmio = handle_mmio_cfg_reg,
372 },
373 {
374 .base = GIC_DIST_SOFTINT,
375 .len = 4,
376 .handle_mmio = handle_mmio_sgi_reg,
377 },
378 {
379 .base = GIC_DIST_SGI_PENDING_CLEAR,
380 .len = VGIC_NR_SGIS,
381 .handle_mmio = handle_mmio_sgi_clear,
382 },
383 {
384 .base = GIC_DIST_SGI_PENDING_SET,
385 .len = VGIC_NR_SGIS,
386 .handle_mmio = handle_mmio_sgi_set,
387 },
388 {}
389};
390
391static bool vgic_v2_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
392 struct kvm_exit_mmio *mmio)
393{
394 unsigned long base = vcpu->kvm->arch.vgic.vgic_dist_base;
395
396 if (!is_in_range(mmio->phys_addr, mmio->len, base,
397 KVM_VGIC_V2_DIST_SIZE))
398 return false;
399
400 /* GICv2 does not support accesses wider than 32 bits */
401 if (mmio->len > 4) {
402 kvm_inject_dabt(vcpu, mmio->phys_addr);
403 return true;
404 }
405
406 return vgic_handle_mmio_range(vcpu, run, mmio, vgic_dist_ranges, base);
407}
408
409static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg)
410{
411 struct kvm *kvm = vcpu->kvm;
412 struct vgic_dist *dist = &kvm->arch.vgic;
413 int nrcpus = atomic_read(&kvm->online_vcpus);
414 u8 target_cpus;
415 int sgi, mode, c, vcpu_id;
416
417 vcpu_id = vcpu->vcpu_id;
418
419 sgi = reg & 0xf;
420 target_cpus = (reg >> 16) & 0xff;
421 mode = (reg >> 24) & 3;
422
423 switch (mode) {
424 case 0:
425 if (!target_cpus)
426 return;
427 break;
428
429 case 1:
430 target_cpus = ((1 << nrcpus) - 1) & ~(1 << vcpu_id) & 0xff;
431 break;
432
433 case 2:
434 target_cpus = 1 << vcpu_id;
435 break;
436 }
437
438 kvm_for_each_vcpu(c, vcpu, kvm) {
439 if (target_cpus & 1) {
440 /* Flag the SGI as pending */
441 vgic_dist_irq_set_pending(vcpu, sgi);
442 *vgic_get_sgi_sources(dist, c, sgi) |= 1 << vcpu_id;
443 kvm_debug("SGI%d from CPU%d to CPU%d\n",
444 sgi, vcpu_id, c);
445 }
446
447 target_cpus >>= 1;
448 }
449}
450
451static bool vgic_v2_queue_sgi(struct kvm_vcpu *vcpu, int irq)
452{
453 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
454 unsigned long sources;
455 int vcpu_id = vcpu->vcpu_id;
456 int c;
457
458 sources = *vgic_get_sgi_sources(dist, vcpu_id, irq);
459
460 for_each_set_bit(c, &sources, dist->nr_cpus) {
461 if (vgic_queue_irq(vcpu, c, irq))
462 clear_bit(c, &sources);
463 }
464
465 *vgic_get_sgi_sources(dist, vcpu_id, irq) = sources;
466
467 /*
468 * If the sources bitmap has been cleared it means that we
469 * could queue all the SGIs onto link registers (see the
470 * clear_bit above), and therefore we are done with them in
471 * our emulated gic and can get rid of them.
472 */
473 if (!sources) {
474 vgic_dist_irq_clear_pending(vcpu, irq);
475 vgic_cpu_irq_clear(vcpu, irq);
476 return true;
477 }
478
479 return false;
480}
481
482/**
483 * kvm_vgic_map_resources - Configure global VGIC state before running any VCPUs
484 * @kvm: pointer to the kvm struct
485 *
486 * Map the virtual CPU interface into the VM before running any VCPUs. We
487 * can't do this at creation time, because user space must first set the
488 * virtual CPU interface address in the guest physical address space.
489 */
490static int vgic_v2_map_resources(struct kvm *kvm,
491 const struct vgic_params *params)
492{
493 int ret = 0;
494
495 if (!irqchip_in_kernel(kvm))
496 return 0;
497
498 mutex_lock(&kvm->lock);
499
500 if (vgic_ready(kvm))
501 goto out;
502
503 if (IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_dist_base) ||
504 IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_cpu_base)) {
505 kvm_err("Need to set vgic cpu and dist addresses first\n");
506 ret = -ENXIO;
507 goto out;
508 }
509
510 /*
511 * Initialize the vgic if this hasn't already been done on demand by
512 * accessing the vgic state from userspace.
513 */
514 ret = vgic_init(kvm);
515 if (ret) {
516 kvm_err("Unable to allocate maps\n");
517 goto out;
518 }
519
520 ret = kvm_phys_addr_ioremap(kvm, kvm->arch.vgic.vgic_cpu_base,
521 params->vcpu_base, KVM_VGIC_V2_CPU_SIZE,
522 true);
523 if (ret) {
524 kvm_err("Unable to remap VGIC CPU to VCPU\n");
525 goto out;
526 }
527
528 kvm->arch.vgic.ready = true;
529out:
530 if (ret)
531 kvm_vgic_destroy(kvm);
532 mutex_unlock(&kvm->lock);
533 return ret;
534}
535
536static void vgic_v2_add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
537{
538 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
539
540 *vgic_get_sgi_sources(dist, vcpu->vcpu_id, irq) |= 1 << source;
541}
542
543static int vgic_v2_init_model(struct kvm *kvm)
544{
545 int i;
546
547 for (i = VGIC_NR_PRIVATE_IRQS; i < kvm->arch.vgic.nr_irqs; i += 4)
548 vgic_set_target_reg(kvm, 0, i);
549
550 return 0;
551}
552
553void vgic_v2_init_emulation(struct kvm *kvm)
554{
555 struct vgic_dist *dist = &kvm->arch.vgic;
556
557 dist->vm_ops.handle_mmio = vgic_v2_handle_mmio;
558 dist->vm_ops.queue_sgi = vgic_v2_queue_sgi;
559 dist->vm_ops.add_sgi_source = vgic_v2_add_sgi_source;
560 dist->vm_ops.init_model = vgic_v2_init_model;
561 dist->vm_ops.map_resources = vgic_v2_map_resources;
562
563 kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS;
564}
565
566static bool handle_cpu_mmio_misc(struct kvm_vcpu *vcpu,
567 struct kvm_exit_mmio *mmio, phys_addr_t offset)
568{
569 bool updated = false;
570 struct vgic_vmcr vmcr;
571 u32 *vmcr_field;
572 u32 reg;
573
574 vgic_get_vmcr(vcpu, &vmcr);
575
576 switch (offset & ~0x3) {
577 case GIC_CPU_CTRL:
578 vmcr_field = &vmcr.ctlr;
579 break;
580 case GIC_CPU_PRIMASK:
581 vmcr_field = &vmcr.pmr;
582 break;
583 case GIC_CPU_BINPOINT:
584 vmcr_field = &vmcr.bpr;
585 break;
586 case GIC_CPU_ALIAS_BINPOINT:
587 vmcr_field = &vmcr.abpr;
588 break;
589 default:
590 BUG();
591 }
592
593 if (!mmio->is_write) {
594 reg = *vmcr_field;
595 mmio_data_write(mmio, ~0, reg);
596 } else {
597 reg = mmio_data_read(mmio, ~0);
598 if (reg != *vmcr_field) {
599 *vmcr_field = reg;
600 vgic_set_vmcr(vcpu, &vmcr);
601 updated = true;
602 }
603 }
604 return updated;
605}
606
607static bool handle_mmio_abpr(struct kvm_vcpu *vcpu,
608 struct kvm_exit_mmio *mmio, phys_addr_t offset)
609{
610 return handle_cpu_mmio_misc(vcpu, mmio, GIC_CPU_ALIAS_BINPOINT);
611}
612
613static bool handle_cpu_mmio_ident(struct kvm_vcpu *vcpu,
614 struct kvm_exit_mmio *mmio,
615 phys_addr_t offset)
616{
617 u32 reg;
618
619 if (mmio->is_write)
620 return false;
621
622 /* GICC_IIDR */
623 reg = (PRODUCT_ID_KVM << 20) |
624 (GICC_ARCH_VERSION_V2 << 16) |
625 (IMPLEMENTER_ARM << 0);
626 mmio_data_write(mmio, ~0, reg);
627 return false;
628}
629
630/*
631 * CPU Interface Register accesses - these are not accessed by the VM, but by
632 * user space for saving and restoring VGIC state.
633 */
634static const struct kvm_mmio_range vgic_cpu_ranges[] = {
635 {
636 .base = GIC_CPU_CTRL,
637 .len = 12,
638 .handle_mmio = handle_cpu_mmio_misc,
639 },
640 {
641 .base = GIC_CPU_ALIAS_BINPOINT,
642 .len = 4,
643 .handle_mmio = handle_mmio_abpr,
644 },
645 {
646 .base = GIC_CPU_ACTIVEPRIO,
647 .len = 16,
648 .handle_mmio = handle_mmio_raz_wi,
649 },
650 {
651 .base = GIC_CPU_IDENT,
652 .len = 4,
653 .handle_mmio = handle_cpu_mmio_ident,
654 },
655};
656
657static int vgic_attr_regs_access(struct kvm_device *dev,
658 struct kvm_device_attr *attr,
659 u32 *reg, bool is_write)
660{
661 const struct kvm_mmio_range *r = NULL, *ranges;
662 phys_addr_t offset;
663 int ret, cpuid, c;
664 struct kvm_vcpu *vcpu, *tmp_vcpu;
665 struct vgic_dist *vgic;
666 struct kvm_exit_mmio mmio;
667
668 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
669 cpuid = (attr->attr & KVM_DEV_ARM_VGIC_CPUID_MASK) >>
670 KVM_DEV_ARM_VGIC_CPUID_SHIFT;
671
672 mutex_lock(&dev->kvm->lock);
673
674 ret = vgic_init(dev->kvm);
675 if (ret)
676 goto out;
677
678 if (cpuid >= atomic_read(&dev->kvm->online_vcpus)) {
679 ret = -EINVAL;
680 goto out;
681 }
682
683 vcpu = kvm_get_vcpu(dev->kvm, cpuid);
684 vgic = &dev->kvm->arch.vgic;
685
686 mmio.len = 4;
687 mmio.is_write = is_write;
688 if (is_write)
689 mmio_data_write(&mmio, ~0, *reg);
690 switch (attr->group) {
691 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
692 mmio.phys_addr = vgic->vgic_dist_base + offset;
693 ranges = vgic_dist_ranges;
694 break;
695 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
696 mmio.phys_addr = vgic->vgic_cpu_base + offset;
697 ranges = vgic_cpu_ranges;
698 break;
699 default:
700 BUG();
701 }
702 r = vgic_find_range(ranges, &mmio, offset);
703
704 if (unlikely(!r || !r->handle_mmio)) {
705 ret = -ENXIO;
706 goto out;
707 }
708
709
710 spin_lock(&vgic->lock);
711
712 /*
713 * Ensure that no other VCPU is running by checking the vcpu->cpu
714 * field. If no other VPCUs are running we can safely access the VGIC
715 * state, because even if another VPU is run after this point, that
716 * VCPU will not touch the vgic state, because it will block on
717 * getting the vgic->lock in kvm_vgic_sync_hwstate().
718 */
719 kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm) {
720 if (unlikely(tmp_vcpu->cpu != -1)) {
721 ret = -EBUSY;
722 goto out_vgic_unlock;
723 }
724 }
725
726 /*
727 * Move all pending IRQs from the LRs on all VCPUs so the pending
728 * state can be properly represented in the register state accessible
729 * through this API.
730 */
731 kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm)
732 vgic_unqueue_irqs(tmp_vcpu);
733
734 offset -= r->base;
735 r->handle_mmio(vcpu, &mmio, offset);
736
737 if (!is_write)
738 *reg = mmio_data_read(&mmio, ~0);
739
740 ret = 0;
741out_vgic_unlock:
742 spin_unlock(&vgic->lock);
743out:
744 mutex_unlock(&dev->kvm->lock);
745 return ret;
746}
747
748static int vgic_v2_create(struct kvm_device *dev, u32 type)
749{
750 return kvm_vgic_create(dev->kvm, type);
751}
752
753static void vgic_v2_destroy(struct kvm_device *dev)
754{
755 kfree(dev);
756}
757
758static int vgic_v2_set_attr(struct kvm_device *dev,
759 struct kvm_device_attr *attr)
760{
761 int ret;
762
763 ret = vgic_set_common_attr(dev, attr);
764 if (ret != -ENXIO)
765 return ret;
766
767 switch (attr->group) {
768 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
769 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
770 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
771 u32 reg;
772
773 if (get_user(reg, uaddr))
774 return -EFAULT;
775
776 return vgic_attr_regs_access(dev, attr, &reg, true);
777 }
778
779 }
780
781 return -ENXIO;
782}
783
784static int vgic_v2_get_attr(struct kvm_device *dev,
785 struct kvm_device_attr *attr)
786{
787 int ret;
788
789 ret = vgic_get_common_attr(dev, attr);
790 if (ret != -ENXIO)
791 return ret;
792
793 switch (attr->group) {
794 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
795 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
796 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
797 u32 reg = 0;
798
799 ret = vgic_attr_regs_access(dev, attr, &reg, false);
800 if (ret)
801 return ret;
802 return put_user(reg, uaddr);
803 }
804
805 }
806
807 return -ENXIO;
808}
809
810static int vgic_v2_has_attr(struct kvm_device *dev,
811 struct kvm_device_attr *attr)
812{
813 phys_addr_t offset;
814
815 switch (attr->group) {
816 case KVM_DEV_ARM_VGIC_GRP_ADDR:
817 switch (attr->attr) {
818 case KVM_VGIC_V2_ADDR_TYPE_DIST:
819 case KVM_VGIC_V2_ADDR_TYPE_CPU:
820 return 0;
821 }
822 break;
823 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
824 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
825 return vgic_has_attr_regs(vgic_dist_ranges, offset);
826 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
827 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
828 return vgic_has_attr_regs(vgic_cpu_ranges, offset);
829 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
830 return 0;
831 case KVM_DEV_ARM_VGIC_GRP_CTRL:
832 switch (attr->attr) {
833 case KVM_DEV_ARM_VGIC_CTRL_INIT:
834 return 0;
835 }
836 }
837 return -ENXIO;
838}
839
840struct kvm_device_ops kvm_arm_vgic_v2_ops = {
841 .name = "kvm-arm-vgic-v2",
842 .create = vgic_v2_create,
843 .destroy = vgic_v2_destroy,
844 .set_attr = vgic_v2_set_attr,
845 .get_attr = vgic_v2_get_attr,
846 .has_attr = vgic_v2_has_attr,
847};
diff --git a/virt/kvm/arm/vgic-v2.c b/virt/kvm/arm/vgic-v2.c
index 2935405ad22f..f9b9c7c51372 100644
--- a/virt/kvm/arm/vgic-v2.c
+++ b/virt/kvm/arm/vgic-v2.c
@@ -72,6 +72,8 @@ static void vgic_v2_sync_lr_elrsr(struct kvm_vcpu *vcpu, int lr,
72{ 72{
73 if (!(lr_desc.state & LR_STATE_MASK)) 73 if (!(lr_desc.state & LR_STATE_MASK))
74 vcpu->arch.vgic_cpu.vgic_v2.vgic_elrsr |= (1ULL << lr); 74 vcpu->arch.vgic_cpu.vgic_v2.vgic_elrsr |= (1ULL << lr);
75 else
76 vcpu->arch.vgic_cpu.vgic_v2.vgic_elrsr &= ~(1ULL << lr);
75} 77}
76 78
77static u64 vgic_v2_get_elrsr(const struct kvm_vcpu *vcpu) 79static u64 vgic_v2_get_elrsr(const struct kvm_vcpu *vcpu)
@@ -84,6 +86,11 @@ static u64 vgic_v2_get_eisr(const struct kvm_vcpu *vcpu)
84 return vcpu->arch.vgic_cpu.vgic_v2.vgic_eisr; 86 return vcpu->arch.vgic_cpu.vgic_v2.vgic_eisr;
85} 87}
86 88
89static void vgic_v2_clear_eisr(struct kvm_vcpu *vcpu)
90{
91 vcpu->arch.vgic_cpu.vgic_v2.vgic_eisr = 0;
92}
93
87static u32 vgic_v2_get_interrupt_status(const struct kvm_vcpu *vcpu) 94static u32 vgic_v2_get_interrupt_status(const struct kvm_vcpu *vcpu)
88{ 95{
89 u32 misr = vcpu->arch.vgic_cpu.vgic_v2.vgic_misr; 96 u32 misr = vcpu->arch.vgic_cpu.vgic_v2.vgic_misr;
@@ -148,6 +155,7 @@ static const struct vgic_ops vgic_v2_ops = {
148 .sync_lr_elrsr = vgic_v2_sync_lr_elrsr, 155 .sync_lr_elrsr = vgic_v2_sync_lr_elrsr,
149 .get_elrsr = vgic_v2_get_elrsr, 156 .get_elrsr = vgic_v2_get_elrsr,
150 .get_eisr = vgic_v2_get_eisr, 157 .get_eisr = vgic_v2_get_eisr,
158 .clear_eisr = vgic_v2_clear_eisr,
151 .get_interrupt_status = vgic_v2_get_interrupt_status, 159 .get_interrupt_status = vgic_v2_get_interrupt_status,
152 .enable_underflow = vgic_v2_enable_underflow, 160 .enable_underflow = vgic_v2_enable_underflow,
153 .disable_underflow = vgic_v2_disable_underflow, 161 .disable_underflow = vgic_v2_disable_underflow,
@@ -229,12 +237,16 @@ int vgic_v2_probe(struct device_node *vgic_node,
229 goto out_unmap; 237 goto out_unmap;
230 } 238 }
231 239
240 vgic->can_emulate_gicv2 = true;
241 kvm_register_device_ops(&kvm_arm_vgic_v2_ops, KVM_DEV_TYPE_ARM_VGIC_V2);
242
232 vgic->vcpu_base = vcpu_res.start; 243 vgic->vcpu_base = vcpu_res.start;
233 244
234 kvm_info("%s@%llx IRQ%d\n", vgic_node->name, 245 kvm_info("%s@%llx IRQ%d\n", vgic_node->name,
235 vctrl_res.start, vgic->maint_irq); 246 vctrl_res.start, vgic->maint_irq);
236 247
237 vgic->type = VGIC_V2; 248 vgic->type = VGIC_V2;
249 vgic->max_gic_vcpus = VGIC_V2_MAX_CPUS;
238 *ops = &vgic_v2_ops; 250 *ops = &vgic_v2_ops;
239 *params = vgic; 251 *params = vgic;
240 goto out; 252 goto out;
diff --git a/virt/kvm/arm/vgic-v3-emul.c b/virt/kvm/arm/vgic-v3-emul.c
new file mode 100644
index 000000000000..b3f154631515
--- /dev/null
+++ b/virt/kvm/arm/vgic-v3-emul.c
@@ -0,0 +1,1036 @@
1/*
2 * GICv3 distributor and redistributor emulation
3 *
4 * GICv3 emulation is currently only supported on a GICv3 host (because
5 * we rely on the hardware's CPU interface virtualization support), but
6 * supports both hardware with or without the optional GICv2 backwards
7 * compatibility features.
8 *
9 * Limitations of the emulation:
10 * (RAZ/WI: read as zero, write ignore, RAO/WI: read as one, write ignore)
11 * - We do not support LPIs (yet). TYPER.LPIS is reported as 0 and is RAZ/WI.
12 * - We do not support the message based interrupts (MBIs) triggered by
13 * writes to the GICD_{SET,CLR}SPI_* registers. TYPER.MBIS is reported as 0.
14 * - We do not support the (optional) backwards compatibility feature.
15 * GICD_CTLR.ARE resets to 1 and is RAO/WI. If the _host_ GIC supports
16 * the compatiblity feature, you can use a GICv2 in the guest, though.
17 * - We only support a single security state. GICD_CTLR.DS is 1 and is RAO/WI.
18 * - Priorities are not emulated (same as the GICv2 emulation). Linux
19 * as a guest is fine with this, because it does not use priorities.
20 * - We only support Group1 interrupts. Again Linux uses only those.
21 *
22 * Copyright (C) 2014 ARM Ltd.
23 * Author: Andre Przywara <andre.przywara@arm.com>
24 *
25 * This program is free software; you can redistribute it and/or modify
26 * it under the terms of the GNU General Public License version 2 as
27 * published by the Free Software Foundation.
28 *
29 * This program is distributed in the hope that it will be useful,
30 * but WITHOUT ANY WARRANTY; without even the implied warranty of
31 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32 * GNU General Public License for more details.
33 *
34 * You should have received a copy of the GNU General Public License
35 * along with this program. If not, see <http://www.gnu.org/licenses/>.
36 */
37
38#include <linux/cpu.h>
39#include <linux/kvm.h>
40#include <linux/kvm_host.h>
41#include <linux/interrupt.h>
42
43#include <linux/irqchip/arm-gic-v3.h>
44#include <kvm/arm_vgic.h>
45
46#include <asm/kvm_emulate.h>
47#include <asm/kvm_arm.h>
48#include <asm/kvm_mmu.h>
49
50#include "vgic.h"
51
52static bool handle_mmio_rao_wi(struct kvm_vcpu *vcpu,
53 struct kvm_exit_mmio *mmio, phys_addr_t offset)
54{
55 u32 reg = 0xffffffff;
56
57 vgic_reg_access(mmio, &reg, offset,
58 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
59
60 return false;
61}
62
63static bool handle_mmio_ctlr(struct kvm_vcpu *vcpu,
64 struct kvm_exit_mmio *mmio, phys_addr_t offset)
65{
66 u32 reg = 0;
67
68 /*
69 * Force ARE and DS to 1, the guest cannot change this.
70 * For the time being we only support Group1 interrupts.
71 */
72 if (vcpu->kvm->arch.vgic.enabled)
73 reg = GICD_CTLR_ENABLE_SS_G1;
74 reg |= GICD_CTLR_ARE_NS | GICD_CTLR_DS;
75
76 vgic_reg_access(mmio, &reg, offset,
77 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
78 if (mmio->is_write) {
79 if (reg & GICD_CTLR_ENABLE_SS_G0)
80 kvm_info("guest tried to enable unsupported Group0 interrupts\n");
81 vcpu->kvm->arch.vgic.enabled = !!(reg & GICD_CTLR_ENABLE_SS_G1);
82 vgic_update_state(vcpu->kvm);
83 return true;
84 }
85 return false;
86}
87
88/*
89 * As this implementation does not provide compatibility
90 * with GICv2 (ARE==1), we report zero CPUs in bits [5..7].
91 * Also LPIs and MBIs are not supported, so we set the respective bits to 0.
92 * Also we report at most 2**10=1024 interrupt IDs (to match 1024 SPIs).
93 */
94#define INTERRUPT_ID_BITS 10
95static bool handle_mmio_typer(struct kvm_vcpu *vcpu,
96 struct kvm_exit_mmio *mmio, phys_addr_t offset)
97{
98 u32 reg;
99
100 reg = (min(vcpu->kvm->arch.vgic.nr_irqs, 1024) >> 5) - 1;
101
102 reg |= (INTERRUPT_ID_BITS - 1) << 19;
103
104 vgic_reg_access(mmio, &reg, offset,
105 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
106
107 return false;
108}
109
110static bool handle_mmio_iidr(struct kvm_vcpu *vcpu,
111 struct kvm_exit_mmio *mmio, phys_addr_t offset)
112{
113 u32 reg;
114
115 reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
116 vgic_reg_access(mmio, &reg, offset,
117 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
118
119 return false;
120}
121
122static bool handle_mmio_set_enable_reg_dist(struct kvm_vcpu *vcpu,
123 struct kvm_exit_mmio *mmio,
124 phys_addr_t offset)
125{
126 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
127 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
128 vcpu->vcpu_id,
129 ACCESS_WRITE_SETBIT);
130
131 vgic_reg_access(mmio, NULL, offset,
132 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
133 return false;
134}
135
136static bool handle_mmio_clear_enable_reg_dist(struct kvm_vcpu *vcpu,
137 struct kvm_exit_mmio *mmio,
138 phys_addr_t offset)
139{
140 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
141 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
142 vcpu->vcpu_id,
143 ACCESS_WRITE_CLEARBIT);
144
145 vgic_reg_access(mmio, NULL, offset,
146 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
147 return false;
148}
149
150static bool handle_mmio_set_pending_reg_dist(struct kvm_vcpu *vcpu,
151 struct kvm_exit_mmio *mmio,
152 phys_addr_t offset)
153{
154 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
155 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
156 vcpu->vcpu_id);
157
158 vgic_reg_access(mmio, NULL, offset,
159 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
160 return false;
161}
162
163static bool handle_mmio_clear_pending_reg_dist(struct kvm_vcpu *vcpu,
164 struct kvm_exit_mmio *mmio,
165 phys_addr_t offset)
166{
167 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8))
168 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
169 vcpu->vcpu_id);
170
171 vgic_reg_access(mmio, NULL, offset,
172 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
173 return false;
174}
175
176static bool handle_mmio_priority_reg_dist(struct kvm_vcpu *vcpu,
177 struct kvm_exit_mmio *mmio,
178 phys_addr_t offset)
179{
180 u32 *reg;
181
182 if (unlikely(offset < VGIC_NR_PRIVATE_IRQS)) {
183 vgic_reg_access(mmio, NULL, offset,
184 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
185 return false;
186 }
187
188 reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
189 vcpu->vcpu_id, offset);
190 vgic_reg_access(mmio, reg, offset,
191 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
192 return false;
193}
194
195static bool handle_mmio_cfg_reg_dist(struct kvm_vcpu *vcpu,
196 struct kvm_exit_mmio *mmio,
197 phys_addr_t offset)
198{
199 u32 *reg;
200
201 if (unlikely(offset < VGIC_NR_PRIVATE_IRQS / 4)) {
202 vgic_reg_access(mmio, NULL, offset,
203 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
204 return false;
205 }
206
207 reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
208 vcpu->vcpu_id, offset >> 1);
209
210 return vgic_handle_cfg_reg(reg, mmio, offset);
211}
212
213/*
214 * We use a compressed version of the MPIDR (all 32 bits in one 32-bit word)
215 * when we store the target MPIDR written by the guest.
216 */
217static u32 compress_mpidr(unsigned long mpidr)
218{
219 u32 ret;
220
221 ret = MPIDR_AFFINITY_LEVEL(mpidr, 0);
222 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8;
223 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16;
224 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24;
225
226 return ret;
227}
228
229static unsigned long uncompress_mpidr(u32 value)
230{
231 unsigned long mpidr;
232
233 mpidr = ((value >> 0) & 0xFF) << MPIDR_LEVEL_SHIFT(0);
234 mpidr |= ((value >> 8) & 0xFF) << MPIDR_LEVEL_SHIFT(1);
235 mpidr |= ((value >> 16) & 0xFF) << MPIDR_LEVEL_SHIFT(2);
236 mpidr |= (u64)((value >> 24) & 0xFF) << MPIDR_LEVEL_SHIFT(3);
237
238 return mpidr;
239}
240
241/*
242 * Lookup the given MPIDR value to get the vcpu_id (if there is one)
243 * and store that in the irq_spi_cpu[] array.
244 * This limits the number of VCPUs to 255 for now, extending the data
245 * type (or storing kvm_vcpu pointers) should lift the limit.
246 * Store the original MPIDR value in an extra array to support read-as-written.
247 * Unallocated MPIDRs are translated to a special value and caught
248 * before any array accesses.
249 */
250static bool handle_mmio_route_reg(struct kvm_vcpu *vcpu,
251 struct kvm_exit_mmio *mmio,
252 phys_addr_t offset)
253{
254 struct kvm *kvm = vcpu->kvm;
255 struct vgic_dist *dist = &kvm->arch.vgic;
256 int spi;
257 u32 reg;
258 int vcpu_id;
259 unsigned long *bmap, mpidr;
260
261 /*
262 * The upper 32 bits of each 64 bit register are zero,
263 * as we don't support Aff3.
264 */
265 if ((offset & 4)) {
266 vgic_reg_access(mmio, NULL, offset,
267 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
268 return false;
269 }
270
271 /* This region only covers SPIs, so no handling of private IRQs here. */
272 spi = offset / 8;
273
274 /* get the stored MPIDR for this IRQ */
275 mpidr = uncompress_mpidr(dist->irq_spi_mpidr[spi]);
276 reg = mpidr;
277
278 vgic_reg_access(mmio, &reg, offset,
279 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
280
281 if (!mmio->is_write)
282 return false;
283
284 /*
285 * Now clear the currently assigned vCPU from the map, making room
286 * for the new one to be written below
287 */
288 vcpu = kvm_mpidr_to_vcpu(kvm, mpidr);
289 if (likely(vcpu)) {
290 vcpu_id = vcpu->vcpu_id;
291 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]);
292 __clear_bit(spi, bmap);
293 }
294
295 dist->irq_spi_mpidr[spi] = compress_mpidr(reg);
296 vcpu = kvm_mpidr_to_vcpu(kvm, reg & MPIDR_HWID_BITMASK);
297
298 /*
299 * The spec says that non-existent MPIDR values should not be
300 * forwarded to any existent (v)CPU, but should be able to become
301 * pending anyway. We simply keep the irq_spi_target[] array empty, so
302 * the interrupt will never be injected.
303 * irq_spi_cpu[irq] gets a magic value in this case.
304 */
305 if (likely(vcpu)) {
306 vcpu_id = vcpu->vcpu_id;
307 dist->irq_spi_cpu[spi] = vcpu_id;
308 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]);
309 __set_bit(spi, bmap);
310 } else {
311 dist->irq_spi_cpu[spi] = VCPU_NOT_ALLOCATED;
312 }
313
314 vgic_update_state(kvm);
315
316 return true;
317}
318
319/*
320 * We should be careful about promising too much when a guest reads
321 * this register. Don't claim to be like any hardware implementation,
322 * but just report the GIC as version 3 - which is what a Linux guest
323 * would check.
324 */
325static bool handle_mmio_idregs(struct kvm_vcpu *vcpu,
326 struct kvm_exit_mmio *mmio,
327 phys_addr_t offset)
328{
329 u32 reg = 0;
330
331 switch (offset + GICD_IDREGS) {
332 case GICD_PIDR2:
333 reg = 0x3b;
334 break;
335 }
336
337 vgic_reg_access(mmio, &reg, offset,
338 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
339
340 return false;
341}
342
343static const struct kvm_mmio_range vgic_v3_dist_ranges[] = {
344 {
345 .base = GICD_CTLR,
346 .len = 0x04,
347 .bits_per_irq = 0,
348 .handle_mmio = handle_mmio_ctlr,
349 },
350 {
351 .base = GICD_TYPER,
352 .len = 0x04,
353 .bits_per_irq = 0,
354 .handle_mmio = handle_mmio_typer,
355 },
356 {
357 .base = GICD_IIDR,
358 .len = 0x04,
359 .bits_per_irq = 0,
360 .handle_mmio = handle_mmio_iidr,
361 },
362 {
363 /* this register is optional, it is RAZ/WI if not implemented */
364 .base = GICD_STATUSR,
365 .len = 0x04,
366 .bits_per_irq = 0,
367 .handle_mmio = handle_mmio_raz_wi,
368 },
369 {
370 /* this write only register is WI when TYPER.MBIS=0 */
371 .base = GICD_SETSPI_NSR,
372 .len = 0x04,
373 .bits_per_irq = 0,
374 .handle_mmio = handle_mmio_raz_wi,
375 },
376 {
377 /* this write only register is WI when TYPER.MBIS=0 */
378 .base = GICD_CLRSPI_NSR,
379 .len = 0x04,
380 .bits_per_irq = 0,
381 .handle_mmio = handle_mmio_raz_wi,
382 },
383 {
384 /* this is RAZ/WI when DS=1 */
385 .base = GICD_SETSPI_SR,
386 .len = 0x04,
387 .bits_per_irq = 0,
388 .handle_mmio = handle_mmio_raz_wi,
389 },
390 {
391 /* this is RAZ/WI when DS=1 */
392 .base = GICD_CLRSPI_SR,
393 .len = 0x04,
394 .bits_per_irq = 0,
395 .handle_mmio = handle_mmio_raz_wi,
396 },
397 {
398 .base = GICD_IGROUPR,
399 .len = 0x80,
400 .bits_per_irq = 1,
401 .handle_mmio = handle_mmio_rao_wi,
402 },
403 {
404 .base = GICD_ISENABLER,
405 .len = 0x80,
406 .bits_per_irq = 1,
407 .handle_mmio = handle_mmio_set_enable_reg_dist,
408 },
409 {
410 .base = GICD_ICENABLER,
411 .len = 0x80,
412 .bits_per_irq = 1,
413 .handle_mmio = handle_mmio_clear_enable_reg_dist,
414 },
415 {
416 .base = GICD_ISPENDR,
417 .len = 0x80,
418 .bits_per_irq = 1,
419 .handle_mmio = handle_mmio_set_pending_reg_dist,
420 },
421 {
422 .base = GICD_ICPENDR,
423 .len = 0x80,
424 .bits_per_irq = 1,
425 .handle_mmio = handle_mmio_clear_pending_reg_dist,
426 },
427 {
428 .base = GICD_ISACTIVER,
429 .len = 0x80,
430 .bits_per_irq = 1,
431 .handle_mmio = handle_mmio_raz_wi,
432 },
433 {
434 .base = GICD_ICACTIVER,
435 .len = 0x80,
436 .bits_per_irq = 1,
437 .handle_mmio = handle_mmio_raz_wi,
438 },
439 {
440 .base = GICD_IPRIORITYR,
441 .len = 0x400,
442 .bits_per_irq = 8,
443 .handle_mmio = handle_mmio_priority_reg_dist,
444 },
445 {
446 /* TARGETSRn is RES0 when ARE=1 */
447 .base = GICD_ITARGETSR,
448 .len = 0x400,
449 .bits_per_irq = 8,
450 .handle_mmio = handle_mmio_raz_wi,
451 },
452 {
453 .base = GICD_ICFGR,
454 .len = 0x100,
455 .bits_per_irq = 2,
456 .handle_mmio = handle_mmio_cfg_reg_dist,
457 },
458 {
459 /* this is RAZ/WI when DS=1 */
460 .base = GICD_IGRPMODR,
461 .len = 0x80,
462 .bits_per_irq = 1,
463 .handle_mmio = handle_mmio_raz_wi,
464 },
465 {
466 /* this is RAZ/WI when DS=1 */
467 .base = GICD_NSACR,
468 .len = 0x100,
469 .bits_per_irq = 2,
470 .handle_mmio = handle_mmio_raz_wi,
471 },
472 {
473 /* this is RAZ/WI when ARE=1 */
474 .base = GICD_SGIR,
475 .len = 0x04,
476 .handle_mmio = handle_mmio_raz_wi,
477 },
478 {
479 /* this is RAZ/WI when ARE=1 */
480 .base = GICD_CPENDSGIR,
481 .len = 0x10,
482 .handle_mmio = handle_mmio_raz_wi,
483 },
484 {
485 /* this is RAZ/WI when ARE=1 */
486 .base = GICD_SPENDSGIR,
487 .len = 0x10,
488 .handle_mmio = handle_mmio_raz_wi,
489 },
490 {
491 .base = GICD_IROUTER + 0x100,
492 .len = 0x1ee0,
493 .bits_per_irq = 64,
494 .handle_mmio = handle_mmio_route_reg,
495 },
496 {
497 .base = GICD_IDREGS,
498 .len = 0x30,
499 .bits_per_irq = 0,
500 .handle_mmio = handle_mmio_idregs,
501 },
502 {},
503};
504
505static bool handle_mmio_set_enable_reg_redist(struct kvm_vcpu *vcpu,
506 struct kvm_exit_mmio *mmio,
507 phys_addr_t offset)
508{
509 struct kvm_vcpu *redist_vcpu = mmio->private;
510
511 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
512 redist_vcpu->vcpu_id,
513 ACCESS_WRITE_SETBIT);
514}
515
516static bool handle_mmio_clear_enable_reg_redist(struct kvm_vcpu *vcpu,
517 struct kvm_exit_mmio *mmio,
518 phys_addr_t offset)
519{
520 struct kvm_vcpu *redist_vcpu = mmio->private;
521
522 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset,
523 redist_vcpu->vcpu_id,
524 ACCESS_WRITE_CLEARBIT);
525}
526
527static bool handle_mmio_set_pending_reg_redist(struct kvm_vcpu *vcpu,
528 struct kvm_exit_mmio *mmio,
529 phys_addr_t offset)
530{
531 struct kvm_vcpu *redist_vcpu = mmio->private;
532
533 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset,
534 redist_vcpu->vcpu_id);
535}
536
537static bool handle_mmio_clear_pending_reg_redist(struct kvm_vcpu *vcpu,
538 struct kvm_exit_mmio *mmio,
539 phys_addr_t offset)
540{
541 struct kvm_vcpu *redist_vcpu = mmio->private;
542
543 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset,
544 redist_vcpu->vcpu_id);
545}
546
547static bool handle_mmio_priority_reg_redist(struct kvm_vcpu *vcpu,
548 struct kvm_exit_mmio *mmio,
549 phys_addr_t offset)
550{
551 struct kvm_vcpu *redist_vcpu = mmio->private;
552 u32 *reg;
553
554 reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
555 redist_vcpu->vcpu_id, offset);
556 vgic_reg_access(mmio, reg, offset,
557 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
558 return false;
559}
560
561static bool handle_mmio_cfg_reg_redist(struct kvm_vcpu *vcpu,
562 struct kvm_exit_mmio *mmio,
563 phys_addr_t offset)
564{
565 struct kvm_vcpu *redist_vcpu = mmio->private;
566
567 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
568 redist_vcpu->vcpu_id, offset >> 1);
569
570 return vgic_handle_cfg_reg(reg, mmio, offset);
571}
572
573static const struct kvm_mmio_range vgic_redist_sgi_ranges[] = {
574 {
575 .base = GICR_IGROUPR0,
576 .len = 0x04,
577 .bits_per_irq = 1,
578 .handle_mmio = handle_mmio_rao_wi,
579 },
580 {
581 .base = GICR_ISENABLER0,
582 .len = 0x04,
583 .bits_per_irq = 1,
584 .handle_mmio = handle_mmio_set_enable_reg_redist,
585 },
586 {
587 .base = GICR_ICENABLER0,
588 .len = 0x04,
589 .bits_per_irq = 1,
590 .handle_mmio = handle_mmio_clear_enable_reg_redist,
591 },
592 {
593 .base = GICR_ISPENDR0,
594 .len = 0x04,
595 .bits_per_irq = 1,
596 .handle_mmio = handle_mmio_set_pending_reg_redist,
597 },
598 {
599 .base = GICR_ICPENDR0,
600 .len = 0x04,
601 .bits_per_irq = 1,
602 .handle_mmio = handle_mmio_clear_pending_reg_redist,
603 },
604 {
605 .base = GICR_ISACTIVER0,
606 .len = 0x04,
607 .bits_per_irq = 1,
608 .handle_mmio = handle_mmio_raz_wi,
609 },
610 {
611 .base = GICR_ICACTIVER0,
612 .len = 0x04,
613 .bits_per_irq = 1,
614 .handle_mmio = handle_mmio_raz_wi,
615 },
616 {
617 .base = GICR_IPRIORITYR0,
618 .len = 0x20,
619 .bits_per_irq = 8,
620 .handle_mmio = handle_mmio_priority_reg_redist,
621 },
622 {
623 .base = GICR_ICFGR0,
624 .len = 0x08,
625 .bits_per_irq = 2,
626 .handle_mmio = handle_mmio_cfg_reg_redist,
627 },
628 {
629 .base = GICR_IGRPMODR0,
630 .len = 0x04,
631 .bits_per_irq = 1,
632 .handle_mmio = handle_mmio_raz_wi,
633 },
634 {
635 .base = GICR_NSACR,
636 .len = 0x04,
637 .handle_mmio = handle_mmio_raz_wi,
638 },
639 {},
640};
641
642static bool handle_mmio_ctlr_redist(struct kvm_vcpu *vcpu,
643 struct kvm_exit_mmio *mmio,
644 phys_addr_t offset)
645{
646 /* since we don't support LPIs, this register is zero for now */
647 vgic_reg_access(mmio, NULL, offset,
648 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
649 return false;
650}
651
652static bool handle_mmio_typer_redist(struct kvm_vcpu *vcpu,
653 struct kvm_exit_mmio *mmio,
654 phys_addr_t offset)
655{
656 u32 reg;
657 u64 mpidr;
658 struct kvm_vcpu *redist_vcpu = mmio->private;
659 int target_vcpu_id = redist_vcpu->vcpu_id;
660
661 /* the upper 32 bits contain the affinity value */
662 if ((offset & ~3) == 4) {
663 mpidr = kvm_vcpu_get_mpidr_aff(redist_vcpu);
664 reg = compress_mpidr(mpidr);
665
666 vgic_reg_access(mmio, &reg, offset,
667 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
668 return false;
669 }
670
671 reg = redist_vcpu->vcpu_id << 8;
672 if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1)
673 reg |= GICR_TYPER_LAST;
674 vgic_reg_access(mmio, &reg, offset,
675 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
676 return false;
677}
678
679static const struct kvm_mmio_range vgic_redist_ranges[] = {
680 {
681 .base = GICR_CTLR,
682 .len = 0x04,
683 .bits_per_irq = 0,
684 .handle_mmio = handle_mmio_ctlr_redist,
685 },
686 {
687 .base = GICR_TYPER,
688 .len = 0x08,
689 .bits_per_irq = 0,
690 .handle_mmio = handle_mmio_typer_redist,
691 },
692 {
693 .base = GICR_IIDR,
694 .len = 0x04,
695 .bits_per_irq = 0,
696 .handle_mmio = handle_mmio_iidr,
697 },
698 {
699 .base = GICR_WAKER,
700 .len = 0x04,
701 .bits_per_irq = 0,
702 .handle_mmio = handle_mmio_raz_wi,
703 },
704 {
705 .base = GICR_IDREGS,
706 .len = 0x30,
707 .bits_per_irq = 0,
708 .handle_mmio = handle_mmio_idregs,
709 },
710 {},
711};
712
713/*
714 * This function splits accesses between the distributor and the two
715 * redistributor parts (private/SPI). As each redistributor is accessible
716 * from any CPU, we have to determine the affected VCPU by taking the faulting
717 * address into account. We then pass this VCPU to the handler function via
718 * the private parameter.
719 */
720#define SGI_BASE_OFFSET SZ_64K
721static bool vgic_v3_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
722 struct kvm_exit_mmio *mmio)
723{
724 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
725 unsigned long dbase = dist->vgic_dist_base;
726 unsigned long rdbase = dist->vgic_redist_base;
727 int nrcpus = atomic_read(&vcpu->kvm->online_vcpus);
728 int vcpu_id;
729 const struct kvm_mmio_range *mmio_range;
730
731 if (is_in_range(mmio->phys_addr, mmio->len, dbase, GIC_V3_DIST_SIZE)) {
732 return vgic_handle_mmio_range(vcpu, run, mmio,
733 vgic_v3_dist_ranges, dbase);
734 }
735
736 if (!is_in_range(mmio->phys_addr, mmio->len, rdbase,
737 GIC_V3_REDIST_SIZE * nrcpus))
738 return false;
739
740 vcpu_id = (mmio->phys_addr - rdbase) / GIC_V3_REDIST_SIZE;
741 rdbase += (vcpu_id * GIC_V3_REDIST_SIZE);
742 mmio->private = kvm_get_vcpu(vcpu->kvm, vcpu_id);
743
744 if (mmio->phys_addr >= rdbase + SGI_BASE_OFFSET) {
745 rdbase += SGI_BASE_OFFSET;
746 mmio_range = vgic_redist_sgi_ranges;
747 } else {
748 mmio_range = vgic_redist_ranges;
749 }
750 return vgic_handle_mmio_range(vcpu, run, mmio, mmio_range, rdbase);
751}
752
753static bool vgic_v3_queue_sgi(struct kvm_vcpu *vcpu, int irq)
754{
755 if (vgic_queue_irq(vcpu, 0, irq)) {
756 vgic_dist_irq_clear_pending(vcpu, irq);
757 vgic_cpu_irq_clear(vcpu, irq);
758 return true;
759 }
760
761 return false;
762}
763
764static int vgic_v3_map_resources(struct kvm *kvm,
765 const struct vgic_params *params)
766{
767 int ret = 0;
768 struct vgic_dist *dist = &kvm->arch.vgic;
769
770 if (!irqchip_in_kernel(kvm))
771 return 0;
772
773 mutex_lock(&kvm->lock);
774
775 if (vgic_ready(kvm))
776 goto out;
777
778 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
779 IS_VGIC_ADDR_UNDEF(dist->vgic_redist_base)) {
780 kvm_err("Need to set vgic distributor addresses first\n");
781 ret = -ENXIO;
782 goto out;
783 }
784
785 /*
786 * For a VGICv3 we require the userland to explicitly initialize
787 * the VGIC before we need to use it.
788 */
789 if (!vgic_initialized(kvm)) {
790 ret = -EBUSY;
791 goto out;
792 }
793
794 kvm->arch.vgic.ready = true;
795out:
796 if (ret)
797 kvm_vgic_destroy(kvm);
798 mutex_unlock(&kvm->lock);
799 return ret;
800}
801
802static int vgic_v3_init_model(struct kvm *kvm)
803{
804 int i;
805 u32 mpidr;
806 struct vgic_dist *dist = &kvm->arch.vgic;
807 int nr_spis = dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
808
809 dist->irq_spi_mpidr = kcalloc(nr_spis, sizeof(dist->irq_spi_mpidr[0]),
810 GFP_KERNEL);
811
812 if (!dist->irq_spi_mpidr)
813 return -ENOMEM;
814
815 /* Initialize the target VCPUs for each IRQ to VCPU 0 */
816 mpidr = compress_mpidr(kvm_vcpu_get_mpidr_aff(kvm_get_vcpu(kvm, 0)));
817 for (i = VGIC_NR_PRIVATE_IRQS; i < dist->nr_irqs; i++) {
818 dist->irq_spi_cpu[i - VGIC_NR_PRIVATE_IRQS] = 0;
819 dist->irq_spi_mpidr[i - VGIC_NR_PRIVATE_IRQS] = mpidr;
820 vgic_bitmap_set_irq_val(dist->irq_spi_target, 0, i, 1);
821 }
822
823 return 0;
824}
825
826/* GICv3 does not keep track of SGI sources anymore. */
827static void vgic_v3_add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
828{
829}
830
831void vgic_v3_init_emulation(struct kvm *kvm)
832{
833 struct vgic_dist *dist = &kvm->arch.vgic;
834
835 dist->vm_ops.handle_mmio = vgic_v3_handle_mmio;
836 dist->vm_ops.queue_sgi = vgic_v3_queue_sgi;
837 dist->vm_ops.add_sgi_source = vgic_v3_add_sgi_source;
838 dist->vm_ops.init_model = vgic_v3_init_model;
839 dist->vm_ops.map_resources = vgic_v3_map_resources;
840
841 kvm->arch.max_vcpus = KVM_MAX_VCPUS;
842}
843
844/*
845 * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI
846 * generation register ICC_SGI1R_EL1) with a given VCPU.
847 * If the VCPU's MPIDR matches, return the level0 affinity, otherwise
848 * return -1.
849 */
850static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu)
851{
852 unsigned long affinity;
853 int level0;
854
855 /*
856 * Split the current VCPU's MPIDR into affinity level 0 and the
857 * rest as this is what we have to compare against.
858 */
859 affinity = kvm_vcpu_get_mpidr_aff(vcpu);
860 level0 = MPIDR_AFFINITY_LEVEL(affinity, 0);
861 affinity &= ~MPIDR_LEVEL_MASK;
862
863 /* bail out if the upper three levels don't match */
864 if (sgi_aff != affinity)
865 return -1;
866
867 /* Is this VCPU's bit set in the mask ? */
868 if (!(sgi_cpu_mask & BIT(level0)))
869 return -1;
870
871 return level0;
872}
873
874#define SGI_AFFINITY_LEVEL(reg, level) \
875 ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \
876 >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
877
878/**
879 * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs
880 * @vcpu: The VCPU requesting a SGI
881 * @reg: The value written into the ICC_SGI1R_EL1 register by that VCPU
882 *
883 * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register.
884 * This will trap in sys_regs.c and call this function.
885 * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the
886 * target processors as well as a bitmask of 16 Aff0 CPUs.
887 * If the interrupt routing mode bit is not set, we iterate over all VCPUs to
888 * check for matching ones. If this bit is set, we signal all, but not the
889 * calling VCPU.
890 */
891void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg)
892{
893 struct kvm *kvm = vcpu->kvm;
894 struct kvm_vcpu *c_vcpu;
895 struct vgic_dist *dist = &kvm->arch.vgic;
896 u16 target_cpus;
897 u64 mpidr;
898 int sgi, c;
899 int vcpu_id = vcpu->vcpu_id;
900 bool broadcast;
901 int updated = 0;
902
903 sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT;
904 broadcast = reg & BIT(ICC_SGI1R_IRQ_ROUTING_MODE_BIT);
905 target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT;
906 mpidr = SGI_AFFINITY_LEVEL(reg, 3);
907 mpidr |= SGI_AFFINITY_LEVEL(reg, 2);
908 mpidr |= SGI_AFFINITY_LEVEL(reg, 1);
909
910 /*
911 * We take the dist lock here, because we come from the sysregs
912 * code path and not from the MMIO one (which already takes the lock).
913 */
914 spin_lock(&dist->lock);
915
916 /*
917 * We iterate over all VCPUs to find the MPIDRs matching the request.
918 * If we have handled one CPU, we clear it's bit to detect early
919 * if we are already finished. This avoids iterating through all
920 * VCPUs when most of the times we just signal a single VCPU.
921 */
922 kvm_for_each_vcpu(c, c_vcpu, kvm) {
923
924 /* Exit early if we have dealt with all requested CPUs */
925 if (!broadcast && target_cpus == 0)
926 break;
927
928 /* Don't signal the calling VCPU */
929 if (broadcast && c == vcpu_id)
930 continue;
931
932 if (!broadcast) {
933 int level0;
934
935 level0 = match_mpidr(mpidr, target_cpus, c_vcpu);
936 if (level0 == -1)
937 continue;
938
939 /* remove this matching VCPU from the mask */
940 target_cpus &= ~BIT(level0);
941 }
942
943 /* Flag the SGI as pending */
944 vgic_dist_irq_set_pending(c_vcpu, sgi);
945 updated = 1;
946 kvm_debug("SGI%d from CPU%d to CPU%d\n", sgi, vcpu_id, c);
947 }
948 if (updated)
949 vgic_update_state(vcpu->kvm);
950 spin_unlock(&dist->lock);
951 if (updated)
952 vgic_kick_vcpus(vcpu->kvm);
953}
954
955static int vgic_v3_create(struct kvm_device *dev, u32 type)
956{
957 return kvm_vgic_create(dev->kvm, type);
958}
959
960static void vgic_v3_destroy(struct kvm_device *dev)
961{
962 kfree(dev);
963}
964
965static int vgic_v3_set_attr(struct kvm_device *dev,
966 struct kvm_device_attr *attr)
967{
968 int ret;
969
970 ret = vgic_set_common_attr(dev, attr);
971 if (ret != -ENXIO)
972 return ret;
973
974 switch (attr->group) {
975 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
976 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
977 return -ENXIO;
978 }
979
980 return -ENXIO;
981}
982
983static int vgic_v3_get_attr(struct kvm_device *dev,
984 struct kvm_device_attr *attr)
985{
986 int ret;
987
988 ret = vgic_get_common_attr(dev, attr);
989 if (ret != -ENXIO)
990 return ret;
991
992 switch (attr->group) {
993 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
994 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
995 return -ENXIO;
996 }
997
998 return -ENXIO;
999}
1000
1001static int vgic_v3_has_attr(struct kvm_device *dev,
1002 struct kvm_device_attr *attr)
1003{
1004 switch (attr->group) {
1005 case KVM_DEV_ARM_VGIC_GRP_ADDR:
1006 switch (attr->attr) {
1007 case KVM_VGIC_V2_ADDR_TYPE_DIST:
1008 case KVM_VGIC_V2_ADDR_TYPE_CPU:
1009 return -ENXIO;
1010 case KVM_VGIC_V3_ADDR_TYPE_DIST:
1011 case KVM_VGIC_V3_ADDR_TYPE_REDIST:
1012 return 0;
1013 }
1014 break;
1015 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
1016 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
1017 return -ENXIO;
1018 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
1019 return 0;
1020 case KVM_DEV_ARM_VGIC_GRP_CTRL:
1021 switch (attr->attr) {
1022 case KVM_DEV_ARM_VGIC_CTRL_INIT:
1023 return 0;
1024 }
1025 }
1026 return -ENXIO;
1027}
1028
1029struct kvm_device_ops kvm_arm_vgic_v3_ops = {
1030 .name = "kvm-arm-vgic-v3",
1031 .create = vgic_v3_create,
1032 .destroy = vgic_v3_destroy,
1033 .set_attr = vgic_v3_set_attr,
1034 .get_attr = vgic_v3_get_attr,
1035 .has_attr = vgic_v3_has_attr,
1036};
diff --git a/virt/kvm/arm/vgic-v3.c b/virt/kvm/arm/vgic-v3.c
index 1c2c8eef0599..dff06021e748 100644
--- a/virt/kvm/arm/vgic-v3.c
+++ b/virt/kvm/arm/vgic-v3.c
@@ -34,6 +34,7 @@
34#define GICH_LR_VIRTUALID (0x3ffUL << 0) 34#define GICH_LR_VIRTUALID (0x3ffUL << 0)
35#define GICH_LR_PHYSID_CPUID_SHIFT (10) 35#define GICH_LR_PHYSID_CPUID_SHIFT (10)
36#define GICH_LR_PHYSID_CPUID (7UL << GICH_LR_PHYSID_CPUID_SHIFT) 36#define GICH_LR_PHYSID_CPUID (7UL << GICH_LR_PHYSID_CPUID_SHIFT)
37#define ICH_LR_VIRTUALID_MASK (BIT_ULL(32) - 1)
37 38
38/* 39/*
39 * LRs are stored in reverse order in memory. make sure we index them 40 * LRs are stored in reverse order in memory. make sure we index them
@@ -48,12 +49,17 @@ static struct vgic_lr vgic_v3_get_lr(const struct kvm_vcpu *vcpu, int lr)
48 struct vgic_lr lr_desc; 49 struct vgic_lr lr_desc;
49 u64 val = vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[LR_INDEX(lr)]; 50 u64 val = vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[LR_INDEX(lr)];
50 51
51 lr_desc.irq = val & GICH_LR_VIRTUALID; 52 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
52 if (lr_desc.irq <= 15) 53 lr_desc.irq = val & ICH_LR_VIRTUALID_MASK;
53 lr_desc.source = (val >> GICH_LR_PHYSID_CPUID_SHIFT) & 0x7;
54 else 54 else
55 lr_desc.source = 0; 55 lr_desc.irq = val & GICH_LR_VIRTUALID;
56 lr_desc.state = 0; 56
57 lr_desc.source = 0;
58 if (lr_desc.irq <= 15 &&
59 vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
60 lr_desc.source = (val >> GICH_LR_PHYSID_CPUID_SHIFT) & 0x7;
61
62 lr_desc.state = 0;
57 63
58 if (val & ICH_LR_PENDING_BIT) 64 if (val & ICH_LR_PENDING_BIT)
59 lr_desc.state |= LR_STATE_PENDING; 65 lr_desc.state |= LR_STATE_PENDING;
@@ -68,8 +74,20 @@ static struct vgic_lr vgic_v3_get_lr(const struct kvm_vcpu *vcpu, int lr)
68static void vgic_v3_set_lr(struct kvm_vcpu *vcpu, int lr, 74static void vgic_v3_set_lr(struct kvm_vcpu *vcpu, int lr,
69 struct vgic_lr lr_desc) 75 struct vgic_lr lr_desc)
70{ 76{
71 u64 lr_val = (((u32)lr_desc.source << GICH_LR_PHYSID_CPUID_SHIFT) | 77 u64 lr_val;
72 lr_desc.irq); 78
79 lr_val = lr_desc.irq;
80
81 /*
82 * Currently all guest IRQs are Group1, as Group0 would result
83 * in a FIQ in the guest, which it wouldn't expect.
84 * Eventually we want to make this configurable, so we may revisit
85 * this in the future.
86 */
87 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
88 lr_val |= ICH_LR_GROUP;
89 else
90 lr_val |= (u32)lr_desc.source << GICH_LR_PHYSID_CPUID_SHIFT;
73 91
74 if (lr_desc.state & LR_STATE_PENDING) 92 if (lr_desc.state & LR_STATE_PENDING)
75 lr_val |= ICH_LR_PENDING_BIT; 93 lr_val |= ICH_LR_PENDING_BIT;
@@ -86,6 +104,8 @@ static void vgic_v3_sync_lr_elrsr(struct kvm_vcpu *vcpu, int lr,
86{ 104{
87 if (!(lr_desc.state & LR_STATE_MASK)) 105 if (!(lr_desc.state & LR_STATE_MASK))
88 vcpu->arch.vgic_cpu.vgic_v3.vgic_elrsr |= (1U << lr); 106 vcpu->arch.vgic_cpu.vgic_v3.vgic_elrsr |= (1U << lr);
107 else
108 vcpu->arch.vgic_cpu.vgic_v3.vgic_elrsr &= ~(1U << lr);
89} 109}
90 110
91static u64 vgic_v3_get_elrsr(const struct kvm_vcpu *vcpu) 111static u64 vgic_v3_get_elrsr(const struct kvm_vcpu *vcpu)
@@ -98,6 +118,11 @@ static u64 vgic_v3_get_eisr(const struct kvm_vcpu *vcpu)
98 return vcpu->arch.vgic_cpu.vgic_v3.vgic_eisr; 118 return vcpu->arch.vgic_cpu.vgic_v3.vgic_eisr;
99} 119}
100 120
121static void vgic_v3_clear_eisr(struct kvm_vcpu *vcpu)
122{
123 vcpu->arch.vgic_cpu.vgic_v3.vgic_eisr = 0;
124}
125
101static u32 vgic_v3_get_interrupt_status(const struct kvm_vcpu *vcpu) 126static u32 vgic_v3_get_interrupt_status(const struct kvm_vcpu *vcpu)
102{ 127{
103 u32 misr = vcpu->arch.vgic_cpu.vgic_v3.vgic_misr; 128 u32 misr = vcpu->arch.vgic_cpu.vgic_v3.vgic_misr;
@@ -145,15 +170,27 @@ static void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
145 170
146static void vgic_v3_enable(struct kvm_vcpu *vcpu) 171static void vgic_v3_enable(struct kvm_vcpu *vcpu)
147{ 172{
173 struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;
174
148 /* 175 /*
149 * By forcing VMCR to zero, the GIC will restore the binary 176 * By forcing VMCR to zero, the GIC will restore the binary
150 * points to their reset values. Anything else resets to zero 177 * points to their reset values. Anything else resets to zero
151 * anyway. 178 * anyway.
152 */ 179 */
153 vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr = 0; 180 vgic_v3->vgic_vmcr = 0;
181
182 /*
183 * If we are emulating a GICv3, we do it in an non-GICv2-compatible
184 * way, so we force SRE to 1 to demonstrate this to the guest.
185 * This goes with the spec allowing the value to be RAO/WI.
186 */
187 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
188 vgic_v3->vgic_sre = ICC_SRE_EL1_SRE;
189 else
190 vgic_v3->vgic_sre = 0;
154 191
155 /* Get the show on the road... */ 192 /* Get the show on the road... */
156 vcpu->arch.vgic_cpu.vgic_v3.vgic_hcr = ICH_HCR_EN; 193 vgic_v3->vgic_hcr = ICH_HCR_EN;
157} 194}
158 195
159static const struct vgic_ops vgic_v3_ops = { 196static const struct vgic_ops vgic_v3_ops = {
@@ -162,6 +199,7 @@ static const struct vgic_ops vgic_v3_ops = {
162 .sync_lr_elrsr = vgic_v3_sync_lr_elrsr, 199 .sync_lr_elrsr = vgic_v3_sync_lr_elrsr,
163 .get_elrsr = vgic_v3_get_elrsr, 200 .get_elrsr = vgic_v3_get_elrsr,
164 .get_eisr = vgic_v3_get_eisr, 201 .get_eisr = vgic_v3_get_eisr,
202 .clear_eisr = vgic_v3_clear_eisr,
165 .get_interrupt_status = vgic_v3_get_interrupt_status, 203 .get_interrupt_status = vgic_v3_get_interrupt_status,
166 .enable_underflow = vgic_v3_enable_underflow, 204 .enable_underflow = vgic_v3_enable_underflow,
167 .disable_underflow = vgic_v3_disable_underflow, 205 .disable_underflow = vgic_v3_disable_underflow,
@@ -205,35 +243,37 @@ int vgic_v3_probe(struct device_node *vgic_node,
205 * maximum of 16 list registers. Just ignore bit 4... 243 * maximum of 16 list registers. Just ignore bit 4...
206 */ 244 */
207 vgic->nr_lr = (ich_vtr_el2 & 0xf) + 1; 245 vgic->nr_lr = (ich_vtr_el2 & 0xf) + 1;
246 vgic->can_emulate_gicv2 = false;
208 247
209 if (of_property_read_u32(vgic_node, "#redistributor-regions", &gicv_idx)) 248 if (of_property_read_u32(vgic_node, "#redistributor-regions", &gicv_idx))
210 gicv_idx = 1; 249 gicv_idx = 1;
211 250
212 gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 251 gicv_idx += 3; /* Also skip GICD, GICC, GICH */
213 if (of_address_to_resource(vgic_node, gicv_idx, &vcpu_res)) { 252 if (of_address_to_resource(vgic_node, gicv_idx, &vcpu_res)) {
214 kvm_err("Cannot obtain GICV region\n"); 253 kvm_info("GICv3: no GICV resource entry\n");
215 ret = -ENXIO; 254 vgic->vcpu_base = 0;
216 goto out; 255 } else if (!PAGE_ALIGNED(vcpu_res.start)) {
217 } 256 pr_warn("GICV physical address 0x%llx not page aligned\n",
218
219 if (!PAGE_ALIGNED(vcpu_res.start)) {
220 kvm_err("GICV physical address 0x%llx not page aligned\n",
221 (unsigned long long)vcpu_res.start); 257 (unsigned long long)vcpu_res.start);
222 ret = -ENXIO; 258 vgic->vcpu_base = 0;
223 goto out; 259 } else if (!PAGE_ALIGNED(resource_size(&vcpu_res))) {
224 } 260 pr_warn("GICV size 0x%llx not a multiple of page size 0x%lx\n",
225
226 if (!PAGE_ALIGNED(resource_size(&vcpu_res))) {
227 kvm_err("GICV size 0x%llx not a multiple of page size 0x%lx\n",
228 (unsigned long long)resource_size(&vcpu_res), 261 (unsigned long long)resource_size(&vcpu_res),
229 PAGE_SIZE); 262 PAGE_SIZE);
230 ret = -ENXIO; 263 vgic->vcpu_base = 0;
231 goto out; 264 } else {
265 vgic->vcpu_base = vcpu_res.start;
266 vgic->can_emulate_gicv2 = true;
267 kvm_register_device_ops(&kvm_arm_vgic_v2_ops,
268 KVM_DEV_TYPE_ARM_VGIC_V2);
232 } 269 }
270 if (vgic->vcpu_base == 0)
271 kvm_info("disabling GICv2 emulation\n");
272 kvm_register_device_ops(&kvm_arm_vgic_v3_ops, KVM_DEV_TYPE_ARM_VGIC_V3);
233 273
234 vgic->vcpu_base = vcpu_res.start;
235 vgic->vctrl_base = NULL; 274 vgic->vctrl_base = NULL;
236 vgic->type = VGIC_V3; 275 vgic->type = VGIC_V3;
276 vgic->max_gic_vcpus = KVM_MAX_VCPUS;
237 277
238 kvm_info("%s@%llx IRQ%d\n", vgic_node->name, 278 kvm_info("%s@%llx IRQ%d\n", vgic_node->name,
239 vcpu_res.start, vgic->maint_irq); 279 vcpu_res.start, vgic->maint_irq);
diff --git a/virt/kvm/arm/vgic.c b/virt/kvm/arm/vgic.c
index 03affc7bf453..c9f60f524588 100644
--- a/virt/kvm/arm/vgic.c
+++ b/virt/kvm/arm/vgic.c
@@ -75,37 +75,31 @@
75 * inactive as long as the external input line is held high. 75 * inactive as long as the external input line is held high.
76 */ 76 */
77 77
78#define VGIC_ADDR_UNDEF (-1) 78#include "vgic.h"
79#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF) 79
80
81#define PRODUCT_ID_KVM 0x4b /* ASCII code K */
82#define IMPLEMENTER_ARM 0x43b
83#define GICC_ARCH_VERSION_V2 0x2
84
85#define ACCESS_READ_VALUE (1 << 0)
86#define ACCESS_READ_RAZ (0 << 0)
87#define ACCESS_READ_MASK(x) ((x) & (1 << 0))
88#define ACCESS_WRITE_IGNORED (0 << 1)
89#define ACCESS_WRITE_SETBIT (1 << 1)
90#define ACCESS_WRITE_CLEARBIT (2 << 1)
91#define ACCESS_WRITE_VALUE (3 << 1)
92#define ACCESS_WRITE_MASK(x) ((x) & (3 << 1))
93
94static int vgic_init(struct kvm *kvm);
95static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu); 80static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu);
96static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu); 81static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu);
97static void vgic_update_state(struct kvm *kvm);
98static void vgic_kick_vcpus(struct kvm *kvm);
99static u8 *vgic_get_sgi_sources(struct vgic_dist *dist, int vcpu_id, int sgi);
100static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg);
101static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr); 82static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr);
102static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr, struct vgic_lr lr_desc); 83static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr, struct vgic_lr lr_desc);
103static void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
104static void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
105 84
106static const struct vgic_ops *vgic_ops; 85static const struct vgic_ops *vgic_ops;
107static const struct vgic_params *vgic; 86static const struct vgic_params *vgic;
108 87
88static void add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
89{
90 vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);
91}
92
93static bool queue_sgi(struct kvm_vcpu *vcpu, int irq)
94{
95 return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);
96}
97
98int kvm_vgic_map_resources(struct kvm *kvm)
99{
100 return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);
101}
102
109/* 103/*
110 * struct vgic_bitmap contains a bitmap made of unsigned longs, but 104 * struct vgic_bitmap contains a bitmap made of unsigned longs, but
111 * extracts u32s out of them. 105 * extracts u32s out of them.
@@ -160,8 +154,7 @@ static unsigned long *u64_to_bitmask(u64 *val)
160 return (unsigned long *)val; 154 return (unsigned long *)val;
161} 155}
162 156
163static u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, 157u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset)
164 int cpuid, u32 offset)
165{ 158{
166 offset >>= 2; 159 offset >>= 2;
167 if (!offset) 160 if (!offset)
@@ -179,8 +172,8 @@ static int vgic_bitmap_get_irq_val(struct vgic_bitmap *x,
179 return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared); 172 return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared);
180} 173}
181 174
182static void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid, 175void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
183 int irq, int val) 176 int irq, int val)
184{ 177{
185 unsigned long *reg; 178 unsigned long *reg;
186 179
@@ -202,7 +195,7 @@ static unsigned long *vgic_bitmap_get_cpu_map(struct vgic_bitmap *x, int cpuid)
202 return x->private + cpuid; 195 return x->private + cpuid;
203} 196}
204 197
205static unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x) 198unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
206{ 199{
207 return x->shared; 200 return x->shared;
208} 201}
@@ -229,7 +222,7 @@ static void vgic_free_bytemap(struct vgic_bytemap *b)
229 b->shared = NULL; 222 b->shared = NULL;
230} 223}
231 224
232static u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset) 225u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
233{ 226{
234 u32 *reg; 227 u32 *reg;
235 228
@@ -326,14 +319,14 @@ static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)
326 return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq); 319 return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq);
327} 320}
328 321
329static void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq) 322void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)
330{ 323{
331 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 324 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
332 325
333 vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1); 326 vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1);
334} 327}
335 328
336static void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq) 329void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)
337{ 330{
338 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 331 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
339 332
@@ -349,7 +342,7 @@ static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)
349 vcpu->arch.vgic_cpu.pending_shared); 342 vcpu->arch.vgic_cpu.pending_shared);
350} 343}
351 344
352static void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq) 345void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
353{ 346{
354 if (irq < VGIC_NR_PRIVATE_IRQS) 347 if (irq < VGIC_NR_PRIVATE_IRQS)
355 clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu); 348 clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
@@ -363,16 +356,6 @@ static bool vgic_can_sample_irq(struct kvm_vcpu *vcpu, int irq)
363 return vgic_irq_is_edge(vcpu, irq) || !vgic_irq_is_queued(vcpu, irq); 356 return vgic_irq_is_edge(vcpu, irq) || !vgic_irq_is_queued(vcpu, irq);
364} 357}
365 358
366static u32 mmio_data_read(struct kvm_exit_mmio *mmio, u32 mask)
367{
368 return le32_to_cpu(*((u32 *)mmio->data)) & mask;
369}
370
371static void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
372{
373 *((u32 *)mmio->data) = cpu_to_le32(value) & mask;
374}
375
376/** 359/**
377 * vgic_reg_access - access vgic register 360 * vgic_reg_access - access vgic register
378 * @mmio: pointer to the data describing the mmio access 361 * @mmio: pointer to the data describing the mmio access
@@ -384,8 +367,8 @@ static void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
384 * modes defined for vgic register access 367 * modes defined for vgic register access
385 * (read,raz,write-ignored,setbit,clearbit,write) 368 * (read,raz,write-ignored,setbit,clearbit,write)
386 */ 369 */
387static void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg, 370void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
388 phys_addr_t offset, int mode) 371 phys_addr_t offset, int mode)
389{ 372{
390 int word_offset = (offset & 3) * 8; 373 int word_offset = (offset & 3) * 8;
391 u32 mask = (1UL << (mmio->len * 8)) - 1; 374 u32 mask = (1UL << (mmio->len * 8)) - 1;
@@ -434,107 +417,58 @@ static void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
434 } 417 }
435} 418}
436 419
437static bool handle_mmio_misc(struct kvm_vcpu *vcpu, 420bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
438 struct kvm_exit_mmio *mmio, phys_addr_t offset) 421 phys_addr_t offset)
439{
440 u32 reg;
441 u32 word_offset = offset & 3;
442
443 switch (offset & ~3) {
444 case 0: /* GICD_CTLR */
445 reg = vcpu->kvm->arch.vgic.enabled;
446 vgic_reg_access(mmio, &reg, word_offset,
447 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
448 if (mmio->is_write) {
449 vcpu->kvm->arch.vgic.enabled = reg & 1;
450 vgic_update_state(vcpu->kvm);
451 return true;
452 }
453 break;
454
455 case 4: /* GICD_TYPER */
456 reg = (atomic_read(&vcpu->kvm->online_vcpus) - 1) << 5;
457 reg |= (vcpu->kvm->arch.vgic.nr_irqs >> 5) - 1;
458 vgic_reg_access(mmio, &reg, word_offset,
459 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
460 break;
461
462 case 8: /* GICD_IIDR */
463 reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
464 vgic_reg_access(mmio, &reg, word_offset,
465 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
466 break;
467 }
468
469 return false;
470}
471
472static bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu,
473 struct kvm_exit_mmio *mmio, phys_addr_t offset)
474{ 422{
475 vgic_reg_access(mmio, NULL, offset, 423 vgic_reg_access(mmio, NULL, offset,
476 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 424 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
477 return false; 425 return false;
478} 426}
479 427
480static bool handle_mmio_set_enable_reg(struct kvm_vcpu *vcpu, 428bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
481 struct kvm_exit_mmio *mmio, 429 phys_addr_t offset, int vcpu_id, int access)
482 phys_addr_t offset)
483{ 430{
484 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_enabled, 431 u32 *reg;
485 vcpu->vcpu_id, offset); 432 int mode = ACCESS_READ_VALUE | access;
486 vgic_reg_access(mmio, reg, offset, 433 struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);
487 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
488 if (mmio->is_write) {
489 vgic_update_state(vcpu->kvm);
490 return true;
491 }
492
493 return false;
494}
495 434
496static bool handle_mmio_clear_enable_reg(struct kvm_vcpu *vcpu, 435 reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);
497 struct kvm_exit_mmio *mmio, 436 vgic_reg_access(mmio, reg, offset, mode);
498 phys_addr_t offset)
499{
500 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_enabled,
501 vcpu->vcpu_id, offset);
502 vgic_reg_access(mmio, reg, offset,
503 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
504 if (mmio->is_write) { 437 if (mmio->is_write) {
505 if (offset < 4) /* Force SGI enabled */ 438 if (access & ACCESS_WRITE_CLEARBIT) {
506 *reg |= 0xffff; 439 if (offset < 4) /* Force SGI enabled */
507 vgic_retire_disabled_irqs(vcpu); 440 *reg |= 0xffff;
508 vgic_update_state(vcpu->kvm); 441 vgic_retire_disabled_irqs(target_vcpu);
442 }
443 vgic_update_state(kvm);
509 return true; 444 return true;
510 } 445 }
511 446
512 return false; 447 return false;
513} 448}
514 449
515static bool handle_mmio_set_pending_reg(struct kvm_vcpu *vcpu, 450bool vgic_handle_set_pending_reg(struct kvm *kvm,
516 struct kvm_exit_mmio *mmio, 451 struct kvm_exit_mmio *mmio,
517 phys_addr_t offset) 452 phys_addr_t offset, int vcpu_id)
518{ 453{
519 u32 *reg, orig; 454 u32 *reg, orig;
520 u32 level_mask; 455 u32 level_mask;
521 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 456 int mode = ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT;
457 struct vgic_dist *dist = &kvm->arch.vgic;
522 458
523 reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu->vcpu_id, offset); 459 reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu_id, offset);
524 level_mask = (~(*reg)); 460 level_mask = (~(*reg));
525 461
526 /* Mark both level and edge triggered irqs as pending */ 462 /* Mark both level and edge triggered irqs as pending */
527 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu->vcpu_id, offset); 463 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
528 orig = *reg; 464 orig = *reg;
529 vgic_reg_access(mmio, reg, offset, 465 vgic_reg_access(mmio, reg, offset, mode);
530 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
531 466
532 if (mmio->is_write) { 467 if (mmio->is_write) {
533 /* Set the soft-pending flag only for level-triggered irqs */ 468 /* Set the soft-pending flag only for level-triggered irqs */
534 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend, 469 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
535 vcpu->vcpu_id, offset); 470 vcpu_id, offset);
536 vgic_reg_access(mmio, reg, offset, 471 vgic_reg_access(mmio, reg, offset, mode);
537 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
538 *reg &= level_mask; 472 *reg &= level_mask;
539 473
540 /* Ignore writes to SGIs */ 474 /* Ignore writes to SGIs */
@@ -543,31 +477,30 @@ static bool handle_mmio_set_pending_reg(struct kvm_vcpu *vcpu,
543 *reg |= orig & 0xffff; 477 *reg |= orig & 0xffff;
544 } 478 }
545 479
546 vgic_update_state(vcpu->kvm); 480 vgic_update_state(kvm);
547 return true; 481 return true;
548 } 482 }
549 483
550 return false; 484 return false;
551} 485}
552 486
553static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu, 487bool vgic_handle_clear_pending_reg(struct kvm *kvm,
554 struct kvm_exit_mmio *mmio, 488 struct kvm_exit_mmio *mmio,
555 phys_addr_t offset) 489 phys_addr_t offset, int vcpu_id)
556{ 490{
557 u32 *level_active; 491 u32 *level_active;
558 u32 *reg, orig; 492 u32 *reg, orig;
559 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 493 int mode = ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT;
494 struct vgic_dist *dist = &kvm->arch.vgic;
560 495
561 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu->vcpu_id, offset); 496 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
562 orig = *reg; 497 orig = *reg;
563 vgic_reg_access(mmio, reg, offset, 498 vgic_reg_access(mmio, reg, offset, mode);
564 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
565 if (mmio->is_write) { 499 if (mmio->is_write) {
566 /* Re-set level triggered level-active interrupts */ 500 /* Re-set level triggered level-active interrupts */
567 level_active = vgic_bitmap_get_reg(&dist->irq_level, 501 level_active = vgic_bitmap_get_reg(&dist->irq_level,
568 vcpu->vcpu_id, offset); 502 vcpu_id, offset);
569 reg = vgic_bitmap_get_reg(&dist->irq_pending, 503 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
570 vcpu->vcpu_id, offset);
571 *reg |= *level_active; 504 *reg |= *level_active;
572 505
573 /* Ignore writes to SGIs */ 506 /* Ignore writes to SGIs */
@@ -578,101 +511,12 @@ static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu,
578 511
579 /* Clear soft-pending flags */ 512 /* Clear soft-pending flags */
580 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend, 513 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
581 vcpu->vcpu_id, offset); 514 vcpu_id, offset);
582 vgic_reg_access(mmio, reg, offset, 515 vgic_reg_access(mmio, reg, offset, mode);
583 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
584
585 vgic_update_state(vcpu->kvm);
586 return true;
587 }
588
589 return false;
590}
591
592static bool handle_mmio_priority_reg(struct kvm_vcpu *vcpu,
593 struct kvm_exit_mmio *mmio,
594 phys_addr_t offset)
595{
596 u32 *reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
597 vcpu->vcpu_id, offset);
598 vgic_reg_access(mmio, reg, offset,
599 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
600 return false;
601}
602
603#define GICD_ITARGETSR_SIZE 32
604#define GICD_CPUTARGETS_BITS 8
605#define GICD_IRQS_PER_ITARGETSR (GICD_ITARGETSR_SIZE / GICD_CPUTARGETS_BITS)
606static u32 vgic_get_target_reg(struct kvm *kvm, int irq)
607{
608 struct vgic_dist *dist = &kvm->arch.vgic;
609 int i;
610 u32 val = 0;
611
612 irq -= VGIC_NR_PRIVATE_IRQS;
613
614 for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++)
615 val |= 1 << (dist->irq_spi_cpu[irq + i] + i * 8);
616
617 return val;
618}
619
620static void vgic_set_target_reg(struct kvm *kvm, u32 val, int irq)
621{
622 struct vgic_dist *dist = &kvm->arch.vgic;
623 struct kvm_vcpu *vcpu;
624 int i, c;
625 unsigned long *bmap;
626 u32 target;
627
628 irq -= VGIC_NR_PRIVATE_IRQS;
629
630 /*
631 * Pick the LSB in each byte. This ensures we target exactly
632 * one vcpu per IRQ. If the byte is null, assume we target
633 * CPU0.
634 */
635 for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++) {
636 int shift = i * GICD_CPUTARGETS_BITS;
637 target = ffs((val >> shift) & 0xffU);
638 target = target ? (target - 1) : 0;
639 dist->irq_spi_cpu[irq + i] = target;
640 kvm_for_each_vcpu(c, vcpu, kvm) {
641 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[c]);
642 if (c == target)
643 set_bit(irq + i, bmap);
644 else
645 clear_bit(irq + i, bmap);
646 }
647 }
648}
649
650static bool handle_mmio_target_reg(struct kvm_vcpu *vcpu,
651 struct kvm_exit_mmio *mmio,
652 phys_addr_t offset)
653{
654 u32 reg;
655
656 /* We treat the banked interrupts targets as read-only */
657 if (offset < 32) {
658 u32 roreg = 1 << vcpu->vcpu_id;
659 roreg |= roreg << 8;
660 roreg |= roreg << 16;
661 516
662 vgic_reg_access(mmio, &roreg, offset, 517 vgic_update_state(kvm);
663 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
664 return false;
665 }
666
667 reg = vgic_get_target_reg(vcpu->kvm, offset & ~3U);
668 vgic_reg_access(mmio, &reg, offset,
669 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
670 if (mmio->is_write) {
671 vgic_set_target_reg(vcpu->kvm, reg, offset & ~3U);
672 vgic_update_state(vcpu->kvm);
673 return true; 518 return true;
674 } 519 }
675
676 return false; 520 return false;
677} 521}
678 522
@@ -711,14 +555,10 @@ static u16 vgic_cfg_compress(u32 val)
711 * LSB is always 0. As such, we only keep the upper bit, and use the 555 * LSB is always 0. As such, we only keep the upper bit, and use the
712 * two above functions to compress/expand the bits 556 * two above functions to compress/expand the bits
713 */ 557 */
714static bool handle_mmio_cfg_reg(struct kvm_vcpu *vcpu, 558bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
715 struct kvm_exit_mmio *mmio, phys_addr_t offset) 559 phys_addr_t offset)
716{ 560{
717 u32 val; 561 u32 val;
718 u32 *reg;
719
720 reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
721 vcpu->vcpu_id, offset >> 1);
722 562
723 if (offset & 4) 563 if (offset & 4)
724 val = *reg >> 16; 564 val = *reg >> 16;
@@ -747,21 +587,6 @@ static bool handle_mmio_cfg_reg(struct kvm_vcpu *vcpu,
747 return false; 587 return false;
748} 588}
749 589
750static bool handle_mmio_sgi_reg(struct kvm_vcpu *vcpu,
751 struct kvm_exit_mmio *mmio, phys_addr_t offset)
752{
753 u32 reg;
754 vgic_reg_access(mmio, &reg, offset,
755 ACCESS_READ_RAZ | ACCESS_WRITE_VALUE);
756 if (mmio->is_write) {
757 vgic_dispatch_sgi(vcpu, reg);
758 vgic_update_state(vcpu->kvm);
759 return true;
760 }
761
762 return false;
763}
764
765/** 590/**
766 * vgic_unqueue_irqs - move pending IRQs from LRs to the distributor 591 * vgic_unqueue_irqs - move pending IRQs from LRs to the distributor
767 * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs 592 * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs
@@ -774,11 +599,9 @@ static bool handle_mmio_sgi_reg(struct kvm_vcpu *vcpu,
774 * to the distributor but the active state stays in the LRs, because we don't 599 * to the distributor but the active state stays in the LRs, because we don't
775 * track the active state on the distributor side. 600 * track the active state on the distributor side.
776 */ 601 */
777static void vgic_unqueue_irqs(struct kvm_vcpu *vcpu) 602void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
778{ 603{
779 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
780 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 604 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
781 int vcpu_id = vcpu->vcpu_id;
782 int i; 605 int i;
783 606
784 for_each_set_bit(i, vgic_cpu->lr_used, vgic_cpu->nr_lr) { 607 for_each_set_bit(i, vgic_cpu->lr_used, vgic_cpu->nr_lr) {
@@ -805,7 +628,7 @@ static void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
805 */ 628 */
806 vgic_dist_irq_set_pending(vcpu, lr.irq); 629 vgic_dist_irq_set_pending(vcpu, lr.irq);
807 if (lr.irq < VGIC_NR_SGIS) 630 if (lr.irq < VGIC_NR_SGIS)
808 *vgic_get_sgi_sources(dist, vcpu_id, lr.irq) |= 1 << lr.source; 631 add_sgi_source(vcpu, lr.irq, lr.source);
809 lr.state &= ~LR_STATE_PENDING; 632 lr.state &= ~LR_STATE_PENDING;
810 vgic_set_lr(vcpu, i, lr); 633 vgic_set_lr(vcpu, i, lr);
811 634
@@ -824,188 +647,12 @@ static void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
824 } 647 }
825} 648}
826 649
827/* Handle reads of GICD_CPENDSGIRn and GICD_SPENDSGIRn */ 650const
828static bool read_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu, 651struct kvm_mmio_range *vgic_find_range(const struct kvm_mmio_range *ranges,
829 struct kvm_exit_mmio *mmio,
830 phys_addr_t offset)
831{
832 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
833 int sgi;
834 int min_sgi = (offset & ~0x3);
835 int max_sgi = min_sgi + 3;
836 int vcpu_id = vcpu->vcpu_id;
837 u32 reg = 0;
838
839 /* Copy source SGIs from distributor side */
840 for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
841 int shift = 8 * (sgi - min_sgi);
842 reg |= ((u32)*vgic_get_sgi_sources(dist, vcpu_id, sgi)) << shift;
843 }
844
845 mmio_data_write(mmio, ~0, reg);
846 return false;
847}
848
849static bool write_set_clear_sgi_pend_reg(struct kvm_vcpu *vcpu,
850 struct kvm_exit_mmio *mmio,
851 phys_addr_t offset, bool set)
852{
853 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
854 int sgi;
855 int min_sgi = (offset & ~0x3);
856 int max_sgi = min_sgi + 3;
857 int vcpu_id = vcpu->vcpu_id;
858 u32 reg;
859 bool updated = false;
860
861 reg = mmio_data_read(mmio, ~0);
862
863 /* Clear pending SGIs on the distributor */
864 for (sgi = min_sgi; sgi <= max_sgi; sgi++) {
865 u8 mask = reg >> (8 * (sgi - min_sgi));
866 u8 *src = vgic_get_sgi_sources(dist, vcpu_id, sgi);
867 if (set) {
868 if ((*src & mask) != mask)
869 updated = true;
870 *src |= mask;
871 } else {
872 if (*src & mask)
873 updated = true;
874 *src &= ~mask;
875 }
876 }
877
878 if (updated)
879 vgic_update_state(vcpu->kvm);
880
881 return updated;
882}
883
884static bool handle_mmio_sgi_set(struct kvm_vcpu *vcpu,
885 struct kvm_exit_mmio *mmio,
886 phys_addr_t offset)
887{
888 if (!mmio->is_write)
889 return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
890 else
891 return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, true);
892}
893
894static bool handle_mmio_sgi_clear(struct kvm_vcpu *vcpu,
895 struct kvm_exit_mmio *mmio,
896 phys_addr_t offset)
897{
898 if (!mmio->is_write)
899 return read_set_clear_sgi_pend_reg(vcpu, mmio, offset);
900 else
901 return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, false);
902}
903
904/*
905 * I would have liked to use the kvm_bus_io_*() API instead, but it
906 * cannot cope with banked registers (only the VM pointer is passed
907 * around, and we need the vcpu). One of these days, someone please
908 * fix it!
909 */
910struct mmio_range {
911 phys_addr_t base;
912 unsigned long len;
913 int bits_per_irq;
914 bool (*handle_mmio)(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
915 phys_addr_t offset);
916};
917
918static const struct mmio_range vgic_dist_ranges[] = {
919 {
920 .base = GIC_DIST_CTRL,
921 .len = 12,
922 .bits_per_irq = 0,
923 .handle_mmio = handle_mmio_misc,
924 },
925 {
926 .base = GIC_DIST_IGROUP,
927 .len = VGIC_MAX_IRQS / 8,
928 .bits_per_irq = 1,
929 .handle_mmio = handle_mmio_raz_wi,
930 },
931 {
932 .base = GIC_DIST_ENABLE_SET,
933 .len = VGIC_MAX_IRQS / 8,
934 .bits_per_irq = 1,
935 .handle_mmio = handle_mmio_set_enable_reg,
936 },
937 {
938 .base = GIC_DIST_ENABLE_CLEAR,
939 .len = VGIC_MAX_IRQS / 8,
940 .bits_per_irq = 1,
941 .handle_mmio = handle_mmio_clear_enable_reg,
942 },
943 {
944 .base = GIC_DIST_PENDING_SET,
945 .len = VGIC_MAX_IRQS / 8,
946 .bits_per_irq = 1,
947 .handle_mmio = handle_mmio_set_pending_reg,
948 },
949 {
950 .base = GIC_DIST_PENDING_CLEAR,
951 .len = VGIC_MAX_IRQS / 8,
952 .bits_per_irq = 1,
953 .handle_mmio = handle_mmio_clear_pending_reg,
954 },
955 {
956 .base = GIC_DIST_ACTIVE_SET,
957 .len = VGIC_MAX_IRQS / 8,
958 .bits_per_irq = 1,
959 .handle_mmio = handle_mmio_raz_wi,
960 },
961 {
962 .base = GIC_DIST_ACTIVE_CLEAR,
963 .len = VGIC_MAX_IRQS / 8,
964 .bits_per_irq = 1,
965 .handle_mmio = handle_mmio_raz_wi,
966 },
967 {
968 .base = GIC_DIST_PRI,
969 .len = VGIC_MAX_IRQS,
970 .bits_per_irq = 8,
971 .handle_mmio = handle_mmio_priority_reg,
972 },
973 {
974 .base = GIC_DIST_TARGET,
975 .len = VGIC_MAX_IRQS,
976 .bits_per_irq = 8,
977 .handle_mmio = handle_mmio_target_reg,
978 },
979 {
980 .base = GIC_DIST_CONFIG,
981 .len = VGIC_MAX_IRQS / 4,
982 .bits_per_irq = 2,
983 .handle_mmio = handle_mmio_cfg_reg,
984 },
985 {
986 .base = GIC_DIST_SOFTINT,
987 .len = 4,
988 .handle_mmio = handle_mmio_sgi_reg,
989 },
990 {
991 .base = GIC_DIST_SGI_PENDING_CLEAR,
992 .len = VGIC_NR_SGIS,
993 .handle_mmio = handle_mmio_sgi_clear,
994 },
995 {
996 .base = GIC_DIST_SGI_PENDING_SET,
997 .len = VGIC_NR_SGIS,
998 .handle_mmio = handle_mmio_sgi_set,
999 },
1000 {}
1001};
1002
1003static const
1004struct mmio_range *find_matching_range(const struct mmio_range *ranges,
1005 struct kvm_exit_mmio *mmio, 652 struct kvm_exit_mmio *mmio,
1006 phys_addr_t offset) 653 phys_addr_t offset)
1007{ 654{
1008 const struct mmio_range *r = ranges; 655 const struct kvm_mmio_range *r = ranges;
1009 656
1010 while (r->len) { 657 while (r->len) {
1011 if (offset >= r->base && 658 if (offset >= r->base &&
@@ -1018,7 +665,7 @@ struct mmio_range *find_matching_range(const struct mmio_range *ranges,
1018} 665}
1019 666
1020static bool vgic_validate_access(const struct vgic_dist *dist, 667static bool vgic_validate_access(const struct vgic_dist *dist,
1021 const struct mmio_range *range, 668 const struct kvm_mmio_range *range,
1022 unsigned long offset) 669 unsigned long offset)
1023{ 670{
1024 int irq; 671 int irq;
@@ -1033,37 +680,76 @@ static bool vgic_validate_access(const struct vgic_dist *dist,
1033 return true; 680 return true;
1034} 681}
1035 682
683/*
684 * Call the respective handler function for the given range.
685 * We split up any 64 bit accesses into two consecutive 32 bit
686 * handler calls and merge the result afterwards.
687 * We do this in a little endian fashion regardless of the host's
688 * or guest's endianness, because the GIC is always LE and the rest of
689 * the code (vgic_reg_access) also puts it in a LE fashion already.
690 * At this point we have already identified the handle function, so
691 * range points to that one entry and offset is relative to this.
692 */
693static bool call_range_handler(struct kvm_vcpu *vcpu,
694 struct kvm_exit_mmio *mmio,
695 unsigned long offset,
696 const struct kvm_mmio_range *range)
697{
698 u32 *data32 = (void *)mmio->data;
699 struct kvm_exit_mmio mmio32;
700 bool ret;
701
702 if (likely(mmio->len <= 4))
703 return range->handle_mmio(vcpu, mmio, offset);
704
705 /*
706 * Any access bigger than 4 bytes (that we currently handle in KVM)
707 * is actually 8 bytes long, caused by a 64-bit access
708 */
709
710 mmio32.len = 4;
711 mmio32.is_write = mmio->is_write;
712 mmio32.private = mmio->private;
713
714 mmio32.phys_addr = mmio->phys_addr + 4;
715 if (mmio->is_write)
716 *(u32 *)mmio32.data = data32[1];
717 ret = range->handle_mmio(vcpu, &mmio32, offset + 4);
718 if (!mmio->is_write)
719 data32[1] = *(u32 *)mmio32.data;
720
721 mmio32.phys_addr = mmio->phys_addr;
722 if (mmio->is_write)
723 *(u32 *)mmio32.data = data32[0];
724 ret |= range->handle_mmio(vcpu, &mmio32, offset);
725 if (!mmio->is_write)
726 data32[0] = *(u32 *)mmio32.data;
727
728 return ret;
729}
730
1036/** 731/**
1037 * vgic_handle_mmio - handle an in-kernel MMIO access 732 * vgic_handle_mmio_range - handle an in-kernel MMIO access
1038 * @vcpu: pointer to the vcpu performing the access 733 * @vcpu: pointer to the vcpu performing the access
1039 * @run: pointer to the kvm_run structure 734 * @run: pointer to the kvm_run structure
1040 * @mmio: pointer to the data describing the access 735 * @mmio: pointer to the data describing the access
736 * @ranges: array of MMIO ranges in a given region
737 * @mmio_base: base address of that region
1041 * 738 *
1042 * returns true if the MMIO access has been performed in kernel space, 739 * returns true if the MMIO access could be performed
1043 * and false if it needs to be emulated in user space.
1044 */ 740 */
1045bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run, 741bool vgic_handle_mmio_range(struct kvm_vcpu *vcpu, struct kvm_run *run,
1046 struct kvm_exit_mmio *mmio) 742 struct kvm_exit_mmio *mmio,
743 const struct kvm_mmio_range *ranges,
744 unsigned long mmio_base)
1047{ 745{
1048 const struct mmio_range *range; 746 const struct kvm_mmio_range *range;
1049 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 747 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1050 unsigned long base = dist->vgic_dist_base;
1051 bool updated_state; 748 bool updated_state;
1052 unsigned long offset; 749 unsigned long offset;
1053 750
1054 if (!irqchip_in_kernel(vcpu->kvm) || 751 offset = mmio->phys_addr - mmio_base;
1055 mmio->phys_addr < base || 752 range = vgic_find_range(ranges, mmio, offset);
1056 (mmio->phys_addr + mmio->len) > (base + KVM_VGIC_V2_DIST_SIZE))
1057 return false;
1058
1059 /* We don't support ldrd / strd or ldm / stm to the emulated vgic */
1060 if (mmio->len > 4) {
1061 kvm_inject_dabt(vcpu, mmio->phys_addr);
1062 return true;
1063 }
1064
1065 offset = mmio->phys_addr - base;
1066 range = find_matching_range(vgic_dist_ranges, mmio, offset);
1067 if (unlikely(!range || !range->handle_mmio)) { 753 if (unlikely(!range || !range->handle_mmio)) {
1068 pr_warn("Unhandled access %d %08llx %d\n", 754 pr_warn("Unhandled access %d %08llx %d\n",
1069 mmio->is_write, mmio->phys_addr, mmio->len); 755 mmio->is_write, mmio->phys_addr, mmio->len);
@@ -1071,12 +757,12 @@ bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
1071 } 757 }
1072 758
1073 spin_lock(&vcpu->kvm->arch.vgic.lock); 759 spin_lock(&vcpu->kvm->arch.vgic.lock);
1074 offset = mmio->phys_addr - range->base - base; 760 offset -= range->base;
1075 if (vgic_validate_access(dist, range, offset)) { 761 if (vgic_validate_access(dist, range, offset)) {
1076 updated_state = range->handle_mmio(vcpu, mmio, offset); 762 updated_state = call_range_handler(vcpu, mmio, offset, range);
1077 } else { 763 } else {
1078 vgic_reg_access(mmio, NULL, offset, 764 if (!mmio->is_write)
1079 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 765 memset(mmio->data, 0, mmio->len);
1080 updated_state = false; 766 updated_state = false;
1081 } 767 }
1082 spin_unlock(&vcpu->kvm->arch.vgic.lock); 768 spin_unlock(&vcpu->kvm->arch.vgic.lock);
@@ -1089,50 +775,28 @@ bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
1089 return true; 775 return true;
1090} 776}
1091 777
1092static u8 *vgic_get_sgi_sources(struct vgic_dist *dist, int vcpu_id, int sgi) 778/**
1093{ 779 * vgic_handle_mmio - handle an in-kernel MMIO access for the GIC emulation
1094 return dist->irq_sgi_sources + vcpu_id * VGIC_NR_SGIS + sgi; 780 * @vcpu: pointer to the vcpu performing the access
1095} 781 * @run: pointer to the kvm_run structure
1096 782 * @mmio: pointer to the data describing the access
1097static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg) 783 *
784 * returns true if the MMIO access has been performed in kernel space,
785 * and false if it needs to be emulated in user space.
786 * Calls the actual handling routine for the selected VGIC model.
787 */
788bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
789 struct kvm_exit_mmio *mmio)
1098{ 790{
1099 struct kvm *kvm = vcpu->kvm; 791 if (!irqchip_in_kernel(vcpu->kvm))
1100 struct vgic_dist *dist = &kvm->arch.vgic; 792 return false;
1101 int nrcpus = atomic_read(&kvm->online_vcpus);
1102 u8 target_cpus;
1103 int sgi, mode, c, vcpu_id;
1104
1105 vcpu_id = vcpu->vcpu_id;
1106
1107 sgi = reg & 0xf;
1108 target_cpus = (reg >> 16) & 0xff;
1109 mode = (reg >> 24) & 3;
1110
1111 switch (mode) {
1112 case 0:
1113 if (!target_cpus)
1114 return;
1115 break;
1116
1117 case 1:
1118 target_cpus = ((1 << nrcpus) - 1) & ~(1 << vcpu_id) & 0xff;
1119 break;
1120
1121 case 2:
1122 target_cpus = 1 << vcpu_id;
1123 break;
1124 }
1125
1126 kvm_for_each_vcpu(c, vcpu, kvm) {
1127 if (target_cpus & 1) {
1128 /* Flag the SGI as pending */
1129 vgic_dist_irq_set_pending(vcpu, sgi);
1130 *vgic_get_sgi_sources(dist, c, sgi) |= 1 << vcpu_id;
1131 kvm_debug("SGI%d from CPU%d to CPU%d\n", sgi, vcpu_id, c);
1132 }
1133 793
1134 target_cpus >>= 1; 794 /*
1135 } 795 * This will currently call either vgic_v2_handle_mmio() or
796 * vgic_v3_handle_mmio(), which in turn will call
797 * vgic_handle_mmio_range() defined above.
798 */
799 return vcpu->kvm->arch.vgic.vm_ops.handle_mmio(vcpu, run, mmio);
1136} 800}
1137 801
1138static int vgic_nr_shared_irqs(struct vgic_dist *dist) 802static int vgic_nr_shared_irqs(struct vgic_dist *dist)
@@ -1173,7 +837,7 @@ static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
1173 * Update the interrupt state and determine which CPUs have pending 837 * Update the interrupt state and determine which CPUs have pending
1174 * interrupts. Must be called with distributor lock held. 838 * interrupts. Must be called with distributor lock held.
1175 */ 839 */
1176static void vgic_update_state(struct kvm *kvm) 840void vgic_update_state(struct kvm *kvm)
1177{ 841{
1178 struct vgic_dist *dist = &kvm->arch.vgic; 842 struct vgic_dist *dist = &kvm->arch.vgic;
1179 struct kvm_vcpu *vcpu; 843 struct kvm_vcpu *vcpu;
@@ -1219,6 +883,11 @@ static inline u64 vgic_get_eisr(struct kvm_vcpu *vcpu)
1219 return vgic_ops->get_eisr(vcpu); 883 return vgic_ops->get_eisr(vcpu);
1220} 884}
1221 885
886static inline void vgic_clear_eisr(struct kvm_vcpu *vcpu)
887{
888 vgic_ops->clear_eisr(vcpu);
889}
890
1222static inline u32 vgic_get_interrupt_status(struct kvm_vcpu *vcpu) 891static inline u32 vgic_get_interrupt_status(struct kvm_vcpu *vcpu)
1223{ 892{
1224 return vgic_ops->get_interrupt_status(vcpu); 893 return vgic_ops->get_interrupt_status(vcpu);
@@ -1234,12 +903,12 @@ static inline void vgic_disable_underflow(struct kvm_vcpu *vcpu)
1234 vgic_ops->disable_underflow(vcpu); 903 vgic_ops->disable_underflow(vcpu);
1235} 904}
1236 905
1237static inline void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) 906void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1238{ 907{
1239 vgic_ops->get_vmcr(vcpu, vmcr); 908 vgic_ops->get_vmcr(vcpu, vmcr);
1240} 909}
1241 910
1242static void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) 911void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1243{ 912{
1244 vgic_ops->set_vmcr(vcpu, vmcr); 913 vgic_ops->set_vmcr(vcpu, vmcr);
1245} 914}
@@ -1258,6 +927,7 @@ static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu)
1258 vgic_set_lr(vcpu, lr_nr, vlr); 927 vgic_set_lr(vcpu, lr_nr, vlr);
1259 clear_bit(lr_nr, vgic_cpu->lr_used); 928 clear_bit(lr_nr, vgic_cpu->lr_used);
1260 vgic_cpu->vgic_irq_lr_map[irq] = LR_EMPTY; 929 vgic_cpu->vgic_irq_lr_map[irq] = LR_EMPTY;
930 vgic_sync_lr_elrsr(vcpu, lr_nr, vlr);
1261} 931}
1262 932
1263/* 933/*
@@ -1288,8 +958,9 @@ static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
1288/* 958/*
1289 * Queue an interrupt to a CPU virtual interface. Return true on success, 959 * Queue an interrupt to a CPU virtual interface. Return true on success,
1290 * or false if it wasn't possible to queue it. 960 * or false if it wasn't possible to queue it.
961 * sgi_source must be zero for any non-SGI interrupts.
1291 */ 962 */
1292static bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq) 963bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
1293{ 964{
1294 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 965 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1295 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 966 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
@@ -1313,6 +984,7 @@ static bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
1313 BUG_ON(!test_bit(lr, vgic_cpu->lr_used)); 984 BUG_ON(!test_bit(lr, vgic_cpu->lr_used));
1314 vlr.state |= LR_STATE_PENDING; 985 vlr.state |= LR_STATE_PENDING;
1315 vgic_set_lr(vcpu, lr, vlr); 986 vgic_set_lr(vcpu, lr, vlr);
987 vgic_sync_lr_elrsr(vcpu, lr, vlr);
1316 return true; 988 return true;
1317 } 989 }
1318 } 990 }
@@ -1334,41 +1006,11 @@ static bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
1334 vlr.state |= LR_EOI_INT; 1006 vlr.state |= LR_EOI_INT;
1335 1007
1336 vgic_set_lr(vcpu, lr, vlr); 1008 vgic_set_lr(vcpu, lr, vlr);
1009 vgic_sync_lr_elrsr(vcpu, lr, vlr);
1337 1010
1338 return true; 1011 return true;
1339} 1012}
1340 1013
1341static bool vgic_queue_sgi(struct kvm_vcpu *vcpu, int irq)
1342{
1343 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1344 unsigned long sources;
1345 int vcpu_id = vcpu->vcpu_id;
1346 int c;
1347
1348 sources = *vgic_get_sgi_sources(dist, vcpu_id, irq);
1349
1350 for_each_set_bit(c, &sources, dist->nr_cpus) {
1351 if (vgic_queue_irq(vcpu, c, irq))
1352 clear_bit(c, &sources);
1353 }
1354
1355 *vgic_get_sgi_sources(dist, vcpu_id, irq) = sources;
1356
1357 /*
1358 * If the sources bitmap has been cleared it means that we
1359 * could queue all the SGIs onto link registers (see the
1360 * clear_bit above), and therefore we are done with them in
1361 * our emulated gic and can get rid of them.
1362 */
1363 if (!sources) {
1364 vgic_dist_irq_clear_pending(vcpu, irq);
1365 vgic_cpu_irq_clear(vcpu, irq);
1366 return true;
1367 }
1368
1369 return false;
1370}
1371
1372static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq) 1014static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)
1373{ 1015{
1374 if (!vgic_can_sample_irq(vcpu, irq)) 1016 if (!vgic_can_sample_irq(vcpu, irq))
@@ -1413,7 +1055,7 @@ static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1413 1055
1414 /* SGIs */ 1056 /* SGIs */
1415 for_each_set_bit(i, vgic_cpu->pending_percpu, VGIC_NR_SGIS) { 1057 for_each_set_bit(i, vgic_cpu->pending_percpu, VGIC_NR_SGIS) {
1416 if (!vgic_queue_sgi(vcpu, i)) 1058 if (!queue_sgi(vcpu, i))
1417 overflow = 1; 1059 overflow = 1;
1418 } 1060 }
1419 1061
@@ -1502,6 +1144,14 @@ static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1502 if (status & INT_STATUS_UNDERFLOW) 1144 if (status & INT_STATUS_UNDERFLOW)
1503 vgic_disable_underflow(vcpu); 1145 vgic_disable_underflow(vcpu);
1504 1146
1147 /*
1148 * In the next iterations of the vcpu loop, if we sync the vgic state
1149 * after flushing it, but before entering the guest (this happens for
1150 * pending signals and vmid rollovers), then make sure we don't pick
1151 * up any old maintenance interrupts here.
1152 */
1153 vgic_clear_eisr(vcpu);
1154
1505 return level_pending; 1155 return level_pending;
1506} 1156}
1507 1157
@@ -1575,7 +1225,7 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
1575 return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu); 1225 return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1576} 1226}
1577 1227
1578static void vgic_kick_vcpus(struct kvm *kvm) 1228void vgic_kick_vcpus(struct kvm *kvm)
1579{ 1229{
1580 struct kvm_vcpu *vcpu; 1230 struct kvm_vcpu *vcpu;
1581 int c; 1231 int c;
@@ -1615,7 +1265,7 @@ static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
1615 struct kvm_vcpu *vcpu; 1265 struct kvm_vcpu *vcpu;
1616 int edge_triggered, level_triggered; 1266 int edge_triggered, level_triggered;
1617 int enabled; 1267 int enabled;
1618 bool ret = true; 1268 bool ret = true, can_inject = true;
1619 1269
1620 spin_lock(&dist->lock); 1270 spin_lock(&dist->lock);
1621 1271
@@ -1630,6 +1280,11 @@ static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
1630 1280
1631 if (irq_num >= VGIC_NR_PRIVATE_IRQS) { 1281 if (irq_num >= VGIC_NR_PRIVATE_IRQS) {
1632 cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS]; 1282 cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS];
1283 if (cpuid == VCPU_NOT_ALLOCATED) {
1284 /* Pretend we use CPU0, and prevent injection */
1285 cpuid = 0;
1286 can_inject = false;
1287 }
1633 vcpu = kvm_get_vcpu(kvm, cpuid); 1288 vcpu = kvm_get_vcpu(kvm, cpuid);
1634 } 1289 }
1635 1290
@@ -1652,7 +1307,7 @@ static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
1652 1307
1653 enabled = vgic_irq_is_enabled(vcpu, irq_num); 1308 enabled = vgic_irq_is_enabled(vcpu, irq_num);
1654 1309
1655 if (!enabled) { 1310 if (!enabled || !can_inject) {
1656 ret = false; 1311 ret = false;
1657 goto out; 1312 goto out;
1658 } 1313 }
@@ -1698,6 +1353,16 @@ int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
1698 int vcpu_id; 1353 int vcpu_id;
1699 1354
1700 if (unlikely(!vgic_initialized(kvm))) { 1355 if (unlikely(!vgic_initialized(kvm))) {
1356 /*
1357 * We only provide the automatic initialization of the VGIC
1358 * for the legacy case of a GICv2. Any other type must
1359 * be explicitly initialized once setup with the respective
1360 * KVM device call.
1361 */
1362 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2) {
1363 ret = -EBUSY;
1364 goto out;
1365 }
1701 mutex_lock(&kvm->lock); 1366 mutex_lock(&kvm->lock);
1702 ret = vgic_init(kvm); 1367 ret = vgic_init(kvm);
1703 mutex_unlock(&kvm->lock); 1368 mutex_unlock(&kvm->lock);
@@ -1762,6 +1427,17 @@ static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs)
1762 return 0; 1427 return 0;
1763} 1428}
1764 1429
1430/**
1431 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
1432 *
1433 * The host's GIC naturally limits the maximum amount of VCPUs a guest
1434 * can use.
1435 */
1436int kvm_vgic_get_max_vcpus(void)
1437{
1438 return vgic->max_gic_vcpus;
1439}
1440
1765void kvm_vgic_destroy(struct kvm *kvm) 1441void kvm_vgic_destroy(struct kvm *kvm)
1766{ 1442{
1767 struct vgic_dist *dist = &kvm->arch.vgic; 1443 struct vgic_dist *dist = &kvm->arch.vgic;
@@ -1784,6 +1460,7 @@ void kvm_vgic_destroy(struct kvm *kvm)
1784 } 1460 }
1785 kfree(dist->irq_sgi_sources); 1461 kfree(dist->irq_sgi_sources);
1786 kfree(dist->irq_spi_cpu); 1462 kfree(dist->irq_spi_cpu);
1463 kfree(dist->irq_spi_mpidr);
1787 kfree(dist->irq_spi_target); 1464 kfree(dist->irq_spi_target);
1788 kfree(dist->irq_pending_on_cpu); 1465 kfree(dist->irq_pending_on_cpu);
1789 dist->irq_sgi_sources = NULL; 1466 dist->irq_sgi_sources = NULL;
@@ -1797,7 +1474,7 @@ void kvm_vgic_destroy(struct kvm *kvm)
1797 * Allocate and initialize the various data structures. Must be called 1474 * Allocate and initialize the various data structures. Must be called
1798 * with kvm->lock held! 1475 * with kvm->lock held!
1799 */ 1476 */
1800static int vgic_init(struct kvm *kvm) 1477int vgic_init(struct kvm *kvm)
1801{ 1478{
1802 struct vgic_dist *dist = &kvm->arch.vgic; 1479 struct vgic_dist *dist = &kvm->arch.vgic;
1803 struct kvm_vcpu *vcpu; 1480 struct kvm_vcpu *vcpu;
@@ -1809,7 +1486,7 @@ static int vgic_init(struct kvm *kvm)
1809 1486
1810 nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus); 1487 nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);
1811 if (!nr_cpus) /* No vcpus? Can't be good... */ 1488 if (!nr_cpus) /* No vcpus? Can't be good... */
1812 return -EINVAL; 1489 return -ENODEV;
1813 1490
1814 /* 1491 /*
1815 * If nobody configured the number of interrupts, use the 1492 * If nobody configured the number of interrupts, use the
@@ -1852,8 +1529,9 @@ static int vgic_init(struct kvm *kvm)
1852 if (ret) 1529 if (ret)
1853 goto out; 1530 goto out;
1854 1531
1855 for (i = VGIC_NR_PRIVATE_IRQS; i < dist->nr_irqs; i += 4) 1532 ret = kvm->arch.vgic.vm_ops.init_model(kvm);
1856 vgic_set_target_reg(kvm, 0, i); 1533 if (ret)
1534 goto out;
1857 1535
1858 kvm_for_each_vcpu(vcpu_id, vcpu, kvm) { 1536 kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {
1859 ret = vgic_vcpu_init_maps(vcpu, nr_irqs); 1537 ret = vgic_vcpu_init_maps(vcpu, nr_irqs);
@@ -1882,72 +1560,51 @@ out:
1882 return ret; 1560 return ret;
1883} 1561}
1884 1562
1885/** 1563static int init_vgic_model(struct kvm *kvm, int type)
1886 * kvm_vgic_map_resources - Configure global VGIC state before running any VCPUs
1887 * @kvm: pointer to the kvm struct
1888 *
1889 * Map the virtual CPU interface into the VM before running any VCPUs. We
1890 * can't do this at creation time, because user space must first set the
1891 * virtual CPU interface address in the guest physical address space.
1892 */
1893int kvm_vgic_map_resources(struct kvm *kvm)
1894{ 1564{
1895 int ret = 0; 1565 switch (type) {
1896 1566 case KVM_DEV_TYPE_ARM_VGIC_V2:
1897 if (!irqchip_in_kernel(kvm)) 1567 vgic_v2_init_emulation(kvm);
1898 return 0; 1568 break;
1899 1569#ifdef CONFIG_ARM_GIC_V3
1900 mutex_lock(&kvm->lock); 1570 case KVM_DEV_TYPE_ARM_VGIC_V3:
1901 1571 vgic_v3_init_emulation(kvm);
1902 if (vgic_ready(kvm)) 1572 break;
1903 goto out; 1573#endif
1904 1574 default:
1905 if (IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_dist_base) || 1575 return -ENODEV;
1906 IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_cpu_base)) {
1907 kvm_err("Need to set vgic cpu and dist addresses first\n");
1908 ret = -ENXIO;
1909 goto out;
1910 }
1911
1912 /*
1913 * Initialize the vgic if this hasn't already been done on demand by
1914 * accessing the vgic state from userspace.
1915 */
1916 ret = vgic_init(kvm);
1917 if (ret) {
1918 kvm_err("Unable to allocate maps\n");
1919 goto out;
1920 } 1576 }
1921 1577
1922 ret = kvm_phys_addr_ioremap(kvm, kvm->arch.vgic.vgic_cpu_base, 1578 if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)
1923 vgic->vcpu_base, KVM_VGIC_V2_CPU_SIZE, 1579 return -E2BIG;
1924 true);
1925 if (ret) {
1926 kvm_err("Unable to remap VGIC CPU to VCPU\n");
1927 goto out;
1928 }
1929 1580
1930 kvm->arch.vgic.ready = true; 1581 return 0;
1931out:
1932 if (ret)
1933 kvm_vgic_destroy(kvm);
1934 mutex_unlock(&kvm->lock);
1935 return ret;
1936} 1582}
1937 1583
1938int kvm_vgic_create(struct kvm *kvm) 1584int kvm_vgic_create(struct kvm *kvm, u32 type)
1939{ 1585{
1940 int i, vcpu_lock_idx = -1, ret; 1586 int i, vcpu_lock_idx = -1, ret;
1941 struct kvm_vcpu *vcpu; 1587 struct kvm_vcpu *vcpu;
1942 1588
1943 mutex_lock(&kvm->lock); 1589 mutex_lock(&kvm->lock);
1944 1590
1945 if (kvm->arch.vgic.vctrl_base) { 1591 if (irqchip_in_kernel(kvm)) {
1946 ret = -EEXIST; 1592 ret = -EEXIST;
1947 goto out; 1593 goto out;
1948 } 1594 }
1949 1595
1950 /* 1596 /*
1597 * This function is also called by the KVM_CREATE_IRQCHIP handler,
1598 * which had no chance yet to check the availability of the GICv2
1599 * emulation. So check this here again. KVM_CREATE_DEVICE does
1600 * the proper checks already.
1601 */
1602 if (type == KVM_DEV_TYPE_ARM_VGIC_V2 && !vgic->can_emulate_gicv2) {
1603 ret = -ENODEV;
1604 goto out;
1605 }
1606
1607 /*
1951 * Any time a vcpu is run, vcpu_load is called which tries to grab the 1608 * Any time a vcpu is run, vcpu_load is called which tries to grab the
1952 * vcpu->mutex. By grabbing the vcpu->mutex of all VCPUs we ensure 1609 * vcpu->mutex. By grabbing the vcpu->mutex of all VCPUs we ensure
1953 * that no other VCPUs are run while we create the vgic. 1610 * that no other VCPUs are run while we create the vgic.
@@ -1965,11 +1622,17 @@ int kvm_vgic_create(struct kvm *kvm)
1965 } 1622 }
1966 ret = 0; 1623 ret = 0;
1967 1624
1625 ret = init_vgic_model(kvm, type);
1626 if (ret)
1627 goto out_unlock;
1628
1968 spin_lock_init(&kvm->arch.vgic.lock); 1629 spin_lock_init(&kvm->arch.vgic.lock);
1969 kvm->arch.vgic.in_kernel = true; 1630 kvm->arch.vgic.in_kernel = true;
1631 kvm->arch.vgic.vgic_model = type;
1970 kvm->arch.vgic.vctrl_base = vgic->vctrl_base; 1632 kvm->arch.vgic.vctrl_base = vgic->vctrl_base;
1971 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF; 1633 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
1972 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF; 1634 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
1635 kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
1973 1636
1974out_unlock: 1637out_unlock:
1975 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) { 1638 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
@@ -2022,7 +1685,7 @@ static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,
2022/** 1685/**
2023 * kvm_vgic_addr - set or get vgic VM base addresses 1686 * kvm_vgic_addr - set or get vgic VM base addresses
2024 * @kvm: pointer to the vm struct 1687 * @kvm: pointer to the vm struct
2025 * @type: the VGIC addr type, one of KVM_VGIC_V2_ADDR_TYPE_XXX 1688 * @type: the VGIC addr type, one of KVM_VGIC_V[23]_ADDR_TYPE_XXX
2026 * @addr: pointer to address value 1689 * @addr: pointer to address value
2027 * @write: if true set the address in the VM address space, if false read the 1690 * @write: if true set the address in the VM address space, if false read the
2028 * address 1691 * address
@@ -2036,216 +1699,64 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
2036{ 1699{
2037 int r = 0; 1700 int r = 0;
2038 struct vgic_dist *vgic = &kvm->arch.vgic; 1701 struct vgic_dist *vgic = &kvm->arch.vgic;
1702 int type_needed;
1703 phys_addr_t *addr_ptr, block_size;
1704 phys_addr_t alignment;
2039 1705
2040 mutex_lock(&kvm->lock); 1706 mutex_lock(&kvm->lock);
2041 switch (type) { 1707 switch (type) {
2042 case KVM_VGIC_V2_ADDR_TYPE_DIST: 1708 case KVM_VGIC_V2_ADDR_TYPE_DIST:
2043 if (write) { 1709 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
2044 r = vgic_ioaddr_assign(kvm, &vgic->vgic_dist_base, 1710 addr_ptr = &vgic->vgic_dist_base;
2045 *addr, KVM_VGIC_V2_DIST_SIZE); 1711 block_size = KVM_VGIC_V2_DIST_SIZE;
2046 } else { 1712 alignment = SZ_4K;
2047 *addr = vgic->vgic_dist_base;
2048 }
2049 break; 1713 break;
2050 case KVM_VGIC_V2_ADDR_TYPE_CPU: 1714 case KVM_VGIC_V2_ADDR_TYPE_CPU:
2051 if (write) { 1715 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
2052 r = vgic_ioaddr_assign(kvm, &vgic->vgic_cpu_base, 1716 addr_ptr = &vgic->vgic_cpu_base;
2053 *addr, KVM_VGIC_V2_CPU_SIZE); 1717 block_size = KVM_VGIC_V2_CPU_SIZE;
2054 } else { 1718 alignment = SZ_4K;
2055 *addr = vgic->vgic_cpu_base;
2056 }
2057 break;
2058 default:
2059 r = -ENODEV;
2060 }
2061
2062 mutex_unlock(&kvm->lock);
2063 return r;
2064}
2065
2066static bool handle_cpu_mmio_misc(struct kvm_vcpu *vcpu,
2067 struct kvm_exit_mmio *mmio, phys_addr_t offset)
2068{
2069 bool updated = false;
2070 struct vgic_vmcr vmcr;
2071 u32 *vmcr_field;
2072 u32 reg;
2073
2074 vgic_get_vmcr(vcpu, &vmcr);
2075
2076 switch (offset & ~0x3) {
2077 case GIC_CPU_CTRL:
2078 vmcr_field = &vmcr.ctlr;
2079 break; 1719 break;
2080 case GIC_CPU_PRIMASK: 1720#ifdef CONFIG_ARM_GIC_V3
2081 vmcr_field = &vmcr.pmr; 1721 case KVM_VGIC_V3_ADDR_TYPE_DIST:
1722 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
1723 addr_ptr = &vgic->vgic_dist_base;
1724 block_size = KVM_VGIC_V3_DIST_SIZE;
1725 alignment = SZ_64K;
2082 break; 1726 break;
2083 case GIC_CPU_BINPOINT: 1727 case KVM_VGIC_V3_ADDR_TYPE_REDIST:
2084 vmcr_field = &vmcr.bpr; 1728 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
2085 break; 1729 addr_ptr = &vgic->vgic_redist_base;
2086 case GIC_CPU_ALIAS_BINPOINT: 1730 block_size = KVM_VGIC_V3_REDIST_SIZE;
2087 vmcr_field = &vmcr.abpr; 1731 alignment = SZ_64K;
2088 break; 1732 break;
1733#endif
2089 default: 1734 default:
2090 BUG(); 1735 r = -ENODEV;
2091 }
2092
2093 if (!mmio->is_write) {
2094 reg = *vmcr_field;
2095 mmio_data_write(mmio, ~0, reg);
2096 } else {
2097 reg = mmio_data_read(mmio, ~0);
2098 if (reg != *vmcr_field) {
2099 *vmcr_field = reg;
2100 vgic_set_vmcr(vcpu, &vmcr);
2101 updated = true;
2102 }
2103 }
2104 return updated;
2105}
2106
2107static bool handle_mmio_abpr(struct kvm_vcpu *vcpu,
2108 struct kvm_exit_mmio *mmio, phys_addr_t offset)
2109{
2110 return handle_cpu_mmio_misc(vcpu, mmio, GIC_CPU_ALIAS_BINPOINT);
2111}
2112
2113static bool handle_cpu_mmio_ident(struct kvm_vcpu *vcpu,
2114 struct kvm_exit_mmio *mmio,
2115 phys_addr_t offset)
2116{
2117 u32 reg;
2118
2119 if (mmio->is_write)
2120 return false;
2121
2122 /* GICC_IIDR */
2123 reg = (PRODUCT_ID_KVM << 20) |
2124 (GICC_ARCH_VERSION_V2 << 16) |
2125 (IMPLEMENTER_ARM << 0);
2126 mmio_data_write(mmio, ~0, reg);
2127 return false;
2128}
2129
2130/*
2131 * CPU Interface Register accesses - these are not accessed by the VM, but by
2132 * user space for saving and restoring VGIC state.
2133 */
2134static const struct mmio_range vgic_cpu_ranges[] = {
2135 {
2136 .base = GIC_CPU_CTRL,
2137 .len = 12,
2138 .handle_mmio = handle_cpu_mmio_misc,
2139 },
2140 {
2141 .base = GIC_CPU_ALIAS_BINPOINT,
2142 .len = 4,
2143 .handle_mmio = handle_mmio_abpr,
2144 },
2145 {
2146 .base = GIC_CPU_ACTIVEPRIO,
2147 .len = 16,
2148 .handle_mmio = handle_mmio_raz_wi,
2149 },
2150 {
2151 .base = GIC_CPU_IDENT,
2152 .len = 4,
2153 .handle_mmio = handle_cpu_mmio_ident,
2154 },
2155};
2156
2157static int vgic_attr_regs_access(struct kvm_device *dev,
2158 struct kvm_device_attr *attr,
2159 u32 *reg, bool is_write)
2160{
2161 const struct mmio_range *r = NULL, *ranges;
2162 phys_addr_t offset;
2163 int ret, cpuid, c;
2164 struct kvm_vcpu *vcpu, *tmp_vcpu;
2165 struct vgic_dist *vgic;
2166 struct kvm_exit_mmio mmio;
2167
2168 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
2169 cpuid = (attr->attr & KVM_DEV_ARM_VGIC_CPUID_MASK) >>
2170 KVM_DEV_ARM_VGIC_CPUID_SHIFT;
2171
2172 mutex_lock(&dev->kvm->lock);
2173
2174 ret = vgic_init(dev->kvm);
2175 if (ret)
2176 goto out;
2177
2178 if (cpuid >= atomic_read(&dev->kvm->online_vcpus)) {
2179 ret = -EINVAL;
2180 goto out; 1736 goto out;
2181 } 1737 }
2182 1738
2183 vcpu = kvm_get_vcpu(dev->kvm, cpuid); 1739 if (vgic->vgic_model != type_needed) {
2184 vgic = &dev->kvm->arch.vgic; 1740 r = -ENODEV;
2185
2186 mmio.len = 4;
2187 mmio.is_write = is_write;
2188 if (is_write)
2189 mmio_data_write(&mmio, ~0, *reg);
2190 switch (attr->group) {
2191 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
2192 mmio.phys_addr = vgic->vgic_dist_base + offset;
2193 ranges = vgic_dist_ranges;
2194 break;
2195 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
2196 mmio.phys_addr = vgic->vgic_cpu_base + offset;
2197 ranges = vgic_cpu_ranges;
2198 break;
2199 default:
2200 BUG();
2201 }
2202 r = find_matching_range(ranges, &mmio, offset);
2203
2204 if (unlikely(!r || !r->handle_mmio)) {
2205 ret = -ENXIO;
2206 goto out; 1741 goto out;
2207 } 1742 }
2208 1743
2209 1744 if (write) {
2210 spin_lock(&vgic->lock); 1745 if (!IS_ALIGNED(*addr, alignment))
2211 1746 r = -EINVAL;
2212 /* 1747 else
2213 * Ensure that no other VCPU is running by checking the vcpu->cpu 1748 r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,
2214 * field. If no other VPCUs are running we can safely access the VGIC 1749 block_size);
2215 * state, because even if another VPU is run after this point, that 1750 } else {
2216 * VCPU will not touch the vgic state, because it will block on 1751 *addr = *addr_ptr;
2217 * getting the vgic->lock in kvm_vgic_sync_hwstate().
2218 */
2219 kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm) {
2220 if (unlikely(tmp_vcpu->cpu != -1)) {
2221 ret = -EBUSY;
2222 goto out_vgic_unlock;
2223 }
2224 } 1752 }
2225 1753
2226 /*
2227 * Move all pending IRQs from the LRs on all VCPUs so the pending
2228 * state can be properly represented in the register state accessible
2229 * through this API.
2230 */
2231 kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm)
2232 vgic_unqueue_irqs(tmp_vcpu);
2233
2234 offset -= r->base;
2235 r->handle_mmio(vcpu, &mmio, offset);
2236
2237 if (!is_write)
2238 *reg = mmio_data_read(&mmio, ~0);
2239
2240 ret = 0;
2241out_vgic_unlock:
2242 spin_unlock(&vgic->lock);
2243out: 1754out:
2244 mutex_unlock(&dev->kvm->lock); 1755 mutex_unlock(&kvm->lock);
2245 return ret; 1756 return r;
2246} 1757}
2247 1758
2248static int vgic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1759int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2249{ 1760{
2250 int r; 1761 int r;
2251 1762
@@ -2261,17 +1772,6 @@ static int vgic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2261 r = kvm_vgic_addr(dev->kvm, type, &addr, true); 1772 r = kvm_vgic_addr(dev->kvm, type, &addr, true);
2262 return (r == -ENODEV) ? -ENXIO : r; 1773 return (r == -ENODEV) ? -ENXIO : r;
2263 } 1774 }
2264
2265 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
2266 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
2267 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2268 u32 reg;
2269
2270 if (get_user(reg, uaddr))
2271 return -EFAULT;
2272
2273 return vgic_attr_regs_access(dev, attr, &reg, true);
2274 }
2275 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: { 1775 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2276 u32 __user *uaddr = (u32 __user *)(long)attr->addr; 1776 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2277 u32 val; 1777 u32 val;
@@ -2302,13 +1802,20 @@ static int vgic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2302 1802
2303 return ret; 1803 return ret;
2304 } 1804 }
2305 1805 case KVM_DEV_ARM_VGIC_GRP_CTRL: {
1806 switch (attr->attr) {
1807 case KVM_DEV_ARM_VGIC_CTRL_INIT:
1808 r = vgic_init(dev->kvm);
1809 return r;
1810 }
1811 break;
1812 }
2306 } 1813 }
2307 1814
2308 return -ENXIO; 1815 return -ENXIO;
2309} 1816}
2310 1817
2311static int vgic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1818int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2312{ 1819{
2313 int r = -ENXIO; 1820 int r = -ENXIO;
2314 1821
@@ -2326,20 +1833,9 @@ static int vgic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2326 return -EFAULT; 1833 return -EFAULT;
2327 break; 1834 break;
2328 } 1835 }
2329
2330 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
2331 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: {
2332 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2333 u32 reg = 0;
2334
2335 r = vgic_attr_regs_access(dev, attr, &reg, false);
2336 if (r)
2337 return r;
2338 r = put_user(reg, uaddr);
2339 break;
2340 }
2341 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: { 1836 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2342 u32 __user *uaddr = (u32 __user *)(long)attr->addr; 1837 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
1838
2343 r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr); 1839 r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);
2344 break; 1840 break;
2345 } 1841 }
@@ -2349,61 +1845,17 @@ static int vgic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2349 return r; 1845 return r;
2350} 1846}
2351 1847
2352static int vgic_has_attr_regs(const struct mmio_range *ranges, 1848int vgic_has_attr_regs(const struct kvm_mmio_range *ranges, phys_addr_t offset)
2353 phys_addr_t offset)
2354{ 1849{
2355 struct kvm_exit_mmio dev_attr_mmio; 1850 struct kvm_exit_mmio dev_attr_mmio;
2356 1851
2357 dev_attr_mmio.len = 4; 1852 dev_attr_mmio.len = 4;
2358 if (find_matching_range(ranges, &dev_attr_mmio, offset)) 1853 if (vgic_find_range(ranges, &dev_attr_mmio, offset))
2359 return 0; 1854 return 0;
2360 else 1855 else
2361 return -ENXIO; 1856 return -ENXIO;
2362} 1857}
2363 1858
2364static int vgic_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2365{
2366 phys_addr_t offset;
2367
2368 switch (attr->group) {
2369 case KVM_DEV_ARM_VGIC_GRP_ADDR:
2370 switch (attr->attr) {
2371 case KVM_VGIC_V2_ADDR_TYPE_DIST:
2372 case KVM_VGIC_V2_ADDR_TYPE_CPU:
2373 return 0;
2374 }
2375 break;
2376 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
2377 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
2378 return vgic_has_attr_regs(vgic_dist_ranges, offset);
2379 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
2380 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
2381 return vgic_has_attr_regs(vgic_cpu_ranges, offset);
2382 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
2383 return 0;
2384 }
2385 return -ENXIO;
2386}
2387
2388static void vgic_destroy(struct kvm_device *dev)
2389{
2390 kfree(dev);
2391}
2392
2393static int vgic_create(struct kvm_device *dev, u32 type)
2394{
2395 return kvm_vgic_create(dev->kvm);
2396}
2397
2398static struct kvm_device_ops kvm_arm_vgic_v2_ops = {
2399 .name = "kvm-arm-vgic",
2400 .create = vgic_create,
2401 .destroy = vgic_destroy,
2402 .set_attr = vgic_set_attr,
2403 .get_attr = vgic_get_attr,
2404 .has_attr = vgic_has_attr,
2405};
2406
2407static void vgic_init_maintenance_interrupt(void *info) 1859static void vgic_init_maintenance_interrupt(void *info)
2408{ 1860{
2409 enable_percpu_irq(vgic->maint_irq, 0); 1861 enable_percpu_irq(vgic->maint_irq, 0);
@@ -2474,8 +1926,7 @@ int kvm_vgic_hyp_init(void)
2474 1926
2475 on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1); 1927 on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);
2476 1928
2477 return kvm_register_device_ops(&kvm_arm_vgic_v2_ops, 1929 return 0;
2478 KVM_DEV_TYPE_ARM_VGIC_V2);
2479 1930
2480out_free_irq: 1931out_free_irq:
2481 free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus()); 1932 free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
diff --git a/virt/kvm/arm/vgic.h b/virt/kvm/arm/vgic.h
new file mode 100644
index 000000000000..1e83bdf5f499
--- /dev/null
+++ b/virt/kvm/arm/vgic.h
@@ -0,0 +1,123 @@
1/*
2 * Copyright (C) 2012-2014 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * Derived from virt/kvm/arm/vgic.c
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#ifndef __KVM_VGIC_H__
21#define __KVM_VGIC_H__
22
23#define VGIC_ADDR_UNDEF (-1)
24#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF)
25
26#define PRODUCT_ID_KVM 0x4b /* ASCII code K */
27#define IMPLEMENTER_ARM 0x43b
28
29#define ACCESS_READ_VALUE (1 << 0)
30#define ACCESS_READ_RAZ (0 << 0)
31#define ACCESS_READ_MASK(x) ((x) & (1 << 0))
32#define ACCESS_WRITE_IGNORED (0 << 1)
33#define ACCESS_WRITE_SETBIT (1 << 1)
34#define ACCESS_WRITE_CLEARBIT (2 << 1)
35#define ACCESS_WRITE_VALUE (3 << 1)
36#define ACCESS_WRITE_MASK(x) ((x) & (3 << 1))
37
38#define VCPU_NOT_ALLOCATED ((u8)-1)
39
40unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x);
41
42void vgic_update_state(struct kvm *kvm);
43int vgic_init_common_maps(struct kvm *kvm);
44
45u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset);
46u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset);
47
48void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq);
49void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq);
50void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq);
51void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
52 int irq, int val);
53
54void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
55void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
56
57bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq);
58void vgic_unqueue_irqs(struct kvm_vcpu *vcpu);
59
60void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
61 phys_addr_t offset, int mode);
62bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
63 phys_addr_t offset);
64
65static inline
66u32 mmio_data_read(struct kvm_exit_mmio *mmio, u32 mask)
67{
68 return le32_to_cpu(*((u32 *)mmio->data)) & mask;
69}
70
71static inline
72void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
73{
74 *((u32 *)mmio->data) = cpu_to_le32(value) & mask;
75}
76
77struct kvm_mmio_range {
78 phys_addr_t base;
79 unsigned long len;
80 int bits_per_irq;
81 bool (*handle_mmio)(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
82 phys_addr_t offset);
83};
84
85static inline bool is_in_range(phys_addr_t addr, unsigned long len,
86 phys_addr_t baseaddr, unsigned long size)
87{
88 return (addr >= baseaddr) && (addr + len <= baseaddr + size);
89}
90
91const
92struct kvm_mmio_range *vgic_find_range(const struct kvm_mmio_range *ranges,
93 struct kvm_exit_mmio *mmio,
94 phys_addr_t offset);
95
96bool vgic_handle_mmio_range(struct kvm_vcpu *vcpu, struct kvm_run *run,
97 struct kvm_exit_mmio *mmio,
98 const struct kvm_mmio_range *ranges,
99 unsigned long mmio_base);
100
101bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
102 phys_addr_t offset, int vcpu_id, int access);
103
104bool vgic_handle_set_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
105 phys_addr_t offset, int vcpu_id);
106
107bool vgic_handle_clear_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
108 phys_addr_t offset, int vcpu_id);
109
110bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
111 phys_addr_t offset);
112
113void vgic_kick_vcpus(struct kvm *kvm);
114
115int vgic_has_attr_regs(const struct kvm_mmio_range *ranges, phys_addr_t offset);
116int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr);
117int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr);
118
119int vgic_init(struct kvm *kvm);
120void vgic_v2_init_emulation(struct kvm *kvm);
121void vgic_v3_init_emulation(struct kvm *kvm);
122
123#endif
diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c
index 5ff7f7f2689a..44660aee335f 100644
--- a/virt/kvm/async_pf.c
+++ b/virt/kvm/async_pf.c
@@ -80,7 +80,7 @@ static void async_pf_execute(struct work_struct *work)
80 80
81 might_sleep(); 81 might_sleep();
82 82
83 kvm_get_user_page_io(NULL, mm, addr, 1, NULL); 83 get_user_pages_unlocked(NULL, mm, addr, 1, 1, 0, NULL);
84 kvm_async_page_present_sync(vcpu, apf); 84 kvm_async_page_present_sync(vcpu, apf);
85 85
86 spin_lock(&vcpu->async_pf.lock); 86 spin_lock(&vcpu->async_pf.lock);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index f5283438ee05..cc6a25d95fbf 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -66,6 +66,9 @@
66MODULE_AUTHOR("Qumranet"); 66MODULE_AUTHOR("Qumranet");
67MODULE_LICENSE("GPL"); 67MODULE_LICENSE("GPL");
68 68
69unsigned int halt_poll_ns = 0;
70module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
71
69/* 72/*
70 * Ordering of locks: 73 * Ordering of locks:
71 * 74 *
@@ -89,7 +92,7 @@ struct dentry *kvm_debugfs_dir;
89 92
90static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 93static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
91 unsigned long arg); 94 unsigned long arg);
92#ifdef CONFIG_COMPAT 95#ifdef CONFIG_KVM_COMPAT
93static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 96static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
94 unsigned long arg); 97 unsigned long arg);
95#endif 98#endif
@@ -176,6 +179,7 @@ bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
176 return called; 179 return called;
177} 180}
178 181
182#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
179void kvm_flush_remote_tlbs(struct kvm *kvm) 183void kvm_flush_remote_tlbs(struct kvm *kvm)
180{ 184{
181 long dirty_count = kvm->tlbs_dirty; 185 long dirty_count = kvm->tlbs_dirty;
@@ -186,6 +190,7 @@ void kvm_flush_remote_tlbs(struct kvm *kvm)
186 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 190 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
187} 191}
188EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 192EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
193#endif
189 194
190void kvm_reload_remote_mmus(struct kvm *kvm) 195void kvm_reload_remote_mmus(struct kvm *kvm)
191{ 196{
@@ -466,7 +471,7 @@ static struct kvm *kvm_create_vm(unsigned long type)
466 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 471 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
467 472
468 r = -ENOMEM; 473 r = -ENOMEM;
469 kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 474 kvm->memslots = kvm_kvzalloc(sizeof(struct kvm_memslots));
470 if (!kvm->memslots) 475 if (!kvm->memslots)
471 goto out_err_no_srcu; 476 goto out_err_no_srcu;
472 477
@@ -517,7 +522,7 @@ out_err_no_srcu:
517out_err_no_disable: 522out_err_no_disable:
518 for (i = 0; i < KVM_NR_BUSES; i++) 523 for (i = 0; i < KVM_NR_BUSES; i++)
519 kfree(kvm->buses[i]); 524 kfree(kvm->buses[i]);
520 kfree(kvm->memslots); 525 kvfree(kvm->memslots);
521 kvm_arch_free_vm(kvm); 526 kvm_arch_free_vm(kvm);
522 return ERR_PTR(r); 527 return ERR_PTR(r);
523} 528}
@@ -573,7 +578,7 @@ static void kvm_free_physmem(struct kvm *kvm)
573 kvm_for_each_memslot(memslot, slots) 578 kvm_for_each_memslot(memslot, slots)
574 kvm_free_physmem_slot(kvm, memslot, NULL); 579 kvm_free_physmem_slot(kvm, memslot, NULL);
575 580
576 kfree(kvm->memslots); 581 kvfree(kvm->memslots);
577} 582}
578 583
579static void kvm_destroy_devices(struct kvm *kvm) 584static void kvm_destroy_devices(struct kvm *kvm)
@@ -671,7 +676,9 @@ static void update_memslots(struct kvm_memslots *slots,
671 676
672 WARN_ON(mslots[i].id != id); 677 WARN_ON(mslots[i].id != id);
673 if (!new->npages) { 678 if (!new->npages) {
679 WARN_ON(!mslots[i].npages);
674 new->base_gfn = 0; 680 new->base_gfn = 0;
681 new->flags = 0;
675 if (mslots[i].npages) 682 if (mslots[i].npages)
676 slots->used_slots--; 683 slots->used_slots--;
677 } else { 684 } else {
@@ -687,12 +694,25 @@ static void update_memslots(struct kvm_memslots *slots,
687 slots->id_to_index[mslots[i].id] = i; 694 slots->id_to_index[mslots[i].id] = i;
688 i++; 695 i++;
689 } 696 }
690 while (i > 0 && 697
691 new->base_gfn > mslots[i - 1].base_gfn) { 698 /*
692 mslots[i] = mslots[i - 1]; 699 * The ">=" is needed when creating a slot with base_gfn == 0,
693 slots->id_to_index[mslots[i].id] = i; 700 * so that it moves before all those with base_gfn == npages == 0.
694 i--; 701 *
695 } 702 * On the other hand, if new->npages is zero, the above loop has
703 * already left i pointing to the beginning of the empty part of
704 * mslots, and the ">=" would move the hole backwards in this
705 * case---which is wrong. So skip the loop when deleting a slot.
706 */
707 if (new->npages) {
708 while (i > 0 &&
709 new->base_gfn >= mslots[i - 1].base_gfn) {
710 mslots[i] = mslots[i - 1];
711 slots->id_to_index[mslots[i].id] = i;
712 i--;
713 }
714 } else
715 WARN_ON_ONCE(i != slots->used_slots);
696 716
697 mslots[i] = *new; 717 mslots[i] = *new;
698 slots->id_to_index[mslots[i].id] = i; 718 slots->id_to_index[mslots[i].id] = i;
@@ -851,10 +871,10 @@ int __kvm_set_memory_region(struct kvm *kvm,
851 goto out_free; 871 goto out_free;
852 } 872 }
853 873
854 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 874 slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
855 GFP_KERNEL);
856 if (!slots) 875 if (!slots)
857 goto out_free; 876 goto out_free;
877 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
858 878
859 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 879 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
860 slot = id_to_memslot(slots, mem->slot); 880 slot = id_to_memslot(slots, mem->slot);
@@ -897,7 +917,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
897 kvm_arch_commit_memory_region(kvm, mem, &old, change); 917 kvm_arch_commit_memory_region(kvm, mem, &old, change);
898 918
899 kvm_free_physmem_slot(kvm, &old, &new); 919 kvm_free_physmem_slot(kvm, &old, &new);
900 kfree(old_memslots); 920 kvfree(old_memslots);
901 921
902 /* 922 /*
903 * IOMMU mapping: New slots need to be mapped. Old slots need to be 923 * IOMMU mapping: New slots need to be mapped. Old slots need to be
@@ -916,7 +936,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
916 return 0; 936 return 0;
917 937
918out_slots: 938out_slots:
919 kfree(slots); 939 kvfree(slots);
920out_free: 940out_free:
921 kvm_free_physmem_slot(kvm, &new, &old); 941 kvm_free_physmem_slot(kvm, &new, &old);
922out: 942out:
@@ -979,6 +999,86 @@ out:
979} 999}
980EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1000EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
981 1001
1002#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1003/**
1004 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
1005 * are dirty write protect them for next write.
1006 * @kvm: pointer to kvm instance
1007 * @log: slot id and address to which we copy the log
1008 * @is_dirty: flag set if any page is dirty
1009 *
1010 * We need to keep it in mind that VCPU threads can write to the bitmap
1011 * concurrently. So, to avoid losing track of dirty pages we keep the
1012 * following order:
1013 *
1014 * 1. Take a snapshot of the bit and clear it if needed.
1015 * 2. Write protect the corresponding page.
1016 * 3. Copy the snapshot to the userspace.
1017 * 4. Upon return caller flushes TLB's if needed.
1018 *
1019 * Between 2 and 4, the guest may write to the page using the remaining TLB
1020 * entry. This is not a problem because the page is reported dirty using
1021 * the snapshot taken before and step 4 ensures that writes done after
1022 * exiting to userspace will be logged for the next call.
1023 *
1024 */
1025int kvm_get_dirty_log_protect(struct kvm *kvm,
1026 struct kvm_dirty_log *log, bool *is_dirty)
1027{
1028 struct kvm_memory_slot *memslot;
1029 int r, i;
1030 unsigned long n;
1031 unsigned long *dirty_bitmap;
1032 unsigned long *dirty_bitmap_buffer;
1033
1034 r = -EINVAL;
1035 if (log->slot >= KVM_USER_MEM_SLOTS)
1036 goto out;
1037
1038 memslot = id_to_memslot(kvm->memslots, log->slot);
1039
1040 dirty_bitmap = memslot->dirty_bitmap;
1041 r = -ENOENT;
1042 if (!dirty_bitmap)
1043 goto out;
1044
1045 n = kvm_dirty_bitmap_bytes(memslot);
1046
1047 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
1048 memset(dirty_bitmap_buffer, 0, n);
1049
1050 spin_lock(&kvm->mmu_lock);
1051 *is_dirty = false;
1052 for (i = 0; i < n / sizeof(long); i++) {
1053 unsigned long mask;
1054 gfn_t offset;
1055
1056 if (!dirty_bitmap[i])
1057 continue;
1058
1059 *is_dirty = true;
1060
1061 mask = xchg(&dirty_bitmap[i], 0);
1062 dirty_bitmap_buffer[i] = mask;
1063
1064 offset = i * BITS_PER_LONG;
1065 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset,
1066 mask);
1067 }
1068
1069 spin_unlock(&kvm->mmu_lock);
1070
1071 r = -EFAULT;
1072 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1073 goto out;
1074
1075 r = 0;
1076out:
1077 return r;
1078}
1079EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
1080#endif
1081
982bool kvm_largepages_enabled(void) 1082bool kvm_largepages_enabled(void)
983{ 1083{
984 return largepages_enabled; 1084 return largepages_enabled;
@@ -1114,43 +1214,6 @@ static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1114 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL); 1214 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
1115} 1215}
1116 1216
1117int kvm_get_user_page_io(struct task_struct *tsk, struct mm_struct *mm,
1118 unsigned long addr, bool write_fault,
1119 struct page **pagep)
1120{
1121 int npages;
1122 int locked = 1;
1123 int flags = FOLL_TOUCH | FOLL_HWPOISON |
1124 (pagep ? FOLL_GET : 0) |
1125 (write_fault ? FOLL_WRITE : 0);
1126
1127 /*
1128 * If retrying the fault, we get here *not* having allowed the filemap
1129 * to wait on the page lock. We should now allow waiting on the IO with
1130 * the mmap semaphore released.
1131 */
1132 down_read(&mm->mmap_sem);
1133 npages = __get_user_pages(tsk, mm, addr, 1, flags, pagep, NULL,
1134 &locked);
1135 if (!locked) {
1136 VM_BUG_ON(npages);
1137
1138 if (!pagep)
1139 return 0;
1140
1141 /*
1142 * The previous call has now waited on the IO. Now we can
1143 * retry and complete. Pass TRIED to ensure we do not re
1144 * schedule async IO (see e.g. filemap_fault).
1145 */
1146 down_read(&mm->mmap_sem);
1147 npages = __get_user_pages(tsk, mm, addr, 1, flags | FOLL_TRIED,
1148 pagep, NULL, NULL);
1149 }
1150 up_read(&mm->mmap_sem);
1151 return npages;
1152}
1153
1154static inline int check_user_page_hwpoison(unsigned long addr) 1217static inline int check_user_page_hwpoison(unsigned long addr)
1155{ 1218{
1156 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1219 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
@@ -1213,15 +1276,10 @@ static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1213 npages = get_user_page_nowait(current, current->mm, 1276 npages = get_user_page_nowait(current, current->mm,
1214 addr, write_fault, page); 1277 addr, write_fault, page);
1215 up_read(&current->mm->mmap_sem); 1278 up_read(&current->mm->mmap_sem);
1216 } else { 1279 } else
1217 /* 1280 npages = __get_user_pages_unlocked(current, current->mm, addr, 1,
1218 * By now we have tried gup_fast, and possibly async_pf, and we 1281 write_fault, 0, page,
1219 * are certainly not atomic. Time to retry the gup, allowing 1282 FOLL_TOUCH|FOLL_HWPOISON);
1220 * mmap semaphore to be relinquished in the case of IO.
1221 */
1222 npages = kvm_get_user_page_io(current, current->mm, addr,
1223 write_fault, page);
1224 }
1225 if (npages != 1) 1283 if (npages != 1)
1226 return npages; 1284 return npages;
1227 1285
@@ -1579,6 +1637,7 @@ int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1579 } 1637 }
1580 return 0; 1638 return 0;
1581} 1639}
1640EXPORT_SYMBOL_GPL(kvm_write_guest);
1582 1641
1583int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1642int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1584 gpa_t gpa, unsigned long len) 1643 gpa_t gpa, unsigned long len)
@@ -1715,29 +1774,60 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1715} 1774}
1716EXPORT_SYMBOL_GPL(mark_page_dirty); 1775EXPORT_SYMBOL_GPL(mark_page_dirty);
1717 1776
1777static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
1778{
1779 if (kvm_arch_vcpu_runnable(vcpu)) {
1780 kvm_make_request(KVM_REQ_UNHALT, vcpu);
1781 return -EINTR;
1782 }
1783 if (kvm_cpu_has_pending_timer(vcpu))
1784 return -EINTR;
1785 if (signal_pending(current))
1786 return -EINTR;
1787
1788 return 0;
1789}
1790
1718/* 1791/*
1719 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 1792 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1720 */ 1793 */
1721void kvm_vcpu_block(struct kvm_vcpu *vcpu) 1794void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1722{ 1795{
1796 ktime_t start, cur;
1723 DEFINE_WAIT(wait); 1797 DEFINE_WAIT(wait);
1798 bool waited = false;
1799
1800 start = cur = ktime_get();
1801 if (halt_poll_ns) {
1802 ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns);
1803 do {
1804 /*
1805 * This sets KVM_REQ_UNHALT if an interrupt
1806 * arrives.
1807 */
1808 if (kvm_vcpu_check_block(vcpu) < 0) {
1809 ++vcpu->stat.halt_successful_poll;
1810 goto out;
1811 }
1812 cur = ktime_get();
1813 } while (single_task_running() && ktime_before(cur, stop));
1814 }
1724 1815
1725 for (;;) { 1816 for (;;) {
1726 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 1817 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1727 1818
1728 if (kvm_arch_vcpu_runnable(vcpu)) { 1819 if (kvm_vcpu_check_block(vcpu) < 0)
1729 kvm_make_request(KVM_REQ_UNHALT, vcpu);
1730 break;
1731 }
1732 if (kvm_cpu_has_pending_timer(vcpu))
1733 break;
1734 if (signal_pending(current))
1735 break; 1820 break;
1736 1821
1822 waited = true;
1737 schedule(); 1823 schedule();
1738 } 1824 }
1739 1825
1740 finish_wait(&vcpu->wq, &wait); 1826 finish_wait(&vcpu->wq, &wait);
1827 cur = ktime_get();
1828
1829out:
1830 trace_kvm_vcpu_wakeup(ktime_to_ns(cur) - ktime_to_ns(start), waited);
1741} 1831}
1742EXPORT_SYMBOL_GPL(kvm_vcpu_block); 1832EXPORT_SYMBOL_GPL(kvm_vcpu_block);
1743 1833
@@ -1920,7 +2010,7 @@ static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1920static struct file_operations kvm_vcpu_fops = { 2010static struct file_operations kvm_vcpu_fops = {
1921 .release = kvm_vcpu_release, 2011 .release = kvm_vcpu_release,
1922 .unlocked_ioctl = kvm_vcpu_ioctl, 2012 .unlocked_ioctl = kvm_vcpu_ioctl,
1923#ifdef CONFIG_COMPAT 2013#ifdef CONFIG_KVM_COMPAT
1924 .compat_ioctl = kvm_vcpu_compat_ioctl, 2014 .compat_ioctl = kvm_vcpu_compat_ioctl,
1925#endif 2015#endif
1926 .mmap = kvm_vcpu_mmap, 2016 .mmap = kvm_vcpu_mmap,
@@ -2210,7 +2300,7 @@ out:
2210 return r; 2300 return r;
2211} 2301}
2212 2302
2213#ifdef CONFIG_COMPAT 2303#ifdef CONFIG_KVM_COMPAT
2214static long kvm_vcpu_compat_ioctl(struct file *filp, 2304static long kvm_vcpu_compat_ioctl(struct file *filp,
2215 unsigned int ioctl, unsigned long arg) 2305 unsigned int ioctl, unsigned long arg)
2216{ 2306{
@@ -2302,7 +2392,7 @@ static int kvm_device_release(struct inode *inode, struct file *filp)
2302 2392
2303static const struct file_operations kvm_device_fops = { 2393static const struct file_operations kvm_device_fops = {
2304 .unlocked_ioctl = kvm_device_ioctl, 2394 .unlocked_ioctl = kvm_device_ioctl,
2305#ifdef CONFIG_COMPAT 2395#ifdef CONFIG_KVM_COMPAT
2306 .compat_ioctl = kvm_device_ioctl, 2396 .compat_ioctl = kvm_device_ioctl,
2307#endif 2397#endif
2308 .release = kvm_device_release, 2398 .release = kvm_device_release,
@@ -2402,6 +2492,7 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
2402 case KVM_CAP_SIGNAL_MSI: 2492 case KVM_CAP_SIGNAL_MSI:
2403#endif 2493#endif
2404#ifdef CONFIG_HAVE_KVM_IRQFD 2494#ifdef CONFIG_HAVE_KVM_IRQFD
2495 case KVM_CAP_IRQFD:
2405 case KVM_CAP_IRQFD_RESAMPLE: 2496 case KVM_CAP_IRQFD_RESAMPLE:
2406#endif 2497#endif
2407 case KVM_CAP_CHECK_EXTENSION_VM: 2498 case KVM_CAP_CHECK_EXTENSION_VM:
@@ -2589,7 +2680,7 @@ out:
2589 return r; 2680 return r;
2590} 2681}
2591 2682
2592#ifdef CONFIG_COMPAT 2683#ifdef CONFIG_KVM_COMPAT
2593struct compat_kvm_dirty_log { 2684struct compat_kvm_dirty_log {
2594 __u32 slot; 2685 __u32 slot;
2595 __u32 padding1; 2686 __u32 padding1;
@@ -2636,7 +2727,7 @@ out:
2636static struct file_operations kvm_vm_fops = { 2727static struct file_operations kvm_vm_fops = {
2637 .release = kvm_vm_release, 2728 .release = kvm_vm_release,
2638 .unlocked_ioctl = kvm_vm_ioctl, 2729 .unlocked_ioctl = kvm_vm_ioctl,
2639#ifdef CONFIG_COMPAT 2730#ifdef CONFIG_KVM_COMPAT
2640 .compat_ioctl = kvm_vm_compat_ioctl, 2731 .compat_ioctl = kvm_vm_compat_ioctl,
2641#endif 2732#endif
2642 .llseek = noop_llseek, 2733 .llseek = noop_llseek,
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-08 20:26:19 -0500