aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm/kvm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/kvm')
-rw-r--r--arch/arm/kvm/Kconfig56
-rw-r--r--arch/arm/kvm/Makefile21
-rw-r--r--arch/arm/kvm/arm.c1015
-rw-r--r--arch/arm/kvm/coproc.c1046
-rw-r--r--arch/arm/kvm/coproc.h153
-rw-r--r--arch/arm/kvm/coproc_a15.c162
-rw-r--r--arch/arm/kvm/emulate.c373
-rw-r--r--arch/arm/kvm/guest.c222
-rw-r--r--arch/arm/kvm/init.S114
-rw-r--r--arch/arm/kvm/interrupts.S478
-rw-r--r--arch/arm/kvm/interrupts_head.S441
-rw-r--r--arch/arm/kvm/mmio.c153
-rw-r--r--arch/arm/kvm/mmu.c787
-rw-r--r--arch/arm/kvm/psci.c108
-rw-r--r--arch/arm/kvm/reset.c74
-rw-r--r--arch/arm/kvm/trace.h235
16 files changed, 5438 insertions, 0 deletions
diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig
new file mode 100644
index 000000000000..05227cb57a7b
--- /dev/null
+++ b/arch/arm/kvm/Kconfig
@@ -0,0 +1,56 @@
1#
2# KVM configuration
3#
4
5source "virt/kvm/Kconfig"
6
7menuconfig VIRTUALIZATION
8 bool "Virtualization"
9 ---help---
10 Say Y here to get to see options for using your Linux host to run
11 other operating systems inside virtual machines (guests).
12 This option alone does not add any kernel code.
13
14 If you say N, all options in this submenu will be skipped and
15 disabled.
16
17if VIRTUALIZATION
18
19config KVM
20 bool "Kernel-based Virtual Machine (KVM) support"
21 select PREEMPT_NOTIFIERS
22 select ANON_INODES
23 select KVM_MMIO
24 select KVM_ARM_HOST
25 depends on ARM_VIRT_EXT && ARM_LPAE
26 ---help---
27 Support hosting virtualized guest machines. You will also
28 need to select one or more of the processor modules below.
29
30 This module provides access to the hardware capabilities through
31 a character device node named /dev/kvm.
32
33 If unsure, say N.
34
35config KVM_ARM_HOST
36 bool "KVM host support for ARM cpus."
37 depends on KVM
38 depends on MMU
39 select MMU_NOTIFIER
40 ---help---
41 Provides host support for ARM processors.
42
43config KVM_ARM_MAX_VCPUS
44 int "Number maximum supported virtual CPUs per VM"
45 depends on KVM_ARM_HOST
46 default 4
47 help
48 Static number of max supported virtual CPUs per VM.
49
50 If you choose a high number, the vcpu structures will be quite
51 large, so only choose a reasonable number that you expect to
52 actually use.
53
54source drivers/virtio/Kconfig
55
56endif # VIRTUALIZATION
diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile
new file mode 100644
index 000000000000..ea27987bd07f
--- /dev/null
+++ b/arch/arm/kvm/Makefile
@@ -0,0 +1,21 @@
1#
2# Makefile for Kernel-based Virtual Machine module
3#
4
5plus_virt := $(call as-instr,.arch_extension virt,+virt)
6ifeq ($(plus_virt),+virt)
7 plus_virt_def := -DREQUIRES_VIRT=1
8endif
9
10ccflags-y += -Ivirt/kvm -Iarch/arm/kvm
11CFLAGS_arm.o := -I. $(plus_virt_def)
12CFLAGS_mmu.o := -I.
13
14AFLAGS_init.o := -Wa,-march=armv7-a$(plus_virt)
15AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt)
16
17kvm-arm-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o)
18
19obj-y += kvm-arm.o init.o interrupts.o
20obj-y += arm.o guest.o mmu.o emulate.o reset.o
21obj-y += coproc.o coproc_a15.o mmio.o psci.o
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
new file mode 100644
index 000000000000..2d30e3afdaf9
--- /dev/null
+++ b/arch/arm/kvm/arm.c
@@ -0,0 +1,1015 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19#include <linux/errno.h>
20#include <linux/err.h>
21#include <linux/kvm_host.h>
22#include <linux/module.h>
23#include <linux/vmalloc.h>
24#include <linux/fs.h>
25#include <linux/mman.h>
26#include <linux/sched.h>
27#include <linux/kvm.h>
28#include <trace/events/kvm.h>
29
30#define CREATE_TRACE_POINTS
31#include "trace.h"
32
33#include <asm/unified.h>
34#include <asm/uaccess.h>
35#include <asm/ptrace.h>
36#include <asm/mman.h>
37#include <asm/cputype.h>
38#include <asm/tlbflush.h>
39#include <asm/cacheflush.h>
40#include <asm/virt.h>
41#include <asm/kvm_arm.h>
42#include <asm/kvm_asm.h>
43#include <asm/kvm_mmu.h>
44#include <asm/kvm_emulate.h>
45#include <asm/kvm_coproc.h>
46#include <asm/kvm_psci.h>
47#include <asm/opcodes.h>
48
49#ifdef REQUIRES_VIRT
50__asm__(".arch_extension virt");
51#endif
52
53static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
54static struct vfp_hard_struct __percpu *kvm_host_vfp_state;
55static unsigned long hyp_default_vectors;
56
57/* The VMID used in the VTTBR */
58static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
59static u8 kvm_next_vmid;
60static DEFINE_SPINLOCK(kvm_vmid_lock);
61
62int kvm_arch_hardware_enable(void *garbage)
63{
64 return 0;
65}
66
67int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
68{
69 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
70}
71
72void kvm_arch_hardware_disable(void *garbage)
73{
74}
75
76int kvm_arch_hardware_setup(void)
77{
78 return 0;
79}
80
81void kvm_arch_hardware_unsetup(void)
82{
83}
84
85void kvm_arch_check_processor_compat(void *rtn)
86{
87 *(int *)rtn = 0;
88}
89
90void kvm_arch_sync_events(struct kvm *kvm)
91{
92}
93
94/**
95 * kvm_arch_init_vm - initializes a VM data structure
96 * @kvm: pointer to the KVM struct
97 */
98int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
99{
100 int ret = 0;
101
102 if (type)
103 return -EINVAL;
104
105 ret = kvm_alloc_stage2_pgd(kvm);
106 if (ret)
107 goto out_fail_alloc;
108
109 ret = create_hyp_mappings(kvm, kvm + 1);
110 if (ret)
111 goto out_free_stage2_pgd;
112
113 /* Mark the initial VMID generation invalid */
114 kvm->arch.vmid_gen = 0;
115
116 return ret;
117out_free_stage2_pgd:
118 kvm_free_stage2_pgd(kvm);
119out_fail_alloc:
120 return ret;
121}
122
123int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
124{
125 return VM_FAULT_SIGBUS;
126}
127
128void kvm_arch_free_memslot(struct kvm_memory_slot *free,
129 struct kvm_memory_slot *dont)
130{
131}
132
133int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
134{
135 return 0;
136}
137
138/**
139 * kvm_arch_destroy_vm - destroy the VM data structure
140 * @kvm: pointer to the KVM struct
141 */
142void kvm_arch_destroy_vm(struct kvm *kvm)
143{
144 int i;
145
146 kvm_free_stage2_pgd(kvm);
147
148 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
149 if (kvm->vcpus[i]) {
150 kvm_arch_vcpu_free(kvm->vcpus[i]);
151 kvm->vcpus[i] = NULL;
152 }
153 }
154}
155
156int kvm_dev_ioctl_check_extension(long ext)
157{
158 int r;
159 switch (ext) {
160 case KVM_CAP_USER_MEMORY:
161 case KVM_CAP_SYNC_MMU:
162 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
163 case KVM_CAP_ONE_REG:
164 case KVM_CAP_ARM_PSCI:
165 r = 1;
166 break;
167 case KVM_CAP_COALESCED_MMIO:
168 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
169 break;
170 case KVM_CAP_NR_VCPUS:
171 r = num_online_cpus();
172 break;
173 case KVM_CAP_MAX_VCPUS:
174 r = KVM_MAX_VCPUS;
175 break;
176 default:
177 r = 0;
178 break;
179 }
180 return r;
181}
182
183long kvm_arch_dev_ioctl(struct file *filp,
184 unsigned int ioctl, unsigned long arg)
185{
186 return -EINVAL;
187}
188
189int kvm_arch_set_memory_region(struct kvm *kvm,
190 struct kvm_userspace_memory_region *mem,
191 struct kvm_memory_slot old,
192 int user_alloc)
193{
194 return 0;
195}
196
197int kvm_arch_prepare_memory_region(struct kvm *kvm,
198 struct kvm_memory_slot *memslot,
199 struct kvm_memory_slot old,
200 struct kvm_userspace_memory_region *mem,
201 int user_alloc)
202{
203 return 0;
204}
205
206void kvm_arch_commit_memory_region(struct kvm *kvm,
207 struct kvm_userspace_memory_region *mem,
208 struct kvm_memory_slot old,
209 int user_alloc)
210{
211}
212
213void kvm_arch_flush_shadow_all(struct kvm *kvm)
214{
215}
216
217void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
218 struct kvm_memory_slot *slot)
219{
220}
221
222struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
223{
224 int err;
225 struct kvm_vcpu *vcpu;
226
227 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
228 if (!vcpu) {
229 err = -ENOMEM;
230 goto out;
231 }
232
233 err = kvm_vcpu_init(vcpu, kvm, id);
234 if (err)
235 goto free_vcpu;
236
237 err = create_hyp_mappings(vcpu, vcpu + 1);
238 if (err)
239 goto vcpu_uninit;
240
241 return vcpu;
242vcpu_uninit:
243 kvm_vcpu_uninit(vcpu);
244free_vcpu:
245 kmem_cache_free(kvm_vcpu_cache, vcpu);
246out:
247 return ERR_PTR(err);
248}
249
250int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
251{
252 return 0;
253}
254
255void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
256{
257 kvm_mmu_free_memory_caches(vcpu);
258 kmem_cache_free(kvm_vcpu_cache, vcpu);
259}
260
261void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
262{
263 kvm_arch_vcpu_free(vcpu);
264}
265
266int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
267{
268 return 0;
269}
270
271int __attribute_const__ kvm_target_cpu(void)
272{
273 unsigned long implementor = read_cpuid_implementor();
274 unsigned long part_number = read_cpuid_part_number();
275
276 if (implementor != ARM_CPU_IMP_ARM)
277 return -EINVAL;
278
279 switch (part_number) {
280 case ARM_CPU_PART_CORTEX_A15:
281 return KVM_ARM_TARGET_CORTEX_A15;
282 default:
283 return -EINVAL;
284 }
285}
286
287int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
288{
289 /* Force users to call KVM_ARM_VCPU_INIT */
290 vcpu->arch.target = -1;
291 return 0;
292}
293
294void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
295{
296}
297
298void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
299{
300 vcpu->cpu = cpu;
301 vcpu->arch.vfp_host = this_cpu_ptr(kvm_host_vfp_state);
302
303 /*
304 * Check whether this vcpu requires the cache to be flushed on
305 * this physical CPU. This is a consequence of doing dcache
306 * operations by set/way on this vcpu. We do it here to be in
307 * a non-preemptible section.
308 */
309 if (cpumask_test_and_clear_cpu(cpu, &vcpu->arch.require_dcache_flush))
310 flush_cache_all(); /* We'd really want v7_flush_dcache_all() */
311}
312
313void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
314{
315}
316
317int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
318 struct kvm_guest_debug *dbg)
319{
320 return -EINVAL;
321}
322
323
324int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
325 struct kvm_mp_state *mp_state)
326{
327 return -EINVAL;
328}
329
330int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
331 struct kvm_mp_state *mp_state)
332{
333 return -EINVAL;
334}
335
336/**
337 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
338 * @v: The VCPU pointer
339 *
340 * If the guest CPU is not waiting for interrupts or an interrupt line is
341 * asserted, the CPU is by definition runnable.
342 */
343int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
344{
345 return !!v->arch.irq_lines;
346}
347
348/* Just ensure a guest exit from a particular CPU */
349static void exit_vm_noop(void *info)
350{
351}
352
353void force_vm_exit(const cpumask_t *mask)
354{
355 smp_call_function_many(mask, exit_vm_noop, NULL, true);
356}
357
358/**
359 * need_new_vmid_gen - check that the VMID is still valid
360 * @kvm: The VM's VMID to checkt
361 *
362 * return true if there is a new generation of VMIDs being used
363 *
364 * The hardware supports only 256 values with the value zero reserved for the
365 * host, so we check if an assigned value belongs to a previous generation,
366 * which which requires us to assign a new value. If we're the first to use a
367 * VMID for the new generation, we must flush necessary caches and TLBs on all
368 * CPUs.
369 */
370static bool need_new_vmid_gen(struct kvm *kvm)
371{
372 return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
373}
374
375/**
376 * update_vttbr - Update the VTTBR with a valid VMID before the guest runs
377 * @kvm The guest that we are about to run
378 *
379 * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
380 * VM has a valid VMID, otherwise assigns a new one and flushes corresponding
381 * caches and TLBs.
382 */
383static void update_vttbr(struct kvm *kvm)
384{
385 phys_addr_t pgd_phys;
386 u64 vmid;
387
388 if (!need_new_vmid_gen(kvm))
389 return;
390
391 spin_lock(&kvm_vmid_lock);
392
393 /*
394 * We need to re-check the vmid_gen here to ensure that if another vcpu
395 * already allocated a valid vmid for this vm, then this vcpu should
396 * use the same vmid.
397 */
398 if (!need_new_vmid_gen(kvm)) {
399 spin_unlock(&kvm_vmid_lock);
400 return;
401 }
402
403 /* First user of a new VMID generation? */
404 if (unlikely(kvm_next_vmid == 0)) {
405 atomic64_inc(&kvm_vmid_gen);
406 kvm_next_vmid = 1;
407
408 /*
409 * On SMP we know no other CPUs can use this CPU's or each
410 * other's VMID after force_vm_exit returns since the
411 * kvm_vmid_lock blocks them from reentry to the guest.
412 */
413 force_vm_exit(cpu_all_mask);
414 /*
415 * Now broadcast TLB + ICACHE invalidation over the inner
416 * shareable domain to make sure all data structures are
417 * clean.
418 */
419 kvm_call_hyp(__kvm_flush_vm_context);
420 }
421
422 kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
423 kvm->arch.vmid = kvm_next_vmid;
424 kvm_next_vmid++;
425
426 /* update vttbr to be used with the new vmid */
427 pgd_phys = virt_to_phys(kvm->arch.pgd);
428 vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK;
429 kvm->arch.vttbr = pgd_phys & VTTBR_BADDR_MASK;
430 kvm->arch.vttbr |= vmid;
431
432 spin_unlock(&kvm_vmid_lock);
433}
434
435static int handle_svc_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run)
436{
437 /* SVC called from Hyp mode should never get here */
438 kvm_debug("SVC called from Hyp mode shouldn't go here\n");
439 BUG();
440 return -EINVAL; /* Squash warning */
441}
442
443static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
444{
445 trace_kvm_hvc(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
446 vcpu->arch.hsr & HSR_HVC_IMM_MASK);
447
448 if (kvm_psci_call(vcpu))
449 return 1;
450
451 kvm_inject_undefined(vcpu);
452 return 1;
453}
454
455static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
456{
457 if (kvm_psci_call(vcpu))
458 return 1;
459
460 kvm_inject_undefined(vcpu);
461 return 1;
462}
463
464static int handle_pabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run)
465{
466 /* The hypervisor should never cause aborts */
467 kvm_err("Prefetch Abort taken from Hyp mode at %#08x (HSR: %#08x)\n",
468 vcpu->arch.hxfar, vcpu->arch.hsr);
469 return -EFAULT;
470}
471
472static int handle_dabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run)
473{
474 /* This is either an error in the ws. code or an external abort */
475 kvm_err("Data Abort taken from Hyp mode at %#08x (HSR: %#08x)\n",
476 vcpu->arch.hxfar, vcpu->arch.hsr);
477 return -EFAULT;
478}
479
480typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *);
481static exit_handle_fn arm_exit_handlers[] = {
482 [HSR_EC_WFI] = kvm_handle_wfi,
483 [HSR_EC_CP15_32] = kvm_handle_cp15_32,
484 [HSR_EC_CP15_64] = kvm_handle_cp15_64,
485 [HSR_EC_CP14_MR] = kvm_handle_cp14_access,
486 [HSR_EC_CP14_LS] = kvm_handle_cp14_load_store,
487 [HSR_EC_CP14_64] = kvm_handle_cp14_access,
488 [HSR_EC_CP_0_13] = kvm_handle_cp_0_13_access,
489 [HSR_EC_CP10_ID] = kvm_handle_cp10_id,
490 [HSR_EC_SVC_HYP] = handle_svc_hyp,
491 [HSR_EC_HVC] = handle_hvc,
492 [HSR_EC_SMC] = handle_smc,
493 [HSR_EC_IABT] = kvm_handle_guest_abort,
494 [HSR_EC_IABT_HYP] = handle_pabt_hyp,
495 [HSR_EC_DABT] = kvm_handle_guest_abort,
496 [HSR_EC_DABT_HYP] = handle_dabt_hyp,
497};
498
499/*
500 * A conditional instruction is allowed to trap, even though it
501 * wouldn't be executed. So let's re-implement the hardware, in
502 * software!
503 */
504static bool kvm_condition_valid(struct kvm_vcpu *vcpu)
505{
506 unsigned long cpsr, cond, insn;
507
508 /*
509 * Exception Code 0 can only happen if we set HCR.TGE to 1, to
510 * catch undefined instructions, and then we won't get past
511 * the arm_exit_handlers test anyway.
512 */
513 BUG_ON(((vcpu->arch.hsr & HSR_EC) >> HSR_EC_SHIFT) == 0);
514
515 /* Top two bits non-zero? Unconditional. */
516 if (vcpu->arch.hsr >> 30)
517 return true;
518
519 cpsr = *vcpu_cpsr(vcpu);
520
521 /* Is condition field valid? */
522 if ((vcpu->arch.hsr & HSR_CV) >> HSR_CV_SHIFT)
523 cond = (vcpu->arch.hsr & HSR_COND) >> HSR_COND_SHIFT;
524 else {
525 /* This can happen in Thumb mode: examine IT state. */
526 unsigned long it;
527
528 it = ((cpsr >> 8) & 0xFC) | ((cpsr >> 25) & 0x3);
529
530 /* it == 0 => unconditional. */
531 if (it == 0)
532 return true;
533
534 /* The cond for this insn works out as the top 4 bits. */
535 cond = (it >> 4);
536 }
537
538 /* Shift makes it look like an ARM-mode instruction */
539 insn = cond << 28;
540 return arm_check_condition(insn, cpsr) != ARM_OPCODE_CONDTEST_FAIL;
541}
542
543/*
544 * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on
545 * proper exit to QEMU.
546 */
547static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
548 int exception_index)
549{
550 unsigned long hsr_ec;
551
552 switch (exception_index) {
553 case ARM_EXCEPTION_IRQ:
554 return 1;
555 case ARM_EXCEPTION_UNDEFINED:
556 kvm_err("Undefined exception in Hyp mode at: %#08x\n",
557 vcpu->arch.hyp_pc);
558 BUG();
559 panic("KVM: Hypervisor undefined exception!\n");
560 case ARM_EXCEPTION_DATA_ABORT:
561 case ARM_EXCEPTION_PREF_ABORT:
562 case ARM_EXCEPTION_HVC:
563 hsr_ec = (vcpu->arch.hsr & HSR_EC) >> HSR_EC_SHIFT;
564
565 if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers)
566 || !arm_exit_handlers[hsr_ec]) {
567 kvm_err("Unkown exception class: %#08lx, "
568 "hsr: %#08x\n", hsr_ec,
569 (unsigned int)vcpu->arch.hsr);
570 BUG();
571 }
572
573 /*
574 * See ARM ARM B1.14.1: "Hyp traps on instructions
575 * that fail their condition code check"
576 */
577 if (!kvm_condition_valid(vcpu)) {
578 bool is_wide = vcpu->arch.hsr & HSR_IL;
579 kvm_skip_instr(vcpu, is_wide);
580 return 1;
581 }
582
583 return arm_exit_handlers[hsr_ec](vcpu, run);
584 default:
585 kvm_pr_unimpl("Unsupported exception type: %d",
586 exception_index);
587 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
588 return 0;
589 }
590}
591
592static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
593{
594 if (likely(vcpu->arch.has_run_once))
595 return 0;
596
597 vcpu->arch.has_run_once = true;
598
599 /*
600 * Handle the "start in power-off" case by calling into the
601 * PSCI code.
602 */
603 if (test_and_clear_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features)) {
604 *vcpu_reg(vcpu, 0) = KVM_PSCI_FN_CPU_OFF;
605 kvm_psci_call(vcpu);
606 }
607
608 return 0;
609}
610
611static void vcpu_pause(struct kvm_vcpu *vcpu)
612{
613 wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
614
615 wait_event_interruptible(*wq, !vcpu->arch.pause);
616}
617
618/**
619 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
620 * @vcpu: The VCPU pointer
621 * @run: The kvm_run structure pointer used for userspace state exchange
622 *
623 * This function is called through the VCPU_RUN ioctl called from user space. It
624 * will execute VM code in a loop until the time slice for the process is used
625 * or some emulation is needed from user space in which case the function will
626 * return with return value 0 and with the kvm_run structure filled in with the
627 * required data for the requested emulation.
628 */
629int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
630{
631 int ret;
632 sigset_t sigsaved;
633
634 /* Make sure they initialize the vcpu with KVM_ARM_VCPU_INIT */
635 if (unlikely(vcpu->arch.target < 0))
636 return -ENOEXEC;
637
638 ret = kvm_vcpu_first_run_init(vcpu);
639 if (ret)
640 return ret;
641
642 if (run->exit_reason == KVM_EXIT_MMIO) {
643 ret = kvm_handle_mmio_return(vcpu, vcpu->run);
644 if (ret)
645 return ret;
646 }
647
648 if (vcpu->sigset_active)
649 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
650
651 ret = 1;
652 run->exit_reason = KVM_EXIT_UNKNOWN;
653 while (ret > 0) {
654 /*
655 * Check conditions before entering the guest
656 */
657 cond_resched();
658
659 update_vttbr(vcpu->kvm);
660
661 if (vcpu->arch.pause)
662 vcpu_pause(vcpu);
663
664 local_irq_disable();
665
666 /*
667 * Re-check atomic conditions
668 */
669 if (signal_pending(current)) {
670 ret = -EINTR;
671 run->exit_reason = KVM_EXIT_INTR;
672 }
673
674 if (ret <= 0 || need_new_vmid_gen(vcpu->kvm)) {
675 local_irq_enable();
676 continue;
677 }
678
679 /**************************************************************
680 * Enter the guest
681 */
682 trace_kvm_entry(*vcpu_pc(vcpu));
683 kvm_guest_enter();
684 vcpu->mode = IN_GUEST_MODE;
685
686 ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
687
688 vcpu->mode = OUTSIDE_GUEST_MODE;
689 vcpu->arch.last_pcpu = smp_processor_id();
690 kvm_guest_exit();
691 trace_kvm_exit(*vcpu_pc(vcpu));
692 /*
693 * We may have taken a host interrupt in HYP mode (ie
694 * while executing the guest). This interrupt is still
695 * pending, as we haven't serviced it yet!
696 *
697 * We're now back in SVC mode, with interrupts
698 * disabled. Enabling the interrupts now will have
699 * the effect of taking the interrupt again, in SVC
700 * mode this time.
701 */
702 local_irq_enable();
703
704 /*
705 * Back from guest
706 *************************************************************/
707
708 ret = handle_exit(vcpu, run, ret);
709 }
710
711 if (vcpu->sigset_active)
712 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
713 return ret;
714}
715
716static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
717{
718 int bit_index;
719 bool set;
720 unsigned long *ptr;
721
722 if (number == KVM_ARM_IRQ_CPU_IRQ)
723 bit_index = __ffs(HCR_VI);
724 else /* KVM_ARM_IRQ_CPU_FIQ */
725 bit_index = __ffs(HCR_VF);
726
727 ptr = (unsigned long *)&vcpu->arch.irq_lines;
728 if (level)
729 set = test_and_set_bit(bit_index, ptr);
730 else
731 set = test_and_clear_bit(bit_index, ptr);
732
733 /*
734 * If we didn't change anything, no need to wake up or kick other CPUs
735 */
736 if (set == level)
737 return 0;
738
739 /*
740 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
741 * trigger a world-switch round on the running physical CPU to set the
742 * virtual IRQ/FIQ fields in the HCR appropriately.
743 */
744 kvm_vcpu_kick(vcpu);
745
746 return 0;
747}
748
749int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level)
750{
751 u32 irq = irq_level->irq;
752 unsigned int irq_type, vcpu_idx, irq_num;
753 int nrcpus = atomic_read(&kvm->online_vcpus);
754 struct kvm_vcpu *vcpu = NULL;
755 bool level = irq_level->level;
756
757 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
758 vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
759 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
760
761 trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
762
763 if (irq_type != KVM_ARM_IRQ_TYPE_CPU)
764 return -EINVAL;
765
766 if (vcpu_idx >= nrcpus)
767 return -EINVAL;
768
769 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
770 if (!vcpu)
771 return -EINVAL;
772
773 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
774 return -EINVAL;
775
776 return vcpu_interrupt_line(vcpu, irq_num, level);
777}
778
779long kvm_arch_vcpu_ioctl(struct file *filp,
780 unsigned int ioctl, unsigned long arg)
781{
782 struct kvm_vcpu *vcpu = filp->private_data;
783 void __user *argp = (void __user *)arg;
784
785 switch (ioctl) {
786 case KVM_ARM_VCPU_INIT: {
787 struct kvm_vcpu_init init;
788
789 if (copy_from_user(&init, argp, sizeof(init)))
790 return -EFAULT;
791
792 return kvm_vcpu_set_target(vcpu, &init);
793
794 }
795 case KVM_SET_ONE_REG:
796 case KVM_GET_ONE_REG: {
797 struct kvm_one_reg reg;
798 if (copy_from_user(&reg, argp, sizeof(reg)))
799 return -EFAULT;
800 if (ioctl == KVM_SET_ONE_REG)
801 return kvm_arm_set_reg(vcpu, &reg);
802 else
803 return kvm_arm_get_reg(vcpu, &reg);
804 }
805 case KVM_GET_REG_LIST: {
806 struct kvm_reg_list __user *user_list = argp;
807 struct kvm_reg_list reg_list;
808 unsigned n;
809
810 if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
811 return -EFAULT;
812 n = reg_list.n;
813 reg_list.n = kvm_arm_num_regs(vcpu);
814 if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
815 return -EFAULT;
816 if (n < reg_list.n)
817 return -E2BIG;
818 return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
819 }
820 default:
821 return -EINVAL;
822 }
823}
824
825int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
826{
827 return -EINVAL;
828}
829
830long kvm_arch_vm_ioctl(struct file *filp,
831 unsigned int ioctl, unsigned long arg)
832{
833 return -EINVAL;
834}
835
836static void cpu_init_hyp_mode(void *vector)
837{
838 unsigned long long pgd_ptr;
839 unsigned long pgd_low, pgd_high;
840 unsigned long hyp_stack_ptr;
841 unsigned long stack_page;
842 unsigned long vector_ptr;
843
844 /* Switch from the HYP stub to our own HYP init vector */
845 __hyp_set_vectors((unsigned long)vector);
846
847 pgd_ptr = (unsigned long long)kvm_mmu_get_httbr();
848 pgd_low = (pgd_ptr & ((1ULL << 32) - 1));
849 pgd_high = (pgd_ptr >> 32ULL);
850 stack_page = __get_cpu_var(kvm_arm_hyp_stack_page);
851 hyp_stack_ptr = stack_page + PAGE_SIZE;
852 vector_ptr = (unsigned long)__kvm_hyp_vector;
853
854 /*
855 * Call initialization code, and switch to the full blown
856 * HYP code. The init code doesn't need to preserve these registers as
857 * r1-r3 and r12 are already callee save according to the AAPCS.
858 * Note that we slightly misuse the prototype by casing the pgd_low to
859 * a void *.
860 */
861 kvm_call_hyp((void *)pgd_low, pgd_high, hyp_stack_ptr, vector_ptr);
862}
863
864/**
865 * Inits Hyp-mode on all online CPUs
866 */
867static int init_hyp_mode(void)
868{
869 phys_addr_t init_phys_addr;
870 int cpu;
871 int err = 0;
872
873 /*
874 * Allocate Hyp PGD and setup Hyp identity mapping
875 */
876 err = kvm_mmu_init();
877 if (err)
878 goto out_err;
879
880 /*
881 * It is probably enough to obtain the default on one
882 * CPU. It's unlikely to be different on the others.
883 */
884 hyp_default_vectors = __hyp_get_vectors();
885
886 /*
887 * Allocate stack pages for Hypervisor-mode
888 */
889 for_each_possible_cpu(cpu) {
890 unsigned long stack_page;
891
892 stack_page = __get_free_page(GFP_KERNEL);
893 if (!stack_page) {
894 err = -ENOMEM;
895 goto out_free_stack_pages;
896 }
897
898 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
899 }
900
901 /*
902 * Execute the init code on each CPU.
903 *
904 * Note: The stack is not mapped yet, so don't do anything else than
905 * initializing the hypervisor mode on each CPU using a local stack
906 * space for temporary storage.
907 */
908 init_phys_addr = virt_to_phys(__kvm_hyp_init);
909 for_each_online_cpu(cpu) {
910 smp_call_function_single(cpu, cpu_init_hyp_mode,
911 (void *)(long)init_phys_addr, 1);
912 }
913
914 /*
915 * Unmap the identity mapping
916 */
917 kvm_clear_hyp_idmap();
918
919 /*
920 * Map the Hyp-code called directly from the host
921 */
922 err = create_hyp_mappings(__kvm_hyp_code_start, __kvm_hyp_code_end);
923 if (err) {
924 kvm_err("Cannot map world-switch code\n");
925 goto out_free_mappings;
926 }
927
928 /*
929 * Map the Hyp stack pages
930 */
931 for_each_possible_cpu(cpu) {
932 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
933 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE);
934
935 if (err) {
936 kvm_err("Cannot map hyp stack\n");
937 goto out_free_mappings;
938 }
939 }
940
941 /*
942 * Map the host VFP structures
943 */
944 kvm_host_vfp_state = alloc_percpu(struct vfp_hard_struct);
945 if (!kvm_host_vfp_state) {
946 err = -ENOMEM;
947 kvm_err("Cannot allocate host VFP state\n");
948 goto out_free_mappings;
949 }
950
951 for_each_possible_cpu(cpu) {
952 struct vfp_hard_struct *vfp;
953
954 vfp = per_cpu_ptr(kvm_host_vfp_state, cpu);
955 err = create_hyp_mappings(vfp, vfp + 1);
956
957 if (err) {
958 kvm_err("Cannot map host VFP state: %d\n", err);
959 goto out_free_vfp;
960 }
961 }
962
963 kvm_info("Hyp mode initialized successfully\n");
964 return 0;
965out_free_vfp:
966 free_percpu(kvm_host_vfp_state);
967out_free_mappings:
968 free_hyp_pmds();
969out_free_stack_pages:
970 for_each_possible_cpu(cpu)
971 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
972out_err:
973 kvm_err("error initializing Hyp mode: %d\n", err);
974 return err;
975}
976
977/**
978 * Initialize Hyp-mode and memory mappings on all CPUs.
979 */
980int kvm_arch_init(void *opaque)
981{
982 int err;
983
984 if (!is_hyp_mode_available()) {
985 kvm_err("HYP mode not available\n");
986 return -ENODEV;
987 }
988
989 if (kvm_target_cpu() < 0) {
990 kvm_err("Target CPU not supported!\n");
991 return -ENODEV;
992 }
993
994 err = init_hyp_mode();
995 if (err)
996 goto out_err;
997
998 kvm_coproc_table_init();
999 return 0;
1000out_err:
1001 return err;
1002}
1003
1004/* NOP: Compiling as a module not supported */
1005void kvm_arch_exit(void)
1006{
1007}
1008
1009static int arm_init(void)
1010{
1011 int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1012 return rc;
1013}
1014
1015module_init(arm_init);
diff --git a/arch/arm/kvm/coproc.c b/arch/arm/kvm/coproc.c
new file mode 100644
index 000000000000..d782638c7ec0
--- /dev/null
+++ b/arch/arm/kvm/coproc.c
@@ -0,0 +1,1046 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Authors: Rusty Russell <rusty@rustcorp.com.au>
4 * Christoffer Dall <c.dall@virtualopensystems.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 */
19#include <linux/mm.h>
20#include <linux/kvm_host.h>
21#include <linux/uaccess.h>
22#include <asm/kvm_arm.h>
23#include <asm/kvm_host.h>
24#include <asm/kvm_emulate.h>
25#include <asm/kvm_coproc.h>
26#include <asm/cacheflush.h>
27#include <asm/cputype.h>
28#include <trace/events/kvm.h>
29#include <asm/vfp.h>
30#include "../vfp/vfpinstr.h"
31
32#include "trace.h"
33#include "coproc.h"
34
35
36/******************************************************************************
37 * Co-processor emulation
38 *****************************************************************************/
39
40/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
41static u32 cache_levels;
42
43/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
44#define CSSELR_MAX 12
45
46int kvm_handle_cp10_id(struct kvm_vcpu *vcpu, struct kvm_run *run)
47{
48 kvm_inject_undefined(vcpu);
49 return 1;
50}
51
52int kvm_handle_cp_0_13_access(struct kvm_vcpu *vcpu, struct kvm_run *run)
53{
54 /*
55 * We can get here, if the host has been built without VFPv3 support,
56 * but the guest attempted a floating point operation.
57 */
58 kvm_inject_undefined(vcpu);
59 return 1;
60}
61
62int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
63{
64 kvm_inject_undefined(vcpu);
65 return 1;
66}
67
68int kvm_handle_cp14_access(struct kvm_vcpu *vcpu, struct kvm_run *run)
69{
70 kvm_inject_undefined(vcpu);
71 return 1;
72}
73
74/* See note at ARM ARM B1.14.4 */
75static bool access_dcsw(struct kvm_vcpu *vcpu,
76 const struct coproc_params *p,
77 const struct coproc_reg *r)
78{
79 u32 val;
80 int cpu;
81
82 cpu = get_cpu();
83
84 if (!p->is_write)
85 return read_from_write_only(vcpu, p);
86
87 cpumask_setall(&vcpu->arch.require_dcache_flush);
88 cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush);
89
90 /* If we were already preempted, take the long way around */
91 if (cpu != vcpu->arch.last_pcpu) {
92 flush_cache_all();
93 goto done;
94 }
95
96 val = *vcpu_reg(vcpu, p->Rt1);
97
98 switch (p->CRm) {
99 case 6: /* Upgrade DCISW to DCCISW, as per HCR.SWIO */
100 case 14: /* DCCISW */
101 asm volatile("mcr p15, 0, %0, c7, c14, 2" : : "r" (val));
102 break;
103
104 case 10: /* DCCSW */
105 asm volatile("mcr p15, 0, %0, c7, c10, 2" : : "r" (val));
106 break;
107 }
108
109done:
110 put_cpu();
111
112 return true;
113}
114
115/*
116 * We could trap ID_DFR0 and tell the guest we don't support performance
117 * monitoring. Unfortunately the patch to make the kernel check ID_DFR0 was
118 * NAKed, so it will read the PMCR anyway.
119 *
120 * Therefore we tell the guest we have 0 counters. Unfortunately, we
121 * must always support PMCCNTR (the cycle counter): we just RAZ/WI for
122 * all PM registers, which doesn't crash the guest kernel at least.
123 */
124static bool pm_fake(struct kvm_vcpu *vcpu,
125 const struct coproc_params *p,
126 const struct coproc_reg *r)
127{
128 if (p->is_write)
129 return ignore_write(vcpu, p);
130 else
131 return read_zero(vcpu, p);
132}
133
134#define access_pmcr pm_fake
135#define access_pmcntenset pm_fake
136#define access_pmcntenclr pm_fake
137#define access_pmovsr pm_fake
138#define access_pmselr pm_fake
139#define access_pmceid0 pm_fake
140#define access_pmceid1 pm_fake
141#define access_pmccntr pm_fake
142#define access_pmxevtyper pm_fake
143#define access_pmxevcntr pm_fake
144#define access_pmuserenr pm_fake
145#define access_pmintenset pm_fake
146#define access_pmintenclr pm_fake
147
148/* Architected CP15 registers.
149 * Important: Must be sorted ascending by CRn, CRM, Op1, Op2
150 */
151static const struct coproc_reg cp15_regs[] = {
152 /* CSSELR: swapped by interrupt.S. */
153 { CRn( 0), CRm( 0), Op1( 2), Op2( 0), is32,
154 NULL, reset_unknown, c0_CSSELR },
155
156 /* TTBR0/TTBR1: swapped by interrupt.S. */
157 { CRm( 2), Op1( 0), is64, NULL, reset_unknown64, c2_TTBR0 },
158 { CRm( 2), Op1( 1), is64, NULL, reset_unknown64, c2_TTBR1 },
159
160 /* TTBCR: swapped by interrupt.S. */
161 { CRn( 2), CRm( 0), Op1( 0), Op2( 2), is32,
162 NULL, reset_val, c2_TTBCR, 0x00000000 },
163
164 /* DACR: swapped by interrupt.S. */
165 { CRn( 3), CRm( 0), Op1( 0), Op2( 0), is32,
166 NULL, reset_unknown, c3_DACR },
167
168 /* DFSR/IFSR/ADFSR/AIFSR: swapped by interrupt.S. */
169 { CRn( 5), CRm( 0), Op1( 0), Op2( 0), is32,
170 NULL, reset_unknown, c5_DFSR },
171 { CRn( 5), CRm( 0), Op1( 0), Op2( 1), is32,
172 NULL, reset_unknown, c5_IFSR },
173 { CRn( 5), CRm( 1), Op1( 0), Op2( 0), is32,
174 NULL, reset_unknown, c5_ADFSR },
175 { CRn( 5), CRm( 1), Op1( 0), Op2( 1), is32,
176 NULL, reset_unknown, c5_AIFSR },
177
178 /* DFAR/IFAR: swapped by interrupt.S. */
179 { CRn( 6), CRm( 0), Op1( 0), Op2( 0), is32,
180 NULL, reset_unknown, c6_DFAR },
181 { CRn( 6), CRm( 0), Op1( 0), Op2( 2), is32,
182 NULL, reset_unknown, c6_IFAR },
183 /*
184 * DC{C,I,CI}SW operations:
185 */
186 { CRn( 7), CRm( 6), Op1( 0), Op2( 2), is32, access_dcsw},
187 { CRn( 7), CRm(10), Op1( 0), Op2( 2), is32, access_dcsw},
188 { CRn( 7), CRm(14), Op1( 0), Op2( 2), is32, access_dcsw},
189 /*
190 * Dummy performance monitor implementation.
191 */
192 { CRn( 9), CRm(12), Op1( 0), Op2( 0), is32, access_pmcr},
193 { CRn( 9), CRm(12), Op1( 0), Op2( 1), is32, access_pmcntenset},
194 { CRn( 9), CRm(12), Op1( 0), Op2( 2), is32, access_pmcntenclr},
195 { CRn( 9), CRm(12), Op1( 0), Op2( 3), is32, access_pmovsr},
196 { CRn( 9), CRm(12), Op1( 0), Op2( 5), is32, access_pmselr},
197 { CRn( 9), CRm(12), Op1( 0), Op2( 6), is32, access_pmceid0},
198 { CRn( 9), CRm(12), Op1( 0), Op2( 7), is32, access_pmceid1},
199 { CRn( 9), CRm(13), Op1( 0), Op2( 0), is32, access_pmccntr},
200 { CRn( 9), CRm(13), Op1( 0), Op2( 1), is32, access_pmxevtyper},
201 { CRn( 9), CRm(13), Op1( 0), Op2( 2), is32, access_pmxevcntr},
202 { CRn( 9), CRm(14), Op1( 0), Op2( 0), is32, access_pmuserenr},
203 { CRn( 9), CRm(14), Op1( 0), Op2( 1), is32, access_pmintenset},
204 { CRn( 9), CRm(14), Op1( 0), Op2( 2), is32, access_pmintenclr},
205
206 /* PRRR/NMRR (aka MAIR0/MAIR1): swapped by interrupt.S. */
207 { CRn(10), CRm( 2), Op1( 0), Op2( 0), is32,
208 NULL, reset_unknown, c10_PRRR},
209 { CRn(10), CRm( 2), Op1( 0), Op2( 1), is32,
210 NULL, reset_unknown, c10_NMRR},
211
212 /* VBAR: swapped by interrupt.S. */
213 { CRn(12), CRm( 0), Op1( 0), Op2( 0), is32,
214 NULL, reset_val, c12_VBAR, 0x00000000 },
215
216 /* CONTEXTIDR/TPIDRURW/TPIDRURO/TPIDRPRW: swapped by interrupt.S. */
217 { CRn(13), CRm( 0), Op1( 0), Op2( 1), is32,
218 NULL, reset_val, c13_CID, 0x00000000 },
219 { CRn(13), CRm( 0), Op1( 0), Op2( 2), is32,
220 NULL, reset_unknown, c13_TID_URW },
221 { CRn(13), CRm( 0), Op1( 0), Op2( 3), is32,
222 NULL, reset_unknown, c13_TID_URO },
223 { CRn(13), CRm( 0), Op1( 0), Op2( 4), is32,
224 NULL, reset_unknown, c13_TID_PRIV },
225};
226
227/* Target specific emulation tables */
228static struct kvm_coproc_target_table *target_tables[KVM_ARM_NUM_TARGETS];
229
230void kvm_register_target_coproc_table(struct kvm_coproc_target_table *table)
231{
232 target_tables[table->target] = table;
233}
234
235/* Get specific register table for this target. */
236static const struct coproc_reg *get_target_table(unsigned target, size_t *num)
237{
238 struct kvm_coproc_target_table *table;
239
240 table = target_tables[target];
241 *num = table->num;
242 return table->table;
243}
244
245static const struct coproc_reg *find_reg(const struct coproc_params *params,
246 const struct coproc_reg table[],
247 unsigned int num)
248{
249 unsigned int i;
250
251 for (i = 0; i < num; i++) {
252 const struct coproc_reg *r = &table[i];
253
254 if (params->is_64bit != r->is_64)
255 continue;
256 if (params->CRn != r->CRn)
257 continue;
258 if (params->CRm != r->CRm)
259 continue;
260 if (params->Op1 != r->Op1)
261 continue;
262 if (params->Op2 != r->Op2)
263 continue;
264
265 return r;
266 }
267 return NULL;
268}
269
270static int emulate_cp15(struct kvm_vcpu *vcpu,
271 const struct coproc_params *params)
272{
273 size_t num;
274 const struct coproc_reg *table, *r;
275
276 trace_kvm_emulate_cp15_imp(params->Op1, params->Rt1, params->CRn,
277 params->CRm, params->Op2, params->is_write);
278
279 table = get_target_table(vcpu->arch.target, &num);
280
281 /* Search target-specific then generic table. */
282 r = find_reg(params, table, num);
283 if (!r)
284 r = find_reg(params, cp15_regs, ARRAY_SIZE(cp15_regs));
285
286 if (likely(r)) {
287 /* If we don't have an accessor, we should never get here! */
288 BUG_ON(!r->access);
289
290 if (likely(r->access(vcpu, params, r))) {
291 /* Skip instruction, since it was emulated */
292 kvm_skip_instr(vcpu, (vcpu->arch.hsr >> 25) & 1);
293 return 1;
294 }
295 /* If access function fails, it should complain. */
296 } else {
297 kvm_err("Unsupported guest CP15 access at: %08x\n",
298 *vcpu_pc(vcpu));
299 print_cp_instr(params);
300 }
301 kvm_inject_undefined(vcpu);
302 return 1;
303}
304
305/**
306 * kvm_handle_cp15_64 -- handles a mrrc/mcrr trap on a guest CP15 access
307 * @vcpu: The VCPU pointer
308 * @run: The kvm_run struct
309 */
310int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
311{
312 struct coproc_params params;
313
314 params.CRm = (vcpu->arch.hsr >> 1) & 0xf;
315 params.Rt1 = (vcpu->arch.hsr >> 5) & 0xf;
316 params.is_write = ((vcpu->arch.hsr & 1) == 0);
317 params.is_64bit = true;
318
319 params.Op1 = (vcpu->arch.hsr >> 16) & 0xf;
320 params.Op2 = 0;
321 params.Rt2 = (vcpu->arch.hsr >> 10) & 0xf;
322 params.CRn = 0;
323
324 return emulate_cp15(vcpu, &params);
325}
326
327static void reset_coproc_regs(struct kvm_vcpu *vcpu,
328 const struct coproc_reg *table, size_t num)
329{
330 unsigned long i;
331
332 for (i = 0; i < num; i++)
333 if (table[i].reset)
334 table[i].reset(vcpu, &table[i]);
335}
336
337/**
338 * kvm_handle_cp15_32 -- handles a mrc/mcr trap on a guest CP15 access
339 * @vcpu: The VCPU pointer
340 * @run: The kvm_run struct
341 */
342int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
343{
344 struct coproc_params params;
345
346 params.CRm = (vcpu->arch.hsr >> 1) & 0xf;
347 params.Rt1 = (vcpu->arch.hsr >> 5) & 0xf;
348 params.is_write = ((vcpu->arch.hsr & 1) == 0);
349 params.is_64bit = false;
350
351 params.CRn = (vcpu->arch.hsr >> 10) & 0xf;
352 params.Op1 = (vcpu->arch.hsr >> 14) & 0x7;
353 params.Op2 = (vcpu->arch.hsr >> 17) & 0x7;
354 params.Rt2 = 0;
355
356 return emulate_cp15(vcpu, &params);
357}
358
359/******************************************************************************
360 * Userspace API
361 *****************************************************************************/
362
363static bool index_to_params(u64 id, struct coproc_params *params)
364{
365 switch (id & KVM_REG_SIZE_MASK) {
366 case KVM_REG_SIZE_U32:
367 /* Any unused index bits means it's not valid. */
368 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
369 | KVM_REG_ARM_COPROC_MASK
370 | KVM_REG_ARM_32_CRN_MASK
371 | KVM_REG_ARM_CRM_MASK
372 | KVM_REG_ARM_OPC1_MASK
373 | KVM_REG_ARM_32_OPC2_MASK))
374 return false;
375
376 params->is_64bit = false;
377 params->CRn = ((id & KVM_REG_ARM_32_CRN_MASK)
378 >> KVM_REG_ARM_32_CRN_SHIFT);
379 params->CRm = ((id & KVM_REG_ARM_CRM_MASK)
380 >> KVM_REG_ARM_CRM_SHIFT);
381 params->Op1 = ((id & KVM_REG_ARM_OPC1_MASK)
382 >> KVM_REG_ARM_OPC1_SHIFT);
383 params->Op2 = ((id & KVM_REG_ARM_32_OPC2_MASK)
384 >> KVM_REG_ARM_32_OPC2_SHIFT);
385 return true;
386 case KVM_REG_SIZE_U64:
387 /* Any unused index bits means it's not valid. */
388 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
389 | KVM_REG_ARM_COPROC_MASK
390 | KVM_REG_ARM_CRM_MASK
391 | KVM_REG_ARM_OPC1_MASK))
392 return false;
393 params->is_64bit = true;
394 params->CRm = ((id & KVM_REG_ARM_CRM_MASK)
395 >> KVM_REG_ARM_CRM_SHIFT);
396 params->Op1 = ((id & KVM_REG_ARM_OPC1_MASK)
397 >> KVM_REG_ARM_OPC1_SHIFT);
398 params->Op2 = 0;
399 params->CRn = 0;
400 return true;
401 default:
402 return false;
403 }
404}
405
406/* Decode an index value, and find the cp15 coproc_reg entry. */
407static const struct coproc_reg *index_to_coproc_reg(struct kvm_vcpu *vcpu,
408 u64 id)
409{
410 size_t num;
411 const struct coproc_reg *table, *r;
412 struct coproc_params params;
413
414 /* We only do cp15 for now. */
415 if ((id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT != 15)
416 return NULL;
417
418 if (!index_to_params(id, &params))
419 return NULL;
420
421 table = get_target_table(vcpu->arch.target, &num);
422 r = find_reg(&params, table, num);
423 if (!r)
424 r = find_reg(&params, cp15_regs, ARRAY_SIZE(cp15_regs));
425
426 /* Not saved in the cp15 array? */
427 if (r && !r->reg)
428 r = NULL;
429
430 return r;
431}
432
433/*
434 * These are the invariant cp15 registers: we let the guest see the host
435 * versions of these, so they're part of the guest state.
436 *
437 * A future CPU may provide a mechanism to present different values to
438 * the guest, or a future kvm may trap them.
439 */
440/* Unfortunately, there's no register-argument for mrc, so generate. */
441#define FUNCTION_FOR32(crn, crm, op1, op2, name) \
442 static void get_##name(struct kvm_vcpu *v, \
443 const struct coproc_reg *r) \
444 { \
445 u32 val; \
446 \
447 asm volatile("mrc p15, " __stringify(op1) \
448 ", %0, c" __stringify(crn) \
449 ", c" __stringify(crm) \
450 ", " __stringify(op2) "\n" : "=r" (val)); \
451 ((struct coproc_reg *)r)->val = val; \
452 }
453
454FUNCTION_FOR32(0, 0, 0, 0, MIDR)
455FUNCTION_FOR32(0, 0, 0, 1, CTR)
456FUNCTION_FOR32(0, 0, 0, 2, TCMTR)
457FUNCTION_FOR32(0, 0, 0, 3, TLBTR)
458FUNCTION_FOR32(0, 0, 0, 6, REVIDR)
459FUNCTION_FOR32(0, 1, 0, 0, ID_PFR0)
460FUNCTION_FOR32(0, 1, 0, 1, ID_PFR1)
461FUNCTION_FOR32(0, 1, 0, 2, ID_DFR0)
462FUNCTION_FOR32(0, 1, 0, 3, ID_AFR0)
463FUNCTION_FOR32(0, 1, 0, 4, ID_MMFR0)
464FUNCTION_FOR32(0, 1, 0, 5, ID_MMFR1)
465FUNCTION_FOR32(0, 1, 0, 6, ID_MMFR2)
466FUNCTION_FOR32(0, 1, 0, 7, ID_MMFR3)
467FUNCTION_FOR32(0, 2, 0, 0, ID_ISAR0)
468FUNCTION_FOR32(0, 2, 0, 1, ID_ISAR1)
469FUNCTION_FOR32(0, 2, 0, 2, ID_ISAR2)
470FUNCTION_FOR32(0, 2, 0, 3, ID_ISAR3)
471FUNCTION_FOR32(0, 2, 0, 4, ID_ISAR4)
472FUNCTION_FOR32(0, 2, 0, 5, ID_ISAR5)
473FUNCTION_FOR32(0, 0, 1, 1, CLIDR)
474FUNCTION_FOR32(0, 0, 1, 7, AIDR)
475
476/* ->val is filled in by kvm_invariant_coproc_table_init() */
477static struct coproc_reg invariant_cp15[] = {
478 { CRn( 0), CRm( 0), Op1( 0), Op2( 0), is32, NULL, get_MIDR },
479 { CRn( 0), CRm( 0), Op1( 0), Op2( 1), is32, NULL, get_CTR },
480 { CRn( 0), CRm( 0), Op1( 0), Op2( 2), is32, NULL, get_TCMTR },
481 { CRn( 0), CRm( 0), Op1( 0), Op2( 3), is32, NULL, get_TLBTR },
482 { CRn( 0), CRm( 0), Op1( 0), Op2( 6), is32, NULL, get_REVIDR },
483
484 { CRn( 0), CRm( 1), Op1( 0), Op2( 0), is32, NULL, get_ID_PFR0 },
485 { CRn( 0), CRm( 1), Op1( 0), Op2( 1), is32, NULL, get_ID_PFR1 },
486 { CRn( 0), CRm( 1), Op1( 0), Op2( 2), is32, NULL, get_ID_DFR0 },
487 { CRn( 0), CRm( 1), Op1( 0), Op2( 3), is32, NULL, get_ID_AFR0 },
488 { CRn( 0), CRm( 1), Op1( 0), Op2( 4), is32, NULL, get_ID_MMFR0 },
489 { CRn( 0), CRm( 1), Op1( 0), Op2( 5), is32, NULL, get_ID_MMFR1 },
490 { CRn( 0), CRm( 1), Op1( 0), Op2( 6), is32, NULL, get_ID_MMFR2 },
491 { CRn( 0), CRm( 1), Op1( 0), Op2( 7), is32, NULL, get_ID_MMFR3 },
492
493 { CRn( 0), CRm( 2), Op1( 0), Op2( 0), is32, NULL, get_ID_ISAR0 },
494 { CRn( 0), CRm( 2), Op1( 0), Op2( 1), is32, NULL, get_ID_ISAR1 },
495 { CRn( 0), CRm( 2), Op1( 0), Op2( 2), is32, NULL, get_ID_ISAR2 },
496 { CRn( 0), CRm( 2), Op1( 0), Op2( 3), is32, NULL, get_ID_ISAR3 },
497 { CRn( 0), CRm( 2), Op1( 0), Op2( 4), is32, NULL, get_ID_ISAR4 },
498 { CRn( 0), CRm( 2), Op1( 0), Op2( 5), is32, NULL, get_ID_ISAR5 },
499
500 { CRn( 0), CRm( 0), Op1( 1), Op2( 1), is32, NULL, get_CLIDR },
501 { CRn( 0), CRm( 0), Op1( 1), Op2( 7), is32, NULL, get_AIDR },
502};
503
504static int reg_from_user(void *val, const void __user *uaddr, u64 id)
505{
506 /* This Just Works because we are little endian. */
507 if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
508 return -EFAULT;
509 return 0;
510}
511
512static int reg_to_user(void __user *uaddr, const void *val, u64 id)
513{
514 /* This Just Works because we are little endian. */
515 if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
516 return -EFAULT;
517 return 0;
518}
519
520static int get_invariant_cp15(u64 id, void __user *uaddr)
521{
522 struct coproc_params params;
523 const struct coproc_reg *r;
524
525 if (!index_to_params(id, &params))
526 return -ENOENT;
527
528 r = find_reg(&params, invariant_cp15, ARRAY_SIZE(invariant_cp15));
529 if (!r)
530 return -ENOENT;
531
532 return reg_to_user(uaddr, &r->val, id);
533}
534
535static int set_invariant_cp15(u64 id, void __user *uaddr)
536{
537 struct coproc_params params;
538 const struct coproc_reg *r;
539 int err;
540 u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
541
542 if (!index_to_params(id, &params))
543 return -ENOENT;
544 r = find_reg(&params, invariant_cp15, ARRAY_SIZE(invariant_cp15));
545 if (!r)
546 return -ENOENT;
547
548 err = reg_from_user(&val, uaddr, id);
549 if (err)
550 return err;
551
552 /* This is what we mean by invariant: you can't change it. */
553 if (r->val != val)
554 return -EINVAL;
555
556 return 0;
557}
558
559static bool is_valid_cache(u32 val)
560{
561 u32 level, ctype;
562
563 if (val >= CSSELR_MAX)
564 return -ENOENT;
565
566 /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */
567 level = (val >> 1);
568 ctype = (cache_levels >> (level * 3)) & 7;
569
570 switch (ctype) {
571 case 0: /* No cache */
572 return false;
573 case 1: /* Instruction cache only */
574 return (val & 1);
575 case 2: /* Data cache only */
576 case 4: /* Unified cache */
577 return !(val & 1);
578 case 3: /* Separate instruction and data caches */
579 return true;
580 default: /* Reserved: we can't know instruction or data. */
581 return false;
582 }
583}
584
585/* Which cache CCSIDR represents depends on CSSELR value. */
586static u32 get_ccsidr(u32 csselr)
587{
588 u32 ccsidr;
589
590 /* Make sure noone else changes CSSELR during this! */
591 local_irq_disable();
592 /* Put value into CSSELR */
593 asm volatile("mcr p15, 2, %0, c0, c0, 0" : : "r" (csselr));
594 isb();
595 /* Read result out of CCSIDR */
596 asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r" (ccsidr));
597 local_irq_enable();
598
599 return ccsidr;
600}
601
602static int demux_c15_get(u64 id, void __user *uaddr)
603{
604 u32 val;
605 u32 __user *uval = uaddr;
606
607 /* Fail if we have unknown bits set. */
608 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
609 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
610 return -ENOENT;
611
612 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
613 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
614 if (KVM_REG_SIZE(id) != 4)
615 return -ENOENT;
616 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
617 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
618 if (!is_valid_cache(val))
619 return -ENOENT;
620
621 return put_user(get_ccsidr(val), uval);
622 default:
623 return -ENOENT;
624 }
625}
626
627static int demux_c15_set(u64 id, void __user *uaddr)
628{
629 u32 val, newval;
630 u32 __user *uval = uaddr;
631
632 /* Fail if we have unknown bits set. */
633 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
634 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
635 return -ENOENT;
636
637 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
638 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
639 if (KVM_REG_SIZE(id) != 4)
640 return -ENOENT;
641 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
642 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
643 if (!is_valid_cache(val))
644 return -ENOENT;
645
646 if (get_user(newval, uval))
647 return -EFAULT;
648
649 /* This is also invariant: you can't change it. */
650 if (newval != get_ccsidr(val))
651 return -EINVAL;
652 return 0;
653 default:
654 return -ENOENT;
655 }
656}
657
658#ifdef CONFIG_VFPv3
659static const int vfp_sysregs[] = { KVM_REG_ARM_VFP_FPEXC,
660 KVM_REG_ARM_VFP_FPSCR,
661 KVM_REG_ARM_VFP_FPINST,
662 KVM_REG_ARM_VFP_FPINST2,
663 KVM_REG_ARM_VFP_MVFR0,
664 KVM_REG_ARM_VFP_MVFR1,
665 KVM_REG_ARM_VFP_FPSID };
666
667static unsigned int num_fp_regs(void)
668{
669 if (((fmrx(MVFR0) & MVFR0_A_SIMD_MASK) >> MVFR0_A_SIMD_BIT) == 2)
670 return 32;
671 else
672 return 16;
673}
674
675static unsigned int num_vfp_regs(void)
676{
677 /* Normal FP regs + control regs. */
678 return num_fp_regs() + ARRAY_SIZE(vfp_sysregs);
679}
680
681static int copy_vfp_regids(u64 __user *uindices)
682{
683 unsigned int i;
684 const u64 u32reg = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_VFP;
685 const u64 u64reg = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP;
686
687 for (i = 0; i < num_fp_regs(); i++) {
688 if (put_user((u64reg | KVM_REG_ARM_VFP_BASE_REG) + i,
689 uindices))
690 return -EFAULT;
691 uindices++;
692 }
693
694 for (i = 0; i < ARRAY_SIZE(vfp_sysregs); i++) {
695 if (put_user(u32reg | vfp_sysregs[i], uindices))
696 return -EFAULT;
697 uindices++;
698 }
699
700 return num_vfp_regs();
701}
702
703static int vfp_get_reg(const struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
704{
705 u32 vfpid = (id & KVM_REG_ARM_VFP_MASK);
706 u32 val;
707
708 /* Fail if we have unknown bits set. */
709 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
710 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
711 return -ENOENT;
712
713 if (vfpid < num_fp_regs()) {
714 if (KVM_REG_SIZE(id) != 8)
715 return -ENOENT;
716 return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpregs[vfpid],
717 id);
718 }
719
720 /* FP control registers are all 32 bit. */
721 if (KVM_REG_SIZE(id) != 4)
722 return -ENOENT;
723
724 switch (vfpid) {
725 case KVM_REG_ARM_VFP_FPEXC:
726 return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpexc, id);
727 case KVM_REG_ARM_VFP_FPSCR:
728 return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpscr, id);
729 case KVM_REG_ARM_VFP_FPINST:
730 return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpinst, id);
731 case KVM_REG_ARM_VFP_FPINST2:
732 return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpinst2, id);
733 case KVM_REG_ARM_VFP_MVFR0:
734 val = fmrx(MVFR0);
735 return reg_to_user(uaddr, &val, id);
736 case KVM_REG_ARM_VFP_MVFR1:
737 val = fmrx(MVFR1);
738 return reg_to_user(uaddr, &val, id);
739 case KVM_REG_ARM_VFP_FPSID:
740 val = fmrx(FPSID);
741 return reg_to_user(uaddr, &val, id);
742 default:
743 return -ENOENT;
744 }
745}
746
747static int vfp_set_reg(struct kvm_vcpu *vcpu, u64 id, const void __user *uaddr)
748{
749 u32 vfpid = (id & KVM_REG_ARM_VFP_MASK);
750 u32 val;
751
752 /* Fail if we have unknown bits set. */
753 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
754 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
755 return -ENOENT;
756
757 if (vfpid < num_fp_regs()) {
758 if (KVM_REG_SIZE(id) != 8)
759 return -ENOENT;
760 return reg_from_user(&vcpu->arch.vfp_guest.fpregs[vfpid],
761 uaddr, id);
762 }
763
764 /* FP control registers are all 32 bit. */
765 if (KVM_REG_SIZE(id) != 4)
766 return -ENOENT;
767
768 switch (vfpid) {
769 case KVM_REG_ARM_VFP_FPEXC:
770 return reg_from_user(&vcpu->arch.vfp_guest.fpexc, uaddr, id);
771 case KVM_REG_ARM_VFP_FPSCR:
772 return reg_from_user(&vcpu->arch.vfp_guest.fpscr, uaddr, id);
773 case KVM_REG_ARM_VFP_FPINST:
774 return reg_from_user(&vcpu->arch.vfp_guest.fpinst, uaddr, id);
775 case KVM_REG_ARM_VFP_FPINST2:
776 return reg_from_user(&vcpu->arch.vfp_guest.fpinst2, uaddr, id);
777 /* These are invariant. */
778 case KVM_REG_ARM_VFP_MVFR0:
779 if (reg_from_user(&val, uaddr, id))
780 return -EFAULT;
781 if (val != fmrx(MVFR0))
782 return -EINVAL;
783 return 0;
784 case KVM_REG_ARM_VFP_MVFR1:
785 if (reg_from_user(&val, uaddr, id))
786 return -EFAULT;
787 if (val != fmrx(MVFR1))
788 return -EINVAL;
789 return 0;
790 case KVM_REG_ARM_VFP_FPSID:
791 if (reg_from_user(&val, uaddr, id))
792 return -EFAULT;
793 if (val != fmrx(FPSID))
794 return -EINVAL;
795 return 0;
796 default:
797 return -ENOENT;
798 }
799}
800#else /* !CONFIG_VFPv3 */
801static unsigned int num_vfp_regs(void)
802{
803 return 0;
804}
805
806static int copy_vfp_regids(u64 __user *uindices)
807{
808 return 0;
809}
810
811static int vfp_get_reg(const struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
812{
813 return -ENOENT;
814}
815
816static int vfp_set_reg(struct kvm_vcpu *vcpu, u64 id, const void __user *uaddr)
817{
818 return -ENOENT;
819}
820#endif /* !CONFIG_VFPv3 */
821
822int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
823{
824 const struct coproc_reg *r;
825 void __user *uaddr = (void __user *)(long)reg->addr;
826
827 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
828 return demux_c15_get(reg->id, uaddr);
829
830 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_VFP)
831 return vfp_get_reg(vcpu, reg->id, uaddr);
832
833 r = index_to_coproc_reg(vcpu, reg->id);
834 if (!r)
835 return get_invariant_cp15(reg->id, uaddr);
836
837 /* Note: copies two regs if size is 64 bit. */
838 return reg_to_user(uaddr, &vcpu->arch.cp15[r->reg], reg->id);
839}
840
841int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
842{
843 const struct coproc_reg *r;
844 void __user *uaddr = (void __user *)(long)reg->addr;
845
846 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
847 return demux_c15_set(reg->id, uaddr);
848
849 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_VFP)
850 return vfp_set_reg(vcpu, reg->id, uaddr);
851
852 r = index_to_coproc_reg(vcpu, reg->id);
853 if (!r)
854 return set_invariant_cp15(reg->id, uaddr);
855
856 /* Note: copies two regs if size is 64 bit */
857 return reg_from_user(&vcpu->arch.cp15[r->reg], uaddr, reg->id);
858}
859
860static unsigned int num_demux_regs(void)
861{
862 unsigned int i, count = 0;
863
864 for (i = 0; i < CSSELR_MAX; i++)
865 if (is_valid_cache(i))
866 count++;
867
868 return count;
869}
870
871static int write_demux_regids(u64 __user *uindices)
872{
873 u64 val = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
874 unsigned int i;
875
876 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
877 for (i = 0; i < CSSELR_MAX; i++) {
878 if (!is_valid_cache(i))
879 continue;
880 if (put_user(val | i, uindices))
881 return -EFAULT;
882 uindices++;
883 }
884 return 0;
885}
886
887static u64 cp15_to_index(const struct coproc_reg *reg)
888{
889 u64 val = KVM_REG_ARM | (15 << KVM_REG_ARM_COPROC_SHIFT);
890 if (reg->is_64) {
891 val |= KVM_REG_SIZE_U64;
892 val |= (reg->Op1 << KVM_REG_ARM_OPC1_SHIFT);
893 val |= (reg->CRm << KVM_REG_ARM_CRM_SHIFT);
894 } else {
895 val |= KVM_REG_SIZE_U32;
896 val |= (reg->Op1 << KVM_REG_ARM_OPC1_SHIFT);
897 val |= (reg->Op2 << KVM_REG_ARM_32_OPC2_SHIFT);
898 val |= (reg->CRm << KVM_REG_ARM_CRM_SHIFT);
899 val |= (reg->CRn << KVM_REG_ARM_32_CRN_SHIFT);
900 }
901 return val;
902}
903
904static bool copy_reg_to_user(const struct coproc_reg *reg, u64 __user **uind)
905{
906 if (!*uind)
907 return true;
908
909 if (put_user(cp15_to_index(reg), *uind))
910 return false;
911
912 (*uind)++;
913 return true;
914}
915
916/* Assumed ordered tables, see kvm_coproc_table_init. */
917static int walk_cp15(struct kvm_vcpu *vcpu, u64 __user *uind)
918{
919 const struct coproc_reg *i1, *i2, *end1, *end2;
920 unsigned int total = 0;
921 size_t num;
922
923 /* We check for duplicates here, to allow arch-specific overrides. */
924 i1 = get_target_table(vcpu->arch.target, &num);
925 end1 = i1 + num;
926 i2 = cp15_regs;
927 end2 = cp15_regs + ARRAY_SIZE(cp15_regs);
928
929 BUG_ON(i1 == end1 || i2 == end2);
930
931 /* Walk carefully, as both tables may refer to the same register. */
932 while (i1 || i2) {
933 int cmp = cmp_reg(i1, i2);
934 /* target-specific overrides generic entry. */
935 if (cmp <= 0) {
936 /* Ignore registers we trap but don't save. */
937 if (i1->reg) {
938 if (!copy_reg_to_user(i1, &uind))
939 return -EFAULT;
940 total++;
941 }
942 } else {
943 /* Ignore registers we trap but don't save. */
944 if (i2->reg) {
945 if (!copy_reg_to_user(i2, &uind))
946 return -EFAULT;
947 total++;
948 }
949 }
950
951 if (cmp <= 0 && ++i1 == end1)
952 i1 = NULL;
953 if (cmp >= 0 && ++i2 == end2)
954 i2 = NULL;
955 }
956 return total;
957}
958
959unsigned long kvm_arm_num_coproc_regs(struct kvm_vcpu *vcpu)
960{
961 return ARRAY_SIZE(invariant_cp15)
962 + num_demux_regs()
963 + num_vfp_regs()
964 + walk_cp15(vcpu, (u64 __user *)NULL);
965}
966
967int kvm_arm_copy_coproc_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
968{
969 unsigned int i;
970 int err;
971
972 /* Then give them all the invariant registers' indices. */
973 for (i = 0; i < ARRAY_SIZE(invariant_cp15); i++) {
974 if (put_user(cp15_to_index(&invariant_cp15[i]), uindices))
975 return -EFAULT;
976 uindices++;
977 }
978
979 err = walk_cp15(vcpu, uindices);
980 if (err < 0)
981 return err;
982 uindices += err;
983
984 err = copy_vfp_regids(uindices);
985 if (err < 0)
986 return err;
987 uindices += err;
988
989 return write_demux_regids(uindices);
990}
991
992void kvm_coproc_table_init(void)
993{
994 unsigned int i;
995
996 /* Make sure tables are unique and in order. */
997 for (i = 1; i < ARRAY_SIZE(cp15_regs); i++)
998 BUG_ON(cmp_reg(&cp15_regs[i-1], &cp15_regs[i]) >= 0);
999
1000 /* We abuse the reset function to overwrite the table itself. */
1001 for (i = 0; i < ARRAY_SIZE(invariant_cp15); i++)
1002 invariant_cp15[i].reset(NULL, &invariant_cp15[i]);
1003
1004 /*
1005 * CLIDR format is awkward, so clean it up. See ARM B4.1.20:
1006 *
1007 * If software reads the Cache Type fields from Ctype1
1008 * upwards, once it has seen a value of 0b000, no caches
1009 * exist at further-out levels of the hierarchy. So, for
1010 * example, if Ctype3 is the first Cache Type field with a
1011 * value of 0b000, the values of Ctype4 to Ctype7 must be
1012 * ignored.
1013 */
1014 asm volatile("mrc p15, 1, %0, c0, c0, 1" : "=r" (cache_levels));
1015 for (i = 0; i < 7; i++)
1016 if (((cache_levels >> (i*3)) & 7) == 0)
1017 break;
1018 /* Clear all higher bits. */
1019 cache_levels &= (1 << (i*3))-1;
1020}
1021
1022/**
1023 * kvm_reset_coprocs - sets cp15 registers to reset value
1024 * @vcpu: The VCPU pointer
1025 *
1026 * This function finds the right table above and sets the registers on the
1027 * virtual CPU struct to their architecturally defined reset values.
1028 */
1029void kvm_reset_coprocs(struct kvm_vcpu *vcpu)
1030{
1031 size_t num;
1032 const struct coproc_reg *table;
1033
1034 /* Catch someone adding a register without putting in reset entry. */
1035 memset(vcpu->arch.cp15, 0x42, sizeof(vcpu->arch.cp15));
1036
1037 /* Generic chip reset first (so target could override). */
1038 reset_coproc_regs(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs));
1039
1040 table = get_target_table(vcpu->arch.target, &num);
1041 reset_coproc_regs(vcpu, table, num);
1042
1043 for (num = 1; num < NR_CP15_REGS; num++)
1044 if (vcpu->arch.cp15[num] == 0x42424242)
1045 panic("Didn't reset vcpu->arch.cp15[%zi]", num);
1046}
diff --git a/arch/arm/kvm/coproc.h b/arch/arm/kvm/coproc.h
new file mode 100644
index 000000000000..992adfafa2ff
--- /dev/null
+++ b/arch/arm/kvm/coproc.h
@@ -0,0 +1,153 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Authors: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19#ifndef __ARM_KVM_COPROC_LOCAL_H__
20#define __ARM_KVM_COPROC_LOCAL_H__
21
22struct coproc_params {
23 unsigned long CRn;
24 unsigned long CRm;
25 unsigned long Op1;
26 unsigned long Op2;
27 unsigned long Rt1;
28 unsigned long Rt2;
29 bool is_64bit;
30 bool is_write;
31};
32
33struct coproc_reg {
34 /* MRC/MCR/MRRC/MCRR instruction which accesses it. */
35 unsigned long CRn;
36 unsigned long CRm;
37 unsigned long Op1;
38 unsigned long Op2;
39
40 bool is_64;
41
42 /* Trapped access from guest, if non-NULL. */
43 bool (*access)(struct kvm_vcpu *,
44 const struct coproc_params *,
45 const struct coproc_reg *);
46
47 /* Initialization for vcpu. */
48 void (*reset)(struct kvm_vcpu *, const struct coproc_reg *);
49
50 /* Index into vcpu->arch.cp15[], or 0 if we don't need to save it. */
51 unsigned long reg;
52
53 /* Value (usually reset value) */
54 u64 val;
55};
56
57static inline void print_cp_instr(const struct coproc_params *p)
58{
59 /* Look, we even formatted it for you to paste into the table! */
60 if (p->is_64bit) {
61 kvm_pr_unimpl(" { CRm(%2lu), Op1(%2lu), is64, func_%s },\n",
62 p->CRm, p->Op1, p->is_write ? "write" : "read");
63 } else {
64 kvm_pr_unimpl(" { CRn(%2lu), CRm(%2lu), Op1(%2lu), Op2(%2lu), is32,"
65 " func_%s },\n",
66 p->CRn, p->CRm, p->Op1, p->Op2,
67 p->is_write ? "write" : "read");
68 }
69}
70
71static inline bool ignore_write(struct kvm_vcpu *vcpu,
72 const struct coproc_params *p)
73{
74 return true;
75}
76
77static inline bool read_zero(struct kvm_vcpu *vcpu,
78 const struct coproc_params *p)
79{
80 *vcpu_reg(vcpu, p->Rt1) = 0;
81 return true;
82}
83
84static inline bool write_to_read_only(struct kvm_vcpu *vcpu,
85 const struct coproc_params *params)
86{
87 kvm_debug("CP15 write to read-only register at: %08x\n",
88 *vcpu_pc(vcpu));
89 print_cp_instr(params);
90 return false;
91}
92
93static inline bool read_from_write_only(struct kvm_vcpu *vcpu,
94 const struct coproc_params *params)
95{
96 kvm_debug("CP15 read to write-only register at: %08x\n",
97 *vcpu_pc(vcpu));
98 print_cp_instr(params);
99 return false;
100}
101
102/* Reset functions */
103static inline void reset_unknown(struct kvm_vcpu *vcpu,
104 const struct coproc_reg *r)
105{
106 BUG_ON(!r->reg);
107 BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.cp15));
108 vcpu->arch.cp15[r->reg] = 0xdecafbad;
109}
110
111static inline void reset_val(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
112{
113 BUG_ON(!r->reg);
114 BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.cp15));
115 vcpu->arch.cp15[r->reg] = r->val;
116}
117
118static inline void reset_unknown64(struct kvm_vcpu *vcpu,
119 const struct coproc_reg *r)
120{
121 BUG_ON(!r->reg);
122 BUG_ON(r->reg + 1 >= ARRAY_SIZE(vcpu->arch.cp15));
123
124 vcpu->arch.cp15[r->reg] = 0xdecafbad;
125 vcpu->arch.cp15[r->reg+1] = 0xd0c0ffee;
126}
127
128static inline int cmp_reg(const struct coproc_reg *i1,
129 const struct coproc_reg *i2)
130{
131 BUG_ON(i1 == i2);
132 if (!i1)
133 return 1;
134 else if (!i2)
135 return -1;
136 if (i1->CRn != i2->CRn)
137 return i1->CRn - i2->CRn;
138 if (i1->CRm != i2->CRm)
139 return i1->CRm - i2->CRm;
140 if (i1->Op1 != i2->Op1)
141 return i1->Op1 - i2->Op1;
142 return i1->Op2 - i2->Op2;
143}
144
145
146#define CRn(_x) .CRn = _x
147#define CRm(_x) .CRm = _x
148#define Op1(_x) .Op1 = _x
149#define Op2(_x) .Op2 = _x
150#define is64 .is_64 = true
151#define is32 .is_64 = false
152
153#endif /* __ARM_KVM_COPROC_LOCAL_H__ */
diff --git a/arch/arm/kvm/coproc_a15.c b/arch/arm/kvm/coproc_a15.c
new file mode 100644
index 000000000000..685063a6d0cf
--- /dev/null
+++ b/arch/arm/kvm/coproc_a15.c
@@ -0,0 +1,162 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Authors: Rusty Russell <rusty@rustcorp.au>
4 * Christoffer Dall <c.dall@virtualopensystems.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 */
19#include <linux/kvm_host.h>
20#include <asm/cputype.h>
21#include <asm/kvm_arm.h>
22#include <asm/kvm_host.h>
23#include <asm/kvm_emulate.h>
24#include <asm/kvm_coproc.h>
25#include <linux/init.h>
26
27static void reset_mpidr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
28{
29 /*
30 * Compute guest MPIDR:
31 * (Even if we present only one VCPU to the guest on an SMP
32 * host we don't set the U bit in the MPIDR, or vice versa, as
33 * revealing the underlying hardware properties is likely to
34 * be the best choice).
35 */
36 vcpu->arch.cp15[c0_MPIDR] = (read_cpuid_mpidr() & ~MPIDR_LEVEL_MASK)
37 | (vcpu->vcpu_id & MPIDR_LEVEL_MASK);
38}
39
40#include "coproc.h"
41
42/* A15 TRM 4.3.28: RO WI */
43static bool access_actlr(struct kvm_vcpu *vcpu,
44 const struct coproc_params *p,
45 const struct coproc_reg *r)
46{
47 if (p->is_write)
48 return ignore_write(vcpu, p);
49
50 *vcpu_reg(vcpu, p->Rt1) = vcpu->arch.cp15[c1_ACTLR];
51 return true;
52}
53
54/* A15 TRM 4.3.60: R/O. */
55static bool access_cbar(struct kvm_vcpu *vcpu,
56 const struct coproc_params *p,
57 const struct coproc_reg *r)
58{
59 if (p->is_write)
60 return write_to_read_only(vcpu, p);
61 return read_zero(vcpu, p);
62}
63
64/* A15 TRM 4.3.48: R/O WI. */
65static bool access_l2ctlr(struct kvm_vcpu *vcpu,
66 const struct coproc_params *p,
67 const struct coproc_reg *r)
68{
69 if (p->is_write)
70 return ignore_write(vcpu, p);
71
72 *vcpu_reg(vcpu, p->Rt1) = vcpu->arch.cp15[c9_L2CTLR];
73 return true;
74}
75
76static void reset_l2ctlr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
77{
78 u32 l2ctlr, ncores;
79
80 asm volatile("mrc p15, 1, %0, c9, c0, 2\n" : "=r" (l2ctlr));
81 l2ctlr &= ~(3 << 24);
82 ncores = atomic_read(&vcpu->kvm->online_vcpus) - 1;
83 l2ctlr |= (ncores & 3) << 24;
84
85 vcpu->arch.cp15[c9_L2CTLR] = l2ctlr;
86}
87
88static void reset_actlr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
89{
90 u32 actlr;
91
92 /* ACTLR contains SMP bit: make sure you create all cpus first! */
93 asm volatile("mrc p15, 0, %0, c1, c0, 1\n" : "=r" (actlr));
94 /* Make the SMP bit consistent with the guest configuration */
95 if (atomic_read(&vcpu->kvm->online_vcpus) > 1)
96 actlr |= 1U << 6;
97 else
98 actlr &= ~(1U << 6);
99
100 vcpu->arch.cp15[c1_ACTLR] = actlr;
101}
102
103/* A15 TRM 4.3.49: R/O WI (even if NSACR.NS_L2ERR, a write of 1 is ignored). */
104static bool access_l2ectlr(struct kvm_vcpu *vcpu,
105 const struct coproc_params *p,
106 const struct coproc_reg *r)
107{
108 if (p->is_write)
109 return ignore_write(vcpu, p);
110
111 *vcpu_reg(vcpu, p->Rt1) = 0;
112 return true;
113}
114
115/*
116 * A15-specific CP15 registers.
117 * Important: Must be sorted ascending by CRn, CRM, Op1, Op2
118 */
119static const struct coproc_reg a15_regs[] = {
120 /* MPIDR: we use VMPIDR for guest access. */
121 { CRn( 0), CRm( 0), Op1( 0), Op2( 5), is32,
122 NULL, reset_mpidr, c0_MPIDR },
123
124 /* SCTLR: swapped by interrupt.S. */
125 { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32,
126 NULL, reset_val, c1_SCTLR, 0x00C50078 },
127 /* ACTLR: trapped by HCR.TAC bit. */
128 { CRn( 1), CRm( 0), Op1( 0), Op2( 1), is32,
129 access_actlr, reset_actlr, c1_ACTLR },
130 /* CPACR: swapped by interrupt.S. */
131 { CRn( 1), CRm( 0), Op1( 0), Op2( 2), is32,
132 NULL, reset_val, c1_CPACR, 0x00000000 },
133
134 /*
135 * L2CTLR access (guest wants to know #CPUs).
136 */
137 { CRn( 9), CRm( 0), Op1( 1), Op2( 2), is32,
138 access_l2ctlr, reset_l2ctlr, c9_L2CTLR },
139 { CRn( 9), CRm( 0), Op1( 1), Op2( 3), is32, access_l2ectlr},
140
141 /* The Configuration Base Address Register. */
142 { CRn(15), CRm( 0), Op1( 4), Op2( 0), is32, access_cbar},
143};
144
145static struct kvm_coproc_target_table a15_target_table = {
146 .target = KVM_ARM_TARGET_CORTEX_A15,
147 .table = a15_regs,
148 .num = ARRAY_SIZE(a15_regs),
149};
150
151static int __init coproc_a15_init(void)
152{
153 unsigned int i;
154
155 for (i = 1; i < ARRAY_SIZE(a15_regs); i++)
156 BUG_ON(cmp_reg(&a15_regs[i-1],
157 &a15_regs[i]) >= 0);
158
159 kvm_register_target_coproc_table(&a15_target_table);
160 return 0;
161}
162late_initcall(coproc_a15_init);
diff --git a/arch/arm/kvm/emulate.c b/arch/arm/kvm/emulate.c
new file mode 100644
index 000000000000..d61450ac6665
--- /dev/null
+++ b/arch/arm/kvm/emulate.c
@@ -0,0 +1,373 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19#include <linux/mm.h>
20#include <linux/kvm_host.h>
21#include <asm/kvm_arm.h>
22#include <asm/kvm_emulate.h>
23#include <trace/events/kvm.h>
24
25#include "trace.h"
26
27#define VCPU_NR_MODES 6
28#define VCPU_REG_OFFSET_USR 0
29#define VCPU_REG_OFFSET_FIQ 1
30#define VCPU_REG_OFFSET_IRQ 2
31#define VCPU_REG_OFFSET_SVC 3
32#define VCPU_REG_OFFSET_ABT 4
33#define VCPU_REG_OFFSET_UND 5
34#define REG_OFFSET(_reg) \
35 (offsetof(struct kvm_regs, _reg) / sizeof(u32))
36
37#define USR_REG_OFFSET(_num) REG_OFFSET(usr_regs.uregs[_num])
38
39static const unsigned long vcpu_reg_offsets[VCPU_NR_MODES][15] = {
40 /* USR/SYS Registers */
41 [VCPU_REG_OFFSET_USR] = {
42 USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
43 USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
44 USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
45 USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
46 USR_REG_OFFSET(12), USR_REG_OFFSET(13), USR_REG_OFFSET(14),
47 },
48
49 /* FIQ Registers */
50 [VCPU_REG_OFFSET_FIQ] = {
51 USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
52 USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
53 USR_REG_OFFSET(6), USR_REG_OFFSET(7),
54 REG_OFFSET(fiq_regs[0]), /* r8 */
55 REG_OFFSET(fiq_regs[1]), /* r9 */
56 REG_OFFSET(fiq_regs[2]), /* r10 */
57 REG_OFFSET(fiq_regs[3]), /* r11 */
58 REG_OFFSET(fiq_regs[4]), /* r12 */
59 REG_OFFSET(fiq_regs[5]), /* r13 */
60 REG_OFFSET(fiq_regs[6]), /* r14 */
61 },
62
63 /* IRQ Registers */
64 [VCPU_REG_OFFSET_IRQ] = {
65 USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
66 USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
67 USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
68 USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
69 USR_REG_OFFSET(12),
70 REG_OFFSET(irq_regs[0]), /* r13 */
71 REG_OFFSET(irq_regs[1]), /* r14 */
72 },
73
74 /* SVC Registers */
75 [VCPU_REG_OFFSET_SVC] = {
76 USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
77 USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
78 USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
79 USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
80 USR_REG_OFFSET(12),
81 REG_OFFSET(svc_regs[0]), /* r13 */
82 REG_OFFSET(svc_regs[1]), /* r14 */
83 },
84
85 /* ABT Registers */
86 [VCPU_REG_OFFSET_ABT] = {
87 USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
88 USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
89 USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
90 USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
91 USR_REG_OFFSET(12),
92 REG_OFFSET(abt_regs[0]), /* r13 */
93 REG_OFFSET(abt_regs[1]), /* r14 */
94 },
95
96 /* UND Registers */
97 [VCPU_REG_OFFSET_UND] = {
98 USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
99 USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
100 USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
101 USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
102 USR_REG_OFFSET(12),
103 REG_OFFSET(und_regs[0]), /* r13 */
104 REG_OFFSET(und_regs[1]), /* r14 */
105 },
106};
107
108/*
109 * Return a pointer to the register number valid in the current mode of
110 * the virtual CPU.
111 */
112u32 *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num)
113{
114 u32 *reg_array = (u32 *)&vcpu->arch.regs;
115 u32 mode = *vcpu_cpsr(vcpu) & MODE_MASK;
116
117 switch (mode) {
118 case USR_MODE...SVC_MODE:
119 mode &= ~MODE32_BIT; /* 0 ... 3 */
120 break;
121
122 case ABT_MODE:
123 mode = VCPU_REG_OFFSET_ABT;
124 break;
125
126 case UND_MODE:
127 mode = VCPU_REG_OFFSET_UND;
128 break;
129
130 case SYSTEM_MODE:
131 mode = VCPU_REG_OFFSET_USR;
132 break;
133
134 default:
135 BUG();
136 }
137
138 return reg_array + vcpu_reg_offsets[mode][reg_num];
139}
140
141/*
142 * Return the SPSR for the current mode of the virtual CPU.
143 */
144u32 *vcpu_spsr(struct kvm_vcpu *vcpu)
145{
146 u32 mode = *vcpu_cpsr(vcpu) & MODE_MASK;
147 switch (mode) {
148 case SVC_MODE:
149 return &vcpu->arch.regs.KVM_ARM_SVC_spsr;
150 case ABT_MODE:
151 return &vcpu->arch.regs.KVM_ARM_ABT_spsr;
152 case UND_MODE:
153 return &vcpu->arch.regs.KVM_ARM_UND_spsr;
154 case IRQ_MODE:
155 return &vcpu->arch.regs.KVM_ARM_IRQ_spsr;
156 case FIQ_MODE:
157 return &vcpu->arch.regs.KVM_ARM_FIQ_spsr;
158 default:
159 BUG();
160 }
161}
162
163/**
164 * kvm_handle_wfi - handle a wait-for-interrupts instruction executed by a guest
165 * @vcpu: the vcpu pointer
166 * @run: the kvm_run structure pointer
167 *
168 * Simply sets the wait_for_interrupts flag on the vcpu structure, which will
169 * halt execution of world-switches and schedule other host processes until
170 * there is an incoming IRQ or FIQ to the VM.
171 */
172int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run)
173{
174 trace_kvm_wfi(*vcpu_pc(vcpu));
175 kvm_vcpu_block(vcpu);
176 return 1;
177}
178
179/**
180 * adjust_itstate - adjust ITSTATE when emulating instructions in IT-block
181 * @vcpu: The VCPU pointer
182 *
183 * When exceptions occur while instructions are executed in Thumb IF-THEN
184 * blocks, the ITSTATE field of the CPSR is not advanved (updated), so we have
185 * to do this little bit of work manually. The fields map like this:
186 *
187 * IT[7:0] -> CPSR[26:25],CPSR[15:10]
188 */
189static void kvm_adjust_itstate(struct kvm_vcpu *vcpu)
190{
191 unsigned long itbits, cond;
192 unsigned long cpsr = *vcpu_cpsr(vcpu);
193 bool is_arm = !(cpsr & PSR_T_BIT);
194
195 BUG_ON(is_arm && (cpsr & PSR_IT_MASK));
196
197 if (!(cpsr & PSR_IT_MASK))
198 return;
199
200 cond = (cpsr & 0xe000) >> 13;
201 itbits = (cpsr & 0x1c00) >> (10 - 2);
202 itbits |= (cpsr & (0x3 << 25)) >> 25;
203
204 /* Perform ITAdvance (see page A-52 in ARM DDI 0406C) */
205 if ((itbits & 0x7) == 0)
206 itbits = cond = 0;
207 else
208 itbits = (itbits << 1) & 0x1f;
209
210 cpsr &= ~PSR_IT_MASK;
211 cpsr |= cond << 13;
212 cpsr |= (itbits & 0x1c) << (10 - 2);
213 cpsr |= (itbits & 0x3) << 25;
214 *vcpu_cpsr(vcpu) = cpsr;
215}
216
217/**
218 * kvm_skip_instr - skip a trapped instruction and proceed to the next
219 * @vcpu: The vcpu pointer
220 */
221void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr)
222{
223 bool is_thumb;
224
225 is_thumb = !!(*vcpu_cpsr(vcpu) & PSR_T_BIT);
226 if (is_thumb && !is_wide_instr)
227 *vcpu_pc(vcpu) += 2;
228 else
229 *vcpu_pc(vcpu) += 4;
230 kvm_adjust_itstate(vcpu);
231}
232
233
234/******************************************************************************
235 * Inject exceptions into the guest
236 */
237
238static u32 exc_vector_base(struct kvm_vcpu *vcpu)
239{
240 u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
241 u32 vbar = vcpu->arch.cp15[c12_VBAR];
242
243 if (sctlr & SCTLR_V)
244 return 0xffff0000;
245 else /* always have security exceptions */
246 return vbar;
247}
248
249/**
250 * kvm_inject_undefined - inject an undefined exception into the guest
251 * @vcpu: The VCPU to receive the undefined exception
252 *
253 * It is assumed that this code is called from the VCPU thread and that the
254 * VCPU therefore is not currently executing guest code.
255 *
256 * Modelled after TakeUndefInstrException() pseudocode.
257 */
258void kvm_inject_undefined(struct kvm_vcpu *vcpu)
259{
260 u32 new_lr_value;
261 u32 new_spsr_value;
262 u32 cpsr = *vcpu_cpsr(vcpu);
263 u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
264 bool is_thumb = (cpsr & PSR_T_BIT);
265 u32 vect_offset = 4;
266 u32 return_offset = (is_thumb) ? 2 : 4;
267
268 new_spsr_value = cpsr;
269 new_lr_value = *vcpu_pc(vcpu) - return_offset;
270
271 *vcpu_cpsr(vcpu) = (cpsr & ~MODE_MASK) | UND_MODE;
272 *vcpu_cpsr(vcpu) |= PSR_I_BIT;
273 *vcpu_cpsr(vcpu) &= ~(PSR_IT_MASK | PSR_J_BIT | PSR_E_BIT | PSR_T_BIT);
274
275 if (sctlr & SCTLR_TE)
276 *vcpu_cpsr(vcpu) |= PSR_T_BIT;
277 if (sctlr & SCTLR_EE)
278 *vcpu_cpsr(vcpu) |= PSR_E_BIT;
279
280 /* Note: These now point to UND banked copies */
281 *vcpu_spsr(vcpu) = cpsr;
282 *vcpu_reg(vcpu, 14) = new_lr_value;
283
284 /* Branch to exception vector */
285 *vcpu_pc(vcpu) = exc_vector_base(vcpu) + vect_offset;
286}
287
288/*
289 * Modelled after TakeDataAbortException() and TakePrefetchAbortException
290 * pseudocode.
291 */
292static void inject_abt(struct kvm_vcpu *vcpu, bool is_pabt, unsigned long addr)
293{
294 u32 new_lr_value;
295 u32 new_spsr_value;
296 u32 cpsr = *vcpu_cpsr(vcpu);
297 u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
298 bool is_thumb = (cpsr & PSR_T_BIT);
299 u32 vect_offset;
300 u32 return_offset = (is_thumb) ? 4 : 0;
301 bool is_lpae;
302
303 new_spsr_value = cpsr;
304 new_lr_value = *vcpu_pc(vcpu) + return_offset;
305
306 *vcpu_cpsr(vcpu) = (cpsr & ~MODE_MASK) | ABT_MODE;
307 *vcpu_cpsr(vcpu) |= PSR_I_BIT | PSR_A_BIT;
308 *vcpu_cpsr(vcpu) &= ~(PSR_IT_MASK | PSR_J_BIT | PSR_E_BIT | PSR_T_BIT);
309
310 if (sctlr & SCTLR_TE)
311 *vcpu_cpsr(vcpu) |= PSR_T_BIT;
312 if (sctlr & SCTLR_EE)
313 *vcpu_cpsr(vcpu) |= PSR_E_BIT;
314
315 /* Note: These now point to ABT banked copies */
316 *vcpu_spsr(vcpu) = cpsr;
317 *vcpu_reg(vcpu, 14) = new_lr_value;
318
319 if (is_pabt)
320 vect_offset = 12;
321 else
322 vect_offset = 16;
323
324 /* Branch to exception vector */
325 *vcpu_pc(vcpu) = exc_vector_base(vcpu) + vect_offset;
326
327 if (is_pabt) {
328 /* Set DFAR and DFSR */
329 vcpu->arch.cp15[c6_IFAR] = addr;
330 is_lpae = (vcpu->arch.cp15[c2_TTBCR] >> 31);
331 /* Always give debug fault for now - should give guest a clue */
332 if (is_lpae)
333 vcpu->arch.cp15[c5_IFSR] = 1 << 9 | 0x22;
334 else
335 vcpu->arch.cp15[c5_IFSR] = 2;
336 } else { /* !iabt */
337 /* Set DFAR and DFSR */
338 vcpu->arch.cp15[c6_DFAR] = addr;
339 is_lpae = (vcpu->arch.cp15[c2_TTBCR] >> 31);
340 /* Always give debug fault for now - should give guest a clue */
341 if (is_lpae)
342 vcpu->arch.cp15[c5_DFSR] = 1 << 9 | 0x22;
343 else
344 vcpu->arch.cp15[c5_DFSR] = 2;
345 }
346
347}
348
349/**
350 * kvm_inject_dabt - inject a data abort into the guest
351 * @vcpu: The VCPU to receive the undefined exception
352 * @addr: The address to report in the DFAR
353 *
354 * It is assumed that this code is called from the VCPU thread and that the
355 * VCPU therefore is not currently executing guest code.
356 */
357void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr)
358{
359 inject_abt(vcpu, false, addr);
360}
361
362/**
363 * kvm_inject_pabt - inject a prefetch abort into the guest
364 * @vcpu: The VCPU to receive the undefined exception
365 * @addr: The address to report in the DFAR
366 *
367 * It is assumed that this code is called from the VCPU thread and that the
368 * VCPU therefore is not currently executing guest code.
369 */
370void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr)
371{
372 inject_abt(vcpu, true, addr);
373}
diff --git a/arch/arm/kvm/guest.c b/arch/arm/kvm/guest.c
new file mode 100644
index 000000000000..2339d9609d36
--- /dev/null
+++ b/arch/arm/kvm/guest.c
@@ -0,0 +1,222 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19#include <linux/errno.h>
20#include <linux/err.h>
21#include <linux/kvm_host.h>
22#include <linux/module.h>
23#include <linux/vmalloc.h>
24#include <linux/fs.h>
25#include <asm/uaccess.h>
26#include <asm/kvm.h>
27#include <asm/kvm_asm.h>
28#include <asm/kvm_emulate.h>
29#include <asm/kvm_coproc.h>
30
31#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
32#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
33
34struct kvm_stats_debugfs_item debugfs_entries[] = {
35 { NULL }
36};
37
38int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
39{
40 return 0;
41}
42
43static u64 core_reg_offset_from_id(u64 id)
44{
45 return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE);
46}
47
48static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
49{
50 u32 __user *uaddr = (u32 __user *)(long)reg->addr;
51 struct kvm_regs *regs = &vcpu->arch.regs;
52 u64 off;
53
54 if (KVM_REG_SIZE(reg->id) != 4)
55 return -ENOENT;
56
57 /* Our ID is an index into the kvm_regs struct. */
58 off = core_reg_offset_from_id(reg->id);
59 if (off >= sizeof(*regs) / KVM_REG_SIZE(reg->id))
60 return -ENOENT;
61
62 return put_user(((u32 *)regs)[off], uaddr);
63}
64
65static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
66{
67 u32 __user *uaddr = (u32 __user *)(long)reg->addr;
68 struct kvm_regs *regs = &vcpu->arch.regs;
69 u64 off, val;
70
71 if (KVM_REG_SIZE(reg->id) != 4)
72 return -ENOENT;
73
74 /* Our ID is an index into the kvm_regs struct. */
75 off = core_reg_offset_from_id(reg->id);
76 if (off >= sizeof(*regs) / KVM_REG_SIZE(reg->id))
77 return -ENOENT;
78
79 if (get_user(val, uaddr) != 0)
80 return -EFAULT;
81
82 if (off == KVM_REG_ARM_CORE_REG(usr_regs.ARM_cpsr)) {
83 unsigned long mode = val & MODE_MASK;
84 switch (mode) {
85 case USR_MODE:
86 case FIQ_MODE:
87 case IRQ_MODE:
88 case SVC_MODE:
89 case ABT_MODE:
90 case UND_MODE:
91 break;
92 default:
93 return -EINVAL;
94 }
95 }
96
97 ((u32 *)regs)[off] = val;
98 return 0;
99}
100
101int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
102{
103 return -EINVAL;
104}
105
106int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
107{
108 return -EINVAL;
109}
110
111static unsigned long num_core_regs(void)
112{
113 return sizeof(struct kvm_regs) / sizeof(u32);
114}
115
116/**
117 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG
118 *
119 * This is for all registers.
120 */
121unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
122{
123 return num_core_regs() + kvm_arm_num_coproc_regs(vcpu);
124}
125
126/**
127 * kvm_arm_copy_reg_indices - get indices of all registers.
128 *
129 * We do core registers right here, then we apppend coproc regs.
130 */
131int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
132{
133 unsigned int i;
134 const u64 core_reg = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE;
135
136 for (i = 0; i < sizeof(struct kvm_regs)/sizeof(u32); i++) {
137 if (put_user(core_reg | i, uindices))
138 return -EFAULT;
139 uindices++;
140 }
141
142 return kvm_arm_copy_coproc_indices(vcpu, uindices);
143}
144
145int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
146{
147 /* We currently use nothing arch-specific in upper 32 bits */
148 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM >> 32)
149 return -EINVAL;
150
151 /* Register group 16 means we want a core register. */
152 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_CORE)
153 return get_core_reg(vcpu, reg);
154
155 return kvm_arm_coproc_get_reg(vcpu, reg);
156}
157
158int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
159{
160 /* We currently use nothing arch-specific in upper 32 bits */
161 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM >> 32)
162 return -EINVAL;
163
164 /* Register group 16 means we set a core register. */
165 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_CORE)
166 return set_core_reg(vcpu, reg);
167
168 return kvm_arm_coproc_set_reg(vcpu, reg);
169}
170
171int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
172 struct kvm_sregs *sregs)
173{
174 return -EINVAL;
175}
176
177int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
178 struct kvm_sregs *sregs)
179{
180 return -EINVAL;
181}
182
183int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
184 const struct kvm_vcpu_init *init)
185{
186 unsigned int i;
187
188 /* We can only do a cortex A15 for now. */
189 if (init->target != kvm_target_cpu())
190 return -EINVAL;
191
192 vcpu->arch.target = init->target;
193 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
194
195 /* -ENOENT for unknown features, -EINVAL for invalid combinations. */
196 for (i = 0; i < sizeof(init->features) * 8; i++) {
197 if (test_bit(i, (void *)init->features)) {
198 if (i >= KVM_VCPU_MAX_FEATURES)
199 return -ENOENT;
200 set_bit(i, vcpu->arch.features);
201 }
202 }
203
204 /* Now we know what it is, we can reset it. */
205 return kvm_reset_vcpu(vcpu);
206}
207
208int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
209{
210 return -EINVAL;
211}
212
213int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
214{
215 return -EINVAL;
216}
217
218int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
219 struct kvm_translation *tr)
220{
221 return -EINVAL;
222}
diff --git a/arch/arm/kvm/init.S b/arch/arm/kvm/init.S
new file mode 100644
index 000000000000..9f37a79b880b
--- /dev/null
+++ b/arch/arm/kvm/init.S
@@ -0,0 +1,114 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19#include <linux/linkage.h>
20#include <asm/unified.h>
21#include <asm/asm-offsets.h>
22#include <asm/kvm_asm.h>
23#include <asm/kvm_arm.h>
24
25/********************************************************************
26 * Hypervisor initialization
27 * - should be called with:
28 * r0,r1 = Hypervisor pgd pointer
29 * r2 = top of Hyp stack (kernel VA)
30 * r3 = pointer to hyp vectors
31 */
32
33 .text
34 .pushsection .hyp.idmap.text,"ax"
35 .align 5
36__kvm_hyp_init:
37 .globl __kvm_hyp_init
38
39 @ Hyp-mode exception vector
40 W(b) .
41 W(b) .
42 W(b) .
43 W(b) .
44 W(b) .
45 W(b) __do_hyp_init
46 W(b) .
47 W(b) .
48
49__do_hyp_init:
50 @ Set the HTTBR to point to the hypervisor PGD pointer passed
51 mcrr p15, 4, r0, r1, c2
52
53 @ Set the HTCR and VTCR to the same shareability and cacheability
54 @ settings as the non-secure TTBCR and with T0SZ == 0.
55 mrc p15, 4, r0, c2, c0, 2 @ HTCR
56 ldr r12, =HTCR_MASK
57 bic r0, r0, r12
58 mrc p15, 0, r1, c2, c0, 2 @ TTBCR
59 and r1, r1, #(HTCR_MASK & ~TTBCR_T0SZ)
60 orr r0, r0, r1
61 mcr p15, 4, r0, c2, c0, 2 @ HTCR
62
63 mrc p15, 4, r1, c2, c1, 2 @ VTCR
64 ldr r12, =VTCR_MASK
65 bic r1, r1, r12
66 bic r0, r0, #(~VTCR_HTCR_SH) @ clear non-reusable HTCR bits
67 orr r1, r0, r1
68 orr r1, r1, #(KVM_VTCR_SL0 | KVM_VTCR_T0SZ | KVM_VTCR_S)
69 mcr p15, 4, r1, c2, c1, 2 @ VTCR
70
71 @ Use the same memory attributes for hyp. accesses as the kernel
72 @ (copy MAIRx ro HMAIRx).
73 mrc p15, 0, r0, c10, c2, 0
74 mcr p15, 4, r0, c10, c2, 0
75 mrc p15, 0, r0, c10, c2, 1
76 mcr p15, 4, r0, c10, c2, 1
77
78 @ Set the HSCTLR to:
79 @ - ARM/THUMB exceptions: Kernel config (Thumb-2 kernel)
80 @ - Endianness: Kernel config
81 @ - Fast Interrupt Features: Kernel config
82 @ - Write permission implies XN: disabled
83 @ - Instruction cache: enabled
84 @ - Data/Unified cache: enabled
85 @ - Memory alignment checks: enabled
86 @ - MMU: enabled (this code must be run from an identity mapping)
87 mrc p15, 4, r0, c1, c0, 0 @ HSCR
88 ldr r12, =HSCTLR_MASK
89 bic r0, r0, r12
90 mrc p15, 0, r1, c1, c0, 0 @ SCTLR
91 ldr r12, =(HSCTLR_EE | HSCTLR_FI | HSCTLR_I | HSCTLR_C)
92 and r1, r1, r12
93 ARM( ldr r12, =(HSCTLR_M | HSCTLR_A) )
94 THUMB( ldr r12, =(HSCTLR_M | HSCTLR_A | HSCTLR_TE) )
95 orr r1, r1, r12
96 orr r0, r0, r1
97 isb
98 mcr p15, 4, r0, c1, c0, 0 @ HSCR
99 isb
100
101 @ Set stack pointer and return to the kernel
102 mov sp, r2
103
104 @ Set HVBAR to point to the HYP vectors
105 mcr p15, 4, r3, c12, c0, 0 @ HVBAR
106
107 eret
108
109 .ltorg
110
111 .globl __kvm_hyp_init_end
112__kvm_hyp_init_end:
113
114 .popsection
diff --git a/arch/arm/kvm/interrupts.S b/arch/arm/kvm/interrupts.S
new file mode 100644
index 000000000000..c5400d2e97ca
--- /dev/null
+++ b/arch/arm/kvm/interrupts.S
@@ -0,0 +1,478 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19#include <linux/linkage.h>
20#include <linux/const.h>
21#include <asm/unified.h>
22#include <asm/page.h>
23#include <asm/ptrace.h>
24#include <asm/asm-offsets.h>
25#include <asm/kvm_asm.h>
26#include <asm/kvm_arm.h>
27#include <asm/vfpmacros.h>
28#include "interrupts_head.S"
29
30 .text
31
32__kvm_hyp_code_start:
33 .globl __kvm_hyp_code_start
34
35/********************************************************************
36 * Flush per-VMID TLBs
37 *
38 * void __kvm_tlb_flush_vmid(struct kvm *kvm);
39 *
40 * We rely on the hardware to broadcast the TLB invalidation to all CPUs
41 * inside the inner-shareable domain (which is the case for all v7
42 * implementations). If we come across a non-IS SMP implementation, we'll
43 * have to use an IPI based mechanism. Until then, we stick to the simple
44 * hardware assisted version.
45 */
46ENTRY(__kvm_tlb_flush_vmid)
47 push {r2, r3}
48
49 add r0, r0, #KVM_VTTBR
50 ldrd r2, r3, [r0]
51 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
52 isb
53 mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
54 dsb
55 isb
56 mov r2, #0
57 mov r3, #0
58 mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
59 isb @ Not necessary if followed by eret
60
61 pop {r2, r3}
62 bx lr
63ENDPROC(__kvm_tlb_flush_vmid)
64
65/********************************************************************
66 * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
67 * domain, for all VMIDs
68 *
69 * void __kvm_flush_vm_context(void);
70 */
71ENTRY(__kvm_flush_vm_context)
72 mov r0, #0 @ rn parameter for c15 flushes is SBZ
73
74 /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
75 mcr p15, 4, r0, c8, c3, 4
76 /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
77 mcr p15, 0, r0, c7, c1, 0
78 dsb
79 isb @ Not necessary if followed by eret
80
81 bx lr
82ENDPROC(__kvm_flush_vm_context)
83
84
85/********************************************************************
86 * Hypervisor world-switch code
87 *
88 *
89 * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
90 */
91ENTRY(__kvm_vcpu_run)
92 @ Save the vcpu pointer
93 mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
94
95 save_host_regs
96
97 @ Store hardware CP15 state and load guest state
98 read_cp15_state store_to_vcpu = 0
99 write_cp15_state read_from_vcpu = 1
100
101 @ If the host kernel has not been configured with VFPv3 support,
102 @ then it is safer if we deny guests from using it as well.
103#ifdef CONFIG_VFPv3
104 @ Set FPEXC_EN so the guest doesn't trap floating point instructions
105 VFPFMRX r2, FPEXC @ VMRS
106 push {r2}
107 orr r2, r2, #FPEXC_EN
108 VFPFMXR FPEXC, r2 @ VMSR
109#endif
110
111 @ Configure Hyp-role
112 configure_hyp_role vmentry
113
114 @ Trap coprocessor CRx accesses
115 set_hstr vmentry
116 set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
117 set_hdcr vmentry
118
119 @ Write configured ID register into MIDR alias
120 ldr r1, [vcpu, #VCPU_MIDR]
121 mcr p15, 4, r1, c0, c0, 0
122
123 @ Write guest view of MPIDR into VMPIDR
124 ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
125 mcr p15, 4, r1, c0, c0, 5
126
127 @ Set up guest memory translation
128 ldr r1, [vcpu, #VCPU_KVM]
129 add r1, r1, #KVM_VTTBR
130 ldrd r2, r3, [r1]
131 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
132
133 @ We're all done, just restore the GPRs and go to the guest
134 restore_guest_regs
135 clrex @ Clear exclusive monitor
136 eret
137
138__kvm_vcpu_return:
139 /*
140 * return convention:
141 * guest r0, r1, r2 saved on the stack
142 * r0: vcpu pointer
143 * r1: exception code
144 */
145 save_guest_regs
146
147 @ Set VMID == 0
148 mov r2, #0
149 mov r3, #0
150 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
151
152 @ Don't trap coprocessor accesses for host kernel
153 set_hstr vmexit
154 set_hdcr vmexit
155 set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
156
157#ifdef CONFIG_VFPv3
158 @ Save floating point registers we if let guest use them.
159 tst r2, #(HCPTR_TCP(10) | HCPTR_TCP(11))
160 bne after_vfp_restore
161
162 @ Switch VFP/NEON hardware state to the host's
163 add r7, vcpu, #VCPU_VFP_GUEST
164 store_vfp_state r7
165 add r7, vcpu, #VCPU_VFP_HOST
166 ldr r7, [r7]
167 restore_vfp_state r7
168
169after_vfp_restore:
170 @ Restore FPEXC_EN which we clobbered on entry
171 pop {r2}
172 VFPFMXR FPEXC, r2
173#endif
174
175 @ Reset Hyp-role
176 configure_hyp_role vmexit
177
178 @ Let host read hardware MIDR
179 mrc p15, 0, r2, c0, c0, 0
180 mcr p15, 4, r2, c0, c0, 0
181
182 @ Back to hardware MPIDR
183 mrc p15, 0, r2, c0, c0, 5
184 mcr p15, 4, r2, c0, c0, 5
185
186 @ Store guest CP15 state and restore host state
187 read_cp15_state store_to_vcpu = 1
188 write_cp15_state read_from_vcpu = 0
189
190 restore_host_regs
191 clrex @ Clear exclusive monitor
192 mov r0, r1 @ Return the return code
193 mov r1, #0 @ Clear upper bits in return value
194 bx lr @ return to IOCTL
195
196/********************************************************************
197 * Call function in Hyp mode
198 *
199 *
200 * u64 kvm_call_hyp(void *hypfn, ...);
201 *
202 * This is not really a variadic function in the classic C-way and care must
203 * be taken when calling this to ensure parameters are passed in registers
204 * only, since the stack will change between the caller and the callee.
205 *
206 * Call the function with the first argument containing a pointer to the
207 * function you wish to call in Hyp mode, and subsequent arguments will be
208 * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
209 * function pointer can be passed). The function being called must be mapped
210 * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
211 * passed in r0 and r1.
212 *
213 * The calling convention follows the standard AAPCS:
214 * r0 - r3: caller save
215 * r12: caller save
216 * rest: callee save
217 */
218ENTRY(kvm_call_hyp)
219 hvc #0
220 bx lr
221
222/********************************************************************
223 * Hypervisor exception vector and handlers
224 *
225 *
226 * The KVM/ARM Hypervisor ABI is defined as follows:
227 *
228 * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
229 * instruction is issued since all traps are disabled when running the host
230 * kernel as per the Hyp-mode initialization at boot time.
231 *
232 * HVC instructions cause a trap to the vector page + offset 0x18 (see hyp_hvc
233 * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
234 * host kernel) and they cause a trap to the vector page + offset 0xc when HVC
235 * instructions are called from within Hyp-mode.
236 *
237 * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
238 * Switching to Hyp mode is done through a simple HVC #0 instruction. The
239 * exception vector code will check that the HVC comes from VMID==0 and if
240 * so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
241 * - r0 contains a pointer to a HYP function
242 * - r1, r2, and r3 contain arguments to the above function.
243 * - The HYP function will be called with its arguments in r0, r1 and r2.
244 * On HYP function return, we return directly to SVC.
245 *
246 * Note that the above is used to execute code in Hyp-mode from a host-kernel
247 * point of view, and is a different concept from performing a world-switch and
248 * executing guest code SVC mode (with a VMID != 0).
249 */
250
251/* Handle undef, svc, pabt, or dabt by crashing with a user notice */
252.macro bad_exception exception_code, panic_str
253 push {r0-r2}
254 mrrc p15, 6, r0, r1, c2 @ Read VTTBR
255 lsr r1, r1, #16
256 ands r1, r1, #0xff
257 beq 99f
258
259 load_vcpu @ Load VCPU pointer
260 .if \exception_code == ARM_EXCEPTION_DATA_ABORT
261 mrc p15, 4, r2, c5, c2, 0 @ HSR
262 mrc p15, 4, r1, c6, c0, 0 @ HDFAR
263 str r2, [vcpu, #VCPU_HSR]
264 str r1, [vcpu, #VCPU_HxFAR]
265 .endif
266 .if \exception_code == ARM_EXCEPTION_PREF_ABORT
267 mrc p15, 4, r2, c5, c2, 0 @ HSR
268 mrc p15, 4, r1, c6, c0, 2 @ HIFAR
269 str r2, [vcpu, #VCPU_HSR]
270 str r1, [vcpu, #VCPU_HxFAR]
271 .endif
272 mov r1, #\exception_code
273 b __kvm_vcpu_return
274
275 @ We were in the host already. Let's craft a panic-ing return to SVC.
27699: mrs r2, cpsr
277 bic r2, r2, #MODE_MASK
278 orr r2, r2, #SVC_MODE
279THUMB( orr r2, r2, #PSR_T_BIT )
280 msr spsr_cxsf, r2
281 mrs r1, ELR_hyp
282 ldr r2, =BSYM(panic)
283 msr ELR_hyp, r2
284 ldr r0, =\panic_str
285 eret
286.endm
287
288 .text
289
290 .align 5
291__kvm_hyp_vector:
292 .globl __kvm_hyp_vector
293
294 @ Hyp-mode exception vector
295 W(b) hyp_reset
296 W(b) hyp_undef
297 W(b) hyp_svc
298 W(b) hyp_pabt
299 W(b) hyp_dabt
300 W(b) hyp_hvc
301 W(b) hyp_irq
302 W(b) hyp_fiq
303
304 .align
305hyp_reset:
306 b hyp_reset
307
308 .align
309hyp_undef:
310 bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
311
312 .align
313hyp_svc:
314 bad_exception ARM_EXCEPTION_HVC, svc_die_str
315
316 .align
317hyp_pabt:
318 bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
319
320 .align
321hyp_dabt:
322 bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
323
324 .align
325hyp_hvc:
326 /*
327 * Getting here is either becuase of a trap from a guest or from calling
328 * HVC from the host kernel, which means "switch to Hyp mode".
329 */
330 push {r0, r1, r2}
331
332 @ Check syndrome register
333 mrc p15, 4, r1, c5, c2, 0 @ HSR
334 lsr r0, r1, #HSR_EC_SHIFT
335#ifdef CONFIG_VFPv3
336 cmp r0, #HSR_EC_CP_0_13
337 beq switch_to_guest_vfp
338#endif
339 cmp r0, #HSR_EC_HVC
340 bne guest_trap @ Not HVC instr.
341
342 /*
343 * Let's check if the HVC came from VMID 0 and allow simple
344 * switch to Hyp mode
345 */
346 mrrc p15, 6, r0, r2, c2
347 lsr r2, r2, #16
348 and r2, r2, #0xff
349 cmp r2, #0
350 bne guest_trap @ Guest called HVC
351
352host_switch_to_hyp:
353 pop {r0, r1, r2}
354
355 push {lr}
356 mrs lr, SPSR
357 push {lr}
358
359 mov lr, r0
360 mov r0, r1
361 mov r1, r2
362 mov r2, r3
363
364THUMB( orr lr, #1)
365 blx lr @ Call the HYP function
366
367 pop {lr}
368 msr SPSR_csxf, lr
369 pop {lr}
370 eret
371
372guest_trap:
373 load_vcpu @ Load VCPU pointer to r0
374 str r1, [vcpu, #VCPU_HSR]
375
376 @ Check if we need the fault information
377 lsr r1, r1, #HSR_EC_SHIFT
378 cmp r1, #HSR_EC_IABT
379 mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
380 beq 2f
381 cmp r1, #HSR_EC_DABT
382 bne 1f
383 mrc p15, 4, r2, c6, c0, 0 @ HDFAR
384
3852: str r2, [vcpu, #VCPU_HxFAR]
386
387 /*
388 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
389 *
390 * Abort on the stage 2 translation for a memory access from a
391 * Non-secure PL1 or PL0 mode:
392 *
393 * For any Access flag fault or Translation fault, and also for any
394 * Permission fault on the stage 2 translation of a memory access
395 * made as part of a translation table walk for a stage 1 translation,
396 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
397 * is UNKNOWN.
398 */
399
400 /* Check for permission fault, and S1PTW */
401 mrc p15, 4, r1, c5, c2, 0 @ HSR
402 and r0, r1, #HSR_FSC_TYPE
403 cmp r0, #FSC_PERM
404 tsteq r1, #(1 << 7) @ S1PTW
405 mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
406 bne 3f
407
408 /* Resolve IPA using the xFAR */
409 mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
410 isb
411 mrrc p15, 0, r0, r1, c7 @ PAR
412 tst r0, #1
413 bne 4f @ Failed translation
414 ubfx r2, r0, #12, #20
415 lsl r2, r2, #4
416 orr r2, r2, r1, lsl #24
417
4183: load_vcpu @ Load VCPU pointer to r0
419 str r2, [r0, #VCPU_HPFAR]
420
4211: mov r1, #ARM_EXCEPTION_HVC
422 b __kvm_vcpu_return
423
4244: pop {r0, r1, r2} @ Failed translation, return to guest
425 eret
426
427/*
428 * If VFPv3 support is not available, then we will not switch the VFP
429 * registers; however cp10 and cp11 accesses will still trap and fallback
430 * to the regular coprocessor emulation code, which currently will
431 * inject an undefined exception to the guest.
432 */
433#ifdef CONFIG_VFPv3
434switch_to_guest_vfp:
435 load_vcpu @ Load VCPU pointer to r0
436 push {r3-r7}
437
438 @ NEON/VFP used. Turn on VFP access.
439 set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11))
440
441 @ Switch VFP/NEON hardware state to the guest's
442 add r7, r0, #VCPU_VFP_HOST
443 ldr r7, [r7]
444 store_vfp_state r7
445 add r7, r0, #VCPU_VFP_GUEST
446 restore_vfp_state r7
447
448 pop {r3-r7}
449 pop {r0-r2}
450 eret
451#endif
452
453 .align
454hyp_irq:
455 push {r0, r1, r2}
456 mov r1, #ARM_EXCEPTION_IRQ
457 load_vcpu @ Load VCPU pointer to r0
458 b __kvm_vcpu_return
459
460 .align
461hyp_fiq:
462 b hyp_fiq
463
464 .ltorg
465
466__kvm_hyp_code_end:
467 .globl __kvm_hyp_code_end
468
469 .section ".rodata"
470
471und_die_str:
472 .ascii "unexpected undefined exception in Hyp mode at: %#08x"
473pabt_die_str:
474 .ascii "unexpected prefetch abort in Hyp mode at: %#08x"
475dabt_die_str:
476 .ascii "unexpected data abort in Hyp mode at: %#08x"
477svc_die_str:
478 .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x"
diff --git a/arch/arm/kvm/interrupts_head.S b/arch/arm/kvm/interrupts_head.S
new file mode 100644
index 000000000000..6a95d341e9c5
--- /dev/null
+++ b/arch/arm/kvm/interrupts_head.S
@@ -0,0 +1,441 @@
1#define VCPU_USR_REG(_reg_nr) (VCPU_USR_REGS + (_reg_nr * 4))
2#define VCPU_USR_SP (VCPU_USR_REG(13))
3#define VCPU_USR_LR (VCPU_USR_REG(14))
4#define CP15_OFFSET(_cp15_reg_idx) (VCPU_CP15 + (_cp15_reg_idx * 4))
5
6/*
7 * Many of these macros need to access the VCPU structure, which is always
8 * held in r0. These macros should never clobber r1, as it is used to hold the
9 * exception code on the return path (except of course the macro that switches
10 * all the registers before the final jump to the VM).
11 */
12vcpu .req r0 @ vcpu pointer always in r0
13
14/* Clobbers {r2-r6} */
15.macro store_vfp_state vfp_base
16 @ The VFPFMRX and VFPFMXR macros are the VMRS and VMSR instructions
17 VFPFMRX r2, FPEXC
18 @ Make sure VFP is enabled so we can touch the registers.
19 orr r6, r2, #FPEXC_EN
20 VFPFMXR FPEXC, r6
21
22 VFPFMRX r3, FPSCR
23 tst r2, #FPEXC_EX @ Check for VFP Subarchitecture
24 beq 1f
25 @ If FPEXC_EX is 0, then FPINST/FPINST2 reads are upredictable, so
26 @ we only need to save them if FPEXC_EX is set.
27 VFPFMRX r4, FPINST
28 tst r2, #FPEXC_FP2V
29 VFPFMRX r5, FPINST2, ne @ vmrsne
30 bic r6, r2, #FPEXC_EX @ FPEXC_EX disable
31 VFPFMXR FPEXC, r6
321:
33 VFPFSTMIA \vfp_base, r6 @ Save VFP registers
34 stm \vfp_base, {r2-r5} @ Save FPEXC, FPSCR, FPINST, FPINST2
35.endm
36
37/* Assume FPEXC_EN is on and FPEXC_EX is off, clobbers {r2-r6} */
38.macro restore_vfp_state vfp_base
39 VFPFLDMIA \vfp_base, r6 @ Load VFP registers
40 ldm \vfp_base, {r2-r5} @ Load FPEXC, FPSCR, FPINST, FPINST2
41
42 VFPFMXR FPSCR, r3
43 tst r2, #FPEXC_EX @ Check for VFP Subarchitecture
44 beq 1f
45 VFPFMXR FPINST, r4
46 tst r2, #FPEXC_FP2V
47 VFPFMXR FPINST2, r5, ne
481:
49 VFPFMXR FPEXC, r2 @ FPEXC (last, in case !EN)
50.endm
51
52/* These are simply for the macros to work - value don't have meaning */
53.equ usr, 0
54.equ svc, 1
55.equ abt, 2
56.equ und, 3
57.equ irq, 4
58.equ fiq, 5
59
60.macro push_host_regs_mode mode
61 mrs r2, SP_\mode
62 mrs r3, LR_\mode
63 mrs r4, SPSR_\mode
64 push {r2, r3, r4}
65.endm
66
67/*
68 * Store all host persistent registers on the stack.
69 * Clobbers all registers, in all modes, except r0 and r1.
70 */
71.macro save_host_regs
72 /* Hyp regs. Only ELR_hyp (SPSR_hyp already saved) */
73 mrs r2, ELR_hyp
74 push {r2}
75
76 /* usr regs */
77 push {r4-r12} @ r0-r3 are always clobbered
78 mrs r2, SP_usr
79 mov r3, lr
80 push {r2, r3}
81
82 push_host_regs_mode svc
83 push_host_regs_mode abt
84 push_host_regs_mode und
85 push_host_regs_mode irq
86
87 /* fiq regs */
88 mrs r2, r8_fiq
89 mrs r3, r9_fiq
90 mrs r4, r10_fiq
91 mrs r5, r11_fiq
92 mrs r6, r12_fiq
93 mrs r7, SP_fiq
94 mrs r8, LR_fiq
95 mrs r9, SPSR_fiq
96 push {r2-r9}
97.endm
98
99.macro pop_host_regs_mode mode
100 pop {r2, r3, r4}
101 msr SP_\mode, r2
102 msr LR_\mode, r3
103 msr SPSR_\mode, r4
104.endm
105
106/*
107 * Restore all host registers from the stack.
108 * Clobbers all registers, in all modes, except r0 and r1.
109 */
110.macro restore_host_regs
111 pop {r2-r9}
112 msr r8_fiq, r2
113 msr r9_fiq, r3
114 msr r10_fiq, r4
115 msr r11_fiq, r5
116 msr r12_fiq, r6
117 msr SP_fiq, r7
118 msr LR_fiq, r8
119 msr SPSR_fiq, r9
120
121 pop_host_regs_mode irq
122 pop_host_regs_mode und
123 pop_host_regs_mode abt
124 pop_host_regs_mode svc
125
126 pop {r2, r3}
127 msr SP_usr, r2
128 mov lr, r3
129 pop {r4-r12}
130
131 pop {r2}
132 msr ELR_hyp, r2
133.endm
134
135/*
136 * Restore SP, LR and SPSR for a given mode. offset is the offset of
137 * this mode's registers from the VCPU base.
138 *
139 * Assumes vcpu pointer in vcpu reg
140 *
141 * Clobbers r1, r2, r3, r4.
142 */
143.macro restore_guest_regs_mode mode, offset
144 add r1, vcpu, \offset
145 ldm r1, {r2, r3, r4}
146 msr SP_\mode, r2
147 msr LR_\mode, r3
148 msr SPSR_\mode, r4
149.endm
150
151/*
152 * Restore all guest registers from the vcpu struct.
153 *
154 * Assumes vcpu pointer in vcpu reg
155 *
156 * Clobbers *all* registers.
157 */
158.macro restore_guest_regs
159 restore_guest_regs_mode svc, #VCPU_SVC_REGS
160 restore_guest_regs_mode abt, #VCPU_ABT_REGS
161 restore_guest_regs_mode und, #VCPU_UND_REGS
162 restore_guest_regs_mode irq, #VCPU_IRQ_REGS
163
164 add r1, vcpu, #VCPU_FIQ_REGS
165 ldm r1, {r2-r9}
166 msr r8_fiq, r2
167 msr r9_fiq, r3
168 msr r10_fiq, r4
169 msr r11_fiq, r5
170 msr r12_fiq, r6
171 msr SP_fiq, r7
172 msr LR_fiq, r8
173 msr SPSR_fiq, r9
174
175 @ Load return state
176 ldr r2, [vcpu, #VCPU_PC]
177 ldr r3, [vcpu, #VCPU_CPSR]
178 msr ELR_hyp, r2
179 msr SPSR_cxsf, r3
180
181 @ Load user registers
182 ldr r2, [vcpu, #VCPU_USR_SP]
183 ldr r3, [vcpu, #VCPU_USR_LR]
184 msr SP_usr, r2
185 mov lr, r3
186 add vcpu, vcpu, #(VCPU_USR_REGS)
187 ldm vcpu, {r0-r12}
188.endm
189
190/*
191 * Save SP, LR and SPSR for a given mode. offset is the offset of
192 * this mode's registers from the VCPU base.
193 *
194 * Assumes vcpu pointer in vcpu reg
195 *
196 * Clobbers r2, r3, r4, r5.
197 */
198.macro save_guest_regs_mode mode, offset
199 add r2, vcpu, \offset
200 mrs r3, SP_\mode
201 mrs r4, LR_\mode
202 mrs r5, SPSR_\mode
203 stm r2, {r3, r4, r5}
204.endm
205
206/*
207 * Save all guest registers to the vcpu struct
208 * Expects guest's r0, r1, r2 on the stack.
209 *
210 * Assumes vcpu pointer in vcpu reg
211 *
212 * Clobbers r2, r3, r4, r5.
213 */
214.macro save_guest_regs
215 @ Store usr registers
216 add r2, vcpu, #VCPU_USR_REG(3)
217 stm r2, {r3-r12}
218 add r2, vcpu, #VCPU_USR_REG(0)
219 pop {r3, r4, r5} @ r0, r1, r2
220 stm r2, {r3, r4, r5}
221 mrs r2, SP_usr
222 mov r3, lr
223 str r2, [vcpu, #VCPU_USR_SP]
224 str r3, [vcpu, #VCPU_USR_LR]
225
226 @ Store return state
227 mrs r2, ELR_hyp
228 mrs r3, spsr
229 str r2, [vcpu, #VCPU_PC]
230 str r3, [vcpu, #VCPU_CPSR]
231
232 @ Store other guest registers
233 save_guest_regs_mode svc, #VCPU_SVC_REGS
234 save_guest_regs_mode abt, #VCPU_ABT_REGS
235 save_guest_regs_mode und, #VCPU_UND_REGS
236 save_guest_regs_mode irq, #VCPU_IRQ_REGS
237.endm
238
239/* Reads cp15 registers from hardware and stores them in memory
240 * @store_to_vcpu: If 0, registers are written in-order to the stack,
241 * otherwise to the VCPU struct pointed to by vcpup
242 *
243 * Assumes vcpu pointer in vcpu reg
244 *
245 * Clobbers r2 - r12
246 */
247.macro read_cp15_state store_to_vcpu
248 mrc p15, 0, r2, c1, c0, 0 @ SCTLR
249 mrc p15, 0, r3, c1, c0, 2 @ CPACR
250 mrc p15, 0, r4, c2, c0, 2 @ TTBCR
251 mrc p15, 0, r5, c3, c0, 0 @ DACR
252 mrrc p15, 0, r6, r7, c2 @ TTBR 0
253 mrrc p15, 1, r8, r9, c2 @ TTBR 1
254 mrc p15, 0, r10, c10, c2, 0 @ PRRR
255 mrc p15, 0, r11, c10, c2, 1 @ NMRR
256 mrc p15, 2, r12, c0, c0, 0 @ CSSELR
257
258 .if \store_to_vcpu == 0
259 push {r2-r12} @ Push CP15 registers
260 .else
261 str r2, [vcpu, #CP15_OFFSET(c1_SCTLR)]
262 str r3, [vcpu, #CP15_OFFSET(c1_CPACR)]
263 str r4, [vcpu, #CP15_OFFSET(c2_TTBCR)]
264 str r5, [vcpu, #CP15_OFFSET(c3_DACR)]
265 add r2, vcpu, #CP15_OFFSET(c2_TTBR0)
266 strd r6, r7, [r2]
267 add r2, vcpu, #CP15_OFFSET(c2_TTBR1)
268 strd r8, r9, [r2]
269 str r10, [vcpu, #CP15_OFFSET(c10_PRRR)]
270 str r11, [vcpu, #CP15_OFFSET(c10_NMRR)]
271 str r12, [vcpu, #CP15_OFFSET(c0_CSSELR)]
272 .endif
273
274 mrc p15, 0, r2, c13, c0, 1 @ CID
275 mrc p15, 0, r3, c13, c0, 2 @ TID_URW
276 mrc p15, 0, r4, c13, c0, 3 @ TID_URO
277 mrc p15, 0, r5, c13, c0, 4 @ TID_PRIV
278 mrc p15, 0, r6, c5, c0, 0 @ DFSR
279 mrc p15, 0, r7, c5, c0, 1 @ IFSR
280 mrc p15, 0, r8, c5, c1, 0 @ ADFSR
281 mrc p15, 0, r9, c5, c1, 1 @ AIFSR
282 mrc p15, 0, r10, c6, c0, 0 @ DFAR
283 mrc p15, 0, r11, c6, c0, 2 @ IFAR
284 mrc p15, 0, r12, c12, c0, 0 @ VBAR
285
286 .if \store_to_vcpu == 0
287 push {r2-r12} @ Push CP15 registers
288 .else
289 str r2, [vcpu, #CP15_OFFSET(c13_CID)]
290 str r3, [vcpu, #CP15_OFFSET(c13_TID_URW)]
291 str r4, [vcpu, #CP15_OFFSET(c13_TID_URO)]
292 str r5, [vcpu, #CP15_OFFSET(c13_TID_PRIV)]
293 str r6, [vcpu, #CP15_OFFSET(c5_DFSR)]
294 str r7, [vcpu, #CP15_OFFSET(c5_IFSR)]
295 str r8, [vcpu, #CP15_OFFSET(c5_ADFSR)]
296 str r9, [vcpu, #CP15_OFFSET(c5_AIFSR)]
297 str r10, [vcpu, #CP15_OFFSET(c6_DFAR)]
298 str r11, [vcpu, #CP15_OFFSET(c6_IFAR)]
299 str r12, [vcpu, #CP15_OFFSET(c12_VBAR)]
300 .endif
301.endm
302
303/*
304 * Reads cp15 registers from memory and writes them to hardware
305 * @read_from_vcpu: If 0, registers are read in-order from the stack,
306 * otherwise from the VCPU struct pointed to by vcpup
307 *
308 * Assumes vcpu pointer in vcpu reg
309 */
310.macro write_cp15_state read_from_vcpu
311 .if \read_from_vcpu == 0
312 pop {r2-r12}
313 .else
314 ldr r2, [vcpu, #CP15_OFFSET(c13_CID)]
315 ldr r3, [vcpu, #CP15_OFFSET(c13_TID_URW)]
316 ldr r4, [vcpu, #CP15_OFFSET(c13_TID_URO)]
317 ldr r5, [vcpu, #CP15_OFFSET(c13_TID_PRIV)]
318 ldr r6, [vcpu, #CP15_OFFSET(c5_DFSR)]
319 ldr r7, [vcpu, #CP15_OFFSET(c5_IFSR)]
320 ldr r8, [vcpu, #CP15_OFFSET(c5_ADFSR)]
321 ldr r9, [vcpu, #CP15_OFFSET(c5_AIFSR)]
322 ldr r10, [vcpu, #CP15_OFFSET(c6_DFAR)]
323 ldr r11, [vcpu, #CP15_OFFSET(c6_IFAR)]
324 ldr r12, [vcpu, #CP15_OFFSET(c12_VBAR)]
325 .endif
326
327 mcr p15, 0, r2, c13, c0, 1 @ CID
328 mcr p15, 0, r3, c13, c0, 2 @ TID_URW
329 mcr p15, 0, r4, c13, c0, 3 @ TID_URO
330 mcr p15, 0, r5, c13, c0, 4 @ TID_PRIV
331 mcr p15, 0, r6, c5, c0, 0 @ DFSR
332 mcr p15, 0, r7, c5, c0, 1 @ IFSR
333 mcr p15, 0, r8, c5, c1, 0 @ ADFSR
334 mcr p15, 0, r9, c5, c1, 1 @ AIFSR
335 mcr p15, 0, r10, c6, c0, 0 @ DFAR
336 mcr p15, 0, r11, c6, c0, 2 @ IFAR
337 mcr p15, 0, r12, c12, c0, 0 @ VBAR
338
339 .if \read_from_vcpu == 0
340 pop {r2-r12}
341 .else
342 ldr r2, [vcpu, #CP15_OFFSET(c1_SCTLR)]
343 ldr r3, [vcpu, #CP15_OFFSET(c1_CPACR)]
344 ldr r4, [vcpu, #CP15_OFFSET(c2_TTBCR)]
345 ldr r5, [vcpu, #CP15_OFFSET(c3_DACR)]
346 add r12, vcpu, #CP15_OFFSET(c2_TTBR0)
347 ldrd r6, r7, [r12]
348 add r12, vcpu, #CP15_OFFSET(c2_TTBR1)
349 ldrd r8, r9, [r12]
350 ldr r10, [vcpu, #CP15_OFFSET(c10_PRRR)]
351 ldr r11, [vcpu, #CP15_OFFSET(c10_NMRR)]
352 ldr r12, [vcpu, #CP15_OFFSET(c0_CSSELR)]
353 .endif
354
355 mcr p15, 0, r2, c1, c0, 0 @ SCTLR
356 mcr p15, 0, r3, c1, c0, 2 @ CPACR
357 mcr p15, 0, r4, c2, c0, 2 @ TTBCR
358 mcr p15, 0, r5, c3, c0, 0 @ DACR
359 mcrr p15, 0, r6, r7, c2 @ TTBR 0
360 mcrr p15, 1, r8, r9, c2 @ TTBR 1
361 mcr p15, 0, r10, c10, c2, 0 @ PRRR
362 mcr p15, 0, r11, c10, c2, 1 @ NMRR
363 mcr p15, 2, r12, c0, c0, 0 @ CSSELR
364.endm
365
366/*
367 * Save the VGIC CPU state into memory
368 *
369 * Assumes vcpu pointer in vcpu reg
370 */
371.macro save_vgic_state
372.endm
373
374/*
375 * Restore the VGIC CPU state from memory
376 *
377 * Assumes vcpu pointer in vcpu reg
378 */
379.macro restore_vgic_state
380.endm
381
382.equ vmentry, 0
383.equ vmexit, 1
384
385/* Configures the HSTR (Hyp System Trap Register) on entry/return
386 * (hardware reset value is 0) */
387.macro set_hstr operation
388 mrc p15, 4, r2, c1, c1, 3
389 ldr r3, =HSTR_T(15)
390 .if \operation == vmentry
391 orr r2, r2, r3 @ Trap CR{15}
392 .else
393 bic r2, r2, r3 @ Don't trap any CRx accesses
394 .endif
395 mcr p15, 4, r2, c1, c1, 3
396.endm
397
398/* Configures the HCPTR (Hyp Coprocessor Trap Register) on entry/return
399 * (hardware reset value is 0). Keep previous value in r2. */
400.macro set_hcptr operation, mask
401 mrc p15, 4, r2, c1, c1, 2
402 ldr r3, =\mask
403 .if \operation == vmentry
404 orr r3, r2, r3 @ Trap coproc-accesses defined in mask
405 .else
406 bic r3, r2, r3 @ Don't trap defined coproc-accesses
407 .endif
408 mcr p15, 4, r3, c1, c1, 2
409.endm
410
411/* Configures the HDCR (Hyp Debug Configuration Register) on entry/return
412 * (hardware reset value is 0) */
413.macro set_hdcr operation
414 mrc p15, 4, r2, c1, c1, 1
415 ldr r3, =(HDCR_TPM|HDCR_TPMCR)
416 .if \operation == vmentry
417 orr r2, r2, r3 @ Trap some perfmon accesses
418 .else
419 bic r2, r2, r3 @ Don't trap any perfmon accesses
420 .endif
421 mcr p15, 4, r2, c1, c1, 1
422.endm
423
424/* Enable/Disable: stage-2 trans., trap interrupts, trap wfi, trap smc */
425.macro configure_hyp_role operation
426 mrc p15, 4, r2, c1, c1, 0 @ HCR
427 bic r2, r2, #HCR_VIRT_EXCP_MASK
428 ldr r3, =HCR_GUEST_MASK
429 .if \operation == vmentry
430 orr r2, r2, r3
431 ldr r3, [vcpu, #VCPU_IRQ_LINES]
432 orr r2, r2, r3
433 .else
434 bic r2, r2, r3
435 .endif
436 mcr p15, 4, r2, c1, c1, 0
437.endm
438
439.macro load_vcpu
440 mrc p15, 4, vcpu, c13, c0, 2 @ HTPIDR
441.endm
diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c
new file mode 100644
index 000000000000..0144baf82904
--- /dev/null
+++ b/arch/arm/kvm/mmio.c
@@ -0,0 +1,153 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19#include <linux/kvm_host.h>
20#include <asm/kvm_mmio.h>
21#include <asm/kvm_emulate.h>
22#include <trace/events/kvm.h>
23
24#include "trace.h"
25
26/**
27 * kvm_handle_mmio_return -- Handle MMIO loads after user space emulation
28 * @vcpu: The VCPU pointer
29 * @run: The VCPU run struct containing the mmio data
30 *
31 * This should only be called after returning from userspace for MMIO load
32 * emulation.
33 */
34int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
35{
36 __u32 *dest;
37 unsigned int len;
38 int mask;
39
40 if (!run->mmio.is_write) {
41 dest = vcpu_reg(vcpu, vcpu->arch.mmio_decode.rt);
42 memset(dest, 0, sizeof(int));
43
44 len = run->mmio.len;
45 if (len > 4)
46 return -EINVAL;
47
48 memcpy(dest, run->mmio.data, len);
49
50 trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr,
51 *((u64 *)run->mmio.data));
52
53 if (vcpu->arch.mmio_decode.sign_extend && len < 4) {
54 mask = 1U << ((len * 8) - 1);
55 *dest = (*dest ^ mask) - mask;
56 }
57 }
58
59 return 0;
60}
61
62static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
63 struct kvm_exit_mmio *mmio)
64{
65 unsigned long rt, len;
66 bool is_write, sign_extend;
67
68 if ((vcpu->arch.hsr >> 8) & 1) {
69 /* cache operation on I/O addr, tell guest unsupported */
70 kvm_inject_dabt(vcpu, vcpu->arch.hxfar);
71 return 1;
72 }
73
74 if ((vcpu->arch.hsr >> 7) & 1) {
75 /* page table accesses IO mem: tell guest to fix its TTBR */
76 kvm_inject_dabt(vcpu, vcpu->arch.hxfar);
77 return 1;
78 }
79
80 switch ((vcpu->arch.hsr >> 22) & 0x3) {
81 case 0:
82 len = 1;
83 break;
84 case 1:
85 len = 2;
86 break;
87 case 2:
88 len = 4;
89 break;
90 default:
91 kvm_err("Hardware is weird: SAS 0b11 is reserved\n");
92 return -EFAULT;
93 }
94
95 is_write = vcpu->arch.hsr & HSR_WNR;
96 sign_extend = vcpu->arch.hsr & HSR_SSE;
97 rt = (vcpu->arch.hsr & HSR_SRT_MASK) >> HSR_SRT_SHIFT;
98
99 if (kvm_vcpu_reg_is_pc(vcpu, rt)) {
100 /* IO memory trying to read/write pc */
101 kvm_inject_pabt(vcpu, vcpu->arch.hxfar);
102 return 1;
103 }
104
105 mmio->is_write = is_write;
106 mmio->phys_addr = fault_ipa;
107 mmio->len = len;
108 vcpu->arch.mmio_decode.sign_extend = sign_extend;
109 vcpu->arch.mmio_decode.rt = rt;
110
111 /*
112 * The MMIO instruction is emulated and should not be re-executed
113 * in the guest.
114 */
115 kvm_skip_instr(vcpu, (vcpu->arch.hsr >> 25) & 1);
116 return 0;
117}
118
119int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
120 phys_addr_t fault_ipa)
121{
122 struct kvm_exit_mmio mmio;
123 unsigned long rt;
124 int ret;
125
126 /*
127 * Prepare MMIO operation. First stash it in a private
128 * structure that we can use for in-kernel emulation. If the
129 * kernel can't handle it, copy it into run->mmio and let user
130 * space do its magic.
131 */
132
133 if (vcpu->arch.hsr & HSR_ISV) {
134 ret = decode_hsr(vcpu, fault_ipa, &mmio);
135 if (ret)
136 return ret;
137 } else {
138 kvm_err("load/store instruction decoding not implemented\n");
139 return -ENOSYS;
140 }
141
142 rt = vcpu->arch.mmio_decode.rt;
143 trace_kvm_mmio((mmio.is_write) ? KVM_TRACE_MMIO_WRITE :
144 KVM_TRACE_MMIO_READ_UNSATISFIED,
145 mmio.len, fault_ipa,
146 (mmio.is_write) ? *vcpu_reg(vcpu, rt) : 0);
147
148 if (mmio.is_write)
149 memcpy(mmio.data, vcpu_reg(vcpu, rt), mmio.len);
150
151 kvm_prepare_mmio(run, &mmio);
152 return 0;
153}
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
new file mode 100644
index 000000000000..f30e13163a96
--- /dev/null
+++ b/arch/arm/kvm/mmu.c
@@ -0,0 +1,787 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19#include <linux/mman.h>
20#include <linux/kvm_host.h>
21#include <linux/io.h>
22#include <trace/events/kvm.h>
23#include <asm/idmap.h>
24#include <asm/pgalloc.h>
25#include <asm/cacheflush.h>
26#include <asm/kvm_arm.h>
27#include <asm/kvm_mmu.h>
28#include <asm/kvm_mmio.h>
29#include <asm/kvm_asm.h>
30#include <asm/kvm_emulate.h>
31#include <asm/mach/map.h>
32#include <trace/events/kvm.h>
33
34#include "trace.h"
35
36extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
37
38static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
39
40static void kvm_tlb_flush_vmid(struct kvm *kvm)
41{
42 kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
43}
44
45static void kvm_set_pte(pte_t *pte, pte_t new_pte)
46{
47 pte_val(*pte) = new_pte;
48 /*
49 * flush_pmd_entry just takes a void pointer and cleans the necessary
50 * cache entries, so we can reuse the function for ptes.
51 */
52 flush_pmd_entry(pte);
53}
54
55static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
56 int min, int max)
57{
58 void *page;
59
60 BUG_ON(max > KVM_NR_MEM_OBJS);
61 if (cache->nobjs >= min)
62 return 0;
63 while (cache->nobjs < max) {
64 page = (void *)__get_free_page(PGALLOC_GFP);
65 if (!page)
66 return -ENOMEM;
67 cache->objects[cache->nobjs++] = page;
68 }
69 return 0;
70}
71
72static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
73{
74 while (mc->nobjs)
75 free_page((unsigned long)mc->objects[--mc->nobjs]);
76}
77
78static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
79{
80 void *p;
81
82 BUG_ON(!mc || !mc->nobjs);
83 p = mc->objects[--mc->nobjs];
84 return p;
85}
86
87static void free_ptes(pmd_t *pmd, unsigned long addr)
88{
89 pte_t *pte;
90 unsigned int i;
91
92 for (i = 0; i < PTRS_PER_PMD; i++, addr += PMD_SIZE) {
93 if (!pmd_none(*pmd) && pmd_table(*pmd)) {
94 pte = pte_offset_kernel(pmd, addr);
95 pte_free_kernel(NULL, pte);
96 }
97 pmd++;
98 }
99}
100
101/**
102 * free_hyp_pmds - free a Hyp-mode level-2 tables and child level-3 tables
103 *
104 * Assumes this is a page table used strictly in Hyp-mode and therefore contains
105 * only mappings in the kernel memory area, which is above PAGE_OFFSET.
106 */
107void free_hyp_pmds(void)
108{
109 pgd_t *pgd;
110 pud_t *pud;
111 pmd_t *pmd;
112 unsigned long addr;
113
114 mutex_lock(&kvm_hyp_pgd_mutex);
115 for (addr = PAGE_OFFSET; addr != 0; addr += PGDIR_SIZE) {
116 pgd = hyp_pgd + pgd_index(addr);
117 pud = pud_offset(pgd, addr);
118
119 if (pud_none(*pud))
120 continue;
121 BUG_ON(pud_bad(*pud));
122
123 pmd = pmd_offset(pud, addr);
124 free_ptes(pmd, addr);
125 pmd_free(NULL, pmd);
126 pud_clear(pud);
127 }
128 mutex_unlock(&kvm_hyp_pgd_mutex);
129}
130
131static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start,
132 unsigned long end)
133{
134 pte_t *pte;
135 unsigned long addr;
136 struct page *page;
137
138 for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
139 pte = pte_offset_kernel(pmd, addr);
140 BUG_ON(!virt_addr_valid(addr));
141 page = virt_to_page(addr);
142 kvm_set_pte(pte, mk_pte(page, PAGE_HYP));
143 }
144}
145
146static void create_hyp_io_pte_mappings(pmd_t *pmd, unsigned long start,
147 unsigned long end,
148 unsigned long *pfn_base)
149{
150 pte_t *pte;
151 unsigned long addr;
152
153 for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
154 pte = pte_offset_kernel(pmd, addr);
155 BUG_ON(pfn_valid(*pfn_base));
156 kvm_set_pte(pte, pfn_pte(*pfn_base, PAGE_HYP_DEVICE));
157 (*pfn_base)++;
158 }
159}
160
161static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
162 unsigned long end, unsigned long *pfn_base)
163{
164 pmd_t *pmd;
165 pte_t *pte;
166 unsigned long addr, next;
167
168 for (addr = start; addr < end; addr = next) {
169 pmd = pmd_offset(pud, addr);
170
171 BUG_ON(pmd_sect(*pmd));
172
173 if (pmd_none(*pmd)) {
174 pte = pte_alloc_one_kernel(NULL, addr);
175 if (!pte) {
176 kvm_err("Cannot allocate Hyp pte\n");
177 return -ENOMEM;
178 }
179 pmd_populate_kernel(NULL, pmd, pte);
180 }
181
182 next = pmd_addr_end(addr, end);
183
184 /*
185 * If pfn_base is NULL, we map kernel pages into HYP with the
186 * virtual address. Otherwise, this is considered an I/O
187 * mapping and we map the physical region starting at
188 * *pfn_base to [start, end[.
189 */
190 if (!pfn_base)
191 create_hyp_pte_mappings(pmd, addr, next);
192 else
193 create_hyp_io_pte_mappings(pmd, addr, next, pfn_base);
194 }
195
196 return 0;
197}
198
199static int __create_hyp_mappings(void *from, void *to, unsigned long *pfn_base)
200{
201 unsigned long start = (unsigned long)from;
202 unsigned long end = (unsigned long)to;
203 pgd_t *pgd;
204 pud_t *pud;
205 pmd_t *pmd;
206 unsigned long addr, next;
207 int err = 0;
208
209 BUG_ON(start > end);
210 if (start < PAGE_OFFSET)
211 return -EINVAL;
212
213 mutex_lock(&kvm_hyp_pgd_mutex);
214 for (addr = start; addr < end; addr = next) {
215 pgd = hyp_pgd + pgd_index(addr);
216 pud = pud_offset(pgd, addr);
217
218 if (pud_none_or_clear_bad(pud)) {
219 pmd = pmd_alloc_one(NULL, addr);
220 if (!pmd) {
221 kvm_err("Cannot allocate Hyp pmd\n");
222 err = -ENOMEM;
223 goto out;
224 }
225 pud_populate(NULL, pud, pmd);
226 }
227
228 next = pgd_addr_end(addr, end);
229 err = create_hyp_pmd_mappings(pud, addr, next, pfn_base);
230 if (err)
231 goto out;
232 }
233out:
234 mutex_unlock(&kvm_hyp_pgd_mutex);
235 return err;
236}
237
238/**
239 * create_hyp_mappings - map a kernel virtual address range in Hyp mode
240 * @from: The virtual kernel start address of the range
241 * @to: The virtual kernel end address of the range (exclusive)
242 *
243 * The same virtual address as the kernel virtual address is also used in
244 * Hyp-mode mapping to the same underlying physical pages.
245 *
246 * Note: Wrapping around zero in the "to" address is not supported.
247 */
248int create_hyp_mappings(void *from, void *to)
249{
250 return __create_hyp_mappings(from, to, NULL);
251}
252
253/**
254 * create_hyp_io_mappings - map a physical IO range in Hyp mode
255 * @from: The virtual HYP start address of the range
256 * @to: The virtual HYP end address of the range (exclusive)
257 * @addr: The physical start address which gets mapped
258 */
259int create_hyp_io_mappings(void *from, void *to, phys_addr_t addr)
260{
261 unsigned long pfn = __phys_to_pfn(addr);
262 return __create_hyp_mappings(from, to, &pfn);
263}
264
265/**
266 * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
267 * @kvm: The KVM struct pointer for the VM.
268 *
269 * Allocates the 1st level table only of size defined by S2_PGD_ORDER (can
270 * support either full 40-bit input addresses or limited to 32-bit input
271 * addresses). Clears the allocated pages.
272 *
273 * Note we don't need locking here as this is only called when the VM is
274 * created, which can only be done once.
275 */
276int kvm_alloc_stage2_pgd(struct kvm *kvm)
277{
278 pgd_t *pgd;
279
280 if (kvm->arch.pgd != NULL) {
281 kvm_err("kvm_arch already initialized?\n");
282 return -EINVAL;
283 }
284
285 pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, S2_PGD_ORDER);
286 if (!pgd)
287 return -ENOMEM;
288
289 /* stage-2 pgd must be aligned to its size */
290 VM_BUG_ON((unsigned long)pgd & (S2_PGD_SIZE - 1));
291
292 memset(pgd, 0, PTRS_PER_S2_PGD * sizeof(pgd_t));
293 clean_dcache_area(pgd, PTRS_PER_S2_PGD * sizeof(pgd_t));
294 kvm->arch.pgd = pgd;
295
296 return 0;
297}
298
299static void clear_pud_entry(pud_t *pud)
300{
301 pmd_t *pmd_table = pmd_offset(pud, 0);
302 pud_clear(pud);
303 pmd_free(NULL, pmd_table);
304 put_page(virt_to_page(pud));
305}
306
307static void clear_pmd_entry(pmd_t *pmd)
308{
309 pte_t *pte_table = pte_offset_kernel(pmd, 0);
310 pmd_clear(pmd);
311 pte_free_kernel(NULL, pte_table);
312 put_page(virt_to_page(pmd));
313}
314
315static bool pmd_empty(pmd_t *pmd)
316{
317 struct page *pmd_page = virt_to_page(pmd);
318 return page_count(pmd_page) == 1;
319}
320
321static void clear_pte_entry(pte_t *pte)
322{
323 if (pte_present(*pte)) {
324 kvm_set_pte(pte, __pte(0));
325 put_page(virt_to_page(pte));
326 }
327}
328
329static bool pte_empty(pte_t *pte)
330{
331 struct page *pte_page = virt_to_page(pte);
332 return page_count(pte_page) == 1;
333}
334
335/**
336 * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
337 * @kvm: The VM pointer
338 * @start: The intermediate physical base address of the range to unmap
339 * @size: The size of the area to unmap
340 *
341 * Clear a range of stage-2 mappings, lowering the various ref-counts. Must
342 * be called while holding mmu_lock (unless for freeing the stage2 pgd before
343 * destroying the VM), otherwise another faulting VCPU may come in and mess
344 * with things behind our backs.
345 */
346static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
347{
348 pgd_t *pgd;
349 pud_t *pud;
350 pmd_t *pmd;
351 pte_t *pte;
352 phys_addr_t addr = start, end = start + size;
353 u64 range;
354
355 while (addr < end) {
356 pgd = kvm->arch.pgd + pgd_index(addr);
357 pud = pud_offset(pgd, addr);
358 if (pud_none(*pud)) {
359 addr += PUD_SIZE;
360 continue;
361 }
362
363 pmd = pmd_offset(pud, addr);
364 if (pmd_none(*pmd)) {
365 addr += PMD_SIZE;
366 continue;
367 }
368
369 pte = pte_offset_kernel(pmd, addr);
370 clear_pte_entry(pte);
371 range = PAGE_SIZE;
372
373 /* If we emptied the pte, walk back up the ladder */
374 if (pte_empty(pte)) {
375 clear_pmd_entry(pmd);
376 range = PMD_SIZE;
377 if (pmd_empty(pmd)) {
378 clear_pud_entry(pud);
379 range = PUD_SIZE;
380 }
381 }
382
383 addr += range;
384 }
385}
386
387/**
388 * kvm_free_stage2_pgd - free all stage-2 tables
389 * @kvm: The KVM struct pointer for the VM.
390 *
391 * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all
392 * underlying level-2 and level-3 tables before freeing the actual level-1 table
393 * and setting the struct pointer to NULL.
394 *
395 * Note we don't need locking here as this is only called when the VM is
396 * destroyed, which can only be done once.
397 */
398void kvm_free_stage2_pgd(struct kvm *kvm)
399{
400 if (kvm->arch.pgd == NULL)
401 return;
402
403 unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
404 free_pages((unsigned long)kvm->arch.pgd, S2_PGD_ORDER);
405 kvm->arch.pgd = NULL;
406}
407
408
409static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
410 phys_addr_t addr, const pte_t *new_pte, bool iomap)
411{
412 pgd_t *pgd;
413 pud_t *pud;
414 pmd_t *pmd;
415 pte_t *pte, old_pte;
416
417 /* Create 2nd stage page table mapping - Level 1 */
418 pgd = kvm->arch.pgd + pgd_index(addr);
419 pud = pud_offset(pgd, addr);
420 if (pud_none(*pud)) {
421 if (!cache)
422 return 0; /* ignore calls from kvm_set_spte_hva */
423 pmd = mmu_memory_cache_alloc(cache);
424 pud_populate(NULL, pud, pmd);
425 pmd += pmd_index(addr);
426 get_page(virt_to_page(pud));
427 } else
428 pmd = pmd_offset(pud, addr);
429
430 /* Create 2nd stage page table mapping - Level 2 */
431 if (pmd_none(*pmd)) {
432 if (!cache)
433 return 0; /* ignore calls from kvm_set_spte_hva */
434 pte = mmu_memory_cache_alloc(cache);
435 clean_pte_table(pte);
436 pmd_populate_kernel(NULL, pmd, pte);
437 pte += pte_index(addr);
438 get_page(virt_to_page(pmd));
439 } else
440 pte = pte_offset_kernel(pmd, addr);
441
442 if (iomap && pte_present(*pte))
443 return -EFAULT;
444
445 /* Create 2nd stage page table mapping - Level 3 */
446 old_pte = *pte;
447 kvm_set_pte(pte, *new_pte);
448 if (pte_present(old_pte))
449 kvm_tlb_flush_vmid(kvm);
450 else
451 get_page(virt_to_page(pte));
452
453 return 0;
454}
455
456/**
457 * kvm_phys_addr_ioremap - map a device range to guest IPA
458 *
459 * @kvm: The KVM pointer
460 * @guest_ipa: The IPA at which to insert the mapping
461 * @pa: The physical address of the device
462 * @size: The size of the mapping
463 */
464int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
465 phys_addr_t pa, unsigned long size)
466{
467 phys_addr_t addr, end;
468 int ret = 0;
469 unsigned long pfn;
470 struct kvm_mmu_memory_cache cache = { 0, };
471
472 end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK;
473 pfn = __phys_to_pfn(pa);
474
475 for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) {
476 pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE | L_PTE_S2_RDWR);
477
478 ret = mmu_topup_memory_cache(&cache, 2, 2);
479 if (ret)
480 goto out;
481 spin_lock(&kvm->mmu_lock);
482 ret = stage2_set_pte(kvm, &cache, addr, &pte, true);
483 spin_unlock(&kvm->mmu_lock);
484 if (ret)
485 goto out;
486
487 pfn++;
488 }
489
490out:
491 mmu_free_memory_cache(&cache);
492 return ret;
493}
494
495static void coherent_icache_guest_page(struct kvm *kvm, gfn_t gfn)
496{
497 /*
498 * If we are going to insert an instruction page and the icache is
499 * either VIPT or PIPT, there is a potential problem where the host
500 * (or another VM) may have used the same page as this guest, and we
501 * read incorrect data from the icache. If we're using a PIPT cache,
502 * we can invalidate just that page, but if we are using a VIPT cache
503 * we need to invalidate the entire icache - damn shame - as written
504 * in the ARM ARM (DDI 0406C.b - Page B3-1393).
505 *
506 * VIVT caches are tagged using both the ASID and the VMID and doesn't
507 * need any kind of flushing (DDI 0406C.b - Page B3-1392).
508 */
509 if (icache_is_pipt()) {
510 unsigned long hva = gfn_to_hva(kvm, gfn);
511 __cpuc_coherent_user_range(hva, hva + PAGE_SIZE);
512 } else if (!icache_is_vivt_asid_tagged()) {
513 /* any kind of VIPT cache */
514 __flush_icache_all();
515 }
516}
517
518static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
519 gfn_t gfn, struct kvm_memory_slot *memslot,
520 unsigned long fault_status)
521{
522 pte_t new_pte;
523 pfn_t pfn;
524 int ret;
525 bool write_fault, writable;
526 unsigned long mmu_seq;
527 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
528
529 write_fault = kvm_is_write_fault(vcpu->arch.hsr);
530 if (fault_status == FSC_PERM && !write_fault) {
531 kvm_err("Unexpected L2 read permission error\n");
532 return -EFAULT;
533 }
534
535 /* We need minimum second+third level pages */
536 ret = mmu_topup_memory_cache(memcache, 2, KVM_NR_MEM_OBJS);
537 if (ret)
538 return ret;
539
540 mmu_seq = vcpu->kvm->mmu_notifier_seq;
541 /*
542 * Ensure the read of mmu_notifier_seq happens before we call
543 * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
544 * the page we just got a reference to gets unmapped before we have a
545 * chance to grab the mmu_lock, which ensure that if the page gets
546 * unmapped afterwards, the call to kvm_unmap_hva will take it away
547 * from us again properly. This smp_rmb() interacts with the smp_wmb()
548 * in kvm_mmu_notifier_invalidate_<page|range_end>.
549 */
550 smp_rmb();
551
552 pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write_fault, &writable);
553 if (is_error_pfn(pfn))
554 return -EFAULT;
555
556 new_pte = pfn_pte(pfn, PAGE_S2);
557 coherent_icache_guest_page(vcpu->kvm, gfn);
558
559 spin_lock(&vcpu->kvm->mmu_lock);
560 if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
561 goto out_unlock;
562 if (writable) {
563 pte_val(new_pte) |= L_PTE_S2_RDWR;
564 kvm_set_pfn_dirty(pfn);
565 }
566 stage2_set_pte(vcpu->kvm, memcache, fault_ipa, &new_pte, false);
567
568out_unlock:
569 spin_unlock(&vcpu->kvm->mmu_lock);
570 kvm_release_pfn_clean(pfn);
571 return 0;
572}
573
574/**
575 * kvm_handle_guest_abort - handles all 2nd stage aborts
576 * @vcpu: the VCPU pointer
577 * @run: the kvm_run structure
578 *
579 * Any abort that gets to the host is almost guaranteed to be caused by a
580 * missing second stage translation table entry, which can mean that either the
581 * guest simply needs more memory and we must allocate an appropriate page or it
582 * can mean that the guest tried to access I/O memory, which is emulated by user
583 * space. The distinction is based on the IPA causing the fault and whether this
584 * memory region has been registered as standard RAM by user space.
585 */
586int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
587{
588 unsigned long hsr_ec;
589 unsigned long fault_status;
590 phys_addr_t fault_ipa;
591 struct kvm_memory_slot *memslot;
592 bool is_iabt;
593 gfn_t gfn;
594 int ret, idx;
595
596 hsr_ec = vcpu->arch.hsr >> HSR_EC_SHIFT;
597 is_iabt = (hsr_ec == HSR_EC_IABT);
598 fault_ipa = ((phys_addr_t)vcpu->arch.hpfar & HPFAR_MASK) << 8;
599
600 trace_kvm_guest_fault(*vcpu_pc(vcpu), vcpu->arch.hsr,
601 vcpu->arch.hxfar, fault_ipa);
602
603 /* Check the stage-2 fault is trans. fault or write fault */
604 fault_status = (vcpu->arch.hsr & HSR_FSC_TYPE);
605 if (fault_status != FSC_FAULT && fault_status != FSC_PERM) {
606 kvm_err("Unsupported fault status: EC=%#lx DFCS=%#lx\n",
607 hsr_ec, fault_status);
608 return -EFAULT;
609 }
610
611 idx = srcu_read_lock(&vcpu->kvm->srcu);
612
613 gfn = fault_ipa >> PAGE_SHIFT;
614 if (!kvm_is_visible_gfn(vcpu->kvm, gfn)) {
615 if (is_iabt) {
616 /* Prefetch Abort on I/O address */
617 kvm_inject_pabt(vcpu, vcpu->arch.hxfar);
618 ret = 1;
619 goto out_unlock;
620 }
621
622 if (fault_status != FSC_FAULT) {
623 kvm_err("Unsupported fault status on io memory: %#lx\n",
624 fault_status);
625 ret = -EFAULT;
626 goto out_unlock;
627 }
628
629 /* Adjust page offset */
630 fault_ipa |= vcpu->arch.hxfar & ~PAGE_MASK;
631 ret = io_mem_abort(vcpu, run, fault_ipa);
632 goto out_unlock;
633 }
634
635 memslot = gfn_to_memslot(vcpu->kvm, gfn);
636 if (!memslot->user_alloc) {
637 kvm_err("non user-alloc memslots not supported\n");
638 ret = -EINVAL;
639 goto out_unlock;
640 }
641
642 ret = user_mem_abort(vcpu, fault_ipa, gfn, memslot, fault_status);
643 if (ret == 0)
644 ret = 1;
645out_unlock:
646 srcu_read_unlock(&vcpu->kvm->srcu, idx);
647 return ret;
648}
649
650static void handle_hva_to_gpa(struct kvm *kvm,
651 unsigned long start,
652 unsigned long end,
653 void (*handler)(struct kvm *kvm,
654 gpa_t gpa, void *data),
655 void *data)
656{
657 struct kvm_memslots *slots;
658 struct kvm_memory_slot *memslot;
659
660 slots = kvm_memslots(kvm);
661
662 /* we only care about the pages that the guest sees */
663 kvm_for_each_memslot(memslot, slots) {
664 unsigned long hva_start, hva_end;
665 gfn_t gfn, gfn_end;
666
667 hva_start = max(start, memslot->userspace_addr);
668 hva_end = min(end, memslot->userspace_addr +
669 (memslot->npages << PAGE_SHIFT));
670 if (hva_start >= hva_end)
671 continue;
672
673 /*
674 * {gfn(page) | page intersects with [hva_start, hva_end)} =
675 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
676 */
677 gfn = hva_to_gfn_memslot(hva_start, memslot);
678 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
679
680 for (; gfn < gfn_end; ++gfn) {
681 gpa_t gpa = gfn << PAGE_SHIFT;
682 handler(kvm, gpa, data);
683 }
684 }
685}
686
687static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
688{
689 unmap_stage2_range(kvm, gpa, PAGE_SIZE);
690 kvm_tlb_flush_vmid(kvm);
691}
692
693int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
694{
695 unsigned long end = hva + PAGE_SIZE;
696
697 if (!kvm->arch.pgd)
698 return 0;
699
700 trace_kvm_unmap_hva(hva);
701 handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL);
702 return 0;
703}
704
705int kvm_unmap_hva_range(struct kvm *kvm,
706 unsigned long start, unsigned long end)
707{
708 if (!kvm->arch.pgd)
709 return 0;
710
711 trace_kvm_unmap_hva_range(start, end);
712 handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
713 return 0;
714}
715
716static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
717{
718 pte_t *pte = (pte_t *)data;
719
720 stage2_set_pte(kvm, NULL, gpa, pte, false);
721}
722
723
724void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
725{
726 unsigned long end = hva + PAGE_SIZE;
727 pte_t stage2_pte;
728
729 if (!kvm->arch.pgd)
730 return;
731
732 trace_kvm_set_spte_hva(hva);
733 stage2_pte = pfn_pte(pte_pfn(pte), PAGE_S2);
734 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
735}
736
737void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
738{
739 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
740}
741
742phys_addr_t kvm_mmu_get_httbr(void)
743{
744 VM_BUG_ON(!virt_addr_valid(hyp_pgd));
745 return virt_to_phys(hyp_pgd);
746}
747
748int kvm_mmu_init(void)
749{
750 if (!hyp_pgd) {
751 kvm_err("Hyp mode PGD not allocated\n");
752 return -ENOMEM;
753 }
754
755 return 0;
756}
757
758/**
759 * kvm_clear_idmap - remove all idmaps from the hyp pgd
760 *
761 * Free the underlying pmds for all pgds in range and clear the pgds (but
762 * don't free them) afterwards.
763 */
764void kvm_clear_hyp_idmap(void)
765{
766 unsigned long addr, end;
767 unsigned long next;
768 pgd_t *pgd = hyp_pgd;
769 pud_t *pud;
770 pmd_t *pmd;
771
772 addr = virt_to_phys(__hyp_idmap_text_start);
773 end = virt_to_phys(__hyp_idmap_text_end);
774
775 pgd += pgd_index(addr);
776 do {
777 next = pgd_addr_end(addr, end);
778 if (pgd_none_or_clear_bad(pgd))
779 continue;
780 pud = pud_offset(pgd, addr);
781 pmd = pmd_offset(pud, addr);
782
783 pud_clear(pud);
784 clean_pmd_entry(pmd);
785 pmd_free(NULL, (pmd_t *)((unsigned long)pmd & PAGE_MASK));
786 } while (pgd++, addr = next, addr < end);
787}
diff --git a/arch/arm/kvm/psci.c b/arch/arm/kvm/psci.c
new file mode 100644
index 000000000000..7ee5bb7a3667
--- /dev/null
+++ b/arch/arm/kvm/psci.c
@@ -0,0 +1,108 @@
1/*
2 * Copyright (C) 2012 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/kvm_host.h>
19#include <linux/wait.h>
20
21#include <asm/kvm_emulate.h>
22#include <asm/kvm_psci.h>
23
24/*
25 * This is an implementation of the Power State Coordination Interface
26 * as described in ARM document number ARM DEN 0022A.
27 */
28
29static void kvm_psci_vcpu_off(struct kvm_vcpu *vcpu)
30{
31 vcpu->arch.pause = true;
32}
33
34static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
35{
36 struct kvm *kvm = source_vcpu->kvm;
37 struct kvm_vcpu *vcpu;
38 wait_queue_head_t *wq;
39 unsigned long cpu_id;
40 phys_addr_t target_pc;
41
42 cpu_id = *vcpu_reg(source_vcpu, 1);
43 if (vcpu_mode_is_32bit(source_vcpu))
44 cpu_id &= ~((u32) 0);
45
46 if (cpu_id >= atomic_read(&kvm->online_vcpus))
47 return KVM_PSCI_RET_INVAL;
48
49 target_pc = *vcpu_reg(source_vcpu, 2);
50
51 vcpu = kvm_get_vcpu(kvm, cpu_id);
52
53 wq = kvm_arch_vcpu_wq(vcpu);
54 if (!waitqueue_active(wq))
55 return KVM_PSCI_RET_INVAL;
56
57 kvm_reset_vcpu(vcpu);
58
59 /* Gracefully handle Thumb2 entry point */
60 if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) {
61 target_pc &= ~((phys_addr_t) 1);
62 vcpu_set_thumb(vcpu);
63 }
64
65 *vcpu_pc(vcpu) = target_pc;
66 vcpu->arch.pause = false;
67 smp_mb(); /* Make sure the above is visible */
68
69 wake_up_interruptible(wq);
70
71 return KVM_PSCI_RET_SUCCESS;
72}
73
74/**
75 * kvm_psci_call - handle PSCI call if r0 value is in range
76 * @vcpu: Pointer to the VCPU struct
77 *
78 * Handle PSCI calls from guests through traps from HVC or SMC instructions.
79 * The calling convention is similar to SMC calls to the secure world where
80 * the function number is placed in r0 and this function returns true if the
81 * function number specified in r0 is withing the PSCI range, and false
82 * otherwise.
83 */
84bool kvm_psci_call(struct kvm_vcpu *vcpu)
85{
86 unsigned long psci_fn = *vcpu_reg(vcpu, 0) & ~((u32) 0);
87 unsigned long val;
88
89 switch (psci_fn) {
90 case KVM_PSCI_FN_CPU_OFF:
91 kvm_psci_vcpu_off(vcpu);
92 val = KVM_PSCI_RET_SUCCESS;
93 break;
94 case KVM_PSCI_FN_CPU_ON:
95 val = kvm_psci_vcpu_on(vcpu);
96 break;
97 case KVM_PSCI_FN_CPU_SUSPEND:
98 case KVM_PSCI_FN_MIGRATE:
99 val = KVM_PSCI_RET_NI;
100 break;
101
102 default:
103 return false;
104 }
105
106 *vcpu_reg(vcpu, 0) = val;
107 return true;
108}
diff --git a/arch/arm/kvm/reset.c b/arch/arm/kvm/reset.c
new file mode 100644
index 000000000000..b80256b554cd
--- /dev/null
+++ b/arch/arm/kvm/reset.c
@@ -0,0 +1,74 @@
1/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18#include <linux/compiler.h>
19#include <linux/errno.h>
20#include <linux/sched.h>
21#include <linux/kvm_host.h>
22#include <linux/kvm.h>
23
24#include <asm/unified.h>
25#include <asm/ptrace.h>
26#include <asm/cputype.h>
27#include <asm/kvm_arm.h>
28#include <asm/kvm_coproc.h>
29
30/******************************************************************************
31 * Cortex-A15 Reset Values
32 */
33
34static const int a15_max_cpu_idx = 3;
35
36static struct kvm_regs a15_regs_reset = {
37 .usr_regs.ARM_cpsr = SVC_MODE | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT,
38};
39
40
41/*******************************************************************************
42 * Exported reset function
43 */
44
45/**
46 * kvm_reset_vcpu - sets core registers and cp15 registers to reset value
47 * @vcpu: The VCPU pointer
48 *
49 * This function finds the right table above and sets the registers on the
50 * virtual CPU struct to their architectually defined reset values.
51 */
52int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
53{
54 struct kvm_regs *cpu_reset;
55
56 switch (vcpu->arch.target) {
57 case KVM_ARM_TARGET_CORTEX_A15:
58 if (vcpu->vcpu_id > a15_max_cpu_idx)
59 return -EINVAL;
60 cpu_reset = &a15_regs_reset;
61 vcpu->arch.midr = read_cpuid_id();
62 break;
63 default:
64 return -ENODEV;
65 }
66
67 /* Reset core registers */
68 memcpy(&vcpu->arch.regs, cpu_reset, sizeof(vcpu->arch.regs));
69
70 /* Reset CP15 registers */
71 kvm_reset_coprocs(vcpu);
72
73 return 0;
74}
diff --git a/arch/arm/kvm/trace.h b/arch/arm/kvm/trace.h
new file mode 100644
index 000000000000..a8e73ed5ad5b
--- /dev/null
+++ b/arch/arm/kvm/trace.h
@@ -0,0 +1,235 @@
1#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
2#define _TRACE_KVM_H
3
4#include <linux/tracepoint.h>
5
6#undef TRACE_SYSTEM
7#define TRACE_SYSTEM kvm
8
9/*
10 * Tracepoints for entry/exit to guest
11 */
12TRACE_EVENT(kvm_entry,
13 TP_PROTO(unsigned long vcpu_pc),
14 TP_ARGS(vcpu_pc),
15
16 TP_STRUCT__entry(
17 __field( unsigned long, vcpu_pc )
18 ),
19
20 TP_fast_assign(
21 __entry->vcpu_pc = vcpu_pc;
22 ),
23
24 TP_printk("PC: 0x%08lx", __entry->vcpu_pc)
25);
26
27TRACE_EVENT(kvm_exit,
28 TP_PROTO(unsigned long vcpu_pc),
29 TP_ARGS(vcpu_pc),
30
31 TP_STRUCT__entry(
32 __field( unsigned long, vcpu_pc )
33 ),
34
35 TP_fast_assign(
36 __entry->vcpu_pc = vcpu_pc;
37 ),
38
39 TP_printk("PC: 0x%08lx", __entry->vcpu_pc)
40);
41
42TRACE_EVENT(kvm_guest_fault,
43 TP_PROTO(unsigned long vcpu_pc, unsigned long hsr,
44 unsigned long hxfar,
45 unsigned long long ipa),
46 TP_ARGS(vcpu_pc, hsr, hxfar, ipa),
47
48 TP_STRUCT__entry(
49 __field( unsigned long, vcpu_pc )
50 __field( unsigned long, hsr )
51 __field( unsigned long, hxfar )
52 __field( unsigned long long, ipa )
53 ),
54
55 TP_fast_assign(
56 __entry->vcpu_pc = vcpu_pc;
57 __entry->hsr = hsr;
58 __entry->hxfar = hxfar;
59 __entry->ipa = ipa;
60 ),
61
62 TP_printk("guest fault at PC %#08lx (hxfar %#08lx, "
63 "ipa %#16llx, hsr %#08lx",
64 __entry->vcpu_pc, __entry->hxfar,
65 __entry->ipa, __entry->hsr)
66);
67
68TRACE_EVENT(kvm_irq_line,
69 TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level),
70 TP_ARGS(type, vcpu_idx, irq_num, level),
71
72 TP_STRUCT__entry(
73 __field( unsigned int, type )
74 __field( int, vcpu_idx )
75 __field( int, irq_num )
76 __field( int, level )
77 ),
78
79 TP_fast_assign(
80 __entry->type = type;
81 __entry->vcpu_idx = vcpu_idx;
82 __entry->irq_num = irq_num;
83 __entry->level = level;
84 ),
85
86 TP_printk("Inject %s interrupt (%d), vcpu->idx: %d, num: %d, level: %d",
87 (__entry->type == KVM_ARM_IRQ_TYPE_CPU) ? "CPU" :
88 (__entry->type == KVM_ARM_IRQ_TYPE_PPI) ? "VGIC PPI" :
89 (__entry->type == KVM_ARM_IRQ_TYPE_SPI) ? "VGIC SPI" : "UNKNOWN",
90 __entry->type, __entry->vcpu_idx, __entry->irq_num, __entry->level)
91);
92
93TRACE_EVENT(kvm_mmio_emulate,
94 TP_PROTO(unsigned long vcpu_pc, unsigned long instr,
95 unsigned long cpsr),
96 TP_ARGS(vcpu_pc, instr, cpsr),
97
98 TP_STRUCT__entry(
99 __field( unsigned long, vcpu_pc )
100 __field( unsigned long, instr )
101 __field( unsigned long, cpsr )
102 ),
103
104 TP_fast_assign(
105 __entry->vcpu_pc = vcpu_pc;
106 __entry->instr = instr;
107 __entry->cpsr = cpsr;
108 ),
109
110 TP_printk("Emulate MMIO at: 0x%08lx (instr: %08lx, cpsr: %08lx)",
111 __entry->vcpu_pc, __entry->instr, __entry->cpsr)
112);
113
114/* Architecturally implementation defined CP15 register access */
115TRACE_EVENT(kvm_emulate_cp15_imp,
116 TP_PROTO(unsigned long Op1, unsigned long Rt1, unsigned long CRn,
117 unsigned long CRm, unsigned long Op2, bool is_write),
118 TP_ARGS(Op1, Rt1, CRn, CRm, Op2, is_write),
119
120 TP_STRUCT__entry(
121 __field( unsigned int, Op1 )
122 __field( unsigned int, Rt1 )
123 __field( unsigned int, CRn )
124 __field( unsigned int, CRm )
125 __field( unsigned int, Op2 )
126 __field( bool, is_write )
127 ),
128
129 TP_fast_assign(
130 __entry->is_write = is_write;
131 __entry->Op1 = Op1;
132 __entry->Rt1 = Rt1;
133 __entry->CRn = CRn;
134 __entry->CRm = CRm;
135 __entry->Op2 = Op2;
136 ),
137
138 TP_printk("Implementation defined CP15: %s\tp15, %u, r%u, c%u, c%u, %u",
139 (__entry->is_write) ? "mcr" : "mrc",
140 __entry->Op1, __entry->Rt1, __entry->CRn,
141 __entry->CRm, __entry->Op2)
142);
143
144TRACE_EVENT(kvm_wfi,
145 TP_PROTO(unsigned long vcpu_pc),
146 TP_ARGS(vcpu_pc),
147
148 TP_STRUCT__entry(
149 __field( unsigned long, vcpu_pc )
150 ),
151
152 TP_fast_assign(
153 __entry->vcpu_pc = vcpu_pc;
154 ),
155
156 TP_printk("guest executed wfi at: 0x%08lx", __entry->vcpu_pc)
157);
158
159TRACE_EVENT(kvm_unmap_hva,
160 TP_PROTO(unsigned long hva),
161 TP_ARGS(hva),
162
163 TP_STRUCT__entry(
164 __field( unsigned long, hva )
165 ),
166
167 TP_fast_assign(
168 __entry->hva = hva;
169 ),
170
171 TP_printk("mmu notifier unmap hva: %#08lx", __entry->hva)
172);
173
174TRACE_EVENT(kvm_unmap_hva_range,
175 TP_PROTO(unsigned long start, unsigned long end),
176 TP_ARGS(start, end),
177
178 TP_STRUCT__entry(
179 __field( unsigned long, start )
180 __field( unsigned long, end )
181 ),
182
183 TP_fast_assign(
184 __entry->start = start;
185 __entry->end = end;
186 ),
187
188 TP_printk("mmu notifier unmap range: %#08lx -- %#08lx",
189 __entry->start, __entry->end)
190);
191
192TRACE_EVENT(kvm_set_spte_hva,
193 TP_PROTO(unsigned long hva),
194 TP_ARGS(hva),
195
196 TP_STRUCT__entry(
197 __field( unsigned long, hva )
198 ),
199
200 TP_fast_assign(
201 __entry->hva = hva;
202 ),
203
204 TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva)
205);
206
207TRACE_EVENT(kvm_hvc,
208 TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
209 TP_ARGS(vcpu_pc, r0, imm),
210
211 TP_STRUCT__entry(
212 __field( unsigned long, vcpu_pc )
213 __field( unsigned long, r0 )
214 __field( unsigned long, imm )
215 ),
216
217 TP_fast_assign(
218 __entry->vcpu_pc = vcpu_pc;
219 __entry->r0 = r0;
220 __entry->imm = imm;
221 ),
222
223 TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx",
224 __entry->vcpu_pc, __entry->r0, __entry->imm)
225);
226
227#endif /* _TRACE_KVM_H */
228
229#undef TRACE_INCLUDE_PATH
230#define TRACE_INCLUDE_PATH arch/arm/kvm
231#undef TRACE_INCLUDE_FILE
232#define TRACE_INCLUDE_FILE trace
233
234/* This part must be outside protection */
235#include <trace/define_trace.h>
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-17 00:35:09 -0400