aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-04-13 12:47:01 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2015-04-13 12:47:01 -0400
commit900360131066f192c82311a098d03d6ac6429e20 (patch)
treee9681537a2d1f75fa5be21d8f1116f9f0ba8a391 /arch
parent4541fec3104bef0c60633f9e180be94ea5ccc2b7 (diff)
parentca3f0874723fad81d0c701b63ae3a17a408d5f25 (diff)
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm
Pull KVM updates from Paolo Bonzini: "First batch of KVM changes for 4.1 The most interesting bit here is irqfd/ioeventfd support for ARM and ARM64. Summary: ARM/ARM64: fixes for live migration, irqfd and ioeventfd support (enabling vhost, too), page aging s390: interrupt handling rework, allowing to inject all local interrupts via new ioctl and to get/set the full local irq state for migration and introspection. New ioctls to access memory by virtual address, and to get/set the guest storage keys. SIMD support. MIPS: FPU and MIPS SIMD Architecture (MSA) support. Includes some patches from Ralf Baechle's MIPS tree. x86: bugfixes (notably for pvclock, the others are small) and cleanups. Another small latency improvement for the TSC deadline timer" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (146 commits) KVM: use slowpath for cross page cached accesses kvm: mmu: lazy collapse small sptes into large sptes KVM: x86: Clear CR2 on VCPU reset KVM: x86: DR0-DR3 are not clear on reset KVM: x86: BSP in MSR_IA32_APICBASE is writable KVM: x86: simplify kvm_apic_map KVM: x86: avoid logical_map when it is invalid KVM: x86: fix mixed APIC mode broadcast KVM: x86: use MDA for interrupt matching kvm/ppc/mpic: drop unused IRQ_testbit KVM: nVMX: remove unnecessary double caching of MAXPHYADDR KVM: nVMX: checks for address bits beyond MAXPHYADDR on VM-entry KVM: x86: cache maxphyaddr CPUID leaf in struct kvm_vcpu KVM: vmx: pass error code with internal error #2 x86: vdso: fix pvclock races with task migration KVM: remove kvm_read_hva and kvm_read_hva_atomic KVM: x86: optimize delivery of TSC deadline timer interrupt KVM: x86: extract blocking logic from __vcpu_run kvm: x86: fix x86 eflags fixed bit KVM: s390: migrate vcpu interrupt state ...
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/include/asm/kvm_arm.h1
-rw-r--r--arch/arm/include/asm/kvm_host.h15
-rw-r--r--arch/arm/include/asm/kvm_mmio.h22
-rw-r--r--arch/arm/include/uapi/asm/kvm.h3
-rw-r--r--arch/arm/kernel/asm-offsets.c4
-rw-r--r--arch/arm/kvm/Kconfig30
-rw-r--r--arch/arm/kvm/Makefile12
-rw-r--r--arch/arm/kvm/arm.c45
-rw-r--r--arch/arm/kvm/guest.c18
-rw-r--r--arch/arm/kvm/interrupts_head.S8
-rw-r--r--arch/arm/kvm/mmio.c64
-rw-r--r--arch/arm/kvm/mmu.c134
-rw-r--r--arch/arm/kvm/trace.h48
-rw-r--r--arch/arm64/include/asm/esr.h1
-rw-r--r--arch/arm64/include/asm/kvm_arm.h1
-rw-r--r--arch/arm64/include/asm/kvm_host.h15
-rw-r--r--arch/arm64/include/asm/kvm_mmio.h22
-rw-r--r--arch/arm64/include/uapi/asm/kvm.h3
-rw-r--r--arch/arm64/kvm/Kconfig18
-rw-r--r--arch/arm64/kvm/Makefile20
-rw-r--r--arch/mips/include/asm/asmmacro-32.h128
-rw-r--r--arch/mips/include/asm/asmmacro.h218
-rw-r--r--arch/mips/include/asm/fpu.h20
-rw-r--r--arch/mips/include/asm/kdebug.h3
-rw-r--r--arch/mips/include/asm/kvm_host.h125
-rw-r--r--arch/mips/include/asm/processor.h2
-rw-r--r--arch/mips/include/uapi/asm/kvm.h164
-rw-r--r--arch/mips/kernel/asm-offsets.c105
-rw-r--r--arch/mips/kernel/genex.S15
-rw-r--r--arch/mips/kernel/ptrace.c30
-rw-r--r--arch/mips/kernel/r4k_fpu.S2
-rw-r--r--arch/mips/kernel/traps.c33
-rw-r--r--arch/mips/kvm/Makefile8
-rw-r--r--arch/mips/kvm/emulate.c332
-rw-r--r--arch/mips/kvm/fpu.S122
-rw-r--r--arch/mips/kvm/locore.S38
-rw-r--r--arch/mips/kvm/mips.c472
-rw-r--r--arch/mips/kvm/msa.S161
-rw-r--r--arch/mips/kvm/stats.c4
-rw-r--r--arch/mips/kvm/tlb.c6
-rw-r--r--arch/mips/kvm/trap_emul.c199
-rw-r--r--arch/powerpc/kvm/mpic.c17
-rw-r--r--arch/powerpc/kvm/powerpc.c4
-rw-r--r--arch/s390/include/asm/kvm_host.h46
-rw-r--r--arch/s390/include/uapi/asm/kvm.h4
-rw-r--r--arch/s390/include/uapi/asm/sie.h4
-rw-r--r--arch/s390/kernel/asm-offsets.c1
-rw-r--r--arch/s390/kvm/diag.c6
-rw-r--r--arch/s390/kvm/gaccess.c296
-rw-r--r--arch/s390/kvm/gaccess.h21
-rw-r--r--arch/s390/kvm/guestdbg.c8
-rw-r--r--arch/s390/kvm/intercept.c5
-rw-r--r--arch/s390/kvm/interrupt.c1101
-rw-r--r--arch/s390/kvm/kvm-s390.c398
-rw-r--r--arch/s390/kvm/kvm-s390.h51
-rw-r--r--arch/s390/kvm/priv.c144
-rw-r--r--arch/s390/kvm/sigp.c7
-rw-r--r--arch/x86/include/asm/kvm_host.h28
-rw-r--r--arch/x86/include/asm/kvm_para.h2
-rw-r--r--arch/x86/include/asm/pvclock.h1
-rw-r--r--arch/x86/include/uapi/asm/vmx.h1
-rw-r--r--arch/x86/kernel/pvclock.c44
-rw-r--r--arch/x86/kvm/Makefile2
-rw-r--r--arch/x86/kvm/cpuid.c33
-rw-r--r--arch/x86/kvm/cpuid.h8
-rw-r--r--arch/x86/kvm/emulate.c193
-rw-r--r--arch/x86/kvm/i8254.c14
-rw-r--r--arch/x86/kvm/i8254.h2
-rw-r--r--arch/x86/kvm/i8259.c12
-rw-r--r--arch/x86/kvm/ioapic.c22
-rw-r--r--arch/x86/kvm/ioapic.h11
-rw-r--r--arch/x86/kvm/irq.h2
-rw-r--r--arch/x86/kvm/lapic.c147
-rw-r--r--arch/x86/kvm/lapic.h17
-rw-r--r--arch/x86/kvm/mmu.c73
-rw-r--r--arch/x86/kvm/pmu.c2
-rw-r--r--arch/x86/kvm/svm.c43
-rw-r--r--arch/x86/kvm/vmx.c146
-rw-r--r--arch/x86/kvm/x86.c157
-rw-r--r--arch/x86/vdso/vclock_gettime.c34
80 files changed, 4250 insertions, 1528 deletions
diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h
index 816db0bf2dd8..d995821f1698 100644
--- a/arch/arm/include/asm/kvm_arm.h
+++ b/arch/arm/include/asm/kvm_arm.h
@@ -185,6 +185,7 @@
185#define HSR_COND (0xfU << HSR_COND_SHIFT) 185#define HSR_COND (0xfU << HSR_COND_SHIFT)
186 186
187#define FSC_FAULT (0x04) 187#define FSC_FAULT (0x04)
188#define FSC_ACCESS (0x08)
188#define FSC_PERM (0x0c) 189#define FSC_PERM (0x0c)
189 190
190/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ 191/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 41008cd7c53f..d71607c16601 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -27,6 +27,8 @@
27#include <asm/fpstate.h> 27#include <asm/fpstate.h>
28#include <kvm/arm_arch_timer.h> 28#include <kvm/arm_arch_timer.h>
29 29
30#define __KVM_HAVE_ARCH_INTC_INITIALIZED
31
30#if defined(CONFIG_KVM_ARM_MAX_VCPUS) 32#if defined(CONFIG_KVM_ARM_MAX_VCPUS)
31#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS 33#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
32#else 34#else
@@ -165,19 +167,10 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
165 167
166unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); 168unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
167int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); 169int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
170int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
171int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
168 172
169/* We do not have shadow page tables, hence the empty hooks */ 173/* We do not have shadow page tables, hence the empty hooks */
170static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
171 unsigned long end)
172{
173 return 0;
174}
175
176static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
177{
178 return 0;
179}
180
181static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm, 174static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
182 unsigned long address) 175 unsigned long address)
183{ 176{
diff --git a/arch/arm/include/asm/kvm_mmio.h b/arch/arm/include/asm/kvm_mmio.h
index 3f83db2f6cf0..d8e90c8cb5fa 100644
--- a/arch/arm/include/asm/kvm_mmio.h
+++ b/arch/arm/include/asm/kvm_mmio.h
@@ -28,28 +28,6 @@ struct kvm_decode {
28 bool sign_extend; 28 bool sign_extend;
29}; 29};
30 30
31/*
32 * The in-kernel MMIO emulation code wants to use a copy of run->mmio,
33 * which is an anonymous type. Use our own type instead.
34 */
35struct kvm_exit_mmio {
36 phys_addr_t phys_addr;
37 u8 data[8];
38 u32 len;
39 bool is_write;
40 void *private;
41};
42
43static inline void kvm_prepare_mmio(struct kvm_run *run,
44 struct kvm_exit_mmio *mmio)
45{
46 run->mmio.phys_addr = mmio->phys_addr;
47 run->mmio.len = mmio->len;
48 run->mmio.is_write = mmio->is_write;
49 memcpy(run->mmio.data, mmio->data, mmio->len);
50 run->exit_reason = KVM_EXIT_MMIO;
51}
52
53int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); 31int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
54int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, 32int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
55 phys_addr_t fault_ipa); 33 phys_addr_t fault_ipa);
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index 0db25bc32864..2499867dd0d8 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -198,6 +198,9 @@ struct kvm_arch_memory_slot {
198/* Highest supported SPI, from VGIC_NR_IRQS */ 198/* Highest supported SPI, from VGIC_NR_IRQS */
199#define KVM_ARM_IRQ_GIC_MAX 127 199#define KVM_ARM_IRQ_GIC_MAX 127
200 200
201/* One single KVM irqchip, ie. the VGIC */
202#define KVM_NR_IRQCHIPS 1
203
201/* PSCI interface */ 204/* PSCI interface */
202#define KVM_PSCI_FN_BASE 0x95c1ba5e 205#define KVM_PSCI_FN_BASE 0x95c1ba5e
203#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n)) 206#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n))
diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c
index 2d2d6087b9b1..488eaac56028 100644
--- a/arch/arm/kernel/asm-offsets.c
+++ b/arch/arm/kernel/asm-offsets.c
@@ -190,7 +190,6 @@ int main(void)
190 DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.fault.hxfar)); 190 DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.fault.hxfar));
191 DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.fault.hpfar)); 191 DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.fault.hpfar));
192 DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.fault.hyp_pc)); 192 DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.fault.hyp_pc));
193#ifdef CONFIG_KVM_ARM_VGIC
194 DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu)); 193 DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu));
195 DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr)); 194 DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr));
196 DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr)); 195 DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr));
@@ -200,14 +199,11 @@ int main(void)
200 DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr)); 199 DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr));
201 DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr)); 200 DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr));
202 DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr)); 201 DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr));
203#ifdef CONFIG_KVM_ARM_TIMER
204 DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl)); 202 DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
205 DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval)); 203 DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval));
206 DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff)); 204 DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff));
207 DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled)); 205 DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled));
208#endif
209 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); 206 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
210#endif
211 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); 207 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
212#endif 208#endif
213 return 0; 209 return 0;
diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig
index 338ace78ed18..f1f79d104309 100644
--- a/arch/arm/kvm/Kconfig
+++ b/arch/arm/kvm/Kconfig
@@ -18,6 +18,7 @@ if VIRTUALIZATION
18 18
19config KVM 19config KVM
20 bool "Kernel-based Virtual Machine (KVM) support" 20 bool "Kernel-based Virtual Machine (KVM) support"
21 depends on MMU && OF
21 select PREEMPT_NOTIFIERS 22 select PREEMPT_NOTIFIERS
22 select ANON_INODES 23 select ANON_INODES
23 select HAVE_KVM_CPU_RELAX_INTERCEPT 24 select HAVE_KVM_CPU_RELAX_INTERCEPT
@@ -26,10 +27,12 @@ config KVM
26 select KVM_ARM_HOST 27 select KVM_ARM_HOST
27 select KVM_GENERIC_DIRTYLOG_READ_PROTECT 28 select KVM_GENERIC_DIRTYLOG_READ_PROTECT
28 select SRCU 29 select SRCU
29 depends on ARM_VIRT_EXT && ARM_LPAE 30 select MMU_NOTIFIER
31 select HAVE_KVM_EVENTFD
32 select HAVE_KVM_IRQFD
33 depends on ARM_VIRT_EXT && ARM_LPAE && ARM_ARCH_TIMER
30 ---help--- 34 ---help---
31 Support hosting virtualized guest machines. You will also 35 Support hosting virtualized guest machines.
32 need to select one or more of the processor modules below.
33 36
34 This module provides access to the hardware capabilities through 37 This module provides access to the hardware capabilities through
35 a character device node named /dev/kvm. 38 a character device node named /dev/kvm.
@@ -37,10 +40,7 @@ config KVM
37 If unsure, say N. 40 If unsure, say N.
38 41
39config KVM_ARM_HOST 42config KVM_ARM_HOST
40 bool "KVM host support for ARM cpus." 43 bool
41 depends on KVM
42 depends on MMU
43 select MMU_NOTIFIER
44 ---help--- 44 ---help---
45 Provides host support for ARM processors. 45 Provides host support for ARM processors.
46 46
@@ -55,20 +55,4 @@ config KVM_ARM_MAX_VCPUS
55 large, so only choose a reasonable number that you expect to 55 large, so only choose a reasonable number that you expect to
56 actually use. 56 actually use.
57 57
58config KVM_ARM_VGIC
59 bool "KVM support for Virtual GIC"
60 depends on KVM_ARM_HOST && OF
61 select HAVE_KVM_IRQCHIP
62 default y
63 ---help---
64 Adds support for a hardware assisted, in-kernel GIC emulation.
65
66config KVM_ARM_TIMER
67 bool "KVM support for Architected Timers"
68 depends on KVM_ARM_VGIC && ARM_ARCH_TIMER
69 select HAVE_KVM_IRQCHIP
70 default y
71 ---help---
72 Adds support for the Architected Timers in virtual machines
73
74endif # VIRTUALIZATION 58endif # VIRTUALIZATION
diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile
index 443b8bea43e9..139e46c08b6e 100644
--- a/arch/arm/kvm/Makefile
+++ b/arch/arm/kvm/Makefile
@@ -7,7 +7,7 @@ ifeq ($(plus_virt),+virt)
7 plus_virt_def := -DREQUIRES_VIRT=1 7 plus_virt_def := -DREQUIRES_VIRT=1
8endif 8endif
9 9
10ccflags-y += -Ivirt/kvm -Iarch/arm/kvm 10ccflags-y += -Iarch/arm/kvm
11CFLAGS_arm.o := -I. $(plus_virt_def) 11CFLAGS_arm.o := -I. $(plus_virt_def)
12CFLAGS_mmu.o := -I. 12CFLAGS_mmu.o := -I.
13 13
@@ -15,12 +15,12 @@ AFLAGS_init.o := -Wa,-march=armv7-a$(plus_virt)
15AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt) 15AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt)
16 16
17KVM := ../../../virt/kvm 17KVM := ../../../virt/kvm
18kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o 18kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o
19 19
20obj-y += kvm-arm.o init.o interrupts.o 20obj-y += kvm-arm.o init.o interrupts.o
21obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o 21obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o
22obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o 22obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o
23obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o 23obj-y += $(KVM)/arm/vgic.o
24obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o 24obj-y += $(KVM)/arm/vgic-v2.o
25obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o 25obj-y += $(KVM)/arm/vgic-v2-emul.o
26obj-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o 26obj-y += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 5560f74f9eee..6f536451ab78 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -61,8 +61,6 @@ static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
61static u8 kvm_next_vmid; 61static u8 kvm_next_vmid;
62static DEFINE_SPINLOCK(kvm_vmid_lock); 62static DEFINE_SPINLOCK(kvm_vmid_lock);
63 63
64static bool vgic_present;
65
66static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu) 64static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
67{ 65{
68 BUG_ON(preemptible()); 66 BUG_ON(preemptible());
@@ -173,8 +171,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
173 int r; 171 int r;
174 switch (ext) { 172 switch (ext) {
175 case KVM_CAP_IRQCHIP: 173 case KVM_CAP_IRQCHIP:
176 r = vgic_present; 174 case KVM_CAP_IRQFD:
177 break; 175 case KVM_CAP_IOEVENTFD:
178 case KVM_CAP_DEVICE_CTRL: 176 case KVM_CAP_DEVICE_CTRL:
179 case KVM_CAP_USER_MEMORY: 177 case KVM_CAP_USER_MEMORY:
180 case KVM_CAP_SYNC_MMU: 178 case KVM_CAP_SYNC_MMU:
@@ -183,6 +181,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
183 case KVM_CAP_ARM_PSCI: 181 case KVM_CAP_ARM_PSCI:
184 case KVM_CAP_ARM_PSCI_0_2: 182 case KVM_CAP_ARM_PSCI_0_2:
185 case KVM_CAP_READONLY_MEM: 183 case KVM_CAP_READONLY_MEM:
184 case KVM_CAP_MP_STATE:
186 r = 1; 185 r = 1;
187 break; 186 break;
188 case KVM_CAP_COALESCED_MMIO: 187 case KVM_CAP_COALESCED_MMIO:
@@ -268,7 +267,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
268 267
269int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 268int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
270{ 269{
271 return 0; 270 return kvm_timer_should_fire(vcpu);
272} 271}
273 272
274int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 273int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
@@ -313,13 +312,29 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
313int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 312int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
314 struct kvm_mp_state *mp_state) 313 struct kvm_mp_state *mp_state)
315{ 314{
316 return -EINVAL; 315 if (vcpu->arch.pause)
316 mp_state->mp_state = KVM_MP_STATE_STOPPED;
317 else
318 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
319
320 return 0;
317} 321}
318 322
319int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 323int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
320 struct kvm_mp_state *mp_state) 324 struct kvm_mp_state *mp_state)
321{ 325{
322 return -EINVAL; 326 switch (mp_state->mp_state) {
327 case KVM_MP_STATE_RUNNABLE:
328 vcpu->arch.pause = false;
329 break;
330 case KVM_MP_STATE_STOPPED:
331 vcpu->arch.pause = true;
332 break;
333 default:
334 return -EINVAL;
335 }
336
337 return 0;
323} 338}
324 339
325/** 340/**
@@ -452,6 +467,11 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
452 return 0; 467 return 0;
453} 468}
454 469
470bool kvm_arch_intc_initialized(struct kvm *kvm)
471{
472 return vgic_initialized(kvm);
473}
474
455static void vcpu_pause(struct kvm_vcpu *vcpu) 475static void vcpu_pause(struct kvm_vcpu *vcpu)
456{ 476{
457 wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu); 477 wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
@@ -831,8 +851,6 @@ static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
831 851
832 switch (dev_id) { 852 switch (dev_id) {
833 case KVM_ARM_DEVICE_VGIC_V2: 853 case KVM_ARM_DEVICE_VGIC_V2:
834 if (!vgic_present)
835 return -ENXIO;
836 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true); 854 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
837 default: 855 default:
838 return -ENODEV; 856 return -ENODEV;
@@ -847,10 +865,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
847 865
848 switch (ioctl) { 866 switch (ioctl) {
849 case KVM_CREATE_IRQCHIP: { 867 case KVM_CREATE_IRQCHIP: {
850 if (vgic_present) 868 return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
851 return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
852 else
853 return -ENXIO;
854 } 869 }
855 case KVM_ARM_SET_DEVICE_ADDR: { 870 case KVM_ARM_SET_DEVICE_ADDR: {
856 struct kvm_arm_device_addr dev_addr; 871 struct kvm_arm_device_addr dev_addr;
@@ -1035,10 +1050,6 @@ static int init_hyp_mode(void)
1035 if (err) 1050 if (err)
1036 goto out_free_context; 1051 goto out_free_context;
1037 1052
1038#ifdef CONFIG_KVM_ARM_VGIC
1039 vgic_present = true;
1040#endif
1041
1042 /* 1053 /*
1043 * Init HYP architected timer support 1054 * Init HYP architected timer support
1044 */ 1055 */
diff --git a/arch/arm/kvm/guest.c b/arch/arm/kvm/guest.c
index 384bab67c462..d503fbb787d3 100644
--- a/arch/arm/kvm/guest.c
+++ b/arch/arm/kvm/guest.c
@@ -109,22 +109,6 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
109 return -EINVAL; 109 return -EINVAL;
110} 110}
111 111
112#ifndef CONFIG_KVM_ARM_TIMER
113
114#define NUM_TIMER_REGS 0
115
116static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
117{
118 return 0;
119}
120
121static bool is_timer_reg(u64 index)
122{
123 return false;
124}
125
126#else
127
128#define NUM_TIMER_REGS 3 112#define NUM_TIMER_REGS 3
129 113
130static bool is_timer_reg(u64 index) 114static bool is_timer_reg(u64 index)
@@ -152,8 +136,6 @@ static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
152 return 0; 136 return 0;
153} 137}
154 138
155#endif
156
157static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 139static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
158{ 140{
159 void __user *uaddr = (void __user *)(long)reg->addr; 141 void __user *uaddr = (void __user *)(long)reg->addr;
diff --git a/arch/arm/kvm/interrupts_head.S b/arch/arm/kvm/interrupts_head.S
index 14d488388480..35e4a3a0c476 100644
--- a/arch/arm/kvm/interrupts_head.S
+++ b/arch/arm/kvm/interrupts_head.S
@@ -402,7 +402,6 @@ vcpu .req r0 @ vcpu pointer always in r0
402 * Assumes vcpu pointer in vcpu reg 402 * Assumes vcpu pointer in vcpu reg
403 */ 403 */
404.macro save_vgic_state 404.macro save_vgic_state
405#ifdef CONFIG_KVM_ARM_VGIC
406 /* Get VGIC VCTRL base into r2 */ 405 /* Get VGIC VCTRL base into r2 */
407 ldr r2, [vcpu, #VCPU_KVM] 406 ldr r2, [vcpu, #VCPU_KVM]
408 ldr r2, [r2, #KVM_VGIC_VCTRL] 407 ldr r2, [r2, #KVM_VGIC_VCTRL]
@@ -460,7 +459,6 @@ ARM_BE8(rev r6, r6 )
460 subs r4, r4, #1 459 subs r4, r4, #1
461 bne 1b 460 bne 1b
4622: 4612:
463#endif
464.endm 462.endm
465 463
466/* 464/*
@@ -469,7 +467,6 @@ ARM_BE8(rev r6, r6 )
469 * Assumes vcpu pointer in vcpu reg 467 * Assumes vcpu pointer in vcpu reg
470 */ 468 */
471.macro restore_vgic_state 469.macro restore_vgic_state
472#ifdef CONFIG_KVM_ARM_VGIC
473 /* Get VGIC VCTRL base into r2 */ 470 /* Get VGIC VCTRL base into r2 */
474 ldr r2, [vcpu, #VCPU_KVM] 471 ldr r2, [vcpu, #VCPU_KVM]
475 ldr r2, [r2, #KVM_VGIC_VCTRL] 472 ldr r2, [r2, #KVM_VGIC_VCTRL]
@@ -501,7 +498,6 @@ ARM_BE8(rev r6, r6 )
501 subs r4, r4, #1 498 subs r4, r4, #1
502 bne 1b 499 bne 1b
5032: 5002:
504#endif
505.endm 501.endm
506 502
507#define CNTHCTL_PL1PCTEN (1 << 0) 503#define CNTHCTL_PL1PCTEN (1 << 0)
@@ -515,7 +511,6 @@ ARM_BE8(rev r6, r6 )
515 * Clobbers r2-r5 511 * Clobbers r2-r5
516 */ 512 */
517.macro save_timer_state 513.macro save_timer_state
518#ifdef CONFIG_KVM_ARM_TIMER
519 ldr r4, [vcpu, #VCPU_KVM] 514 ldr r4, [vcpu, #VCPU_KVM]
520 ldr r2, [r4, #KVM_TIMER_ENABLED] 515 ldr r2, [r4, #KVM_TIMER_ENABLED]
521 cmp r2, #0 516 cmp r2, #0
@@ -537,7 +532,6 @@ ARM_BE8(rev r6, r6 )
537 mcrr p15, 4, r2, r2, c14 @ CNTVOFF 532 mcrr p15, 4, r2, r2, c14 @ CNTVOFF
538 533
5391: 5341:
540#endif
541 @ Allow physical timer/counter access for the host 535 @ Allow physical timer/counter access for the host
542 mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL 536 mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL
543 orr r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN) 537 orr r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN)
@@ -559,7 +553,6 @@ ARM_BE8(rev r6, r6 )
559 bic r2, r2, #CNTHCTL_PL1PCEN 553 bic r2, r2, #CNTHCTL_PL1PCEN
560 mcr p15, 4, r2, c14, c1, 0 @ CNTHCTL 554 mcr p15, 4, r2, c14, c1, 0 @ CNTHCTL
561 555
562#ifdef CONFIG_KVM_ARM_TIMER
563 ldr r4, [vcpu, #VCPU_KVM] 556 ldr r4, [vcpu, #VCPU_KVM]
564 ldr r2, [r4, #KVM_TIMER_ENABLED] 557 ldr r2, [r4, #KVM_TIMER_ENABLED]
565 cmp r2, #0 558 cmp r2, #0
@@ -579,7 +572,6 @@ ARM_BE8(rev r6, r6 )
579 and r2, r2, #3 572 and r2, r2, #3
580 mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL 573 mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL
5811: 5741:
582#endif
583.endm 575.endm
584 576
585.equ vmentry, 0 577.equ vmentry, 0
diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c
index 5d3bfc0eb3f0..974b1c606d04 100644
--- a/arch/arm/kvm/mmio.c
+++ b/arch/arm/kvm/mmio.c
@@ -121,12 +121,11 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
121 return 0; 121 return 0;
122} 122}
123 123
124static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 124static int decode_hsr(struct kvm_vcpu *vcpu, bool *is_write, int *len)
125 struct kvm_exit_mmio *mmio)
126{ 125{
127 unsigned long rt; 126 unsigned long rt;
128 int len; 127 int access_size;
129 bool is_write, sign_extend; 128 bool sign_extend;
130 129
131 if (kvm_vcpu_dabt_isextabt(vcpu)) { 130 if (kvm_vcpu_dabt_isextabt(vcpu)) {
132 /* cache operation on I/O addr, tell guest unsupported */ 131 /* cache operation on I/O addr, tell guest unsupported */
@@ -140,17 +139,15 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
140 return 1; 139 return 1;
141 } 140 }
142 141
143 len = kvm_vcpu_dabt_get_as(vcpu); 142 access_size = kvm_vcpu_dabt_get_as(vcpu);
144 if (unlikely(len < 0)) 143 if (unlikely(access_size < 0))
145 return len; 144 return access_size;
146 145
147 is_write = kvm_vcpu_dabt_iswrite(vcpu); 146 *is_write = kvm_vcpu_dabt_iswrite(vcpu);
148 sign_extend = kvm_vcpu_dabt_issext(vcpu); 147 sign_extend = kvm_vcpu_dabt_issext(vcpu);
149 rt = kvm_vcpu_dabt_get_rd(vcpu); 148 rt = kvm_vcpu_dabt_get_rd(vcpu);
150 149
151 mmio->is_write = is_write; 150 *len = access_size;
152 mmio->phys_addr = fault_ipa;
153 mmio->len = len;
154 vcpu->arch.mmio_decode.sign_extend = sign_extend; 151 vcpu->arch.mmio_decode.sign_extend = sign_extend;
155 vcpu->arch.mmio_decode.rt = rt; 152 vcpu->arch.mmio_decode.rt = rt;
156 153
@@ -165,20 +162,20 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
165int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, 162int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
166 phys_addr_t fault_ipa) 163 phys_addr_t fault_ipa)
167{ 164{
168 struct kvm_exit_mmio mmio;
169 unsigned long data; 165 unsigned long data;
170 unsigned long rt; 166 unsigned long rt;
171 int ret; 167 int ret;
168 bool is_write;
169 int len;
170 u8 data_buf[8];
172 171
173 /* 172 /*
174 * Prepare MMIO operation. First stash it in a private 173 * Prepare MMIO operation. First decode the syndrome data we get
175 * structure that we can use for in-kernel emulation. If the 174 * from the CPU. Then try if some in-kernel emulation feels
176 * kernel can't handle it, copy it into run->mmio and let user 175 * responsible, otherwise let user space do its magic.
177 * space do its magic.
178 */ 176 */
179
180 if (kvm_vcpu_dabt_isvalid(vcpu)) { 177 if (kvm_vcpu_dabt_isvalid(vcpu)) {
181 ret = decode_hsr(vcpu, fault_ipa, &mmio); 178 ret = decode_hsr(vcpu, &is_write, &len);
182 if (ret) 179 if (ret)
183 return ret; 180 return ret;
184 } else { 181 } else {
@@ -188,21 +185,34 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
188 185
189 rt = vcpu->arch.mmio_decode.rt; 186 rt = vcpu->arch.mmio_decode.rt;
190 187
191 if (mmio.is_write) { 188 if (is_write) {
192 data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), 189 data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), len);
193 mmio.len); 190
191 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, fault_ipa, data);
192 mmio_write_buf(data_buf, len, data);
194 193
195 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, mmio.len, 194 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, fault_ipa, len,
196 fault_ipa, data); 195 data_buf);
197 mmio_write_buf(mmio.data, mmio.len, data);
198 } else { 196 } else {
199 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, mmio.len, 197 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len,
200 fault_ipa, 0); 198 fault_ipa, 0);
199
200 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_ipa, len,
201 data_buf);
201 } 202 }
202 203
203 if (vgic_handle_mmio(vcpu, run, &mmio)) 204 /* Now prepare kvm_run for the potential return to userland. */
205 run->mmio.is_write = is_write;
206 run->mmio.phys_addr = fault_ipa;
207 run->mmio.len = len;
208 memcpy(run->mmio.data, data_buf, len);
209
210 if (!ret) {
211 /* We handled the access successfully in the kernel. */
212 kvm_handle_mmio_return(vcpu, run);
204 return 1; 213 return 1;
214 }
205 215
206 kvm_prepare_mmio(run, &mmio); 216 run->exit_reason = KVM_EXIT_MMIO;
207 return 0; 217 return 0;
208} 218}
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 5656d79c5a44..15b050d46fc9 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1330,10 +1330,51 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
1330 1330
1331out_unlock: 1331out_unlock:
1332 spin_unlock(&kvm->mmu_lock); 1332 spin_unlock(&kvm->mmu_lock);
1333 kvm_set_pfn_accessed(pfn);
1333 kvm_release_pfn_clean(pfn); 1334 kvm_release_pfn_clean(pfn);
1334 return ret; 1335 return ret;
1335} 1336}
1336 1337
1338/*
1339 * Resolve the access fault by making the page young again.
1340 * Note that because the faulting entry is guaranteed not to be
1341 * cached in the TLB, we don't need to invalidate anything.
1342 */
1343static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
1344{
1345 pmd_t *pmd;
1346 pte_t *pte;
1347 pfn_t pfn;
1348 bool pfn_valid = false;
1349
1350 trace_kvm_access_fault(fault_ipa);
1351
1352 spin_lock(&vcpu->kvm->mmu_lock);
1353
1354 pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);
1355 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1356 goto out;
1357
1358 if (kvm_pmd_huge(*pmd)) { /* THP, HugeTLB */
1359 *pmd = pmd_mkyoung(*pmd);
1360 pfn = pmd_pfn(*pmd);
1361 pfn_valid = true;
1362 goto out;
1363 }
1364
1365 pte = pte_offset_kernel(pmd, fault_ipa);
1366 if (pte_none(*pte)) /* Nothing there either */
1367 goto out;
1368
1369 *pte = pte_mkyoung(*pte); /* Just a page... */
1370 pfn = pte_pfn(*pte);
1371 pfn_valid = true;
1372out:
1373 spin_unlock(&vcpu->kvm->mmu_lock);
1374 if (pfn_valid)
1375 kvm_set_pfn_accessed(pfn);
1376}
1377
1337/** 1378/**
1338 * kvm_handle_guest_abort - handles all 2nd stage aborts 1379 * kvm_handle_guest_abort - handles all 2nd stage aborts
1339 * @vcpu: the VCPU pointer 1380 * @vcpu: the VCPU pointer
@@ -1364,7 +1405,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
1364 1405
1365 /* Check the stage-2 fault is trans. fault or write fault */ 1406 /* Check the stage-2 fault is trans. fault or write fault */
1366 fault_status = kvm_vcpu_trap_get_fault_type(vcpu); 1407 fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
1367 if (fault_status != FSC_FAULT && fault_status != FSC_PERM) { 1408 if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
1409 fault_status != FSC_ACCESS) {
1368 kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", 1410 kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
1369 kvm_vcpu_trap_get_class(vcpu), 1411 kvm_vcpu_trap_get_class(vcpu),
1370 (unsigned long)kvm_vcpu_trap_get_fault(vcpu), 1412 (unsigned long)kvm_vcpu_trap_get_fault(vcpu),
@@ -1400,6 +1442,12 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
1400 /* Userspace should not be able to register out-of-bounds IPAs */ 1442 /* Userspace should not be able to register out-of-bounds IPAs */
1401 VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE); 1443 VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE);
1402 1444
1445 if (fault_status == FSC_ACCESS) {
1446 handle_access_fault(vcpu, fault_ipa);
1447 ret = 1;
1448 goto out_unlock;
1449 }
1450
1403 ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status); 1451 ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
1404 if (ret == 0) 1452 if (ret == 0)
1405 ret = 1; 1453 ret = 1;
@@ -1408,15 +1456,16 @@ out_unlock:
1408 return ret; 1456 return ret;
1409} 1457}
1410 1458
1411static void handle_hva_to_gpa(struct kvm *kvm, 1459static int handle_hva_to_gpa(struct kvm *kvm,
1412 unsigned long start, 1460 unsigned long start,
1413 unsigned long end, 1461 unsigned long end,
1414 void (*handler)(struct kvm *kvm, 1462 int (*handler)(struct kvm *kvm,
1415 gpa_t gpa, void *data), 1463 gpa_t gpa, void *data),
1416 void *data) 1464 void *data)
1417{ 1465{
1418 struct kvm_memslots *slots; 1466 struct kvm_memslots *slots;
1419 struct kvm_memory_slot *memslot; 1467 struct kvm_memory_slot *memslot;
1468 int ret = 0;
1420 1469
1421 slots = kvm_memslots(kvm); 1470 slots = kvm_memslots(kvm);
1422 1471
@@ -1440,14 +1489,17 @@ static void handle_hva_to_gpa(struct kvm *kvm,
1440 1489
1441 for (; gfn < gfn_end; ++gfn) { 1490 for (; gfn < gfn_end; ++gfn) {
1442 gpa_t gpa = gfn << PAGE_SHIFT; 1491 gpa_t gpa = gfn << PAGE_SHIFT;
1443 handler(kvm, gpa, data); 1492 ret |= handler(kvm, gpa, data);
1444 } 1493 }
1445 } 1494 }
1495
1496 return ret;
1446} 1497}
1447 1498
1448static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data) 1499static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1449{ 1500{
1450 unmap_stage2_range(kvm, gpa, PAGE_SIZE); 1501 unmap_stage2_range(kvm, gpa, PAGE_SIZE);
1502 return 0;
1451} 1503}
1452 1504
1453int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) 1505int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
@@ -1473,7 +1525,7 @@ int kvm_unmap_hva_range(struct kvm *kvm,
1473 return 0; 1525 return 0;
1474} 1526}
1475 1527
1476static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data) 1528static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
1477{ 1529{
1478 pte_t *pte = (pte_t *)data; 1530 pte_t *pte = (pte_t *)data;
1479 1531
@@ -1485,6 +1537,7 @@ static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
1485 * through this calling path. 1537 * through this calling path.
1486 */ 1538 */
1487 stage2_set_pte(kvm, NULL, gpa, pte, 0); 1539 stage2_set_pte(kvm, NULL, gpa, pte, 0);
1540 return 0;
1488} 1541}
1489 1542
1490 1543
@@ -1501,6 +1554,67 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1501 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte); 1554 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
1502} 1555}
1503 1556
1557static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1558{
1559 pmd_t *pmd;
1560 pte_t *pte;
1561
1562 pmd = stage2_get_pmd(kvm, NULL, gpa);
1563 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1564 return 0;
1565
1566 if (kvm_pmd_huge(*pmd)) { /* THP, HugeTLB */
1567 if (pmd_young(*pmd)) {
1568 *pmd = pmd_mkold(*pmd);
1569 return 1;
1570 }
1571
1572 return 0;
1573 }
1574
1575 pte = pte_offset_kernel(pmd, gpa);
1576 if (pte_none(*pte))
1577 return 0;
1578
1579 if (pte_young(*pte)) {
1580 *pte = pte_mkold(*pte); /* Just a page... */
1581 return 1;
1582 }
1583
1584 return 0;
1585}
1586
1587static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1588{
1589 pmd_t *pmd;
1590 pte_t *pte;
1591
1592 pmd = stage2_get_pmd(kvm, NULL, gpa);
1593 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1594 return 0;
1595
1596 if (kvm_pmd_huge(*pmd)) /* THP, HugeTLB */
1597 return pmd_young(*pmd);
1598
1599 pte = pte_offset_kernel(pmd, gpa);
1600 if (!pte_none(*pte)) /* Just a page... */
1601 return pte_young(*pte);
1602
1603 return 0;
1604}
1605
1606int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1607{
1608 trace_kvm_age_hva(start, end);
1609 return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
1610}
1611
1612int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1613{
1614 trace_kvm_test_age_hva(hva);
1615 return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
1616}
1617
1504void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) 1618void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
1505{ 1619{
1506 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 1620 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
diff --git a/arch/arm/kvm/trace.h b/arch/arm/kvm/trace.h
index 6817664b46b8..0ec35392d208 100644
--- a/arch/arm/kvm/trace.h
+++ b/arch/arm/kvm/trace.h
@@ -68,6 +68,21 @@ TRACE_EVENT(kvm_guest_fault,
68 __entry->hxfar, __entry->vcpu_pc) 68 __entry->hxfar, __entry->vcpu_pc)
69); 69);
70 70
71TRACE_EVENT(kvm_access_fault,
72 TP_PROTO(unsigned long ipa),
73 TP_ARGS(ipa),
74
75 TP_STRUCT__entry(
76 __field( unsigned long, ipa )
77 ),
78
79 TP_fast_assign(
80 __entry->ipa = ipa;
81 ),
82
83 TP_printk("IPA: %lx", __entry->ipa)
84);
85
71TRACE_EVENT(kvm_irq_line, 86TRACE_EVENT(kvm_irq_line,
72 TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level), 87 TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level),
73 TP_ARGS(type, vcpu_idx, irq_num, level), 88 TP_ARGS(type, vcpu_idx, irq_num, level),
@@ -210,6 +225,39 @@ TRACE_EVENT(kvm_set_spte_hva,
210 TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva) 225 TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva)
211); 226);
212 227
228TRACE_EVENT(kvm_age_hva,
229 TP_PROTO(unsigned long start, unsigned long end),
230 TP_ARGS(start, end),
231
232 TP_STRUCT__entry(
233 __field( unsigned long, start )
234 __field( unsigned long, end )
235 ),
236
237 TP_fast_assign(
238 __entry->start = start;
239 __entry->end = end;
240 ),
241
242 TP_printk("mmu notifier age hva: %#08lx -- %#08lx",
243 __entry->start, __entry->end)
244);
245
246TRACE_EVENT(kvm_test_age_hva,
247 TP_PROTO(unsigned long hva),
248 TP_ARGS(hva),
249
250 TP_STRUCT__entry(
251 __field( unsigned long, hva )
252 ),
253
254 TP_fast_assign(
255 __entry->hva = hva;
256 ),
257
258 TP_printk("mmu notifier test age hva: %#08lx", __entry->hva)
259);
260
213TRACE_EVENT(kvm_hvc, 261TRACE_EVENT(kvm_hvc,
214 TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm), 262 TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
215 TP_ARGS(vcpu_pc, r0, imm), 263 TP_ARGS(vcpu_pc, r0, imm),
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index 92bbae381598..70522450ca23 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -90,6 +90,7 @@
90#define ESR_ELx_FSC (0x3F) 90#define ESR_ELx_FSC (0x3F)
91#define ESR_ELx_FSC_TYPE (0x3C) 91#define ESR_ELx_FSC_TYPE (0x3C)
92#define ESR_ELx_FSC_EXTABT (0x10) 92#define ESR_ELx_FSC_EXTABT (0x10)
93#define ESR_ELx_FSC_ACCESS (0x08)
93#define ESR_ELx_FSC_FAULT (0x04) 94#define ESR_ELx_FSC_FAULT (0x04)
94#define ESR_ELx_FSC_PERM (0x0C) 95#define ESR_ELx_FSC_PERM (0x0C)
95#define ESR_ELx_CV (UL(1) << 24) 96#define ESR_ELx_CV (UL(1) << 24)
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 54bb4ba97441..ac6fafb95fe7 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -188,6 +188,7 @@
188 188
189/* For compatibility with fault code shared with 32-bit */ 189/* For compatibility with fault code shared with 32-bit */
190#define FSC_FAULT ESR_ELx_FSC_FAULT 190#define FSC_FAULT ESR_ELx_FSC_FAULT
191#define FSC_ACCESS ESR_ELx_FSC_ACCESS
191#define FSC_PERM ESR_ELx_FSC_PERM 192#define FSC_PERM ESR_ELx_FSC_PERM
192 193
193/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ 194/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 8ac3c70fe3c6..f0f58c9beec0 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -28,6 +28,8 @@
28#include <asm/kvm_asm.h> 28#include <asm/kvm_asm.h>
29#include <asm/kvm_mmio.h> 29#include <asm/kvm_mmio.h>
30 30
31#define __KVM_HAVE_ARCH_INTC_INITIALIZED
32
31#if defined(CONFIG_KVM_ARM_MAX_VCPUS) 33#if defined(CONFIG_KVM_ARM_MAX_VCPUS)
32#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS 34#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
33#else 35#else
@@ -177,19 +179,10 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
177int kvm_unmap_hva_range(struct kvm *kvm, 179int kvm_unmap_hva_range(struct kvm *kvm,
178 unsigned long start, unsigned long end); 180 unsigned long start, unsigned long end);
179void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); 181void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
182int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
183int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
180 184
181/* We do not have shadow page tables, hence the empty hooks */ 185/* We do not have shadow page tables, hence the empty hooks */
182static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
183 unsigned long end)
184{
185 return 0;
186}
187
188static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
189{
190 return 0;
191}
192
193static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm, 186static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
194 unsigned long address) 187 unsigned long address)
195{ 188{
diff --git a/arch/arm64/include/asm/kvm_mmio.h b/arch/arm64/include/asm/kvm_mmio.h
index 9f52beb7cb13..889c908ee631 100644
--- a/arch/arm64/include/asm/kvm_mmio.h
+++ b/arch/arm64/include/asm/kvm_mmio.h
@@ -31,28 +31,6 @@ struct kvm_decode {
31 bool sign_extend; 31 bool sign_extend;
32}; 32};
33 33
34/*
35 * The in-kernel MMIO emulation code wants to use a copy of run->mmio,
36 * which is an anonymous type. Use our own type instead.
37 */
38struct kvm_exit_mmio {
39 phys_addr_t phys_addr;
40 u8 data[8];
41 u32 len;
42 bool is_write;
43 void *private;
44};
45
46static inline void kvm_prepare_mmio(struct kvm_run *run,
47 struct kvm_exit_mmio *mmio)
48{
49 run->mmio.phys_addr = mmio->phys_addr;
50 run->mmio.len = mmio->len;
51 run->mmio.is_write = mmio->is_write;
52 memcpy(run->mmio.data, mmio->data, mmio->len);
53 run->exit_reason = KVM_EXIT_MMIO;
54}
55
56int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); 34int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
57int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, 35int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
58 phys_addr_t fault_ipa); 36 phys_addr_t fault_ipa);
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 3ef77a466018..c154c0b7eb60 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -191,6 +191,9 @@ struct kvm_arch_memory_slot {
191/* Highest supported SPI, from VGIC_NR_IRQS */ 191/* Highest supported SPI, from VGIC_NR_IRQS */
192#define KVM_ARM_IRQ_GIC_MAX 127 192#define KVM_ARM_IRQ_GIC_MAX 127
193 193
194/* One single KVM irqchip, ie. the VGIC */
195#define KVM_NR_IRQCHIPS 1
196
194/* PSCI interface */ 197/* PSCI interface */
195#define KVM_PSCI_FN_BASE 0x95c1ba5e 198#define KVM_PSCI_FN_BASE 0x95c1ba5e
196#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n)) 199#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n))
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index f5590c81d95f..5105e297ed5f 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -18,6 +18,7 @@ if VIRTUALIZATION
18 18
19config KVM 19config KVM
20 bool "Kernel-based Virtual Machine (KVM) support" 20 bool "Kernel-based Virtual Machine (KVM) support"
21 depends on OF
21 select MMU_NOTIFIER 22 select MMU_NOTIFIER
22 select PREEMPT_NOTIFIERS 23 select PREEMPT_NOTIFIERS
23 select ANON_INODES 24 select ANON_INODES
@@ -25,10 +26,10 @@ config KVM
25 select HAVE_KVM_ARCH_TLB_FLUSH_ALL 26 select HAVE_KVM_ARCH_TLB_FLUSH_ALL
26 select KVM_MMIO 27 select KVM_MMIO
27 select KVM_ARM_HOST 28 select KVM_ARM_HOST
28 select KVM_ARM_VGIC
29 select KVM_ARM_TIMER
30 select KVM_GENERIC_DIRTYLOG_READ_PROTECT 29 select KVM_GENERIC_DIRTYLOG_READ_PROTECT
31 select SRCU 30 select SRCU
31 select HAVE_KVM_EVENTFD
32 select HAVE_KVM_IRQFD
32 ---help--- 33 ---help---
33 Support hosting virtualized guest machines. 34 Support hosting virtualized guest machines.
34 35
@@ -50,17 +51,4 @@ config KVM_ARM_MAX_VCPUS
50 large, so only choose a reasonable number that you expect to 51 large, so only choose a reasonable number that you expect to
51 actually use. 52 actually use.
52 53
53config KVM_ARM_VGIC
54 bool
55 depends on KVM_ARM_HOST && OF
56 select HAVE_KVM_IRQCHIP
57 ---help---
58 Adds support for a hardware assisted, in-kernel GIC emulation.
59
60config KVM_ARM_TIMER
61 bool
62 depends on KVM_ARM_VGIC
63 ---help---
64 Adds support for the Architected Timers in virtual machines.
65
66endif # VIRTUALIZATION 54endif # VIRTUALIZATION
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 4e6e09ee4033..d5904f876cdb 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -2,7 +2,7 @@
2# Makefile for Kernel-based Virtual Machine module 2# Makefile for Kernel-based Virtual Machine module
3# 3#
4 4
5ccflags-y += -Ivirt/kvm -Iarch/arm64/kvm 5ccflags-y += -Iarch/arm64/kvm
6CFLAGS_arm.o := -I. 6CFLAGS_arm.o := -I.
7CFLAGS_mmu.o := -I. 7CFLAGS_mmu.o := -I.
8 8
@@ -11,7 +11,7 @@ ARM=../../../arch/arm/kvm
11 11
12obj-$(CONFIG_KVM_ARM_HOST) += kvm.o 12obj-$(CONFIG_KVM_ARM_HOST) += kvm.o
13 13
14kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o 14kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o
15kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/arm.o $(ARM)/mmu.o $(ARM)/mmio.o 15kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/arm.o $(ARM)/mmu.o $(ARM)/mmio.o
16kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o 16kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o
17 17
@@ -19,11 +19,11 @@ kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o
19kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o 19kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
20kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o 20kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
21 21
22kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o 22kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o
23kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o 23kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o
24kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o 24kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2-emul.o
25kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v2-switch.o 25kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v2-switch.o
26kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3.o 26kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
27kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3-emul.o 27kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
28kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v3-switch.o 28kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
29kvm-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o 29kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
diff --git a/arch/mips/include/asm/asmmacro-32.h b/arch/mips/include/asm/asmmacro-32.h
index cdac7b3eeaf7..80386470d3a4 100644
--- a/arch/mips/include/asm/asmmacro-32.h
+++ b/arch/mips/include/asm/asmmacro-32.h
@@ -16,38 +16,38 @@
16 .set push 16 .set push
17 SET_HARDFLOAT 17 SET_HARDFLOAT
18 cfc1 \tmp, fcr31 18 cfc1 \tmp, fcr31
19 swc1 $f0, THREAD_FPR0_LS64(\thread) 19 swc1 $f0, THREAD_FPR0(\thread)
20 swc1 $f1, THREAD_FPR1_LS64(\thread) 20 swc1 $f1, THREAD_FPR1(\thread)
21 swc1 $f2, THREAD_FPR2_LS64(\thread) 21 swc1 $f2, THREAD_FPR2(\thread)
22 swc1 $f3, THREAD_FPR3_LS64(\thread) 22 swc1 $f3, THREAD_FPR3(\thread)
23 swc1 $f4, THREAD_FPR4_LS64(\thread) 23 swc1 $f4, THREAD_FPR4(\thread)
24 swc1 $f5, THREAD_FPR5_LS64(\thread) 24 swc1 $f5, THREAD_FPR5(\thread)
25 swc1 $f6, THREAD_FPR6_LS64(\thread) 25 swc1 $f6, THREAD_FPR6(\thread)
26 swc1 $f7, THREAD_FPR7_LS64(\thread) 26 swc1 $f7, THREAD_FPR7(\thread)
27 swc1 $f8, THREAD_FPR8_LS64(\thread) 27 swc1 $f8, THREAD_FPR8(\thread)
28 swc1 $f9, THREAD_FPR9_LS64(\thread) 28 swc1 $f9, THREAD_FPR9(\thread)
29 swc1 $f10, THREAD_FPR10_LS64(\thread) 29 swc1 $f10, THREAD_FPR10(\thread)
30 swc1 $f11, THREAD_FPR11_LS64(\thread) 30 swc1 $f11, THREAD_FPR11(\thread)
31 swc1 $f12, THREAD_FPR12_LS64(\thread) 31 swc1 $f12, THREAD_FPR12(\thread)
32 swc1 $f13, THREAD_FPR13_LS64(\thread) 32 swc1 $f13, THREAD_FPR13(\thread)
33 swc1 $f14, THREAD_FPR14_LS64(\thread) 33 swc1 $f14, THREAD_FPR14(\thread)
34 swc1 $f15, THREAD_FPR15_LS64(\thread) 34 swc1 $f15, THREAD_FPR15(\thread)
35 swc1 $f16, THREAD_FPR16_LS64(\thread) 35 swc1 $f16, THREAD_FPR16(\thread)
36 swc1 $f17, THREAD_FPR17_LS64(\thread) 36 swc1 $f17, THREAD_FPR17(\thread)
37 swc1 $f18, THREAD_FPR18_LS64(\thread) 37 swc1 $f18, THREAD_FPR18(\thread)
38 swc1 $f19, THREAD_FPR19_LS64(\thread) 38 swc1 $f19, THREAD_FPR19(\thread)
39 swc1 $f20, THREAD_FPR20_LS64(\thread) 39 swc1 $f20, THREAD_FPR20(\thread)
40 swc1 $f21, THREAD_FPR21_LS64(\thread) 40 swc1 $f21, THREAD_FPR21(\thread)
41 swc1 $f22, THREAD_FPR22_LS64(\thread) 41 swc1 $f22, THREAD_FPR22(\thread)
42 swc1 $f23, THREAD_FPR23_LS64(\thread) 42 swc1 $f23, THREAD_FPR23(\thread)
43 swc1 $f24, THREAD_FPR24_LS64(\thread) 43 swc1 $f24, THREAD_FPR24(\thread)
44 swc1 $f25, THREAD_FPR25_LS64(\thread) 44 swc1 $f25, THREAD_FPR25(\thread)
45 swc1 $f26, THREAD_FPR26_LS64(\thread) 45 swc1 $f26, THREAD_FPR26(\thread)
46 swc1 $f27, THREAD_FPR27_LS64(\thread) 46 swc1 $f27, THREAD_FPR27(\thread)
47 swc1 $f28, THREAD_FPR28_LS64(\thread) 47 swc1 $f28, THREAD_FPR28(\thread)
48 swc1 $f29, THREAD_FPR29_LS64(\thread) 48 swc1 $f29, THREAD_FPR29(\thread)
49 swc1 $f30, THREAD_FPR30_LS64(\thread) 49 swc1 $f30, THREAD_FPR30(\thread)
50 swc1 $f31, THREAD_FPR31_LS64(\thread) 50 swc1 $f31, THREAD_FPR31(\thread)
51 sw \tmp, THREAD_FCR31(\thread) 51 sw \tmp, THREAD_FCR31(\thread)
52 .set pop 52 .set pop
53 .endm 53 .endm
@@ -56,38 +56,38 @@
56 .set push 56 .set push
57 SET_HARDFLOAT 57 SET_HARDFLOAT
58 lw \tmp, THREAD_FCR31(\thread) 58 lw \tmp, THREAD_FCR31(\thread)
59 lwc1 $f0, THREAD_FPR0_LS64(\thread) 59 lwc1 $f0, THREAD_FPR0(\thread)
60 lwc1 $f1, THREAD_FPR1_LS64(\thread) 60 lwc1 $f1, THREAD_FPR1(\thread)
61 lwc1 $f2, THREAD_FPR2_LS64(\thread) 61 lwc1 $f2, THREAD_FPR2(\thread)
62 lwc1 $f3, THREAD_FPR3_LS64(\thread) 62 lwc1 $f3, THREAD_FPR3(\thread)
63 lwc1 $f4, THREAD_FPR4_LS64(\thread) 63 lwc1 $f4, THREAD_FPR4(\thread)
64 lwc1 $f5, THREAD_FPR5_LS64(\thread) 64 lwc1 $f5, THREAD_FPR5(\thread)
65 lwc1 $f6, THREAD_FPR6_LS64(\thread) 65 lwc1 $f6, THREAD_FPR6(\thread)
66 lwc1 $f7, THREAD_FPR7_LS64(\thread) 66 lwc1 $f7, THREAD_FPR7(\thread)
67 lwc1 $f8, THREAD_FPR8_LS64(\thread) 67 lwc1 $f8, THREAD_FPR8(\thread)
68 lwc1 $f9, THREAD_FPR9_LS64(\thread) 68 lwc1 $f9, THREAD_FPR9(\thread)
69 lwc1 $f10, THREAD_FPR10_LS64(\thread) 69 lwc1 $f10, THREAD_FPR10(\thread)
70 lwc1 $f11, THREAD_FPR11_LS64(\thread) 70 lwc1 $f11, THREAD_FPR11(\thread)
71 lwc1 $f12, THREAD_FPR12_LS64(\thread) 71 lwc1 $f12, THREAD_FPR12(\thread)
72 lwc1 $f13, THREAD_FPR13_LS64(\thread) 72 lwc1 $f13, THREAD_FPR13(\thread)
73 lwc1 $f14, THREAD_FPR14_LS64(\thread) 73 lwc1 $f14, THREAD_FPR14(\thread)
74 lwc1 $f15, THREAD_FPR15_LS64(\thread) 74 lwc1 $f15, THREAD_FPR15(\thread)
75 lwc1 $f16, THREAD_FPR16_LS64(\thread) 75 lwc1 $f16, THREAD_FPR16(\thread)
76 lwc1 $f17, THREAD_FPR17_LS64(\thread) 76 lwc1 $f17, THREAD_FPR17(\thread)
77 lwc1 $f18, THREAD_FPR18_LS64(\thread) 77 lwc1 $f18, THREAD_FPR18(\thread)
78 lwc1 $f19, THREAD_FPR19_LS64(\thread) 78 lwc1 $f19, THREAD_FPR19(\thread)
79 lwc1 $f20, THREAD_FPR20_LS64(\thread) 79 lwc1 $f20, THREAD_FPR20(\thread)
80 lwc1 $f21, THREAD_FPR21_LS64(\thread) 80 lwc1 $f21, THREAD_FPR21(\thread)
81 lwc1 $f22, THREAD_FPR22_LS64(\thread) 81 lwc1 $f22, THREAD_FPR22(\thread)
82 lwc1 $f23, THREAD_FPR23_LS64(\thread) 82 lwc1 $f23, THREAD_FPR23(\thread)
83 lwc1 $f24, THREAD_FPR24_LS64(\thread) 83 lwc1 $f24, THREAD_FPR24(\thread)
84 lwc1 $f25, THREAD_FPR25_LS64(\thread) 84 lwc1 $f25, THREAD_FPR25(\thread)
85 lwc1 $f26, THREAD_FPR26_LS64(\thread) 85 lwc1 $f26, THREAD_FPR26(\thread)
86 lwc1 $f27, THREAD_FPR27_LS64(\thread) 86 lwc1 $f27, THREAD_FPR27(\thread)
87 lwc1 $f28, THREAD_FPR28_LS64(\thread) 87 lwc1 $f28, THREAD_FPR28(\thread)
88 lwc1 $f29, THREAD_FPR29_LS64(\thread) 88 lwc1 $f29, THREAD_FPR29(\thread)
89 lwc1 $f30, THREAD_FPR30_LS64(\thread) 89 lwc1 $f30, THREAD_FPR30(\thread)
90 lwc1 $f31, THREAD_FPR31_LS64(\thread) 90 lwc1 $f31, THREAD_FPR31(\thread)
91 ctc1 \tmp, fcr31 91 ctc1 \tmp, fcr31
92 .set pop 92 .set pop
93 .endm 93 .endm
diff --git a/arch/mips/include/asm/asmmacro.h b/arch/mips/include/asm/asmmacro.h
index 0cae4595e985..6156ac8c4cfb 100644
--- a/arch/mips/include/asm/asmmacro.h
+++ b/arch/mips/include/asm/asmmacro.h
@@ -60,22 +60,22 @@
60 .set push 60 .set push
61 SET_HARDFLOAT 61 SET_HARDFLOAT
62 cfc1 \tmp, fcr31 62 cfc1 \tmp, fcr31
63 sdc1 $f0, THREAD_FPR0_LS64(\thread) 63 sdc1 $f0, THREAD_FPR0(\thread)
64 sdc1 $f2, THREAD_FPR2_LS64(\thread) 64 sdc1 $f2, THREAD_FPR2(\thread)
65 sdc1 $f4, THREAD_FPR4_LS64(\thread) 65 sdc1 $f4, THREAD_FPR4(\thread)
66 sdc1 $f6, THREAD_FPR6_LS64(\thread) 66 sdc1 $f6, THREAD_FPR6(\thread)
67 sdc1 $f8, THREAD_FPR8_LS64(\thread) 67 sdc1 $f8, THREAD_FPR8(\thread)
68 sdc1 $f10, THREAD_FPR10_LS64(\thread) 68 sdc1 $f10, THREAD_FPR10(\thread)
69 sdc1 $f12, THREAD_FPR12_LS64(\thread) 69 sdc1 $f12, THREAD_FPR12(\thread)
70 sdc1 $f14, THREAD_FPR14_LS64(\thread) 70 sdc1 $f14, THREAD_FPR14(\thread)
71 sdc1 $f16, THREAD_FPR16_LS64(\thread) 71 sdc1 $f16, THREAD_FPR16(\thread)
72 sdc1 $f18, THREAD_FPR18_LS64(\thread) 72 sdc1 $f18, THREAD_FPR18(\thread)
73 sdc1 $f20, THREAD_FPR20_LS64(\thread) 73 sdc1 $f20, THREAD_FPR20(\thread)
74 sdc1 $f22, THREAD_FPR22_LS64(\thread) 74 sdc1 $f22, THREAD_FPR22(\thread)
75 sdc1 $f24, THREAD_FPR24_LS64(\thread) 75 sdc1 $f24, THREAD_FPR24(\thread)
76 sdc1 $f26, THREAD_FPR26_LS64(\thread) 76 sdc1 $f26, THREAD_FPR26(\thread)
77 sdc1 $f28, THREAD_FPR28_LS64(\thread) 77 sdc1 $f28, THREAD_FPR28(\thread)
78 sdc1 $f30, THREAD_FPR30_LS64(\thread) 78 sdc1 $f30, THREAD_FPR30(\thread)
79 sw \tmp, THREAD_FCR31(\thread) 79 sw \tmp, THREAD_FCR31(\thread)
80 .set pop 80 .set pop
81 .endm 81 .endm
@@ -84,22 +84,22 @@
84 .set push 84 .set push
85 .set mips64r2 85 .set mips64r2
86 SET_HARDFLOAT 86 SET_HARDFLOAT
87 sdc1 $f1, THREAD_FPR1_LS64(\thread) 87 sdc1 $f1, THREAD_FPR1(\thread)
88 sdc1 $f3, THREAD_FPR3_LS64(\thread) 88 sdc1 $f3, THREAD_FPR3(\thread)
89 sdc1 $f5, THREAD_FPR5_LS64(\thread) 89 sdc1 $f5, THREAD_FPR5(\thread)
90 sdc1 $f7, THREAD_FPR7_LS64(\thread) 90 sdc1 $f7, THREAD_FPR7(\thread)
91 sdc1 $f9, THREAD_FPR9_LS64(\thread) 91 sdc1 $f9, THREAD_FPR9(\thread)
92 sdc1 $f11, THREAD_FPR11_LS64(\thread) 92 sdc1 $f11, THREAD_FPR11(\thread)
93 sdc1 $f13, THREAD_FPR13_LS64(\thread) 93 sdc1 $f13, THREAD_FPR13(\thread)
94 sdc1 $f15, THREAD_FPR15_LS64(\thread) 94 sdc1 $f15, THREAD_FPR15(\thread)
95 sdc1 $f17, THREAD_FPR17_LS64(\thread) 95 sdc1 $f17, THREAD_FPR17(\thread)
96 sdc1 $f19, THREAD_FPR19_LS64(\thread) 96 sdc1 $f19, THREAD_FPR19(\thread)
97 sdc1 $f21, THREAD_FPR21_LS64(\thread) 97 sdc1 $f21, THREAD_FPR21(\thread)
98 sdc1 $f23, THREAD_FPR23_LS64(\thread) 98 sdc1 $f23, THREAD_FPR23(\thread)
99 sdc1 $f25, THREAD_FPR25_LS64(\thread) 99 sdc1 $f25, THREAD_FPR25(\thread)
100 sdc1 $f27, THREAD_FPR27_LS64(\thread) 100 sdc1 $f27, THREAD_FPR27(\thread)
101 sdc1 $f29, THREAD_FPR29_LS64(\thread) 101 sdc1 $f29, THREAD_FPR29(\thread)
102 sdc1 $f31, THREAD_FPR31_LS64(\thread) 102 sdc1 $f31, THREAD_FPR31(\thread)
103 .set pop 103 .set pop
104 .endm 104 .endm
105 105
@@ -118,22 +118,22 @@
118 .set push 118 .set push
119 SET_HARDFLOAT 119 SET_HARDFLOAT
120 lw \tmp, THREAD_FCR31(\thread) 120 lw \tmp, THREAD_FCR31(\thread)
121 ldc1 $f0, THREAD_FPR0_LS64(\thread) 121 ldc1 $f0, THREAD_FPR0(\thread)
122 ldc1 $f2, THREAD_FPR2_LS64(\thread) 122 ldc1 $f2, THREAD_FPR2(\thread)
123 ldc1 $f4, THREAD_FPR4_LS64(\thread) 123 ldc1 $f4, THREAD_FPR4(\thread)
124 ldc1 $f6, THREAD_FPR6_LS64(\thread) 124 ldc1 $f6, THREAD_FPR6(\thread)
125 ldc1 $f8, THREAD_FPR8_LS64(\thread) 125 ldc1 $f8, THREAD_FPR8(\thread)
126 ldc1 $f10, THREAD_FPR10_LS64(\thread) 126 ldc1 $f10, THREAD_FPR10(\thread)
127 ldc1 $f12, THREAD_FPR12_LS64(\thread) 127 ldc1 $f12, THREAD_FPR12(\thread)
128 ldc1 $f14, THREAD_FPR14_LS64(\thread) 128 ldc1 $f14, THREAD_FPR14(\thread)
129 ldc1 $f16, THREAD_FPR16_LS64(\thread) 129 ldc1 $f16, THREAD_FPR16(\thread)
130 ldc1 $f18, THREAD_FPR18_LS64(\thread) 130 ldc1 $f18, THREAD_FPR18(\thread)
131 ldc1 $f20, THREAD_FPR20_LS64(\thread) 131 ldc1 $f20, THREAD_FPR20(\thread)
132 ldc1 $f22, THREAD_FPR22_LS64(\thread) 132 ldc1 $f22, THREAD_FPR22(\thread)
133 ldc1 $f24, THREAD_FPR24_LS64(\thread) 133 ldc1 $f24, THREAD_FPR24(\thread)
134 ldc1 $f26, THREAD_FPR26_LS64(\thread) 134 ldc1 $f26, THREAD_FPR26(\thread)
135 ldc1 $f28, THREAD_FPR28_LS64(\thread) 135 ldc1 $f28, THREAD_FPR28(\thread)
136 ldc1 $f30, THREAD_FPR30_LS64(\thread) 136 ldc1 $f30, THREAD_FPR30(\thread)
137 ctc1 \tmp, fcr31 137 ctc1 \tmp, fcr31
138 .endm 138 .endm
139 139
@@ -141,22 +141,22 @@
141 .set push 141 .set push
142 .set mips64r2 142 .set mips64r2
143 SET_HARDFLOAT 143 SET_HARDFLOAT
144 ldc1 $f1, THREAD_FPR1_LS64(\thread) 144 ldc1 $f1, THREAD_FPR1(\thread)
145 ldc1 $f3, THREAD_FPR3_LS64(\thread) 145 ldc1 $f3, THREAD_FPR3(\thread)
146 ldc1 $f5, THREAD_FPR5_LS64(\thread) 146 ldc1 $f5, THREAD_FPR5(\thread)
147 ldc1 $f7, THREAD_FPR7_LS64(\thread) 147 ldc1 $f7, THREAD_FPR7(\thread)
148 ldc1 $f9, THREAD_FPR9_LS64(\thread) 148 ldc1 $f9, THREAD_FPR9(\thread)
149 ldc1 $f11, THREAD_FPR11_LS64(\thread) 149 ldc1 $f11, THREAD_FPR11(\thread)
150 ldc1 $f13, THREAD_FPR13_LS64(\thread) 150 ldc1 $f13, THREAD_FPR13(\thread)
151 ldc1 $f15, THREAD_FPR15_LS64(\thread) 151 ldc1 $f15, THREAD_FPR15(\thread)
152 ldc1 $f17, THREAD_FPR17_LS64(\thread) 152 ldc1 $f17, THREAD_FPR17(\thread)
153 ldc1 $f19, THREAD_FPR19_LS64(\thread) 153 ldc1 $f19, THREAD_FPR19(\thread)
154 ldc1 $f21, THREAD_FPR21_LS64(\thread) 154 ldc1 $f21, THREAD_FPR21(\thread)
155 ldc1 $f23, THREAD_FPR23_LS64(\thread) 155 ldc1 $f23, THREAD_FPR23(\thread)
156 ldc1 $f25, THREAD_FPR25_LS64(\thread) 156 ldc1 $f25, THREAD_FPR25(\thread)
157 ldc1 $f27, THREAD_FPR27_LS64(\thread) 157 ldc1 $f27, THREAD_FPR27(\thread)
158 ldc1 $f29, THREAD_FPR29_LS64(\thread) 158 ldc1 $f29, THREAD_FPR29(\thread)
159 ldc1 $f31, THREAD_FPR31_LS64(\thread) 159 ldc1 $f31, THREAD_FPR31(\thread)
160 .set pop 160 .set pop
161 .endm 161 .endm
162 162
@@ -211,6 +211,22 @@
211 .endm 211 .endm
212 212
213#ifdef TOOLCHAIN_SUPPORTS_MSA 213#ifdef TOOLCHAIN_SUPPORTS_MSA
214 .macro _cfcmsa rd, cs
215 .set push
216 .set mips32r2
217 .set msa
218 cfcmsa \rd, $\cs
219 .set pop
220 .endm
221
222 .macro _ctcmsa cd, rs
223 .set push
224 .set mips32r2
225 .set msa
226 ctcmsa $\cd, \rs
227 .set pop
228 .endm
229
214 .macro ld_d wd, off, base 230 .macro ld_d wd, off, base
215 .set push 231 .set push
216 .set mips32r2 232 .set mips32r2
@@ -227,35 +243,35 @@
227 .set pop 243 .set pop
228 .endm 244 .endm
229 245
230 .macro copy_u_w rd, ws, n 246 .macro copy_u_w ws, n
231 .set push 247 .set push
232 .set mips32r2 248 .set mips32r2
233 .set msa 249 .set msa
234 copy_u.w \rd, $w\ws[\n] 250 copy_u.w $1, $w\ws[\n]
235 .set pop 251 .set pop
236 .endm 252 .endm
237 253
238 .macro copy_u_d rd, ws, n 254 .macro copy_u_d ws, n
239 .set push 255 .set push
240 .set mips64r2 256 .set mips64r2
241 .set msa 257 .set msa
242 copy_u.d \rd, $w\ws[\n] 258 copy_u.d $1, $w\ws[\n]
243 .set pop 259 .set pop
244 .endm 260 .endm
245 261
246 .macro insert_w wd, n, rs 262 .macro insert_w wd, n
247 .set push 263 .set push
248 .set mips32r2 264 .set mips32r2
249 .set msa 265 .set msa
250 insert.w $w\wd[\n], \rs 266 insert.w $w\wd[\n], $1
251 .set pop 267 .set pop
252 .endm 268 .endm
253 269
254 .macro insert_d wd, n, rs 270 .macro insert_d wd, n
255 .set push 271 .set push
256 .set mips64r2 272 .set mips64r2
257 .set msa 273 .set msa
258 insert.d $w\wd[\n], \rs 274 insert.d $w\wd[\n], $1
259 .set pop 275 .set pop
260 .endm 276 .endm
261#else 277#else
@@ -283,7 +299,7 @@
283 /* 299 /*
284 * Temporary until all toolchains in use include MSA support. 300 * Temporary until all toolchains in use include MSA support.
285 */ 301 */
286 .macro cfcmsa rd, cs 302 .macro _cfcmsa rd, cs
287 .set push 303 .set push
288 .set noat 304 .set noat
289 SET_HARDFLOAT 305 SET_HARDFLOAT
@@ -293,7 +309,7 @@
293 .set pop 309 .set pop
294 .endm 310 .endm
295 311
296 .macro ctcmsa cd, rs 312 .macro _ctcmsa cd, rs
297 .set push 313 .set push
298 .set noat 314 .set noat
299 SET_HARDFLOAT 315 SET_HARDFLOAT
@@ -320,44 +336,36 @@
320 .set pop 336 .set pop
321 .endm 337 .endm
322 338
323 .macro copy_u_w rd, ws, n 339 .macro copy_u_w ws, n
324 .set push 340 .set push
325 .set noat 341 .set noat
326 SET_HARDFLOAT 342 SET_HARDFLOAT
327 .insn 343 .insn
328 .word COPY_UW_MSA_INSN | (\n << 16) | (\ws << 11) 344 .word COPY_UW_MSA_INSN | (\n << 16) | (\ws << 11)
329 /* move triggers an assembler bug... */
330 or \rd, $1, zero
331 .set pop 345 .set pop
332 .endm 346 .endm
333 347
334 .macro copy_u_d rd, ws, n 348 .macro copy_u_d ws, n
335 .set push 349 .set push
336 .set noat 350 .set noat
337 SET_HARDFLOAT 351 SET_HARDFLOAT
338 .insn 352 .insn
339 .word COPY_UD_MSA_INSN | (\n << 16) | (\ws << 11) 353 .word COPY_UD_MSA_INSN | (\n << 16) | (\ws << 11)
340 /* move triggers an assembler bug... */
341 or \rd, $1, zero
342 .set pop 354 .set pop
343 .endm 355 .endm
344 356
345 .macro insert_w wd, n, rs 357 .macro insert_w wd, n
346 .set push 358 .set push
347 .set noat 359 .set noat
348 SET_HARDFLOAT 360 SET_HARDFLOAT
349 /* move triggers an assembler bug... */
350 or $1, \rs, zero
351 .word INSERT_W_MSA_INSN | (\n << 16) | (\wd << 6) 361 .word INSERT_W_MSA_INSN | (\n << 16) | (\wd << 6)
352 .set pop 362 .set pop
353 .endm 363 .endm
354 364
355 .macro insert_d wd, n, rs 365 .macro insert_d wd, n
356 .set push 366 .set push
357 .set noat 367 .set noat
358 SET_HARDFLOAT 368 SET_HARDFLOAT
359 /* move triggers an assembler bug... */
360 or $1, \rs, zero
361 .word INSERT_D_MSA_INSN | (\n << 16) | (\wd << 6) 369 .word INSERT_D_MSA_INSN | (\n << 16) | (\wd << 6)
362 .set pop 370 .set pop
363 .endm 371 .endm
@@ -399,7 +407,7 @@
399 .set push 407 .set push
400 .set noat 408 .set noat
401 SET_HARDFLOAT 409 SET_HARDFLOAT
402 cfcmsa $1, MSA_CSR 410 _cfcmsa $1, MSA_CSR
403 sw $1, THREAD_MSA_CSR(\thread) 411 sw $1, THREAD_MSA_CSR(\thread)
404 .set pop 412 .set pop
405 .endm 413 .endm
@@ -409,7 +417,7 @@
409 .set noat 417 .set noat
410 SET_HARDFLOAT 418 SET_HARDFLOAT
411 lw $1, THREAD_MSA_CSR(\thread) 419 lw $1, THREAD_MSA_CSR(\thread)
412 ctcmsa MSA_CSR, $1 420 _ctcmsa MSA_CSR, $1
413 .set pop 421 .set pop
414 ld_d 0, THREAD_FPR0, \thread 422 ld_d 0, THREAD_FPR0, \thread
415 ld_d 1, THREAD_FPR1, \thread 423 ld_d 1, THREAD_FPR1, \thread
@@ -452,9 +460,6 @@
452 insert_w \wd, 2 460 insert_w \wd, 2
453 insert_w \wd, 3 461 insert_w \wd, 3
454#endif 462#endif
455 .if 31-\wd
456 msa_init_upper (\wd+1)
457 .endif
458 .endm 463 .endm
459 464
460 .macro msa_init_all_upper 465 .macro msa_init_all_upper
@@ -463,6 +468,37 @@
463 SET_HARDFLOAT 468 SET_HARDFLOAT
464 not $1, zero 469 not $1, zero
465 msa_init_upper 0 470 msa_init_upper 0
471 msa_init_upper 1
472 msa_init_upper 2
473 msa_init_upper 3
474 msa_init_upper 4
475 msa_init_upper 5
476 msa_init_upper 6
477 msa_init_upper 7
478 msa_init_upper 8
479 msa_init_upper 9
480 msa_init_upper 10
481 msa_init_upper 11
482 msa_init_upper 12
483 msa_init_upper 13
484 msa_init_upper 14
485 msa_init_upper 15
486 msa_init_upper 16
487 msa_init_upper 17
488 msa_init_upper 18
489 msa_init_upper 19
490 msa_init_upper 20
491 msa_init_upper 21
492 msa_init_upper 22
493 msa_init_upper 23
494 msa_init_upper 24
495 msa_init_upper 25
496 msa_init_upper 26
497 msa_init_upper 27
498 msa_init_upper 28
499 msa_init_upper 29
500 msa_init_upper 30
501 msa_init_upper 31
466 .set pop 502 .set pop
467 .endm 503 .endm
468 504
diff --git a/arch/mips/include/asm/fpu.h b/arch/mips/include/asm/fpu.h
index dd083e999b08..b104ad9d655f 100644
--- a/arch/mips/include/asm/fpu.h
+++ b/arch/mips/include/asm/fpu.h
@@ -48,6 +48,12 @@ enum fpu_mode {
48#define FPU_FR_MASK 0x1 48#define FPU_FR_MASK 0x1
49}; 49};
50 50
51#define __disable_fpu() \
52do { \
53 clear_c0_status(ST0_CU1); \
54 disable_fpu_hazard(); \
55} while (0)
56
51static inline int __enable_fpu(enum fpu_mode mode) 57static inline int __enable_fpu(enum fpu_mode mode)
52{ 58{
53 int fr; 59 int fr;
@@ -86,7 +92,12 @@ fr_common:
86 enable_fpu_hazard(); 92 enable_fpu_hazard();
87 93
88 /* check FR has the desired value */ 94 /* check FR has the desired value */
89 return (!!(read_c0_status() & ST0_FR) == !!fr) ? 0 : SIGFPE; 95 if (!!(read_c0_status() & ST0_FR) == !!fr)
96 return 0;
97
98 /* unsupported FR value */
99 __disable_fpu();
100 return SIGFPE;
90 101
91 default: 102 default:
92 BUG(); 103 BUG();
@@ -95,12 +106,6 @@ fr_common:
95 return SIGFPE; 106 return SIGFPE;
96} 107}
97 108
98#define __disable_fpu() \
99do { \
100 clear_c0_status(ST0_CU1); \
101 disable_fpu_hazard(); \
102} while (0)
103
104#define clear_fpu_owner() clear_thread_flag(TIF_USEDFPU) 109#define clear_fpu_owner() clear_thread_flag(TIF_USEDFPU)
105 110
106static inline int __is_fpu_owner(void) 111static inline int __is_fpu_owner(void)
@@ -170,6 +175,7 @@ static inline void lose_fpu(int save)
170 } 175 }
171 disable_msa(); 176 disable_msa();
172 clear_thread_flag(TIF_USEDMSA); 177 clear_thread_flag(TIF_USEDMSA);
178 __disable_fpu();
173 } else if (is_fpu_owner()) { 179 } else if (is_fpu_owner()) {
174 if (save) 180 if (save)
175 _save_fp(current); 181 _save_fp(current);
diff --git a/arch/mips/include/asm/kdebug.h b/arch/mips/include/asm/kdebug.h
index 6a9af5fcb5d7..cba22ab7ad4d 100644
--- a/arch/mips/include/asm/kdebug.h
+++ b/arch/mips/include/asm/kdebug.h
@@ -10,7 +10,8 @@ enum die_val {
10 DIE_RI, 10 DIE_RI,
11 DIE_PAGE_FAULT, 11 DIE_PAGE_FAULT,
12 DIE_BREAK, 12 DIE_BREAK,
13 DIE_SSTEPBP 13 DIE_SSTEPBP,
14 DIE_MSAFP
14}; 15};
15 16
16#endif /* _ASM_MIPS_KDEBUG_H */ 17#endif /* _ASM_MIPS_KDEBUG_H */
diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h
index ac4fc716062b..4c25823563fe 100644
--- a/arch/mips/include/asm/kvm_host.h
+++ b/arch/mips/include/asm/kvm_host.h
@@ -21,10 +21,10 @@
21 21
22/* MIPS KVM register ids */ 22/* MIPS KVM register ids */
23#define MIPS_CP0_32(_R, _S) \ 23#define MIPS_CP0_32(_R, _S) \
24 (KVM_REG_MIPS | KVM_REG_SIZE_U32 | 0x10000 | (8 * (_R) + (_S))) 24 (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U32 | (8 * (_R) + (_S)))
25 25
26#define MIPS_CP0_64(_R, _S) \ 26#define MIPS_CP0_64(_R, _S) \
27 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0x10000 | (8 * (_R) + (_S))) 27 (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U64 | (8 * (_R) + (_S)))
28 28
29#define KVM_REG_MIPS_CP0_INDEX MIPS_CP0_32(0, 0) 29#define KVM_REG_MIPS_CP0_INDEX MIPS_CP0_32(0, 0)
30#define KVM_REG_MIPS_CP0_ENTRYLO0 MIPS_CP0_64(2, 0) 30#define KVM_REG_MIPS_CP0_ENTRYLO0 MIPS_CP0_64(2, 0)
@@ -42,11 +42,14 @@
42#define KVM_REG_MIPS_CP0_STATUS MIPS_CP0_32(12, 0) 42#define KVM_REG_MIPS_CP0_STATUS MIPS_CP0_32(12, 0)
43#define KVM_REG_MIPS_CP0_CAUSE MIPS_CP0_32(13, 0) 43#define KVM_REG_MIPS_CP0_CAUSE MIPS_CP0_32(13, 0)
44#define KVM_REG_MIPS_CP0_EPC MIPS_CP0_64(14, 0) 44#define KVM_REG_MIPS_CP0_EPC MIPS_CP0_64(14, 0)
45#define KVM_REG_MIPS_CP0_PRID MIPS_CP0_32(15, 0)
45#define KVM_REG_MIPS_CP0_EBASE MIPS_CP0_64(15, 1) 46#define KVM_REG_MIPS_CP0_EBASE MIPS_CP0_64(15, 1)
46#define KVM_REG_MIPS_CP0_CONFIG MIPS_CP0_32(16, 0) 47#define KVM_REG_MIPS_CP0_CONFIG MIPS_CP0_32(16, 0)
47#define KVM_REG_MIPS_CP0_CONFIG1 MIPS_CP0_32(16, 1) 48#define KVM_REG_MIPS_CP0_CONFIG1 MIPS_CP0_32(16, 1)
48#define KVM_REG_MIPS_CP0_CONFIG2 MIPS_CP0_32(16, 2) 49#define KVM_REG_MIPS_CP0_CONFIG2 MIPS_CP0_32(16, 2)
49#define KVM_REG_MIPS_CP0_CONFIG3 MIPS_CP0_32(16, 3) 50#define KVM_REG_MIPS_CP0_CONFIG3 MIPS_CP0_32(16, 3)
51#define KVM_REG_MIPS_CP0_CONFIG4 MIPS_CP0_32(16, 4)
52#define KVM_REG_MIPS_CP0_CONFIG5 MIPS_CP0_32(16, 5)
50#define KVM_REG_MIPS_CP0_CONFIG7 MIPS_CP0_32(16, 7) 53#define KVM_REG_MIPS_CP0_CONFIG7 MIPS_CP0_32(16, 7)
51#define KVM_REG_MIPS_CP0_XCONTEXT MIPS_CP0_64(20, 0) 54#define KVM_REG_MIPS_CP0_XCONTEXT MIPS_CP0_64(20, 0)
52#define KVM_REG_MIPS_CP0_ERROREPC MIPS_CP0_64(30, 0) 55#define KVM_REG_MIPS_CP0_ERROREPC MIPS_CP0_64(30, 0)
@@ -119,6 +122,10 @@ struct kvm_vcpu_stat {
119 u32 syscall_exits; 122 u32 syscall_exits;
120 u32 resvd_inst_exits; 123 u32 resvd_inst_exits;
121 u32 break_inst_exits; 124 u32 break_inst_exits;
125 u32 trap_inst_exits;
126 u32 msa_fpe_exits;
127 u32 fpe_exits;
128 u32 msa_disabled_exits;
122 u32 flush_dcache_exits; 129 u32 flush_dcache_exits;
123 u32 halt_successful_poll; 130 u32 halt_successful_poll;
124 u32 halt_wakeup; 131 u32 halt_wakeup;
@@ -138,6 +145,10 @@ enum kvm_mips_exit_types {
138 SYSCALL_EXITS, 145 SYSCALL_EXITS,
139 RESVD_INST_EXITS, 146 RESVD_INST_EXITS,
140 BREAK_INST_EXITS, 147 BREAK_INST_EXITS,
148 TRAP_INST_EXITS,
149 MSA_FPE_EXITS,
150 FPE_EXITS,
151 MSA_DISABLED_EXITS,
141 FLUSH_DCACHE_EXITS, 152 FLUSH_DCACHE_EXITS,
142 MAX_KVM_MIPS_EXIT_TYPES 153 MAX_KVM_MIPS_EXIT_TYPES
143}; 154};
@@ -206,6 +217,8 @@ struct mips_coproc {
206#define MIPS_CP0_CONFIG1_SEL 1 217#define MIPS_CP0_CONFIG1_SEL 1
207#define MIPS_CP0_CONFIG2_SEL 2 218#define MIPS_CP0_CONFIG2_SEL 2
208#define MIPS_CP0_CONFIG3_SEL 3 219#define MIPS_CP0_CONFIG3_SEL 3
220#define MIPS_CP0_CONFIG4_SEL 4
221#define MIPS_CP0_CONFIG5_SEL 5
209 222
210/* Config0 register bits */ 223/* Config0 register bits */
211#define CP0C0_M 31 224#define CP0C0_M 31
@@ -262,31 +275,6 @@ struct mips_coproc {
262#define CP0C3_SM 1 275#define CP0C3_SM 1
263#define CP0C3_TL 0 276#define CP0C3_TL 0
264 277
265/* Have config1, Cacheable, noncoherent, write-back, write allocate*/
266#define MIPS_CONFIG0 \
267 ((1 << CP0C0_M) | (0x3 << CP0C0_K0))
268
269/* Have config2, no coprocessor2 attached, no MDMX support attached,
270 no performance counters, watch registers present,
271 no code compression, EJTAG present, no FPU, no watch registers */
272#define MIPS_CONFIG1 \
273((1 << CP0C1_M) | \
274 (0 << CP0C1_C2) | (0 << CP0C1_MD) | (0 << CP0C1_PC) | \
275 (0 << CP0C1_WR) | (0 << CP0C1_CA) | (1 << CP0C1_EP) | \
276 (0 << CP0C1_FP))
277
278/* Have config3, no tertiary/secondary caches implemented */
279#define MIPS_CONFIG2 \
280((1 << CP0C2_M))
281
282/* No config4, no DSP ASE, no large physaddr (PABITS),
283 no external interrupt controller, no vectored interrupts,
284 no 1kb pages, no SmartMIPS ASE, no trace logic */
285#define MIPS_CONFIG3 \
286((0 << CP0C3_M) | (0 << CP0C3_DSPP) | (0 << CP0C3_LPA) | \
287 (0 << CP0C3_VEIC) | (0 << CP0C3_VInt) | (0 << CP0C3_SP) | \
288 (0 << CP0C3_SM) | (0 << CP0C3_TL))
289
290/* MMU types, the first four entries have the same layout as the 278/* MMU types, the first four entries have the same layout as the
291 CP0C0_MT field. */ 279 CP0C0_MT field. */
292enum mips_mmu_types { 280enum mips_mmu_types {
@@ -321,7 +309,9 @@ enum mips_mmu_types {
321 */ 309 */
322#define T_TRAP 13 /* Trap instruction */ 310#define T_TRAP 13 /* Trap instruction */
323#define T_VCEI 14 /* Virtual coherency exception */ 311#define T_VCEI 14 /* Virtual coherency exception */
312#define T_MSAFPE 14 /* MSA floating point exception */
324#define T_FPE 15 /* Floating point exception */ 313#define T_FPE 15 /* Floating point exception */
314#define T_MSADIS 21 /* MSA disabled exception */
325#define T_WATCH 23 /* Watch address reference */ 315#define T_WATCH 23 /* Watch address reference */
326#define T_VCED 31 /* Virtual coherency data */ 316#define T_VCED 31 /* Virtual coherency data */
327 317
@@ -374,6 +364,9 @@ struct kvm_mips_tlb {
374 long tlb_lo1; 364 long tlb_lo1;
375}; 365};
376 366
367#define KVM_MIPS_FPU_FPU 0x1
368#define KVM_MIPS_FPU_MSA 0x2
369
377#define KVM_MIPS_GUEST_TLB_SIZE 64 370#define KVM_MIPS_GUEST_TLB_SIZE 64
378struct kvm_vcpu_arch { 371struct kvm_vcpu_arch {
379 void *host_ebase, *guest_ebase; 372 void *host_ebase, *guest_ebase;
@@ -395,6 +388,8 @@ struct kvm_vcpu_arch {
395 388
396 /* FPU State */ 389 /* FPU State */
397 struct mips_fpu_struct fpu; 390 struct mips_fpu_struct fpu;
391 /* Which FPU state is loaded (KVM_MIPS_FPU_*) */
392 unsigned int fpu_inuse;
398 393
399 /* COP0 State */ 394 /* COP0 State */
400 struct mips_coproc *cop0; 395 struct mips_coproc *cop0;
@@ -441,6 +436,9 @@ struct kvm_vcpu_arch {
441 436
442 /* WAIT executed */ 437 /* WAIT executed */
443 int wait; 438 int wait;
439
440 u8 fpu_enabled;
441 u8 msa_enabled;
444}; 442};
445 443
446 444
@@ -482,11 +480,15 @@ struct kvm_vcpu_arch {
482#define kvm_read_c0_guest_config1(cop0) (cop0->reg[MIPS_CP0_CONFIG][1]) 480#define kvm_read_c0_guest_config1(cop0) (cop0->reg[MIPS_CP0_CONFIG][1])
483#define kvm_read_c0_guest_config2(cop0) (cop0->reg[MIPS_CP0_CONFIG][2]) 481#define kvm_read_c0_guest_config2(cop0) (cop0->reg[MIPS_CP0_CONFIG][2])
484#define kvm_read_c0_guest_config3(cop0) (cop0->reg[MIPS_CP0_CONFIG][3]) 482#define kvm_read_c0_guest_config3(cop0) (cop0->reg[MIPS_CP0_CONFIG][3])
483#define kvm_read_c0_guest_config4(cop0) (cop0->reg[MIPS_CP0_CONFIG][4])
484#define kvm_read_c0_guest_config5(cop0) (cop0->reg[MIPS_CP0_CONFIG][5])
485#define kvm_read_c0_guest_config7(cop0) (cop0->reg[MIPS_CP0_CONFIG][7]) 485#define kvm_read_c0_guest_config7(cop0) (cop0->reg[MIPS_CP0_CONFIG][7])
486#define kvm_write_c0_guest_config(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][0] = (val)) 486#define kvm_write_c0_guest_config(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][0] = (val))
487#define kvm_write_c0_guest_config1(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][1] = (val)) 487#define kvm_write_c0_guest_config1(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][1] = (val))
488#define kvm_write_c0_guest_config2(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][2] = (val)) 488#define kvm_write_c0_guest_config2(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][2] = (val))
489#define kvm_write_c0_guest_config3(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][3] = (val)) 489#define kvm_write_c0_guest_config3(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][3] = (val))
490#define kvm_write_c0_guest_config4(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][4] = (val))
491#define kvm_write_c0_guest_config5(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][5] = (val))
490#define kvm_write_c0_guest_config7(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][7] = (val)) 492#define kvm_write_c0_guest_config7(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][7] = (val))
491#define kvm_read_c0_guest_errorepc(cop0) (cop0->reg[MIPS_CP0_ERROR_PC][0]) 493#define kvm_read_c0_guest_errorepc(cop0) (cop0->reg[MIPS_CP0_ERROR_PC][0])
492#define kvm_write_c0_guest_errorepc(cop0, val) (cop0->reg[MIPS_CP0_ERROR_PC][0] = (val)) 494#define kvm_write_c0_guest_errorepc(cop0, val) (cop0->reg[MIPS_CP0_ERROR_PC][0] = (val))
@@ -567,6 +569,31 @@ static inline void _kvm_atomic_change_c0_guest_reg(unsigned long *reg,
567 kvm_set_c0_guest_ebase(cop0, ((val) & (change))); \ 569 kvm_set_c0_guest_ebase(cop0, ((val) & (change))); \
568} 570}
569 571
572/* Helpers */
573
574static inline bool kvm_mips_guest_can_have_fpu(struct kvm_vcpu_arch *vcpu)
575{
576 return (!__builtin_constant_p(cpu_has_fpu) || cpu_has_fpu) &&
577 vcpu->fpu_enabled;
578}
579
580static inline bool kvm_mips_guest_has_fpu(struct kvm_vcpu_arch *vcpu)
581{
582 return kvm_mips_guest_can_have_fpu(vcpu) &&
583 kvm_read_c0_guest_config1(vcpu->cop0) & MIPS_CONF1_FP;
584}
585
586static inline bool kvm_mips_guest_can_have_msa(struct kvm_vcpu_arch *vcpu)
587{
588 return (!__builtin_constant_p(cpu_has_msa) || cpu_has_msa) &&
589 vcpu->msa_enabled;
590}
591
592static inline bool kvm_mips_guest_has_msa(struct kvm_vcpu_arch *vcpu)
593{
594 return kvm_mips_guest_can_have_msa(vcpu) &&
595 kvm_read_c0_guest_config3(vcpu->cop0) & MIPS_CONF3_MSA;
596}
570 597
571struct kvm_mips_callbacks { 598struct kvm_mips_callbacks {
572 int (*handle_cop_unusable)(struct kvm_vcpu *vcpu); 599 int (*handle_cop_unusable)(struct kvm_vcpu *vcpu);
@@ -578,6 +605,10 @@ struct kvm_mips_callbacks {
578 int (*handle_syscall)(struct kvm_vcpu *vcpu); 605 int (*handle_syscall)(struct kvm_vcpu *vcpu);
579 int (*handle_res_inst)(struct kvm_vcpu *vcpu); 606 int (*handle_res_inst)(struct kvm_vcpu *vcpu);
580 int (*handle_break)(struct kvm_vcpu *vcpu); 607 int (*handle_break)(struct kvm_vcpu *vcpu);
608 int (*handle_trap)(struct kvm_vcpu *vcpu);
609 int (*handle_msa_fpe)(struct kvm_vcpu *vcpu);
610 int (*handle_fpe)(struct kvm_vcpu *vcpu);
611 int (*handle_msa_disabled)(struct kvm_vcpu *vcpu);
581 int (*vm_init)(struct kvm *kvm); 612 int (*vm_init)(struct kvm *kvm);
582 int (*vcpu_init)(struct kvm_vcpu *vcpu); 613 int (*vcpu_init)(struct kvm_vcpu *vcpu);
583 int (*vcpu_setup)(struct kvm_vcpu *vcpu); 614 int (*vcpu_setup)(struct kvm_vcpu *vcpu);
@@ -596,6 +627,8 @@ struct kvm_mips_callbacks {
596 const struct kvm_one_reg *reg, s64 *v); 627 const struct kvm_one_reg *reg, s64 *v);
597 int (*set_one_reg)(struct kvm_vcpu *vcpu, 628 int (*set_one_reg)(struct kvm_vcpu *vcpu,
598 const struct kvm_one_reg *reg, s64 v); 629 const struct kvm_one_reg *reg, s64 v);
630 int (*vcpu_get_regs)(struct kvm_vcpu *vcpu);
631 int (*vcpu_set_regs)(struct kvm_vcpu *vcpu);
599}; 632};
600extern struct kvm_mips_callbacks *kvm_mips_callbacks; 633extern struct kvm_mips_callbacks *kvm_mips_callbacks;
601int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks); 634int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks);
@@ -606,6 +639,19 @@ int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);
606/* Trampoline ASM routine to start running in "Guest" context */ 639/* Trampoline ASM routine to start running in "Guest" context */
607extern int __kvm_mips_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu); 640extern int __kvm_mips_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu);
608 641
642/* FPU/MSA context management */
643void __kvm_save_fpu(struct kvm_vcpu_arch *vcpu);
644void __kvm_restore_fpu(struct kvm_vcpu_arch *vcpu);
645void __kvm_restore_fcsr(struct kvm_vcpu_arch *vcpu);
646void __kvm_save_msa(struct kvm_vcpu_arch *vcpu);
647void __kvm_restore_msa(struct kvm_vcpu_arch *vcpu);
648void __kvm_restore_msa_upper(struct kvm_vcpu_arch *vcpu);
649void __kvm_restore_msacsr(struct kvm_vcpu_arch *vcpu);
650void kvm_own_fpu(struct kvm_vcpu *vcpu);
651void kvm_own_msa(struct kvm_vcpu *vcpu);
652void kvm_drop_fpu(struct kvm_vcpu *vcpu);
653void kvm_lose_fpu(struct kvm_vcpu *vcpu);
654
609/* TLB handling */ 655/* TLB handling */
610uint32_t kvm_get_kernel_asid(struct kvm_vcpu *vcpu); 656uint32_t kvm_get_kernel_asid(struct kvm_vcpu *vcpu);
611 657
@@ -711,6 +757,26 @@ extern enum emulation_result kvm_mips_emulate_bp_exc(unsigned long cause,
711 struct kvm_run *run, 757 struct kvm_run *run,
712 struct kvm_vcpu *vcpu); 758 struct kvm_vcpu *vcpu);
713 759
760extern enum emulation_result kvm_mips_emulate_trap_exc(unsigned long cause,
761 uint32_t *opc,
762 struct kvm_run *run,
763 struct kvm_vcpu *vcpu);
764
765extern enum emulation_result kvm_mips_emulate_msafpe_exc(unsigned long cause,
766 uint32_t *opc,
767 struct kvm_run *run,
768 struct kvm_vcpu *vcpu);
769
770extern enum emulation_result kvm_mips_emulate_fpe_exc(unsigned long cause,
771 uint32_t *opc,
772 struct kvm_run *run,
773 struct kvm_vcpu *vcpu);
774
775extern enum emulation_result kvm_mips_emulate_msadis_exc(unsigned long cause,
776 uint32_t *opc,
777 struct kvm_run *run,
778 struct kvm_vcpu *vcpu);
779
714extern enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, 780extern enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
715 struct kvm_run *run); 781 struct kvm_run *run);
716 782
@@ -749,6 +815,11 @@ enum emulation_result kvm_mips_emulate_load(uint32_t inst,
749 struct kvm_run *run, 815 struct kvm_run *run,
750 struct kvm_vcpu *vcpu); 816 struct kvm_vcpu *vcpu);
751 817
818unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu);
819unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu);
820unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu);
821unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu);
822
752/* Dynamic binary translation */ 823/* Dynamic binary translation */
753extern int kvm_mips_trans_cache_index(uint32_t inst, uint32_t *opc, 824extern int kvm_mips_trans_cache_index(uint32_t inst, uint32_t *opc,
754 struct kvm_vcpu *vcpu); 825 struct kvm_vcpu *vcpu);
diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h
index b5dcbee01fd7..9b3b48e21c22 100644
--- a/arch/mips/include/asm/processor.h
+++ b/arch/mips/include/asm/processor.h
@@ -105,7 +105,7 @@ union fpureg {
105#ifdef CONFIG_CPU_LITTLE_ENDIAN 105#ifdef CONFIG_CPU_LITTLE_ENDIAN
106# define FPR_IDX(width, idx) (idx) 106# define FPR_IDX(width, idx) (idx)
107#else 107#else
108# define FPR_IDX(width, idx) ((FPU_REG_WIDTH / (width)) - 1 - (idx)) 108# define FPR_IDX(width, idx) ((idx) ^ ((64 / (width)) - 1))
109#endif 109#endif
110 110
111#define BUILD_FPR_ACCESS(width) \ 111#define BUILD_FPR_ACCESS(width) \
diff --git a/arch/mips/include/uapi/asm/kvm.h b/arch/mips/include/uapi/asm/kvm.h
index 2c04b6d9ff85..6985eb59b085 100644
--- a/arch/mips/include/uapi/asm/kvm.h
+++ b/arch/mips/include/uapi/asm/kvm.h
@@ -36,77 +36,85 @@ struct kvm_regs {
36 36
37/* 37/*
38 * for KVM_GET_FPU and KVM_SET_FPU 38 * for KVM_GET_FPU and KVM_SET_FPU
39 *
40 * If Status[FR] is zero (32-bit FPU), the upper 32-bits of the FPRs
41 * are zero filled.
42 */ 39 */
43struct kvm_fpu { 40struct kvm_fpu {
44 __u64 fpr[32];
45 __u32 fir;
46 __u32 fccr;
47 __u32 fexr;
48 __u32 fenr;
49 __u32 fcsr;
50 __u32 pad;
51}; 41};
52 42
53 43
54/* 44/*
55 * For MIPS, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access CP0 45 * For MIPS, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access various
56 * registers. The id field is broken down as follows: 46 * registers. The id field is broken down as follows:
57 * 47 *
58 * bits[2..0] - Register 'sel' index.
59 * bits[7..3] - Register 'rd' index.
60 * bits[15..8] - Must be zero.
61 * bits[31..16] - 1 -> CP0 registers.
62 * bits[51..32] - Must be zero.
63 * bits[63..52] - As per linux/kvm.h 48 * bits[63..52] - As per linux/kvm.h
49 * bits[51..32] - Must be zero.
50 * bits[31..16] - Register set.
51 *
52 * Register set = 0: GP registers from kvm_regs (see definitions below).
53 *
54 * Register set = 1: CP0 registers.
55 * bits[15..8] - Must be zero.
56 * bits[7..3] - Register 'rd' index.
57 * bits[2..0] - Register 'sel' index.
58 *
59 * Register set = 2: KVM specific registers (see definitions below).
60 *
61 * Register set = 3: FPU / MSA registers (see definitions below).
64 * 62 *
65 * Other sets registers may be added in the future. Each set would 63 * Other sets registers may be added in the future. Each set would
66 * have its own identifier in bits[31..16]. 64 * have its own identifier in bits[31..16].
67 *
68 * The registers defined in struct kvm_regs are also accessible, the
69 * id values for these are below.
70 */ 65 */
71 66
72#define KVM_REG_MIPS_R0 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0) 67#define KVM_REG_MIPS_GP (KVM_REG_MIPS | 0x0000000000000000ULL)
73#define KVM_REG_MIPS_R1 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 1) 68#define KVM_REG_MIPS_CP0 (KVM_REG_MIPS | 0x0000000000010000ULL)
74#define KVM_REG_MIPS_R2 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 2) 69#define KVM_REG_MIPS_KVM (KVM_REG_MIPS | 0x0000000000020000ULL)
75#define KVM_REG_MIPS_R3 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 3) 70#define KVM_REG_MIPS_FPU (KVM_REG_MIPS | 0x0000000000030000ULL)
76#define KVM_REG_MIPS_R4 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 4) 71
77#define KVM_REG_MIPS_R5 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 5) 72
78#define KVM_REG_MIPS_R6 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 6) 73/*
79#define KVM_REG_MIPS_R7 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 7) 74 * KVM_REG_MIPS_GP - General purpose registers from kvm_regs.
80#define KVM_REG_MIPS_R8 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 8) 75 */
81#define KVM_REG_MIPS_R9 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 9) 76
82#define KVM_REG_MIPS_R10 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 10) 77#define KVM_REG_MIPS_R0 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 0)
83#define KVM_REG_MIPS_R11 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 11) 78#define KVM_REG_MIPS_R1 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 1)
84#define KVM_REG_MIPS_R12 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 12) 79#define KVM_REG_MIPS_R2 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 2)
85#define KVM_REG_MIPS_R13 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 13) 80#define KVM_REG_MIPS_R3 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 3)
86#define KVM_REG_MIPS_R14 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 14) 81#define KVM_REG_MIPS_R4 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 4)
87#define KVM_REG_MIPS_R15 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 15) 82#define KVM_REG_MIPS_R5 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 5)
88#define KVM_REG_MIPS_R16 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 16) 83#define KVM_REG_MIPS_R6 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 6)
89#define KVM_REG_MIPS_R17 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 17) 84#define KVM_REG_MIPS_R7 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 7)
90#define KVM_REG_MIPS_R18 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 18) 85#define KVM_REG_MIPS_R8 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 8)
91#define KVM_REG_MIPS_R19 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 19) 86#define KVM_REG_MIPS_R9 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 9)
92#define KVM_REG_MIPS_R20 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 20) 87#define KVM_REG_MIPS_R10 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 10)
93#define KVM_REG_MIPS_R21 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 21) 88#define KVM_REG_MIPS_R11 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 11)
94#define KVM_REG_MIPS_R22 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 22) 89#define KVM_REG_MIPS_R12 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 12)
95#define KVM_REG_MIPS_R23 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 23) 90#define KVM_REG_MIPS_R13 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 13)
96#define KVM_REG_MIPS_R24 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 24) 91#define KVM_REG_MIPS_R14 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 14)
97#define KVM_REG_MIPS_R25 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 25) 92#define KVM_REG_MIPS_R15 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 15)
98#define KVM_REG_MIPS_R26 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 26) 93#define KVM_REG_MIPS_R16 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 16)
99#define KVM_REG_MIPS_R27 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 27) 94#define KVM_REG_MIPS_R17 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 17)
100#define KVM_REG_MIPS_R28 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 28) 95#define KVM_REG_MIPS_R18 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 18)
101#define KVM_REG_MIPS_R29 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 29) 96#define KVM_REG_MIPS_R19 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 19)
102#define KVM_REG_MIPS_R30 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 30) 97#define KVM_REG_MIPS_R20 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 20)
103#define KVM_REG_MIPS_R31 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 31) 98#define KVM_REG_MIPS_R21 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 21)
104 99#define KVM_REG_MIPS_R22 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 22)
105#define KVM_REG_MIPS_HI (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 32) 100#define KVM_REG_MIPS_R23 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 23)
106#define KVM_REG_MIPS_LO (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 33) 101#define KVM_REG_MIPS_R24 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 24)
107#define KVM_REG_MIPS_PC (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 34) 102#define KVM_REG_MIPS_R25 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 25)
108 103#define KVM_REG_MIPS_R26 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 26)
109/* KVM specific control registers */ 104#define KVM_REG_MIPS_R27 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 27)
105#define KVM_REG_MIPS_R28 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 28)
106#define KVM_REG_MIPS_R29 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 29)
107#define KVM_REG_MIPS_R30 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 30)
108#define KVM_REG_MIPS_R31 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 31)
109
110#define KVM_REG_MIPS_HI (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 32)
111#define KVM_REG_MIPS_LO (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 33)
112#define KVM_REG_MIPS_PC (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 34)
113
114
115/*
116 * KVM_REG_MIPS_KVM - KVM specific control registers.
117 */
110 118
111/* 119/*
112 * CP0_Count control 120 * CP0_Count control
@@ -118,8 +126,7 @@ struct kvm_fpu {
118 * safely without losing time or guest timer interrupts. 126 * safely without losing time or guest timer interrupts.
119 * Other: Reserved, do not change. 127 * Other: Reserved, do not change.
120 */ 128 */
121#define KVM_REG_MIPS_COUNT_CTL (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \ 129#define KVM_REG_MIPS_COUNT_CTL (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 0)
122 0x20000 | 0)
123#define KVM_REG_MIPS_COUNT_CTL_DC 0x00000001 130#define KVM_REG_MIPS_COUNT_CTL_DC 0x00000001
124 131
125/* 132/*
@@ -131,15 +138,46 @@ struct kvm_fpu {
131 * emulated. 138 * emulated.
132 * Modifications to times in the future are rejected. 139 * Modifications to times in the future are rejected.
133 */ 140 */
134#define KVM_REG_MIPS_COUNT_RESUME (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \ 141#define KVM_REG_MIPS_COUNT_RESUME (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 1)
135 0x20000 | 1)
136/* 142/*
137 * CP0_Count rate in Hz 143 * CP0_Count rate in Hz
138 * Specifies the rate of the CP0_Count timer in Hz. Modifications occur without 144 * Specifies the rate of the CP0_Count timer in Hz. Modifications occur without
139 * discontinuities in CP0_Count. 145 * discontinuities in CP0_Count.
140 */ 146 */
141#define KVM_REG_MIPS_COUNT_HZ (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \ 147#define KVM_REG_MIPS_COUNT_HZ (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 2)
142 0x20000 | 2) 148
149
150/*
151 * KVM_REG_MIPS_FPU - Floating Point and MIPS SIMD Architecture (MSA) registers.
152 *
153 * bits[15..8] - Register subset (see definitions below).
154 * bits[7..5] - Must be zero.
155 * bits[4..0] - Register number within register subset.
156 */
157
158#define KVM_REG_MIPS_FPR (KVM_REG_MIPS_FPU | 0x0000000000000000ULL)
159#define KVM_REG_MIPS_FCR (KVM_REG_MIPS_FPU | 0x0000000000000100ULL)
160#define KVM_REG_MIPS_MSACR (KVM_REG_MIPS_FPU | 0x0000000000000200ULL)
161
162/*
163 * KVM_REG_MIPS_FPR - Floating point / Vector registers.
164 */
165#define KVM_REG_MIPS_FPR_32(n) (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U32 | (n))
166#define KVM_REG_MIPS_FPR_64(n) (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U64 | (n))
167#define KVM_REG_MIPS_VEC_128(n) (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U128 | (n))
168
169/*
170 * KVM_REG_MIPS_FCR - Floating point control registers.
171 */
172#define KVM_REG_MIPS_FCR_IR (KVM_REG_MIPS_FCR | KVM_REG_SIZE_U32 | 0)
173#define KVM_REG_MIPS_FCR_CSR (KVM_REG_MIPS_FCR | KVM_REG_SIZE_U32 | 31)
174
175/*
176 * KVM_REG_MIPS_MSACR - MIPS SIMD Architecture (MSA) control registers.
177 */
178#define KVM_REG_MIPS_MSA_IR (KVM_REG_MIPS_MSACR | KVM_REG_SIZE_U32 | 0)
179#define KVM_REG_MIPS_MSA_CSR (KVM_REG_MIPS_MSACR | KVM_REG_SIZE_U32 | 1)
180
143 181
144/* 182/*
145 * KVM MIPS specific structures and definitions 183 * KVM MIPS specific structures and definitions
diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c
index 750d67ac41e9..e59fd7cfac9e 100644
--- a/arch/mips/kernel/asm-offsets.c
+++ b/arch/mips/kernel/asm-offsets.c
@@ -167,72 +167,6 @@ void output_thread_fpu_defines(void)
167 OFFSET(THREAD_FPR30, task_struct, thread.fpu.fpr[30]); 167 OFFSET(THREAD_FPR30, task_struct, thread.fpu.fpr[30]);
168 OFFSET(THREAD_FPR31, task_struct, thread.fpu.fpr[31]); 168 OFFSET(THREAD_FPR31, task_struct, thread.fpu.fpr[31]);
169 169
170 /* the least significant 64 bits of each FP register */
171 OFFSET(THREAD_FPR0_LS64, task_struct,
172 thread.fpu.fpr[0].val64[FPR_IDX(64, 0)]);
173 OFFSET(THREAD_FPR1_LS64, task_struct,
174 thread.fpu.fpr[1].val64[FPR_IDX(64, 0)]);
175 OFFSET(THREAD_FPR2_LS64, task_struct,
176 thread.fpu.fpr[2].val64[FPR_IDX(64, 0)]);
177 OFFSET(THREAD_FPR3_LS64, task_struct,
178 thread.fpu.fpr[3].val64[FPR_IDX(64, 0)]);
179 OFFSET(THREAD_FPR4_LS64, task_struct,
180 thread.fpu.fpr[4].val64[FPR_IDX(64, 0)]);
181 OFFSET(THREAD_FPR5_LS64, task_struct,
182 thread.fpu.fpr[5].val64[FPR_IDX(64, 0)]);
183 OFFSET(THREAD_FPR6_LS64, task_struct,
184 thread.fpu.fpr[6].val64[FPR_IDX(64, 0)]);
185 OFFSET(THREAD_FPR7_LS64, task_struct,
186 thread.fpu.fpr[7].val64[FPR_IDX(64, 0)]);
187 OFFSET(THREAD_FPR8_LS64, task_struct,
188 thread.fpu.fpr[8].val64[FPR_IDX(64, 0)]);
189 OFFSET(THREAD_FPR9_LS64, task_struct,
190 thread.fpu.fpr[9].val64[FPR_IDX(64, 0)]);
191 OFFSET(THREAD_FPR10_LS64, task_struct,
192 thread.fpu.fpr[10].val64[FPR_IDX(64, 0)]);
193 OFFSET(THREAD_FPR11_LS64, task_struct,
194 thread.fpu.fpr[11].val64[FPR_IDX(64, 0)]);
195 OFFSET(THREAD_FPR12_LS64, task_struct,
196 thread.fpu.fpr[12].val64[FPR_IDX(64, 0)]);
197 OFFSET(THREAD_FPR13_LS64, task_struct,
198 thread.fpu.fpr[13].val64[FPR_IDX(64, 0)]);
199 OFFSET(THREAD_FPR14_LS64, task_struct,
200 thread.fpu.fpr[14].val64[FPR_IDX(64, 0)]);
201 OFFSET(THREAD_FPR15_LS64, task_struct,
202 thread.fpu.fpr[15].val64[FPR_IDX(64, 0)]);
203 OFFSET(THREAD_FPR16_LS64, task_struct,
204 thread.fpu.fpr[16].val64[FPR_IDX(64, 0)]);
205 OFFSET(THREAD_FPR17_LS64, task_struct,
206 thread.fpu.fpr[17].val64[FPR_IDX(64, 0)]);
207 OFFSET(THREAD_FPR18_LS64, task_struct,
208 thread.fpu.fpr[18].val64[FPR_IDX(64, 0)]);
209 OFFSET(THREAD_FPR19_LS64, task_struct,
210 thread.fpu.fpr[19].val64[FPR_IDX(64, 0)]);
211 OFFSET(THREAD_FPR20_LS64, task_struct,
212 thread.fpu.fpr[20].val64[FPR_IDX(64, 0)]);
213 OFFSET(THREAD_FPR21_LS64, task_struct,
214 thread.fpu.fpr[21].val64[FPR_IDX(64, 0)]);
215 OFFSET(THREAD_FPR22_LS64, task_struct,
216 thread.fpu.fpr[22].val64[FPR_IDX(64, 0)]);
217 OFFSET(THREAD_FPR23_LS64, task_struct,
218 thread.fpu.fpr[23].val64[FPR_IDX(64, 0)]);
219 OFFSET(THREAD_FPR24_LS64, task_struct,
220 thread.fpu.fpr[24].val64[FPR_IDX(64, 0)]);
221 OFFSET(THREAD_FPR25_LS64, task_struct,
222 thread.fpu.fpr[25].val64[FPR_IDX(64, 0)]);
223 OFFSET(THREAD_FPR26_LS64, task_struct,
224 thread.fpu.fpr[26].val64[FPR_IDX(64, 0)]);
225 OFFSET(THREAD_FPR27_LS64, task_struct,
226 thread.fpu.fpr[27].val64[FPR_IDX(64, 0)]);
227 OFFSET(THREAD_FPR28_LS64, task_struct,
228 thread.fpu.fpr[28].val64[FPR_IDX(64, 0)]);
229 OFFSET(THREAD_FPR29_LS64, task_struct,
230 thread.fpu.fpr[29].val64[FPR_IDX(64, 0)]);
231 OFFSET(THREAD_FPR30_LS64, task_struct,
232 thread.fpu.fpr[30].val64[FPR_IDX(64, 0)]);
233 OFFSET(THREAD_FPR31_LS64, task_struct,
234 thread.fpu.fpr[31].val64[FPR_IDX(64, 0)]);
235
236 OFFSET(THREAD_FCR31, task_struct, thread.fpu.fcr31); 170 OFFSET(THREAD_FCR31, task_struct, thread.fpu.fcr31);
237 OFFSET(THREAD_MSA_CSR, task_struct, thread.fpu.msacsr); 171 OFFSET(THREAD_MSA_CSR, task_struct, thread.fpu.msacsr);
238 BLANK(); 172 BLANK();
@@ -470,6 +404,45 @@ void output_kvm_defines(void)
470 OFFSET(VCPU_LO, kvm_vcpu_arch, lo); 404 OFFSET(VCPU_LO, kvm_vcpu_arch, lo);
471 OFFSET(VCPU_HI, kvm_vcpu_arch, hi); 405 OFFSET(VCPU_HI, kvm_vcpu_arch, hi);
472 OFFSET(VCPU_PC, kvm_vcpu_arch, pc); 406 OFFSET(VCPU_PC, kvm_vcpu_arch, pc);
407 BLANK();
408
409 OFFSET(VCPU_FPR0, kvm_vcpu_arch, fpu.fpr[0]);
410 OFFSET(VCPU_FPR1, kvm_vcpu_arch, fpu.fpr[1]);
411 OFFSET(VCPU_FPR2, kvm_vcpu_arch, fpu.fpr[2]);
412 OFFSET(VCPU_FPR3, kvm_vcpu_arch, fpu.fpr[3]);
413 OFFSET(VCPU_FPR4, kvm_vcpu_arch, fpu.fpr[4]);
414 OFFSET(VCPU_FPR5, kvm_vcpu_arch, fpu.fpr[5]);
415 OFFSET(VCPU_FPR6, kvm_vcpu_arch, fpu.fpr[6]);
416 OFFSET(VCPU_FPR7, kvm_vcpu_arch, fpu.fpr[7]);
417 OFFSET(VCPU_FPR8, kvm_vcpu_arch, fpu.fpr[8]);
418 OFFSET(VCPU_FPR9, kvm_vcpu_arch, fpu.fpr[9]);
419 OFFSET(VCPU_FPR10, kvm_vcpu_arch, fpu.fpr[10]);
420 OFFSET(VCPU_FPR11, kvm_vcpu_arch, fpu.fpr[11]);
421 OFFSET(VCPU_FPR12, kvm_vcpu_arch, fpu.fpr[12]);
422 OFFSET(VCPU_FPR13, kvm_vcpu_arch, fpu.fpr[13]);
423 OFFSET(VCPU_FPR14, kvm_vcpu_arch, fpu.fpr[14]);
424 OFFSET(VCPU_FPR15, kvm_vcpu_arch, fpu.fpr[15]);
425 OFFSET(VCPU_FPR16, kvm_vcpu_arch, fpu.fpr[16]);
426 OFFSET(VCPU_FPR17, kvm_vcpu_arch, fpu.fpr[17]);
427 OFFSET(VCPU_FPR18, kvm_vcpu_arch, fpu.fpr[18]);
428 OFFSET(VCPU_FPR19, kvm_vcpu_arch, fpu.fpr[19]);
429 OFFSET(VCPU_FPR20, kvm_vcpu_arch, fpu.fpr[20]);
430 OFFSET(VCPU_FPR21, kvm_vcpu_arch, fpu.fpr[21]);
431 OFFSET(VCPU_FPR22, kvm_vcpu_arch, fpu.fpr[22]);
432 OFFSET(VCPU_FPR23, kvm_vcpu_arch, fpu.fpr[23]);
433 OFFSET(VCPU_FPR24, kvm_vcpu_arch, fpu.fpr[24]);
434 OFFSET(VCPU_FPR25, kvm_vcpu_arch, fpu.fpr[25]);
435 OFFSET(VCPU_FPR26, kvm_vcpu_arch, fpu.fpr[26]);
436 OFFSET(VCPU_FPR27, kvm_vcpu_arch, fpu.fpr[27]);
437 OFFSET(VCPU_FPR28, kvm_vcpu_arch, fpu.fpr[28]);
438 OFFSET(VCPU_FPR29, kvm_vcpu_arch, fpu.fpr[29]);
439 OFFSET(VCPU_FPR30, kvm_vcpu_arch, fpu.fpr[30]);
440 OFFSET(VCPU_FPR31, kvm_vcpu_arch, fpu.fpr[31]);
441
442 OFFSET(VCPU_FCR31, kvm_vcpu_arch, fpu.fcr31);
443 OFFSET(VCPU_MSA_CSR, kvm_vcpu_arch, fpu.msacsr);
444 BLANK();
445
473 OFFSET(VCPU_COP0, kvm_vcpu_arch, cop0); 446 OFFSET(VCPU_COP0, kvm_vcpu_arch, cop0);
474 OFFSET(VCPU_GUEST_KERNEL_ASID, kvm_vcpu_arch, guest_kernel_asid); 447 OFFSET(VCPU_GUEST_KERNEL_ASID, kvm_vcpu_arch, guest_kernel_asid);
475 OFFSET(VCPU_GUEST_USER_ASID, kvm_vcpu_arch, guest_user_asid); 448 OFFSET(VCPU_GUEST_USER_ASID, kvm_vcpu_arch, guest_user_asid);
diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S
index 2ebaabe3af15..af42e7003f12 100644
--- a/arch/mips/kernel/genex.S
+++ b/arch/mips/kernel/genex.S
@@ -360,12 +360,15 @@ NESTED(nmi_handler, PT_SIZE, sp)
360 .set mips1 360 .set mips1
361 SET_HARDFLOAT 361 SET_HARDFLOAT
362 cfc1 a1, fcr31 362 cfc1 a1, fcr31
363 li a2, ~(0x3f << 12)
364 and a2, a1
365 ctc1 a2, fcr31
366 .set pop 363 .set pop
367 TRACE_IRQS_ON 364 CLI
368 STI 365 TRACE_IRQS_OFF
366 .endm
367
368 .macro __build_clear_msa_fpe
369 _cfcmsa a1, MSA_CSR
370 CLI
371 TRACE_IRQS_OFF
369 .endm 372 .endm
370 373
371 .macro __build_clear_ade 374 .macro __build_clear_ade
@@ -426,7 +429,7 @@ NESTED(nmi_handler, PT_SIZE, sp)
426 BUILD_HANDLER cpu cpu sti silent /* #11 */ 429 BUILD_HANDLER cpu cpu sti silent /* #11 */
427 BUILD_HANDLER ov ov sti silent /* #12 */ 430 BUILD_HANDLER ov ov sti silent /* #12 */
428 BUILD_HANDLER tr tr sti silent /* #13 */ 431 BUILD_HANDLER tr tr sti silent /* #13 */
429 BUILD_HANDLER msa_fpe msa_fpe sti silent /* #14 */ 432 BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent /* #14 */
430 BUILD_HANDLER fpe fpe fpe silent /* #15 */ 433 BUILD_HANDLER fpe fpe fpe silent /* #15 */
431 BUILD_HANDLER ftlb ftlb none silent /* #16 */ 434 BUILD_HANDLER ftlb ftlb none silent /* #16 */
432 BUILD_HANDLER msa msa sti silent /* #21 */ 435 BUILD_HANDLER msa msa sti silent /* #21 */
diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c
index 510452812594..7da6e324dd35 100644
--- a/arch/mips/kernel/ptrace.c
+++ b/arch/mips/kernel/ptrace.c
@@ -46,6 +46,26 @@
46#define CREATE_TRACE_POINTS 46#define CREATE_TRACE_POINTS
47#include <trace/events/syscalls.h> 47#include <trace/events/syscalls.h>
48 48
49static void init_fp_ctx(struct task_struct *target)
50{
51 /* If FP has been used then the target already has context */
52 if (tsk_used_math(target))
53 return;
54
55 /* Begin with data registers set to all 1s... */
56 memset(&target->thread.fpu.fpr, ~0, sizeof(target->thread.fpu.fpr));
57
58 /* ...and FCSR zeroed */
59 target->thread.fpu.fcr31 = 0;
60
61 /*
62 * Record that the target has "used" math, such that the context
63 * just initialised, and any modifications made by the caller,
64 * aren't discarded.
65 */
66 set_stopped_child_used_math(target);
67}
68
49/* 69/*
50 * Called by kernel/ptrace.c when detaching.. 70 * Called by kernel/ptrace.c when detaching..
51 * 71 *
@@ -142,6 +162,7 @@ int ptrace_setfpregs(struct task_struct *child, __u32 __user *data)
142 if (!access_ok(VERIFY_READ, data, 33 * 8)) 162 if (!access_ok(VERIFY_READ, data, 33 * 8))
143 return -EIO; 163 return -EIO;
144 164
165 init_fp_ctx(child);
145 fregs = get_fpu_regs(child); 166 fregs = get_fpu_regs(child);
146 167
147 for (i = 0; i < 32; i++) { 168 for (i = 0; i < 32; i++) {
@@ -439,6 +460,8 @@ static int fpr_set(struct task_struct *target,
439 460
440 /* XXX fcr31 */ 461 /* XXX fcr31 */
441 462
463 init_fp_ctx(target);
464
442 if (sizeof(target->thread.fpu.fpr[i]) == sizeof(elf_fpreg_t)) 465 if (sizeof(target->thread.fpu.fpr[i]) == sizeof(elf_fpreg_t))
443 return user_regset_copyin(&pos, &count, &kbuf, &ubuf, 466 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
444 &target->thread.fpu, 467 &target->thread.fpu,
@@ -660,12 +683,7 @@ long arch_ptrace(struct task_struct *child, long request,
660 case FPR_BASE ... FPR_BASE + 31: { 683 case FPR_BASE ... FPR_BASE + 31: {
661 union fpureg *fregs = get_fpu_regs(child); 684 union fpureg *fregs = get_fpu_regs(child);
662 685
663 if (!tsk_used_math(child)) { 686 init_fp_ctx(child);
664 /* FP not yet used */
665 memset(&child->thread.fpu, ~0,
666 sizeof(child->thread.fpu));
667 child->thread.fpu.fcr31 = 0;
668 }
669#ifdef CONFIG_32BIT 687#ifdef CONFIG_32BIT
670 if (test_thread_flag(TIF_32BIT_FPREGS)) { 688 if (test_thread_flag(TIF_32BIT_FPREGS)) {
671 /* 689 /*
diff --git a/arch/mips/kernel/r4k_fpu.S b/arch/mips/kernel/r4k_fpu.S
index 676c5030a953..1d88af26ba82 100644
--- a/arch/mips/kernel/r4k_fpu.S
+++ b/arch/mips/kernel/r4k_fpu.S
@@ -34,7 +34,6 @@
34 .endm 34 .endm
35 35
36 .set noreorder 36 .set noreorder
37 .set MIPS_ISA_ARCH_LEVEL_RAW
38 37
39LEAF(_save_fp_context) 38LEAF(_save_fp_context)
40 .set push 39 .set push
@@ -103,6 +102,7 @@ LEAF(_save_fp_context)
103 /* Save 32-bit process floating point context */ 102 /* Save 32-bit process floating point context */
104LEAF(_save_fp_context32) 103LEAF(_save_fp_context32)
105 .set push 104 .set push
105 .set MIPS_ISA_ARCH_LEVEL_RAW
106 SET_HARDFLOAT 106 SET_HARDFLOAT
107 cfc1 t1, fcr31 107 cfc1 t1, fcr31
108 108
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 33984c04b60b..5b4d711f878d 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -701,6 +701,13 @@ asmlinkage void do_ov(struct pt_regs *regs)
701 701
702int process_fpemu_return(int sig, void __user *fault_addr) 702int process_fpemu_return(int sig, void __user *fault_addr)
703{ 703{
704 /*
705 * We can't allow the emulated instruction to leave any of the cause
706 * bits set in FCSR. If they were then the kernel would take an FP
707 * exception when restoring FP context.
708 */
709 current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X;
710
704 if (sig == SIGSEGV || sig == SIGBUS) { 711 if (sig == SIGSEGV || sig == SIGBUS) {
705 struct siginfo si = {0}; 712 struct siginfo si = {0};
706 si.si_addr = fault_addr; 713 si.si_addr = fault_addr;
@@ -781,6 +788,11 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
781 if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs), 788 if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs),
782 SIGFPE) == NOTIFY_STOP) 789 SIGFPE) == NOTIFY_STOP)
783 goto out; 790 goto out;
791
792 /* Clear FCSR.Cause before enabling interrupts */
793 write_32bit_cp1_register(CP1_STATUS, fcr31 & ~FPU_CSR_ALL_X);
794 local_irq_enable();
795
784 die_if_kernel("FP exception in kernel code", regs); 796 die_if_kernel("FP exception in kernel code", regs);
785 797
786 if (fcr31 & FPU_CSR_UNI_X) { 798 if (fcr31 & FPU_CSR_UNI_X) {
@@ -804,18 +816,12 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
804 sig = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1, 816 sig = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
805 &fault_addr); 817 &fault_addr);
806 818
807 /* 819 /* If something went wrong, signal */
808 * We can't allow the emulated instruction to leave any of 820 process_fpemu_return(sig, fault_addr);
809 * the cause bit set in $fcr31.
810 */
811 current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X;
812 821
813 /* Restore the hardware register state */ 822 /* Restore the hardware register state */
814 own_fpu(1); /* Using the FPU again. */ 823 own_fpu(1); /* Using the FPU again. */
815 824
816 /* If something went wrong, signal */
817 process_fpemu_return(sig, fault_addr);
818
819 goto out; 825 goto out;
820 } else if (fcr31 & FPU_CSR_INV_X) 826 } else if (fcr31 & FPU_CSR_INV_X)
821 info.si_code = FPE_FLTINV; 827 info.si_code = FPE_FLTINV;
@@ -1392,13 +1398,22 @@ out:
1392 exception_exit(prev_state); 1398 exception_exit(prev_state);
1393} 1399}
1394 1400
1395asmlinkage void do_msa_fpe(struct pt_regs *regs) 1401asmlinkage void do_msa_fpe(struct pt_regs *regs, unsigned int msacsr)
1396{ 1402{
1397 enum ctx_state prev_state; 1403 enum ctx_state prev_state;
1398 1404
1399 prev_state = exception_enter(); 1405 prev_state = exception_enter();
1406 if (notify_die(DIE_MSAFP, "MSA FP exception", regs, 0,
1407 regs_to_trapnr(regs), SIGFPE) == NOTIFY_STOP)
1408 goto out;
1409
1410 /* Clear MSACSR.Cause before enabling interrupts */
1411 write_msa_csr(msacsr & ~MSA_CSR_CAUSEF);
1412 local_irq_enable();
1413
1400 die_if_kernel("do_msa_fpe invoked from kernel context!", regs); 1414 die_if_kernel("do_msa_fpe invoked from kernel context!", regs);
1401 force_sig(SIGFPE, current); 1415 force_sig(SIGFPE, current);
1416out:
1402 exception_exit(prev_state); 1417 exception_exit(prev_state);
1403} 1418}
1404 1419
diff --git a/arch/mips/kvm/Makefile b/arch/mips/kvm/Makefile
index 401fe027c261..637ebbebd549 100644
--- a/arch/mips/kvm/Makefile
+++ b/arch/mips/kvm/Makefile
@@ -1,13 +1,15 @@
1# Makefile for KVM support for MIPS 1# Makefile for KVM support for MIPS
2# 2#
3 3
4common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o) 4common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o)
5 5
6EXTRA_CFLAGS += -Ivirt/kvm -Iarch/mips/kvm 6EXTRA_CFLAGS += -Ivirt/kvm -Iarch/mips/kvm
7 7
8kvm-objs := $(common-objs) mips.o emulate.o locore.o \ 8common-objs-$(CONFIG_CPU_HAS_MSA) += msa.o
9
10kvm-objs := $(common-objs-y) mips.o emulate.o locore.o \
9 interrupt.o stats.o commpage.o \ 11 interrupt.o stats.o commpage.o \
10 dyntrans.o trap_emul.o 12 dyntrans.o trap_emul.o fpu.o
11 13
12obj-$(CONFIG_KVM) += kvm.o 14obj-$(CONFIG_KVM) += kvm.o
13obj-y += callback.o tlb.o 15obj-y += callback.o tlb.o
diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c
index fb3e8dfd1ff6..6230f376a44e 100644
--- a/arch/mips/kvm/emulate.c
+++ b/arch/mips/kvm/emulate.c
@@ -884,6 +884,84 @@ enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
884 return EMULATE_DONE; 884 return EMULATE_DONE;
885} 885}
886 886
887/**
888 * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
889 * @vcpu: Virtual CPU.
890 *
891 * Finds the mask of bits which are writable in the guest's Config1 CP0
892 * register, by userland (currently read-only to the guest).
893 */
894unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
895{
896 unsigned int mask = 0;
897
898 /* Permit FPU to be present if FPU is supported */
899 if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
900 mask |= MIPS_CONF1_FP;
901
902 return mask;
903}
904
905/**
906 * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
907 * @vcpu: Virtual CPU.
908 *
909 * Finds the mask of bits which are writable in the guest's Config3 CP0
910 * register, by userland (currently read-only to the guest).
911 */
912unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
913{
914 /* Config4 is optional */
915 unsigned int mask = MIPS_CONF_M;
916
917 /* Permit MSA to be present if MSA is supported */
918 if (kvm_mips_guest_can_have_msa(&vcpu->arch))
919 mask |= MIPS_CONF3_MSA;
920
921 return mask;
922}
923
924/**
925 * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
926 * @vcpu: Virtual CPU.
927 *
928 * Finds the mask of bits which are writable in the guest's Config4 CP0
929 * register, by userland (currently read-only to the guest).
930 */
931unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
932{
933 /* Config5 is optional */
934 return MIPS_CONF_M;
935}
936
937/**
938 * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
939 * @vcpu: Virtual CPU.
940 *
941 * Finds the mask of bits which are writable in the guest's Config5 CP0
942 * register, by the guest itself.
943 */
944unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
945{
946 unsigned int mask = 0;
947
948 /* Permit MSAEn changes if MSA supported and enabled */
949 if (kvm_mips_guest_has_msa(&vcpu->arch))
950 mask |= MIPS_CONF5_MSAEN;
951
952 /*
953 * Permit guest FPU mode changes if FPU is enabled and the relevant
954 * feature exists according to FIR register.
955 */
956 if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
957 if (cpu_has_fre)
958 mask |= MIPS_CONF5_FRE;
959 /* We don't support UFR or UFE */
960 }
961
962 return mask;
963}
964
887enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc, 965enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc,
888 uint32_t cause, struct kvm_run *run, 966 uint32_t cause, struct kvm_run *run,
889 struct kvm_vcpu *vcpu) 967 struct kvm_vcpu *vcpu)
@@ -1021,18 +1099,114 @@ enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc,
1021 kvm_mips_write_compare(vcpu, 1099 kvm_mips_write_compare(vcpu,
1022 vcpu->arch.gprs[rt]); 1100 vcpu->arch.gprs[rt]);
1023 } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) { 1101 } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1024 kvm_write_c0_guest_status(cop0, 1102 unsigned int old_val, val, change;
1025 vcpu->arch.gprs[rt]); 1103
1104 old_val = kvm_read_c0_guest_status(cop0);
1105 val = vcpu->arch.gprs[rt];
1106 change = val ^ old_val;
1107
1108 /* Make sure that the NMI bit is never set */
1109 val &= ~ST0_NMI;
1110
1111 /*
1112 * Don't allow CU1 or FR to be set unless FPU
1113 * capability enabled and exists in guest
1114 * configuration.
1115 */
1116 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1117 val &= ~(ST0_CU1 | ST0_FR);
1118
1119 /*
1120 * Also don't allow FR to be set if host doesn't
1121 * support it.
1122 */
1123 if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1124 val &= ~ST0_FR;
1125
1126
1127 /* Handle changes in FPU mode */
1128 preempt_disable();
1129
1130 /*
1131 * FPU and Vector register state is made
1132 * UNPREDICTABLE by a change of FR, so don't
1133 * even bother saving it.
1134 */
1135 if (change & ST0_FR)
1136 kvm_drop_fpu(vcpu);
1137
1138 /*
1139 * If MSA state is already live, it is undefined
1140 * how it interacts with FR=0 FPU state, and we
1141 * don't want to hit reserved instruction
1142 * exceptions trying to save the MSA state later
1143 * when CU=1 && FR=1, so play it safe and save
1144 * it first.
1145 */
1146 if (change & ST0_CU1 && !(val & ST0_FR) &&
1147 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
1148 kvm_lose_fpu(vcpu);
1149
1026 /* 1150 /*
1027 * Make sure that CU1 and NMI bits are 1151 * Propagate CU1 (FPU enable) changes
1028 * never set 1152 * immediately if the FPU context is already
1153 * loaded. When disabling we leave the context
1154 * loaded so it can be quickly enabled again in
1155 * the near future.
1029 */ 1156 */
1030 kvm_clear_c0_guest_status(cop0, 1157 if (change & ST0_CU1 &&
1031 (ST0_CU1 | ST0_NMI)); 1158 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
1159 change_c0_status(ST0_CU1, val);
1160
1161 preempt_enable();
1162
1163 kvm_write_c0_guest_status(cop0, val);
1032 1164
1033#ifdef CONFIG_KVM_MIPS_DYN_TRANS 1165#ifdef CONFIG_KVM_MIPS_DYN_TRANS
1034 kvm_mips_trans_mtc0(inst, opc, vcpu); 1166 /*
1167 * If FPU present, we need CU1/FR bits to take
1168 * effect fairly soon.
1169 */
1170 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1171 kvm_mips_trans_mtc0(inst, opc, vcpu);
1035#endif 1172#endif
1173 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1174 unsigned int old_val, val, change, wrmask;
1175
1176 old_val = kvm_read_c0_guest_config5(cop0);
1177 val = vcpu->arch.gprs[rt];
1178
1179 /* Only a few bits are writable in Config5 */
1180 wrmask = kvm_mips_config5_wrmask(vcpu);
1181 change = (val ^ old_val) & wrmask;
1182 val = old_val ^ change;
1183
1184
1185 /* Handle changes in FPU/MSA modes */
1186 preempt_disable();
1187
1188 /*
1189 * Propagate FRE changes immediately if the FPU
1190 * context is already loaded.
1191 */
1192 if (change & MIPS_CONF5_FRE &&
1193 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
1194 change_c0_config5(MIPS_CONF5_FRE, val);
1195
1196 /*
1197 * Propagate MSAEn changes immediately if the
1198 * MSA context is already loaded. When disabling
1199 * we leave the context loaded so it can be
1200 * quickly enabled again in the near future.
1201 */
1202 if (change & MIPS_CONF5_MSAEN &&
1203 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
1204 change_c0_config5(MIPS_CONF5_MSAEN,
1205 val);
1206
1207 preempt_enable();
1208
1209 kvm_write_c0_guest_config5(cop0, val);
1036 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) { 1210 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1037 uint32_t old_cause, new_cause; 1211 uint32_t old_cause, new_cause;
1038 1212
@@ -1970,6 +2144,146 @@ enum emulation_result kvm_mips_emulate_bp_exc(unsigned long cause,
1970 return er; 2144 return er;
1971} 2145}
1972 2146
2147enum emulation_result kvm_mips_emulate_trap_exc(unsigned long cause,
2148 uint32_t *opc,
2149 struct kvm_run *run,
2150 struct kvm_vcpu *vcpu)
2151{
2152 struct mips_coproc *cop0 = vcpu->arch.cop0;
2153 struct kvm_vcpu_arch *arch = &vcpu->arch;
2154 enum emulation_result er = EMULATE_DONE;
2155
2156 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2157 /* save old pc */
2158 kvm_write_c0_guest_epc(cop0, arch->pc);
2159 kvm_set_c0_guest_status(cop0, ST0_EXL);
2160
2161 if (cause & CAUSEF_BD)
2162 kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2163 else
2164 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2165
2166 kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2167
2168 kvm_change_c0_guest_cause(cop0, (0xff),
2169 (T_TRAP << CAUSEB_EXCCODE));
2170
2171 /* Set PC to the exception entry point */
2172 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2173
2174 } else {
2175 kvm_err("Trying to deliver TRAP when EXL is already set\n");
2176 er = EMULATE_FAIL;
2177 }
2178
2179 return er;
2180}
2181
2182enum emulation_result kvm_mips_emulate_msafpe_exc(unsigned long cause,
2183 uint32_t *opc,
2184 struct kvm_run *run,
2185 struct kvm_vcpu *vcpu)
2186{
2187 struct mips_coproc *cop0 = vcpu->arch.cop0;
2188 struct kvm_vcpu_arch *arch = &vcpu->arch;
2189 enum emulation_result er = EMULATE_DONE;
2190
2191 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2192 /* save old pc */
2193 kvm_write_c0_guest_epc(cop0, arch->pc);
2194 kvm_set_c0_guest_status(cop0, ST0_EXL);
2195
2196 if (cause & CAUSEF_BD)
2197 kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2198 else
2199 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2200
2201 kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2202
2203 kvm_change_c0_guest_cause(cop0, (0xff),
2204 (T_MSAFPE << CAUSEB_EXCCODE));
2205
2206 /* Set PC to the exception entry point */
2207 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2208
2209 } else {
2210 kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2211 er = EMULATE_FAIL;
2212 }
2213
2214 return er;
2215}
2216
2217enum emulation_result kvm_mips_emulate_fpe_exc(unsigned long cause,
2218 uint32_t *opc,
2219 struct kvm_run *run,
2220 struct kvm_vcpu *vcpu)
2221{
2222 struct mips_coproc *cop0 = vcpu->arch.cop0;
2223 struct kvm_vcpu_arch *arch = &vcpu->arch;
2224 enum emulation_result er = EMULATE_DONE;
2225
2226 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2227 /* save old pc */
2228 kvm_write_c0_guest_epc(cop0, arch->pc);
2229 kvm_set_c0_guest_status(cop0, ST0_EXL);
2230
2231 if (cause & CAUSEF_BD)
2232 kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2233 else
2234 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2235
2236 kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2237
2238 kvm_change_c0_guest_cause(cop0, (0xff),
2239 (T_FPE << CAUSEB_EXCCODE));
2240
2241 /* Set PC to the exception entry point */
2242 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2243
2244 } else {
2245 kvm_err("Trying to deliver FPE when EXL is already set\n");
2246 er = EMULATE_FAIL;
2247 }
2248
2249 return er;
2250}
2251
2252enum emulation_result kvm_mips_emulate_msadis_exc(unsigned long cause,
2253 uint32_t *opc,
2254 struct kvm_run *run,
2255 struct kvm_vcpu *vcpu)
2256{
2257 struct mips_coproc *cop0 = vcpu->arch.cop0;
2258 struct kvm_vcpu_arch *arch = &vcpu->arch;
2259 enum emulation_result er = EMULATE_DONE;
2260
2261 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2262 /* save old pc */
2263 kvm_write_c0_guest_epc(cop0, arch->pc);
2264 kvm_set_c0_guest_status(cop0, ST0_EXL);
2265
2266 if (cause & CAUSEF_BD)
2267 kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2268 else
2269 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2270
2271 kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2272
2273 kvm_change_c0_guest_cause(cop0, (0xff),
2274 (T_MSADIS << CAUSEB_EXCCODE));
2275
2276 /* Set PC to the exception entry point */
2277 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2278
2279 } else {
2280 kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2281 er = EMULATE_FAIL;
2282 }
2283
2284 return er;
2285}
2286
1973/* ll/sc, rdhwr, sync emulation */ 2287/* ll/sc, rdhwr, sync emulation */
1974 2288
1975#define OPCODE 0xfc000000 2289#define OPCODE 0xfc000000
@@ -2176,6 +2490,10 @@ enum emulation_result kvm_mips_check_privilege(unsigned long cause,
2176 case T_SYSCALL: 2490 case T_SYSCALL:
2177 case T_BREAK: 2491 case T_BREAK:
2178 case T_RES_INST: 2492 case T_RES_INST:
2493 case T_TRAP:
2494 case T_MSAFPE:
2495 case T_FPE:
2496 case T_MSADIS:
2179 break; 2497 break;
2180 2498
2181 case T_COP_UNUSABLE: 2499 case T_COP_UNUSABLE:
diff --git a/arch/mips/kvm/fpu.S b/arch/mips/kvm/fpu.S
new file mode 100644
index 000000000000..531fbf5131c0
--- /dev/null
+++ b/arch/mips/kvm/fpu.S
@@ -0,0 +1,122 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * FPU context handling code for KVM.
7 *
8 * Copyright (C) 2015 Imagination Technologies Ltd.
9 */
10
11#include <asm/asm.h>
12#include <asm/asm-offsets.h>
13#include <asm/fpregdef.h>
14#include <asm/mipsregs.h>
15#include <asm/regdef.h>
16
17 .set noreorder
18 .set noat
19
20LEAF(__kvm_save_fpu)
21 .set push
22 .set mips64r2
23 SET_HARDFLOAT
24 mfc0 t0, CP0_STATUS
25 sll t0, t0, 5 # is Status.FR set?
26 bgez t0, 1f # no: skip odd doubles
27 nop
28 sdc1 $f1, VCPU_FPR1(a0)
29 sdc1 $f3, VCPU_FPR3(a0)
30 sdc1 $f5, VCPU_FPR5(a0)
31 sdc1 $f7, VCPU_FPR7(a0)
32 sdc1 $f9, VCPU_FPR9(a0)
33 sdc1 $f11, VCPU_FPR11(a0)
34 sdc1 $f13, VCPU_FPR13(a0)
35 sdc1 $f15, VCPU_FPR15(a0)
36 sdc1 $f17, VCPU_FPR17(a0)
37 sdc1 $f19, VCPU_FPR19(a0)
38 sdc1 $f21, VCPU_FPR21(a0)
39 sdc1 $f23, VCPU_FPR23(a0)
40 sdc1 $f25, VCPU_FPR25(a0)
41 sdc1 $f27, VCPU_FPR27(a0)
42 sdc1 $f29, VCPU_FPR29(a0)
43 sdc1 $f31, VCPU_FPR31(a0)
441: sdc1 $f0, VCPU_FPR0(a0)
45 sdc1 $f2, VCPU_FPR2(a0)
46 sdc1 $f4, VCPU_FPR4(a0)
47 sdc1 $f6, VCPU_FPR6(a0)
48 sdc1 $f8, VCPU_FPR8(a0)
49 sdc1 $f10, VCPU_FPR10(a0)
50 sdc1 $f12, VCPU_FPR12(a0)
51 sdc1 $f14, VCPU_FPR14(a0)
52 sdc1 $f16, VCPU_FPR16(a0)
53 sdc1 $f18, VCPU_FPR18(a0)
54 sdc1 $f20, VCPU_FPR20(a0)
55 sdc1 $f22, VCPU_FPR22(a0)
56 sdc1 $f24, VCPU_FPR24(a0)
57 sdc1 $f26, VCPU_FPR26(a0)
58 sdc1 $f28, VCPU_FPR28(a0)
59 jr ra
60 sdc1 $f30, VCPU_FPR30(a0)
61 .set pop
62 END(__kvm_save_fpu)
63
64LEAF(__kvm_restore_fpu)
65 .set push
66 .set mips64r2
67 SET_HARDFLOAT
68 mfc0 t0, CP0_STATUS
69 sll t0, t0, 5 # is Status.FR set?
70 bgez t0, 1f # no: skip odd doubles
71 nop
72 ldc1 $f1, VCPU_FPR1(a0)
73 ldc1 $f3, VCPU_FPR3(a0)
74 ldc1 $f5, VCPU_FPR5(a0)
75 ldc1 $f7, VCPU_FPR7(a0)
76 ldc1 $f9, VCPU_FPR9(a0)
77 ldc1 $f11, VCPU_FPR11(a0)
78 ldc1 $f13, VCPU_FPR13(a0)
79 ldc1 $f15, VCPU_FPR15(a0)
80 ldc1 $f17, VCPU_FPR17(a0)
81 ldc1 $f19, VCPU_FPR19(a0)
82 ldc1 $f21, VCPU_FPR21(a0)
83 ldc1 $f23, VCPU_FPR23(a0)
84 ldc1 $f25, VCPU_FPR25(a0)
85 ldc1 $f27, VCPU_FPR27(a0)
86 ldc1 $f29, VCPU_FPR29(a0)
87 ldc1 $f31, VCPU_FPR31(a0)
881: ldc1 $f0, VCPU_FPR0(a0)
89 ldc1 $f2, VCPU_FPR2(a0)
90 ldc1 $f4, VCPU_FPR4(a0)
91 ldc1 $f6, VCPU_FPR6(a0)
92 ldc1 $f8, VCPU_FPR8(a0)
93 ldc1 $f10, VCPU_FPR10(a0)
94 ldc1 $f12, VCPU_FPR12(a0)
95 ldc1 $f14, VCPU_FPR14(a0)
96 ldc1 $f16, VCPU_FPR16(a0)
97 ldc1 $f18, VCPU_FPR18(a0)
98 ldc1 $f20, VCPU_FPR20(a0)
99 ldc1 $f22, VCPU_FPR22(a0)
100 ldc1 $f24, VCPU_FPR24(a0)
101 ldc1 $f26, VCPU_FPR26(a0)
102 ldc1 $f28, VCPU_FPR28(a0)
103 jr ra
104 ldc1 $f30, VCPU_FPR30(a0)
105 .set pop
106 END(__kvm_restore_fpu)
107
108LEAF(__kvm_restore_fcsr)
109 .set push
110 SET_HARDFLOAT
111 lw t0, VCPU_FCR31(a0)
112 /*
113 * The ctc1 must stay at this offset in __kvm_restore_fcsr.
114 * See kvm_mips_csr_die_notify() which handles t0 containing a value
115 * which triggers an FP Exception, which must be stepped over and
116 * ignored since the set cause bits must remain there for the guest.
117 */
118 ctc1 t0, fcr31
119 jr ra
120 nop
121 .set pop
122 END(__kvm_restore_fcsr)
diff --git a/arch/mips/kvm/locore.S b/arch/mips/kvm/locore.S
index 4a68b176d6e4..c567240386a0 100644
--- a/arch/mips/kvm/locore.S
+++ b/arch/mips/kvm/locore.S
@@ -36,6 +36,8 @@
36#define PT_HOST_USERLOCAL PT_EPC 36#define PT_HOST_USERLOCAL PT_EPC
37 37
38#define CP0_DDATA_LO $28,3 38#define CP0_DDATA_LO $28,3
39#define CP0_CONFIG3 $16,3
40#define CP0_CONFIG5 $16,5
39#define CP0_EBASE $15,1 41#define CP0_EBASE $15,1
40 42
41#define CP0_INTCTL $12,1 43#define CP0_INTCTL $12,1
@@ -353,6 +355,42 @@ NESTED (MIPSX(GuestException), CALLFRAME_SIZ, ra)
353 LONG_L k0, VCPU_HOST_EBASE(k1) 355 LONG_L k0, VCPU_HOST_EBASE(k1)
354 mtc0 k0,CP0_EBASE 356 mtc0 k0,CP0_EBASE
355 357
358 /*
359 * If FPU is enabled, save FCR31 and clear it so that later ctc1's don't
360 * trigger FPE for pending exceptions.
361 */
362 .set at
363 and v1, v0, ST0_CU1
364 beqz v1, 1f
365 nop
366 .set push
367 SET_HARDFLOAT
368 cfc1 t0, fcr31
369 sw t0, VCPU_FCR31(k1)
370 ctc1 zero,fcr31
371 .set pop
372 .set noat
3731:
374
375#ifdef CONFIG_CPU_HAS_MSA
376 /*
377 * If MSA is enabled, save MSACSR and clear it so that later
378 * instructions don't trigger MSAFPE for pending exceptions.
379 */
380 mfc0 t0, CP0_CONFIG3
381 ext t0, t0, 28, 1 /* MIPS_CONF3_MSAP */
382 beqz t0, 1f
383 nop
384 mfc0 t0, CP0_CONFIG5
385 ext t0, t0, 27, 1 /* MIPS_CONF5_MSAEN */
386 beqz t0, 1f
387 nop
388 _cfcmsa t0, MSA_CSR
389 sw t0, VCPU_MSA_CSR(k1)
390 _ctcmsa MSA_CSR, zero
3911:
392#endif
393
356 /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */ 394 /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */
357 .set at 395 .set at
358 and v0, v0, ~(ST0_EXL | KSU_USER | ST0_IE) 396 and v0, v0, ~(ST0_EXL | KSU_USER | ST0_IE)
diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index c9eccf5df912..bb68e8d520e8 100644
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -11,6 +11,7 @@
11 11
12#include <linux/errno.h> 12#include <linux/errno.h>
13#include <linux/err.h> 13#include <linux/err.h>
14#include <linux/kdebug.h>
14#include <linux/module.h> 15#include <linux/module.h>
15#include <linux/vmalloc.h> 16#include <linux/vmalloc.h>
16#include <linux/fs.h> 17#include <linux/fs.h>
@@ -48,6 +49,10 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
48 { "syscall", VCPU_STAT(syscall_exits), KVM_STAT_VCPU }, 49 { "syscall", VCPU_STAT(syscall_exits), KVM_STAT_VCPU },
49 { "resvd_inst", VCPU_STAT(resvd_inst_exits), KVM_STAT_VCPU }, 50 { "resvd_inst", VCPU_STAT(resvd_inst_exits), KVM_STAT_VCPU },
50 { "break_inst", VCPU_STAT(break_inst_exits), KVM_STAT_VCPU }, 51 { "break_inst", VCPU_STAT(break_inst_exits), KVM_STAT_VCPU },
52 { "trap_inst", VCPU_STAT(trap_inst_exits), KVM_STAT_VCPU },
53 { "msa_fpe", VCPU_STAT(msa_fpe_exits), KVM_STAT_VCPU },
54 { "fpe", VCPU_STAT(fpe_exits), KVM_STAT_VCPU },
55 { "msa_disabled", VCPU_STAT(msa_disabled_exits), KVM_STAT_VCPU },
51 { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU }, 56 { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
52 { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU }, 57 { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
53 { "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU }, 58 { "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU },
@@ -504,10 +509,13 @@ static u64 kvm_mips_get_one_regs[] = {
504 KVM_REG_MIPS_CP0_STATUS, 509 KVM_REG_MIPS_CP0_STATUS,
505 KVM_REG_MIPS_CP0_CAUSE, 510 KVM_REG_MIPS_CP0_CAUSE,
506 KVM_REG_MIPS_CP0_EPC, 511 KVM_REG_MIPS_CP0_EPC,
512 KVM_REG_MIPS_CP0_PRID,
507 KVM_REG_MIPS_CP0_CONFIG, 513 KVM_REG_MIPS_CP0_CONFIG,
508 KVM_REG_MIPS_CP0_CONFIG1, 514 KVM_REG_MIPS_CP0_CONFIG1,
509 KVM_REG_MIPS_CP0_CONFIG2, 515 KVM_REG_MIPS_CP0_CONFIG2,
510 KVM_REG_MIPS_CP0_CONFIG3, 516 KVM_REG_MIPS_CP0_CONFIG3,
517 KVM_REG_MIPS_CP0_CONFIG4,
518 KVM_REG_MIPS_CP0_CONFIG5,
511 KVM_REG_MIPS_CP0_CONFIG7, 519 KVM_REG_MIPS_CP0_CONFIG7,
512 KVM_REG_MIPS_CP0_ERROREPC, 520 KVM_REG_MIPS_CP0_ERROREPC,
513 521
@@ -520,10 +528,14 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
520 const struct kvm_one_reg *reg) 528 const struct kvm_one_reg *reg)
521{ 529{
522 struct mips_coproc *cop0 = vcpu->arch.cop0; 530 struct mips_coproc *cop0 = vcpu->arch.cop0;
531 struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
523 int ret; 532 int ret;
524 s64 v; 533 s64 v;
534 s64 vs[2];
535 unsigned int idx;
525 536
526 switch (reg->id) { 537 switch (reg->id) {
538 /* General purpose registers */
527 case KVM_REG_MIPS_R0 ... KVM_REG_MIPS_R31: 539 case KVM_REG_MIPS_R0 ... KVM_REG_MIPS_R31:
528 v = (long)vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0]; 540 v = (long)vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0];
529 break; 541 break;
@@ -537,6 +549,67 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
537 v = (long)vcpu->arch.pc; 549 v = (long)vcpu->arch.pc;
538 break; 550 break;
539 551
552 /* Floating point registers */
553 case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
554 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
555 return -EINVAL;
556 idx = reg->id - KVM_REG_MIPS_FPR_32(0);
557 /* Odd singles in top of even double when FR=0 */
558 if (kvm_read_c0_guest_status(cop0) & ST0_FR)
559 v = get_fpr32(&fpu->fpr[idx], 0);
560 else
561 v = get_fpr32(&fpu->fpr[idx & ~1], idx & 1);
562 break;
563 case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
564 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
565 return -EINVAL;
566 idx = reg->id - KVM_REG_MIPS_FPR_64(0);
567 /* Can't access odd doubles in FR=0 mode */
568 if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
569 return -EINVAL;
570 v = get_fpr64(&fpu->fpr[idx], 0);
571 break;
572 case KVM_REG_MIPS_FCR_IR:
573 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
574 return -EINVAL;
575 v = boot_cpu_data.fpu_id;
576 break;
577 case KVM_REG_MIPS_FCR_CSR:
578 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
579 return -EINVAL;
580 v = fpu->fcr31;
581 break;
582
583 /* MIPS SIMD Architecture (MSA) registers */
584 case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
585 if (!kvm_mips_guest_has_msa(&vcpu->arch))
586 return -EINVAL;
587 /* Can't access MSA registers in FR=0 mode */
588 if (!(kvm_read_c0_guest_status(cop0) & ST0_FR))
589 return -EINVAL;
590 idx = reg->id - KVM_REG_MIPS_VEC_128(0);
591#ifdef CONFIG_CPU_LITTLE_ENDIAN
592 /* least significant byte first */
593 vs[0] = get_fpr64(&fpu->fpr[idx], 0);
594 vs[1] = get_fpr64(&fpu->fpr[idx], 1);
595#else
596 /* most significant byte first */
597 vs[0] = get_fpr64(&fpu->fpr[idx], 1);
598 vs[1] = get_fpr64(&fpu->fpr[idx], 0);
599#endif
600 break;
601 case KVM_REG_MIPS_MSA_IR:
602 if (!kvm_mips_guest_has_msa(&vcpu->arch))
603 return -EINVAL;
604 v = boot_cpu_data.msa_id;
605 break;
606 case KVM_REG_MIPS_MSA_CSR:
607 if (!kvm_mips_guest_has_msa(&vcpu->arch))
608 return -EINVAL;
609 v = fpu->msacsr;
610 break;
611
612 /* Co-processor 0 registers */
540 case KVM_REG_MIPS_CP0_INDEX: 613 case KVM_REG_MIPS_CP0_INDEX:
541 v = (long)kvm_read_c0_guest_index(cop0); 614 v = (long)kvm_read_c0_guest_index(cop0);
542 break; 615 break;
@@ -573,8 +646,8 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
573 case KVM_REG_MIPS_CP0_EPC: 646 case KVM_REG_MIPS_CP0_EPC:
574 v = (long)kvm_read_c0_guest_epc(cop0); 647 v = (long)kvm_read_c0_guest_epc(cop0);
575 break; 648 break;
576 case KVM_REG_MIPS_CP0_ERROREPC: 649 case KVM_REG_MIPS_CP0_PRID:
577 v = (long)kvm_read_c0_guest_errorepc(cop0); 650 v = (long)kvm_read_c0_guest_prid(cop0);
578 break; 651 break;
579 case KVM_REG_MIPS_CP0_CONFIG: 652 case KVM_REG_MIPS_CP0_CONFIG:
580 v = (long)kvm_read_c0_guest_config(cop0); 653 v = (long)kvm_read_c0_guest_config(cop0);
@@ -588,9 +661,18 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
588 case KVM_REG_MIPS_CP0_CONFIG3: 661 case KVM_REG_MIPS_CP0_CONFIG3:
589 v = (long)kvm_read_c0_guest_config3(cop0); 662 v = (long)kvm_read_c0_guest_config3(cop0);
590 break; 663 break;
664 case KVM_REG_MIPS_CP0_CONFIG4:
665 v = (long)kvm_read_c0_guest_config4(cop0);
666 break;
667 case KVM_REG_MIPS_CP0_CONFIG5:
668 v = (long)kvm_read_c0_guest_config5(cop0);
669 break;
591 case KVM_REG_MIPS_CP0_CONFIG7: 670 case KVM_REG_MIPS_CP0_CONFIG7:
592 v = (long)kvm_read_c0_guest_config7(cop0); 671 v = (long)kvm_read_c0_guest_config7(cop0);
593 break; 672 break;
673 case KVM_REG_MIPS_CP0_ERROREPC:
674 v = (long)kvm_read_c0_guest_errorepc(cop0);
675 break;
594 /* registers to be handled specially */ 676 /* registers to be handled specially */
595 case KVM_REG_MIPS_CP0_COUNT: 677 case KVM_REG_MIPS_CP0_COUNT:
596 case KVM_REG_MIPS_COUNT_CTL: 678 case KVM_REG_MIPS_COUNT_CTL:
@@ -612,6 +694,10 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
612 u32 v32 = (u32)v; 694 u32 v32 = (u32)v;
613 695
614 return put_user(v32, uaddr32); 696 return put_user(v32, uaddr32);
697 } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
698 void __user *uaddr = (void __user *)(long)reg->addr;
699
700 return copy_to_user(uaddr, vs, 16);
615 } else { 701 } else {
616 return -EINVAL; 702 return -EINVAL;
617 } 703 }
@@ -621,7 +707,10 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
621 const struct kvm_one_reg *reg) 707 const struct kvm_one_reg *reg)
622{ 708{
623 struct mips_coproc *cop0 = vcpu->arch.cop0; 709 struct mips_coproc *cop0 = vcpu->arch.cop0;
624 u64 v; 710 struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
711 s64 v;
712 s64 vs[2];
713 unsigned int idx;
625 714
626 if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) { 715 if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
627 u64 __user *uaddr64 = (u64 __user *)(long)reg->addr; 716 u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;
@@ -635,11 +724,16 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
635 if (get_user(v32, uaddr32) != 0) 724 if (get_user(v32, uaddr32) != 0)
636 return -EFAULT; 725 return -EFAULT;
637 v = (s64)v32; 726 v = (s64)v32;
727 } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
728 void __user *uaddr = (void __user *)(long)reg->addr;
729
730 return copy_from_user(vs, uaddr, 16);
638 } else { 731 } else {
639 return -EINVAL; 732 return -EINVAL;
640 } 733 }
641 734
642 switch (reg->id) { 735 switch (reg->id) {
736 /* General purpose registers */
643 case KVM_REG_MIPS_R0: 737 case KVM_REG_MIPS_R0:
644 /* Silently ignore requests to set $0 */ 738 /* Silently ignore requests to set $0 */
645 break; 739 break;
@@ -656,6 +750,64 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
656 vcpu->arch.pc = v; 750 vcpu->arch.pc = v;
657 break; 751 break;
658 752
753 /* Floating point registers */
754 case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
755 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
756 return -EINVAL;
757 idx = reg->id - KVM_REG_MIPS_FPR_32(0);
758 /* Odd singles in top of even double when FR=0 */
759 if (kvm_read_c0_guest_status(cop0) & ST0_FR)
760 set_fpr32(&fpu->fpr[idx], 0, v);
761 else
762 set_fpr32(&fpu->fpr[idx & ~1], idx & 1, v);
763 break;
764 case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
765 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
766 return -EINVAL;
767 idx = reg->id - KVM_REG_MIPS_FPR_64(0);
768 /* Can't access odd doubles in FR=0 mode */
769 if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
770 return -EINVAL;
771 set_fpr64(&fpu->fpr[idx], 0, v);
772 break;
773 case KVM_REG_MIPS_FCR_IR:
774 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
775 return -EINVAL;
776 /* Read-only */
777 break;
778 case KVM_REG_MIPS_FCR_CSR:
779 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
780 return -EINVAL;
781 fpu->fcr31 = v;
782 break;
783
784 /* MIPS SIMD Architecture (MSA) registers */
785 case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
786 if (!kvm_mips_guest_has_msa(&vcpu->arch))
787 return -EINVAL;
788 idx = reg->id - KVM_REG_MIPS_VEC_128(0);
789#ifdef CONFIG_CPU_LITTLE_ENDIAN
790 /* least significant byte first */
791 set_fpr64(&fpu->fpr[idx], 0, vs[0]);
792 set_fpr64(&fpu->fpr[idx], 1, vs[1]);
793#else
794 /* most significant byte first */
795 set_fpr64(&fpu->fpr[idx], 1, vs[0]);
796 set_fpr64(&fpu->fpr[idx], 0, vs[1]);
797#endif
798 break;
799 case KVM_REG_MIPS_MSA_IR:
800 if (!kvm_mips_guest_has_msa(&vcpu->arch))
801 return -EINVAL;
802 /* Read-only */
803 break;
804 case KVM_REG_MIPS_MSA_CSR:
805 if (!kvm_mips_guest_has_msa(&vcpu->arch))
806 return -EINVAL;
807 fpu->msacsr = v;
808 break;
809
810 /* Co-processor 0 registers */
659 case KVM_REG_MIPS_CP0_INDEX: 811 case KVM_REG_MIPS_CP0_INDEX:
660 kvm_write_c0_guest_index(cop0, v); 812 kvm_write_c0_guest_index(cop0, v);
661 break; 813 break;
@@ -686,6 +838,9 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
686 case KVM_REG_MIPS_CP0_EPC: 838 case KVM_REG_MIPS_CP0_EPC:
687 kvm_write_c0_guest_epc(cop0, v); 839 kvm_write_c0_guest_epc(cop0, v);
688 break; 840 break;
841 case KVM_REG_MIPS_CP0_PRID:
842 kvm_write_c0_guest_prid(cop0, v);
843 break;
689 case KVM_REG_MIPS_CP0_ERROREPC: 844 case KVM_REG_MIPS_CP0_ERROREPC:
690 kvm_write_c0_guest_errorepc(cop0, v); 845 kvm_write_c0_guest_errorepc(cop0, v);
691 break; 846 break;
@@ -693,6 +848,12 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
693 case KVM_REG_MIPS_CP0_COUNT: 848 case KVM_REG_MIPS_CP0_COUNT:
694 case KVM_REG_MIPS_CP0_COMPARE: 849 case KVM_REG_MIPS_CP0_COMPARE:
695 case KVM_REG_MIPS_CP0_CAUSE: 850 case KVM_REG_MIPS_CP0_CAUSE:
851 case KVM_REG_MIPS_CP0_CONFIG:
852 case KVM_REG_MIPS_CP0_CONFIG1:
853 case KVM_REG_MIPS_CP0_CONFIG2:
854 case KVM_REG_MIPS_CP0_CONFIG3:
855 case KVM_REG_MIPS_CP0_CONFIG4:
856 case KVM_REG_MIPS_CP0_CONFIG5:
696 case KVM_REG_MIPS_COUNT_CTL: 857 case KVM_REG_MIPS_COUNT_CTL:
697 case KVM_REG_MIPS_COUNT_RESUME: 858 case KVM_REG_MIPS_COUNT_RESUME:
698 case KVM_REG_MIPS_COUNT_HZ: 859 case KVM_REG_MIPS_COUNT_HZ:
@@ -703,6 +864,33 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
703 return 0; 864 return 0;
704} 865}
705 866
867static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
868 struct kvm_enable_cap *cap)
869{
870 int r = 0;
871
872 if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap))
873 return -EINVAL;
874 if (cap->flags)
875 return -EINVAL;
876 if (cap->args[0])
877 return -EINVAL;
878
879 switch (cap->cap) {
880 case KVM_CAP_MIPS_FPU:
881 vcpu->arch.fpu_enabled = true;
882 break;
883 case KVM_CAP_MIPS_MSA:
884 vcpu->arch.msa_enabled = true;
885 break;
886 default:
887 r = -EINVAL;
888 break;
889 }
890
891 return r;
892}
893
706long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, 894long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
707 unsigned long arg) 895 unsigned long arg)
708{ 896{
@@ -760,6 +948,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
760 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); 948 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
761 break; 949 break;
762 } 950 }
951 case KVM_ENABLE_CAP: {
952 struct kvm_enable_cap cap;
953
954 r = -EFAULT;
955 if (copy_from_user(&cap, argp, sizeof(cap)))
956 goto out;
957 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
958 break;
959 }
763 default: 960 default:
764 r = -ENOIOCTLCMD; 961 r = -ENOIOCTLCMD;
765 } 962 }
@@ -868,11 +1065,30 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
868 1065
869 switch (ext) { 1066 switch (ext) {
870 case KVM_CAP_ONE_REG: 1067 case KVM_CAP_ONE_REG:
1068 case KVM_CAP_ENABLE_CAP:
871 r = 1; 1069 r = 1;
872 break; 1070 break;
873 case KVM_CAP_COALESCED_MMIO: 1071 case KVM_CAP_COALESCED_MMIO:
874 r = KVM_COALESCED_MMIO_PAGE_OFFSET; 1072 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
875 break; 1073 break;
1074 case KVM_CAP_MIPS_FPU:
1075 r = !!cpu_has_fpu;
1076 break;
1077 case KVM_CAP_MIPS_MSA:
1078 /*
1079 * We don't support MSA vector partitioning yet:
1080 * 1) It would require explicit support which can't be tested
1081 * yet due to lack of support in current hardware.
1082 * 2) It extends the state that would need to be saved/restored
1083 * by e.g. QEMU for migration.
1084 *
1085 * When vector partitioning hardware becomes available, support
1086 * could be added by requiring a flag when enabling
1087 * KVM_CAP_MIPS_MSA capability to indicate that userland knows
1088 * to save/restore the appropriate extra state.
1089 */
1090 r = cpu_has_msa && !(boot_cpu_data.msa_id & MSA_IR_WRPF);
1091 break;
876 default: 1092 default:
877 r = 0; 1093 r = 0;
878 break; 1094 break;
@@ -1119,6 +1335,30 @@ int kvm_mips_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
1119 ret = kvm_mips_callbacks->handle_break(vcpu); 1335 ret = kvm_mips_callbacks->handle_break(vcpu);
1120 break; 1336 break;
1121 1337
1338 case T_TRAP:
1339 ++vcpu->stat.trap_inst_exits;
1340 trace_kvm_exit(vcpu, TRAP_INST_EXITS);
1341 ret = kvm_mips_callbacks->handle_trap(vcpu);
1342 break;
1343
1344 case T_MSAFPE:
1345 ++vcpu->stat.msa_fpe_exits;
1346 trace_kvm_exit(vcpu, MSA_FPE_EXITS);
1347 ret = kvm_mips_callbacks->handle_msa_fpe(vcpu);
1348 break;
1349
1350 case T_FPE:
1351 ++vcpu->stat.fpe_exits;
1352 trace_kvm_exit(vcpu, FPE_EXITS);
1353 ret = kvm_mips_callbacks->handle_fpe(vcpu);
1354 break;
1355
1356 case T_MSADIS:
1357 ++vcpu->stat.msa_disabled_exits;
1358 trace_kvm_exit(vcpu, MSA_DISABLED_EXITS);
1359 ret = kvm_mips_callbacks->handle_msa_disabled(vcpu);
1360 break;
1361
1122 default: 1362 default:
1123 kvm_err("Exception Code: %d, not yet handled, @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#lx\n", 1363 kvm_err("Exception Code: %d, not yet handled, @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#lx\n",
1124 exccode, opc, kvm_get_inst(opc, vcpu), badvaddr, 1364 exccode, opc, kvm_get_inst(opc, vcpu), badvaddr,
@@ -1146,12 +1386,233 @@ skip_emul:
1146 } 1386 }
1147 } 1387 }
1148 1388
1389 if (ret == RESUME_GUEST) {
1390 /*
1391 * If FPU / MSA are enabled (i.e. the guest's FPU / MSA context
1392 * is live), restore FCR31 / MSACSR.
1393 *
1394 * This should be before returning to the guest exception
1395 * vector, as it may well cause an [MSA] FP exception if there
1396 * are pending exception bits unmasked. (see
1397 * kvm_mips_csr_die_notifier() for how that is handled).
1398 */
1399 if (kvm_mips_guest_has_fpu(&vcpu->arch) &&
1400 read_c0_status() & ST0_CU1)
1401 __kvm_restore_fcsr(&vcpu->arch);
1402
1403 if (kvm_mips_guest_has_msa(&vcpu->arch) &&
1404 read_c0_config5() & MIPS_CONF5_MSAEN)
1405 __kvm_restore_msacsr(&vcpu->arch);
1406 }
1407
1149 /* Disable HTW before returning to guest or host */ 1408 /* Disable HTW before returning to guest or host */
1150 htw_stop(); 1409 htw_stop();
1151 1410
1152 return ret; 1411 return ret;
1153} 1412}
1154 1413
1414/* Enable FPU for guest and restore context */
1415void kvm_own_fpu(struct kvm_vcpu *vcpu)
1416{
1417 struct mips_coproc *cop0 = vcpu->arch.cop0;
1418 unsigned int sr, cfg5;
1419
1420 preempt_disable();
1421
1422 sr = kvm_read_c0_guest_status(cop0);
1423
1424 /*
1425 * If MSA state is already live, it is undefined how it interacts with
1426 * FR=0 FPU state, and we don't want to hit reserved instruction
1427 * exceptions trying to save the MSA state later when CU=1 && FR=1, so
1428 * play it safe and save it first.
1429 *
1430 * In theory we shouldn't ever hit this case since kvm_lose_fpu() should
1431 * get called when guest CU1 is set, however we can't trust the guest
1432 * not to clobber the status register directly via the commpage.
1433 */
1434 if (cpu_has_msa && sr & ST0_CU1 && !(sr & ST0_FR) &&
1435 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
1436 kvm_lose_fpu(vcpu);
1437
1438 /*
1439 * Enable FPU for guest
1440 * We set FR and FRE according to guest context
1441 */
1442 change_c0_status(ST0_CU1 | ST0_FR, sr);
1443 if (cpu_has_fre) {
1444 cfg5 = kvm_read_c0_guest_config5(cop0);
1445 change_c0_config5(MIPS_CONF5_FRE, cfg5);
1446 }
1447 enable_fpu_hazard();
1448
1449 /* If guest FPU state not active, restore it now */
1450 if (!(vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)) {
1451 __kvm_restore_fpu(&vcpu->arch);
1452 vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_FPU;
1453 }
1454
1455 preempt_enable();
1456}
1457
1458#ifdef CONFIG_CPU_HAS_MSA
1459/* Enable MSA for guest and restore context */
1460void kvm_own_msa(struct kvm_vcpu *vcpu)
1461{
1462 struct mips_coproc *cop0 = vcpu->arch.cop0;
1463 unsigned int sr, cfg5;
1464
1465 preempt_disable();
1466
1467 /*
1468 * Enable FPU if enabled in guest, since we're restoring FPU context
1469 * anyway. We set FR and FRE according to guest context.
1470 */
1471 if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
1472 sr = kvm_read_c0_guest_status(cop0);
1473
1474 /*
1475 * If FR=0 FPU state is already live, it is undefined how it
1476 * interacts with MSA state, so play it safe and save it first.
1477 */
1478 if (!(sr & ST0_FR) &&
1479 (vcpu->arch.fpu_inuse & (KVM_MIPS_FPU_FPU |
1480 KVM_MIPS_FPU_MSA)) == KVM_MIPS_FPU_FPU)
1481 kvm_lose_fpu(vcpu);
1482
1483 change_c0_status(ST0_CU1 | ST0_FR, sr);
1484 if (sr & ST0_CU1 && cpu_has_fre) {
1485 cfg5 = kvm_read_c0_guest_config5(cop0);
1486 change_c0_config5(MIPS_CONF5_FRE, cfg5);
1487 }
1488 }
1489
1490 /* Enable MSA for guest */
1491 set_c0_config5(MIPS_CONF5_MSAEN);
1492 enable_fpu_hazard();
1493
1494 switch (vcpu->arch.fpu_inuse & (KVM_MIPS_FPU_FPU | KVM_MIPS_FPU_MSA)) {
1495 case KVM_MIPS_FPU_FPU:
1496 /*
1497 * Guest FPU state already loaded, only restore upper MSA state
1498 */
1499 __kvm_restore_msa_upper(&vcpu->arch);
1500 vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_MSA;
1501 break;
1502 case 0:
1503 /* Neither FPU or MSA already active, restore full MSA state */
1504 __kvm_restore_msa(&vcpu->arch);
1505 vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_MSA;
1506 if (kvm_mips_guest_has_fpu(&vcpu->arch))
1507 vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_FPU;
1508 break;
1509 default:
1510 break;
1511 }
1512
1513 preempt_enable();
1514}
1515#endif
1516
1517/* Drop FPU & MSA without saving it */
1518void kvm_drop_fpu(struct kvm_vcpu *vcpu)
1519{
1520 preempt_disable();
1521 if (cpu_has_msa && vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA) {
1522 disable_msa();
1523 vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_MSA;
1524 }
1525 if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU) {
1526 clear_c0_status(ST0_CU1 | ST0_FR);
1527 vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_FPU;
1528 }
1529 preempt_enable();
1530}
1531
1532/* Save and disable FPU & MSA */
1533void kvm_lose_fpu(struct kvm_vcpu *vcpu)
1534{
1535 /*
1536 * FPU & MSA get disabled in root context (hardware) when it is disabled
1537 * in guest context (software), but the register state in the hardware
1538 * may still be in use. This is why we explicitly re-enable the hardware
1539 * before saving.
1540 */
1541
1542 preempt_disable();
1543 if (cpu_has_msa && vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA) {
1544 set_c0_config5(MIPS_CONF5_MSAEN);
1545 enable_fpu_hazard();
1546
1547 __kvm_save_msa(&vcpu->arch);
1548
1549 /* Disable MSA & FPU */
1550 disable_msa();
1551 if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
1552 clear_c0_status(ST0_CU1 | ST0_FR);
1553 vcpu->arch.fpu_inuse &= ~(KVM_MIPS_FPU_FPU | KVM_MIPS_FPU_MSA);
1554 } else if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU) {
1555 set_c0_status(ST0_CU1);
1556 enable_fpu_hazard();
1557
1558 __kvm_save_fpu(&vcpu->arch);
1559 vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_FPU;
1560
1561 /* Disable FPU */
1562 clear_c0_status(ST0_CU1 | ST0_FR);
1563 }
1564 preempt_enable();
1565}
1566
1567/*
1568 * Step over a specific ctc1 to FCSR and a specific ctcmsa to MSACSR which are
1569 * used to restore guest FCSR/MSACSR state and may trigger a "harmless" FP/MSAFP
1570 * exception if cause bits are set in the value being written.
1571 */
1572static int kvm_mips_csr_die_notify(struct notifier_block *self,
1573 unsigned long cmd, void *ptr)
1574{
1575 struct die_args *args = (struct die_args *)ptr;
1576 struct pt_regs *regs = args->regs;
1577 unsigned long pc;
1578
1579 /* Only interested in FPE and MSAFPE */
1580 if (cmd != DIE_FP && cmd != DIE_MSAFP)
1581 return NOTIFY_DONE;
1582
1583 /* Return immediately if guest context isn't active */
1584 if (!(current->flags & PF_VCPU))
1585 return NOTIFY_DONE;
1586
1587 /* Should never get here from user mode */
1588 BUG_ON(user_mode(regs));
1589
1590 pc = instruction_pointer(regs);
1591 switch (cmd) {
1592 case DIE_FP:
1593 /* match 2nd instruction in __kvm_restore_fcsr */
1594 if (pc != (unsigned long)&__kvm_restore_fcsr + 4)
1595 return NOTIFY_DONE;
1596 break;
1597 case DIE_MSAFP:
1598 /* match 2nd/3rd instruction in __kvm_restore_msacsr */
1599 if (!cpu_has_msa ||
1600 pc < (unsigned long)&__kvm_restore_msacsr + 4 ||
1601 pc > (unsigned long)&__kvm_restore_msacsr + 8)
1602 return NOTIFY_DONE;
1603 break;
1604 }
1605
1606 /* Move PC forward a little and continue executing */
1607 instruction_pointer(regs) += 4;
1608
1609 return NOTIFY_STOP;
1610}
1611
1612static struct notifier_block kvm_mips_csr_die_notifier = {
1613 .notifier_call = kvm_mips_csr_die_notify,
1614};
1615
1155int __init kvm_mips_init(void) 1616int __init kvm_mips_init(void)
1156{ 1617{
1157 int ret; 1618 int ret;
@@ -1161,6 +1622,8 @@ int __init kvm_mips_init(void)
1161 if (ret) 1622 if (ret)
1162 return ret; 1623 return ret;
1163 1624
1625 register_die_notifier(&kvm_mips_csr_die_notifier);
1626
1164 /* 1627 /*
1165 * On MIPS, kernel modules are executed from "mapped space", which 1628 * On MIPS, kernel modules are executed from "mapped space", which
1166 * requires TLBs. The TLB handling code is statically linked with 1629 * requires TLBs. The TLB handling code is statically linked with
@@ -1173,7 +1636,6 @@ int __init kvm_mips_init(void)
1173 kvm_mips_release_pfn_clean = kvm_release_pfn_clean; 1636 kvm_mips_release_pfn_clean = kvm_release_pfn_clean;
1174 kvm_mips_is_error_pfn = is_error_pfn; 1637 kvm_mips_is_error_pfn = is_error_pfn;
1175 1638
1176 pr_info("KVM/MIPS Initialized\n");
1177 return 0; 1639 return 0;
1178} 1640}
1179 1641
@@ -1185,7 +1647,7 @@ void __exit kvm_mips_exit(void)
1185 kvm_mips_release_pfn_clean = NULL; 1647 kvm_mips_release_pfn_clean = NULL;
1186 kvm_mips_is_error_pfn = NULL; 1648 kvm_mips_is_error_pfn = NULL;
1187 1649
1188 pr_info("KVM/MIPS unloaded\n"); 1650 unregister_die_notifier(&kvm_mips_csr_die_notifier);
1189} 1651}
1190 1652
1191module_init(kvm_mips_init); 1653module_init(kvm_mips_init);
diff --git a/arch/mips/kvm/msa.S b/arch/mips/kvm/msa.S
new file mode 100644
index 000000000000..d02f0c6cc2cc
--- /dev/null
+++ b/arch/mips/kvm/msa.S
@@ -0,0 +1,161 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * MIPS SIMD Architecture (MSA) context handling code for KVM.
7 *
8 * Copyright (C) 2015 Imagination Technologies Ltd.
9 */
10
11#include <asm/asm.h>
12#include <asm/asm-offsets.h>
13#include <asm/asmmacro.h>
14#include <asm/regdef.h>
15
16 .set noreorder
17 .set noat
18
19LEAF(__kvm_save_msa)
20 st_d 0, VCPU_FPR0, a0
21 st_d 1, VCPU_FPR1, a0
22 st_d 2, VCPU_FPR2, a0
23 st_d 3, VCPU_FPR3, a0
24 st_d 4, VCPU_FPR4, a0
25 st_d 5, VCPU_FPR5, a0
26 st_d 6, VCPU_FPR6, a0
27 st_d 7, VCPU_FPR7, a0
28 st_d 8, VCPU_FPR8, a0
29 st_d 9, VCPU_FPR9, a0
30 st_d 10, VCPU_FPR10, a0
31 st_d 11, VCPU_FPR11, a0
32 st_d 12, VCPU_FPR12, a0
33 st_d 13, VCPU_FPR13, a0
34 st_d 14, VCPU_FPR14, a0
35 st_d 15, VCPU_FPR15, a0
36 st_d 16, VCPU_FPR16, a0
37 st_d 17, VCPU_FPR17, a0
38 st_d 18, VCPU_FPR18, a0
39 st_d 19, VCPU_FPR19, a0
40 st_d 20, VCPU_FPR20, a0
41 st_d 21, VCPU_FPR21, a0
42 st_d 22, VCPU_FPR22, a0
43 st_d 23, VCPU_FPR23, a0
44 st_d 24, VCPU_FPR24, a0
45 st_d 25, VCPU_FPR25, a0
46 st_d 26, VCPU_FPR26, a0
47 st_d 27, VCPU_FPR27, a0
48 st_d 28, VCPU_FPR28, a0
49 st_d 29, VCPU_FPR29, a0
50 st_d 30, VCPU_FPR30, a0
51 st_d 31, VCPU_FPR31, a0
52 jr ra
53 nop
54 END(__kvm_save_msa)
55
56LEAF(__kvm_restore_msa)
57 ld_d 0, VCPU_FPR0, a0
58 ld_d 1, VCPU_FPR1, a0
59 ld_d 2, VCPU_FPR2, a0
60 ld_d 3, VCPU_FPR3, a0
61 ld_d 4, VCPU_FPR4, a0
62 ld_d 5, VCPU_FPR5, a0
63 ld_d 6, VCPU_FPR6, a0
64 ld_d 7, VCPU_FPR7, a0
65 ld_d 8, VCPU_FPR8, a0
66 ld_d 9, VCPU_FPR9, a0
67 ld_d 10, VCPU_FPR10, a0
68 ld_d 11, VCPU_FPR11, a0
69 ld_d 12, VCPU_FPR12, a0
70 ld_d 13, VCPU_FPR13, a0
71 ld_d 14, VCPU_FPR14, a0
72 ld_d 15, VCPU_FPR15, a0
73 ld_d 16, VCPU_FPR16, a0
74 ld_d 17, VCPU_FPR17, a0
75 ld_d 18, VCPU_FPR18, a0
76 ld_d 19, VCPU_FPR19, a0
77 ld_d 20, VCPU_FPR20, a0
78 ld_d 21, VCPU_FPR21, a0
79 ld_d 22, VCPU_FPR22, a0
80 ld_d 23, VCPU_FPR23, a0
81 ld_d 24, VCPU_FPR24, a0
82 ld_d 25, VCPU_FPR25, a0
83 ld_d 26, VCPU_FPR26, a0
84 ld_d 27, VCPU_FPR27, a0
85 ld_d 28, VCPU_FPR28, a0
86 ld_d 29, VCPU_FPR29, a0
87 ld_d 30, VCPU_FPR30, a0
88 ld_d 31, VCPU_FPR31, a0
89 jr ra
90 nop
91 END(__kvm_restore_msa)
92
93 .macro kvm_restore_msa_upper wr, off, base
94 .set push
95 .set noat
96#ifdef CONFIG_64BIT
97 ld $1, \off(\base)
98 insert_d \wr, 1
99#elif defined(CONFIG_CPU_LITTLE_ENDIAN)
100 lw $1, \off(\base)
101 insert_w \wr, 2
102 lw $1, (\off+4)(\base)
103 insert_w \wr, 3
104#else /* CONFIG_CPU_BIG_ENDIAN */
105 lw $1, (\off+4)(\base)
106 insert_w \wr, 2
107 lw $1, \off(\base)
108 insert_w \wr, 3
109#endif
110 .set pop
111 .endm
112
113LEAF(__kvm_restore_msa_upper)
114 kvm_restore_msa_upper 0, VCPU_FPR0 +8, a0
115 kvm_restore_msa_upper 1, VCPU_FPR1 +8, a0
116 kvm_restore_msa_upper 2, VCPU_FPR2 +8, a0
117 kvm_restore_msa_upper 3, VCPU_FPR3 +8, a0
118 kvm_restore_msa_upper 4, VCPU_FPR4 +8, a0
119 kvm_restore_msa_upper 5, VCPU_FPR5 +8, a0
120 kvm_restore_msa_upper 6, VCPU_FPR6 +8, a0
121 kvm_restore_msa_upper 7, VCPU_FPR7 +8, a0
122 kvm_restore_msa_upper 8, VCPU_FPR8 +8, a0
123 kvm_restore_msa_upper 9, VCPU_FPR9 +8, a0
124 kvm_restore_msa_upper 10, VCPU_FPR10+8, a0
125 kvm_restore_msa_upper 11, VCPU_FPR11+8, a0
126 kvm_restore_msa_upper 12, VCPU_FPR12+8, a0
127 kvm_restore_msa_upper 13, VCPU_FPR13+8, a0
128 kvm_restore_msa_upper 14, VCPU_FPR14+8, a0
129 kvm_restore_msa_upper 15, VCPU_FPR15+8, a0
130 kvm_restore_msa_upper 16, VCPU_FPR16+8, a0
131 kvm_restore_msa_upper 17, VCPU_FPR17+8, a0
132 kvm_restore_msa_upper 18, VCPU_FPR18+8, a0
133 kvm_restore_msa_upper 19, VCPU_FPR19+8, a0
134 kvm_restore_msa_upper 20, VCPU_FPR20+8, a0
135 kvm_restore_msa_upper 21, VCPU_FPR21+8, a0
136 kvm_restore_msa_upper 22, VCPU_FPR22+8, a0
137 kvm_restore_msa_upper 23, VCPU_FPR23+8, a0
138 kvm_restore_msa_upper 24, VCPU_FPR24+8, a0
139 kvm_restore_msa_upper 25, VCPU_FPR25+8, a0
140 kvm_restore_msa_upper 26, VCPU_FPR26+8, a0
141 kvm_restore_msa_upper 27, VCPU_FPR27+8, a0
142 kvm_restore_msa_upper 28, VCPU_FPR28+8, a0
143 kvm_restore_msa_upper 29, VCPU_FPR29+8, a0
144 kvm_restore_msa_upper 30, VCPU_FPR30+8, a0
145 kvm_restore_msa_upper 31, VCPU_FPR31+8, a0
146 jr ra
147 nop
148 END(__kvm_restore_msa_upper)
149
150LEAF(__kvm_restore_msacsr)
151 lw t0, VCPU_MSA_CSR(a0)
152 /*
153 * The ctcmsa must stay at this offset in __kvm_restore_msacsr.
154 * See kvm_mips_csr_die_notify() which handles t0 containing a value
155 * which triggers an MSA FP Exception, which must be stepped over and
156 * ignored since the set cause bits must remain there for the guest.
157 */
158 _ctcmsa MSA_CSR, t0
159 jr ra
160 nop
161 END(__kvm_restore_msacsr)
diff --git a/arch/mips/kvm/stats.c b/arch/mips/kvm/stats.c
index a74d6024c5ad..888bb67070ac 100644
--- a/arch/mips/kvm/stats.c
+++ b/arch/mips/kvm/stats.c
@@ -25,6 +25,10 @@ char *kvm_mips_exit_types_str[MAX_KVM_MIPS_EXIT_TYPES] = {
25 "System Call", 25 "System Call",
26 "Reserved Inst", 26 "Reserved Inst",
27 "Break Inst", 27 "Break Inst",
28 "Trap Inst",
29 "MSA FPE",
30 "FPE",
31 "MSA Disabled",
28 "D-Cache Flushes", 32 "D-Cache Flushes",
29}; 33};
30 34
diff --git a/arch/mips/kvm/tlb.c b/arch/mips/kvm/tlb.c
index b6beb0e07b1b..aed0ac2a4972 100644
--- a/arch/mips/kvm/tlb.c
+++ b/arch/mips/kvm/tlb.c
@@ -733,6 +733,9 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
733 } 733 }
734 } 734 }
735 735
736 /* restore guest state to registers */
737 kvm_mips_callbacks->vcpu_set_regs(vcpu);
738
736 local_irq_restore(flags); 739 local_irq_restore(flags);
737 740
738} 741}
@@ -751,6 +754,9 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
751 vcpu->arch.preempt_entryhi = read_c0_entryhi(); 754 vcpu->arch.preempt_entryhi = read_c0_entryhi();
752 vcpu->arch.last_sched_cpu = cpu; 755 vcpu->arch.last_sched_cpu = cpu;
753 756
757 /* save guest state in registers */
758 kvm_mips_callbacks->vcpu_get_regs(vcpu);
759
754 if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) & 760 if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
755 ASID_VERSION_MASK)) { 761 ASID_VERSION_MASK)) {
756 kvm_debug("%s: Dropping MMU Context: %#lx\n", __func__, 762 kvm_debug("%s: Dropping MMU Context: %#lx\n", __func__,
diff --git a/arch/mips/kvm/trap_emul.c b/arch/mips/kvm/trap_emul.c
index fd7257b70e65..d836ed5b0bc7 100644
--- a/arch/mips/kvm/trap_emul.c
+++ b/arch/mips/kvm/trap_emul.c
@@ -39,16 +39,30 @@ static gpa_t kvm_trap_emul_gva_to_gpa_cb(gva_t gva)
39 39
40static int kvm_trap_emul_handle_cop_unusable(struct kvm_vcpu *vcpu) 40static int kvm_trap_emul_handle_cop_unusable(struct kvm_vcpu *vcpu)
41{ 41{
42 struct mips_coproc *cop0 = vcpu->arch.cop0;
42 struct kvm_run *run = vcpu->run; 43 struct kvm_run *run = vcpu->run;
43 uint32_t __user *opc = (uint32_t __user *) vcpu->arch.pc; 44 uint32_t __user *opc = (uint32_t __user *) vcpu->arch.pc;
44 unsigned long cause = vcpu->arch.host_cp0_cause; 45 unsigned long cause = vcpu->arch.host_cp0_cause;
45 enum emulation_result er = EMULATE_DONE; 46 enum emulation_result er = EMULATE_DONE;
46 int ret = RESUME_GUEST; 47 int ret = RESUME_GUEST;
47 48
48 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) 49 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) {
49 er = kvm_mips_emulate_fpu_exc(cause, opc, run, vcpu); 50 /* FPU Unusable */
50 else 51 if (!kvm_mips_guest_has_fpu(&vcpu->arch) ||
52 (kvm_read_c0_guest_status(cop0) & ST0_CU1) == 0) {
53 /*
54 * Unusable/no FPU in guest:
55 * deliver guest COP1 Unusable Exception
56 */
57 er = kvm_mips_emulate_fpu_exc(cause, opc, run, vcpu);
58 } else {
59 /* Restore FPU state */
60 kvm_own_fpu(vcpu);
61 er = EMULATE_DONE;
62 }
63 } else {
51 er = kvm_mips_emulate_inst(cause, opc, run, vcpu); 64 er = kvm_mips_emulate_inst(cause, opc, run, vcpu);
65 }
52 66
53 switch (er) { 67 switch (er) {
54 case EMULATE_DONE: 68 case EMULATE_DONE:
@@ -330,6 +344,107 @@ static int kvm_trap_emul_handle_break(struct kvm_vcpu *vcpu)
330 return ret; 344 return ret;
331} 345}
332 346
347static int kvm_trap_emul_handle_trap(struct kvm_vcpu *vcpu)
348{
349 struct kvm_run *run = vcpu->run;
350 uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
351 unsigned long cause = vcpu->arch.host_cp0_cause;
352 enum emulation_result er = EMULATE_DONE;
353 int ret = RESUME_GUEST;
354
355 er = kvm_mips_emulate_trap_exc(cause, opc, run, vcpu);
356 if (er == EMULATE_DONE) {
357 ret = RESUME_GUEST;
358 } else {
359 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
360 ret = RESUME_HOST;
361 }
362 return ret;
363}
364
365static int kvm_trap_emul_handle_msa_fpe(struct kvm_vcpu *vcpu)
366{
367 struct kvm_run *run = vcpu->run;
368 uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
369 unsigned long cause = vcpu->arch.host_cp0_cause;
370 enum emulation_result er = EMULATE_DONE;
371 int ret = RESUME_GUEST;
372
373 er = kvm_mips_emulate_msafpe_exc(cause, opc, run, vcpu);
374 if (er == EMULATE_DONE) {
375 ret = RESUME_GUEST;
376 } else {
377 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
378 ret = RESUME_HOST;
379 }
380 return ret;
381}
382
383static int kvm_trap_emul_handle_fpe(struct kvm_vcpu *vcpu)
384{
385 struct kvm_run *run = vcpu->run;
386 uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
387 unsigned long cause = vcpu->arch.host_cp0_cause;
388 enum emulation_result er = EMULATE_DONE;
389 int ret = RESUME_GUEST;
390
391 er = kvm_mips_emulate_fpe_exc(cause, opc, run, vcpu);
392 if (er == EMULATE_DONE) {
393 ret = RESUME_GUEST;
394 } else {
395 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
396 ret = RESUME_HOST;
397 }
398 return ret;
399}
400
401/**
402 * kvm_trap_emul_handle_msa_disabled() - Guest used MSA while disabled in root.
403 * @vcpu: Virtual CPU context.
404 *
405 * Handle when the guest attempts to use MSA when it is disabled.
406 */
407static int kvm_trap_emul_handle_msa_disabled(struct kvm_vcpu *vcpu)
408{
409 struct mips_coproc *cop0 = vcpu->arch.cop0;
410 struct kvm_run *run = vcpu->run;
411 uint32_t __user *opc = (uint32_t __user *) vcpu->arch.pc;
412 unsigned long cause = vcpu->arch.host_cp0_cause;
413 enum emulation_result er = EMULATE_DONE;
414 int ret = RESUME_GUEST;
415
416 if (!kvm_mips_guest_has_msa(&vcpu->arch) ||
417 (kvm_read_c0_guest_status(cop0) & (ST0_CU1 | ST0_FR)) == ST0_CU1) {
418 /*
419 * No MSA in guest, or FPU enabled and not in FR=1 mode,
420 * guest reserved instruction exception
421 */
422 er = kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
423 } else if (!(kvm_read_c0_guest_config5(cop0) & MIPS_CONF5_MSAEN)) {
424 /* MSA disabled by guest, guest MSA disabled exception */
425 er = kvm_mips_emulate_msadis_exc(cause, opc, run, vcpu);
426 } else {
427 /* Restore MSA/FPU state */
428 kvm_own_msa(vcpu);
429 er = EMULATE_DONE;
430 }
431
432 switch (er) {
433 case EMULATE_DONE:
434 ret = RESUME_GUEST;
435 break;
436
437 case EMULATE_FAIL:
438 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
439 ret = RESUME_HOST;
440 break;
441
442 default:
443 BUG();
444 }
445 return ret;
446}
447
333static int kvm_trap_emul_vm_init(struct kvm *kvm) 448static int kvm_trap_emul_vm_init(struct kvm *kvm)
334{ 449{
335 return 0; 450 return 0;
@@ -351,8 +466,9 @@ static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu)
351 * guest will come up as expected, for now we simulate a MIPS 24kc 466 * guest will come up as expected, for now we simulate a MIPS 24kc
352 */ 467 */
353 kvm_write_c0_guest_prid(cop0, 0x00019300); 468 kvm_write_c0_guest_prid(cop0, 0x00019300);
354 kvm_write_c0_guest_config(cop0, 469 /* Have config1, Cacheable, noncoherent, write-back, write allocate */
355 MIPS_CONFIG0 | (0x1 << CP0C0_AR) | 470 kvm_write_c0_guest_config(cop0, MIPS_CONF_M | (0x3 << CP0C0_K0) |
471 (0x1 << CP0C0_AR) |
356 (MMU_TYPE_R4000 << CP0C0_MT)); 472 (MMU_TYPE_R4000 << CP0C0_MT));
357 473
358 /* Read the cache characteristics from the host Config1 Register */ 474 /* Read the cache characteristics from the host Config1 Register */
@@ -368,10 +484,18 @@ static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu)
368 (1 << CP0C1_WR) | (1 << CP0C1_CA)); 484 (1 << CP0C1_WR) | (1 << CP0C1_CA));
369 kvm_write_c0_guest_config1(cop0, config1); 485 kvm_write_c0_guest_config1(cop0, config1);
370 486
371 kvm_write_c0_guest_config2(cop0, MIPS_CONFIG2); 487 /* Have config3, no tertiary/secondary caches implemented */
372 /* MIPS_CONFIG2 | (read_c0_config2() & 0xfff) */ 488 kvm_write_c0_guest_config2(cop0, MIPS_CONF_M);
373 kvm_write_c0_guest_config3(cop0, MIPS_CONFIG3 | (0 << CP0C3_VInt) | 489 /* MIPS_CONF_M | (read_c0_config2() & 0xfff) */
374 (1 << CP0C3_ULRI)); 490
491 /* Have config4, UserLocal */
492 kvm_write_c0_guest_config3(cop0, MIPS_CONF_M | MIPS_CONF3_ULRI);
493
494 /* Have config5 */
495 kvm_write_c0_guest_config4(cop0, MIPS_CONF_M);
496
497 /* No config6 */
498 kvm_write_c0_guest_config5(cop0, 0);
375 499
376 /* Set Wait IE/IXMT Ignore in Config7, IAR, AR */ 500 /* Set Wait IE/IXMT Ignore in Config7, IAR, AR */
377 kvm_write_c0_guest_config7(cop0, (MIPS_CONF7_WII) | (1 << 10)); 501 kvm_write_c0_guest_config7(cop0, (MIPS_CONF7_WII) | (1 << 10));
@@ -416,6 +540,7 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
416{ 540{
417 struct mips_coproc *cop0 = vcpu->arch.cop0; 541 struct mips_coproc *cop0 = vcpu->arch.cop0;
418 int ret = 0; 542 int ret = 0;
543 unsigned int cur, change;
419 544
420 switch (reg->id) { 545 switch (reg->id) {
421 case KVM_REG_MIPS_CP0_COUNT: 546 case KVM_REG_MIPS_CP0_COUNT:
@@ -444,6 +569,44 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
444 kvm_write_c0_guest_cause(cop0, v); 569 kvm_write_c0_guest_cause(cop0, v);
445 } 570 }
446 break; 571 break;
572 case KVM_REG_MIPS_CP0_CONFIG:
573 /* read-only for now */
574 break;
575 case KVM_REG_MIPS_CP0_CONFIG1:
576 cur = kvm_read_c0_guest_config1(cop0);
577 change = (cur ^ v) & kvm_mips_config1_wrmask(vcpu);
578 if (change) {
579 v = cur ^ change;
580 kvm_write_c0_guest_config1(cop0, v);
581 }
582 break;
583 case KVM_REG_MIPS_CP0_CONFIG2:
584 /* read-only for now */
585 break;
586 case KVM_REG_MIPS_CP0_CONFIG3:
587 cur = kvm_read_c0_guest_config3(cop0);
588 change = (cur ^ v) & kvm_mips_config3_wrmask(vcpu);
589 if (change) {
590 v = cur ^ change;
591 kvm_write_c0_guest_config3(cop0, v);
592 }
593 break;
594 case KVM_REG_MIPS_CP0_CONFIG4:
595 cur = kvm_read_c0_guest_config4(cop0);
596 change = (cur ^ v) & kvm_mips_config4_wrmask(vcpu);
597 if (change) {
598 v = cur ^ change;
599 kvm_write_c0_guest_config4(cop0, v);
600 }
601 break;
602 case KVM_REG_MIPS_CP0_CONFIG5:
603 cur = kvm_read_c0_guest_config5(cop0);
604 change = (cur ^ v) & kvm_mips_config5_wrmask(vcpu);
605 if (change) {
606 v = cur ^ change;
607 kvm_write_c0_guest_config5(cop0, v);
608 }
609 break;
447 case KVM_REG_MIPS_COUNT_CTL: 610 case KVM_REG_MIPS_COUNT_CTL:
448 ret = kvm_mips_set_count_ctl(vcpu, v); 611 ret = kvm_mips_set_count_ctl(vcpu, v);
449 break; 612 break;
@@ -459,6 +622,18 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
459 return ret; 622 return ret;
460} 623}
461 624
625static int kvm_trap_emul_vcpu_get_regs(struct kvm_vcpu *vcpu)
626{
627 kvm_lose_fpu(vcpu);
628
629 return 0;
630}
631
632static int kvm_trap_emul_vcpu_set_regs(struct kvm_vcpu *vcpu)
633{
634 return 0;
635}
636
462static struct kvm_mips_callbacks kvm_trap_emul_callbacks = { 637static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
463 /* exit handlers */ 638 /* exit handlers */
464 .handle_cop_unusable = kvm_trap_emul_handle_cop_unusable, 639 .handle_cop_unusable = kvm_trap_emul_handle_cop_unusable,
@@ -470,6 +645,10 @@ static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
470 .handle_syscall = kvm_trap_emul_handle_syscall, 645 .handle_syscall = kvm_trap_emul_handle_syscall,
471 .handle_res_inst = kvm_trap_emul_handle_res_inst, 646 .handle_res_inst = kvm_trap_emul_handle_res_inst,
472 .handle_break = kvm_trap_emul_handle_break, 647 .handle_break = kvm_trap_emul_handle_break,
648 .handle_trap = kvm_trap_emul_handle_trap,
649 .handle_msa_fpe = kvm_trap_emul_handle_msa_fpe,
650 .handle_fpe = kvm_trap_emul_handle_fpe,
651 .handle_msa_disabled = kvm_trap_emul_handle_msa_disabled,
473 652
474 .vm_init = kvm_trap_emul_vm_init, 653 .vm_init = kvm_trap_emul_vm_init,
475 .vcpu_init = kvm_trap_emul_vcpu_init, 654 .vcpu_init = kvm_trap_emul_vcpu_init,
@@ -483,6 +662,8 @@ static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
483 .irq_clear = kvm_mips_irq_clear_cb, 662 .irq_clear = kvm_mips_irq_clear_cb,
484 .get_one_reg = kvm_trap_emul_get_one_reg, 663 .get_one_reg = kvm_trap_emul_get_one_reg,
485 .set_one_reg = kvm_trap_emul_set_one_reg, 664 .set_one_reg = kvm_trap_emul_set_one_reg,
665 .vcpu_get_regs = kvm_trap_emul_vcpu_get_regs,
666 .vcpu_set_regs = kvm_trap_emul_vcpu_set_regs,
486}; 667};
487 668
488int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks) 669int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks)
diff --git a/arch/powerpc/kvm/mpic.c b/arch/powerpc/kvm/mpic.c
index 39b3a8f816f2..6249cdc834d1 100644
--- a/arch/powerpc/kvm/mpic.c
+++ b/arch/powerpc/kvm/mpic.c
@@ -34,7 +34,7 @@
34#include <asm/kvm_para.h> 34#include <asm/kvm_para.h>
35#include <asm/kvm_host.h> 35#include <asm/kvm_host.h>
36#include <asm/kvm_ppc.h> 36#include <asm/kvm_ppc.h>
37#include "iodev.h" 37#include <kvm/iodev.h>
38 38
39#define MAX_CPU 32 39#define MAX_CPU 32
40#define MAX_SRC 256 40#define MAX_SRC 256
@@ -289,11 +289,6 @@ static inline void IRQ_resetbit(struct irq_queue *q, int n_IRQ)
289 clear_bit(n_IRQ, q->queue); 289 clear_bit(n_IRQ, q->queue);
290} 290}
291 291
292static inline int IRQ_testbit(struct irq_queue *q, int n_IRQ)
293{
294 return test_bit(n_IRQ, q->queue);
295}
296
297static void IRQ_check(struct openpic *opp, struct irq_queue *q) 292static void IRQ_check(struct openpic *opp, struct irq_queue *q)
298{ 293{
299 int irq = -1; 294 int irq = -1;
@@ -1374,8 +1369,9 @@ static int kvm_mpic_write_internal(struct openpic *opp, gpa_t addr, u32 val)
1374 return -ENXIO; 1369 return -ENXIO;
1375} 1370}
1376 1371
1377static int kvm_mpic_read(struct kvm_io_device *this, gpa_t addr, 1372static int kvm_mpic_read(struct kvm_vcpu *vcpu,
1378 int len, void *ptr) 1373 struct kvm_io_device *this,
1374 gpa_t addr, int len, void *ptr)
1379{ 1375{
1380 struct openpic *opp = container_of(this, struct openpic, mmio); 1376 struct openpic *opp = container_of(this, struct openpic, mmio);
1381 int ret; 1377 int ret;
@@ -1415,8 +1411,9 @@ static int kvm_mpic_read(struct kvm_io_device *this, gpa_t addr,
1415 return ret; 1411 return ret;
1416} 1412}
1417 1413
1418static int kvm_mpic_write(struct kvm_io_device *this, gpa_t addr, 1414static int kvm_mpic_write(struct kvm_vcpu *vcpu,
1419 int len, const void *ptr) 1415 struct kvm_io_device *this,
1416 gpa_t addr, int len, const void *ptr)
1420{ 1417{
1421 struct openpic *opp = container_of(this, struct openpic, mmio); 1418 struct openpic *opp = container_of(this, struct openpic, mmio);
1422 int ret; 1419 int ret;
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 27c0face86f4..24bfe401373e 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -807,7 +807,7 @@ int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
807 807
808 idx = srcu_read_lock(&vcpu->kvm->srcu); 808 idx = srcu_read_lock(&vcpu->kvm->srcu);
809 809
810 ret = kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, run->mmio.phys_addr, 810 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
811 bytes, &run->mmio.data); 811 bytes, &run->mmio.data);
812 812
813 srcu_read_unlock(&vcpu->kvm->srcu, idx); 813 srcu_read_unlock(&vcpu->kvm->srcu, idx);
@@ -880,7 +880,7 @@ int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
880 880
881 idx = srcu_read_lock(&vcpu->kvm->srcu); 881 idx = srcu_read_lock(&vcpu->kvm->srcu);
882 882
883 ret = kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, run->mmio.phys_addr, 883 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
884 bytes, &run->mmio.data); 884 bytes, &run->mmio.data);
885 885
886 srcu_read_unlock(&vcpu->kvm->srcu, idx); 886 srcu_read_unlock(&vcpu->kvm->srcu, idx);
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index f407bbf5ee94..d01fc588b5c3 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -172,7 +172,9 @@ struct kvm_s390_sie_block {
172 __u32 fac; /* 0x01a0 */ 172 __u32 fac; /* 0x01a0 */
173 __u8 reserved1a4[20]; /* 0x01a4 */ 173 __u8 reserved1a4[20]; /* 0x01a4 */
174 __u64 cbrlo; /* 0x01b8 */ 174 __u64 cbrlo; /* 0x01b8 */
175 __u8 reserved1c0[30]; /* 0x01c0 */ 175 __u8 reserved1c0[8]; /* 0x01c0 */
176 __u32 ecd; /* 0x01c8 */
177 __u8 reserved1cc[18]; /* 0x01cc */
176 __u64 pp; /* 0x01de */ 178 __u64 pp; /* 0x01de */
177 __u8 reserved1e6[2]; /* 0x01e6 */ 179 __u8 reserved1e6[2]; /* 0x01e6 */
178 __u64 itdba; /* 0x01e8 */ 180 __u64 itdba; /* 0x01e8 */
@@ -183,11 +185,17 @@ struct kvm_s390_itdb {
183 __u8 data[256]; 185 __u8 data[256];
184} __packed; 186} __packed;
185 187
188struct kvm_s390_vregs {
189 __vector128 vrs[32];
190 __u8 reserved200[512]; /* for future vector expansion */
191} __packed;
192
186struct sie_page { 193struct sie_page {
187 struct kvm_s390_sie_block sie_block; 194 struct kvm_s390_sie_block sie_block;
188 __u8 reserved200[1024]; /* 0x0200 */ 195 __u8 reserved200[1024]; /* 0x0200 */
189 struct kvm_s390_itdb itdb; /* 0x0600 */ 196 struct kvm_s390_itdb itdb; /* 0x0600 */
190 __u8 reserved700[2304]; /* 0x0700 */ 197 __u8 reserved700[1280]; /* 0x0700 */
198 struct kvm_s390_vregs vregs; /* 0x0c00 */
191} __packed; 199} __packed;
192 200
193struct kvm_vcpu_stat { 201struct kvm_vcpu_stat {
@@ -238,6 +246,7 @@ struct kvm_vcpu_stat {
238 u32 instruction_sigp_stop; 246 u32 instruction_sigp_stop;
239 u32 instruction_sigp_stop_store_status; 247 u32 instruction_sigp_stop_store_status;
240 u32 instruction_sigp_store_status; 248 u32 instruction_sigp_store_status;
249 u32 instruction_sigp_store_adtl_status;
241 u32 instruction_sigp_arch; 250 u32 instruction_sigp_arch;
242 u32 instruction_sigp_prefix; 251 u32 instruction_sigp_prefix;
243 u32 instruction_sigp_restart; 252 u32 instruction_sigp_restart;
@@ -270,6 +279,7 @@ struct kvm_vcpu_stat {
270#define PGM_SPECIAL_OPERATION 0x13 279#define PGM_SPECIAL_OPERATION 0x13
271#define PGM_OPERAND 0x15 280#define PGM_OPERAND 0x15
272#define PGM_TRACE_TABEL 0x16 281#define PGM_TRACE_TABEL 0x16
282#define PGM_VECTOR_PROCESSING 0x1b
273#define PGM_SPACE_SWITCH 0x1c 283#define PGM_SPACE_SWITCH 0x1c
274#define PGM_HFP_SQUARE_ROOT 0x1d 284#define PGM_HFP_SQUARE_ROOT 0x1d
275#define PGM_PC_TRANSLATION_SPEC 0x1f 285#define PGM_PC_TRANSLATION_SPEC 0x1f
@@ -334,6 +344,11 @@ enum irq_types {
334 IRQ_PEND_COUNT 344 IRQ_PEND_COUNT
335}; 345};
336 346
347/* We have 2M for virtio device descriptor pages. Smallest amount of
348 * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381
349 */
350#define KVM_S390_MAX_VIRTIO_IRQS 87381
351
337/* 352/*
338 * Repressible (non-floating) machine check interrupts 353 * Repressible (non-floating) machine check interrupts
339 * subclass bits in MCIC 354 * subclass bits in MCIC
@@ -411,13 +426,32 @@ struct kvm_s390_local_interrupt {
411 unsigned long pending_irqs; 426 unsigned long pending_irqs;
412}; 427};
413 428
429#define FIRQ_LIST_IO_ISC_0 0
430#define FIRQ_LIST_IO_ISC_1 1
431#define FIRQ_LIST_IO_ISC_2 2
432#define FIRQ_LIST_IO_ISC_3 3
433#define FIRQ_LIST_IO_ISC_4 4
434#define FIRQ_LIST_IO_ISC_5 5
435#define FIRQ_LIST_IO_ISC_6 6
436#define FIRQ_LIST_IO_ISC_7 7
437#define FIRQ_LIST_PFAULT 8
438#define FIRQ_LIST_VIRTIO 9
439#define FIRQ_LIST_COUNT 10
440#define FIRQ_CNTR_IO 0
441#define FIRQ_CNTR_SERVICE 1
442#define FIRQ_CNTR_VIRTIO 2
443#define FIRQ_CNTR_PFAULT 3
444#define FIRQ_MAX_COUNT 4
445
414struct kvm_s390_float_interrupt { 446struct kvm_s390_float_interrupt {
447 unsigned long pending_irqs;
415 spinlock_t lock; 448 spinlock_t lock;
416 struct list_head list; 449 struct list_head lists[FIRQ_LIST_COUNT];
417 atomic_t active; 450 int counters[FIRQ_MAX_COUNT];
451 struct kvm_s390_mchk_info mchk;
452 struct kvm_s390_ext_info srv_signal;
418 int next_rr_cpu; 453 int next_rr_cpu;
419 unsigned long idle_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 454 unsigned long idle_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)];
420 unsigned int irq_count;
421}; 455};
422 456
423struct kvm_hw_wp_info_arch { 457struct kvm_hw_wp_info_arch {
@@ -465,6 +499,7 @@ struct kvm_vcpu_arch {
465 s390_fp_regs host_fpregs; 499 s390_fp_regs host_fpregs;
466 unsigned int host_acrs[NUM_ACRS]; 500 unsigned int host_acrs[NUM_ACRS];
467 s390_fp_regs guest_fpregs; 501 s390_fp_regs guest_fpregs;
502 struct kvm_s390_vregs *host_vregs;
468 struct kvm_s390_local_interrupt local_int; 503 struct kvm_s390_local_interrupt local_int;
469 struct hrtimer ckc_timer; 504 struct hrtimer ckc_timer;
470 struct kvm_s390_pgm_info pgm; 505 struct kvm_s390_pgm_info pgm;
@@ -553,6 +588,7 @@ struct kvm_arch{
553 int use_cmma; 588 int use_cmma;
554 int user_cpu_state_ctrl; 589 int user_cpu_state_ctrl;
555 int user_sigp; 590 int user_sigp;
591 int user_stsi;
556 struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS]; 592 struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS];
557 wait_queue_head_t ipte_wq; 593 wait_queue_head_t ipte_wq;
558 int ipte_lock_count; 594 int ipte_lock_count;
diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
index 9c77e60b9a26..ef1a5fcc6c66 100644
--- a/arch/s390/include/uapi/asm/kvm.h
+++ b/arch/s390/include/uapi/asm/kvm.h
@@ -150,6 +150,7 @@ struct kvm_guest_debug_arch {
150#define KVM_SYNC_CRS (1UL << 3) 150#define KVM_SYNC_CRS (1UL << 3)
151#define KVM_SYNC_ARCH0 (1UL << 4) 151#define KVM_SYNC_ARCH0 (1UL << 4)
152#define KVM_SYNC_PFAULT (1UL << 5) 152#define KVM_SYNC_PFAULT (1UL << 5)
153#define KVM_SYNC_VRS (1UL << 6)
153/* definition of registers in kvm_run */ 154/* definition of registers in kvm_run */
154struct kvm_sync_regs { 155struct kvm_sync_regs {
155 __u64 prefix; /* prefix register */ 156 __u64 prefix; /* prefix register */
@@ -164,6 +165,9 @@ struct kvm_sync_regs {
164 __u64 pft; /* pfault token [PFAULT] */ 165 __u64 pft; /* pfault token [PFAULT] */
165 __u64 pfs; /* pfault select [PFAULT] */ 166 __u64 pfs; /* pfault select [PFAULT] */
166 __u64 pfc; /* pfault compare [PFAULT] */ 167 __u64 pfc; /* pfault compare [PFAULT] */
168 __u64 vrs[32][2]; /* vector registers */
169 __u8 reserved[512]; /* for future vector expansion */
170 __u32 fpc; /* only valid with vector registers */
167}; 171};
168 172
169#define KVM_REG_S390_TODPR (KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1) 173#define KVM_REG_S390_TODPR (KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1)
diff --git a/arch/s390/include/uapi/asm/sie.h b/arch/s390/include/uapi/asm/sie.h
index d4096fdfc6ab..ee69c0854c88 100644
--- a/arch/s390/include/uapi/asm/sie.h
+++ b/arch/s390/include/uapi/asm/sie.h
@@ -230,7 +230,7 @@
230 * and returns a key, which can be used to find a mnemonic name 230 * and returns a key, which can be used to find a mnemonic name
231 * of the instruction in the icpt_insn_codes table. 231 * of the instruction in the icpt_insn_codes table.
232 */ 232 */
233#define icpt_insn_decoder(insn) \ 233#define icpt_insn_decoder(insn) ( \
234 INSN_DECODE_IPA0(0x01, insn, 48, 0xff) \ 234 INSN_DECODE_IPA0(0x01, insn, 48, 0xff) \
235 INSN_DECODE_IPA0(0xaa, insn, 48, 0x0f) \ 235 INSN_DECODE_IPA0(0xaa, insn, 48, 0x0f) \
236 INSN_DECODE_IPA0(0xb2, insn, 48, 0xff) \ 236 INSN_DECODE_IPA0(0xb2, insn, 48, 0xff) \
@@ -239,6 +239,6 @@
239 INSN_DECODE_IPA0(0xe5, insn, 48, 0xff) \ 239 INSN_DECODE_IPA0(0xe5, insn, 48, 0xff) \
240 INSN_DECODE_IPA0(0xeb, insn, 16, 0xff) \ 240 INSN_DECODE_IPA0(0xeb, insn, 16, 0xff) \
241 INSN_DECODE_IPA0(0xc8, insn, 48, 0x0f) \ 241 INSN_DECODE_IPA0(0xc8, insn, 48, 0x0f) \
242 INSN_DECODE(insn) 242 INSN_DECODE(insn))
243 243
244#endif /* _UAPI_ASM_S390_SIE_H */ 244#endif /* _UAPI_ASM_S390_SIE_H */
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index e07e91605353..8dc4db10d160 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -171,6 +171,7 @@ int main(void)
171#else /* CONFIG_32BIT */ 171#else /* CONFIG_32BIT */
172 DEFINE(__LC_DATA_EXC_CODE, offsetof(struct _lowcore, data_exc_code)); 172 DEFINE(__LC_DATA_EXC_CODE, offsetof(struct _lowcore, data_exc_code));
173 DEFINE(__LC_MCCK_FAIL_STOR_ADDR, offsetof(struct _lowcore, failing_storage_address)); 173 DEFINE(__LC_MCCK_FAIL_STOR_ADDR, offsetof(struct _lowcore, failing_storage_address));
174 DEFINE(__LC_VX_SAVE_AREA_ADDR, offsetof(struct _lowcore, vector_save_area_addr));
174 DEFINE(__LC_EXT_PARAMS2, offsetof(struct _lowcore, ext_params2)); 175 DEFINE(__LC_EXT_PARAMS2, offsetof(struct _lowcore, ext_params2));
175 DEFINE(SAVE_AREA_BASE, offsetof(struct _lowcore, floating_pt_save_area)); 176 DEFINE(SAVE_AREA_BASE, offsetof(struct _lowcore, floating_pt_save_area));
176 DEFINE(__LC_PASTE, offsetof(struct _lowcore, paste)); 177 DEFINE(__LC_PASTE, offsetof(struct _lowcore, paste));
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c
index 9254afff250c..fc7ec95848c3 100644
--- a/arch/s390/kvm/diag.c
+++ b/arch/s390/kvm/diag.c
@@ -77,7 +77,7 @@ static int __diag_page_ref_service(struct kvm_vcpu *vcpu)
77 77
78 if (vcpu->run->s.regs.gprs[rx] & 7) 78 if (vcpu->run->s.regs.gprs[rx] & 7)
79 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 79 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
80 rc = read_guest(vcpu, vcpu->run->s.regs.gprs[rx], &parm, sizeof(parm)); 80 rc = read_guest(vcpu, vcpu->run->s.regs.gprs[rx], rx, &parm, sizeof(parm));
81 if (rc) 81 if (rc)
82 return kvm_s390_inject_prog_cond(vcpu, rc); 82 return kvm_s390_inject_prog_cond(vcpu, rc);
83 if (parm.parm_version != 2 || parm.parm_len < 5 || parm.code != 0x258) 83 if (parm.parm_version != 2 || parm.parm_len < 5 || parm.code != 0x258)
@@ -213,7 +213,7 @@ static int __diag_virtio_hypercall(struct kvm_vcpu *vcpu)
213 * - gpr 3 contains the virtqueue index (passed as datamatch) 213 * - gpr 3 contains the virtqueue index (passed as datamatch)
214 * - gpr 4 contains the index on the bus (optionally) 214 * - gpr 4 contains the index on the bus (optionally)
215 */ 215 */
216 ret = kvm_io_bus_write_cookie(vcpu->kvm, KVM_VIRTIO_CCW_NOTIFY_BUS, 216 ret = kvm_io_bus_write_cookie(vcpu, KVM_VIRTIO_CCW_NOTIFY_BUS,
217 vcpu->run->s.regs.gprs[2] & 0xffffffff, 217 vcpu->run->s.regs.gprs[2] & 0xffffffff,
218 8, &vcpu->run->s.regs.gprs[3], 218 8, &vcpu->run->s.regs.gprs[3],
219 vcpu->run->s.regs.gprs[4]); 219 vcpu->run->s.regs.gprs[4]);
@@ -230,7 +230,7 @@ static int __diag_virtio_hypercall(struct kvm_vcpu *vcpu)
230 230
231int kvm_s390_handle_diag(struct kvm_vcpu *vcpu) 231int kvm_s390_handle_diag(struct kvm_vcpu *vcpu)
232{ 232{
233 int code = kvm_s390_get_base_disp_rs(vcpu) & 0xffff; 233 int code = kvm_s390_get_base_disp_rs(vcpu, NULL) & 0xffff;
234 234
235 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 235 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
236 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 236 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index 267523cac6de..a7559f7207df 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -10,6 +10,7 @@
10#include <asm/pgtable.h> 10#include <asm/pgtable.h>
11#include "kvm-s390.h" 11#include "kvm-s390.h"
12#include "gaccess.h" 12#include "gaccess.h"
13#include <asm/switch_to.h>
13 14
14union asce { 15union asce {
15 unsigned long val; 16 unsigned long val;
@@ -207,6 +208,54 @@ union raddress {
207 unsigned long pfra : 52; /* Page-Frame Real Address */ 208 unsigned long pfra : 52; /* Page-Frame Real Address */
208}; 209};
209 210
211union alet {
212 u32 val;
213 struct {
214 u32 reserved : 7;
215 u32 p : 1;
216 u32 alesn : 8;
217 u32 alen : 16;
218 };
219};
220
221union ald {
222 u32 val;
223 struct {
224 u32 : 1;
225 u32 alo : 24;
226 u32 all : 7;
227 };
228};
229
230struct ale {
231 unsigned long i : 1; /* ALEN-Invalid Bit */
232 unsigned long : 5;
233 unsigned long fo : 1; /* Fetch-Only Bit */
234 unsigned long p : 1; /* Private Bit */
235 unsigned long alesn : 8; /* Access-List-Entry Sequence Number */
236 unsigned long aleax : 16; /* Access-List-Entry Authorization Index */
237 unsigned long : 32;
238 unsigned long : 1;
239 unsigned long asteo : 25; /* ASN-Second-Table-Entry Origin */
240 unsigned long : 6;
241 unsigned long astesn : 32; /* ASTE Sequence Number */
242} __packed;
243
244struct aste {
245 unsigned long i : 1; /* ASX-Invalid Bit */
246 unsigned long ato : 29; /* Authority-Table Origin */
247 unsigned long : 1;
248 unsigned long b : 1; /* Base-Space Bit */
249 unsigned long ax : 16; /* Authorization Index */
250 unsigned long atl : 12; /* Authority-Table Length */
251 unsigned long : 2;
252 unsigned long ca : 1; /* Controlled-ASN Bit */
253 unsigned long ra : 1; /* Reusable-ASN Bit */
254 unsigned long asce : 64; /* Address-Space-Control Element */
255 unsigned long ald : 32;
256 unsigned long astesn : 32;
257 /* .. more fields there */
258} __packed;
210 259
211int ipte_lock_held(struct kvm_vcpu *vcpu) 260int ipte_lock_held(struct kvm_vcpu *vcpu)
212{ 261{
@@ -307,15 +356,157 @@ void ipte_unlock(struct kvm_vcpu *vcpu)
307 ipte_unlock_simple(vcpu); 356 ipte_unlock_simple(vcpu);
308} 357}
309 358
310static unsigned long get_vcpu_asce(struct kvm_vcpu *vcpu) 359static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, ar_t ar,
360 int write)
361{
362 union alet alet;
363 struct ale ale;
364 struct aste aste;
365 unsigned long ald_addr, authority_table_addr;
366 union ald ald;
367 int eax, rc;
368 u8 authority_table;
369
370 if (ar >= NUM_ACRS)
371 return -EINVAL;
372
373 save_access_regs(vcpu->run->s.regs.acrs);
374 alet.val = vcpu->run->s.regs.acrs[ar];
375
376 if (ar == 0 || alet.val == 0) {
377 asce->val = vcpu->arch.sie_block->gcr[1];
378 return 0;
379 } else if (alet.val == 1) {
380 asce->val = vcpu->arch.sie_block->gcr[7];
381 return 0;
382 }
383
384 if (alet.reserved)
385 return PGM_ALET_SPECIFICATION;
386
387 if (alet.p)
388 ald_addr = vcpu->arch.sie_block->gcr[5];
389 else
390 ald_addr = vcpu->arch.sie_block->gcr[2];
391 ald_addr &= 0x7fffffc0;
392
393 rc = read_guest_real(vcpu, ald_addr + 16, &ald.val, sizeof(union ald));
394 if (rc)
395 return rc;
396
397 if (alet.alen / 8 > ald.all)
398 return PGM_ALEN_TRANSLATION;
399
400 if (0x7fffffff - ald.alo * 128 < alet.alen * 16)
401 return PGM_ADDRESSING;
402
403 rc = read_guest_real(vcpu, ald.alo * 128 + alet.alen * 16, &ale,
404 sizeof(struct ale));
405 if (rc)
406 return rc;
407
408 if (ale.i == 1)
409 return PGM_ALEN_TRANSLATION;
410 if (ale.alesn != alet.alesn)
411 return PGM_ALE_SEQUENCE;
412
413 rc = read_guest_real(vcpu, ale.asteo * 64, &aste, sizeof(struct aste));
414 if (rc)
415 return rc;
416
417 if (aste.i)
418 return PGM_ASTE_VALIDITY;
419 if (aste.astesn != ale.astesn)
420 return PGM_ASTE_SEQUENCE;
421
422 if (ale.p == 1) {
423 eax = (vcpu->arch.sie_block->gcr[8] >> 16) & 0xffff;
424 if (ale.aleax != eax) {
425 if (eax / 16 > aste.atl)
426 return PGM_EXTENDED_AUTHORITY;
427
428 authority_table_addr = aste.ato * 4 + eax / 4;
429
430 rc = read_guest_real(vcpu, authority_table_addr,
431 &authority_table,
432 sizeof(u8));
433 if (rc)
434 return rc;
435
436 if ((authority_table & (0x40 >> ((eax & 3) * 2))) == 0)
437 return PGM_EXTENDED_AUTHORITY;
438 }
439 }
440
441 if (ale.fo == 1 && write)
442 return PGM_PROTECTION;
443
444 asce->val = aste.asce;
445 return 0;
446}
447
448struct trans_exc_code_bits {
449 unsigned long addr : 52; /* Translation-exception Address */
450 unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */
451 unsigned long : 6;
452 unsigned long b60 : 1;
453 unsigned long b61 : 1;
454 unsigned long as : 2; /* ASCE Identifier */
455};
456
457enum {
458 FSI_UNKNOWN = 0, /* Unknown wether fetch or store */
459 FSI_STORE = 1, /* Exception was due to store operation */
460 FSI_FETCH = 2 /* Exception was due to fetch operation */
461};
462
463static int get_vcpu_asce(struct kvm_vcpu *vcpu, union asce *asce,
464 ar_t ar, int write)
311{ 465{
466 int rc;
467 psw_t *psw = &vcpu->arch.sie_block->gpsw;
468 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
469 struct trans_exc_code_bits *tec_bits;
470
471 memset(pgm, 0, sizeof(*pgm));
472 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
473 tec_bits->fsi = write ? FSI_STORE : FSI_FETCH;
474 tec_bits->as = psw_bits(*psw).as;
475
476 if (!psw_bits(*psw).t) {
477 asce->val = 0;
478 asce->r = 1;
479 return 0;
480 }
481
312 switch (psw_bits(vcpu->arch.sie_block->gpsw).as) { 482 switch (psw_bits(vcpu->arch.sie_block->gpsw).as) {
313 case PSW_AS_PRIMARY: 483 case PSW_AS_PRIMARY:
314 return vcpu->arch.sie_block->gcr[1]; 484 asce->val = vcpu->arch.sie_block->gcr[1];
485 return 0;
315 case PSW_AS_SECONDARY: 486 case PSW_AS_SECONDARY:
316 return vcpu->arch.sie_block->gcr[7]; 487 asce->val = vcpu->arch.sie_block->gcr[7];
488 return 0;
317 case PSW_AS_HOME: 489 case PSW_AS_HOME:
318 return vcpu->arch.sie_block->gcr[13]; 490 asce->val = vcpu->arch.sie_block->gcr[13];
491 return 0;
492 case PSW_AS_ACCREG:
493 rc = ar_translation(vcpu, asce, ar, write);
494 switch (rc) {
495 case PGM_ALEN_TRANSLATION:
496 case PGM_ALE_SEQUENCE:
497 case PGM_ASTE_VALIDITY:
498 case PGM_ASTE_SEQUENCE:
499 case PGM_EXTENDED_AUTHORITY:
500 vcpu->arch.pgm.exc_access_id = ar;
501 break;
502 case PGM_PROTECTION:
503 tec_bits->b60 = 1;
504 tec_bits->b61 = 1;
505 break;
506 }
507 if (rc > 0)
508 pgm->code = rc;
509 return rc;
319 } 510 }
320 return 0; 511 return 0;
321} 512}
@@ -330,10 +521,11 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
330 * @vcpu: virtual cpu 521 * @vcpu: virtual cpu
331 * @gva: guest virtual address 522 * @gva: guest virtual address
332 * @gpa: points to where guest physical (absolute) address should be stored 523 * @gpa: points to where guest physical (absolute) address should be stored
524 * @asce: effective asce
333 * @write: indicates if access is a write access 525 * @write: indicates if access is a write access
334 * 526 *
335 * Translate a guest virtual address into a guest absolute address by means 527 * Translate a guest virtual address into a guest absolute address by means
336 * of dynamic address translation as specified by the architecuture. 528 * of dynamic address translation as specified by the architecture.
337 * If the resulting absolute address is not available in the configuration 529 * If the resulting absolute address is not available in the configuration
338 * an addressing exception is indicated and @gpa will not be changed. 530 * an addressing exception is indicated and @gpa will not be changed.
339 * 531 *
@@ -345,7 +537,8 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
345 * by the architecture 537 * by the architecture
346 */ 538 */
347static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, 539static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
348 unsigned long *gpa, int write) 540 unsigned long *gpa, const union asce asce,
541 int write)
349{ 542{
350 union vaddress vaddr = {.addr = gva}; 543 union vaddress vaddr = {.addr = gva};
351 union raddress raddr = {.addr = gva}; 544 union raddress raddr = {.addr = gva};
@@ -354,12 +547,10 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
354 union ctlreg0 ctlreg0; 547 union ctlreg0 ctlreg0;
355 unsigned long ptr; 548 unsigned long ptr;
356 int edat1, edat2; 549 int edat1, edat2;
357 union asce asce;
358 550
359 ctlreg0.val = vcpu->arch.sie_block->gcr[0]; 551 ctlreg0.val = vcpu->arch.sie_block->gcr[0];
360 edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8); 552 edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
361 edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78); 553 edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
362 asce.val = get_vcpu_asce(vcpu);
363 if (asce.r) 554 if (asce.r)
364 goto real_address; 555 goto real_address;
365 ptr = asce.origin * 4096; 556 ptr = asce.origin * 4096;
@@ -506,48 +697,30 @@ static inline int is_low_address(unsigned long ga)
506 return (ga & ~0x11fful) == 0; 697 return (ga & ~0x11fful) == 0;
507} 698}
508 699
509static int low_address_protection_enabled(struct kvm_vcpu *vcpu) 700static int low_address_protection_enabled(struct kvm_vcpu *vcpu,
701 const union asce asce)
510{ 702{
511 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]}; 703 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
512 psw_t *psw = &vcpu->arch.sie_block->gpsw; 704 psw_t *psw = &vcpu->arch.sie_block->gpsw;
513 union asce asce;
514 705
515 if (!ctlreg0.lap) 706 if (!ctlreg0.lap)
516 return 0; 707 return 0;
517 asce.val = get_vcpu_asce(vcpu);
518 if (psw_bits(*psw).t && asce.p) 708 if (psw_bits(*psw).t && asce.p)
519 return 0; 709 return 0;
520 return 1; 710 return 1;
521} 711}
522 712
523struct trans_exc_code_bits {
524 unsigned long addr : 52; /* Translation-exception Address */
525 unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */
526 unsigned long : 7;
527 unsigned long b61 : 1;
528 unsigned long as : 2; /* ASCE Identifier */
529};
530
531enum {
532 FSI_UNKNOWN = 0, /* Unknown wether fetch or store */
533 FSI_STORE = 1, /* Exception was due to store operation */
534 FSI_FETCH = 2 /* Exception was due to fetch operation */
535};
536
537static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga, 713static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
538 unsigned long *pages, unsigned long nr_pages, 714 unsigned long *pages, unsigned long nr_pages,
539 int write) 715 const union asce asce, int write)
540{ 716{
541 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; 717 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
542 psw_t *psw = &vcpu->arch.sie_block->gpsw; 718 psw_t *psw = &vcpu->arch.sie_block->gpsw;
543 struct trans_exc_code_bits *tec_bits; 719 struct trans_exc_code_bits *tec_bits;
544 int lap_enabled, rc; 720 int lap_enabled, rc;
545 721
546 memset(pgm, 0, sizeof(*pgm));
547 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code; 722 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
548 tec_bits->fsi = write ? FSI_STORE : FSI_FETCH; 723 lap_enabled = low_address_protection_enabled(vcpu, asce);
549 tec_bits->as = psw_bits(*psw).as;
550 lap_enabled = low_address_protection_enabled(vcpu);
551 while (nr_pages) { 724 while (nr_pages) {
552 ga = kvm_s390_logical_to_effective(vcpu, ga); 725 ga = kvm_s390_logical_to_effective(vcpu, ga);
553 tec_bits->addr = ga >> PAGE_SHIFT; 726 tec_bits->addr = ga >> PAGE_SHIFT;
@@ -557,7 +730,7 @@ static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
557 } 730 }
558 ga &= PAGE_MASK; 731 ga &= PAGE_MASK;
559 if (psw_bits(*psw).t) { 732 if (psw_bits(*psw).t) {
560 rc = guest_translate(vcpu, ga, pages, write); 733 rc = guest_translate(vcpu, ga, pages, asce, write);
561 if (rc < 0) 734 if (rc < 0)
562 return rc; 735 return rc;
563 if (rc == PGM_PROTECTION) 736 if (rc == PGM_PROTECTION)
@@ -578,7 +751,7 @@ static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
578 return 0; 751 return 0;
579} 752}
580 753
581int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data, 754int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
582 unsigned long len, int write) 755 unsigned long len, int write)
583{ 756{
584 psw_t *psw = &vcpu->arch.sie_block->gpsw; 757 psw_t *psw = &vcpu->arch.sie_block->gpsw;
@@ -591,20 +764,19 @@ int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
591 764
592 if (!len) 765 if (!len)
593 return 0; 766 return 0;
594 /* Access register mode is not supported yet. */ 767 rc = get_vcpu_asce(vcpu, &asce, ar, write);
595 if (psw_bits(*psw).t && psw_bits(*psw).as == PSW_AS_ACCREG) 768 if (rc)
596 return -EOPNOTSUPP; 769 return rc;
597 nr_pages = (((ga & ~PAGE_MASK) + len - 1) >> PAGE_SHIFT) + 1; 770 nr_pages = (((ga & ~PAGE_MASK) + len - 1) >> PAGE_SHIFT) + 1;
598 pages = pages_array; 771 pages = pages_array;
599 if (nr_pages > ARRAY_SIZE(pages_array)) 772 if (nr_pages > ARRAY_SIZE(pages_array))
600 pages = vmalloc(nr_pages * sizeof(unsigned long)); 773 pages = vmalloc(nr_pages * sizeof(unsigned long));
601 if (!pages) 774 if (!pages)
602 return -ENOMEM; 775 return -ENOMEM;
603 asce.val = get_vcpu_asce(vcpu);
604 need_ipte_lock = psw_bits(*psw).t && !asce.r; 776 need_ipte_lock = psw_bits(*psw).t && !asce.r;
605 if (need_ipte_lock) 777 if (need_ipte_lock)
606 ipte_lock(vcpu); 778 ipte_lock(vcpu);
607 rc = guest_page_range(vcpu, ga, pages, nr_pages, write); 779 rc = guest_page_range(vcpu, ga, pages, nr_pages, asce, write);
608 for (idx = 0; idx < nr_pages && !rc; idx++) { 780 for (idx = 0; idx < nr_pages && !rc; idx++) {
609 gpa = *(pages + idx) + (ga & ~PAGE_MASK); 781 gpa = *(pages + idx) + (ga & ~PAGE_MASK);
610 _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len); 782 _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len);
@@ -652,7 +824,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
652 * Note: The IPTE lock is not taken during this function, so the caller 824 * Note: The IPTE lock is not taken during this function, so the caller
653 * has to take care of this. 825 * has to take care of this.
654 */ 826 */
655int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, 827int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
656 unsigned long *gpa, int write) 828 unsigned long *gpa, int write)
657{ 829{
658 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; 830 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
@@ -661,26 +833,21 @@ int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
661 union asce asce; 833 union asce asce;
662 int rc; 834 int rc;
663 835
664 /* Access register mode is not supported yet. */
665 if (psw_bits(*psw).t && psw_bits(*psw).as == PSW_AS_ACCREG)
666 return -EOPNOTSUPP;
667
668 gva = kvm_s390_logical_to_effective(vcpu, gva); 836 gva = kvm_s390_logical_to_effective(vcpu, gva);
669 memset(pgm, 0, sizeof(*pgm));
670 tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code; 837 tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
671 tec->as = psw_bits(*psw).as; 838 rc = get_vcpu_asce(vcpu, &asce, ar, write);
672 tec->fsi = write ? FSI_STORE : FSI_FETCH;
673 tec->addr = gva >> PAGE_SHIFT; 839 tec->addr = gva >> PAGE_SHIFT;
674 if (is_low_address(gva) && low_address_protection_enabled(vcpu)) { 840 if (rc)
841 return rc;
842 if (is_low_address(gva) && low_address_protection_enabled(vcpu, asce)) {
675 if (write) { 843 if (write) {
676 rc = pgm->code = PGM_PROTECTION; 844 rc = pgm->code = PGM_PROTECTION;
677 return rc; 845 return rc;
678 } 846 }
679 } 847 }
680 848
681 asce.val = get_vcpu_asce(vcpu);
682 if (psw_bits(*psw).t && !asce.r) { /* Use DAT? */ 849 if (psw_bits(*psw).t && !asce.r) { /* Use DAT? */
683 rc = guest_translate(vcpu, gva, gpa, write); 850 rc = guest_translate(vcpu, gva, gpa, asce, write);
684 if (rc > 0) { 851 if (rc > 0) {
685 if (rc == PGM_PROTECTION) 852 if (rc == PGM_PROTECTION)
686 tec->b61 = 1; 853 tec->b61 = 1;
@@ -697,28 +864,51 @@ int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
697} 864}
698 865
699/** 866/**
700 * kvm_s390_check_low_addr_protection - check for low-address protection 867 * check_gva_range - test a range of guest virtual addresses for accessibility
701 * @ga: Guest address 868 */
869int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
870 unsigned long length, int is_write)
871{
872 unsigned long gpa;
873 unsigned long currlen;
874 int rc = 0;
875
876 ipte_lock(vcpu);
877 while (length > 0 && !rc) {
878 currlen = min(length, PAGE_SIZE - (gva % PAGE_SIZE));
879 rc = guest_translate_address(vcpu, gva, ar, &gpa, is_write);
880 gva += currlen;
881 length -= currlen;
882 }
883 ipte_unlock(vcpu);
884
885 return rc;
886}
887
888/**
889 * kvm_s390_check_low_addr_prot_real - check for low-address protection
890 * @gra: Guest real address
702 * 891 *
703 * Checks whether an address is subject to low-address protection and set 892 * Checks whether an address is subject to low-address protection and set
704 * up vcpu->arch.pgm accordingly if necessary. 893 * up vcpu->arch.pgm accordingly if necessary.
705 * 894 *
706 * Return: 0 if no protection exception, or PGM_PROTECTION if protected. 895 * Return: 0 if no protection exception, or PGM_PROTECTION if protected.
707 */ 896 */
708int kvm_s390_check_low_addr_protection(struct kvm_vcpu *vcpu, unsigned long ga) 897int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra)
709{ 898{
710 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; 899 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
711 psw_t *psw = &vcpu->arch.sie_block->gpsw; 900 psw_t *psw = &vcpu->arch.sie_block->gpsw;
712 struct trans_exc_code_bits *tec_bits; 901 struct trans_exc_code_bits *tec_bits;
902 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
713 903
714 if (!is_low_address(ga) || !low_address_protection_enabled(vcpu)) 904 if (!ctlreg0.lap || !is_low_address(gra))
715 return 0; 905 return 0;
716 906
717 memset(pgm, 0, sizeof(*pgm)); 907 memset(pgm, 0, sizeof(*pgm));
718 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code; 908 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
719 tec_bits->fsi = FSI_STORE; 909 tec_bits->fsi = FSI_STORE;
720 tec_bits->as = psw_bits(*psw).as; 910 tec_bits->as = psw_bits(*psw).as;
721 tec_bits->addr = ga >> PAGE_SHIFT; 911 tec_bits->addr = gra >> PAGE_SHIFT;
722 pgm->code = PGM_PROTECTION; 912 pgm->code = PGM_PROTECTION;
723 913
724 return pgm->code; 914 return pgm->code;
diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/gaccess.h
index 0149cf15058a..ef03726cc661 100644
--- a/arch/s390/kvm/gaccess.h
+++ b/arch/s390/kvm/gaccess.h
@@ -156,9 +156,11 @@ int read_guest_lc(struct kvm_vcpu *vcpu, unsigned long gra, void *data,
156} 156}
157 157
158int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, 158int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
159 unsigned long *gpa, int write); 159 ar_t ar, unsigned long *gpa, int write);
160int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
161 unsigned long length, int is_write);
160 162
161int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data, 163int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
162 unsigned long len, int write); 164 unsigned long len, int write);
163 165
164int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, 166int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
@@ -168,6 +170,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
168 * write_guest - copy data from kernel space to guest space 170 * write_guest - copy data from kernel space to guest space
169 * @vcpu: virtual cpu 171 * @vcpu: virtual cpu
170 * @ga: guest address 172 * @ga: guest address
173 * @ar: access register
171 * @data: source address in kernel space 174 * @data: source address in kernel space
172 * @len: number of bytes to copy 175 * @len: number of bytes to copy
173 * 176 *
@@ -176,8 +179,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
176 * If DAT is off data will be copied to guest real or absolute memory. 179 * If DAT is off data will be copied to guest real or absolute memory.
177 * If DAT is on data will be copied to the address space as specified by 180 * If DAT is on data will be copied to the address space as specified by
178 * the address space bits of the PSW: 181 * the address space bits of the PSW:
179 * Primary, secondory or home space (access register mode is currently not 182 * Primary, secondary, home space or access register mode.
180 * implemented).
181 * The addressing mode of the PSW is also inspected, so that address wrap 183 * The addressing mode of the PSW is also inspected, so that address wrap
182 * around is taken into account for 24-, 31- and 64-bit addressing mode, 184 * around is taken into account for 24-, 31- and 64-bit addressing mode,
183 * if the to be copied data crosses page boundaries in guest address space. 185 * if the to be copied data crosses page boundaries in guest address space.
@@ -210,16 +212,17 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
210 * if data has been changed in guest space in case of an exception. 212 * if data has been changed in guest space in case of an exception.
211 */ 213 */
212static inline __must_check 214static inline __must_check
213int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data, 215int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
214 unsigned long len) 216 unsigned long len)
215{ 217{
216 return access_guest(vcpu, ga, data, len, 1); 218 return access_guest(vcpu, ga, ar, data, len, 1);
217} 219}
218 220
219/** 221/**
220 * read_guest - copy data from guest space to kernel space 222 * read_guest - copy data from guest space to kernel space
221 * @vcpu: virtual cpu 223 * @vcpu: virtual cpu
222 * @ga: guest address 224 * @ga: guest address
225 * @ar: access register
223 * @data: destination address in kernel space 226 * @data: destination address in kernel space
224 * @len: number of bytes to copy 227 * @len: number of bytes to copy
225 * 228 *
@@ -229,10 +232,10 @@ int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
229 * data will be copied from guest space to kernel space. 232 * data will be copied from guest space to kernel space.
230 */ 233 */
231static inline __must_check 234static inline __must_check
232int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data, 235int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
233 unsigned long len) 236 unsigned long len)
234{ 237{
235 return access_guest(vcpu, ga, data, len, 0); 238 return access_guest(vcpu, ga, ar, data, len, 0);
236} 239}
237 240
238/** 241/**
@@ -330,6 +333,6 @@ int read_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, void *data,
330void ipte_lock(struct kvm_vcpu *vcpu); 333void ipte_lock(struct kvm_vcpu *vcpu);
331void ipte_unlock(struct kvm_vcpu *vcpu); 334void ipte_unlock(struct kvm_vcpu *vcpu);
332int ipte_lock_held(struct kvm_vcpu *vcpu); 335int ipte_lock_held(struct kvm_vcpu *vcpu);
333int kvm_s390_check_low_addr_protection(struct kvm_vcpu *vcpu, unsigned long ga); 336int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra);
334 337
335#endif /* __KVM_S390_GACCESS_H */ 338#endif /* __KVM_S390_GACCESS_H */
diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c
index 3e8d4092ce30..e97b3455d7e6 100644
--- a/arch/s390/kvm/guestdbg.c
+++ b/arch/s390/kvm/guestdbg.c
@@ -191,8 +191,8 @@ static int __import_wp_info(struct kvm_vcpu *vcpu,
191 if (!wp_info->old_data) 191 if (!wp_info->old_data)
192 return -ENOMEM; 192 return -ENOMEM;
193 /* try to backup the original value */ 193 /* try to backup the original value */
194 ret = read_guest(vcpu, wp_info->phys_addr, wp_info->old_data, 194 ret = read_guest_abs(vcpu, wp_info->phys_addr, wp_info->old_data,
195 wp_info->len); 195 wp_info->len);
196 if (ret) { 196 if (ret) {
197 kfree(wp_info->old_data); 197 kfree(wp_info->old_data);
198 wp_info->old_data = NULL; 198 wp_info->old_data = NULL;
@@ -362,8 +362,8 @@ static struct kvm_hw_wp_info_arch *any_wp_changed(struct kvm_vcpu *vcpu)
362 continue; 362 continue;
363 363
364 /* refetch the wp data and compare it to the old value */ 364 /* refetch the wp data and compare it to the old value */
365 if (!read_guest(vcpu, wp_info->phys_addr, temp, 365 if (!read_guest_abs(vcpu, wp_info->phys_addr, temp,
366 wp_info->len)) { 366 wp_info->len)) {
367 if (memcmp(temp, wp_info->old_data, wp_info->len)) { 367 if (memcmp(temp, wp_info->old_data, wp_info->len)) {
368 kfree(temp); 368 kfree(temp);
369 return wp_info; 369 return wp_info;
diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c
index bebd2157edd0..9e3779e3e496 100644
--- a/arch/s390/kvm/intercept.c
+++ b/arch/s390/kvm/intercept.c
@@ -165,6 +165,7 @@ static void __extract_prog_irq(struct kvm_vcpu *vcpu,
165 pgm_info->mon_class_nr = vcpu->arch.sie_block->mcn; 165 pgm_info->mon_class_nr = vcpu->arch.sie_block->mcn;
166 pgm_info->mon_code = vcpu->arch.sie_block->tecmc; 166 pgm_info->mon_code = vcpu->arch.sie_block->tecmc;
167 break; 167 break;
168 case PGM_VECTOR_PROCESSING:
168 case PGM_DATA: 169 case PGM_DATA:
169 pgm_info->data_exc_code = vcpu->arch.sie_block->dxc; 170 pgm_info->data_exc_code = vcpu->arch.sie_block->dxc;
170 break; 171 break;
@@ -319,7 +320,7 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
319 320
320 /* Make sure that the source is paged-in */ 321 /* Make sure that the source is paged-in */
321 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg2], 322 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg2],
322 &srcaddr, 0); 323 reg2, &srcaddr, 0);
323 if (rc) 324 if (rc)
324 return kvm_s390_inject_prog_cond(vcpu, rc); 325 return kvm_s390_inject_prog_cond(vcpu, rc);
325 rc = kvm_arch_fault_in_page(vcpu, srcaddr, 0); 326 rc = kvm_arch_fault_in_page(vcpu, srcaddr, 0);
@@ -328,7 +329,7 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
328 329
329 /* Make sure that the destination is paged-in */ 330 /* Make sure that the destination is paged-in */
330 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg1], 331 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg1],
331 &dstaddr, 1); 332 reg1, &dstaddr, 1);
332 if (rc) 333 if (rc)
333 return kvm_s390_inject_prog_cond(vcpu, rc); 334 return kvm_s390_inject_prog_cond(vcpu, rc);
334 rc = kvm_arch_fault_in_page(vcpu, dstaddr, 1); 335 rc = kvm_arch_fault_in_page(vcpu, dstaddr, 1);
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 073b5f387d1d..9de47265ef73 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * handling kvm guest interrupts 2 * handling kvm guest interrupts
3 * 3 *
4 * Copyright IBM Corp. 2008,2014 4 * Copyright IBM Corp. 2008, 2015
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only) 7 * it under the terms of the GNU General Public License (version 2 only)
@@ -17,9 +17,12 @@
17#include <linux/signal.h> 17#include <linux/signal.h>
18#include <linux/slab.h> 18#include <linux/slab.h>
19#include <linux/bitmap.h> 19#include <linux/bitmap.h>
20#include <linux/vmalloc.h>
20#include <asm/asm-offsets.h> 21#include <asm/asm-offsets.h>
22#include <asm/dis.h>
21#include <asm/uaccess.h> 23#include <asm/uaccess.h>
22#include <asm/sclp.h> 24#include <asm/sclp.h>
25#include <asm/isc.h>
23#include "kvm-s390.h" 26#include "kvm-s390.h"
24#include "gaccess.h" 27#include "gaccess.h"
25#include "trace-s390.h" 28#include "trace-s390.h"
@@ -32,11 +35,6 @@
32#define PFAULT_DONE 0x0680 35#define PFAULT_DONE 0x0680
33#define VIRTIO_PARAM 0x0d00 36#define VIRTIO_PARAM 0x0d00
34 37
35static int is_ioint(u64 type)
36{
37 return ((type & 0xfffe0000u) != 0xfffe0000u);
38}
39
40int psw_extint_disabled(struct kvm_vcpu *vcpu) 38int psw_extint_disabled(struct kvm_vcpu *vcpu)
41{ 39{
42 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT); 40 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
@@ -72,70 +70,45 @@ static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
72 return 1; 70 return 1;
73} 71}
74 72
75static u64 int_word_to_isc_bits(u32 int_word) 73static int ckc_irq_pending(struct kvm_vcpu *vcpu)
74{
75 if (!(vcpu->arch.sie_block->ckc <
76 get_tod_clock_fast() + vcpu->arch.sie_block->epoch))
77 return 0;
78 return ckc_interrupts_enabled(vcpu);
79}
80
81static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
82{
83 return !psw_extint_disabled(vcpu) &&
84 (vcpu->arch.sie_block->gcr[0] & 0x400ul);
85}
86
87static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
88{
89 return (vcpu->arch.sie_block->cputm >> 63) &&
90 cpu_timer_interrupts_enabled(vcpu);
91}
92
93static inline int is_ioirq(unsigned long irq_type)
76{ 94{
77 u8 isc = (int_word & 0x38000000) >> 27; 95 return ((irq_type >= IRQ_PEND_IO_ISC_0) &&
96 (irq_type <= IRQ_PEND_IO_ISC_7));
97}
78 98
99static uint64_t isc_to_isc_bits(int isc)
100{
79 return (0x80 >> isc) << 24; 101 return (0x80 >> isc) << 24;
80} 102}
81 103
82static int __must_check __interrupt_is_deliverable(struct kvm_vcpu *vcpu, 104static inline u8 int_word_to_isc(u32 int_word)
83 struct kvm_s390_interrupt_info *inti)
84{ 105{
85 switch (inti->type) { 106 return (int_word & 0x38000000) >> 27;
86 case KVM_S390_INT_EXTERNAL_CALL: 107}
87 if (psw_extint_disabled(vcpu)) 108
88 return 0; 109static inline unsigned long pending_floating_irqs(struct kvm_vcpu *vcpu)
89 if (vcpu->arch.sie_block->gcr[0] & 0x2000ul) 110{
90 return 1; 111 return vcpu->kvm->arch.float_int.pending_irqs;
91 return 0;
92 case KVM_S390_INT_EMERGENCY:
93 if (psw_extint_disabled(vcpu))
94 return 0;
95 if (vcpu->arch.sie_block->gcr[0] & 0x4000ul)
96 return 1;
97 return 0;
98 case KVM_S390_INT_CLOCK_COMP:
99 return ckc_interrupts_enabled(vcpu);
100 case KVM_S390_INT_CPU_TIMER:
101 if (psw_extint_disabled(vcpu))
102 return 0;
103 if (vcpu->arch.sie_block->gcr[0] & 0x400ul)
104 return 1;
105 return 0;
106 case KVM_S390_INT_SERVICE:
107 case KVM_S390_INT_PFAULT_INIT:
108 case KVM_S390_INT_PFAULT_DONE:
109 case KVM_S390_INT_VIRTIO:
110 if (psw_extint_disabled(vcpu))
111 return 0;
112 if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
113 return 1;
114 return 0;
115 case KVM_S390_PROGRAM_INT:
116 case KVM_S390_SIGP_STOP:
117 case KVM_S390_SIGP_SET_PREFIX:
118 case KVM_S390_RESTART:
119 return 1;
120 case KVM_S390_MCHK:
121 if (psw_mchk_disabled(vcpu))
122 return 0;
123 if (vcpu->arch.sie_block->gcr[14] & inti->mchk.cr14)
124 return 1;
125 return 0;
126 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
127 if (psw_ioint_disabled(vcpu))
128 return 0;
129 if (vcpu->arch.sie_block->gcr[6] &
130 int_word_to_isc_bits(inti->io.io_int_word))
131 return 1;
132 return 0;
133 default:
134 printk(KERN_WARNING "illegal interrupt type %llx\n",
135 inti->type);
136 BUG();
137 }
138 return 0;
139} 112}
140 113
141static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu) 114static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu)
@@ -143,12 +116,31 @@ static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu)
143 return vcpu->arch.local_int.pending_irqs; 116 return vcpu->arch.local_int.pending_irqs;
144} 117}
145 118
146static unsigned long deliverable_local_irqs(struct kvm_vcpu *vcpu) 119static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
120 unsigned long active_mask)
121{
122 int i;
123
124 for (i = 0; i <= MAX_ISC; i++)
125 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
126 active_mask &= ~(1UL << (IRQ_PEND_IO_ISC_0 + i));
127
128 return active_mask;
129}
130
131static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
147{ 132{
148 unsigned long active_mask = pending_local_irqs(vcpu); 133 unsigned long active_mask;
134
135 active_mask = pending_local_irqs(vcpu);
136 active_mask |= pending_floating_irqs(vcpu);
149 137
150 if (psw_extint_disabled(vcpu)) 138 if (psw_extint_disabled(vcpu))
151 active_mask &= ~IRQ_PEND_EXT_MASK; 139 active_mask &= ~IRQ_PEND_EXT_MASK;
140 if (psw_ioint_disabled(vcpu))
141 active_mask &= ~IRQ_PEND_IO_MASK;
142 else
143 active_mask = disable_iscs(vcpu, active_mask);
152 if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 144 if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul))
153 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask); 145 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
154 if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul)) 146 if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul))
@@ -157,8 +149,13 @@ static unsigned long deliverable_local_irqs(struct kvm_vcpu *vcpu)
157 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); 149 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
158 if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul)) 150 if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul))
159 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); 151 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
152 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
153 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
160 if (psw_mchk_disabled(vcpu)) 154 if (psw_mchk_disabled(vcpu))
161 active_mask &= ~IRQ_PEND_MCHK_MASK; 155 active_mask &= ~IRQ_PEND_MCHK_MASK;
156 if (!(vcpu->arch.sie_block->gcr[14] &
157 vcpu->kvm->arch.float_int.mchk.cr14))
158 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
162 159
163 /* 160 /*
164 * STOP irqs will never be actively delivered. They are triggered via 161 * STOP irqs will never be actively delivered. They are triggered via
@@ -200,6 +197,16 @@ static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
200 atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags); 197 atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags);
201} 198}
202 199
200static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
201{
202 if (!(pending_floating_irqs(vcpu) & IRQ_PEND_IO_MASK))
203 return;
204 else if (psw_ioint_disabled(vcpu))
205 __set_cpuflag(vcpu, CPUSTAT_IO_INT);
206 else
207 vcpu->arch.sie_block->lctl |= LCTL_CR6;
208}
209
203static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) 210static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
204{ 211{
205 if (!(pending_local_irqs(vcpu) & IRQ_PEND_EXT_MASK)) 212 if (!(pending_local_irqs(vcpu) & IRQ_PEND_EXT_MASK))
@@ -226,47 +233,17 @@ static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
226 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 233 __set_cpuflag(vcpu, CPUSTAT_STOP_INT);
227} 234}
228 235
229/* Set interception request for non-deliverable local interrupts */ 236/* Set interception request for non-deliverable interrupts */
230static void set_intercept_indicators_local(struct kvm_vcpu *vcpu) 237static void set_intercept_indicators(struct kvm_vcpu *vcpu)
231{ 238{
239 set_intercept_indicators_io(vcpu);
232 set_intercept_indicators_ext(vcpu); 240 set_intercept_indicators_ext(vcpu);
233 set_intercept_indicators_mchk(vcpu); 241 set_intercept_indicators_mchk(vcpu);
234 set_intercept_indicators_stop(vcpu); 242 set_intercept_indicators_stop(vcpu);
235} 243}
236 244
237static void __set_intercept_indicator(struct kvm_vcpu *vcpu,
238 struct kvm_s390_interrupt_info *inti)
239{
240 switch (inti->type) {
241 case KVM_S390_INT_SERVICE:
242 case KVM_S390_INT_PFAULT_DONE:
243 case KVM_S390_INT_VIRTIO:
244 if (psw_extint_disabled(vcpu))
245 __set_cpuflag(vcpu, CPUSTAT_EXT_INT);
246 else
247 vcpu->arch.sie_block->lctl |= LCTL_CR0;
248 break;
249 case KVM_S390_MCHK:
250 if (psw_mchk_disabled(vcpu))
251 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
252 else
253 vcpu->arch.sie_block->lctl |= LCTL_CR14;
254 break;
255 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
256 if (psw_ioint_disabled(vcpu))
257 __set_cpuflag(vcpu, CPUSTAT_IO_INT);
258 else
259 vcpu->arch.sie_block->lctl |= LCTL_CR6;
260 break;
261 default:
262 BUG();
263 }
264}
265
266static u16 get_ilc(struct kvm_vcpu *vcpu) 245static u16 get_ilc(struct kvm_vcpu *vcpu)
267{ 246{
268 const unsigned short table[] = { 2, 4, 4, 6 };
269
270 switch (vcpu->arch.sie_block->icptcode) { 247 switch (vcpu->arch.sie_block->icptcode) {
271 case ICPT_INST: 248 case ICPT_INST:
272 case ICPT_INSTPROGI: 249 case ICPT_INSTPROGI:
@@ -274,7 +251,7 @@ static u16 get_ilc(struct kvm_vcpu *vcpu)
274 case ICPT_PARTEXEC: 251 case ICPT_PARTEXEC:
275 case ICPT_IOINST: 252 case ICPT_IOINST:
276 /* last instruction only stored for these icptcodes */ 253 /* last instruction only stored for these icptcodes */
277 return table[vcpu->arch.sie_block->ipa >> 14]; 254 return insn_length(vcpu->arch.sie_block->ipa >> 8);
278 case ICPT_PROGI: 255 case ICPT_PROGI:
279 return vcpu->arch.sie_block->pgmilc; 256 return vcpu->arch.sie_block->pgmilc;
280 default: 257 default:
@@ -350,38 +327,72 @@ static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
350 327
351static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) 328static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
352{ 329{
330 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
353 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 331 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
354 struct kvm_s390_mchk_info mchk; 332 struct kvm_s390_mchk_info mchk = {};
355 int rc; 333 unsigned long adtl_status_addr;
334 int deliver = 0;
335 int rc = 0;
356 336
337 spin_lock(&fi->lock);
357 spin_lock(&li->lock); 338 spin_lock(&li->lock);
358 mchk = li->irq.mchk; 339 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
340 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
341 /*
342 * If there was an exigent machine check pending, then any
343 * repressible machine checks that might have been pending
344 * are indicated along with it, so always clear bits for
345 * repressible and exigent interrupts
346 */
347 mchk = li->irq.mchk;
348 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
349 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
350 memset(&li->irq.mchk, 0, sizeof(mchk));
351 deliver = 1;
352 }
359 /* 353 /*
360 * If there was an exigent machine check pending, then any repressible 354 * We indicate floating repressible conditions along with
361 * machine checks that might have been pending are indicated along 355 * other pending conditions. Channel Report Pending and Channel
362 * with it, so always clear both bits 356 * Subsystem damage are the only two and and are indicated by
357 * bits in mcic and masked in cr14.
363 */ 358 */
364 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 359 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
365 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 360 mchk.mcic |= fi->mchk.mcic;
366 memset(&li->irq.mchk, 0, sizeof(mchk)); 361 mchk.cr14 |= fi->mchk.cr14;
362 memset(&fi->mchk, 0, sizeof(mchk));
363 deliver = 1;
364 }
367 spin_unlock(&li->lock); 365 spin_unlock(&li->lock);
366 spin_unlock(&fi->lock);
368 367
369 VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx", 368 if (deliver) {
370 mchk.mcic); 369 VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
371 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_MCHK, 370 mchk.mcic);
372 mchk.cr14, mchk.mcic); 371 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
373 372 KVM_S390_MCHK,
374 rc = kvm_s390_vcpu_store_status(vcpu, KVM_S390_STORE_STATUS_PREFIXED); 373 mchk.cr14, mchk.mcic);
375 rc |= put_guest_lc(vcpu, mchk.mcic, 374
376 (u64 __user *) __LC_MCCK_CODE); 375 rc = kvm_s390_vcpu_store_status(vcpu,
377 rc |= put_guest_lc(vcpu, mchk.failing_storage_address, 376 KVM_S390_STORE_STATUS_PREFIXED);
378 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR); 377 rc |= read_guest_lc(vcpu, __LC_VX_SAVE_AREA_ADDR,
379 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, 378 &adtl_status_addr,
380 &mchk.fixed_logout, sizeof(mchk.fixed_logout)); 379 sizeof(unsigned long));
381 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW, 380 rc |= kvm_s390_vcpu_store_adtl_status(vcpu,
382 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 381 adtl_status_addr);
383 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW, 382 rc |= put_guest_lc(vcpu, mchk.mcic,
384 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 383 (u64 __user *) __LC_MCCK_CODE);
384 rc |= put_guest_lc(vcpu, mchk.failing_storage_address,
385 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
386 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA,
387 &mchk.fixed_logout,
388 sizeof(mchk.fixed_logout));
389 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
390 &vcpu->arch.sie_block->gpsw,
391 sizeof(psw_t));
392 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
393 &vcpu->arch.sie_block->gpsw,
394 sizeof(psw_t));
395 }
385 return rc ? -EFAULT : 0; 396 return rc ? -EFAULT : 0;
386} 397}
387 398
@@ -484,7 +495,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
484{ 495{
485 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 496 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
486 struct kvm_s390_pgm_info pgm_info; 497 struct kvm_s390_pgm_info pgm_info;
487 int rc = 0; 498 int rc = 0, nullifying = false;
488 u16 ilc = get_ilc(vcpu); 499 u16 ilc = get_ilc(vcpu);
489 500
490 spin_lock(&li->lock); 501 spin_lock(&li->lock);
@@ -509,6 +520,8 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
509 case PGM_LX_TRANSLATION: 520 case PGM_LX_TRANSLATION:
510 case PGM_PRIMARY_AUTHORITY: 521 case PGM_PRIMARY_AUTHORITY:
511 case PGM_SECONDARY_AUTHORITY: 522 case PGM_SECONDARY_AUTHORITY:
523 nullifying = true;
524 /* fall through */
512 case PGM_SPACE_SWITCH: 525 case PGM_SPACE_SWITCH:
513 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 526 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
514 (u64 *)__LC_TRANS_EXC_CODE); 527 (u64 *)__LC_TRANS_EXC_CODE);
@@ -521,6 +534,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
521 case PGM_EXTENDED_AUTHORITY: 534 case PGM_EXTENDED_AUTHORITY:
522 rc = put_guest_lc(vcpu, pgm_info.exc_access_id, 535 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
523 (u8 *)__LC_EXC_ACCESS_ID); 536 (u8 *)__LC_EXC_ACCESS_ID);
537 nullifying = true;
524 break; 538 break;
525 case PGM_ASCE_TYPE: 539 case PGM_ASCE_TYPE:
526 case PGM_PAGE_TRANSLATION: 540 case PGM_PAGE_TRANSLATION:
@@ -534,6 +548,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
534 (u8 *)__LC_EXC_ACCESS_ID); 548 (u8 *)__LC_EXC_ACCESS_ID);
535 rc |= put_guest_lc(vcpu, pgm_info.op_access_id, 549 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
536 (u8 *)__LC_OP_ACCESS_ID); 550 (u8 *)__LC_OP_ACCESS_ID);
551 nullifying = true;
537 break; 552 break;
538 case PGM_MONITOR: 553 case PGM_MONITOR:
539 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr, 554 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
@@ -541,6 +556,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
541 rc |= put_guest_lc(vcpu, pgm_info.mon_code, 556 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
542 (u64 *)__LC_MON_CODE); 557 (u64 *)__LC_MON_CODE);
543 break; 558 break;
559 case PGM_VECTOR_PROCESSING:
544 case PGM_DATA: 560 case PGM_DATA:
545 rc = put_guest_lc(vcpu, pgm_info.data_exc_code, 561 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
546 (u32 *)__LC_DATA_EXC_CODE); 562 (u32 *)__LC_DATA_EXC_CODE);
@@ -551,6 +567,15 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
551 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 567 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
552 (u8 *)__LC_EXC_ACCESS_ID); 568 (u8 *)__LC_EXC_ACCESS_ID);
553 break; 569 break;
570 case PGM_STACK_FULL:
571 case PGM_STACK_EMPTY:
572 case PGM_STACK_SPECIFICATION:
573 case PGM_STACK_TYPE:
574 case PGM_STACK_OPERATION:
575 case PGM_TRACE_TABEL:
576 case PGM_CRYPTO_OPERATION:
577 nullifying = true;
578 break;
554 } 579 }
555 580
556 if (pgm_info.code & PGM_PER) { 581 if (pgm_info.code & PGM_PER) {
@@ -564,7 +589,12 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
564 (u8 *) __LC_PER_ACCESS_ID); 589 (u8 *) __LC_PER_ACCESS_ID);
565 } 590 }
566 591
592 if (nullifying && vcpu->arch.sie_block->icptcode == ICPT_INST)
593 kvm_s390_rewind_psw(vcpu, ilc);
594
567 rc |= put_guest_lc(vcpu, ilc, (u16 *) __LC_PGM_ILC); 595 rc |= put_guest_lc(vcpu, ilc, (u16 *) __LC_PGM_ILC);
596 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
597 (u64 *) __LC_LAST_BREAK);
568 rc |= put_guest_lc(vcpu, pgm_info.code, 598 rc |= put_guest_lc(vcpu, pgm_info.code,
569 (u16 *)__LC_PGM_INT_CODE); 599 (u16 *)__LC_PGM_INT_CODE);
570 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW, 600 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
@@ -574,16 +604,27 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
574 return rc ? -EFAULT : 0; 604 return rc ? -EFAULT : 0;
575} 605}
576 606
577static int __must_check __deliver_service(struct kvm_vcpu *vcpu, 607static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
578 struct kvm_s390_interrupt_info *inti)
579{ 608{
580 int rc; 609 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
610 struct kvm_s390_ext_info ext;
611 int rc = 0;
612
613 spin_lock(&fi->lock);
614 if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
615 spin_unlock(&fi->lock);
616 return 0;
617 }
618 ext = fi->srv_signal;
619 memset(&fi->srv_signal, 0, sizeof(ext));
620 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
621 spin_unlock(&fi->lock);
581 622
582 VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x", 623 VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x",
583 inti->ext.ext_params); 624 ext.ext_params);
584 vcpu->stat.deliver_service_signal++; 625 vcpu->stat.deliver_service_signal++;
585 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, 626 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
586 inti->ext.ext_params, 0); 627 ext.ext_params, 0);
587 628
588 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); 629 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
589 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 630 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
@@ -591,106 +632,146 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu,
591 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 632 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
592 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 633 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
593 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 634 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
594 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 635 rc |= put_guest_lc(vcpu, ext.ext_params,
595 (u32 *)__LC_EXT_PARAMS); 636 (u32 *)__LC_EXT_PARAMS);
637
596 return rc ? -EFAULT : 0; 638 return rc ? -EFAULT : 0;
597} 639}
598 640
599static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu, 641static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
600 struct kvm_s390_interrupt_info *inti)
601{ 642{
602 int rc; 643 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
644 struct kvm_s390_interrupt_info *inti;
645 int rc = 0;
603 646
604 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 647 spin_lock(&fi->lock);
605 KVM_S390_INT_PFAULT_DONE, 0, 648 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
606 inti->ext.ext_params2); 649 struct kvm_s390_interrupt_info,
650 list);
651 if (inti) {
652 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
653 KVM_S390_INT_PFAULT_DONE, 0,
654 inti->ext.ext_params2);
655 list_del(&inti->list);
656 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
657 }
658 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
659 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
660 spin_unlock(&fi->lock);
607 661
608 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *)__LC_EXT_INT_CODE); 662 if (inti) {
609 rc |= put_guest_lc(vcpu, PFAULT_DONE, (u16 *)__LC_EXT_CPU_ADDR); 663 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
610 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 664 (u16 *)__LC_EXT_INT_CODE);
611 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 665 rc |= put_guest_lc(vcpu, PFAULT_DONE,
612 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 666 (u16 *)__LC_EXT_CPU_ADDR);
613 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 667 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
614 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 668 &vcpu->arch.sie_block->gpsw,
615 (u64 *)__LC_EXT_PARAMS2); 669 sizeof(psw_t));
670 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
671 &vcpu->arch.sie_block->gpsw,
672 sizeof(psw_t));
673 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
674 (u64 *)__LC_EXT_PARAMS2);
675 kfree(inti);
676 }
616 return rc ? -EFAULT : 0; 677 return rc ? -EFAULT : 0;
617} 678}
618 679
619static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu, 680static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
620 struct kvm_s390_interrupt_info *inti)
621{ 681{
622 int rc; 682 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
683 struct kvm_s390_interrupt_info *inti;
684 int rc = 0;
623 685
624 VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx", 686 spin_lock(&fi->lock);
625 inti->ext.ext_params, inti->ext.ext_params2); 687 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
626 vcpu->stat.deliver_virtio_interrupt++; 688 struct kvm_s390_interrupt_info,
627 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, 689 list);
628 inti->ext.ext_params, 690 if (inti) {
629 inti->ext.ext_params2); 691 VCPU_EVENT(vcpu, 4,
692 "interrupt: virtio parm:%x,parm64:%llx",
693 inti->ext.ext_params, inti->ext.ext_params2);
694 vcpu->stat.deliver_virtio_interrupt++;
695 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
696 inti->type,
697 inti->ext.ext_params,
698 inti->ext.ext_params2);
699 list_del(&inti->list);
700 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
701 }
702 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
703 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
704 spin_unlock(&fi->lock);
630 705
631 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *)__LC_EXT_INT_CODE); 706 if (inti) {
632 rc |= put_guest_lc(vcpu, VIRTIO_PARAM, (u16 *)__LC_EXT_CPU_ADDR); 707 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
633 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 708 (u16 *)__LC_EXT_INT_CODE);
634 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 709 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
635 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 710 (u16 *)__LC_EXT_CPU_ADDR);
636 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 711 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
637 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 712 &vcpu->arch.sie_block->gpsw,
638 (u32 *)__LC_EXT_PARAMS); 713 sizeof(psw_t));
639 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 714 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
640 (u64 *)__LC_EXT_PARAMS2); 715 &vcpu->arch.sie_block->gpsw,
716 sizeof(psw_t));
717 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
718 (u32 *)__LC_EXT_PARAMS);
719 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
720 (u64 *)__LC_EXT_PARAMS2);
721 kfree(inti);
722 }
641 return rc ? -EFAULT : 0; 723 return rc ? -EFAULT : 0;
642} 724}
643 725
644static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 726static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
645 struct kvm_s390_interrupt_info *inti) 727 unsigned long irq_type)
646{ 728{
647 int rc; 729 struct list_head *isc_list;
730 struct kvm_s390_float_interrupt *fi;
731 struct kvm_s390_interrupt_info *inti = NULL;
732 int rc = 0;
648 733
649 VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type); 734 fi = &vcpu->kvm->arch.float_int;
650 vcpu->stat.deliver_io_int++;
651 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
652 ((__u32)inti->io.subchannel_id << 16) |
653 inti->io.subchannel_nr,
654 ((__u64)inti->io.io_int_parm << 32) |
655 inti->io.io_int_word);
656
657 rc = put_guest_lc(vcpu, inti->io.subchannel_id,
658 (u16 *)__LC_SUBCHANNEL_ID);
659 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
660 (u16 *)__LC_SUBCHANNEL_NR);
661 rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
662 (u32 *)__LC_IO_INT_PARM);
663 rc |= put_guest_lc(vcpu, inti->io.io_int_word,
664 (u32 *)__LC_IO_INT_WORD);
665 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
666 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
667 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
668 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
669 return rc ? -EFAULT : 0;
670}
671 735
672static int __must_check __deliver_mchk_floating(struct kvm_vcpu *vcpu, 736 spin_lock(&fi->lock);
673 struct kvm_s390_interrupt_info *inti) 737 isc_list = &fi->lists[irq_type - IRQ_PEND_IO_ISC_0];
674{ 738 inti = list_first_entry_or_null(isc_list,
675 struct kvm_s390_mchk_info *mchk = &inti->mchk; 739 struct kvm_s390_interrupt_info,
676 int rc; 740 list);
741 if (inti) {
742 VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type);
743 vcpu->stat.deliver_io_int++;
744 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
745 inti->type,
746 ((__u32)inti->io.subchannel_id << 16) |
747 inti->io.subchannel_nr,
748 ((__u64)inti->io.io_int_parm << 32) |
749 inti->io.io_int_word);
750 list_del(&inti->list);
751 fi->counters[FIRQ_CNTR_IO] -= 1;
752 }
753 if (list_empty(isc_list))
754 clear_bit(irq_type, &fi->pending_irqs);
755 spin_unlock(&fi->lock);
756
757 if (inti) {
758 rc = put_guest_lc(vcpu, inti->io.subchannel_id,
759 (u16 *)__LC_SUBCHANNEL_ID);
760 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
761 (u16 *)__LC_SUBCHANNEL_NR);
762 rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
763 (u32 *)__LC_IO_INT_PARM);
764 rc |= put_guest_lc(vcpu, inti->io.io_int_word,
765 (u32 *)__LC_IO_INT_WORD);
766 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
767 &vcpu->arch.sie_block->gpsw,
768 sizeof(psw_t));
769 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
770 &vcpu->arch.sie_block->gpsw,
771 sizeof(psw_t));
772 kfree(inti);
773 }
677 774
678 VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
679 mchk->mcic);
680 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_MCHK,
681 mchk->cr14, mchk->mcic);
682
683 rc = kvm_s390_vcpu_store_status(vcpu, KVM_S390_STORE_STATUS_PREFIXED);
684 rc |= put_guest_lc(vcpu, mchk->mcic,
685 (u64 __user *) __LC_MCCK_CODE);
686 rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
687 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
688 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA,
689 &mchk->fixed_logout, sizeof(mchk->fixed_logout));
690 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
691 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
692 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
693 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
694 return rc ? -EFAULT : 0; 775 return rc ? -EFAULT : 0;
695} 776}
696 777
@@ -698,6 +779,7 @@ typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu);
698 779
699static const deliver_irq_t deliver_irq_funcs[] = { 780static const deliver_irq_t deliver_irq_funcs[] = {
700 [IRQ_PEND_MCHK_EX] = __deliver_machine_check, 781 [IRQ_PEND_MCHK_EX] = __deliver_machine_check,
782 [IRQ_PEND_MCHK_REP] = __deliver_machine_check,
701 [IRQ_PEND_PROG] = __deliver_prog, 783 [IRQ_PEND_PROG] = __deliver_prog,
702 [IRQ_PEND_EXT_EMERGENCY] = __deliver_emergency_signal, 784 [IRQ_PEND_EXT_EMERGENCY] = __deliver_emergency_signal,
703 [IRQ_PEND_EXT_EXTERNAL] = __deliver_external_call, 785 [IRQ_PEND_EXT_EXTERNAL] = __deliver_external_call,
@@ -706,36 +788,11 @@ static const deliver_irq_t deliver_irq_funcs[] = {
706 [IRQ_PEND_RESTART] = __deliver_restart, 788 [IRQ_PEND_RESTART] = __deliver_restart,
707 [IRQ_PEND_SET_PREFIX] = __deliver_set_prefix, 789 [IRQ_PEND_SET_PREFIX] = __deliver_set_prefix,
708 [IRQ_PEND_PFAULT_INIT] = __deliver_pfault_init, 790 [IRQ_PEND_PFAULT_INIT] = __deliver_pfault_init,
791 [IRQ_PEND_EXT_SERVICE] = __deliver_service,
792 [IRQ_PEND_PFAULT_DONE] = __deliver_pfault_done,
793 [IRQ_PEND_VIRTIO] = __deliver_virtio,
709}; 794};
710 795
711static int __must_check __deliver_floating_interrupt(struct kvm_vcpu *vcpu,
712 struct kvm_s390_interrupt_info *inti)
713{
714 int rc;
715
716 switch (inti->type) {
717 case KVM_S390_INT_SERVICE:
718 rc = __deliver_service(vcpu, inti);
719 break;
720 case KVM_S390_INT_PFAULT_DONE:
721 rc = __deliver_pfault_done(vcpu, inti);
722 break;
723 case KVM_S390_INT_VIRTIO:
724 rc = __deliver_virtio(vcpu, inti);
725 break;
726 case KVM_S390_MCHK:
727 rc = __deliver_mchk_floating(vcpu, inti);
728 break;
729 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
730 rc = __deliver_io(vcpu, inti);
731 break;
732 default:
733 BUG();
734 }
735
736 return rc;
737}
738
739/* Check whether an external call is pending (deliverable or not) */ 796/* Check whether an external call is pending (deliverable or not) */
740int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu) 797int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
741{ 798{
@@ -751,21 +808,9 @@ int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
751 808
752int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop) 809int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
753{ 810{
754 struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
755 struct kvm_s390_interrupt_info *inti;
756 int rc; 811 int rc;
757 812
758 rc = !!deliverable_local_irqs(vcpu); 813 rc = !!deliverable_irqs(vcpu);
759
760 if ((!rc) && atomic_read(&fi->active)) {
761 spin_lock(&fi->lock);
762 list_for_each_entry(inti, &fi->list, list)
763 if (__interrupt_is_deliverable(vcpu, inti)) {
764 rc = 1;
765 break;
766 }
767 spin_unlock(&fi->lock);
768 }
769 814
770 if (!rc && kvm_cpu_has_pending_timer(vcpu)) 815 if (!rc && kvm_cpu_has_pending_timer(vcpu))
771 rc = 1; 816 rc = 1;
@@ -784,12 +829,7 @@ int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
784 829
785int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 830int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
786{ 831{
787 if (!(vcpu->arch.sie_block->ckc < 832 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
788 get_tod_clock_fast() + vcpu->arch.sie_block->epoch))
789 return 0;
790 if (!ckc_interrupts_enabled(vcpu))
791 return 0;
792 return 1;
793} 833}
794 834
795int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) 835int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
@@ -884,60 +924,45 @@ void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
884int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) 924int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
885{ 925{
886 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 926 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
887 struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
888 struct kvm_s390_interrupt_info *n, *inti = NULL;
889 deliver_irq_t func; 927 deliver_irq_t func;
890 int deliver;
891 int rc = 0; 928 int rc = 0;
892 unsigned long irq_type; 929 unsigned long irq_type;
893 unsigned long deliverable_irqs; 930 unsigned long irqs;
894 931
895 __reset_intercept_indicators(vcpu); 932 __reset_intercept_indicators(vcpu);
896 933
897 /* pending ckc conditions might have been invalidated */ 934 /* pending ckc conditions might have been invalidated */
898 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 935 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
899 if (kvm_cpu_has_pending_timer(vcpu)) 936 if (ckc_irq_pending(vcpu))
900 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 937 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
901 938
939 /* pending cpu timer conditions might have been invalidated */
940 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
941 if (cpu_timer_irq_pending(vcpu))
942 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
943
902 do { 944 do {
903 deliverable_irqs = deliverable_local_irqs(vcpu); 945 irqs = deliverable_irqs(vcpu);
904 /* bits are in the order of interrupt priority */ 946 /* bits are in the order of interrupt priority */
905 irq_type = find_first_bit(&deliverable_irqs, IRQ_PEND_COUNT); 947 irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT);
906 if (irq_type == IRQ_PEND_COUNT) 948 if (irq_type == IRQ_PEND_COUNT)
907 break; 949 break;
908 func = deliver_irq_funcs[irq_type]; 950 if (is_ioirq(irq_type)) {
909 if (!func) { 951 rc = __deliver_io(vcpu, irq_type);
910 WARN_ON_ONCE(func == NULL); 952 } else {
911 clear_bit(irq_type, &li->pending_irqs); 953 func = deliver_irq_funcs[irq_type];
912 continue; 954 if (!func) {
955 WARN_ON_ONCE(func == NULL);
956 clear_bit(irq_type, &li->pending_irqs);
957 continue;
958 }
959 rc = func(vcpu);
913 } 960 }
914 rc = func(vcpu); 961 if (rc)
915 } while (!rc && irq_type != IRQ_PEND_COUNT); 962 break;
963 } while (!rc);
916 964
917 set_intercept_indicators_local(vcpu); 965 set_intercept_indicators(vcpu);
918
919 if (!rc && atomic_read(&fi->active)) {
920 do {
921 deliver = 0;
922 spin_lock(&fi->lock);
923 list_for_each_entry_safe(inti, n, &fi->list, list) {
924 if (__interrupt_is_deliverable(vcpu, inti)) {
925 list_del(&inti->list);
926 fi->irq_count--;
927 deliver = 1;
928 break;
929 }
930 __set_intercept_indicator(vcpu, inti);
931 }
932 if (list_empty(&fi->list))
933 atomic_set(&fi->active, 0);
934 spin_unlock(&fi->lock);
935 if (deliver) {
936 rc = __deliver_floating_interrupt(vcpu, inti);
937 kfree(inti);
938 }
939 } while (!rc && deliver);
940 }
941 966
942 return rc; 967 return rc;
943} 968}
@@ -1172,80 +1197,182 @@ static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1172 return 0; 1197 return 0;
1173} 1198}
1174 1199
1200static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1201 int isc, u32 schid)
1202{
1203 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1204 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1205 struct kvm_s390_interrupt_info *iter;
1206 u16 id = (schid & 0xffff0000U) >> 16;
1207 u16 nr = schid & 0x0000ffffU;
1175 1208
1209 spin_lock(&fi->lock);
1210 list_for_each_entry(iter, isc_list, list) {
1211 if (schid && (id != iter->io.subchannel_id ||
1212 nr != iter->io.subchannel_nr))
1213 continue;
1214 /* found an appropriate entry */
1215 list_del_init(&iter->list);
1216 fi->counters[FIRQ_CNTR_IO] -= 1;
1217 if (list_empty(isc_list))
1218 clear_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs);
1219 spin_unlock(&fi->lock);
1220 return iter;
1221 }
1222 spin_unlock(&fi->lock);
1223 return NULL;
1224}
1225
1226/*
1227 * Dequeue and return an I/O interrupt matching any of the interruption
1228 * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1229 */
1176struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1230struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1177 u64 cr6, u64 schid) 1231 u64 isc_mask, u32 schid)
1232{
1233 struct kvm_s390_interrupt_info *inti = NULL;
1234 int isc;
1235
1236 for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1237 if (isc_mask & isc_to_isc_bits(isc))
1238 inti = get_io_int(kvm, isc, schid);
1239 }
1240 return inti;
1241}
1242
1243#define SCCB_MASK 0xFFFFFFF8
1244#define SCCB_EVENT_PENDING 0x3
1245
1246static int __inject_service(struct kvm *kvm,
1247 struct kvm_s390_interrupt_info *inti)
1248{
1249 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1250
1251 spin_lock(&fi->lock);
1252 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1253 /*
1254 * Early versions of the QEMU s390 bios will inject several
1255 * service interrupts after another without handling a
1256 * condition code indicating busy.
1257 * We will silently ignore those superfluous sccb values.
1258 * A future version of QEMU will take care of serialization
1259 * of servc requests
1260 */
1261 if (fi->srv_signal.ext_params & SCCB_MASK)
1262 goto out;
1263 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1264 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1265out:
1266 spin_unlock(&fi->lock);
1267 kfree(inti);
1268 return 0;
1269}
1270
1271static int __inject_virtio(struct kvm *kvm,
1272 struct kvm_s390_interrupt_info *inti)
1273{
1274 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1275
1276 spin_lock(&fi->lock);
1277 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1278 spin_unlock(&fi->lock);
1279 return -EBUSY;
1280 }
1281 fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1282 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1283 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1284 spin_unlock(&fi->lock);
1285 return 0;
1286}
1287
1288static int __inject_pfault_done(struct kvm *kvm,
1289 struct kvm_s390_interrupt_info *inti)
1290{
1291 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1292
1293 spin_lock(&fi->lock);
1294 if (fi->counters[FIRQ_CNTR_PFAULT] >=
1295 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1296 spin_unlock(&fi->lock);
1297 return -EBUSY;
1298 }
1299 fi->counters[FIRQ_CNTR_PFAULT] += 1;
1300 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1301 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1302 spin_unlock(&fi->lock);
1303 return 0;
1304}
1305
1306#define CR_PENDING_SUBCLASS 28
1307static int __inject_float_mchk(struct kvm *kvm,
1308 struct kvm_s390_interrupt_info *inti)
1309{
1310 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1311
1312 spin_lock(&fi->lock);
1313 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1314 fi->mchk.mcic |= inti->mchk.mcic;
1315 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1316 spin_unlock(&fi->lock);
1317 kfree(inti);
1318 return 0;
1319}
1320
1321static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1178{ 1322{
1179 struct kvm_s390_float_interrupt *fi; 1323 struct kvm_s390_float_interrupt *fi;
1180 struct kvm_s390_interrupt_info *inti, *iter; 1324 struct list_head *list;
1325 int isc;
1181 1326
1182 if ((!schid && !cr6) || (schid && cr6))
1183 return NULL;
1184 fi = &kvm->arch.float_int; 1327 fi = &kvm->arch.float_int;
1185 spin_lock(&fi->lock); 1328 spin_lock(&fi->lock);
1186 inti = NULL; 1329 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1187 list_for_each_entry(iter, &fi->list, list) { 1330 spin_unlock(&fi->lock);
1188 if (!is_ioint(iter->type)) 1331 return -EBUSY;
1189 continue;
1190 if (cr6 &&
1191 ((cr6 & int_word_to_isc_bits(iter->io.io_int_word)) == 0))
1192 continue;
1193 if (schid) {
1194 if (((schid & 0x00000000ffff0000) >> 16) !=
1195 iter->io.subchannel_id)
1196 continue;
1197 if ((schid & 0x000000000000ffff) !=
1198 iter->io.subchannel_nr)
1199 continue;
1200 }
1201 inti = iter;
1202 break;
1203 }
1204 if (inti) {
1205 list_del_init(&inti->list);
1206 fi->irq_count--;
1207 } 1332 }
1208 if (list_empty(&fi->list)) 1333 fi->counters[FIRQ_CNTR_IO] += 1;
1209 atomic_set(&fi->active, 0); 1334
1335 isc = int_word_to_isc(inti->io.io_int_word);
1336 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1337 list_add_tail(&inti->list, list);
1338 set_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs);
1210 spin_unlock(&fi->lock); 1339 spin_unlock(&fi->lock);
1211 return inti; 1340 return 0;
1212} 1341}
1213 1342
1214static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1343static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1215{ 1344{
1216 struct kvm_s390_local_interrupt *li; 1345 struct kvm_s390_local_interrupt *li;
1217 struct kvm_s390_float_interrupt *fi; 1346 struct kvm_s390_float_interrupt *fi;
1218 struct kvm_s390_interrupt_info *iter;
1219 struct kvm_vcpu *dst_vcpu = NULL; 1347 struct kvm_vcpu *dst_vcpu = NULL;
1220 int sigcpu; 1348 int sigcpu;
1221 int rc = 0; 1349 u64 type = READ_ONCE(inti->type);
1350 int rc;
1222 1351
1223 fi = &kvm->arch.float_int; 1352 fi = &kvm->arch.float_int;
1224 spin_lock(&fi->lock); 1353
1225 if (fi->irq_count >= KVM_S390_MAX_FLOAT_IRQS) { 1354 switch (type) {
1355 case KVM_S390_MCHK:
1356 rc = __inject_float_mchk(kvm, inti);
1357 break;
1358 case KVM_S390_INT_VIRTIO:
1359 rc = __inject_virtio(kvm, inti);
1360 break;
1361 case KVM_S390_INT_SERVICE:
1362 rc = __inject_service(kvm, inti);
1363 break;
1364 case KVM_S390_INT_PFAULT_DONE:
1365 rc = __inject_pfault_done(kvm, inti);
1366 break;
1367 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1368 rc = __inject_io(kvm, inti);
1369 break;
1370 default:
1226 rc = -EINVAL; 1371 rc = -EINVAL;
1227 goto unlock_fi;
1228 } 1372 }
1229 fi->irq_count++; 1373 if (rc)
1230 if (!is_ioint(inti->type)) { 1374 return rc;
1231 list_add_tail(&inti->list, &fi->list);
1232 } else {
1233 u64 isc_bits = int_word_to_isc_bits(inti->io.io_int_word);
1234 1375
1235 /* Keep I/O interrupts sorted in isc order. */
1236 list_for_each_entry(iter, &fi->list, list) {
1237 if (!is_ioint(iter->type))
1238 continue;
1239 if (int_word_to_isc_bits(iter->io.io_int_word)
1240 <= isc_bits)
1241 continue;
1242 break;
1243 }
1244 list_add_tail(&inti->list, &iter->list);
1245 }
1246 atomic_set(&fi->active, 1);
1247 if (atomic_read(&kvm->online_vcpus) == 0)
1248 goto unlock_fi;
1249 sigcpu = find_first_bit(fi->idle_mask, KVM_MAX_VCPUS); 1376 sigcpu = find_first_bit(fi->idle_mask, KVM_MAX_VCPUS);
1250 if (sigcpu == KVM_MAX_VCPUS) { 1377 if (sigcpu == KVM_MAX_VCPUS) {
1251 do { 1378 do {
@@ -1257,7 +1384,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1257 dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1384 dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1258 li = &dst_vcpu->arch.local_int; 1385 li = &dst_vcpu->arch.local_int;
1259 spin_lock(&li->lock); 1386 spin_lock(&li->lock);
1260 switch (inti->type) { 1387 switch (type) {
1261 case KVM_S390_MCHK: 1388 case KVM_S390_MCHK:
1262 atomic_set_mask(CPUSTAT_STOP_INT, li->cpuflags); 1389 atomic_set_mask(CPUSTAT_STOP_INT, li->cpuflags);
1263 break; 1390 break;
@@ -1270,9 +1397,8 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1270 } 1397 }
1271 spin_unlock(&li->lock); 1398 spin_unlock(&li->lock);
1272 kvm_s390_vcpu_wakeup(kvm_get_vcpu(kvm, sigcpu)); 1399 kvm_s390_vcpu_wakeup(kvm_get_vcpu(kvm, sigcpu));
1273unlock_fi: 1400 return 0;
1274 spin_unlock(&fi->lock); 1401
1275 return rc;
1276} 1402}
1277 1403
1278int kvm_s390_inject_vm(struct kvm *kvm, 1404int kvm_s390_inject_vm(struct kvm *kvm,
@@ -1332,10 +1458,10 @@ int kvm_s390_inject_vm(struct kvm *kvm,
1332 return rc; 1458 return rc;
1333} 1459}
1334 1460
1335void kvm_s390_reinject_io_int(struct kvm *kvm, 1461int kvm_s390_reinject_io_int(struct kvm *kvm,
1336 struct kvm_s390_interrupt_info *inti) 1462 struct kvm_s390_interrupt_info *inti)
1337{ 1463{
1338 __inject_vm(kvm, inti); 1464 return __inject_vm(kvm, inti);
1339} 1465}
1340 1466
1341int s390int_to_s390irq(struct kvm_s390_interrupt *s390int, 1467int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
@@ -1388,12 +1514,10 @@ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
1388 spin_unlock(&li->lock); 1514 spin_unlock(&li->lock);
1389} 1515}
1390 1516
1391int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1517static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1392{ 1518{
1393 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1394 int rc; 1519 int rc;
1395 1520
1396 spin_lock(&li->lock);
1397 switch (irq->type) { 1521 switch (irq->type) {
1398 case KVM_S390_PROGRAM_INT: 1522 case KVM_S390_PROGRAM_INT:
1399 VCPU_EVENT(vcpu, 3, "inject: program check %d (from user)", 1523 VCPU_EVENT(vcpu, 3, "inject: program check %d (from user)",
@@ -1433,83 +1557,130 @@ int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1433 default: 1557 default:
1434 rc = -EINVAL; 1558 rc = -EINVAL;
1435 } 1559 }
1560
1561 return rc;
1562}
1563
1564int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1565{
1566 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1567 int rc;
1568
1569 spin_lock(&li->lock);
1570 rc = do_inject_vcpu(vcpu, irq);
1436 spin_unlock(&li->lock); 1571 spin_unlock(&li->lock);
1437 if (!rc) 1572 if (!rc)
1438 kvm_s390_vcpu_wakeup(vcpu); 1573 kvm_s390_vcpu_wakeup(vcpu);
1439 return rc; 1574 return rc;
1440} 1575}
1441 1576
1442void kvm_s390_clear_float_irqs(struct kvm *kvm) 1577static inline void clear_irq_list(struct list_head *_list)
1443{ 1578{
1444 struct kvm_s390_float_interrupt *fi; 1579 struct kvm_s390_interrupt_info *inti, *n;
1445 struct kvm_s390_interrupt_info *n, *inti = NULL;
1446 1580
1447 fi = &kvm->arch.float_int; 1581 list_for_each_entry_safe(inti, n, _list, list) {
1448 spin_lock(&fi->lock);
1449 list_for_each_entry_safe(inti, n, &fi->list, list) {
1450 list_del(&inti->list); 1582 list_del(&inti->list);
1451 kfree(inti); 1583 kfree(inti);
1452 } 1584 }
1453 fi->irq_count = 0;
1454 atomic_set(&fi->active, 0);
1455 spin_unlock(&fi->lock);
1456} 1585}
1457 1586
1458static inline int copy_irq_to_user(struct kvm_s390_interrupt_info *inti, 1587static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
1459 u8 *addr) 1588 struct kvm_s390_irq *irq)
1460{ 1589{
1461 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr; 1590 irq->type = inti->type;
1462 struct kvm_s390_irq irq = {0};
1463
1464 irq.type = inti->type;
1465 switch (inti->type) { 1591 switch (inti->type) {
1466 case KVM_S390_INT_PFAULT_INIT: 1592 case KVM_S390_INT_PFAULT_INIT:
1467 case KVM_S390_INT_PFAULT_DONE: 1593 case KVM_S390_INT_PFAULT_DONE:
1468 case KVM_S390_INT_VIRTIO: 1594 case KVM_S390_INT_VIRTIO:
1469 case KVM_S390_INT_SERVICE: 1595 irq->u.ext = inti->ext;
1470 irq.u.ext = inti->ext;
1471 break; 1596 break;
1472 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1597 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1473 irq.u.io = inti->io; 1598 irq->u.io = inti->io;
1474 break; 1599 break;
1475 case KVM_S390_MCHK:
1476 irq.u.mchk = inti->mchk;
1477 break;
1478 default:
1479 return -EINVAL;
1480 } 1600 }
1601}
1481 1602
1482 if (copy_to_user(uptr, &irq, sizeof(irq))) 1603void kvm_s390_clear_float_irqs(struct kvm *kvm)
1483 return -EFAULT; 1604{
1605 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1606 int i;
1484 1607
1485 return 0; 1608 spin_lock(&fi->lock);
1486} 1609 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1610 clear_irq_list(&fi->lists[i]);
1611 for (i = 0; i < FIRQ_MAX_COUNT; i++)
1612 fi->counters[i] = 0;
1613 spin_unlock(&fi->lock);
1614};
1487 1615
1488static int get_all_floating_irqs(struct kvm *kvm, __u8 *buf, __u64 len) 1616static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
1489{ 1617{
1490 struct kvm_s390_interrupt_info *inti; 1618 struct kvm_s390_interrupt_info *inti;
1491 struct kvm_s390_float_interrupt *fi; 1619 struct kvm_s390_float_interrupt *fi;
1620 struct kvm_s390_irq *buf;
1621 struct kvm_s390_irq *irq;
1622 int max_irqs;
1492 int ret = 0; 1623 int ret = 0;
1493 int n = 0; 1624 int n = 0;
1625 int i;
1626
1627 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
1628 return -EINVAL;
1629
1630 /*
1631 * We are already using -ENOMEM to signal
1632 * userspace it may retry with a bigger buffer,
1633 * so we need to use something else for this case
1634 */
1635 buf = vzalloc(len);
1636 if (!buf)
1637 return -ENOBUFS;
1638
1639 max_irqs = len / sizeof(struct kvm_s390_irq);
1494 1640
1495 fi = &kvm->arch.float_int; 1641 fi = &kvm->arch.float_int;
1496 spin_lock(&fi->lock); 1642 spin_lock(&fi->lock);
1497 1643 for (i = 0; i < FIRQ_LIST_COUNT; i++) {
1498 list_for_each_entry(inti, &fi->list, list) { 1644 list_for_each_entry(inti, &fi->lists[i], list) {
1499 if (len < sizeof(struct kvm_s390_irq)) { 1645 if (n == max_irqs) {
1646 /* signal userspace to try again */
1647 ret = -ENOMEM;
1648 goto out;
1649 }
1650 inti_to_irq(inti, &buf[n]);
1651 n++;
1652 }
1653 }
1654 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) {
1655 if (n == max_irqs) {
1500 /* signal userspace to try again */ 1656 /* signal userspace to try again */
1501 ret = -ENOMEM; 1657 ret = -ENOMEM;
1502 break; 1658 goto out;
1503 } 1659 }
1504 ret = copy_irq_to_user(inti, buf); 1660 irq = (struct kvm_s390_irq *) &buf[n];
1505 if (ret) 1661 irq->type = KVM_S390_INT_SERVICE;
1506 break; 1662 irq->u.ext = fi->srv_signal;
1507 buf += sizeof(struct kvm_s390_irq);
1508 len -= sizeof(struct kvm_s390_irq);
1509 n++; 1663 n++;
1510 } 1664 }
1665 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
1666 if (n == max_irqs) {
1667 /* signal userspace to try again */
1668 ret = -ENOMEM;
1669 goto out;
1670 }
1671 irq = (struct kvm_s390_irq *) &buf[n];
1672 irq->type = KVM_S390_MCHK;
1673 irq->u.mchk = fi->mchk;
1674 n++;
1675}
1511 1676
1677out:
1512 spin_unlock(&fi->lock); 1678 spin_unlock(&fi->lock);
1679 if (!ret && n > 0) {
1680 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
1681 ret = -EFAULT;
1682 }
1683 vfree(buf);
1513 1684
1514 return ret < 0 ? ret : n; 1685 return ret < 0 ? ret : n;
1515} 1686}
@@ -1520,7 +1691,7 @@ static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1520 1691
1521 switch (attr->group) { 1692 switch (attr->group) {
1522 case KVM_DEV_FLIC_GET_ALL_IRQS: 1693 case KVM_DEV_FLIC_GET_ALL_IRQS:
1523 r = get_all_floating_irqs(dev->kvm, (u8 *) attr->addr, 1694 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
1524 attr->attr); 1695 attr->attr);
1525 break; 1696 break;
1526 default: 1697 default:
@@ -1952,3 +2123,143 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
1952{ 2123{
1953 return -EINVAL; 2124 return -EINVAL;
1954} 2125}
2126
2127int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2128{
2129 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2130 struct kvm_s390_irq *buf;
2131 int r = 0;
2132 int n;
2133
2134 buf = vmalloc(len);
2135 if (!buf)
2136 return -ENOMEM;
2137
2138 if (copy_from_user((void *) buf, irqstate, len)) {
2139 r = -EFAULT;
2140 goto out_free;
2141 }
2142
2143 /*
2144 * Don't allow setting the interrupt state
2145 * when there are already interrupts pending
2146 */
2147 spin_lock(&li->lock);
2148 if (li->pending_irqs) {
2149 r = -EBUSY;
2150 goto out_unlock;
2151 }
2152
2153 for (n = 0; n < len / sizeof(*buf); n++) {
2154 r = do_inject_vcpu(vcpu, &buf[n]);
2155 if (r)
2156 break;
2157 }
2158
2159out_unlock:
2160 spin_unlock(&li->lock);
2161out_free:
2162 vfree(buf);
2163
2164 return r;
2165}
2166
2167static void store_local_irq(struct kvm_s390_local_interrupt *li,
2168 struct kvm_s390_irq *irq,
2169 unsigned long irq_type)
2170{
2171 switch (irq_type) {
2172 case IRQ_PEND_MCHK_EX:
2173 case IRQ_PEND_MCHK_REP:
2174 irq->type = KVM_S390_MCHK;
2175 irq->u.mchk = li->irq.mchk;
2176 break;
2177 case IRQ_PEND_PROG:
2178 irq->type = KVM_S390_PROGRAM_INT;
2179 irq->u.pgm = li->irq.pgm;
2180 break;
2181 case IRQ_PEND_PFAULT_INIT:
2182 irq->type = KVM_S390_INT_PFAULT_INIT;
2183 irq->u.ext = li->irq.ext;
2184 break;
2185 case IRQ_PEND_EXT_EXTERNAL:
2186 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2187 irq->u.extcall = li->irq.extcall;
2188 break;
2189 case IRQ_PEND_EXT_CLOCK_COMP:
2190 irq->type = KVM_S390_INT_CLOCK_COMP;
2191 break;
2192 case IRQ_PEND_EXT_CPU_TIMER:
2193 irq->type = KVM_S390_INT_CPU_TIMER;
2194 break;
2195 case IRQ_PEND_SIGP_STOP:
2196 irq->type = KVM_S390_SIGP_STOP;
2197 irq->u.stop = li->irq.stop;
2198 break;
2199 case IRQ_PEND_RESTART:
2200 irq->type = KVM_S390_RESTART;
2201 break;
2202 case IRQ_PEND_SET_PREFIX:
2203 irq->type = KVM_S390_SIGP_SET_PREFIX;
2204 irq->u.prefix = li->irq.prefix;
2205 break;
2206 }
2207}
2208
2209int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
2210{
2211 uint8_t sigp_ctrl = vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl;
2212 unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)];
2213 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2214 unsigned long pending_irqs;
2215 struct kvm_s390_irq irq;
2216 unsigned long irq_type;
2217 int cpuaddr;
2218 int n = 0;
2219
2220 spin_lock(&li->lock);
2221 pending_irqs = li->pending_irqs;
2222 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
2223 sizeof(sigp_emerg_pending));
2224 spin_unlock(&li->lock);
2225
2226 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
2227 memset(&irq, 0, sizeof(irq));
2228 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
2229 continue;
2230 if (n + sizeof(irq) > len)
2231 return -ENOBUFS;
2232 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
2233 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2234 return -EFAULT;
2235 n += sizeof(irq);
2236 }
2237
2238 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
2239 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
2240 memset(&irq, 0, sizeof(irq));
2241 if (n + sizeof(irq) > len)
2242 return -ENOBUFS;
2243 irq.type = KVM_S390_INT_EMERGENCY;
2244 irq.u.emerg.code = cpuaddr;
2245 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2246 return -EFAULT;
2247 n += sizeof(irq);
2248 }
2249 }
2250
2251 if ((sigp_ctrl & SIGP_CTRL_C) &&
2252 (atomic_read(&vcpu->arch.sie_block->cpuflags) &
2253 CPUSTAT_ECALL_PEND)) {
2254 if (n + sizeof(irq) > len)
2255 return -ENOBUFS;
2256 memset(&irq, 0, sizeof(irq));
2257 irq.type = KVM_S390_INT_EXTERNAL_CALL;
2258 irq.u.extcall.code = sigp_ctrl & SIGP_CTRL_SCN_MASK;
2259 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2260 return -EFAULT;
2261 n += sizeof(irq);
2262 }
2263
2264 return n;
2265}
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 19e17bd7aec0..afa2bd750ffc 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -25,11 +25,13 @@
25#include <linux/random.h> 25#include <linux/random.h>
26#include <linux/slab.h> 26#include <linux/slab.h>
27#include <linux/timer.h> 27#include <linux/timer.h>
28#include <linux/vmalloc.h>
28#include <asm/asm-offsets.h> 29#include <asm/asm-offsets.h>
29#include <asm/lowcore.h> 30#include <asm/lowcore.h>
30#include <asm/pgtable.h> 31#include <asm/pgtable.h>
31#include <asm/nmi.h> 32#include <asm/nmi.h>
32#include <asm/switch_to.h> 33#include <asm/switch_to.h>
34#include <asm/isc.h>
33#include <asm/sclp.h> 35#include <asm/sclp.h>
34#include "kvm-s390.h" 36#include "kvm-s390.h"
35#include "gaccess.h" 37#include "gaccess.h"
@@ -38,6 +40,11 @@
38#include "trace.h" 40#include "trace.h"
39#include "trace-s390.h" 41#include "trace-s390.h"
40 42
43#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
44#define LOCAL_IRQS 32
45#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
46 (KVM_MAX_VCPUS + LOCAL_IRQS))
47
41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 48#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42 49
43struct kvm_stats_debugfs_item debugfs_entries[] = { 50struct kvm_stats_debugfs_item debugfs_entries[] = {
@@ -87,6 +94,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
87 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 94 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
88 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) }, 95 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
89 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) }, 96 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
97 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
90 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 98 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
91 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 99 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
92 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 100 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
@@ -101,8 +109,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
101 109
102/* upper facilities limit for kvm */ 110/* upper facilities limit for kvm */
103unsigned long kvm_s390_fac_list_mask[] = { 111unsigned long kvm_s390_fac_list_mask[] = {
104 0xff82fffbf4fc2000UL, 112 0xffe6fffbfcfdfc40UL,
105 0x005c000000000000UL, 113 0x205c800000000000UL,
106}; 114};
107 115
108unsigned long kvm_s390_fac_list_mask_size(void) 116unsigned long kvm_s390_fac_list_mask_size(void)
@@ -171,9 +179,16 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
171 case KVM_CAP_S390_IRQCHIP: 179 case KVM_CAP_S390_IRQCHIP:
172 case KVM_CAP_VM_ATTRIBUTES: 180 case KVM_CAP_VM_ATTRIBUTES:
173 case KVM_CAP_MP_STATE: 181 case KVM_CAP_MP_STATE:
182 case KVM_CAP_S390_INJECT_IRQ:
174 case KVM_CAP_S390_USER_SIGP: 183 case KVM_CAP_S390_USER_SIGP:
184 case KVM_CAP_S390_USER_STSI:
185 case KVM_CAP_S390_SKEYS:
186 case KVM_CAP_S390_IRQ_STATE:
175 r = 1; 187 r = 1;
176 break; 188 break;
189 case KVM_CAP_S390_MEM_OP:
190 r = MEM_OP_MAX_SIZE;
191 break;
177 case KVM_CAP_NR_VCPUS: 192 case KVM_CAP_NR_VCPUS:
178 case KVM_CAP_MAX_VCPUS: 193 case KVM_CAP_MAX_VCPUS:
179 r = KVM_MAX_VCPUS; 194 r = KVM_MAX_VCPUS;
@@ -184,6 +199,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
184 case KVM_CAP_S390_COW: 199 case KVM_CAP_S390_COW:
185 r = MACHINE_HAS_ESOP; 200 r = MACHINE_HAS_ESOP;
186 break; 201 break;
202 case KVM_CAP_S390_VECTOR_REGISTERS:
203 r = MACHINE_HAS_VX;
204 break;
187 default: 205 default:
188 r = 0; 206 r = 0;
189 } 207 }
@@ -264,6 +282,18 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
264 kvm->arch.user_sigp = 1; 282 kvm->arch.user_sigp = 1;
265 r = 0; 283 r = 0;
266 break; 284 break;
285 case KVM_CAP_S390_VECTOR_REGISTERS:
286 if (MACHINE_HAS_VX) {
287 set_kvm_facility(kvm->arch.model.fac->mask, 129);
288 set_kvm_facility(kvm->arch.model.fac->list, 129);
289 r = 0;
290 } else
291 r = -EINVAL;
292 break;
293 case KVM_CAP_S390_USER_STSI:
294 kvm->arch.user_stsi = 1;
295 r = 0;
296 break;
267 default: 297 default:
268 r = -EINVAL; 298 r = -EINVAL;
269 break; 299 break;
@@ -708,6 +738,108 @@ static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
708 return ret; 738 return ret;
709} 739}
710 740
741static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
742{
743 uint8_t *keys;
744 uint64_t hva;
745 unsigned long curkey;
746 int i, r = 0;
747
748 if (args->flags != 0)
749 return -EINVAL;
750
751 /* Is this guest using storage keys? */
752 if (!mm_use_skey(current->mm))
753 return KVM_S390_GET_SKEYS_NONE;
754
755 /* Enforce sane limit on memory allocation */
756 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
757 return -EINVAL;
758
759 keys = kmalloc_array(args->count, sizeof(uint8_t),
760 GFP_KERNEL | __GFP_NOWARN);
761 if (!keys)
762 keys = vmalloc(sizeof(uint8_t) * args->count);
763 if (!keys)
764 return -ENOMEM;
765
766 for (i = 0; i < args->count; i++) {
767 hva = gfn_to_hva(kvm, args->start_gfn + i);
768 if (kvm_is_error_hva(hva)) {
769 r = -EFAULT;
770 goto out;
771 }
772
773 curkey = get_guest_storage_key(current->mm, hva);
774 if (IS_ERR_VALUE(curkey)) {
775 r = curkey;
776 goto out;
777 }
778 keys[i] = curkey;
779 }
780
781 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
782 sizeof(uint8_t) * args->count);
783 if (r)
784 r = -EFAULT;
785out:
786 kvfree(keys);
787 return r;
788}
789
790static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
791{
792 uint8_t *keys;
793 uint64_t hva;
794 int i, r = 0;
795
796 if (args->flags != 0)
797 return -EINVAL;
798
799 /* Enforce sane limit on memory allocation */
800 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
801 return -EINVAL;
802
803 keys = kmalloc_array(args->count, sizeof(uint8_t),
804 GFP_KERNEL | __GFP_NOWARN);
805 if (!keys)
806 keys = vmalloc(sizeof(uint8_t) * args->count);
807 if (!keys)
808 return -ENOMEM;
809
810 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
811 sizeof(uint8_t) * args->count);
812 if (r) {
813 r = -EFAULT;
814 goto out;
815 }
816
817 /* Enable storage key handling for the guest */
818 s390_enable_skey();
819
820 for (i = 0; i < args->count; i++) {
821 hva = gfn_to_hva(kvm, args->start_gfn + i);
822 if (kvm_is_error_hva(hva)) {
823 r = -EFAULT;
824 goto out;
825 }
826
827 /* Lowest order bit is reserved */
828 if (keys[i] & 0x01) {
829 r = -EINVAL;
830 goto out;
831 }
832
833 r = set_guest_storage_key(current->mm, hva,
834 (unsigned long)keys[i], 0);
835 if (r)
836 goto out;
837 }
838out:
839 kvfree(keys);
840 return r;
841}
842
711long kvm_arch_vm_ioctl(struct file *filp, 843long kvm_arch_vm_ioctl(struct file *filp,
712 unsigned int ioctl, unsigned long arg) 844 unsigned int ioctl, unsigned long arg)
713{ 845{
@@ -767,6 +899,26 @@ long kvm_arch_vm_ioctl(struct file *filp,
767 r = kvm_s390_vm_has_attr(kvm, &attr); 899 r = kvm_s390_vm_has_attr(kvm, &attr);
768 break; 900 break;
769 } 901 }
902 case KVM_S390_GET_SKEYS: {
903 struct kvm_s390_skeys args;
904
905 r = -EFAULT;
906 if (copy_from_user(&args, argp,
907 sizeof(struct kvm_s390_skeys)))
908 break;
909 r = kvm_s390_get_skeys(kvm, &args);
910 break;
911 }
912 case KVM_S390_SET_SKEYS: {
913 struct kvm_s390_skeys args;
914
915 r = -EFAULT;
916 if (copy_from_user(&args, argp,
917 sizeof(struct kvm_s390_skeys)))
918 break;
919 r = kvm_s390_set_skeys(kvm, &args);
920 break;
921 }
770 default: 922 default:
771 r = -ENOTTY; 923 r = -ENOTTY;
772 } 924 }
@@ -887,7 +1039,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
887 1039
888 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long)); 1040 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
889 if (!kvm->arch.dbf) 1041 if (!kvm->arch.dbf)
890 goto out_nodbf; 1042 goto out_err;
891 1043
892 /* 1044 /*
893 * The architectural maximum amount of facilities is 16 kbit. To store 1045 * The architectural maximum amount of facilities is 16 kbit. To store
@@ -899,7 +1051,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
899 kvm->arch.model.fac = 1051 kvm->arch.model.fac =
900 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1052 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
901 if (!kvm->arch.model.fac) 1053 if (!kvm->arch.model.fac)
902 goto out_nofac; 1054 goto out_err;
903 1055
904 /* Populate the facility mask initially. */ 1056 /* Populate the facility mask initially. */
905 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list, 1057 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
@@ -919,10 +1071,11 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
919 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff; 1071 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
920 1072
921 if (kvm_s390_crypto_init(kvm) < 0) 1073 if (kvm_s390_crypto_init(kvm) < 0)
922 goto out_crypto; 1074 goto out_err;
923 1075
924 spin_lock_init(&kvm->arch.float_int.lock); 1076 spin_lock_init(&kvm->arch.float_int.lock);
925 INIT_LIST_HEAD(&kvm->arch.float_int.list); 1077 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1078 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
926 init_waitqueue_head(&kvm->arch.ipte_wq); 1079 init_waitqueue_head(&kvm->arch.ipte_wq);
927 mutex_init(&kvm->arch.ipte_mutex); 1080 mutex_init(&kvm->arch.ipte_mutex);
928 1081
@@ -934,7 +1087,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
934 } else { 1087 } else {
935 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1); 1088 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
936 if (!kvm->arch.gmap) 1089 if (!kvm->arch.gmap)
937 goto out_nogmap; 1090 goto out_err;
938 kvm->arch.gmap->private = kvm; 1091 kvm->arch.gmap->private = kvm;
939 kvm->arch.gmap->pfault_enabled = 0; 1092 kvm->arch.gmap->pfault_enabled = 0;
940 } 1093 }
@@ -946,15 +1099,11 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
946 spin_lock_init(&kvm->arch.start_stop_lock); 1099 spin_lock_init(&kvm->arch.start_stop_lock);
947 1100
948 return 0; 1101 return 0;
949out_nogmap: 1102out_err:
950 kfree(kvm->arch.crypto.crycb); 1103 kfree(kvm->arch.crypto.crycb);
951out_crypto:
952 free_page((unsigned long)kvm->arch.model.fac); 1104 free_page((unsigned long)kvm->arch.model.fac);
953out_nofac:
954 debug_unregister(kvm->arch.dbf); 1105 debug_unregister(kvm->arch.dbf);
955out_nodbf:
956 free_page((unsigned long)(kvm->arch.sca)); 1106 free_page((unsigned long)(kvm->arch.sca));
957out_err:
958 return rc; 1107 return rc;
959} 1108}
960 1109
@@ -1034,6 +1183,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1034 KVM_SYNC_CRS | 1183 KVM_SYNC_CRS |
1035 KVM_SYNC_ARCH0 | 1184 KVM_SYNC_ARCH0 |
1036 KVM_SYNC_PFAULT; 1185 KVM_SYNC_PFAULT;
1186 if (test_kvm_facility(vcpu->kvm, 129))
1187 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1037 1188
1038 if (kvm_is_ucontrol(vcpu->kvm)) 1189 if (kvm_is_ucontrol(vcpu->kvm))
1039 return __kvm_ucontrol_vcpu_init(vcpu); 1190 return __kvm_ucontrol_vcpu_init(vcpu);
@@ -1044,10 +1195,18 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1044void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1195void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1045{ 1196{
1046 save_fp_ctl(&vcpu->arch.host_fpregs.fpc); 1197 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1047 save_fp_regs(vcpu->arch.host_fpregs.fprs); 1198 if (test_kvm_facility(vcpu->kvm, 129))
1199 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1200 else
1201 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1048 save_access_regs(vcpu->arch.host_acrs); 1202 save_access_regs(vcpu->arch.host_acrs);
1049 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1203 if (test_kvm_facility(vcpu->kvm, 129)) {
1050 restore_fp_regs(vcpu->arch.guest_fpregs.fprs); 1204 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1205 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1206 } else {
1207 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1208 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1209 }
1051 restore_access_regs(vcpu->run->s.regs.acrs); 1210 restore_access_regs(vcpu->run->s.regs.acrs);
1052 gmap_enable(vcpu->arch.gmap); 1211 gmap_enable(vcpu->arch.gmap);
1053 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 1212 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
@@ -1057,11 +1216,19 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1057{ 1216{
1058 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 1217 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1059 gmap_disable(vcpu->arch.gmap); 1218 gmap_disable(vcpu->arch.gmap);
1060 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1219 if (test_kvm_facility(vcpu->kvm, 129)) {
1061 save_fp_regs(vcpu->arch.guest_fpregs.fprs); 1220 save_fp_ctl(&vcpu->run->s.regs.fpc);
1221 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1222 } else {
1223 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1224 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1225 }
1062 save_access_regs(vcpu->run->s.regs.acrs); 1226 save_access_regs(vcpu->run->s.regs.acrs);
1063 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc); 1227 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1064 restore_fp_regs(vcpu->arch.host_fpregs.fprs); 1228 if (test_kvm_facility(vcpu->kvm, 129))
1229 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1230 else
1231 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1065 restore_access_regs(vcpu->arch.host_acrs); 1232 restore_access_regs(vcpu->arch.host_acrs);
1066} 1233}
1067 1234
@@ -1129,6 +1296,15 @@ int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1129 return 0; 1296 return 0;
1130} 1297}
1131 1298
1299static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1300{
1301 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1302
1303 vcpu->arch.cpu_id = model->cpu_id;
1304 vcpu->arch.sie_block->ibc = model->ibc;
1305 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1306}
1307
1132int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 1308int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1133{ 1309{
1134 int rc = 0; 1310 int rc = 0;
@@ -1137,6 +1313,8 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1137 CPUSTAT_SM | 1313 CPUSTAT_SM |
1138 CPUSTAT_STOPPED | 1314 CPUSTAT_STOPPED |
1139 CPUSTAT_GED); 1315 CPUSTAT_GED);
1316 kvm_s390_vcpu_setup_model(vcpu);
1317
1140 vcpu->arch.sie_block->ecb = 6; 1318 vcpu->arch.sie_block->ecb = 6;
1141 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73)) 1319 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1142 vcpu->arch.sie_block->ecb |= 0x10; 1320 vcpu->arch.sie_block->ecb |= 0x10;
@@ -1147,8 +1325,11 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1147 vcpu->arch.sie_block->eca |= 1; 1325 vcpu->arch.sie_block->eca |= 1;
1148 if (sclp_has_sigpif()) 1326 if (sclp_has_sigpif())
1149 vcpu->arch.sie_block->eca |= 0x10000000U; 1327 vcpu->arch.sie_block->eca |= 0x10000000U;
1150 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE | 1328 if (test_kvm_facility(vcpu->kvm, 129)) {
1151 ICTL_TPROT; 1329 vcpu->arch.sie_block->eca |= 0x00020000;
1330 vcpu->arch.sie_block->ecd |= 0x20000000;
1331 }
1332 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1152 1333
1153 if (kvm_s390_cmma_enabled(vcpu->kvm)) { 1334 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1154 rc = kvm_s390_vcpu_setup_cmma(vcpu); 1335 rc = kvm_s390_vcpu_setup_cmma(vcpu);
@@ -1158,11 +1339,6 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1158 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1339 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1159 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 1340 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1160 1341
1161 mutex_lock(&vcpu->kvm->lock);
1162 vcpu->arch.cpu_id = vcpu->kvm->arch.model.cpu_id;
1163 vcpu->arch.sie_block->ibc = vcpu->kvm->arch.model.ibc;
1164 mutex_unlock(&vcpu->kvm->lock);
1165
1166 kvm_s390_vcpu_crypto_setup(vcpu); 1342 kvm_s390_vcpu_crypto_setup(vcpu);
1167 1343
1168 return rc; 1344 return rc;
@@ -1190,6 +1366,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1190 1366
1191 vcpu->arch.sie_block = &sie_page->sie_block; 1367 vcpu->arch.sie_block = &sie_page->sie_block;
1192 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb; 1368 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1369 vcpu->arch.host_vregs = &sie_page->vregs;
1193 1370
1194 vcpu->arch.sie_block->icpua = id; 1371 vcpu->arch.sie_block->icpua = id;
1195 if (!kvm_is_ucontrol(kvm)) { 1372 if (!kvm_is_ucontrol(kvm)) {
@@ -1205,7 +1382,6 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1205 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca; 1382 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1206 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn); 1383 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1207 } 1384 }
1208 vcpu->arch.sie_block->fac = (int) (long) kvm->arch.model.fac->list;
1209 1385
1210 spin_lock_init(&vcpu->arch.local_int.lock); 1386 spin_lock_init(&vcpu->arch.local_int.lock);
1211 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 1387 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
@@ -1725,6 +1901,31 @@ static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1725 return 0; 1901 return 0;
1726} 1902}
1727 1903
1904static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1905{
1906 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1907 u8 opcode;
1908 int rc;
1909
1910 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1911 trace_kvm_s390_sie_fault(vcpu);
1912
1913 /*
1914 * We want to inject an addressing exception, which is defined as a
1915 * suppressing or terminating exception. However, since we came here
1916 * by a DAT access exception, the PSW still points to the faulting
1917 * instruction since DAT exceptions are nullifying. So we've got
1918 * to look up the current opcode to get the length of the instruction
1919 * to be able to forward the PSW.
1920 */
1921 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
1922 if (rc)
1923 return kvm_s390_inject_prog_cond(vcpu, rc);
1924 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1925
1926 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1927}
1928
1728static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) 1929static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1729{ 1930{
1730 int rc = -1; 1931 int rc = -1;
@@ -1756,11 +1957,8 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1756 } 1957 }
1757 } 1958 }
1758 1959
1759 if (rc == -1) { 1960 if (rc == -1)
1760 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 1961 rc = vcpu_post_run_fault_in_sie(vcpu);
1761 trace_kvm_s390_sie_fault(vcpu);
1762 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1763 }
1764 1962
1765 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16); 1963 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1766 1964
@@ -1976,6 +2174,35 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1976 return kvm_s390_store_status_unloaded(vcpu, addr); 2174 return kvm_s390_store_status_unloaded(vcpu, addr);
1977} 2175}
1978 2176
2177/*
2178 * store additional status at address
2179 */
2180int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2181 unsigned long gpa)
2182{
2183 /* Only bits 0-53 are used for address formation */
2184 if (!(gpa & ~0x3ff))
2185 return 0;
2186
2187 return write_guest_abs(vcpu, gpa & ~0x3ff,
2188 (void *)&vcpu->run->s.regs.vrs, 512);
2189}
2190
2191int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2192{
2193 if (!test_kvm_facility(vcpu->kvm, 129))
2194 return 0;
2195
2196 /*
2197 * The guest VXRS are in the host VXRs due to the lazy
2198 * copying in vcpu load/put. Let's update our copies before we save
2199 * it into the save area.
2200 */
2201 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2202
2203 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2204}
2205
1979static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 2206static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1980{ 2207{
1981 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu); 2208 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
@@ -2100,6 +2327,65 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2100 return r; 2327 return r;
2101} 2328}
2102 2329
2330static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2331 struct kvm_s390_mem_op *mop)
2332{
2333 void __user *uaddr = (void __user *)mop->buf;
2334 void *tmpbuf = NULL;
2335 int r, srcu_idx;
2336 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2337 | KVM_S390_MEMOP_F_CHECK_ONLY;
2338
2339 if (mop->flags & ~supported_flags)
2340 return -EINVAL;
2341
2342 if (mop->size > MEM_OP_MAX_SIZE)
2343 return -E2BIG;
2344
2345 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2346 tmpbuf = vmalloc(mop->size);
2347 if (!tmpbuf)
2348 return -ENOMEM;
2349 }
2350
2351 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2352
2353 switch (mop->op) {
2354 case KVM_S390_MEMOP_LOGICAL_READ:
2355 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2356 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2357 break;
2358 }
2359 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2360 if (r == 0) {
2361 if (copy_to_user(uaddr, tmpbuf, mop->size))
2362 r = -EFAULT;
2363 }
2364 break;
2365 case KVM_S390_MEMOP_LOGICAL_WRITE:
2366 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2367 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2368 break;
2369 }
2370 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2371 r = -EFAULT;
2372 break;
2373 }
2374 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2375 break;
2376 default:
2377 r = -EINVAL;
2378 }
2379
2380 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2381
2382 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2383 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2384
2385 vfree(tmpbuf);
2386 return r;
2387}
2388
2103long kvm_arch_vcpu_ioctl(struct file *filp, 2389long kvm_arch_vcpu_ioctl(struct file *filp,
2104 unsigned int ioctl, unsigned long arg) 2390 unsigned int ioctl, unsigned long arg)
2105{ 2391{
@@ -2109,6 +2395,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
2109 long r; 2395 long r;
2110 2396
2111 switch (ioctl) { 2397 switch (ioctl) {
2398 case KVM_S390_IRQ: {
2399 struct kvm_s390_irq s390irq;
2400
2401 r = -EFAULT;
2402 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2403 break;
2404 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2405 break;
2406 }
2112 case KVM_S390_INTERRUPT: { 2407 case KVM_S390_INTERRUPT: {
2113 struct kvm_s390_interrupt s390int; 2408 struct kvm_s390_interrupt s390int;
2114 struct kvm_s390_irq s390irq; 2409 struct kvm_s390_irq s390irq;
@@ -2199,6 +2494,47 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
2199 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 2494 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2200 break; 2495 break;
2201 } 2496 }
2497 case KVM_S390_MEM_OP: {
2498 struct kvm_s390_mem_op mem_op;
2499
2500 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2501 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2502 else
2503 r = -EFAULT;
2504 break;
2505 }
2506 case KVM_S390_SET_IRQ_STATE: {
2507 struct kvm_s390_irq_state irq_state;
2508
2509 r = -EFAULT;
2510 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2511 break;
2512 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2513 irq_state.len == 0 ||
2514 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2515 r = -EINVAL;
2516 break;
2517 }
2518 r = kvm_s390_set_irq_state(vcpu,
2519 (void __user *) irq_state.buf,
2520 irq_state.len);
2521 break;
2522 }
2523 case KVM_S390_GET_IRQ_STATE: {
2524 struct kvm_s390_irq_state irq_state;
2525
2526 r = -EFAULT;
2527 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2528 break;
2529 if (irq_state.len == 0) {
2530 r = -EINVAL;
2531 break;
2532 }
2533 r = kvm_s390_get_irq_state(vcpu,
2534 (__u8 __user *) irq_state.buf,
2535 irq_state.len);
2536 break;
2537 }
2202 default: 2538 default:
2203 r = -ENOTTY; 2539 r = -ENOTTY;
2204 } 2540 }
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index c34109aa552d..ca108b90ae56 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -70,16 +70,22 @@ static inline void kvm_s390_set_prefix(struct kvm_vcpu *vcpu, u32 prefix)
70 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 70 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
71} 71}
72 72
73static inline u64 kvm_s390_get_base_disp_s(struct kvm_vcpu *vcpu) 73typedef u8 __bitwise ar_t;
74
75static inline u64 kvm_s390_get_base_disp_s(struct kvm_vcpu *vcpu, ar_t *ar)
74{ 76{
75 u32 base2 = vcpu->arch.sie_block->ipb >> 28; 77 u32 base2 = vcpu->arch.sie_block->ipb >> 28;
76 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16); 78 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16);
77 79
80 if (ar)
81 *ar = base2;
82
78 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2; 83 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2;
79} 84}
80 85
81static inline void kvm_s390_get_base_disp_sse(struct kvm_vcpu *vcpu, 86static inline void kvm_s390_get_base_disp_sse(struct kvm_vcpu *vcpu,
82 u64 *address1, u64 *address2) 87 u64 *address1, u64 *address2,
88 ar_t *ar_b1, ar_t *ar_b2)
83{ 89{
84 u32 base1 = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28; 90 u32 base1 = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28;
85 u32 disp1 = (vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16; 91 u32 disp1 = (vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16;
@@ -88,6 +94,11 @@ static inline void kvm_s390_get_base_disp_sse(struct kvm_vcpu *vcpu,
88 94
89 *address1 = (base1 ? vcpu->run->s.regs.gprs[base1] : 0) + disp1; 95 *address1 = (base1 ? vcpu->run->s.regs.gprs[base1] : 0) + disp1;
90 *address2 = (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2; 96 *address2 = (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2;
97
98 if (ar_b1)
99 *ar_b1 = base1;
100 if (ar_b2)
101 *ar_b2 = base2;
91} 102}
92 103
93static inline void kvm_s390_get_regs_rre(struct kvm_vcpu *vcpu, int *r1, int *r2) 104static inline void kvm_s390_get_regs_rre(struct kvm_vcpu *vcpu, int *r1, int *r2)
@@ -98,7 +109,7 @@ static inline void kvm_s390_get_regs_rre(struct kvm_vcpu *vcpu, int *r1, int *r2
98 *r2 = (vcpu->arch.sie_block->ipb & 0x000f0000) >> 16; 109 *r2 = (vcpu->arch.sie_block->ipb & 0x000f0000) >> 16;
99} 110}
100 111
101static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu) 112static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu, ar_t *ar)
102{ 113{
103 u32 base2 = vcpu->arch.sie_block->ipb >> 28; 114 u32 base2 = vcpu->arch.sie_block->ipb >> 28;
104 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16) + 115 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16) +
@@ -107,14 +118,20 @@ static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu)
107 if (disp2 & 0x80000) 118 if (disp2 & 0x80000)
108 disp2+=0xfff00000; 119 disp2+=0xfff00000;
109 120
121 if (ar)
122 *ar = base2;
123
110 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + (long)(int)disp2; 124 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + (long)(int)disp2;
111} 125}
112 126
113static inline u64 kvm_s390_get_base_disp_rs(struct kvm_vcpu *vcpu) 127static inline u64 kvm_s390_get_base_disp_rs(struct kvm_vcpu *vcpu, ar_t *ar)
114{ 128{
115 u32 base2 = vcpu->arch.sie_block->ipb >> 28; 129 u32 base2 = vcpu->arch.sie_block->ipb >> 28;
116 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16); 130 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16);
117 131
132 if (ar)
133 *ar = base2;
134
118 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2; 135 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2;
119} 136}
120 137
@@ -125,13 +142,24 @@ static inline void kvm_s390_set_psw_cc(struct kvm_vcpu *vcpu, unsigned long cc)
125 vcpu->arch.sie_block->gpsw.mask |= cc << 44; 142 vcpu->arch.sie_block->gpsw.mask |= cc << 44;
126} 143}
127 144
128/* test availability of facility in a kvm intance */ 145/* test availability of facility in a kvm instance */
129static inline int test_kvm_facility(struct kvm *kvm, unsigned long nr) 146static inline int test_kvm_facility(struct kvm *kvm, unsigned long nr)
130{ 147{
131 return __test_facility(nr, kvm->arch.model.fac->mask) && 148 return __test_facility(nr, kvm->arch.model.fac->mask) &&
132 __test_facility(nr, kvm->arch.model.fac->list); 149 __test_facility(nr, kvm->arch.model.fac->list);
133} 150}
134 151
152static inline int set_kvm_facility(u64 *fac_list, unsigned long nr)
153{
154 unsigned char *ptr;
155
156 if (nr >= MAX_FACILITY_BIT)
157 return -EINVAL;
158 ptr = (unsigned char *) fac_list + (nr >> 3);
159 *ptr |= (0x80UL >> (nr & 7));
160 return 0;
161}
162
135/* are cpu states controlled by user space */ 163/* are cpu states controlled by user space */
136static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm) 164static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm)
137{ 165{
@@ -150,9 +178,9 @@ int __must_check kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu,
150 struct kvm_s390_irq *irq); 178 struct kvm_s390_irq *irq);
151int __must_check kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code); 179int __must_check kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code);
152struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 180struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
153 u64 cr6, u64 schid); 181 u64 isc_mask, u32 schid);
154void kvm_s390_reinject_io_int(struct kvm *kvm, 182int kvm_s390_reinject_io_int(struct kvm *kvm,
155 struct kvm_s390_interrupt_info *inti); 183 struct kvm_s390_interrupt_info *inti);
156int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked); 184int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked);
157 185
158/* implemented in intercept.c */ 186/* implemented in intercept.c */
@@ -177,7 +205,10 @@ int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu);
177/* implemented in kvm-s390.c */ 205/* implemented in kvm-s390.c */
178long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable); 206long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable);
179int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr); 207int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr);
208int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
209 unsigned long addr);
180int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr); 210int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr);
211int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr);
181void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu); 212void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu);
182void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu); 213void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu);
183void s390_vcpu_block(struct kvm_vcpu *vcpu); 214void s390_vcpu_block(struct kvm_vcpu *vcpu);
@@ -241,6 +272,10 @@ int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu);
241extern struct kvm_device_ops kvm_flic_ops; 272extern struct kvm_device_ops kvm_flic_ops;
242int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu); 273int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu);
243void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu); 274void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu);
275int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu,
276 void __user *buf, int len);
277int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu,
278 __u8 __user *buf, int len);
244 279
245/* implemented in guestdbg.c */ 280/* implemented in guestdbg.c */
246void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu); 281void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu);
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index 351116939ea2..d22d8ee1ff9d 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -36,15 +36,16 @@ static int handle_set_clock(struct kvm_vcpu *vcpu)
36 struct kvm_vcpu *cpup; 36 struct kvm_vcpu *cpup;
37 s64 hostclk, val; 37 s64 hostclk, val;
38 int i, rc; 38 int i, rc;
39 ar_t ar;
39 u64 op2; 40 u64 op2;
40 41
41 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 42 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
42 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 43 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
43 44
44 op2 = kvm_s390_get_base_disp_s(vcpu); 45 op2 = kvm_s390_get_base_disp_s(vcpu, &ar);
45 if (op2 & 7) /* Operand must be on a doubleword boundary */ 46 if (op2 & 7) /* Operand must be on a doubleword boundary */
46 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 47 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
47 rc = read_guest(vcpu, op2, &val, sizeof(val)); 48 rc = read_guest(vcpu, op2, ar, &val, sizeof(val));
48 if (rc) 49 if (rc)
49 return kvm_s390_inject_prog_cond(vcpu, rc); 50 return kvm_s390_inject_prog_cond(vcpu, rc);
50 51
@@ -68,20 +69,21 @@ static int handle_set_prefix(struct kvm_vcpu *vcpu)
68 u64 operand2; 69 u64 operand2;
69 u32 address; 70 u32 address;
70 int rc; 71 int rc;
72 ar_t ar;
71 73
72 vcpu->stat.instruction_spx++; 74 vcpu->stat.instruction_spx++;
73 75
74 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 76 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
75 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 77 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
76 78
77 operand2 = kvm_s390_get_base_disp_s(vcpu); 79 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
78 80
79 /* must be word boundary */ 81 /* must be word boundary */
80 if (operand2 & 3) 82 if (operand2 & 3)
81 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 83 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
82 84
83 /* get the value */ 85 /* get the value */
84 rc = read_guest(vcpu, operand2, &address, sizeof(address)); 86 rc = read_guest(vcpu, operand2, ar, &address, sizeof(address));
85 if (rc) 87 if (rc)
86 return kvm_s390_inject_prog_cond(vcpu, rc); 88 return kvm_s390_inject_prog_cond(vcpu, rc);
87 89
@@ -107,13 +109,14 @@ static int handle_store_prefix(struct kvm_vcpu *vcpu)
107 u64 operand2; 109 u64 operand2;
108 u32 address; 110 u32 address;
109 int rc; 111 int rc;
112 ar_t ar;
110 113
111 vcpu->stat.instruction_stpx++; 114 vcpu->stat.instruction_stpx++;
112 115
113 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 116 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
114 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 117 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
115 118
116 operand2 = kvm_s390_get_base_disp_s(vcpu); 119 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
117 120
118 /* must be word boundary */ 121 /* must be word boundary */
119 if (operand2 & 3) 122 if (operand2 & 3)
@@ -122,7 +125,7 @@ static int handle_store_prefix(struct kvm_vcpu *vcpu)
122 address = kvm_s390_get_prefix(vcpu); 125 address = kvm_s390_get_prefix(vcpu);
123 126
124 /* get the value */ 127 /* get the value */
125 rc = write_guest(vcpu, operand2, &address, sizeof(address)); 128 rc = write_guest(vcpu, operand2, ar, &address, sizeof(address));
126 if (rc) 129 if (rc)
127 return kvm_s390_inject_prog_cond(vcpu, rc); 130 return kvm_s390_inject_prog_cond(vcpu, rc);
128 131
@@ -136,18 +139,19 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu)
136 u16 vcpu_id = vcpu->vcpu_id; 139 u16 vcpu_id = vcpu->vcpu_id;
137 u64 ga; 140 u64 ga;
138 int rc; 141 int rc;
142 ar_t ar;
139 143
140 vcpu->stat.instruction_stap++; 144 vcpu->stat.instruction_stap++;
141 145
142 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 146 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
143 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 147 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
144 148
145 ga = kvm_s390_get_base_disp_s(vcpu); 149 ga = kvm_s390_get_base_disp_s(vcpu, &ar);
146 150
147 if (ga & 1) 151 if (ga & 1)
148 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 152 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
149 153
150 rc = write_guest(vcpu, ga, &vcpu_id, sizeof(vcpu_id)); 154 rc = write_guest(vcpu, ga, ar, &vcpu_id, sizeof(vcpu_id));
151 if (rc) 155 if (rc)
152 return kvm_s390_inject_prog_cond(vcpu, rc); 156 return kvm_s390_inject_prog_cond(vcpu, rc);
153 157
@@ -207,7 +211,7 @@ static int handle_test_block(struct kvm_vcpu *vcpu)
207 kvm_s390_get_regs_rre(vcpu, NULL, &reg2); 211 kvm_s390_get_regs_rre(vcpu, NULL, &reg2);
208 addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 212 addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
209 addr = kvm_s390_logical_to_effective(vcpu, addr); 213 addr = kvm_s390_logical_to_effective(vcpu, addr);
210 if (kvm_s390_check_low_addr_protection(vcpu, addr)) 214 if (kvm_s390_check_low_addr_prot_real(vcpu, addr))
211 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 215 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
212 addr = kvm_s390_real_to_abs(vcpu, addr); 216 addr = kvm_s390_real_to_abs(vcpu, addr);
213 217
@@ -229,18 +233,20 @@ static int handle_tpi(struct kvm_vcpu *vcpu)
229 struct kvm_s390_interrupt_info *inti; 233 struct kvm_s390_interrupt_info *inti;
230 unsigned long len; 234 unsigned long len;
231 u32 tpi_data[3]; 235 u32 tpi_data[3];
232 int cc, rc; 236 int rc;
233 u64 addr; 237 u64 addr;
238 ar_t ar;
234 239
235 rc = 0; 240 addr = kvm_s390_get_base_disp_s(vcpu, &ar);
236 addr = kvm_s390_get_base_disp_s(vcpu);
237 if (addr & 3) 241 if (addr & 3)
238 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 242 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
239 cc = 0; 243
240 inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0); 244 inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0);
241 if (!inti) 245 if (!inti) {
242 goto no_interrupt; 246 kvm_s390_set_psw_cc(vcpu, 0);
243 cc = 1; 247 return 0;
248 }
249
244 tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr; 250 tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr;
245 tpi_data[1] = inti->io.io_int_parm; 251 tpi_data[1] = inti->io.io_int_parm;
246 tpi_data[2] = inti->io.io_int_word; 252 tpi_data[2] = inti->io.io_int_word;
@@ -250,40 +256,51 @@ static int handle_tpi(struct kvm_vcpu *vcpu)
250 * provided area. 256 * provided area.
251 */ 257 */
252 len = sizeof(tpi_data) - 4; 258 len = sizeof(tpi_data) - 4;
253 rc = write_guest(vcpu, addr, &tpi_data, len); 259 rc = write_guest(vcpu, addr, ar, &tpi_data, len);
254 if (rc) 260 if (rc) {
255 return kvm_s390_inject_prog_cond(vcpu, rc); 261 rc = kvm_s390_inject_prog_cond(vcpu, rc);
262 goto reinject_interrupt;
263 }
256 } else { 264 } else {
257 /* 265 /*
258 * Store the three-word I/O interruption code into 266 * Store the three-word I/O interruption code into
259 * the appropriate lowcore area. 267 * the appropriate lowcore area.
260 */ 268 */
261 len = sizeof(tpi_data); 269 len = sizeof(tpi_data);
262 if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) 270 if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) {
271 /* failed writes to the low core are not recoverable */
263 rc = -EFAULT; 272 rc = -EFAULT;
273 goto reinject_interrupt;
274 }
264 } 275 }
276
277 /* irq was successfully handed to the guest */
278 kfree(inti);
279 kvm_s390_set_psw_cc(vcpu, 1);
280 return 0;
281reinject_interrupt:
265 /* 282 /*
266 * If we encounter a problem storing the interruption code, the 283 * If we encounter a problem storing the interruption code, the
267 * instruction is suppressed from the guest's view: reinject the 284 * instruction is suppressed from the guest's view: reinject the
268 * interrupt. 285 * interrupt.
269 */ 286 */
270 if (!rc) 287 if (kvm_s390_reinject_io_int(vcpu->kvm, inti)) {
271 kfree(inti); 288 kfree(inti);
272 else 289 rc = -EFAULT;
273 kvm_s390_reinject_io_int(vcpu->kvm, inti); 290 }
274no_interrupt: 291 /* don't set the cc, a pgm irq was injected or we drop to user space */
275 /* Set condition code and we're done. */
276 if (!rc)
277 kvm_s390_set_psw_cc(vcpu, cc);
278 return rc ? -EFAULT : 0; 292 return rc ? -EFAULT : 0;
279} 293}
280 294
281static int handle_tsch(struct kvm_vcpu *vcpu) 295static int handle_tsch(struct kvm_vcpu *vcpu)
282{ 296{
283 struct kvm_s390_interrupt_info *inti; 297 struct kvm_s390_interrupt_info *inti = NULL;
298 const u64 isc_mask = 0xffUL << 24; /* all iscs set */
284 299
285 inti = kvm_s390_get_io_int(vcpu->kvm, 0, 300 /* a valid schid has at least one bit set */
286 vcpu->run->s.regs.gprs[1]); 301 if (vcpu->run->s.regs.gprs[1])
302 inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask,
303 vcpu->run->s.regs.gprs[1]);
287 304
288 /* 305 /*
289 * Prepare exit to userspace. 306 * Prepare exit to userspace.
@@ -386,15 +403,16 @@ int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu)
386 psw_compat_t new_psw; 403 psw_compat_t new_psw;
387 u64 addr; 404 u64 addr;
388 int rc; 405 int rc;
406 ar_t ar;
389 407
390 if (gpsw->mask & PSW_MASK_PSTATE) 408 if (gpsw->mask & PSW_MASK_PSTATE)
391 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 409 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
392 410
393 addr = kvm_s390_get_base_disp_s(vcpu); 411 addr = kvm_s390_get_base_disp_s(vcpu, &ar);
394 if (addr & 7) 412 if (addr & 7)
395 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 413 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
396 414
397 rc = read_guest(vcpu, addr, &new_psw, sizeof(new_psw)); 415 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
398 if (rc) 416 if (rc)
399 return kvm_s390_inject_prog_cond(vcpu, rc); 417 return kvm_s390_inject_prog_cond(vcpu, rc);
400 if (!(new_psw.mask & PSW32_MASK_BASE)) 418 if (!(new_psw.mask & PSW32_MASK_BASE))
@@ -412,14 +430,15 @@ static int handle_lpswe(struct kvm_vcpu *vcpu)
412 psw_t new_psw; 430 psw_t new_psw;
413 u64 addr; 431 u64 addr;
414 int rc; 432 int rc;
433 ar_t ar;
415 434
416 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 435 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
417 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 436 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
418 437
419 addr = kvm_s390_get_base_disp_s(vcpu); 438 addr = kvm_s390_get_base_disp_s(vcpu, &ar);
420 if (addr & 7) 439 if (addr & 7)
421 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 440 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
422 rc = read_guest(vcpu, addr, &new_psw, sizeof(new_psw)); 441 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
423 if (rc) 442 if (rc)
424 return kvm_s390_inject_prog_cond(vcpu, rc); 443 return kvm_s390_inject_prog_cond(vcpu, rc);
425 vcpu->arch.sie_block->gpsw = new_psw; 444 vcpu->arch.sie_block->gpsw = new_psw;
@@ -433,18 +452,19 @@ static int handle_stidp(struct kvm_vcpu *vcpu)
433 u64 stidp_data = vcpu->arch.stidp_data; 452 u64 stidp_data = vcpu->arch.stidp_data;
434 u64 operand2; 453 u64 operand2;
435 int rc; 454 int rc;
455 ar_t ar;
436 456
437 vcpu->stat.instruction_stidp++; 457 vcpu->stat.instruction_stidp++;
438 458
439 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 459 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
440 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 460 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
441 461
442 operand2 = kvm_s390_get_base_disp_s(vcpu); 462 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
443 463
444 if (operand2 & 7) 464 if (operand2 & 7)
445 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 465 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
446 466
447 rc = write_guest(vcpu, operand2, &stidp_data, sizeof(stidp_data)); 467 rc = write_guest(vcpu, operand2, ar, &stidp_data, sizeof(stidp_data));
448 if (rc) 468 if (rc)
449 return kvm_s390_inject_prog_cond(vcpu, rc); 469 return kvm_s390_inject_prog_cond(vcpu, rc);
450 470
@@ -467,6 +487,7 @@ static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem)
467 for (n = mem->count - 1; n > 0 ; n--) 487 for (n = mem->count - 1; n > 0 ; n--)
468 memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0])); 488 memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0]));
469 489
490 memset(&mem->vm[0], 0, sizeof(mem->vm[0]));
470 mem->vm[0].cpus_total = cpus; 491 mem->vm[0].cpus_total = cpus;
471 mem->vm[0].cpus_configured = cpus; 492 mem->vm[0].cpus_configured = cpus;
472 mem->vm[0].cpus_standby = 0; 493 mem->vm[0].cpus_standby = 0;
@@ -478,6 +499,17 @@ static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem)
478 ASCEBC(mem->vm[0].cpi, 16); 499 ASCEBC(mem->vm[0].cpi, 16);
479} 500}
480 501
502static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, ar_t ar,
503 u8 fc, u8 sel1, u16 sel2)
504{
505 vcpu->run->exit_reason = KVM_EXIT_S390_STSI;
506 vcpu->run->s390_stsi.addr = addr;
507 vcpu->run->s390_stsi.ar = ar;
508 vcpu->run->s390_stsi.fc = fc;
509 vcpu->run->s390_stsi.sel1 = sel1;
510 vcpu->run->s390_stsi.sel2 = sel2;
511}
512
481static int handle_stsi(struct kvm_vcpu *vcpu) 513static int handle_stsi(struct kvm_vcpu *vcpu)
482{ 514{
483 int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; 515 int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28;
@@ -486,6 +518,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
486 unsigned long mem = 0; 518 unsigned long mem = 0;
487 u64 operand2; 519 u64 operand2;
488 int rc = 0; 520 int rc = 0;
521 ar_t ar;
489 522
490 vcpu->stat.instruction_stsi++; 523 vcpu->stat.instruction_stsi++;
491 VCPU_EVENT(vcpu, 4, "stsi: fc: %x sel1: %x sel2: %x", fc, sel1, sel2); 524 VCPU_EVENT(vcpu, 4, "stsi: fc: %x sel1: %x sel2: %x", fc, sel1, sel2);
@@ -508,7 +541,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
508 return 0; 541 return 0;
509 } 542 }
510 543
511 operand2 = kvm_s390_get_base_disp_s(vcpu); 544 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
512 545
513 if (operand2 & 0xfff) 546 if (operand2 & 0xfff)
514 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 547 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -532,16 +565,20 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
532 break; 565 break;
533 } 566 }
534 567
535 rc = write_guest(vcpu, operand2, (void *)mem, PAGE_SIZE); 568 rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE);
536 if (rc) { 569 if (rc) {
537 rc = kvm_s390_inject_prog_cond(vcpu, rc); 570 rc = kvm_s390_inject_prog_cond(vcpu, rc);
538 goto out; 571 goto out;
539 } 572 }
573 if (vcpu->kvm->arch.user_stsi) {
574 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2);
575 rc = -EREMOTE;
576 }
540 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 577 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2);
541 free_page(mem); 578 free_page(mem);
542 kvm_s390_set_psw_cc(vcpu, 0); 579 kvm_s390_set_psw_cc(vcpu, 0);
543 vcpu->run->s.regs.gprs[0] = 0; 580 vcpu->run->s.regs.gprs[0] = 0;
544 return 0; 581 return rc;
545out_no_data: 582out_no_data:
546 kvm_s390_set_psw_cc(vcpu, 3); 583 kvm_s390_set_psw_cc(vcpu, 3);
547out: 584out:
@@ -670,7 +707,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
670 } 707 }
671 708
672 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 709 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
673 if (kvm_s390_check_low_addr_protection(vcpu, start)) 710 if (kvm_s390_check_low_addr_prot_real(vcpu, start))
674 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 711 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
675 } 712 }
676 713
@@ -776,13 +813,14 @@ int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
776 int reg, rc, nr_regs; 813 int reg, rc, nr_regs;
777 u32 ctl_array[16]; 814 u32 ctl_array[16];
778 u64 ga; 815 u64 ga;
816 ar_t ar;
779 817
780 vcpu->stat.instruction_lctl++; 818 vcpu->stat.instruction_lctl++;
781 819
782 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 820 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
783 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 821 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
784 822
785 ga = kvm_s390_get_base_disp_rs(vcpu); 823 ga = kvm_s390_get_base_disp_rs(vcpu, &ar);
786 824
787 if (ga & 3) 825 if (ga & 3)
788 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 826 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -791,7 +829,7 @@ int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
791 trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga); 829 trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga);
792 830
793 nr_regs = ((reg3 - reg1) & 0xf) + 1; 831 nr_regs = ((reg3 - reg1) & 0xf) + 1;
794 rc = read_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u32)); 832 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32));
795 if (rc) 833 if (rc)
796 return kvm_s390_inject_prog_cond(vcpu, rc); 834 return kvm_s390_inject_prog_cond(vcpu, rc);
797 reg = reg1; 835 reg = reg1;
@@ -814,13 +852,14 @@ int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu)
814 int reg, rc, nr_regs; 852 int reg, rc, nr_regs;
815 u32 ctl_array[16]; 853 u32 ctl_array[16];
816 u64 ga; 854 u64 ga;
855 ar_t ar;
817 856
818 vcpu->stat.instruction_stctl++; 857 vcpu->stat.instruction_stctl++;
819 858
820 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 859 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
821 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 860 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
822 861
823 ga = kvm_s390_get_base_disp_rs(vcpu); 862 ga = kvm_s390_get_base_disp_rs(vcpu, &ar);
824 863
825 if (ga & 3) 864 if (ga & 3)
826 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 865 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -836,7 +875,7 @@ int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu)
836 break; 875 break;
837 reg = (reg + 1) % 16; 876 reg = (reg + 1) % 16;
838 } while (1); 877 } while (1);
839 rc = write_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u32)); 878 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32));
840 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 879 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
841} 880}
842 881
@@ -847,13 +886,14 @@ static int handle_lctlg(struct kvm_vcpu *vcpu)
847 int reg, rc, nr_regs; 886 int reg, rc, nr_regs;
848 u64 ctl_array[16]; 887 u64 ctl_array[16];
849 u64 ga; 888 u64 ga;
889 ar_t ar;
850 890
851 vcpu->stat.instruction_lctlg++; 891 vcpu->stat.instruction_lctlg++;
852 892
853 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 893 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
854 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 894 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
855 895
856 ga = kvm_s390_get_base_disp_rsy(vcpu); 896 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar);
857 897
858 if (ga & 7) 898 if (ga & 7)
859 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 899 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -862,7 +902,7 @@ static int handle_lctlg(struct kvm_vcpu *vcpu)
862 trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga); 902 trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga);
863 903
864 nr_regs = ((reg3 - reg1) & 0xf) + 1; 904 nr_regs = ((reg3 - reg1) & 0xf) + 1;
865 rc = read_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u64)); 905 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64));
866 if (rc) 906 if (rc)
867 return kvm_s390_inject_prog_cond(vcpu, rc); 907 return kvm_s390_inject_prog_cond(vcpu, rc);
868 reg = reg1; 908 reg = reg1;
@@ -884,13 +924,14 @@ static int handle_stctg(struct kvm_vcpu *vcpu)
884 int reg, rc, nr_regs; 924 int reg, rc, nr_regs;
885 u64 ctl_array[16]; 925 u64 ctl_array[16];
886 u64 ga; 926 u64 ga;
927 ar_t ar;
887 928
888 vcpu->stat.instruction_stctg++; 929 vcpu->stat.instruction_stctg++;
889 930
890 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 931 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
891 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 932 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
892 933
893 ga = kvm_s390_get_base_disp_rsy(vcpu); 934 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar);
894 935
895 if (ga & 7) 936 if (ga & 7)
896 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 937 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -906,7 +947,7 @@ static int handle_stctg(struct kvm_vcpu *vcpu)
906 break; 947 break;
907 reg = (reg + 1) % 16; 948 reg = (reg + 1) % 16;
908 } while (1); 949 } while (1);
909 rc = write_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u64)); 950 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64));
910 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 951 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
911} 952}
912 953
@@ -931,13 +972,14 @@ static int handle_tprot(struct kvm_vcpu *vcpu)
931 unsigned long hva, gpa; 972 unsigned long hva, gpa;
932 int ret = 0, cc = 0; 973 int ret = 0, cc = 0;
933 bool writable; 974 bool writable;
975 ar_t ar;
934 976
935 vcpu->stat.instruction_tprot++; 977 vcpu->stat.instruction_tprot++;
936 978
937 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 979 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
938 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 980 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
939 981
940 kvm_s390_get_base_disp_sse(vcpu, &address1, &address2); 982 kvm_s390_get_base_disp_sse(vcpu, &address1, &address2, &ar, NULL);
941 983
942 /* we only handle the Linux memory detection case: 984 /* we only handle the Linux memory detection case:
943 * access key == 0 985 * access key == 0
@@ -946,11 +988,11 @@ static int handle_tprot(struct kvm_vcpu *vcpu)
946 return -EOPNOTSUPP; 988 return -EOPNOTSUPP;
947 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 989 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT)
948 ipte_lock(vcpu); 990 ipte_lock(vcpu);
949 ret = guest_translate_address(vcpu, address1, &gpa, 1); 991 ret = guest_translate_address(vcpu, address1, ar, &gpa, 1);
950 if (ret == PGM_PROTECTION) { 992 if (ret == PGM_PROTECTION) {
951 /* Write protected? Try again with read-only... */ 993 /* Write protected? Try again with read-only... */
952 cc = 1; 994 cc = 1;
953 ret = guest_translate_address(vcpu, address1, &gpa, 0); 995 ret = guest_translate_address(vcpu, address1, ar, &gpa, 0);
954 } 996 }
955 if (ret) { 997 if (ret) {
956 if (ret == PGM_ADDRESSING || ret == PGM_TRANSLATION_SPEC) { 998 if (ret == PGM_ADDRESSING || ret == PGM_TRANSLATION_SPEC) {
diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c
index 23b1e86b2122..72e58bd2bee7 100644
--- a/arch/s390/kvm/sigp.c
+++ b/arch/s390/kvm/sigp.c
@@ -393,6 +393,9 @@ static int handle_sigp_order_in_user_space(struct kvm_vcpu *vcpu, u8 order_code)
393 case SIGP_STORE_STATUS_AT_ADDRESS: 393 case SIGP_STORE_STATUS_AT_ADDRESS:
394 vcpu->stat.instruction_sigp_store_status++; 394 vcpu->stat.instruction_sigp_store_status++;
395 break; 395 break;
396 case SIGP_STORE_ADDITIONAL_STATUS:
397 vcpu->stat.instruction_sigp_store_adtl_status++;
398 break;
396 case SIGP_SET_PREFIX: 399 case SIGP_SET_PREFIX:
397 vcpu->stat.instruction_sigp_prefix++; 400 vcpu->stat.instruction_sigp_prefix++;
398 break; 401 break;
@@ -431,7 +434,7 @@ int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu)
431 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 434 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
432 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 435 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
433 436
434 order_code = kvm_s390_get_base_disp_rs(vcpu); 437 order_code = kvm_s390_get_base_disp_rs(vcpu, NULL);
435 if (handle_sigp_order_in_user_space(vcpu, order_code)) 438 if (handle_sigp_order_in_user_space(vcpu, order_code))
436 return -EOPNOTSUPP; 439 return -EOPNOTSUPP;
437 440
@@ -473,7 +476,7 @@ int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu)
473 int r3 = vcpu->arch.sie_block->ipa & 0x000f; 476 int r3 = vcpu->arch.sie_block->ipa & 0x000f;
474 u16 cpu_addr = vcpu->run->s.regs.gprs[r3]; 477 u16 cpu_addr = vcpu->run->s.regs.gprs[r3];
475 struct kvm_vcpu *dest_vcpu; 478 struct kvm_vcpu *dest_vcpu;
476 u8 order_code = kvm_s390_get_base_disp_rs(vcpu); 479 u8 order_code = kvm_s390_get_base_disp_rs(vcpu, NULL);
477 480
478 trace_kvm_s390_handle_sigp_pei(vcpu, order_code, cpu_addr); 481 trace_kvm_s390_handle_sigp_pei(vcpu, order_code, cpu_addr);
479 482
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index a236e39cc385..dea2e7e962e3 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -81,11 +81,6 @@ static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
81 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); 81 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
82} 82}
83 83
84#define SELECTOR_TI_MASK (1 << 2)
85#define SELECTOR_RPL_MASK 0x03
86
87#define IOPL_SHIFT 12
88
89#define KVM_PERMILLE_MMU_PAGES 20 84#define KVM_PERMILLE_MMU_PAGES 20
90#define KVM_MIN_ALLOC_MMU_PAGES 64 85#define KVM_MIN_ALLOC_MMU_PAGES 64
91#define KVM_MMU_HASH_SHIFT 10 86#define KVM_MMU_HASH_SHIFT 10
@@ -345,6 +340,7 @@ struct kvm_pmu {
345enum { 340enum {
346 KVM_DEBUGREG_BP_ENABLED = 1, 341 KVM_DEBUGREG_BP_ENABLED = 1,
347 KVM_DEBUGREG_WONT_EXIT = 2, 342 KVM_DEBUGREG_WONT_EXIT = 2,
343 KVM_DEBUGREG_RELOAD = 4,
348}; 344};
349 345
350struct kvm_vcpu_arch { 346struct kvm_vcpu_arch {
@@ -431,6 +427,9 @@ struct kvm_vcpu_arch {
431 427
432 int cpuid_nent; 428 int cpuid_nent;
433 struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; 429 struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES];
430
431 int maxphyaddr;
432
434 /* emulate context */ 433 /* emulate context */
435 434
436 struct x86_emulate_ctxt emulate_ctxt; 435 struct x86_emulate_ctxt emulate_ctxt;
@@ -550,11 +549,20 @@ struct kvm_arch_memory_slot {
550 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; 549 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
551}; 550};
552 551
552/*
553 * We use as the mode the number of bits allocated in the LDR for the
554 * logical processor ID. It happens that these are all powers of two.
555 * This makes it is very easy to detect cases where the APICs are
556 * configured for multiple modes; in that case, we cannot use the map and
557 * hence cannot use kvm_irq_delivery_to_apic_fast either.
558 */
559#define KVM_APIC_MODE_XAPIC_CLUSTER 4
560#define KVM_APIC_MODE_XAPIC_FLAT 8
561#define KVM_APIC_MODE_X2APIC 16
562
553struct kvm_apic_map { 563struct kvm_apic_map {
554 struct rcu_head rcu; 564 struct rcu_head rcu;
555 u8 ldr_bits; 565 u8 mode;
556 /* fields bellow are used to decode ldr values in different modes */
557 u32 cid_shift, cid_mask, lid_mask, broadcast;
558 struct kvm_lapic *phys_map[256]; 566 struct kvm_lapic *phys_map[256];
559 /* first index is cluster id second is cpu id in a cluster */ 567 /* first index is cluster id second is cpu id in a cluster */
560 struct kvm_lapic *logical_map[16][16]; 568 struct kvm_lapic *logical_map[16][16];
@@ -859,6 +867,8 @@ void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
859void kvm_mmu_reset_context(struct kvm_vcpu *vcpu); 867void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
860void kvm_mmu_slot_remove_write_access(struct kvm *kvm, 868void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
861 struct kvm_memory_slot *memslot); 869 struct kvm_memory_slot *memslot);
870void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
871 struct kvm_memory_slot *memslot);
862void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, 872void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
863 struct kvm_memory_slot *memslot); 873 struct kvm_memory_slot *memslot);
864void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, 874void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
@@ -933,6 +943,7 @@ struct x86_emulate_ctxt;
933int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port); 943int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port);
934void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); 944void kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
935int kvm_emulate_halt(struct kvm_vcpu *vcpu); 945int kvm_emulate_halt(struct kvm_vcpu *vcpu);
946int kvm_vcpu_halt(struct kvm_vcpu *vcpu);
936int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); 947int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
937 948
938void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); 949void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
@@ -1128,7 +1139,6 @@ int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1128int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); 1139int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
1129int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); 1140int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
1130void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); 1141void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
1131int cpuid_maxphyaddr(struct kvm_vcpu *vcpu);
1132int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v); 1142int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
1133int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu); 1143int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
1134int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu); 1144int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index e62cf897f781..c1adf33fdd0d 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -115,7 +115,7 @@ static inline void kvm_spinlock_init(void)
115 115
116static inline bool kvm_para_available(void) 116static inline bool kvm_para_available(void)
117{ 117{
118 return 0; 118 return false;
119} 119}
120 120
121static inline unsigned int kvm_arch_para_features(void) 121static inline unsigned int kvm_arch_para_features(void)
diff --git a/arch/x86/include/asm/pvclock.h b/arch/x86/include/asm/pvclock.h
index d6b078e9fa28..25b1cc07d496 100644
--- a/arch/x86/include/asm/pvclock.h
+++ b/arch/x86/include/asm/pvclock.h
@@ -95,6 +95,7 @@ unsigned __pvclock_read_cycles(const struct pvclock_vcpu_time_info *src,
95 95
96struct pvclock_vsyscall_time_info { 96struct pvclock_vsyscall_time_info {
97 struct pvclock_vcpu_time_info pvti; 97 struct pvclock_vcpu_time_info pvti;
98 u32 migrate_count;
98} __attribute__((__aligned__(SMP_CACHE_BYTES))); 99} __attribute__((__aligned__(SMP_CACHE_BYTES)));
99 100
100#define PVTI_SIZE sizeof(struct pvclock_vsyscall_time_info) 101#define PVTI_SIZE sizeof(struct pvclock_vsyscall_time_info)
diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h
index c5f1a1deb91a..1fe92181ee9e 100644
--- a/arch/x86/include/uapi/asm/vmx.h
+++ b/arch/x86/include/uapi/asm/vmx.h
@@ -67,6 +67,7 @@
67#define EXIT_REASON_EPT_VIOLATION 48 67#define EXIT_REASON_EPT_VIOLATION 48
68#define EXIT_REASON_EPT_MISCONFIG 49 68#define EXIT_REASON_EPT_MISCONFIG 49
69#define EXIT_REASON_INVEPT 50 69#define EXIT_REASON_INVEPT 50
70#define EXIT_REASON_RDTSCP 51
70#define EXIT_REASON_PREEMPTION_TIMER 52 71#define EXIT_REASON_PREEMPTION_TIMER 52
71#define EXIT_REASON_INVVPID 53 72#define EXIT_REASON_INVVPID 53
72#define EXIT_REASON_WBINVD 54 73#define EXIT_REASON_WBINVD 54
diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c
index 2f355d229a58..e5ecd20e72dd 100644
--- a/arch/x86/kernel/pvclock.c
+++ b/arch/x86/kernel/pvclock.c
@@ -141,7 +141,46 @@ void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock,
141 set_normalized_timespec(ts, now.tv_sec, now.tv_nsec); 141 set_normalized_timespec(ts, now.tv_sec, now.tv_nsec);
142} 142}
143 143
144static struct pvclock_vsyscall_time_info *pvclock_vdso_info;
145
146static struct pvclock_vsyscall_time_info *
147pvclock_get_vsyscall_user_time_info(int cpu)
148{
149 if (!pvclock_vdso_info) {
150 BUG();
151 return NULL;
152 }
153
154 return &pvclock_vdso_info[cpu];
155}
156
157struct pvclock_vcpu_time_info *pvclock_get_vsyscall_time_info(int cpu)
158{
159 return &pvclock_get_vsyscall_user_time_info(cpu)->pvti;
160}
161
144#ifdef CONFIG_X86_64 162#ifdef CONFIG_X86_64
163static int pvclock_task_migrate(struct notifier_block *nb, unsigned long l,
164 void *v)
165{
166 struct task_migration_notifier *mn = v;
167 struct pvclock_vsyscall_time_info *pvti;
168
169 pvti = pvclock_get_vsyscall_user_time_info(mn->from_cpu);
170
171 /* this is NULL when pvclock vsyscall is not initialized */
172 if (unlikely(pvti == NULL))
173 return NOTIFY_DONE;
174
175 pvti->migrate_count++;
176
177 return NOTIFY_DONE;
178}
179
180static struct notifier_block pvclock_migrate = {
181 .notifier_call = pvclock_task_migrate,
182};
183
145/* 184/*
146 * Initialize the generic pvclock vsyscall state. This will allocate 185 * Initialize the generic pvclock vsyscall state. This will allocate
147 * a/some page(s) for the per-vcpu pvclock information, set up a 186 * a/some page(s) for the per-vcpu pvclock information, set up a
@@ -155,12 +194,17 @@ int __init pvclock_init_vsyscall(struct pvclock_vsyscall_time_info *i,
155 194
156 WARN_ON (size != PVCLOCK_VSYSCALL_NR_PAGES*PAGE_SIZE); 195 WARN_ON (size != PVCLOCK_VSYSCALL_NR_PAGES*PAGE_SIZE);
157 196
197 pvclock_vdso_info = i;
198
158 for (idx = 0; idx <= (PVCLOCK_FIXMAP_END-PVCLOCK_FIXMAP_BEGIN); idx++) { 199 for (idx = 0; idx <= (PVCLOCK_FIXMAP_END-PVCLOCK_FIXMAP_BEGIN); idx++) {
159 __set_fixmap(PVCLOCK_FIXMAP_BEGIN + idx, 200 __set_fixmap(PVCLOCK_FIXMAP_BEGIN + idx,
160 __pa(i) + (idx*PAGE_SIZE), 201 __pa(i) + (idx*PAGE_SIZE),
161 PAGE_KERNEL_VVAR); 202 PAGE_KERNEL_VVAR);
162 } 203 }
163 204
205
206 register_task_migration_notifier(&pvclock_migrate);
207
164 return 0; 208 return 0;
165} 209}
166#endif 210#endif
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 08f790dfadc9..16e8f962eaad 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -1,5 +1,5 @@
1 1
2ccflags-y += -Ivirt/kvm -Iarch/x86/kvm 2ccflags-y += -Iarch/x86/kvm
3 3
4CFLAGS_x86.o := -I. 4CFLAGS_x86.o := -I.
5CFLAGS_svm.o := -I. 5CFLAGS_svm.o := -I.
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 8a80737ee6e6..59b69f6a2844 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -104,6 +104,9 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu)
104 ((best->eax & 0xff00) >> 8) != 0) 104 ((best->eax & 0xff00) >> 8) != 0)
105 return -EINVAL; 105 return -EINVAL;
106 106
107 /* Update physical-address width */
108 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
109
107 kvm_pmu_cpuid_update(vcpu); 110 kvm_pmu_cpuid_update(vcpu);
108 return 0; 111 return 0;
109} 112}
@@ -135,6 +138,21 @@ static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
135 } 138 }
136} 139}
137 140
141int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu)
142{
143 struct kvm_cpuid_entry2 *best;
144
145 best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
146 if (!best || best->eax < 0x80000008)
147 goto not_found;
148 best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
149 if (best)
150 return best->eax & 0xff;
151not_found:
152 return 36;
153}
154EXPORT_SYMBOL_GPL(cpuid_query_maxphyaddr);
155
138/* when an old userspace process fills a new kernel module */ 156/* when an old userspace process fills a new kernel module */
139int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, 157int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
140 struct kvm_cpuid *cpuid, 158 struct kvm_cpuid *cpuid,
@@ -757,21 +775,6 @@ struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
757} 775}
758EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry); 776EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
759 777
760int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
761{
762 struct kvm_cpuid_entry2 *best;
763
764 best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
765 if (!best || best->eax < 0x80000008)
766 goto not_found;
767 best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
768 if (best)
769 return best->eax & 0xff;
770not_found:
771 return 36;
772}
773EXPORT_SYMBOL_GPL(cpuid_maxphyaddr);
774
775/* 778/*
776 * If no match is found, check whether we exceed the vCPU's limit 779 * If no match is found, check whether we exceed the vCPU's limit
777 * and return the content of the highest valid _standard_ leaf instead. 780 * and return the content of the highest valid _standard_ leaf instead.
diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h
index 4452eedfaedd..c3b1ad9fca81 100644
--- a/arch/x86/kvm/cpuid.h
+++ b/arch/x86/kvm/cpuid.h
@@ -20,13 +20,19 @@ int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
20 struct kvm_cpuid_entry2 __user *entries); 20 struct kvm_cpuid_entry2 __user *entries);
21void kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx); 21void kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx);
22 22
23int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu);
24
25static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
26{
27 return vcpu->arch.maxphyaddr;
28}
23 29
24static inline bool guest_cpuid_has_xsave(struct kvm_vcpu *vcpu) 30static inline bool guest_cpuid_has_xsave(struct kvm_vcpu *vcpu)
25{ 31{
26 struct kvm_cpuid_entry2 *best; 32 struct kvm_cpuid_entry2 *best;
27 33
28 if (!static_cpu_has(X86_FEATURE_XSAVE)) 34 if (!static_cpu_has(X86_FEATURE_XSAVE))
29 return 0; 35 return false;
30 36
31 best = kvm_find_cpuid_entry(vcpu, 1, 0); 37 best = kvm_find_cpuid_entry(vcpu, 1, 0);
32 return best && (best->ecx & bit(X86_FEATURE_XSAVE)); 38 return best && (best->ecx & bit(X86_FEATURE_XSAVE));
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index 106c01557f2b..630bcb0d7a04 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -248,27 +248,7 @@ struct mode_dual {
248 struct opcode mode64; 248 struct opcode mode64;
249}; 249};
250 250
251/* EFLAGS bit definitions. */
252#define EFLG_ID (1<<21)
253#define EFLG_VIP (1<<20)
254#define EFLG_VIF (1<<19)
255#define EFLG_AC (1<<18)
256#define EFLG_VM (1<<17)
257#define EFLG_RF (1<<16)
258#define EFLG_IOPL (3<<12)
259#define EFLG_NT (1<<14)
260#define EFLG_OF (1<<11)
261#define EFLG_DF (1<<10)
262#define EFLG_IF (1<<9)
263#define EFLG_TF (1<<8)
264#define EFLG_SF (1<<7)
265#define EFLG_ZF (1<<6)
266#define EFLG_AF (1<<4)
267#define EFLG_PF (1<<2)
268#define EFLG_CF (1<<0)
269
270#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a 251#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
271#define EFLG_RESERVED_ONE_MASK 2
272 252
273enum x86_transfer_type { 253enum x86_transfer_type {
274 X86_TRANSFER_NONE, 254 X86_TRANSFER_NONE,
@@ -317,7 +297,8 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
317 * These EFLAGS bits are restored from saved value during emulation, and 297 * These EFLAGS bits are restored from saved value during emulation, and
318 * any changes are written back to the saved value after emulation. 298 * any changes are written back to the saved value after emulation.
319 */ 299 */
320#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF) 300#define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
301 X86_EFLAGS_PF|X86_EFLAGS_CF)
321 302
322#ifdef CONFIG_X86_64 303#ifdef CONFIG_X86_64
323#define ON64(x) x 304#define ON64(x) x
@@ -478,6 +459,25 @@ static void assign_masked(ulong *dest, ulong src, ulong mask)
478 *dest = (*dest & ~mask) | (src & mask); 459 *dest = (*dest & ~mask) | (src & mask);
479} 460}
480 461
462static void assign_register(unsigned long *reg, u64 val, int bytes)
463{
464 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
465 switch (bytes) {
466 case 1:
467 *(u8 *)reg = (u8)val;
468 break;
469 case 2:
470 *(u16 *)reg = (u16)val;
471 break;
472 case 4:
473 *reg = (u32)val;
474 break; /* 64b: zero-extend */
475 case 8:
476 *reg = val;
477 break;
478 }
479}
480
481static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt) 481static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
482{ 482{
483 return (1UL << (ctxt->ad_bytes << 3)) - 1; 483 return (1UL << (ctxt->ad_bytes << 3)) - 1;
@@ -943,6 +943,22 @@ FASTOP2(xadd);
943 943
944FASTOP2R(cmp, cmp_r); 944FASTOP2R(cmp, cmp_r);
945 945
946static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
947{
948 /* If src is zero, do not writeback, but update flags */
949 if (ctxt->src.val == 0)
950 ctxt->dst.type = OP_NONE;
951 return fastop(ctxt, em_bsf);
952}
953
954static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
955{
956 /* If src is zero, do not writeback, but update flags */
957 if (ctxt->src.val == 0)
958 ctxt->dst.type = OP_NONE;
959 return fastop(ctxt, em_bsr);
960}
961
946static u8 test_cc(unsigned int condition, unsigned long flags) 962static u8 test_cc(unsigned int condition, unsigned long flags)
947{ 963{
948 u8 rc; 964 u8 rc;
@@ -1399,7 +1415,7 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1399 unsigned int in_page, n; 1415 unsigned int in_page, n;
1400 unsigned int count = ctxt->rep_prefix ? 1416 unsigned int count = ctxt->rep_prefix ?
1401 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1; 1417 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1402 in_page = (ctxt->eflags & EFLG_DF) ? 1418 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1403 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) : 1419 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1404 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)); 1420 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1405 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count); 1421 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
@@ -1412,7 +1428,7 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1412 } 1428 }
1413 1429
1414 if (ctxt->rep_prefix && (ctxt->d & String) && 1430 if (ctxt->rep_prefix && (ctxt->d & String) &&
1415 !(ctxt->eflags & EFLG_DF)) { 1431 !(ctxt->eflags & X86_EFLAGS_DF)) {
1416 ctxt->dst.data = rc->data + rc->pos; 1432 ctxt->dst.data = rc->data + rc->pos;
1417 ctxt->dst.type = OP_MEM_STR; 1433 ctxt->dst.type = OP_MEM_STR;
1418 ctxt->dst.count = (rc->end - rc->pos) / size; 1434 ctxt->dst.count = (rc->end - rc->pos) / size;
@@ -1691,21 +1707,7 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1691 1707
1692static void write_register_operand(struct operand *op) 1708static void write_register_operand(struct operand *op)
1693{ 1709{
1694 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ 1710 return assign_register(op->addr.reg, op->val, op->bytes);
1695 switch (op->bytes) {
1696 case 1:
1697 *(u8 *)op->addr.reg = (u8)op->val;
1698 break;
1699 case 2:
1700 *(u16 *)op->addr.reg = (u16)op->val;
1701 break;
1702 case 4:
1703 *op->addr.reg = (u32)op->val;
1704 break; /* 64b: zero-extend */
1705 case 8:
1706 *op->addr.reg = op->val;
1707 break;
1708 }
1709} 1711}
1710 1712
1711static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op) 1713static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
@@ -1792,32 +1794,34 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1792{ 1794{
1793 int rc; 1795 int rc;
1794 unsigned long val, change_mask; 1796 unsigned long val, change_mask;
1795 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; 1797 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1796 int cpl = ctxt->ops->cpl(ctxt); 1798 int cpl = ctxt->ops->cpl(ctxt);
1797 1799
1798 rc = emulate_pop(ctxt, &val, len); 1800 rc = emulate_pop(ctxt, &val, len);
1799 if (rc != X86EMUL_CONTINUE) 1801 if (rc != X86EMUL_CONTINUE)
1800 return rc; 1802 return rc;
1801 1803
1802 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF 1804 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1803 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID; 1805 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1806 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1807 X86_EFLAGS_AC | X86_EFLAGS_ID;
1804 1808
1805 switch(ctxt->mode) { 1809 switch(ctxt->mode) {
1806 case X86EMUL_MODE_PROT64: 1810 case X86EMUL_MODE_PROT64:
1807 case X86EMUL_MODE_PROT32: 1811 case X86EMUL_MODE_PROT32:
1808 case X86EMUL_MODE_PROT16: 1812 case X86EMUL_MODE_PROT16:
1809 if (cpl == 0) 1813 if (cpl == 0)
1810 change_mask |= EFLG_IOPL; 1814 change_mask |= X86_EFLAGS_IOPL;
1811 if (cpl <= iopl) 1815 if (cpl <= iopl)
1812 change_mask |= EFLG_IF; 1816 change_mask |= X86_EFLAGS_IF;
1813 break; 1817 break;
1814 case X86EMUL_MODE_VM86: 1818 case X86EMUL_MODE_VM86:
1815 if (iopl < 3) 1819 if (iopl < 3)
1816 return emulate_gp(ctxt, 0); 1820 return emulate_gp(ctxt, 0);
1817 change_mask |= EFLG_IF; 1821 change_mask |= X86_EFLAGS_IF;
1818 break; 1822 break;
1819 default: /* real mode */ 1823 default: /* real mode */
1820 change_mask |= (EFLG_IOPL | EFLG_IF); 1824 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1821 break; 1825 break;
1822 } 1826 }
1823 1827
@@ -1918,7 +1922,7 @@ static int em_pusha(struct x86_emulate_ctxt *ctxt)
1918 1922
1919static int em_pushf(struct x86_emulate_ctxt *ctxt) 1923static int em_pushf(struct x86_emulate_ctxt *ctxt)
1920{ 1924{
1921 ctxt->src.val = (unsigned long)ctxt->eflags & ~EFLG_VM; 1925 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1922 return em_push(ctxt); 1926 return em_push(ctxt);
1923} 1927}
1924 1928
@@ -1926,6 +1930,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
1926{ 1930{
1927 int rc = X86EMUL_CONTINUE; 1931 int rc = X86EMUL_CONTINUE;
1928 int reg = VCPU_REGS_RDI; 1932 int reg = VCPU_REGS_RDI;
1933 u32 val;
1929 1934
1930 while (reg >= VCPU_REGS_RAX) { 1935 while (reg >= VCPU_REGS_RAX) {
1931 if (reg == VCPU_REGS_RSP) { 1936 if (reg == VCPU_REGS_RSP) {
@@ -1933,9 +1938,10 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
1933 --reg; 1938 --reg;
1934 } 1939 }
1935 1940
1936 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes); 1941 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1937 if (rc != X86EMUL_CONTINUE) 1942 if (rc != X86EMUL_CONTINUE)
1938 break; 1943 break;
1944 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1939 --reg; 1945 --reg;
1940 } 1946 }
1941 return rc; 1947 return rc;
@@ -1956,7 +1962,7 @@ static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1956 if (rc != X86EMUL_CONTINUE) 1962 if (rc != X86EMUL_CONTINUE)
1957 return rc; 1963 return rc;
1958 1964
1959 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC); 1965 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
1960 1966
1961 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS); 1967 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1962 rc = em_push(ctxt); 1968 rc = em_push(ctxt);
@@ -2022,10 +2028,14 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2022 unsigned long temp_eip = 0; 2028 unsigned long temp_eip = 0;
2023 unsigned long temp_eflags = 0; 2029 unsigned long temp_eflags = 0;
2024 unsigned long cs = 0; 2030 unsigned long cs = 0;
2025 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF | 2031 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2026 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF | 2032 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2027 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */ 2033 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2028 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP; 2034 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2035 X86_EFLAGS_AC | X86_EFLAGS_ID |
2036 X86_EFLAGS_FIXED;
2037 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2038 X86_EFLAGS_VIP;
2029 2039
2030 /* TODO: Add stack limit check */ 2040 /* TODO: Add stack limit check */
2031 2041
@@ -2054,7 +2064,6 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2054 2064
2055 ctxt->_eip = temp_eip; 2065 ctxt->_eip = temp_eip;
2056 2066
2057
2058 if (ctxt->op_bytes == 4) 2067 if (ctxt->op_bytes == 4)
2059 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask)); 2068 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2060 else if (ctxt->op_bytes == 2) { 2069 else if (ctxt->op_bytes == 2) {
@@ -2063,7 +2072,7 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2063 } 2072 }
2064 2073
2065 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */ 2074 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2066 ctxt->eflags |= EFLG_RESERVED_ONE_MASK; 2075 ctxt->eflags |= X86_EFLAGS_FIXED;
2067 ctxt->ops->set_nmi_mask(ctxt, false); 2076 ctxt->ops->set_nmi_mask(ctxt, false);
2068 2077
2069 return rc; 2078 return rc;
@@ -2145,12 +2154,12 @@ static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2145 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) { 2154 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2146 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0); 2155 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2147 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32); 2156 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2148 ctxt->eflags &= ~EFLG_ZF; 2157 ctxt->eflags &= ~X86_EFLAGS_ZF;
2149 } else { 2158 } else {
2150 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) | 2159 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2151 (u32) reg_read(ctxt, VCPU_REGS_RBX); 2160 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2152 2161
2153 ctxt->eflags |= EFLG_ZF; 2162 ctxt->eflags |= X86_EFLAGS_ZF;
2154 } 2163 }
2155 return X86EMUL_CONTINUE; 2164 return X86EMUL_CONTINUE;
2156} 2165}
@@ -2222,7 +2231,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2222 ctxt->src.val = ctxt->dst.orig_val; 2231 ctxt->src.val = ctxt->dst.orig_val;
2223 fastop(ctxt, em_cmp); 2232 fastop(ctxt, em_cmp);
2224 2233
2225 if (ctxt->eflags & EFLG_ZF) { 2234 if (ctxt->eflags & X86_EFLAGS_ZF) {
2226 /* Success: write back to memory; no update of EAX */ 2235 /* Success: write back to memory; no update of EAX */
2227 ctxt->src.type = OP_NONE; 2236 ctxt->src.type = OP_NONE;
2228 ctxt->dst.val = ctxt->src.orig_val; 2237 ctxt->dst.val = ctxt->src.orig_val;
@@ -2381,14 +2390,14 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt)
2381 2390
2382 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data); 2391 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2383 ctxt->eflags &= ~msr_data; 2392 ctxt->eflags &= ~msr_data;
2384 ctxt->eflags |= EFLG_RESERVED_ONE_MASK; 2393 ctxt->eflags |= X86_EFLAGS_FIXED;
2385#endif 2394#endif
2386 } else { 2395 } else {
2387 /* legacy mode */ 2396 /* legacy mode */
2388 ops->get_msr(ctxt, MSR_STAR, &msr_data); 2397 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2389 ctxt->_eip = (u32)msr_data; 2398 ctxt->_eip = (u32)msr_data;
2390 2399
2391 ctxt->eflags &= ~(EFLG_VM | EFLG_IF); 2400 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2392 } 2401 }
2393 2402
2394 return X86EMUL_CONTINUE; 2403 return X86EMUL_CONTINUE;
@@ -2425,8 +2434,8 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2425 if ((msr_data & 0xfffc) == 0x0) 2434 if ((msr_data & 0xfffc) == 0x0)
2426 return emulate_gp(ctxt, 0); 2435 return emulate_gp(ctxt, 0);
2427 2436
2428 ctxt->eflags &= ~(EFLG_VM | EFLG_IF); 2437 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2429 cs_sel = (u16)msr_data & ~SELECTOR_RPL_MASK; 2438 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2430 ss_sel = cs_sel + 8; 2439 ss_sel = cs_sel + 8;
2431 if (efer & EFER_LMA) { 2440 if (efer & EFER_LMA) {
2432 cs.d = 0; 2441 cs.d = 0;
@@ -2493,8 +2502,8 @@ static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2493 return emulate_gp(ctxt, 0); 2502 return emulate_gp(ctxt, 0);
2494 break; 2503 break;
2495 } 2504 }
2496 cs_sel |= SELECTOR_RPL_MASK; 2505 cs_sel |= SEGMENT_RPL_MASK;
2497 ss_sel |= SELECTOR_RPL_MASK; 2506 ss_sel |= SEGMENT_RPL_MASK;
2498 2507
2499 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); 2508 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2500 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); 2509 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
@@ -2512,7 +2521,7 @@ static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2512 return false; 2521 return false;
2513 if (ctxt->mode == X86EMUL_MODE_VM86) 2522 if (ctxt->mode == X86EMUL_MODE_VM86)
2514 return true; 2523 return true;
2515 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; 2524 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2516 return ctxt->ops->cpl(ctxt) > iopl; 2525 return ctxt->ops->cpl(ctxt) > iopl;
2517} 2526}
2518 2527
@@ -2782,10 +2791,8 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2782 return ret; 2791 return ret;
2783 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, 2792 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2784 X86_TRANSFER_TASK_SWITCH, NULL); 2793 X86_TRANSFER_TASK_SWITCH, NULL);
2785 if (ret != X86EMUL_CONTINUE)
2786 return ret;
2787 2794
2788 return X86EMUL_CONTINUE; 2795 return ret;
2789} 2796}
2790 2797
2791static int task_switch_32(struct x86_emulate_ctxt *ctxt, 2798static int task_switch_32(struct x86_emulate_ctxt *ctxt,
@@ -2954,7 +2961,7 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2954static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg, 2961static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2955 struct operand *op) 2962 struct operand *op)
2956{ 2963{
2957 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count; 2964 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
2958 2965
2959 register_address_increment(ctxt, reg, df * op->bytes); 2966 register_address_increment(ctxt, reg, df * op->bytes);
2960 op->addr.mem.ea = register_address(ctxt, reg); 2967 op->addr.mem.ea = register_address(ctxt, reg);
@@ -3323,7 +3330,7 @@ static int em_clts(struct x86_emulate_ctxt *ctxt)
3323 return X86EMUL_CONTINUE; 3330 return X86EMUL_CONTINUE;
3324} 3331}
3325 3332
3326static int em_vmcall(struct x86_emulate_ctxt *ctxt) 3333static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3327{ 3334{
3328 int rc = ctxt->ops->fix_hypercall(ctxt); 3335 int rc = ctxt->ops->fix_hypercall(ctxt);
3329 3336
@@ -3395,17 +3402,6 @@ static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3395 return em_lgdt_lidt(ctxt, true); 3402 return em_lgdt_lidt(ctxt, true);
3396} 3403}
3397 3404
3398static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3399{
3400 int rc;
3401
3402 rc = ctxt->ops->fix_hypercall(ctxt);
3403
3404 /* Disable writeback. */
3405 ctxt->dst.type = OP_NONE;
3406 return rc;
3407}
3408
3409static int em_lidt(struct x86_emulate_ctxt *ctxt) 3405static int em_lidt(struct x86_emulate_ctxt *ctxt)
3410{ 3406{
3411 return em_lgdt_lidt(ctxt, false); 3407 return em_lgdt_lidt(ctxt, false);
@@ -3504,7 +3500,8 @@ static int em_sahf(struct x86_emulate_ctxt *ctxt)
3504{ 3500{
3505 u32 flags; 3501 u32 flags;
3506 3502
3507 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF; 3503 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3504 X86_EFLAGS_SF;
3508 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8; 3505 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3509 3506
3510 ctxt->eflags &= ~0xffUL; 3507 ctxt->eflags &= ~0xffUL;
@@ -3769,7 +3766,7 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3769 3766
3770static const struct opcode group7_rm0[] = { 3767static const struct opcode group7_rm0[] = {
3771 N, 3768 N,
3772 I(SrcNone | Priv | EmulateOnUD, em_vmcall), 3769 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
3773 N, N, N, N, N, N, 3770 N, N, N, N, N, N,
3774}; 3771};
3775 3772
@@ -3781,7 +3778,7 @@ static const struct opcode group7_rm1[] = {
3781 3778
3782static const struct opcode group7_rm3[] = { 3779static const struct opcode group7_rm3[] = {
3783 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa), 3780 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3784 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall), 3781 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
3785 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa), 3782 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3786 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa), 3783 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3787 DIP(SrcNone | Prot | Priv, stgi, check_svme), 3784 DIP(SrcNone | Prot | Priv, stgi, check_svme),
@@ -4192,7 +4189,8 @@ static const struct opcode twobyte_table[256] = {
4192 N, N, 4189 N, N,
4193 G(BitOp, group8), 4190 G(BitOp, group8),
4194 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc), 4191 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4195 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr), 4192 I(DstReg | SrcMem | ModRM, em_bsf_c),
4193 I(DstReg | SrcMem | ModRM, em_bsr_c),
4196 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov), 4194 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4197 /* 0xC0 - 0xC7 */ 4195 /* 0xC0 - 0xC7 */
4198 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd), 4196 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
@@ -4759,9 +4757,9 @@ static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4759 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) || 4757 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4760 (ctxt->b == 0xae) || (ctxt->b == 0xaf)) 4758 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4761 && (((ctxt->rep_prefix == REPE_PREFIX) && 4759 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4762 ((ctxt->eflags & EFLG_ZF) == 0)) 4760 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
4763 || ((ctxt->rep_prefix == REPNE_PREFIX) && 4761 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4764 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)))) 4762 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
4765 return true; 4763 return true;
4766 4764
4767 return false; 4765 return false;
@@ -4913,7 +4911,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4913 /* All REP prefixes have the same first termination condition */ 4911 /* All REP prefixes have the same first termination condition */
4914 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) { 4912 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4915 ctxt->eip = ctxt->_eip; 4913 ctxt->eip = ctxt->_eip;
4916 ctxt->eflags &= ~EFLG_RF; 4914 ctxt->eflags &= ~X86_EFLAGS_RF;
4917 goto done; 4915 goto done;
4918 } 4916 }
4919 } 4917 }
@@ -4963,9 +4961,9 @@ special_insn:
4963 } 4961 }
4964 4962
4965 if (ctxt->rep_prefix && (ctxt->d & String)) 4963 if (ctxt->rep_prefix && (ctxt->d & String))
4966 ctxt->eflags |= EFLG_RF; 4964 ctxt->eflags |= X86_EFLAGS_RF;
4967 else 4965 else
4968 ctxt->eflags &= ~EFLG_RF; 4966 ctxt->eflags &= ~X86_EFLAGS_RF;
4969 4967
4970 if (ctxt->execute) { 4968 if (ctxt->execute) {
4971 if (ctxt->d & Fastop) { 4969 if (ctxt->d & Fastop) {
@@ -5014,7 +5012,7 @@ special_insn:
5014 rc = emulate_int(ctxt, ctxt->src.val); 5012 rc = emulate_int(ctxt, ctxt->src.val);
5015 break; 5013 break;
5016 case 0xce: /* into */ 5014 case 0xce: /* into */
5017 if (ctxt->eflags & EFLG_OF) 5015 if (ctxt->eflags & X86_EFLAGS_OF)
5018 rc = emulate_int(ctxt, 4); 5016 rc = emulate_int(ctxt, 4);
5019 break; 5017 break;
5020 case 0xe9: /* jmp rel */ 5018 case 0xe9: /* jmp rel */
@@ -5027,19 +5025,19 @@ special_insn:
5027 break; 5025 break;
5028 case 0xf5: /* cmc */ 5026 case 0xf5: /* cmc */
5029 /* complement carry flag from eflags reg */ 5027 /* complement carry flag from eflags reg */
5030 ctxt->eflags ^= EFLG_CF; 5028 ctxt->eflags ^= X86_EFLAGS_CF;
5031 break; 5029 break;
5032 case 0xf8: /* clc */ 5030 case 0xf8: /* clc */
5033 ctxt->eflags &= ~EFLG_CF; 5031 ctxt->eflags &= ~X86_EFLAGS_CF;
5034 break; 5032 break;
5035 case 0xf9: /* stc */ 5033 case 0xf9: /* stc */
5036 ctxt->eflags |= EFLG_CF; 5034 ctxt->eflags |= X86_EFLAGS_CF;
5037 break; 5035 break;
5038 case 0xfc: /* cld */ 5036 case 0xfc: /* cld */
5039 ctxt->eflags &= ~EFLG_DF; 5037 ctxt->eflags &= ~X86_EFLAGS_DF;
5040 break; 5038 break;
5041 case 0xfd: /* std */ 5039 case 0xfd: /* std */
5042 ctxt->eflags |= EFLG_DF; 5040 ctxt->eflags |= X86_EFLAGS_DF;
5043 break; 5041 break;
5044 default: 5042 default:
5045 goto cannot_emulate; 5043 goto cannot_emulate;
@@ -5100,7 +5098,7 @@ writeback:
5100 } 5098 }
5101 goto done; /* skip rip writeback */ 5099 goto done; /* skip rip writeback */
5102 } 5100 }
5103 ctxt->eflags &= ~EFLG_RF; 5101 ctxt->eflags &= ~X86_EFLAGS_RF;
5104 } 5102 }
5105 5103
5106 ctxt->eip = ctxt->_eip; 5104 ctxt->eip = ctxt->_eip;
@@ -5137,8 +5135,7 @@ twobyte_insn:
5137 case 0x40 ... 0x4f: /* cmov */ 5135 case 0x40 ... 0x4f: /* cmov */
5138 if (test_cc(ctxt->b, ctxt->eflags)) 5136 if (test_cc(ctxt->b, ctxt->eflags))
5139 ctxt->dst.val = ctxt->src.val; 5137 ctxt->dst.val = ctxt->src.val;
5140 else if (ctxt->mode != X86EMUL_MODE_PROT64 || 5138 else if (ctxt->op_bytes != 4)
5141 ctxt->op_bytes != 4)
5142 ctxt->dst.type = OP_NONE; /* no writeback */ 5139 ctxt->dst.type = OP_NONE; /* no writeback */
5143 break; 5140 break;
5144 case 0x80 ... 0x8f: /* jnz rel, etc*/ 5141 case 0x80 ... 0x8f: /* jnz rel, etc*/
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index 298781d4cfb4..4dce6f8b6129 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -443,7 +443,8 @@ static inline int pit_in_range(gpa_t addr)
443 (addr < KVM_PIT_BASE_ADDRESS + KVM_PIT_MEM_LENGTH)); 443 (addr < KVM_PIT_BASE_ADDRESS + KVM_PIT_MEM_LENGTH));
444} 444}
445 445
446static int pit_ioport_write(struct kvm_io_device *this, 446static int pit_ioport_write(struct kvm_vcpu *vcpu,
447 struct kvm_io_device *this,
447 gpa_t addr, int len, const void *data) 448 gpa_t addr, int len, const void *data)
448{ 449{
449 struct kvm_pit *pit = dev_to_pit(this); 450 struct kvm_pit *pit = dev_to_pit(this);
@@ -519,7 +520,8 @@ static int pit_ioport_write(struct kvm_io_device *this,
519 return 0; 520 return 0;
520} 521}
521 522
522static int pit_ioport_read(struct kvm_io_device *this, 523static int pit_ioport_read(struct kvm_vcpu *vcpu,
524 struct kvm_io_device *this,
523 gpa_t addr, int len, void *data) 525 gpa_t addr, int len, void *data)
524{ 526{
525 struct kvm_pit *pit = dev_to_pit(this); 527 struct kvm_pit *pit = dev_to_pit(this);
@@ -589,7 +591,8 @@ static int pit_ioport_read(struct kvm_io_device *this,
589 return 0; 591 return 0;
590} 592}
591 593
592static int speaker_ioport_write(struct kvm_io_device *this, 594static int speaker_ioport_write(struct kvm_vcpu *vcpu,
595 struct kvm_io_device *this,
593 gpa_t addr, int len, const void *data) 596 gpa_t addr, int len, const void *data)
594{ 597{
595 struct kvm_pit *pit = speaker_to_pit(this); 598 struct kvm_pit *pit = speaker_to_pit(this);
@@ -606,8 +609,9 @@ static int speaker_ioport_write(struct kvm_io_device *this,
606 return 0; 609 return 0;
607} 610}
608 611
609static int speaker_ioport_read(struct kvm_io_device *this, 612static int speaker_ioport_read(struct kvm_vcpu *vcpu,
610 gpa_t addr, int len, void *data) 613 struct kvm_io_device *this,
614 gpa_t addr, int len, void *data)
611{ 615{
612 struct kvm_pit *pit = speaker_to_pit(this); 616 struct kvm_pit *pit = speaker_to_pit(this);
613 struct kvm_kpit_state *pit_state = &pit->pit_state; 617 struct kvm_kpit_state *pit_state = &pit->pit_state;
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h
index dd1b16b611b0..c84990b42b5b 100644
--- a/arch/x86/kvm/i8254.h
+++ b/arch/x86/kvm/i8254.h
@@ -3,7 +3,7 @@
3 3
4#include <linux/kthread.h> 4#include <linux/kthread.h>
5 5
6#include "iodev.h" 6#include <kvm/iodev.h>
7 7
8struct kvm_kpit_channel_state { 8struct kvm_kpit_channel_state {
9 u32 count; /* can be 65536 */ 9 u32 count; /* can be 65536 */
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c
index 9541ba34126b..fef922ff2635 100644
--- a/arch/x86/kvm/i8259.c
+++ b/arch/x86/kvm/i8259.c
@@ -529,42 +529,42 @@ static int picdev_read(struct kvm_pic *s,
529 return 0; 529 return 0;
530} 530}
531 531
532static int picdev_master_write(struct kvm_io_device *dev, 532static int picdev_master_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
533 gpa_t addr, int len, const void *val) 533 gpa_t addr, int len, const void *val)
534{ 534{
535 return picdev_write(container_of(dev, struct kvm_pic, dev_master), 535 return picdev_write(container_of(dev, struct kvm_pic, dev_master),
536 addr, len, val); 536 addr, len, val);
537} 537}
538 538
539static int picdev_master_read(struct kvm_io_device *dev, 539static int picdev_master_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
540 gpa_t addr, int len, void *val) 540 gpa_t addr, int len, void *val)
541{ 541{
542 return picdev_read(container_of(dev, struct kvm_pic, dev_master), 542 return picdev_read(container_of(dev, struct kvm_pic, dev_master),
543 addr, len, val); 543 addr, len, val);
544} 544}
545 545
546static int picdev_slave_write(struct kvm_io_device *dev, 546static int picdev_slave_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
547 gpa_t addr, int len, const void *val) 547 gpa_t addr, int len, const void *val)
548{ 548{
549 return picdev_write(container_of(dev, struct kvm_pic, dev_slave), 549 return picdev_write(container_of(dev, struct kvm_pic, dev_slave),
550 addr, len, val); 550 addr, len, val);
551} 551}
552 552
553static int picdev_slave_read(struct kvm_io_device *dev, 553static int picdev_slave_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
554 gpa_t addr, int len, void *val) 554 gpa_t addr, int len, void *val)
555{ 555{
556 return picdev_read(container_of(dev, struct kvm_pic, dev_slave), 556 return picdev_read(container_of(dev, struct kvm_pic, dev_slave),
557 addr, len, val); 557 addr, len, val);
558} 558}
559 559
560static int picdev_eclr_write(struct kvm_io_device *dev, 560static int picdev_eclr_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
561 gpa_t addr, int len, const void *val) 561 gpa_t addr, int len, const void *val)
562{ 562{
563 return picdev_write(container_of(dev, struct kvm_pic, dev_eclr), 563 return picdev_write(container_of(dev, struct kvm_pic, dev_eclr),
564 addr, len, val); 564 addr, len, val);
565} 565}
566 566
567static int picdev_eclr_read(struct kvm_io_device *dev, 567static int picdev_eclr_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
568 gpa_t addr, int len, void *val) 568 gpa_t addr, int len, void *val)
569{ 569{
570 return picdev_read(container_of(dev, struct kvm_pic, dev_eclr), 570 return picdev_read(container_of(dev, struct kvm_pic, dev_eclr),
diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c
index 46d4449772bc..28146f03c514 100644
--- a/arch/x86/kvm/ioapic.c
+++ b/arch/x86/kvm/ioapic.c
@@ -206,6 +206,8 @@ static int ioapic_set_irq(struct kvm_ioapic *ioapic, unsigned int irq,
206 206
207 old_irr = ioapic->irr; 207 old_irr = ioapic->irr;
208 ioapic->irr |= mask; 208 ioapic->irr |= mask;
209 if (edge)
210 ioapic->irr_delivered &= ~mask;
209 if ((edge && old_irr == ioapic->irr) || 211 if ((edge && old_irr == ioapic->irr) ||
210 (!edge && entry.fields.remote_irr)) { 212 (!edge && entry.fields.remote_irr)) {
211 ret = 0; 213 ret = 0;
@@ -349,7 +351,7 @@ static int ioapic_service(struct kvm_ioapic *ioapic, int irq, bool line_status)
349 irqe.shorthand = 0; 351 irqe.shorthand = 0;
350 352
351 if (irqe.trig_mode == IOAPIC_EDGE_TRIG) 353 if (irqe.trig_mode == IOAPIC_EDGE_TRIG)
352 ioapic->irr &= ~(1 << irq); 354 ioapic->irr_delivered |= 1 << irq;
353 355
354 if (irq == RTC_GSI && line_status) { 356 if (irq == RTC_GSI && line_status) {
355 /* 357 /*
@@ -473,13 +475,6 @@ static void __kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu,
473 } 475 }
474} 476}
475 477
476bool kvm_ioapic_handles_vector(struct kvm *kvm, int vector)
477{
478 struct kvm_ioapic *ioapic = kvm->arch.vioapic;
479 smp_rmb();
480 return test_bit(vector, ioapic->handled_vectors);
481}
482
483void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, int trigger_mode) 478void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, int trigger_mode)
484{ 479{
485 struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic; 480 struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
@@ -500,8 +495,8 @@ static inline int ioapic_in_range(struct kvm_ioapic *ioapic, gpa_t addr)
500 (addr < ioapic->base_address + IOAPIC_MEM_LENGTH))); 495 (addr < ioapic->base_address + IOAPIC_MEM_LENGTH)));
501} 496}
502 497
503static int ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len, 498static int ioapic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
504 void *val) 499 gpa_t addr, int len, void *val)
505{ 500{
506 struct kvm_ioapic *ioapic = to_ioapic(this); 501 struct kvm_ioapic *ioapic = to_ioapic(this);
507 u32 result; 502 u32 result;
@@ -543,8 +538,8 @@ static int ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len,
543 return 0; 538 return 0;
544} 539}
545 540
546static int ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, 541static int ioapic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
547 const void *val) 542 gpa_t addr, int len, const void *val)
548{ 543{
549 struct kvm_ioapic *ioapic = to_ioapic(this); 544 struct kvm_ioapic *ioapic = to_ioapic(this);
550 u32 data; 545 u32 data;
@@ -599,6 +594,7 @@ static void kvm_ioapic_reset(struct kvm_ioapic *ioapic)
599 ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS; 594 ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS;
600 ioapic->ioregsel = 0; 595 ioapic->ioregsel = 0;
601 ioapic->irr = 0; 596 ioapic->irr = 0;
597 ioapic->irr_delivered = 0;
602 ioapic->id = 0; 598 ioapic->id = 0;
603 memset(ioapic->irq_eoi, 0x00, IOAPIC_NUM_PINS); 599 memset(ioapic->irq_eoi, 0x00, IOAPIC_NUM_PINS);
604 rtc_irq_eoi_tracking_reset(ioapic); 600 rtc_irq_eoi_tracking_reset(ioapic);
@@ -656,6 +652,7 @@ int kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state)
656 652
657 spin_lock(&ioapic->lock); 653 spin_lock(&ioapic->lock);
658 memcpy(state, ioapic, sizeof(struct kvm_ioapic_state)); 654 memcpy(state, ioapic, sizeof(struct kvm_ioapic_state));
655 state->irr &= ~ioapic->irr_delivered;
659 spin_unlock(&ioapic->lock); 656 spin_unlock(&ioapic->lock);
660 return 0; 657 return 0;
661} 658}
@@ -669,6 +666,7 @@ int kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state)
669 spin_lock(&ioapic->lock); 666 spin_lock(&ioapic->lock);
670 memcpy(ioapic, state, sizeof(struct kvm_ioapic_state)); 667 memcpy(ioapic, state, sizeof(struct kvm_ioapic_state));
671 ioapic->irr = 0; 668 ioapic->irr = 0;
669 ioapic->irr_delivered = 0;
672 update_handled_vectors(ioapic); 670 update_handled_vectors(ioapic);
673 kvm_vcpu_request_scan_ioapic(kvm); 671 kvm_vcpu_request_scan_ioapic(kvm);
674 kvm_ioapic_inject_all(ioapic, state->irr); 672 kvm_ioapic_inject_all(ioapic, state->irr);
diff --git a/arch/x86/kvm/ioapic.h b/arch/x86/kvm/ioapic.h
index c2e36d934af4..ca0b0b4e6256 100644
--- a/arch/x86/kvm/ioapic.h
+++ b/arch/x86/kvm/ioapic.h
@@ -3,7 +3,7 @@
3 3
4#include <linux/kvm_host.h> 4#include <linux/kvm_host.h>
5 5
6#include "iodev.h" 6#include <kvm/iodev.h>
7 7
8struct kvm; 8struct kvm;
9struct kvm_vcpu; 9struct kvm_vcpu;
@@ -77,6 +77,7 @@ struct kvm_ioapic {
77 struct rtc_status rtc_status; 77 struct rtc_status rtc_status;
78 struct delayed_work eoi_inject; 78 struct delayed_work eoi_inject;
79 u32 irq_eoi[IOAPIC_NUM_PINS]; 79 u32 irq_eoi[IOAPIC_NUM_PINS];
80 u32 irr_delivered;
80}; 81};
81 82
82#ifdef DEBUG 83#ifdef DEBUG
@@ -97,13 +98,19 @@ static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm)
97 return kvm->arch.vioapic; 98 return kvm->arch.vioapic;
98} 99}
99 100
101static inline bool kvm_ioapic_handles_vector(struct kvm *kvm, int vector)
102{
103 struct kvm_ioapic *ioapic = kvm->arch.vioapic;
104 smp_rmb();
105 return test_bit(vector, ioapic->handled_vectors);
106}
107
100void kvm_rtc_eoi_tracking_restore_one(struct kvm_vcpu *vcpu); 108void kvm_rtc_eoi_tracking_restore_one(struct kvm_vcpu *vcpu);
101bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 109bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
102 int short_hand, unsigned int dest, int dest_mode); 110 int short_hand, unsigned int dest, int dest_mode);
103int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2); 111int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2);
104void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, 112void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector,
105 int trigger_mode); 113 int trigger_mode);
106bool kvm_ioapic_handles_vector(struct kvm *kvm, int vector);
107int kvm_ioapic_init(struct kvm *kvm); 114int kvm_ioapic_init(struct kvm *kvm);
108void kvm_ioapic_destroy(struct kvm *kvm); 115void kvm_ioapic_destroy(struct kvm *kvm);
109int kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int irq_source_id, 116int kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int irq_source_id,
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h
index 2d03568e9498..ad68c73008c5 100644
--- a/arch/x86/kvm/irq.h
+++ b/arch/x86/kvm/irq.h
@@ -27,7 +27,7 @@
27#include <linux/kvm_host.h> 27#include <linux/kvm_host.h>
28#include <linux/spinlock.h> 28#include <linux/spinlock.h>
29 29
30#include "iodev.h" 30#include <kvm/iodev.h>
31#include "ioapic.h" 31#include "ioapic.h"
32#include "lapic.h" 32#include "lapic.h"
33 33
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 4ee827d7bf36..d67206a7b99a 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -133,6 +133,28 @@ static inline int kvm_apic_id(struct kvm_lapic *apic)
133 return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff; 133 return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
134} 134}
135 135
136/* The logical map is definitely wrong if we have multiple
137 * modes at the same time. (Physical map is always right.)
138 */
139static inline bool kvm_apic_logical_map_valid(struct kvm_apic_map *map)
140{
141 return !(map->mode & (map->mode - 1));
142}
143
144static inline void
145apic_logical_id(struct kvm_apic_map *map, u32 dest_id, u16 *cid, u16 *lid)
146{
147 unsigned lid_bits;
148
149 BUILD_BUG_ON(KVM_APIC_MODE_XAPIC_CLUSTER != 4);
150 BUILD_BUG_ON(KVM_APIC_MODE_XAPIC_FLAT != 8);
151 BUILD_BUG_ON(KVM_APIC_MODE_X2APIC != 16);
152 lid_bits = map->mode;
153
154 *cid = dest_id >> lid_bits;
155 *lid = dest_id & ((1 << lid_bits) - 1);
156}
157
136static void recalculate_apic_map(struct kvm *kvm) 158static void recalculate_apic_map(struct kvm *kvm)
137{ 159{
138 struct kvm_apic_map *new, *old = NULL; 160 struct kvm_apic_map *new, *old = NULL;
@@ -146,48 +168,6 @@ static void recalculate_apic_map(struct kvm *kvm)
146 if (!new) 168 if (!new)
147 goto out; 169 goto out;
148 170
149 new->ldr_bits = 8;
150 /* flat mode is default */
151 new->cid_shift = 8;
152 new->cid_mask = 0;
153 new->lid_mask = 0xff;
154 new->broadcast = APIC_BROADCAST;
155
156 kvm_for_each_vcpu(i, vcpu, kvm) {
157 struct kvm_lapic *apic = vcpu->arch.apic;
158
159 if (!kvm_apic_present(vcpu))
160 continue;
161
162 if (apic_x2apic_mode(apic)) {
163 new->ldr_bits = 32;
164 new->cid_shift = 16;
165 new->cid_mask = new->lid_mask = 0xffff;
166 new->broadcast = X2APIC_BROADCAST;
167 } else if (kvm_apic_get_reg(apic, APIC_LDR)) {
168 if (kvm_apic_get_reg(apic, APIC_DFR) ==
169 APIC_DFR_CLUSTER) {
170 new->cid_shift = 4;
171 new->cid_mask = 0xf;
172 new->lid_mask = 0xf;
173 } else {
174 new->cid_shift = 8;
175 new->cid_mask = 0;
176 new->lid_mask = 0xff;
177 }
178 }
179
180 /*
181 * All APICs have to be configured in the same mode by an OS.
182 * We take advatage of this while building logical id loockup
183 * table. After reset APICs are in software disabled mode, so if
184 * we find apic with different setting we assume this is the mode
185 * OS wants all apics to be in; build lookup table accordingly.
186 */
187 if (kvm_apic_sw_enabled(apic))
188 break;
189 }
190
191 kvm_for_each_vcpu(i, vcpu, kvm) { 171 kvm_for_each_vcpu(i, vcpu, kvm) {
192 struct kvm_lapic *apic = vcpu->arch.apic; 172 struct kvm_lapic *apic = vcpu->arch.apic;
193 u16 cid, lid; 173 u16 cid, lid;
@@ -198,11 +178,25 @@ static void recalculate_apic_map(struct kvm *kvm)
198 178
199 aid = kvm_apic_id(apic); 179 aid = kvm_apic_id(apic);
200 ldr = kvm_apic_get_reg(apic, APIC_LDR); 180 ldr = kvm_apic_get_reg(apic, APIC_LDR);
201 cid = apic_cluster_id(new, ldr);
202 lid = apic_logical_id(new, ldr);
203 181
204 if (aid < ARRAY_SIZE(new->phys_map)) 182 if (aid < ARRAY_SIZE(new->phys_map))
205 new->phys_map[aid] = apic; 183 new->phys_map[aid] = apic;
184
185 if (apic_x2apic_mode(apic)) {
186 new->mode |= KVM_APIC_MODE_X2APIC;
187 } else if (ldr) {
188 ldr = GET_APIC_LOGICAL_ID(ldr);
189 if (kvm_apic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
190 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
191 else
192 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
193 }
194
195 if (!kvm_apic_logical_map_valid(new))
196 continue;
197
198 apic_logical_id(new, ldr, &cid, &lid);
199
206 if (lid && cid < ARRAY_SIZE(new->logical_map)) 200 if (lid && cid < ARRAY_SIZE(new->logical_map))
207 new->logical_map[cid][ffs(lid) - 1] = apic; 201 new->logical_map[cid][ffs(lid) - 1] = apic;
208 } 202 }
@@ -588,15 +582,23 @@ static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
588 apic_update_ppr(apic); 582 apic_update_ppr(apic);
589} 583}
590 584
591static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 dest) 585static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
592{ 586{
593 return dest == (apic_x2apic_mode(apic) ? 587 if (apic_x2apic_mode(apic))
594 X2APIC_BROADCAST : APIC_BROADCAST); 588 return mda == X2APIC_BROADCAST;
589
590 return GET_APIC_DEST_FIELD(mda) == APIC_BROADCAST;
595} 591}
596 592
597static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 dest) 593static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
598{ 594{
599 return kvm_apic_id(apic) == dest || kvm_apic_broadcast(apic, dest); 595 if (kvm_apic_broadcast(apic, mda))
596 return true;
597
598 if (apic_x2apic_mode(apic))
599 return mda == kvm_apic_id(apic);
600
601 return mda == SET_APIC_DEST_FIELD(kvm_apic_id(apic));
600} 602}
601 603
602static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 604static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
@@ -613,6 +615,7 @@ static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
613 && (logical_id & mda & 0xffff) != 0; 615 && (logical_id & mda & 0xffff) != 0;
614 616
615 logical_id = GET_APIC_LOGICAL_ID(logical_id); 617 logical_id = GET_APIC_LOGICAL_ID(logical_id);
618 mda = GET_APIC_DEST_FIELD(mda);
616 619
617 switch (kvm_apic_get_reg(apic, APIC_DFR)) { 620 switch (kvm_apic_get_reg(apic, APIC_DFR)) {
618 case APIC_DFR_FLAT: 621 case APIC_DFR_FLAT:
@@ -627,10 +630,27 @@ static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
627 } 630 }
628} 631}
629 632
633/* KVM APIC implementation has two quirks
634 * - dest always begins at 0 while xAPIC MDA has offset 24,
635 * - IOxAPIC messages have to be delivered (directly) to x2APIC.
636 */
637static u32 kvm_apic_mda(unsigned int dest_id, struct kvm_lapic *source,
638 struct kvm_lapic *target)
639{
640 bool ipi = source != NULL;
641 bool x2apic_mda = apic_x2apic_mode(ipi ? source : target);
642
643 if (!ipi && dest_id == APIC_BROADCAST && x2apic_mda)
644 return X2APIC_BROADCAST;
645
646 return x2apic_mda ? dest_id : SET_APIC_DEST_FIELD(dest_id);
647}
648
630bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 649bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
631 int short_hand, unsigned int dest, int dest_mode) 650 int short_hand, unsigned int dest, int dest_mode)
632{ 651{
633 struct kvm_lapic *target = vcpu->arch.apic; 652 struct kvm_lapic *target = vcpu->arch.apic;
653 u32 mda = kvm_apic_mda(dest, source, target);
634 654
635 apic_debug("target %p, source %p, dest 0x%x, " 655 apic_debug("target %p, source %p, dest 0x%x, "
636 "dest_mode 0x%x, short_hand 0x%x\n", 656 "dest_mode 0x%x, short_hand 0x%x\n",
@@ -640,9 +660,9 @@ bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
640 switch (short_hand) { 660 switch (short_hand) {
641 case APIC_DEST_NOSHORT: 661 case APIC_DEST_NOSHORT:
642 if (dest_mode == APIC_DEST_PHYSICAL) 662 if (dest_mode == APIC_DEST_PHYSICAL)
643 return kvm_apic_match_physical_addr(target, dest); 663 return kvm_apic_match_physical_addr(target, mda);
644 else 664 else
645 return kvm_apic_match_logical_addr(target, dest); 665 return kvm_apic_match_logical_addr(target, mda);
646 case APIC_DEST_SELF: 666 case APIC_DEST_SELF:
647 return target == source; 667 return target == source;
648 case APIC_DEST_ALLINC: 668 case APIC_DEST_ALLINC:
@@ -664,6 +684,7 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
664 struct kvm_lapic **dst; 684 struct kvm_lapic **dst;
665 int i; 685 int i;
666 bool ret = false; 686 bool ret = false;
687 bool x2apic_ipi = src && apic_x2apic_mode(src);
667 688
668 *r = -1; 689 *r = -1;
669 690
@@ -675,15 +696,15 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
675 if (irq->shorthand) 696 if (irq->shorthand)
676 return false; 697 return false;
677 698
699 if (irq->dest_id == (x2apic_ipi ? X2APIC_BROADCAST : APIC_BROADCAST))
700 return false;
701
678 rcu_read_lock(); 702 rcu_read_lock();
679 map = rcu_dereference(kvm->arch.apic_map); 703 map = rcu_dereference(kvm->arch.apic_map);
680 704
681 if (!map) 705 if (!map)
682 goto out; 706 goto out;
683 707
684 if (irq->dest_id == map->broadcast)
685 goto out;
686
687 ret = true; 708 ret = true;
688 709
689 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 710 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
@@ -692,16 +713,20 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
692 713
693 dst = &map->phys_map[irq->dest_id]; 714 dst = &map->phys_map[irq->dest_id];
694 } else { 715 } else {
695 u32 mda = irq->dest_id << (32 - map->ldr_bits); 716 u16 cid;
696 u16 cid = apic_cluster_id(map, mda); 717
718 if (!kvm_apic_logical_map_valid(map)) {
719 ret = false;
720 goto out;
721 }
722
723 apic_logical_id(map, irq->dest_id, &cid, (u16 *)&bitmap);
697 724
698 if (cid >= ARRAY_SIZE(map->logical_map)) 725 if (cid >= ARRAY_SIZE(map->logical_map))
699 goto out; 726 goto out;
700 727
701 dst = map->logical_map[cid]; 728 dst = map->logical_map[cid];
702 729
703 bitmap = apic_logical_id(map, mda);
704
705 if (irq->delivery_mode == APIC_DM_LOWEST) { 730 if (irq->delivery_mode == APIC_DM_LOWEST) {
706 int l = -1; 731 int l = -1;
707 for_each_set_bit(i, &bitmap, 16) { 732 for_each_set_bit(i, &bitmap, 16) {
@@ -1037,7 +1062,7 @@ static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1037 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1062 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1038} 1063}
1039 1064
1040static int apic_mmio_read(struct kvm_io_device *this, 1065static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1041 gpa_t address, int len, void *data) 1066 gpa_t address, int len, void *data)
1042{ 1067{
1043 struct kvm_lapic *apic = to_lapic(this); 1068 struct kvm_lapic *apic = to_lapic(this);
@@ -1357,7 +1382,7 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
1357 return ret; 1382 return ret;
1358} 1383}
1359 1384
1360static int apic_mmio_write(struct kvm_io_device *this, 1385static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1361 gpa_t address, int len, const void *data) 1386 gpa_t address, int len, const void *data)
1362{ 1387{
1363 struct kvm_lapic *apic = to_lapic(this); 1388 struct kvm_lapic *apic = to_lapic(this);
@@ -1497,8 +1522,6 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
1497 return; 1522 return;
1498 } 1523 }
1499 1524
1500 if (!kvm_vcpu_is_bsp(apic->vcpu))
1501 value &= ~MSR_IA32_APICBASE_BSP;
1502 vcpu->arch.apic_base = value; 1525 vcpu->arch.apic_base = value;
1503 1526
1504 /* update jump label if enable bit changes */ 1527 /* update jump label if enable bit changes */
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index 0bc6c656625b..9d28383fc1e7 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -1,7 +1,7 @@
1#ifndef __KVM_X86_LAPIC_H 1#ifndef __KVM_X86_LAPIC_H
2#define __KVM_X86_LAPIC_H 2#define __KVM_X86_LAPIC_H
3 3
4#include "iodev.h" 4#include <kvm/iodev.h>
5 5
6#include <linux/kvm_host.h> 6#include <linux/kvm_host.h>
7 7
@@ -148,21 +148,6 @@ static inline bool kvm_apic_vid_enabled(struct kvm *kvm)
148 return kvm_x86_ops->vm_has_apicv(kvm); 148 return kvm_x86_ops->vm_has_apicv(kvm);
149} 149}
150 150
151static inline u16 apic_cluster_id(struct kvm_apic_map *map, u32 ldr)
152{
153 u16 cid;
154 ldr >>= 32 - map->ldr_bits;
155 cid = (ldr >> map->cid_shift) & map->cid_mask;
156
157 return cid;
158}
159
160static inline u16 apic_logical_id(struct kvm_apic_map *map, u32 ldr)
161{
162 ldr >>= (32 - map->ldr_bits);
163 return ldr & map->lid_mask;
164}
165
166static inline bool kvm_apic_has_events(struct kvm_vcpu *vcpu) 151static inline bool kvm_apic_has_events(struct kvm_vcpu *vcpu)
167{ 152{
168 return vcpu->arch.apic->pending_events; 153 return vcpu->arch.apic->pending_events;
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index cee759299a35..146f295ee322 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -4465,6 +4465,79 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
4465 kvm_flush_remote_tlbs(kvm); 4465 kvm_flush_remote_tlbs(kvm);
4466} 4466}
4467 4467
4468static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
4469 unsigned long *rmapp)
4470{
4471 u64 *sptep;
4472 struct rmap_iterator iter;
4473 int need_tlb_flush = 0;
4474 pfn_t pfn;
4475 struct kvm_mmu_page *sp;
4476
4477 for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
4478 BUG_ON(!(*sptep & PT_PRESENT_MASK));
4479
4480 sp = page_header(__pa(sptep));
4481 pfn = spte_to_pfn(*sptep);
4482
4483 /*
4484 * Only EPT supported for now; otherwise, one would need to
4485 * find out efficiently whether the guest page tables are
4486 * also using huge pages.
4487 */
4488 if (sp->role.direct &&
4489 !kvm_is_reserved_pfn(pfn) &&
4490 PageTransCompound(pfn_to_page(pfn))) {
4491 drop_spte(kvm, sptep);
4492 sptep = rmap_get_first(*rmapp, &iter);
4493 need_tlb_flush = 1;
4494 } else
4495 sptep = rmap_get_next(&iter);
4496 }
4497
4498 return need_tlb_flush;
4499}
4500
4501void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
4502 struct kvm_memory_slot *memslot)
4503{
4504 bool flush = false;
4505 unsigned long *rmapp;
4506 unsigned long last_index, index;
4507 gfn_t gfn_start, gfn_end;
4508
4509 spin_lock(&kvm->mmu_lock);
4510
4511 gfn_start = memslot->base_gfn;
4512 gfn_end = memslot->base_gfn + memslot->npages - 1;
4513
4514 if (gfn_start >= gfn_end)
4515 goto out;
4516
4517 rmapp = memslot->arch.rmap[0];
4518 last_index = gfn_to_index(gfn_end, memslot->base_gfn,
4519 PT_PAGE_TABLE_LEVEL);
4520
4521 for (index = 0; index <= last_index; ++index, ++rmapp) {
4522 if (*rmapp)
4523 flush |= kvm_mmu_zap_collapsible_spte(kvm, rmapp);
4524
4525 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
4526 if (flush) {
4527 kvm_flush_remote_tlbs(kvm);
4528 flush = false;
4529 }
4530 cond_resched_lock(&kvm->mmu_lock);
4531 }
4532 }
4533
4534 if (flush)
4535 kvm_flush_remote_tlbs(kvm);
4536
4537out:
4538 spin_unlock(&kvm->mmu_lock);
4539}
4540
4468void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, 4541void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
4469 struct kvm_memory_slot *memslot) 4542 struct kvm_memory_slot *memslot)
4470{ 4543{
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 8e6b7d869d2f..29fbf9dfdc54 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -38,7 +38,7 @@ static struct kvm_arch_event_perf_mapping {
38}; 38};
39 39
40/* mapping between fixed pmc index and arch_events array */ 40/* mapping between fixed pmc index and arch_events array */
41int fixed_pmc_events[] = {1, 0, 7}; 41static int fixed_pmc_events[] = {1, 0, 7};
42 42
43static bool pmc_is_gp(struct kvm_pmc *pmc) 43static bool pmc_is_gp(struct kvm_pmc *pmc)
44{ 44{
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index cc618c882f90..ce741b8650f6 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -1261,7 +1261,7 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1261 1261
1262 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE | 1262 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1263 MSR_IA32_APICBASE_ENABLE; 1263 MSR_IA32_APICBASE_ENABLE;
1264 if (kvm_vcpu_is_bsp(&svm->vcpu)) 1264 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1265 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP; 1265 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1266 1266
1267 svm_init_osvw(&svm->vcpu); 1267 svm_init_osvw(&svm->vcpu);
@@ -1929,14 +1929,12 @@ static int nop_on_interception(struct vcpu_svm *svm)
1929static int halt_interception(struct vcpu_svm *svm) 1929static int halt_interception(struct vcpu_svm *svm)
1930{ 1930{
1931 svm->next_rip = kvm_rip_read(&svm->vcpu) + 1; 1931 svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
1932 skip_emulated_instruction(&svm->vcpu);
1933 return kvm_emulate_halt(&svm->vcpu); 1932 return kvm_emulate_halt(&svm->vcpu);
1934} 1933}
1935 1934
1936static int vmmcall_interception(struct vcpu_svm *svm) 1935static int vmmcall_interception(struct vcpu_svm *svm)
1937{ 1936{
1938 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; 1937 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
1939 skip_emulated_instruction(&svm->vcpu);
1940 kvm_emulate_hypercall(&svm->vcpu); 1938 kvm_emulate_hypercall(&svm->vcpu);
1941 return 1; 1939 return 1;
1942} 1940}
@@ -2757,11 +2755,11 @@ static int invlpga_interception(struct vcpu_svm *svm)
2757{ 2755{
2758 struct kvm_vcpu *vcpu = &svm->vcpu; 2756 struct kvm_vcpu *vcpu = &svm->vcpu;
2759 2757
2760 trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX], 2758 trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
2761 vcpu->arch.regs[VCPU_REGS_RAX]); 2759 kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2762 2760
2763 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ 2761 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2764 kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]); 2762 kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2765 2763
2766 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; 2764 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2767 skip_emulated_instruction(&svm->vcpu); 2765 skip_emulated_instruction(&svm->vcpu);
@@ -2770,12 +2768,18 @@ static int invlpga_interception(struct vcpu_svm *svm)
2770 2768
2771static int skinit_interception(struct vcpu_svm *svm) 2769static int skinit_interception(struct vcpu_svm *svm)
2772{ 2770{
2773 trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]); 2771 trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2774 2772
2775 kvm_queue_exception(&svm->vcpu, UD_VECTOR); 2773 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2776 return 1; 2774 return 1;
2777} 2775}
2778 2776
2777static int wbinvd_interception(struct vcpu_svm *svm)
2778{
2779 kvm_emulate_wbinvd(&svm->vcpu);
2780 return 1;
2781}
2782
2779static int xsetbv_interception(struct vcpu_svm *svm) 2783static int xsetbv_interception(struct vcpu_svm *svm)
2780{ 2784{
2781 u64 new_bv = kvm_read_edx_eax(&svm->vcpu); 2785 u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
@@ -2902,7 +2906,8 @@ static int rdpmc_interception(struct vcpu_svm *svm)
2902 return 1; 2906 return 1;
2903} 2907}
2904 2908
2905bool check_selective_cr0_intercepted(struct vcpu_svm *svm, unsigned long val) 2909static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
2910 unsigned long val)
2906{ 2911{
2907 unsigned long cr0 = svm->vcpu.arch.cr0; 2912 unsigned long cr0 = svm->vcpu.arch.cr0;
2908 bool ret = false; 2913 bool ret = false;
@@ -2940,7 +2945,10 @@ static int cr_interception(struct vcpu_svm *svm)
2940 return emulate_on_interception(svm); 2945 return emulate_on_interception(svm);
2941 2946
2942 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2947 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2943 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; 2948 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2949 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2950 else
2951 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2944 2952
2945 err = 0; 2953 err = 0;
2946 if (cr >= 16) { /* mov to cr */ 2954 if (cr >= 16) { /* mov to cr */
@@ -3133,7 +3141,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
3133 3141
3134static int rdmsr_interception(struct vcpu_svm *svm) 3142static int rdmsr_interception(struct vcpu_svm *svm)
3135{ 3143{
3136 u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 3144 u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3137 u64 data; 3145 u64 data;
3138 3146
3139 if (svm_get_msr(&svm->vcpu, ecx, &data)) { 3147 if (svm_get_msr(&svm->vcpu, ecx, &data)) {
@@ -3142,8 +3150,8 @@ static int rdmsr_interception(struct vcpu_svm *svm)
3142 } else { 3150 } else {
3143 trace_kvm_msr_read(ecx, data); 3151 trace_kvm_msr_read(ecx, data);
3144 3152
3145 svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff; 3153 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, data & 0xffffffff);
3146 svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32; 3154 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX, data >> 32);
3147 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2; 3155 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3148 skip_emulated_instruction(&svm->vcpu); 3156 skip_emulated_instruction(&svm->vcpu);
3149 } 3157 }
@@ -3246,9 +3254,8 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3246static int wrmsr_interception(struct vcpu_svm *svm) 3254static int wrmsr_interception(struct vcpu_svm *svm)
3247{ 3255{
3248 struct msr_data msr; 3256 struct msr_data msr;
3249 u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 3257 u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3250 u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u) 3258 u64 data = kvm_read_edx_eax(&svm->vcpu);
3251 | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
3252 3259
3253 msr.data = data; 3260 msr.data = data;
3254 msr.index = ecx; 3261 msr.index = ecx;
@@ -3325,7 +3332,7 @@ static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3325 [SVM_EXIT_READ_CR3] = cr_interception, 3332 [SVM_EXIT_READ_CR3] = cr_interception,
3326 [SVM_EXIT_READ_CR4] = cr_interception, 3333 [SVM_EXIT_READ_CR4] = cr_interception,
3327 [SVM_EXIT_READ_CR8] = cr_interception, 3334 [SVM_EXIT_READ_CR8] = cr_interception,
3328 [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, 3335 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception,
3329 [SVM_EXIT_WRITE_CR0] = cr_interception, 3336 [SVM_EXIT_WRITE_CR0] = cr_interception,
3330 [SVM_EXIT_WRITE_CR3] = cr_interception, 3337 [SVM_EXIT_WRITE_CR3] = cr_interception,
3331 [SVM_EXIT_WRITE_CR4] = cr_interception, 3338 [SVM_EXIT_WRITE_CR4] = cr_interception,
@@ -3376,7 +3383,7 @@ static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3376 [SVM_EXIT_STGI] = stgi_interception, 3383 [SVM_EXIT_STGI] = stgi_interception,
3377 [SVM_EXIT_CLGI] = clgi_interception, 3384 [SVM_EXIT_CLGI] = clgi_interception,
3378 [SVM_EXIT_SKINIT] = skinit_interception, 3385 [SVM_EXIT_SKINIT] = skinit_interception,
3379 [SVM_EXIT_WBINVD] = emulate_on_interception, 3386 [SVM_EXIT_WBINVD] = wbinvd_interception,
3380 [SVM_EXIT_MONITOR] = monitor_interception, 3387 [SVM_EXIT_MONITOR] = monitor_interception,
3381 [SVM_EXIT_MWAIT] = mwait_interception, 3388 [SVM_EXIT_MWAIT] = mwait_interception,
3382 [SVM_EXIT_XSETBV] = xsetbv_interception, 3389 [SVM_EXIT_XSETBV] = xsetbv_interception,
@@ -3555,7 +3562,7 @@ static int handle_exit(struct kvm_vcpu *vcpu)
3555 3562
3556 if (exit_code >= ARRAY_SIZE(svm_exit_handlers) 3563 if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3557 || !svm_exit_handlers[exit_code]) { 3564 || !svm_exit_handlers[exit_code]) {
3558 WARN_ONCE(1, "vmx: unexpected exit reason 0x%x\n", exit_code); 3565 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
3559 kvm_queue_exception(vcpu, UD_VECTOR); 3566 kvm_queue_exception(vcpu, UD_VECTOR);
3560 return 1; 3567 return 1;
3561 } 3568 }
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index ae4f6d35d19c..f5e8dce8046c 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -2470,6 +2470,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
2470 vmx->nested.nested_vmx_secondary_ctls_low = 0; 2470 vmx->nested.nested_vmx_secondary_ctls_low = 0;
2471 vmx->nested.nested_vmx_secondary_ctls_high &= 2471 vmx->nested.nested_vmx_secondary_ctls_high &=
2472 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2472 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2473 SECONDARY_EXEC_RDTSCP |
2473 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2474 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2474 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2475 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2475 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2476 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
@@ -3268,8 +3269,8 @@ static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
3268 * default value. 3269 * default value.
3269 */ 3270 */
3270 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 3271 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
3271 save->selector &= ~SELECTOR_RPL_MASK; 3272 save->selector &= ~SEGMENT_RPL_MASK;
3272 save->dpl = save->selector & SELECTOR_RPL_MASK; 3273 save->dpl = save->selector & SEGMENT_RPL_MASK;
3273 save->s = 1; 3274 save->s = 1;
3274 } 3275 }
3275 vmx_set_segment(vcpu, save, seg); 3276 vmx_set_segment(vcpu, save, seg);
@@ -3842,7 +3843,7 @@ static bool code_segment_valid(struct kvm_vcpu *vcpu)
3842 unsigned int cs_rpl; 3843 unsigned int cs_rpl;
3843 3844
3844 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3845 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3845 cs_rpl = cs.selector & SELECTOR_RPL_MASK; 3846 cs_rpl = cs.selector & SEGMENT_RPL_MASK;
3846 3847
3847 if (cs.unusable) 3848 if (cs.unusable)
3848 return false; 3849 return false;
@@ -3870,7 +3871,7 @@ static bool stack_segment_valid(struct kvm_vcpu *vcpu)
3870 unsigned int ss_rpl; 3871 unsigned int ss_rpl;
3871 3872
3872 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3873 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3873 ss_rpl = ss.selector & SELECTOR_RPL_MASK; 3874 ss_rpl = ss.selector & SEGMENT_RPL_MASK;
3874 3875
3875 if (ss.unusable) 3876 if (ss.unusable)
3876 return true; 3877 return true;
@@ -3892,7 +3893,7 @@ static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
3892 unsigned int rpl; 3893 unsigned int rpl;
3893 3894
3894 vmx_get_segment(vcpu, &var, seg); 3895 vmx_get_segment(vcpu, &var, seg);
3895 rpl = var.selector & SELECTOR_RPL_MASK; 3896 rpl = var.selector & SEGMENT_RPL_MASK;
3896 3897
3897 if (var.unusable) 3898 if (var.unusable)
3898 return true; 3899 return true;
@@ -3919,7 +3920,7 @@ static bool tr_valid(struct kvm_vcpu *vcpu)
3919 3920
3920 if (tr.unusable) 3921 if (tr.unusable)
3921 return false; 3922 return false;
3922 if (tr.selector & SELECTOR_TI_MASK) /* TI = 1 */ 3923 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */
3923 return false; 3924 return false;
3924 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3925 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
3925 return false; 3926 return false;
@@ -3937,7 +3938,7 @@ static bool ldtr_valid(struct kvm_vcpu *vcpu)
3937 3938
3938 if (ldtr.unusable) 3939 if (ldtr.unusable)
3939 return true; 3940 return true;
3940 if (ldtr.selector & SELECTOR_TI_MASK) /* TI = 1 */ 3941 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */
3941 return false; 3942 return false;
3942 if (ldtr.type != 2) 3943 if (ldtr.type != 2)
3943 return false; 3944 return false;
@@ -3954,8 +3955,8 @@ static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
3954 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3955 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3955 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3956 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3956 3957
3957 return ((cs.selector & SELECTOR_RPL_MASK) == 3958 return ((cs.selector & SEGMENT_RPL_MASK) ==
3958 (ss.selector & SELECTOR_RPL_MASK)); 3959 (ss.selector & SEGMENT_RPL_MASK));
3959} 3960}
3960 3961
3961/* 3962/*
@@ -4711,7 +4712,7 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu)
4711 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val(); 4712 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
4712 kvm_set_cr8(&vmx->vcpu, 0); 4713 kvm_set_cr8(&vmx->vcpu, 0);
4713 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE; 4714 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE;
4714 if (kvm_vcpu_is_bsp(&vmx->vcpu)) 4715 if (kvm_vcpu_is_reset_bsp(&vmx->vcpu))
4715 apic_base_msr.data |= MSR_IA32_APICBASE_BSP; 4716 apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
4716 apic_base_msr.host_initiated = true; 4717 apic_base_msr.host_initiated = true;
4717 kvm_set_apic_base(&vmx->vcpu, &apic_base_msr); 4718 kvm_set_apic_base(&vmx->vcpu, &apic_base_msr);
@@ -5006,7 +5007,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu,
5006 if (emulate_instruction(vcpu, 0) == EMULATE_DONE) { 5007 if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
5007 if (vcpu->arch.halt_request) { 5008 if (vcpu->arch.halt_request) {
5008 vcpu->arch.halt_request = 0; 5009 vcpu->arch.halt_request = 0;
5009 return kvm_emulate_halt(vcpu); 5010 return kvm_vcpu_halt(vcpu);
5010 } 5011 }
5011 return 1; 5012 return 1;
5012 } 5013 }
@@ -5071,6 +5072,10 @@ static int handle_exception(struct kvm_vcpu *vcpu)
5071 } 5072 }
5072 5073
5073 if (is_invalid_opcode(intr_info)) { 5074 if (is_invalid_opcode(intr_info)) {
5075 if (is_guest_mode(vcpu)) {
5076 kvm_queue_exception(vcpu, UD_VECTOR);
5077 return 1;
5078 }
5074 er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD); 5079 er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
5075 if (er != EMULATE_DONE) 5080 if (er != EMULATE_DONE)
5076 kvm_queue_exception(vcpu, UD_VECTOR); 5081 kvm_queue_exception(vcpu, UD_VECTOR);
@@ -5090,9 +5095,10 @@ static int handle_exception(struct kvm_vcpu *vcpu)
5090 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 5095 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
5091 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5096 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5092 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 5097 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
5093 vcpu->run->internal.ndata = 2; 5098 vcpu->run->internal.ndata = 3;
5094 vcpu->run->internal.data[0] = vect_info; 5099 vcpu->run->internal.data[0] = vect_info;
5095 vcpu->run->internal.data[1] = intr_info; 5100 vcpu->run->internal.data[1] = intr_info;
5101 vcpu->run->internal.data[2] = error_code;
5096 return 0; 5102 return 0;
5097 } 5103 }
5098 5104
@@ -5533,13 +5539,11 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
5533 5539
5534static int handle_halt(struct kvm_vcpu *vcpu) 5540static int handle_halt(struct kvm_vcpu *vcpu)
5535{ 5541{
5536 skip_emulated_instruction(vcpu);
5537 return kvm_emulate_halt(vcpu); 5542 return kvm_emulate_halt(vcpu);
5538} 5543}
5539 5544
5540static int handle_vmcall(struct kvm_vcpu *vcpu) 5545static int handle_vmcall(struct kvm_vcpu *vcpu)
5541{ 5546{
5542 skip_emulated_instruction(vcpu);
5543 kvm_emulate_hypercall(vcpu); 5547 kvm_emulate_hypercall(vcpu);
5544 return 1; 5548 return 1;
5545} 5549}
@@ -5570,7 +5574,6 @@ static int handle_rdpmc(struct kvm_vcpu *vcpu)
5570 5574
5571static int handle_wbinvd(struct kvm_vcpu *vcpu) 5575static int handle_wbinvd(struct kvm_vcpu *vcpu)
5572{ 5576{
5573 skip_emulated_instruction(vcpu);
5574 kvm_emulate_wbinvd(vcpu); 5577 kvm_emulate_wbinvd(vcpu);
5575 return 1; 5578 return 1;
5576} 5579}
@@ -5828,7 +5831,7 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
5828 gpa_t gpa; 5831 gpa_t gpa;
5829 5832
5830 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5833 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5831 if (!kvm_io_bus_write(vcpu->kvm, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 5834 if (!kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
5832 skip_emulated_instruction(vcpu); 5835 skip_emulated_instruction(vcpu);
5833 return 1; 5836 return 1;
5834 } 5837 }
@@ -5909,7 +5912,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
5909 5912
5910 if (vcpu->arch.halt_request) { 5913 if (vcpu->arch.halt_request) {
5911 vcpu->arch.halt_request = 0; 5914 vcpu->arch.halt_request = 0;
5912 ret = kvm_emulate_halt(vcpu); 5915 ret = kvm_vcpu_halt(vcpu);
5913 goto out; 5916 goto out;
5914 } 5917 }
5915 5918
@@ -7318,21 +7321,21 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
7318 else if (port < 0x10000) 7321 else if (port < 0x10000)
7319 bitmap = vmcs12->io_bitmap_b; 7322 bitmap = vmcs12->io_bitmap_b;
7320 else 7323 else
7321 return 1; 7324 return true;
7322 bitmap += (port & 0x7fff) / 8; 7325 bitmap += (port & 0x7fff) / 8;
7323 7326
7324 if (last_bitmap != bitmap) 7327 if (last_bitmap != bitmap)
7325 if (kvm_read_guest(vcpu->kvm, bitmap, &b, 1)) 7328 if (kvm_read_guest(vcpu->kvm, bitmap, &b, 1))
7326 return 1; 7329 return true;
7327 if (b & (1 << (port & 7))) 7330 if (b & (1 << (port & 7)))
7328 return 1; 7331 return true;
7329 7332
7330 port++; 7333 port++;
7331 size--; 7334 size--;
7332 last_bitmap = bitmap; 7335 last_bitmap = bitmap;
7333 } 7336 }
7334 7337
7335 return 0; 7338 return false;
7336} 7339}
7337 7340
7338/* 7341/*
@@ -7348,7 +7351,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
7348 gpa_t bitmap; 7351 gpa_t bitmap;
7349 7352
7350 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 7353 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
7351 return 1; 7354 return true;
7352 7355
7353 /* 7356 /*
7354 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 7357 * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
@@ -7367,10 +7370,10 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
7367 if (msr_index < 1024*8) { 7370 if (msr_index < 1024*8) {
7368 unsigned char b; 7371 unsigned char b;
7369 if (kvm_read_guest(vcpu->kvm, bitmap + msr_index/8, &b, 1)) 7372 if (kvm_read_guest(vcpu->kvm, bitmap + msr_index/8, &b, 1))
7370 return 1; 7373 return true;
7371 return 1 & (b >> (msr_index & 7)); 7374 return 1 & (b >> (msr_index & 7));
7372 } else 7375 } else
7373 return 1; /* let L1 handle the wrong parameter */ 7376 return true; /* let L1 handle the wrong parameter */
7374} 7377}
7375 7378
7376/* 7379/*
@@ -7392,7 +7395,7 @@ static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
7392 case 0: 7395 case 0:
7393 if (vmcs12->cr0_guest_host_mask & 7396 if (vmcs12->cr0_guest_host_mask &
7394 (val ^ vmcs12->cr0_read_shadow)) 7397 (val ^ vmcs12->cr0_read_shadow))
7395 return 1; 7398 return true;
7396 break; 7399 break;
7397 case 3: 7400 case 3:
7398 if ((vmcs12->cr3_target_count >= 1 && 7401 if ((vmcs12->cr3_target_count >= 1 &&
@@ -7403,37 +7406,37 @@ static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
7403 vmcs12->cr3_target_value2 == val) || 7406 vmcs12->cr3_target_value2 == val) ||
7404 (vmcs12->cr3_target_count >= 4 && 7407 (vmcs12->cr3_target_count >= 4 &&
7405 vmcs12->cr3_target_value3 == val)) 7408 vmcs12->cr3_target_value3 == val))
7406 return 0; 7409 return false;
7407 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 7410 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
7408 return 1; 7411 return true;
7409 break; 7412 break;
7410 case 4: 7413 case 4:
7411 if (vmcs12->cr4_guest_host_mask & 7414 if (vmcs12->cr4_guest_host_mask &
7412 (vmcs12->cr4_read_shadow ^ val)) 7415 (vmcs12->cr4_read_shadow ^ val))
7413 return 1; 7416 return true;
7414 break; 7417 break;
7415 case 8: 7418 case 8:
7416 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 7419 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
7417 return 1; 7420 return true;
7418 break; 7421 break;
7419 } 7422 }
7420 break; 7423 break;
7421 case 2: /* clts */ 7424 case 2: /* clts */
7422 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 7425 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
7423 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 7426 (vmcs12->cr0_read_shadow & X86_CR0_TS))
7424 return 1; 7427 return true;
7425 break; 7428 break;
7426 case 1: /* mov from cr */ 7429 case 1: /* mov from cr */
7427 switch (cr) { 7430 switch (cr) {
7428 case 3: 7431 case 3:
7429 if (vmcs12->cpu_based_vm_exec_control & 7432 if (vmcs12->cpu_based_vm_exec_control &
7430 CPU_BASED_CR3_STORE_EXITING) 7433 CPU_BASED_CR3_STORE_EXITING)
7431 return 1; 7434 return true;
7432 break; 7435 break;
7433 case 8: 7436 case 8:
7434 if (vmcs12->cpu_based_vm_exec_control & 7437 if (vmcs12->cpu_based_vm_exec_control &
7435 CPU_BASED_CR8_STORE_EXITING) 7438 CPU_BASED_CR8_STORE_EXITING)
7436 return 1; 7439 return true;
7437 break; 7440 break;
7438 } 7441 }
7439 break; 7442 break;
@@ -7444,14 +7447,14 @@ static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
7444 */ 7447 */
7445 if (vmcs12->cr0_guest_host_mask & 0xe & 7448 if (vmcs12->cr0_guest_host_mask & 0xe &
7446 (val ^ vmcs12->cr0_read_shadow)) 7449 (val ^ vmcs12->cr0_read_shadow))
7447 return 1; 7450 return true;
7448 if ((vmcs12->cr0_guest_host_mask & 0x1) && 7451 if ((vmcs12->cr0_guest_host_mask & 0x1) &&
7449 !(vmcs12->cr0_read_shadow & 0x1) && 7452 !(vmcs12->cr0_read_shadow & 0x1) &&
7450 (val & 0x1)) 7453 (val & 0x1))
7451 return 1; 7454 return true;
7452 break; 7455 break;
7453 } 7456 }
7454 return 0; 7457 return false;
7455} 7458}
7456 7459
7457/* 7460/*
@@ -7474,48 +7477,48 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7474 KVM_ISA_VMX); 7477 KVM_ISA_VMX);
7475 7478
7476 if (vmx->nested.nested_run_pending) 7479 if (vmx->nested.nested_run_pending)
7477 return 0; 7480 return false;
7478 7481
7479 if (unlikely(vmx->fail)) { 7482 if (unlikely(vmx->fail)) {
7480 pr_info_ratelimited("%s failed vm entry %x\n", __func__, 7483 pr_info_ratelimited("%s failed vm entry %x\n", __func__,
7481 vmcs_read32(VM_INSTRUCTION_ERROR)); 7484 vmcs_read32(VM_INSTRUCTION_ERROR));
7482 return 1; 7485 return true;
7483 } 7486 }
7484 7487
7485 switch (exit_reason) { 7488 switch (exit_reason) {
7486 case EXIT_REASON_EXCEPTION_NMI: 7489 case EXIT_REASON_EXCEPTION_NMI:
7487 if (!is_exception(intr_info)) 7490 if (!is_exception(intr_info))
7488 return 0; 7491 return false;
7489 else if (is_page_fault(intr_info)) 7492 else if (is_page_fault(intr_info))
7490 return enable_ept; 7493 return enable_ept;
7491 else if (is_no_device(intr_info) && 7494 else if (is_no_device(intr_info) &&
7492 !(vmcs12->guest_cr0 & X86_CR0_TS)) 7495 !(vmcs12->guest_cr0 & X86_CR0_TS))
7493 return 0; 7496 return false;
7494 return vmcs12->exception_bitmap & 7497 return vmcs12->exception_bitmap &
7495 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 7498 (1u << (intr_info & INTR_INFO_VECTOR_MASK));
7496 case EXIT_REASON_EXTERNAL_INTERRUPT: 7499 case EXIT_REASON_EXTERNAL_INTERRUPT:
7497 return 0; 7500 return false;
7498 case EXIT_REASON_TRIPLE_FAULT: 7501 case EXIT_REASON_TRIPLE_FAULT:
7499 return 1; 7502 return true;
7500 case EXIT_REASON_PENDING_INTERRUPT: 7503 case EXIT_REASON_PENDING_INTERRUPT:
7501 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING); 7504 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
7502 case EXIT_REASON_NMI_WINDOW: 7505 case EXIT_REASON_NMI_WINDOW:
7503 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING); 7506 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
7504 case EXIT_REASON_TASK_SWITCH: 7507 case EXIT_REASON_TASK_SWITCH:
7505 return 1; 7508 return true;
7506 case EXIT_REASON_CPUID: 7509 case EXIT_REASON_CPUID:
7507 if (kvm_register_read(vcpu, VCPU_REGS_RAX) == 0xa) 7510 if (kvm_register_read(vcpu, VCPU_REGS_RAX) == 0xa)
7508 return 0; 7511 return false;
7509 return 1; 7512 return true;
7510 case EXIT_REASON_HLT: 7513 case EXIT_REASON_HLT:
7511 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 7514 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
7512 case EXIT_REASON_INVD: 7515 case EXIT_REASON_INVD:
7513 return 1; 7516 return true;
7514 case EXIT_REASON_INVLPG: 7517 case EXIT_REASON_INVLPG:
7515 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 7518 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
7516 case EXIT_REASON_RDPMC: 7519 case EXIT_REASON_RDPMC:
7517 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 7520 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
7518 case EXIT_REASON_RDTSC: 7521 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
7519 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 7522 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
7520 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 7523 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
7521 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 7524 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
@@ -7527,7 +7530,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7527 * VMX instructions trap unconditionally. This allows L1 to 7530 * VMX instructions trap unconditionally. This allows L1 to
7528 * emulate them for its L2 guest, i.e., allows 3-level nesting! 7531 * emulate them for its L2 guest, i.e., allows 3-level nesting!
7529 */ 7532 */
7530 return 1; 7533 return true;
7531 case EXIT_REASON_CR_ACCESS: 7534 case EXIT_REASON_CR_ACCESS:
7532 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 7535 return nested_vmx_exit_handled_cr(vcpu, vmcs12);
7533 case EXIT_REASON_DR_ACCESS: 7536 case EXIT_REASON_DR_ACCESS:
@@ -7538,7 +7541,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7538 case EXIT_REASON_MSR_WRITE: 7541 case EXIT_REASON_MSR_WRITE:
7539 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 7542 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
7540 case EXIT_REASON_INVALID_STATE: 7543 case EXIT_REASON_INVALID_STATE:
7541 return 1; 7544 return true;
7542 case EXIT_REASON_MWAIT_INSTRUCTION: 7545 case EXIT_REASON_MWAIT_INSTRUCTION:
7543 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 7546 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
7544 case EXIT_REASON_MONITOR_INSTRUCTION: 7547 case EXIT_REASON_MONITOR_INSTRUCTION:
@@ -7548,7 +7551,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7548 nested_cpu_has2(vmcs12, 7551 nested_cpu_has2(vmcs12,
7549 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 7552 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
7550 case EXIT_REASON_MCE_DURING_VMENTRY: 7553 case EXIT_REASON_MCE_DURING_VMENTRY:
7551 return 0; 7554 return false;
7552 case EXIT_REASON_TPR_BELOW_THRESHOLD: 7555 case EXIT_REASON_TPR_BELOW_THRESHOLD:
7553 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 7556 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
7554 case EXIT_REASON_APIC_ACCESS: 7557 case EXIT_REASON_APIC_ACCESS:
@@ -7557,7 +7560,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7557 case EXIT_REASON_APIC_WRITE: 7560 case EXIT_REASON_APIC_WRITE:
7558 case EXIT_REASON_EOI_INDUCED: 7561 case EXIT_REASON_EOI_INDUCED:
7559 /* apic_write and eoi_induced should exit unconditionally. */ 7562 /* apic_write and eoi_induced should exit unconditionally. */
7560 return 1; 7563 return true;
7561 case EXIT_REASON_EPT_VIOLATION: 7564 case EXIT_REASON_EPT_VIOLATION:
7562 /* 7565 /*
7563 * L0 always deals with the EPT violation. If nested EPT is 7566 * L0 always deals with the EPT violation. If nested EPT is
@@ -7565,7 +7568,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7565 * missing in the guest EPT table (EPT12), the EPT violation 7568 * missing in the guest EPT table (EPT12), the EPT violation
7566 * will be injected with nested_ept_inject_page_fault() 7569 * will be injected with nested_ept_inject_page_fault()
7567 */ 7570 */
7568 return 0; 7571 return false;
7569 case EXIT_REASON_EPT_MISCONFIG: 7572 case EXIT_REASON_EPT_MISCONFIG:
7570 /* 7573 /*
7571 * L2 never uses directly L1's EPT, but rather L0's own EPT 7574 * L2 never uses directly L1's EPT, but rather L0's own EPT
@@ -7573,11 +7576,11 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7573 * (EPT on EPT). So any problems with the structure of the 7576 * (EPT on EPT). So any problems with the structure of the
7574 * table is L0's fault. 7577 * table is L0's fault.
7575 */ 7578 */
7576 return 0; 7579 return false;
7577 case EXIT_REASON_WBINVD: 7580 case EXIT_REASON_WBINVD:
7578 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 7581 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
7579 case EXIT_REASON_XSETBV: 7582 case EXIT_REASON_XSETBV:
7580 return 1; 7583 return true;
7581 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 7584 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
7582 /* 7585 /*
7583 * This should never happen, since it is not possible to 7586 * This should never happen, since it is not possible to
@@ -7587,7 +7590,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7587 */ 7590 */
7588 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 7591 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
7589 default: 7592 default:
7590 return 1; 7593 return true;
7591 } 7594 }
7592} 7595}
7593 7596
@@ -8522,6 +8525,9 @@ static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
8522 exec_control); 8525 exec_control);
8523 } 8526 }
8524 } 8527 }
8528 if (nested && !vmx->rdtscp_enabled)
8529 vmx->nested.nested_vmx_secondary_ctls_high &=
8530 ~SECONDARY_EXEC_RDTSCP;
8525 } 8531 }
8526 8532
8527 /* Exposing INVPCID only when PCID is exposed */ 8533 /* Exposing INVPCID only when PCID is exposed */
@@ -8622,10 +8628,11 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
8622 struct vmcs12 *vmcs12) 8628 struct vmcs12 *vmcs12)
8623{ 8629{
8624 struct vcpu_vmx *vmx = to_vmx(vcpu); 8630 struct vcpu_vmx *vmx = to_vmx(vcpu);
8631 int maxphyaddr = cpuid_maxphyaddr(vcpu);
8625 8632
8626 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 8633 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
8627 /* TODO: Also verify bits beyond physical address width are 0 */ 8634 if (!PAGE_ALIGNED(vmcs12->apic_access_addr) ||
8628 if (!PAGE_ALIGNED(vmcs12->apic_access_addr)) 8635 vmcs12->apic_access_addr >> maxphyaddr)
8629 return false; 8636 return false;
8630 8637
8631 /* 8638 /*
@@ -8641,8 +8648,8 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
8641 } 8648 }
8642 8649
8643 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 8650 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
8644 /* TODO: Also verify bits beyond physical address width are 0 */ 8651 if (!PAGE_ALIGNED(vmcs12->virtual_apic_page_addr) ||
8645 if (!PAGE_ALIGNED(vmcs12->virtual_apic_page_addr)) 8652 vmcs12->virtual_apic_page_addr >> maxphyaddr)
8646 return false; 8653 return false;
8647 8654
8648 if (vmx->nested.virtual_apic_page) /* shouldn't happen */ 8655 if (vmx->nested.virtual_apic_page) /* shouldn't happen */
@@ -8665,7 +8672,8 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
8665 } 8672 }
8666 8673
8667 if (nested_cpu_has_posted_intr(vmcs12)) { 8674 if (nested_cpu_has_posted_intr(vmcs12)) {
8668 if (!IS_ALIGNED(vmcs12->posted_intr_desc_addr, 64)) 8675 if (!IS_ALIGNED(vmcs12->posted_intr_desc_addr, 64) ||
8676 vmcs12->posted_intr_desc_addr >> maxphyaddr)
8669 return false; 8677 return false;
8670 8678
8671 if (vmx->nested.pi_desc_page) { /* shouldn't happen */ 8679 if (vmx->nested.pi_desc_page) { /* shouldn't happen */
@@ -8864,9 +8872,9 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
8864 8872
8865static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 8873static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
8866 unsigned long count_field, 8874 unsigned long count_field,
8867 unsigned long addr_field, 8875 unsigned long addr_field)
8868 int maxphyaddr)
8869{ 8876{
8877 int maxphyaddr;
8870 u64 count, addr; 8878 u64 count, addr;
8871 8879
8872 if (vmcs12_read_any(vcpu, count_field, &count) || 8880 if (vmcs12_read_any(vcpu, count_field, &count) ||
@@ -8876,6 +8884,7 @@ static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
8876 } 8884 }
8877 if (count == 0) 8885 if (count == 0)
8878 return 0; 8886 return 0;
8887 maxphyaddr = cpuid_maxphyaddr(vcpu);
8879 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr || 8888 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
8880 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) { 8889 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) {
8881 pr_warn_ratelimited( 8890 pr_warn_ratelimited(
@@ -8889,19 +8898,16 @@ static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
8889static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu, 8898static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu,
8890 struct vmcs12 *vmcs12) 8899 struct vmcs12 *vmcs12)
8891{ 8900{
8892 int maxphyaddr;
8893
8894 if (vmcs12->vm_exit_msr_load_count == 0 && 8901 if (vmcs12->vm_exit_msr_load_count == 0 &&
8895 vmcs12->vm_exit_msr_store_count == 0 && 8902 vmcs12->vm_exit_msr_store_count == 0 &&
8896 vmcs12->vm_entry_msr_load_count == 0) 8903 vmcs12->vm_entry_msr_load_count == 0)
8897 return 0; /* Fast path */ 8904 return 0; /* Fast path */
8898 maxphyaddr = cpuid_maxphyaddr(vcpu);
8899 if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT, 8905 if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT,
8900 VM_EXIT_MSR_LOAD_ADDR, maxphyaddr) || 8906 VM_EXIT_MSR_LOAD_ADDR) ||
8901 nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT, 8907 nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT,
8902 VM_EXIT_MSR_STORE_ADDR, maxphyaddr) || 8908 VM_EXIT_MSR_STORE_ADDR) ||
8903 nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT, 8909 nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT,
8904 VM_ENTRY_MSR_LOAD_ADDR, maxphyaddr)) 8910 VM_ENTRY_MSR_LOAD_ADDR))
8905 return -EINVAL; 8911 return -EINVAL;
8906 return 0; 8912 return 0;
8907} 8913}
@@ -9151,8 +9157,9 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
9151 exec_control &= ~SECONDARY_EXEC_RDTSCP; 9157 exec_control &= ~SECONDARY_EXEC_RDTSCP;
9152 /* Take the following fields only from vmcs12 */ 9158 /* Take the following fields only from vmcs12 */
9153 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 9159 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
9160 SECONDARY_EXEC_RDTSCP |
9154 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 9161 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
9155 SECONDARY_EXEC_APIC_REGISTER_VIRT); 9162 SECONDARY_EXEC_APIC_REGISTER_VIRT);
9156 if (nested_cpu_has(vmcs12, 9163 if (nested_cpu_has(vmcs12,
9157 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) 9164 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
9158 exec_control |= vmcs12->secondary_vm_exec_control; 9165 exec_control |= vmcs12->secondary_vm_exec_control;
@@ -9385,7 +9392,6 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
9385 } 9392 }
9386 9393
9387 if (!nested_get_vmcs12_pages(vcpu, vmcs12)) { 9394 if (!nested_get_vmcs12_pages(vcpu, vmcs12)) {
9388 /*TODO: Also verify bits beyond physical address width are 0*/
9389 nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 9395 nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
9390 return 1; 9396 return 1;
9391 } 9397 }
@@ -9524,7 +9530,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
9524 vmcs12->launch_state = 1; 9530 vmcs12->launch_state = 1;
9525 9531
9526 if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) 9532 if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT)
9527 return kvm_emulate_halt(vcpu); 9533 return kvm_vcpu_halt(vcpu);
9528 9534
9529 vmx->nested.nested_run_pending = 1; 9535 vmx->nested.nested_run_pending = 1;
9530 9536
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 32bf19ef3115..2b2dd030ea3b 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -801,6 +801,17 @@ unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
801} 801}
802EXPORT_SYMBOL_GPL(kvm_get_cr8); 802EXPORT_SYMBOL_GPL(kvm_get_cr8);
803 803
804static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
805{
806 int i;
807
808 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
809 for (i = 0; i < KVM_NR_DB_REGS; i++)
810 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
811 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
812 }
813}
814
804static void kvm_update_dr6(struct kvm_vcpu *vcpu) 815static void kvm_update_dr6(struct kvm_vcpu *vcpu)
805{ 816{
806 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) 817 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
@@ -3149,6 +3160,7 @@ static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
3149 return -EINVAL; 3160 return -EINVAL;
3150 3161
3151 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db)); 3162 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
3163 kvm_update_dr0123(vcpu);
3152 vcpu->arch.dr6 = dbgregs->dr6; 3164 vcpu->arch.dr6 = dbgregs->dr6;
3153 kvm_update_dr6(vcpu); 3165 kvm_update_dr6(vcpu);
3154 vcpu->arch.dr7 = dbgregs->dr7; 3166 vcpu->arch.dr7 = dbgregs->dr7;
@@ -4114,8 +4126,8 @@ static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
4114 do { 4126 do {
4115 n = min(len, 8); 4127 n = min(len, 8);
4116 if (!(vcpu->arch.apic && 4128 if (!(vcpu->arch.apic &&
4117 !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v)) 4129 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
4118 && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v)) 4130 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
4119 break; 4131 break;
4120 handled += n; 4132 handled += n;
4121 addr += n; 4133 addr += n;
@@ -4134,8 +4146,9 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
4134 do { 4146 do {
4135 n = min(len, 8); 4147 n = min(len, 8);
4136 if (!(vcpu->arch.apic && 4148 if (!(vcpu->arch.apic &&
4137 !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v)) 4149 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
4138 && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v)) 4150 addr, n, v))
4151 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
4139 break; 4152 break;
4140 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v); 4153 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v);
4141 handled += n; 4154 handled += n;
@@ -4475,7 +4488,8 @@ mmio:
4475 return X86EMUL_CONTINUE; 4488 return X86EMUL_CONTINUE;
4476} 4489}
4477 4490
4478int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr, 4491static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
4492 unsigned long addr,
4479 void *val, unsigned int bytes, 4493 void *val, unsigned int bytes,
4480 struct x86_exception *exception, 4494 struct x86_exception *exception,
4481 const struct read_write_emulator_ops *ops) 4495 const struct read_write_emulator_ops *ops)
@@ -4538,7 +4552,7 @@ static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
4538 exception, &read_emultor); 4552 exception, &read_emultor);
4539} 4553}
4540 4554
4541int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, 4555static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
4542 unsigned long addr, 4556 unsigned long addr,
4543 const void *val, 4557 const void *val,
4544 unsigned int bytes, 4558 unsigned int bytes,
@@ -4629,10 +4643,10 @@ static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
4629 int r; 4643 int r;
4630 4644
4631 if (vcpu->arch.pio.in) 4645 if (vcpu->arch.pio.in)
4632 r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port, 4646 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
4633 vcpu->arch.pio.size, pd); 4647 vcpu->arch.pio.size, pd);
4634 else 4648 else
4635 r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS, 4649 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
4636 vcpu->arch.pio.port, vcpu->arch.pio.size, 4650 vcpu->arch.pio.port, vcpu->arch.pio.size,
4637 pd); 4651 pd);
4638 return r; 4652 return r;
@@ -4705,7 +4719,7 @@ static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
4705 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); 4719 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
4706} 4720}
4707 4721
4708int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) 4722int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
4709{ 4723{
4710 if (!need_emulate_wbinvd(vcpu)) 4724 if (!need_emulate_wbinvd(vcpu))
4711 return X86EMUL_CONTINUE; 4725 return X86EMUL_CONTINUE;
@@ -4722,19 +4736,29 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
4722 wbinvd(); 4736 wbinvd();
4723 return X86EMUL_CONTINUE; 4737 return X86EMUL_CONTINUE;
4724} 4738}
4739
4740int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
4741{
4742 kvm_x86_ops->skip_emulated_instruction(vcpu);
4743 return kvm_emulate_wbinvd_noskip(vcpu);
4744}
4725EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd); 4745EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
4726 4746
4747
4748
4727static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) 4749static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
4728{ 4750{
4729 kvm_emulate_wbinvd(emul_to_vcpu(ctxt)); 4751 kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
4730} 4752}
4731 4753
4732int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) 4754static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
4755 unsigned long *dest)
4733{ 4756{
4734 return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest); 4757 return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
4735} 4758}
4736 4759
4737int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) 4760static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
4761 unsigned long value)
4738{ 4762{
4739 4763
4740 return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value); 4764 return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
@@ -5816,7 +5840,7 @@ void kvm_arch_exit(void)
5816 free_percpu(shared_msrs); 5840 free_percpu(shared_msrs);
5817} 5841}
5818 5842
5819int kvm_emulate_halt(struct kvm_vcpu *vcpu) 5843int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
5820{ 5844{
5821 ++vcpu->stat.halt_exits; 5845 ++vcpu->stat.halt_exits;
5822 if (irqchip_in_kernel(vcpu->kvm)) { 5846 if (irqchip_in_kernel(vcpu->kvm)) {
@@ -5827,6 +5851,13 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu)
5827 return 0; 5851 return 0;
5828 } 5852 }
5829} 5853}
5854EXPORT_SYMBOL_GPL(kvm_vcpu_halt);
5855
5856int kvm_emulate_halt(struct kvm_vcpu *vcpu)
5857{
5858 kvm_x86_ops->skip_emulated_instruction(vcpu);
5859 return kvm_vcpu_halt(vcpu);
5860}
5830EXPORT_SYMBOL_GPL(kvm_emulate_halt); 5861EXPORT_SYMBOL_GPL(kvm_emulate_halt);
5831 5862
5832int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 5863int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
@@ -5903,7 +5934,7 @@ static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
5903 lapic_irq.dest_id = apicid; 5934 lapic_irq.dest_id = apicid;
5904 5935
5905 lapic_irq.delivery_mode = APIC_DM_REMRD; 5936 lapic_irq.delivery_mode = APIC_DM_REMRD;
5906 kvm_irq_delivery_to_apic(kvm, 0, &lapic_irq, NULL); 5937 kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
5907} 5938}
5908 5939
5909int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) 5940int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
@@ -5911,6 +5942,8 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
5911 unsigned long nr, a0, a1, a2, a3, ret; 5942 unsigned long nr, a0, a1, a2, a3, ret;
5912 int op_64_bit, r = 1; 5943 int op_64_bit, r = 1;
5913 5944
5945 kvm_x86_ops->skip_emulated_instruction(vcpu);
5946
5914 if (kvm_hv_hypercall_enabled(vcpu->kvm)) 5947 if (kvm_hv_hypercall_enabled(vcpu->kvm))
5915 return kvm_hv_hypercall(vcpu); 5948 return kvm_hv_hypercall(vcpu);
5916 5949
@@ -6164,7 +6197,7 @@ void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
6164} 6197}
6165 6198
6166/* 6199/*
6167 * Returns 1 to let __vcpu_run() continue the guest execution loop without 6200 * Returns 1 to let vcpu_run() continue the guest execution loop without
6168 * exiting to the userspace. Otherwise, the value will be returned to the 6201 * exiting to the userspace. Otherwise, the value will be returned to the
6169 * userspace. 6202 * userspace.
6170 */ 6203 */
@@ -6301,6 +6334,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
6301 set_debugreg(vcpu->arch.eff_db[2], 2); 6334 set_debugreg(vcpu->arch.eff_db[2], 2);
6302 set_debugreg(vcpu->arch.eff_db[3], 3); 6335 set_debugreg(vcpu->arch.eff_db[3], 3);
6303 set_debugreg(vcpu->arch.dr6, 6); 6336 set_debugreg(vcpu->arch.dr6, 6);
6337 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
6304 } 6338 }
6305 6339
6306 trace_kvm_entry(vcpu->vcpu_id); 6340 trace_kvm_entry(vcpu->vcpu_id);
@@ -6382,42 +6416,47 @@ out:
6382 return r; 6416 return r;
6383} 6417}
6384 6418
6419static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
6420{
6421 if (!kvm_arch_vcpu_runnable(vcpu)) {
6422 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
6423 kvm_vcpu_block(vcpu);
6424 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6425 if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
6426 return 1;
6427 }
6428
6429 kvm_apic_accept_events(vcpu);
6430 switch(vcpu->arch.mp_state) {
6431 case KVM_MP_STATE_HALTED:
6432 vcpu->arch.pv.pv_unhalted = false;
6433 vcpu->arch.mp_state =
6434 KVM_MP_STATE_RUNNABLE;
6435 case KVM_MP_STATE_RUNNABLE:
6436 vcpu->arch.apf.halted = false;
6437 break;
6438 case KVM_MP_STATE_INIT_RECEIVED:
6439 break;
6440 default:
6441 return -EINTR;
6442 break;
6443 }
6444 return 1;
6445}
6385 6446
6386static int __vcpu_run(struct kvm_vcpu *vcpu) 6447static int vcpu_run(struct kvm_vcpu *vcpu)
6387{ 6448{
6388 int r; 6449 int r;
6389 struct kvm *kvm = vcpu->kvm; 6450 struct kvm *kvm = vcpu->kvm;
6390 6451
6391 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 6452 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6392 6453
6393 r = 1; 6454 for (;;) {
6394 while (r > 0) {
6395 if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 6455 if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
6396 !vcpu->arch.apf.halted) 6456 !vcpu->arch.apf.halted)
6397 r = vcpu_enter_guest(vcpu); 6457 r = vcpu_enter_guest(vcpu);
6398 else { 6458 else
6399 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 6459 r = vcpu_block(kvm, vcpu);
6400 kvm_vcpu_block(vcpu);
6401 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6402 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
6403 kvm_apic_accept_events(vcpu);
6404 switch(vcpu->arch.mp_state) {
6405 case KVM_MP_STATE_HALTED:
6406 vcpu->arch.pv.pv_unhalted = false;
6407 vcpu->arch.mp_state =
6408 KVM_MP_STATE_RUNNABLE;
6409 case KVM_MP_STATE_RUNNABLE:
6410 vcpu->arch.apf.halted = false;
6411 break;
6412 case KVM_MP_STATE_INIT_RECEIVED:
6413 break;
6414 default:
6415 r = -EINTR;
6416 break;
6417 }
6418 }
6419 }
6420
6421 if (r <= 0) 6460 if (r <= 0)
6422 break; 6461 break;
6423 6462
@@ -6429,6 +6468,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
6429 r = -EINTR; 6468 r = -EINTR;
6430 vcpu->run->exit_reason = KVM_EXIT_INTR; 6469 vcpu->run->exit_reason = KVM_EXIT_INTR;
6431 ++vcpu->stat.request_irq_exits; 6470 ++vcpu->stat.request_irq_exits;
6471 break;
6432 } 6472 }
6433 6473
6434 kvm_check_async_pf_completion(vcpu); 6474 kvm_check_async_pf_completion(vcpu);
@@ -6437,6 +6477,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
6437 r = -EINTR; 6477 r = -EINTR;
6438 vcpu->run->exit_reason = KVM_EXIT_INTR; 6478 vcpu->run->exit_reason = KVM_EXIT_INTR;
6439 ++vcpu->stat.signal_exits; 6479 ++vcpu->stat.signal_exits;
6480 break;
6440 } 6481 }
6441 if (need_resched()) { 6482 if (need_resched()) {
6442 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 6483 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
@@ -6568,7 +6609,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
6568 } else 6609 } else
6569 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed); 6610 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
6570 6611
6571 r = __vcpu_run(vcpu); 6612 r = vcpu_run(vcpu);
6572 6613
6573out: 6614out:
6574 post_kvm_run_save(vcpu); 6615 post_kvm_run_save(vcpu);
@@ -7075,11 +7116,14 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu)
7075 kvm_clear_exception_queue(vcpu); 7116 kvm_clear_exception_queue(vcpu);
7076 7117
7077 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); 7118 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
7119 kvm_update_dr0123(vcpu);
7078 vcpu->arch.dr6 = DR6_INIT; 7120 vcpu->arch.dr6 = DR6_INIT;
7079 kvm_update_dr6(vcpu); 7121 kvm_update_dr6(vcpu);
7080 vcpu->arch.dr7 = DR7_FIXED_1; 7122 vcpu->arch.dr7 = DR7_FIXED_1;
7081 kvm_update_dr7(vcpu); 7123 kvm_update_dr7(vcpu);
7082 7124
7125 vcpu->arch.cr2 = 0;
7126
7083 kvm_make_request(KVM_REQ_EVENT, vcpu); 7127 kvm_make_request(KVM_REQ_EVENT, vcpu);
7084 vcpu->arch.apf.msr_val = 0; 7128 vcpu->arch.apf.msr_val = 0;
7085 vcpu->arch.st.msr_val = 0; 7129 vcpu->arch.st.msr_val = 0;
@@ -7240,7 +7284,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
7240 7284
7241 vcpu->arch.pv.pv_unhalted = false; 7285 vcpu->arch.pv.pv_unhalted = false;
7242 vcpu->arch.emulate_ctxt.ops = &emulate_ops; 7286 vcpu->arch.emulate_ctxt.ops = &emulate_ops;
7243 if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu)) 7287 if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_reset_bsp(vcpu))
7244 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 7288 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
7245 else 7289 else
7246 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED; 7290 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
@@ -7288,6 +7332,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
7288 vcpu->arch.guest_supported_xcr0 = 0; 7332 vcpu->arch.guest_supported_xcr0 = 0;
7289 vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; 7333 vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
7290 7334
7335 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
7336
7291 kvm_async_pf_hash_reset(vcpu); 7337 kvm_async_pf_hash_reset(vcpu);
7292 kvm_pmu_init(vcpu); 7338 kvm_pmu_init(vcpu);
7293 7339
@@ -7428,7 +7474,7 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
7428 7474
7429 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 7475 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
7430 if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) { 7476 if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
7431 kvm_kvfree(free->arch.rmap[i]); 7477 kvfree(free->arch.rmap[i]);
7432 free->arch.rmap[i] = NULL; 7478 free->arch.rmap[i] = NULL;
7433 } 7479 }
7434 if (i == 0) 7480 if (i == 0)
@@ -7436,7 +7482,7 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
7436 7482
7437 if (!dont || free->arch.lpage_info[i - 1] != 7483 if (!dont || free->arch.lpage_info[i - 1] !=
7438 dont->arch.lpage_info[i - 1]) { 7484 dont->arch.lpage_info[i - 1]) {
7439 kvm_kvfree(free->arch.lpage_info[i - 1]); 7485 kvfree(free->arch.lpage_info[i - 1]);
7440 free->arch.lpage_info[i - 1] = NULL; 7486 free->arch.lpage_info[i - 1] = NULL;
7441 } 7487 }
7442 } 7488 }
@@ -7490,12 +7536,12 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
7490 7536
7491out_free: 7537out_free:
7492 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 7538 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
7493 kvm_kvfree(slot->arch.rmap[i]); 7539 kvfree(slot->arch.rmap[i]);
7494 slot->arch.rmap[i] = NULL; 7540 slot->arch.rmap[i] = NULL;
7495 if (i == 0) 7541 if (i == 0)
7496 continue; 7542 continue;
7497 7543
7498 kvm_kvfree(slot->arch.lpage_info[i - 1]); 7544 kvfree(slot->arch.lpage_info[i - 1]);
7499 slot->arch.lpage_info[i - 1] = NULL; 7545 slot->arch.lpage_info[i - 1] = NULL;
7500 } 7546 }
7501 return -ENOMEM; 7547 return -ENOMEM;
@@ -7618,6 +7664,23 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
7618 new = id_to_memslot(kvm->memslots, mem->slot); 7664 new = id_to_memslot(kvm->memslots, mem->slot);
7619 7665
7620 /* 7666 /*
7667 * Dirty logging tracks sptes in 4k granularity, meaning that large
7668 * sptes have to be split. If live migration is successful, the guest
7669 * in the source machine will be destroyed and large sptes will be
7670 * created in the destination. However, if the guest continues to run
7671 * in the source machine (for example if live migration fails), small
7672 * sptes will remain around and cause bad performance.
7673 *
7674 * Scan sptes if dirty logging has been stopped, dropping those
7675 * which can be collapsed into a single large-page spte. Later
7676 * page faults will create the large-page sptes.
7677 */
7678 if ((change != KVM_MR_DELETE) &&
7679 (old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
7680 !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
7681 kvm_mmu_zap_collapsible_sptes(kvm, new);
7682
7683 /*
7621 * Set up write protection and/or dirty logging for the new slot. 7684 * Set up write protection and/or dirty logging for the new slot.
7622 * 7685 *
7623 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have 7686 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 9793322751e0..40d2473836c9 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -82,18 +82,15 @@ static notrace cycle_t vread_pvclock(int *mode)
82 cycle_t ret; 82 cycle_t ret;
83 u64 last; 83 u64 last;
84 u32 version; 84 u32 version;
85 u32 migrate_count;
85 u8 flags; 86 u8 flags;
86 unsigned cpu, cpu1; 87 unsigned cpu, cpu1;
87 88
88 89
89 /* 90 /*
90 * Note: hypervisor must guarantee that: 91 * When looping to get a consistent (time-info, tsc) pair, we
91 * 1. cpu ID number maps 1:1 to per-CPU pvclock time info. 92 * also need to deal with the possibility we can switch vcpus,
92 * 2. that per-CPU pvclock time info is updated if the 93 * so make sure we always re-fetch time-info for the current vcpu.
93 * underlying CPU changes.
94 * 3. that version is increased whenever underlying CPU
95 * changes.
96 *
97 */ 94 */
98 do { 95 do {
99 cpu = __getcpu() & VGETCPU_CPU_MASK; 96 cpu = __getcpu() & VGETCPU_CPU_MASK;
@@ -102,20 +99,27 @@ static notrace cycle_t vread_pvclock(int *mode)
102 * __getcpu() calls (Gleb). 99 * __getcpu() calls (Gleb).
103 */ 100 */
104 101
105 pvti = get_pvti(cpu); 102 /* Make sure migrate_count will change if we leave the VCPU. */
103 do {
104 pvti = get_pvti(cpu);
105 migrate_count = pvti->migrate_count;
106
107 cpu1 = cpu;
108 cpu = __getcpu() & VGETCPU_CPU_MASK;
109 } while (unlikely(cpu != cpu1));
106 110
107 version = __pvclock_read_cycles(&pvti->pvti, &ret, &flags); 111 version = __pvclock_read_cycles(&pvti->pvti, &ret, &flags);
108 112
109 /* 113 /*
110 * Test we're still on the cpu as well as the version. 114 * Test we're still on the cpu as well as the version.
111 * We could have been migrated just after the first 115 * - We must read TSC of pvti's VCPU.
112 * vgetcpu but before fetching the version, so we 116 * - KVM doesn't follow the versioning protocol, so data could
113 * wouldn't notice a version change. 117 * change before version if we left the VCPU.
114 */ 118 */
115 cpu1 = __getcpu() & VGETCPU_CPU_MASK; 119 smp_rmb();
116 } while (unlikely(cpu != cpu1 || 120 } while (unlikely((pvti->pvti.version & 1) ||
117 (pvti->pvti.version & 1) || 121 pvti->pvti.version != version ||
118 pvti->pvti.version != version)); 122 pvti->migrate_count != migrate_count));
119 123
120 if (unlikely(!(flags & PVCLOCK_TSC_STABLE_BIT))) 124 if (unlikely(!(flags & PVCLOCK_TSC_STABLE_BIT)))
121 *mode = VCLOCK_NONE; 125 *mode = VCLOCK_NONE;
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-08 12:12:15 -0500