aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/virtual/kvm/api.txt335
-rw-r--r--Documentation/virtual/kvm/devices/s390_flic.txt3
-rw-r--r--MAINTAINERS2
-rw-r--r--arch/arm/include/asm/kvm_arm.h1
-rw-r--r--arch/arm/include/asm/kvm_host.h15
-rw-r--r--arch/arm/include/asm/kvm_mmio.h22
-rw-r--r--arch/arm/include/uapi/asm/kvm.h3
-rw-r--r--arch/arm/kernel/asm-offsets.c4
-rw-r--r--arch/arm/kvm/Kconfig30
-rw-r--r--arch/arm/kvm/Makefile12
-rw-r--r--arch/arm/kvm/arm.c45
-rw-r--r--arch/arm/kvm/guest.c18
-rw-r--r--arch/arm/kvm/interrupts_head.S8
-rw-r--r--arch/arm/kvm/mmio.c64
-rw-r--r--arch/arm/kvm/mmu.c134
-rw-r--r--arch/arm/kvm/trace.h48
-rw-r--r--arch/arm64/include/asm/esr.h1
-rw-r--r--arch/arm64/include/asm/kvm_arm.h1
-rw-r--r--arch/arm64/include/asm/kvm_host.h15
-rw-r--r--arch/arm64/include/asm/kvm_mmio.h22
-rw-r--r--arch/arm64/include/uapi/asm/kvm.h3
-rw-r--r--arch/arm64/kvm/Kconfig18
-rw-r--r--arch/arm64/kvm/Makefile20
-rw-r--r--arch/mips/include/asm/asmmacro-32.h128
-rw-r--r--arch/mips/include/asm/asmmacro.h218
-rw-r--r--arch/mips/include/asm/fpu.h20
-rw-r--r--arch/mips/include/asm/kdebug.h3
-rw-r--r--arch/mips/include/asm/kvm_host.h125
-rw-r--r--arch/mips/include/asm/processor.h2
-rw-r--r--arch/mips/include/uapi/asm/kvm.h164
-rw-r--r--arch/mips/kernel/asm-offsets.c105
-rw-r--r--arch/mips/kernel/genex.S15
-rw-r--r--arch/mips/kernel/ptrace.c30
-rw-r--r--arch/mips/kernel/r4k_fpu.S2
-rw-r--r--arch/mips/kernel/traps.c33
-rw-r--r--arch/mips/kvm/Makefile8
-rw-r--r--arch/mips/kvm/emulate.c332
-rw-r--r--arch/mips/kvm/fpu.S122
-rw-r--r--arch/mips/kvm/locore.S38
-rw-r--r--arch/mips/kvm/mips.c472
-rw-r--r--arch/mips/kvm/msa.S161
-rw-r--r--arch/mips/kvm/stats.c4
-rw-r--r--arch/mips/kvm/tlb.c6
-rw-r--r--arch/mips/kvm/trap_emul.c199
-rw-r--r--arch/powerpc/kvm/mpic.c17
-rw-r--r--arch/powerpc/kvm/powerpc.c4
-rw-r--r--arch/s390/include/asm/kvm_host.h46
-rw-r--r--arch/s390/include/uapi/asm/kvm.h4
-rw-r--r--arch/s390/include/uapi/asm/sie.h4
-rw-r--r--arch/s390/kernel/asm-offsets.c1
-rw-r--r--arch/s390/kvm/diag.c6
-rw-r--r--arch/s390/kvm/gaccess.c296
-rw-r--r--arch/s390/kvm/gaccess.h21
-rw-r--r--arch/s390/kvm/guestdbg.c8
-rw-r--r--arch/s390/kvm/intercept.c5
-rw-r--r--arch/s390/kvm/interrupt.c1101
-rw-r--r--arch/s390/kvm/kvm-s390.c398
-rw-r--r--arch/s390/kvm/kvm-s390.h51
-rw-r--r--arch/s390/kvm/priv.c144
-rw-r--r--arch/s390/kvm/sigp.c7
-rw-r--r--arch/x86/include/asm/kvm_host.h28
-rw-r--r--arch/x86/include/asm/kvm_para.h2
-rw-r--r--arch/x86/include/asm/pvclock.h1
-rw-r--r--arch/x86/include/uapi/asm/vmx.h1
-rw-r--r--arch/x86/kernel/pvclock.c44
-rw-r--r--arch/x86/kvm/Makefile2
-rw-r--r--arch/x86/kvm/cpuid.c33
-rw-r--r--arch/x86/kvm/cpuid.h8
-rw-r--r--arch/x86/kvm/emulate.c193
-rw-r--r--arch/x86/kvm/i8254.c14
-rw-r--r--arch/x86/kvm/i8254.h2
-rw-r--r--arch/x86/kvm/i8259.c12
-rw-r--r--arch/x86/kvm/ioapic.c22
-rw-r--r--arch/x86/kvm/ioapic.h11
-rw-r--r--arch/x86/kvm/irq.h2
-rw-r--r--arch/x86/kvm/lapic.c147
-rw-r--r--arch/x86/kvm/lapic.h17
-rw-r--r--arch/x86/kvm/mmu.c73
-rw-r--r--arch/x86/kvm/pmu.c2
-rw-r--r--arch/x86/kvm/svm.c43
-rw-r--r--arch/x86/kvm/vmx.c146
-rw-r--r--arch/x86/kvm/x86.c157
-rw-r--r--arch/x86/vdso/vclock_gettime.c34
-rw-r--r--include/kvm/arm_arch_timer.h31
-rw-r--r--include/kvm/arm_vgic.h117
-rw-r--r--include/kvm/iodev.h (renamed from virt/kvm/iodev.h)28
-rw-r--r--include/linux/kvm_host.h32
-rw-r--r--include/linux/sched.h8
-rw-r--r--include/uapi/linux/kvm.h65
-rw-r--r--kernel/sched/core.c15
-rw-r--r--virt/kvm/arm/arch_timer.c45
-rw-r--r--virt/kvm/arm/vgic-v2-emul.c71
-rw-r--r--virt/kvm/arm/vgic-v3-emul.c246
-rw-r--r--virt/kvm/arm/vgic.c479
-rw-r--r--virt/kvm/arm/vgic.h37
-rw-r--r--virt/kvm/coalesced_mmio.c7
-rw-r--r--virt/kvm/eventfd.c9
-rw-r--r--virt/kvm/irqchip.c2
-rw-r--r--virt/kvm/kvm_main.c148
99 files changed, 5401 insertions, 2057 deletions
diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index b112efc816f1..bc9f6fe44e27 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -997,7 +997,7 @@ for vm-wide capabilities.
9974.38 KVM_GET_MP_STATE 9974.38 KVM_GET_MP_STATE
998 998
999Capability: KVM_CAP_MP_STATE 999Capability: KVM_CAP_MP_STATE
1000Architectures: x86, s390 1000Architectures: x86, s390, arm, arm64
1001Type: vcpu ioctl 1001Type: vcpu ioctl
1002Parameters: struct kvm_mp_state (out) 1002Parameters: struct kvm_mp_state (out)
1003Returns: 0 on success; -1 on error 1003Returns: 0 on success; -1 on error
@@ -1011,7 +1011,7 @@ uniprocessor guests).
1011 1011
1012Possible values are: 1012Possible values are:
1013 1013
1014 - KVM_MP_STATE_RUNNABLE: the vcpu is currently running [x86] 1014 - KVM_MP_STATE_RUNNABLE: the vcpu is currently running [x86,arm/arm64]
1015 - KVM_MP_STATE_UNINITIALIZED: the vcpu is an application processor (AP) 1015 - KVM_MP_STATE_UNINITIALIZED: the vcpu is an application processor (AP)
1016 which has not yet received an INIT signal [x86] 1016 which has not yet received an INIT signal [x86]
1017 - KVM_MP_STATE_INIT_RECEIVED: the vcpu has received an INIT signal, and is 1017 - KVM_MP_STATE_INIT_RECEIVED: the vcpu has received an INIT signal, and is
@@ -1020,7 +1020,7 @@ Possible values are:
1020 is waiting for an interrupt [x86] 1020 is waiting for an interrupt [x86]
1021 - KVM_MP_STATE_SIPI_RECEIVED: the vcpu has just received a SIPI (vector 1021 - KVM_MP_STATE_SIPI_RECEIVED: the vcpu has just received a SIPI (vector
1022 accessible via KVM_GET_VCPU_EVENTS) [x86] 1022 accessible via KVM_GET_VCPU_EVENTS) [x86]
1023 - KVM_MP_STATE_STOPPED: the vcpu is stopped [s390] 1023 - KVM_MP_STATE_STOPPED: the vcpu is stopped [s390,arm/arm64]
1024 - KVM_MP_STATE_CHECK_STOP: the vcpu is in a special error state [s390] 1024 - KVM_MP_STATE_CHECK_STOP: the vcpu is in a special error state [s390]
1025 - KVM_MP_STATE_OPERATING: the vcpu is operating (running or halted) 1025 - KVM_MP_STATE_OPERATING: the vcpu is operating (running or halted)
1026 [s390] 1026 [s390]
@@ -1031,11 +1031,15 @@ On x86, this ioctl is only useful after KVM_CREATE_IRQCHIP. Without an
1031in-kernel irqchip, the multiprocessing state must be maintained by userspace on 1031in-kernel irqchip, the multiprocessing state must be maintained by userspace on
1032these architectures. 1032these architectures.
1033 1033
1034For arm/arm64:
1035
1036The only states that are valid are KVM_MP_STATE_STOPPED and
1037KVM_MP_STATE_RUNNABLE which reflect if the vcpu is paused or not.
1034 1038
10354.39 KVM_SET_MP_STATE 10394.39 KVM_SET_MP_STATE
1036 1040
1037Capability: KVM_CAP_MP_STATE 1041Capability: KVM_CAP_MP_STATE
1038Architectures: x86, s390 1042Architectures: x86, s390, arm, arm64
1039Type: vcpu ioctl 1043Type: vcpu ioctl
1040Parameters: struct kvm_mp_state (in) 1044Parameters: struct kvm_mp_state (in)
1041Returns: 0 on success; -1 on error 1045Returns: 0 on success; -1 on error
@@ -1047,6 +1051,10 @@ On x86, this ioctl is only useful after KVM_CREATE_IRQCHIP. Without an
1047in-kernel irqchip, the multiprocessing state must be maintained by userspace on 1051in-kernel irqchip, the multiprocessing state must be maintained by userspace on
1048these architectures. 1052these architectures.
1049 1053
1054For arm/arm64:
1055
1056The only states that are valid are KVM_MP_STATE_STOPPED and
1057KVM_MP_STATE_RUNNABLE which reflect if the vcpu should be paused or not.
1050 1058
10514.40 KVM_SET_IDENTITY_MAP_ADDR 10594.40 KVM_SET_IDENTITY_MAP_ADDR
1052 1060
@@ -1967,15 +1975,25 @@ registers, find a list below:
1967 MIPS | KVM_REG_MIPS_CP0_STATUS | 32 1975 MIPS | KVM_REG_MIPS_CP0_STATUS | 32
1968 MIPS | KVM_REG_MIPS_CP0_CAUSE | 32 1976 MIPS | KVM_REG_MIPS_CP0_CAUSE | 32
1969 MIPS | KVM_REG_MIPS_CP0_EPC | 64 1977 MIPS | KVM_REG_MIPS_CP0_EPC | 64
1978 MIPS | KVM_REG_MIPS_CP0_PRID | 32
1970 MIPS | KVM_REG_MIPS_CP0_CONFIG | 32 1979 MIPS | KVM_REG_MIPS_CP0_CONFIG | 32
1971 MIPS | KVM_REG_MIPS_CP0_CONFIG1 | 32 1980 MIPS | KVM_REG_MIPS_CP0_CONFIG1 | 32
1972 MIPS | KVM_REG_MIPS_CP0_CONFIG2 | 32 1981 MIPS | KVM_REG_MIPS_CP0_CONFIG2 | 32
1973 MIPS | KVM_REG_MIPS_CP0_CONFIG3 | 32 1982 MIPS | KVM_REG_MIPS_CP0_CONFIG3 | 32
1983 MIPS | KVM_REG_MIPS_CP0_CONFIG4 | 32
1984 MIPS | KVM_REG_MIPS_CP0_CONFIG5 | 32
1974 MIPS | KVM_REG_MIPS_CP0_CONFIG7 | 32 1985 MIPS | KVM_REG_MIPS_CP0_CONFIG7 | 32
1975 MIPS | KVM_REG_MIPS_CP0_ERROREPC | 64 1986 MIPS | KVM_REG_MIPS_CP0_ERROREPC | 64
1976 MIPS | KVM_REG_MIPS_COUNT_CTL | 64 1987 MIPS | KVM_REG_MIPS_COUNT_CTL | 64
1977 MIPS | KVM_REG_MIPS_COUNT_RESUME | 64 1988 MIPS | KVM_REG_MIPS_COUNT_RESUME | 64
1978 MIPS | KVM_REG_MIPS_COUNT_HZ | 64 1989 MIPS | KVM_REG_MIPS_COUNT_HZ | 64
1990 MIPS | KVM_REG_MIPS_FPR_32(0..31) | 32
1991 MIPS | KVM_REG_MIPS_FPR_64(0..31) | 64
1992 MIPS | KVM_REG_MIPS_VEC_128(0..31) | 128
1993 MIPS | KVM_REG_MIPS_FCR_IR | 32
1994 MIPS | KVM_REG_MIPS_FCR_CSR | 32
1995 MIPS | KVM_REG_MIPS_MSA_IR | 32
1996 MIPS | KVM_REG_MIPS_MSA_CSR | 32
1979 1997
1980ARM registers are mapped using the lower 32 bits. The upper 16 of that 1998ARM registers are mapped using the lower 32 bits. The upper 16 of that
1981is the register group type, or coprocessor number: 1999is the register group type, or coprocessor number:
@@ -2029,6 +2047,25 @@ patterns depending on whether they're 32-bit or 64-bit registers:
2029MIPS KVM control registers (see above) have the following id bit patterns: 2047MIPS KVM control registers (see above) have the following id bit patterns:
2030 0x7030 0000 0002 <reg:16> 2048 0x7030 0000 0002 <reg:16>
2031 2049
2050MIPS FPU registers (see KVM_REG_MIPS_FPR_{32,64}() above) have the following
2051id bit patterns depending on the size of the register being accessed. They are
2052always accessed according to the current guest FPU mode (Status.FR and
2053Config5.FRE), i.e. as the guest would see them, and they become unpredictable
2054if the guest FPU mode is changed. MIPS SIMD Architecture (MSA) vector
2055registers (see KVM_REG_MIPS_VEC_128() above) have similar patterns as they
2056overlap the FPU registers:
2057 0x7020 0000 0003 00 <0:3> <reg:5> (32-bit FPU registers)
2058 0x7030 0000 0003 00 <0:3> <reg:5> (64-bit FPU registers)
2059 0x7040 0000 0003 00 <0:3> <reg:5> (128-bit MSA vector registers)
2060
2061MIPS FPU control registers (see KVM_REG_MIPS_FCR_{IR,CSR} above) have the
2062following id bit patterns:
2063 0x7020 0000 0003 01 <0:3> <reg:5>
2064
2065MIPS MSA control registers (see KVM_REG_MIPS_MSA_{IR,CSR} above) have the
2066following id bit patterns:
2067 0x7020 0000 0003 02 <0:3> <reg:5>
2068
2032 2069
20334.69 KVM_GET_ONE_REG 20704.69 KVM_GET_ONE_REG
2034 2071
@@ -2234,7 +2271,7 @@ into the hash PTE second double word).
22344.75 KVM_IRQFD 22714.75 KVM_IRQFD
2235 2272
2236Capability: KVM_CAP_IRQFD 2273Capability: KVM_CAP_IRQFD
2237Architectures: x86 s390 2274Architectures: x86 s390 arm arm64
2238Type: vm ioctl 2275Type: vm ioctl
2239Parameters: struct kvm_irqfd (in) 2276Parameters: struct kvm_irqfd (in)
2240Returns: 0 on success, -1 on error 2277Returns: 0 on success, -1 on error
@@ -2260,6 +2297,10 @@ Note that closing the resamplefd is not sufficient to disable the
2260irqfd. The KVM_IRQFD_FLAG_RESAMPLE is only necessary on assignment 2297irqfd. The KVM_IRQFD_FLAG_RESAMPLE is only necessary on assignment
2261and need not be specified with KVM_IRQFD_FLAG_DEASSIGN. 2298and need not be specified with KVM_IRQFD_FLAG_DEASSIGN.
2262 2299
2300On ARM/ARM64, the gsi field in the kvm_irqfd struct specifies the Shared
2301Peripheral Interrupt (SPI) index, such that the GIC interrupt ID is
2302given by gsi + 32.
2303
22634.76 KVM_PPC_ALLOCATE_HTAB 23044.76 KVM_PPC_ALLOCATE_HTAB
2264 2305
2265Capability: KVM_CAP_PPC_ALLOC_HTAB 2306Capability: KVM_CAP_PPC_ALLOC_HTAB
@@ -2716,6 +2757,227 @@ The fields in each entry are defined as follows:
2716 eax, ebx, ecx, edx: the values returned by the cpuid instruction for 2757 eax, ebx, ecx, edx: the values returned by the cpuid instruction for
2717 this function/index combination 2758 this function/index combination
2718 2759
27604.89 KVM_S390_MEM_OP
2761
2762Capability: KVM_CAP_S390_MEM_OP
2763Architectures: s390
2764Type: vcpu ioctl
2765Parameters: struct kvm_s390_mem_op (in)
2766Returns: = 0 on success,
2767 < 0 on generic error (e.g. -EFAULT or -ENOMEM),
2768 > 0 if an exception occurred while walking the page tables
2769
2770Read or write data from/to the logical (virtual) memory of a VPCU.
2771
2772Parameters are specified via the following structure:
2773
2774struct kvm_s390_mem_op {
2775 __u64 gaddr; /* the guest address */
2776 __u64 flags; /* flags */
2777 __u32 size; /* amount of bytes */
2778 __u32 op; /* type of operation */
2779 __u64 buf; /* buffer in userspace */
2780 __u8 ar; /* the access register number */
2781 __u8 reserved[31]; /* should be set to 0 */
2782};
2783
2784The type of operation is specified in the "op" field. It is either
2785KVM_S390_MEMOP_LOGICAL_READ for reading from logical memory space or
2786KVM_S390_MEMOP_LOGICAL_WRITE for writing to logical memory space. The
2787KVM_S390_MEMOP_F_CHECK_ONLY flag can be set in the "flags" field to check
2788whether the corresponding memory access would create an access exception
2789(without touching the data in the memory at the destination). In case an
2790access exception occurred while walking the MMU tables of the guest, the
2791ioctl returns a positive error number to indicate the type of exception.
2792This exception is also raised directly at the corresponding VCPU if the
2793flag KVM_S390_MEMOP_F_INJECT_EXCEPTION is set in the "flags" field.
2794
2795The start address of the memory region has to be specified in the "gaddr"
2796field, and the length of the region in the "size" field. "buf" is the buffer
2797supplied by the userspace application where the read data should be written
2798to for KVM_S390_MEMOP_LOGICAL_READ, or where the data that should be written
2799is stored for a KVM_S390_MEMOP_LOGICAL_WRITE. "buf" is unused and can be NULL
2800when KVM_S390_MEMOP_F_CHECK_ONLY is specified. "ar" designates the access
2801register number to be used.
2802
2803The "reserved" field is meant for future extensions. It is not used by
2804KVM with the currently defined set of flags.
2805
28064.90 KVM_S390_GET_SKEYS
2807
2808Capability: KVM_CAP_S390_SKEYS
2809Architectures: s390
2810Type: vm ioctl
2811Parameters: struct kvm_s390_skeys
2812Returns: 0 on success, KVM_S390_GET_KEYS_NONE if guest is not using storage
2813 keys, negative value on error
2814
2815This ioctl is used to get guest storage key values on the s390
2816architecture. The ioctl takes parameters via the kvm_s390_skeys struct.
2817
2818struct kvm_s390_skeys {
2819 __u64 start_gfn;
2820 __u64 count;
2821 __u64 skeydata_addr;
2822 __u32 flags;
2823 __u32 reserved[9];
2824};
2825
2826The start_gfn field is the number of the first guest frame whose storage keys
2827you want to get.
2828
2829The count field is the number of consecutive frames (starting from start_gfn)
2830whose storage keys to get. The count field must be at least 1 and the maximum
2831allowed value is defined as KVM_S390_SKEYS_ALLOC_MAX. Values outside this range
2832will cause the ioctl to return -EINVAL.
2833
2834The skeydata_addr field is the address to a buffer large enough to hold count
2835bytes. This buffer will be filled with storage key data by the ioctl.
2836
28374.91 KVM_S390_SET_SKEYS
2838
2839Capability: KVM_CAP_S390_SKEYS
2840Architectures: s390
2841Type: vm ioctl
2842Parameters: struct kvm_s390_skeys
2843Returns: 0 on success, negative value on error
2844
2845This ioctl is used to set guest storage key values on the s390
2846architecture. The ioctl takes parameters via the kvm_s390_skeys struct.
2847See section on KVM_S390_GET_SKEYS for struct definition.
2848
2849The start_gfn field is the number of the first guest frame whose storage keys
2850you want to set.
2851
2852The count field is the number of consecutive frames (starting from start_gfn)
2853whose storage keys to get. The count field must be at least 1 and the maximum
2854allowed value is defined as KVM_S390_SKEYS_ALLOC_MAX. Values outside this range
2855will cause the ioctl to return -EINVAL.
2856
2857The skeydata_addr field is the address to a buffer containing count bytes of
2858storage keys. Each byte in the buffer will be set as the storage key for a
2859single frame starting at start_gfn for count frames.
2860
2861Note: If any architecturally invalid key value is found in the given data then
2862the ioctl will return -EINVAL.
2863
28644.92 KVM_S390_IRQ
2865
2866Capability: KVM_CAP_S390_INJECT_IRQ
2867Architectures: s390
2868Type: vcpu ioctl
2869Parameters: struct kvm_s390_irq (in)
2870Returns: 0 on success, -1 on error
2871Errors:
2872 EINVAL: interrupt type is invalid
2873 type is KVM_S390_SIGP_STOP and flag parameter is invalid value
2874 type is KVM_S390_INT_EXTERNAL_CALL and code is bigger
2875 than the maximum of VCPUs
2876 EBUSY: type is KVM_S390_SIGP_SET_PREFIX and vcpu is not stopped
2877 type is KVM_S390_SIGP_STOP and a stop irq is already pending
2878 type is KVM_S390_INT_EXTERNAL_CALL and an external call interrupt
2879 is already pending
2880
2881Allows to inject an interrupt to the guest.
2882
2883Using struct kvm_s390_irq as a parameter allows
2884to inject additional payload which is not
2885possible via KVM_S390_INTERRUPT.
2886
2887Interrupt parameters are passed via kvm_s390_irq:
2888
2889struct kvm_s390_irq {
2890 __u64 type;
2891 union {
2892 struct kvm_s390_io_info io;
2893 struct kvm_s390_ext_info ext;
2894 struct kvm_s390_pgm_info pgm;
2895 struct kvm_s390_emerg_info emerg;
2896 struct kvm_s390_extcall_info extcall;
2897 struct kvm_s390_prefix_info prefix;
2898 struct kvm_s390_stop_info stop;
2899 struct kvm_s390_mchk_info mchk;
2900 char reserved[64];
2901 } u;
2902};
2903
2904type can be one of the following:
2905
2906KVM_S390_SIGP_STOP - sigp stop; parameter in .stop
2907KVM_S390_PROGRAM_INT - program check; parameters in .pgm
2908KVM_S390_SIGP_SET_PREFIX - sigp set prefix; parameters in .prefix
2909KVM_S390_RESTART - restart; no parameters
2910KVM_S390_INT_CLOCK_COMP - clock comparator interrupt; no parameters
2911KVM_S390_INT_CPU_TIMER - CPU timer interrupt; no parameters
2912KVM_S390_INT_EMERGENCY - sigp emergency; parameters in .emerg
2913KVM_S390_INT_EXTERNAL_CALL - sigp external call; parameters in .extcall
2914KVM_S390_MCHK - machine check interrupt; parameters in .mchk
2915
2916
2917Note that the vcpu ioctl is asynchronous to vcpu execution.
2918
29194.94 KVM_S390_GET_IRQ_STATE
2920
2921Capability: KVM_CAP_S390_IRQ_STATE
2922Architectures: s390
2923Type: vcpu ioctl
2924Parameters: struct kvm_s390_irq_state (out)
2925Returns: >= number of bytes copied into buffer,
2926 -EINVAL if buffer size is 0,
2927 -ENOBUFS if buffer size is too small to fit all pending interrupts,
2928 -EFAULT if the buffer address was invalid
2929
2930This ioctl allows userspace to retrieve the complete state of all currently
2931pending interrupts in a single buffer. Use cases include migration
2932and introspection. The parameter structure contains the address of a
2933userspace buffer and its length:
2934
2935struct kvm_s390_irq_state {
2936 __u64 buf;
2937 __u32 flags;
2938 __u32 len;
2939 __u32 reserved[4];
2940};
2941
2942Userspace passes in the above struct and for each pending interrupt a
2943struct kvm_s390_irq is copied to the provided buffer.
2944
2945If -ENOBUFS is returned the buffer provided was too small and userspace
2946may retry with a bigger buffer.
2947
29484.95 KVM_S390_SET_IRQ_STATE
2949
2950Capability: KVM_CAP_S390_IRQ_STATE
2951Architectures: s390
2952Type: vcpu ioctl
2953Parameters: struct kvm_s390_irq_state (in)
2954Returns: 0 on success,
2955 -EFAULT if the buffer address was invalid,
2956 -EINVAL for an invalid buffer length (see below),
2957 -EBUSY if there were already interrupts pending,
2958 errors occurring when actually injecting the
2959 interrupt. See KVM_S390_IRQ.
2960
2961This ioctl allows userspace to set the complete state of all cpu-local
2962interrupts currently pending for the vcpu. It is intended for restoring
2963interrupt state after a migration. The input parameter is a userspace buffer
2964containing a struct kvm_s390_irq_state:
2965
2966struct kvm_s390_irq_state {
2967 __u64 buf;
2968 __u32 len;
2969 __u32 pad;
2970};
2971
2972The userspace memory referenced by buf contains a struct kvm_s390_irq
2973for each interrupt to be injected into the guest.
2974If one of the interrupts could not be injected for some reason the
2975ioctl aborts.
2976
2977len must be a multiple of sizeof(struct kvm_s390_irq). It must be > 0
2978and it must not exceed (max_vcpus + 32) * sizeof(struct kvm_s390_irq),
2979which is the maximum number of possibly pending cpu-local interrupts.
2980
27195. The kvm_run structure 29815. The kvm_run structure
2720------------------------ 2982------------------------
2721 2983
@@ -3189,6 +3451,31 @@ Parameters: none
3189This capability enables the in-kernel irqchip for s390. Please refer to 3451This capability enables the in-kernel irqchip for s390. Please refer to
3190"4.24 KVM_CREATE_IRQCHIP" for details. 3452"4.24 KVM_CREATE_IRQCHIP" for details.
3191 3453
34546.9 KVM_CAP_MIPS_FPU
3455
3456Architectures: mips
3457Target: vcpu
3458Parameters: args[0] is reserved for future use (should be 0).
3459
3460This capability allows the use of the host Floating Point Unit by the guest. It
3461allows the Config1.FP bit to be set to enable the FPU in the guest. Once this is
3462done the KVM_REG_MIPS_FPR_* and KVM_REG_MIPS_FCR_* registers can be accessed
3463(depending on the current guest FPU register mode), and the Status.FR,
3464Config5.FRE bits are accessible via the KVM API and also from the guest,
3465depending on them being supported by the FPU.
3466
34676.10 KVM_CAP_MIPS_MSA
3468
3469Architectures: mips
3470Target: vcpu
3471Parameters: args[0] is reserved for future use (should be 0).
3472
3473This capability allows the use of the MIPS SIMD Architecture (MSA) by the guest.
3474It allows the Config3.MSAP bit to be set to enable the use of MSA by the guest.
3475Once this is done the KVM_REG_MIPS_VEC_* and KVM_REG_MIPS_MSA_* registers can be
3476accessed, and the Config5.MSAEn bit is accessible via the KVM API and also from
3477the guest.
3478
31927. Capabilities that can be enabled on VMs 34797. Capabilities that can be enabled on VMs
3193------------------------------------------ 3480------------------------------------------
3194 3481
@@ -3248,3 +3535,41 @@ All other orders will be handled completely in user space.
3248Only privileged operation exceptions will be checked for in the kernel (or even 3535Only privileged operation exceptions will be checked for in the kernel (or even
3249in the hardware prior to interception). If this capability is not enabled, the 3536in the hardware prior to interception). If this capability is not enabled, the
3250old way of handling SIGP orders is used (partially in kernel and user space). 3537old way of handling SIGP orders is used (partially in kernel and user space).
3538
35397.3 KVM_CAP_S390_VECTOR_REGISTERS
3540
3541Architectures: s390
3542Parameters: none
3543Returns: 0 on success, negative value on error
3544
3545Allows use of the vector registers introduced with z13 processor, and
3546provides for the synchronization between host and user space. Will
3547return -EINVAL if the machine does not support vectors.
3548
35497.4 KVM_CAP_S390_USER_STSI
3550
3551Architectures: s390
3552Parameters: none
3553
3554This capability allows post-handlers for the STSI instruction. After
3555initial handling in the kernel, KVM exits to user space with
3556KVM_EXIT_S390_STSI to allow user space to insert further data.
3557
3558Before exiting to userspace, kvm handlers should fill in s390_stsi field of
3559vcpu->run:
3560struct {
3561 __u64 addr;
3562 __u8 ar;
3563 __u8 reserved;
3564 __u8 fc;
3565 __u8 sel1;
3566 __u16 sel2;
3567} s390_stsi;
3568
3569@addr - guest address of STSI SYSIB
3570@fc - function code
3571@sel1 - selector 1
3572@sel2 - selector 2
3573@ar - access register number
3574
3575KVM handlers should exit to userspace with rc = -EREMOTE.
diff --git a/Documentation/virtual/kvm/devices/s390_flic.txt b/Documentation/virtual/kvm/devices/s390_flic.txt
index 4ceef53164b0..d1ad9d5cae46 100644
--- a/Documentation/virtual/kvm/devices/s390_flic.txt
+++ b/Documentation/virtual/kvm/devices/s390_flic.txt
@@ -27,6 +27,9 @@ Groups:
27 Copies all floating interrupts into a buffer provided by userspace. 27 Copies all floating interrupts into a buffer provided by userspace.
28 When the buffer is too small it returns -ENOMEM, which is the indication 28 When the buffer is too small it returns -ENOMEM, which is the indication
29 for userspace to try again with a bigger buffer. 29 for userspace to try again with a bigger buffer.
30 -ENOBUFS is returned when the allocation of a kernelspace buffer has
31 failed.
32 -EFAULT is returned when copying data to userspace failed.
30 All interrupts remain pending, i.e. are not deleted from the list of 33 All interrupts remain pending, i.e. are not deleted from the list of
31 currently pending interrupts. 34 currently pending interrupts.
32 attr->addr contains the userspace address of the buffer into which all 35 attr->addr contains the userspace address of the buffer into which all
diff --git a/MAINTAINERS b/MAINTAINERS
index e8bdf1b17cdb..b84686826b23 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -5591,6 +5591,8 @@ S: Supported
5591F: Documentation/*/kvm*.txt 5591F: Documentation/*/kvm*.txt
5592F: Documentation/virtual/kvm/ 5592F: Documentation/virtual/kvm/
5593F: arch/*/kvm/ 5593F: arch/*/kvm/
5594F: arch/x86/kernel/kvm.c
5595F: arch/x86/kernel/kvmclock.c
5594F: arch/*/include/asm/kvm* 5596F: arch/*/include/asm/kvm*
5595F: include/linux/kvm* 5597F: include/linux/kvm*
5596F: include/uapi/linux/kvm* 5598F: include/uapi/linux/kvm*
diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h
index 816db0bf2dd8..d995821f1698 100644
--- a/arch/arm/include/asm/kvm_arm.h
+++ b/arch/arm/include/asm/kvm_arm.h
@@ -185,6 +185,7 @@
185#define HSR_COND (0xfU << HSR_COND_SHIFT) 185#define HSR_COND (0xfU << HSR_COND_SHIFT)
186 186
187#define FSC_FAULT (0x04) 187#define FSC_FAULT (0x04)
188#define FSC_ACCESS (0x08)
188#define FSC_PERM (0x0c) 189#define FSC_PERM (0x0c)
189 190
190/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ 191/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 41008cd7c53f..d71607c16601 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -27,6 +27,8 @@
27#include <asm/fpstate.h> 27#include <asm/fpstate.h>
28#include <kvm/arm_arch_timer.h> 28#include <kvm/arm_arch_timer.h>
29 29
30#define __KVM_HAVE_ARCH_INTC_INITIALIZED
31
30#if defined(CONFIG_KVM_ARM_MAX_VCPUS) 32#if defined(CONFIG_KVM_ARM_MAX_VCPUS)
31#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS 33#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
32#else 34#else
@@ -165,19 +167,10 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
165 167
166unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); 168unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
167int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); 169int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
170int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
171int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
168 172
169/* We do not have shadow page tables, hence the empty hooks */ 173/* We do not have shadow page tables, hence the empty hooks */
170static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
171 unsigned long end)
172{
173 return 0;
174}
175
176static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
177{
178 return 0;
179}
180
181static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm, 174static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
182 unsigned long address) 175 unsigned long address)
183{ 176{
diff --git a/arch/arm/include/asm/kvm_mmio.h b/arch/arm/include/asm/kvm_mmio.h
index 3f83db2f6cf0..d8e90c8cb5fa 100644
--- a/arch/arm/include/asm/kvm_mmio.h
+++ b/arch/arm/include/asm/kvm_mmio.h
@@ -28,28 +28,6 @@ struct kvm_decode {
28 bool sign_extend; 28 bool sign_extend;
29}; 29};
30 30
31/*
32 * The in-kernel MMIO emulation code wants to use a copy of run->mmio,
33 * which is an anonymous type. Use our own type instead.
34 */
35struct kvm_exit_mmio {
36 phys_addr_t phys_addr;
37 u8 data[8];
38 u32 len;
39 bool is_write;
40 void *private;
41};
42
43static inline void kvm_prepare_mmio(struct kvm_run *run,
44 struct kvm_exit_mmio *mmio)
45{
46 run->mmio.phys_addr = mmio->phys_addr;
47 run->mmio.len = mmio->len;
48 run->mmio.is_write = mmio->is_write;
49 memcpy(run->mmio.data, mmio->data, mmio->len);
50 run->exit_reason = KVM_EXIT_MMIO;
51}
52
53int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); 31int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
54int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, 32int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
55 phys_addr_t fault_ipa); 33 phys_addr_t fault_ipa);
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index 0db25bc32864..2499867dd0d8 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -198,6 +198,9 @@ struct kvm_arch_memory_slot {
198/* Highest supported SPI, from VGIC_NR_IRQS */ 198/* Highest supported SPI, from VGIC_NR_IRQS */
199#define KVM_ARM_IRQ_GIC_MAX 127 199#define KVM_ARM_IRQ_GIC_MAX 127
200 200
201/* One single KVM irqchip, ie. the VGIC */
202#define KVM_NR_IRQCHIPS 1
203
201/* PSCI interface */ 204/* PSCI interface */
202#define KVM_PSCI_FN_BASE 0x95c1ba5e 205#define KVM_PSCI_FN_BASE 0x95c1ba5e
203#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n)) 206#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n))
diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c
index 2d2d6087b9b1..488eaac56028 100644
--- a/arch/arm/kernel/asm-offsets.c
+++ b/arch/arm/kernel/asm-offsets.c
@@ -190,7 +190,6 @@ int main(void)
190 DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.fault.hxfar)); 190 DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.fault.hxfar));
191 DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.fault.hpfar)); 191 DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.fault.hpfar));
192 DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.fault.hyp_pc)); 192 DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.fault.hyp_pc));
193#ifdef CONFIG_KVM_ARM_VGIC
194 DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu)); 193 DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu));
195 DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr)); 194 DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr));
196 DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr)); 195 DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr));
@@ -200,14 +199,11 @@ int main(void)
200 DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr)); 199 DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr));
201 DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr)); 200 DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr));
202 DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr)); 201 DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr));
203#ifdef CONFIG_KVM_ARM_TIMER
204 DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl)); 202 DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
205 DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval)); 203 DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval));
206 DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff)); 204 DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff));
207 DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled)); 205 DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled));
208#endif
209 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); 206 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
210#endif
211 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); 207 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
212#endif 208#endif
213 return 0; 209 return 0;
diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig
index 338ace78ed18..f1f79d104309 100644
--- a/arch/arm/kvm/Kconfig
+++ b/arch/arm/kvm/Kconfig
@@ -18,6 +18,7 @@ if VIRTUALIZATION
18 18
19config KVM 19config KVM
20 bool "Kernel-based Virtual Machine (KVM) support" 20 bool "Kernel-based Virtual Machine (KVM) support"
21 depends on MMU && OF
21 select PREEMPT_NOTIFIERS 22 select PREEMPT_NOTIFIERS
22 select ANON_INODES 23 select ANON_INODES
23 select HAVE_KVM_CPU_RELAX_INTERCEPT 24 select HAVE_KVM_CPU_RELAX_INTERCEPT
@@ -26,10 +27,12 @@ config KVM
26 select KVM_ARM_HOST 27 select KVM_ARM_HOST
27 select KVM_GENERIC_DIRTYLOG_READ_PROTECT 28 select KVM_GENERIC_DIRTYLOG_READ_PROTECT
28 select SRCU 29 select SRCU
29 depends on ARM_VIRT_EXT && ARM_LPAE 30 select MMU_NOTIFIER
31 select HAVE_KVM_EVENTFD
32 select HAVE_KVM_IRQFD
33 depends on ARM_VIRT_EXT && ARM_LPAE && ARM_ARCH_TIMER
30 ---help--- 34 ---help---
31 Support hosting virtualized guest machines. You will also 35 Support hosting virtualized guest machines.
32 need to select one or more of the processor modules below.
33 36
34 This module provides access to the hardware capabilities through 37 This module provides access to the hardware capabilities through
35 a character device node named /dev/kvm. 38 a character device node named /dev/kvm.
@@ -37,10 +40,7 @@ config KVM
37 If unsure, say N. 40 If unsure, say N.
38 41
39config KVM_ARM_HOST 42config KVM_ARM_HOST
40 bool "KVM host support for ARM cpus." 43 bool
41 depends on KVM
42 depends on MMU
43 select MMU_NOTIFIER
44 ---help--- 44 ---help---
45 Provides host support for ARM processors. 45 Provides host support for ARM processors.
46 46
@@ -55,20 +55,4 @@ config KVM_ARM_MAX_VCPUS
55 large, so only choose a reasonable number that you expect to 55 large, so only choose a reasonable number that you expect to
56 actually use. 56 actually use.
57 57
58config KVM_ARM_VGIC
59 bool "KVM support for Virtual GIC"
60 depends on KVM_ARM_HOST && OF
61 select HAVE_KVM_IRQCHIP
62 default y
63 ---help---
64 Adds support for a hardware assisted, in-kernel GIC emulation.
65
66config KVM_ARM_TIMER
67 bool "KVM support for Architected Timers"
68 depends on KVM_ARM_VGIC && ARM_ARCH_TIMER
69 select HAVE_KVM_IRQCHIP
70 default y
71 ---help---
72 Adds support for the Architected Timers in virtual machines
73
74endif # VIRTUALIZATION 58endif # VIRTUALIZATION
diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile
index 443b8bea43e9..139e46c08b6e 100644
--- a/arch/arm/kvm/Makefile
+++ b/arch/arm/kvm/Makefile
@@ -7,7 +7,7 @@ ifeq ($(plus_virt),+virt)
7 plus_virt_def := -DREQUIRES_VIRT=1 7 plus_virt_def := -DREQUIRES_VIRT=1
8endif 8endif
9 9
10ccflags-y += -Ivirt/kvm -Iarch/arm/kvm 10ccflags-y += -Iarch/arm/kvm
11CFLAGS_arm.o := -I. $(plus_virt_def) 11CFLAGS_arm.o := -I. $(plus_virt_def)
12CFLAGS_mmu.o := -I. 12CFLAGS_mmu.o := -I.
13 13
@@ -15,12 +15,12 @@ AFLAGS_init.o := -Wa,-march=armv7-a$(plus_virt)
15AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt) 15AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt)
16 16
17KVM := ../../../virt/kvm 17KVM := ../../../virt/kvm
18kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o 18kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o
19 19
20obj-y += kvm-arm.o init.o interrupts.o 20obj-y += kvm-arm.o init.o interrupts.o
21obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o 21obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o
22obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o 22obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o
23obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o 23obj-y += $(KVM)/arm/vgic.o
24obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o 24obj-y += $(KVM)/arm/vgic-v2.o
25obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o 25obj-y += $(KVM)/arm/vgic-v2-emul.o
26obj-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o 26obj-y += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 5560f74f9eee..6f536451ab78 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -61,8 +61,6 @@ static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
61static u8 kvm_next_vmid; 61static u8 kvm_next_vmid;
62static DEFINE_SPINLOCK(kvm_vmid_lock); 62static DEFINE_SPINLOCK(kvm_vmid_lock);
63 63
64static bool vgic_present;
65
66static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu) 64static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
67{ 65{
68 BUG_ON(preemptible()); 66 BUG_ON(preemptible());
@@ -173,8 +171,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
173 int r; 171 int r;
174 switch (ext) { 172 switch (ext) {
175 case KVM_CAP_IRQCHIP: 173 case KVM_CAP_IRQCHIP:
176 r = vgic_present; 174 case KVM_CAP_IRQFD:
177 break; 175 case KVM_CAP_IOEVENTFD:
178 case KVM_CAP_DEVICE_CTRL: 176 case KVM_CAP_DEVICE_CTRL:
179 case KVM_CAP_USER_MEMORY: 177 case KVM_CAP_USER_MEMORY:
180 case KVM_CAP_SYNC_MMU: 178 case KVM_CAP_SYNC_MMU:
@@ -183,6 +181,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
183 case KVM_CAP_ARM_PSCI: 181 case KVM_CAP_ARM_PSCI:
184 case KVM_CAP_ARM_PSCI_0_2: 182 case KVM_CAP_ARM_PSCI_0_2:
185 case KVM_CAP_READONLY_MEM: 183 case KVM_CAP_READONLY_MEM:
184 case KVM_CAP_MP_STATE:
186 r = 1; 185 r = 1;
187 break; 186 break;
188 case KVM_CAP_COALESCED_MMIO: 187 case KVM_CAP_COALESCED_MMIO:
@@ -268,7 +267,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
268 267
269int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 268int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
270{ 269{
271 return 0; 270 return kvm_timer_should_fire(vcpu);
272} 271}
273 272
274int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 273int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
@@ -313,13 +312,29 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
313int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 312int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
314 struct kvm_mp_state *mp_state) 313 struct kvm_mp_state *mp_state)
315{ 314{
316 return -EINVAL; 315 if (vcpu->arch.pause)
316 mp_state->mp_state = KVM_MP_STATE_STOPPED;
317 else
318 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
319
320 return 0;
317} 321}
318 322
319int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 323int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
320 struct kvm_mp_state *mp_state) 324 struct kvm_mp_state *mp_state)
321{ 325{
322 return -EINVAL; 326 switch (mp_state->mp_state) {
327 case KVM_MP_STATE_RUNNABLE:
328 vcpu->arch.pause = false;
329 break;
330 case KVM_MP_STATE_STOPPED:
331 vcpu->arch.pause = true;
332 break;
333 default:
334 return -EINVAL;
335 }
336
337 return 0;
323} 338}
324 339
325/** 340/**
@@ -452,6 +467,11 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
452 return 0; 467 return 0;
453} 468}
454 469
470bool kvm_arch_intc_initialized(struct kvm *kvm)
471{
472 return vgic_initialized(kvm);
473}
474
455static void vcpu_pause(struct kvm_vcpu *vcpu) 475static void vcpu_pause(struct kvm_vcpu *vcpu)
456{ 476{
457 wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu); 477 wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
@@ -831,8 +851,6 @@ static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
831 851
832 switch (dev_id) { 852 switch (dev_id) {
833 case KVM_ARM_DEVICE_VGIC_V2: 853 case KVM_ARM_DEVICE_VGIC_V2:
834 if (!vgic_present)
835 return -ENXIO;
836 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true); 854 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
837 default: 855 default:
838 return -ENODEV; 856 return -ENODEV;
@@ -847,10 +865,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
847 865
848 switch (ioctl) { 866 switch (ioctl) {
849 case KVM_CREATE_IRQCHIP: { 867 case KVM_CREATE_IRQCHIP: {
850 if (vgic_present) 868 return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
851 return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
852 else
853 return -ENXIO;
854 } 869 }
855 case KVM_ARM_SET_DEVICE_ADDR: { 870 case KVM_ARM_SET_DEVICE_ADDR: {
856 struct kvm_arm_device_addr dev_addr; 871 struct kvm_arm_device_addr dev_addr;
@@ -1035,10 +1050,6 @@ static int init_hyp_mode(void)
1035 if (err) 1050 if (err)
1036 goto out_free_context; 1051 goto out_free_context;
1037 1052
1038#ifdef CONFIG_KVM_ARM_VGIC
1039 vgic_present = true;
1040#endif
1041
1042 /* 1053 /*
1043 * Init HYP architected timer support 1054 * Init HYP architected timer support
1044 */ 1055 */
diff --git a/arch/arm/kvm/guest.c b/arch/arm/kvm/guest.c
index 384bab67c462..d503fbb787d3 100644
--- a/arch/arm/kvm/guest.c
+++ b/arch/arm/kvm/guest.c
@@ -109,22 +109,6 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
109 return -EINVAL; 109 return -EINVAL;
110} 110}
111 111
112#ifndef CONFIG_KVM_ARM_TIMER
113
114#define NUM_TIMER_REGS 0
115
116static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
117{
118 return 0;
119}
120
121static bool is_timer_reg(u64 index)
122{
123 return false;
124}
125
126#else
127
128#define NUM_TIMER_REGS 3 112#define NUM_TIMER_REGS 3
129 113
130static bool is_timer_reg(u64 index) 114static bool is_timer_reg(u64 index)
@@ -152,8 +136,6 @@ static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
152 return 0; 136 return 0;
153} 137}
154 138
155#endif
156
157static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 139static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
158{ 140{
159 void __user *uaddr = (void __user *)(long)reg->addr; 141 void __user *uaddr = (void __user *)(long)reg->addr;
diff --git a/arch/arm/kvm/interrupts_head.S b/arch/arm/kvm/interrupts_head.S
index 14d488388480..35e4a3a0c476 100644
--- a/arch/arm/kvm/interrupts_head.S
+++ b/arch/arm/kvm/interrupts_head.S
@@ -402,7 +402,6 @@ vcpu .req r0 @ vcpu pointer always in r0
402 * Assumes vcpu pointer in vcpu reg 402 * Assumes vcpu pointer in vcpu reg
403 */ 403 */
404.macro save_vgic_state 404.macro save_vgic_state
405#ifdef CONFIG_KVM_ARM_VGIC
406 /* Get VGIC VCTRL base into r2 */ 405 /* Get VGIC VCTRL base into r2 */
407 ldr r2, [vcpu, #VCPU_KVM] 406 ldr r2, [vcpu, #VCPU_KVM]
408 ldr r2, [r2, #KVM_VGIC_VCTRL] 407 ldr r2, [r2, #KVM_VGIC_VCTRL]
@@ -460,7 +459,6 @@ ARM_BE8(rev r6, r6 )
460 subs r4, r4, #1 459 subs r4, r4, #1
461 bne 1b 460 bne 1b
4622: 4612:
463#endif
464.endm 462.endm
465 463
466/* 464/*
@@ -469,7 +467,6 @@ ARM_BE8(rev r6, r6 )
469 * Assumes vcpu pointer in vcpu reg 467 * Assumes vcpu pointer in vcpu reg
470 */ 468 */
471.macro restore_vgic_state 469.macro restore_vgic_state
472#ifdef CONFIG_KVM_ARM_VGIC
473 /* Get VGIC VCTRL base into r2 */ 470 /* Get VGIC VCTRL base into r2 */
474 ldr r2, [vcpu, #VCPU_KVM] 471 ldr r2, [vcpu, #VCPU_KVM]
475 ldr r2, [r2, #KVM_VGIC_VCTRL] 472 ldr r2, [r2, #KVM_VGIC_VCTRL]
@@ -501,7 +498,6 @@ ARM_BE8(rev r6, r6 )
501 subs r4, r4, #1 498 subs r4, r4, #1
502 bne 1b 499 bne 1b
5032: 5002:
504#endif
505.endm 501.endm
506 502
507#define CNTHCTL_PL1PCTEN (1 << 0) 503#define CNTHCTL_PL1PCTEN (1 << 0)
@@ -515,7 +511,6 @@ ARM_BE8(rev r6, r6 )
515 * Clobbers r2-r5 511 * Clobbers r2-r5
516 */ 512 */
517.macro save_timer_state 513.macro save_timer_state
518#ifdef CONFIG_KVM_ARM_TIMER
519 ldr r4, [vcpu, #VCPU_KVM] 514 ldr r4, [vcpu, #VCPU_KVM]
520 ldr r2, [r4, #KVM_TIMER_ENABLED] 515 ldr r2, [r4, #KVM_TIMER_ENABLED]
521 cmp r2, #0 516 cmp r2, #0
@@ -537,7 +532,6 @@ ARM_BE8(rev r6, r6 )
537 mcrr p15, 4, r2, r2, c14 @ CNTVOFF 532 mcrr p15, 4, r2, r2, c14 @ CNTVOFF
538 533
5391: 5341:
540#endif
541 @ Allow physical timer/counter access for the host 535 @ Allow physical timer/counter access for the host
542 mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL 536 mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL
543 orr r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN) 537 orr r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN)
@@ -559,7 +553,6 @@ ARM_BE8(rev r6, r6 )
559 bic r2, r2, #CNTHCTL_PL1PCEN 553 bic r2, r2, #CNTHCTL_PL1PCEN
560 mcr p15, 4, r2, c14, c1, 0 @ CNTHCTL 554 mcr p15, 4, r2, c14, c1, 0 @ CNTHCTL
561 555
562#ifdef CONFIG_KVM_ARM_TIMER
563 ldr r4, [vcpu, #VCPU_KVM] 556 ldr r4, [vcpu, #VCPU_KVM]
564 ldr r2, [r4, #KVM_TIMER_ENABLED] 557 ldr r2, [r4, #KVM_TIMER_ENABLED]
565 cmp r2, #0 558 cmp r2, #0
@@ -579,7 +572,6 @@ ARM_BE8(rev r6, r6 )
579 and r2, r2, #3 572 and r2, r2, #3
580 mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL 573 mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL
5811: 5741:
582#endif
583.endm 575.endm
584 576
585.equ vmentry, 0 577.equ vmentry, 0
diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c
index 5d3bfc0eb3f0..974b1c606d04 100644
--- a/arch/arm/kvm/mmio.c
+++ b/arch/arm/kvm/mmio.c
@@ -121,12 +121,11 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
121 return 0; 121 return 0;
122} 122}
123 123
124static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 124static int decode_hsr(struct kvm_vcpu *vcpu, bool *is_write, int *len)
125 struct kvm_exit_mmio *mmio)
126{ 125{
127 unsigned long rt; 126 unsigned long rt;
128 int len; 127 int access_size;
129 bool is_write, sign_extend; 128 bool sign_extend;
130 129
131 if (kvm_vcpu_dabt_isextabt(vcpu)) { 130 if (kvm_vcpu_dabt_isextabt(vcpu)) {
132 /* cache operation on I/O addr, tell guest unsupported */ 131 /* cache operation on I/O addr, tell guest unsupported */
@@ -140,17 +139,15 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
140 return 1; 139 return 1;
141 } 140 }
142 141
143 len = kvm_vcpu_dabt_get_as(vcpu); 142 access_size = kvm_vcpu_dabt_get_as(vcpu);
144 if (unlikely(len < 0)) 143 if (unlikely(access_size < 0))
145 return len; 144 return access_size;
146 145
147 is_write = kvm_vcpu_dabt_iswrite(vcpu); 146 *is_write = kvm_vcpu_dabt_iswrite(vcpu);
148 sign_extend = kvm_vcpu_dabt_issext(vcpu); 147 sign_extend = kvm_vcpu_dabt_issext(vcpu);
149 rt = kvm_vcpu_dabt_get_rd(vcpu); 148 rt = kvm_vcpu_dabt_get_rd(vcpu);
150 149
151 mmio->is_write = is_write; 150 *len = access_size;
152 mmio->phys_addr = fault_ipa;
153 mmio->len = len;
154 vcpu->arch.mmio_decode.sign_extend = sign_extend; 151 vcpu->arch.mmio_decode.sign_extend = sign_extend;
155 vcpu->arch.mmio_decode.rt = rt; 152 vcpu->arch.mmio_decode.rt = rt;
156 153
@@ -165,20 +162,20 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
165int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, 162int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
166 phys_addr_t fault_ipa) 163 phys_addr_t fault_ipa)
167{ 164{
168 struct kvm_exit_mmio mmio;
169 unsigned long data; 165 unsigned long data;
170 unsigned long rt; 166 unsigned long rt;
171 int ret; 167 int ret;
168 bool is_write;
169 int len;
170 u8 data_buf[8];
172 171
173 /* 172 /*
174 * Prepare MMIO operation. First stash it in a private 173 * Prepare MMIO operation. First decode the syndrome data we get
175 * structure that we can use for in-kernel emulation. If the 174 * from the CPU. Then try if some in-kernel emulation feels
176 * kernel can't handle it, copy it into run->mmio and let user 175 * responsible, otherwise let user space do its magic.
177 * space do its magic.
178 */ 176 */
179
180 if (kvm_vcpu_dabt_isvalid(vcpu)) { 177 if (kvm_vcpu_dabt_isvalid(vcpu)) {
181 ret = decode_hsr(vcpu, fault_ipa, &mmio); 178 ret = decode_hsr(vcpu, &is_write, &len);
182 if (ret) 179 if (ret)
183 return ret; 180 return ret;
184 } else { 181 } else {
@@ -188,21 +185,34 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
188 185
189 rt = vcpu->arch.mmio_decode.rt; 186 rt = vcpu->arch.mmio_decode.rt;
190 187
191 if (mmio.is_write) { 188 if (is_write) {
192 data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), 189 data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), len);
193 mmio.len); 190
191 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, fault_ipa, data);
192 mmio_write_buf(data_buf, len, data);
194 193
195 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, mmio.len, 194 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, fault_ipa, len,
196 fault_ipa, data); 195 data_buf);
197 mmio_write_buf(mmio.data, mmio.len, data);
198 } else { 196 } else {
199 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, mmio.len, 197 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len,
200 fault_ipa, 0); 198 fault_ipa, 0);
199
200 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_ipa, len,
201 data_buf);
201 } 202 }
202 203
203 if (vgic_handle_mmio(vcpu, run, &mmio)) 204 /* Now prepare kvm_run for the potential return to userland. */
205 run->mmio.is_write = is_write;
206 run->mmio.phys_addr = fault_ipa;
207 run->mmio.len = len;
208 memcpy(run->mmio.data, data_buf, len);
209
210 if (!ret) {
211 /* We handled the access successfully in the kernel. */
212 kvm_handle_mmio_return(vcpu, run);
204 return 1; 213 return 1;
214 }
205 215
206 kvm_prepare_mmio(run, &mmio); 216 run->exit_reason = KVM_EXIT_MMIO;
207 return 0; 217 return 0;
208} 218}
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 5656d79c5a44..15b050d46fc9 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1330,10 +1330,51 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
1330 1330
1331out_unlock: 1331out_unlock:
1332 spin_unlock(&kvm->mmu_lock); 1332 spin_unlock(&kvm->mmu_lock);
1333 kvm_set_pfn_accessed(pfn);
1333 kvm_release_pfn_clean(pfn); 1334 kvm_release_pfn_clean(pfn);
1334 return ret; 1335 return ret;
1335} 1336}
1336 1337
1338/*
1339 * Resolve the access fault by making the page young again.
1340 * Note that because the faulting entry is guaranteed not to be
1341 * cached in the TLB, we don't need to invalidate anything.
1342 */
1343static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
1344{
1345 pmd_t *pmd;
1346 pte_t *pte;
1347 pfn_t pfn;
1348 bool pfn_valid = false;
1349
1350 trace_kvm_access_fault(fault_ipa);
1351
1352 spin_lock(&vcpu->kvm->mmu_lock);
1353
1354 pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);
1355 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1356 goto out;
1357
1358 if (kvm_pmd_huge(*pmd)) { /* THP, HugeTLB */
1359 *pmd = pmd_mkyoung(*pmd);
1360 pfn = pmd_pfn(*pmd);
1361 pfn_valid = true;
1362 goto out;
1363 }
1364
1365 pte = pte_offset_kernel(pmd, fault_ipa);
1366 if (pte_none(*pte)) /* Nothing there either */
1367 goto out;
1368
1369 *pte = pte_mkyoung(*pte); /* Just a page... */
1370 pfn = pte_pfn(*pte);
1371 pfn_valid = true;
1372out:
1373 spin_unlock(&vcpu->kvm->mmu_lock);
1374 if (pfn_valid)
1375 kvm_set_pfn_accessed(pfn);
1376}
1377
1337/** 1378/**
1338 * kvm_handle_guest_abort - handles all 2nd stage aborts 1379 * kvm_handle_guest_abort - handles all 2nd stage aborts
1339 * @vcpu: the VCPU pointer 1380 * @vcpu: the VCPU pointer
@@ -1364,7 +1405,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
1364 1405
1365 /* Check the stage-2 fault is trans. fault or write fault */ 1406 /* Check the stage-2 fault is trans. fault or write fault */
1366 fault_status = kvm_vcpu_trap_get_fault_type(vcpu); 1407 fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
1367 if (fault_status != FSC_FAULT && fault_status != FSC_PERM) { 1408 if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
1409 fault_status != FSC_ACCESS) {
1368 kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", 1410 kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
1369 kvm_vcpu_trap_get_class(vcpu), 1411 kvm_vcpu_trap_get_class(vcpu),
1370 (unsigned long)kvm_vcpu_trap_get_fault(vcpu), 1412 (unsigned long)kvm_vcpu_trap_get_fault(vcpu),
@@ -1400,6 +1442,12 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
1400 /* Userspace should not be able to register out-of-bounds IPAs */ 1442 /* Userspace should not be able to register out-of-bounds IPAs */
1401 VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE); 1443 VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE);
1402 1444
1445 if (fault_status == FSC_ACCESS) {
1446 handle_access_fault(vcpu, fault_ipa);
1447 ret = 1;
1448 goto out_unlock;
1449 }
1450
1403 ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status); 1451 ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
1404 if (ret == 0) 1452 if (ret == 0)
1405 ret = 1; 1453 ret = 1;
@@ -1408,15 +1456,16 @@ out_unlock:
1408 return ret; 1456 return ret;
1409} 1457}
1410 1458
1411static void handle_hva_to_gpa(struct kvm *kvm, 1459static int handle_hva_to_gpa(struct kvm *kvm,
1412 unsigned long start, 1460 unsigned long start,
1413 unsigned long end, 1461 unsigned long end,
1414 void (*handler)(struct kvm *kvm, 1462 int (*handler)(struct kvm *kvm,
1415 gpa_t gpa, void *data), 1463 gpa_t gpa, void *data),
1416 void *data) 1464 void *data)
1417{ 1465{
1418 struct kvm_memslots *slots; 1466 struct kvm_memslots *slots;
1419 struct kvm_memory_slot *memslot; 1467 struct kvm_memory_slot *memslot;
1468 int ret = 0;
1420 1469
1421 slots = kvm_memslots(kvm); 1470 slots = kvm_memslots(kvm);
1422 1471
@@ -1440,14 +1489,17 @@ static void handle_hva_to_gpa(struct kvm *kvm,
1440 1489
1441 for (; gfn < gfn_end; ++gfn) { 1490 for (; gfn < gfn_end; ++gfn) {
1442 gpa_t gpa = gfn << PAGE_SHIFT; 1491 gpa_t gpa = gfn << PAGE_SHIFT;
1443 handler(kvm, gpa, data); 1492 ret |= handler(kvm, gpa, data);
1444 } 1493 }
1445 } 1494 }
1495
1496 return ret;
1446} 1497}
1447 1498
1448static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data) 1499static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1449{ 1500{
1450 unmap_stage2_range(kvm, gpa, PAGE_SIZE); 1501 unmap_stage2_range(kvm, gpa, PAGE_SIZE);
1502 return 0;
1451} 1503}
1452 1504
1453int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) 1505int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
@@ -1473,7 +1525,7 @@ int kvm_unmap_hva_range(struct kvm *kvm,
1473 return 0; 1525 return 0;
1474} 1526}
1475 1527
1476static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data) 1528static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
1477{ 1529{
1478 pte_t *pte = (pte_t *)data; 1530 pte_t *pte = (pte_t *)data;
1479 1531
@@ -1485,6 +1537,7 @@ static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
1485 * through this calling path. 1537 * through this calling path.
1486 */ 1538 */
1487 stage2_set_pte(kvm, NULL, gpa, pte, 0); 1539 stage2_set_pte(kvm, NULL, gpa, pte, 0);
1540 return 0;
1488} 1541}
1489 1542
1490 1543
@@ -1501,6 +1554,67 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1501 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte); 1554 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
1502} 1555}
1503 1556
1557static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1558{
1559 pmd_t *pmd;
1560 pte_t *pte;
1561
1562 pmd = stage2_get_pmd(kvm, NULL, gpa);
1563 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1564 return 0;
1565
1566 if (kvm_pmd_huge(*pmd)) { /* THP, HugeTLB */
1567 if (pmd_young(*pmd)) {
1568 *pmd = pmd_mkold(*pmd);
1569 return 1;
1570 }
1571
1572 return 0;
1573 }
1574
1575 pte = pte_offset_kernel(pmd, gpa);
1576 if (pte_none(*pte))
1577 return 0;
1578
1579 if (pte_young(*pte)) {
1580 *pte = pte_mkold(*pte); /* Just a page... */
1581 return 1;
1582 }
1583
1584 return 0;
1585}
1586
1587static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1588{
1589 pmd_t *pmd;
1590 pte_t *pte;
1591
1592 pmd = stage2_get_pmd(kvm, NULL, gpa);
1593 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1594 return 0;
1595
1596 if (kvm_pmd_huge(*pmd)) /* THP, HugeTLB */
1597 return pmd_young(*pmd);
1598
1599 pte = pte_offset_kernel(pmd, gpa);
1600 if (!pte_none(*pte)) /* Just a page... */
1601 return pte_young(*pte);
1602
1603 return 0;
1604}
1605
1606int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1607{
1608 trace_kvm_age_hva(start, end);
1609 return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
1610}
1611
1612int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1613{
1614 trace_kvm_test_age_hva(hva);
1615 return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
1616}
1617
1504void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) 1618void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
1505{ 1619{
1506 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 1620 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
diff --git a/arch/arm/kvm/trace.h b/arch/arm/kvm/trace.h
index 6817664b46b8..0ec35392d208 100644
--- a/arch/arm/kvm/trace.h
+++ b/arch/arm/kvm/trace.h
@@ -68,6 +68,21 @@ TRACE_EVENT(kvm_guest_fault,
68 __entry->hxfar, __entry->vcpu_pc) 68 __entry->hxfar, __entry->vcpu_pc)
69); 69);
70 70
71TRACE_EVENT(kvm_access_fault,
72 TP_PROTO(unsigned long ipa),
73 TP_ARGS(ipa),
74
75 TP_STRUCT__entry(
76 __field( unsigned long, ipa )
77 ),
78
79 TP_fast_assign(
80 __entry->ipa = ipa;
81 ),
82
83 TP_printk("IPA: %lx", __entry->ipa)
84);
85
71TRACE_EVENT(kvm_irq_line, 86TRACE_EVENT(kvm_irq_line,
72 TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level), 87 TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level),
73 TP_ARGS(type, vcpu_idx, irq_num, level), 88 TP_ARGS(type, vcpu_idx, irq_num, level),
@@ -210,6 +225,39 @@ TRACE_EVENT(kvm_set_spte_hva,
210 TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva) 225 TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva)
211); 226);
212 227
228TRACE_EVENT(kvm_age_hva,
229 TP_PROTO(unsigned long start, unsigned long end),
230 TP_ARGS(start, end),
231
232 TP_STRUCT__entry(
233 __field( unsigned long, start )
234 __field( unsigned long, end )
235 ),
236
237 TP_fast_assign(
238 __entry->start = start;
239 __entry->end = end;
240 ),
241
242 TP_printk("mmu notifier age hva: %#08lx -- %#08lx",
243 __entry->start, __entry->end)
244);
245
246TRACE_EVENT(kvm_test_age_hva,
247 TP_PROTO(unsigned long hva),
248 TP_ARGS(hva),
249
250 TP_STRUCT__entry(
251 __field( unsigned long, hva )
252 ),
253
254 TP_fast_assign(
255 __entry->hva = hva;
256 ),
257
258 TP_printk("mmu notifier test age hva: %#08lx", __entry->hva)
259);
260
213TRACE_EVENT(kvm_hvc, 261TRACE_EVENT(kvm_hvc,
214 TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm), 262 TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
215 TP_ARGS(vcpu_pc, r0, imm), 263 TP_ARGS(vcpu_pc, r0, imm),
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index 92bbae381598..70522450ca23 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -90,6 +90,7 @@
90#define ESR_ELx_FSC (0x3F) 90#define ESR_ELx_FSC (0x3F)
91#define ESR_ELx_FSC_TYPE (0x3C) 91#define ESR_ELx_FSC_TYPE (0x3C)
92#define ESR_ELx_FSC_EXTABT (0x10) 92#define ESR_ELx_FSC_EXTABT (0x10)
93#define ESR_ELx_FSC_ACCESS (0x08)
93#define ESR_ELx_FSC_FAULT (0x04) 94#define ESR_ELx_FSC_FAULT (0x04)
94#define ESR_ELx_FSC_PERM (0x0C) 95#define ESR_ELx_FSC_PERM (0x0C)
95#define ESR_ELx_CV (UL(1) << 24) 96#define ESR_ELx_CV (UL(1) << 24)
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 54bb4ba97441..ac6fafb95fe7 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -188,6 +188,7 @@
188 188
189/* For compatibility with fault code shared with 32-bit */ 189/* For compatibility with fault code shared with 32-bit */
190#define FSC_FAULT ESR_ELx_FSC_FAULT 190#define FSC_FAULT ESR_ELx_FSC_FAULT
191#define FSC_ACCESS ESR_ELx_FSC_ACCESS
191#define FSC_PERM ESR_ELx_FSC_PERM 192#define FSC_PERM ESR_ELx_FSC_PERM
192 193
193/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ 194/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 8ac3c70fe3c6..f0f58c9beec0 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -28,6 +28,8 @@
28#include <asm/kvm_asm.h> 28#include <asm/kvm_asm.h>
29#include <asm/kvm_mmio.h> 29#include <asm/kvm_mmio.h>
30 30
31#define __KVM_HAVE_ARCH_INTC_INITIALIZED
32
31#if defined(CONFIG_KVM_ARM_MAX_VCPUS) 33#if defined(CONFIG_KVM_ARM_MAX_VCPUS)
32#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS 34#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
33#else 35#else
@@ -177,19 +179,10 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
177int kvm_unmap_hva_range(struct kvm *kvm, 179int kvm_unmap_hva_range(struct kvm *kvm,
178 unsigned long start, unsigned long end); 180 unsigned long start, unsigned long end);
179void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); 181void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
182int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
183int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
180 184
181/* We do not have shadow page tables, hence the empty hooks */ 185/* We do not have shadow page tables, hence the empty hooks */
182static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
183 unsigned long end)
184{
185 return 0;
186}
187
188static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
189{
190 return 0;
191}
192
193static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm, 186static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
194 unsigned long address) 187 unsigned long address)
195{ 188{
diff --git a/arch/arm64/include/asm/kvm_mmio.h b/arch/arm64/include/asm/kvm_mmio.h
index 9f52beb7cb13..889c908ee631 100644
--- a/arch/arm64/include/asm/kvm_mmio.h
+++ b/arch/arm64/include/asm/kvm_mmio.h
@@ -31,28 +31,6 @@ struct kvm_decode {
31 bool sign_extend; 31 bool sign_extend;
32}; 32};
33 33
34/*
35 * The in-kernel MMIO emulation code wants to use a copy of run->mmio,
36 * which is an anonymous type. Use our own type instead.
37 */
38struct kvm_exit_mmio {
39 phys_addr_t phys_addr;
40 u8 data[8];
41 u32 len;
42 bool is_write;
43 void *private;
44};
45
46static inline void kvm_prepare_mmio(struct kvm_run *run,
47 struct kvm_exit_mmio *mmio)
48{
49 run->mmio.phys_addr = mmio->phys_addr;
50 run->mmio.len = mmio->len;
51 run->mmio.is_write = mmio->is_write;
52 memcpy(run->mmio.data, mmio->data, mmio->len);
53 run->exit_reason = KVM_EXIT_MMIO;
54}
55
56int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); 34int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
57int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, 35int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
58 phys_addr_t fault_ipa); 36 phys_addr_t fault_ipa);
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 3ef77a466018..c154c0b7eb60 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -191,6 +191,9 @@ struct kvm_arch_memory_slot {
191/* Highest supported SPI, from VGIC_NR_IRQS */ 191/* Highest supported SPI, from VGIC_NR_IRQS */
192#define KVM_ARM_IRQ_GIC_MAX 127 192#define KVM_ARM_IRQ_GIC_MAX 127
193 193
194/* One single KVM irqchip, ie. the VGIC */
195#define KVM_NR_IRQCHIPS 1
196
194/* PSCI interface */ 197/* PSCI interface */
195#define KVM_PSCI_FN_BASE 0x95c1ba5e 198#define KVM_PSCI_FN_BASE 0x95c1ba5e
196#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n)) 199#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n))
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index f5590c81d95f..5105e297ed5f 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -18,6 +18,7 @@ if VIRTUALIZATION
18 18
19config KVM 19config KVM
20 bool "Kernel-based Virtual Machine (KVM) support" 20 bool "Kernel-based Virtual Machine (KVM) support"
21 depends on OF
21 select MMU_NOTIFIER 22 select MMU_NOTIFIER
22 select PREEMPT_NOTIFIERS 23 select PREEMPT_NOTIFIERS
23 select ANON_INODES 24 select ANON_INODES
@@ -25,10 +26,10 @@ config KVM
25 select HAVE_KVM_ARCH_TLB_FLUSH_ALL 26 select HAVE_KVM_ARCH_TLB_FLUSH_ALL
26 select KVM_MMIO 27 select KVM_MMIO
27 select KVM_ARM_HOST 28 select KVM_ARM_HOST
28 select KVM_ARM_VGIC
29 select KVM_ARM_TIMER
30 select KVM_GENERIC_DIRTYLOG_READ_PROTECT 29 select KVM_GENERIC_DIRTYLOG_READ_PROTECT
31 select SRCU 30 select SRCU
31 select HAVE_KVM_EVENTFD
32 select HAVE_KVM_IRQFD
32 ---help--- 33 ---help---
33 Support hosting virtualized guest machines. 34 Support hosting virtualized guest machines.
34 35
@@ -50,17 +51,4 @@ config KVM_ARM_MAX_VCPUS
50 large, so only choose a reasonable number that you expect to 51 large, so only choose a reasonable number that you expect to
51 actually use. 52 actually use.
52 53
53config KVM_ARM_VGIC
54 bool
55 depends on KVM_ARM_HOST && OF
56 select HAVE_KVM_IRQCHIP
57 ---help---
58 Adds support for a hardware assisted, in-kernel GIC emulation.
59
60config KVM_ARM_TIMER
61 bool
62 depends on KVM_ARM_VGIC
63 ---help---
64 Adds support for the Architected Timers in virtual machines.
65
66endif # VIRTUALIZATION 54endif # VIRTUALIZATION
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 4e6e09ee4033..d5904f876cdb 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -2,7 +2,7 @@
2# Makefile for Kernel-based Virtual Machine module 2# Makefile for Kernel-based Virtual Machine module
3# 3#
4 4
5ccflags-y += -Ivirt/kvm -Iarch/arm64/kvm 5ccflags-y += -Iarch/arm64/kvm
6CFLAGS_arm.o := -I. 6CFLAGS_arm.o := -I.
7CFLAGS_mmu.o := -I. 7CFLAGS_mmu.o := -I.
8 8
@@ -11,7 +11,7 @@ ARM=../../../arch/arm/kvm
11 11
12obj-$(CONFIG_KVM_ARM_HOST) += kvm.o 12obj-$(CONFIG_KVM_ARM_HOST) += kvm.o
13 13
14kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o 14kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o
15kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/arm.o $(ARM)/mmu.o $(ARM)/mmio.o 15kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/arm.o $(ARM)/mmu.o $(ARM)/mmio.o
16kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o 16kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o
17 17
@@ -19,11 +19,11 @@ kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o
19kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o 19kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
20kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o 20kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
21 21
22kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o 22kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o
23kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o 23kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o
24kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o 24kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2-emul.o
25kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v2-switch.o 25kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v2-switch.o
26kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3.o 26kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
27kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3-emul.o 27kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
28kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v3-switch.o 28kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
29kvm-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o 29kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
diff --git a/arch/mips/include/asm/asmmacro-32.h b/arch/mips/include/asm/asmmacro-32.h
index cdac7b3eeaf7..80386470d3a4 100644
--- a/arch/mips/include/asm/asmmacro-32.h
+++ b/arch/mips/include/asm/asmmacro-32.h
@@ -16,38 +16,38 @@
16 .set push 16 .set push
17 SET_HARDFLOAT 17 SET_HARDFLOAT
18 cfc1 \tmp, fcr31 18 cfc1 \tmp, fcr31
19 swc1 $f0, THREAD_FPR0_LS64(\thread) 19 swc1 $f0, THREAD_FPR0(\thread)
20 swc1 $f1, THREAD_FPR1_LS64(\thread) 20 swc1 $f1, THREAD_FPR1(\thread)
21 swc1 $f2, THREAD_FPR2_LS64(\thread) 21 swc1 $f2, THREAD_FPR2(\thread)
22 swc1 $f3, THREAD_FPR3_LS64(\thread) 22 swc1 $f3, THREAD_FPR3(\thread)
23 swc1 $f4, THREAD_FPR4_LS64(\thread) 23 swc1 $f4, THREAD_FPR4(\thread)
24 swc1 $f5, THREAD_FPR5_LS64(\thread) 24 swc1 $f5, THREAD_FPR5(\thread)
25 swc1 $f6, THREAD_FPR6_LS64(\thread) 25 swc1 $f6, THREAD_FPR6(\thread)
26 swc1 $f7, THREAD_FPR7_LS64(\thread) 26 swc1 $f7, THREAD_FPR7(\thread)
27 swc1 $f8, THREAD_FPR8_LS64(\thread) 27 swc1 $f8, THREAD_FPR8(\thread)
28 swc1 $f9, THREAD_FPR9_LS64(\thread) 28 swc1 $f9, THREAD_FPR9(\thread)
29 swc1 $f10, THREAD_FPR10_LS64(\thread) 29 swc1 $f10, THREAD_FPR10(\thread)
30 swc1 $f11, THREAD_FPR11_LS64(\thread) 30 swc1 $f11, THREAD_FPR11(\thread)
31 swc1 $f12, THREAD_FPR12_LS64(\thread) 31 swc1 $f12, THREAD_FPR12(\thread)
32 swc1 $f13, THREAD_FPR13_LS64(\thread) 32 swc1 $f13, THREAD_FPR13(\thread)
33 swc1 $f14, THREAD_FPR14_LS64(\thread) 33 swc1 $f14, THREAD_FPR14(\thread)
34 swc1 $f15, THREAD_FPR15_LS64(\thread) 34 swc1 $f15, THREAD_FPR15(\thread)
35 swc1 $f16, THREAD_FPR16_LS64(\thread) 35 swc1 $f16, THREAD_FPR16(\thread)
36 swc1 $f17, THREAD_FPR17_LS64(\thread) 36 swc1 $f17, THREAD_FPR17(\thread)
37 swc1 $f18, THREAD_FPR18_LS64(\thread) 37 swc1 $f18, THREAD_FPR18(\thread)
38 swc1 $f19, THREAD_FPR19_LS64(\thread) 38 swc1 $f19, THREAD_FPR19(\thread)
39 swc1 $f20, THREAD_FPR20_LS64(\thread) 39 swc1 $f20, THREAD_FPR20(\thread)
40 swc1 $f21, THREAD_FPR21_LS64(\thread) 40 swc1 $f21, THREAD_FPR21(\thread)
41 swc1 $f22, THREAD_FPR22_LS64(\thread) 41 swc1 $f22, THREAD_FPR22(\thread)
42 swc1 $f23, THREAD_FPR23_LS64(\thread) 42 swc1 $f23, THREAD_FPR23(\thread)
43 swc1 $f24, THREAD_FPR24_LS64(\thread) 43 swc1 $f24, THREAD_FPR24(\thread)
44 swc1 $f25, THREAD_FPR25_LS64(\thread) 44 swc1 $f25, THREAD_FPR25(\thread)
45 swc1 $f26, THREAD_FPR26_LS64(\thread) 45 swc1 $f26, THREAD_FPR26(\thread)
46 swc1 $f27, THREAD_FPR27_LS64(\thread) 46 swc1 $f27, THREAD_FPR27(\thread)
47 swc1 $f28, THREAD_FPR28_LS64(\thread) 47 swc1 $f28, THREAD_FPR28(\thread)
48 swc1 $f29, THREAD_FPR29_LS64(\thread) 48 swc1 $f29, THREAD_FPR29(\thread)
49 swc1 $f30, THREAD_FPR30_LS64(\thread) 49 swc1 $f30, THREAD_FPR30(\thread)
50 swc1 $f31, THREAD_FPR31_LS64(\thread) 50 swc1 $f31, THREAD_FPR31(\thread)
51 sw \tmp, THREAD_FCR31(\thread) 51 sw \tmp, THREAD_FCR31(\thread)
52 .set pop 52 .set pop
53 .endm 53 .endm
@@ -56,38 +56,38 @@
56 .set push 56 .set push
57 SET_HARDFLOAT 57 SET_HARDFLOAT
58 lw \tmp, THREAD_FCR31(\thread) 58 lw \tmp, THREAD_FCR31(\thread)
59 lwc1 $f0, THREAD_FPR0_LS64(\thread) 59 lwc1 $f0, THREAD_FPR0(\thread)
60 lwc1 $f1, THREAD_FPR1_LS64(\thread) 60 lwc1 $f1, THREAD_FPR1(\thread)
61 lwc1 $f2, THREAD_FPR2_LS64(\thread) 61 lwc1 $f2, THREAD_FPR2(\thread)
62 lwc1 $f3, THREAD_FPR3_LS64(\thread) 62 lwc1 $f3, THREAD_FPR3(\thread)
63 lwc1 $f4, THREAD_FPR4_LS64(\thread) 63 lwc1 $f4, THREAD_FPR4(\thread)
64 lwc1 $f5, THREAD_FPR5_LS64(\thread) 64 lwc1 $f5, THREAD_FPR5(\thread)
65 lwc1 $f6, THREAD_FPR6_LS64(\thread) 65 lwc1 $f6, THREAD_FPR6(\thread)
66 lwc1 $f7, THREAD_FPR7_LS64(\thread) 66 lwc1 $f7, THREAD_FPR7(\thread)
67 lwc1 $f8, THREAD_FPR8_LS64(\thread) 67 lwc1 $f8, THREAD_FPR8(\thread)
68 lwc1 $f9, THREAD_FPR9_LS64(\thread) 68 lwc1 $f9, THREAD_FPR9(\thread)
69 lwc1 $f10, THREAD_FPR10_LS64(\thread) 69 lwc1 $f10, THREAD_FPR10(\thread)
70 lwc1 $f11, THREAD_FPR11_LS64(\thread) 70 lwc1 $f11, THREAD_FPR11(\thread)
71 lwc1 $f12, THREAD_FPR12_LS64(\thread) 71 lwc1 $f12, THREAD_FPR12(\thread)
72 lwc1 $f13, THREAD_FPR13_LS64(\thread) 72 lwc1 $f13, THREAD_FPR13(\thread)
73 lwc1 $f14, THREAD_FPR14_LS64(\thread) 73 lwc1 $f14, THREAD_FPR14(\thread)
74 lwc1 $f15, THREAD_FPR15_LS64(\thread) 74 lwc1 $f15, THREAD_FPR15(\thread)
75 lwc1 $f16, THREAD_FPR16_LS64(\thread) 75 lwc1 $f16, THREAD_FPR16(\thread)
76 lwc1 $f17, THREAD_FPR17_LS64(\thread) 76 lwc1 $f17, THREAD_FPR17(\thread)
77 lwc1 $f18, THREAD_FPR18_LS64(\thread) 77 lwc1 $f18, THREAD_FPR18(\thread)
78 lwc1 $f19, THREAD_FPR19_LS64(\thread) 78 lwc1 $f19, THREAD_FPR19(\thread)
79 lwc1 $f20, THREAD_FPR20_LS64(\thread) 79 lwc1 $f20, THREAD_FPR20(\thread)
80 lwc1 $f21, THREAD_FPR21_LS64(\thread) 80 lwc1 $f21, THREAD_FPR21(\thread)
81 lwc1 $f22, THREAD_FPR22_LS64(\thread) 81 lwc1 $f22, THREAD_FPR22(\thread)
82 lwc1 $f23, THREAD_FPR23_LS64(\thread) 82 lwc1 $f23, THREAD_FPR23(\thread)
83 lwc1 $f24, THREAD_FPR24_LS64(\thread) 83 lwc1 $f24, THREAD_FPR24(\thread)
84 lwc1 $f25, THREAD_FPR25_LS64(\thread) 84 lwc1 $f25, THREAD_FPR25(\thread)
85 lwc1 $f26, THREAD_FPR26_LS64(\thread) 85 lwc1 $f26, THREAD_FPR26(\thread)
86 lwc1 $f27, THREAD_FPR27_LS64(\thread) 86 lwc1 $f27, THREAD_FPR27(\thread)
87 lwc1 $f28, THREAD_FPR28_LS64(\thread) 87 lwc1 $f28, THREAD_FPR28(\thread)
88 lwc1 $f29, THREAD_FPR29_LS64(\thread) 88 lwc1 $f29, THREAD_FPR29(\thread)
89 lwc1 $f30, THREAD_FPR30_LS64(\thread) 89 lwc1 $f30, THREAD_FPR30(\thread)
90 lwc1 $f31, THREAD_FPR31_LS64(\thread) 90 lwc1 $f31, THREAD_FPR31(\thread)
91 ctc1 \tmp, fcr31 91 ctc1 \tmp, fcr31
92 .set pop 92 .set pop
93 .endm 93 .endm
diff --git a/arch/mips/include/asm/asmmacro.h b/arch/mips/include/asm/asmmacro.h
index 0cae4595e985..6156ac8c4cfb 100644
--- a/arch/mips/include/asm/asmmacro.h
+++ b/arch/mips/include/asm/asmmacro.h
@@ -60,22 +60,22 @@
60 .set push 60 .set push
61 SET_HARDFLOAT 61 SET_HARDFLOAT
62 cfc1 \tmp, fcr31 62 cfc1 \tmp, fcr31
63 sdc1 $f0, THREAD_FPR0_LS64(\thread) 63 sdc1 $f0, THREAD_FPR0(\thread)
64 sdc1 $f2, THREAD_FPR2_LS64(\thread) 64 sdc1 $f2, THREAD_FPR2(\thread)
65 sdc1 $f4, THREAD_FPR4_LS64(\thread) 65 sdc1 $f4, THREAD_FPR4(\thread)
66 sdc1 $f6, THREAD_FPR6_LS64(\thread) 66 sdc1 $f6, THREAD_FPR6(\thread)
67 sdc1 $f8, THREAD_FPR8_LS64(\thread) 67 sdc1 $f8, THREAD_FPR8(\thread)
68 sdc1 $f10, THREAD_FPR10_LS64(\thread) 68 sdc1 $f10, THREAD_FPR10(\thread)
69 sdc1 $f12, THREAD_FPR12_LS64(\thread) 69 sdc1 $f12, THREAD_FPR12(\thread)
70 sdc1 $f14, THREAD_FPR14_LS64(\thread) 70 sdc1 $f14, THREAD_FPR14(\thread)
71 sdc1 $f16, THREAD_FPR16_LS64(\thread) 71 sdc1 $f16, THREAD_FPR16(\thread)
72 sdc1 $f18, THREAD_FPR18_LS64(\thread) 72 sdc1 $f18, THREAD_FPR18(\thread)
73 sdc1 $f20, THREAD_FPR20_LS64(\thread) 73 sdc1 $f20, THREAD_FPR20(\thread)
74 sdc1 $f22, THREAD_FPR22_LS64(\thread) 74 sdc1 $f22, THREAD_FPR22(\thread)
75 sdc1 $f24, THREAD_FPR24_LS64(\thread) 75 sdc1 $f24, THREAD_FPR24(\thread)
76 sdc1 $f26, THREAD_FPR26_LS64(\thread) 76 sdc1 $f26, THREAD_FPR26(\thread)
77 sdc1 $f28, THREAD_FPR28_LS64(\thread) 77 sdc1 $f28, THREAD_FPR28(\thread)
78 sdc1 $f30, THREAD_FPR30_LS64(\thread) 78 sdc1 $f30, THREAD_FPR30(\thread)
79 sw \tmp, THREAD_FCR31(\thread) 79 sw \tmp, THREAD_FCR31(\thread)
80 .set pop 80 .set pop
81 .endm 81 .endm
@@ -84,22 +84,22 @@
84 .set push 84 .set push
85 .set mips64r2 85 .set mips64r2
86 SET_HARDFLOAT 86 SET_HARDFLOAT
87 sdc1 $f1, THREAD_FPR1_LS64(\thread) 87 sdc1 $f1, THREAD_FPR1(\thread)
88 sdc1 $f3, THREAD_FPR3_LS64(\thread) 88 sdc1 $f3, THREAD_FPR3(\thread)
89 sdc1 $f5, THREAD_FPR5_LS64(\thread) 89 sdc1 $f5, THREAD_FPR5(\thread)
90 sdc1 $f7, THREAD_FPR7_LS64(\thread) 90 sdc1 $f7, THREAD_FPR7(\thread)
91 sdc1 $f9, THREAD_FPR9_LS64(\thread) 91 sdc1 $f9, THREAD_FPR9(\thread)
92 sdc1 $f11, THREAD_FPR11_LS64(\thread) 92 sdc1 $f11, THREAD_FPR11(\thread)
93 sdc1 $f13, THREAD_FPR13_LS64(\thread) 93 sdc1 $f13, THREAD_FPR13(\thread)
94 sdc1 $f15, THREAD_FPR15_LS64(\thread) 94 sdc1 $f15, THREAD_FPR15(\thread)
95 sdc1 $f17, THREAD_FPR17_LS64(\thread) 95 sdc1 $f17, THREAD_FPR17(\thread)
96 sdc1 $f19, THREAD_FPR19_LS64(\thread) 96 sdc1 $f19, THREAD_FPR19(\thread)
97 sdc1 $f21, THREAD_FPR21_LS64(\thread) 97 sdc1 $f21, THREAD_FPR21(\thread)
98 sdc1 $f23, THREAD_FPR23_LS64(\thread) 98 sdc1 $f23, THREAD_FPR23(\thread)
99 sdc1 $f25, THREAD_FPR25_LS64(\thread) 99 sdc1 $f25, THREAD_FPR25(\thread)
100 sdc1 $f27, THREAD_FPR27_LS64(\thread) 100 sdc1 $f27, THREAD_FPR27(\thread)
101 sdc1 $f29, THREAD_FPR29_LS64(\thread) 101 sdc1 $f29, THREAD_FPR29(\thread)
102 sdc1 $f31, THREAD_FPR31_LS64(\thread) 102 sdc1 $f31, THREAD_FPR31(\thread)
103 .set pop 103 .set pop
104 .endm 104 .endm
105 105
@@ -118,22 +118,22 @@
118 .set push 118 .set push
119 SET_HARDFLOAT 119 SET_HARDFLOAT
120 lw \tmp, THREAD_FCR31(\thread) 120 lw \tmp, THREAD_FCR31(\thread)
121 ldc1 $f0, THREAD_FPR0_LS64(\thread) 121 ldc1 $f0, THREAD_FPR0(\thread)
122 ldc1 $f2, THREAD_FPR2_LS64(\thread) 122 ldc1 $f2, THREAD_FPR2(\thread)
123 ldc1 $f4, THREAD_FPR4_LS64(\thread) 123 ldc1 $f4, THREAD_FPR4(\thread)
124 ldc1 $f6, THREAD_FPR6_LS64(\thread) 124 ldc1 $f6, THREAD_FPR6(\thread)
125 ldc1 $f8, THREAD_FPR8_LS64(\thread) 125 ldc1 $f8, THREAD_FPR8(\thread)
126 ldc1 $f10, THREAD_FPR10_LS64(\thread) 126 ldc1 $f10, THREAD_FPR10(\thread)
127 ldc1 $f12, THREAD_FPR12_LS64(\thread) 127 ldc1 $f12, THREAD_FPR12(\thread)
128 ldc1 $f14, THREAD_FPR14_LS64(\thread) 128 ldc1 $f14, THREAD_FPR14(\thread)
129 ldc1 $f16, THREAD_FPR16_LS64(\thread) 129 ldc1 $f16, THREAD_FPR16(\thread)
130 ldc1 $f18, THREAD_FPR18_LS64(\thread) 130 ldc1 $f18, THREAD_FPR18(\thread)
131 ldc1 $f20, THREAD_FPR20_LS64(\thread) 131 ldc1 $f20, THREAD_FPR20(\thread)
132 ldc1 $f22, THREAD_FPR22_LS64(\thread) 132 ldc1 $f22, THREAD_FPR22(\thread)
133 ldc1 $f24, THREAD_FPR24_LS64(\thread) 133 ldc1 $f24, THREAD_FPR24(\thread)
134 ldc1 $f26, THREAD_FPR26_LS64(\thread) 134 ldc1 $f26, THREAD_FPR26(\thread)
135 ldc1 $f28, THREAD_FPR28_LS64(\thread) 135 ldc1 $f28, THREAD_FPR28(\thread)
136 ldc1 $f30, THREAD_FPR30_LS64(\thread) 136 ldc1 $f30, THREAD_FPR30(\thread)
137 ctc1 \tmp, fcr31 137 ctc1 \tmp, fcr31
138 .endm 138 .endm
139 139
@@ -141,22 +141,22 @@
141 .set push 141 .set push
142 .set mips64r2 142 .set mips64r2
143 SET_HARDFLOAT 143 SET_HARDFLOAT
144 ldc1 $f1, THREAD_FPR1_LS64(\thread) 144 ldc1 $f1, THREAD_FPR1(\thread)
145 ldc1 $f3, THREAD_FPR3_LS64(\thread) 145 ldc1 $f3, THREAD_FPR3(\thread)
146 ldc1 $f5, THREAD_FPR5_LS64(\thread) 146 ldc1 $f5, THREAD_FPR5(\thread)
147 ldc1 $f7, THREAD_FPR7_LS64(\thread) 147 ldc1 $f7, THREAD_FPR7(\thread)
148 ldc1 $f9, THREAD_FPR9_LS64(\thread) 148 ldc1 $f9, THREAD_FPR9(\thread)
149 ldc1 $f11, THREAD_FPR11_LS64(\thread) 149 ldc1 $f11, THREAD_FPR11(\thread)
150 ldc1 $f13, THREAD_FPR13_LS64(\thread) 150 ldc1 $f13, THREAD_FPR13(\thread)
151 ldc1 $f15, THREAD_FPR15_LS64(\thread) 151 ldc1 $f15, THREAD_FPR15(\thread)
152 ldc1 $f17, THREAD_FPR17_LS64(\thread) 152 ldc1 $f17, THREAD_FPR17(\thread)
153 ldc1 $f19, THREAD_FPR19_LS64(\thread) 153 ldc1 $f19, THREAD_FPR19(\thread)
154 ldc1 $f21, THREAD_FPR21_LS64(\thread) 154 ldc1 $f21, THREAD_FPR21(\thread)
155 ldc1 $f23, THREAD_FPR23_LS64(\thread) 155 ldc1 $f23, THREAD_FPR23(\thread)
156 ldc1 $f25, THREAD_FPR25_LS64(\thread) 156 ldc1 $f25, THREAD_FPR25(\thread)
157 ldc1 $f27, THREAD_FPR27_LS64(\thread) 157 ldc1 $f27, THREAD_FPR27(\thread)
158 ldc1 $f29, THREAD_FPR29_LS64(\thread) 158 ldc1 $f29, THREAD_FPR29(\thread)
159 ldc1 $f31, THREAD_FPR31_LS64(\thread) 159 ldc1 $f31, THREAD_FPR31(\thread)
160 .set pop 160 .set pop
161 .endm 161 .endm
162 162
@@ -211,6 +211,22 @@
211 .endm 211 .endm
212 212
213#ifdef TOOLCHAIN_SUPPORTS_MSA 213#ifdef TOOLCHAIN_SUPPORTS_MSA
214 .macro _cfcmsa rd, cs
215 .set push
216 .set mips32r2
217 .set msa
218 cfcmsa \rd, $\cs
219 .set pop
220 .endm
221
222 .macro _ctcmsa cd, rs
223 .set push
224 .set mips32r2
225 .set msa
226 ctcmsa $\cd, \rs
227 .set pop
228 .endm
229
214 .macro ld_d wd, off, base 230 .macro ld_d wd, off, base
215 .set push 231 .set push
216 .set mips32r2 232 .set mips32r2
@@ -227,35 +243,35 @@
227 .set pop 243 .set pop
228 .endm 244 .endm
229 245
230 .macro copy_u_w rd, ws, n 246 .macro copy_u_w ws, n
231 .set push 247 .set push
232 .set mips32r2 248 .set mips32r2
233 .set msa 249 .set msa
234 copy_u.w \rd, $w\ws[\n] 250 copy_u.w $1, $w\ws[\n]
235 .set pop 251 .set pop
236 .endm 252 .endm
237 253
238 .macro copy_u_d rd, ws, n 254 .macro copy_u_d ws, n
239 .set push 255 .set push
240 .set mips64r2 256 .set mips64r2
241 .set msa 257 .set msa
242 copy_u.d \rd, $w\ws[\n] 258 copy_u.d $1, $w\ws[\n]
243 .set pop 259 .set pop
244 .endm 260 .endm
245 261
246 .macro insert_w wd, n, rs 262 .macro insert_w wd, n
247 .set push 263 .set push
248 .set mips32r2 264 .set mips32r2
249 .set msa 265 .set msa
250 insert.w $w\wd[\n], \rs 266 insert.w $w\wd[\n], $1
251 .set pop 267 .set pop
252 .endm 268 .endm
253 269
254 .macro insert_d wd, n, rs 270 .macro insert_d wd, n
255 .set push 271 .set push
256 .set mips64r2 272 .set mips64r2
257 .set msa 273 .set msa
258 insert.d $w\wd[\n], \rs 274 insert.d $w\wd[\n], $1
259 .set pop 275 .set pop
260 .endm 276 .endm
261#else 277#else
@@ -283,7 +299,7 @@
283 /* 299 /*
284 * Temporary until all toolchains in use include MSA support. 300 * Temporary until all toolchains in use include MSA support.
285 */ 301 */
286 .macro cfcmsa rd, cs 302 .macro _cfcmsa rd, cs
287 .set push 303 .set push
288 .set noat 304 .set noat
289 SET_HARDFLOAT 305 SET_HARDFLOAT
@@ -293,7 +309,7 @@
293 .set pop 309 .set pop
294 .endm 310 .endm
295 311
296 .macro ctcmsa cd, rs 312 .macro _ctcmsa cd, rs
297 .set push 313 .set push
298 .set noat 314 .set noat
299 SET_HARDFLOAT 315 SET_HARDFLOAT
@@ -320,44 +336,36 @@
320 .set pop 336 .set pop
321 .endm 337 .endm
322 338
323 .macro copy_u_w rd, ws, n 339 .macro copy_u_w ws, n
324 .set push 340 .set push
325 .set noat 341 .set noat
326 SET_HARDFLOAT 342 SET_HARDFLOAT
327 .insn 343 .insn
328 .word COPY_UW_MSA_INSN | (\n << 16) | (\ws << 11) 344 .word COPY_UW_MSA_INSN | (\n << 16) | (\ws << 11)
329 /* move triggers an assembler bug... */
330 or \rd, $1, zero
331 .set pop 345 .set pop
332 .endm 346 .endm
333 347
334 .macro copy_u_d rd, ws, n 348 .macro copy_u_d ws, n
335 .set push 349 .set push
336 .set noat 350 .set noat
337 SET_HARDFLOAT 351 SET_HARDFLOAT
338 .insn 352 .insn
339 .word COPY_UD_MSA_INSN | (\n << 16) | (\ws << 11) 353 .word COPY_UD_MSA_INSN | (\n << 16) | (\ws << 11)
340 /* move triggers an assembler bug... */
341 or \rd, $1, zero
342 .set pop 354 .set pop
343 .endm 355 .endm
344 356
345 .macro insert_w wd, n, rs 357 .macro insert_w wd, n
346 .set push 358 .set push
347 .set noat 359 .set noat
348 SET_HARDFLOAT 360 SET_HARDFLOAT
349 /* move triggers an assembler bug... */
350 or $1, \rs, zero
351 .word INSERT_W_MSA_INSN | (\n << 16) | (\wd << 6) 361 .word INSERT_W_MSA_INSN | (\n << 16) | (\wd << 6)
352 .set pop 362 .set pop
353 .endm 363 .endm
354 364
355 .macro insert_d wd, n, rs 365 .macro insert_d wd, n
356 .set push 366 .set push
357 .set noat 367 .set noat
358 SET_HARDFLOAT 368 SET_HARDFLOAT
359 /* move triggers an assembler bug... */
360 or $1, \rs, zero
361 .word INSERT_D_MSA_INSN | (\n << 16) | (\wd << 6) 369 .word INSERT_D_MSA_INSN | (\n << 16) | (\wd << 6)
362 .set pop 370 .set pop
363 .endm 371 .endm
@@ -399,7 +407,7 @@
399 .set push 407 .set push
400 .set noat 408 .set noat
401 SET_HARDFLOAT 409 SET_HARDFLOAT
402 cfcmsa $1, MSA_CSR 410 _cfcmsa $1, MSA_CSR
403 sw $1, THREAD_MSA_CSR(\thread) 411 sw $1, THREAD_MSA_CSR(\thread)
404 .set pop 412 .set pop
405 .endm 413 .endm
@@ -409,7 +417,7 @@
409 .set noat 417 .set noat
410 SET_HARDFLOAT 418 SET_HARDFLOAT
411 lw $1, THREAD_MSA_CSR(\thread) 419 lw $1, THREAD_MSA_CSR(\thread)
412 ctcmsa MSA_CSR, $1 420 _ctcmsa MSA_CSR, $1
413 .set pop 421 .set pop
414 ld_d 0, THREAD_FPR0, \thread 422 ld_d 0, THREAD_FPR0, \thread
415 ld_d 1, THREAD_FPR1, \thread 423 ld_d 1, THREAD_FPR1, \thread
@@ -452,9 +460,6 @@
452 insert_w \wd, 2 460 insert_w \wd, 2
453 insert_w \wd, 3 461 insert_w \wd, 3
454#endif 462#endif
455 .if 31-\wd
456 msa_init_upper (\wd+1)
457 .endif
458 .endm 463 .endm
459 464
460 .macro msa_init_all_upper 465 .macro msa_init_all_upper
@@ -463,6 +468,37 @@
463 SET_HARDFLOAT 468 SET_HARDFLOAT
464 not $1, zero 469 not $1, zero
465 msa_init_upper 0 470 msa_init_upper 0
471 msa_init_upper 1
472 msa_init_upper 2
473 msa_init_upper 3
474 msa_init_upper 4
475 msa_init_upper 5
476 msa_init_upper 6
477 msa_init_upper 7
478 msa_init_upper 8
479 msa_init_upper 9
480 msa_init_upper 10
481 msa_init_upper 11
482 msa_init_upper 12
483 msa_init_upper 13
484 msa_init_upper 14
485 msa_init_upper 15
486 msa_init_upper 16
487 msa_init_upper 17
488 msa_init_upper 18
489 msa_init_upper 19
490 msa_init_upper 20
491 msa_init_upper 21
492 msa_init_upper 22
493 msa_init_upper 23
494 msa_init_upper 24
495 msa_init_upper 25
496 msa_init_upper 26
497 msa_init_upper 27
498 msa_init_upper 28
499 msa_init_upper 29
500 msa_init_upper 30
501 msa_init_upper 31
466 .set pop 502 .set pop
467 .endm 503 .endm
468 504
diff --git a/arch/mips/include/asm/fpu.h b/arch/mips/include/asm/fpu.h
index dd083e999b08..b104ad9d655f 100644
--- a/arch/mips/include/asm/fpu.h
+++ b/arch/mips/include/asm/fpu.h
@@ -48,6 +48,12 @@ enum fpu_mode {
48#define FPU_FR_MASK 0x1 48#define FPU_FR_MASK 0x1
49}; 49};
50 50
51#define __disable_fpu() \
52do { \
53 clear_c0_status(ST0_CU1); \
54 disable_fpu_hazard(); \
55} while (0)
56
51static inline int __enable_fpu(enum fpu_mode mode) 57static inline int __enable_fpu(enum fpu_mode mode)
52{ 58{
53 int fr; 59 int fr;
@@ -86,7 +92,12 @@ fr_common:
86 enable_fpu_hazard(); 92 enable_fpu_hazard();
87 93
88 /* check FR has the desired value */ 94 /* check FR has the desired value */
89 return (!!(read_c0_status() & ST0_FR) == !!fr) ? 0 : SIGFPE; 95 if (!!(read_c0_status() & ST0_FR) == !!fr)
96 return 0;
97
98 /* unsupported FR value */
99 __disable_fpu();
100 return SIGFPE;
90 101
91 default: 102 default:
92 BUG(); 103 BUG();
@@ -95,12 +106,6 @@ fr_common:
95 return SIGFPE; 106 return SIGFPE;
96} 107}
97 108
98#define __disable_fpu() \
99do { \
100 clear_c0_status(ST0_CU1); \
101 disable_fpu_hazard(); \
102} while (0)
103
104#define clear_fpu_owner() clear_thread_flag(TIF_USEDFPU) 109#define clear_fpu_owner() clear_thread_flag(TIF_USEDFPU)
105 110
106static inline int __is_fpu_owner(void) 111static inline int __is_fpu_owner(void)
@@ -170,6 +175,7 @@ static inline void lose_fpu(int save)
170 } 175 }
171 disable_msa(); 176 disable_msa();
172 clear_thread_flag(TIF_USEDMSA); 177 clear_thread_flag(TIF_USEDMSA);
178 __disable_fpu();
173 } else if (is_fpu_owner()) { 179 } else if (is_fpu_owner()) {
174 if (save) 180 if (save)
175 _save_fp(current); 181 _save_fp(current);
diff --git a/arch/mips/include/asm/kdebug.h b/arch/mips/include/asm/kdebug.h
index 6a9af5fcb5d7..cba22ab7ad4d 100644
--- a/arch/mips/include/asm/kdebug.h
+++ b/arch/mips/include/asm/kdebug.h
@@ -10,7 +10,8 @@ enum die_val {
10 DIE_RI, 10 DIE_RI,
11 DIE_PAGE_FAULT, 11 DIE_PAGE_FAULT,
12 DIE_BREAK, 12 DIE_BREAK,
13 DIE_SSTEPBP 13 DIE_SSTEPBP,
14 DIE_MSAFP
14}; 15};
15 16
16#endif /* _ASM_MIPS_KDEBUG_H */ 17#endif /* _ASM_MIPS_KDEBUG_H */
diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h
index ac4fc716062b..4c25823563fe 100644
--- a/arch/mips/include/asm/kvm_host.h
+++ b/arch/mips/include/asm/kvm_host.h
@@ -21,10 +21,10 @@
21 21
22/* MIPS KVM register ids */ 22/* MIPS KVM register ids */
23#define MIPS_CP0_32(_R, _S) \ 23#define MIPS_CP0_32(_R, _S) \
24 (KVM_REG_MIPS | KVM_REG_SIZE_U32 | 0x10000 | (8 * (_R) + (_S))) 24 (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U32 | (8 * (_R) + (_S)))
25 25
26#define MIPS_CP0_64(_R, _S) \ 26#define MIPS_CP0_64(_R, _S) \
27 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0x10000 | (8 * (_R) + (_S))) 27 (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U64 | (8 * (_R) + (_S)))
28 28
29#define KVM_REG_MIPS_CP0_INDEX MIPS_CP0_32(0, 0) 29#define KVM_REG_MIPS_CP0_INDEX MIPS_CP0_32(0, 0)
30#define KVM_REG_MIPS_CP0_ENTRYLO0 MIPS_CP0_64(2, 0) 30#define KVM_REG_MIPS_CP0_ENTRYLO0 MIPS_CP0_64(2, 0)
@@ -42,11 +42,14 @@
42#define KVM_REG_MIPS_CP0_STATUS MIPS_CP0_32(12, 0) 42#define KVM_REG_MIPS_CP0_STATUS MIPS_CP0_32(12, 0)
43#define KVM_REG_MIPS_CP0_CAUSE MIPS_CP0_32(13, 0) 43#define KVM_REG_MIPS_CP0_CAUSE MIPS_CP0_32(13, 0)
44#define KVM_REG_MIPS_CP0_EPC MIPS_CP0_64(14, 0) 44#define KVM_REG_MIPS_CP0_EPC MIPS_CP0_64(14, 0)
45#define KVM_REG_MIPS_CP0_PRID MIPS_CP0_32(15, 0)
45#define KVM_REG_MIPS_CP0_EBASE MIPS_CP0_64(15, 1) 46#define KVM_REG_MIPS_CP0_EBASE MIPS_CP0_64(15, 1)
46#define KVM_REG_MIPS_CP0_CONFIG MIPS_CP0_32(16, 0) 47#define KVM_REG_MIPS_CP0_CONFIG MIPS_CP0_32(16, 0)
47#define KVM_REG_MIPS_CP0_CONFIG1 MIPS_CP0_32(16, 1) 48#define KVM_REG_MIPS_CP0_CONFIG1 MIPS_CP0_32(16, 1)
48#define KVM_REG_MIPS_CP0_CONFIG2 MIPS_CP0_32(16, 2) 49#define KVM_REG_MIPS_CP0_CONFIG2 MIPS_CP0_32(16, 2)
49#define KVM_REG_MIPS_CP0_CONFIG3 MIPS_CP0_32(16, 3) 50#define KVM_REG_MIPS_CP0_CONFIG3 MIPS_CP0_32(16, 3)
51#define KVM_REG_MIPS_CP0_CONFIG4 MIPS_CP0_32(16, 4)
52#define KVM_REG_MIPS_CP0_CONFIG5 MIPS_CP0_32(16, 5)
50#define KVM_REG_MIPS_CP0_CONFIG7 MIPS_CP0_32(16, 7) 53#define KVM_REG_MIPS_CP0_CONFIG7 MIPS_CP0_32(16, 7)
51#define KVM_REG_MIPS_CP0_XCONTEXT MIPS_CP0_64(20, 0) 54#define KVM_REG_MIPS_CP0_XCONTEXT MIPS_CP0_64(20, 0)
52#define KVM_REG_MIPS_CP0_ERROREPC MIPS_CP0_64(30, 0) 55#define KVM_REG_MIPS_CP0_ERROREPC MIPS_CP0_64(30, 0)
@@ -119,6 +122,10 @@ struct kvm_vcpu_stat {
119 u32 syscall_exits; 122 u32 syscall_exits;
120 u32 resvd_inst_exits; 123 u32 resvd_inst_exits;
121 u32 break_inst_exits; 124 u32 break_inst_exits;
125 u32 trap_inst_exits;
126 u32 msa_fpe_exits;
127 u32 fpe_exits;
128 u32 msa_disabled_exits;
122 u32 flush_dcache_exits; 129 u32 flush_dcache_exits;
123 u32 halt_successful_poll; 130 u32 halt_successful_poll;
124 u32 halt_wakeup; 131 u32 halt_wakeup;
@@ -138,6 +145,10 @@ enum kvm_mips_exit_types {
138 SYSCALL_EXITS, 145 SYSCALL_EXITS,
139 RESVD_INST_EXITS, 146 RESVD_INST_EXITS,
140 BREAK_INST_EXITS, 147 BREAK_INST_EXITS,
148 TRAP_INST_EXITS,
149 MSA_FPE_EXITS,
150 FPE_EXITS,
151 MSA_DISABLED_EXITS,
141 FLUSH_DCACHE_EXITS, 152 FLUSH_DCACHE_EXITS,
142 MAX_KVM_MIPS_EXIT_TYPES 153 MAX_KVM_MIPS_EXIT_TYPES
143}; 154};
@@ -206,6 +217,8 @@ struct mips_coproc {
206#define MIPS_CP0_CONFIG1_SEL 1 217#define MIPS_CP0_CONFIG1_SEL 1
207#define MIPS_CP0_CONFIG2_SEL 2 218#define MIPS_CP0_CONFIG2_SEL 2
208#define MIPS_CP0_CONFIG3_SEL 3 219#define MIPS_CP0_CONFIG3_SEL 3
220#define MIPS_CP0_CONFIG4_SEL 4
221#define MIPS_CP0_CONFIG5_SEL 5
209 222
210/* Config0 register bits */ 223/* Config0 register bits */
211#define CP0C0_M 31 224#define CP0C0_M 31
@@ -262,31 +275,6 @@ struct mips_coproc {
262#define CP0C3_SM 1 275#define CP0C3_SM 1
263#define CP0C3_TL 0 276#define CP0C3_TL 0
264 277
265/* Have config1, Cacheable, noncoherent, write-back, write allocate*/
266#define MIPS_CONFIG0 \
267 ((1 << CP0C0_M) | (0x3 << CP0C0_K0))
268
269/* Have config2, no coprocessor2 attached, no MDMX support attached,
270 no performance counters, watch registers present,
271 no code compression, EJTAG present, no FPU, no watch registers */
272#define MIPS_CONFIG1 \
273((1 << CP0C1_M) | \
274 (0 << CP0C1_C2) | (0 << CP0C1_MD) | (0 << CP0C1_PC) | \
275 (0 << CP0C1_WR) | (0 << CP0C1_CA) | (1 << CP0C1_EP) | \
276 (0 << CP0C1_FP))
277
278/* Have config3, no tertiary/secondary caches implemented */
279#define MIPS_CONFIG2 \
280((1 << CP0C2_M))
281
282/* No config4, no DSP ASE, no large physaddr (PABITS),
283 no external interrupt controller, no vectored interrupts,
284 no 1kb pages, no SmartMIPS ASE, no trace logic */
285#define MIPS_CONFIG3 \
286((0 << CP0C3_M) | (0 << CP0C3_DSPP) | (0 << CP0C3_LPA) | \
287 (0 << CP0C3_VEIC) | (0 << CP0C3_VInt) | (0 << CP0C3_SP) | \
288 (0 << CP0C3_SM) | (0 << CP0C3_TL))
289
290/* MMU types, the first four entries have the same layout as the 278/* MMU types, the first four entries have the same layout as the
291 CP0C0_MT field. */ 279 CP0C0_MT field. */
292enum mips_mmu_types { 280enum mips_mmu_types {
@@ -321,7 +309,9 @@ enum mips_mmu_types {
321 */ 309 */
322#define T_TRAP 13 /* Trap instruction */ 310#define T_TRAP 13 /* Trap instruction */
323#define T_VCEI 14 /* Virtual coherency exception */ 311#define T_VCEI 14 /* Virtual coherency exception */
312#define T_MSAFPE 14 /* MSA floating point exception */
324#define T_FPE 15 /* Floating point exception */ 313#define T_FPE 15 /* Floating point exception */
314#define T_MSADIS 21 /* MSA disabled exception */
325#define T_WATCH 23 /* Watch address reference */ 315#define T_WATCH 23 /* Watch address reference */
326#define T_VCED 31 /* Virtual coherency data */ 316#define T_VCED 31 /* Virtual coherency data */
327 317
@@ -374,6 +364,9 @@ struct kvm_mips_tlb {
374 long tlb_lo1; 364 long tlb_lo1;
375}; 365};
376 366
367#define KVM_MIPS_FPU_FPU 0x1
368#define KVM_MIPS_FPU_MSA 0x2
369
377#define KVM_MIPS_GUEST_TLB_SIZE 64 370#define KVM_MIPS_GUEST_TLB_SIZE 64
378struct kvm_vcpu_arch { 371struct kvm_vcpu_arch {
379 void *host_ebase, *guest_ebase; 372 void *host_ebase, *guest_ebase;
@@ -395,6 +388,8 @@ struct kvm_vcpu_arch {
395 388
396 /* FPU State */ 389 /* FPU State */
397 struct mips_fpu_struct fpu; 390 struct mips_fpu_struct fpu;
391 /* Which FPU state is loaded (KVM_MIPS_FPU_*) */
392 unsigned int fpu_inuse;
398 393
399 /* COP0 State */ 394 /* COP0 State */
400 struct mips_coproc *cop0; 395 struct mips_coproc *cop0;
@@ -441,6 +436,9 @@ struct kvm_vcpu_arch {
441 436
442 /* WAIT executed */ 437 /* WAIT executed */
443 int wait; 438 int wait;
439
440 u8 fpu_enabled;
441 u8 msa_enabled;
444}; 442};
445 443
446 444
@@ -482,11 +480,15 @@ struct kvm_vcpu_arch {
482#define kvm_read_c0_guest_config1(cop0) (cop0->reg[MIPS_CP0_CONFIG][1]) 480#define kvm_read_c0_guest_config1(cop0) (cop0->reg[MIPS_CP0_CONFIG][1])
483#define kvm_read_c0_guest_config2(cop0) (cop0->reg[MIPS_CP0_CONFIG][2]) 481#define kvm_read_c0_guest_config2(cop0) (cop0->reg[MIPS_CP0_CONFIG][2])
484#define kvm_read_c0_guest_config3(cop0) (cop0->reg[MIPS_CP0_CONFIG][3]) 482#define kvm_read_c0_guest_config3(cop0) (cop0->reg[MIPS_CP0_CONFIG][3])
483#define kvm_read_c0_guest_config4(cop0) (cop0->reg[MIPS_CP0_CONFIG][4])
484#define kvm_read_c0_guest_config5(cop0) (cop0->reg[MIPS_CP0_CONFIG][5])
485#define kvm_read_c0_guest_config7(cop0) (cop0->reg[MIPS_CP0_CONFIG][7]) 485#define kvm_read_c0_guest_config7(cop0) (cop0->reg[MIPS_CP0_CONFIG][7])
486#define kvm_write_c0_guest_config(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][0] = (val)) 486#define kvm_write_c0_guest_config(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][0] = (val))
487#define kvm_write_c0_guest_config1(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][1] = (val)) 487#define kvm_write_c0_guest_config1(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][1] = (val))
488#define kvm_write_c0_guest_config2(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][2] = (val)) 488#define kvm_write_c0_guest_config2(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][2] = (val))
489#define kvm_write_c0_guest_config3(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][3] = (val)) 489#define kvm_write_c0_guest_config3(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][3] = (val))
490#define kvm_write_c0_guest_config4(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][4] = (val))
491#define kvm_write_c0_guest_config5(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][5] = (val))
490#define kvm_write_c0_guest_config7(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][7] = (val)) 492#define kvm_write_c0_guest_config7(cop0, val) (cop0->reg[MIPS_CP0_CONFIG][7] = (val))
491#define kvm_read_c0_guest_errorepc(cop0) (cop0->reg[MIPS_CP0_ERROR_PC][0]) 493#define kvm_read_c0_guest_errorepc(cop0) (cop0->reg[MIPS_CP0_ERROR_PC][0])
492#define kvm_write_c0_guest_errorepc(cop0, val) (cop0->reg[MIPS_CP0_ERROR_PC][0] = (val)) 494#define kvm_write_c0_guest_errorepc(cop0, val) (cop0->reg[MIPS_CP0_ERROR_PC][0] = (val))
@@ -567,6 +569,31 @@ static inline void _kvm_atomic_change_c0_guest_reg(unsigned long *reg,
567 kvm_set_c0_guest_ebase(cop0, ((val) & (change))); \ 569 kvm_set_c0_guest_ebase(cop0, ((val) & (change))); \
568} 570}
569 571
572/* Helpers */
573
574static inline bool kvm_mips_guest_can_have_fpu(struct kvm_vcpu_arch *vcpu)
575{
576 return (!__builtin_constant_p(cpu_has_fpu) || cpu_has_fpu) &&
577 vcpu->fpu_enabled;
578}
579
580static inline bool kvm_mips_guest_has_fpu(struct kvm_vcpu_arch *vcpu)
581{
582 return kvm_mips_guest_can_have_fpu(vcpu) &&
583 kvm_read_c0_guest_config1(vcpu->cop0) & MIPS_CONF1_FP;
584}
585
586static inline bool kvm_mips_guest_can_have_msa(struct kvm_vcpu_arch *vcpu)
587{
588 return (!__builtin_constant_p(cpu_has_msa) || cpu_has_msa) &&
589 vcpu->msa_enabled;
590}
591
592static inline bool kvm_mips_guest_has_msa(struct kvm_vcpu_arch *vcpu)
593{
594 return kvm_mips_guest_can_have_msa(vcpu) &&
595 kvm_read_c0_guest_config3(vcpu->cop0) & MIPS_CONF3_MSA;
596}
570 597
571struct kvm_mips_callbacks { 598struct kvm_mips_callbacks {
572 int (*handle_cop_unusable)(struct kvm_vcpu *vcpu); 599 int (*handle_cop_unusable)(struct kvm_vcpu *vcpu);
@@ -578,6 +605,10 @@ struct kvm_mips_callbacks {
578 int (*handle_syscall)(struct kvm_vcpu *vcpu); 605 int (*handle_syscall)(struct kvm_vcpu *vcpu);
579 int (*handle_res_inst)(struct kvm_vcpu *vcpu); 606 int (*handle_res_inst)(struct kvm_vcpu *vcpu);
580 int (*handle_break)(struct kvm_vcpu *vcpu); 607 int (*handle_break)(struct kvm_vcpu *vcpu);
608 int (*handle_trap)(struct kvm_vcpu *vcpu);
609 int (*handle_msa_fpe)(struct kvm_vcpu *vcpu);
610 int (*handle_fpe)(struct kvm_vcpu *vcpu);
611 int (*handle_msa_disabled)(struct kvm_vcpu *vcpu);
581 int (*vm_init)(struct kvm *kvm); 612 int (*vm_init)(struct kvm *kvm);
582 int (*vcpu_init)(struct kvm_vcpu *vcpu); 613 int (*vcpu_init)(struct kvm_vcpu *vcpu);
583 int (*vcpu_setup)(struct kvm_vcpu *vcpu); 614 int (*vcpu_setup)(struct kvm_vcpu *vcpu);
@@ -596,6 +627,8 @@ struct kvm_mips_callbacks {
596 const struct kvm_one_reg *reg, s64 *v); 627 const struct kvm_one_reg *reg, s64 *v);
597 int (*set_one_reg)(struct kvm_vcpu *vcpu, 628 int (*set_one_reg)(struct kvm_vcpu *vcpu,
598 const struct kvm_one_reg *reg, s64 v); 629 const struct kvm_one_reg *reg, s64 v);
630 int (*vcpu_get_regs)(struct kvm_vcpu *vcpu);
631 int (*vcpu_set_regs)(struct kvm_vcpu *vcpu);
599}; 632};
600extern struct kvm_mips_callbacks *kvm_mips_callbacks; 633extern struct kvm_mips_callbacks *kvm_mips_callbacks;
601int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks); 634int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks);
@@ -606,6 +639,19 @@ int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);
606/* Trampoline ASM routine to start running in "Guest" context */ 639/* Trampoline ASM routine to start running in "Guest" context */
607extern int __kvm_mips_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu); 640extern int __kvm_mips_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu);
608 641
642/* FPU/MSA context management */
643void __kvm_save_fpu(struct kvm_vcpu_arch *vcpu);
644void __kvm_restore_fpu(struct kvm_vcpu_arch *vcpu);
645void __kvm_restore_fcsr(struct kvm_vcpu_arch *vcpu);
646void __kvm_save_msa(struct kvm_vcpu_arch *vcpu);
647void __kvm_restore_msa(struct kvm_vcpu_arch *vcpu);
648void __kvm_restore_msa_upper(struct kvm_vcpu_arch *vcpu);
649void __kvm_restore_msacsr(struct kvm_vcpu_arch *vcpu);
650void kvm_own_fpu(struct kvm_vcpu *vcpu);
651void kvm_own_msa(struct kvm_vcpu *vcpu);
652void kvm_drop_fpu(struct kvm_vcpu *vcpu);
653void kvm_lose_fpu(struct kvm_vcpu *vcpu);
654
609/* TLB handling */ 655/* TLB handling */
610uint32_t kvm_get_kernel_asid(struct kvm_vcpu *vcpu); 656uint32_t kvm_get_kernel_asid(struct kvm_vcpu *vcpu);
611 657
@@ -711,6 +757,26 @@ extern enum emulation_result kvm_mips_emulate_bp_exc(unsigned long cause,
711 struct kvm_run *run, 757 struct kvm_run *run,
712 struct kvm_vcpu *vcpu); 758 struct kvm_vcpu *vcpu);
713 759
760extern enum emulation_result kvm_mips_emulate_trap_exc(unsigned long cause,
761 uint32_t *opc,
762 struct kvm_run *run,
763 struct kvm_vcpu *vcpu);
764
765extern enum emulation_result kvm_mips_emulate_msafpe_exc(unsigned long cause,
766 uint32_t *opc,
767 struct kvm_run *run,
768 struct kvm_vcpu *vcpu);
769
770extern enum emulation_result kvm_mips_emulate_fpe_exc(unsigned long cause,
771 uint32_t *opc,
772 struct kvm_run *run,
773 struct kvm_vcpu *vcpu);
774
775extern enum emulation_result kvm_mips_emulate_msadis_exc(unsigned long cause,
776 uint32_t *opc,
777 struct kvm_run *run,
778 struct kvm_vcpu *vcpu);
779
714extern enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, 780extern enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
715 struct kvm_run *run); 781 struct kvm_run *run);
716 782
@@ -749,6 +815,11 @@ enum emulation_result kvm_mips_emulate_load(uint32_t inst,
749 struct kvm_run *run, 815 struct kvm_run *run,
750 struct kvm_vcpu *vcpu); 816 struct kvm_vcpu *vcpu);
751 817
818unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu);
819unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu);
820unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu);
821unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu);
822
752/* Dynamic binary translation */ 823/* Dynamic binary translation */
753extern int kvm_mips_trans_cache_index(uint32_t inst, uint32_t *opc, 824extern int kvm_mips_trans_cache_index(uint32_t inst, uint32_t *opc,
754 struct kvm_vcpu *vcpu); 825 struct kvm_vcpu *vcpu);
diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h
index b5dcbee01fd7..9b3b48e21c22 100644
--- a/arch/mips/include/asm/processor.h
+++ b/arch/mips/include/asm/processor.h
@@ -105,7 +105,7 @@ union fpureg {
105#ifdef CONFIG_CPU_LITTLE_ENDIAN 105#ifdef CONFIG_CPU_LITTLE_ENDIAN
106# define FPR_IDX(width, idx) (idx) 106# define FPR_IDX(width, idx) (idx)
107#else 107#else
108# define FPR_IDX(width, idx) ((FPU_REG_WIDTH / (width)) - 1 - (idx)) 108# define FPR_IDX(width, idx) ((idx) ^ ((64 / (width)) - 1))
109#endif 109#endif
110 110
111#define BUILD_FPR_ACCESS(width) \ 111#define BUILD_FPR_ACCESS(width) \
diff --git a/arch/mips/include/uapi/asm/kvm.h b/arch/mips/include/uapi/asm/kvm.h
index 2c04b6d9ff85..6985eb59b085 100644
--- a/arch/mips/include/uapi/asm/kvm.h
+++ b/arch/mips/include/uapi/asm/kvm.h
@@ -36,77 +36,85 @@ struct kvm_regs {
36 36
37/* 37/*
38 * for KVM_GET_FPU and KVM_SET_FPU 38 * for KVM_GET_FPU and KVM_SET_FPU
39 *
40 * If Status[FR] is zero (32-bit FPU), the upper 32-bits of the FPRs
41 * are zero filled.
42 */ 39 */
43struct kvm_fpu { 40struct kvm_fpu {
44 __u64 fpr[32];
45 __u32 fir;
46 __u32 fccr;
47 __u32 fexr;
48 __u32 fenr;
49 __u32 fcsr;
50 __u32 pad;
51}; 41};
52 42
53 43
54/* 44/*
55 * For MIPS, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access CP0 45 * For MIPS, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access various
56 * registers. The id field is broken down as follows: 46 * registers. The id field is broken down as follows:
57 * 47 *
58 * bits[2..0] - Register 'sel' index.
59 * bits[7..3] - Register 'rd' index.
60 * bits[15..8] - Must be zero.
61 * bits[31..16] - 1 -> CP0 registers.
62 * bits[51..32] - Must be zero.
63 * bits[63..52] - As per linux/kvm.h 48 * bits[63..52] - As per linux/kvm.h
49 * bits[51..32] - Must be zero.
50 * bits[31..16] - Register set.
51 *
52 * Register set = 0: GP registers from kvm_regs (see definitions below).
53 *
54 * Register set = 1: CP0 registers.
55 * bits[15..8] - Must be zero.
56 * bits[7..3] - Register 'rd' index.
57 * bits[2..0] - Register 'sel' index.
58 *
59 * Register set = 2: KVM specific registers (see definitions below).
60 *
61 * Register set = 3: FPU / MSA registers (see definitions below).
64 * 62 *
65 * Other sets registers may be added in the future. Each set would 63 * Other sets registers may be added in the future. Each set would
66 * have its own identifier in bits[31..16]. 64 * have its own identifier in bits[31..16].
67 *
68 * The registers defined in struct kvm_regs are also accessible, the
69 * id values for these are below.
70 */ 65 */
71 66
72#define KVM_REG_MIPS_R0 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0) 67#define KVM_REG_MIPS_GP (KVM_REG_MIPS | 0x0000000000000000ULL)
73#define KVM_REG_MIPS_R1 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 1) 68#define KVM_REG_MIPS_CP0 (KVM_REG_MIPS | 0x0000000000010000ULL)
74#define KVM_REG_MIPS_R2 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 2) 69#define KVM_REG_MIPS_KVM (KVM_REG_MIPS | 0x0000000000020000ULL)
75#define KVM_REG_MIPS_R3 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 3) 70#define KVM_REG_MIPS_FPU (KVM_REG_MIPS | 0x0000000000030000ULL)
76#define KVM_REG_MIPS_R4 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 4) 71
77#define KVM_REG_MIPS_R5 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 5) 72
78#define KVM_REG_MIPS_R6 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 6) 73/*
79#define KVM_REG_MIPS_R7 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 7) 74 * KVM_REG_MIPS_GP - General purpose registers from kvm_regs.
80#define KVM_REG_MIPS_R8 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 8) 75 */
81#define KVM_REG_MIPS_R9 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 9) 76
82#define KVM_REG_MIPS_R10 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 10) 77#define KVM_REG_MIPS_R0 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 0)
83#define KVM_REG_MIPS_R11 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 11) 78#define KVM_REG_MIPS_R1 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 1)
84#define KVM_REG_MIPS_R12 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 12) 79#define KVM_REG_MIPS_R2 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 2)
85#define KVM_REG_MIPS_R13 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 13) 80#define KVM_REG_MIPS_R3 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 3)
86#define KVM_REG_MIPS_R14 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 14) 81#define KVM_REG_MIPS_R4 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 4)
87#define KVM_REG_MIPS_R15 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 15) 82#define KVM_REG_MIPS_R5 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 5)
88#define KVM_REG_MIPS_R16 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 16) 83#define KVM_REG_MIPS_R6 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 6)
89#define KVM_REG_MIPS_R17 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 17) 84#define KVM_REG_MIPS_R7 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 7)
90#define KVM_REG_MIPS_R18 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 18) 85#define KVM_REG_MIPS_R8 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 8)
91#define KVM_REG_MIPS_R19 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 19) 86#define KVM_REG_MIPS_R9 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 9)
92#define KVM_REG_MIPS_R20 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 20) 87#define KVM_REG_MIPS_R10 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 10)
93#define KVM_REG_MIPS_R21 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 21) 88#define KVM_REG_MIPS_R11 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 11)
94#define KVM_REG_MIPS_R22 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 22) 89#define KVM_REG_MIPS_R12 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 12)
95#define KVM_REG_MIPS_R23 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 23) 90#define KVM_REG_MIPS_R13 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 13)
96#define KVM_REG_MIPS_R24 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 24) 91#define KVM_REG_MIPS_R14 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 14)
97#define KVM_REG_MIPS_R25 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 25) 92#define KVM_REG_MIPS_R15 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 15)
98#define KVM_REG_MIPS_R26 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 26) 93#define KVM_REG_MIPS_R16 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 16)
99#define KVM_REG_MIPS_R27 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 27) 94#define KVM_REG_MIPS_R17 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 17)
100#define KVM_REG_MIPS_R28 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 28) 95#define KVM_REG_MIPS_R18 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 18)
101#define KVM_REG_MIPS_R29 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 29) 96#define KVM_REG_MIPS_R19 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 19)
102#define KVM_REG_MIPS_R30 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 30) 97#define KVM_REG_MIPS_R20 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 20)
103#define KVM_REG_MIPS_R31 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 31) 98#define KVM_REG_MIPS_R21 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 21)
104 99#define KVM_REG_MIPS_R22 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 22)
105#define KVM_REG_MIPS_HI (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 32) 100#define KVM_REG_MIPS_R23 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 23)
106#define KVM_REG_MIPS_LO (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 33) 101#define KVM_REG_MIPS_R24 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 24)
107#define KVM_REG_MIPS_PC (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 34) 102#define KVM_REG_MIPS_R25 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 25)
108 103#define KVM_REG_MIPS_R26 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 26)
109/* KVM specific control registers */ 104#define KVM_REG_MIPS_R27 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 27)
105#define KVM_REG_MIPS_R28 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 28)
106#define KVM_REG_MIPS_R29 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 29)
107#define KVM_REG_MIPS_R30 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 30)
108#define KVM_REG_MIPS_R31 (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 31)
109
110#define KVM_REG_MIPS_HI (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 32)
111#define KVM_REG_MIPS_LO (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 33)
112#define KVM_REG_MIPS_PC (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 34)
113
114
115/*
116 * KVM_REG_MIPS_KVM - KVM specific control registers.
117 */
110 118
111/* 119/*
112 * CP0_Count control 120 * CP0_Count control
@@ -118,8 +126,7 @@ struct kvm_fpu {
118 * safely without losing time or guest timer interrupts. 126 * safely without losing time or guest timer interrupts.
119 * Other: Reserved, do not change. 127 * Other: Reserved, do not change.
120 */ 128 */
121#define KVM_REG_MIPS_COUNT_CTL (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \ 129#define KVM_REG_MIPS_COUNT_CTL (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 0)
122 0x20000 | 0)
123#define KVM_REG_MIPS_COUNT_CTL_DC 0x00000001 130#define KVM_REG_MIPS_COUNT_CTL_DC 0x00000001
124 131
125/* 132/*
@@ -131,15 +138,46 @@ struct kvm_fpu {
131 * emulated. 138 * emulated.
132 * Modifications to times in the future are rejected. 139 * Modifications to times in the future are rejected.
133 */ 140 */
134#define KVM_REG_MIPS_COUNT_RESUME (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \ 141#define KVM_REG_MIPS_COUNT_RESUME (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 1)
135 0x20000 | 1)
136/* 142/*
137 * CP0_Count rate in Hz 143 * CP0_Count rate in Hz
138 * Specifies the rate of the CP0_Count timer in Hz. Modifications occur without 144 * Specifies the rate of the CP0_Count timer in Hz. Modifications occur without
139 * discontinuities in CP0_Count. 145 * discontinuities in CP0_Count.
140 */ 146 */
141#define KVM_REG_MIPS_COUNT_HZ (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \ 147#define KVM_REG_MIPS_COUNT_HZ (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 2)
142 0x20000 | 2) 148
149
150/*
151 * KVM_REG_MIPS_FPU - Floating Point and MIPS SIMD Architecture (MSA) registers.
152 *
153 * bits[15..8] - Register subset (see definitions below).
154 * bits[7..5] - Must be zero.
155 * bits[4..0] - Register number within register subset.
156 */
157
158#define KVM_REG_MIPS_FPR (KVM_REG_MIPS_FPU | 0x0000000000000000ULL)
159#define KVM_REG_MIPS_FCR (KVM_REG_MIPS_FPU | 0x0000000000000100ULL)
160#define KVM_REG_MIPS_MSACR (KVM_REG_MIPS_FPU | 0x0000000000000200ULL)
161
162/*
163 * KVM_REG_MIPS_FPR - Floating point / Vector registers.
164 */
165#define KVM_REG_MIPS_FPR_32(n) (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U32 | (n))
166#define KVM_REG_MIPS_FPR_64(n) (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U64 | (n))
167#define KVM_REG_MIPS_VEC_128(n) (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U128 | (n))
168
169/*
170 * KVM_REG_MIPS_FCR - Floating point control registers.
171 */
172#define KVM_REG_MIPS_FCR_IR (KVM_REG_MIPS_FCR | KVM_REG_SIZE_U32 | 0)
173#define KVM_REG_MIPS_FCR_CSR (KVM_REG_MIPS_FCR | KVM_REG_SIZE_U32 | 31)
174
175/*
176 * KVM_REG_MIPS_MSACR - MIPS SIMD Architecture (MSA) control registers.
177 */
178#define KVM_REG_MIPS_MSA_IR (KVM_REG_MIPS_MSACR | KVM_REG_SIZE_U32 | 0)
179#define KVM_REG_MIPS_MSA_CSR (KVM_REG_MIPS_MSACR | KVM_REG_SIZE_U32 | 1)
180
143 181
144/* 182/*
145 * KVM MIPS specific structures and definitions 183 * KVM MIPS specific structures and definitions
diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c
index 750d67ac41e9..e59fd7cfac9e 100644
--- a/arch/mips/kernel/asm-offsets.c
+++ b/arch/mips/kernel/asm-offsets.c
@@ -167,72 +167,6 @@ void output_thread_fpu_defines(void)
167 OFFSET(THREAD_FPR30, task_struct, thread.fpu.fpr[30]); 167 OFFSET(THREAD_FPR30, task_struct, thread.fpu.fpr[30]);
168 OFFSET(THREAD_FPR31, task_struct, thread.fpu.fpr[31]); 168 OFFSET(THREAD_FPR31, task_struct, thread.fpu.fpr[31]);
169 169
170 /* the least significant 64 bits of each FP register */
171 OFFSET(THREAD_FPR0_LS64, task_struct,
172 thread.fpu.fpr[0].val64[FPR_IDX(64, 0)]);
173 OFFSET(THREAD_FPR1_LS64, task_struct,
174 thread.fpu.fpr[1].val64[FPR_IDX(64, 0)]);
175 OFFSET(THREAD_FPR2_LS64, task_struct,
176 thread.fpu.fpr[2].val64[FPR_IDX(64, 0)]);
177 OFFSET(THREAD_FPR3_LS64, task_struct,
178 thread.fpu.fpr[3].val64[FPR_IDX(64, 0)]);
179 OFFSET(THREAD_FPR4_LS64, task_struct,
180 thread.fpu.fpr[4].val64[FPR_IDX(64, 0)]);
181 OFFSET(THREAD_FPR5_LS64, task_struct,
182 thread.fpu.fpr[5].val64[FPR_IDX(64, 0)]);
183 OFFSET(THREAD_FPR6_LS64, task_struct,
184 thread.fpu.fpr[6].val64[FPR_IDX(64, 0)]);
185 OFFSET(THREAD_FPR7_LS64, task_struct,
186 thread.fpu.fpr[7].val64[FPR_IDX(64, 0)]);
187 OFFSET(THREAD_FPR8_LS64, task_struct,
188 thread.fpu.fpr[8].val64[FPR_IDX(64, 0)]);
189 OFFSET(THREAD_FPR9_LS64, task_struct,
190 thread.fpu.fpr[9].val64[FPR_IDX(64, 0)]);
191 OFFSET(THREAD_FPR10_LS64, task_struct,
192 thread.fpu.fpr[10].val64[FPR_IDX(64, 0)]);
193 OFFSET(THREAD_FPR11_LS64, task_struct,
194 thread.fpu.fpr[11].val64[FPR_IDX(64, 0)]);
195 OFFSET(THREAD_FPR12_LS64, task_struct,
196 thread.fpu.fpr[12].val64[FPR_IDX(64, 0)]);
197 OFFSET(THREAD_FPR13_LS64, task_struct,
198 thread.fpu.fpr[13].val64[FPR_IDX(64, 0)]);
199 OFFSET(THREAD_FPR14_LS64, task_struct,
200 thread.fpu.fpr[14].val64[FPR_IDX(64, 0)]);
201 OFFSET(THREAD_FPR15_LS64, task_struct,
202 thread.fpu.fpr[15].val64[FPR_IDX(64, 0)]);
203 OFFSET(THREAD_FPR16_LS64, task_struct,
204 thread.fpu.fpr[16].val64[FPR_IDX(64, 0)]);
205 OFFSET(THREAD_FPR17_LS64, task_struct,
206 thread.fpu.fpr[17].val64[FPR_IDX(64, 0)]);
207 OFFSET(THREAD_FPR18_LS64, task_struct,
208 thread.fpu.fpr[18].val64[FPR_IDX(64, 0)]);
209 OFFSET(THREAD_FPR19_LS64, task_struct,
210 thread.fpu.fpr[19].val64[FPR_IDX(64, 0)]);
211 OFFSET(THREAD_FPR20_LS64, task_struct,
212 thread.fpu.fpr[20].val64[FPR_IDX(64, 0)]);
213 OFFSET(THREAD_FPR21_LS64, task_struct,
214 thread.fpu.fpr[21].val64[FPR_IDX(64, 0)]);
215 OFFSET(THREAD_FPR22_LS64, task_struct,
216 thread.fpu.fpr[22].val64[FPR_IDX(64, 0)]);
217 OFFSET(THREAD_FPR23_LS64, task_struct,
218 thread.fpu.fpr[23].val64[FPR_IDX(64, 0)]);
219 OFFSET(THREAD_FPR24_LS64, task_struct,
220 thread.fpu.fpr[24].val64[FPR_IDX(64, 0)]);
221 OFFSET(THREAD_FPR25_LS64, task_struct,
222 thread.fpu.fpr[25].val64[FPR_IDX(64, 0)]);
223 OFFSET(THREAD_FPR26_LS64, task_struct,
224 thread.fpu.fpr[26].val64[FPR_IDX(64, 0)]);
225 OFFSET(THREAD_FPR27_LS64, task_struct,
226 thread.fpu.fpr[27].val64[FPR_IDX(64, 0)]);
227 OFFSET(THREAD_FPR28_LS64, task_struct,
228 thread.fpu.fpr[28].val64[FPR_IDX(64, 0)]);
229 OFFSET(THREAD_FPR29_LS64, task_struct,
230 thread.fpu.fpr[29].val64[FPR_IDX(64, 0)]);
231 OFFSET(THREAD_FPR30_LS64, task_struct,
232 thread.fpu.fpr[30].val64[FPR_IDX(64, 0)]);
233 OFFSET(THREAD_FPR31_LS64, task_struct,
234 thread.fpu.fpr[31].val64[FPR_IDX(64, 0)]);
235
236 OFFSET(THREAD_FCR31, task_struct, thread.fpu.fcr31); 170 OFFSET(THREAD_FCR31, task_struct, thread.fpu.fcr31);
237 OFFSET(THREAD_MSA_CSR, task_struct, thread.fpu.msacsr); 171 OFFSET(THREAD_MSA_CSR, task_struct, thread.fpu.msacsr);
238 BLANK(); 172 BLANK();
@@ -470,6 +404,45 @@ void output_kvm_defines(void)
470 OFFSET(VCPU_LO, kvm_vcpu_arch, lo); 404 OFFSET(VCPU_LO, kvm_vcpu_arch, lo);
471 OFFSET(VCPU_HI, kvm_vcpu_arch, hi); 405 OFFSET(VCPU_HI, kvm_vcpu_arch, hi);
472 OFFSET(VCPU_PC, kvm_vcpu_arch, pc); 406 OFFSET(VCPU_PC, kvm_vcpu_arch, pc);
407 BLANK();
408
409 OFFSET(VCPU_FPR0, kvm_vcpu_arch, fpu.fpr[0]);
410 OFFSET(VCPU_FPR1, kvm_vcpu_arch, fpu.fpr[1]);
411 OFFSET(VCPU_FPR2, kvm_vcpu_arch, fpu.fpr[2]);
412 OFFSET(VCPU_FPR3, kvm_vcpu_arch, fpu.fpr[3]);
413 OFFSET(VCPU_FPR4, kvm_vcpu_arch, fpu.fpr[4]);
414 OFFSET(VCPU_FPR5, kvm_vcpu_arch, fpu.fpr[5]);
415 OFFSET(VCPU_FPR6, kvm_vcpu_arch, fpu.fpr[6]);
416 OFFSET(VCPU_FPR7, kvm_vcpu_arch, fpu.fpr[7]);
417 OFFSET(VCPU_FPR8, kvm_vcpu_arch, fpu.fpr[8]);
418 OFFSET(VCPU_FPR9, kvm_vcpu_arch, fpu.fpr[9]);
419 OFFSET(VCPU_FPR10, kvm_vcpu_arch, fpu.fpr[10]);
420 OFFSET(VCPU_FPR11, kvm_vcpu_arch, fpu.fpr[11]);
421 OFFSET(VCPU_FPR12, kvm_vcpu_arch, fpu.fpr[12]);
422 OFFSET(VCPU_FPR13, kvm_vcpu_arch, fpu.fpr[13]);
423 OFFSET(VCPU_FPR14, kvm_vcpu_arch, fpu.fpr[14]);
424 OFFSET(VCPU_FPR15, kvm_vcpu_arch, fpu.fpr[15]);
425 OFFSET(VCPU_FPR16, kvm_vcpu_arch, fpu.fpr[16]);
426 OFFSET(VCPU_FPR17, kvm_vcpu_arch, fpu.fpr[17]);
427 OFFSET(VCPU_FPR18, kvm_vcpu_arch, fpu.fpr[18]);
428 OFFSET(VCPU_FPR19, kvm_vcpu_arch, fpu.fpr[19]);
429 OFFSET(VCPU_FPR20, kvm_vcpu_arch, fpu.fpr[20]);
430 OFFSET(VCPU_FPR21, kvm_vcpu_arch, fpu.fpr[21]);
431 OFFSET(VCPU_FPR22, kvm_vcpu_arch, fpu.fpr[22]);
432 OFFSET(VCPU_FPR23, kvm_vcpu_arch, fpu.fpr[23]);
433 OFFSET(VCPU_FPR24, kvm_vcpu_arch, fpu.fpr[24]);
434 OFFSET(VCPU_FPR25, kvm_vcpu_arch, fpu.fpr[25]);
435 OFFSET(VCPU_FPR26, kvm_vcpu_arch, fpu.fpr[26]);
436 OFFSET(VCPU_FPR27, kvm_vcpu_arch, fpu.fpr[27]);
437 OFFSET(VCPU_FPR28, kvm_vcpu_arch, fpu.fpr[28]);
438 OFFSET(VCPU_FPR29, kvm_vcpu_arch, fpu.fpr[29]);
439 OFFSET(VCPU_FPR30, kvm_vcpu_arch, fpu.fpr[30]);
440 OFFSET(VCPU_FPR31, kvm_vcpu_arch, fpu.fpr[31]);
441
442 OFFSET(VCPU_FCR31, kvm_vcpu_arch, fpu.fcr31);
443 OFFSET(VCPU_MSA_CSR, kvm_vcpu_arch, fpu.msacsr);
444 BLANK();
445
473 OFFSET(VCPU_COP0, kvm_vcpu_arch, cop0); 446 OFFSET(VCPU_COP0, kvm_vcpu_arch, cop0);
474 OFFSET(VCPU_GUEST_KERNEL_ASID, kvm_vcpu_arch, guest_kernel_asid); 447 OFFSET(VCPU_GUEST_KERNEL_ASID, kvm_vcpu_arch, guest_kernel_asid);
475 OFFSET(VCPU_GUEST_USER_ASID, kvm_vcpu_arch, guest_user_asid); 448 OFFSET(VCPU_GUEST_USER_ASID, kvm_vcpu_arch, guest_user_asid);
diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S
index 2ebaabe3af15..af42e7003f12 100644
--- a/arch/mips/kernel/genex.S
+++ b/arch/mips/kernel/genex.S
@@ -360,12 +360,15 @@ NESTED(nmi_handler, PT_SIZE, sp)
360 .set mips1 360 .set mips1
361 SET_HARDFLOAT 361 SET_HARDFLOAT
362 cfc1 a1, fcr31 362 cfc1 a1, fcr31
363 li a2, ~(0x3f << 12)
364 and a2, a1
365 ctc1 a2, fcr31
366 .set pop 363 .set pop
367 TRACE_IRQS_ON 364 CLI
368 STI 365 TRACE_IRQS_OFF
366 .endm
367
368 .macro __build_clear_msa_fpe
369 _cfcmsa a1, MSA_CSR
370 CLI
371 TRACE_IRQS_OFF
369 .endm 372 .endm
370 373
371 .macro __build_clear_ade 374 .macro __build_clear_ade
@@ -426,7 +429,7 @@ NESTED(nmi_handler, PT_SIZE, sp)
426 BUILD_HANDLER cpu cpu sti silent /* #11 */ 429 BUILD_HANDLER cpu cpu sti silent /* #11 */
427 BUILD_HANDLER ov ov sti silent /* #12 */ 430 BUILD_HANDLER ov ov sti silent /* #12 */
428 BUILD_HANDLER tr tr sti silent /* #13 */ 431 BUILD_HANDLER tr tr sti silent /* #13 */
429 BUILD_HANDLER msa_fpe msa_fpe sti silent /* #14 */ 432 BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent /* #14 */
430 BUILD_HANDLER fpe fpe fpe silent /* #15 */ 433 BUILD_HANDLER fpe fpe fpe silent /* #15 */
431 BUILD_HANDLER ftlb ftlb none silent /* #16 */ 434 BUILD_HANDLER ftlb ftlb none silent /* #16 */
432 BUILD_HANDLER msa msa sti silent /* #21 */ 435 BUILD_HANDLER msa msa sti silent /* #21 */
diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c
index 510452812594..7da6e324dd35 100644
--- a/arch/mips/kernel/ptrace.c
+++ b/arch/mips/kernel/ptrace.c
@@ -46,6 +46,26 @@
46#define CREATE_TRACE_POINTS 46#define CREATE_TRACE_POINTS
47#include <trace/events/syscalls.h> 47#include <trace/events/syscalls.h>
48 48
49static void init_fp_ctx(struct task_struct *target)
50{
51 /* If FP has been used then the target already has context */
52 if (tsk_used_math(target))
53 return;
54
55 /* Begin with data registers set to all 1s... */
56 memset(&target->thread.fpu.fpr, ~0, sizeof(target->thread.fpu.fpr));
57
58 /* ...and FCSR zeroed */
59 target->thread.fpu.fcr31 = 0;
60
61 /*
62 * Record that the target has "used" math, such that the context
63 * just initialised, and any modifications made by the caller,
64 * aren't discarded.
65 */
66 set_stopped_child_used_math(target);
67}
68
49/* 69/*
50 * Called by kernel/ptrace.c when detaching.. 70 * Called by kernel/ptrace.c when detaching..
51 * 71 *
@@ -142,6 +162,7 @@ int ptrace_setfpregs(struct task_struct *child, __u32 __user *data)
142 if (!access_ok(VERIFY_READ, data, 33 * 8)) 162 if (!access_ok(VERIFY_READ, data, 33 * 8))
143 return -EIO; 163 return -EIO;
144 164
165 init_fp_ctx(child);
145 fregs = get_fpu_regs(child); 166 fregs = get_fpu_regs(child);
146 167
147 for (i = 0; i < 32; i++) { 168 for (i = 0; i < 32; i++) {
@@ -439,6 +460,8 @@ static int fpr_set(struct task_struct *target,
439 460
440 /* XXX fcr31 */ 461 /* XXX fcr31 */
441 462
463 init_fp_ctx(target);
464
442 if (sizeof(target->thread.fpu.fpr[i]) == sizeof(elf_fpreg_t)) 465 if (sizeof(target->thread.fpu.fpr[i]) == sizeof(elf_fpreg_t))
443 return user_regset_copyin(&pos, &count, &kbuf, &ubuf, 466 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
444 &target->thread.fpu, 467 &target->thread.fpu,
@@ -660,12 +683,7 @@ long arch_ptrace(struct task_struct *child, long request,
660 case FPR_BASE ... FPR_BASE + 31: { 683 case FPR_BASE ... FPR_BASE + 31: {
661 union fpureg *fregs = get_fpu_regs(child); 684 union fpureg *fregs = get_fpu_regs(child);
662 685
663 if (!tsk_used_math(child)) { 686 init_fp_ctx(child);
664 /* FP not yet used */
665 memset(&child->thread.fpu, ~0,
666 sizeof(child->thread.fpu));
667 child->thread.fpu.fcr31 = 0;
668 }
669#ifdef CONFIG_32BIT 687#ifdef CONFIG_32BIT
670 if (test_thread_flag(TIF_32BIT_FPREGS)) { 688 if (test_thread_flag(TIF_32BIT_FPREGS)) {
671 /* 689 /*
diff --git a/arch/mips/kernel/r4k_fpu.S b/arch/mips/kernel/r4k_fpu.S
index 676c5030a953..1d88af26ba82 100644
--- a/arch/mips/kernel/r4k_fpu.S
+++ b/arch/mips/kernel/r4k_fpu.S
@@ -34,7 +34,6 @@
34 .endm 34 .endm
35 35
36 .set noreorder 36 .set noreorder
37 .set MIPS_ISA_ARCH_LEVEL_RAW
38 37
39LEAF(_save_fp_context) 38LEAF(_save_fp_context)
40 .set push 39 .set push
@@ -103,6 +102,7 @@ LEAF(_save_fp_context)
103 /* Save 32-bit process floating point context */ 102 /* Save 32-bit process floating point context */
104LEAF(_save_fp_context32) 103LEAF(_save_fp_context32)
105 .set push 104 .set push
105 .set MIPS_ISA_ARCH_LEVEL_RAW
106 SET_HARDFLOAT 106 SET_HARDFLOAT
107 cfc1 t1, fcr31 107 cfc1 t1, fcr31
108 108
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 33984c04b60b..5b4d711f878d 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -701,6 +701,13 @@ asmlinkage void do_ov(struct pt_regs *regs)
701 701
702int process_fpemu_return(int sig, void __user *fault_addr) 702int process_fpemu_return(int sig, void __user *fault_addr)
703{ 703{
704 /*
705 * We can't allow the emulated instruction to leave any of the cause
706 * bits set in FCSR. If they were then the kernel would take an FP
707 * exception when restoring FP context.
708 */
709 current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X;
710
704 if (sig == SIGSEGV || sig == SIGBUS) { 711 if (sig == SIGSEGV || sig == SIGBUS) {
705 struct siginfo si = {0}; 712 struct siginfo si = {0};
706 si.si_addr = fault_addr; 713 si.si_addr = fault_addr;
@@ -781,6 +788,11 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
781 if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs), 788 if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs),
782 SIGFPE) == NOTIFY_STOP) 789 SIGFPE) == NOTIFY_STOP)
783 goto out; 790 goto out;
791
792 /* Clear FCSR.Cause before enabling interrupts */
793 write_32bit_cp1_register(CP1_STATUS, fcr31 & ~FPU_CSR_ALL_X);
794 local_irq_enable();
795
784 die_if_kernel("FP exception in kernel code", regs); 796 die_if_kernel("FP exception in kernel code", regs);
785 797
786 if (fcr31 & FPU_CSR_UNI_X) { 798 if (fcr31 & FPU_CSR_UNI_X) {
@@ -804,18 +816,12 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
804 sig = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1, 816 sig = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
805 &fault_addr); 817 &fault_addr);
806 818
807 /* 819 /* If something went wrong, signal */
808 * We can't allow the emulated instruction to leave any of 820 process_fpemu_return(sig, fault_addr);
809 * the cause bit set in $fcr31.
810 */
811 current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X;
812 821
813 /* Restore the hardware register state */ 822 /* Restore the hardware register state */
814 own_fpu(1); /* Using the FPU again. */ 823 own_fpu(1); /* Using the FPU again. */
815 824
816 /* If something went wrong, signal */
817 process_fpemu_return(sig, fault_addr);
818
819 goto out; 825 goto out;
820 } else if (fcr31 & FPU_CSR_INV_X) 826 } else if (fcr31 & FPU_CSR_INV_X)
821 info.si_code = FPE_FLTINV; 827 info.si_code = FPE_FLTINV;
@@ -1392,13 +1398,22 @@ out:
1392 exception_exit(prev_state); 1398 exception_exit(prev_state);
1393} 1399}
1394 1400
1395asmlinkage void do_msa_fpe(struct pt_regs *regs) 1401asmlinkage void do_msa_fpe(struct pt_regs *regs, unsigned int msacsr)
1396{ 1402{
1397 enum ctx_state prev_state; 1403 enum ctx_state prev_state;
1398 1404
1399 prev_state = exception_enter(); 1405 prev_state = exception_enter();
1406 if (notify_die(DIE_MSAFP, "MSA FP exception", regs, 0,
1407 regs_to_trapnr(regs), SIGFPE) == NOTIFY_STOP)
1408 goto out;
1409
1410 /* Clear MSACSR.Cause before enabling interrupts */
1411 write_msa_csr(msacsr & ~MSA_CSR_CAUSEF);
1412 local_irq_enable();
1413
1400 die_if_kernel("do_msa_fpe invoked from kernel context!", regs); 1414 die_if_kernel("do_msa_fpe invoked from kernel context!", regs);
1401 force_sig(SIGFPE, current); 1415 force_sig(SIGFPE, current);
1416out:
1402 exception_exit(prev_state); 1417 exception_exit(prev_state);
1403} 1418}
1404 1419
diff --git a/arch/mips/kvm/Makefile b/arch/mips/kvm/Makefile
index 401fe027c261..637ebbebd549 100644
--- a/arch/mips/kvm/Makefile
+++ b/arch/mips/kvm/Makefile
@@ -1,13 +1,15 @@
1# Makefile for KVM support for MIPS 1# Makefile for KVM support for MIPS
2# 2#
3 3
4common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o) 4common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o)
5 5
6EXTRA_CFLAGS += -Ivirt/kvm -Iarch/mips/kvm 6EXTRA_CFLAGS += -Ivirt/kvm -Iarch/mips/kvm
7 7
8kvm-objs := $(common-objs) mips.o emulate.o locore.o \ 8common-objs-$(CONFIG_CPU_HAS_MSA) += msa.o
9
10kvm-objs := $(common-objs-y) mips.o emulate.o locore.o \
9 interrupt.o stats.o commpage.o \ 11 interrupt.o stats.o commpage.o \
10 dyntrans.o trap_emul.o 12 dyntrans.o trap_emul.o fpu.o
11 13
12obj-$(CONFIG_KVM) += kvm.o 14obj-$(CONFIG_KVM) += kvm.o
13obj-y += callback.o tlb.o 15obj-y += callback.o tlb.o
diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c
index fb3e8dfd1ff6..6230f376a44e 100644
--- a/arch/mips/kvm/emulate.c
+++ b/arch/mips/kvm/emulate.c
@@ -884,6 +884,84 @@ enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
884 return EMULATE_DONE; 884 return EMULATE_DONE;
885} 885}
886 886
887/**
888 * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
889 * @vcpu: Virtual CPU.
890 *
891 * Finds the mask of bits which are writable in the guest's Config1 CP0
892 * register, by userland (currently read-only to the guest).
893 */
894unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
895{
896 unsigned int mask = 0;
897
898 /* Permit FPU to be present if FPU is supported */
899 if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
900 mask |= MIPS_CONF1_FP;
901
902 return mask;
903}
904
905/**
906 * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
907 * @vcpu: Virtual CPU.
908 *
909 * Finds the mask of bits which are writable in the guest's Config3 CP0
910 * register, by userland (currently read-only to the guest).
911 */
912unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
913{
914 /* Config4 is optional */
915 unsigned int mask = MIPS_CONF_M;
916
917 /* Permit MSA to be present if MSA is supported */
918 if (kvm_mips_guest_can_have_msa(&vcpu->arch))
919 mask |= MIPS_CONF3_MSA;
920
921 return mask;
922}
923
924/**
925 * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
926 * @vcpu: Virtual CPU.
927 *
928 * Finds the mask of bits which are writable in the guest's Config4 CP0
929 * register, by userland (currently read-only to the guest).
930 */
931unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
932{
933 /* Config5 is optional */
934 return MIPS_CONF_M;
935}
936
937/**
938 * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
939 * @vcpu: Virtual CPU.
940 *
941 * Finds the mask of bits which are writable in the guest's Config5 CP0
942 * register, by the guest itself.
943 */
944unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
945{
946 unsigned int mask = 0;
947
948 /* Permit MSAEn changes if MSA supported and enabled */
949 if (kvm_mips_guest_has_msa(&vcpu->arch))
950 mask |= MIPS_CONF5_MSAEN;
951
952 /*
953 * Permit guest FPU mode changes if FPU is enabled and the relevant
954 * feature exists according to FIR register.
955 */
956 if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
957 if (cpu_has_fre)
958 mask |= MIPS_CONF5_FRE;
959 /* We don't support UFR or UFE */
960 }
961
962 return mask;
963}
964
887enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc, 965enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc,
888 uint32_t cause, struct kvm_run *run, 966 uint32_t cause, struct kvm_run *run,
889 struct kvm_vcpu *vcpu) 967 struct kvm_vcpu *vcpu)
@@ -1021,18 +1099,114 @@ enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc,
1021 kvm_mips_write_compare(vcpu, 1099 kvm_mips_write_compare(vcpu,
1022 vcpu->arch.gprs[rt]); 1100 vcpu->arch.gprs[rt]);
1023 } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) { 1101 } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1024 kvm_write_c0_guest_status(cop0, 1102 unsigned int old_val, val, change;
1025 vcpu->arch.gprs[rt]); 1103
1104 old_val = kvm_read_c0_guest_status(cop0);
1105 val = vcpu->arch.gprs[rt];
1106 change = val ^ old_val;
1107
1108 /* Make sure that the NMI bit is never set */
1109 val &= ~ST0_NMI;
1110
1111 /*
1112 * Don't allow CU1 or FR to be set unless FPU
1113 * capability enabled and exists in guest
1114 * configuration.
1115 */
1116 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1117 val &= ~(ST0_CU1 | ST0_FR);
1118
1119 /*
1120 * Also don't allow FR to be set if host doesn't
1121 * support it.
1122 */
1123 if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1124 val &= ~ST0_FR;
1125
1126
1127 /* Handle changes in FPU mode */
1128 preempt_disable();
1129
1130 /*
1131 * FPU and Vector register state is made
1132 * UNPREDICTABLE by a change of FR, so don't
1133 * even bother saving it.
1134 */
1135 if (change & ST0_FR)
1136 kvm_drop_fpu(vcpu);
1137
1138 /*
1139 * If MSA state is already live, it is undefined
1140 * how it interacts with FR=0 FPU state, and we
1141 * don't want to hit reserved instruction
1142 * exceptions trying to save the MSA state later
1143 * when CU=1 && FR=1, so play it safe and save
1144 * it first.
1145 */
1146 if (change & ST0_CU1 && !(val & ST0_FR) &&
1147 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
1148 kvm_lose_fpu(vcpu);
1149
1026 /* 1150 /*
1027 * Make sure that CU1 and NMI bits are 1151 * Propagate CU1 (FPU enable) changes
1028 * never set 1152 * immediately if the FPU context is already
1153 * loaded. When disabling we leave the context
1154 * loaded so it can be quickly enabled again in
1155 * the near future.
1029 */ 1156 */
1030 kvm_clear_c0_guest_status(cop0, 1157 if (change & ST0_CU1 &&
1031 (ST0_CU1 | ST0_NMI)); 1158 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
1159 change_c0_status(ST0_CU1, val);
1160
1161 preempt_enable();
1162
1163 kvm_write_c0_guest_status(cop0, val);
1032 1164
1033#ifdef CONFIG_KVM_MIPS_DYN_TRANS 1165#ifdef CONFIG_KVM_MIPS_DYN_TRANS
1034 kvm_mips_trans_mtc0(inst, opc, vcpu); 1166 /*
1167 * If FPU present, we need CU1/FR bits to take
1168 * effect fairly soon.
1169 */
1170 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1171 kvm_mips_trans_mtc0(inst, opc, vcpu);
1035#endif 1172#endif
1173 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1174 unsigned int old_val, val, change, wrmask;
1175
1176 old_val = kvm_read_c0_guest_config5(cop0);
1177 val = vcpu->arch.gprs[rt];
1178
1179 /* Only a few bits are writable in Config5 */
1180 wrmask = kvm_mips_config5_wrmask(vcpu);
1181 change = (val ^ old_val) & wrmask;
1182 val = old_val ^ change;
1183
1184
1185 /* Handle changes in FPU/MSA modes */
1186 preempt_disable();
1187
1188 /*
1189 * Propagate FRE changes immediately if the FPU
1190 * context is already loaded.
1191 */
1192 if (change & MIPS_CONF5_FRE &&
1193 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
1194 change_c0_config5(MIPS_CONF5_FRE, val);
1195
1196 /*
1197 * Propagate MSAEn changes immediately if the
1198 * MSA context is already loaded. When disabling
1199 * we leave the context loaded so it can be
1200 * quickly enabled again in the near future.
1201 */
1202 if (change & MIPS_CONF5_MSAEN &&
1203 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
1204 change_c0_config5(MIPS_CONF5_MSAEN,
1205 val);
1206
1207 preempt_enable();
1208
1209 kvm_write_c0_guest_config5(cop0, val);
1036 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) { 1210 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1037 uint32_t old_cause, new_cause; 1211 uint32_t old_cause, new_cause;
1038 1212
@@ -1970,6 +2144,146 @@ enum emulation_result kvm_mips_emulate_bp_exc(unsigned long cause,
1970 return er; 2144 return er;
1971} 2145}
1972 2146
2147enum emulation_result kvm_mips_emulate_trap_exc(unsigned long cause,
2148 uint32_t *opc,
2149 struct kvm_run *run,
2150 struct kvm_vcpu *vcpu)
2151{
2152 struct mips_coproc *cop0 = vcpu->arch.cop0;
2153 struct kvm_vcpu_arch *arch = &vcpu->arch;
2154 enum emulation_result er = EMULATE_DONE;
2155
2156 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2157 /* save old pc */
2158 kvm_write_c0_guest_epc(cop0, arch->pc);
2159 kvm_set_c0_guest_status(cop0, ST0_EXL);
2160
2161 if (cause & CAUSEF_BD)
2162 kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2163 else
2164 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2165
2166 kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2167
2168 kvm_change_c0_guest_cause(cop0, (0xff),
2169 (T_TRAP << CAUSEB_EXCCODE));
2170
2171 /* Set PC to the exception entry point */
2172 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2173
2174 } else {
2175 kvm_err("Trying to deliver TRAP when EXL is already set\n");
2176 er = EMULATE_FAIL;
2177 }
2178
2179 return er;
2180}
2181
2182enum emulation_result kvm_mips_emulate_msafpe_exc(unsigned long cause,
2183 uint32_t *opc,
2184 struct kvm_run *run,
2185 struct kvm_vcpu *vcpu)
2186{
2187 struct mips_coproc *cop0 = vcpu->arch.cop0;
2188 struct kvm_vcpu_arch *arch = &vcpu->arch;
2189 enum emulation_result er = EMULATE_DONE;
2190
2191 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2192 /* save old pc */
2193 kvm_write_c0_guest_epc(cop0, arch->pc);
2194 kvm_set_c0_guest_status(cop0, ST0_EXL);
2195
2196 if (cause & CAUSEF_BD)
2197 kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2198 else
2199 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2200
2201 kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2202
2203 kvm_change_c0_guest_cause(cop0, (0xff),
2204 (T_MSAFPE << CAUSEB_EXCCODE));
2205
2206 /* Set PC to the exception entry point */
2207 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2208
2209 } else {
2210 kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2211 er = EMULATE_FAIL;
2212 }
2213
2214 return er;
2215}
2216
2217enum emulation_result kvm_mips_emulate_fpe_exc(unsigned long cause,
2218 uint32_t *opc,
2219 struct kvm_run *run,
2220 struct kvm_vcpu *vcpu)
2221{
2222 struct mips_coproc *cop0 = vcpu->arch.cop0;
2223 struct kvm_vcpu_arch *arch = &vcpu->arch;
2224 enum emulation_result er = EMULATE_DONE;
2225
2226 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2227 /* save old pc */
2228 kvm_write_c0_guest_epc(cop0, arch->pc);
2229 kvm_set_c0_guest_status(cop0, ST0_EXL);
2230
2231 if (cause & CAUSEF_BD)
2232 kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2233 else
2234 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2235
2236 kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2237
2238 kvm_change_c0_guest_cause(cop0, (0xff),
2239 (T_FPE << CAUSEB_EXCCODE));
2240
2241 /* Set PC to the exception entry point */
2242 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2243
2244 } else {
2245 kvm_err("Trying to deliver FPE when EXL is already set\n");
2246 er = EMULATE_FAIL;
2247 }
2248
2249 return er;
2250}
2251
2252enum emulation_result kvm_mips_emulate_msadis_exc(unsigned long cause,
2253 uint32_t *opc,
2254 struct kvm_run *run,
2255 struct kvm_vcpu *vcpu)
2256{
2257 struct mips_coproc *cop0 = vcpu->arch.cop0;
2258 struct kvm_vcpu_arch *arch = &vcpu->arch;
2259 enum emulation_result er = EMULATE_DONE;
2260
2261 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2262 /* save old pc */
2263 kvm_write_c0_guest_epc(cop0, arch->pc);
2264 kvm_set_c0_guest_status(cop0, ST0_EXL);
2265
2266 if (cause & CAUSEF_BD)
2267 kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2268 else
2269 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2270
2271 kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2272
2273 kvm_change_c0_guest_cause(cop0, (0xff),
2274 (T_MSADIS << CAUSEB_EXCCODE));
2275
2276 /* Set PC to the exception entry point */
2277 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2278
2279 } else {
2280 kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2281 er = EMULATE_FAIL;
2282 }
2283
2284 return er;
2285}
2286
1973/* ll/sc, rdhwr, sync emulation */ 2287/* ll/sc, rdhwr, sync emulation */
1974 2288
1975#define OPCODE 0xfc000000 2289#define OPCODE 0xfc000000
@@ -2176,6 +2490,10 @@ enum emulation_result kvm_mips_check_privilege(unsigned long cause,
2176 case T_SYSCALL: 2490 case T_SYSCALL:
2177 case T_BREAK: 2491 case T_BREAK:
2178 case T_RES_INST: 2492 case T_RES_INST:
2493 case T_TRAP:
2494 case T_MSAFPE:
2495 case T_FPE:
2496 case T_MSADIS:
2179 break; 2497 break;
2180 2498
2181 case T_COP_UNUSABLE: 2499 case T_COP_UNUSABLE:
diff --git a/arch/mips/kvm/fpu.S b/arch/mips/kvm/fpu.S
new file mode 100644
index 000000000000..531fbf5131c0
--- /dev/null
+++ b/arch/mips/kvm/fpu.S
@@ -0,0 +1,122 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * FPU context handling code for KVM.
7 *
8 * Copyright (C) 2015 Imagination Technologies Ltd.
9 */
10
11#include <asm/asm.h>
12#include <asm/asm-offsets.h>
13#include <asm/fpregdef.h>
14#include <asm/mipsregs.h>
15#include <asm/regdef.h>
16
17 .set noreorder
18 .set noat
19
20LEAF(__kvm_save_fpu)
21 .set push
22 .set mips64r2
23 SET_HARDFLOAT
24 mfc0 t0, CP0_STATUS
25 sll t0, t0, 5 # is Status.FR set?
26 bgez t0, 1f # no: skip odd doubles
27 nop
28 sdc1 $f1, VCPU_FPR1(a0)
29 sdc1 $f3, VCPU_FPR3(a0)
30 sdc1 $f5, VCPU_FPR5(a0)
31 sdc1 $f7, VCPU_FPR7(a0)
32 sdc1 $f9, VCPU_FPR9(a0)
33 sdc1 $f11, VCPU_FPR11(a0)
34 sdc1 $f13, VCPU_FPR13(a0)
35 sdc1 $f15, VCPU_FPR15(a0)
36 sdc1 $f17, VCPU_FPR17(a0)
37 sdc1 $f19, VCPU_FPR19(a0)
38 sdc1 $f21, VCPU_FPR21(a0)
39 sdc1 $f23, VCPU_FPR23(a0)
40 sdc1 $f25, VCPU_FPR25(a0)
41 sdc1 $f27, VCPU_FPR27(a0)
42 sdc1 $f29, VCPU_FPR29(a0)
43 sdc1 $f31, VCPU_FPR31(a0)
441: sdc1 $f0, VCPU_FPR0(a0)
45 sdc1 $f2, VCPU_FPR2(a0)
46 sdc1 $f4, VCPU_FPR4(a0)
47 sdc1 $f6, VCPU_FPR6(a0)
48 sdc1 $f8, VCPU_FPR8(a0)
49 sdc1 $f10, VCPU_FPR10(a0)
50 sdc1 $f12, VCPU_FPR12(a0)
51 sdc1 $f14, VCPU_FPR14(a0)
52 sdc1 $f16, VCPU_FPR16(a0)
53 sdc1 $f18, VCPU_FPR18(a0)
54 sdc1 $f20, VCPU_FPR20(a0)
55 sdc1 $f22, VCPU_FPR22(a0)
56 sdc1 $f24, VCPU_FPR24(a0)
57 sdc1 $f26, VCPU_FPR26(a0)
58 sdc1 $f28, VCPU_FPR28(a0)
59 jr ra
60 sdc1 $f30, VCPU_FPR30(a0)
61 .set pop
62 END(__kvm_save_fpu)
63
64LEAF(__kvm_restore_fpu)
65 .set push
66 .set mips64r2
67 SET_HARDFLOAT
68 mfc0 t0, CP0_STATUS
69 sll t0, t0, 5 # is Status.FR set?
70 bgez t0, 1f # no: skip odd doubles
71 nop
72 ldc1 $f1, VCPU_FPR1(a0)
73 ldc1 $f3, VCPU_FPR3(a0)
74 ldc1 $f5, VCPU_FPR5(a0)
75 ldc1 $f7, VCPU_FPR7(a0)
76 ldc1 $f9, VCPU_FPR9(a0)
77 ldc1 $f11, VCPU_FPR11(a0)
78 ldc1 $f13, VCPU_FPR13(a0)
79 ldc1 $f15, VCPU_FPR15(a0)
80 ldc1 $f17, VCPU_FPR17(a0)
81 ldc1 $f19, VCPU_FPR19(a0)
82 ldc1 $f21, VCPU_FPR21(a0)
83 ldc1 $f23, VCPU_FPR23(a0)
84 ldc1 $f25, VCPU_FPR25(a0)
85 ldc1 $f27, VCPU_FPR27(a0)
86 ldc1 $f29, VCPU_FPR29(a0)
87 ldc1 $f31, VCPU_FPR31(a0)
881: ldc1 $f0, VCPU_FPR0(a0)
89 ldc1 $f2, VCPU_FPR2(a0)
90 ldc1 $f4, VCPU_FPR4(a0)
91 ldc1 $f6, VCPU_FPR6(a0)
92 ldc1 $f8, VCPU_FPR8(a0)
93 ldc1 $f10, VCPU_FPR10(a0)
94 ldc1 $f12, VCPU_FPR12(a0)
95 ldc1 $f14, VCPU_FPR14(a0)
96 ldc1 $f16, VCPU_FPR16(a0)
97 ldc1 $f18, VCPU_FPR18(a0)
98 ldc1 $f20, VCPU_FPR20(a0)
99 ldc1 $f22, VCPU_FPR22(a0)
100 ldc1 $f24, VCPU_FPR24(a0)
101 ldc1 $f26, VCPU_FPR26(a0)
102 ldc1 $f28, VCPU_FPR28(a0)
103 jr ra
104 ldc1 $f30, VCPU_FPR30(a0)
105 .set pop
106 END(__kvm_restore_fpu)
107
108LEAF(__kvm_restore_fcsr)
109 .set push
110 SET_HARDFLOAT
111 lw t0, VCPU_FCR31(a0)
112 /*
113 * The ctc1 must stay at this offset in __kvm_restore_fcsr.
114 * See kvm_mips_csr_die_notify() which handles t0 containing a value
115 * which triggers an FP Exception, which must be stepped over and
116 * ignored since the set cause bits must remain there for the guest.
117 */
118 ctc1 t0, fcr31
119 jr ra
120 nop
121 .set pop
122 END(__kvm_restore_fcsr)
diff --git a/arch/mips/kvm/locore.S b/arch/mips/kvm/locore.S
index 4a68b176d6e4..c567240386a0 100644
--- a/arch/mips/kvm/locore.S
+++ b/arch/mips/kvm/locore.S
@@ -36,6 +36,8 @@
36#define PT_HOST_USERLOCAL PT_EPC 36#define PT_HOST_USERLOCAL PT_EPC
37 37
38#define CP0_DDATA_LO $28,3 38#define CP0_DDATA_LO $28,3
39#define CP0_CONFIG3 $16,3
40#define CP0_CONFIG5 $16,5
39#define CP0_EBASE $15,1 41#define CP0_EBASE $15,1
40 42
41#define CP0_INTCTL $12,1 43#define CP0_INTCTL $12,1
@@ -353,6 +355,42 @@ NESTED (MIPSX(GuestException), CALLFRAME_SIZ, ra)
353 LONG_L k0, VCPU_HOST_EBASE(k1) 355 LONG_L k0, VCPU_HOST_EBASE(k1)
354 mtc0 k0,CP0_EBASE 356 mtc0 k0,CP0_EBASE
355 357
358 /*
359 * If FPU is enabled, save FCR31 and clear it so that later ctc1's don't
360 * trigger FPE for pending exceptions.
361 */
362 .set at
363 and v1, v0, ST0_CU1
364 beqz v1, 1f
365 nop
366 .set push
367 SET_HARDFLOAT
368 cfc1 t0, fcr31
369 sw t0, VCPU_FCR31(k1)
370 ctc1 zero,fcr31
371 .set pop
372 .set noat
3731:
374
375#ifdef CONFIG_CPU_HAS_MSA
376 /*
377 * If MSA is enabled, save MSACSR and clear it so that later
378 * instructions don't trigger MSAFPE for pending exceptions.
379 */
380 mfc0 t0, CP0_CONFIG3
381 ext t0, t0, 28, 1 /* MIPS_CONF3_MSAP */
382 beqz t0, 1f
383 nop
384 mfc0 t0, CP0_CONFIG5
385 ext t0, t0, 27, 1 /* MIPS_CONF5_MSAEN */
386 beqz t0, 1f
387 nop
388 _cfcmsa t0, MSA_CSR
389 sw t0, VCPU_MSA_CSR(k1)
390 _ctcmsa MSA_CSR, zero
3911:
392#endif
393
356 /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */ 394 /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */
357 .set at 395 .set at
358 and v0, v0, ~(ST0_EXL | KSU_USER | ST0_IE) 396 and v0, v0, ~(ST0_EXL | KSU_USER | ST0_IE)
diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index c9eccf5df912..bb68e8d520e8 100644
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -11,6 +11,7 @@
11 11
12#include <linux/errno.h> 12#include <linux/errno.h>
13#include <linux/err.h> 13#include <linux/err.h>
14#include <linux/kdebug.h>
14#include <linux/module.h> 15#include <linux/module.h>
15#include <linux/vmalloc.h> 16#include <linux/vmalloc.h>
16#include <linux/fs.h> 17#include <linux/fs.h>
@@ -48,6 +49,10 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
48 { "syscall", VCPU_STAT(syscall_exits), KVM_STAT_VCPU }, 49 { "syscall", VCPU_STAT(syscall_exits), KVM_STAT_VCPU },
49 { "resvd_inst", VCPU_STAT(resvd_inst_exits), KVM_STAT_VCPU }, 50 { "resvd_inst", VCPU_STAT(resvd_inst_exits), KVM_STAT_VCPU },
50 { "break_inst", VCPU_STAT(break_inst_exits), KVM_STAT_VCPU }, 51 { "break_inst", VCPU_STAT(break_inst_exits), KVM_STAT_VCPU },
52 { "trap_inst", VCPU_STAT(trap_inst_exits), KVM_STAT_VCPU },
53 { "msa_fpe", VCPU_STAT(msa_fpe_exits), KVM_STAT_VCPU },
54 { "fpe", VCPU_STAT(fpe_exits), KVM_STAT_VCPU },
55 { "msa_disabled", VCPU_STAT(msa_disabled_exits), KVM_STAT_VCPU },
51 { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU }, 56 { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
52 { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU }, 57 { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
53 { "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU }, 58 { "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU },
@@ -504,10 +509,13 @@ static u64 kvm_mips_get_one_regs[] = {
504 KVM_REG_MIPS_CP0_STATUS, 509 KVM_REG_MIPS_CP0_STATUS,
505 KVM_REG_MIPS_CP0_CAUSE, 510 KVM_REG_MIPS_CP0_CAUSE,
506 KVM_REG_MIPS_CP0_EPC, 511 KVM_REG_MIPS_CP0_EPC,
512 KVM_REG_MIPS_CP0_PRID,
507 KVM_REG_MIPS_CP0_CONFIG, 513 KVM_REG_MIPS_CP0_CONFIG,
508 KVM_REG_MIPS_CP0_CONFIG1, 514 KVM_REG_MIPS_CP0_CONFIG1,
509 KVM_REG_MIPS_CP0_CONFIG2, 515 KVM_REG_MIPS_CP0_CONFIG2,
510 KVM_REG_MIPS_CP0_CONFIG3, 516 KVM_REG_MIPS_CP0_CONFIG3,
517 KVM_REG_MIPS_CP0_CONFIG4,
518 KVM_REG_MIPS_CP0_CONFIG5,
511 KVM_REG_MIPS_CP0_CONFIG7, 519 KVM_REG_MIPS_CP0_CONFIG7,
512 KVM_REG_MIPS_CP0_ERROREPC, 520 KVM_REG_MIPS_CP0_ERROREPC,
513 521
@@ -520,10 +528,14 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
520 const struct kvm_one_reg *reg) 528 const struct kvm_one_reg *reg)
521{ 529{
522 struct mips_coproc *cop0 = vcpu->arch.cop0; 530 struct mips_coproc *cop0 = vcpu->arch.cop0;
531 struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
523 int ret; 532 int ret;
524 s64 v; 533 s64 v;
534 s64 vs[2];
535 unsigned int idx;
525 536
526 switch (reg->id) { 537 switch (reg->id) {
538 /* General purpose registers */
527 case KVM_REG_MIPS_R0 ... KVM_REG_MIPS_R31: 539 case KVM_REG_MIPS_R0 ... KVM_REG_MIPS_R31:
528 v = (long)vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0]; 540 v = (long)vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0];
529 break; 541 break;
@@ -537,6 +549,67 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
537 v = (long)vcpu->arch.pc; 549 v = (long)vcpu->arch.pc;
538 break; 550 break;
539 551
552 /* Floating point registers */
553 case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
554 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
555 return -EINVAL;
556 idx = reg->id - KVM_REG_MIPS_FPR_32(0);
557 /* Odd singles in top of even double when FR=0 */
558 if (kvm_read_c0_guest_status(cop0) & ST0_FR)
559 v = get_fpr32(&fpu->fpr[idx], 0);
560 else
561 v = get_fpr32(&fpu->fpr[idx & ~1], idx & 1);
562 break;
563 case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
564 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
565 return -EINVAL;
566 idx = reg->id - KVM_REG_MIPS_FPR_64(0);
567 /* Can't access odd doubles in FR=0 mode */
568 if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
569 return -EINVAL;
570 v = get_fpr64(&fpu->fpr[idx], 0);
571 break;
572 case KVM_REG_MIPS_FCR_IR:
573 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
574 return -EINVAL;
575 v = boot_cpu_data.fpu_id;
576 break;
577 case KVM_REG_MIPS_FCR_CSR:
578 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
579 return -EINVAL;
580 v = fpu->fcr31;
581 break;
582
583 /* MIPS SIMD Architecture (MSA) registers */
584 case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
585 if (!kvm_mips_guest_has_msa(&vcpu->arch))
586 return -EINVAL;
587 /* Can't access MSA registers in FR=0 mode */
588 if (!(kvm_read_c0_guest_status(cop0) & ST0_FR))
589 return -EINVAL;
590 idx = reg->id - KVM_REG_MIPS_VEC_128(0);
591#ifdef CONFIG_CPU_LITTLE_ENDIAN
592 /* least significant byte first */
593 vs[0] = get_fpr64(&fpu->fpr[idx], 0);
594 vs[1] = get_fpr64(&fpu->fpr[idx], 1);
595#else
596 /* most significant byte first */
597 vs[0] = get_fpr64(&fpu->fpr[idx], 1);
598 vs[1] = get_fpr64(&fpu->fpr[idx], 0);
599#endif
600 break;
601 case KVM_REG_MIPS_MSA_IR:
602 if (!kvm_mips_guest_has_msa(&vcpu->arch))
603 return -EINVAL;
604 v = boot_cpu_data.msa_id;
605 break;
606 case KVM_REG_MIPS_MSA_CSR:
607 if (!kvm_mips_guest_has_msa(&vcpu->arch))
608 return -EINVAL;
609 v = fpu->msacsr;
610 break;
611
612 /* Co-processor 0 registers */
540 case KVM_REG_MIPS_CP0_INDEX: 613 case KVM_REG_MIPS_CP0_INDEX:
541 v = (long)kvm_read_c0_guest_index(cop0); 614 v = (long)kvm_read_c0_guest_index(cop0);
542 break; 615 break;
@@ -573,8 +646,8 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
573 case KVM_REG_MIPS_CP0_EPC: 646 case KVM_REG_MIPS_CP0_EPC:
574 v = (long)kvm_read_c0_guest_epc(cop0); 647 v = (long)kvm_read_c0_guest_epc(cop0);
575 break; 648 break;
576 case KVM_REG_MIPS_CP0_ERROREPC: 649 case KVM_REG_MIPS_CP0_PRID:
577 v = (long)kvm_read_c0_guest_errorepc(cop0); 650 v = (long)kvm_read_c0_guest_prid(cop0);
578 break; 651 break;
579 case KVM_REG_MIPS_CP0_CONFIG: 652 case KVM_REG_MIPS_CP0_CONFIG:
580 v = (long)kvm_read_c0_guest_config(cop0); 653 v = (long)kvm_read_c0_guest_config(cop0);
@@ -588,9 +661,18 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
588 case KVM_REG_MIPS_CP0_CONFIG3: 661 case KVM_REG_MIPS_CP0_CONFIG3:
589 v = (long)kvm_read_c0_guest_config3(cop0); 662 v = (long)kvm_read_c0_guest_config3(cop0);
590 break; 663 break;
664 case KVM_REG_MIPS_CP0_CONFIG4:
665 v = (long)kvm_read_c0_guest_config4(cop0);
666 break;
667 case KVM_REG_MIPS_CP0_CONFIG5:
668 v = (long)kvm_read_c0_guest_config5(cop0);
669 break;
591 case KVM_REG_MIPS_CP0_CONFIG7: 670 case KVM_REG_MIPS_CP0_CONFIG7:
592 v = (long)kvm_read_c0_guest_config7(cop0); 671 v = (long)kvm_read_c0_guest_config7(cop0);
593 break; 672 break;
673 case KVM_REG_MIPS_CP0_ERROREPC:
674 v = (long)kvm_read_c0_guest_errorepc(cop0);
675 break;
594 /* registers to be handled specially */ 676 /* registers to be handled specially */
595 case KVM_REG_MIPS_CP0_COUNT: 677 case KVM_REG_MIPS_CP0_COUNT:
596 case KVM_REG_MIPS_COUNT_CTL: 678 case KVM_REG_MIPS_COUNT_CTL:
@@ -612,6 +694,10 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
612 u32 v32 = (u32)v; 694 u32 v32 = (u32)v;
613 695
614 return put_user(v32, uaddr32); 696 return put_user(v32, uaddr32);
697 } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
698 void __user *uaddr = (void __user *)(long)reg->addr;
699
700 return copy_to_user(uaddr, vs, 16);
615 } else { 701 } else {
616 return -EINVAL; 702 return -EINVAL;
617 } 703 }
@@ -621,7 +707,10 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
621 const struct kvm_one_reg *reg) 707 const struct kvm_one_reg *reg)
622{ 708{
623 struct mips_coproc *cop0 = vcpu->arch.cop0; 709 struct mips_coproc *cop0 = vcpu->arch.cop0;
624 u64 v; 710 struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
711 s64 v;
712 s64 vs[2];
713 unsigned int idx;
625 714
626 if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) { 715 if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
627 u64 __user *uaddr64 = (u64 __user *)(long)reg->addr; 716 u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;
@@ -635,11 +724,16 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
635 if (get_user(v32, uaddr32) != 0) 724 if (get_user(v32, uaddr32) != 0)
636 return -EFAULT; 725 return -EFAULT;
637 v = (s64)v32; 726 v = (s64)v32;
727 } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
728 void __user *uaddr = (void __user *)(long)reg->addr;
729
730 return copy_from_user(vs, uaddr, 16);
638 } else { 731 } else {
639 return -EINVAL; 732 return -EINVAL;
640 } 733 }
641 734
642 switch (reg->id) { 735 switch (reg->id) {
736 /* General purpose registers */
643 case KVM_REG_MIPS_R0: 737 case KVM_REG_MIPS_R0:
644 /* Silently ignore requests to set $0 */ 738 /* Silently ignore requests to set $0 */
645 break; 739 break;
@@ -656,6 +750,64 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
656 vcpu->arch.pc = v; 750 vcpu->arch.pc = v;
657 break; 751 break;
658 752
753 /* Floating point registers */
754 case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
755 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
756 return -EINVAL;
757 idx = reg->id - KVM_REG_MIPS_FPR_32(0);
758 /* Odd singles in top of even double when FR=0 */
759 if (kvm_read_c0_guest_status(cop0) & ST0_FR)
760 set_fpr32(&fpu->fpr[idx], 0, v);
761 else
762 set_fpr32(&fpu->fpr[idx & ~1], idx & 1, v);
763 break;
764 case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
765 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
766 return -EINVAL;
767 idx = reg->id - KVM_REG_MIPS_FPR_64(0);
768 /* Can't access odd doubles in FR=0 mode */
769 if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
770 return -EINVAL;
771 set_fpr64(&fpu->fpr[idx], 0, v);
772 break;
773 case KVM_REG_MIPS_FCR_IR:
774 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
775 return -EINVAL;
776 /* Read-only */
777 break;
778 case KVM_REG_MIPS_FCR_CSR:
779 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
780 return -EINVAL;
781 fpu->fcr31 = v;
782 break;
783
784 /* MIPS SIMD Architecture (MSA) registers */
785 case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
786 if (!kvm_mips_guest_has_msa(&vcpu->arch))
787 return -EINVAL;
788 idx = reg->id - KVM_REG_MIPS_VEC_128(0);
789#ifdef CONFIG_CPU_LITTLE_ENDIAN
790 /* least significant byte first */
791 set_fpr64(&fpu->fpr[idx], 0, vs[0]);
792 set_fpr64(&fpu->fpr[idx], 1, vs[1]);
793#else
794 /* most significant byte first */
795 set_fpr64(&fpu->fpr[idx], 1, vs[0]);
796 set_fpr64(&fpu->fpr[idx], 0, vs[1]);
797#endif
798 break;
799 case KVM_REG_MIPS_MSA_IR:
800 if (!kvm_mips_guest_has_msa(&vcpu->arch))
801 return -EINVAL;
802 /* Read-only */
803 break;
804 case KVM_REG_MIPS_MSA_CSR:
805 if (!kvm_mips_guest_has_msa(&vcpu->arch))
806 return -EINVAL;
807 fpu->msacsr = v;
808 break;
809
810 /* Co-processor 0 registers */
659 case KVM_REG_MIPS_CP0_INDEX: 811 case KVM_REG_MIPS_CP0_INDEX:
660 kvm_write_c0_guest_index(cop0, v); 812 kvm_write_c0_guest_index(cop0, v);
661 break; 813 break;
@@ -686,6 +838,9 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
686 case KVM_REG_MIPS_CP0_EPC: 838 case KVM_REG_MIPS_CP0_EPC:
687 kvm_write_c0_guest_epc(cop0, v); 839 kvm_write_c0_guest_epc(cop0, v);
688 break; 840 break;
841 case KVM_REG_MIPS_CP0_PRID:
842 kvm_write_c0_guest_prid(cop0, v);
843 break;
689 case KVM_REG_MIPS_CP0_ERROREPC: 844 case KVM_REG_MIPS_CP0_ERROREPC:
690 kvm_write_c0_guest_errorepc(cop0, v); 845 kvm_write_c0_guest_errorepc(cop0, v);
691 break; 846 break;
@@ -693,6 +848,12 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
693 case KVM_REG_MIPS_CP0_COUNT: 848 case KVM_REG_MIPS_CP0_COUNT:
694 case KVM_REG_MIPS_CP0_COMPARE: 849 case KVM_REG_MIPS_CP0_COMPARE:
695 case KVM_REG_MIPS_CP0_CAUSE: 850 case KVM_REG_MIPS_CP0_CAUSE:
851 case KVM_REG_MIPS_CP0_CONFIG:
852 case KVM_REG_MIPS_CP0_CONFIG1:
853 case KVM_REG_MIPS_CP0_CONFIG2:
854 case KVM_REG_MIPS_CP0_CONFIG3:
855 case KVM_REG_MIPS_CP0_CONFIG4:
856 case KVM_REG_MIPS_CP0_CONFIG5:
696 case KVM_REG_MIPS_COUNT_CTL: 857 case KVM_REG_MIPS_COUNT_CTL:
697 case KVM_REG_MIPS_COUNT_RESUME: 858 case KVM_REG_MIPS_COUNT_RESUME:
698 case KVM_REG_MIPS_COUNT_HZ: 859 case KVM_REG_MIPS_COUNT_HZ:
@@ -703,6 +864,33 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
703 return 0; 864 return 0;
704} 865}
705 866
867static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
868 struct kvm_enable_cap *cap)
869{
870 int r = 0;
871
872 if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap))
873 return -EINVAL;
874 if (cap->flags)
875 return -EINVAL;
876 if (cap->args[0])
877 return -EINVAL;
878
879 switch (cap->cap) {
880 case KVM_CAP_MIPS_FPU:
881 vcpu->arch.fpu_enabled = true;
882 break;
883 case KVM_CAP_MIPS_MSA:
884 vcpu->arch.msa_enabled = true;
885 break;
886 default:
887 r = -EINVAL;
888 break;
889 }
890
891 return r;
892}
893
706long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, 894long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
707 unsigned long arg) 895 unsigned long arg)
708{ 896{
@@ -760,6 +948,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
760 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); 948 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
761 break; 949 break;
762 } 950 }
951 case KVM_ENABLE_CAP: {
952 struct kvm_enable_cap cap;
953
954 r = -EFAULT;
955 if (copy_from_user(&cap, argp, sizeof(cap)))
956 goto out;
957 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
958 break;
959 }
763 default: 960 default:
764 r = -ENOIOCTLCMD; 961 r = -ENOIOCTLCMD;
765 } 962 }
@@ -868,11 +1065,30 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
868 1065
869 switch (ext) { 1066 switch (ext) {
870 case KVM_CAP_ONE_REG: 1067 case KVM_CAP_ONE_REG:
1068 case KVM_CAP_ENABLE_CAP:
871 r = 1; 1069 r = 1;
872 break; 1070 break;
873 case KVM_CAP_COALESCED_MMIO: 1071 case KVM_CAP_COALESCED_MMIO:
874 r = KVM_COALESCED_MMIO_PAGE_OFFSET; 1072 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
875 break; 1073 break;
1074 case KVM_CAP_MIPS_FPU:
1075 r = !!cpu_has_fpu;
1076 break;
1077 case KVM_CAP_MIPS_MSA:
1078 /*
1079 * We don't support MSA vector partitioning yet:
1080 * 1) It would require explicit support which can't be tested
1081 * yet due to lack of support in current hardware.
1082 * 2) It extends the state that would need to be saved/restored
1083 * by e.g. QEMU for migration.
1084 *
1085 * When vector partitioning hardware becomes available, support
1086 * could be added by requiring a flag when enabling
1087 * KVM_CAP_MIPS_MSA capability to indicate that userland knows
1088 * to save/restore the appropriate extra state.
1089 */
1090 r = cpu_has_msa && !(boot_cpu_data.msa_id & MSA_IR_WRPF);
1091 break;
876 default: 1092 default:
877 r = 0; 1093 r = 0;
878 break; 1094 break;
@@ -1119,6 +1335,30 @@ int kvm_mips_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
1119 ret = kvm_mips_callbacks->handle_break(vcpu); 1335 ret = kvm_mips_callbacks->handle_break(vcpu);
1120 break; 1336 break;
1121 1337
1338 case T_TRAP:
1339 ++vcpu->stat.trap_inst_exits;
1340 trace_kvm_exit(vcpu, TRAP_INST_EXITS);
1341 ret = kvm_mips_callbacks->handle_trap(vcpu);
1342 break;
1343
1344 case T_MSAFPE:
1345 ++vcpu->stat.msa_fpe_exits;
1346 trace_kvm_exit(vcpu, MSA_FPE_EXITS);
1347 ret = kvm_mips_callbacks->handle_msa_fpe(vcpu);
1348 break;
1349
1350 case T_FPE:
1351 ++vcpu->stat.fpe_exits;
1352 trace_kvm_exit(vcpu, FPE_EXITS);
1353 ret = kvm_mips_callbacks->handle_fpe(vcpu);
1354 break;
1355
1356 case T_MSADIS:
1357 ++vcpu->stat.msa_disabled_exits;
1358 trace_kvm_exit(vcpu, MSA_DISABLED_EXITS);
1359 ret = kvm_mips_callbacks->handle_msa_disabled(vcpu);
1360 break;
1361
1122 default: 1362 default:
1123 kvm_err("Exception Code: %d, not yet handled, @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#lx\n", 1363 kvm_err("Exception Code: %d, not yet handled, @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#lx\n",
1124 exccode, opc, kvm_get_inst(opc, vcpu), badvaddr, 1364 exccode, opc, kvm_get_inst(opc, vcpu), badvaddr,
@@ -1146,12 +1386,233 @@ skip_emul:
1146 } 1386 }
1147 } 1387 }
1148 1388
1389 if (ret == RESUME_GUEST) {
1390 /*
1391 * If FPU / MSA are enabled (i.e. the guest's FPU / MSA context
1392 * is live), restore FCR31 / MSACSR.
1393 *
1394 * This should be before returning to the guest exception
1395 * vector, as it may well cause an [MSA] FP exception if there
1396 * are pending exception bits unmasked. (see
1397 * kvm_mips_csr_die_notifier() for how that is handled).
1398 */
1399 if (kvm_mips_guest_has_fpu(&vcpu->arch) &&
1400 read_c0_status() & ST0_CU1)
1401 __kvm_restore_fcsr(&vcpu->arch);
1402
1403 if (kvm_mips_guest_has_msa(&vcpu->arch) &&
1404 read_c0_config5() & MIPS_CONF5_MSAEN)
1405 __kvm_restore_msacsr(&vcpu->arch);
1406 }
1407
1149 /* Disable HTW before returning to guest or host */ 1408 /* Disable HTW before returning to guest or host */
1150 htw_stop(); 1409 htw_stop();
1151 1410
1152 return ret; 1411 return ret;
1153} 1412}
1154 1413
1414/* Enable FPU for guest and restore context */
1415void kvm_own_fpu(struct kvm_vcpu *vcpu)
1416{
1417 struct mips_coproc *cop0 = vcpu->arch.cop0;
1418 unsigned int sr, cfg5;
1419
1420 preempt_disable();
1421
1422 sr = kvm_read_c0_guest_status(cop0);
1423
1424 /*
1425 * If MSA state is already live, it is undefined how it interacts with
1426 * FR=0 FPU state, and we don't want to hit reserved instruction
1427 * exceptions trying to save the MSA state later when CU=1 && FR=1, so
1428 * play it safe and save it first.
1429 *
1430 * In theory we shouldn't ever hit this case since kvm_lose_fpu() should
1431 * get called when guest CU1 is set, however we can't trust the guest
1432 * not to clobber the status register directly via the commpage.
1433 */
1434 if (cpu_has_msa && sr & ST0_CU1 && !(sr & ST0_FR) &&
1435 vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
1436 kvm_lose_fpu(vcpu);
1437
1438 /*
1439 * Enable FPU for guest
1440 * We set FR and FRE according to guest context
1441 */
1442 change_c0_status(ST0_CU1 | ST0_FR, sr);
1443 if (cpu_has_fre) {
1444 cfg5 = kvm_read_c0_guest_config5(cop0);
1445 change_c0_config5(MIPS_CONF5_FRE, cfg5);
1446 }
1447 enable_fpu_hazard();
1448
1449 /* If guest FPU state not active, restore it now */
1450 if (!(vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)) {
1451 __kvm_restore_fpu(&vcpu->arch);
1452 vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_FPU;
1453 }
1454
1455 preempt_enable();
1456}
1457
1458#ifdef CONFIG_CPU_HAS_MSA
1459/* Enable MSA for guest and restore context */
1460void kvm_own_msa(struct kvm_vcpu *vcpu)
1461{
1462 struct mips_coproc *cop0 = vcpu->arch.cop0;
1463 unsigned int sr, cfg5;
1464
1465 preempt_disable();
1466
1467 /*
1468 * Enable FPU if enabled in guest, since we're restoring FPU context
1469 * anyway. We set FR and FRE according to guest context.
1470 */
1471 if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
1472 sr = kvm_read_c0_guest_status(cop0);
1473
1474 /*
1475 * If FR=0 FPU state is already live, it is undefined how it
1476 * interacts with MSA state, so play it safe and save it first.
1477 */
1478 if (!(sr & ST0_FR) &&
1479 (vcpu->arch.fpu_inuse & (KVM_MIPS_FPU_FPU |
1480 KVM_MIPS_FPU_MSA)) == KVM_MIPS_FPU_FPU)
1481 kvm_lose_fpu(vcpu);
1482
1483 change_c0_status(ST0_CU1 | ST0_FR, sr);
1484 if (sr & ST0_CU1 && cpu_has_fre) {
1485 cfg5 = kvm_read_c0_guest_config5(cop0);
1486 change_c0_config5(MIPS_CONF5_FRE, cfg5);
1487 }
1488 }
1489
1490 /* Enable MSA for guest */
1491 set_c0_config5(MIPS_CONF5_MSAEN);
1492 enable_fpu_hazard();
1493
1494 switch (vcpu->arch.fpu_inuse & (KVM_MIPS_FPU_FPU | KVM_MIPS_FPU_MSA)) {
1495 case KVM_MIPS_FPU_FPU:
1496 /*
1497 * Guest FPU state already loaded, only restore upper MSA state
1498 */
1499 __kvm_restore_msa_upper(&vcpu->arch);
1500 vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_MSA;
1501 break;
1502 case 0:
1503 /* Neither FPU or MSA already active, restore full MSA state */
1504 __kvm_restore_msa(&vcpu->arch);
1505 vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_MSA;
1506 if (kvm_mips_guest_has_fpu(&vcpu->arch))
1507 vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_FPU;
1508 break;
1509 default:
1510 break;
1511 }
1512
1513 preempt_enable();
1514}
1515#endif
1516
1517/* Drop FPU & MSA without saving it */
1518void kvm_drop_fpu(struct kvm_vcpu *vcpu)
1519{
1520 preempt_disable();
1521 if (cpu_has_msa && vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA) {
1522 disable_msa();
1523 vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_MSA;
1524 }
1525 if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU) {
1526 clear_c0_status(ST0_CU1 | ST0_FR);
1527 vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_FPU;
1528 }
1529 preempt_enable();
1530}
1531
1532/* Save and disable FPU & MSA */
1533void kvm_lose_fpu(struct kvm_vcpu *vcpu)
1534{
1535 /*
1536 * FPU & MSA get disabled in root context (hardware) when it is disabled
1537 * in guest context (software), but the register state in the hardware
1538 * may still be in use. This is why we explicitly re-enable the hardware
1539 * before saving.
1540 */
1541
1542 preempt_disable();
1543 if (cpu_has_msa && vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA) {
1544 set_c0_config5(MIPS_CONF5_MSAEN);
1545 enable_fpu_hazard();
1546
1547 __kvm_save_msa(&vcpu->arch);
1548
1549 /* Disable MSA & FPU */
1550 disable_msa();
1551 if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
1552 clear_c0_status(ST0_CU1 | ST0_FR);
1553 vcpu->arch.fpu_inuse &= ~(KVM_MIPS_FPU_FPU | KVM_MIPS_FPU_MSA);
1554 } else if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU) {
1555 set_c0_status(ST0_CU1);
1556 enable_fpu_hazard();
1557
1558 __kvm_save_fpu(&vcpu->arch);
1559 vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_FPU;
1560
1561 /* Disable FPU */
1562 clear_c0_status(ST0_CU1 | ST0_FR);
1563 }
1564 preempt_enable();
1565}
1566
1567/*
1568 * Step over a specific ctc1 to FCSR and a specific ctcmsa to MSACSR which are
1569 * used to restore guest FCSR/MSACSR state and may trigger a "harmless" FP/MSAFP
1570 * exception if cause bits are set in the value being written.
1571 */
1572static int kvm_mips_csr_die_notify(struct notifier_block *self,
1573 unsigned long cmd, void *ptr)
1574{
1575 struct die_args *args = (struct die_args *)ptr;
1576 struct pt_regs *regs = args->regs;
1577 unsigned long pc;
1578
1579 /* Only interested in FPE and MSAFPE */
1580 if (cmd != DIE_FP && cmd != DIE_MSAFP)
1581 return NOTIFY_DONE;
1582
1583 /* Return immediately if guest context isn't active */
1584 if (!(current->flags & PF_VCPU))
1585 return NOTIFY_DONE;
1586
1587 /* Should never get here from user mode */
1588 BUG_ON(user_mode(regs));
1589
1590 pc = instruction_pointer(regs);
1591 switch (cmd) {
1592 case DIE_FP:
1593 /* match 2nd instruction in __kvm_restore_fcsr */
1594 if (pc != (unsigned long)&__kvm_restore_fcsr + 4)
1595 return NOTIFY_DONE;
1596 break;
1597 case DIE_MSAFP:
1598 /* match 2nd/3rd instruction in __kvm_restore_msacsr */
1599 if (!cpu_has_msa ||
1600 pc < (unsigned long)&__kvm_restore_msacsr + 4 ||
1601 pc > (unsigned long)&__kvm_restore_msacsr + 8)
1602 return NOTIFY_DONE;
1603 break;
1604 }
1605
1606 /* Move PC forward a little and continue executing */
1607 instruction_pointer(regs) += 4;
1608
1609 return NOTIFY_STOP;
1610}
1611
1612static struct notifier_block kvm_mips_csr_die_notifier = {
1613 .notifier_call = kvm_mips_csr_die_notify,
1614};
1615
1155int __init kvm_mips_init(void) 1616int __init kvm_mips_init(void)
1156{ 1617{
1157 int ret; 1618 int ret;
@@ -1161,6 +1622,8 @@ int __init kvm_mips_init(void)
1161 if (ret) 1622 if (ret)
1162 return ret; 1623 return ret;
1163 1624
1625 register_die_notifier(&kvm_mips_csr_die_notifier);
1626
1164 /* 1627 /*
1165 * On MIPS, kernel modules are executed from "mapped space", which 1628 * On MIPS, kernel modules are executed from "mapped space", which
1166 * requires TLBs. The TLB handling code is statically linked with 1629 * requires TLBs. The TLB handling code is statically linked with
@@ -1173,7 +1636,6 @@ int __init kvm_mips_init(void)
1173 kvm_mips_release_pfn_clean = kvm_release_pfn_clean; 1636 kvm_mips_release_pfn_clean = kvm_release_pfn_clean;
1174 kvm_mips_is_error_pfn = is_error_pfn; 1637 kvm_mips_is_error_pfn = is_error_pfn;
1175 1638
1176 pr_info("KVM/MIPS Initialized\n");
1177 return 0; 1639 return 0;
1178} 1640}
1179 1641
@@ -1185,7 +1647,7 @@ void __exit kvm_mips_exit(void)
1185 kvm_mips_release_pfn_clean = NULL; 1647 kvm_mips_release_pfn_clean = NULL;
1186 kvm_mips_is_error_pfn = NULL; 1648 kvm_mips_is_error_pfn = NULL;
1187 1649
1188 pr_info("KVM/MIPS unloaded\n"); 1650 unregister_die_notifier(&kvm_mips_csr_die_notifier);
1189} 1651}
1190 1652
1191module_init(kvm_mips_init); 1653module_init(kvm_mips_init);
diff --git a/arch/mips/kvm/msa.S b/arch/mips/kvm/msa.S
new file mode 100644
index 000000000000..d02f0c6cc2cc
--- /dev/null
+++ b/arch/mips/kvm/msa.S
@@ -0,0 +1,161 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * MIPS SIMD Architecture (MSA) context handling code for KVM.
7 *
8 * Copyright (C) 2015 Imagination Technologies Ltd.
9 */
10
11#include <asm/asm.h>
12#include <asm/asm-offsets.h>
13#include <asm/asmmacro.h>
14#include <asm/regdef.h>
15
16 .set noreorder
17 .set noat
18
19LEAF(__kvm_save_msa)
20 st_d 0, VCPU_FPR0, a0
21 st_d 1, VCPU_FPR1, a0
22 st_d 2, VCPU_FPR2, a0
23 st_d 3, VCPU_FPR3, a0
24 st_d 4, VCPU_FPR4, a0
25 st_d 5, VCPU_FPR5, a0
26 st_d 6, VCPU_FPR6, a0
27 st_d 7, VCPU_FPR7, a0
28 st_d 8, VCPU_FPR8, a0
29 st_d 9, VCPU_FPR9, a0
30 st_d 10, VCPU_FPR10, a0
31 st_d 11, VCPU_FPR11, a0
32 st_d 12, VCPU_FPR12, a0
33 st_d 13, VCPU_FPR13, a0
34 st_d 14, VCPU_FPR14, a0
35 st_d 15, VCPU_FPR15, a0
36 st_d 16, VCPU_FPR16, a0
37 st_d 17, VCPU_FPR17, a0
38 st_d 18, VCPU_FPR18, a0
39 st_d 19, VCPU_FPR19, a0
40 st_d 20, VCPU_FPR20, a0
41 st_d 21, VCPU_FPR21, a0
42 st_d 22, VCPU_FPR22, a0
43 st_d 23, VCPU_FPR23, a0
44 st_d 24, VCPU_FPR24, a0
45 st_d 25, VCPU_FPR25, a0
46 st_d 26, VCPU_FPR26, a0
47 st_d 27, VCPU_FPR27, a0
48 st_d 28, VCPU_FPR28, a0
49 st_d 29, VCPU_FPR29, a0
50 st_d 30, VCPU_FPR30, a0
51 st_d 31, VCPU_FPR31, a0
52 jr ra
53 nop
54 END(__kvm_save_msa)
55
56LEAF(__kvm_restore_msa)
57 ld_d 0, VCPU_FPR0, a0
58 ld_d 1, VCPU_FPR1, a0
59 ld_d 2, VCPU_FPR2, a0
60 ld_d 3, VCPU_FPR3, a0
61 ld_d 4, VCPU_FPR4, a0
62 ld_d 5, VCPU_FPR5, a0
63 ld_d 6, VCPU_FPR6, a0
64 ld_d 7, VCPU_FPR7, a0
65 ld_d 8, VCPU_FPR8, a0
66 ld_d 9, VCPU_FPR9, a0
67 ld_d 10, VCPU_FPR10, a0
68 ld_d 11, VCPU_FPR11, a0
69 ld_d 12, VCPU_FPR12, a0
70 ld_d 13, VCPU_FPR13, a0
71 ld_d 14, VCPU_FPR14, a0
72 ld_d 15, VCPU_FPR15, a0
73 ld_d 16, VCPU_FPR16, a0
74 ld_d 17, VCPU_FPR17, a0
75 ld_d 18, VCPU_FPR18, a0
76 ld_d 19, VCPU_FPR19, a0
77 ld_d 20, VCPU_FPR20, a0
78 ld_d 21, VCPU_FPR21, a0
79 ld_d 22, VCPU_FPR22, a0
80 ld_d 23, VCPU_FPR23, a0
81 ld_d 24, VCPU_FPR24, a0
82 ld_d 25, VCPU_FPR25, a0
83 ld_d 26, VCPU_FPR26, a0
84 ld_d 27, VCPU_FPR27, a0
85 ld_d 28, VCPU_FPR28, a0
86 ld_d 29, VCPU_FPR29, a0
87 ld_d 30, VCPU_FPR30, a0
88 ld_d 31, VCPU_FPR31, a0
89 jr ra
90 nop
91 END(__kvm_restore_msa)
92
93 .macro kvm_restore_msa_upper wr, off, base
94 .set push
95 .set noat
96#ifdef CONFIG_64BIT
97 ld $1, \off(\base)
98 insert_d \wr, 1
99#elif defined(CONFIG_CPU_LITTLE_ENDIAN)
100 lw $1, \off(\base)
101 insert_w \wr, 2
102 lw $1, (\off+4)(\base)
103 insert_w \wr, 3
104#else /* CONFIG_CPU_BIG_ENDIAN */
105 lw $1, (\off+4)(\base)
106 insert_w \wr, 2
107 lw $1, \off(\base)
108 insert_w \wr, 3
109#endif
110 .set pop
111 .endm
112
113LEAF(__kvm_restore_msa_upper)
114 kvm_restore_msa_upper 0, VCPU_FPR0 +8, a0
115 kvm_restore_msa_upper 1, VCPU_FPR1 +8, a0
116 kvm_restore_msa_upper 2, VCPU_FPR2 +8, a0
117 kvm_restore_msa_upper 3, VCPU_FPR3 +8, a0
118 kvm_restore_msa_upper 4, VCPU_FPR4 +8, a0
119 kvm_restore_msa_upper 5, VCPU_FPR5 +8, a0
120 kvm_restore_msa_upper 6, VCPU_FPR6 +8, a0
121 kvm_restore_msa_upper 7, VCPU_FPR7 +8, a0
122 kvm_restore_msa_upper 8, VCPU_FPR8 +8, a0
123 kvm_restore_msa_upper 9, VCPU_FPR9 +8, a0
124 kvm_restore_msa_upper 10, VCPU_FPR10+8, a0
125 kvm_restore_msa_upper 11, VCPU_FPR11+8, a0
126 kvm_restore_msa_upper 12, VCPU_FPR12+8, a0
127 kvm_restore_msa_upper 13, VCPU_FPR13+8, a0
128 kvm_restore_msa_upper 14, VCPU_FPR14+8, a0
129 kvm_restore_msa_upper 15, VCPU_FPR15+8, a0
130 kvm_restore_msa_upper 16, VCPU_FPR16+8, a0
131 kvm_restore_msa_upper 17, VCPU_FPR17+8, a0
132 kvm_restore_msa_upper 18, VCPU_FPR18+8, a0
133 kvm_restore_msa_upper 19, VCPU_FPR19+8, a0
134 kvm_restore_msa_upper 20, VCPU_FPR20+8, a0
135 kvm_restore_msa_upper 21, VCPU_FPR21+8, a0
136 kvm_restore_msa_upper 22, VCPU_FPR22+8, a0
137 kvm_restore_msa_upper 23, VCPU_FPR23+8, a0
138 kvm_restore_msa_upper 24, VCPU_FPR24+8, a0
139 kvm_restore_msa_upper 25, VCPU_FPR25+8, a0
140 kvm_restore_msa_upper 26, VCPU_FPR26+8, a0
141 kvm_restore_msa_upper 27, VCPU_FPR27+8, a0
142 kvm_restore_msa_upper 28, VCPU_FPR28+8, a0
143 kvm_restore_msa_upper 29, VCPU_FPR29+8, a0
144 kvm_restore_msa_upper 30, VCPU_FPR30+8, a0
145 kvm_restore_msa_upper 31, VCPU_FPR31+8, a0
146 jr ra
147 nop
148 END(__kvm_restore_msa_upper)
149
150LEAF(__kvm_restore_msacsr)
151 lw t0, VCPU_MSA_CSR(a0)
152 /*
153 * The ctcmsa must stay at this offset in __kvm_restore_msacsr.
154 * See kvm_mips_csr_die_notify() which handles t0 containing a value
155 * which triggers an MSA FP Exception, which must be stepped over and
156 * ignored since the set cause bits must remain there for the guest.
157 */
158 _ctcmsa MSA_CSR, t0
159 jr ra
160 nop
161 END(__kvm_restore_msacsr)
diff --git a/arch/mips/kvm/stats.c b/arch/mips/kvm/stats.c
index a74d6024c5ad..888bb67070ac 100644
--- a/arch/mips/kvm/stats.c
+++ b/arch/mips/kvm/stats.c
@@ -25,6 +25,10 @@ char *kvm_mips_exit_types_str[MAX_KVM_MIPS_EXIT_TYPES] = {
25 "System Call", 25 "System Call",
26 "Reserved Inst", 26 "Reserved Inst",
27 "Break Inst", 27 "Break Inst",
28 "Trap Inst",
29 "MSA FPE",
30 "FPE",
31 "MSA Disabled",
28 "D-Cache Flushes", 32 "D-Cache Flushes",
29}; 33};
30 34
diff --git a/arch/mips/kvm/tlb.c b/arch/mips/kvm/tlb.c
index b6beb0e07b1b..aed0ac2a4972 100644
--- a/arch/mips/kvm/tlb.c
+++ b/arch/mips/kvm/tlb.c
@@ -733,6 +733,9 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
733 } 733 }
734 } 734 }
735 735
736 /* restore guest state to registers */
737 kvm_mips_callbacks->vcpu_set_regs(vcpu);
738
736 local_irq_restore(flags); 739 local_irq_restore(flags);
737 740
738} 741}
@@ -751,6 +754,9 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
751 vcpu->arch.preempt_entryhi = read_c0_entryhi(); 754 vcpu->arch.preempt_entryhi = read_c0_entryhi();
752 vcpu->arch.last_sched_cpu = cpu; 755 vcpu->arch.last_sched_cpu = cpu;
753 756
757 /* save guest state in registers */
758 kvm_mips_callbacks->vcpu_get_regs(vcpu);
759
754 if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) & 760 if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
755 ASID_VERSION_MASK)) { 761 ASID_VERSION_MASK)) {
756 kvm_debug("%s: Dropping MMU Context: %#lx\n", __func__, 762 kvm_debug("%s: Dropping MMU Context: %#lx\n", __func__,
diff --git a/arch/mips/kvm/trap_emul.c b/arch/mips/kvm/trap_emul.c
index fd7257b70e65..d836ed5b0bc7 100644
--- a/arch/mips/kvm/trap_emul.c
+++ b/arch/mips/kvm/trap_emul.c
@@ -39,16 +39,30 @@ static gpa_t kvm_trap_emul_gva_to_gpa_cb(gva_t gva)
39 39
40static int kvm_trap_emul_handle_cop_unusable(struct kvm_vcpu *vcpu) 40static int kvm_trap_emul_handle_cop_unusable(struct kvm_vcpu *vcpu)
41{ 41{
42 struct mips_coproc *cop0 = vcpu->arch.cop0;
42 struct kvm_run *run = vcpu->run; 43 struct kvm_run *run = vcpu->run;
43 uint32_t __user *opc = (uint32_t __user *) vcpu->arch.pc; 44 uint32_t __user *opc = (uint32_t __user *) vcpu->arch.pc;
44 unsigned long cause = vcpu->arch.host_cp0_cause; 45 unsigned long cause = vcpu->arch.host_cp0_cause;
45 enum emulation_result er = EMULATE_DONE; 46 enum emulation_result er = EMULATE_DONE;
46 int ret = RESUME_GUEST; 47 int ret = RESUME_GUEST;
47 48
48 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) 49 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) {
49 er = kvm_mips_emulate_fpu_exc(cause, opc, run, vcpu); 50 /* FPU Unusable */
50 else 51 if (!kvm_mips_guest_has_fpu(&vcpu->arch) ||
52 (kvm_read_c0_guest_status(cop0) & ST0_CU1) == 0) {
53 /*
54 * Unusable/no FPU in guest:
55 * deliver guest COP1 Unusable Exception
56 */
57 er = kvm_mips_emulate_fpu_exc(cause, opc, run, vcpu);
58 } else {
59 /* Restore FPU state */
60 kvm_own_fpu(vcpu);
61 er = EMULATE_DONE;
62 }
63 } else {
51 er = kvm_mips_emulate_inst(cause, opc, run, vcpu); 64 er = kvm_mips_emulate_inst(cause, opc, run, vcpu);
65 }
52 66
53 switch (er) { 67 switch (er) {
54 case EMULATE_DONE: 68 case EMULATE_DONE:
@@ -330,6 +344,107 @@ static int kvm_trap_emul_handle_break(struct kvm_vcpu *vcpu)
330 return ret; 344 return ret;
331} 345}
332 346
347static int kvm_trap_emul_handle_trap(struct kvm_vcpu *vcpu)
348{
349 struct kvm_run *run = vcpu->run;
350 uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
351 unsigned long cause = vcpu->arch.host_cp0_cause;
352 enum emulation_result er = EMULATE_DONE;
353 int ret = RESUME_GUEST;
354
355 er = kvm_mips_emulate_trap_exc(cause, opc, run, vcpu);
356 if (er == EMULATE_DONE) {
357 ret = RESUME_GUEST;
358 } else {
359 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
360 ret = RESUME_HOST;
361 }
362 return ret;
363}
364
365static int kvm_trap_emul_handle_msa_fpe(struct kvm_vcpu *vcpu)
366{
367 struct kvm_run *run = vcpu->run;
368 uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
369 unsigned long cause = vcpu->arch.host_cp0_cause;
370 enum emulation_result er = EMULATE_DONE;
371 int ret = RESUME_GUEST;
372
373 er = kvm_mips_emulate_msafpe_exc(cause, opc, run, vcpu);
374 if (er == EMULATE_DONE) {
375 ret = RESUME_GUEST;
376 } else {
377 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
378 ret = RESUME_HOST;
379 }
380 return ret;
381}
382
383static int kvm_trap_emul_handle_fpe(struct kvm_vcpu *vcpu)
384{
385 struct kvm_run *run = vcpu->run;
386 uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
387 unsigned long cause = vcpu->arch.host_cp0_cause;
388 enum emulation_result er = EMULATE_DONE;
389 int ret = RESUME_GUEST;
390
391 er = kvm_mips_emulate_fpe_exc(cause, opc, run, vcpu);
392 if (er == EMULATE_DONE) {
393 ret = RESUME_GUEST;
394 } else {
395 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
396 ret = RESUME_HOST;
397 }
398 return ret;
399}
400
401/**
402 * kvm_trap_emul_handle_msa_disabled() - Guest used MSA while disabled in root.
403 * @vcpu: Virtual CPU context.
404 *
405 * Handle when the guest attempts to use MSA when it is disabled.
406 */
407static int kvm_trap_emul_handle_msa_disabled(struct kvm_vcpu *vcpu)
408{
409 struct mips_coproc *cop0 = vcpu->arch.cop0;
410 struct kvm_run *run = vcpu->run;
411 uint32_t __user *opc = (uint32_t __user *) vcpu->arch.pc;
412 unsigned long cause = vcpu->arch.host_cp0_cause;
413 enum emulation_result er = EMULATE_DONE;
414 int ret = RESUME_GUEST;
415
416 if (!kvm_mips_guest_has_msa(&vcpu->arch) ||
417 (kvm_read_c0_guest_status(cop0) & (ST0_CU1 | ST0_FR)) == ST0_CU1) {
418 /*
419 * No MSA in guest, or FPU enabled and not in FR=1 mode,
420 * guest reserved instruction exception
421 */
422 er = kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
423 } else if (!(kvm_read_c0_guest_config5(cop0) & MIPS_CONF5_MSAEN)) {
424 /* MSA disabled by guest, guest MSA disabled exception */
425 er = kvm_mips_emulate_msadis_exc(cause, opc, run, vcpu);
426 } else {
427 /* Restore MSA/FPU state */
428 kvm_own_msa(vcpu);
429 er = EMULATE_DONE;
430 }
431
432 switch (er) {
433 case EMULATE_DONE:
434 ret = RESUME_GUEST;
435 break;
436
437 case EMULATE_FAIL:
438 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
439 ret = RESUME_HOST;
440 break;
441
442 default:
443 BUG();
444 }
445 return ret;
446}
447
333static int kvm_trap_emul_vm_init(struct kvm *kvm) 448static int kvm_trap_emul_vm_init(struct kvm *kvm)
334{ 449{
335 return 0; 450 return 0;
@@ -351,8 +466,9 @@ static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu)
351 * guest will come up as expected, for now we simulate a MIPS 24kc 466 * guest will come up as expected, for now we simulate a MIPS 24kc
352 */ 467 */
353 kvm_write_c0_guest_prid(cop0, 0x00019300); 468 kvm_write_c0_guest_prid(cop0, 0x00019300);
354 kvm_write_c0_guest_config(cop0, 469 /* Have config1, Cacheable, noncoherent, write-back, write allocate */
355 MIPS_CONFIG0 | (0x1 << CP0C0_AR) | 470 kvm_write_c0_guest_config(cop0, MIPS_CONF_M | (0x3 << CP0C0_K0) |
471 (0x1 << CP0C0_AR) |
356 (MMU_TYPE_R4000 << CP0C0_MT)); 472 (MMU_TYPE_R4000 << CP0C0_MT));
357 473
358 /* Read the cache characteristics from the host Config1 Register */ 474 /* Read the cache characteristics from the host Config1 Register */
@@ -368,10 +484,18 @@ static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu)
368 (1 << CP0C1_WR) | (1 << CP0C1_CA)); 484 (1 << CP0C1_WR) | (1 << CP0C1_CA));
369 kvm_write_c0_guest_config1(cop0, config1); 485 kvm_write_c0_guest_config1(cop0, config1);
370 486
371 kvm_write_c0_guest_config2(cop0, MIPS_CONFIG2); 487 /* Have config3, no tertiary/secondary caches implemented */
372 /* MIPS_CONFIG2 | (read_c0_config2() & 0xfff) */ 488 kvm_write_c0_guest_config2(cop0, MIPS_CONF_M);
373 kvm_write_c0_guest_config3(cop0, MIPS_CONFIG3 | (0 << CP0C3_VInt) | 489 /* MIPS_CONF_M | (read_c0_config2() & 0xfff) */
374 (1 << CP0C3_ULRI)); 490
491 /* Have config4, UserLocal */
492 kvm_write_c0_guest_config3(cop0, MIPS_CONF_M | MIPS_CONF3_ULRI);
493
494 /* Have config5 */
495 kvm_write_c0_guest_config4(cop0, MIPS_CONF_M);
496
497 /* No config6 */
498 kvm_write_c0_guest_config5(cop0, 0);
375 499
376 /* Set Wait IE/IXMT Ignore in Config7, IAR, AR */ 500 /* Set Wait IE/IXMT Ignore in Config7, IAR, AR */
377 kvm_write_c0_guest_config7(cop0, (MIPS_CONF7_WII) | (1 << 10)); 501 kvm_write_c0_guest_config7(cop0, (MIPS_CONF7_WII) | (1 << 10));
@@ -416,6 +540,7 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
416{ 540{
417 struct mips_coproc *cop0 = vcpu->arch.cop0; 541 struct mips_coproc *cop0 = vcpu->arch.cop0;
418 int ret = 0; 542 int ret = 0;
543 unsigned int cur, change;
419 544
420 switch (reg->id) { 545 switch (reg->id) {
421 case KVM_REG_MIPS_CP0_COUNT: 546 case KVM_REG_MIPS_CP0_COUNT:
@@ -444,6 +569,44 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
444 kvm_write_c0_guest_cause(cop0, v); 569 kvm_write_c0_guest_cause(cop0, v);
445 } 570 }
446 break; 571 break;
572 case KVM_REG_MIPS_CP0_CONFIG:
573 /* read-only for now */
574 break;
575 case KVM_REG_MIPS_CP0_CONFIG1:
576 cur = kvm_read_c0_guest_config1(cop0);
577 change = (cur ^ v) & kvm_mips_config1_wrmask(vcpu);
578 if (change) {
579 v = cur ^ change;
580 kvm_write_c0_guest_config1(cop0, v);
581 }
582 break;
583 case KVM_REG_MIPS_CP0_CONFIG2:
584 /* read-only for now */
585 break;
586 case KVM_REG_MIPS_CP0_CONFIG3:
587 cur = kvm_read_c0_guest_config3(cop0);
588 change = (cur ^ v) & kvm_mips_config3_wrmask(vcpu);
589 if (change) {
590 v = cur ^ change;
591 kvm_write_c0_guest_config3(cop0, v);
592 }
593 break;
594 case KVM_REG_MIPS_CP0_CONFIG4:
595 cur = kvm_read_c0_guest_config4(cop0);
596 change = (cur ^ v) & kvm_mips_config4_wrmask(vcpu);
597 if (change) {
598 v = cur ^ change;
599 kvm_write_c0_guest_config4(cop0, v);
600 }
601 break;
602 case KVM_REG_MIPS_CP0_CONFIG5:
603 cur = kvm_read_c0_guest_config5(cop0);
604 change = (cur ^ v) & kvm_mips_config5_wrmask(vcpu);
605 if (change) {
606 v = cur ^ change;
607 kvm_write_c0_guest_config5(cop0, v);
608 }
609 break;
447 case KVM_REG_MIPS_COUNT_CTL: 610 case KVM_REG_MIPS_COUNT_CTL:
448 ret = kvm_mips_set_count_ctl(vcpu, v); 611 ret = kvm_mips_set_count_ctl(vcpu, v);
449 break; 612 break;
@@ -459,6 +622,18 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
459 return ret; 622 return ret;
460} 623}
461 624
625static int kvm_trap_emul_vcpu_get_regs(struct kvm_vcpu *vcpu)
626{
627 kvm_lose_fpu(vcpu);
628
629 return 0;
630}
631
632static int kvm_trap_emul_vcpu_set_regs(struct kvm_vcpu *vcpu)
633{
634 return 0;
635}
636
462static struct kvm_mips_callbacks kvm_trap_emul_callbacks = { 637static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
463 /* exit handlers */ 638 /* exit handlers */
464 .handle_cop_unusable = kvm_trap_emul_handle_cop_unusable, 639 .handle_cop_unusable = kvm_trap_emul_handle_cop_unusable,
@@ -470,6 +645,10 @@ static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
470 .handle_syscall = kvm_trap_emul_handle_syscall, 645 .handle_syscall = kvm_trap_emul_handle_syscall,
471 .handle_res_inst = kvm_trap_emul_handle_res_inst, 646 .handle_res_inst = kvm_trap_emul_handle_res_inst,
472 .handle_break = kvm_trap_emul_handle_break, 647 .handle_break = kvm_trap_emul_handle_break,
648 .handle_trap = kvm_trap_emul_handle_trap,
649 .handle_msa_fpe = kvm_trap_emul_handle_msa_fpe,
650 .handle_fpe = kvm_trap_emul_handle_fpe,
651 .handle_msa_disabled = kvm_trap_emul_handle_msa_disabled,
473 652
474 .vm_init = kvm_trap_emul_vm_init, 653 .vm_init = kvm_trap_emul_vm_init,
475 .vcpu_init = kvm_trap_emul_vcpu_init, 654 .vcpu_init = kvm_trap_emul_vcpu_init,
@@ -483,6 +662,8 @@ static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
483 .irq_clear = kvm_mips_irq_clear_cb, 662 .irq_clear = kvm_mips_irq_clear_cb,
484 .get_one_reg = kvm_trap_emul_get_one_reg, 663 .get_one_reg = kvm_trap_emul_get_one_reg,
485 .set_one_reg = kvm_trap_emul_set_one_reg, 664 .set_one_reg = kvm_trap_emul_set_one_reg,
665 .vcpu_get_regs = kvm_trap_emul_vcpu_get_regs,
666 .vcpu_set_regs = kvm_trap_emul_vcpu_set_regs,
486}; 667};
487 668
488int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks) 669int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks)
diff --git a/arch/powerpc/kvm/mpic.c b/arch/powerpc/kvm/mpic.c
index 39b3a8f816f2..6249cdc834d1 100644
--- a/arch/powerpc/kvm/mpic.c
+++ b/arch/powerpc/kvm/mpic.c
@@ -34,7 +34,7 @@
34#include <asm/kvm_para.h> 34#include <asm/kvm_para.h>
35#include <asm/kvm_host.h> 35#include <asm/kvm_host.h>
36#include <asm/kvm_ppc.h> 36#include <asm/kvm_ppc.h>
37#include "iodev.h" 37#include <kvm/iodev.h>
38 38
39#define MAX_CPU 32 39#define MAX_CPU 32
40#define MAX_SRC 256 40#define MAX_SRC 256
@@ -289,11 +289,6 @@ static inline void IRQ_resetbit(struct irq_queue *q, int n_IRQ)
289 clear_bit(n_IRQ, q->queue); 289 clear_bit(n_IRQ, q->queue);
290} 290}
291 291
292static inline int IRQ_testbit(struct irq_queue *q, int n_IRQ)
293{
294 return test_bit(n_IRQ, q->queue);
295}
296
297static void IRQ_check(struct openpic *opp, struct irq_queue *q) 292static void IRQ_check(struct openpic *opp, struct irq_queue *q)
298{ 293{
299 int irq = -1; 294 int irq = -1;
@@ -1374,8 +1369,9 @@ static int kvm_mpic_write_internal(struct openpic *opp, gpa_t addr, u32 val)
1374 return -ENXIO; 1369 return -ENXIO;
1375} 1370}
1376 1371
1377static int kvm_mpic_read(struct kvm_io_device *this, gpa_t addr, 1372static int kvm_mpic_read(struct kvm_vcpu *vcpu,
1378 int len, void *ptr) 1373 struct kvm_io_device *this,
1374 gpa_t addr, int len, void *ptr)
1379{ 1375{
1380 struct openpic *opp = container_of(this, struct openpic, mmio); 1376 struct openpic *opp = container_of(this, struct openpic, mmio);
1381 int ret; 1377 int ret;
@@ -1415,8 +1411,9 @@ static int kvm_mpic_read(struct kvm_io_device *this, gpa_t addr,
1415 return ret; 1411 return ret;
1416} 1412}
1417 1413
1418static int kvm_mpic_write(struct kvm_io_device *this, gpa_t addr, 1414static int kvm_mpic_write(struct kvm_vcpu *vcpu,
1419 int len, const void *ptr) 1415 struct kvm_io_device *this,
1416 gpa_t addr, int len, const void *ptr)
1420{ 1417{
1421 struct openpic *opp = container_of(this, struct openpic, mmio); 1418 struct openpic *opp = container_of(this, struct openpic, mmio);
1422 int ret; 1419 int ret;
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 27c0face86f4..24bfe401373e 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -807,7 +807,7 @@ int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
807 807
808 idx = srcu_read_lock(&vcpu->kvm->srcu); 808 idx = srcu_read_lock(&vcpu->kvm->srcu);
809 809
810 ret = kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, run->mmio.phys_addr, 810 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
811 bytes, &run->mmio.data); 811 bytes, &run->mmio.data);
812 812
813 srcu_read_unlock(&vcpu->kvm->srcu, idx); 813 srcu_read_unlock(&vcpu->kvm->srcu, idx);
@@ -880,7 +880,7 @@ int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
880 880
881 idx = srcu_read_lock(&vcpu->kvm->srcu); 881 idx = srcu_read_lock(&vcpu->kvm->srcu);
882 882
883 ret = kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, run->mmio.phys_addr, 883 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
884 bytes, &run->mmio.data); 884 bytes, &run->mmio.data);
885 885
886 srcu_read_unlock(&vcpu->kvm->srcu, idx); 886 srcu_read_unlock(&vcpu->kvm->srcu, idx);
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index f407bbf5ee94..d01fc588b5c3 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -172,7 +172,9 @@ struct kvm_s390_sie_block {
172 __u32 fac; /* 0x01a0 */ 172 __u32 fac; /* 0x01a0 */
173 __u8 reserved1a4[20]; /* 0x01a4 */ 173 __u8 reserved1a4[20]; /* 0x01a4 */
174 __u64 cbrlo; /* 0x01b8 */ 174 __u64 cbrlo; /* 0x01b8 */
175 __u8 reserved1c0[30]; /* 0x01c0 */ 175 __u8 reserved1c0[8]; /* 0x01c0 */
176 __u32 ecd; /* 0x01c8 */
177 __u8 reserved1cc[18]; /* 0x01cc */
176 __u64 pp; /* 0x01de */ 178 __u64 pp; /* 0x01de */
177 __u8 reserved1e6[2]; /* 0x01e6 */ 179 __u8 reserved1e6[2]; /* 0x01e6 */
178 __u64 itdba; /* 0x01e8 */ 180 __u64 itdba; /* 0x01e8 */
@@ -183,11 +185,17 @@ struct kvm_s390_itdb {
183 __u8 data[256]; 185 __u8 data[256];
184} __packed; 186} __packed;
185 187
188struct kvm_s390_vregs {
189 __vector128 vrs[32];
190 __u8 reserved200[512]; /* for future vector expansion */
191} __packed;
192
186struct sie_page { 193struct sie_page {
187 struct kvm_s390_sie_block sie_block; 194 struct kvm_s390_sie_block sie_block;
188 __u8 reserved200[1024]; /* 0x0200 */ 195 __u8 reserved200[1024]; /* 0x0200 */
189 struct kvm_s390_itdb itdb; /* 0x0600 */ 196 struct kvm_s390_itdb itdb; /* 0x0600 */
190 __u8 reserved700[2304]; /* 0x0700 */ 197 __u8 reserved700[1280]; /* 0x0700 */
198 struct kvm_s390_vregs vregs; /* 0x0c00 */
191} __packed; 199} __packed;
192 200
193struct kvm_vcpu_stat { 201struct kvm_vcpu_stat {
@@ -238,6 +246,7 @@ struct kvm_vcpu_stat {
238 u32 instruction_sigp_stop; 246 u32 instruction_sigp_stop;
239 u32 instruction_sigp_stop_store_status; 247 u32 instruction_sigp_stop_store_status;
240 u32 instruction_sigp_store_status; 248 u32 instruction_sigp_store_status;
249 u32 instruction_sigp_store_adtl_status;
241 u32 instruction_sigp_arch; 250 u32 instruction_sigp_arch;
242 u32 instruction_sigp_prefix; 251 u32 instruction_sigp_prefix;
243 u32 instruction_sigp_restart; 252 u32 instruction_sigp_restart;
@@ -270,6 +279,7 @@ struct kvm_vcpu_stat {
270#define PGM_SPECIAL_OPERATION 0x13 279#define PGM_SPECIAL_OPERATION 0x13
271#define PGM_OPERAND 0x15 280#define PGM_OPERAND 0x15
272#define PGM_TRACE_TABEL 0x16 281#define PGM_TRACE_TABEL 0x16
282#define PGM_VECTOR_PROCESSING 0x1b
273#define PGM_SPACE_SWITCH 0x1c 283#define PGM_SPACE_SWITCH 0x1c
274#define PGM_HFP_SQUARE_ROOT 0x1d 284#define PGM_HFP_SQUARE_ROOT 0x1d
275#define PGM_PC_TRANSLATION_SPEC 0x1f 285#define PGM_PC_TRANSLATION_SPEC 0x1f
@@ -334,6 +344,11 @@ enum irq_types {
334 IRQ_PEND_COUNT 344 IRQ_PEND_COUNT
335}; 345};
336 346
347/* We have 2M for virtio device descriptor pages. Smallest amount of
348 * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381
349 */
350#define KVM_S390_MAX_VIRTIO_IRQS 87381
351
337/* 352/*
338 * Repressible (non-floating) machine check interrupts 353 * Repressible (non-floating) machine check interrupts
339 * subclass bits in MCIC 354 * subclass bits in MCIC
@@ -411,13 +426,32 @@ struct kvm_s390_local_interrupt {
411 unsigned long pending_irqs; 426 unsigned long pending_irqs;
412}; 427};
413 428
429#define FIRQ_LIST_IO_ISC_0 0
430#define FIRQ_LIST_IO_ISC_1 1
431#define FIRQ_LIST_IO_ISC_2 2
432#define FIRQ_LIST_IO_ISC_3 3
433#define FIRQ_LIST_IO_ISC_4 4
434#define FIRQ_LIST_IO_ISC_5 5
435#define FIRQ_LIST_IO_ISC_6 6
436#define FIRQ_LIST_IO_ISC_7 7
437#define FIRQ_LIST_PFAULT 8
438#define FIRQ_LIST_VIRTIO 9
439#define FIRQ_LIST_COUNT 10
440#define FIRQ_CNTR_IO 0
441#define FIRQ_CNTR_SERVICE 1
442#define FIRQ_CNTR_VIRTIO 2
443#define FIRQ_CNTR_PFAULT 3
444#define FIRQ_MAX_COUNT 4
445
414struct kvm_s390_float_interrupt { 446struct kvm_s390_float_interrupt {
447 unsigned long pending_irqs;
415 spinlock_t lock; 448 spinlock_t lock;
416 struct list_head list; 449 struct list_head lists[FIRQ_LIST_COUNT];
417 atomic_t active; 450 int counters[FIRQ_MAX_COUNT];
451 struct kvm_s390_mchk_info mchk;
452 struct kvm_s390_ext_info srv_signal;
418 int next_rr_cpu; 453 int next_rr_cpu;
419 unsigned long idle_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 454 unsigned long idle_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)];
420 unsigned int irq_count;
421}; 455};
422 456
423struct kvm_hw_wp_info_arch { 457struct kvm_hw_wp_info_arch {
@@ -465,6 +499,7 @@ struct kvm_vcpu_arch {
465 s390_fp_regs host_fpregs; 499 s390_fp_regs host_fpregs;
466 unsigned int host_acrs[NUM_ACRS]; 500 unsigned int host_acrs[NUM_ACRS];
467 s390_fp_regs guest_fpregs; 501 s390_fp_regs guest_fpregs;
502 struct kvm_s390_vregs *host_vregs;
468 struct kvm_s390_local_interrupt local_int; 503 struct kvm_s390_local_interrupt local_int;
469 struct hrtimer ckc_timer; 504 struct hrtimer ckc_timer;
470 struct kvm_s390_pgm_info pgm; 505 struct kvm_s390_pgm_info pgm;
@@ -553,6 +588,7 @@ struct kvm_arch{
553 int use_cmma; 588 int use_cmma;
554 int user_cpu_state_ctrl; 589 int user_cpu_state_ctrl;
555 int user_sigp; 590 int user_sigp;
591 int user_stsi;
556 struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS]; 592 struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS];
557 wait_queue_head_t ipte_wq; 593 wait_queue_head_t ipte_wq;
558 int ipte_lock_count; 594 int ipte_lock_count;
diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
index 9c77e60b9a26..ef1a5fcc6c66 100644
--- a/arch/s390/include/uapi/asm/kvm.h
+++ b/arch/s390/include/uapi/asm/kvm.h
@@ -150,6 +150,7 @@ struct kvm_guest_debug_arch {
150#define KVM_SYNC_CRS (1UL << 3) 150#define KVM_SYNC_CRS (1UL << 3)
151#define KVM_SYNC_ARCH0 (1UL << 4) 151#define KVM_SYNC_ARCH0 (1UL << 4)
152#define KVM_SYNC_PFAULT (1UL << 5) 152#define KVM_SYNC_PFAULT (1UL << 5)
153#define KVM_SYNC_VRS (1UL << 6)
153/* definition of registers in kvm_run */ 154/* definition of registers in kvm_run */
154struct kvm_sync_regs { 155struct kvm_sync_regs {
155 __u64 prefix; /* prefix register */ 156 __u64 prefix; /* prefix register */
@@ -164,6 +165,9 @@ struct kvm_sync_regs {
164 __u64 pft; /* pfault token [PFAULT] */ 165 __u64 pft; /* pfault token [PFAULT] */
165 __u64 pfs; /* pfault select [PFAULT] */ 166 __u64 pfs; /* pfault select [PFAULT] */
166 __u64 pfc; /* pfault compare [PFAULT] */ 167 __u64 pfc; /* pfault compare [PFAULT] */
168 __u64 vrs[32][2]; /* vector registers */
169 __u8 reserved[512]; /* for future vector expansion */
170 __u32 fpc; /* only valid with vector registers */
167}; 171};
168 172
169#define KVM_REG_S390_TODPR (KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1) 173#define KVM_REG_S390_TODPR (KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1)
diff --git a/arch/s390/include/uapi/asm/sie.h b/arch/s390/include/uapi/asm/sie.h
index d4096fdfc6ab..ee69c0854c88 100644
--- a/arch/s390/include/uapi/asm/sie.h
+++ b/arch/s390/include/uapi/asm/sie.h
@@ -230,7 +230,7 @@
230 * and returns a key, which can be used to find a mnemonic name 230 * and returns a key, which can be used to find a mnemonic name
231 * of the instruction in the icpt_insn_codes table. 231 * of the instruction in the icpt_insn_codes table.
232 */ 232 */
233#define icpt_insn_decoder(insn) \ 233#define icpt_insn_decoder(insn) ( \
234 INSN_DECODE_IPA0(0x01, insn, 48, 0xff) \ 234 INSN_DECODE_IPA0(0x01, insn, 48, 0xff) \
235 INSN_DECODE_IPA0(0xaa, insn, 48, 0x0f) \ 235 INSN_DECODE_IPA0(0xaa, insn, 48, 0x0f) \
236 INSN_DECODE_IPA0(0xb2, insn, 48, 0xff) \ 236 INSN_DECODE_IPA0(0xb2, insn, 48, 0xff) \
@@ -239,6 +239,6 @@
239 INSN_DECODE_IPA0(0xe5, insn, 48, 0xff) \ 239 INSN_DECODE_IPA0(0xe5, insn, 48, 0xff) \
240 INSN_DECODE_IPA0(0xeb, insn, 16, 0xff) \ 240 INSN_DECODE_IPA0(0xeb, insn, 16, 0xff) \
241 INSN_DECODE_IPA0(0xc8, insn, 48, 0x0f) \ 241 INSN_DECODE_IPA0(0xc8, insn, 48, 0x0f) \
242 INSN_DECODE(insn) 242 INSN_DECODE(insn))
243 243
244#endif /* _UAPI_ASM_S390_SIE_H */ 244#endif /* _UAPI_ASM_S390_SIE_H */
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index e07e91605353..8dc4db10d160 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -171,6 +171,7 @@ int main(void)
171#else /* CONFIG_32BIT */ 171#else /* CONFIG_32BIT */
172 DEFINE(__LC_DATA_EXC_CODE, offsetof(struct _lowcore, data_exc_code)); 172 DEFINE(__LC_DATA_EXC_CODE, offsetof(struct _lowcore, data_exc_code));
173 DEFINE(__LC_MCCK_FAIL_STOR_ADDR, offsetof(struct _lowcore, failing_storage_address)); 173 DEFINE(__LC_MCCK_FAIL_STOR_ADDR, offsetof(struct _lowcore, failing_storage_address));
174 DEFINE(__LC_VX_SAVE_AREA_ADDR, offsetof(struct _lowcore, vector_save_area_addr));
174 DEFINE(__LC_EXT_PARAMS2, offsetof(struct _lowcore, ext_params2)); 175 DEFINE(__LC_EXT_PARAMS2, offsetof(struct _lowcore, ext_params2));
175 DEFINE(SAVE_AREA_BASE, offsetof(struct _lowcore, floating_pt_save_area)); 176 DEFINE(SAVE_AREA_BASE, offsetof(struct _lowcore, floating_pt_save_area));
176 DEFINE(__LC_PASTE, offsetof(struct _lowcore, paste)); 177 DEFINE(__LC_PASTE, offsetof(struct _lowcore, paste));
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c
index 9254afff250c..fc7ec95848c3 100644
--- a/arch/s390/kvm/diag.c
+++ b/arch/s390/kvm/diag.c
@@ -77,7 +77,7 @@ static int __diag_page_ref_service(struct kvm_vcpu *vcpu)
77 77
78 if (vcpu->run->s.regs.gprs[rx] & 7) 78 if (vcpu->run->s.regs.gprs[rx] & 7)
79 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 79 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
80 rc = read_guest(vcpu, vcpu->run->s.regs.gprs[rx], &parm, sizeof(parm)); 80 rc = read_guest(vcpu, vcpu->run->s.regs.gprs[rx], rx, &parm, sizeof(parm));
81 if (rc) 81 if (rc)
82 return kvm_s390_inject_prog_cond(vcpu, rc); 82 return kvm_s390_inject_prog_cond(vcpu, rc);
83 if (parm.parm_version != 2 || parm.parm_len < 5 || parm.code != 0x258) 83 if (parm.parm_version != 2 || parm.parm_len < 5 || parm.code != 0x258)
@@ -213,7 +213,7 @@ static int __diag_virtio_hypercall(struct kvm_vcpu *vcpu)
213 * - gpr 3 contains the virtqueue index (passed as datamatch) 213 * - gpr 3 contains the virtqueue index (passed as datamatch)
214 * - gpr 4 contains the index on the bus (optionally) 214 * - gpr 4 contains the index on the bus (optionally)
215 */ 215 */
216 ret = kvm_io_bus_write_cookie(vcpu->kvm, KVM_VIRTIO_CCW_NOTIFY_BUS, 216 ret = kvm_io_bus_write_cookie(vcpu, KVM_VIRTIO_CCW_NOTIFY_BUS,
217 vcpu->run->s.regs.gprs[2] & 0xffffffff, 217 vcpu->run->s.regs.gprs[2] & 0xffffffff,
218 8, &vcpu->run->s.regs.gprs[3], 218 8, &vcpu->run->s.regs.gprs[3],
219 vcpu->run->s.regs.gprs[4]); 219 vcpu->run->s.regs.gprs[4]);
@@ -230,7 +230,7 @@ static int __diag_virtio_hypercall(struct kvm_vcpu *vcpu)
230 230
231int kvm_s390_handle_diag(struct kvm_vcpu *vcpu) 231int kvm_s390_handle_diag(struct kvm_vcpu *vcpu)
232{ 232{
233 int code = kvm_s390_get_base_disp_rs(vcpu) & 0xffff; 233 int code = kvm_s390_get_base_disp_rs(vcpu, NULL) & 0xffff;
234 234
235 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 235 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
236 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 236 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index 267523cac6de..a7559f7207df 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -10,6 +10,7 @@
10#include <asm/pgtable.h> 10#include <asm/pgtable.h>
11#include "kvm-s390.h" 11#include "kvm-s390.h"
12#include "gaccess.h" 12#include "gaccess.h"
13#include <asm/switch_to.h>
13 14
14union asce { 15union asce {
15 unsigned long val; 16 unsigned long val;
@@ -207,6 +208,54 @@ union raddress {
207 unsigned long pfra : 52; /* Page-Frame Real Address */ 208 unsigned long pfra : 52; /* Page-Frame Real Address */
208}; 209};
209 210
211union alet {
212 u32 val;
213 struct {
214 u32 reserved : 7;
215 u32 p : 1;
216 u32 alesn : 8;
217 u32 alen : 16;
218 };
219};
220
221union ald {
222 u32 val;
223 struct {
224 u32 : 1;
225 u32 alo : 24;
226 u32 all : 7;
227 };
228};
229
230struct ale {
231 unsigned long i : 1; /* ALEN-Invalid Bit */
232 unsigned long : 5;
233 unsigned long fo : 1; /* Fetch-Only Bit */
234 unsigned long p : 1; /* Private Bit */
235 unsigned long alesn : 8; /* Access-List-Entry Sequence Number */
236 unsigned long aleax : 16; /* Access-List-Entry Authorization Index */
237 unsigned long : 32;
238 unsigned long : 1;
239 unsigned long asteo : 25; /* ASN-Second-Table-Entry Origin */
240 unsigned long : 6;
241 unsigned long astesn : 32; /* ASTE Sequence Number */
242} __packed;
243
244struct aste {
245 unsigned long i : 1; /* ASX-Invalid Bit */
246 unsigned long ato : 29; /* Authority-Table Origin */
247 unsigned long : 1;
248 unsigned long b : 1; /* Base-Space Bit */
249 unsigned long ax : 16; /* Authorization Index */
250 unsigned long atl : 12; /* Authority-Table Length */
251 unsigned long : 2;
252 unsigned long ca : 1; /* Controlled-ASN Bit */
253 unsigned long ra : 1; /* Reusable-ASN Bit */
254 unsigned long asce : 64; /* Address-Space-Control Element */
255 unsigned long ald : 32;
256 unsigned long astesn : 32;
257 /* .. more fields there */
258} __packed;
210 259
211int ipte_lock_held(struct kvm_vcpu *vcpu) 260int ipte_lock_held(struct kvm_vcpu *vcpu)
212{ 261{
@@ -307,15 +356,157 @@ void ipte_unlock(struct kvm_vcpu *vcpu)
307 ipte_unlock_simple(vcpu); 356 ipte_unlock_simple(vcpu);
308} 357}
309 358
310static unsigned long get_vcpu_asce(struct kvm_vcpu *vcpu) 359static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, ar_t ar,
360 int write)
361{
362 union alet alet;
363 struct ale ale;
364 struct aste aste;
365 unsigned long ald_addr, authority_table_addr;
366 union ald ald;
367 int eax, rc;
368 u8 authority_table;
369
370 if (ar >= NUM_ACRS)
371 return -EINVAL;
372
373 save_access_regs(vcpu->run->s.regs.acrs);
374 alet.val = vcpu->run->s.regs.acrs[ar];
375
376 if (ar == 0 || alet.val == 0) {
377 asce->val = vcpu->arch.sie_block->gcr[1];
378 return 0;
379 } else if (alet.val == 1) {
380 asce->val = vcpu->arch.sie_block->gcr[7];
381 return 0;
382 }
383
384 if (alet.reserved)
385 return PGM_ALET_SPECIFICATION;
386
387 if (alet.p)
388 ald_addr = vcpu->arch.sie_block->gcr[5];
389 else
390 ald_addr = vcpu->arch.sie_block->gcr[2];
391 ald_addr &= 0x7fffffc0;
392
393 rc = read_guest_real(vcpu, ald_addr + 16, &ald.val, sizeof(union ald));
394 if (rc)
395 return rc;
396
397 if (alet.alen / 8 > ald.all)
398 return PGM_ALEN_TRANSLATION;
399
400 if (0x7fffffff - ald.alo * 128 < alet.alen * 16)
401 return PGM_ADDRESSING;
402
403 rc = read_guest_real(vcpu, ald.alo * 128 + alet.alen * 16, &ale,
404 sizeof(struct ale));
405 if (rc)
406 return rc;
407
408 if (ale.i == 1)
409 return PGM_ALEN_TRANSLATION;
410 if (ale.alesn != alet.alesn)
411 return PGM_ALE_SEQUENCE;
412
413 rc = read_guest_real(vcpu, ale.asteo * 64, &aste, sizeof(struct aste));
414 if (rc)
415 return rc;
416
417 if (aste.i)
418 return PGM_ASTE_VALIDITY;
419 if (aste.astesn != ale.astesn)
420 return PGM_ASTE_SEQUENCE;
421
422 if (ale.p == 1) {
423 eax = (vcpu->arch.sie_block->gcr[8] >> 16) & 0xffff;
424 if (ale.aleax != eax) {
425 if (eax / 16 > aste.atl)
426 return PGM_EXTENDED_AUTHORITY;
427
428 authority_table_addr = aste.ato * 4 + eax / 4;
429
430 rc = read_guest_real(vcpu, authority_table_addr,
431 &authority_table,
432 sizeof(u8));
433 if (rc)
434 return rc;
435
436 if ((authority_table & (0x40 >> ((eax & 3) * 2))) == 0)
437 return PGM_EXTENDED_AUTHORITY;
438 }
439 }
440
441 if (ale.fo == 1 && write)
442 return PGM_PROTECTION;
443
444 asce->val = aste.asce;
445 return 0;
446}
447
448struct trans_exc_code_bits {
449 unsigned long addr : 52; /* Translation-exception Address */
450 unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */
451 unsigned long : 6;
452 unsigned long b60 : 1;
453 unsigned long b61 : 1;
454 unsigned long as : 2; /* ASCE Identifier */
455};
456
457enum {
458 FSI_UNKNOWN = 0, /* Unknown wether fetch or store */
459 FSI_STORE = 1, /* Exception was due to store operation */
460 FSI_FETCH = 2 /* Exception was due to fetch operation */
461};
462
463static int get_vcpu_asce(struct kvm_vcpu *vcpu, union asce *asce,
464 ar_t ar, int write)
311{ 465{
466 int rc;
467 psw_t *psw = &vcpu->arch.sie_block->gpsw;
468 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
469 struct trans_exc_code_bits *tec_bits;
470
471 memset(pgm, 0, sizeof(*pgm));
472 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
473 tec_bits->fsi = write ? FSI_STORE : FSI_FETCH;
474 tec_bits->as = psw_bits(*psw).as;
475
476 if (!psw_bits(*psw).t) {
477 asce->val = 0;
478 asce->r = 1;
479 return 0;
480 }
481
312 switch (psw_bits(vcpu->arch.sie_block->gpsw).as) { 482 switch (psw_bits(vcpu->arch.sie_block->gpsw).as) {
313 case PSW_AS_PRIMARY: 483 case PSW_AS_PRIMARY:
314 return vcpu->arch.sie_block->gcr[1]; 484 asce->val = vcpu->arch.sie_block->gcr[1];
485 return 0;
315 case PSW_AS_SECONDARY: 486 case PSW_AS_SECONDARY:
316 return vcpu->arch.sie_block->gcr[7]; 487 asce->val = vcpu->arch.sie_block->gcr[7];
488 return 0;
317 case PSW_AS_HOME: 489 case PSW_AS_HOME:
318 return vcpu->arch.sie_block->gcr[13]; 490 asce->val = vcpu->arch.sie_block->gcr[13];
491 return 0;
492 case PSW_AS_ACCREG:
493 rc = ar_translation(vcpu, asce, ar, write);
494 switch (rc) {
495 case PGM_ALEN_TRANSLATION:
496 case PGM_ALE_SEQUENCE:
497 case PGM_ASTE_VALIDITY:
498 case PGM_ASTE_SEQUENCE:
499 case PGM_EXTENDED_AUTHORITY:
500 vcpu->arch.pgm.exc_access_id = ar;
501 break;
502 case PGM_PROTECTION:
503 tec_bits->b60 = 1;
504 tec_bits->b61 = 1;
505 break;
506 }
507 if (rc > 0)
508 pgm->code = rc;
509 return rc;
319 } 510 }
320 return 0; 511 return 0;
321} 512}
@@ -330,10 +521,11 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
330 * @vcpu: virtual cpu 521 * @vcpu: virtual cpu
331 * @gva: guest virtual address 522 * @gva: guest virtual address
332 * @gpa: points to where guest physical (absolute) address should be stored 523 * @gpa: points to where guest physical (absolute) address should be stored
524 * @asce: effective asce
333 * @write: indicates if access is a write access 525 * @write: indicates if access is a write access
334 * 526 *
335 * Translate a guest virtual address into a guest absolute address by means 527 * Translate a guest virtual address into a guest absolute address by means
336 * of dynamic address translation as specified by the architecuture. 528 * of dynamic address translation as specified by the architecture.
337 * If the resulting absolute address is not available in the configuration 529 * If the resulting absolute address is not available in the configuration
338 * an addressing exception is indicated and @gpa will not be changed. 530 * an addressing exception is indicated and @gpa will not be changed.
339 * 531 *
@@ -345,7 +537,8 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
345 * by the architecture 537 * by the architecture
346 */ 538 */
347static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, 539static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
348 unsigned long *gpa, int write) 540 unsigned long *gpa, const union asce asce,
541 int write)
349{ 542{
350 union vaddress vaddr = {.addr = gva}; 543 union vaddress vaddr = {.addr = gva};
351 union raddress raddr = {.addr = gva}; 544 union raddress raddr = {.addr = gva};
@@ -354,12 +547,10 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
354 union ctlreg0 ctlreg0; 547 union ctlreg0 ctlreg0;
355 unsigned long ptr; 548 unsigned long ptr;
356 int edat1, edat2; 549 int edat1, edat2;
357 union asce asce;
358 550
359 ctlreg0.val = vcpu->arch.sie_block->gcr[0]; 551 ctlreg0.val = vcpu->arch.sie_block->gcr[0];
360 edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8); 552 edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
361 edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78); 553 edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
362 asce.val = get_vcpu_asce(vcpu);
363 if (asce.r) 554 if (asce.r)
364 goto real_address; 555 goto real_address;
365 ptr = asce.origin * 4096; 556 ptr = asce.origin * 4096;
@@ -506,48 +697,30 @@ static inline int is_low_address(unsigned long ga)
506 return (ga & ~0x11fful) == 0; 697 return (ga & ~0x11fful) == 0;
507} 698}
508 699
509static int low_address_protection_enabled(struct kvm_vcpu *vcpu) 700static int low_address_protection_enabled(struct kvm_vcpu *vcpu,
701 const union asce asce)
510{ 702{
511 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]}; 703 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
512 psw_t *psw = &vcpu->arch.sie_block->gpsw; 704 psw_t *psw = &vcpu->arch.sie_block->gpsw;
513 union asce asce;
514 705
515 if (!ctlreg0.lap) 706 if (!ctlreg0.lap)
516 return 0; 707 return 0;
517 asce.val = get_vcpu_asce(vcpu);
518 if (psw_bits(*psw).t && asce.p) 708 if (psw_bits(*psw).t && asce.p)
519 return 0; 709 return 0;
520 return 1; 710 return 1;
521} 711}
522 712
523struct trans_exc_code_bits {
524 unsigned long addr : 52; /* Translation-exception Address */
525 unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */
526 unsigned long : 7;
527 unsigned long b61 : 1;
528 unsigned long as : 2; /* ASCE Identifier */
529};
530
531enum {
532 FSI_UNKNOWN = 0, /* Unknown wether fetch or store */
533 FSI_STORE = 1, /* Exception was due to store operation */
534 FSI_FETCH = 2 /* Exception was due to fetch operation */
535};
536
537static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga, 713static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
538 unsigned long *pages, unsigned long nr_pages, 714 unsigned long *pages, unsigned long nr_pages,
539 int write) 715 const union asce asce, int write)
540{ 716{
541 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; 717 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
542 psw_t *psw = &vcpu->arch.sie_block->gpsw; 718 psw_t *psw = &vcpu->arch.sie_block->gpsw;
543 struct trans_exc_code_bits *tec_bits; 719 struct trans_exc_code_bits *tec_bits;
544 int lap_enabled, rc; 720 int lap_enabled, rc;
545 721
546 memset(pgm, 0, sizeof(*pgm));
547 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code; 722 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
548 tec_bits->fsi = write ? FSI_STORE : FSI_FETCH; 723 lap_enabled = low_address_protection_enabled(vcpu, asce);
549 tec_bits->as = psw_bits(*psw).as;
550 lap_enabled = low_address_protection_enabled(vcpu);
551 while (nr_pages) { 724 while (nr_pages) {
552 ga = kvm_s390_logical_to_effective(vcpu, ga); 725 ga = kvm_s390_logical_to_effective(vcpu, ga);
553 tec_bits->addr = ga >> PAGE_SHIFT; 726 tec_bits->addr = ga >> PAGE_SHIFT;
@@ -557,7 +730,7 @@ static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
557 } 730 }
558 ga &= PAGE_MASK; 731 ga &= PAGE_MASK;
559 if (psw_bits(*psw).t) { 732 if (psw_bits(*psw).t) {
560 rc = guest_translate(vcpu, ga, pages, write); 733 rc = guest_translate(vcpu, ga, pages, asce, write);
561 if (rc < 0) 734 if (rc < 0)
562 return rc; 735 return rc;
563 if (rc == PGM_PROTECTION) 736 if (rc == PGM_PROTECTION)
@@ -578,7 +751,7 @@ static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
578 return 0; 751 return 0;
579} 752}
580 753
581int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data, 754int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
582 unsigned long len, int write) 755 unsigned long len, int write)
583{ 756{
584 psw_t *psw = &vcpu->arch.sie_block->gpsw; 757 psw_t *psw = &vcpu->arch.sie_block->gpsw;
@@ -591,20 +764,19 @@ int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
591 764
592 if (!len) 765 if (!len)
593 return 0; 766 return 0;
594 /* Access register mode is not supported yet. */ 767 rc = get_vcpu_asce(vcpu, &asce, ar, write);
595 if (psw_bits(*psw).t && psw_bits(*psw).as == PSW_AS_ACCREG) 768 if (rc)
596 return -EOPNOTSUPP; 769 return rc;
597 nr_pages = (((ga & ~PAGE_MASK) + len - 1) >> PAGE_SHIFT) + 1; 770 nr_pages = (((ga & ~PAGE_MASK) + len - 1) >> PAGE_SHIFT) + 1;
598 pages = pages_array; 771 pages = pages_array;
599 if (nr_pages > ARRAY_SIZE(pages_array)) 772 if (nr_pages > ARRAY_SIZE(pages_array))
600 pages = vmalloc(nr_pages * sizeof(unsigned long)); 773 pages = vmalloc(nr_pages * sizeof(unsigned long));
601 if (!pages) 774 if (!pages)
602 return -ENOMEM; 775 return -ENOMEM;
603 asce.val = get_vcpu_asce(vcpu);
604 need_ipte_lock = psw_bits(*psw).t && !asce.r; 776 need_ipte_lock = psw_bits(*psw).t && !asce.r;
605 if (need_ipte_lock) 777 if (need_ipte_lock)
606 ipte_lock(vcpu); 778 ipte_lock(vcpu);
607 rc = guest_page_range(vcpu, ga, pages, nr_pages, write); 779 rc = guest_page_range(vcpu, ga, pages, nr_pages, asce, write);
608 for (idx = 0; idx < nr_pages && !rc; idx++) { 780 for (idx = 0; idx < nr_pages && !rc; idx++) {
609 gpa = *(pages + idx) + (ga & ~PAGE_MASK); 781 gpa = *(pages + idx) + (ga & ~PAGE_MASK);
610 _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len); 782 _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len);
@@ -652,7 +824,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
652 * Note: The IPTE lock is not taken during this function, so the caller 824 * Note: The IPTE lock is not taken during this function, so the caller
653 * has to take care of this. 825 * has to take care of this.
654 */ 826 */
655int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, 827int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
656 unsigned long *gpa, int write) 828 unsigned long *gpa, int write)
657{ 829{
658 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; 830 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
@@ -661,26 +833,21 @@ int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
661 union asce asce; 833 union asce asce;
662 int rc; 834 int rc;
663 835
664 /* Access register mode is not supported yet. */
665 if (psw_bits(*psw).t && psw_bits(*psw).as == PSW_AS_ACCREG)
666 return -EOPNOTSUPP;
667
668 gva = kvm_s390_logical_to_effective(vcpu, gva); 836 gva = kvm_s390_logical_to_effective(vcpu, gva);
669 memset(pgm, 0, sizeof(*pgm));
670 tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code; 837 tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
671 tec->as = psw_bits(*psw).as; 838 rc = get_vcpu_asce(vcpu, &asce, ar, write);
672 tec->fsi = write ? FSI_STORE : FSI_FETCH;
673 tec->addr = gva >> PAGE_SHIFT; 839 tec->addr = gva >> PAGE_SHIFT;
674 if (is_low_address(gva) && low_address_protection_enabled(vcpu)) { 840 if (rc)
841 return rc;
842 if (is_low_address(gva) && low_address_protection_enabled(vcpu, asce)) {
675 if (write) { 843 if (write) {
676 rc = pgm->code = PGM_PROTECTION; 844 rc = pgm->code = PGM_PROTECTION;
677 return rc; 845 return rc;
678 } 846 }
679 } 847 }
680 848
681 asce.val = get_vcpu_asce(vcpu);
682 if (psw_bits(*psw).t && !asce.r) { /* Use DAT? */ 849 if (psw_bits(*psw).t && !asce.r) { /* Use DAT? */
683 rc = guest_translate(vcpu, gva, gpa, write); 850 rc = guest_translate(vcpu, gva, gpa, asce, write);
684 if (rc > 0) { 851 if (rc > 0) {
685 if (rc == PGM_PROTECTION) 852 if (rc == PGM_PROTECTION)
686 tec->b61 = 1; 853 tec->b61 = 1;
@@ -697,28 +864,51 @@ int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
697} 864}
698 865
699/** 866/**
700 * kvm_s390_check_low_addr_protection - check for low-address protection 867 * check_gva_range - test a range of guest virtual addresses for accessibility
701 * @ga: Guest address 868 */
869int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
870 unsigned long length, int is_write)
871{
872 unsigned long gpa;
873 unsigned long currlen;
874 int rc = 0;
875
876 ipte_lock(vcpu);
877 while (length > 0 && !rc) {
878 currlen = min(length, PAGE_SIZE - (gva % PAGE_SIZE));
879 rc = guest_translate_address(vcpu, gva, ar, &gpa, is_write);
880 gva += currlen;
881 length -= currlen;
882 }
883 ipte_unlock(vcpu);
884
885 return rc;
886}
887
888/**
889 * kvm_s390_check_low_addr_prot_real - check for low-address protection
890 * @gra: Guest real address
702 * 891 *
703 * Checks whether an address is subject to low-address protection and set 892 * Checks whether an address is subject to low-address protection and set
704 * up vcpu->arch.pgm accordingly if necessary. 893 * up vcpu->arch.pgm accordingly if necessary.
705 * 894 *
706 * Return: 0 if no protection exception, or PGM_PROTECTION if protected. 895 * Return: 0 if no protection exception, or PGM_PROTECTION if protected.
707 */ 896 */
708int kvm_s390_check_low_addr_protection(struct kvm_vcpu *vcpu, unsigned long ga) 897int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra)
709{ 898{
710 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; 899 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
711 psw_t *psw = &vcpu->arch.sie_block->gpsw; 900 psw_t *psw = &vcpu->arch.sie_block->gpsw;
712 struct trans_exc_code_bits *tec_bits; 901 struct trans_exc_code_bits *tec_bits;
902 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
713 903
714 if (!is_low_address(ga) || !low_address_protection_enabled(vcpu)) 904 if (!ctlreg0.lap || !is_low_address(gra))
715 return 0; 905 return 0;
716 906
717 memset(pgm, 0, sizeof(*pgm)); 907 memset(pgm, 0, sizeof(*pgm));
718 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code; 908 tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
719 tec_bits->fsi = FSI_STORE; 909 tec_bits->fsi = FSI_STORE;
720 tec_bits->as = psw_bits(*psw).as; 910 tec_bits->as = psw_bits(*psw).as;
721 tec_bits->addr = ga >> PAGE_SHIFT; 911 tec_bits->addr = gra >> PAGE_SHIFT;
722 pgm->code = PGM_PROTECTION; 912 pgm->code = PGM_PROTECTION;
723 913
724 return pgm->code; 914 return pgm->code;
diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/gaccess.h
index 0149cf15058a..ef03726cc661 100644
--- a/arch/s390/kvm/gaccess.h
+++ b/arch/s390/kvm/gaccess.h
@@ -156,9 +156,11 @@ int read_guest_lc(struct kvm_vcpu *vcpu, unsigned long gra, void *data,
156} 156}
157 157
158int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, 158int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
159 unsigned long *gpa, int write); 159 ar_t ar, unsigned long *gpa, int write);
160int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
161 unsigned long length, int is_write);
160 162
161int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data, 163int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
162 unsigned long len, int write); 164 unsigned long len, int write);
163 165
164int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, 166int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
@@ -168,6 +170,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
168 * write_guest - copy data from kernel space to guest space 170 * write_guest - copy data from kernel space to guest space
169 * @vcpu: virtual cpu 171 * @vcpu: virtual cpu
170 * @ga: guest address 172 * @ga: guest address
173 * @ar: access register
171 * @data: source address in kernel space 174 * @data: source address in kernel space
172 * @len: number of bytes to copy 175 * @len: number of bytes to copy
173 * 176 *
@@ -176,8 +179,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
176 * If DAT is off data will be copied to guest real or absolute memory. 179 * If DAT is off data will be copied to guest real or absolute memory.
177 * If DAT is on data will be copied to the address space as specified by 180 * If DAT is on data will be copied to the address space as specified by
178 * the address space bits of the PSW: 181 * the address space bits of the PSW:
179 * Primary, secondory or home space (access register mode is currently not 182 * Primary, secondary, home space or access register mode.
180 * implemented).
181 * The addressing mode of the PSW is also inspected, so that address wrap 183 * The addressing mode of the PSW is also inspected, so that address wrap
182 * around is taken into account for 24-, 31- and 64-bit addressing mode, 184 * around is taken into account for 24-, 31- and 64-bit addressing mode,
183 * if the to be copied data crosses page boundaries in guest address space. 185 * if the to be copied data crosses page boundaries in guest address space.
@@ -210,16 +212,17 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
210 * if data has been changed in guest space in case of an exception. 212 * if data has been changed in guest space in case of an exception.
211 */ 213 */
212static inline __must_check 214static inline __must_check
213int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data, 215int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
214 unsigned long len) 216 unsigned long len)
215{ 217{
216 return access_guest(vcpu, ga, data, len, 1); 218 return access_guest(vcpu, ga, ar, data, len, 1);
217} 219}
218 220
219/** 221/**
220 * read_guest - copy data from guest space to kernel space 222 * read_guest - copy data from guest space to kernel space
221 * @vcpu: virtual cpu 223 * @vcpu: virtual cpu
222 * @ga: guest address 224 * @ga: guest address
225 * @ar: access register
223 * @data: destination address in kernel space 226 * @data: destination address in kernel space
224 * @len: number of bytes to copy 227 * @len: number of bytes to copy
225 * 228 *
@@ -229,10 +232,10 @@ int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
229 * data will be copied from guest space to kernel space. 232 * data will be copied from guest space to kernel space.
230 */ 233 */
231static inline __must_check 234static inline __must_check
232int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data, 235int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
233 unsigned long len) 236 unsigned long len)
234{ 237{
235 return access_guest(vcpu, ga, data, len, 0); 238 return access_guest(vcpu, ga, ar, data, len, 0);
236} 239}
237 240
238/** 241/**
@@ -330,6 +333,6 @@ int read_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, void *data,
330void ipte_lock(struct kvm_vcpu *vcpu); 333void ipte_lock(struct kvm_vcpu *vcpu);
331void ipte_unlock(struct kvm_vcpu *vcpu); 334void ipte_unlock(struct kvm_vcpu *vcpu);
332int ipte_lock_held(struct kvm_vcpu *vcpu); 335int ipte_lock_held(struct kvm_vcpu *vcpu);
333int kvm_s390_check_low_addr_protection(struct kvm_vcpu *vcpu, unsigned long ga); 336int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra);
334 337
335#endif /* __KVM_S390_GACCESS_H */ 338#endif /* __KVM_S390_GACCESS_H */
diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c
index 3e8d4092ce30..e97b3455d7e6 100644
--- a/arch/s390/kvm/guestdbg.c
+++ b/arch/s390/kvm/guestdbg.c
@@ -191,8 +191,8 @@ static int __import_wp_info(struct kvm_vcpu *vcpu,
191 if (!wp_info->old_data) 191 if (!wp_info->old_data)
192 return -ENOMEM; 192 return -ENOMEM;
193 /* try to backup the original value */ 193 /* try to backup the original value */
194 ret = read_guest(vcpu, wp_info->phys_addr, wp_info->old_data, 194 ret = read_guest_abs(vcpu, wp_info->phys_addr, wp_info->old_data,
195 wp_info->len); 195 wp_info->len);
196 if (ret) { 196 if (ret) {
197 kfree(wp_info->old_data); 197 kfree(wp_info->old_data);
198 wp_info->old_data = NULL; 198 wp_info->old_data = NULL;
@@ -362,8 +362,8 @@ static struct kvm_hw_wp_info_arch *any_wp_changed(struct kvm_vcpu *vcpu)
362 continue; 362 continue;
363 363
364 /* refetch the wp data and compare it to the old value */ 364 /* refetch the wp data and compare it to the old value */
365 if (!read_guest(vcpu, wp_info->phys_addr, temp, 365 if (!read_guest_abs(vcpu, wp_info->phys_addr, temp,
366 wp_info->len)) { 366 wp_info->len)) {
367 if (memcmp(temp, wp_info->old_data, wp_info->len)) { 367 if (memcmp(temp, wp_info->old_data, wp_info->len)) {
368 kfree(temp); 368 kfree(temp);
369 return wp_info; 369 return wp_info;
diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c
index bebd2157edd0..9e3779e3e496 100644
--- a/arch/s390/kvm/intercept.c
+++ b/arch/s390/kvm/intercept.c
@@ -165,6 +165,7 @@ static void __extract_prog_irq(struct kvm_vcpu *vcpu,
165 pgm_info->mon_class_nr = vcpu->arch.sie_block->mcn; 165 pgm_info->mon_class_nr = vcpu->arch.sie_block->mcn;
166 pgm_info->mon_code = vcpu->arch.sie_block->tecmc; 166 pgm_info->mon_code = vcpu->arch.sie_block->tecmc;
167 break; 167 break;
168 case PGM_VECTOR_PROCESSING:
168 case PGM_DATA: 169 case PGM_DATA:
169 pgm_info->data_exc_code = vcpu->arch.sie_block->dxc; 170 pgm_info->data_exc_code = vcpu->arch.sie_block->dxc;
170 break; 171 break;
@@ -319,7 +320,7 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
319 320
320 /* Make sure that the source is paged-in */ 321 /* Make sure that the source is paged-in */
321 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg2], 322 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg2],
322 &srcaddr, 0); 323 reg2, &srcaddr, 0);
323 if (rc) 324 if (rc)
324 return kvm_s390_inject_prog_cond(vcpu, rc); 325 return kvm_s390_inject_prog_cond(vcpu, rc);
325 rc = kvm_arch_fault_in_page(vcpu, srcaddr, 0); 326 rc = kvm_arch_fault_in_page(vcpu, srcaddr, 0);
@@ -328,7 +329,7 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
328 329
329 /* Make sure that the destination is paged-in */ 330 /* Make sure that the destination is paged-in */
330 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg1], 331 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg1],
331 &dstaddr, 1); 332 reg1, &dstaddr, 1);
332 if (rc) 333 if (rc)
333 return kvm_s390_inject_prog_cond(vcpu, rc); 334 return kvm_s390_inject_prog_cond(vcpu, rc);
334 rc = kvm_arch_fault_in_page(vcpu, dstaddr, 1); 335 rc = kvm_arch_fault_in_page(vcpu, dstaddr, 1);
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 073b5f387d1d..9de47265ef73 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * handling kvm guest interrupts 2 * handling kvm guest interrupts
3 * 3 *
4 * Copyright IBM Corp. 2008,2014 4 * Copyright IBM Corp. 2008, 2015
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only) 7 * it under the terms of the GNU General Public License (version 2 only)
@@ -17,9 +17,12 @@
17#include <linux/signal.h> 17#include <linux/signal.h>
18#include <linux/slab.h> 18#include <linux/slab.h>
19#include <linux/bitmap.h> 19#include <linux/bitmap.h>
20#include <linux/vmalloc.h>
20#include <asm/asm-offsets.h> 21#include <asm/asm-offsets.h>
22#include <asm/dis.h>
21#include <asm/uaccess.h> 23#include <asm/uaccess.h>
22#include <asm/sclp.h> 24#include <asm/sclp.h>
25#include <asm/isc.h>
23#include "kvm-s390.h" 26#include "kvm-s390.h"
24#include "gaccess.h" 27#include "gaccess.h"
25#include "trace-s390.h" 28#include "trace-s390.h"
@@ -32,11 +35,6 @@
32#define PFAULT_DONE 0x0680 35#define PFAULT_DONE 0x0680
33#define VIRTIO_PARAM 0x0d00 36#define VIRTIO_PARAM 0x0d00
34 37
35static int is_ioint(u64 type)
36{
37 return ((type & 0xfffe0000u) != 0xfffe0000u);
38}
39
40int psw_extint_disabled(struct kvm_vcpu *vcpu) 38int psw_extint_disabled(struct kvm_vcpu *vcpu)
41{ 39{
42 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT); 40 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
@@ -72,70 +70,45 @@ static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
72 return 1; 70 return 1;
73} 71}
74 72
75static u64 int_word_to_isc_bits(u32 int_word) 73static int ckc_irq_pending(struct kvm_vcpu *vcpu)
74{
75 if (!(vcpu->arch.sie_block->ckc <
76 get_tod_clock_fast() + vcpu->arch.sie_block->epoch))
77 return 0;
78 return ckc_interrupts_enabled(vcpu);
79}
80
81static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
82{
83 return !psw_extint_disabled(vcpu) &&
84 (vcpu->arch.sie_block->gcr[0] & 0x400ul);
85}
86
87static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
88{
89 return (vcpu->arch.sie_block->cputm >> 63) &&
90 cpu_timer_interrupts_enabled(vcpu);
91}
92
93static inline int is_ioirq(unsigned long irq_type)
76{ 94{
77 u8 isc = (int_word & 0x38000000) >> 27; 95 return ((irq_type >= IRQ_PEND_IO_ISC_0) &&
96 (irq_type <= IRQ_PEND_IO_ISC_7));
97}
78 98
99static uint64_t isc_to_isc_bits(int isc)
100{
79 return (0x80 >> isc) << 24; 101 return (0x80 >> isc) << 24;
80} 102}
81 103
82static int __must_check __interrupt_is_deliverable(struct kvm_vcpu *vcpu, 104static inline u8 int_word_to_isc(u32 int_word)
83 struct kvm_s390_interrupt_info *inti)
84{ 105{
85 switch (inti->type) { 106 return (int_word & 0x38000000) >> 27;
86 case KVM_S390_INT_EXTERNAL_CALL: 107}
87 if (psw_extint_disabled(vcpu)) 108
88 return 0; 109static inline unsigned long pending_floating_irqs(struct kvm_vcpu *vcpu)
89 if (vcpu->arch.sie_block->gcr[0] & 0x2000ul) 110{
90 return 1; 111 return vcpu->kvm->arch.float_int.pending_irqs;
91 return 0;
92 case KVM_S390_INT_EMERGENCY:
93 if (psw_extint_disabled(vcpu))
94 return 0;
95 if (vcpu->arch.sie_block->gcr[0] & 0x4000ul)
96 return 1;
97 return 0;
98 case KVM_S390_INT_CLOCK_COMP:
99 return ckc_interrupts_enabled(vcpu);
100 case KVM_S390_INT_CPU_TIMER:
101 if (psw_extint_disabled(vcpu))
102 return 0;
103 if (vcpu->arch.sie_block->gcr[0] & 0x400ul)
104 return 1;
105 return 0;
106 case KVM_S390_INT_SERVICE:
107 case KVM_S390_INT_PFAULT_INIT:
108 case KVM_S390_INT_PFAULT_DONE:
109 case KVM_S390_INT_VIRTIO:
110 if (psw_extint_disabled(vcpu))
111 return 0;
112 if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
113 return 1;
114 return 0;
115 case KVM_S390_PROGRAM_INT:
116 case KVM_S390_SIGP_STOP:
117 case KVM_S390_SIGP_SET_PREFIX:
118 case KVM_S390_RESTART:
119 return 1;
120 case KVM_S390_MCHK:
121 if (psw_mchk_disabled(vcpu))
122 return 0;
123 if (vcpu->arch.sie_block->gcr[14] & inti->mchk.cr14)
124 return 1;
125 return 0;
126 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
127 if (psw_ioint_disabled(vcpu))
128 return 0;
129 if (vcpu->arch.sie_block->gcr[6] &
130 int_word_to_isc_bits(inti->io.io_int_word))
131 return 1;
132 return 0;
133 default:
134 printk(KERN_WARNING "illegal interrupt type %llx\n",
135 inti->type);
136 BUG();
137 }
138 return 0;
139} 112}
140 113
141static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu) 114static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu)
@@ -143,12 +116,31 @@ static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu)
143 return vcpu->arch.local_int.pending_irqs; 116 return vcpu->arch.local_int.pending_irqs;
144} 117}
145 118
146static unsigned long deliverable_local_irqs(struct kvm_vcpu *vcpu) 119static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
120 unsigned long active_mask)
121{
122 int i;
123
124 for (i = 0; i <= MAX_ISC; i++)
125 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
126 active_mask &= ~(1UL << (IRQ_PEND_IO_ISC_0 + i));
127
128 return active_mask;
129}
130
131static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
147{ 132{
148 unsigned long active_mask = pending_local_irqs(vcpu); 133 unsigned long active_mask;
134
135 active_mask = pending_local_irqs(vcpu);
136 active_mask |= pending_floating_irqs(vcpu);
149 137
150 if (psw_extint_disabled(vcpu)) 138 if (psw_extint_disabled(vcpu))
151 active_mask &= ~IRQ_PEND_EXT_MASK; 139 active_mask &= ~IRQ_PEND_EXT_MASK;
140 if (psw_ioint_disabled(vcpu))
141 active_mask &= ~IRQ_PEND_IO_MASK;
142 else
143 active_mask = disable_iscs(vcpu, active_mask);
152 if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 144 if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul))
153 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask); 145 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
154 if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul)) 146 if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul))
@@ -157,8 +149,13 @@ static unsigned long deliverable_local_irqs(struct kvm_vcpu *vcpu)
157 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); 149 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
158 if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul)) 150 if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul))
159 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); 151 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
152 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
153 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
160 if (psw_mchk_disabled(vcpu)) 154 if (psw_mchk_disabled(vcpu))
161 active_mask &= ~IRQ_PEND_MCHK_MASK; 155 active_mask &= ~IRQ_PEND_MCHK_MASK;
156 if (!(vcpu->arch.sie_block->gcr[14] &
157 vcpu->kvm->arch.float_int.mchk.cr14))
158 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
162 159
163 /* 160 /*
164 * STOP irqs will never be actively delivered. They are triggered via 161 * STOP irqs will never be actively delivered. They are triggered via
@@ -200,6 +197,16 @@ static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
200 atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags); 197 atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags);
201} 198}
202 199
200static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
201{
202 if (!(pending_floating_irqs(vcpu) & IRQ_PEND_IO_MASK))
203 return;
204 else if (psw_ioint_disabled(vcpu))
205 __set_cpuflag(vcpu, CPUSTAT_IO_INT);
206 else
207 vcpu->arch.sie_block->lctl |= LCTL_CR6;
208}
209
203static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) 210static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
204{ 211{
205 if (!(pending_local_irqs(vcpu) & IRQ_PEND_EXT_MASK)) 212 if (!(pending_local_irqs(vcpu) & IRQ_PEND_EXT_MASK))
@@ -226,47 +233,17 @@ static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
226 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 233 __set_cpuflag(vcpu, CPUSTAT_STOP_INT);
227} 234}
228 235
229/* Set interception request for non-deliverable local interrupts */ 236/* Set interception request for non-deliverable interrupts */
230static void set_intercept_indicators_local(struct kvm_vcpu *vcpu) 237static void set_intercept_indicators(struct kvm_vcpu *vcpu)
231{ 238{
239 set_intercept_indicators_io(vcpu);
232 set_intercept_indicators_ext(vcpu); 240 set_intercept_indicators_ext(vcpu);
233 set_intercept_indicators_mchk(vcpu); 241 set_intercept_indicators_mchk(vcpu);
234 set_intercept_indicators_stop(vcpu); 242 set_intercept_indicators_stop(vcpu);
235} 243}
236 244
237static void __set_intercept_indicator(struct kvm_vcpu *vcpu,
238 struct kvm_s390_interrupt_info *inti)
239{
240 switch (inti->type) {
241 case KVM_S390_INT_SERVICE:
242 case KVM_S390_INT_PFAULT_DONE:
243 case KVM_S390_INT_VIRTIO:
244 if (psw_extint_disabled(vcpu))
245 __set_cpuflag(vcpu, CPUSTAT_EXT_INT);
246 else
247 vcpu->arch.sie_block->lctl |= LCTL_CR0;
248 break;
249 case KVM_S390_MCHK:
250 if (psw_mchk_disabled(vcpu))
251 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
252 else
253 vcpu->arch.sie_block->lctl |= LCTL_CR14;
254 break;
255 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
256 if (psw_ioint_disabled(vcpu))
257 __set_cpuflag(vcpu, CPUSTAT_IO_INT);
258 else
259 vcpu->arch.sie_block->lctl |= LCTL_CR6;
260 break;
261 default:
262 BUG();
263 }
264}
265
266static u16 get_ilc(struct kvm_vcpu *vcpu) 245static u16 get_ilc(struct kvm_vcpu *vcpu)
267{ 246{
268 const unsigned short table[] = { 2, 4, 4, 6 };
269
270 switch (vcpu->arch.sie_block->icptcode) { 247 switch (vcpu->arch.sie_block->icptcode) {
271 case ICPT_INST: 248 case ICPT_INST:
272 case ICPT_INSTPROGI: 249 case ICPT_INSTPROGI:
@@ -274,7 +251,7 @@ static u16 get_ilc(struct kvm_vcpu *vcpu)
274 case ICPT_PARTEXEC: 251 case ICPT_PARTEXEC:
275 case ICPT_IOINST: 252 case ICPT_IOINST:
276 /* last instruction only stored for these icptcodes */ 253 /* last instruction only stored for these icptcodes */
277 return table[vcpu->arch.sie_block->ipa >> 14]; 254 return insn_length(vcpu->arch.sie_block->ipa >> 8);
278 case ICPT_PROGI: 255 case ICPT_PROGI:
279 return vcpu->arch.sie_block->pgmilc; 256 return vcpu->arch.sie_block->pgmilc;
280 default: 257 default:
@@ -350,38 +327,72 @@ static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
350 327
351static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) 328static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
352{ 329{
330 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
353 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 331 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
354 struct kvm_s390_mchk_info mchk; 332 struct kvm_s390_mchk_info mchk = {};
355 int rc; 333 unsigned long adtl_status_addr;
334 int deliver = 0;
335 int rc = 0;
356 336
337 spin_lock(&fi->lock);
357 spin_lock(&li->lock); 338 spin_lock(&li->lock);
358 mchk = li->irq.mchk; 339 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
340 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
341 /*
342 * If there was an exigent machine check pending, then any
343 * repressible machine checks that might have been pending
344 * are indicated along with it, so always clear bits for
345 * repressible and exigent interrupts
346 */
347 mchk = li->irq.mchk;
348 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
349 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
350 memset(&li->irq.mchk, 0, sizeof(mchk));
351 deliver = 1;
352 }
359 /* 353 /*
360 * If there was an exigent machine check pending, then any repressible 354 * We indicate floating repressible conditions along with
361 * machine checks that might have been pending are indicated along 355 * other pending conditions. Channel Report Pending and Channel
362 * with it, so always clear both bits 356 * Subsystem damage are the only two and and are indicated by
357 * bits in mcic and masked in cr14.
363 */ 358 */
364 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 359 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
365 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 360 mchk.mcic |= fi->mchk.mcic;
366 memset(&li->irq.mchk, 0, sizeof(mchk)); 361 mchk.cr14 |= fi->mchk.cr14;
362 memset(&fi->mchk, 0, sizeof(mchk));
363 deliver = 1;
364 }
367 spin_unlock(&li->lock); 365 spin_unlock(&li->lock);
366 spin_unlock(&fi->lock);
368 367
369 VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx", 368 if (deliver) {
370 mchk.mcic); 369 VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
371 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_MCHK, 370 mchk.mcic);
372 mchk.cr14, mchk.mcic); 371 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
373 372 KVM_S390_MCHK,
374 rc = kvm_s390_vcpu_store_status(vcpu, KVM_S390_STORE_STATUS_PREFIXED); 373 mchk.cr14, mchk.mcic);
375 rc |= put_guest_lc(vcpu, mchk.mcic, 374
376 (u64 __user *) __LC_MCCK_CODE); 375 rc = kvm_s390_vcpu_store_status(vcpu,
377 rc |= put_guest_lc(vcpu, mchk.failing_storage_address, 376 KVM_S390_STORE_STATUS_PREFIXED);
378 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR); 377 rc |= read_guest_lc(vcpu, __LC_VX_SAVE_AREA_ADDR,
379 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, 378 &adtl_status_addr,
380 &mchk.fixed_logout, sizeof(mchk.fixed_logout)); 379 sizeof(unsigned long));
381 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW, 380 rc |= kvm_s390_vcpu_store_adtl_status(vcpu,
382 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 381 adtl_status_addr);
383 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW, 382 rc |= put_guest_lc(vcpu, mchk.mcic,
384 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 383 (u64 __user *) __LC_MCCK_CODE);
384 rc |= put_guest_lc(vcpu, mchk.failing_storage_address,
385 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
386 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA,
387 &mchk.fixed_logout,
388 sizeof(mchk.fixed_logout));
389 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
390 &vcpu->arch.sie_block->gpsw,
391 sizeof(psw_t));
392 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
393 &vcpu->arch.sie_block->gpsw,
394 sizeof(psw_t));
395 }
385 return rc ? -EFAULT : 0; 396 return rc ? -EFAULT : 0;
386} 397}
387 398
@@ -484,7 +495,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
484{ 495{
485 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 496 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
486 struct kvm_s390_pgm_info pgm_info; 497 struct kvm_s390_pgm_info pgm_info;
487 int rc = 0; 498 int rc = 0, nullifying = false;
488 u16 ilc = get_ilc(vcpu); 499 u16 ilc = get_ilc(vcpu);
489 500
490 spin_lock(&li->lock); 501 spin_lock(&li->lock);
@@ -509,6 +520,8 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
509 case PGM_LX_TRANSLATION: 520 case PGM_LX_TRANSLATION:
510 case PGM_PRIMARY_AUTHORITY: 521 case PGM_PRIMARY_AUTHORITY:
511 case PGM_SECONDARY_AUTHORITY: 522 case PGM_SECONDARY_AUTHORITY:
523 nullifying = true;
524 /* fall through */
512 case PGM_SPACE_SWITCH: 525 case PGM_SPACE_SWITCH:
513 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 526 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
514 (u64 *)__LC_TRANS_EXC_CODE); 527 (u64 *)__LC_TRANS_EXC_CODE);
@@ -521,6 +534,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
521 case PGM_EXTENDED_AUTHORITY: 534 case PGM_EXTENDED_AUTHORITY:
522 rc = put_guest_lc(vcpu, pgm_info.exc_access_id, 535 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
523 (u8 *)__LC_EXC_ACCESS_ID); 536 (u8 *)__LC_EXC_ACCESS_ID);
537 nullifying = true;
524 break; 538 break;
525 case PGM_ASCE_TYPE: 539 case PGM_ASCE_TYPE:
526 case PGM_PAGE_TRANSLATION: 540 case PGM_PAGE_TRANSLATION:
@@ -534,6 +548,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
534 (u8 *)__LC_EXC_ACCESS_ID); 548 (u8 *)__LC_EXC_ACCESS_ID);
535 rc |= put_guest_lc(vcpu, pgm_info.op_access_id, 549 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
536 (u8 *)__LC_OP_ACCESS_ID); 550 (u8 *)__LC_OP_ACCESS_ID);
551 nullifying = true;
537 break; 552 break;
538 case PGM_MONITOR: 553 case PGM_MONITOR:
539 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr, 554 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
@@ -541,6 +556,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
541 rc |= put_guest_lc(vcpu, pgm_info.mon_code, 556 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
542 (u64 *)__LC_MON_CODE); 557 (u64 *)__LC_MON_CODE);
543 break; 558 break;
559 case PGM_VECTOR_PROCESSING:
544 case PGM_DATA: 560 case PGM_DATA:
545 rc = put_guest_lc(vcpu, pgm_info.data_exc_code, 561 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
546 (u32 *)__LC_DATA_EXC_CODE); 562 (u32 *)__LC_DATA_EXC_CODE);
@@ -551,6 +567,15 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
551 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 567 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
552 (u8 *)__LC_EXC_ACCESS_ID); 568 (u8 *)__LC_EXC_ACCESS_ID);
553 break; 569 break;
570 case PGM_STACK_FULL:
571 case PGM_STACK_EMPTY:
572 case PGM_STACK_SPECIFICATION:
573 case PGM_STACK_TYPE:
574 case PGM_STACK_OPERATION:
575 case PGM_TRACE_TABEL:
576 case PGM_CRYPTO_OPERATION:
577 nullifying = true;
578 break;
554 } 579 }
555 580
556 if (pgm_info.code & PGM_PER) { 581 if (pgm_info.code & PGM_PER) {
@@ -564,7 +589,12 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
564 (u8 *) __LC_PER_ACCESS_ID); 589 (u8 *) __LC_PER_ACCESS_ID);
565 } 590 }
566 591
592 if (nullifying && vcpu->arch.sie_block->icptcode == ICPT_INST)
593 kvm_s390_rewind_psw(vcpu, ilc);
594
567 rc |= put_guest_lc(vcpu, ilc, (u16 *) __LC_PGM_ILC); 595 rc |= put_guest_lc(vcpu, ilc, (u16 *) __LC_PGM_ILC);
596 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
597 (u64 *) __LC_LAST_BREAK);
568 rc |= put_guest_lc(vcpu, pgm_info.code, 598 rc |= put_guest_lc(vcpu, pgm_info.code,
569 (u16 *)__LC_PGM_INT_CODE); 599 (u16 *)__LC_PGM_INT_CODE);
570 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW, 600 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
@@ -574,16 +604,27 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
574 return rc ? -EFAULT : 0; 604 return rc ? -EFAULT : 0;
575} 605}
576 606
577static int __must_check __deliver_service(struct kvm_vcpu *vcpu, 607static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
578 struct kvm_s390_interrupt_info *inti)
579{ 608{
580 int rc; 609 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
610 struct kvm_s390_ext_info ext;
611 int rc = 0;
612
613 spin_lock(&fi->lock);
614 if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
615 spin_unlock(&fi->lock);
616 return 0;
617 }
618 ext = fi->srv_signal;
619 memset(&fi->srv_signal, 0, sizeof(ext));
620 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
621 spin_unlock(&fi->lock);
581 622
582 VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x", 623 VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x",
583 inti->ext.ext_params); 624 ext.ext_params);
584 vcpu->stat.deliver_service_signal++; 625 vcpu->stat.deliver_service_signal++;
585 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, 626 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
586 inti->ext.ext_params, 0); 627 ext.ext_params, 0);
587 628
588 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); 629 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
589 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 630 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
@@ -591,106 +632,146 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu,
591 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 632 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
592 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 633 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
593 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 634 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
594 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 635 rc |= put_guest_lc(vcpu, ext.ext_params,
595 (u32 *)__LC_EXT_PARAMS); 636 (u32 *)__LC_EXT_PARAMS);
637
596 return rc ? -EFAULT : 0; 638 return rc ? -EFAULT : 0;
597} 639}
598 640
599static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu, 641static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
600 struct kvm_s390_interrupt_info *inti)
601{ 642{
602 int rc; 643 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
644 struct kvm_s390_interrupt_info *inti;
645 int rc = 0;
603 646
604 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 647 spin_lock(&fi->lock);
605 KVM_S390_INT_PFAULT_DONE, 0, 648 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
606 inti->ext.ext_params2); 649 struct kvm_s390_interrupt_info,
650 list);
651 if (inti) {
652 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
653 KVM_S390_INT_PFAULT_DONE, 0,
654 inti->ext.ext_params2);
655 list_del(&inti->list);
656 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
657 }
658 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
659 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
660 spin_unlock(&fi->lock);
607 661
608 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *)__LC_EXT_INT_CODE); 662 if (inti) {
609 rc |= put_guest_lc(vcpu, PFAULT_DONE, (u16 *)__LC_EXT_CPU_ADDR); 663 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
610 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 664 (u16 *)__LC_EXT_INT_CODE);
611 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 665 rc |= put_guest_lc(vcpu, PFAULT_DONE,
612 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 666 (u16 *)__LC_EXT_CPU_ADDR);
613 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 667 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
614 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 668 &vcpu->arch.sie_block->gpsw,
615 (u64 *)__LC_EXT_PARAMS2); 669 sizeof(psw_t));
670 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
671 &vcpu->arch.sie_block->gpsw,
672 sizeof(psw_t));
673 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
674 (u64 *)__LC_EXT_PARAMS2);
675 kfree(inti);
676 }
616 return rc ? -EFAULT : 0; 677 return rc ? -EFAULT : 0;
617} 678}
618 679
619static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu, 680static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
620 struct kvm_s390_interrupt_info *inti)
621{ 681{
622 int rc; 682 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
683 struct kvm_s390_interrupt_info *inti;
684 int rc = 0;
623 685
624 VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx", 686 spin_lock(&fi->lock);
625 inti->ext.ext_params, inti->ext.ext_params2); 687 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
626 vcpu->stat.deliver_virtio_interrupt++; 688 struct kvm_s390_interrupt_info,
627 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, 689 list);
628 inti->ext.ext_params, 690 if (inti) {
629 inti->ext.ext_params2); 691 VCPU_EVENT(vcpu, 4,
692 "interrupt: virtio parm:%x,parm64:%llx",
693 inti->ext.ext_params, inti->ext.ext_params2);
694 vcpu->stat.deliver_virtio_interrupt++;
695 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
696 inti->type,
697 inti->ext.ext_params,
698 inti->ext.ext_params2);
699 list_del(&inti->list);
700 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
701 }
702 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
703 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
704 spin_unlock(&fi->lock);
630 705
631 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *)__LC_EXT_INT_CODE); 706 if (inti) {
632 rc |= put_guest_lc(vcpu, VIRTIO_PARAM, (u16 *)__LC_EXT_CPU_ADDR); 707 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
633 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 708 (u16 *)__LC_EXT_INT_CODE);
634 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 709 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
635 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 710 (u16 *)__LC_EXT_CPU_ADDR);
636 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 711 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
637 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 712 &vcpu->arch.sie_block->gpsw,
638 (u32 *)__LC_EXT_PARAMS); 713 sizeof(psw_t));
639 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 714 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
640 (u64 *)__LC_EXT_PARAMS2); 715 &vcpu->arch.sie_block->gpsw,
716 sizeof(psw_t));
717 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
718 (u32 *)__LC_EXT_PARAMS);
719 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
720 (u64 *)__LC_EXT_PARAMS2);
721 kfree(inti);
722 }
641 return rc ? -EFAULT : 0; 723 return rc ? -EFAULT : 0;
642} 724}
643 725
644static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 726static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
645 struct kvm_s390_interrupt_info *inti) 727 unsigned long irq_type)
646{ 728{
647 int rc; 729 struct list_head *isc_list;
730 struct kvm_s390_float_interrupt *fi;
731 struct kvm_s390_interrupt_info *inti = NULL;
732 int rc = 0;
648 733
649 VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type); 734 fi = &vcpu->kvm->arch.float_int;
650 vcpu->stat.deliver_io_int++;
651 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
652 ((__u32)inti->io.subchannel_id << 16) |
653 inti->io.subchannel_nr,
654 ((__u64)inti->io.io_int_parm << 32) |
655 inti->io.io_int_word);
656
657 rc = put_guest_lc(vcpu, inti->io.subchannel_id,
658 (u16 *)__LC_SUBCHANNEL_ID);
659 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
660 (u16 *)__LC_SUBCHANNEL_NR);
661 rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
662 (u32 *)__LC_IO_INT_PARM);
663 rc |= put_guest_lc(vcpu, inti->io.io_int_word,
664 (u32 *)__LC_IO_INT_WORD);
665 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
666 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
667 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
668 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
669 return rc ? -EFAULT : 0;
670}
671 735
672static int __must_check __deliver_mchk_floating(struct kvm_vcpu *vcpu, 736 spin_lock(&fi->lock);
673 struct kvm_s390_interrupt_info *inti) 737 isc_list = &fi->lists[irq_type - IRQ_PEND_IO_ISC_0];
674{ 738 inti = list_first_entry_or_null(isc_list,
675 struct kvm_s390_mchk_info *mchk = &inti->mchk; 739 struct kvm_s390_interrupt_info,
676 int rc; 740 list);
741 if (inti) {
742 VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type);
743 vcpu->stat.deliver_io_int++;
744 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
745 inti->type,
746 ((__u32)inti->io.subchannel_id << 16) |
747 inti->io.subchannel_nr,
748 ((__u64)inti->io.io_int_parm << 32) |
749 inti->io.io_int_word);
750 list_del(&inti->list);
751 fi->counters[FIRQ_CNTR_IO] -= 1;
752 }
753 if (list_empty(isc_list))
754 clear_bit(irq_type, &fi->pending_irqs);
755 spin_unlock(&fi->lock);
756
757 if (inti) {
758 rc = put_guest_lc(vcpu, inti->io.subchannel_id,
759 (u16 *)__LC_SUBCHANNEL_ID);
760 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
761 (u16 *)__LC_SUBCHANNEL_NR);
762 rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
763 (u32 *)__LC_IO_INT_PARM);
764 rc |= put_guest_lc(vcpu, inti->io.io_int_word,
765 (u32 *)__LC_IO_INT_WORD);
766 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
767 &vcpu->arch.sie_block->gpsw,
768 sizeof(psw_t));
769 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
770 &vcpu->arch.sie_block->gpsw,
771 sizeof(psw_t));
772 kfree(inti);
773 }
677 774
678 VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
679 mchk->mcic);
680 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_MCHK,
681 mchk->cr14, mchk->mcic);
682
683 rc = kvm_s390_vcpu_store_status(vcpu, KVM_S390_STORE_STATUS_PREFIXED);
684 rc |= put_guest_lc(vcpu, mchk->mcic,
685 (u64 __user *) __LC_MCCK_CODE);
686 rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
687 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
688 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA,
689 &mchk->fixed_logout, sizeof(mchk->fixed_logout));
690 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
691 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
692 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
693 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
694 return rc ? -EFAULT : 0; 775 return rc ? -EFAULT : 0;
695} 776}
696 777
@@ -698,6 +779,7 @@ typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu);
698 779
699static const deliver_irq_t deliver_irq_funcs[] = { 780static const deliver_irq_t deliver_irq_funcs[] = {
700 [IRQ_PEND_MCHK_EX] = __deliver_machine_check, 781 [IRQ_PEND_MCHK_EX] = __deliver_machine_check,
782 [IRQ_PEND_MCHK_REP] = __deliver_machine_check,
701 [IRQ_PEND_PROG] = __deliver_prog, 783 [IRQ_PEND_PROG] = __deliver_prog,
702 [IRQ_PEND_EXT_EMERGENCY] = __deliver_emergency_signal, 784 [IRQ_PEND_EXT_EMERGENCY] = __deliver_emergency_signal,
703 [IRQ_PEND_EXT_EXTERNAL] = __deliver_external_call, 785 [IRQ_PEND_EXT_EXTERNAL] = __deliver_external_call,
@@ -706,36 +788,11 @@ static const deliver_irq_t deliver_irq_funcs[] = {
706 [IRQ_PEND_RESTART] = __deliver_restart, 788 [IRQ_PEND_RESTART] = __deliver_restart,
707 [IRQ_PEND_SET_PREFIX] = __deliver_set_prefix, 789 [IRQ_PEND_SET_PREFIX] = __deliver_set_prefix,
708 [IRQ_PEND_PFAULT_INIT] = __deliver_pfault_init, 790 [IRQ_PEND_PFAULT_INIT] = __deliver_pfault_init,
791 [IRQ_PEND_EXT_SERVICE] = __deliver_service,
792 [IRQ_PEND_PFAULT_DONE] = __deliver_pfault_done,
793 [IRQ_PEND_VIRTIO] = __deliver_virtio,
709}; 794};
710 795
711static int __must_check __deliver_floating_interrupt(struct kvm_vcpu *vcpu,
712 struct kvm_s390_interrupt_info *inti)
713{
714 int rc;
715
716 switch (inti->type) {
717 case KVM_S390_INT_SERVICE:
718 rc = __deliver_service(vcpu, inti);
719 break;
720 case KVM_S390_INT_PFAULT_DONE:
721 rc = __deliver_pfault_done(vcpu, inti);
722 break;
723 case KVM_S390_INT_VIRTIO:
724 rc = __deliver_virtio(vcpu, inti);
725 break;
726 case KVM_S390_MCHK:
727 rc = __deliver_mchk_floating(vcpu, inti);
728 break;
729 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
730 rc = __deliver_io(vcpu, inti);
731 break;
732 default:
733 BUG();
734 }
735
736 return rc;
737}
738
739/* Check whether an external call is pending (deliverable or not) */ 796/* Check whether an external call is pending (deliverable or not) */
740int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu) 797int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
741{ 798{
@@ -751,21 +808,9 @@ int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
751 808
752int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop) 809int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
753{ 810{
754 struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
755 struct kvm_s390_interrupt_info *inti;
756 int rc; 811 int rc;
757 812
758 rc = !!deliverable_local_irqs(vcpu); 813 rc = !!deliverable_irqs(vcpu);
759
760 if ((!rc) && atomic_read(&fi->active)) {
761 spin_lock(&fi->lock);
762 list_for_each_entry(inti, &fi->list, list)
763 if (__interrupt_is_deliverable(vcpu, inti)) {
764 rc = 1;
765 break;
766 }
767 spin_unlock(&fi->lock);
768 }
769 814
770 if (!rc && kvm_cpu_has_pending_timer(vcpu)) 815 if (!rc && kvm_cpu_has_pending_timer(vcpu))
771 rc = 1; 816 rc = 1;
@@ -784,12 +829,7 @@ int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
784 829
785int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 830int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
786{ 831{
787 if (!(vcpu->arch.sie_block->ckc < 832 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
788 get_tod_clock_fast() + vcpu->arch.sie_block->epoch))
789 return 0;
790 if (!ckc_interrupts_enabled(vcpu))
791 return 0;
792 return 1;
793} 833}
794 834
795int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) 835int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
@@ -884,60 +924,45 @@ void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
884int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) 924int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
885{ 925{
886 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 926 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
887 struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
888 struct kvm_s390_interrupt_info *n, *inti = NULL;
889 deliver_irq_t func; 927 deliver_irq_t func;
890 int deliver;
891 int rc = 0; 928 int rc = 0;
892 unsigned long irq_type; 929 unsigned long irq_type;
893 unsigned long deliverable_irqs; 930 unsigned long irqs;
894 931
895 __reset_intercept_indicators(vcpu); 932 __reset_intercept_indicators(vcpu);
896 933
897 /* pending ckc conditions might have been invalidated */ 934 /* pending ckc conditions might have been invalidated */
898 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 935 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
899 if (kvm_cpu_has_pending_timer(vcpu)) 936 if (ckc_irq_pending(vcpu))
900 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 937 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
901 938
939 /* pending cpu timer conditions might have been invalidated */
940 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
941 if (cpu_timer_irq_pending(vcpu))
942 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
943
902 do { 944 do {
903 deliverable_irqs = deliverable_local_irqs(vcpu); 945 irqs = deliverable_irqs(vcpu);
904 /* bits are in the order of interrupt priority */ 946 /* bits are in the order of interrupt priority */
905 irq_type = find_first_bit(&deliverable_irqs, IRQ_PEND_COUNT); 947 irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT);
906 if (irq_type == IRQ_PEND_COUNT) 948 if (irq_type == IRQ_PEND_COUNT)
907 break; 949 break;
908 func = deliver_irq_funcs[irq_type]; 950 if (is_ioirq(irq_type)) {
909 if (!func) { 951 rc = __deliver_io(vcpu, irq_type);
910 WARN_ON_ONCE(func == NULL); 952 } else {
911 clear_bit(irq_type, &li->pending_irqs); 953 func = deliver_irq_funcs[irq_type];
912 continue; 954 if (!func) {
955 WARN_ON_ONCE(func == NULL);
956 clear_bit(irq_type, &li->pending_irqs);
957 continue;
958 }
959 rc = func(vcpu);
913 } 960 }
914 rc = func(vcpu); 961 if (rc)
915 } while (!rc && irq_type != IRQ_PEND_COUNT); 962 break;
963 } while (!rc);
916 964
917 set_intercept_indicators_local(vcpu); 965 set_intercept_indicators(vcpu);
918
919 if (!rc && atomic_read(&fi->active)) {
920 do {
921 deliver = 0;
922 spin_lock(&fi->lock);
923 list_for_each_entry_safe(inti, n, &fi->list, list) {
924 if (__interrupt_is_deliverable(vcpu, inti)) {
925 list_del(&inti->list);
926 fi->irq_count--;
927 deliver = 1;
928 break;
929 }
930 __set_intercept_indicator(vcpu, inti);
931 }
932 if (list_empty(&fi->list))
933 atomic_set(&fi->active, 0);
934 spin_unlock(&fi->lock);
935 if (deliver) {
936 rc = __deliver_floating_interrupt(vcpu, inti);
937 kfree(inti);
938 }
939 } while (!rc && deliver);
940 }
941 966
942 return rc; 967 return rc;
943} 968}
@@ -1172,80 +1197,182 @@ static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1172 return 0; 1197 return 0;
1173} 1198}
1174 1199
1200static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1201 int isc, u32 schid)
1202{
1203 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1204 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1205 struct kvm_s390_interrupt_info *iter;
1206 u16 id = (schid & 0xffff0000U) >> 16;
1207 u16 nr = schid & 0x0000ffffU;
1175 1208
1209 spin_lock(&fi->lock);
1210 list_for_each_entry(iter, isc_list, list) {
1211 if (schid && (id != iter->io.subchannel_id ||
1212 nr != iter->io.subchannel_nr))
1213 continue;
1214 /* found an appropriate entry */
1215 list_del_init(&iter->list);
1216 fi->counters[FIRQ_CNTR_IO] -= 1;
1217 if (list_empty(isc_list))
1218 clear_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs);
1219 spin_unlock(&fi->lock);
1220 return iter;
1221 }
1222 spin_unlock(&fi->lock);
1223 return NULL;
1224}
1225
1226/*
1227 * Dequeue and return an I/O interrupt matching any of the interruption
1228 * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1229 */
1176struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1230struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1177 u64 cr6, u64 schid) 1231 u64 isc_mask, u32 schid)
1232{
1233 struct kvm_s390_interrupt_info *inti = NULL;
1234 int isc;
1235
1236 for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1237 if (isc_mask & isc_to_isc_bits(isc))
1238 inti = get_io_int(kvm, isc, schid);
1239 }
1240 return inti;
1241}
1242
1243#define SCCB_MASK 0xFFFFFFF8
1244#define SCCB_EVENT_PENDING 0x3
1245
1246static int __inject_service(struct kvm *kvm,
1247 struct kvm_s390_interrupt_info *inti)
1248{
1249 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1250
1251 spin_lock(&fi->lock);
1252 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1253 /*
1254 * Early versions of the QEMU s390 bios will inject several
1255 * service interrupts after another without handling a
1256 * condition code indicating busy.
1257 * We will silently ignore those superfluous sccb values.
1258 * A future version of QEMU will take care of serialization
1259 * of servc requests
1260 */
1261 if (fi->srv_signal.ext_params & SCCB_MASK)
1262 goto out;
1263 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1264 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1265out:
1266 spin_unlock(&fi->lock);
1267 kfree(inti);
1268 return 0;
1269}
1270
1271static int __inject_virtio(struct kvm *kvm,
1272 struct kvm_s390_interrupt_info *inti)
1273{
1274 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1275
1276 spin_lock(&fi->lock);
1277 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1278 spin_unlock(&fi->lock);
1279 return -EBUSY;
1280 }
1281 fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1282 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1283 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1284 spin_unlock(&fi->lock);
1285 return 0;
1286}
1287
1288static int __inject_pfault_done(struct kvm *kvm,
1289 struct kvm_s390_interrupt_info *inti)
1290{
1291 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1292
1293 spin_lock(&fi->lock);
1294 if (fi->counters[FIRQ_CNTR_PFAULT] >=
1295 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1296 spin_unlock(&fi->lock);
1297 return -EBUSY;
1298 }
1299 fi->counters[FIRQ_CNTR_PFAULT] += 1;
1300 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1301 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1302 spin_unlock(&fi->lock);
1303 return 0;
1304}
1305
1306#define CR_PENDING_SUBCLASS 28
1307static int __inject_float_mchk(struct kvm *kvm,
1308 struct kvm_s390_interrupt_info *inti)
1309{
1310 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1311
1312 spin_lock(&fi->lock);
1313 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1314 fi->mchk.mcic |= inti->mchk.mcic;
1315 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1316 spin_unlock(&fi->lock);
1317 kfree(inti);
1318 return 0;
1319}
1320
1321static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1178{ 1322{
1179 struct kvm_s390_float_interrupt *fi; 1323 struct kvm_s390_float_interrupt *fi;
1180 struct kvm_s390_interrupt_info *inti, *iter; 1324 struct list_head *list;
1325 int isc;
1181 1326
1182 if ((!schid && !cr6) || (schid && cr6))
1183 return NULL;
1184 fi = &kvm->arch.float_int; 1327 fi = &kvm->arch.float_int;
1185 spin_lock(&fi->lock); 1328 spin_lock(&fi->lock);
1186 inti = NULL; 1329 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1187 list_for_each_entry(iter, &fi->list, list) { 1330 spin_unlock(&fi->lock);
1188 if (!is_ioint(iter->type)) 1331 return -EBUSY;
1189 continue;
1190 if (cr6 &&
1191 ((cr6 & int_word_to_isc_bits(iter->io.io_int_word)) == 0))
1192 continue;
1193 if (schid) {
1194 if (((schid & 0x00000000ffff0000) >> 16) !=
1195 iter->io.subchannel_id)
1196 continue;
1197 if ((schid & 0x000000000000ffff) !=
1198 iter->io.subchannel_nr)
1199 continue;
1200 }
1201 inti = iter;
1202 break;
1203 }
1204 if (inti) {
1205 list_del_init(&inti->list);
1206 fi->irq_count--;
1207 } 1332 }
1208 if (list_empty(&fi->list)) 1333 fi->counters[FIRQ_CNTR_IO] += 1;
1209 atomic_set(&fi->active, 0); 1334
1335 isc = int_word_to_isc(inti->io.io_int_word);
1336 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1337 list_add_tail(&inti->list, list);
1338 set_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs);
1210 spin_unlock(&fi->lock); 1339 spin_unlock(&fi->lock);
1211 return inti; 1340 return 0;
1212} 1341}
1213 1342
1214static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1343static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1215{ 1344{
1216 struct kvm_s390_local_interrupt *li; 1345 struct kvm_s390_local_interrupt *li;
1217 struct kvm_s390_float_interrupt *fi; 1346 struct kvm_s390_float_interrupt *fi;
1218 struct kvm_s390_interrupt_info *iter;
1219 struct kvm_vcpu *dst_vcpu = NULL; 1347 struct kvm_vcpu *dst_vcpu = NULL;
1220 int sigcpu; 1348 int sigcpu;
1221 int rc = 0; 1349 u64 type = READ_ONCE(inti->type);
1350 int rc;
1222 1351
1223 fi = &kvm->arch.float_int; 1352 fi = &kvm->arch.float_int;
1224 spin_lock(&fi->lock); 1353
1225 if (fi->irq_count >= KVM_S390_MAX_FLOAT_IRQS) { 1354 switch (type) {
1355 case KVM_S390_MCHK:
1356 rc = __inject_float_mchk(kvm, inti);
1357 break;
1358 case KVM_S390_INT_VIRTIO:
1359 rc = __inject_virtio(kvm, inti);
1360 break;
1361 case KVM_S390_INT_SERVICE:
1362 rc = __inject_service(kvm, inti);
1363 break;
1364 case KVM_S390_INT_PFAULT_DONE:
1365 rc = __inject_pfault_done(kvm, inti);
1366 break;
1367 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1368 rc = __inject_io(kvm, inti);
1369 break;
1370 default:
1226 rc = -EINVAL; 1371 rc = -EINVAL;
1227 goto unlock_fi;
1228 } 1372 }
1229 fi->irq_count++; 1373 if (rc)
1230 if (!is_ioint(inti->type)) { 1374 return rc;
1231 list_add_tail(&inti->list, &fi->list);
1232 } else {
1233 u64 isc_bits = int_word_to_isc_bits(inti->io.io_int_word);
1234 1375
1235 /* Keep I/O interrupts sorted in isc order. */
1236 list_for_each_entry(iter, &fi->list, list) {
1237 if (!is_ioint(iter->type))
1238 continue;
1239 if (int_word_to_isc_bits(iter->io.io_int_word)
1240 <= isc_bits)
1241 continue;
1242 break;
1243 }
1244 list_add_tail(&inti->list, &iter->list);
1245 }
1246 atomic_set(&fi->active, 1);
1247 if (atomic_read(&kvm->online_vcpus) == 0)
1248 goto unlock_fi;
1249 sigcpu = find_first_bit(fi->idle_mask, KVM_MAX_VCPUS); 1376 sigcpu = find_first_bit(fi->idle_mask, KVM_MAX_VCPUS);
1250 if (sigcpu == KVM_MAX_VCPUS) { 1377 if (sigcpu == KVM_MAX_VCPUS) {
1251 do { 1378 do {
@@ -1257,7 +1384,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1257 dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1384 dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1258 li = &dst_vcpu->arch.local_int; 1385 li = &dst_vcpu->arch.local_int;
1259 spin_lock(&li->lock); 1386 spin_lock(&li->lock);
1260 switch (inti->type) { 1387 switch (type) {
1261 case KVM_S390_MCHK: 1388 case KVM_S390_MCHK:
1262 atomic_set_mask(CPUSTAT_STOP_INT, li->cpuflags); 1389 atomic_set_mask(CPUSTAT_STOP_INT, li->cpuflags);
1263 break; 1390 break;
@@ -1270,9 +1397,8 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1270 } 1397 }
1271 spin_unlock(&li->lock); 1398 spin_unlock(&li->lock);
1272 kvm_s390_vcpu_wakeup(kvm_get_vcpu(kvm, sigcpu)); 1399 kvm_s390_vcpu_wakeup(kvm_get_vcpu(kvm, sigcpu));
1273unlock_fi: 1400 return 0;
1274 spin_unlock(&fi->lock); 1401
1275 return rc;
1276} 1402}
1277 1403
1278int kvm_s390_inject_vm(struct kvm *kvm, 1404int kvm_s390_inject_vm(struct kvm *kvm,
@@ -1332,10 +1458,10 @@ int kvm_s390_inject_vm(struct kvm *kvm,
1332 return rc; 1458 return rc;
1333} 1459}
1334 1460
1335void kvm_s390_reinject_io_int(struct kvm *kvm, 1461int kvm_s390_reinject_io_int(struct kvm *kvm,
1336 struct kvm_s390_interrupt_info *inti) 1462 struct kvm_s390_interrupt_info *inti)
1337{ 1463{
1338 __inject_vm(kvm, inti); 1464 return __inject_vm(kvm, inti);
1339} 1465}
1340 1466
1341int s390int_to_s390irq(struct kvm_s390_interrupt *s390int, 1467int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
@@ -1388,12 +1514,10 @@ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
1388 spin_unlock(&li->lock); 1514 spin_unlock(&li->lock);
1389} 1515}
1390 1516
1391int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1517static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1392{ 1518{
1393 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1394 int rc; 1519 int rc;
1395 1520
1396 spin_lock(&li->lock);
1397 switch (irq->type) { 1521 switch (irq->type) {
1398 case KVM_S390_PROGRAM_INT: 1522 case KVM_S390_PROGRAM_INT:
1399 VCPU_EVENT(vcpu, 3, "inject: program check %d (from user)", 1523 VCPU_EVENT(vcpu, 3, "inject: program check %d (from user)",
@@ -1433,83 +1557,130 @@ int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1433 default: 1557 default:
1434 rc = -EINVAL; 1558 rc = -EINVAL;
1435 } 1559 }
1560
1561 return rc;
1562}
1563
1564int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1565{
1566 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1567 int rc;
1568
1569 spin_lock(&li->lock);
1570 rc = do_inject_vcpu(vcpu, irq);
1436 spin_unlock(&li->lock); 1571 spin_unlock(&li->lock);
1437 if (!rc) 1572 if (!rc)
1438 kvm_s390_vcpu_wakeup(vcpu); 1573 kvm_s390_vcpu_wakeup(vcpu);
1439 return rc; 1574 return rc;
1440} 1575}
1441 1576
1442void kvm_s390_clear_float_irqs(struct kvm *kvm) 1577static inline void clear_irq_list(struct list_head *_list)
1443{ 1578{
1444 struct kvm_s390_float_interrupt *fi; 1579 struct kvm_s390_interrupt_info *inti, *n;
1445 struct kvm_s390_interrupt_info *n, *inti = NULL;
1446 1580
1447 fi = &kvm->arch.float_int; 1581 list_for_each_entry_safe(inti, n, _list, list) {
1448 spin_lock(&fi->lock);
1449 list_for_each_entry_safe(inti, n, &fi->list, list) {
1450 list_del(&inti->list); 1582 list_del(&inti->list);
1451 kfree(inti); 1583 kfree(inti);
1452 } 1584 }
1453 fi->irq_count = 0;
1454 atomic_set(&fi->active, 0);
1455 spin_unlock(&fi->lock);
1456} 1585}
1457 1586
1458static inline int copy_irq_to_user(struct kvm_s390_interrupt_info *inti, 1587static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
1459 u8 *addr) 1588 struct kvm_s390_irq *irq)
1460{ 1589{
1461 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr; 1590 irq->type = inti->type;
1462 struct kvm_s390_irq irq = {0};
1463
1464 irq.type = inti->type;
1465 switch (inti->type) { 1591 switch (inti->type) {
1466 case KVM_S390_INT_PFAULT_INIT: 1592 case KVM_S390_INT_PFAULT_INIT:
1467 case KVM_S390_INT_PFAULT_DONE: 1593 case KVM_S390_INT_PFAULT_DONE:
1468 case KVM_S390_INT_VIRTIO: 1594 case KVM_S390_INT_VIRTIO:
1469 case KVM_S390_INT_SERVICE: 1595 irq->u.ext = inti->ext;
1470 irq.u.ext = inti->ext;
1471 break; 1596 break;
1472 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1597 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1473 irq.u.io = inti->io; 1598 irq->u.io = inti->io;
1474 break; 1599 break;
1475 case KVM_S390_MCHK:
1476 irq.u.mchk = inti->mchk;
1477 break;
1478 default:
1479 return -EINVAL;
1480 } 1600 }
1601}
1481 1602
1482 if (copy_to_user(uptr, &irq, sizeof(irq))) 1603void kvm_s390_clear_float_irqs(struct kvm *kvm)
1483 return -EFAULT; 1604{
1605 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1606 int i;
1484 1607
1485 return 0; 1608 spin_lock(&fi->lock);
1486} 1609 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1610 clear_irq_list(&fi->lists[i]);
1611 for (i = 0; i < FIRQ_MAX_COUNT; i++)
1612 fi->counters[i] = 0;
1613 spin_unlock(&fi->lock);
1614};
1487 1615
1488static int get_all_floating_irqs(struct kvm *kvm, __u8 *buf, __u64 len) 1616static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
1489{ 1617{
1490 struct kvm_s390_interrupt_info *inti; 1618 struct kvm_s390_interrupt_info *inti;
1491 struct kvm_s390_float_interrupt *fi; 1619 struct kvm_s390_float_interrupt *fi;
1620 struct kvm_s390_irq *buf;
1621 struct kvm_s390_irq *irq;
1622 int max_irqs;
1492 int ret = 0; 1623 int ret = 0;
1493 int n = 0; 1624 int n = 0;
1625 int i;
1626
1627 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
1628 return -EINVAL;
1629
1630 /*
1631 * We are already using -ENOMEM to signal
1632 * userspace it may retry with a bigger buffer,
1633 * so we need to use something else for this case
1634 */
1635 buf = vzalloc(len);
1636 if (!buf)
1637 return -ENOBUFS;
1638
1639 max_irqs = len / sizeof(struct kvm_s390_irq);
1494 1640
1495 fi = &kvm->arch.float_int; 1641 fi = &kvm->arch.float_int;
1496 spin_lock(&fi->lock); 1642 spin_lock(&fi->lock);
1497 1643 for (i = 0; i < FIRQ_LIST_COUNT; i++) {
1498 list_for_each_entry(inti, &fi->list, list) { 1644 list_for_each_entry(inti, &fi->lists[i], list) {
1499 if (len < sizeof(struct kvm_s390_irq)) { 1645 if (n == max_irqs) {
1646 /* signal userspace to try again */
1647 ret = -ENOMEM;
1648 goto out;
1649 }
1650 inti_to_irq(inti, &buf[n]);
1651 n++;
1652 }
1653 }
1654 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) {
1655 if (n == max_irqs) {
1500 /* signal userspace to try again */ 1656 /* signal userspace to try again */
1501 ret = -ENOMEM; 1657 ret = -ENOMEM;
1502 break; 1658 goto out;
1503 } 1659 }
1504 ret = copy_irq_to_user(inti, buf); 1660 irq = (struct kvm_s390_irq *) &buf[n];
1505 if (ret) 1661 irq->type = KVM_S390_INT_SERVICE;
1506 break; 1662 irq->u.ext = fi->srv_signal;
1507 buf += sizeof(struct kvm_s390_irq);
1508 len -= sizeof(struct kvm_s390_irq);
1509 n++; 1663 n++;
1510 } 1664 }
1665 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
1666 if (n == max_irqs) {
1667 /* signal userspace to try again */
1668 ret = -ENOMEM;
1669 goto out;
1670 }
1671 irq = (struct kvm_s390_irq *) &buf[n];
1672 irq->type = KVM_S390_MCHK;
1673 irq->u.mchk = fi->mchk;
1674 n++;
1675}
1511 1676
1677out:
1512 spin_unlock(&fi->lock); 1678 spin_unlock(&fi->lock);
1679 if (!ret && n > 0) {
1680 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
1681 ret = -EFAULT;
1682 }
1683 vfree(buf);
1513 1684
1514 return ret < 0 ? ret : n; 1685 return ret < 0 ? ret : n;
1515} 1686}
@@ -1520,7 +1691,7 @@ static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1520 1691
1521 switch (attr->group) { 1692 switch (attr->group) {
1522 case KVM_DEV_FLIC_GET_ALL_IRQS: 1693 case KVM_DEV_FLIC_GET_ALL_IRQS:
1523 r = get_all_floating_irqs(dev->kvm, (u8 *) attr->addr, 1694 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
1524 attr->attr); 1695 attr->attr);
1525 break; 1696 break;
1526 default: 1697 default:
@@ -1952,3 +2123,143 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
1952{ 2123{
1953 return -EINVAL; 2124 return -EINVAL;
1954} 2125}
2126
2127int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2128{
2129 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2130 struct kvm_s390_irq *buf;
2131 int r = 0;
2132 int n;
2133
2134 buf = vmalloc(len);
2135 if (!buf)
2136 return -ENOMEM;
2137
2138 if (copy_from_user((void *) buf, irqstate, len)) {
2139 r = -EFAULT;
2140 goto out_free;
2141 }
2142
2143 /*
2144 * Don't allow setting the interrupt state
2145 * when there are already interrupts pending
2146 */
2147 spin_lock(&li->lock);
2148 if (li->pending_irqs) {
2149 r = -EBUSY;
2150 goto out_unlock;
2151 }
2152
2153 for (n = 0; n < len / sizeof(*buf); n++) {
2154 r = do_inject_vcpu(vcpu, &buf[n]);
2155 if (r)
2156 break;
2157 }
2158
2159out_unlock:
2160 spin_unlock(&li->lock);
2161out_free:
2162 vfree(buf);
2163
2164 return r;
2165}
2166
2167static void store_local_irq(struct kvm_s390_local_interrupt *li,
2168 struct kvm_s390_irq *irq,
2169 unsigned long irq_type)
2170{
2171 switch (irq_type) {
2172 case IRQ_PEND_MCHK_EX:
2173 case IRQ_PEND_MCHK_REP:
2174 irq->type = KVM_S390_MCHK;
2175 irq->u.mchk = li->irq.mchk;
2176 break;
2177 case IRQ_PEND_PROG:
2178 irq->type = KVM_S390_PROGRAM_INT;
2179 irq->u.pgm = li->irq.pgm;
2180 break;
2181 case IRQ_PEND_PFAULT_INIT:
2182 irq->type = KVM_S390_INT_PFAULT_INIT;
2183 irq->u.ext = li->irq.ext;
2184 break;
2185 case IRQ_PEND_EXT_EXTERNAL:
2186 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2187 irq->u.extcall = li->irq.extcall;
2188 break;
2189 case IRQ_PEND_EXT_CLOCK_COMP:
2190 irq->type = KVM_S390_INT_CLOCK_COMP;
2191 break;
2192 case IRQ_PEND_EXT_CPU_TIMER:
2193 irq->type = KVM_S390_INT_CPU_TIMER;
2194 break;
2195 case IRQ_PEND_SIGP_STOP:
2196 irq->type = KVM_S390_SIGP_STOP;
2197 irq->u.stop = li->irq.stop;
2198 break;
2199 case IRQ_PEND_RESTART:
2200 irq->type = KVM_S390_RESTART;
2201 break;
2202 case IRQ_PEND_SET_PREFIX:
2203 irq->type = KVM_S390_SIGP_SET_PREFIX;
2204 irq->u.prefix = li->irq.prefix;
2205 break;
2206 }
2207}
2208
2209int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
2210{
2211 uint8_t sigp_ctrl = vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl;
2212 unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)];
2213 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2214 unsigned long pending_irqs;
2215 struct kvm_s390_irq irq;
2216 unsigned long irq_type;
2217 int cpuaddr;
2218 int n = 0;
2219
2220 spin_lock(&li->lock);
2221 pending_irqs = li->pending_irqs;
2222 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
2223 sizeof(sigp_emerg_pending));
2224 spin_unlock(&li->lock);
2225
2226 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
2227 memset(&irq, 0, sizeof(irq));
2228 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
2229 continue;
2230 if (n + sizeof(irq) > len)
2231 return -ENOBUFS;
2232 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
2233 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2234 return -EFAULT;
2235 n += sizeof(irq);
2236 }
2237
2238 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
2239 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
2240 memset(&irq, 0, sizeof(irq));
2241 if (n + sizeof(irq) > len)
2242 return -ENOBUFS;
2243 irq.type = KVM_S390_INT_EMERGENCY;
2244 irq.u.emerg.code = cpuaddr;
2245 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2246 return -EFAULT;
2247 n += sizeof(irq);
2248 }
2249 }
2250
2251 if ((sigp_ctrl & SIGP_CTRL_C) &&
2252 (atomic_read(&vcpu->arch.sie_block->cpuflags) &
2253 CPUSTAT_ECALL_PEND)) {
2254 if (n + sizeof(irq) > len)
2255 return -ENOBUFS;
2256 memset(&irq, 0, sizeof(irq));
2257 irq.type = KVM_S390_INT_EXTERNAL_CALL;
2258 irq.u.extcall.code = sigp_ctrl & SIGP_CTRL_SCN_MASK;
2259 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2260 return -EFAULT;
2261 n += sizeof(irq);
2262 }
2263
2264 return n;
2265}
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 19e17bd7aec0..afa2bd750ffc 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -25,11 +25,13 @@
25#include <linux/random.h> 25#include <linux/random.h>
26#include <linux/slab.h> 26#include <linux/slab.h>
27#include <linux/timer.h> 27#include <linux/timer.h>
28#include <linux/vmalloc.h>
28#include <asm/asm-offsets.h> 29#include <asm/asm-offsets.h>
29#include <asm/lowcore.h> 30#include <asm/lowcore.h>
30#include <asm/pgtable.h> 31#include <asm/pgtable.h>
31#include <asm/nmi.h> 32#include <asm/nmi.h>
32#include <asm/switch_to.h> 33#include <asm/switch_to.h>
34#include <asm/isc.h>
33#include <asm/sclp.h> 35#include <asm/sclp.h>
34#include "kvm-s390.h" 36#include "kvm-s390.h"
35#include "gaccess.h" 37#include "gaccess.h"
@@ -38,6 +40,11 @@
38#include "trace.h" 40#include "trace.h"
39#include "trace-s390.h" 41#include "trace-s390.h"
40 42
43#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
44#define LOCAL_IRQS 32
45#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
46 (KVM_MAX_VCPUS + LOCAL_IRQS))
47
41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 48#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42 49
43struct kvm_stats_debugfs_item debugfs_entries[] = { 50struct kvm_stats_debugfs_item debugfs_entries[] = {
@@ -87,6 +94,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
87 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 94 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
88 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) }, 95 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
89 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) }, 96 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
97 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
90 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 98 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
91 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 99 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
92 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 100 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
@@ -101,8 +109,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
101 109
102/* upper facilities limit for kvm */ 110/* upper facilities limit for kvm */
103unsigned long kvm_s390_fac_list_mask[] = { 111unsigned long kvm_s390_fac_list_mask[] = {
104 0xff82fffbf4fc2000UL, 112 0xffe6fffbfcfdfc40UL,
105 0x005c000000000000UL, 113 0x205c800000000000UL,
106}; 114};
107 115
108unsigned long kvm_s390_fac_list_mask_size(void) 116unsigned long kvm_s390_fac_list_mask_size(void)
@@ -171,9 +179,16 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
171 case KVM_CAP_S390_IRQCHIP: 179 case KVM_CAP_S390_IRQCHIP:
172 case KVM_CAP_VM_ATTRIBUTES: 180 case KVM_CAP_VM_ATTRIBUTES:
173 case KVM_CAP_MP_STATE: 181 case KVM_CAP_MP_STATE:
182 case KVM_CAP_S390_INJECT_IRQ:
174 case KVM_CAP_S390_USER_SIGP: 183 case KVM_CAP_S390_USER_SIGP:
184 case KVM_CAP_S390_USER_STSI:
185 case KVM_CAP_S390_SKEYS:
186 case KVM_CAP_S390_IRQ_STATE:
175 r = 1; 187 r = 1;
176 break; 188 break;
189 case KVM_CAP_S390_MEM_OP:
190 r = MEM_OP_MAX_SIZE;
191 break;
177 case KVM_CAP_NR_VCPUS: 192 case KVM_CAP_NR_VCPUS:
178 case KVM_CAP_MAX_VCPUS: 193 case KVM_CAP_MAX_VCPUS:
179 r = KVM_MAX_VCPUS; 194 r = KVM_MAX_VCPUS;
@@ -184,6 +199,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
184 case KVM_CAP_S390_COW: 199 case KVM_CAP_S390_COW:
185 r = MACHINE_HAS_ESOP; 200 r = MACHINE_HAS_ESOP;
186 break; 201 break;
202 case KVM_CAP_S390_VECTOR_REGISTERS:
203 r = MACHINE_HAS_VX;
204 break;
187 default: 205 default:
188 r = 0; 206 r = 0;
189 } 207 }
@@ -264,6 +282,18 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
264 kvm->arch.user_sigp = 1; 282 kvm->arch.user_sigp = 1;
265 r = 0; 283 r = 0;
266 break; 284 break;
285 case KVM_CAP_S390_VECTOR_REGISTERS:
286 if (MACHINE_HAS_VX) {
287 set_kvm_facility(kvm->arch.model.fac->mask, 129);
288 set_kvm_facility(kvm->arch.model.fac->list, 129);
289 r = 0;
290 } else
291 r = -EINVAL;
292 break;
293 case KVM_CAP_S390_USER_STSI:
294 kvm->arch.user_stsi = 1;
295 r = 0;
296 break;
267 default: 297 default:
268 r = -EINVAL; 298 r = -EINVAL;
269 break; 299 break;
@@ -708,6 +738,108 @@ static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
708 return ret; 738 return ret;
709} 739}
710 740
741static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
742{
743 uint8_t *keys;
744 uint64_t hva;
745 unsigned long curkey;
746 int i, r = 0;
747
748 if (args->flags != 0)
749 return -EINVAL;
750
751 /* Is this guest using storage keys? */
752 if (!mm_use_skey(current->mm))
753 return KVM_S390_GET_SKEYS_NONE;
754
755 /* Enforce sane limit on memory allocation */
756 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
757 return -EINVAL;
758
759 keys = kmalloc_array(args->count, sizeof(uint8_t),
760 GFP_KERNEL | __GFP_NOWARN);
761 if (!keys)
762 keys = vmalloc(sizeof(uint8_t) * args->count);
763 if (!keys)
764 return -ENOMEM;
765
766 for (i = 0; i < args->count; i++) {
767 hva = gfn_to_hva(kvm, args->start_gfn + i);
768 if (kvm_is_error_hva(hva)) {
769 r = -EFAULT;
770 goto out;
771 }
772
773 curkey = get_guest_storage_key(current->mm, hva);
774 if (IS_ERR_VALUE(curkey)) {
775 r = curkey;
776 goto out;
777 }
778 keys[i] = curkey;
779 }
780
781 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
782 sizeof(uint8_t) * args->count);
783 if (r)
784 r = -EFAULT;
785out:
786 kvfree(keys);
787 return r;
788}
789
790static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
791{
792 uint8_t *keys;
793 uint64_t hva;
794 int i, r = 0;
795
796 if (args->flags != 0)
797 return -EINVAL;
798
799 /* Enforce sane limit on memory allocation */
800 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
801 return -EINVAL;
802
803 keys = kmalloc_array(args->count, sizeof(uint8_t),
804 GFP_KERNEL | __GFP_NOWARN);
805 if (!keys)
806 keys = vmalloc(sizeof(uint8_t) * args->count);
807 if (!keys)
808 return -ENOMEM;
809
810 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
811 sizeof(uint8_t) * args->count);
812 if (r) {
813 r = -EFAULT;
814 goto out;
815 }
816
817 /* Enable storage key handling for the guest */
818 s390_enable_skey();
819
820 for (i = 0; i < args->count; i++) {
821 hva = gfn_to_hva(kvm, args->start_gfn + i);
822 if (kvm_is_error_hva(hva)) {
823 r = -EFAULT;
824 goto out;
825 }
826
827 /* Lowest order bit is reserved */
828 if (keys[i] & 0x01) {
829 r = -EINVAL;
830 goto out;
831 }
832
833 r = set_guest_storage_key(current->mm, hva,
834 (unsigned long)keys[i], 0);
835 if (r)
836 goto out;
837 }
838out:
839 kvfree(keys);
840 return r;
841}
842
711long kvm_arch_vm_ioctl(struct file *filp, 843long kvm_arch_vm_ioctl(struct file *filp,
712 unsigned int ioctl, unsigned long arg) 844 unsigned int ioctl, unsigned long arg)
713{ 845{
@@ -767,6 +899,26 @@ long kvm_arch_vm_ioctl(struct file *filp,
767 r = kvm_s390_vm_has_attr(kvm, &attr); 899 r = kvm_s390_vm_has_attr(kvm, &attr);
768 break; 900 break;
769 } 901 }
902 case KVM_S390_GET_SKEYS: {
903 struct kvm_s390_skeys args;
904
905 r = -EFAULT;
906 if (copy_from_user(&args, argp,
907 sizeof(struct kvm_s390_skeys)))
908 break;
909 r = kvm_s390_get_skeys(kvm, &args);
910 break;
911 }
912 case KVM_S390_SET_SKEYS: {
913 struct kvm_s390_skeys args;
914
915 r = -EFAULT;
916 if (copy_from_user(&args, argp,
917 sizeof(struct kvm_s390_skeys)))
918 break;
919 r = kvm_s390_set_skeys(kvm, &args);
920 break;
921 }
770 default: 922 default:
771 r = -ENOTTY; 923 r = -ENOTTY;
772 } 924 }
@@ -887,7 +1039,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
887 1039
888 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long)); 1040 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
889 if (!kvm->arch.dbf) 1041 if (!kvm->arch.dbf)
890 goto out_nodbf; 1042 goto out_err;
891 1043
892 /* 1044 /*
893 * The architectural maximum amount of facilities is 16 kbit. To store 1045 * The architectural maximum amount of facilities is 16 kbit. To store
@@ -899,7 +1051,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
899 kvm->arch.model.fac = 1051 kvm->arch.model.fac =
900 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1052 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
901 if (!kvm->arch.model.fac) 1053 if (!kvm->arch.model.fac)
902 goto out_nofac; 1054 goto out_err;
903 1055
904 /* Populate the facility mask initially. */ 1056 /* Populate the facility mask initially. */
905 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list, 1057 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
@@ -919,10 +1071,11 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
919 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff; 1071 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
920 1072
921 if (kvm_s390_crypto_init(kvm) < 0) 1073 if (kvm_s390_crypto_init(kvm) < 0)
922 goto out_crypto; 1074 goto out_err;
923 1075
924 spin_lock_init(&kvm->arch.float_int.lock); 1076 spin_lock_init(&kvm->arch.float_int.lock);
925 INIT_LIST_HEAD(&kvm->arch.float_int.list); 1077 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1078 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
926 init_waitqueue_head(&kvm->arch.ipte_wq); 1079 init_waitqueue_head(&kvm->arch.ipte_wq);
927 mutex_init(&kvm->arch.ipte_mutex); 1080 mutex_init(&kvm->arch.ipte_mutex);
928 1081
@@ -934,7 +1087,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
934 } else { 1087 } else {
935 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1); 1088 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
936 if (!kvm->arch.gmap) 1089 if (!kvm->arch.gmap)
937 goto out_nogmap; 1090 goto out_err;
938 kvm->arch.gmap->private = kvm; 1091 kvm->arch.gmap->private = kvm;
939 kvm->arch.gmap->pfault_enabled = 0; 1092 kvm->arch.gmap->pfault_enabled = 0;
940 } 1093 }
@@ -946,15 +1099,11 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
946 spin_lock_init(&kvm->arch.start_stop_lock); 1099 spin_lock_init(&kvm->arch.start_stop_lock);
947 1100
948 return 0; 1101 return 0;
949out_nogmap: 1102out_err:
950 kfree(kvm->arch.crypto.crycb); 1103 kfree(kvm->arch.crypto.crycb);
951out_crypto:
952 free_page((unsigned long)kvm->arch.model.fac); 1104 free_page((unsigned long)kvm->arch.model.fac);
953out_nofac:
954 debug_unregister(kvm->arch.dbf); 1105 debug_unregister(kvm->arch.dbf);
955out_nodbf:
956 free_page((unsigned long)(kvm->arch.sca)); 1106 free_page((unsigned long)(kvm->arch.sca));
957out_err:
958 return rc; 1107 return rc;
959} 1108}
960 1109
@@ -1034,6 +1183,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1034 KVM_SYNC_CRS | 1183 KVM_SYNC_CRS |
1035 KVM_SYNC_ARCH0 | 1184 KVM_SYNC_ARCH0 |
1036 KVM_SYNC_PFAULT; 1185 KVM_SYNC_PFAULT;
1186 if (test_kvm_facility(vcpu->kvm, 129))
1187 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1037 1188
1038 if (kvm_is_ucontrol(vcpu->kvm)) 1189 if (kvm_is_ucontrol(vcpu->kvm))
1039 return __kvm_ucontrol_vcpu_init(vcpu); 1190 return __kvm_ucontrol_vcpu_init(vcpu);
@@ -1044,10 +1195,18 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1044void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1195void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1045{ 1196{
1046 save_fp_ctl(&vcpu->arch.host_fpregs.fpc); 1197 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1047 save_fp_regs(vcpu->arch.host_fpregs.fprs); 1198 if (test_kvm_facility(vcpu->kvm, 129))
1199 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1200 else
1201 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1048 save_access_regs(vcpu->arch.host_acrs); 1202 save_access_regs(vcpu->arch.host_acrs);
1049 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1203 if (test_kvm_facility(vcpu->kvm, 129)) {
1050 restore_fp_regs(vcpu->arch.guest_fpregs.fprs); 1204 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1205 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1206 } else {
1207 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1208 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1209 }
1051 restore_access_regs(vcpu->run->s.regs.acrs); 1210 restore_access_regs(vcpu->run->s.regs.acrs);
1052 gmap_enable(vcpu->arch.gmap); 1211 gmap_enable(vcpu->arch.gmap);
1053 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 1212 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
@@ -1057,11 +1216,19 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1057{ 1216{
1058 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 1217 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1059 gmap_disable(vcpu->arch.gmap); 1218 gmap_disable(vcpu->arch.gmap);
1060 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc); 1219 if (test_kvm_facility(vcpu->kvm, 129)) {
1061 save_fp_regs(vcpu->arch.guest_fpregs.fprs); 1220 save_fp_ctl(&vcpu->run->s.regs.fpc);
1221 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1222 } else {
1223 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1224 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1225 }
1062 save_access_regs(vcpu->run->s.regs.acrs); 1226 save_access_regs(vcpu->run->s.regs.acrs);
1063 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc); 1227 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1064 restore_fp_regs(vcpu->arch.host_fpregs.fprs); 1228 if (test_kvm_facility(vcpu->kvm, 129))
1229 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1230 else
1231 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1065 restore_access_regs(vcpu->arch.host_acrs); 1232 restore_access_regs(vcpu->arch.host_acrs);
1066} 1233}
1067 1234
@@ -1129,6 +1296,15 @@ int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1129 return 0; 1296 return 0;
1130} 1297}
1131 1298
1299static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1300{
1301 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1302
1303 vcpu->arch.cpu_id = model->cpu_id;
1304 vcpu->arch.sie_block->ibc = model->ibc;
1305 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1306}
1307
1132int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 1308int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1133{ 1309{
1134 int rc = 0; 1310 int rc = 0;
@@ -1137,6 +1313,8 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1137 CPUSTAT_SM | 1313 CPUSTAT_SM |
1138 CPUSTAT_STOPPED | 1314 CPUSTAT_STOPPED |
1139 CPUSTAT_GED); 1315 CPUSTAT_GED);
1316 kvm_s390_vcpu_setup_model(vcpu);
1317
1140 vcpu->arch.sie_block->ecb = 6; 1318 vcpu->arch.sie_block->ecb = 6;
1141 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73)) 1319 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1142 vcpu->arch.sie_block->ecb |= 0x10; 1320 vcpu->arch.sie_block->ecb |= 0x10;
@@ -1147,8 +1325,11 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1147 vcpu->arch.sie_block->eca |= 1; 1325 vcpu->arch.sie_block->eca |= 1;
1148 if (sclp_has_sigpif()) 1326 if (sclp_has_sigpif())
1149 vcpu->arch.sie_block->eca |= 0x10000000U; 1327 vcpu->arch.sie_block->eca |= 0x10000000U;
1150 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE | 1328 if (test_kvm_facility(vcpu->kvm, 129)) {
1151 ICTL_TPROT; 1329 vcpu->arch.sie_block->eca |= 0x00020000;
1330 vcpu->arch.sie_block->ecd |= 0x20000000;
1331 }
1332 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1152 1333
1153 if (kvm_s390_cmma_enabled(vcpu->kvm)) { 1334 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1154 rc = kvm_s390_vcpu_setup_cmma(vcpu); 1335 rc = kvm_s390_vcpu_setup_cmma(vcpu);
@@ -1158,11 +1339,6 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1158 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1339 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1159 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 1340 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1160 1341
1161 mutex_lock(&vcpu->kvm->lock);
1162 vcpu->arch.cpu_id = vcpu->kvm->arch.model.cpu_id;
1163 vcpu->arch.sie_block->ibc = vcpu->kvm->arch.model.ibc;
1164 mutex_unlock(&vcpu->kvm->lock);
1165
1166 kvm_s390_vcpu_crypto_setup(vcpu); 1342 kvm_s390_vcpu_crypto_setup(vcpu);
1167 1343
1168 return rc; 1344 return rc;
@@ -1190,6 +1366,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1190 1366
1191 vcpu->arch.sie_block = &sie_page->sie_block; 1367 vcpu->arch.sie_block = &sie_page->sie_block;
1192 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb; 1368 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1369 vcpu->arch.host_vregs = &sie_page->vregs;
1193 1370
1194 vcpu->arch.sie_block->icpua = id; 1371 vcpu->arch.sie_block->icpua = id;
1195 if (!kvm_is_ucontrol(kvm)) { 1372 if (!kvm_is_ucontrol(kvm)) {
@@ -1205,7 +1382,6 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1205 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca; 1382 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1206 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn); 1383 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1207 } 1384 }
1208 vcpu->arch.sie_block->fac = (int) (long) kvm->arch.model.fac->list;
1209 1385
1210 spin_lock_init(&vcpu->arch.local_int.lock); 1386 spin_lock_init(&vcpu->arch.local_int.lock);
1211 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 1387 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
@@ -1725,6 +1901,31 @@ static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1725 return 0; 1901 return 0;
1726} 1902}
1727 1903
1904static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1905{
1906 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1907 u8 opcode;
1908 int rc;
1909
1910 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1911 trace_kvm_s390_sie_fault(vcpu);
1912
1913 /*
1914 * We want to inject an addressing exception, which is defined as a
1915 * suppressing or terminating exception. However, since we came here
1916 * by a DAT access exception, the PSW still points to the faulting
1917 * instruction since DAT exceptions are nullifying. So we've got
1918 * to look up the current opcode to get the length of the instruction
1919 * to be able to forward the PSW.
1920 */
1921 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
1922 if (rc)
1923 return kvm_s390_inject_prog_cond(vcpu, rc);
1924 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1925
1926 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1927}
1928
1728static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) 1929static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1729{ 1930{
1730 int rc = -1; 1931 int rc = -1;
@@ -1756,11 +1957,8 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1756 } 1957 }
1757 } 1958 }
1758 1959
1759 if (rc == -1) { 1960 if (rc == -1)
1760 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 1961 rc = vcpu_post_run_fault_in_sie(vcpu);
1761 trace_kvm_s390_sie_fault(vcpu);
1762 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1763 }
1764 1962
1765 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16); 1963 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1766 1964
@@ -1976,6 +2174,35 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1976 return kvm_s390_store_status_unloaded(vcpu, addr); 2174 return kvm_s390_store_status_unloaded(vcpu, addr);
1977} 2175}
1978 2176
2177/*
2178 * store additional status at address
2179 */
2180int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2181 unsigned long gpa)
2182{
2183 /* Only bits 0-53 are used for address formation */
2184 if (!(gpa & ~0x3ff))
2185 return 0;
2186
2187 return write_guest_abs(vcpu, gpa & ~0x3ff,
2188 (void *)&vcpu->run->s.regs.vrs, 512);
2189}
2190
2191int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2192{
2193 if (!test_kvm_facility(vcpu->kvm, 129))
2194 return 0;
2195
2196 /*
2197 * The guest VXRS are in the host VXRs due to the lazy
2198 * copying in vcpu load/put. Let's update our copies before we save
2199 * it into the save area.
2200 */
2201 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2202
2203 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2204}
2205
1979static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 2206static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1980{ 2207{
1981 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu); 2208 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
@@ -2100,6 +2327,65 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2100 return r; 2327 return r;
2101} 2328}
2102 2329
2330static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2331 struct kvm_s390_mem_op *mop)
2332{
2333 void __user *uaddr = (void __user *)mop->buf;
2334 void *tmpbuf = NULL;
2335 int r, srcu_idx;
2336 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2337 | KVM_S390_MEMOP_F_CHECK_ONLY;
2338
2339 if (mop->flags & ~supported_flags)
2340 return -EINVAL;
2341
2342 if (mop->size > MEM_OP_MAX_SIZE)
2343 return -E2BIG;
2344
2345 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2346 tmpbuf = vmalloc(mop->size);
2347 if (!tmpbuf)
2348 return -ENOMEM;
2349 }
2350
2351 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2352
2353 switch (mop->op) {
2354 case KVM_S390_MEMOP_LOGICAL_READ:
2355 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2356 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2357 break;
2358 }
2359 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2360 if (r == 0) {
2361 if (copy_to_user(uaddr, tmpbuf, mop->size))
2362 r = -EFAULT;
2363 }
2364 break;
2365 case KVM_S390_MEMOP_LOGICAL_WRITE:
2366 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2367 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2368 break;
2369 }
2370 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2371 r = -EFAULT;
2372 break;
2373 }
2374 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2375 break;
2376 default:
2377 r = -EINVAL;
2378 }
2379
2380 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2381
2382 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2383 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2384
2385 vfree(tmpbuf);
2386 return r;
2387}
2388
2103long kvm_arch_vcpu_ioctl(struct file *filp, 2389long kvm_arch_vcpu_ioctl(struct file *filp,
2104 unsigned int ioctl, unsigned long arg) 2390 unsigned int ioctl, unsigned long arg)
2105{ 2391{
@@ -2109,6 +2395,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
2109 long r; 2395 long r;
2110 2396
2111 switch (ioctl) { 2397 switch (ioctl) {
2398 case KVM_S390_IRQ: {
2399 struct kvm_s390_irq s390irq;
2400
2401 r = -EFAULT;
2402 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2403 break;
2404 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2405 break;
2406 }
2112 case KVM_S390_INTERRUPT: { 2407 case KVM_S390_INTERRUPT: {
2113 struct kvm_s390_interrupt s390int; 2408 struct kvm_s390_interrupt s390int;
2114 struct kvm_s390_irq s390irq; 2409 struct kvm_s390_irq s390irq;
@@ -2199,6 +2494,47 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
2199 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 2494 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2200 break; 2495 break;
2201 } 2496 }
2497 case KVM_S390_MEM_OP: {
2498 struct kvm_s390_mem_op mem_op;
2499
2500 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2501 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2502 else
2503 r = -EFAULT;
2504 break;
2505 }
2506 case KVM_S390_SET_IRQ_STATE: {
2507 struct kvm_s390_irq_state irq_state;
2508
2509 r = -EFAULT;
2510 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2511 break;
2512 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2513 irq_state.len == 0 ||
2514 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2515 r = -EINVAL;
2516 break;
2517 }
2518 r = kvm_s390_set_irq_state(vcpu,
2519 (void __user *) irq_state.buf,
2520 irq_state.len);
2521 break;
2522 }
2523 case KVM_S390_GET_IRQ_STATE: {
2524 struct kvm_s390_irq_state irq_state;
2525
2526 r = -EFAULT;
2527 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2528 break;
2529 if (irq_state.len == 0) {
2530 r = -EINVAL;
2531 break;
2532 }
2533 r = kvm_s390_get_irq_state(vcpu,
2534 (__u8 __user *) irq_state.buf,
2535 irq_state.len);
2536 break;
2537 }
2202 default: 2538 default:
2203 r = -ENOTTY; 2539 r = -ENOTTY;
2204 } 2540 }
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index c34109aa552d..ca108b90ae56 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -70,16 +70,22 @@ static inline void kvm_s390_set_prefix(struct kvm_vcpu *vcpu, u32 prefix)
70 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 70 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
71} 71}
72 72
73static inline u64 kvm_s390_get_base_disp_s(struct kvm_vcpu *vcpu) 73typedef u8 __bitwise ar_t;
74
75static inline u64 kvm_s390_get_base_disp_s(struct kvm_vcpu *vcpu, ar_t *ar)
74{ 76{
75 u32 base2 = vcpu->arch.sie_block->ipb >> 28; 77 u32 base2 = vcpu->arch.sie_block->ipb >> 28;
76 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16); 78 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16);
77 79
80 if (ar)
81 *ar = base2;
82
78 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2; 83 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2;
79} 84}
80 85
81static inline void kvm_s390_get_base_disp_sse(struct kvm_vcpu *vcpu, 86static inline void kvm_s390_get_base_disp_sse(struct kvm_vcpu *vcpu,
82 u64 *address1, u64 *address2) 87 u64 *address1, u64 *address2,
88 ar_t *ar_b1, ar_t *ar_b2)
83{ 89{
84 u32 base1 = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28; 90 u32 base1 = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28;
85 u32 disp1 = (vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16; 91 u32 disp1 = (vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16;
@@ -88,6 +94,11 @@ static inline void kvm_s390_get_base_disp_sse(struct kvm_vcpu *vcpu,
88 94
89 *address1 = (base1 ? vcpu->run->s.regs.gprs[base1] : 0) + disp1; 95 *address1 = (base1 ? vcpu->run->s.regs.gprs[base1] : 0) + disp1;
90 *address2 = (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2; 96 *address2 = (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2;
97
98 if (ar_b1)
99 *ar_b1 = base1;
100 if (ar_b2)
101 *ar_b2 = base2;
91} 102}
92 103
93static inline void kvm_s390_get_regs_rre(struct kvm_vcpu *vcpu, int *r1, int *r2) 104static inline void kvm_s390_get_regs_rre(struct kvm_vcpu *vcpu, int *r1, int *r2)
@@ -98,7 +109,7 @@ static inline void kvm_s390_get_regs_rre(struct kvm_vcpu *vcpu, int *r1, int *r2
98 *r2 = (vcpu->arch.sie_block->ipb & 0x000f0000) >> 16; 109 *r2 = (vcpu->arch.sie_block->ipb & 0x000f0000) >> 16;
99} 110}
100 111
101static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu) 112static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu, ar_t *ar)
102{ 113{
103 u32 base2 = vcpu->arch.sie_block->ipb >> 28; 114 u32 base2 = vcpu->arch.sie_block->ipb >> 28;
104 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16) + 115 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16) +
@@ -107,14 +118,20 @@ static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu)
107 if (disp2 & 0x80000) 118 if (disp2 & 0x80000)
108 disp2+=0xfff00000; 119 disp2+=0xfff00000;
109 120
121 if (ar)
122 *ar = base2;
123
110 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + (long)(int)disp2; 124 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + (long)(int)disp2;
111} 125}
112 126
113static inline u64 kvm_s390_get_base_disp_rs(struct kvm_vcpu *vcpu) 127static inline u64 kvm_s390_get_base_disp_rs(struct kvm_vcpu *vcpu, ar_t *ar)
114{ 128{
115 u32 base2 = vcpu->arch.sie_block->ipb >> 28; 129 u32 base2 = vcpu->arch.sie_block->ipb >> 28;
116 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16); 130 u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16);
117 131
132 if (ar)
133 *ar = base2;
134
118 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2; 135 return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + disp2;
119} 136}
120 137
@@ -125,13 +142,24 @@ static inline void kvm_s390_set_psw_cc(struct kvm_vcpu *vcpu, unsigned long cc)
125 vcpu->arch.sie_block->gpsw.mask |= cc << 44; 142 vcpu->arch.sie_block->gpsw.mask |= cc << 44;
126} 143}
127 144
128/* test availability of facility in a kvm intance */ 145/* test availability of facility in a kvm instance */
129static inline int test_kvm_facility(struct kvm *kvm, unsigned long nr) 146static inline int test_kvm_facility(struct kvm *kvm, unsigned long nr)
130{ 147{
131 return __test_facility(nr, kvm->arch.model.fac->mask) && 148 return __test_facility(nr, kvm->arch.model.fac->mask) &&
132 __test_facility(nr, kvm->arch.model.fac->list); 149 __test_facility(nr, kvm->arch.model.fac->list);
133} 150}
134 151
152static inline int set_kvm_facility(u64 *fac_list, unsigned long nr)
153{
154 unsigned char *ptr;
155
156 if (nr >= MAX_FACILITY_BIT)
157 return -EINVAL;
158 ptr = (unsigned char *) fac_list + (nr >> 3);
159 *ptr |= (0x80UL >> (nr & 7));
160 return 0;
161}
162
135/* are cpu states controlled by user space */ 163/* are cpu states controlled by user space */
136static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm) 164static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm)
137{ 165{
@@ -150,9 +178,9 @@ int __must_check kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu,
150 struct kvm_s390_irq *irq); 178 struct kvm_s390_irq *irq);
151int __must_check kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code); 179int __must_check kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code);
152struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 180struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
153 u64 cr6, u64 schid); 181 u64 isc_mask, u32 schid);
154void kvm_s390_reinject_io_int(struct kvm *kvm, 182int kvm_s390_reinject_io_int(struct kvm *kvm,
155 struct kvm_s390_interrupt_info *inti); 183 struct kvm_s390_interrupt_info *inti);
156int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked); 184int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked);
157 185
158/* implemented in intercept.c */ 186/* implemented in intercept.c */
@@ -177,7 +205,10 @@ int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu);
177/* implemented in kvm-s390.c */ 205/* implemented in kvm-s390.c */
178long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable); 206long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable);
179int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr); 207int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr);
208int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
209 unsigned long addr);
180int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr); 210int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr);
211int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr);
181void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu); 212void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu);
182void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu); 213void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu);
183void s390_vcpu_block(struct kvm_vcpu *vcpu); 214void s390_vcpu_block(struct kvm_vcpu *vcpu);
@@ -241,6 +272,10 @@ int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu);
241extern struct kvm_device_ops kvm_flic_ops; 272extern struct kvm_device_ops kvm_flic_ops;
242int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu); 273int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu);
243void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu); 274void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu);
275int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu,
276 void __user *buf, int len);
277int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu,
278 __u8 __user *buf, int len);
244 279
245/* implemented in guestdbg.c */ 280/* implemented in guestdbg.c */
246void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu); 281void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu);
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index 351116939ea2..d22d8ee1ff9d 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -36,15 +36,16 @@ static int handle_set_clock(struct kvm_vcpu *vcpu)
36 struct kvm_vcpu *cpup; 36 struct kvm_vcpu *cpup;
37 s64 hostclk, val; 37 s64 hostclk, val;
38 int i, rc; 38 int i, rc;
39 ar_t ar;
39 u64 op2; 40 u64 op2;
40 41
41 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 42 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
42 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 43 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
43 44
44 op2 = kvm_s390_get_base_disp_s(vcpu); 45 op2 = kvm_s390_get_base_disp_s(vcpu, &ar);
45 if (op2 & 7) /* Operand must be on a doubleword boundary */ 46 if (op2 & 7) /* Operand must be on a doubleword boundary */
46 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 47 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
47 rc = read_guest(vcpu, op2, &val, sizeof(val)); 48 rc = read_guest(vcpu, op2, ar, &val, sizeof(val));
48 if (rc) 49 if (rc)
49 return kvm_s390_inject_prog_cond(vcpu, rc); 50 return kvm_s390_inject_prog_cond(vcpu, rc);
50 51
@@ -68,20 +69,21 @@ static int handle_set_prefix(struct kvm_vcpu *vcpu)
68 u64 operand2; 69 u64 operand2;
69 u32 address; 70 u32 address;
70 int rc; 71 int rc;
72 ar_t ar;
71 73
72 vcpu->stat.instruction_spx++; 74 vcpu->stat.instruction_spx++;
73 75
74 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 76 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
75 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 77 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
76 78
77 operand2 = kvm_s390_get_base_disp_s(vcpu); 79 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
78 80
79 /* must be word boundary */ 81 /* must be word boundary */
80 if (operand2 & 3) 82 if (operand2 & 3)
81 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 83 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
82 84
83 /* get the value */ 85 /* get the value */
84 rc = read_guest(vcpu, operand2, &address, sizeof(address)); 86 rc = read_guest(vcpu, operand2, ar, &address, sizeof(address));
85 if (rc) 87 if (rc)
86 return kvm_s390_inject_prog_cond(vcpu, rc); 88 return kvm_s390_inject_prog_cond(vcpu, rc);
87 89
@@ -107,13 +109,14 @@ static int handle_store_prefix(struct kvm_vcpu *vcpu)
107 u64 operand2; 109 u64 operand2;
108 u32 address; 110 u32 address;
109 int rc; 111 int rc;
112 ar_t ar;
110 113
111 vcpu->stat.instruction_stpx++; 114 vcpu->stat.instruction_stpx++;
112 115
113 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 116 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
114 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 117 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
115 118
116 operand2 = kvm_s390_get_base_disp_s(vcpu); 119 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
117 120
118 /* must be word boundary */ 121 /* must be word boundary */
119 if (operand2 & 3) 122 if (operand2 & 3)
@@ -122,7 +125,7 @@ static int handle_store_prefix(struct kvm_vcpu *vcpu)
122 address = kvm_s390_get_prefix(vcpu); 125 address = kvm_s390_get_prefix(vcpu);
123 126
124 /* get the value */ 127 /* get the value */
125 rc = write_guest(vcpu, operand2, &address, sizeof(address)); 128 rc = write_guest(vcpu, operand2, ar, &address, sizeof(address));
126 if (rc) 129 if (rc)
127 return kvm_s390_inject_prog_cond(vcpu, rc); 130 return kvm_s390_inject_prog_cond(vcpu, rc);
128 131
@@ -136,18 +139,19 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu)
136 u16 vcpu_id = vcpu->vcpu_id; 139 u16 vcpu_id = vcpu->vcpu_id;
137 u64 ga; 140 u64 ga;
138 int rc; 141 int rc;
142 ar_t ar;
139 143
140 vcpu->stat.instruction_stap++; 144 vcpu->stat.instruction_stap++;
141 145
142 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 146 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
143 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 147 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
144 148
145 ga = kvm_s390_get_base_disp_s(vcpu); 149 ga = kvm_s390_get_base_disp_s(vcpu, &ar);
146 150
147 if (ga & 1) 151 if (ga & 1)
148 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 152 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
149 153
150 rc = write_guest(vcpu, ga, &vcpu_id, sizeof(vcpu_id)); 154 rc = write_guest(vcpu, ga, ar, &vcpu_id, sizeof(vcpu_id));
151 if (rc) 155 if (rc)
152 return kvm_s390_inject_prog_cond(vcpu, rc); 156 return kvm_s390_inject_prog_cond(vcpu, rc);
153 157
@@ -207,7 +211,7 @@ static int handle_test_block(struct kvm_vcpu *vcpu)
207 kvm_s390_get_regs_rre(vcpu, NULL, &reg2); 211 kvm_s390_get_regs_rre(vcpu, NULL, &reg2);
208 addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 212 addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
209 addr = kvm_s390_logical_to_effective(vcpu, addr); 213 addr = kvm_s390_logical_to_effective(vcpu, addr);
210 if (kvm_s390_check_low_addr_protection(vcpu, addr)) 214 if (kvm_s390_check_low_addr_prot_real(vcpu, addr))
211 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 215 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
212 addr = kvm_s390_real_to_abs(vcpu, addr); 216 addr = kvm_s390_real_to_abs(vcpu, addr);
213 217
@@ -229,18 +233,20 @@ static int handle_tpi(struct kvm_vcpu *vcpu)
229 struct kvm_s390_interrupt_info *inti; 233 struct kvm_s390_interrupt_info *inti;
230 unsigned long len; 234 unsigned long len;
231 u32 tpi_data[3]; 235 u32 tpi_data[3];
232 int cc, rc; 236 int rc;
233 u64 addr; 237 u64 addr;
238 ar_t ar;
234 239
235 rc = 0; 240 addr = kvm_s390_get_base_disp_s(vcpu, &ar);
236 addr = kvm_s390_get_base_disp_s(vcpu);
237 if (addr & 3) 241 if (addr & 3)
238 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 242 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
239 cc = 0; 243
240 inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0); 244 inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0);
241 if (!inti) 245 if (!inti) {
242 goto no_interrupt; 246 kvm_s390_set_psw_cc(vcpu, 0);
243 cc = 1; 247 return 0;
248 }
249
244 tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr; 250 tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr;
245 tpi_data[1] = inti->io.io_int_parm; 251 tpi_data[1] = inti->io.io_int_parm;
246 tpi_data[2] = inti->io.io_int_word; 252 tpi_data[2] = inti->io.io_int_word;
@@ -250,40 +256,51 @@ static int handle_tpi(struct kvm_vcpu *vcpu)
250 * provided area. 256 * provided area.
251 */ 257 */
252 len = sizeof(tpi_data) - 4; 258 len = sizeof(tpi_data) - 4;
253 rc = write_guest(vcpu, addr, &tpi_data, len); 259 rc = write_guest(vcpu, addr, ar, &tpi_data, len);
254 if (rc) 260 if (rc) {
255 return kvm_s390_inject_prog_cond(vcpu, rc); 261 rc = kvm_s390_inject_prog_cond(vcpu, rc);
262 goto reinject_interrupt;
263 }
256 } else { 264 } else {
257 /* 265 /*
258 * Store the three-word I/O interruption code into 266 * Store the three-word I/O interruption code into
259 * the appropriate lowcore area. 267 * the appropriate lowcore area.
260 */ 268 */
261 len = sizeof(tpi_data); 269 len = sizeof(tpi_data);
262 if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) 270 if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) {
271 /* failed writes to the low core are not recoverable */
263 rc = -EFAULT; 272 rc = -EFAULT;
273 goto reinject_interrupt;
274 }
264 } 275 }
276
277 /* irq was successfully handed to the guest */
278 kfree(inti);
279 kvm_s390_set_psw_cc(vcpu, 1);
280 return 0;
281reinject_interrupt:
265 /* 282 /*
266 * If we encounter a problem storing the interruption code, the 283 * If we encounter a problem storing the interruption code, the
267 * instruction is suppressed from the guest's view: reinject the 284 * instruction is suppressed from the guest's view: reinject the
268 * interrupt. 285 * interrupt.
269 */ 286 */
270 if (!rc) 287 if (kvm_s390_reinject_io_int(vcpu->kvm, inti)) {
271 kfree(inti); 288 kfree(inti);
272 else 289 rc = -EFAULT;
273 kvm_s390_reinject_io_int(vcpu->kvm, inti); 290 }
274no_interrupt: 291 /* don't set the cc, a pgm irq was injected or we drop to user space */
275 /* Set condition code and we're done. */
276 if (!rc)
277 kvm_s390_set_psw_cc(vcpu, cc);
278 return rc ? -EFAULT : 0; 292 return rc ? -EFAULT : 0;
279} 293}
280 294
281static int handle_tsch(struct kvm_vcpu *vcpu) 295static int handle_tsch(struct kvm_vcpu *vcpu)
282{ 296{
283 struct kvm_s390_interrupt_info *inti; 297 struct kvm_s390_interrupt_info *inti = NULL;
298 const u64 isc_mask = 0xffUL << 24; /* all iscs set */
284 299
285 inti = kvm_s390_get_io_int(vcpu->kvm, 0, 300 /* a valid schid has at least one bit set */
286 vcpu->run->s.regs.gprs[1]); 301 if (vcpu->run->s.regs.gprs[1])
302 inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask,
303 vcpu->run->s.regs.gprs[1]);
287 304
288 /* 305 /*
289 * Prepare exit to userspace. 306 * Prepare exit to userspace.
@@ -386,15 +403,16 @@ int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu)
386 psw_compat_t new_psw; 403 psw_compat_t new_psw;
387 u64 addr; 404 u64 addr;
388 int rc; 405 int rc;
406 ar_t ar;
389 407
390 if (gpsw->mask & PSW_MASK_PSTATE) 408 if (gpsw->mask & PSW_MASK_PSTATE)
391 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 409 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
392 410
393 addr = kvm_s390_get_base_disp_s(vcpu); 411 addr = kvm_s390_get_base_disp_s(vcpu, &ar);
394 if (addr & 7) 412 if (addr & 7)
395 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 413 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
396 414
397 rc = read_guest(vcpu, addr, &new_psw, sizeof(new_psw)); 415 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
398 if (rc) 416 if (rc)
399 return kvm_s390_inject_prog_cond(vcpu, rc); 417 return kvm_s390_inject_prog_cond(vcpu, rc);
400 if (!(new_psw.mask & PSW32_MASK_BASE)) 418 if (!(new_psw.mask & PSW32_MASK_BASE))
@@ -412,14 +430,15 @@ static int handle_lpswe(struct kvm_vcpu *vcpu)
412 psw_t new_psw; 430 psw_t new_psw;
413 u64 addr; 431 u64 addr;
414 int rc; 432 int rc;
433 ar_t ar;
415 434
416 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 435 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
417 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 436 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
418 437
419 addr = kvm_s390_get_base_disp_s(vcpu); 438 addr = kvm_s390_get_base_disp_s(vcpu, &ar);
420 if (addr & 7) 439 if (addr & 7)
421 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 440 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
422 rc = read_guest(vcpu, addr, &new_psw, sizeof(new_psw)); 441 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
423 if (rc) 442 if (rc)
424 return kvm_s390_inject_prog_cond(vcpu, rc); 443 return kvm_s390_inject_prog_cond(vcpu, rc);
425 vcpu->arch.sie_block->gpsw = new_psw; 444 vcpu->arch.sie_block->gpsw = new_psw;
@@ -433,18 +452,19 @@ static int handle_stidp(struct kvm_vcpu *vcpu)
433 u64 stidp_data = vcpu->arch.stidp_data; 452 u64 stidp_data = vcpu->arch.stidp_data;
434 u64 operand2; 453 u64 operand2;
435 int rc; 454 int rc;
455 ar_t ar;
436 456
437 vcpu->stat.instruction_stidp++; 457 vcpu->stat.instruction_stidp++;
438 458
439 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 459 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
440 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 460 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
441 461
442 operand2 = kvm_s390_get_base_disp_s(vcpu); 462 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
443 463
444 if (operand2 & 7) 464 if (operand2 & 7)
445 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 465 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
446 466
447 rc = write_guest(vcpu, operand2, &stidp_data, sizeof(stidp_data)); 467 rc = write_guest(vcpu, operand2, ar, &stidp_data, sizeof(stidp_data));
448 if (rc) 468 if (rc)
449 return kvm_s390_inject_prog_cond(vcpu, rc); 469 return kvm_s390_inject_prog_cond(vcpu, rc);
450 470
@@ -467,6 +487,7 @@ static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem)
467 for (n = mem->count - 1; n > 0 ; n--) 487 for (n = mem->count - 1; n > 0 ; n--)
468 memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0])); 488 memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0]));
469 489
490 memset(&mem->vm[0], 0, sizeof(mem->vm[0]));
470 mem->vm[0].cpus_total = cpus; 491 mem->vm[0].cpus_total = cpus;
471 mem->vm[0].cpus_configured = cpus; 492 mem->vm[0].cpus_configured = cpus;
472 mem->vm[0].cpus_standby = 0; 493 mem->vm[0].cpus_standby = 0;
@@ -478,6 +499,17 @@ static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem)
478 ASCEBC(mem->vm[0].cpi, 16); 499 ASCEBC(mem->vm[0].cpi, 16);
479} 500}
480 501
502static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, ar_t ar,
503 u8 fc, u8 sel1, u16 sel2)
504{
505 vcpu->run->exit_reason = KVM_EXIT_S390_STSI;
506 vcpu->run->s390_stsi.addr = addr;
507 vcpu->run->s390_stsi.ar = ar;
508 vcpu->run->s390_stsi.fc = fc;
509 vcpu->run->s390_stsi.sel1 = sel1;
510 vcpu->run->s390_stsi.sel2 = sel2;
511}
512
481static int handle_stsi(struct kvm_vcpu *vcpu) 513static int handle_stsi(struct kvm_vcpu *vcpu)
482{ 514{
483 int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; 515 int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28;
@@ -486,6 +518,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
486 unsigned long mem = 0; 518 unsigned long mem = 0;
487 u64 operand2; 519 u64 operand2;
488 int rc = 0; 520 int rc = 0;
521 ar_t ar;
489 522
490 vcpu->stat.instruction_stsi++; 523 vcpu->stat.instruction_stsi++;
491 VCPU_EVENT(vcpu, 4, "stsi: fc: %x sel1: %x sel2: %x", fc, sel1, sel2); 524 VCPU_EVENT(vcpu, 4, "stsi: fc: %x sel1: %x sel2: %x", fc, sel1, sel2);
@@ -508,7 +541,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
508 return 0; 541 return 0;
509 } 542 }
510 543
511 operand2 = kvm_s390_get_base_disp_s(vcpu); 544 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
512 545
513 if (operand2 & 0xfff) 546 if (operand2 & 0xfff)
514 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 547 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -532,16 +565,20 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
532 break; 565 break;
533 } 566 }
534 567
535 rc = write_guest(vcpu, operand2, (void *)mem, PAGE_SIZE); 568 rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE);
536 if (rc) { 569 if (rc) {
537 rc = kvm_s390_inject_prog_cond(vcpu, rc); 570 rc = kvm_s390_inject_prog_cond(vcpu, rc);
538 goto out; 571 goto out;
539 } 572 }
573 if (vcpu->kvm->arch.user_stsi) {
574 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2);
575 rc = -EREMOTE;
576 }
540 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 577 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2);
541 free_page(mem); 578 free_page(mem);
542 kvm_s390_set_psw_cc(vcpu, 0); 579 kvm_s390_set_psw_cc(vcpu, 0);
543 vcpu->run->s.regs.gprs[0] = 0; 580 vcpu->run->s.regs.gprs[0] = 0;
544 return 0; 581 return rc;
545out_no_data: 582out_no_data:
546 kvm_s390_set_psw_cc(vcpu, 3); 583 kvm_s390_set_psw_cc(vcpu, 3);
547out: 584out:
@@ -670,7 +707,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
670 } 707 }
671 708
672 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 709 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
673 if (kvm_s390_check_low_addr_protection(vcpu, start)) 710 if (kvm_s390_check_low_addr_prot_real(vcpu, start))
674 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 711 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
675 } 712 }
676 713
@@ -776,13 +813,14 @@ int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
776 int reg, rc, nr_regs; 813 int reg, rc, nr_regs;
777 u32 ctl_array[16]; 814 u32 ctl_array[16];
778 u64 ga; 815 u64 ga;
816 ar_t ar;
779 817
780 vcpu->stat.instruction_lctl++; 818 vcpu->stat.instruction_lctl++;
781 819
782 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 820 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
783 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 821 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
784 822
785 ga = kvm_s390_get_base_disp_rs(vcpu); 823 ga = kvm_s390_get_base_disp_rs(vcpu, &ar);
786 824
787 if (ga & 3) 825 if (ga & 3)
788 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 826 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -791,7 +829,7 @@ int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
791 trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga); 829 trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga);
792 830
793 nr_regs = ((reg3 - reg1) & 0xf) + 1; 831 nr_regs = ((reg3 - reg1) & 0xf) + 1;
794 rc = read_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u32)); 832 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32));
795 if (rc) 833 if (rc)
796 return kvm_s390_inject_prog_cond(vcpu, rc); 834 return kvm_s390_inject_prog_cond(vcpu, rc);
797 reg = reg1; 835 reg = reg1;
@@ -814,13 +852,14 @@ int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu)
814 int reg, rc, nr_regs; 852 int reg, rc, nr_regs;
815 u32 ctl_array[16]; 853 u32 ctl_array[16];
816 u64 ga; 854 u64 ga;
855 ar_t ar;
817 856
818 vcpu->stat.instruction_stctl++; 857 vcpu->stat.instruction_stctl++;
819 858
820 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 859 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
821 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 860 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
822 861
823 ga = kvm_s390_get_base_disp_rs(vcpu); 862 ga = kvm_s390_get_base_disp_rs(vcpu, &ar);
824 863
825 if (ga & 3) 864 if (ga & 3)
826 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 865 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -836,7 +875,7 @@ int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu)
836 break; 875 break;
837 reg = (reg + 1) % 16; 876 reg = (reg + 1) % 16;
838 } while (1); 877 } while (1);
839 rc = write_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u32)); 878 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32));
840 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 879 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
841} 880}
842 881
@@ -847,13 +886,14 @@ static int handle_lctlg(struct kvm_vcpu *vcpu)
847 int reg, rc, nr_regs; 886 int reg, rc, nr_regs;
848 u64 ctl_array[16]; 887 u64 ctl_array[16];
849 u64 ga; 888 u64 ga;
889 ar_t ar;
850 890
851 vcpu->stat.instruction_lctlg++; 891 vcpu->stat.instruction_lctlg++;
852 892
853 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 893 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
854 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 894 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
855 895
856 ga = kvm_s390_get_base_disp_rsy(vcpu); 896 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar);
857 897
858 if (ga & 7) 898 if (ga & 7)
859 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 899 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -862,7 +902,7 @@ static int handle_lctlg(struct kvm_vcpu *vcpu)
862 trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga); 902 trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga);
863 903
864 nr_regs = ((reg3 - reg1) & 0xf) + 1; 904 nr_regs = ((reg3 - reg1) & 0xf) + 1;
865 rc = read_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u64)); 905 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64));
866 if (rc) 906 if (rc)
867 return kvm_s390_inject_prog_cond(vcpu, rc); 907 return kvm_s390_inject_prog_cond(vcpu, rc);
868 reg = reg1; 908 reg = reg1;
@@ -884,13 +924,14 @@ static int handle_stctg(struct kvm_vcpu *vcpu)
884 int reg, rc, nr_regs; 924 int reg, rc, nr_regs;
885 u64 ctl_array[16]; 925 u64 ctl_array[16];
886 u64 ga; 926 u64 ga;
927 ar_t ar;
887 928
888 vcpu->stat.instruction_stctg++; 929 vcpu->stat.instruction_stctg++;
889 930
890 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 931 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
891 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 932 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
892 933
893 ga = kvm_s390_get_base_disp_rsy(vcpu); 934 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar);
894 935
895 if (ga & 7) 936 if (ga & 7)
896 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 937 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -906,7 +947,7 @@ static int handle_stctg(struct kvm_vcpu *vcpu)
906 break; 947 break;
907 reg = (reg + 1) % 16; 948 reg = (reg + 1) % 16;
908 } while (1); 949 } while (1);
909 rc = write_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u64)); 950 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64));
910 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 951 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
911} 952}
912 953
@@ -931,13 +972,14 @@ static int handle_tprot(struct kvm_vcpu *vcpu)
931 unsigned long hva, gpa; 972 unsigned long hva, gpa;
932 int ret = 0, cc = 0; 973 int ret = 0, cc = 0;
933 bool writable; 974 bool writable;
975 ar_t ar;
934 976
935 vcpu->stat.instruction_tprot++; 977 vcpu->stat.instruction_tprot++;
936 978
937 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 979 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
938 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 980 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
939 981
940 kvm_s390_get_base_disp_sse(vcpu, &address1, &address2); 982 kvm_s390_get_base_disp_sse(vcpu, &address1, &address2, &ar, NULL);
941 983
942 /* we only handle the Linux memory detection case: 984 /* we only handle the Linux memory detection case:
943 * access key == 0 985 * access key == 0
@@ -946,11 +988,11 @@ static int handle_tprot(struct kvm_vcpu *vcpu)
946 return -EOPNOTSUPP; 988 return -EOPNOTSUPP;
947 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 989 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT)
948 ipte_lock(vcpu); 990 ipte_lock(vcpu);
949 ret = guest_translate_address(vcpu, address1, &gpa, 1); 991 ret = guest_translate_address(vcpu, address1, ar, &gpa, 1);
950 if (ret == PGM_PROTECTION) { 992 if (ret == PGM_PROTECTION) {
951 /* Write protected? Try again with read-only... */ 993 /* Write protected? Try again with read-only... */
952 cc = 1; 994 cc = 1;
953 ret = guest_translate_address(vcpu, address1, &gpa, 0); 995 ret = guest_translate_address(vcpu, address1, ar, &gpa, 0);
954 } 996 }
955 if (ret) { 997 if (ret) {
956 if (ret == PGM_ADDRESSING || ret == PGM_TRANSLATION_SPEC) { 998 if (ret == PGM_ADDRESSING || ret == PGM_TRANSLATION_SPEC) {
diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c
index 23b1e86b2122..72e58bd2bee7 100644
--- a/arch/s390/kvm/sigp.c
+++ b/arch/s390/kvm/sigp.c
@@ -393,6 +393,9 @@ static int handle_sigp_order_in_user_space(struct kvm_vcpu *vcpu, u8 order_code)
393 case SIGP_STORE_STATUS_AT_ADDRESS: 393 case SIGP_STORE_STATUS_AT_ADDRESS:
394 vcpu->stat.instruction_sigp_store_status++; 394 vcpu->stat.instruction_sigp_store_status++;
395 break; 395 break;
396 case SIGP_STORE_ADDITIONAL_STATUS:
397 vcpu->stat.instruction_sigp_store_adtl_status++;
398 break;
396 case SIGP_SET_PREFIX: 399 case SIGP_SET_PREFIX:
397 vcpu->stat.instruction_sigp_prefix++; 400 vcpu->stat.instruction_sigp_prefix++;
398 break; 401 break;
@@ -431,7 +434,7 @@ int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu)
431 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 434 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
432 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 435 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
433 436
434 order_code = kvm_s390_get_base_disp_rs(vcpu); 437 order_code = kvm_s390_get_base_disp_rs(vcpu, NULL);
435 if (handle_sigp_order_in_user_space(vcpu, order_code)) 438 if (handle_sigp_order_in_user_space(vcpu, order_code))
436 return -EOPNOTSUPP; 439 return -EOPNOTSUPP;
437 440
@@ -473,7 +476,7 @@ int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu)
473 int r3 = vcpu->arch.sie_block->ipa & 0x000f; 476 int r3 = vcpu->arch.sie_block->ipa & 0x000f;
474 u16 cpu_addr = vcpu->run->s.regs.gprs[r3]; 477 u16 cpu_addr = vcpu->run->s.regs.gprs[r3];
475 struct kvm_vcpu *dest_vcpu; 478 struct kvm_vcpu *dest_vcpu;
476 u8 order_code = kvm_s390_get_base_disp_rs(vcpu); 479 u8 order_code = kvm_s390_get_base_disp_rs(vcpu, NULL);
477 480
478 trace_kvm_s390_handle_sigp_pei(vcpu, order_code, cpu_addr); 481 trace_kvm_s390_handle_sigp_pei(vcpu, order_code, cpu_addr);
479 482
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index a236e39cc385..dea2e7e962e3 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -81,11 +81,6 @@ static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
81 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); 81 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
82} 82}
83 83
84#define SELECTOR_TI_MASK (1 << 2)
85#define SELECTOR_RPL_MASK 0x03
86
87#define IOPL_SHIFT 12
88
89#define KVM_PERMILLE_MMU_PAGES 20 84#define KVM_PERMILLE_MMU_PAGES 20
90#define KVM_MIN_ALLOC_MMU_PAGES 64 85#define KVM_MIN_ALLOC_MMU_PAGES 64
91#define KVM_MMU_HASH_SHIFT 10 86#define KVM_MMU_HASH_SHIFT 10
@@ -345,6 +340,7 @@ struct kvm_pmu {
345enum { 340enum {
346 KVM_DEBUGREG_BP_ENABLED = 1, 341 KVM_DEBUGREG_BP_ENABLED = 1,
347 KVM_DEBUGREG_WONT_EXIT = 2, 342 KVM_DEBUGREG_WONT_EXIT = 2,
343 KVM_DEBUGREG_RELOAD = 4,
348}; 344};
349 345
350struct kvm_vcpu_arch { 346struct kvm_vcpu_arch {
@@ -431,6 +427,9 @@ struct kvm_vcpu_arch {
431 427
432 int cpuid_nent; 428 int cpuid_nent;
433 struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; 429 struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES];
430
431 int maxphyaddr;
432
434 /* emulate context */ 433 /* emulate context */
435 434
436 struct x86_emulate_ctxt emulate_ctxt; 435 struct x86_emulate_ctxt emulate_ctxt;
@@ -550,11 +549,20 @@ struct kvm_arch_memory_slot {
550 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; 549 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
551}; 550};
552 551
552/*
553 * We use as the mode the number of bits allocated in the LDR for the
554 * logical processor ID. It happens that these are all powers of two.
555 * This makes it is very easy to detect cases where the APICs are
556 * configured for multiple modes; in that case, we cannot use the map and
557 * hence cannot use kvm_irq_delivery_to_apic_fast either.
558 */
559#define KVM_APIC_MODE_XAPIC_CLUSTER 4
560#define KVM_APIC_MODE_XAPIC_FLAT 8
561#define KVM_APIC_MODE_X2APIC 16
562
553struct kvm_apic_map { 563struct kvm_apic_map {
554 struct rcu_head rcu; 564 struct rcu_head rcu;
555 u8 ldr_bits; 565 u8 mode;
556 /* fields bellow are used to decode ldr values in different modes */
557 u32 cid_shift, cid_mask, lid_mask, broadcast;
558 struct kvm_lapic *phys_map[256]; 566 struct kvm_lapic *phys_map[256];
559 /* first index is cluster id second is cpu id in a cluster */ 567 /* first index is cluster id second is cpu id in a cluster */
560 struct kvm_lapic *logical_map[16][16]; 568 struct kvm_lapic *logical_map[16][16];
@@ -859,6 +867,8 @@ void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
859void kvm_mmu_reset_context(struct kvm_vcpu *vcpu); 867void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
860void kvm_mmu_slot_remove_write_access(struct kvm *kvm, 868void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
861 struct kvm_memory_slot *memslot); 869 struct kvm_memory_slot *memslot);
870void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
871 struct kvm_memory_slot *memslot);
862void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, 872void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
863 struct kvm_memory_slot *memslot); 873 struct kvm_memory_slot *memslot);
864void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, 874void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
@@ -933,6 +943,7 @@ struct x86_emulate_ctxt;
933int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port); 943int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port);
934void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); 944void kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
935int kvm_emulate_halt(struct kvm_vcpu *vcpu); 945int kvm_emulate_halt(struct kvm_vcpu *vcpu);
946int kvm_vcpu_halt(struct kvm_vcpu *vcpu);
936int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); 947int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
937 948
938void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); 949void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
@@ -1128,7 +1139,6 @@ int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1128int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); 1139int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
1129int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); 1140int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
1130void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); 1141void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
1131int cpuid_maxphyaddr(struct kvm_vcpu *vcpu);
1132int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v); 1142int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
1133int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu); 1143int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
1134int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu); 1144int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index e62cf897f781..c1adf33fdd0d 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -115,7 +115,7 @@ static inline void kvm_spinlock_init(void)
115 115
116static inline bool kvm_para_available(void) 116static inline bool kvm_para_available(void)
117{ 117{
118 return 0; 118 return false;
119} 119}
120 120
121static inline unsigned int kvm_arch_para_features(void) 121static inline unsigned int kvm_arch_para_features(void)
diff --git a/arch/x86/include/asm/pvclock.h b/arch/x86/include/asm/pvclock.h
index d6b078e9fa28..25b1cc07d496 100644
--- a/arch/x86/include/asm/pvclock.h
+++ b/arch/x86/include/asm/pvclock.h
@@ -95,6 +95,7 @@ unsigned __pvclock_read_cycles(const struct pvclock_vcpu_time_info *src,
95 95
96struct pvclock_vsyscall_time_info { 96struct pvclock_vsyscall_time_info {
97 struct pvclock_vcpu_time_info pvti; 97 struct pvclock_vcpu_time_info pvti;
98 u32 migrate_count;
98} __attribute__((__aligned__(SMP_CACHE_BYTES))); 99} __attribute__((__aligned__(SMP_CACHE_BYTES)));
99 100
100#define PVTI_SIZE sizeof(struct pvclock_vsyscall_time_info) 101#define PVTI_SIZE sizeof(struct pvclock_vsyscall_time_info)
diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h
index c5f1a1deb91a..1fe92181ee9e 100644
--- a/arch/x86/include/uapi/asm/vmx.h
+++ b/arch/x86/include/uapi/asm/vmx.h
@@ -67,6 +67,7 @@
67#define EXIT_REASON_EPT_VIOLATION 48 67#define EXIT_REASON_EPT_VIOLATION 48
68#define EXIT_REASON_EPT_MISCONFIG 49 68#define EXIT_REASON_EPT_MISCONFIG 49
69#define EXIT_REASON_INVEPT 50 69#define EXIT_REASON_INVEPT 50
70#define EXIT_REASON_RDTSCP 51
70#define EXIT_REASON_PREEMPTION_TIMER 52 71#define EXIT_REASON_PREEMPTION_TIMER 52
71#define EXIT_REASON_INVVPID 53 72#define EXIT_REASON_INVVPID 53
72#define EXIT_REASON_WBINVD 54 73#define EXIT_REASON_WBINVD 54
diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c
index 2f355d229a58..e5ecd20e72dd 100644
--- a/arch/x86/kernel/pvclock.c
+++ b/arch/x86/kernel/pvclock.c
@@ -141,7 +141,46 @@ void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock,
141 set_normalized_timespec(ts, now.tv_sec, now.tv_nsec); 141 set_normalized_timespec(ts, now.tv_sec, now.tv_nsec);
142} 142}
143 143
144static struct pvclock_vsyscall_time_info *pvclock_vdso_info;
145
146static struct pvclock_vsyscall_time_info *
147pvclock_get_vsyscall_user_time_info(int cpu)
148{
149 if (!pvclock_vdso_info) {
150 BUG();
151 return NULL;
152 }
153
154 return &pvclock_vdso_info[cpu];
155}
156
157struct pvclock_vcpu_time_info *pvclock_get_vsyscall_time_info(int cpu)
158{
159 return &pvclock_get_vsyscall_user_time_info(cpu)->pvti;
160}
161
144#ifdef CONFIG_X86_64 162#ifdef CONFIG_X86_64
163static int pvclock_task_migrate(struct notifier_block *nb, unsigned long l,
164 void *v)
165{
166 struct task_migration_notifier *mn = v;
167 struct pvclock_vsyscall_time_info *pvti;
168
169 pvti = pvclock_get_vsyscall_user_time_info(mn->from_cpu);
170
171 /* this is NULL when pvclock vsyscall is not initialized */
172 if (unlikely(pvti == NULL))
173 return NOTIFY_DONE;
174
175 pvti->migrate_count++;
176
177 return NOTIFY_DONE;
178}
179
180static struct notifier_block pvclock_migrate = {
181 .notifier_call = pvclock_task_migrate,
182};
183
145/* 184/*
146 * Initialize the generic pvclock vsyscall state. This will allocate 185 * Initialize the generic pvclock vsyscall state. This will allocate
147 * a/some page(s) for the per-vcpu pvclock information, set up a 186 * a/some page(s) for the per-vcpu pvclock information, set up a
@@ -155,12 +194,17 @@ int __init pvclock_init_vsyscall(struct pvclock_vsyscall_time_info *i,
155 194
156 WARN_ON (size != PVCLOCK_VSYSCALL_NR_PAGES*PAGE_SIZE); 195 WARN_ON (size != PVCLOCK_VSYSCALL_NR_PAGES*PAGE_SIZE);
157 196
197 pvclock_vdso_info = i;
198
158 for (idx = 0; idx <= (PVCLOCK_FIXMAP_END-PVCLOCK_FIXMAP_BEGIN); idx++) { 199 for (idx = 0; idx <= (PVCLOCK_FIXMAP_END-PVCLOCK_FIXMAP_BEGIN); idx++) {
159 __set_fixmap(PVCLOCK_FIXMAP_BEGIN + idx, 200 __set_fixmap(PVCLOCK_FIXMAP_BEGIN + idx,
160 __pa(i) + (idx*PAGE_SIZE), 201 __pa(i) + (idx*PAGE_SIZE),
161 PAGE_KERNEL_VVAR); 202 PAGE_KERNEL_VVAR);
162 } 203 }
163 204
205
206 register_task_migration_notifier(&pvclock_migrate);
207
164 return 0; 208 return 0;
165} 209}
166#endif 210#endif
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 08f790dfadc9..16e8f962eaad 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -1,5 +1,5 @@
1 1
2ccflags-y += -Ivirt/kvm -Iarch/x86/kvm 2ccflags-y += -Iarch/x86/kvm
3 3
4CFLAGS_x86.o := -I. 4CFLAGS_x86.o := -I.
5CFLAGS_svm.o := -I. 5CFLAGS_svm.o := -I.
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 8a80737ee6e6..59b69f6a2844 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -104,6 +104,9 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu)
104 ((best->eax & 0xff00) >> 8) != 0) 104 ((best->eax & 0xff00) >> 8) != 0)
105 return -EINVAL; 105 return -EINVAL;
106 106
107 /* Update physical-address width */
108 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
109
107 kvm_pmu_cpuid_update(vcpu); 110 kvm_pmu_cpuid_update(vcpu);
108 return 0; 111 return 0;
109} 112}
@@ -135,6 +138,21 @@ static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
135 } 138 }
136} 139}
137 140
141int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu)
142{
143 struct kvm_cpuid_entry2 *best;
144
145 best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
146 if (!best || best->eax < 0x80000008)
147 goto not_found;
148 best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
149 if (best)
150 return best->eax & 0xff;
151not_found:
152 return 36;
153}
154EXPORT_SYMBOL_GPL(cpuid_query_maxphyaddr);
155
138/* when an old userspace process fills a new kernel module */ 156/* when an old userspace process fills a new kernel module */
139int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, 157int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
140 struct kvm_cpuid *cpuid, 158 struct kvm_cpuid *cpuid,
@@ -757,21 +775,6 @@ struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
757} 775}
758EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry); 776EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
759 777
760int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
761{
762 struct kvm_cpuid_entry2 *best;
763
764 best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
765 if (!best || best->eax < 0x80000008)
766 goto not_found;
767 best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
768 if (best)
769 return best->eax & 0xff;
770not_found:
771 return 36;
772}
773EXPORT_SYMBOL_GPL(cpuid_maxphyaddr);
774
775/* 778/*
776 * If no match is found, check whether we exceed the vCPU's limit 779 * If no match is found, check whether we exceed the vCPU's limit
777 * and return the content of the highest valid _standard_ leaf instead. 780 * and return the content of the highest valid _standard_ leaf instead.
diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h
index 4452eedfaedd..c3b1ad9fca81 100644
--- a/arch/x86/kvm/cpuid.h
+++ b/arch/x86/kvm/cpuid.h
@@ -20,13 +20,19 @@ int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
20 struct kvm_cpuid_entry2 __user *entries); 20 struct kvm_cpuid_entry2 __user *entries);
21void kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx); 21void kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx);
22 22
23int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu);
24
25static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
26{
27 return vcpu->arch.maxphyaddr;
28}
23 29
24static inline bool guest_cpuid_has_xsave(struct kvm_vcpu *vcpu) 30static inline bool guest_cpuid_has_xsave(struct kvm_vcpu *vcpu)
25{ 31{
26 struct kvm_cpuid_entry2 *best; 32 struct kvm_cpuid_entry2 *best;
27 33
28 if (!static_cpu_has(X86_FEATURE_XSAVE)) 34 if (!static_cpu_has(X86_FEATURE_XSAVE))
29 return 0; 35 return false;
30 36
31 best = kvm_find_cpuid_entry(vcpu, 1, 0); 37 best = kvm_find_cpuid_entry(vcpu, 1, 0);
32 return best && (best->ecx & bit(X86_FEATURE_XSAVE)); 38 return best && (best->ecx & bit(X86_FEATURE_XSAVE));
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index 106c01557f2b..630bcb0d7a04 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -248,27 +248,7 @@ struct mode_dual {
248 struct opcode mode64; 248 struct opcode mode64;
249}; 249};
250 250
251/* EFLAGS bit definitions. */
252#define EFLG_ID (1<<21)
253#define EFLG_VIP (1<<20)
254#define EFLG_VIF (1<<19)
255#define EFLG_AC (1<<18)
256#define EFLG_VM (1<<17)
257#define EFLG_RF (1<<16)
258#define EFLG_IOPL (3<<12)
259#define EFLG_NT (1<<14)
260#define EFLG_OF (1<<11)
261#define EFLG_DF (1<<10)
262#define EFLG_IF (1<<9)
263#define EFLG_TF (1<<8)
264#define EFLG_SF (1<<7)
265#define EFLG_ZF (1<<6)
266#define EFLG_AF (1<<4)
267#define EFLG_PF (1<<2)
268#define EFLG_CF (1<<0)
269
270#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a 251#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
271#define EFLG_RESERVED_ONE_MASK 2
272 252
273enum x86_transfer_type { 253enum x86_transfer_type {
274 X86_TRANSFER_NONE, 254 X86_TRANSFER_NONE,
@@ -317,7 +297,8 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
317 * These EFLAGS bits are restored from saved value during emulation, and 297 * These EFLAGS bits are restored from saved value during emulation, and
318 * any changes are written back to the saved value after emulation. 298 * any changes are written back to the saved value after emulation.
319 */ 299 */
320#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF) 300#define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
301 X86_EFLAGS_PF|X86_EFLAGS_CF)
321 302
322#ifdef CONFIG_X86_64 303#ifdef CONFIG_X86_64
323#define ON64(x) x 304#define ON64(x) x
@@ -478,6 +459,25 @@ static void assign_masked(ulong *dest, ulong src, ulong mask)
478 *dest = (*dest & ~mask) | (src & mask); 459 *dest = (*dest & ~mask) | (src & mask);
479} 460}
480 461
462static void assign_register(unsigned long *reg, u64 val, int bytes)
463{
464 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
465 switch (bytes) {
466 case 1:
467 *(u8 *)reg = (u8)val;
468 break;
469 case 2:
470 *(u16 *)reg = (u16)val;
471 break;
472 case 4:
473 *reg = (u32)val;
474 break; /* 64b: zero-extend */
475 case 8:
476 *reg = val;
477 break;
478 }
479}
480
481static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt) 481static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
482{ 482{
483 return (1UL << (ctxt->ad_bytes << 3)) - 1; 483 return (1UL << (ctxt->ad_bytes << 3)) - 1;
@@ -943,6 +943,22 @@ FASTOP2(xadd);
943 943
944FASTOP2R(cmp, cmp_r); 944FASTOP2R(cmp, cmp_r);
945 945
946static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
947{
948 /* If src is zero, do not writeback, but update flags */
949 if (ctxt->src.val == 0)
950 ctxt->dst.type = OP_NONE;
951 return fastop(ctxt, em_bsf);
952}
953
954static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
955{
956 /* If src is zero, do not writeback, but update flags */
957 if (ctxt->src.val == 0)
958 ctxt->dst.type = OP_NONE;
959 return fastop(ctxt, em_bsr);
960}
961
946static u8 test_cc(unsigned int condition, unsigned long flags) 962static u8 test_cc(unsigned int condition, unsigned long flags)
947{ 963{
948 u8 rc; 964 u8 rc;
@@ -1399,7 +1415,7 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1399 unsigned int in_page, n; 1415 unsigned int in_page, n;
1400 unsigned int count = ctxt->rep_prefix ? 1416 unsigned int count = ctxt->rep_prefix ?
1401 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1; 1417 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1402 in_page = (ctxt->eflags & EFLG_DF) ? 1418 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1403 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) : 1419 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1404 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)); 1420 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1405 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count); 1421 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
@@ -1412,7 +1428,7 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1412 } 1428 }
1413 1429
1414 if (ctxt->rep_prefix && (ctxt->d & String) && 1430 if (ctxt->rep_prefix && (ctxt->d & String) &&
1415 !(ctxt->eflags & EFLG_DF)) { 1431 !(ctxt->eflags & X86_EFLAGS_DF)) {
1416 ctxt->dst.data = rc->data + rc->pos; 1432 ctxt->dst.data = rc->data + rc->pos;
1417 ctxt->dst.type = OP_MEM_STR; 1433 ctxt->dst.type = OP_MEM_STR;
1418 ctxt->dst.count = (rc->end - rc->pos) / size; 1434 ctxt->dst.count = (rc->end - rc->pos) / size;
@@ -1691,21 +1707,7 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1691 1707
1692static void write_register_operand(struct operand *op) 1708static void write_register_operand(struct operand *op)
1693{ 1709{
1694 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ 1710 return assign_register(op->addr.reg, op->val, op->bytes);
1695 switch (op->bytes) {
1696 case 1:
1697 *(u8 *)op->addr.reg = (u8)op->val;
1698 break;
1699 case 2:
1700 *(u16 *)op->addr.reg = (u16)op->val;
1701 break;
1702 case 4:
1703 *op->addr.reg = (u32)op->val;
1704 break; /* 64b: zero-extend */
1705 case 8:
1706 *op->addr.reg = op->val;
1707 break;
1708 }
1709} 1711}
1710 1712
1711static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op) 1713static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
@@ -1792,32 +1794,34 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1792{ 1794{
1793 int rc; 1795 int rc;
1794 unsigned long val, change_mask; 1796 unsigned long val, change_mask;
1795 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; 1797 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1796 int cpl = ctxt->ops->cpl(ctxt); 1798 int cpl = ctxt->ops->cpl(ctxt);
1797 1799
1798 rc = emulate_pop(ctxt, &val, len); 1800 rc = emulate_pop(ctxt, &val, len);
1799 if (rc != X86EMUL_CONTINUE) 1801 if (rc != X86EMUL_CONTINUE)
1800 return rc; 1802 return rc;
1801 1803
1802 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF 1804 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1803 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID; 1805 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1806 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1807 X86_EFLAGS_AC | X86_EFLAGS_ID;
1804 1808
1805 switch(ctxt->mode) { 1809 switch(ctxt->mode) {
1806 case X86EMUL_MODE_PROT64: 1810 case X86EMUL_MODE_PROT64:
1807 case X86EMUL_MODE_PROT32: 1811 case X86EMUL_MODE_PROT32:
1808 case X86EMUL_MODE_PROT16: 1812 case X86EMUL_MODE_PROT16:
1809 if (cpl == 0) 1813 if (cpl == 0)
1810 change_mask |= EFLG_IOPL; 1814 change_mask |= X86_EFLAGS_IOPL;
1811 if (cpl <= iopl) 1815 if (cpl <= iopl)
1812 change_mask |= EFLG_IF; 1816 change_mask |= X86_EFLAGS_IF;
1813 break; 1817 break;
1814 case X86EMUL_MODE_VM86: 1818 case X86EMUL_MODE_VM86:
1815 if (iopl < 3) 1819 if (iopl < 3)
1816 return emulate_gp(ctxt, 0); 1820 return emulate_gp(ctxt, 0);
1817 change_mask |= EFLG_IF; 1821 change_mask |= X86_EFLAGS_IF;
1818 break; 1822 break;
1819 default: /* real mode */ 1823 default: /* real mode */
1820 change_mask |= (EFLG_IOPL | EFLG_IF); 1824 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1821 break; 1825 break;
1822 } 1826 }
1823 1827
@@ -1918,7 +1922,7 @@ static int em_pusha(struct x86_emulate_ctxt *ctxt)
1918 1922
1919static int em_pushf(struct x86_emulate_ctxt *ctxt) 1923static int em_pushf(struct x86_emulate_ctxt *ctxt)
1920{ 1924{
1921 ctxt->src.val = (unsigned long)ctxt->eflags & ~EFLG_VM; 1925 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1922 return em_push(ctxt); 1926 return em_push(ctxt);
1923} 1927}
1924 1928
@@ -1926,6 +1930,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
1926{ 1930{
1927 int rc = X86EMUL_CONTINUE; 1931 int rc = X86EMUL_CONTINUE;
1928 int reg = VCPU_REGS_RDI; 1932 int reg = VCPU_REGS_RDI;
1933 u32 val;
1929 1934
1930 while (reg >= VCPU_REGS_RAX) { 1935 while (reg >= VCPU_REGS_RAX) {
1931 if (reg == VCPU_REGS_RSP) { 1936 if (reg == VCPU_REGS_RSP) {
@@ -1933,9 +1938,10 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
1933 --reg; 1938 --reg;
1934 } 1939 }
1935 1940
1936 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes); 1941 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1937 if (rc != X86EMUL_CONTINUE) 1942 if (rc != X86EMUL_CONTINUE)
1938 break; 1943 break;
1944 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1939 --reg; 1945 --reg;
1940 } 1946 }
1941 return rc; 1947 return rc;
@@ -1956,7 +1962,7 @@ static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1956 if (rc != X86EMUL_CONTINUE) 1962 if (rc != X86EMUL_CONTINUE)
1957 return rc; 1963 return rc;
1958 1964
1959 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC); 1965 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
1960 1966
1961 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS); 1967 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1962 rc = em_push(ctxt); 1968 rc = em_push(ctxt);
@@ -2022,10 +2028,14 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2022 unsigned long temp_eip = 0; 2028 unsigned long temp_eip = 0;
2023 unsigned long temp_eflags = 0; 2029 unsigned long temp_eflags = 0;
2024 unsigned long cs = 0; 2030 unsigned long cs = 0;
2025 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF | 2031 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2026 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF | 2032 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2027 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */ 2033 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2028 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP; 2034 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2035 X86_EFLAGS_AC | X86_EFLAGS_ID |
2036 X86_EFLAGS_FIXED;
2037 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2038 X86_EFLAGS_VIP;
2029 2039
2030 /* TODO: Add stack limit check */ 2040 /* TODO: Add stack limit check */
2031 2041
@@ -2054,7 +2064,6 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2054 2064
2055 ctxt->_eip = temp_eip; 2065 ctxt->_eip = temp_eip;
2056 2066
2057
2058 if (ctxt->op_bytes == 4) 2067 if (ctxt->op_bytes == 4)
2059 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask)); 2068 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2060 else if (ctxt->op_bytes == 2) { 2069 else if (ctxt->op_bytes == 2) {
@@ -2063,7 +2072,7 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2063 } 2072 }
2064 2073
2065 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */ 2074 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2066 ctxt->eflags |= EFLG_RESERVED_ONE_MASK; 2075 ctxt->eflags |= X86_EFLAGS_FIXED;
2067 ctxt->ops->set_nmi_mask(ctxt, false); 2076 ctxt->ops->set_nmi_mask(ctxt, false);
2068 2077
2069 return rc; 2078 return rc;
@@ -2145,12 +2154,12 @@ static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2145 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) { 2154 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2146 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0); 2155 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2147 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32); 2156 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2148 ctxt->eflags &= ~EFLG_ZF; 2157 ctxt->eflags &= ~X86_EFLAGS_ZF;
2149 } else { 2158 } else {
2150 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) | 2159 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2151 (u32) reg_read(ctxt, VCPU_REGS_RBX); 2160 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2152 2161
2153 ctxt->eflags |= EFLG_ZF; 2162 ctxt->eflags |= X86_EFLAGS_ZF;
2154 } 2163 }
2155 return X86EMUL_CONTINUE; 2164 return X86EMUL_CONTINUE;
2156} 2165}
@@ -2222,7 +2231,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2222 ctxt->src.val = ctxt->dst.orig_val; 2231 ctxt->src.val = ctxt->dst.orig_val;
2223 fastop(ctxt, em_cmp); 2232 fastop(ctxt, em_cmp);
2224 2233
2225 if (ctxt->eflags & EFLG_ZF) { 2234 if (ctxt->eflags & X86_EFLAGS_ZF) {
2226 /* Success: write back to memory; no update of EAX */ 2235 /* Success: write back to memory; no update of EAX */
2227 ctxt->src.type = OP_NONE; 2236 ctxt->src.type = OP_NONE;
2228 ctxt->dst.val = ctxt->src.orig_val; 2237 ctxt->dst.val = ctxt->src.orig_val;
@@ -2381,14 +2390,14 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt)
2381 2390
2382 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data); 2391 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2383 ctxt->eflags &= ~msr_data; 2392 ctxt->eflags &= ~msr_data;
2384 ctxt->eflags |= EFLG_RESERVED_ONE_MASK; 2393 ctxt->eflags |= X86_EFLAGS_FIXED;
2385#endif 2394#endif
2386 } else { 2395 } else {
2387 /* legacy mode */ 2396 /* legacy mode */
2388 ops->get_msr(ctxt, MSR_STAR, &msr_data); 2397 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2389 ctxt->_eip = (u32)msr_data; 2398 ctxt->_eip = (u32)msr_data;
2390 2399
2391 ctxt->eflags &= ~(EFLG_VM | EFLG_IF); 2400 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2392 } 2401 }
2393 2402
2394 return X86EMUL_CONTINUE; 2403 return X86EMUL_CONTINUE;
@@ -2425,8 +2434,8 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2425 if ((msr_data & 0xfffc) == 0x0) 2434 if ((msr_data & 0xfffc) == 0x0)
2426 return emulate_gp(ctxt, 0); 2435 return emulate_gp(ctxt, 0);
2427 2436
2428 ctxt->eflags &= ~(EFLG_VM | EFLG_IF); 2437 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2429 cs_sel = (u16)msr_data & ~SELECTOR_RPL_MASK; 2438 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2430 ss_sel = cs_sel + 8; 2439 ss_sel = cs_sel + 8;
2431 if (efer & EFER_LMA) { 2440 if (efer & EFER_LMA) {
2432 cs.d = 0; 2441 cs.d = 0;
@@ -2493,8 +2502,8 @@ static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2493 return emulate_gp(ctxt, 0); 2502 return emulate_gp(ctxt, 0);
2494 break; 2503 break;
2495 } 2504 }
2496 cs_sel |= SELECTOR_RPL_MASK; 2505 cs_sel |= SEGMENT_RPL_MASK;
2497 ss_sel |= SELECTOR_RPL_MASK; 2506 ss_sel |= SEGMENT_RPL_MASK;
2498 2507
2499 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); 2508 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2500 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); 2509 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
@@ -2512,7 +2521,7 @@ static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2512 return false; 2521 return false;
2513 if (ctxt->mode == X86EMUL_MODE_VM86) 2522 if (ctxt->mode == X86EMUL_MODE_VM86)
2514 return true; 2523 return true;
2515 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; 2524 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2516 return ctxt->ops->cpl(ctxt) > iopl; 2525 return ctxt->ops->cpl(ctxt) > iopl;
2517} 2526}
2518 2527
@@ -2782,10 +2791,8 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2782 return ret; 2791 return ret;
2783 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, 2792 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2784 X86_TRANSFER_TASK_SWITCH, NULL); 2793 X86_TRANSFER_TASK_SWITCH, NULL);
2785 if (ret != X86EMUL_CONTINUE)
2786 return ret;
2787 2794
2788 return X86EMUL_CONTINUE; 2795 return ret;
2789} 2796}
2790 2797
2791static int task_switch_32(struct x86_emulate_ctxt *ctxt, 2798static int task_switch_32(struct x86_emulate_ctxt *ctxt,
@@ -2954,7 +2961,7 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2954static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg, 2961static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2955 struct operand *op) 2962 struct operand *op)
2956{ 2963{
2957 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count; 2964 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
2958 2965
2959 register_address_increment(ctxt, reg, df * op->bytes); 2966 register_address_increment(ctxt, reg, df * op->bytes);
2960 op->addr.mem.ea = register_address(ctxt, reg); 2967 op->addr.mem.ea = register_address(ctxt, reg);
@@ -3323,7 +3330,7 @@ static int em_clts(struct x86_emulate_ctxt *ctxt)
3323 return X86EMUL_CONTINUE; 3330 return X86EMUL_CONTINUE;
3324} 3331}
3325 3332
3326static int em_vmcall(struct x86_emulate_ctxt *ctxt) 3333static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3327{ 3334{
3328 int rc = ctxt->ops->fix_hypercall(ctxt); 3335 int rc = ctxt->ops->fix_hypercall(ctxt);
3329 3336
@@ -3395,17 +3402,6 @@ static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3395 return em_lgdt_lidt(ctxt, true); 3402 return em_lgdt_lidt(ctxt, true);
3396} 3403}
3397 3404
3398static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3399{
3400 int rc;
3401
3402 rc = ctxt->ops->fix_hypercall(ctxt);
3403
3404 /* Disable writeback. */
3405 ctxt->dst.type = OP_NONE;
3406 return rc;
3407}
3408
3409static int em_lidt(struct x86_emulate_ctxt *ctxt) 3405static int em_lidt(struct x86_emulate_ctxt *ctxt)
3410{ 3406{
3411 return em_lgdt_lidt(ctxt, false); 3407 return em_lgdt_lidt(ctxt, false);
@@ -3504,7 +3500,8 @@ static int em_sahf(struct x86_emulate_ctxt *ctxt)
3504{ 3500{
3505 u32 flags; 3501 u32 flags;
3506 3502
3507 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF; 3503 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3504 X86_EFLAGS_SF;
3508 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8; 3505 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3509 3506
3510 ctxt->eflags &= ~0xffUL; 3507 ctxt->eflags &= ~0xffUL;
@@ -3769,7 +3766,7 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3769 3766
3770static const struct opcode group7_rm0[] = { 3767static const struct opcode group7_rm0[] = {
3771 N, 3768 N,
3772 I(SrcNone | Priv | EmulateOnUD, em_vmcall), 3769 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
3773 N, N, N, N, N, N, 3770 N, N, N, N, N, N,
3774}; 3771};
3775 3772
@@ -3781,7 +3778,7 @@ static const struct opcode group7_rm1[] = {
3781 3778
3782static const struct opcode group7_rm3[] = { 3779static const struct opcode group7_rm3[] = {
3783 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa), 3780 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3784 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall), 3781 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
3785 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa), 3782 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3786 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa), 3783 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3787 DIP(SrcNone | Prot | Priv, stgi, check_svme), 3784 DIP(SrcNone | Prot | Priv, stgi, check_svme),
@@ -4192,7 +4189,8 @@ static const struct opcode twobyte_table[256] = {
4192 N, N, 4189 N, N,
4193 G(BitOp, group8), 4190 G(BitOp, group8),
4194 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc), 4191 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4195 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr), 4192 I(DstReg | SrcMem | ModRM, em_bsf_c),
4193 I(DstReg | SrcMem | ModRM, em_bsr_c),
4196 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov), 4194 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4197 /* 0xC0 - 0xC7 */ 4195 /* 0xC0 - 0xC7 */
4198 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd), 4196 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
@@ -4759,9 +4757,9 @@ static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4759 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) || 4757 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4760 (ctxt->b == 0xae) || (ctxt->b == 0xaf)) 4758 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4761 && (((ctxt->rep_prefix == REPE_PREFIX) && 4759 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4762 ((ctxt->eflags & EFLG_ZF) == 0)) 4760 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
4763 || ((ctxt->rep_prefix == REPNE_PREFIX) && 4761 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4764 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)))) 4762 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
4765 return true; 4763 return true;
4766 4764
4767 return false; 4765 return false;
@@ -4913,7 +4911,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4913 /* All REP prefixes have the same first termination condition */ 4911 /* All REP prefixes have the same first termination condition */
4914 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) { 4912 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4915 ctxt->eip = ctxt->_eip; 4913 ctxt->eip = ctxt->_eip;
4916 ctxt->eflags &= ~EFLG_RF; 4914 ctxt->eflags &= ~X86_EFLAGS_RF;
4917 goto done; 4915 goto done;
4918 } 4916 }
4919 } 4917 }
@@ -4963,9 +4961,9 @@ special_insn:
4963 } 4961 }
4964 4962
4965 if (ctxt->rep_prefix && (ctxt->d & String)) 4963 if (ctxt->rep_prefix && (ctxt->d & String))
4966 ctxt->eflags |= EFLG_RF; 4964 ctxt->eflags |= X86_EFLAGS_RF;
4967 else 4965 else
4968 ctxt->eflags &= ~EFLG_RF; 4966 ctxt->eflags &= ~X86_EFLAGS_RF;
4969 4967
4970 if (ctxt->execute) { 4968 if (ctxt->execute) {
4971 if (ctxt->d & Fastop) { 4969 if (ctxt->d & Fastop) {
@@ -5014,7 +5012,7 @@ special_insn:
5014 rc = emulate_int(ctxt, ctxt->src.val); 5012 rc = emulate_int(ctxt, ctxt->src.val);
5015 break; 5013 break;
5016 case 0xce: /* into */ 5014 case 0xce: /* into */
5017 if (ctxt->eflags & EFLG_OF) 5015 if (ctxt->eflags & X86_EFLAGS_OF)
5018 rc = emulate_int(ctxt, 4); 5016 rc = emulate_int(ctxt, 4);
5019 break; 5017 break;
5020 case 0xe9: /* jmp rel */ 5018 case 0xe9: /* jmp rel */
@@ -5027,19 +5025,19 @@ special_insn:
5027 break; 5025 break;
5028 case 0xf5: /* cmc */ 5026 case 0xf5: /* cmc */
5029 /* complement carry flag from eflags reg */ 5027 /* complement carry flag from eflags reg */
5030 ctxt->eflags ^= EFLG_CF; 5028 ctxt->eflags ^= X86_EFLAGS_CF;
5031 break; 5029 break;
5032 case 0xf8: /* clc */ 5030 case 0xf8: /* clc */
5033 ctxt->eflags &= ~EFLG_CF; 5031 ctxt->eflags &= ~X86_EFLAGS_CF;
5034 break; 5032 break;
5035 case 0xf9: /* stc */ 5033 case 0xf9: /* stc */
5036 ctxt->eflags |= EFLG_CF; 5034 ctxt->eflags |= X86_EFLAGS_CF;
5037 break; 5035 break;
5038 case 0xfc: /* cld */ 5036 case 0xfc: /* cld */
5039 ctxt->eflags &= ~EFLG_DF; 5037 ctxt->eflags &= ~X86_EFLAGS_DF;
5040 break; 5038 break;
5041 case 0xfd: /* std */ 5039 case 0xfd: /* std */
5042 ctxt->eflags |= EFLG_DF; 5040 ctxt->eflags |= X86_EFLAGS_DF;
5043 break; 5041 break;
5044 default: 5042 default:
5045 goto cannot_emulate; 5043 goto cannot_emulate;
@@ -5100,7 +5098,7 @@ writeback:
5100 } 5098 }
5101 goto done; /* skip rip writeback */ 5099 goto done; /* skip rip writeback */
5102 } 5100 }
5103 ctxt->eflags &= ~EFLG_RF; 5101 ctxt->eflags &= ~X86_EFLAGS_RF;
5104 } 5102 }
5105 5103
5106 ctxt->eip = ctxt->_eip; 5104 ctxt->eip = ctxt->_eip;
@@ -5137,8 +5135,7 @@ twobyte_insn:
5137 case 0x40 ... 0x4f: /* cmov */ 5135 case 0x40 ... 0x4f: /* cmov */
5138 if (test_cc(ctxt->b, ctxt->eflags)) 5136 if (test_cc(ctxt->b, ctxt->eflags))
5139 ctxt->dst.val = ctxt->src.val; 5137 ctxt->dst.val = ctxt->src.val;
5140 else if (ctxt->mode != X86EMUL_MODE_PROT64 || 5138 else if (ctxt->op_bytes != 4)
5141 ctxt->op_bytes != 4)
5142 ctxt->dst.type = OP_NONE; /* no writeback */ 5139 ctxt->dst.type = OP_NONE; /* no writeback */
5143 break; 5140 break;
5144 case 0x80 ... 0x8f: /* jnz rel, etc*/ 5141 case 0x80 ... 0x8f: /* jnz rel, etc*/
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index 298781d4cfb4..4dce6f8b6129 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -443,7 +443,8 @@ static inline int pit_in_range(gpa_t addr)
443 (addr < KVM_PIT_BASE_ADDRESS + KVM_PIT_MEM_LENGTH)); 443 (addr < KVM_PIT_BASE_ADDRESS + KVM_PIT_MEM_LENGTH));
444} 444}
445 445
446static int pit_ioport_write(struct kvm_io_device *this, 446static int pit_ioport_write(struct kvm_vcpu *vcpu,
447 struct kvm_io_device *this,
447 gpa_t addr, int len, const void *data) 448 gpa_t addr, int len, const void *data)
448{ 449{
449 struct kvm_pit *pit = dev_to_pit(this); 450 struct kvm_pit *pit = dev_to_pit(this);
@@ -519,7 +520,8 @@ static int pit_ioport_write(struct kvm_io_device *this,
519 return 0; 520 return 0;
520} 521}
521 522
522static int pit_ioport_read(struct kvm_io_device *this, 523static int pit_ioport_read(struct kvm_vcpu *vcpu,
524 struct kvm_io_device *this,
523 gpa_t addr, int len, void *data) 525 gpa_t addr, int len, void *data)
524{ 526{
525 struct kvm_pit *pit = dev_to_pit(this); 527 struct kvm_pit *pit = dev_to_pit(this);
@@ -589,7 +591,8 @@ static int pit_ioport_read(struct kvm_io_device *this,
589 return 0; 591 return 0;
590} 592}
591 593
592static int speaker_ioport_write(struct kvm_io_device *this, 594static int speaker_ioport_write(struct kvm_vcpu *vcpu,
595 struct kvm_io_device *this,
593 gpa_t addr, int len, const void *data) 596 gpa_t addr, int len, const void *data)
594{ 597{
595 struct kvm_pit *pit = speaker_to_pit(this); 598 struct kvm_pit *pit = speaker_to_pit(this);
@@ -606,8 +609,9 @@ static int speaker_ioport_write(struct kvm_io_device *this,
606 return 0; 609 return 0;
607} 610}
608 611
609static int speaker_ioport_read(struct kvm_io_device *this, 612static int speaker_ioport_read(struct kvm_vcpu *vcpu,
610 gpa_t addr, int len, void *data) 613 struct kvm_io_device *this,
614 gpa_t addr, int len, void *data)
611{ 615{
612 struct kvm_pit *pit = speaker_to_pit(this); 616 struct kvm_pit *pit = speaker_to_pit(this);
613 struct kvm_kpit_state *pit_state = &pit->pit_state; 617 struct kvm_kpit_state *pit_state = &pit->pit_state;
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h
index dd1b16b611b0..c84990b42b5b 100644
--- a/arch/x86/kvm/i8254.h
+++ b/arch/x86/kvm/i8254.h
@@ -3,7 +3,7 @@
3 3
4#include <linux/kthread.h> 4#include <linux/kthread.h>
5 5
6#include "iodev.h" 6#include <kvm/iodev.h>
7 7
8struct kvm_kpit_channel_state { 8struct kvm_kpit_channel_state {
9 u32 count; /* can be 65536 */ 9 u32 count; /* can be 65536 */
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c
index 9541ba34126b..fef922ff2635 100644
--- a/arch/x86/kvm/i8259.c
+++ b/arch/x86/kvm/i8259.c
@@ -529,42 +529,42 @@ static int picdev_read(struct kvm_pic *s,
529 return 0; 529 return 0;
530} 530}
531 531
532static int picdev_master_write(struct kvm_io_device *dev, 532static int picdev_master_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
533 gpa_t addr, int len, const void *val) 533 gpa_t addr, int len, const void *val)
534{ 534{
535 return picdev_write(container_of(dev, struct kvm_pic, dev_master), 535 return picdev_write(container_of(dev, struct kvm_pic, dev_master),
536 addr, len, val); 536 addr, len, val);
537} 537}
538 538
539static int picdev_master_read(struct kvm_io_device *dev, 539static int picdev_master_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
540 gpa_t addr, int len, void *val) 540 gpa_t addr, int len, void *val)
541{ 541{
542 return picdev_read(container_of(dev, struct kvm_pic, dev_master), 542 return picdev_read(container_of(dev, struct kvm_pic, dev_master),
543 addr, len, val); 543 addr, len, val);
544} 544}
545 545
546static int picdev_slave_write(struct kvm_io_device *dev, 546static int picdev_slave_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
547 gpa_t addr, int len, const void *val) 547 gpa_t addr, int len, const void *val)
548{ 548{
549 return picdev_write(container_of(dev, struct kvm_pic, dev_slave), 549 return picdev_write(container_of(dev, struct kvm_pic, dev_slave),
550 addr, len, val); 550 addr, len, val);
551} 551}
552 552
553static int picdev_slave_read(struct kvm_io_device *dev, 553static int picdev_slave_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
554 gpa_t addr, int len, void *val) 554 gpa_t addr, int len, void *val)
555{ 555{
556 return picdev_read(container_of(dev, struct kvm_pic, dev_slave), 556 return picdev_read(container_of(dev, struct kvm_pic, dev_slave),
557 addr, len, val); 557 addr, len, val);
558} 558}
559 559
560static int picdev_eclr_write(struct kvm_io_device *dev, 560static int picdev_eclr_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
561 gpa_t addr, int len, const void *val) 561 gpa_t addr, int len, const void *val)
562{ 562{
563 return picdev_write(container_of(dev, struct kvm_pic, dev_eclr), 563 return picdev_write(container_of(dev, struct kvm_pic, dev_eclr),
564 addr, len, val); 564 addr, len, val);
565} 565}
566 566
567static int picdev_eclr_read(struct kvm_io_device *dev, 567static int picdev_eclr_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
568 gpa_t addr, int len, void *val) 568 gpa_t addr, int len, void *val)
569{ 569{
570 return picdev_read(container_of(dev, struct kvm_pic, dev_eclr), 570 return picdev_read(container_of(dev, struct kvm_pic, dev_eclr),
diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c
index 46d4449772bc..28146f03c514 100644
--- a/arch/x86/kvm/ioapic.c
+++ b/arch/x86/kvm/ioapic.c
@@ -206,6 +206,8 @@ static int ioapic_set_irq(struct kvm_ioapic *ioapic, unsigned int irq,
206 206
207 old_irr = ioapic->irr; 207 old_irr = ioapic->irr;
208 ioapic->irr |= mask; 208 ioapic->irr |= mask;
209 if (edge)
210 ioapic->irr_delivered &= ~mask;
209 if ((edge && old_irr == ioapic->irr) || 211 if ((edge && old_irr == ioapic->irr) ||
210 (!edge && entry.fields.remote_irr)) { 212 (!edge && entry.fields.remote_irr)) {
211 ret = 0; 213 ret = 0;
@@ -349,7 +351,7 @@ static int ioapic_service(struct kvm_ioapic *ioapic, int irq, bool line_status)
349 irqe.shorthand = 0; 351 irqe.shorthand = 0;
350 352
351 if (irqe.trig_mode == IOAPIC_EDGE_TRIG) 353 if (irqe.trig_mode == IOAPIC_EDGE_TRIG)
352 ioapic->irr &= ~(1 << irq); 354 ioapic->irr_delivered |= 1 << irq;
353 355
354 if (irq == RTC_GSI && line_status) { 356 if (irq == RTC_GSI && line_status) {
355 /* 357 /*
@@ -473,13 +475,6 @@ static void __kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu,
473 } 475 }
474} 476}
475 477
476bool kvm_ioapic_handles_vector(struct kvm *kvm, int vector)
477{
478 struct kvm_ioapic *ioapic = kvm->arch.vioapic;
479 smp_rmb();
480 return test_bit(vector, ioapic->handled_vectors);
481}
482
483void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, int trigger_mode) 478void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, int trigger_mode)
484{ 479{
485 struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic; 480 struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
@@ -500,8 +495,8 @@ static inline int ioapic_in_range(struct kvm_ioapic *ioapic, gpa_t addr)
500 (addr < ioapic->base_address + IOAPIC_MEM_LENGTH))); 495 (addr < ioapic->base_address + IOAPIC_MEM_LENGTH)));
501} 496}
502 497
503static int ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len, 498static int ioapic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
504 void *val) 499 gpa_t addr, int len, void *val)
505{ 500{
506 struct kvm_ioapic *ioapic = to_ioapic(this); 501 struct kvm_ioapic *ioapic = to_ioapic(this);
507 u32 result; 502 u32 result;
@@ -543,8 +538,8 @@ static int ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len,
543 return 0; 538 return 0;
544} 539}
545 540
546static int ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, 541static int ioapic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
547 const void *val) 542 gpa_t addr, int len, const void *val)
548{ 543{
549 struct kvm_ioapic *ioapic = to_ioapic(this); 544 struct kvm_ioapic *ioapic = to_ioapic(this);
550 u32 data; 545 u32 data;
@@ -599,6 +594,7 @@ static void kvm_ioapic_reset(struct kvm_ioapic *ioapic)
599 ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS; 594 ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS;
600 ioapic->ioregsel = 0; 595 ioapic->ioregsel = 0;
601 ioapic->irr = 0; 596 ioapic->irr = 0;
597 ioapic->irr_delivered = 0;
602 ioapic->id = 0; 598 ioapic->id = 0;
603 memset(ioapic->irq_eoi, 0x00, IOAPIC_NUM_PINS); 599 memset(ioapic->irq_eoi, 0x00, IOAPIC_NUM_PINS);
604 rtc_irq_eoi_tracking_reset(ioapic); 600 rtc_irq_eoi_tracking_reset(ioapic);
@@ -656,6 +652,7 @@ int kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state)
656 652
657 spin_lock(&ioapic->lock); 653 spin_lock(&ioapic->lock);
658 memcpy(state, ioapic, sizeof(struct kvm_ioapic_state)); 654 memcpy(state, ioapic, sizeof(struct kvm_ioapic_state));
655 state->irr &= ~ioapic->irr_delivered;
659 spin_unlock(&ioapic->lock); 656 spin_unlock(&ioapic->lock);
660 return 0; 657 return 0;
661} 658}
@@ -669,6 +666,7 @@ int kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state)
669 spin_lock(&ioapic->lock); 666 spin_lock(&ioapic->lock);
670 memcpy(ioapic, state, sizeof(struct kvm_ioapic_state)); 667 memcpy(ioapic, state, sizeof(struct kvm_ioapic_state));
671 ioapic->irr = 0; 668 ioapic->irr = 0;
669 ioapic->irr_delivered = 0;
672 update_handled_vectors(ioapic); 670 update_handled_vectors(ioapic);
673 kvm_vcpu_request_scan_ioapic(kvm); 671 kvm_vcpu_request_scan_ioapic(kvm);
674 kvm_ioapic_inject_all(ioapic, state->irr); 672 kvm_ioapic_inject_all(ioapic, state->irr);
diff --git a/arch/x86/kvm/ioapic.h b/arch/x86/kvm/ioapic.h
index c2e36d934af4..ca0b0b4e6256 100644
--- a/arch/x86/kvm/ioapic.h
+++ b/arch/x86/kvm/ioapic.h
@@ -3,7 +3,7 @@
3 3
4#include <linux/kvm_host.h> 4#include <linux/kvm_host.h>
5 5
6#include "iodev.h" 6#include <kvm/iodev.h>
7 7
8struct kvm; 8struct kvm;
9struct kvm_vcpu; 9struct kvm_vcpu;
@@ -77,6 +77,7 @@ struct kvm_ioapic {
77 struct rtc_status rtc_status; 77 struct rtc_status rtc_status;
78 struct delayed_work eoi_inject; 78 struct delayed_work eoi_inject;
79 u32 irq_eoi[IOAPIC_NUM_PINS]; 79 u32 irq_eoi[IOAPIC_NUM_PINS];
80 u32 irr_delivered;
80}; 81};
81 82
82#ifdef DEBUG 83#ifdef DEBUG
@@ -97,13 +98,19 @@ static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm)
97 return kvm->arch.vioapic; 98 return kvm->arch.vioapic;
98} 99}
99 100
101static inline bool kvm_ioapic_handles_vector(struct kvm *kvm, int vector)
102{
103 struct kvm_ioapic *ioapic = kvm->arch.vioapic;
104 smp_rmb();
105 return test_bit(vector, ioapic->handled_vectors);
106}
107
100void kvm_rtc_eoi_tracking_restore_one(struct kvm_vcpu *vcpu); 108void kvm_rtc_eoi_tracking_restore_one(struct kvm_vcpu *vcpu);
101bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 109bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
102 int short_hand, unsigned int dest, int dest_mode); 110 int short_hand, unsigned int dest, int dest_mode);
103int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2); 111int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2);
104void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, 112void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector,
105 int trigger_mode); 113 int trigger_mode);
106bool kvm_ioapic_handles_vector(struct kvm *kvm, int vector);
107int kvm_ioapic_init(struct kvm *kvm); 114int kvm_ioapic_init(struct kvm *kvm);
108void kvm_ioapic_destroy(struct kvm *kvm); 115void kvm_ioapic_destroy(struct kvm *kvm);
109int kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int irq_source_id, 116int kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int irq_source_id,
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h
index 2d03568e9498..ad68c73008c5 100644
--- a/arch/x86/kvm/irq.h
+++ b/arch/x86/kvm/irq.h
@@ -27,7 +27,7 @@
27#include <linux/kvm_host.h> 27#include <linux/kvm_host.h>
28#include <linux/spinlock.h> 28#include <linux/spinlock.h>
29 29
30#include "iodev.h" 30#include <kvm/iodev.h>
31#include "ioapic.h" 31#include "ioapic.h"
32#include "lapic.h" 32#include "lapic.h"
33 33
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 4ee827d7bf36..d67206a7b99a 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -133,6 +133,28 @@ static inline int kvm_apic_id(struct kvm_lapic *apic)
133 return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff; 133 return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
134} 134}
135 135
136/* The logical map is definitely wrong if we have multiple
137 * modes at the same time. (Physical map is always right.)
138 */
139static inline bool kvm_apic_logical_map_valid(struct kvm_apic_map *map)
140{
141 return !(map->mode & (map->mode - 1));
142}
143
144static inline void
145apic_logical_id(struct kvm_apic_map *map, u32 dest_id, u16 *cid, u16 *lid)
146{
147 unsigned lid_bits;
148
149 BUILD_BUG_ON(KVM_APIC_MODE_XAPIC_CLUSTER != 4);
150 BUILD_BUG_ON(KVM_APIC_MODE_XAPIC_FLAT != 8);
151 BUILD_BUG_ON(KVM_APIC_MODE_X2APIC != 16);
152 lid_bits = map->mode;
153
154 *cid = dest_id >> lid_bits;
155 *lid = dest_id & ((1 << lid_bits) - 1);
156}
157
136static void recalculate_apic_map(struct kvm *kvm) 158static void recalculate_apic_map(struct kvm *kvm)
137{ 159{
138 struct kvm_apic_map *new, *old = NULL; 160 struct kvm_apic_map *new, *old = NULL;
@@ -146,48 +168,6 @@ static void recalculate_apic_map(struct kvm *kvm)
146 if (!new) 168 if (!new)
147 goto out; 169 goto out;
148 170
149 new->ldr_bits = 8;
150 /* flat mode is default */
151 new->cid_shift = 8;
152 new->cid_mask = 0;
153 new->lid_mask = 0xff;
154 new->broadcast = APIC_BROADCAST;
155
156 kvm_for_each_vcpu(i, vcpu, kvm) {
157 struct kvm_lapic *apic = vcpu->arch.apic;
158
159 if (!kvm_apic_present(vcpu))
160 continue;
161
162 if (apic_x2apic_mode(apic)) {
163 new->ldr_bits = 32;
164 new->cid_shift = 16;
165 new->cid_mask = new->lid_mask = 0xffff;
166 new->broadcast = X2APIC_BROADCAST;
167 } else if (kvm_apic_get_reg(apic, APIC_LDR)) {
168 if (kvm_apic_get_reg(apic, APIC_DFR) ==
169 APIC_DFR_CLUSTER) {
170 new->cid_shift = 4;
171 new->cid_mask = 0xf;
172 new->lid_mask = 0xf;
173 } else {
174 new->cid_shift = 8;
175 new->cid_mask = 0;
176 new->lid_mask = 0xff;
177 }
178 }
179
180 /*
181 * All APICs have to be configured in the same mode by an OS.
182 * We take advatage of this while building logical id loockup
183 * table. After reset APICs are in software disabled mode, so if
184 * we find apic with different setting we assume this is the mode
185 * OS wants all apics to be in; build lookup table accordingly.
186 */
187 if (kvm_apic_sw_enabled(apic))
188 break;
189 }
190
191 kvm_for_each_vcpu(i, vcpu, kvm) { 171 kvm_for_each_vcpu(i, vcpu, kvm) {
192 struct kvm_lapic *apic = vcpu->arch.apic; 172 struct kvm_lapic *apic = vcpu->arch.apic;
193 u16 cid, lid; 173 u16 cid, lid;
@@ -198,11 +178,25 @@ static void recalculate_apic_map(struct kvm *kvm)
198 178
199 aid = kvm_apic_id(apic); 179 aid = kvm_apic_id(apic);
200 ldr = kvm_apic_get_reg(apic, APIC_LDR); 180 ldr = kvm_apic_get_reg(apic, APIC_LDR);
201 cid = apic_cluster_id(new, ldr);
202 lid = apic_logical_id(new, ldr);
203 181
204 if (aid < ARRAY_SIZE(new->phys_map)) 182 if (aid < ARRAY_SIZE(new->phys_map))
205 new->phys_map[aid] = apic; 183 new->phys_map[aid] = apic;
184
185 if (apic_x2apic_mode(apic)) {
186 new->mode |= KVM_APIC_MODE_X2APIC;
187 } else if (ldr) {
188 ldr = GET_APIC_LOGICAL_ID(ldr);
189 if (kvm_apic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
190 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
191 else
192 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
193 }
194
195 if (!kvm_apic_logical_map_valid(new))
196 continue;
197
198 apic_logical_id(new, ldr, &cid, &lid);
199
206 if (lid && cid < ARRAY_SIZE(new->logical_map)) 200 if (lid && cid < ARRAY_SIZE(new->logical_map))
207 new->logical_map[cid][ffs(lid) - 1] = apic; 201 new->logical_map[cid][ffs(lid) - 1] = apic;
208 } 202 }
@@ -588,15 +582,23 @@ static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
588 apic_update_ppr(apic); 582 apic_update_ppr(apic);
589} 583}
590 584
591static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 dest) 585static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
592{ 586{
593 return dest == (apic_x2apic_mode(apic) ? 587 if (apic_x2apic_mode(apic))
594 X2APIC_BROADCAST : APIC_BROADCAST); 588 return mda == X2APIC_BROADCAST;
589
590 return GET_APIC_DEST_FIELD(mda) == APIC_BROADCAST;
595} 591}
596 592
597static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 dest) 593static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
598{ 594{
599 return kvm_apic_id(apic) == dest || kvm_apic_broadcast(apic, dest); 595 if (kvm_apic_broadcast(apic, mda))
596 return true;
597
598 if (apic_x2apic_mode(apic))
599 return mda == kvm_apic_id(apic);
600
601 return mda == SET_APIC_DEST_FIELD(kvm_apic_id(apic));
600} 602}
601 603
602static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 604static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
@@ -613,6 +615,7 @@ static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
613 && (logical_id & mda & 0xffff) != 0; 615 && (logical_id & mda & 0xffff) != 0;
614 616
615 logical_id = GET_APIC_LOGICAL_ID(logical_id); 617 logical_id = GET_APIC_LOGICAL_ID(logical_id);
618 mda = GET_APIC_DEST_FIELD(mda);
616 619
617 switch (kvm_apic_get_reg(apic, APIC_DFR)) { 620 switch (kvm_apic_get_reg(apic, APIC_DFR)) {
618 case APIC_DFR_FLAT: 621 case APIC_DFR_FLAT:
@@ -627,10 +630,27 @@ static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
627 } 630 }
628} 631}
629 632
633/* KVM APIC implementation has two quirks
634 * - dest always begins at 0 while xAPIC MDA has offset 24,
635 * - IOxAPIC messages have to be delivered (directly) to x2APIC.
636 */
637static u32 kvm_apic_mda(unsigned int dest_id, struct kvm_lapic *source,
638 struct kvm_lapic *target)
639{
640 bool ipi = source != NULL;
641 bool x2apic_mda = apic_x2apic_mode(ipi ? source : target);
642
643 if (!ipi && dest_id == APIC_BROADCAST && x2apic_mda)
644 return X2APIC_BROADCAST;
645
646 return x2apic_mda ? dest_id : SET_APIC_DEST_FIELD(dest_id);
647}
648
630bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 649bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
631 int short_hand, unsigned int dest, int dest_mode) 650 int short_hand, unsigned int dest, int dest_mode)
632{ 651{
633 struct kvm_lapic *target = vcpu->arch.apic; 652 struct kvm_lapic *target = vcpu->arch.apic;
653 u32 mda = kvm_apic_mda(dest, source, target);
634 654
635 apic_debug("target %p, source %p, dest 0x%x, " 655 apic_debug("target %p, source %p, dest 0x%x, "
636 "dest_mode 0x%x, short_hand 0x%x\n", 656 "dest_mode 0x%x, short_hand 0x%x\n",
@@ -640,9 +660,9 @@ bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
640 switch (short_hand) { 660 switch (short_hand) {
641 case APIC_DEST_NOSHORT: 661 case APIC_DEST_NOSHORT:
642 if (dest_mode == APIC_DEST_PHYSICAL) 662 if (dest_mode == APIC_DEST_PHYSICAL)
643 return kvm_apic_match_physical_addr(target, dest); 663 return kvm_apic_match_physical_addr(target, mda);
644 else 664 else
645 return kvm_apic_match_logical_addr(target, dest); 665 return kvm_apic_match_logical_addr(target, mda);
646 case APIC_DEST_SELF: 666 case APIC_DEST_SELF:
647 return target == source; 667 return target == source;
648 case APIC_DEST_ALLINC: 668 case APIC_DEST_ALLINC:
@@ -664,6 +684,7 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
664 struct kvm_lapic **dst; 684 struct kvm_lapic **dst;
665 int i; 685 int i;
666 bool ret = false; 686 bool ret = false;
687 bool x2apic_ipi = src && apic_x2apic_mode(src);
667 688
668 *r = -1; 689 *r = -1;
669 690
@@ -675,15 +696,15 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
675 if (irq->shorthand) 696 if (irq->shorthand)
676 return false; 697 return false;
677 698
699 if (irq->dest_id == (x2apic_ipi ? X2APIC_BROADCAST : APIC_BROADCAST))
700 return false;
701
678 rcu_read_lock(); 702 rcu_read_lock();
679 map = rcu_dereference(kvm->arch.apic_map); 703 map = rcu_dereference(kvm->arch.apic_map);
680 704
681 if (!map) 705 if (!map)
682 goto out; 706 goto out;
683 707
684 if (irq->dest_id == map->broadcast)
685 goto out;
686
687 ret = true; 708 ret = true;
688 709
689 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 710 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
@@ -692,16 +713,20 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
692 713
693 dst = &map->phys_map[irq->dest_id]; 714 dst = &map->phys_map[irq->dest_id];
694 } else { 715 } else {
695 u32 mda = irq->dest_id << (32 - map->ldr_bits); 716 u16 cid;
696 u16 cid = apic_cluster_id(map, mda); 717
718 if (!kvm_apic_logical_map_valid(map)) {
719 ret = false;
720 goto out;
721 }
722
723 apic_logical_id(map, irq->dest_id, &cid, (u16 *)&bitmap);
697 724
698 if (cid >= ARRAY_SIZE(map->logical_map)) 725 if (cid >= ARRAY_SIZE(map->logical_map))
699 goto out; 726 goto out;
700 727
701 dst = map->logical_map[cid]; 728 dst = map->logical_map[cid];
702 729
703 bitmap = apic_logical_id(map, mda);
704
705 if (irq->delivery_mode == APIC_DM_LOWEST) { 730 if (irq->delivery_mode == APIC_DM_LOWEST) {
706 int l = -1; 731 int l = -1;
707 for_each_set_bit(i, &bitmap, 16) { 732 for_each_set_bit(i, &bitmap, 16) {
@@ -1037,7 +1062,7 @@ static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1037 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1062 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1038} 1063}
1039 1064
1040static int apic_mmio_read(struct kvm_io_device *this, 1065static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1041 gpa_t address, int len, void *data) 1066 gpa_t address, int len, void *data)
1042{ 1067{
1043 struct kvm_lapic *apic = to_lapic(this); 1068 struct kvm_lapic *apic = to_lapic(this);
@@ -1357,7 +1382,7 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
1357 return ret; 1382 return ret;
1358} 1383}
1359 1384
1360static int apic_mmio_write(struct kvm_io_device *this, 1385static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1361 gpa_t address, int len, const void *data) 1386 gpa_t address, int len, const void *data)
1362{ 1387{
1363 struct kvm_lapic *apic = to_lapic(this); 1388 struct kvm_lapic *apic = to_lapic(this);
@@ -1497,8 +1522,6 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
1497 return; 1522 return;
1498 } 1523 }
1499 1524
1500 if (!kvm_vcpu_is_bsp(apic->vcpu))
1501 value &= ~MSR_IA32_APICBASE_BSP;
1502 vcpu->arch.apic_base = value; 1525 vcpu->arch.apic_base = value;
1503 1526
1504 /* update jump label if enable bit changes */ 1527 /* update jump label if enable bit changes */
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index 0bc6c656625b..9d28383fc1e7 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -1,7 +1,7 @@
1#ifndef __KVM_X86_LAPIC_H 1#ifndef __KVM_X86_LAPIC_H
2#define __KVM_X86_LAPIC_H 2#define __KVM_X86_LAPIC_H
3 3
4#include "iodev.h" 4#include <kvm/iodev.h>
5 5
6#include <linux/kvm_host.h> 6#include <linux/kvm_host.h>
7 7
@@ -148,21 +148,6 @@ static inline bool kvm_apic_vid_enabled(struct kvm *kvm)
148 return kvm_x86_ops->vm_has_apicv(kvm); 148 return kvm_x86_ops->vm_has_apicv(kvm);
149} 149}
150 150
151static inline u16 apic_cluster_id(struct kvm_apic_map *map, u32 ldr)
152{
153 u16 cid;
154 ldr >>= 32 - map->ldr_bits;
155 cid = (ldr >> map->cid_shift) & map->cid_mask;
156
157 return cid;
158}
159
160static inline u16 apic_logical_id(struct kvm_apic_map *map, u32 ldr)
161{
162 ldr >>= (32 - map->ldr_bits);
163 return ldr & map->lid_mask;
164}
165
166static inline bool kvm_apic_has_events(struct kvm_vcpu *vcpu) 151static inline bool kvm_apic_has_events(struct kvm_vcpu *vcpu)
167{ 152{
168 return vcpu->arch.apic->pending_events; 153 return vcpu->arch.apic->pending_events;
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index cee759299a35..146f295ee322 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -4465,6 +4465,79 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
4465 kvm_flush_remote_tlbs(kvm); 4465 kvm_flush_remote_tlbs(kvm);
4466} 4466}
4467 4467
4468static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
4469 unsigned long *rmapp)
4470{
4471 u64 *sptep;
4472 struct rmap_iterator iter;
4473 int need_tlb_flush = 0;
4474 pfn_t pfn;
4475 struct kvm_mmu_page *sp;
4476
4477 for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
4478 BUG_ON(!(*sptep & PT_PRESENT_MASK));
4479
4480 sp = page_header(__pa(sptep));
4481 pfn = spte_to_pfn(*sptep);
4482
4483 /*
4484 * Only EPT supported for now; otherwise, one would need to
4485 * find out efficiently whether the guest page tables are
4486 * also using huge pages.
4487 */
4488 if (sp->role.direct &&
4489 !kvm_is_reserved_pfn(pfn) &&
4490 PageTransCompound(pfn_to_page(pfn))) {
4491 drop_spte(kvm, sptep);
4492 sptep = rmap_get_first(*rmapp, &iter);
4493 need_tlb_flush = 1;
4494 } else
4495 sptep = rmap_get_next(&iter);
4496 }
4497
4498 return need_tlb_flush;
4499}
4500
4501void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
4502 struct kvm_memory_slot *memslot)
4503{
4504 bool flush = false;
4505 unsigned long *rmapp;
4506 unsigned long last_index, index;
4507 gfn_t gfn_start, gfn_end;
4508
4509 spin_lock(&kvm->mmu_lock);
4510
4511 gfn_start = memslot->base_gfn;
4512 gfn_end = memslot->base_gfn + memslot->npages - 1;
4513
4514 if (gfn_start >= gfn_end)
4515 goto out;
4516
4517 rmapp = memslot->arch.rmap[0];
4518 last_index = gfn_to_index(gfn_end, memslot->base_gfn,
4519 PT_PAGE_TABLE_LEVEL);
4520
4521 for (index = 0; index <= last_index; ++index, ++rmapp) {
4522 if (*rmapp)
4523 flush |= kvm_mmu_zap_collapsible_spte(kvm, rmapp);
4524
4525 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
4526 if (flush) {
4527 kvm_flush_remote_tlbs(kvm);
4528 flush = false;
4529 }
4530 cond_resched_lock(&kvm->mmu_lock);
4531 }
4532 }
4533
4534 if (flush)
4535 kvm_flush_remote_tlbs(kvm);
4536
4537out:
4538 spin_unlock(&kvm->mmu_lock);
4539}
4540
4468void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, 4541void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
4469 struct kvm_memory_slot *memslot) 4542 struct kvm_memory_slot *memslot)
4470{ 4543{
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 8e6b7d869d2f..29fbf9dfdc54 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -38,7 +38,7 @@ static struct kvm_arch_event_perf_mapping {
38}; 38};
39 39
40/* mapping between fixed pmc index and arch_events array */ 40/* mapping between fixed pmc index and arch_events array */
41int fixed_pmc_events[] = {1, 0, 7}; 41static int fixed_pmc_events[] = {1, 0, 7};
42 42
43static bool pmc_is_gp(struct kvm_pmc *pmc) 43static bool pmc_is_gp(struct kvm_pmc *pmc)
44{ 44{
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index cc618c882f90..ce741b8650f6 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -1261,7 +1261,7 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1261 1261
1262 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE | 1262 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1263 MSR_IA32_APICBASE_ENABLE; 1263 MSR_IA32_APICBASE_ENABLE;
1264 if (kvm_vcpu_is_bsp(&svm->vcpu)) 1264 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1265 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP; 1265 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1266 1266
1267 svm_init_osvw(&svm->vcpu); 1267 svm_init_osvw(&svm->vcpu);
@@ -1929,14 +1929,12 @@ static int nop_on_interception(struct vcpu_svm *svm)
1929static int halt_interception(struct vcpu_svm *svm) 1929static int halt_interception(struct vcpu_svm *svm)
1930{ 1930{
1931 svm->next_rip = kvm_rip_read(&svm->vcpu) + 1; 1931 svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
1932 skip_emulated_instruction(&svm->vcpu);
1933 return kvm_emulate_halt(&svm->vcpu); 1932 return kvm_emulate_halt(&svm->vcpu);
1934} 1933}
1935 1934
1936static int vmmcall_interception(struct vcpu_svm *svm) 1935static int vmmcall_interception(struct vcpu_svm *svm)
1937{ 1936{
1938 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; 1937 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
1939 skip_emulated_instruction(&svm->vcpu);
1940 kvm_emulate_hypercall(&svm->vcpu); 1938 kvm_emulate_hypercall(&svm->vcpu);
1941 return 1; 1939 return 1;
1942} 1940}
@@ -2757,11 +2755,11 @@ static int invlpga_interception(struct vcpu_svm *svm)
2757{ 2755{
2758 struct kvm_vcpu *vcpu = &svm->vcpu; 2756 struct kvm_vcpu *vcpu = &svm->vcpu;
2759 2757
2760 trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX], 2758 trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
2761 vcpu->arch.regs[VCPU_REGS_RAX]); 2759 kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2762 2760
2763 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ 2761 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2764 kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]); 2762 kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2765 2763
2766 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; 2764 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2767 skip_emulated_instruction(&svm->vcpu); 2765 skip_emulated_instruction(&svm->vcpu);
@@ -2770,12 +2768,18 @@ static int invlpga_interception(struct vcpu_svm *svm)
2770 2768
2771static int skinit_interception(struct vcpu_svm *svm) 2769static int skinit_interception(struct vcpu_svm *svm)
2772{ 2770{
2773 trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]); 2771 trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2774 2772
2775 kvm_queue_exception(&svm->vcpu, UD_VECTOR); 2773 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2776 return 1; 2774 return 1;
2777} 2775}
2778 2776
2777static int wbinvd_interception(struct vcpu_svm *svm)
2778{
2779 kvm_emulate_wbinvd(&svm->vcpu);
2780 return 1;
2781}
2782
2779static int xsetbv_interception(struct vcpu_svm *svm) 2783static int xsetbv_interception(struct vcpu_svm *svm)
2780{ 2784{
2781 u64 new_bv = kvm_read_edx_eax(&svm->vcpu); 2785 u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
@@ -2902,7 +2906,8 @@ static int rdpmc_interception(struct vcpu_svm *svm)
2902 return 1; 2906 return 1;
2903} 2907}
2904 2908
2905bool check_selective_cr0_intercepted(struct vcpu_svm *svm, unsigned long val) 2909static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
2910 unsigned long val)
2906{ 2911{
2907 unsigned long cr0 = svm->vcpu.arch.cr0; 2912 unsigned long cr0 = svm->vcpu.arch.cr0;
2908 bool ret = false; 2913 bool ret = false;
@@ -2940,7 +2945,10 @@ static int cr_interception(struct vcpu_svm *svm)
2940 return emulate_on_interception(svm); 2945 return emulate_on_interception(svm);
2941 2946
2942 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2947 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2943 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; 2948 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2949 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2950 else
2951 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2944 2952
2945 err = 0; 2953 err = 0;
2946 if (cr >= 16) { /* mov to cr */ 2954 if (cr >= 16) { /* mov to cr */
@@ -3133,7 +3141,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
3133 3141
3134static int rdmsr_interception(struct vcpu_svm *svm) 3142static int rdmsr_interception(struct vcpu_svm *svm)
3135{ 3143{
3136 u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 3144 u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3137 u64 data; 3145 u64 data;
3138 3146
3139 if (svm_get_msr(&svm->vcpu, ecx, &data)) { 3147 if (svm_get_msr(&svm->vcpu, ecx, &data)) {
@@ -3142,8 +3150,8 @@ static int rdmsr_interception(struct vcpu_svm *svm)
3142 } else { 3150 } else {
3143 trace_kvm_msr_read(ecx, data); 3151 trace_kvm_msr_read(ecx, data);
3144 3152
3145 svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff; 3153 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, data & 0xffffffff);
3146 svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32; 3154 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX, data >> 32);
3147 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2; 3155 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3148 skip_emulated_instruction(&svm->vcpu); 3156 skip_emulated_instruction(&svm->vcpu);
3149 } 3157 }
@@ -3246,9 +3254,8 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3246static int wrmsr_interception(struct vcpu_svm *svm) 3254static int wrmsr_interception(struct vcpu_svm *svm)
3247{ 3255{
3248 struct msr_data msr; 3256 struct msr_data msr;
3249 u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 3257 u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3250 u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u) 3258 u64 data = kvm_read_edx_eax(&svm->vcpu);
3251 | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
3252 3259
3253 msr.data = data; 3260 msr.data = data;
3254 msr.index = ecx; 3261 msr.index = ecx;
@@ -3325,7 +3332,7 @@ static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3325 [SVM_EXIT_READ_CR3] = cr_interception, 3332 [SVM_EXIT_READ_CR3] = cr_interception,
3326 [SVM_EXIT_READ_CR4] = cr_interception, 3333 [SVM_EXIT_READ_CR4] = cr_interception,
3327 [SVM_EXIT_READ_CR8] = cr_interception, 3334 [SVM_EXIT_READ_CR8] = cr_interception,
3328 [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, 3335 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception,
3329 [SVM_EXIT_WRITE_CR0] = cr_interception, 3336 [SVM_EXIT_WRITE_CR0] = cr_interception,
3330 [SVM_EXIT_WRITE_CR3] = cr_interception, 3337 [SVM_EXIT_WRITE_CR3] = cr_interception,
3331 [SVM_EXIT_WRITE_CR4] = cr_interception, 3338 [SVM_EXIT_WRITE_CR4] = cr_interception,
@@ -3376,7 +3383,7 @@ static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3376 [SVM_EXIT_STGI] = stgi_interception, 3383 [SVM_EXIT_STGI] = stgi_interception,
3377 [SVM_EXIT_CLGI] = clgi_interception, 3384 [SVM_EXIT_CLGI] = clgi_interception,
3378 [SVM_EXIT_SKINIT] = skinit_interception, 3385 [SVM_EXIT_SKINIT] = skinit_interception,
3379 [SVM_EXIT_WBINVD] = emulate_on_interception, 3386 [SVM_EXIT_WBINVD] = wbinvd_interception,
3380 [SVM_EXIT_MONITOR] = monitor_interception, 3387 [SVM_EXIT_MONITOR] = monitor_interception,
3381 [SVM_EXIT_MWAIT] = mwait_interception, 3388 [SVM_EXIT_MWAIT] = mwait_interception,
3382 [SVM_EXIT_XSETBV] = xsetbv_interception, 3389 [SVM_EXIT_XSETBV] = xsetbv_interception,
@@ -3555,7 +3562,7 @@ static int handle_exit(struct kvm_vcpu *vcpu)
3555 3562
3556 if (exit_code >= ARRAY_SIZE(svm_exit_handlers) 3563 if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3557 || !svm_exit_handlers[exit_code]) { 3564 || !svm_exit_handlers[exit_code]) {
3558 WARN_ONCE(1, "vmx: unexpected exit reason 0x%x\n", exit_code); 3565 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
3559 kvm_queue_exception(vcpu, UD_VECTOR); 3566 kvm_queue_exception(vcpu, UD_VECTOR);
3560 return 1; 3567 return 1;
3561 } 3568 }
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index ae4f6d35d19c..f5e8dce8046c 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -2470,6 +2470,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
2470 vmx->nested.nested_vmx_secondary_ctls_low = 0; 2470 vmx->nested.nested_vmx_secondary_ctls_low = 0;
2471 vmx->nested.nested_vmx_secondary_ctls_high &= 2471 vmx->nested.nested_vmx_secondary_ctls_high &=
2472 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2472 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2473 SECONDARY_EXEC_RDTSCP |
2473 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2474 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2474 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2475 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2475 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2476 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
@@ -3268,8 +3269,8 @@ static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
3268 * default value. 3269 * default value.
3269 */ 3270 */
3270 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 3271 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
3271 save->selector &= ~SELECTOR_RPL_MASK; 3272 save->selector &= ~SEGMENT_RPL_MASK;
3272 save->dpl = save->selector & SELECTOR_RPL_MASK; 3273 save->dpl = save->selector & SEGMENT_RPL_MASK;
3273 save->s = 1; 3274 save->s = 1;
3274 } 3275 }
3275 vmx_set_segment(vcpu, save, seg); 3276 vmx_set_segment(vcpu, save, seg);
@@ -3842,7 +3843,7 @@ static bool code_segment_valid(struct kvm_vcpu *vcpu)
3842 unsigned int cs_rpl; 3843 unsigned int cs_rpl;
3843 3844
3844 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3845 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3845 cs_rpl = cs.selector & SELECTOR_RPL_MASK; 3846 cs_rpl = cs.selector & SEGMENT_RPL_MASK;
3846 3847
3847 if (cs.unusable) 3848 if (cs.unusable)
3848 return false; 3849 return false;
@@ -3870,7 +3871,7 @@ static bool stack_segment_valid(struct kvm_vcpu *vcpu)
3870 unsigned int ss_rpl; 3871 unsigned int ss_rpl;
3871 3872
3872 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3873 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3873 ss_rpl = ss.selector & SELECTOR_RPL_MASK; 3874 ss_rpl = ss.selector & SEGMENT_RPL_MASK;
3874 3875
3875 if (ss.unusable) 3876 if (ss.unusable)
3876 return true; 3877 return true;
@@ -3892,7 +3893,7 @@ static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
3892 unsigned int rpl; 3893 unsigned int rpl;
3893 3894
3894 vmx_get_segment(vcpu, &var, seg); 3895 vmx_get_segment(vcpu, &var, seg);
3895 rpl = var.selector & SELECTOR_RPL_MASK; 3896 rpl = var.selector & SEGMENT_RPL_MASK;
3896 3897
3897 if (var.unusable) 3898 if (var.unusable)
3898 return true; 3899 return true;
@@ -3919,7 +3920,7 @@ static bool tr_valid(struct kvm_vcpu *vcpu)
3919 3920
3920 if (tr.unusable) 3921 if (tr.unusable)
3921 return false; 3922 return false;
3922 if (tr.selector & SELECTOR_TI_MASK) /* TI = 1 */ 3923 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */
3923 return false; 3924 return false;
3924 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3925 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
3925 return false; 3926 return false;
@@ -3937,7 +3938,7 @@ static bool ldtr_valid(struct kvm_vcpu *vcpu)
3937 3938
3938 if (ldtr.unusable) 3939 if (ldtr.unusable)
3939 return true; 3940 return true;
3940 if (ldtr.selector & SELECTOR_TI_MASK) /* TI = 1 */ 3941 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */
3941 return false; 3942 return false;
3942 if (ldtr.type != 2) 3943 if (ldtr.type != 2)
3943 return false; 3944 return false;
@@ -3954,8 +3955,8 @@ static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
3954 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3955 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3955 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3956 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3956 3957
3957 return ((cs.selector & SELECTOR_RPL_MASK) == 3958 return ((cs.selector & SEGMENT_RPL_MASK) ==
3958 (ss.selector & SELECTOR_RPL_MASK)); 3959 (ss.selector & SEGMENT_RPL_MASK));
3959} 3960}
3960 3961
3961/* 3962/*
@@ -4711,7 +4712,7 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu)
4711 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val(); 4712 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
4712 kvm_set_cr8(&vmx->vcpu, 0); 4713 kvm_set_cr8(&vmx->vcpu, 0);
4713 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE; 4714 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE;
4714 if (kvm_vcpu_is_bsp(&vmx->vcpu)) 4715 if (kvm_vcpu_is_reset_bsp(&vmx->vcpu))
4715 apic_base_msr.data |= MSR_IA32_APICBASE_BSP; 4716 apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
4716 apic_base_msr.host_initiated = true; 4717 apic_base_msr.host_initiated = true;
4717 kvm_set_apic_base(&vmx->vcpu, &apic_base_msr); 4718 kvm_set_apic_base(&vmx->vcpu, &apic_base_msr);
@@ -5006,7 +5007,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu,
5006 if (emulate_instruction(vcpu, 0) == EMULATE_DONE) { 5007 if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
5007 if (vcpu->arch.halt_request) { 5008 if (vcpu->arch.halt_request) {
5008 vcpu->arch.halt_request = 0; 5009 vcpu->arch.halt_request = 0;
5009 return kvm_emulate_halt(vcpu); 5010 return kvm_vcpu_halt(vcpu);
5010 } 5011 }
5011 return 1; 5012 return 1;
5012 } 5013 }
@@ -5071,6 +5072,10 @@ static int handle_exception(struct kvm_vcpu *vcpu)
5071 } 5072 }
5072 5073
5073 if (is_invalid_opcode(intr_info)) { 5074 if (is_invalid_opcode(intr_info)) {
5075 if (is_guest_mode(vcpu)) {
5076 kvm_queue_exception(vcpu, UD_VECTOR);
5077 return 1;
5078 }
5074 er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD); 5079 er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
5075 if (er != EMULATE_DONE) 5080 if (er != EMULATE_DONE)
5076 kvm_queue_exception(vcpu, UD_VECTOR); 5081 kvm_queue_exception(vcpu, UD_VECTOR);
@@ -5090,9 +5095,10 @@ static int handle_exception(struct kvm_vcpu *vcpu)
5090 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 5095 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
5091 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5096 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5092 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 5097 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
5093 vcpu->run->internal.ndata = 2; 5098 vcpu->run->internal.ndata = 3;
5094 vcpu->run->internal.data[0] = vect_info; 5099 vcpu->run->internal.data[0] = vect_info;
5095 vcpu->run->internal.data[1] = intr_info; 5100 vcpu->run->internal.data[1] = intr_info;
5101 vcpu->run->internal.data[2] = error_code;
5096 return 0; 5102 return 0;
5097 } 5103 }
5098 5104
@@ -5533,13 +5539,11 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
5533 5539
5534static int handle_halt(struct kvm_vcpu *vcpu) 5540static int handle_halt(struct kvm_vcpu *vcpu)
5535{ 5541{
5536 skip_emulated_instruction(vcpu);
5537 return kvm_emulate_halt(vcpu); 5542 return kvm_emulate_halt(vcpu);
5538} 5543}
5539 5544
5540static int handle_vmcall(struct kvm_vcpu *vcpu) 5545static int handle_vmcall(struct kvm_vcpu *vcpu)
5541{ 5546{
5542 skip_emulated_instruction(vcpu);
5543 kvm_emulate_hypercall(vcpu); 5547 kvm_emulate_hypercall(vcpu);
5544 return 1; 5548 return 1;
5545} 5549}
@@ -5570,7 +5574,6 @@ static int handle_rdpmc(struct kvm_vcpu *vcpu)
5570 5574
5571static int handle_wbinvd(struct kvm_vcpu *vcpu) 5575static int handle_wbinvd(struct kvm_vcpu *vcpu)
5572{ 5576{
5573 skip_emulated_instruction(vcpu);
5574 kvm_emulate_wbinvd(vcpu); 5577 kvm_emulate_wbinvd(vcpu);
5575 return 1; 5578 return 1;
5576} 5579}
@@ -5828,7 +5831,7 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
5828 gpa_t gpa; 5831 gpa_t gpa;
5829 5832
5830 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5833 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5831 if (!kvm_io_bus_write(vcpu->kvm, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 5834 if (!kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
5832 skip_emulated_instruction(vcpu); 5835 skip_emulated_instruction(vcpu);
5833 return 1; 5836 return 1;
5834 } 5837 }
@@ -5909,7 +5912,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
5909 5912
5910 if (vcpu->arch.halt_request) { 5913 if (vcpu->arch.halt_request) {
5911 vcpu->arch.halt_request = 0; 5914 vcpu->arch.halt_request = 0;
5912 ret = kvm_emulate_halt(vcpu); 5915 ret = kvm_vcpu_halt(vcpu);
5913 goto out; 5916 goto out;
5914 } 5917 }
5915 5918
@@ -7318,21 +7321,21 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
7318 else if (port < 0x10000) 7321 else if (port < 0x10000)
7319 bitmap = vmcs12->io_bitmap_b; 7322 bitmap = vmcs12->io_bitmap_b;
7320 else 7323 else
7321 return 1; 7324 return true;
7322 bitmap += (port & 0x7fff) / 8; 7325 bitmap += (port & 0x7fff) / 8;
7323 7326
7324 if (last_bitmap != bitmap) 7327 if (last_bitmap != bitmap)
7325 if (kvm_read_guest(vcpu->kvm, bitmap, &b, 1)) 7328 if (kvm_read_guest(vcpu->kvm, bitmap, &b, 1))
7326 return 1; 7329 return true;
7327 if (b & (1 << (port & 7))) 7330 if (b & (1 << (port & 7)))
7328 return 1; 7331 return true;
7329 7332
7330 port++; 7333 port++;
7331 size--; 7334 size--;
7332 last_bitmap = bitmap; 7335 last_bitmap = bitmap;
7333 } 7336 }
7334 7337
7335 return 0; 7338 return false;
7336} 7339}
7337 7340
7338/* 7341/*
@@ -7348,7 +7351,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
7348 gpa_t bitmap; 7351 gpa_t bitmap;
7349 7352
7350 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 7353 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
7351 return 1; 7354 return true;
7352 7355
7353 /* 7356 /*
7354 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 7357 * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
@@ -7367,10 +7370,10 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
7367 if (msr_index < 1024*8) { 7370 if (msr_index < 1024*8) {
7368 unsigned char b; 7371 unsigned char b;
7369 if (kvm_read_guest(vcpu->kvm, bitmap + msr_index/8, &b, 1)) 7372 if (kvm_read_guest(vcpu->kvm, bitmap + msr_index/8, &b, 1))
7370 return 1; 7373 return true;
7371 return 1 & (b >> (msr_index & 7)); 7374 return 1 & (b >> (msr_index & 7));
7372 } else 7375 } else
7373 return 1; /* let L1 handle the wrong parameter */ 7376 return true; /* let L1 handle the wrong parameter */
7374} 7377}
7375 7378
7376/* 7379/*
@@ -7392,7 +7395,7 @@ static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
7392 case 0: 7395 case 0:
7393 if (vmcs12->cr0_guest_host_mask & 7396 if (vmcs12->cr0_guest_host_mask &
7394 (val ^ vmcs12->cr0_read_shadow)) 7397 (val ^ vmcs12->cr0_read_shadow))
7395 return 1; 7398 return true;
7396 break; 7399 break;
7397 case 3: 7400 case 3:
7398 if ((vmcs12->cr3_target_count >= 1 && 7401 if ((vmcs12->cr3_target_count >= 1 &&
@@ -7403,37 +7406,37 @@ static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
7403 vmcs12->cr3_target_value2 == val) || 7406 vmcs12->cr3_target_value2 == val) ||
7404 (vmcs12->cr3_target_count >= 4 && 7407 (vmcs12->cr3_target_count >= 4 &&
7405 vmcs12->cr3_target_value3 == val)) 7408 vmcs12->cr3_target_value3 == val))
7406 return 0; 7409 return false;
7407 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 7410 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
7408 return 1; 7411 return true;
7409 break; 7412 break;
7410 case 4: 7413 case 4:
7411 if (vmcs12->cr4_guest_host_mask & 7414 if (vmcs12->cr4_guest_host_mask &
7412 (vmcs12->cr4_read_shadow ^ val)) 7415 (vmcs12->cr4_read_shadow ^ val))
7413 return 1; 7416 return true;
7414 break; 7417 break;
7415 case 8: 7418 case 8:
7416 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 7419 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
7417 return 1; 7420 return true;
7418 break; 7421 break;
7419 } 7422 }
7420 break; 7423 break;
7421 case 2: /* clts */ 7424 case 2: /* clts */
7422 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 7425 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
7423 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 7426 (vmcs12->cr0_read_shadow & X86_CR0_TS))
7424 return 1; 7427 return true;
7425 break; 7428 break;
7426 case 1: /* mov from cr */ 7429 case 1: /* mov from cr */
7427 switch (cr) { 7430 switch (cr) {
7428 case 3: 7431 case 3:
7429 if (vmcs12->cpu_based_vm_exec_control & 7432 if (vmcs12->cpu_based_vm_exec_control &
7430 CPU_BASED_CR3_STORE_EXITING) 7433 CPU_BASED_CR3_STORE_EXITING)
7431 return 1; 7434 return true;
7432 break; 7435 break;
7433 case 8: 7436 case 8:
7434 if (vmcs12->cpu_based_vm_exec_control & 7437 if (vmcs12->cpu_based_vm_exec_control &
7435 CPU_BASED_CR8_STORE_EXITING) 7438 CPU_BASED_CR8_STORE_EXITING)
7436 return 1; 7439 return true;
7437 break; 7440 break;
7438 } 7441 }
7439 break; 7442 break;
@@ -7444,14 +7447,14 @@ static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
7444 */ 7447 */
7445 if (vmcs12->cr0_guest_host_mask & 0xe & 7448 if (vmcs12->cr0_guest_host_mask & 0xe &
7446 (val ^ vmcs12->cr0_read_shadow)) 7449 (val ^ vmcs12->cr0_read_shadow))
7447 return 1; 7450 return true;
7448 if ((vmcs12->cr0_guest_host_mask & 0x1) && 7451 if ((vmcs12->cr0_guest_host_mask & 0x1) &&
7449 !(vmcs12->cr0_read_shadow & 0x1) && 7452 !(vmcs12->cr0_read_shadow & 0x1) &&
7450 (val & 0x1)) 7453 (val & 0x1))
7451 return 1; 7454 return true;
7452 break; 7455 break;
7453 } 7456 }
7454 return 0; 7457 return false;
7455} 7458}
7456 7459
7457/* 7460/*
@@ -7474,48 +7477,48 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7474 KVM_ISA_VMX); 7477 KVM_ISA_VMX);
7475 7478
7476 if (vmx->nested.nested_run_pending) 7479 if (vmx->nested.nested_run_pending)
7477 return 0; 7480 return false;
7478 7481
7479 if (unlikely(vmx->fail)) { 7482 if (unlikely(vmx->fail)) {
7480 pr_info_ratelimited("%s failed vm entry %x\n", __func__, 7483 pr_info_ratelimited("%s failed vm entry %x\n", __func__,
7481 vmcs_read32(VM_INSTRUCTION_ERROR)); 7484 vmcs_read32(VM_INSTRUCTION_ERROR));
7482 return 1; 7485 return true;
7483 } 7486 }
7484 7487
7485 switch (exit_reason) { 7488 switch (exit_reason) {
7486 case EXIT_REASON_EXCEPTION_NMI: 7489 case EXIT_REASON_EXCEPTION_NMI:
7487 if (!is_exception(intr_info)) 7490 if (!is_exception(intr_info))
7488 return 0; 7491 return false;
7489 else if (is_page_fault(intr_info)) 7492 else if (is_page_fault(intr_info))
7490 return enable_ept; 7493 return enable_ept;
7491 else if (is_no_device(intr_info) && 7494 else if (is_no_device(intr_info) &&
7492 !(vmcs12->guest_cr0 & X86_CR0_TS)) 7495 !(vmcs12->guest_cr0 & X86_CR0_TS))
7493 return 0; 7496 return false;
7494 return vmcs12->exception_bitmap & 7497 return vmcs12->exception_bitmap &
7495 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 7498 (1u << (intr_info & INTR_INFO_VECTOR_MASK));
7496 case EXIT_REASON_EXTERNAL_INTERRUPT: 7499 case EXIT_REASON_EXTERNAL_INTERRUPT:
7497 return 0; 7500 return false;
7498 case EXIT_REASON_TRIPLE_FAULT: 7501 case EXIT_REASON_TRIPLE_FAULT:
7499 return 1; 7502 return true;
7500 case EXIT_REASON_PENDING_INTERRUPT: 7503 case EXIT_REASON_PENDING_INTERRUPT:
7501 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING); 7504 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
7502 case EXIT_REASON_NMI_WINDOW: 7505 case EXIT_REASON_NMI_WINDOW:
7503 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING); 7506 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
7504 case EXIT_REASON_TASK_SWITCH: 7507 case EXIT_REASON_TASK_SWITCH:
7505 return 1; 7508 return true;
7506 case EXIT_REASON_CPUID: 7509 case EXIT_REASON_CPUID:
7507 if (kvm_register_read(vcpu, VCPU_REGS_RAX) == 0xa) 7510 if (kvm_register_read(vcpu, VCPU_REGS_RAX) == 0xa)
7508 return 0; 7511 return false;
7509 return 1; 7512 return true;
7510 case EXIT_REASON_HLT: 7513 case EXIT_REASON_HLT:
7511 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 7514 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
7512 case EXIT_REASON_INVD: 7515 case EXIT_REASON_INVD:
7513 return 1; 7516 return true;
7514 case EXIT_REASON_INVLPG: 7517 case EXIT_REASON_INVLPG:
7515 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 7518 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
7516 case EXIT_REASON_RDPMC: 7519 case EXIT_REASON_RDPMC:
7517 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 7520 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
7518 case EXIT_REASON_RDTSC: 7521 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
7519 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 7522 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
7520 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 7523 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
7521 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 7524 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
@@ -7527,7 +7530,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7527 * VMX instructions trap unconditionally. This allows L1 to 7530 * VMX instructions trap unconditionally. This allows L1 to
7528 * emulate them for its L2 guest, i.e., allows 3-level nesting! 7531 * emulate them for its L2 guest, i.e., allows 3-level nesting!
7529 */ 7532 */
7530 return 1; 7533 return true;
7531 case EXIT_REASON_CR_ACCESS: 7534 case EXIT_REASON_CR_ACCESS:
7532 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 7535 return nested_vmx_exit_handled_cr(vcpu, vmcs12);
7533 case EXIT_REASON_DR_ACCESS: 7536 case EXIT_REASON_DR_ACCESS:
@@ -7538,7 +7541,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7538 case EXIT_REASON_MSR_WRITE: 7541 case EXIT_REASON_MSR_WRITE:
7539 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 7542 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
7540 case EXIT_REASON_INVALID_STATE: 7543 case EXIT_REASON_INVALID_STATE:
7541 return 1; 7544 return true;
7542 case EXIT_REASON_MWAIT_INSTRUCTION: 7545 case EXIT_REASON_MWAIT_INSTRUCTION:
7543 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 7546 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
7544 case EXIT_REASON_MONITOR_INSTRUCTION: 7547 case EXIT_REASON_MONITOR_INSTRUCTION:
@@ -7548,7 +7551,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7548 nested_cpu_has2(vmcs12, 7551 nested_cpu_has2(vmcs12,
7549 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 7552 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
7550 case EXIT_REASON_MCE_DURING_VMENTRY: 7553 case EXIT_REASON_MCE_DURING_VMENTRY:
7551 return 0; 7554 return false;
7552 case EXIT_REASON_TPR_BELOW_THRESHOLD: 7555 case EXIT_REASON_TPR_BELOW_THRESHOLD:
7553 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 7556 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
7554 case EXIT_REASON_APIC_ACCESS: 7557 case EXIT_REASON_APIC_ACCESS:
@@ -7557,7 +7560,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7557 case EXIT_REASON_APIC_WRITE: 7560 case EXIT_REASON_APIC_WRITE:
7558 case EXIT_REASON_EOI_INDUCED: 7561 case EXIT_REASON_EOI_INDUCED:
7559 /* apic_write and eoi_induced should exit unconditionally. */ 7562 /* apic_write and eoi_induced should exit unconditionally. */
7560 return 1; 7563 return true;
7561 case EXIT_REASON_EPT_VIOLATION: 7564 case EXIT_REASON_EPT_VIOLATION:
7562 /* 7565 /*
7563 * L0 always deals with the EPT violation. If nested EPT is 7566 * L0 always deals with the EPT violation. If nested EPT is
@@ -7565,7 +7568,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7565 * missing in the guest EPT table (EPT12), the EPT violation 7568 * missing in the guest EPT table (EPT12), the EPT violation
7566 * will be injected with nested_ept_inject_page_fault() 7569 * will be injected with nested_ept_inject_page_fault()
7567 */ 7570 */
7568 return 0; 7571 return false;
7569 case EXIT_REASON_EPT_MISCONFIG: 7572 case EXIT_REASON_EPT_MISCONFIG:
7570 /* 7573 /*
7571 * L2 never uses directly L1's EPT, but rather L0's own EPT 7574 * L2 never uses directly L1's EPT, but rather L0's own EPT
@@ -7573,11 +7576,11 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7573 * (EPT on EPT). So any problems with the structure of the 7576 * (EPT on EPT). So any problems with the structure of the
7574 * table is L0's fault. 7577 * table is L0's fault.
7575 */ 7578 */
7576 return 0; 7579 return false;
7577 case EXIT_REASON_WBINVD: 7580 case EXIT_REASON_WBINVD:
7578 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 7581 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
7579 case EXIT_REASON_XSETBV: 7582 case EXIT_REASON_XSETBV:
7580 return 1; 7583 return true;
7581 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 7584 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
7582 /* 7585 /*
7583 * This should never happen, since it is not possible to 7586 * This should never happen, since it is not possible to
@@ -7587,7 +7590,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
7587 */ 7590 */
7588 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 7591 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
7589 default: 7592 default:
7590 return 1; 7593 return true;
7591 } 7594 }
7592} 7595}
7593 7596
@@ -8522,6 +8525,9 @@ static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
8522 exec_control); 8525 exec_control);
8523 } 8526 }
8524 } 8527 }
8528 if (nested && !vmx->rdtscp_enabled)
8529 vmx->nested.nested_vmx_secondary_ctls_high &=
8530 ~SECONDARY_EXEC_RDTSCP;
8525 } 8531 }
8526 8532
8527 /* Exposing INVPCID only when PCID is exposed */ 8533 /* Exposing INVPCID only when PCID is exposed */
@@ -8622,10 +8628,11 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
8622 struct vmcs12 *vmcs12) 8628 struct vmcs12 *vmcs12)
8623{ 8629{
8624 struct vcpu_vmx *vmx = to_vmx(vcpu); 8630 struct vcpu_vmx *vmx = to_vmx(vcpu);
8631 int maxphyaddr = cpuid_maxphyaddr(vcpu);
8625 8632
8626 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 8633 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
8627 /* TODO: Also verify bits beyond physical address width are 0 */ 8634 if (!PAGE_ALIGNED(vmcs12->apic_access_addr) ||
8628 if (!PAGE_ALIGNED(vmcs12->apic_access_addr)) 8635 vmcs12->apic_access_addr >> maxphyaddr)
8629 return false; 8636 return false;
8630 8637
8631 /* 8638 /*
@@ -8641,8 +8648,8 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
8641 } 8648 }
8642 8649
8643 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 8650 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
8644 /* TODO: Also verify bits beyond physical address width are 0 */ 8651 if (!PAGE_ALIGNED(vmcs12->virtual_apic_page_addr) ||
8645 if (!PAGE_ALIGNED(vmcs12->virtual_apic_page_addr)) 8652 vmcs12->virtual_apic_page_addr >> maxphyaddr)
8646 return false; 8653 return false;
8647 8654
8648 if (vmx->nested.virtual_apic_page) /* shouldn't happen */ 8655 if (vmx->nested.virtual_apic_page) /* shouldn't happen */
@@ -8665,7 +8672,8 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
8665 } 8672 }
8666 8673
8667 if (nested_cpu_has_posted_intr(vmcs12)) { 8674 if (nested_cpu_has_posted_intr(vmcs12)) {
8668 if (!IS_ALIGNED(vmcs12->posted_intr_desc_addr, 64)) 8675 if (!IS_ALIGNED(vmcs12->posted_intr_desc_addr, 64) ||
8676 vmcs12->posted_intr_desc_addr >> maxphyaddr)
8669 return false; 8677 return false;
8670 8678
8671 if (vmx->nested.pi_desc_page) { /* shouldn't happen */ 8679 if (vmx->nested.pi_desc_page) { /* shouldn't happen */
@@ -8864,9 +8872,9 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
8864 8872
8865static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 8873static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
8866 unsigned long count_field, 8874 unsigned long count_field,
8867 unsigned long addr_field, 8875 unsigned long addr_field)
8868 int maxphyaddr)
8869{ 8876{
8877 int maxphyaddr;
8870 u64 count, addr; 8878 u64 count, addr;
8871 8879
8872 if (vmcs12_read_any(vcpu, count_field, &count) || 8880 if (vmcs12_read_any(vcpu, count_field, &count) ||
@@ -8876,6 +8884,7 @@ static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
8876 } 8884 }
8877 if (count == 0) 8885 if (count == 0)
8878 return 0; 8886 return 0;
8887 maxphyaddr = cpuid_maxphyaddr(vcpu);
8879 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr || 8888 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
8880 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) { 8889 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) {
8881 pr_warn_ratelimited( 8890 pr_warn_ratelimited(
@@ -8889,19 +8898,16 @@ static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
8889static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu, 8898static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu,
8890 struct vmcs12 *vmcs12) 8899 struct vmcs12 *vmcs12)
8891{ 8900{
8892 int maxphyaddr;
8893
8894 if (vmcs12->vm_exit_msr_load_count == 0 && 8901 if (vmcs12->vm_exit_msr_load_count == 0 &&
8895 vmcs12->vm_exit_msr_store_count == 0 && 8902 vmcs12->vm_exit_msr_store_count == 0 &&
8896 vmcs12->vm_entry_msr_load_count == 0) 8903 vmcs12->vm_entry_msr_load_count == 0)
8897 return 0; /* Fast path */ 8904 return 0; /* Fast path */
8898 maxphyaddr = cpuid_maxphyaddr(vcpu);
8899 if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT, 8905 if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT,
8900 VM_EXIT_MSR_LOAD_ADDR, maxphyaddr) || 8906 VM_EXIT_MSR_LOAD_ADDR) ||
8901 nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT, 8907 nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT,
8902 VM_EXIT_MSR_STORE_ADDR, maxphyaddr) || 8908 VM_EXIT_MSR_STORE_ADDR) ||
8903 nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT, 8909 nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT,
8904 VM_ENTRY_MSR_LOAD_ADDR, maxphyaddr)) 8910 VM_ENTRY_MSR_LOAD_ADDR))
8905 return -EINVAL; 8911 return -EINVAL;
8906 return 0; 8912 return 0;
8907} 8913}
@@ -9151,8 +9157,9 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
9151 exec_control &= ~SECONDARY_EXEC_RDTSCP; 9157 exec_control &= ~SECONDARY_EXEC_RDTSCP;
9152 /* Take the following fields only from vmcs12 */ 9158 /* Take the following fields only from vmcs12 */
9153 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 9159 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
9160 SECONDARY_EXEC_RDTSCP |
9154 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 9161 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
9155 SECONDARY_EXEC_APIC_REGISTER_VIRT); 9162 SECONDARY_EXEC_APIC_REGISTER_VIRT);
9156 if (nested_cpu_has(vmcs12, 9163 if (nested_cpu_has(vmcs12,
9157 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) 9164 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
9158 exec_control |= vmcs12->secondary_vm_exec_control; 9165 exec_control |= vmcs12->secondary_vm_exec_control;
@@ -9385,7 +9392,6 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
9385 } 9392 }
9386 9393
9387 if (!nested_get_vmcs12_pages(vcpu, vmcs12)) { 9394 if (!nested_get_vmcs12_pages(vcpu, vmcs12)) {
9388 /*TODO: Also verify bits beyond physical address width are 0*/
9389 nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 9395 nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
9390 return 1; 9396 return 1;
9391 } 9397 }
@@ -9524,7 +9530,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
9524 vmcs12->launch_state = 1; 9530 vmcs12->launch_state = 1;
9525 9531
9526 if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) 9532 if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT)
9527 return kvm_emulate_halt(vcpu); 9533 return kvm_vcpu_halt(vcpu);
9528 9534
9529 vmx->nested.nested_run_pending = 1; 9535 vmx->nested.nested_run_pending = 1;
9530 9536
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 32bf19ef3115..2b2dd030ea3b 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -801,6 +801,17 @@ unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
801} 801}
802EXPORT_SYMBOL_GPL(kvm_get_cr8); 802EXPORT_SYMBOL_GPL(kvm_get_cr8);
803 803
804static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
805{
806 int i;
807
808 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
809 for (i = 0; i < KVM_NR_DB_REGS; i++)
810 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
811 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
812 }
813}
814
804static void kvm_update_dr6(struct kvm_vcpu *vcpu) 815static void kvm_update_dr6(struct kvm_vcpu *vcpu)
805{ 816{
806 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) 817 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
@@ -3149,6 +3160,7 @@ static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
3149 return -EINVAL; 3160 return -EINVAL;
3150 3161
3151 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db)); 3162 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
3163 kvm_update_dr0123(vcpu);
3152 vcpu->arch.dr6 = dbgregs->dr6; 3164 vcpu->arch.dr6 = dbgregs->dr6;
3153 kvm_update_dr6(vcpu); 3165 kvm_update_dr6(vcpu);
3154 vcpu->arch.dr7 = dbgregs->dr7; 3166 vcpu->arch.dr7 = dbgregs->dr7;
@@ -4114,8 +4126,8 @@ static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
4114 do { 4126 do {
4115 n = min(len, 8); 4127 n = min(len, 8);
4116 if (!(vcpu->arch.apic && 4128 if (!(vcpu->arch.apic &&
4117 !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v)) 4129 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
4118 && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v)) 4130 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
4119 break; 4131 break;
4120 handled += n; 4132 handled += n;
4121 addr += n; 4133 addr += n;
@@ -4134,8 +4146,9 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
4134 do { 4146 do {
4135 n = min(len, 8); 4147 n = min(len, 8);
4136 if (!(vcpu->arch.apic && 4148 if (!(vcpu->arch.apic &&
4137 !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v)) 4149 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
4138 && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v)) 4150 addr, n, v))
4151 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
4139 break; 4152 break;
4140 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v); 4153 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v);
4141 handled += n; 4154 handled += n;
@@ -4475,7 +4488,8 @@ mmio:
4475 return X86EMUL_CONTINUE; 4488 return X86EMUL_CONTINUE;
4476} 4489}
4477 4490
4478int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr, 4491static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
4492 unsigned long addr,
4479 void *val, unsigned int bytes, 4493 void *val, unsigned int bytes,
4480 struct x86_exception *exception, 4494 struct x86_exception *exception,
4481 const struct read_write_emulator_ops *ops) 4495 const struct read_write_emulator_ops *ops)
@@ -4538,7 +4552,7 @@ static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
4538 exception, &read_emultor); 4552 exception, &read_emultor);
4539} 4553}
4540 4554
4541int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, 4555static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
4542 unsigned long addr, 4556 unsigned long addr,
4543 const void *val, 4557 const void *val,
4544 unsigned int bytes, 4558 unsigned int bytes,
@@ -4629,10 +4643,10 @@ static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
4629 int r; 4643 int r;
4630 4644
4631 if (vcpu->arch.pio.in) 4645 if (vcpu->arch.pio.in)
4632 r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port, 4646 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
4633 vcpu->arch.pio.size, pd); 4647 vcpu->arch.pio.size, pd);
4634 else 4648 else
4635 r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS, 4649 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
4636 vcpu->arch.pio.port, vcpu->arch.pio.size, 4650 vcpu->arch.pio.port, vcpu->arch.pio.size,
4637 pd); 4651 pd);
4638 return r; 4652 return r;
@@ -4705,7 +4719,7 @@ static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
4705 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); 4719 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
4706} 4720}
4707 4721
4708int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) 4722int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
4709{ 4723{
4710 if (!need_emulate_wbinvd(vcpu)) 4724 if (!need_emulate_wbinvd(vcpu))
4711 return X86EMUL_CONTINUE; 4725 return X86EMUL_CONTINUE;
@@ -4722,19 +4736,29 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
4722 wbinvd(); 4736 wbinvd();
4723 return X86EMUL_CONTINUE; 4737 return X86EMUL_CONTINUE;
4724} 4738}
4739
4740int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
4741{
4742 kvm_x86_ops->skip_emulated_instruction(vcpu);
4743 return kvm_emulate_wbinvd_noskip(vcpu);
4744}
4725EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd); 4745EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
4726 4746
4747
4748
4727static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) 4749static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
4728{ 4750{
4729 kvm_emulate_wbinvd(emul_to_vcpu(ctxt)); 4751 kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
4730} 4752}
4731 4753
4732int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) 4754static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
4755 unsigned long *dest)
4733{ 4756{
4734 return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest); 4757 return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
4735} 4758}
4736 4759
4737int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) 4760static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
4761 unsigned long value)
4738{ 4762{
4739 4763
4740 return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value); 4764 return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
@@ -5816,7 +5840,7 @@ void kvm_arch_exit(void)
5816 free_percpu(shared_msrs); 5840 free_percpu(shared_msrs);
5817} 5841}
5818 5842
5819int kvm_emulate_halt(struct kvm_vcpu *vcpu) 5843int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
5820{ 5844{
5821 ++vcpu->stat.halt_exits; 5845 ++vcpu->stat.halt_exits;
5822 if (irqchip_in_kernel(vcpu->kvm)) { 5846 if (irqchip_in_kernel(vcpu->kvm)) {
@@ -5827,6 +5851,13 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu)
5827 return 0; 5851 return 0;
5828 } 5852 }
5829} 5853}
5854EXPORT_SYMBOL_GPL(kvm_vcpu_halt);
5855
5856int kvm_emulate_halt(struct kvm_vcpu *vcpu)
5857{
5858 kvm_x86_ops->skip_emulated_instruction(vcpu);
5859 return kvm_vcpu_halt(vcpu);
5860}
5830EXPORT_SYMBOL_GPL(kvm_emulate_halt); 5861EXPORT_SYMBOL_GPL(kvm_emulate_halt);
5831 5862
5832int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 5863int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
@@ -5903,7 +5934,7 @@ static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
5903 lapic_irq.dest_id = apicid; 5934 lapic_irq.dest_id = apicid;
5904 5935
5905 lapic_irq.delivery_mode = APIC_DM_REMRD; 5936 lapic_irq.delivery_mode = APIC_DM_REMRD;
5906 kvm_irq_delivery_to_apic(kvm, 0, &lapic_irq, NULL); 5937 kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
5907} 5938}
5908 5939
5909int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) 5940int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
@@ -5911,6 +5942,8 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
5911 unsigned long nr, a0, a1, a2, a3, ret; 5942 unsigned long nr, a0, a1, a2, a3, ret;
5912 int op_64_bit, r = 1; 5943 int op_64_bit, r = 1;
5913 5944
5945 kvm_x86_ops->skip_emulated_instruction(vcpu);
5946
5914 if (kvm_hv_hypercall_enabled(vcpu->kvm)) 5947 if (kvm_hv_hypercall_enabled(vcpu->kvm))
5915 return kvm_hv_hypercall(vcpu); 5948 return kvm_hv_hypercall(vcpu);
5916 5949
@@ -6164,7 +6197,7 @@ void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
6164} 6197}
6165 6198
6166/* 6199/*
6167 * Returns 1 to let __vcpu_run() continue the guest execution loop without 6200 * Returns 1 to let vcpu_run() continue the guest execution loop without
6168 * exiting to the userspace. Otherwise, the value will be returned to the 6201 * exiting to the userspace. Otherwise, the value will be returned to the
6169 * userspace. 6202 * userspace.
6170 */ 6203 */
@@ -6301,6 +6334,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
6301 set_debugreg(vcpu->arch.eff_db[2], 2); 6334 set_debugreg(vcpu->arch.eff_db[2], 2);
6302 set_debugreg(vcpu->arch.eff_db[3], 3); 6335 set_debugreg(vcpu->arch.eff_db[3], 3);
6303 set_debugreg(vcpu->arch.dr6, 6); 6336 set_debugreg(vcpu->arch.dr6, 6);
6337 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
6304 } 6338 }
6305 6339
6306 trace_kvm_entry(vcpu->vcpu_id); 6340 trace_kvm_entry(vcpu->vcpu_id);
@@ -6382,42 +6416,47 @@ out:
6382 return r; 6416 return r;
6383} 6417}
6384 6418
6419static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
6420{
6421 if (!kvm_arch_vcpu_runnable(vcpu)) {
6422 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
6423 kvm_vcpu_block(vcpu);
6424 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6425 if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
6426 return 1;
6427 }
6428
6429 kvm_apic_accept_events(vcpu);
6430 switch(vcpu->arch.mp_state) {
6431 case KVM_MP_STATE_HALTED:
6432 vcpu->arch.pv.pv_unhalted = false;
6433 vcpu->arch.mp_state =
6434 KVM_MP_STATE_RUNNABLE;
6435 case KVM_MP_STATE_RUNNABLE:
6436 vcpu->arch.apf.halted = false;
6437 break;
6438 case KVM_MP_STATE_INIT_RECEIVED:
6439 break;
6440 default:
6441 return -EINTR;
6442 break;
6443 }
6444 return 1;
6445}
6385 6446
6386static int __vcpu_run(struct kvm_vcpu *vcpu) 6447static int vcpu_run(struct kvm_vcpu *vcpu)
6387{ 6448{
6388 int r; 6449 int r;
6389 struct kvm *kvm = vcpu->kvm; 6450 struct kvm *kvm = vcpu->kvm;
6390 6451
6391 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 6452 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6392 6453
6393 r = 1; 6454 for (;;) {
6394 while (r > 0) {
6395 if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 6455 if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
6396 !vcpu->arch.apf.halted) 6456 !vcpu->arch.apf.halted)
6397 r = vcpu_enter_guest(vcpu); 6457 r = vcpu_enter_guest(vcpu);
6398 else { 6458 else
6399 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 6459 r = vcpu_block(kvm, vcpu);
6400 kvm_vcpu_block(vcpu);
6401 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6402 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
6403 kvm_apic_accept_events(vcpu);
6404 switch(vcpu->arch.mp_state) {
6405 case KVM_MP_STATE_HALTED:
6406 vcpu->arch.pv.pv_unhalted = false;
6407 vcpu->arch.mp_state =
6408 KVM_MP_STATE_RUNNABLE;
6409 case KVM_MP_STATE_RUNNABLE:
6410 vcpu->arch.apf.halted = false;
6411 break;
6412 case KVM_MP_STATE_INIT_RECEIVED:
6413 break;
6414 default:
6415 r = -EINTR;
6416 break;
6417 }
6418 }
6419 }
6420
6421 if (r <= 0) 6460 if (r <= 0)
6422 break; 6461 break;
6423 6462
@@ -6429,6 +6468,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
6429 r = -EINTR; 6468 r = -EINTR;
6430 vcpu->run->exit_reason = KVM_EXIT_INTR; 6469 vcpu->run->exit_reason = KVM_EXIT_INTR;
6431 ++vcpu->stat.request_irq_exits; 6470 ++vcpu->stat.request_irq_exits;
6471 break;
6432 } 6472 }
6433 6473
6434 kvm_check_async_pf_completion(vcpu); 6474 kvm_check_async_pf_completion(vcpu);
@@ -6437,6 +6477,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
6437 r = -EINTR; 6477 r = -EINTR;
6438 vcpu->run->exit_reason = KVM_EXIT_INTR; 6478 vcpu->run->exit_reason = KVM_EXIT_INTR;
6439 ++vcpu->stat.signal_exits; 6479 ++vcpu->stat.signal_exits;
6480 break;
6440 } 6481 }
6441 if (need_resched()) { 6482 if (need_resched()) {
6442 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 6483 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
@@ -6568,7 +6609,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
6568 } else 6609 } else
6569 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed); 6610 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
6570 6611
6571 r = __vcpu_run(vcpu); 6612 r = vcpu_run(vcpu);
6572 6613
6573out: 6614out:
6574 post_kvm_run_save(vcpu); 6615 post_kvm_run_save(vcpu);
@@ -7075,11 +7116,14 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu)
7075 kvm_clear_exception_queue(vcpu); 7116 kvm_clear_exception_queue(vcpu);
7076 7117
7077 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); 7118 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
7119 kvm_update_dr0123(vcpu);
7078 vcpu->arch.dr6 = DR6_INIT; 7120 vcpu->arch.dr6 = DR6_INIT;
7079 kvm_update_dr6(vcpu); 7121 kvm_update_dr6(vcpu);
7080 vcpu->arch.dr7 = DR7_FIXED_1; 7122 vcpu->arch.dr7 = DR7_FIXED_1;
7081 kvm_update_dr7(vcpu); 7123 kvm_update_dr7(vcpu);
7082 7124
7125 vcpu->arch.cr2 = 0;
7126
7083 kvm_make_request(KVM_REQ_EVENT, vcpu); 7127 kvm_make_request(KVM_REQ_EVENT, vcpu);
7084 vcpu->arch.apf.msr_val = 0; 7128 vcpu->arch.apf.msr_val = 0;
7085 vcpu->arch.st.msr_val = 0; 7129 vcpu->arch.st.msr_val = 0;
@@ -7240,7 +7284,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
7240 7284
7241 vcpu->arch.pv.pv_unhalted = false; 7285 vcpu->arch.pv.pv_unhalted = false;
7242 vcpu->arch.emulate_ctxt.ops = &emulate_ops; 7286 vcpu->arch.emulate_ctxt.ops = &emulate_ops;
7243 if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu)) 7287 if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_reset_bsp(vcpu))
7244 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 7288 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
7245 else 7289 else
7246 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED; 7290 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
@@ -7288,6 +7332,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
7288 vcpu->arch.guest_supported_xcr0 = 0; 7332 vcpu->arch.guest_supported_xcr0 = 0;
7289 vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; 7333 vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
7290 7334
7335 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
7336
7291 kvm_async_pf_hash_reset(vcpu); 7337 kvm_async_pf_hash_reset(vcpu);
7292 kvm_pmu_init(vcpu); 7338 kvm_pmu_init(vcpu);
7293 7339
@@ -7428,7 +7474,7 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
7428 7474
7429 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 7475 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
7430 if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) { 7476 if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
7431 kvm_kvfree(free->arch.rmap[i]); 7477 kvfree(free->arch.rmap[i]);
7432 free->arch.rmap[i] = NULL; 7478 free->arch.rmap[i] = NULL;
7433 } 7479 }
7434 if (i == 0) 7480 if (i == 0)
@@ -7436,7 +7482,7 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
7436 7482
7437 if (!dont || free->arch.lpage_info[i - 1] != 7483 if (!dont || free->arch.lpage_info[i - 1] !=
7438 dont->arch.lpage_info[i - 1]) { 7484 dont->arch.lpage_info[i - 1]) {
7439 kvm_kvfree(free->arch.lpage_info[i - 1]); 7485 kvfree(free->arch.lpage_info[i - 1]);
7440 free->arch.lpage_info[i - 1] = NULL; 7486 free->arch.lpage_info[i - 1] = NULL;
7441 } 7487 }
7442 } 7488 }
@@ -7490,12 +7536,12 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
7490 7536
7491out_free: 7537out_free:
7492 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 7538 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
7493 kvm_kvfree(slot->arch.rmap[i]); 7539 kvfree(slot->arch.rmap[i]);
7494 slot->arch.rmap[i] = NULL; 7540 slot->arch.rmap[i] = NULL;
7495 if (i == 0) 7541 if (i == 0)
7496 continue; 7542 continue;
7497 7543
7498 kvm_kvfree(slot->arch.lpage_info[i - 1]); 7544 kvfree(slot->arch.lpage_info[i - 1]);
7499 slot->arch.lpage_info[i - 1] = NULL; 7545 slot->arch.lpage_info[i - 1] = NULL;
7500 } 7546 }
7501 return -ENOMEM; 7547 return -ENOMEM;
@@ -7618,6 +7664,23 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
7618 new = id_to_memslot(kvm->memslots, mem->slot); 7664 new = id_to_memslot(kvm->memslots, mem->slot);
7619 7665
7620 /* 7666 /*
7667 * Dirty logging tracks sptes in 4k granularity, meaning that large
7668 * sptes have to be split. If live migration is successful, the guest
7669 * in the source machine will be destroyed and large sptes will be
7670 * created in the destination. However, if the guest continues to run
7671 * in the source machine (for example if live migration fails), small
7672 * sptes will remain around and cause bad performance.
7673 *
7674 * Scan sptes if dirty logging has been stopped, dropping those
7675 * which can be collapsed into a single large-page spte. Later
7676 * page faults will create the large-page sptes.
7677 */
7678 if ((change != KVM_MR_DELETE) &&
7679 (old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
7680 !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
7681 kvm_mmu_zap_collapsible_sptes(kvm, new);
7682
7683 /*
7621 * Set up write protection and/or dirty logging for the new slot. 7684 * Set up write protection and/or dirty logging for the new slot.
7622 * 7685 *
7623 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have 7686 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 9793322751e0..40d2473836c9 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -82,18 +82,15 @@ static notrace cycle_t vread_pvclock(int *mode)
82 cycle_t ret; 82 cycle_t ret;
83 u64 last; 83 u64 last;
84 u32 version; 84 u32 version;
85 u32 migrate_count;
85 u8 flags; 86 u8 flags;
86 unsigned cpu, cpu1; 87 unsigned cpu, cpu1;
87 88
88 89
89 /* 90 /*
90 * Note: hypervisor must guarantee that: 91 * When looping to get a consistent (time-info, tsc) pair, we
91 * 1. cpu ID number maps 1:1 to per-CPU pvclock time info. 92 * also need to deal with the possibility we can switch vcpus,
92 * 2. that per-CPU pvclock time info is updated if the 93 * so make sure we always re-fetch time-info for the current vcpu.
93 * underlying CPU changes.
94 * 3. that version is increased whenever underlying CPU
95 * changes.
96 *
97 */ 94 */
98 do { 95 do {
99 cpu = __getcpu() & VGETCPU_CPU_MASK; 96 cpu = __getcpu() & VGETCPU_CPU_MASK;
@@ -102,20 +99,27 @@ static notrace cycle_t vread_pvclock(int *mode)
102 * __getcpu() calls (Gleb). 99 * __getcpu() calls (Gleb).
103 */ 100 */
104 101
105 pvti = get_pvti(cpu); 102 /* Make sure migrate_count will change if we leave the VCPU. */
103 do {
104 pvti = get_pvti(cpu);
105 migrate_count = pvti->migrate_count;
106
107 cpu1 = cpu;
108 cpu = __getcpu() & VGETCPU_CPU_MASK;
109 } while (unlikely(cpu != cpu1));
106 110
107 version = __pvclock_read_cycles(&pvti->pvti, &ret, &flags); 111 version = __pvclock_read_cycles(&pvti->pvti, &ret, &flags);
108 112
109 /* 113 /*
110 * Test we're still on the cpu as well as the version. 114 * Test we're still on the cpu as well as the version.
111 * We could have been migrated just after the first 115 * - We must read TSC of pvti's VCPU.
112 * vgetcpu but before fetching the version, so we 116 * - KVM doesn't follow the versioning protocol, so data could
113 * wouldn't notice a version change. 117 * change before version if we left the VCPU.
114 */ 118 */
115 cpu1 = __getcpu() & VGETCPU_CPU_MASK; 119 smp_rmb();
116 } while (unlikely(cpu != cpu1 || 120 } while (unlikely((pvti->pvti.version & 1) ||
117 (pvti->pvti.version & 1) || 121 pvti->pvti.version != version ||
118 pvti->pvti.version != version)); 122 pvti->migrate_count != migrate_count));
119 123
120 if (unlikely(!(flags & PVCLOCK_TSC_STABLE_BIT))) 124 if (unlikely(!(flags & PVCLOCK_TSC_STABLE_BIT)))
121 *mode = VCLOCK_NONE; 125 *mode = VCLOCK_NONE;
diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h
index b3f45a578344..e5966758c093 100644
--- a/include/kvm/arm_arch_timer.h
+++ b/include/kvm/arm_arch_timer.h
@@ -24,17 +24,14 @@
24#include <linux/workqueue.h> 24#include <linux/workqueue.h>
25 25
26struct arch_timer_kvm { 26struct arch_timer_kvm {
27#ifdef CONFIG_KVM_ARM_TIMER
28 /* Is the timer enabled */ 27 /* Is the timer enabled */
29 bool enabled; 28 bool enabled;
30 29
31 /* Virtual offset */ 30 /* Virtual offset */
32 cycle_t cntvoff; 31 cycle_t cntvoff;
33#endif
34}; 32};
35 33
36struct arch_timer_cpu { 34struct arch_timer_cpu {
37#ifdef CONFIG_KVM_ARM_TIMER
38 /* Registers: control register, timer value */ 35 /* Registers: control register, timer value */
39 u32 cntv_ctl; /* Saved/restored */ 36 u32 cntv_ctl; /* Saved/restored */
40 cycle_t cntv_cval; /* Saved/restored */ 37 cycle_t cntv_cval; /* Saved/restored */
@@ -55,10 +52,8 @@ struct arch_timer_cpu {
55 52
56 /* Timer IRQ */ 53 /* Timer IRQ */
57 const struct kvm_irq_level *irq; 54 const struct kvm_irq_level *irq;
58#endif
59}; 55};
60 56
61#ifdef CONFIG_KVM_ARM_TIMER
62int kvm_timer_hyp_init(void); 57int kvm_timer_hyp_init(void);
63void kvm_timer_enable(struct kvm *kvm); 58void kvm_timer_enable(struct kvm *kvm);
64void kvm_timer_init(struct kvm *kvm); 59void kvm_timer_init(struct kvm *kvm);
@@ -72,30 +67,6 @@ void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu);
72u64 kvm_arm_timer_get_reg(struct kvm_vcpu *, u64 regid); 67u64 kvm_arm_timer_get_reg(struct kvm_vcpu *, u64 regid);
73int kvm_arm_timer_set_reg(struct kvm_vcpu *, u64 regid, u64 value); 68int kvm_arm_timer_set_reg(struct kvm_vcpu *, u64 regid, u64 value);
74 69
75#else 70bool kvm_timer_should_fire(struct kvm_vcpu *vcpu);
76static inline int kvm_timer_hyp_init(void)
77{
78 return 0;
79};
80
81static inline void kvm_timer_enable(struct kvm *kvm) {}
82static inline void kvm_timer_init(struct kvm *kvm) {}
83static inline void kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu,
84 const struct kvm_irq_level *irq) {}
85static inline void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) {}
86static inline void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu) {}
87static inline void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu) {}
88static inline void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu) {}
89
90static inline int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
91{
92 return 0;
93}
94
95static inline u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
96{
97 return 0;
98}
99#endif
100 71
101#endif 72#endif
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 66203b268984..133ea00aa83b 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -24,6 +24,7 @@
24#include <linux/irqreturn.h> 24#include <linux/irqreturn.h>
25#include <linux/spinlock.h> 25#include <linux/spinlock.h>
26#include <linux/types.h> 26#include <linux/types.h>
27#include <kvm/iodev.h>
27 28
28#define VGIC_NR_IRQS_LEGACY 256 29#define VGIC_NR_IRQS_LEGACY 256
29#define VGIC_NR_SGIS 16 30#define VGIC_NR_SGIS 16
@@ -140,16 +141,21 @@ struct vgic_params {
140}; 141};
141 142
142struct vgic_vm_ops { 143struct vgic_vm_ops {
143 bool (*handle_mmio)(struct kvm_vcpu *, struct kvm_run *,
144 struct kvm_exit_mmio *);
145 bool (*queue_sgi)(struct kvm_vcpu *, int irq); 144 bool (*queue_sgi)(struct kvm_vcpu *, int irq);
146 void (*add_sgi_source)(struct kvm_vcpu *, int irq, int source); 145 void (*add_sgi_source)(struct kvm_vcpu *, int irq, int source);
147 int (*init_model)(struct kvm *); 146 int (*init_model)(struct kvm *);
148 int (*map_resources)(struct kvm *, const struct vgic_params *); 147 int (*map_resources)(struct kvm *, const struct vgic_params *);
149}; 148};
150 149
150struct vgic_io_device {
151 gpa_t addr;
152 int len;
153 const struct vgic_io_range *reg_ranges;
154 struct kvm_vcpu *redist_vcpu;
155 struct kvm_io_device dev;
156};
157
151struct vgic_dist { 158struct vgic_dist {
152#ifdef CONFIG_KVM_ARM_VGIC
153 spinlock_t lock; 159 spinlock_t lock;
154 bool in_kernel; 160 bool in_kernel;
155 bool ready; 161 bool ready;
@@ -197,6 +203,9 @@ struct vgic_dist {
197 /* Level-triggered interrupt queued on VCPU interface */ 203 /* Level-triggered interrupt queued on VCPU interface */
198 struct vgic_bitmap irq_queued; 204 struct vgic_bitmap irq_queued;
199 205
206 /* Interrupt was active when unqueue from VCPU interface */
207 struct vgic_bitmap irq_active;
208
200 /* Interrupt priority. Not used yet. */ 209 /* Interrupt priority. Not used yet. */
201 struct vgic_bytemap irq_priority; 210 struct vgic_bytemap irq_priority;
202 211
@@ -237,8 +246,12 @@ struct vgic_dist {
237 /* Bitmap indicating which CPU has something pending */ 246 /* Bitmap indicating which CPU has something pending */
238 unsigned long *irq_pending_on_cpu; 247 unsigned long *irq_pending_on_cpu;
239 248
249 /* Bitmap indicating which CPU has active IRQs */
250 unsigned long *irq_active_on_cpu;
251
240 struct vgic_vm_ops vm_ops; 252 struct vgic_vm_ops vm_ops;
241#endif 253 struct vgic_io_device dist_iodev;
254 struct vgic_io_device *redist_iodevs;
242}; 255};
243 256
244struct vgic_v2_cpu_if { 257struct vgic_v2_cpu_if {
@@ -266,13 +279,18 @@ struct vgic_v3_cpu_if {
266}; 279};
267 280
268struct vgic_cpu { 281struct vgic_cpu {
269#ifdef CONFIG_KVM_ARM_VGIC
270 /* per IRQ to LR mapping */ 282 /* per IRQ to LR mapping */
271 u8 *vgic_irq_lr_map; 283 u8 *vgic_irq_lr_map;
272 284
273 /* Pending interrupts on this VCPU */ 285 /* Pending/active/both interrupts on this VCPU */
274 DECLARE_BITMAP( pending_percpu, VGIC_NR_PRIVATE_IRQS); 286 DECLARE_BITMAP( pending_percpu, VGIC_NR_PRIVATE_IRQS);
287 DECLARE_BITMAP( active_percpu, VGIC_NR_PRIVATE_IRQS);
288 DECLARE_BITMAP( pend_act_percpu, VGIC_NR_PRIVATE_IRQS);
289
290 /* Pending/active/both shared interrupts, dynamically sized */
275 unsigned long *pending_shared; 291 unsigned long *pending_shared;
292 unsigned long *active_shared;
293 unsigned long *pend_act_shared;
276 294
277 /* Bitmap of used/free list registers */ 295 /* Bitmap of used/free list registers */
278 DECLARE_BITMAP( lr_used, VGIC_V2_MAX_LRS); 296 DECLARE_BITMAP( lr_used, VGIC_V2_MAX_LRS);
@@ -285,7 +303,6 @@ struct vgic_cpu {
285 struct vgic_v2_cpu_if vgic_v2; 303 struct vgic_v2_cpu_if vgic_v2;
286 struct vgic_v3_cpu_if vgic_v3; 304 struct vgic_v3_cpu_if vgic_v3;
287 }; 305 };
288#endif
289}; 306};
290 307
291#define LR_EMPTY 0xff 308#define LR_EMPTY 0xff
@@ -295,10 +312,7 @@ struct vgic_cpu {
295 312
296struct kvm; 313struct kvm;
297struct kvm_vcpu; 314struct kvm_vcpu;
298struct kvm_run;
299struct kvm_exit_mmio;
300 315
301#ifdef CONFIG_KVM_ARM_VGIC
302int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write); 316int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write);
303int kvm_vgic_hyp_init(void); 317int kvm_vgic_hyp_init(void);
304int kvm_vgic_map_resources(struct kvm *kvm); 318int kvm_vgic_map_resources(struct kvm *kvm);
@@ -312,8 +326,7 @@ int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
312 bool level); 326 bool level);
313void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg); 327void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg);
314int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu); 328int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
315bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run, 329int kvm_vgic_vcpu_active_irq(struct kvm_vcpu *vcpu);
316 struct kvm_exit_mmio *mmio);
317 330
318#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel)) 331#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel))
319#define vgic_initialized(k) (!!((k)->arch.vgic.nr_cpus)) 332#define vgic_initialized(k) (!!((k)->arch.vgic.nr_cpus))
@@ -335,84 +348,4 @@ static inline int vgic_v3_probe(struct device_node *vgic_node,
335} 348}
336#endif 349#endif
337 350
338#else
339static inline int kvm_vgic_hyp_init(void)
340{
341 return 0;
342}
343
344static inline int kvm_vgic_set_addr(struct kvm *kvm, unsigned long type, u64 addr)
345{
346 return 0;
347}
348
349static inline int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
350{
351 return -ENXIO;
352}
353
354static inline int kvm_vgic_map_resources(struct kvm *kvm)
355{
356 return 0;
357}
358
359static inline int kvm_vgic_create(struct kvm *kvm, u32 type)
360{
361 return 0;
362}
363
364static inline void kvm_vgic_destroy(struct kvm *kvm)
365{
366}
367
368static inline void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
369{
370}
371
372static inline int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
373{
374 return 0;
375}
376
377static inline void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) {}
378static inline void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) {}
379
380static inline int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid,
381 unsigned int irq_num, bool level)
382{
383 return 0;
384}
385
386static inline int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
387{
388 return 0;
389}
390
391static inline bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
392 struct kvm_exit_mmio *mmio)
393{
394 return false;
395}
396
397static inline int irqchip_in_kernel(struct kvm *kvm)
398{
399 return 0;
400}
401
402static inline bool vgic_initialized(struct kvm *kvm)
403{
404 return true;
405}
406
407static inline bool vgic_ready(struct kvm *kvm)
408{
409 return true;
410}
411
412static inline int kvm_vgic_get_max_vcpus(void)
413{
414 return KVM_MAX_VCPUS;
415}
416#endif
417
418#endif 351#endif
diff --git a/virt/kvm/iodev.h b/include/kvm/iodev.h
index 12fd3caffd2b..a6d208b916f5 100644
--- a/virt/kvm/iodev.h
+++ b/include/kvm/iodev.h
@@ -9,17 +9,17 @@
9 * GNU General Public License for more details. 9 * GNU General Public License for more details.
10 * 10 *
11 * You should have received a copy of the GNU General Public License 11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software 12 * along with this program. If not, see <http://www.gnu.org/licenses/>.
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 */ 13 */
15 14
16#ifndef __KVM_IODEV_H__ 15#ifndef __KVM_IODEV_H__
17#define __KVM_IODEV_H__ 16#define __KVM_IODEV_H__
18 17
19#include <linux/kvm_types.h> 18#include <linux/kvm_types.h>
20#include <asm/errno.h> 19#include <linux/errno.h>
21 20
22struct kvm_io_device; 21struct kvm_io_device;
22struct kvm_vcpu;
23 23
24/** 24/**
25 * kvm_io_device_ops are called under kvm slots_lock. 25 * kvm_io_device_ops are called under kvm slots_lock.
@@ -27,11 +27,13 @@ struct kvm_io_device;
27 * or non-zero to have it passed to the next device. 27 * or non-zero to have it passed to the next device.
28 **/ 28 **/
29struct kvm_io_device_ops { 29struct kvm_io_device_ops {
30 int (*read)(struct kvm_io_device *this, 30 int (*read)(struct kvm_vcpu *vcpu,
31 struct kvm_io_device *this,
31 gpa_t addr, 32 gpa_t addr,
32 int len, 33 int len,
33 void *val); 34 void *val);
34 int (*write)(struct kvm_io_device *this, 35 int (*write)(struct kvm_vcpu *vcpu,
36 struct kvm_io_device *this,
35 gpa_t addr, 37 gpa_t addr,
36 int len, 38 int len,
37 const void *val); 39 const void *val);
@@ -49,16 +51,20 @@ static inline void kvm_iodevice_init(struct kvm_io_device *dev,
49 dev->ops = ops; 51 dev->ops = ops;
50} 52}
51 53
52static inline int kvm_iodevice_read(struct kvm_io_device *dev, 54static inline int kvm_iodevice_read(struct kvm_vcpu *vcpu,
53 gpa_t addr, int l, void *v) 55 struct kvm_io_device *dev, gpa_t addr,
56 int l, void *v)
54{ 57{
55 return dev->ops->read ? dev->ops->read(dev, addr, l, v) : -EOPNOTSUPP; 58 return dev->ops->read ? dev->ops->read(vcpu, dev, addr, l, v)
59 : -EOPNOTSUPP;
56} 60}
57 61
58static inline int kvm_iodevice_write(struct kvm_io_device *dev, 62static inline int kvm_iodevice_write(struct kvm_vcpu *vcpu,
59 gpa_t addr, int l, const void *v) 63 struct kvm_io_device *dev, gpa_t addr,
64 int l, const void *v)
60{ 65{
61 return dev->ops->write ? dev->ops->write(dev, addr, l, v) : -EOPNOTSUPP; 66 return dev->ops->write ? dev->ops->write(vcpu, dev, addr, l, v)
67 : -EOPNOTSUPP;
62} 68}
63 69
64static inline void kvm_iodevice_destructor(struct kvm_io_device *dev) 70static inline void kvm_iodevice_destructor(struct kvm_io_device *dev)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index d12b2104d19b..82af5d0b996e 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -165,12 +165,12 @@ enum kvm_bus {
165 KVM_NR_BUSES 165 KVM_NR_BUSES
166}; 166};
167 167
168int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 168int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
169 int len, const void *val); 169 int len, const void *val);
170int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 170int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
171 int len, const void *val, long cookie); 171 gpa_t addr, int len, const void *val, long cookie);
172int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len, 172int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
173 void *val); 173 int len, void *val);
174int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 174int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
175 int len, struct kvm_io_device *dev); 175 int len, struct kvm_io_device *dev);
176int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 176int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
@@ -658,7 +658,6 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
658int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu); 658int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
659 659
660void *kvm_kvzalloc(unsigned long size); 660void *kvm_kvzalloc(unsigned long size);
661void kvm_kvfree(const void *addr);
662 661
663#ifndef __KVM_HAVE_ARCH_VM_ALLOC 662#ifndef __KVM_HAVE_ARCH_VM_ALLOC
664static inline struct kvm *kvm_arch_alloc_vm(void) 663static inline struct kvm *kvm_arch_alloc_vm(void)
@@ -700,6 +699,20 @@ static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
700#endif 699#endif
701} 700}
702 701
702#ifdef __KVM_HAVE_ARCH_INTC_INITIALIZED
703/*
704 * returns true if the virtual interrupt controller is initialized and
705 * ready to accept virtual IRQ. On some architectures the virtual interrupt
706 * controller is dynamically instantiated and this is not always true.
707 */
708bool kvm_arch_intc_initialized(struct kvm *kvm);
709#else
710static inline bool kvm_arch_intc_initialized(struct kvm *kvm)
711{
712 return true;
713}
714#endif
715
703int kvm_arch_init_vm(struct kvm *kvm, unsigned long type); 716int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
704void kvm_arch_destroy_vm(struct kvm *kvm); 717void kvm_arch_destroy_vm(struct kvm *kvm);
705void kvm_arch_sync_events(struct kvm *kvm); 718void kvm_arch_sync_events(struct kvm *kvm);
@@ -969,11 +982,16 @@ static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
969#endif /* CONFIG_HAVE_KVM_EVENTFD */ 982#endif /* CONFIG_HAVE_KVM_EVENTFD */
970 983
971#ifdef CONFIG_KVM_APIC_ARCHITECTURE 984#ifdef CONFIG_KVM_APIC_ARCHITECTURE
972static inline bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) 985static inline bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
973{ 986{
974 return vcpu->kvm->bsp_vcpu_id == vcpu->vcpu_id; 987 return vcpu->kvm->bsp_vcpu_id == vcpu->vcpu_id;
975} 988}
976 989
990static inline bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
991{
992 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
993}
994
977bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu); 995bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu);
978 996
979#else 997#else
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a419b65770d6..51348f77e431 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -176,6 +176,14 @@ extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
176extern void calc_global_load(unsigned long ticks); 176extern void calc_global_load(unsigned long ticks);
177extern void update_cpu_load_nohz(void); 177extern void update_cpu_load_nohz(void);
178 178
179/* Notifier for when a task gets migrated to a new CPU */
180struct task_migration_notifier {
181 struct task_struct *task;
182 int from_cpu;
183 int to_cpu;
184};
185extern void register_task_migration_notifier(struct notifier_block *n);
186
179extern unsigned long get_parent_ip(unsigned long addr); 187extern unsigned long get_parent_ip(unsigned long addr);
180 188
181extern void dump_cpu_task(int cpu); 189extern void dump_cpu_task(int cpu);
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 805570650062..f574d7be7631 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -147,6 +147,16 @@ struct kvm_pit_config {
147 147
148#define KVM_PIT_SPEAKER_DUMMY 1 148#define KVM_PIT_SPEAKER_DUMMY 1
149 149
150struct kvm_s390_skeys {
151 __u64 start_gfn;
152 __u64 count;
153 __u64 skeydata_addr;
154 __u32 flags;
155 __u32 reserved[9];
156};
157#define KVM_S390_GET_SKEYS_NONE 1
158#define KVM_S390_SKEYS_MAX 1048576
159
150#define KVM_EXIT_UNKNOWN 0 160#define KVM_EXIT_UNKNOWN 0
151#define KVM_EXIT_EXCEPTION 1 161#define KVM_EXIT_EXCEPTION 1
152#define KVM_EXIT_IO 2 162#define KVM_EXIT_IO 2
@@ -172,6 +182,7 @@ struct kvm_pit_config {
172#define KVM_EXIT_S390_TSCH 22 182#define KVM_EXIT_S390_TSCH 22
173#define KVM_EXIT_EPR 23 183#define KVM_EXIT_EPR 23
174#define KVM_EXIT_SYSTEM_EVENT 24 184#define KVM_EXIT_SYSTEM_EVENT 24
185#define KVM_EXIT_S390_STSI 25
175 186
176/* For KVM_EXIT_INTERNAL_ERROR */ 187/* For KVM_EXIT_INTERNAL_ERROR */
177/* Emulate instruction failed. */ 188/* Emulate instruction failed. */
@@ -309,6 +320,15 @@ struct kvm_run {
309 __u32 type; 320 __u32 type;
310 __u64 flags; 321 __u64 flags;
311 } system_event; 322 } system_event;
323 /* KVM_EXIT_S390_STSI */
324 struct {
325 __u64 addr;
326 __u8 ar;
327 __u8 reserved;
328 __u8 fc;
329 __u8 sel1;
330 __u16 sel2;
331 } s390_stsi;
312 /* Fix the size of the union. */ 332 /* Fix the size of the union. */
313 char padding[256]; 333 char padding[256];
314 }; 334 };
@@ -324,7 +344,7 @@ struct kvm_run {
324 __u64 kvm_dirty_regs; 344 __u64 kvm_dirty_regs;
325 union { 345 union {
326 struct kvm_sync_regs regs; 346 struct kvm_sync_regs regs;
327 char padding[1024]; 347 char padding[2048];
328 } s; 348 } s;
329}; 349};
330 350
@@ -365,6 +385,24 @@ struct kvm_translation {
365 __u8 pad[5]; 385 __u8 pad[5];
366}; 386};
367 387
388/* for KVM_S390_MEM_OP */
389struct kvm_s390_mem_op {
390 /* in */
391 __u64 gaddr; /* the guest address */
392 __u64 flags; /* flags */
393 __u32 size; /* amount of bytes */
394 __u32 op; /* type of operation */
395 __u64 buf; /* buffer in userspace */
396 __u8 ar; /* the access register number */
397 __u8 reserved[31]; /* should be set to 0 */
398};
399/* types for kvm_s390_mem_op->op */
400#define KVM_S390_MEMOP_LOGICAL_READ 0
401#define KVM_S390_MEMOP_LOGICAL_WRITE 1
402/* flags for kvm_s390_mem_op->flags */
403#define KVM_S390_MEMOP_F_CHECK_ONLY (1ULL << 0)
404#define KVM_S390_MEMOP_F_INJECT_EXCEPTION (1ULL << 1)
405
368/* for KVM_INTERRUPT */ 406/* for KVM_INTERRUPT */
369struct kvm_interrupt { 407struct kvm_interrupt {
370 /* in */ 408 /* in */
@@ -520,6 +558,13 @@ struct kvm_s390_irq {
520 } u; 558 } u;
521}; 559};
522 560
561struct kvm_s390_irq_state {
562 __u64 buf;
563 __u32 flags;
564 __u32 len;
565 __u32 reserved[4];
566};
567
523/* for KVM_SET_GUEST_DEBUG */ 568/* for KVM_SET_GUEST_DEBUG */
524 569
525#define KVM_GUESTDBG_ENABLE 0x00000001 570#define KVM_GUESTDBG_ENABLE 0x00000001
@@ -760,6 +805,14 @@ struct kvm_ppc_smmu_info {
760#define KVM_CAP_PPC_ENABLE_HCALL 104 805#define KVM_CAP_PPC_ENABLE_HCALL 104
761#define KVM_CAP_CHECK_EXTENSION_VM 105 806#define KVM_CAP_CHECK_EXTENSION_VM 105
762#define KVM_CAP_S390_USER_SIGP 106 807#define KVM_CAP_S390_USER_SIGP 106
808#define KVM_CAP_S390_VECTOR_REGISTERS 107
809#define KVM_CAP_S390_MEM_OP 108
810#define KVM_CAP_S390_USER_STSI 109
811#define KVM_CAP_S390_SKEYS 110
812#define KVM_CAP_MIPS_FPU 111
813#define KVM_CAP_MIPS_MSA 112
814#define KVM_CAP_S390_INJECT_IRQ 113
815#define KVM_CAP_S390_IRQ_STATE 114
763 816
764#ifdef KVM_CAP_IRQ_ROUTING 817#ifdef KVM_CAP_IRQ_ROUTING
765 818
@@ -1135,6 +1188,16 @@ struct kvm_s390_ucas_mapping {
1135#define KVM_ARM_VCPU_INIT _IOW(KVMIO, 0xae, struct kvm_vcpu_init) 1188#define KVM_ARM_VCPU_INIT _IOW(KVMIO, 0xae, struct kvm_vcpu_init)
1136#define KVM_ARM_PREFERRED_TARGET _IOR(KVMIO, 0xaf, struct kvm_vcpu_init) 1189#define KVM_ARM_PREFERRED_TARGET _IOR(KVMIO, 0xaf, struct kvm_vcpu_init)
1137#define KVM_GET_REG_LIST _IOWR(KVMIO, 0xb0, struct kvm_reg_list) 1190#define KVM_GET_REG_LIST _IOWR(KVMIO, 0xb0, struct kvm_reg_list)
1191/* Available with KVM_CAP_S390_MEM_OP */
1192#define KVM_S390_MEM_OP _IOW(KVMIO, 0xb1, struct kvm_s390_mem_op)
1193/* Available with KVM_CAP_S390_SKEYS */
1194#define KVM_S390_GET_SKEYS _IOW(KVMIO, 0xb2, struct kvm_s390_skeys)
1195#define KVM_S390_SET_SKEYS _IOW(KVMIO, 0xb3, struct kvm_s390_skeys)
1196/* Available with KVM_CAP_S390_INJECT_IRQ */
1197#define KVM_S390_IRQ _IOW(KVMIO, 0xb4, struct kvm_s390_irq)
1198/* Available with KVM_CAP_S390_IRQ_STATE */
1199#define KVM_S390_SET_IRQ_STATE _IOW(KVMIO, 0xb5, struct kvm_s390_irq_state)
1200#define KVM_S390_GET_IRQ_STATE _IOW(KVMIO, 0xb6, struct kvm_s390_irq_state)
1138 1201
1139#define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0) 1202#define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0)
1140#define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1) 1203#define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 62671f53202a..3d5f6f6d14c2 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -996,6 +996,13 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
996 rq_clock_skip_update(rq, true); 996 rq_clock_skip_update(rq, true);
997} 997}
998 998
999static ATOMIC_NOTIFIER_HEAD(task_migration_notifier);
1000
1001void register_task_migration_notifier(struct notifier_block *n)
1002{
1003 atomic_notifier_chain_register(&task_migration_notifier, n);
1004}
1005
999#ifdef CONFIG_SMP 1006#ifdef CONFIG_SMP
1000void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 1007void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1001{ 1008{
@@ -1026,10 +1033,18 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1026 trace_sched_migrate_task(p, new_cpu); 1033 trace_sched_migrate_task(p, new_cpu);
1027 1034
1028 if (task_cpu(p) != new_cpu) { 1035 if (task_cpu(p) != new_cpu) {
1036 struct task_migration_notifier tmn;
1037
1029 if (p->sched_class->migrate_task_rq) 1038 if (p->sched_class->migrate_task_rq)
1030 p->sched_class->migrate_task_rq(p, new_cpu); 1039 p->sched_class->migrate_task_rq(p, new_cpu);
1031 p->se.nr_migrations++; 1040 p->se.nr_migrations++;
1032 perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0); 1041 perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0);
1042
1043 tmn.task = p;
1044 tmn.from_cpu = task_cpu(p);
1045 tmn.to_cpu = new_cpu;
1046
1047 atomic_notifier_call_chain(&task_migration_notifier, 0, &tmn);
1033 } 1048 }
1034 1049
1035 __set_task_cpu(p, new_cpu); 1050 __set_task_cpu(p, new_cpu);
diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c
index 6e54f3542126..98c95f2fcba4 100644
--- a/virt/kvm/arm/arch_timer.c
+++ b/virt/kvm/arm/arch_timer.c
@@ -85,13 +85,22 @@ static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
85 return IRQ_HANDLED; 85 return IRQ_HANDLED;
86} 86}
87 87
88/*
89 * Work function for handling the backup timer that we schedule when a vcpu is
90 * no longer running, but had a timer programmed to fire in the future.
91 */
88static void kvm_timer_inject_irq_work(struct work_struct *work) 92static void kvm_timer_inject_irq_work(struct work_struct *work)
89{ 93{
90 struct kvm_vcpu *vcpu; 94 struct kvm_vcpu *vcpu;
91 95
92 vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired); 96 vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
93 vcpu->arch.timer_cpu.armed = false; 97 vcpu->arch.timer_cpu.armed = false;
94 kvm_timer_inject_irq(vcpu); 98
99 /*
100 * If the vcpu is blocked we want to wake it up so that it will see
101 * the timer has expired when entering the guest.
102 */
103 kvm_vcpu_kick(vcpu);
95} 104}
96 105
97static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt) 106static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
@@ -102,6 +111,21 @@ static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
102 return HRTIMER_NORESTART; 111 return HRTIMER_NORESTART;
103} 112}
104 113
114bool kvm_timer_should_fire(struct kvm_vcpu *vcpu)
115{
116 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
117 cycle_t cval, now;
118
119 if ((timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) ||
120 !(timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE))
121 return false;
122
123 cval = timer->cntv_cval;
124 now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
125
126 return cval <= now;
127}
128
105/** 129/**
106 * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu 130 * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu
107 * @vcpu: The vcpu pointer 131 * @vcpu: The vcpu pointer
@@ -119,6 +143,13 @@ void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
119 * populate the CPU timer again. 143 * populate the CPU timer again.
120 */ 144 */
121 timer_disarm(timer); 145 timer_disarm(timer);
146
147 /*
148 * If the timer expired while we were not scheduled, now is the time
149 * to inject it.
150 */
151 if (kvm_timer_should_fire(vcpu))
152 kvm_timer_inject_irq(vcpu);
122} 153}
123 154
124/** 155/**
@@ -134,16 +165,9 @@ void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
134 cycle_t cval, now; 165 cycle_t cval, now;
135 u64 ns; 166 u64 ns;
136 167
137 if ((timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) ||
138 !(timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE))
139 return;
140
141 cval = timer->cntv_cval;
142 now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
143
144 BUG_ON(timer_is_armed(timer)); 168 BUG_ON(timer_is_armed(timer));
145 169
146 if (cval <= now) { 170 if (kvm_timer_should_fire(vcpu)) {
147 /* 171 /*
148 * Timer has already expired while we were not 172 * Timer has already expired while we were not
149 * looking. Inject the interrupt and carry on. 173 * looking. Inject the interrupt and carry on.
@@ -152,6 +176,9 @@ void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
152 return; 176 return;
153 } 177 }
154 178
179 cval = timer->cntv_cval;
180 now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
181
155 ns = cyclecounter_cyc2ns(timecounter->cc, cval - now, timecounter->mask, 182 ns = cyclecounter_cyc2ns(timecounter->cc, cval - now, timecounter->mask,
156 &timecounter->frac); 183 &timecounter->frac);
157 timer_arm(timer, ns); 184 timer_arm(timer, ns);
diff --git a/virt/kvm/arm/vgic-v2-emul.c b/virt/kvm/arm/vgic-v2-emul.c
index 19c6210f02cf..13907970d11c 100644
--- a/virt/kvm/arm/vgic-v2-emul.c
+++ b/virt/kvm/arm/vgic-v2-emul.c
@@ -107,6 +107,22 @@ static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu,
107 vcpu->vcpu_id); 107 vcpu->vcpu_id);
108} 108}
109 109
110static bool handle_mmio_set_active_reg(struct kvm_vcpu *vcpu,
111 struct kvm_exit_mmio *mmio,
112 phys_addr_t offset)
113{
114 return vgic_handle_set_active_reg(vcpu->kvm, mmio, offset,
115 vcpu->vcpu_id);
116}
117
118static bool handle_mmio_clear_active_reg(struct kvm_vcpu *vcpu,
119 struct kvm_exit_mmio *mmio,
120 phys_addr_t offset)
121{
122 return vgic_handle_clear_active_reg(vcpu->kvm, mmio, offset,
123 vcpu->vcpu_id);
124}
125
110static bool handle_mmio_priority_reg(struct kvm_vcpu *vcpu, 126static bool handle_mmio_priority_reg(struct kvm_vcpu *vcpu,
111 struct kvm_exit_mmio *mmio, 127 struct kvm_exit_mmio *mmio,
112 phys_addr_t offset) 128 phys_addr_t offset)
@@ -303,7 +319,7 @@ static bool handle_mmio_sgi_clear(struct kvm_vcpu *vcpu,
303 return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, false); 319 return write_set_clear_sgi_pend_reg(vcpu, mmio, offset, false);
304} 320}
305 321
306static const struct kvm_mmio_range vgic_dist_ranges[] = { 322static const struct vgic_io_range vgic_dist_ranges[] = {
307 { 323 {
308 .base = GIC_DIST_CTRL, 324 .base = GIC_DIST_CTRL,
309 .len = 12, 325 .len = 12,
@@ -344,13 +360,13 @@ static const struct kvm_mmio_range vgic_dist_ranges[] = {
344 .base = GIC_DIST_ACTIVE_SET, 360 .base = GIC_DIST_ACTIVE_SET,
345 .len = VGIC_MAX_IRQS / 8, 361 .len = VGIC_MAX_IRQS / 8,
346 .bits_per_irq = 1, 362 .bits_per_irq = 1,
347 .handle_mmio = handle_mmio_raz_wi, 363 .handle_mmio = handle_mmio_set_active_reg,
348 }, 364 },
349 { 365 {
350 .base = GIC_DIST_ACTIVE_CLEAR, 366 .base = GIC_DIST_ACTIVE_CLEAR,
351 .len = VGIC_MAX_IRQS / 8, 367 .len = VGIC_MAX_IRQS / 8,
352 .bits_per_irq = 1, 368 .bits_per_irq = 1,
353 .handle_mmio = handle_mmio_raz_wi, 369 .handle_mmio = handle_mmio_clear_active_reg,
354 }, 370 },
355 { 371 {
356 .base = GIC_DIST_PRI, 372 .base = GIC_DIST_PRI,
@@ -388,24 +404,6 @@ static const struct kvm_mmio_range vgic_dist_ranges[] = {
388 {} 404 {}
389}; 405};
390 406
391static bool vgic_v2_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
392 struct kvm_exit_mmio *mmio)
393{
394 unsigned long base = vcpu->kvm->arch.vgic.vgic_dist_base;
395
396 if (!is_in_range(mmio->phys_addr, mmio->len, base,
397 KVM_VGIC_V2_DIST_SIZE))
398 return false;
399
400 /* GICv2 does not support accesses wider than 32 bits */
401 if (mmio->len > 4) {
402 kvm_inject_dabt(vcpu, mmio->phys_addr);
403 return true;
404 }
405
406 return vgic_handle_mmio_range(vcpu, run, mmio, vgic_dist_ranges, base);
407}
408
409static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg) 407static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg)
410{ 408{
411 struct kvm *kvm = vcpu->kvm; 409 struct kvm *kvm = vcpu->kvm;
@@ -490,6 +488,7 @@ static bool vgic_v2_queue_sgi(struct kvm_vcpu *vcpu, int irq)
490static int vgic_v2_map_resources(struct kvm *kvm, 488static int vgic_v2_map_resources(struct kvm *kvm,
491 const struct vgic_params *params) 489 const struct vgic_params *params)
492{ 490{
491 struct vgic_dist *dist = &kvm->arch.vgic;
493 int ret = 0; 492 int ret = 0;
494 493
495 if (!irqchip_in_kernel(kvm)) 494 if (!irqchip_in_kernel(kvm))
@@ -500,13 +499,17 @@ static int vgic_v2_map_resources(struct kvm *kvm,
500 if (vgic_ready(kvm)) 499 if (vgic_ready(kvm))
501 goto out; 500 goto out;
502 501
503 if (IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_dist_base) || 502 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
504 IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_cpu_base)) { 503 IS_VGIC_ADDR_UNDEF(dist->vgic_cpu_base)) {
505 kvm_err("Need to set vgic cpu and dist addresses first\n"); 504 kvm_err("Need to set vgic cpu and dist addresses first\n");
506 ret = -ENXIO; 505 ret = -ENXIO;
507 goto out; 506 goto out;
508 } 507 }
509 508
509 vgic_register_kvm_io_dev(kvm, dist->vgic_dist_base,
510 KVM_VGIC_V2_DIST_SIZE,
511 vgic_dist_ranges, -1, &dist->dist_iodev);
512
510 /* 513 /*
511 * Initialize the vgic if this hasn't already been done on demand by 514 * Initialize the vgic if this hasn't already been done on demand by
512 * accessing the vgic state from userspace. 515 * accessing the vgic state from userspace.
@@ -514,18 +517,23 @@ static int vgic_v2_map_resources(struct kvm *kvm,
514 ret = vgic_init(kvm); 517 ret = vgic_init(kvm);
515 if (ret) { 518 if (ret) {
516 kvm_err("Unable to allocate maps\n"); 519 kvm_err("Unable to allocate maps\n");
517 goto out; 520 goto out_unregister;
518 } 521 }
519 522
520 ret = kvm_phys_addr_ioremap(kvm, kvm->arch.vgic.vgic_cpu_base, 523 ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,
521 params->vcpu_base, KVM_VGIC_V2_CPU_SIZE, 524 params->vcpu_base, KVM_VGIC_V2_CPU_SIZE,
522 true); 525 true);
523 if (ret) { 526 if (ret) {
524 kvm_err("Unable to remap VGIC CPU to VCPU\n"); 527 kvm_err("Unable to remap VGIC CPU to VCPU\n");
525 goto out; 528 goto out_unregister;
526 } 529 }
527 530
528 kvm->arch.vgic.ready = true; 531 dist->ready = true;
532 goto out;
533
534out_unregister:
535 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &dist->dist_iodev.dev);
536
529out: 537out:
530 if (ret) 538 if (ret)
531 kvm_vgic_destroy(kvm); 539 kvm_vgic_destroy(kvm);
@@ -554,7 +562,6 @@ void vgic_v2_init_emulation(struct kvm *kvm)
554{ 562{
555 struct vgic_dist *dist = &kvm->arch.vgic; 563 struct vgic_dist *dist = &kvm->arch.vgic;
556 564
557 dist->vm_ops.handle_mmio = vgic_v2_handle_mmio;
558 dist->vm_ops.queue_sgi = vgic_v2_queue_sgi; 565 dist->vm_ops.queue_sgi = vgic_v2_queue_sgi;
559 dist->vm_ops.add_sgi_source = vgic_v2_add_sgi_source; 566 dist->vm_ops.add_sgi_source = vgic_v2_add_sgi_source;
560 dist->vm_ops.init_model = vgic_v2_init_model; 567 dist->vm_ops.init_model = vgic_v2_init_model;
@@ -631,7 +638,7 @@ static bool handle_cpu_mmio_ident(struct kvm_vcpu *vcpu,
631 * CPU Interface Register accesses - these are not accessed by the VM, but by 638 * CPU Interface Register accesses - these are not accessed by the VM, but by
632 * user space for saving and restoring VGIC state. 639 * user space for saving and restoring VGIC state.
633 */ 640 */
634static const struct kvm_mmio_range vgic_cpu_ranges[] = { 641static const struct vgic_io_range vgic_cpu_ranges[] = {
635 { 642 {
636 .base = GIC_CPU_CTRL, 643 .base = GIC_CPU_CTRL,
637 .len = 12, 644 .len = 12,
@@ -658,12 +665,13 @@ static int vgic_attr_regs_access(struct kvm_device *dev,
658 struct kvm_device_attr *attr, 665 struct kvm_device_attr *attr,
659 u32 *reg, bool is_write) 666 u32 *reg, bool is_write)
660{ 667{
661 const struct kvm_mmio_range *r = NULL, *ranges; 668 const struct vgic_io_range *r = NULL, *ranges;
662 phys_addr_t offset; 669 phys_addr_t offset;
663 int ret, cpuid, c; 670 int ret, cpuid, c;
664 struct kvm_vcpu *vcpu, *tmp_vcpu; 671 struct kvm_vcpu *vcpu, *tmp_vcpu;
665 struct vgic_dist *vgic; 672 struct vgic_dist *vgic;
666 struct kvm_exit_mmio mmio; 673 struct kvm_exit_mmio mmio;
674 u32 data;
667 675
668 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK; 676 offset = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
669 cpuid = (attr->attr & KVM_DEV_ARM_VGIC_CPUID_MASK) >> 677 cpuid = (attr->attr & KVM_DEV_ARM_VGIC_CPUID_MASK) >>
@@ -685,6 +693,7 @@ static int vgic_attr_regs_access(struct kvm_device *dev,
685 693
686 mmio.len = 4; 694 mmio.len = 4;
687 mmio.is_write = is_write; 695 mmio.is_write = is_write;
696 mmio.data = &data;
688 if (is_write) 697 if (is_write)
689 mmio_data_write(&mmio, ~0, *reg); 698 mmio_data_write(&mmio, ~0, *reg);
690 switch (attr->group) { 699 switch (attr->group) {
@@ -699,7 +708,7 @@ static int vgic_attr_regs_access(struct kvm_device *dev,
699 default: 708 default:
700 BUG(); 709 BUG();
701 } 710 }
702 r = vgic_find_range(ranges, &mmio, offset); 711 r = vgic_find_range(ranges, 4, offset);
703 712
704 if (unlikely(!r || !r->handle_mmio)) { 713 if (unlikely(!r || !r->handle_mmio)) {
705 ret = -ENXIO; 714 ret = -ENXIO;
diff --git a/virt/kvm/arm/vgic-v3-emul.c b/virt/kvm/arm/vgic-v3-emul.c
index b3f154631515..e9c3a7a83833 100644
--- a/virt/kvm/arm/vgic-v3-emul.c
+++ b/virt/kvm/arm/vgic-v3-emul.c
@@ -340,7 +340,7 @@ static bool handle_mmio_idregs(struct kvm_vcpu *vcpu,
340 return false; 340 return false;
341} 341}
342 342
343static const struct kvm_mmio_range vgic_v3_dist_ranges[] = { 343static const struct vgic_io_range vgic_v3_dist_ranges[] = {
344 { 344 {
345 .base = GICD_CTLR, 345 .base = GICD_CTLR,
346 .len = 0x04, 346 .len = 0x04,
@@ -502,6 +502,43 @@ static const struct kvm_mmio_range vgic_v3_dist_ranges[] = {
502 {}, 502 {},
503}; 503};
504 504
505static bool handle_mmio_ctlr_redist(struct kvm_vcpu *vcpu,
506 struct kvm_exit_mmio *mmio,
507 phys_addr_t offset)
508{
509 /* since we don't support LPIs, this register is zero for now */
510 vgic_reg_access(mmio, NULL, offset,
511 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
512 return false;
513}
514
515static bool handle_mmio_typer_redist(struct kvm_vcpu *vcpu,
516 struct kvm_exit_mmio *mmio,
517 phys_addr_t offset)
518{
519 u32 reg;
520 u64 mpidr;
521 struct kvm_vcpu *redist_vcpu = mmio->private;
522 int target_vcpu_id = redist_vcpu->vcpu_id;
523
524 /* the upper 32 bits contain the affinity value */
525 if ((offset & ~3) == 4) {
526 mpidr = kvm_vcpu_get_mpidr_aff(redist_vcpu);
527 reg = compress_mpidr(mpidr);
528
529 vgic_reg_access(mmio, &reg, offset,
530 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
531 return false;
532 }
533
534 reg = redist_vcpu->vcpu_id << 8;
535 if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1)
536 reg |= GICR_TYPER_LAST;
537 vgic_reg_access(mmio, &reg, offset,
538 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
539 return false;
540}
541
505static bool handle_mmio_set_enable_reg_redist(struct kvm_vcpu *vcpu, 542static bool handle_mmio_set_enable_reg_redist(struct kvm_vcpu *vcpu,
506 struct kvm_exit_mmio *mmio, 543 struct kvm_exit_mmio *mmio,
507 phys_addr_t offset) 544 phys_addr_t offset)
@@ -570,186 +607,107 @@ static bool handle_mmio_cfg_reg_redist(struct kvm_vcpu *vcpu,
570 return vgic_handle_cfg_reg(reg, mmio, offset); 607 return vgic_handle_cfg_reg(reg, mmio, offset);
571} 608}
572 609
573static const struct kvm_mmio_range vgic_redist_sgi_ranges[] = { 610#define SGI_base(x) ((x) + SZ_64K)
611
612static const struct vgic_io_range vgic_redist_ranges[] = {
613 {
614 .base = GICR_CTLR,
615 .len = 0x04,
616 .bits_per_irq = 0,
617 .handle_mmio = handle_mmio_ctlr_redist,
618 },
619 {
620 .base = GICR_TYPER,
621 .len = 0x08,
622 .bits_per_irq = 0,
623 .handle_mmio = handle_mmio_typer_redist,
624 },
625 {
626 .base = GICR_IIDR,
627 .len = 0x04,
628 .bits_per_irq = 0,
629 .handle_mmio = handle_mmio_iidr,
630 },
631 {
632 .base = GICR_WAKER,
633 .len = 0x04,
634 .bits_per_irq = 0,
635 .handle_mmio = handle_mmio_raz_wi,
636 },
574 { 637 {
575 .base = GICR_IGROUPR0, 638 .base = GICR_IDREGS,
639 .len = 0x30,
640 .bits_per_irq = 0,
641 .handle_mmio = handle_mmio_idregs,
642 },
643 {
644 .base = SGI_base(GICR_IGROUPR0),
576 .len = 0x04, 645 .len = 0x04,
577 .bits_per_irq = 1, 646 .bits_per_irq = 1,
578 .handle_mmio = handle_mmio_rao_wi, 647 .handle_mmio = handle_mmio_rao_wi,
579 }, 648 },
580 { 649 {
581 .base = GICR_ISENABLER0, 650 .base = SGI_base(GICR_ISENABLER0),
582 .len = 0x04, 651 .len = 0x04,
583 .bits_per_irq = 1, 652 .bits_per_irq = 1,
584 .handle_mmio = handle_mmio_set_enable_reg_redist, 653 .handle_mmio = handle_mmio_set_enable_reg_redist,
585 }, 654 },
586 { 655 {
587 .base = GICR_ICENABLER0, 656 .base = SGI_base(GICR_ICENABLER0),
588 .len = 0x04, 657 .len = 0x04,
589 .bits_per_irq = 1, 658 .bits_per_irq = 1,
590 .handle_mmio = handle_mmio_clear_enable_reg_redist, 659 .handle_mmio = handle_mmio_clear_enable_reg_redist,
591 }, 660 },
592 { 661 {
593 .base = GICR_ISPENDR0, 662 .base = SGI_base(GICR_ISPENDR0),
594 .len = 0x04, 663 .len = 0x04,
595 .bits_per_irq = 1, 664 .bits_per_irq = 1,
596 .handle_mmio = handle_mmio_set_pending_reg_redist, 665 .handle_mmio = handle_mmio_set_pending_reg_redist,
597 }, 666 },
598 { 667 {
599 .base = GICR_ICPENDR0, 668 .base = SGI_base(GICR_ICPENDR0),
600 .len = 0x04, 669 .len = 0x04,
601 .bits_per_irq = 1, 670 .bits_per_irq = 1,
602 .handle_mmio = handle_mmio_clear_pending_reg_redist, 671 .handle_mmio = handle_mmio_clear_pending_reg_redist,
603 }, 672 },
604 { 673 {
605 .base = GICR_ISACTIVER0, 674 .base = SGI_base(GICR_ISACTIVER0),
606 .len = 0x04, 675 .len = 0x04,
607 .bits_per_irq = 1, 676 .bits_per_irq = 1,
608 .handle_mmio = handle_mmio_raz_wi, 677 .handle_mmio = handle_mmio_raz_wi,
609 }, 678 },
610 { 679 {
611 .base = GICR_ICACTIVER0, 680 .base = SGI_base(GICR_ICACTIVER0),
612 .len = 0x04, 681 .len = 0x04,
613 .bits_per_irq = 1, 682 .bits_per_irq = 1,
614 .handle_mmio = handle_mmio_raz_wi, 683 .handle_mmio = handle_mmio_raz_wi,
615 }, 684 },
616 { 685 {
617 .base = GICR_IPRIORITYR0, 686 .base = SGI_base(GICR_IPRIORITYR0),
618 .len = 0x20, 687 .len = 0x20,
619 .bits_per_irq = 8, 688 .bits_per_irq = 8,
620 .handle_mmio = handle_mmio_priority_reg_redist, 689 .handle_mmio = handle_mmio_priority_reg_redist,
621 }, 690 },
622 { 691 {
623 .base = GICR_ICFGR0, 692 .base = SGI_base(GICR_ICFGR0),
624 .len = 0x08, 693 .len = 0x08,
625 .bits_per_irq = 2, 694 .bits_per_irq = 2,
626 .handle_mmio = handle_mmio_cfg_reg_redist, 695 .handle_mmio = handle_mmio_cfg_reg_redist,
627 }, 696 },
628 { 697 {
629 .base = GICR_IGRPMODR0, 698 .base = SGI_base(GICR_IGRPMODR0),
630 .len = 0x04, 699 .len = 0x04,
631 .bits_per_irq = 1, 700 .bits_per_irq = 1,
632 .handle_mmio = handle_mmio_raz_wi, 701 .handle_mmio = handle_mmio_raz_wi,
633 }, 702 },
634 { 703 {
635 .base = GICR_NSACR, 704 .base = SGI_base(GICR_NSACR),
636 .len = 0x04, 705 .len = 0x04,
637 .handle_mmio = handle_mmio_raz_wi, 706 .handle_mmio = handle_mmio_raz_wi,
638 }, 707 },
639 {}, 708 {},
640}; 709};
641 710
642static bool handle_mmio_ctlr_redist(struct kvm_vcpu *vcpu,
643 struct kvm_exit_mmio *mmio,
644 phys_addr_t offset)
645{
646 /* since we don't support LPIs, this register is zero for now */
647 vgic_reg_access(mmio, NULL, offset,
648 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
649 return false;
650}
651
652static bool handle_mmio_typer_redist(struct kvm_vcpu *vcpu,
653 struct kvm_exit_mmio *mmio,
654 phys_addr_t offset)
655{
656 u32 reg;
657 u64 mpidr;
658 struct kvm_vcpu *redist_vcpu = mmio->private;
659 int target_vcpu_id = redist_vcpu->vcpu_id;
660
661 /* the upper 32 bits contain the affinity value */
662 if ((offset & ~3) == 4) {
663 mpidr = kvm_vcpu_get_mpidr_aff(redist_vcpu);
664 reg = compress_mpidr(mpidr);
665
666 vgic_reg_access(mmio, &reg, offset,
667 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
668 return false;
669 }
670
671 reg = redist_vcpu->vcpu_id << 8;
672 if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1)
673 reg |= GICR_TYPER_LAST;
674 vgic_reg_access(mmio, &reg, offset,
675 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
676 return false;
677}
678
679static const struct kvm_mmio_range vgic_redist_ranges[] = {
680 {
681 .base = GICR_CTLR,
682 .len = 0x04,
683 .bits_per_irq = 0,
684 .handle_mmio = handle_mmio_ctlr_redist,
685 },
686 {
687 .base = GICR_TYPER,
688 .len = 0x08,
689 .bits_per_irq = 0,
690 .handle_mmio = handle_mmio_typer_redist,
691 },
692 {
693 .base = GICR_IIDR,
694 .len = 0x04,
695 .bits_per_irq = 0,
696 .handle_mmio = handle_mmio_iidr,
697 },
698 {
699 .base = GICR_WAKER,
700 .len = 0x04,
701 .bits_per_irq = 0,
702 .handle_mmio = handle_mmio_raz_wi,
703 },
704 {
705 .base = GICR_IDREGS,
706 .len = 0x30,
707 .bits_per_irq = 0,
708 .handle_mmio = handle_mmio_idregs,
709 },
710 {},
711};
712
713/*
714 * This function splits accesses between the distributor and the two
715 * redistributor parts (private/SPI). As each redistributor is accessible
716 * from any CPU, we have to determine the affected VCPU by taking the faulting
717 * address into account. We then pass this VCPU to the handler function via
718 * the private parameter.
719 */
720#define SGI_BASE_OFFSET SZ_64K
721static bool vgic_v3_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
722 struct kvm_exit_mmio *mmio)
723{
724 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
725 unsigned long dbase = dist->vgic_dist_base;
726 unsigned long rdbase = dist->vgic_redist_base;
727 int nrcpus = atomic_read(&vcpu->kvm->online_vcpus);
728 int vcpu_id;
729 const struct kvm_mmio_range *mmio_range;
730
731 if (is_in_range(mmio->phys_addr, mmio->len, dbase, GIC_V3_DIST_SIZE)) {
732 return vgic_handle_mmio_range(vcpu, run, mmio,
733 vgic_v3_dist_ranges, dbase);
734 }
735
736 if (!is_in_range(mmio->phys_addr, mmio->len, rdbase,
737 GIC_V3_REDIST_SIZE * nrcpus))
738 return false;
739
740 vcpu_id = (mmio->phys_addr - rdbase) / GIC_V3_REDIST_SIZE;
741 rdbase += (vcpu_id * GIC_V3_REDIST_SIZE);
742 mmio->private = kvm_get_vcpu(vcpu->kvm, vcpu_id);
743
744 if (mmio->phys_addr >= rdbase + SGI_BASE_OFFSET) {
745 rdbase += SGI_BASE_OFFSET;
746 mmio_range = vgic_redist_sgi_ranges;
747 } else {
748 mmio_range = vgic_redist_ranges;
749 }
750 return vgic_handle_mmio_range(vcpu, run, mmio, mmio_range, rdbase);
751}
752
753static bool vgic_v3_queue_sgi(struct kvm_vcpu *vcpu, int irq) 711static bool vgic_v3_queue_sgi(struct kvm_vcpu *vcpu, int irq)
754{ 712{
755 if (vgic_queue_irq(vcpu, 0, irq)) { 713 if (vgic_queue_irq(vcpu, 0, irq)) {
@@ -766,6 +724,9 @@ static int vgic_v3_map_resources(struct kvm *kvm,
766{ 724{
767 int ret = 0; 725 int ret = 0;
768 struct vgic_dist *dist = &kvm->arch.vgic; 726 struct vgic_dist *dist = &kvm->arch.vgic;
727 gpa_t rdbase = dist->vgic_redist_base;
728 struct vgic_io_device *iodevs = NULL;
729 int i;
769 730
770 if (!irqchip_in_kernel(kvm)) 731 if (!irqchip_in_kernel(kvm))
771 return 0; 732 return 0;
@@ -791,7 +752,41 @@ static int vgic_v3_map_resources(struct kvm *kvm,
791 goto out; 752 goto out;
792 } 753 }
793 754
794 kvm->arch.vgic.ready = true; 755 ret = vgic_register_kvm_io_dev(kvm, dist->vgic_dist_base,
756 GIC_V3_DIST_SIZE, vgic_v3_dist_ranges,
757 -1, &dist->dist_iodev);
758 if (ret)
759 goto out;
760
761 iodevs = kcalloc(dist->nr_cpus, sizeof(iodevs[0]), GFP_KERNEL);
762 if (!iodevs) {
763 ret = -ENOMEM;
764 goto out_unregister;
765 }
766
767 for (i = 0; i < dist->nr_cpus; i++) {
768 ret = vgic_register_kvm_io_dev(kvm, rdbase,
769 SZ_128K, vgic_redist_ranges,
770 i, &iodevs[i]);
771 if (ret)
772 goto out_unregister;
773 rdbase += GIC_V3_REDIST_SIZE;
774 }
775
776 dist->redist_iodevs = iodevs;
777 dist->ready = true;
778 goto out;
779
780out_unregister:
781 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &dist->dist_iodev.dev);
782 if (iodevs) {
783 for (i = 0; i < dist->nr_cpus; i++) {
784 if (iodevs[i].dev.ops)
785 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
786 &iodevs[i].dev);
787 }
788 }
789
795out: 790out:
796 if (ret) 791 if (ret)
797 kvm_vgic_destroy(kvm); 792 kvm_vgic_destroy(kvm);
@@ -832,7 +827,6 @@ void vgic_v3_init_emulation(struct kvm *kvm)
832{ 827{
833 struct vgic_dist *dist = &kvm->arch.vgic; 828 struct vgic_dist *dist = &kvm->arch.vgic;
834 829
835 dist->vm_ops.handle_mmio = vgic_v3_handle_mmio;
836 dist->vm_ops.queue_sgi = vgic_v3_queue_sgi; 830 dist->vm_ops.queue_sgi = vgic_v3_queue_sgi;
837 dist->vm_ops.add_sgi_source = vgic_v3_add_sgi_source; 831 dist->vm_ops.add_sgi_source = vgic_v3_add_sgi_source;
838 dist->vm_ops.init_model = vgic_v3_init_model; 832 dist->vm_ops.init_model = vgic_v3_init_model;
diff --git a/virt/kvm/arm/vgic.c b/virt/kvm/arm/vgic.c
index c9f60f524588..8d550ff14700 100644
--- a/virt/kvm/arm/vgic.c
+++ b/virt/kvm/arm/vgic.c
@@ -31,6 +31,9 @@
31#include <asm/kvm_emulate.h> 31#include <asm/kvm_emulate.h>
32#include <asm/kvm_arm.h> 32#include <asm/kvm_arm.h>
33#include <asm/kvm_mmu.h> 33#include <asm/kvm_mmu.h>
34#include <trace/events/kvm.h>
35#include <asm/kvm.h>
36#include <kvm/iodev.h>
34 37
35/* 38/*
36 * How the whole thing works (courtesy of Christoffer Dall): 39 * How the whole thing works (courtesy of Christoffer Dall):
@@ -263,6 +266,13 @@ static int vgic_irq_is_queued(struct kvm_vcpu *vcpu, int irq)
263 return vgic_bitmap_get_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq); 266 return vgic_bitmap_get_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq);
264} 267}
265 268
269static int vgic_irq_is_active(struct kvm_vcpu *vcpu, int irq)
270{
271 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
272
273 return vgic_bitmap_get_irq_val(&dist->irq_active, vcpu->vcpu_id, irq);
274}
275
266static void vgic_irq_set_queued(struct kvm_vcpu *vcpu, int irq) 276static void vgic_irq_set_queued(struct kvm_vcpu *vcpu, int irq)
267{ 277{
268 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 278 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
@@ -277,6 +287,20 @@ static void vgic_irq_clear_queued(struct kvm_vcpu *vcpu, int irq)
277 vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 0); 287 vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 0);
278} 288}
279 289
290static void vgic_irq_set_active(struct kvm_vcpu *vcpu, int irq)
291{
292 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
293
294 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 1);
295}
296
297static void vgic_irq_clear_active(struct kvm_vcpu *vcpu, int irq)
298{
299 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
300
301 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 0);
302}
303
280static int vgic_dist_irq_get_level(struct kvm_vcpu *vcpu, int irq) 304static int vgic_dist_irq_get_level(struct kvm_vcpu *vcpu, int irq)
281{ 305{
282 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 306 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
@@ -520,6 +544,44 @@ bool vgic_handle_clear_pending_reg(struct kvm *kvm,
520 return false; 544 return false;
521} 545}
522 546
547bool vgic_handle_set_active_reg(struct kvm *kvm,
548 struct kvm_exit_mmio *mmio,
549 phys_addr_t offset, int vcpu_id)
550{
551 u32 *reg;
552 struct vgic_dist *dist = &kvm->arch.vgic;
553
554 reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
555 vgic_reg_access(mmio, reg, offset,
556 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
557
558 if (mmio->is_write) {
559 vgic_update_state(kvm);
560 return true;
561 }
562
563 return false;
564}
565
566bool vgic_handle_clear_active_reg(struct kvm *kvm,
567 struct kvm_exit_mmio *mmio,
568 phys_addr_t offset, int vcpu_id)
569{
570 u32 *reg;
571 struct vgic_dist *dist = &kvm->arch.vgic;
572
573 reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
574 vgic_reg_access(mmio, reg, offset,
575 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
576
577 if (mmio->is_write) {
578 vgic_update_state(kvm);
579 return true;
580 }
581
582 return false;
583}
584
523static u32 vgic_cfg_expand(u16 val) 585static u32 vgic_cfg_expand(u16 val)
524{ 586{
525 u32 res = 0; 587 u32 res = 0;
@@ -588,16 +650,12 @@ bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
588} 650}
589 651
590/** 652/**
591 * vgic_unqueue_irqs - move pending IRQs from LRs to the distributor 653 * vgic_unqueue_irqs - move pending/active IRQs from LRs to the distributor
592 * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs 654 * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs
593 * 655 *
594 * Move any pending IRQs that have already been assigned to LRs back to the 656 * Move any IRQs that have already been assigned to LRs back to the
595 * emulated distributor state so that the complete emulated state can be read 657 * emulated distributor state so that the complete emulated state can be read
596 * from the main emulation structures without investigating the LRs. 658 * from the main emulation structures without investigating the LRs.
597 *
598 * Note that IRQs in the active state in the LRs get their pending state moved
599 * to the distributor but the active state stays in the LRs, because we don't
600 * track the active state on the distributor side.
601 */ 659 */
602void vgic_unqueue_irqs(struct kvm_vcpu *vcpu) 660void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
603{ 661{
@@ -613,12 +671,22 @@ void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
613 * 01: pending 671 * 01: pending
614 * 10: active 672 * 10: active
615 * 11: pending and active 673 * 11: pending and active
616 *
617 * If the LR holds only an active interrupt (not pending) then
618 * just leave it alone.
619 */ 674 */
620 if ((lr.state & LR_STATE_MASK) == LR_STATE_ACTIVE) 675 BUG_ON(!(lr.state & LR_STATE_MASK));
621 continue; 676
677 /* Reestablish SGI source for pending and active IRQs */
678 if (lr.irq < VGIC_NR_SGIS)
679 add_sgi_source(vcpu, lr.irq, lr.source);
680
681 /*
682 * If the LR holds an active (10) or a pending and active (11)
683 * interrupt then move the active state to the
684 * distributor tracking bit.
685 */
686 if (lr.state & LR_STATE_ACTIVE) {
687 vgic_irq_set_active(vcpu, lr.irq);
688 lr.state &= ~LR_STATE_ACTIVE;
689 }
622 690
623 /* 691 /*
624 * Reestablish the pending state on the distributor and the 692 * Reestablish the pending state on the distributor and the
@@ -626,21 +694,19 @@ void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
626 * is fine, then we are only setting a few bits that were 694 * is fine, then we are only setting a few bits that were
627 * already set. 695 * already set.
628 */ 696 */
629 vgic_dist_irq_set_pending(vcpu, lr.irq); 697 if (lr.state & LR_STATE_PENDING) {
630 if (lr.irq < VGIC_NR_SGIS) 698 vgic_dist_irq_set_pending(vcpu, lr.irq);
631 add_sgi_source(vcpu, lr.irq, lr.source); 699 lr.state &= ~LR_STATE_PENDING;
632 lr.state &= ~LR_STATE_PENDING; 700 }
701
633 vgic_set_lr(vcpu, i, lr); 702 vgic_set_lr(vcpu, i, lr);
634 703
635 /* 704 /*
636 * If there's no state left on the LR (it could still be 705 * Mark the LR as free for other use.
637 * active), then the LR does not hold any useful info and can
638 * be marked as free for other use.
639 */ 706 */
640 if (!(lr.state & LR_STATE_MASK)) { 707 BUG_ON(lr.state & LR_STATE_MASK);
641 vgic_retire_lr(i, lr.irq, vcpu); 708 vgic_retire_lr(i, lr.irq, vcpu);
642 vgic_irq_clear_queued(vcpu, lr.irq); 709 vgic_irq_clear_queued(vcpu, lr.irq);
643 }
644 710
645 /* Finally update the VGIC state. */ 711 /* Finally update the VGIC state. */
646 vgic_update_state(vcpu->kvm); 712 vgic_update_state(vcpu->kvm);
@@ -648,24 +714,21 @@ void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
648} 714}
649 715
650const 716const
651struct kvm_mmio_range *vgic_find_range(const struct kvm_mmio_range *ranges, 717struct vgic_io_range *vgic_find_range(const struct vgic_io_range *ranges,
652 struct kvm_exit_mmio *mmio, 718 int len, gpa_t offset)
653 phys_addr_t offset) 719{
654{ 720 while (ranges->len) {
655 const struct kvm_mmio_range *r = ranges; 721 if (offset >= ranges->base &&
656 722 (offset + len) <= (ranges->base + ranges->len))
657 while (r->len) { 723 return ranges;
658 if (offset >= r->base && 724 ranges++;
659 (offset + mmio->len) <= (r->base + r->len))
660 return r;
661 r++;
662 } 725 }
663 726
664 return NULL; 727 return NULL;
665} 728}
666 729
667static bool vgic_validate_access(const struct vgic_dist *dist, 730static bool vgic_validate_access(const struct vgic_dist *dist,
668 const struct kvm_mmio_range *range, 731 const struct vgic_io_range *range,
669 unsigned long offset) 732 unsigned long offset)
670{ 733{
671 int irq; 734 int irq;
@@ -693,9 +756,8 @@ static bool vgic_validate_access(const struct vgic_dist *dist,
693static bool call_range_handler(struct kvm_vcpu *vcpu, 756static bool call_range_handler(struct kvm_vcpu *vcpu,
694 struct kvm_exit_mmio *mmio, 757 struct kvm_exit_mmio *mmio,
695 unsigned long offset, 758 unsigned long offset,
696 const struct kvm_mmio_range *range) 759 const struct vgic_io_range *range)
697{ 760{
698 u32 *data32 = (void *)mmio->data;
699 struct kvm_exit_mmio mmio32; 761 struct kvm_exit_mmio mmio32;
700 bool ret; 762 bool ret;
701 763
@@ -712,91 +774,142 @@ static bool call_range_handler(struct kvm_vcpu *vcpu,
712 mmio32.private = mmio->private; 774 mmio32.private = mmio->private;
713 775
714 mmio32.phys_addr = mmio->phys_addr + 4; 776 mmio32.phys_addr = mmio->phys_addr + 4;
715 if (mmio->is_write) 777 mmio32.data = &((u32 *)mmio->data)[1];
716 *(u32 *)mmio32.data = data32[1];
717 ret = range->handle_mmio(vcpu, &mmio32, offset + 4); 778 ret = range->handle_mmio(vcpu, &mmio32, offset + 4);
718 if (!mmio->is_write)
719 data32[1] = *(u32 *)mmio32.data;
720 779
721 mmio32.phys_addr = mmio->phys_addr; 780 mmio32.phys_addr = mmio->phys_addr;
722 if (mmio->is_write) 781 mmio32.data = &((u32 *)mmio->data)[0];
723 *(u32 *)mmio32.data = data32[0];
724 ret |= range->handle_mmio(vcpu, &mmio32, offset); 782 ret |= range->handle_mmio(vcpu, &mmio32, offset);
725 if (!mmio->is_write)
726 data32[0] = *(u32 *)mmio32.data;
727 783
728 return ret; 784 return ret;
729} 785}
730 786
731/** 787/**
732 * vgic_handle_mmio_range - handle an in-kernel MMIO access 788 * vgic_handle_mmio_access - handle an in-kernel MMIO access
789 * This is called by the read/write KVM IO device wrappers below.
733 * @vcpu: pointer to the vcpu performing the access 790 * @vcpu: pointer to the vcpu performing the access
734 * @run: pointer to the kvm_run structure 791 * @this: pointer to the KVM IO device in charge
735 * @mmio: pointer to the data describing the access 792 * @addr: guest physical address of the access
736 * @ranges: array of MMIO ranges in a given region 793 * @len: size of the access
737 * @mmio_base: base address of that region 794 * @val: pointer to the data region
795 * @is_write: read or write access
738 * 796 *
739 * returns true if the MMIO access could be performed 797 * returns true if the MMIO access could be performed
740 */ 798 */
741bool vgic_handle_mmio_range(struct kvm_vcpu *vcpu, struct kvm_run *run, 799static int vgic_handle_mmio_access(struct kvm_vcpu *vcpu,
742 struct kvm_exit_mmio *mmio, 800 struct kvm_io_device *this, gpa_t addr,
743 const struct kvm_mmio_range *ranges, 801 int len, void *val, bool is_write)
744 unsigned long mmio_base)
745{ 802{
746 const struct kvm_mmio_range *range;
747 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 803 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
804 struct vgic_io_device *iodev = container_of(this,
805 struct vgic_io_device, dev);
806 struct kvm_run *run = vcpu->run;
807 const struct vgic_io_range *range;
808 struct kvm_exit_mmio mmio;
748 bool updated_state; 809 bool updated_state;
749 unsigned long offset; 810 gpa_t offset;
750 811
751 offset = mmio->phys_addr - mmio_base; 812 offset = addr - iodev->addr;
752 range = vgic_find_range(ranges, mmio, offset); 813 range = vgic_find_range(iodev->reg_ranges, len, offset);
753 if (unlikely(!range || !range->handle_mmio)) { 814 if (unlikely(!range || !range->handle_mmio)) {
754 pr_warn("Unhandled access %d %08llx %d\n", 815 pr_warn("Unhandled access %d %08llx %d\n", is_write, addr, len);
755 mmio->is_write, mmio->phys_addr, mmio->len); 816 return -ENXIO;
756 return false;
757 } 817 }
758 818
759 spin_lock(&vcpu->kvm->arch.vgic.lock); 819 mmio.phys_addr = addr;
820 mmio.len = len;
821 mmio.is_write = is_write;
822 mmio.data = val;
823 mmio.private = iodev->redist_vcpu;
824
825 spin_lock(&dist->lock);
760 offset -= range->base; 826 offset -= range->base;
761 if (vgic_validate_access(dist, range, offset)) { 827 if (vgic_validate_access(dist, range, offset)) {
762 updated_state = call_range_handler(vcpu, mmio, offset, range); 828 updated_state = call_range_handler(vcpu, &mmio, offset, range);
763 } else { 829 } else {
764 if (!mmio->is_write) 830 if (!is_write)
765 memset(mmio->data, 0, mmio->len); 831 memset(val, 0, len);
766 updated_state = false; 832 updated_state = false;
767 } 833 }
768 spin_unlock(&vcpu->kvm->arch.vgic.lock); 834 spin_unlock(&dist->lock);
769 kvm_prepare_mmio(run, mmio); 835 run->mmio.is_write = is_write;
836 run->mmio.len = len;
837 run->mmio.phys_addr = addr;
838 memcpy(run->mmio.data, val, len);
839
770 kvm_handle_mmio_return(vcpu, run); 840 kvm_handle_mmio_return(vcpu, run);
771 841
772 if (updated_state) 842 if (updated_state)
773 vgic_kick_vcpus(vcpu->kvm); 843 vgic_kick_vcpus(vcpu->kvm);
774 844
775 return true; 845 return 0;
846}
847
848static int vgic_handle_mmio_read(struct kvm_vcpu *vcpu,
849 struct kvm_io_device *this,
850 gpa_t addr, int len, void *val)
851{
852 return vgic_handle_mmio_access(vcpu, this, addr, len, val, false);
776} 853}
777 854
855static int vgic_handle_mmio_write(struct kvm_vcpu *vcpu,
856 struct kvm_io_device *this,
857 gpa_t addr, int len, const void *val)
858{
859 return vgic_handle_mmio_access(vcpu, this, addr, len, (void *)val,
860 true);
861}
862
863struct kvm_io_device_ops vgic_io_ops = {
864 .read = vgic_handle_mmio_read,
865 .write = vgic_handle_mmio_write,
866};
867
778/** 868/**
779 * vgic_handle_mmio - handle an in-kernel MMIO access for the GIC emulation 869 * vgic_register_kvm_io_dev - register VGIC register frame on the KVM I/O bus
780 * @vcpu: pointer to the vcpu performing the access 870 * @kvm: The VM structure pointer
781 * @run: pointer to the kvm_run structure 871 * @base: The (guest) base address for the register frame
782 * @mmio: pointer to the data describing the access 872 * @len: Length of the register frame window
873 * @ranges: Describing the handler functions for each register
874 * @redist_vcpu_id: The VCPU ID to pass on to the handlers on call
875 * @iodev: Points to memory to be passed on to the handler
783 * 876 *
784 * returns true if the MMIO access has been performed in kernel space, 877 * @iodev stores the parameters of this function to be usable by the handler
785 * and false if it needs to be emulated in user space. 878 * respectively the dispatcher function (since the KVM I/O bus framework lacks
786 * Calls the actual handling routine for the selected VGIC model. 879 * an opaque parameter). Initialization is done in this function, but the
880 * reference should be valid and unique for the whole VGIC lifetime.
881 * If the register frame is not mapped for a specific VCPU, pass -1 to
882 * @redist_vcpu_id.
787 */ 883 */
788bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run, 884int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,
789 struct kvm_exit_mmio *mmio) 885 const struct vgic_io_range *ranges,
886 int redist_vcpu_id,
887 struct vgic_io_device *iodev)
790{ 888{
791 if (!irqchip_in_kernel(vcpu->kvm)) 889 struct kvm_vcpu *vcpu = NULL;
792 return false; 890 int ret;
793 891
794 /* 892 if (redist_vcpu_id >= 0)
795 * This will currently call either vgic_v2_handle_mmio() or 893 vcpu = kvm_get_vcpu(kvm, redist_vcpu_id);
796 * vgic_v3_handle_mmio(), which in turn will call 894
797 * vgic_handle_mmio_range() defined above. 895 iodev->addr = base;
798 */ 896 iodev->len = len;
799 return vcpu->kvm->arch.vgic.vm_ops.handle_mmio(vcpu, run, mmio); 897 iodev->reg_ranges = ranges;
898 iodev->redist_vcpu = vcpu;
899
900 kvm_iodevice_init(&iodev->dev, &vgic_io_ops);
901
902 mutex_lock(&kvm->slots_lock);
903
904 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, base, len,
905 &iodev->dev);
906 mutex_unlock(&kvm->slots_lock);
907
908 /* Mark the iodev as invalid if registration fails. */
909 if (ret)
910 iodev->dev.ops = NULL;
911
912 return ret;
800} 913}
801 914
802static int vgic_nr_shared_irqs(struct vgic_dist *dist) 915static int vgic_nr_shared_irqs(struct vgic_dist *dist)
@@ -804,6 +917,36 @@ static int vgic_nr_shared_irqs(struct vgic_dist *dist)
804 return dist->nr_irqs - VGIC_NR_PRIVATE_IRQS; 917 return dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
805} 918}
806 919
920static int compute_active_for_cpu(struct kvm_vcpu *vcpu)
921{
922 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
923 unsigned long *active, *enabled, *act_percpu, *act_shared;
924 unsigned long active_private, active_shared;
925 int nr_shared = vgic_nr_shared_irqs(dist);
926 int vcpu_id;
927
928 vcpu_id = vcpu->vcpu_id;
929 act_percpu = vcpu->arch.vgic_cpu.active_percpu;
930 act_shared = vcpu->arch.vgic_cpu.active_shared;
931
932 active = vgic_bitmap_get_cpu_map(&dist->irq_active, vcpu_id);
933 enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
934 bitmap_and(act_percpu, active, enabled, VGIC_NR_PRIVATE_IRQS);
935
936 active = vgic_bitmap_get_shared_map(&dist->irq_active);
937 enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
938 bitmap_and(act_shared, active, enabled, nr_shared);
939 bitmap_and(act_shared, act_shared,
940 vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
941 nr_shared);
942
943 active_private = find_first_bit(act_percpu, VGIC_NR_PRIVATE_IRQS);
944 active_shared = find_first_bit(act_shared, nr_shared);
945
946 return (active_private < VGIC_NR_PRIVATE_IRQS ||
947 active_shared < nr_shared);
948}
949
807static int compute_pending_for_cpu(struct kvm_vcpu *vcpu) 950static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
808{ 951{
809 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 952 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
@@ -835,7 +978,7 @@ static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
835 978
836/* 979/*
837 * Update the interrupt state and determine which CPUs have pending 980 * Update the interrupt state and determine which CPUs have pending
838 * interrupts. Must be called with distributor lock held. 981 * or active interrupts. Must be called with distributor lock held.
839 */ 982 */
840void vgic_update_state(struct kvm *kvm) 983void vgic_update_state(struct kvm *kvm)
841{ 984{
@@ -849,10 +992,13 @@ void vgic_update_state(struct kvm *kvm)
849 } 992 }
850 993
851 kvm_for_each_vcpu(c, vcpu, kvm) { 994 kvm_for_each_vcpu(c, vcpu, kvm) {
852 if (compute_pending_for_cpu(vcpu)) { 995 if (compute_pending_for_cpu(vcpu))
853 pr_debug("CPU%d has pending interrupts\n", c);
854 set_bit(c, dist->irq_pending_on_cpu); 996 set_bit(c, dist->irq_pending_on_cpu);
855 } 997
998 if (compute_active_for_cpu(vcpu))
999 set_bit(c, dist->irq_active_on_cpu);
1000 else
1001 clear_bit(c, dist->irq_active_on_cpu);
856 } 1002 }
857} 1003}
858 1004
@@ -955,6 +1101,26 @@ static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
955 } 1101 }
956} 1102}
957 1103
1104static void vgic_queue_irq_to_lr(struct kvm_vcpu *vcpu, int irq,
1105 int lr_nr, struct vgic_lr vlr)
1106{
1107 if (vgic_irq_is_active(vcpu, irq)) {
1108 vlr.state |= LR_STATE_ACTIVE;
1109 kvm_debug("Set active, clear distributor: 0x%x\n", vlr.state);
1110 vgic_irq_clear_active(vcpu, irq);
1111 vgic_update_state(vcpu->kvm);
1112 } else if (vgic_dist_irq_is_pending(vcpu, irq)) {
1113 vlr.state |= LR_STATE_PENDING;
1114 kvm_debug("Set pending: 0x%x\n", vlr.state);
1115 }
1116
1117 if (!vgic_irq_is_edge(vcpu, irq))
1118 vlr.state |= LR_EOI_INT;
1119
1120 vgic_set_lr(vcpu, lr_nr, vlr);
1121 vgic_sync_lr_elrsr(vcpu, lr_nr, vlr);
1122}
1123
958/* 1124/*
959 * Queue an interrupt to a CPU virtual interface. Return true on success, 1125 * Queue an interrupt to a CPU virtual interface. Return true on success,
960 * or false if it wasn't possible to queue it. 1126 * or false if it wasn't possible to queue it.
@@ -982,9 +1148,7 @@ bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
982 if (vlr.source == sgi_source_id) { 1148 if (vlr.source == sgi_source_id) {
983 kvm_debug("LR%d piggyback for IRQ%d\n", lr, vlr.irq); 1149 kvm_debug("LR%d piggyback for IRQ%d\n", lr, vlr.irq);
984 BUG_ON(!test_bit(lr, vgic_cpu->lr_used)); 1150 BUG_ON(!test_bit(lr, vgic_cpu->lr_used));
985 vlr.state |= LR_STATE_PENDING; 1151 vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
986 vgic_set_lr(vcpu, lr, vlr);
987 vgic_sync_lr_elrsr(vcpu, lr, vlr);
988 return true; 1152 return true;
989 } 1153 }
990 } 1154 }
@@ -1001,12 +1165,8 @@ bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
1001 1165
1002 vlr.irq = irq; 1166 vlr.irq = irq;
1003 vlr.source = sgi_source_id; 1167 vlr.source = sgi_source_id;
1004 vlr.state = LR_STATE_PENDING; 1168 vlr.state = 0;
1005 if (!vgic_irq_is_edge(vcpu, irq)) 1169 vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1006 vlr.state |= LR_EOI_INT;
1007
1008 vgic_set_lr(vcpu, lr, vlr);
1009 vgic_sync_lr_elrsr(vcpu, lr, vlr);
1010 1170
1011 return true; 1171 return true;
1012} 1172}
@@ -1038,39 +1198,49 @@ static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1038{ 1198{
1039 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 1199 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1040 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 1200 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1201 unsigned long *pa_percpu, *pa_shared;
1041 int i, vcpu_id; 1202 int i, vcpu_id;
1042 int overflow = 0; 1203 int overflow = 0;
1204 int nr_shared = vgic_nr_shared_irqs(dist);
1043 1205
1044 vcpu_id = vcpu->vcpu_id; 1206 vcpu_id = vcpu->vcpu_id;
1045 1207
1208 pa_percpu = vcpu->arch.vgic_cpu.pend_act_percpu;
1209 pa_shared = vcpu->arch.vgic_cpu.pend_act_shared;
1210
1211 bitmap_or(pa_percpu, vgic_cpu->pending_percpu, vgic_cpu->active_percpu,
1212 VGIC_NR_PRIVATE_IRQS);
1213 bitmap_or(pa_shared, vgic_cpu->pending_shared, vgic_cpu->active_shared,
1214 nr_shared);
1046 /* 1215 /*
1047 * We may not have any pending interrupt, or the interrupts 1216 * We may not have any pending interrupt, or the interrupts
1048 * may have been serviced from another vcpu. In all cases, 1217 * may have been serviced from another vcpu. In all cases,
1049 * move along. 1218 * move along.
1050 */ 1219 */
1051 if (!kvm_vgic_vcpu_pending_irq(vcpu)) { 1220 if (!kvm_vgic_vcpu_pending_irq(vcpu) && !kvm_vgic_vcpu_active_irq(vcpu))
1052 pr_debug("CPU%d has no pending interrupt\n", vcpu_id);
1053 goto epilog; 1221 goto epilog;
1054 }
1055 1222
1056 /* SGIs */ 1223 /* SGIs */
1057 for_each_set_bit(i, vgic_cpu->pending_percpu, VGIC_NR_SGIS) { 1224 for_each_set_bit(i, pa_percpu, VGIC_NR_SGIS) {
1058 if (!queue_sgi(vcpu, i)) 1225 if (!queue_sgi(vcpu, i))
1059 overflow = 1; 1226 overflow = 1;
1060 } 1227 }
1061 1228
1062 /* PPIs */ 1229 /* PPIs */
1063 for_each_set_bit_from(i, vgic_cpu->pending_percpu, VGIC_NR_PRIVATE_IRQS) { 1230 for_each_set_bit_from(i, pa_percpu, VGIC_NR_PRIVATE_IRQS) {
1064 if (!vgic_queue_hwirq(vcpu, i)) 1231 if (!vgic_queue_hwirq(vcpu, i))
1065 overflow = 1; 1232 overflow = 1;
1066 } 1233 }
1067 1234
1068 /* SPIs */ 1235 /* SPIs */
1069 for_each_set_bit(i, vgic_cpu->pending_shared, vgic_nr_shared_irqs(dist)) { 1236 for_each_set_bit(i, pa_shared, nr_shared) {
1070 if (!vgic_queue_hwirq(vcpu, i + VGIC_NR_PRIVATE_IRQS)) 1237 if (!vgic_queue_hwirq(vcpu, i + VGIC_NR_PRIVATE_IRQS))
1071 overflow = 1; 1238 overflow = 1;
1072 } 1239 }
1073 1240
1241
1242
1243
1074epilog: 1244epilog:
1075 if (overflow) { 1245 if (overflow) {
1076 vgic_enable_underflow(vcpu); 1246 vgic_enable_underflow(vcpu);
@@ -1089,7 +1259,9 @@ epilog:
1089static bool vgic_process_maintenance(struct kvm_vcpu *vcpu) 1259static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1090{ 1260{
1091 u32 status = vgic_get_interrupt_status(vcpu); 1261 u32 status = vgic_get_interrupt_status(vcpu);
1262 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1092 bool level_pending = false; 1263 bool level_pending = false;
1264 struct kvm *kvm = vcpu->kvm;
1093 1265
1094 kvm_debug("STATUS = %08x\n", status); 1266 kvm_debug("STATUS = %08x\n", status);
1095 1267
@@ -1106,6 +1278,7 @@ static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1106 struct vgic_lr vlr = vgic_get_lr(vcpu, lr); 1278 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1107 WARN_ON(vgic_irq_is_edge(vcpu, vlr.irq)); 1279 WARN_ON(vgic_irq_is_edge(vcpu, vlr.irq));
1108 1280
1281 spin_lock(&dist->lock);
1109 vgic_irq_clear_queued(vcpu, vlr.irq); 1282 vgic_irq_clear_queued(vcpu, vlr.irq);
1110 WARN_ON(vlr.state & LR_STATE_MASK); 1283 WARN_ON(vlr.state & LR_STATE_MASK);
1111 vlr.state = 0; 1284 vlr.state = 0;
@@ -1124,6 +1297,17 @@ static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1124 */ 1297 */
1125 vgic_dist_irq_clear_soft_pend(vcpu, vlr.irq); 1298 vgic_dist_irq_clear_soft_pend(vcpu, vlr.irq);
1126 1299
1300 /*
1301 * kvm_notify_acked_irq calls kvm_set_irq()
1302 * to reset the IRQ level. Need to release the
1303 * lock for kvm_set_irq to grab it.
1304 */
1305 spin_unlock(&dist->lock);
1306
1307 kvm_notify_acked_irq(kvm, 0,
1308 vlr.irq - VGIC_NR_PRIVATE_IRQS);
1309 spin_lock(&dist->lock);
1310
1127 /* Any additional pending interrupt? */ 1311 /* Any additional pending interrupt? */
1128 if (vgic_dist_irq_get_level(vcpu, vlr.irq)) { 1312 if (vgic_dist_irq_get_level(vcpu, vlr.irq)) {
1129 vgic_cpu_irq_set(vcpu, vlr.irq); 1313 vgic_cpu_irq_set(vcpu, vlr.irq);
@@ -1133,6 +1317,8 @@ static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1133 vgic_cpu_irq_clear(vcpu, vlr.irq); 1317 vgic_cpu_irq_clear(vcpu, vlr.irq);
1134 } 1318 }
1135 1319
1320 spin_unlock(&dist->lock);
1321
1136 /* 1322 /*
1137 * Despite being EOIed, the LR may not have 1323 * Despite being EOIed, the LR may not have
1138 * been marked as empty. 1324 * been marked as empty.
@@ -1155,10 +1341,7 @@ static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1155 return level_pending; 1341 return level_pending;
1156} 1342}
1157 1343
1158/* 1344/* Sync back the VGIC state after a guest run */
1159 * Sync back the VGIC state after a guest run. The distributor lock is
1160 * needed so we don't get preempted in the middle of the state processing.
1161 */
1162static void __kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) 1345static void __kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1163{ 1346{
1164 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 1347 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
@@ -1205,14 +1388,10 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1205 1388
1206void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) 1389void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1207{ 1390{
1208 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1209
1210 if (!irqchip_in_kernel(vcpu->kvm)) 1391 if (!irqchip_in_kernel(vcpu->kvm))
1211 return; 1392 return;
1212 1393
1213 spin_lock(&dist->lock);
1214 __kvm_vgic_sync_hwstate(vcpu); 1394 __kvm_vgic_sync_hwstate(vcpu);
1215 spin_unlock(&dist->lock);
1216} 1395}
1217 1396
1218int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu) 1397int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
@@ -1225,6 +1404,17 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
1225 return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu); 1404 return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1226} 1405}
1227 1406
1407int kvm_vgic_vcpu_active_irq(struct kvm_vcpu *vcpu)
1408{
1409 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1410
1411 if (!irqchip_in_kernel(vcpu->kvm))
1412 return 0;
1413
1414 return test_bit(vcpu->vcpu_id, dist->irq_active_on_cpu);
1415}
1416
1417
1228void vgic_kick_vcpus(struct kvm *kvm) 1418void vgic_kick_vcpus(struct kvm *kvm)
1229{ 1419{
1230 struct kvm_vcpu *vcpu; 1420 struct kvm_vcpu *vcpu;
@@ -1397,8 +1587,12 @@ void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
1397 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 1587 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1398 1588
1399 kfree(vgic_cpu->pending_shared); 1589 kfree(vgic_cpu->pending_shared);
1590 kfree(vgic_cpu->active_shared);
1591 kfree(vgic_cpu->pend_act_shared);
1400 kfree(vgic_cpu->vgic_irq_lr_map); 1592 kfree(vgic_cpu->vgic_irq_lr_map);
1401 vgic_cpu->pending_shared = NULL; 1593 vgic_cpu->pending_shared = NULL;
1594 vgic_cpu->active_shared = NULL;
1595 vgic_cpu->pend_act_shared = NULL;
1402 vgic_cpu->vgic_irq_lr_map = NULL; 1596 vgic_cpu->vgic_irq_lr_map = NULL;
1403} 1597}
1404 1598
@@ -1408,9 +1602,14 @@ static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs)
1408 1602
1409 int sz = (nr_irqs - VGIC_NR_PRIVATE_IRQS) / 8; 1603 int sz = (nr_irqs - VGIC_NR_PRIVATE_IRQS) / 8;
1410 vgic_cpu->pending_shared = kzalloc(sz, GFP_KERNEL); 1604 vgic_cpu->pending_shared = kzalloc(sz, GFP_KERNEL);
1605 vgic_cpu->active_shared = kzalloc(sz, GFP_KERNEL);
1606 vgic_cpu->pend_act_shared = kzalloc(sz, GFP_KERNEL);
1411 vgic_cpu->vgic_irq_lr_map = kmalloc(nr_irqs, GFP_KERNEL); 1607 vgic_cpu->vgic_irq_lr_map = kmalloc(nr_irqs, GFP_KERNEL);
1412 1608
1413 if (!vgic_cpu->pending_shared || !vgic_cpu->vgic_irq_lr_map) { 1609 if (!vgic_cpu->pending_shared
1610 || !vgic_cpu->active_shared
1611 || !vgic_cpu->pend_act_shared
1612 || !vgic_cpu->vgic_irq_lr_map) {
1414 kvm_vgic_vcpu_destroy(vcpu); 1613 kvm_vgic_vcpu_destroy(vcpu);
1415 return -ENOMEM; 1614 return -ENOMEM;
1416 } 1615 }
@@ -1463,10 +1662,12 @@ void kvm_vgic_destroy(struct kvm *kvm)
1463 kfree(dist->irq_spi_mpidr); 1662 kfree(dist->irq_spi_mpidr);
1464 kfree(dist->irq_spi_target); 1663 kfree(dist->irq_spi_target);
1465 kfree(dist->irq_pending_on_cpu); 1664 kfree(dist->irq_pending_on_cpu);
1665 kfree(dist->irq_active_on_cpu);
1466 dist->irq_sgi_sources = NULL; 1666 dist->irq_sgi_sources = NULL;
1467 dist->irq_spi_cpu = NULL; 1667 dist->irq_spi_cpu = NULL;
1468 dist->irq_spi_target = NULL; 1668 dist->irq_spi_target = NULL;
1469 dist->irq_pending_on_cpu = NULL; 1669 dist->irq_pending_on_cpu = NULL;
1670 dist->irq_active_on_cpu = NULL;
1470 dist->nr_cpus = 0; 1671 dist->nr_cpus = 0;
1471} 1672}
1472 1673
@@ -1502,6 +1703,7 @@ int vgic_init(struct kvm *kvm)
1502 ret |= vgic_init_bitmap(&dist->irq_pending, nr_cpus, nr_irqs); 1703 ret |= vgic_init_bitmap(&dist->irq_pending, nr_cpus, nr_irqs);
1503 ret |= vgic_init_bitmap(&dist->irq_soft_pend, nr_cpus, nr_irqs); 1704 ret |= vgic_init_bitmap(&dist->irq_soft_pend, nr_cpus, nr_irqs);
1504 ret |= vgic_init_bitmap(&dist->irq_queued, nr_cpus, nr_irqs); 1705 ret |= vgic_init_bitmap(&dist->irq_queued, nr_cpus, nr_irqs);
1706 ret |= vgic_init_bitmap(&dist->irq_active, nr_cpus, nr_irqs);
1505 ret |= vgic_init_bitmap(&dist->irq_cfg, nr_cpus, nr_irqs); 1707 ret |= vgic_init_bitmap(&dist->irq_cfg, nr_cpus, nr_irqs);
1506 ret |= vgic_init_bytemap(&dist->irq_priority, nr_cpus, nr_irqs); 1708 ret |= vgic_init_bytemap(&dist->irq_priority, nr_cpus, nr_irqs);
1507 1709
@@ -1514,10 +1716,13 @@ int vgic_init(struct kvm *kvm)
1514 GFP_KERNEL); 1716 GFP_KERNEL);
1515 dist->irq_pending_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long), 1717 dist->irq_pending_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
1516 GFP_KERNEL); 1718 GFP_KERNEL);
1719 dist->irq_active_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
1720 GFP_KERNEL);
1517 if (!dist->irq_sgi_sources || 1721 if (!dist->irq_sgi_sources ||
1518 !dist->irq_spi_cpu || 1722 !dist->irq_spi_cpu ||
1519 !dist->irq_spi_target || 1723 !dist->irq_spi_target ||
1520 !dist->irq_pending_on_cpu) { 1724 !dist->irq_pending_on_cpu ||
1725 !dist->irq_active_on_cpu) {
1521 ret = -ENOMEM; 1726 ret = -ENOMEM;
1522 goto out; 1727 goto out;
1523 } 1728 }
@@ -1845,12 +2050,9 @@ int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1845 return r; 2050 return r;
1846} 2051}
1847 2052
1848int vgic_has_attr_regs(const struct kvm_mmio_range *ranges, phys_addr_t offset) 2053int vgic_has_attr_regs(const struct vgic_io_range *ranges, phys_addr_t offset)
1849{ 2054{
1850 struct kvm_exit_mmio dev_attr_mmio; 2055 if (vgic_find_range(ranges, 4, offset))
1851
1852 dev_attr_mmio.len = 4;
1853 if (vgic_find_range(ranges, &dev_attr_mmio, offset))
1854 return 0; 2056 return 0;
1855 else 2057 else
1856 return -ENXIO; 2058 return -ENXIO;
@@ -1883,8 +2085,10 @@ static struct notifier_block vgic_cpu_nb = {
1883}; 2085};
1884 2086
1885static const struct of_device_id vgic_ids[] = { 2087static const struct of_device_id vgic_ids[] = {
1886 { .compatible = "arm,cortex-a15-gic", .data = vgic_v2_probe, }, 2088 { .compatible = "arm,cortex-a15-gic", .data = vgic_v2_probe, },
1887 { .compatible = "arm,gic-v3", .data = vgic_v3_probe, }, 2089 { .compatible = "arm,cortex-a7-gic", .data = vgic_v2_probe, },
2090 { .compatible = "arm,gic-400", .data = vgic_v2_probe, },
2091 { .compatible = "arm,gic-v3", .data = vgic_v3_probe, },
1888 {}, 2092 {},
1889}; 2093};
1890 2094
@@ -1932,3 +2136,38 @@ out_free_irq:
1932 free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus()); 2136 free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
1933 return ret; 2137 return ret;
1934} 2138}
2139
2140int kvm_irq_map_gsi(struct kvm *kvm,
2141 struct kvm_kernel_irq_routing_entry *entries,
2142 int gsi)
2143{
2144 return gsi;
2145}
2146
2147int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
2148{
2149 return pin;
2150}
2151
2152int kvm_set_irq(struct kvm *kvm, int irq_source_id,
2153 u32 irq, int level, bool line_status)
2154{
2155 unsigned int spi = irq + VGIC_NR_PRIVATE_IRQS;
2156
2157 trace_kvm_set_irq(irq, level, irq_source_id);
2158
2159 BUG_ON(!vgic_initialized(kvm));
2160
2161 if (spi > kvm->arch.vgic.nr_irqs)
2162 return -EINVAL;
2163 return kvm_vgic_inject_irq(kvm, 0, spi, level);
2164
2165}
2166
2167/* MSI not implemented yet */
2168int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
2169 struct kvm *kvm, int irq_source_id,
2170 int level, bool line_status)
2171{
2172 return 0;
2173}
diff --git a/virt/kvm/arm/vgic.h b/virt/kvm/arm/vgic.h
index 1e83bdf5f499..0df74cbb6200 100644
--- a/virt/kvm/arm/vgic.h
+++ b/virt/kvm/arm/vgic.h
@@ -20,6 +20,8 @@
20#ifndef __KVM_VGIC_H__ 20#ifndef __KVM_VGIC_H__
21#define __KVM_VGIC_H__ 21#define __KVM_VGIC_H__
22 22
23#include <kvm/iodev.h>
24
23#define VGIC_ADDR_UNDEF (-1) 25#define VGIC_ADDR_UNDEF (-1)
24#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF) 26#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF)
25 27
@@ -57,6 +59,14 @@ void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
57bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq); 59bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq);
58void vgic_unqueue_irqs(struct kvm_vcpu *vcpu); 60void vgic_unqueue_irqs(struct kvm_vcpu *vcpu);
59 61
62struct kvm_exit_mmio {
63 phys_addr_t phys_addr;
64 void *data;
65 u32 len;
66 bool is_write;
67 void *private;
68};
69
60void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg, 70void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
61 phys_addr_t offset, int mode); 71 phys_addr_t offset, int mode);
62bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio, 72bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
@@ -74,7 +84,7 @@ void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
74 *((u32 *)mmio->data) = cpu_to_le32(value) & mask; 84 *((u32 *)mmio->data) = cpu_to_le32(value) & mask;
75} 85}
76 86
77struct kvm_mmio_range { 87struct vgic_io_range {
78 phys_addr_t base; 88 phys_addr_t base;
79 unsigned long len; 89 unsigned long len;
80 int bits_per_irq; 90 int bits_per_irq;
@@ -82,6 +92,11 @@ struct kvm_mmio_range {
82 phys_addr_t offset); 92 phys_addr_t offset);
83}; 93};
84 94
95int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,
96 const struct vgic_io_range *ranges,
97 int redist_id,
98 struct vgic_io_device *iodev);
99
85static inline bool is_in_range(phys_addr_t addr, unsigned long len, 100static inline bool is_in_range(phys_addr_t addr, unsigned long len,
86 phys_addr_t baseaddr, unsigned long size) 101 phys_addr_t baseaddr, unsigned long size)
87{ 102{
@@ -89,14 +104,8 @@ static inline bool is_in_range(phys_addr_t addr, unsigned long len,
89} 104}
90 105
91const 106const
92struct kvm_mmio_range *vgic_find_range(const struct kvm_mmio_range *ranges, 107struct vgic_io_range *vgic_find_range(const struct vgic_io_range *ranges,
93 struct kvm_exit_mmio *mmio, 108 int len, gpa_t offset);
94 phys_addr_t offset);
95
96bool vgic_handle_mmio_range(struct kvm_vcpu *vcpu, struct kvm_run *run,
97 struct kvm_exit_mmio *mmio,
98 const struct kvm_mmio_range *ranges,
99 unsigned long mmio_base);
100 109
101bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio, 110bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
102 phys_addr_t offset, int vcpu_id, int access); 111 phys_addr_t offset, int vcpu_id, int access);
@@ -107,12 +116,20 @@ bool vgic_handle_set_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
107bool vgic_handle_clear_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio, 116bool vgic_handle_clear_pending_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
108 phys_addr_t offset, int vcpu_id); 117 phys_addr_t offset, int vcpu_id);
109 118
119bool vgic_handle_set_active_reg(struct kvm *kvm,
120 struct kvm_exit_mmio *mmio,
121 phys_addr_t offset, int vcpu_id);
122
123bool vgic_handle_clear_active_reg(struct kvm *kvm,
124 struct kvm_exit_mmio *mmio,
125 phys_addr_t offset, int vcpu_id);
126
110bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio, 127bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
111 phys_addr_t offset); 128 phys_addr_t offset);
112 129
113void vgic_kick_vcpus(struct kvm *kvm); 130void vgic_kick_vcpus(struct kvm *kvm);
114 131
115int vgic_has_attr_regs(const struct kvm_mmio_range *ranges, phys_addr_t offset); 132int vgic_has_attr_regs(const struct vgic_io_range *ranges, phys_addr_t offset);
116int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr); 133int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr);
117int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr); 134int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr);
118 135
diff --git a/virt/kvm/coalesced_mmio.c b/virt/kvm/coalesced_mmio.c
index 00d86427af0f..571c1ce37d15 100644
--- a/virt/kvm/coalesced_mmio.c
+++ b/virt/kvm/coalesced_mmio.c
@@ -8,7 +8,7 @@
8 * 8 *
9 */ 9 */
10 10
11#include "iodev.h" 11#include <kvm/iodev.h>
12 12
13#include <linux/kvm_host.h> 13#include <linux/kvm_host.h>
14#include <linux/slab.h> 14#include <linux/slab.h>
@@ -60,8 +60,9 @@ static int coalesced_mmio_has_room(struct kvm_coalesced_mmio_dev *dev)
60 return 1; 60 return 1;
61} 61}
62 62
63static int coalesced_mmio_write(struct kvm_io_device *this, 63static int coalesced_mmio_write(struct kvm_vcpu *vcpu,
64 gpa_t addr, int len, const void *val) 64 struct kvm_io_device *this, gpa_t addr,
65 int len, const void *val)
65{ 66{
66 struct kvm_coalesced_mmio_dev *dev = to_mmio(this); 67 struct kvm_coalesced_mmio_dev *dev = to_mmio(this);
67 struct kvm_coalesced_mmio_ring *ring = dev->kvm->coalesced_mmio_ring; 68 struct kvm_coalesced_mmio_ring *ring = dev->kvm->coalesced_mmio_ring;
diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c
index 148b2392c762..9ff4193dfa49 100644
--- a/virt/kvm/eventfd.c
+++ b/virt/kvm/eventfd.c
@@ -36,7 +36,7 @@
36#include <linux/seqlock.h> 36#include <linux/seqlock.h>
37#include <trace/events/kvm.h> 37#include <trace/events/kvm.h>
38 38
39#include "iodev.h" 39#include <kvm/iodev.h>
40 40
41#ifdef CONFIG_HAVE_KVM_IRQFD 41#ifdef CONFIG_HAVE_KVM_IRQFD
42/* 42/*
@@ -311,6 +311,9 @@ kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
311 unsigned int events; 311 unsigned int events;
312 int idx; 312 int idx;
313 313
314 if (!kvm_arch_intc_initialized(kvm))
315 return -EAGAIN;
316
314 irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL); 317 irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL);
315 if (!irqfd) 318 if (!irqfd)
316 return -ENOMEM; 319 return -ENOMEM;
@@ -712,8 +715,8 @@ ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val)
712 715
713/* MMIO/PIO writes trigger an event if the addr/val match */ 716/* MMIO/PIO writes trigger an event if the addr/val match */
714static int 717static int
715ioeventfd_write(struct kvm_io_device *this, gpa_t addr, int len, 718ioeventfd_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, gpa_t addr,
716 const void *val) 719 int len, const void *val)
717{ 720{
718 struct _ioeventfd *p = to_ioeventfd(this); 721 struct _ioeventfd *p = to_ioeventfd(this);
719 722
diff --git a/virt/kvm/irqchip.c b/virt/kvm/irqchip.c
index 7f256f31df10..1d56a901e791 100644
--- a/virt/kvm/irqchip.c
+++ b/virt/kvm/irqchip.c
@@ -105,7 +105,7 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
105 i = kvm_irq_map_gsi(kvm, irq_set, irq); 105 i = kvm_irq_map_gsi(kvm, irq_set, irq);
106 srcu_read_unlock(&kvm->irq_srcu, idx); 106 srcu_read_unlock(&kvm->irq_srcu, idx);
107 107
108 while(i--) { 108 while (i--) {
109 int r; 109 int r;
110 r = irq_set[i].set(&irq_set[i], kvm, irq_source_id, level, 110 r = irq_set[i].set(&irq_set[i], kvm, irq_source_id, level,
111 line_status); 111 line_status);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index cc6a25d95fbf..d3fc9399062a 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -16,7 +16,7 @@
16 * 16 *
17 */ 17 */
18 18
19#include "iodev.h" 19#include <kvm/iodev.h>
20 20
21#include <linux/kvm_host.h> 21#include <linux/kvm_host.h>
22#include <linux/kvm.h> 22#include <linux/kvm.h>
@@ -66,13 +66,13 @@
66MODULE_AUTHOR("Qumranet"); 66MODULE_AUTHOR("Qumranet");
67MODULE_LICENSE("GPL"); 67MODULE_LICENSE("GPL");
68 68
69unsigned int halt_poll_ns = 0; 69static unsigned int halt_poll_ns;
70module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR); 70module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
71 71
72/* 72/*
73 * Ordering of locks: 73 * Ordering of locks:
74 * 74 *
75 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 75 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
76 */ 76 */
77 77
78DEFINE_SPINLOCK(kvm_lock); 78DEFINE_SPINLOCK(kvm_lock);
@@ -80,7 +80,7 @@ static DEFINE_RAW_SPINLOCK(kvm_count_lock);
80LIST_HEAD(vm_list); 80LIST_HEAD(vm_list);
81 81
82static cpumask_var_t cpus_hardware_enabled; 82static cpumask_var_t cpus_hardware_enabled;
83static int kvm_usage_count = 0; 83static int kvm_usage_count;
84static atomic_t hardware_enable_failed; 84static atomic_t hardware_enable_failed;
85 85
86struct kmem_cache *kvm_vcpu_cache; 86struct kmem_cache *kvm_vcpu_cache;
@@ -539,20 +539,12 @@ void *kvm_kvzalloc(unsigned long size)
539 return kzalloc(size, GFP_KERNEL); 539 return kzalloc(size, GFP_KERNEL);
540} 540}
541 541
542void kvm_kvfree(const void *addr)
543{
544 if (is_vmalloc_addr(addr))
545 vfree(addr);
546 else
547 kfree(addr);
548}
549
550static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 542static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
551{ 543{
552 if (!memslot->dirty_bitmap) 544 if (!memslot->dirty_bitmap)
553 return; 545 return;
554 546
555 kvm_kvfree(memslot->dirty_bitmap); 547 kvfree(memslot->dirty_bitmap);
556 memslot->dirty_bitmap = NULL; 548 memslot->dirty_bitmap = NULL;
557} 549}
558 550
@@ -888,8 +880,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
888 * or moved, memslot will be created. 880 * or moved, memslot will be created.
889 * 881 *
890 * validation of sp->gfn happens in: 882 * validation of sp->gfn happens in:
891 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 883 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
892 * - kvm_is_visible_gfn (mmu_check_roots) 884 * - kvm_is_visible_gfn (mmu_check_roots)
893 */ 885 */
894 kvm_arch_flush_shadow_memslot(kvm, slot); 886 kvm_arch_flush_shadow_memslot(kvm, slot);
895 887
@@ -1061,9 +1053,11 @@ int kvm_get_dirty_log_protect(struct kvm *kvm,
1061 mask = xchg(&dirty_bitmap[i], 0); 1053 mask = xchg(&dirty_bitmap[i], 0);
1062 dirty_bitmap_buffer[i] = mask; 1054 dirty_bitmap_buffer[i] = mask;
1063 1055
1064 offset = i * BITS_PER_LONG; 1056 if (mask) {
1065 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, 1057 offset = i * BITS_PER_LONG;
1066 mask); 1058 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1059 offset, mask);
1060 }
1067 } 1061 }
1068 1062
1069 spin_unlock(&kvm->mmu_lock); 1063 spin_unlock(&kvm->mmu_lock);
@@ -1193,16 +1187,6 @@ unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1193 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1187 return gfn_to_hva_memslot_prot(slot, gfn, writable);
1194} 1188}
1195 1189
1196static int kvm_read_hva(void *data, void __user *hva, int len)
1197{
1198 return __copy_from_user(data, hva, len);
1199}
1200
1201static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
1202{
1203 return __copy_from_user_inatomic(data, hva, len);
1204}
1205
1206static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm, 1190static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1207 unsigned long start, int write, struct page **page) 1191 unsigned long start, int write, struct page **page)
1208{ 1192{
@@ -1481,7 +1465,6 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1481 1465
1482 return kvm_pfn_to_page(pfn); 1466 return kvm_pfn_to_page(pfn);
1483} 1467}
1484
1485EXPORT_SYMBOL_GPL(gfn_to_page); 1468EXPORT_SYMBOL_GPL(gfn_to_page);
1486 1469
1487void kvm_release_page_clean(struct page *page) 1470void kvm_release_page_clean(struct page *page)
@@ -1517,6 +1500,7 @@ void kvm_set_pfn_dirty(pfn_t pfn)
1517{ 1500{
1518 if (!kvm_is_reserved_pfn(pfn)) { 1501 if (!kvm_is_reserved_pfn(pfn)) {
1519 struct page *page = pfn_to_page(pfn); 1502 struct page *page = pfn_to_page(pfn);
1503
1520 if (!PageReserved(page)) 1504 if (!PageReserved(page))
1521 SetPageDirty(page); 1505 SetPageDirty(page);
1522 } 1506 }
@@ -1554,7 +1538,7 @@ int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1554 addr = gfn_to_hva_prot(kvm, gfn, NULL); 1538 addr = gfn_to_hva_prot(kvm, gfn, NULL);
1555 if (kvm_is_error_hva(addr)) 1539 if (kvm_is_error_hva(addr))
1556 return -EFAULT; 1540 return -EFAULT;
1557 r = kvm_read_hva(data, (void __user *)addr + offset, len); 1541 r = __copy_from_user(data, (void __user *)addr + offset, len);
1558 if (r) 1542 if (r)
1559 return -EFAULT; 1543 return -EFAULT;
1560 return 0; 1544 return 0;
@@ -1593,7 +1577,7 @@ int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1593 if (kvm_is_error_hva(addr)) 1577 if (kvm_is_error_hva(addr))
1594 return -EFAULT; 1578 return -EFAULT;
1595 pagefault_disable(); 1579 pagefault_disable();
1596 r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len); 1580 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1597 pagefault_enable(); 1581 pagefault_enable();
1598 if (r) 1582 if (r)
1599 return -EFAULT; 1583 return -EFAULT;
@@ -1653,8 +1637,8 @@ int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1653 ghc->generation = slots->generation; 1637 ghc->generation = slots->generation;
1654 ghc->len = len; 1638 ghc->len = len;
1655 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1639 ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1656 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, &nr_pages_avail); 1640 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
1657 if (!kvm_is_error_hva(ghc->hva) && nr_pages_avail >= nr_pages_needed) { 1641 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
1658 ghc->hva += offset; 1642 ghc->hva += offset;
1659 } else { 1643 } else {
1660 /* 1644 /*
@@ -1742,7 +1726,7 @@ int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1742 int offset = offset_in_page(gpa); 1726 int offset = offset_in_page(gpa);
1743 int ret; 1727 int ret;
1744 1728
1745 while ((seg = next_segment(len, offset)) != 0) { 1729 while ((seg = next_segment(len, offset)) != 0) {
1746 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1730 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1747 if (ret < 0) 1731 if (ret < 0)
1748 return ret; 1732 return ret;
@@ -1800,6 +1784,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1800 start = cur = ktime_get(); 1784 start = cur = ktime_get();
1801 if (halt_poll_ns) { 1785 if (halt_poll_ns) {
1802 ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns); 1786 ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns);
1787
1803 do { 1788 do {
1804 /* 1789 /*
1805 * This sets KVM_REQ_UNHALT if an interrupt 1790 * This sets KVM_REQ_UNHALT if an interrupt
@@ -2118,7 +2103,7 @@ static long kvm_vcpu_ioctl(struct file *filp,
2118 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2103 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
2119 * so vcpu_load() would break it. 2104 * so vcpu_load() would break it.
2120 */ 2105 */
2121 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT) 2106 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT)
2122 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2107 return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2123#endif 2108#endif
2124 2109
@@ -2135,6 +2120,7 @@ static long kvm_vcpu_ioctl(struct file *filp,
2135 /* The thread running this VCPU changed. */ 2120 /* The thread running this VCPU changed. */
2136 struct pid *oldpid = vcpu->pid; 2121 struct pid *oldpid = vcpu->pid;
2137 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 2122 struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
2123
2138 rcu_assign_pointer(vcpu->pid, newpid); 2124 rcu_assign_pointer(vcpu->pid, newpid);
2139 if (oldpid) 2125 if (oldpid)
2140 synchronize_rcu(); 2126 synchronize_rcu();
@@ -2205,7 +2191,7 @@ out_free1:
2205 if (r) 2191 if (r)
2206 goto out; 2192 goto out;
2207 r = -EFAULT; 2193 r = -EFAULT;
2208 if (copy_to_user(argp, &mp_state, sizeof mp_state)) 2194 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2209 goto out; 2195 goto out;
2210 r = 0; 2196 r = 0;
2211 break; 2197 break;
@@ -2214,7 +2200,7 @@ out_free1:
2214 struct kvm_mp_state mp_state; 2200 struct kvm_mp_state mp_state;
2215 2201
2216 r = -EFAULT; 2202 r = -EFAULT;
2217 if (copy_from_user(&mp_state, argp, sizeof mp_state)) 2203 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2218 goto out; 2204 goto out;
2219 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2205 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2220 break; 2206 break;
@@ -2223,13 +2209,13 @@ out_free1:
2223 struct kvm_translation tr; 2209 struct kvm_translation tr;
2224 2210
2225 r = -EFAULT; 2211 r = -EFAULT;
2226 if (copy_from_user(&tr, argp, sizeof tr)) 2212 if (copy_from_user(&tr, argp, sizeof(tr)))
2227 goto out; 2213 goto out;
2228 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2214 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2229 if (r) 2215 if (r)
2230 goto out; 2216 goto out;
2231 r = -EFAULT; 2217 r = -EFAULT;
2232 if (copy_to_user(argp, &tr, sizeof tr)) 2218 if (copy_to_user(argp, &tr, sizeof(tr)))
2233 goto out; 2219 goto out;
2234 r = 0; 2220 r = 0;
2235 break; 2221 break;
@@ -2238,7 +2224,7 @@ out_free1:
2238 struct kvm_guest_debug dbg; 2224 struct kvm_guest_debug dbg;
2239 2225
2240 r = -EFAULT; 2226 r = -EFAULT;
2241 if (copy_from_user(&dbg, argp, sizeof dbg)) 2227 if (copy_from_user(&dbg, argp, sizeof(dbg)))
2242 goto out; 2228 goto out;
2243 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2229 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2244 break; 2230 break;
@@ -2252,14 +2238,14 @@ out_free1:
2252 if (argp) { 2238 if (argp) {
2253 r = -EFAULT; 2239 r = -EFAULT;
2254 if (copy_from_user(&kvm_sigmask, argp, 2240 if (copy_from_user(&kvm_sigmask, argp,
2255 sizeof kvm_sigmask)) 2241 sizeof(kvm_sigmask)))
2256 goto out; 2242 goto out;
2257 r = -EINVAL; 2243 r = -EINVAL;
2258 if (kvm_sigmask.len != sizeof sigset) 2244 if (kvm_sigmask.len != sizeof(sigset))
2259 goto out; 2245 goto out;
2260 r = -EFAULT; 2246 r = -EFAULT;
2261 if (copy_from_user(&sigset, sigmask_arg->sigset, 2247 if (copy_from_user(&sigset, sigmask_arg->sigset,
2262 sizeof sigset)) 2248 sizeof(sigset)))
2263 goto out; 2249 goto out;
2264 p = &sigset; 2250 p = &sigset;
2265 } 2251 }
@@ -2321,14 +2307,14 @@ static long kvm_vcpu_compat_ioctl(struct file *filp,
2321 if (argp) { 2307 if (argp) {
2322 r = -EFAULT; 2308 r = -EFAULT;
2323 if (copy_from_user(&kvm_sigmask, argp, 2309 if (copy_from_user(&kvm_sigmask, argp,
2324 sizeof kvm_sigmask)) 2310 sizeof(kvm_sigmask)))
2325 goto out; 2311 goto out;
2326 r = -EINVAL; 2312 r = -EINVAL;
2327 if (kvm_sigmask.len != sizeof csigset) 2313 if (kvm_sigmask.len != sizeof(csigset))
2328 goto out; 2314 goto out;
2329 r = -EFAULT; 2315 r = -EFAULT;
2330 if (copy_from_user(&csigset, sigmask_arg->sigset, 2316 if (copy_from_user(&csigset, sigmask_arg->sigset,
2331 sizeof csigset)) 2317 sizeof(csigset)))
2332 goto out; 2318 goto out;
2333 sigset_from_compat(&sigset, &csigset); 2319 sigset_from_compat(&sigset, &csigset);
2334 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2320 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
@@ -2525,7 +2511,7 @@ static long kvm_vm_ioctl(struct file *filp,
2525 2511
2526 r = -EFAULT; 2512 r = -EFAULT;
2527 if (copy_from_user(&kvm_userspace_mem, argp, 2513 if (copy_from_user(&kvm_userspace_mem, argp,
2528 sizeof kvm_userspace_mem)) 2514 sizeof(kvm_userspace_mem)))
2529 goto out; 2515 goto out;
2530 2516
2531 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2517 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
@@ -2535,7 +2521,7 @@ static long kvm_vm_ioctl(struct file *filp,
2535 struct kvm_dirty_log log; 2521 struct kvm_dirty_log log;
2536 2522
2537 r = -EFAULT; 2523 r = -EFAULT;
2538 if (copy_from_user(&log, argp, sizeof log)) 2524 if (copy_from_user(&log, argp, sizeof(log)))
2539 goto out; 2525 goto out;
2540 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2526 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2541 break; 2527 break;
@@ -2543,16 +2529,18 @@ static long kvm_vm_ioctl(struct file *filp,
2543#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2529#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2544 case KVM_REGISTER_COALESCED_MMIO: { 2530 case KVM_REGISTER_COALESCED_MMIO: {
2545 struct kvm_coalesced_mmio_zone zone; 2531 struct kvm_coalesced_mmio_zone zone;
2532
2546 r = -EFAULT; 2533 r = -EFAULT;
2547 if (copy_from_user(&zone, argp, sizeof zone)) 2534 if (copy_from_user(&zone, argp, sizeof(zone)))
2548 goto out; 2535 goto out;
2549 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2536 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2550 break; 2537 break;
2551 } 2538 }
2552 case KVM_UNREGISTER_COALESCED_MMIO: { 2539 case KVM_UNREGISTER_COALESCED_MMIO: {
2553 struct kvm_coalesced_mmio_zone zone; 2540 struct kvm_coalesced_mmio_zone zone;
2541
2554 r = -EFAULT; 2542 r = -EFAULT;
2555 if (copy_from_user(&zone, argp, sizeof zone)) 2543 if (copy_from_user(&zone, argp, sizeof(zone)))
2556 goto out; 2544 goto out;
2557 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2545 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2558 break; 2546 break;
@@ -2562,7 +2550,7 @@ static long kvm_vm_ioctl(struct file *filp,
2562 struct kvm_irqfd data; 2550 struct kvm_irqfd data;
2563 2551
2564 r = -EFAULT; 2552 r = -EFAULT;
2565 if (copy_from_user(&data, argp, sizeof data)) 2553 if (copy_from_user(&data, argp, sizeof(data)))
2566 goto out; 2554 goto out;
2567 r = kvm_irqfd(kvm, &data); 2555 r = kvm_irqfd(kvm, &data);
2568 break; 2556 break;
@@ -2571,7 +2559,7 @@ static long kvm_vm_ioctl(struct file *filp,
2571 struct kvm_ioeventfd data; 2559 struct kvm_ioeventfd data;
2572 2560
2573 r = -EFAULT; 2561 r = -EFAULT;
2574 if (copy_from_user(&data, argp, sizeof data)) 2562 if (copy_from_user(&data, argp, sizeof(data)))
2575 goto out; 2563 goto out;
2576 r = kvm_ioeventfd(kvm, &data); 2564 r = kvm_ioeventfd(kvm, &data);
2577 break; 2565 break;
@@ -2592,7 +2580,7 @@ static long kvm_vm_ioctl(struct file *filp,
2592 struct kvm_msi msi; 2580 struct kvm_msi msi;
2593 2581
2594 r = -EFAULT; 2582 r = -EFAULT;
2595 if (copy_from_user(&msi, argp, sizeof msi)) 2583 if (copy_from_user(&msi, argp, sizeof(msi)))
2596 goto out; 2584 goto out;
2597 r = kvm_send_userspace_msi(kvm, &msi); 2585 r = kvm_send_userspace_msi(kvm, &msi);
2598 break; 2586 break;
@@ -2604,7 +2592,7 @@ static long kvm_vm_ioctl(struct file *filp,
2604 struct kvm_irq_level irq_event; 2592 struct kvm_irq_level irq_event;
2605 2593
2606 r = -EFAULT; 2594 r = -EFAULT;
2607 if (copy_from_user(&irq_event, argp, sizeof irq_event)) 2595 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
2608 goto out; 2596 goto out;
2609 2597
2610 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 2598 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
@@ -2614,7 +2602,7 @@ static long kvm_vm_ioctl(struct file *filp,
2614 2602
2615 r = -EFAULT; 2603 r = -EFAULT;
2616 if (ioctl == KVM_IRQ_LINE_STATUS) { 2604 if (ioctl == KVM_IRQ_LINE_STATUS) {
2617 if (copy_to_user(argp, &irq_event, sizeof irq_event)) 2605 if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
2618 goto out; 2606 goto out;
2619 } 2607 }
2620 2608
@@ -2647,7 +2635,7 @@ static long kvm_vm_ioctl(struct file *filp,
2647 goto out_free_irq_routing; 2635 goto out_free_irq_routing;
2648 r = kvm_set_irq_routing(kvm, entries, routing.nr, 2636 r = kvm_set_irq_routing(kvm, entries, routing.nr,
2649 routing.flags); 2637 routing.flags);
2650 out_free_irq_routing: 2638out_free_irq_routing:
2651 vfree(entries); 2639 vfree(entries);
2652 break; 2640 break;
2653 } 2641 }
@@ -2822,8 +2810,7 @@ static void hardware_enable_nolock(void *junk)
2822 if (r) { 2810 if (r) {
2823 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2811 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2824 atomic_inc(&hardware_enable_failed); 2812 atomic_inc(&hardware_enable_failed);
2825 printk(KERN_INFO "kvm: enabling virtualization on " 2813 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
2826 "CPU%d failed\n", cpu);
2827 } 2814 }
2828} 2815}
2829 2816
@@ -2899,12 +2886,12 @@ static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2899 val &= ~CPU_TASKS_FROZEN; 2886 val &= ~CPU_TASKS_FROZEN;
2900 switch (val) { 2887 switch (val) {
2901 case CPU_DYING: 2888 case CPU_DYING:
2902 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", 2889 pr_info("kvm: disabling virtualization on CPU%d\n",
2903 cpu); 2890 cpu);
2904 hardware_disable(); 2891 hardware_disable();
2905 break; 2892 break;
2906 case CPU_STARTING: 2893 case CPU_STARTING:
2907 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", 2894 pr_info("kvm: enabling virtualization on CPU%d\n",
2908 cpu); 2895 cpu);
2909 hardware_enable(); 2896 hardware_enable();
2910 break; 2897 break;
@@ -2921,7 +2908,7 @@ static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2921 * 2908 *
2922 * And Intel TXT required VMX off for all cpu when system shutdown. 2909 * And Intel TXT required VMX off for all cpu when system shutdown.
2923 */ 2910 */
2924 printk(KERN_INFO "kvm: exiting hardware virtualization\n"); 2911 pr_info("kvm: exiting hardware virtualization\n");
2925 kvm_rebooting = true; 2912 kvm_rebooting = true;
2926 on_each_cpu(hardware_disable_nolock, NULL, 1); 2913 on_each_cpu(hardware_disable_nolock, NULL, 1);
2927 return NOTIFY_OK; 2914 return NOTIFY_OK;
@@ -2945,7 +2932,7 @@ static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2945} 2932}
2946 2933
2947static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 2934static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
2948 const struct kvm_io_range *r2) 2935 const struct kvm_io_range *r2)
2949{ 2936{
2950 if (r1->addr < r2->addr) 2937 if (r1->addr < r2->addr)
2951 return -1; 2938 return -1;
@@ -2998,7 +2985,7 @@ static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
2998 return off; 2985 return off;
2999} 2986}
3000 2987
3001static int __kvm_io_bus_write(struct kvm_io_bus *bus, 2988static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3002 struct kvm_io_range *range, const void *val) 2989 struct kvm_io_range *range, const void *val)
3003{ 2990{
3004 int idx; 2991 int idx;
@@ -3009,7 +2996,7 @@ static int __kvm_io_bus_write(struct kvm_io_bus *bus,
3009 2996
3010 while (idx < bus->dev_count && 2997 while (idx < bus->dev_count &&
3011 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 2998 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3012 if (!kvm_iodevice_write(bus->range[idx].dev, range->addr, 2999 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
3013 range->len, val)) 3000 range->len, val))
3014 return idx; 3001 return idx;
3015 idx++; 3002 idx++;
@@ -3019,7 +3006,7 @@ static int __kvm_io_bus_write(struct kvm_io_bus *bus,
3019} 3006}
3020 3007
3021/* kvm_io_bus_write - called under kvm->slots_lock */ 3008/* kvm_io_bus_write - called under kvm->slots_lock */
3022int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3009int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3023 int len, const void *val) 3010 int len, const void *val)
3024{ 3011{
3025 struct kvm_io_bus *bus; 3012 struct kvm_io_bus *bus;
@@ -3031,14 +3018,14 @@ int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3031 .len = len, 3018 .len = len,
3032 }; 3019 };
3033 3020
3034 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 3021 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3035 r = __kvm_io_bus_write(bus, &range, val); 3022 r = __kvm_io_bus_write(vcpu, bus, &range, val);
3036 return r < 0 ? r : 0; 3023 return r < 0 ? r : 0;
3037} 3024}
3038 3025
3039/* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3026/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3040int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3027int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3041 int len, const void *val, long cookie) 3028 gpa_t addr, int len, const void *val, long cookie)
3042{ 3029{
3043 struct kvm_io_bus *bus; 3030 struct kvm_io_bus *bus;
3044 struct kvm_io_range range; 3031 struct kvm_io_range range;
@@ -3048,12 +3035,12 @@ int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3048 .len = len, 3035 .len = len,
3049 }; 3036 };
3050 3037
3051 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 3038 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3052 3039
3053 /* First try the device referenced by cookie. */ 3040 /* First try the device referenced by cookie. */
3054 if ((cookie >= 0) && (cookie < bus->dev_count) && 3041 if ((cookie >= 0) && (cookie < bus->dev_count) &&
3055 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3042 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3056 if (!kvm_iodevice_write(bus->range[cookie].dev, addr, len, 3043 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
3057 val)) 3044 val))
3058 return cookie; 3045 return cookie;
3059 3046
@@ -3061,11 +3048,11 @@ int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3061 * cookie contained garbage; fall back to search and return the 3048 * cookie contained garbage; fall back to search and return the
3062 * correct cookie value. 3049 * correct cookie value.
3063 */ 3050 */
3064 return __kvm_io_bus_write(bus, &range, val); 3051 return __kvm_io_bus_write(vcpu, bus, &range, val);
3065} 3052}
3066 3053
3067static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range, 3054static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3068 void *val) 3055 struct kvm_io_range *range, void *val)
3069{ 3056{
3070 int idx; 3057 int idx;
3071 3058
@@ -3075,7 +3062,7 @@ static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range,
3075 3062
3076 while (idx < bus->dev_count && 3063 while (idx < bus->dev_count &&
3077 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3064 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3078 if (!kvm_iodevice_read(bus->range[idx].dev, range->addr, 3065 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
3079 range->len, val)) 3066 range->len, val))
3080 return idx; 3067 return idx;
3081 idx++; 3068 idx++;
@@ -3086,7 +3073,7 @@ static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range,
3086EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3073EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3087 3074
3088/* kvm_io_bus_read - called under kvm->slots_lock */ 3075/* kvm_io_bus_read - called under kvm->slots_lock */
3089int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3076int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3090 int len, void *val) 3077 int len, void *val)
3091{ 3078{
3092 struct kvm_io_bus *bus; 3079 struct kvm_io_bus *bus;
@@ -3098,8 +3085,8 @@ int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3098 .len = len, 3085 .len = len,
3099 }; 3086 };
3100 3087
3101 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 3088 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3102 r = __kvm_io_bus_read(bus, &range, val); 3089 r = __kvm_io_bus_read(vcpu, bus, &range, val);
3103 return r < 0 ? r : 0; 3090 return r < 0 ? r : 0;
3104} 3091}
3105 3092
@@ -3269,6 +3256,7 @@ struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3269static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3256static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3270{ 3257{
3271 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3258 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3259
3272 if (vcpu->preempted) 3260 if (vcpu->preempted)
3273 vcpu->preempted = false; 3261 vcpu->preempted = false;
3274 3262
@@ -3350,7 +3338,7 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3350 3338
3351 r = misc_register(&kvm_dev); 3339 r = misc_register(&kvm_dev);
3352 if (r) { 3340 if (r) {
3353 printk(KERN_ERR "kvm: misc device register failed\n"); 3341 pr_err("kvm: misc device register failed\n");
3354 goto out_unreg; 3342 goto out_unreg;
3355 } 3343 }
3356 3344
@@ -3361,7 +3349,7 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3361 3349
3362 r = kvm_init_debug(); 3350 r = kvm_init_debug();
3363 if (r) { 3351 if (r) {
3364 printk(KERN_ERR "kvm: create debugfs files failed\n"); 3352 pr_err("kvm: create debugfs files failed\n");
3365 goto out_undebugfs; 3353 goto out_undebugfs;
3366 } 3354 }
3367 3355