aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRadim Krčmář <rkrcmar@redhat.com>2018-01-30 11:42:40 -0500
committerRadim Krčmář <rkrcmar@redhat.com>2018-01-30 11:42:40 -0500
commit92ea2b3381d7dc4223375f4337dcadb1fd556576 (patch)
treead403262229ec273bad951fd3e6ace01235927de
parentd7231e75f73f424068ad205358bd8ba6a7a2e113 (diff)
parent4b9f952577fb40875a2a163d80515a8daa0d6bef (diff)
Merge tag 'kvm-s390-next-4.16-2' of git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux
KVM: s390: Fixes and features for 4.16 part 2 - exitless interrupts for emulated devices (Michael Mueller) - cleanup of cpuflag handling (David Hildenbrand) - kvm stat counter improvements (Christian Borntraeger) - vsie improvements (David Hildenbrand) - mm cleanup (Janosch Frank)
Diffstat
-rw-r--r--arch/s390/include/asm/bitops.h5
-rw-r--r--arch/s390/include/asm/css_chars.h4
-rw-r--r--arch/s390/include/asm/kvm_host.h123
-rw-r--r--arch/s390/include/asm/sclp.h1
-rw-r--r--arch/s390/kvm/diag.c1
-rw-r--r--arch/s390/kvm/interrupt.c279
-rw-r--r--arch/s390/kvm/kvm-s390.c74
-rw-r--r--arch/s390/kvm/kvm-s390.h20
-rw-r--r--arch/s390/kvm/priv.c38
-rw-r--r--arch/s390/kvm/sigp.c14
-rw-r--r--arch/s390/kvm/vsie.c91
-rw-r--r--arch/s390/mm/gmap.c21
-rw-r--r--drivers/s390/char/sclp_early.c3
13 files changed, 496 insertions, 178 deletions
diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h
index 31e400c1a1f3..86e5b2fdee3c 100644
--- a/arch/s390/include/asm/bitops.h
+++ b/arch/s390/include/asm/bitops.h
@@ -261,6 +261,11 @@ static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
261 return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr); 261 return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
262} 262}
263 263
264static inline int test_and_clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
265{
266 return test_and_clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
267}
268
264static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr) 269static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
265{ 270{
266 return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr); 271 return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
diff --git a/arch/s390/include/asm/css_chars.h b/arch/s390/include/asm/css_chars.h
index a478eb61aaf7..fb56fa3283a2 100644
--- a/arch/s390/include/asm/css_chars.h
+++ b/arch/s390/include/asm/css_chars.h
@@ -20,7 +20,9 @@ struct css_general_char {
20 u32 aif_tdd : 1; /* bit 56 */ 20 u32 aif_tdd : 1; /* bit 56 */
21 u32 : 1; 21 u32 : 1;
22 u32 qebsm : 1; /* bit 58 */ 22 u32 qebsm : 1; /* bit 58 */
23 u32 : 8; 23 u32 : 2;
24 u32 aiv : 1; /* bit 61 */
25 u32 : 5;
24 u32 aif_osa : 1; /* bit 67 */ 26 u32 aif_osa : 1; /* bit 67 */
25 u32 : 12; 27 u32 : 12;
26 u32 eadm_rf : 1; /* bit 80 */ 28 u32 eadm_rf : 1; /* bit 80 */
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index e16a9f2a44ad..59dd46adf0e8 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -2,7 +2,7 @@
2/* 2/*
3 * definition for kernel virtual machines on s390 3 * definition for kernel virtual machines on s390
4 * 4 *
5 * Copyright IBM Corp. 2008, 2009 5 * Copyright IBM Corp. 2008, 2018
6 * 6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com> 7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 */ 8 */
@@ -183,6 +183,7 @@ struct kvm_s390_sie_block {
183#define ECA_IB 0x40000000 183#define ECA_IB 0x40000000
184#define ECA_SIGPI 0x10000000 184#define ECA_SIGPI 0x10000000
185#define ECA_MVPGI 0x01000000 185#define ECA_MVPGI 0x01000000
186#define ECA_AIV 0x00200000
186#define ECA_VX 0x00020000 187#define ECA_VX 0x00020000
187#define ECA_PROTEXCI 0x00002000 188#define ECA_PROTEXCI 0x00002000
188#define ECA_SII 0x00000001 189#define ECA_SII 0x00000001
@@ -227,7 +228,9 @@ struct kvm_s390_sie_block {
227 __u8 epdx; /* 0x0069 */ 228 __u8 epdx; /* 0x0069 */
228 __u8 reserved6a[2]; /* 0x006a */ 229 __u8 reserved6a[2]; /* 0x006a */
229 __u32 todpr; /* 0x006c */ 230 __u32 todpr; /* 0x006c */
230 __u8 reserved70[16]; /* 0x0070 */ 231#define GISA_FORMAT1 0x00000001
232 __u32 gd; /* 0x0070 */
233 __u8 reserved74[12]; /* 0x0074 */
231 __u64 mso; /* 0x0080 */ 234 __u64 mso; /* 0x0080 */
232 __u64 msl; /* 0x0088 */ 235 __u64 msl; /* 0x0088 */
233 psw_t gpsw; /* 0x0090 */ 236 psw_t gpsw; /* 0x0090 */
@@ -316,18 +319,30 @@ struct kvm_vcpu_stat {
316 u64 deliver_program_int; 319 u64 deliver_program_int;
317 u64 deliver_io_int; 320 u64 deliver_io_int;
318 u64 exit_wait_state; 321 u64 exit_wait_state;
322 u64 instruction_epsw;
323 u64 instruction_gs;
324 u64 instruction_io_other;
325 u64 instruction_lpsw;
326 u64 instruction_lpswe;
319 u64 instruction_pfmf; 327 u64 instruction_pfmf;
328 u64 instruction_ptff;
329 u64 instruction_sck;
330 u64 instruction_sckpf;
320 u64 instruction_stidp; 331 u64 instruction_stidp;
321 u64 instruction_spx; 332 u64 instruction_spx;
322 u64 instruction_stpx; 333 u64 instruction_stpx;
323 u64 instruction_stap; 334 u64 instruction_stap;
324 u64 instruction_storage_key; 335 u64 instruction_iske;
336 u64 instruction_ri;
337 u64 instruction_rrbe;
338 u64 instruction_sske;
325 u64 instruction_ipte_interlock; 339 u64 instruction_ipte_interlock;
326 u64 instruction_stsch;
327 u64 instruction_chsc;
328 u64 instruction_stsi; 340 u64 instruction_stsi;
329 u64 instruction_stfl; 341 u64 instruction_stfl;
342 u64 instruction_tb;
343 u64 instruction_tpi;
330 u64 instruction_tprot; 344 u64 instruction_tprot;
345 u64 instruction_tsch;
331 u64 instruction_sie; 346 u64 instruction_sie;
332 u64 instruction_essa; 347 u64 instruction_essa;
333 u64 instruction_sthyi; 348 u64 instruction_sthyi;
@@ -353,6 +368,7 @@ struct kvm_vcpu_stat {
353 u64 diagnose_258; 368 u64 diagnose_258;
354 u64 diagnose_308; 369 u64 diagnose_308;
355 u64 diagnose_500; 370 u64 diagnose_500;
371 u64 diagnose_other;
356}; 372};
357 373
358#define PGM_OPERATION 0x01 374#define PGM_OPERATION 0x01
@@ -409,35 +425,35 @@ struct kvm_vcpu_stat {
409#define PGM_PER 0x80 425#define PGM_PER 0x80
410#define PGM_CRYPTO_OPERATION 0x119 426#define PGM_CRYPTO_OPERATION 0x119
411 427
412/* irq types in order of priority */ 428/* irq types in ascend order of priorities */
413enum irq_types { 429enum irq_types {
414 IRQ_PEND_MCHK_EX = 0, 430 IRQ_PEND_SET_PREFIX = 0,
415 IRQ_PEND_SVC,
416 IRQ_PEND_PROG,
417 IRQ_PEND_MCHK_REP,
418 IRQ_PEND_EXT_IRQ_KEY,
419 IRQ_PEND_EXT_MALFUNC,
420 IRQ_PEND_EXT_EMERGENCY,
421 IRQ_PEND_EXT_EXTERNAL,
422 IRQ_PEND_EXT_CLOCK_COMP,
423 IRQ_PEND_EXT_CPU_TIMER,
424 IRQ_PEND_EXT_TIMING,
425 IRQ_PEND_EXT_SERVICE,
426 IRQ_PEND_EXT_HOST,
427 IRQ_PEND_PFAULT_INIT,
428 IRQ_PEND_PFAULT_DONE,
429 IRQ_PEND_VIRTIO,
430 IRQ_PEND_IO_ISC_0,
431 IRQ_PEND_IO_ISC_1,
432 IRQ_PEND_IO_ISC_2,
433 IRQ_PEND_IO_ISC_3,
434 IRQ_PEND_IO_ISC_4,
435 IRQ_PEND_IO_ISC_5,
436 IRQ_PEND_IO_ISC_6,
437 IRQ_PEND_IO_ISC_7,
438 IRQ_PEND_SIGP_STOP,
439 IRQ_PEND_RESTART, 431 IRQ_PEND_RESTART,
440 IRQ_PEND_SET_PREFIX, 432 IRQ_PEND_SIGP_STOP,
433 IRQ_PEND_IO_ISC_7,
434 IRQ_PEND_IO_ISC_6,
435 IRQ_PEND_IO_ISC_5,
436 IRQ_PEND_IO_ISC_4,
437 IRQ_PEND_IO_ISC_3,
438 IRQ_PEND_IO_ISC_2,
439 IRQ_PEND_IO_ISC_1,
440 IRQ_PEND_IO_ISC_0,
441 IRQ_PEND_VIRTIO,
442 IRQ_PEND_PFAULT_DONE,
443 IRQ_PEND_PFAULT_INIT,
444 IRQ_PEND_EXT_HOST,
445 IRQ_PEND_EXT_SERVICE,
446 IRQ_PEND_EXT_TIMING,
447 IRQ_PEND_EXT_CPU_TIMER,
448 IRQ_PEND_EXT_CLOCK_COMP,
449 IRQ_PEND_EXT_EXTERNAL,
450 IRQ_PEND_EXT_EMERGENCY,
451 IRQ_PEND_EXT_MALFUNC,
452 IRQ_PEND_EXT_IRQ_KEY,
453 IRQ_PEND_MCHK_REP,
454 IRQ_PEND_PROG,
455 IRQ_PEND_SVC,
456 IRQ_PEND_MCHK_EX,
441 IRQ_PEND_COUNT 457 IRQ_PEND_COUNT
442}; 458};
443 459
@@ -703,14 +719,50 @@ struct kvm_s390_crypto_cb {
703 struct kvm_s390_apcb1 apcb1; /* 0x0080 */ 719 struct kvm_s390_apcb1 apcb1; /* 0x0080 */
704}; 720};
705 721
722struct kvm_s390_gisa {
723 union {
724 struct { /* common to all formats */
725 u32 next_alert;
726 u8 ipm;
727 u8 reserved01[2];
728 u8 iam;
729 };
730 struct { /* format 0 */
731 u32 next_alert;
732 u8 ipm;
733 u8 reserved01;
734 u8 : 6;
735 u8 g : 1;
736 u8 c : 1;
737 u8 iam;
738 u8 reserved02[4];
739 u32 airq_count;
740 } g0;
741 struct { /* format 1 */
742 u32 next_alert;
743 u8 ipm;
744 u8 simm;
745 u8 nimm;
746 u8 iam;
747 u8 aism[8];
748 u8 : 6;
749 u8 g : 1;
750 u8 c : 1;
751 u8 reserved03[11];
752 u32 airq_count;
753 } g1;
754 };
755};
756
706/* 757/*
707 * sie_page2 has to be allocated as DMA because fac_list and crycb need 758 * sie_page2 has to be allocated as DMA because fac_list, crycb and
708 * 31bit addresses in the sie control block. 759 * gisa need 31bit addresses in the sie control block.
709 */ 760 */
710struct sie_page2 { 761struct sie_page2 {
711 __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */ 762 __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */
712 struct kvm_s390_crypto_cb crycb; /* 0x0800 */ 763 struct kvm_s390_crypto_cb crycb; /* 0x0800 */
713 u8 reserved900[0x1000 - 0x900]; /* 0x0900 */ 764 struct kvm_s390_gisa gisa; /* 0x0900 */
765 u8 reserved920[0x1000 - 0x920]; /* 0x0920 */
714}; 766};
715 767
716struct kvm_s390_vsie { 768struct kvm_s390_vsie {
@@ -757,6 +809,7 @@ struct kvm_arch{
757 struct kvm_s390_migration_state *migration_state; 809 struct kvm_s390_migration_state *migration_state;
758 /* subset of available cpu features enabled by user space */ 810 /* subset of available cpu features enabled by user space */
759 DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); 811 DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
812 struct kvm_s390_gisa *gisa;
760}; 813};
761 814
762#define KVM_HVA_ERR_BAD (-1UL) 815#define KVM_HVA_ERR_BAD (-1UL)
diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h
index d3c1a8a2e3ad..3cae9168f63c 100644
--- a/arch/s390/include/asm/sclp.h
+++ b/arch/s390/include/asm/sclp.h
@@ -77,6 +77,7 @@ struct sclp_info {
77 unsigned char has_ibs : 1; 77 unsigned char has_ibs : 1;
78 unsigned char has_skey : 1; 78 unsigned char has_skey : 1;
79 unsigned char has_kss : 1; 79 unsigned char has_kss : 1;
80 unsigned char has_gisaf : 1;
80 unsigned int ibc; 81 unsigned int ibc;
81 unsigned int mtid; 82 unsigned int mtid;
82 unsigned int mtid_cp; 83 unsigned int mtid_cp;
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c
index 89aa114a2cba..45634b3d2e0a 100644
--- a/arch/s390/kvm/diag.c
+++ b/arch/s390/kvm/diag.c
@@ -257,6 +257,7 @@ int kvm_s390_handle_diag(struct kvm_vcpu *vcpu)
257 case 0x500: 257 case 0x500:
258 return __diag_virtio_hypercall(vcpu); 258 return __diag_virtio_hypercall(vcpu);
259 default: 259 default:
260 vcpu->stat.diagnose_other++;
260 return -EOPNOTSUPP; 261 return -EOPNOTSUPP;
261 } 262 }
262} 263}
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index f8eb2cfa763a..aabf46f5f883 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -36,7 +36,7 @@ static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
36{ 36{
37 int c, scn; 37 int c, scn;
38 38
39 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND)) 39 if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
40 return 0; 40 return 0;
41 41
42 BUG_ON(!kvm_s390_use_sca_entries()); 42 BUG_ON(!kvm_s390_use_sca_entries());
@@ -101,7 +101,7 @@ static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
101 /* another external call is pending */ 101 /* another external call is pending */
102 return -EBUSY; 102 return -EBUSY;
103 } 103 }
104 atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); 104 kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
105 return 0; 105 return 0;
106} 106}
107 107
@@ -111,7 +111,7 @@ static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
111 111
112 if (!kvm_s390_use_sca_entries()) 112 if (!kvm_s390_use_sca_entries())
113 return; 113 return;
114 atomic_andnot(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); 114 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
115 read_lock(&vcpu->kvm->arch.sca_lock); 115 read_lock(&vcpu->kvm->arch.sca_lock);
116 if (vcpu->kvm->arch.use_esca) { 116 if (vcpu->kvm->arch.use_esca) {
117 struct esca_block *sca = vcpu->kvm->arch.sca; 117 struct esca_block *sca = vcpu->kvm->arch.sca;
@@ -189,8 +189,8 @@ static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
189 189
190static inline int is_ioirq(unsigned long irq_type) 190static inline int is_ioirq(unsigned long irq_type)
191{ 191{
192 return ((irq_type >= IRQ_PEND_IO_ISC_0) && 192 return ((irq_type >= IRQ_PEND_IO_ISC_7) &&
193 (irq_type <= IRQ_PEND_IO_ISC_7)); 193 (irq_type <= IRQ_PEND_IO_ISC_0));
194} 194}
195 195
196static uint64_t isc_to_isc_bits(int isc) 196static uint64_t isc_to_isc_bits(int isc)
@@ -198,25 +198,59 @@ static uint64_t isc_to_isc_bits(int isc)
198 return (0x80 >> isc) << 24; 198 return (0x80 >> isc) << 24;
199} 199}
200 200
201static inline u32 isc_to_int_word(u8 isc)
202{
203 return ((u32)isc << 27) | 0x80000000;
204}
205
201static inline u8 int_word_to_isc(u32 int_word) 206static inline u8 int_word_to_isc(u32 int_word)
202{ 207{
203 return (int_word & 0x38000000) >> 27; 208 return (int_word & 0x38000000) >> 27;
204} 209}
205 210
211/*
212 * To use atomic bitmap functions, we have to provide a bitmap address
213 * that is u64 aligned. However, the ipm might be u32 aligned.
214 * Therefore, we logically start the bitmap at the very beginning of the
215 * struct and fixup the bit number.
216 */
217#define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
218
219static inline void kvm_s390_gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
220{
221 set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
222}
223
224static inline u8 kvm_s390_gisa_get_ipm(struct kvm_s390_gisa *gisa)
225{
226 return READ_ONCE(gisa->ipm);
227}
228
229static inline void kvm_s390_gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
230{
231 clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
232}
233
234static inline int kvm_s390_gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
235{
236 return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
237}
238
206static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) 239static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
207{ 240{
208 return vcpu->kvm->arch.float_int.pending_irqs | 241 return vcpu->kvm->arch.float_int.pending_irqs |
209 vcpu->arch.local_int.pending_irqs; 242 vcpu->arch.local_int.pending_irqs |
243 kvm_s390_gisa_get_ipm(vcpu->kvm->arch.gisa) << IRQ_PEND_IO_ISC_7;
210} 244}
211 245
212static inline int isc_to_irq_type(unsigned long isc) 246static inline int isc_to_irq_type(unsigned long isc)
213{ 247{
214 return IRQ_PEND_IO_ISC_0 + isc; 248 return IRQ_PEND_IO_ISC_0 - isc;
215} 249}
216 250
217static inline int irq_type_to_isc(unsigned long irq_type) 251static inline int irq_type_to_isc(unsigned long irq_type)
218{ 252{
219 return irq_type - IRQ_PEND_IO_ISC_0; 253 return IRQ_PEND_IO_ISC_0 - irq_type;
220} 254}
221 255
222static unsigned long disable_iscs(struct kvm_vcpu *vcpu, 256static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
@@ -277,20 +311,20 @@ static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
277 311
278static void __set_cpu_idle(struct kvm_vcpu *vcpu) 312static void __set_cpu_idle(struct kvm_vcpu *vcpu)
279{ 313{
280 atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 314 kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
281 set_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask); 315 set_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask);
282} 316}
283 317
284static void __unset_cpu_idle(struct kvm_vcpu *vcpu) 318static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
285{ 319{
286 atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 320 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
287 clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask); 321 clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask);
288} 322}
289 323
290static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) 324static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
291{ 325{
292 atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, 326 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
293 &vcpu->arch.sie_block->cpuflags); 327 CPUSTAT_STOP_INT);
294 vcpu->arch.sie_block->lctl = 0x0000; 328 vcpu->arch.sie_block->lctl = 0x0000;
295 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); 329 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
296 330
@@ -301,17 +335,12 @@ static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
301 } 335 }
302} 336}
303 337
304static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
305{
306 atomic_or(flag, &vcpu->arch.sie_block->cpuflags);
307}
308
309static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) 338static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
310{ 339{
311 if (!(pending_irqs(vcpu) & IRQ_PEND_IO_MASK)) 340 if (!(pending_irqs(vcpu) & IRQ_PEND_IO_MASK))
312 return; 341 return;
313 else if (psw_ioint_disabled(vcpu)) 342 else if (psw_ioint_disabled(vcpu))
314 __set_cpuflag(vcpu, CPUSTAT_IO_INT); 343 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
315 else 344 else
316 vcpu->arch.sie_block->lctl |= LCTL_CR6; 345 vcpu->arch.sie_block->lctl |= LCTL_CR6;
317} 346}
@@ -321,7 +350,7 @@ static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
321 if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK)) 350 if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK))
322 return; 351 return;
323 if (psw_extint_disabled(vcpu)) 352 if (psw_extint_disabled(vcpu))
324 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 353 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
325 else 354 else
326 vcpu->arch.sie_block->lctl |= LCTL_CR0; 355 vcpu->arch.sie_block->lctl |= LCTL_CR0;
327} 356}
@@ -339,7 +368,7 @@ static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
339static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu) 368static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
340{ 369{
341 if (kvm_s390_is_stop_irq_pending(vcpu)) 370 if (kvm_s390_is_stop_irq_pending(vcpu))
342 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 371 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
343} 372}
344 373
345/* Set interception request for non-deliverable interrupts */ 374/* Set interception request for non-deliverable interrupts */
@@ -896,18 +925,38 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
896 return rc ? -EFAULT : 0; 925 return rc ? -EFAULT : 0;
897} 926}
898 927
928static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
929{
930 int rc;
931
932 rc = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
933 rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
934 rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
935 rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
936 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
937 &vcpu->arch.sie_block->gpsw,
938 sizeof(psw_t));
939 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
940 &vcpu->arch.sie_block->gpsw,
941 sizeof(psw_t));
942 return rc ? -EFAULT : 0;
943}
944
899static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 945static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
900 unsigned long irq_type) 946 unsigned long irq_type)
901{ 947{
902 struct list_head *isc_list; 948 struct list_head *isc_list;
903 struct kvm_s390_float_interrupt *fi; 949 struct kvm_s390_float_interrupt *fi;
904 struct kvm_s390_interrupt_info *inti = NULL; 950 struct kvm_s390_interrupt_info *inti = NULL;
951 struct kvm_s390_io_info io;
952 u32 isc;
905 int rc = 0; 953 int rc = 0;
906 954
907 fi = &vcpu->kvm->arch.float_int; 955 fi = &vcpu->kvm->arch.float_int;
908 956
909 spin_lock(&fi->lock); 957 spin_lock(&fi->lock);
910 isc_list = &fi->lists[irq_type_to_isc(irq_type)]; 958 isc = irq_type_to_isc(irq_type);
959 isc_list = &fi->lists[isc];
911 inti = list_first_entry_or_null(isc_list, 960 inti = list_first_entry_or_null(isc_list,
912 struct kvm_s390_interrupt_info, 961 struct kvm_s390_interrupt_info,
913 list); 962 list);
@@ -935,24 +984,31 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
935 spin_unlock(&fi->lock); 984 spin_unlock(&fi->lock);
936 985
937 if (inti) { 986 if (inti) {
938 rc = put_guest_lc(vcpu, inti->io.subchannel_id, 987 rc = __do_deliver_io(vcpu, &(inti->io));
939 (u16 *)__LC_SUBCHANNEL_ID);
940 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
941 (u16 *)__LC_SUBCHANNEL_NR);
942 rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
943 (u32 *)__LC_IO_INT_PARM);
944 rc |= put_guest_lc(vcpu, inti->io.io_int_word,
945 (u32 *)__LC_IO_INT_WORD);
946 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
947 &vcpu->arch.sie_block->gpsw,
948 sizeof(psw_t));
949 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
950 &vcpu->arch.sie_block->gpsw,
951 sizeof(psw_t));
952 kfree(inti); 988 kfree(inti);
989 goto out;
953 } 990 }
954 991
955 return rc ? -EFAULT : 0; 992 if (vcpu->kvm->arch.gisa &&
993 kvm_s390_gisa_tac_ipm_gisc(vcpu->kvm->arch.gisa, isc)) {
994 /*
995 * in case an adapter interrupt was not delivered
996 * in SIE context KVM will handle the delivery
997 */
998 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
999 memset(&io, 0, sizeof(io));
1000 io.io_int_word = isc_to_int_word(isc);
1001 vcpu->stat.deliver_io_int++;
1002 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1003 KVM_S390_INT_IO(1, 0, 0, 0),
1004 ((__u32)io.subchannel_id << 16) |
1005 io.subchannel_nr,
1006 ((__u64)io.io_int_parm << 32) |
1007 io.io_int_word);
1008 rc = __do_deliver_io(vcpu, &io);
1009 }
1010out:
1011 return rc;
956} 1012}
957 1013
958typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu); 1014typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu);
@@ -1154,8 +1210,8 @@ int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1154 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1210 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1155 1211
1156 while ((irqs = deliverable_irqs(vcpu)) && !rc) { 1212 while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1157 /* bits are in the order of interrupt priority */ 1213 /* bits are in the reverse order of interrupt priority */
1158 irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT); 1214 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1159 if (is_ioirq(irq_type)) { 1215 if (is_ioirq(irq_type)) {
1160 rc = __deliver_io(vcpu, irq_type); 1216 rc = __deliver_io(vcpu, irq_type);
1161 } else { 1217 } else {
@@ -1227,7 +1283,7 @@ static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1227 1283
1228 li->irq.ext = irq->u.ext; 1284 li->irq.ext = irq->u.ext;
1229 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 1285 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1230 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 1286 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1231 return 0; 1287 return 0;
1232} 1288}
1233 1289
@@ -1252,7 +1308,7 @@ static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1252 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) 1308 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1253 return -EBUSY; 1309 return -EBUSY;
1254 *extcall = irq->u.extcall; 1310 *extcall = irq->u.extcall;
1255 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 1311 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1256 return 0; 1312 return 0;
1257} 1313}
1258 1314
@@ -1296,7 +1352,7 @@ static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1296 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) 1352 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1297 return -EBUSY; 1353 return -EBUSY;
1298 stop->flags = irq->u.stop.flags; 1354 stop->flags = irq->u.stop.flags;
1299 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 1355 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1300 return 0; 1356 return 0;
1301} 1357}
1302 1358
@@ -1328,7 +1384,7 @@ static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1328 1384
1329 set_bit(irq->u.emerg.code, li->sigp_emerg_pending); 1385 set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1330 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 1386 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1331 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 1387 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1332 return 0; 1388 return 0;
1333} 1389}
1334 1390
@@ -1372,7 +1428,7 @@ static int __inject_ckc(struct kvm_vcpu *vcpu)
1372 0, 0); 1428 0, 0);
1373 1429
1374 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1430 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1375 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 1431 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1376 return 0; 1432 return 0;
1377} 1433}
1378 1434
@@ -1385,7 +1441,7 @@ static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1385 0, 0); 1441 0, 0);
1386 1442
1387 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1443 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1388 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 1444 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1389 return 0; 1445 return 0;
1390} 1446}
1391 1447
@@ -1415,20 +1471,86 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1415 return NULL; 1471 return NULL;
1416} 1472}
1417 1473
1474static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1475 u64 isc_mask, u32 schid)
1476{
1477 struct kvm_s390_interrupt_info *inti = NULL;
1478 int isc;
1479
1480 for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1481 if (isc_mask & isc_to_isc_bits(isc))
1482 inti = get_io_int(kvm, isc, schid);
1483 }
1484 return inti;
1485}
1486
1487static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1488{
1489 unsigned long active_mask;
1490 int isc;
1491
1492 if (schid)
1493 goto out;
1494 if (!kvm->arch.gisa)
1495 goto out;
1496
1497 active_mask = (isc_mask & kvm_s390_gisa_get_ipm(kvm->arch.gisa) << 24) << 32;
1498 while (active_mask) {
1499 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
1500 if (kvm_s390_gisa_tac_ipm_gisc(kvm->arch.gisa, isc))
1501 return isc;
1502 clear_bit_inv(isc, &active_mask);
1503 }
1504out:
1505 return -EINVAL;
1506}
1507
1418/* 1508/*
1419 * Dequeue and return an I/O interrupt matching any of the interruption 1509 * Dequeue and return an I/O interrupt matching any of the interruption
1420 * subclasses as designated by the isc mask in cr6 and the schid (if != 0). 1510 * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1511 * Take into account the interrupts pending in the interrupt list and in GISA.
1512 *
1513 * Note that for a guest that does not enable I/O interrupts
1514 * but relies on TPI, a flood of classic interrupts may starve
1515 * out adapter interrupts on the same isc. Linux does not do
1516 * that, and it is possible to work around the issue by configuring
1517 * different iscs for classic and adapter interrupts in the guest,
1518 * but we may want to revisit this in the future.
1421 */ 1519 */
1422struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1520struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1423 u64 isc_mask, u32 schid) 1521 u64 isc_mask, u32 schid)
1424{ 1522{
1425 struct kvm_s390_interrupt_info *inti = NULL; 1523 struct kvm_s390_interrupt_info *inti, *tmp_inti;
1426 int isc; 1524 int isc;
1427 1525
1428 for (isc = 0; isc <= MAX_ISC && !inti; isc++) { 1526 inti = get_top_io_int(kvm, isc_mask, schid);
1429 if (isc_mask & isc_to_isc_bits(isc)) 1527
1430 inti = get_io_int(kvm, isc, schid); 1528 isc = get_top_gisa_isc(kvm, isc_mask, schid);
1529 if (isc < 0)
1530 /* no AI in GISA */
1531 goto out;
1532
1533 if (!inti)
1534 /* AI in GISA but no classical IO int */
1535 goto gisa_out;
1536
1537 /* both types of interrupts present */
1538 if (int_word_to_isc(inti->io.io_int_word) <= isc) {
1539 /* classical IO int with higher priority */
1540 kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
1541 goto out;
1431 } 1542 }
1543gisa_out:
1544 tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL);
1545 if (tmp_inti) {
1546 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1547 tmp_inti->io.io_int_word = isc_to_int_word(isc);
1548 if (inti)
1549 kvm_s390_reinject_io_int(kvm, inti);
1550 inti = tmp_inti;
1551 } else
1552 kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
1553out:
1432 return inti; 1554 return inti;
1433} 1555}
1434 1556
@@ -1516,6 +1638,15 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1516 struct list_head *list; 1638 struct list_head *list;
1517 int isc; 1639 int isc;
1518 1640
1641 isc = int_word_to_isc(inti->io.io_int_word);
1642
1643 if (kvm->arch.gisa && inti->type & KVM_S390_INT_IO_AI_MASK) {
1644 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1645 kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
1646 kfree(inti);
1647 return 0;
1648 }
1649
1519 fi = &kvm->arch.float_int; 1650 fi = &kvm->arch.float_int;
1520 spin_lock(&fi->lock); 1651 spin_lock(&fi->lock);
1521 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { 1652 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
@@ -1531,7 +1662,6 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1531 inti->io.subchannel_id >> 8, 1662 inti->io.subchannel_id >> 8,
1532 inti->io.subchannel_id >> 1 & 0x3, 1663 inti->io.subchannel_id >> 1 & 0x3,
1533 inti->io.subchannel_nr); 1664 inti->io.subchannel_nr);
1534 isc = int_word_to_isc(inti->io.io_int_word);
1535 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1665 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1536 list_add_tail(&inti->list, list); 1666 list_add_tail(&inti->list, list);
1537 set_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1667 set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
@@ -1568,13 +1698,13 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type)
1568 /* make the VCPU drop out of the SIE, or wake it up if sleeping */ 1698 /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1569 switch (type) { 1699 switch (type) {
1570 case KVM_S390_MCHK: 1700 case KVM_S390_MCHK:
1571 __set_cpuflag(dst_vcpu, CPUSTAT_STOP_INT); 1701 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1572 break; 1702 break;
1573 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1703 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1574 __set_cpuflag(dst_vcpu, CPUSTAT_IO_INT); 1704 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1575 break; 1705 break;
1576 default: 1706 default:
1577 __set_cpuflag(dst_vcpu, CPUSTAT_EXT_INT); 1707 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1578 break; 1708 break;
1579 } 1709 }
1580 kvm_s390_vcpu_wakeup(dst_vcpu); 1710 kvm_s390_vcpu_wakeup(dst_vcpu);
@@ -1815,6 +1945,7 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm)
1815 for (i = 0; i < FIRQ_MAX_COUNT; i++) 1945 for (i = 0; i < FIRQ_MAX_COUNT; i++)
1816 fi->counters[i] = 0; 1946 fi->counters[i] = 0;
1817 spin_unlock(&fi->lock); 1947 spin_unlock(&fi->lock);
1948 kvm_s390_gisa_clear(kvm);
1818}; 1949};
1819 1950
1820static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) 1951static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
@@ -1842,6 +1973,22 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
1842 1973
1843 max_irqs = len / sizeof(struct kvm_s390_irq); 1974 max_irqs = len / sizeof(struct kvm_s390_irq);
1844 1975
1976 if (kvm->arch.gisa &&
1977 kvm_s390_gisa_get_ipm(kvm->arch.gisa)) {
1978 for (i = 0; i <= MAX_ISC; i++) {
1979 if (n == max_irqs) {
1980 /* signal userspace to try again */
1981 ret = -ENOMEM;
1982 goto out_nolock;
1983 }
1984 if (kvm_s390_gisa_tac_ipm_gisc(kvm->arch.gisa, i)) {
1985 irq = (struct kvm_s390_irq *) &buf[n];
1986 irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
1987 irq->u.io.io_int_word = isc_to_int_word(i);
1988 n++;
1989 }
1990 }
1991 }
1845 fi = &kvm->arch.float_int; 1992 fi = &kvm->arch.float_int;
1846 spin_lock(&fi->lock); 1993 spin_lock(&fi->lock);
1847 for (i = 0; i < FIRQ_LIST_COUNT; i++) { 1994 for (i = 0; i < FIRQ_LIST_COUNT; i++) {
@@ -1880,6 +2027,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
1880 2027
1881out: 2028out:
1882 spin_unlock(&fi->lock); 2029 spin_unlock(&fi->lock);
2030out_nolock:
1883 if (!ret && n > 0) { 2031 if (!ret && n > 0) {
1884 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) 2032 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
1885 ret = -EFAULT; 2033 ret = -EFAULT;
@@ -2240,7 +2388,7 @@ static int kvm_s390_inject_airq(struct kvm *kvm,
2240 struct kvm_s390_interrupt s390int = { 2388 struct kvm_s390_interrupt s390int = {
2241 .type = KVM_S390_INT_IO(1, 0, 0, 0), 2389 .type = KVM_S390_INT_IO(1, 0, 0, 0),
2242 .parm = 0, 2390 .parm = 0,
2243 .parm64 = (adapter->isc << 27) | 0x80000000, 2391 .parm64 = isc_to_int_word(adapter->isc),
2244 }; 2392 };
2245 int ret = 0; 2393 int ret = 0;
2246 2394
@@ -2682,3 +2830,28 @@ int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
2682 2830
2683 return n; 2831 return n;
2684} 2832}
2833
2834void kvm_s390_gisa_clear(struct kvm *kvm)
2835{
2836 if (kvm->arch.gisa) {
2837 memset(kvm->arch.gisa, 0, sizeof(struct kvm_s390_gisa));
2838 kvm->arch.gisa->next_alert = (u32)(u64)kvm->arch.gisa;
2839 VM_EVENT(kvm, 3, "gisa 0x%pK cleared", kvm->arch.gisa);
2840 }
2841}
2842
2843void kvm_s390_gisa_init(struct kvm *kvm)
2844{
2845 if (css_general_characteristics.aiv) {
2846 kvm->arch.gisa = &kvm->arch.sie_page2->gisa;
2847 VM_EVENT(kvm, 3, "gisa 0x%pK initialized", kvm->arch.gisa);
2848 kvm_s390_gisa_clear(kvm);
2849 }
2850}
2851
2852void kvm_s390_gisa_destroy(struct kvm *kvm)
2853{
2854 if (!kvm->arch.gisa)
2855 return;
2856 kvm->arch.gisa = NULL;
2857}
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index de16c224319c..58bee42d7a9d 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -2,7 +2,7 @@
2/* 2/*
3 * hosting IBM Z kernel virtual machines (s390x) 3 * hosting IBM Z kernel virtual machines (s390x)
4 * 4 *
5 * Copyright IBM Corp. 2008, 2017 5 * Copyright IBM Corp. 2008, 2018
6 * 6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com> 7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com>
@@ -87,19 +87,31 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
87 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 87 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
88 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) }, 88 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
89 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 89 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
90 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
91 { "instruction_gs", VCPU_STAT(instruction_gs) },
92 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
93 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
94 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
90 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 95 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
96 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
91 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 97 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
98 { "instruction_sck", VCPU_STAT(instruction_sck) },
99 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
92 { "instruction_spx", VCPU_STAT(instruction_spx) }, 100 { "instruction_spx", VCPU_STAT(instruction_spx) },
93 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 101 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
94 { "instruction_stap", VCPU_STAT(instruction_stap) }, 102 { "instruction_stap", VCPU_STAT(instruction_stap) },
95 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) }, 103 { "instruction_iske", VCPU_STAT(instruction_iske) },
104 { "instruction_ri", VCPU_STAT(instruction_ri) },
105 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
106 { "instruction_sske", VCPU_STAT(instruction_sske) },
96 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) }, 107 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
97 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
98 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
99 { "instruction_essa", VCPU_STAT(instruction_essa) }, 108 { "instruction_essa", VCPU_STAT(instruction_essa) },
100 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 109 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
101 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 110 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
111 { "instruction_tb", VCPU_STAT(instruction_tb) },
112 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
102 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 113 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
114 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
103 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) }, 115 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
104 { "instruction_sie", VCPU_STAT(instruction_sie) }, 116 { "instruction_sie", VCPU_STAT(instruction_sie) },
105 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 117 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
@@ -118,12 +130,13 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
118 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) }, 130 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
119 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) }, 131 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
120 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) }, 132 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
121 { "diagnose_10", VCPU_STAT(diagnose_10) }, 133 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
122 { "diagnose_44", VCPU_STAT(diagnose_44) }, 134 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
123 { "diagnose_9c", VCPU_STAT(diagnose_9c) }, 135 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
124 { "diagnose_258", VCPU_STAT(diagnose_258) }, 136 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
125 { "diagnose_308", VCPU_STAT(diagnose_308) }, 137 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
126 { "diagnose_500", VCPU_STAT(diagnose_500) }, 138 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
139 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
127 { NULL } 140 { NULL }
128}; 141};
129 142
@@ -1915,6 +1928,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1915 if (!kvm->arch.dbf) 1928 if (!kvm->arch.dbf)
1916 goto out_err; 1929 goto out_err;
1917 1930
1931 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
1918 kvm->arch.sie_page2 = 1932 kvm->arch.sie_page2 =
1919 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1933 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1920 if (!kvm->arch.sie_page2) 1934 if (!kvm->arch.sie_page2)
@@ -1985,6 +1999,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1985 1999
1986 spin_lock_init(&kvm->arch.start_stop_lock); 2000 spin_lock_init(&kvm->arch.start_stop_lock);
1987 kvm_s390_vsie_init(kvm); 2001 kvm_s390_vsie_init(kvm);
2002 kvm_s390_gisa_init(kvm);
1988 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid); 2003 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1989 2004
1990 return 0; 2005 return 0;
@@ -2047,6 +2062,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
2047 kvm_free_vcpus(kvm); 2062 kvm_free_vcpus(kvm);
2048 sca_dispose(kvm); 2063 sca_dispose(kvm);
2049 debug_unregister(kvm->arch.dbf); 2064 debug_unregister(kvm->arch.dbf);
2065 kvm_s390_gisa_destroy(kvm);
2050 free_page((unsigned long)kvm->arch.sie_page2); 2066 free_page((unsigned long)kvm->arch.sie_page2);
2051 if (!kvm_is_ucontrol(kvm)) 2067 if (!kvm_is_ucontrol(kvm))
2052 gmap_remove(kvm->arch.gmap); 2068 gmap_remove(kvm->arch.gmap);
@@ -2316,7 +2332,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2316{ 2332{
2317 2333
2318 gmap_enable(vcpu->arch.enabled_gmap); 2334 gmap_enable(vcpu->arch.enabled_gmap);
2319 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 2335 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
2320 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) 2336 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2321 __start_cpu_timer_accounting(vcpu); 2337 __start_cpu_timer_accounting(vcpu);
2322 vcpu->cpu = cpu; 2338 vcpu->cpu = cpu;
@@ -2327,7 +2343,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2327 vcpu->cpu = -1; 2343 vcpu->cpu = -1;
2328 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) 2344 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2329 __stop_cpu_timer_accounting(vcpu); 2345 __stop_cpu_timer_accounting(vcpu);
2330 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 2346 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
2331 vcpu->arch.enabled_gmap = gmap_get_enabled(); 2347 vcpu->arch.enabled_gmap = gmap_get_enabled();
2332 gmap_disable(vcpu->arch.enabled_gmap); 2348 gmap_disable(vcpu->arch.enabled_gmap);
2333 2349
@@ -2423,9 +2439,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2423 CPUSTAT_STOPPED); 2439 CPUSTAT_STOPPED);
2424 2440
2425 if (test_kvm_facility(vcpu->kvm, 78)) 2441 if (test_kvm_facility(vcpu->kvm, 78))
2426 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags); 2442 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
2427 else if (test_kvm_facility(vcpu->kvm, 8)) 2443 else if (test_kvm_facility(vcpu->kvm, 8))
2428 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags); 2444 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
2429 2445
2430 kvm_s390_vcpu_setup_model(vcpu); 2446 kvm_s390_vcpu_setup_model(vcpu);
2431 2447
@@ -2457,12 +2473,17 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2457 if (test_kvm_facility(vcpu->kvm, 139)) 2473 if (test_kvm_facility(vcpu->kvm, 139))
2458 vcpu->arch.sie_block->ecd |= ECD_MEF; 2474 vcpu->arch.sie_block->ecd |= ECD_MEF;
2459 2475
2476 if (vcpu->arch.sie_block->gd) {
2477 vcpu->arch.sie_block->eca |= ECA_AIV;
2478 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2479 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2480 }
2460 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx) 2481 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2461 | SDNXC; 2482 | SDNXC;
2462 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb; 2483 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2463 2484
2464 if (sclp.has_kss) 2485 if (sclp.has_kss)
2465 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags); 2486 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
2466 else 2487 else
2467 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE; 2488 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2468 2489
@@ -2509,6 +2530,9 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2509 2530
2510 vcpu->arch.sie_block->icpua = id; 2531 vcpu->arch.sie_block->icpua = id;
2511 spin_lock_init(&vcpu->arch.local_int.lock); 2532 spin_lock_init(&vcpu->arch.local_int.lock);
2533 vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa;
2534 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
2535 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
2512 seqcount_init(&vcpu->arch.cputm_seqcount); 2536 seqcount_init(&vcpu->arch.cputm_seqcount);
2513 2537
2514 rc = kvm_vcpu_init(vcpu, kvm, id); 2538 rc = kvm_vcpu_init(vcpu, kvm, id);
@@ -2565,7 +2589,7 @@ static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2565 * return immediately. */ 2589 * return immediately. */
2566void exit_sie(struct kvm_vcpu *vcpu) 2590void exit_sie(struct kvm_vcpu *vcpu)
2567{ 2591{
2568 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); 2592 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
2569 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 2593 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2570 cpu_relax(); 2594 cpu_relax();
2571} 2595}
@@ -2840,19 +2864,19 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2840 if (dbg->control & KVM_GUESTDBG_ENABLE) { 2864 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2841 vcpu->guest_debug = dbg->control; 2865 vcpu->guest_debug = dbg->control;
2842 /* enforce guest PER */ 2866 /* enforce guest PER */
2843 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2867 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
2844 2868
2845 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) 2869 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2846 rc = kvm_s390_import_bp_data(vcpu, dbg); 2870 rc = kvm_s390_import_bp_data(vcpu, dbg);
2847 } else { 2871 } else {
2848 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2872 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
2849 vcpu->arch.guestdbg.last_bp = 0; 2873 vcpu->arch.guestdbg.last_bp = 0;
2850 } 2874 }
2851 2875
2852 if (rc) { 2876 if (rc) {
2853 vcpu->guest_debug = 0; 2877 vcpu->guest_debug = 0;
2854 kvm_s390_clear_bp_data(vcpu); 2878 kvm_s390_clear_bp_data(vcpu);
2855 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2879 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
2856 } 2880 }
2857 2881
2858out: 2882out:
@@ -2905,7 +2929,7 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2905 2929
2906static bool ibs_enabled(struct kvm_vcpu *vcpu) 2930static bool ibs_enabled(struct kvm_vcpu *vcpu)
2907{ 2931{
2908 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS; 2932 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
2909} 2933}
2910 2934
2911static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 2935static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
@@ -2941,8 +2965,7 @@ retry:
2941 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { 2965 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2942 if (!ibs_enabled(vcpu)) { 2966 if (!ibs_enabled(vcpu)) {
2943 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); 2967 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2944 atomic_or(CPUSTAT_IBS, 2968 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
2945 &vcpu->arch.sie_block->cpuflags);
2946 } 2969 }
2947 goto retry; 2970 goto retry;
2948 } 2971 }
@@ -2950,8 +2973,7 @@ retry:
2950 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { 2973 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2951 if (ibs_enabled(vcpu)) { 2974 if (ibs_enabled(vcpu)) {
2952 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); 2975 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2953 atomic_andnot(CPUSTAT_IBS, 2976 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
2954 &vcpu->arch.sie_block->cpuflags);
2955 } 2977 }
2956 goto retry; 2978 goto retry;
2957 } 2979 }
@@ -3601,7 +3623,7 @@ void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3601 __disable_ibs_on_all_vcpus(vcpu->kvm); 3623 __disable_ibs_on_all_vcpus(vcpu->kvm);
3602 } 3624 }
3603 3625
3604 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 3626 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
3605 /* 3627 /*
3606 * Another VCPU might have used IBS while we were offline. 3628 * Another VCPU might have used IBS while we were offline.
3607 * Let's play safe and flush the VCPU at startup. 3629 * Let's play safe and flush the VCPU at startup.
@@ -3627,7 +3649,7 @@ void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3627 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ 3649 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3628 kvm_s390_clear_stop_irq(vcpu); 3650 kvm_s390_clear_stop_irq(vcpu);
3629 3651
3630 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 3652 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
3631 __disable_ibs_on_vcpu(vcpu); 3653 __disable_ibs_on_vcpu(vcpu);
3632 3654
3633 for (i = 0; i < online_vcpus; i++) { 3655 for (i = 0; i < online_vcpus; i++) {
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index 8877116f0159..bd31b37b0e6f 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -47,9 +47,24 @@ do { \
47 d_args); \ 47 d_args); \
48} while (0) 48} while (0)
49 49
50static inline void kvm_s390_set_cpuflags(struct kvm_vcpu *vcpu, u32 flags)
51{
52 atomic_or(flags, &vcpu->arch.sie_block->cpuflags);
53}
54
55static inline void kvm_s390_clear_cpuflags(struct kvm_vcpu *vcpu, u32 flags)
56{
57 atomic_andnot(flags, &vcpu->arch.sie_block->cpuflags);
58}
59
60static inline bool kvm_s390_test_cpuflags(struct kvm_vcpu *vcpu, u32 flags)
61{
62 return (atomic_read(&vcpu->arch.sie_block->cpuflags) & flags) == flags;
63}
64
50static inline int is_vcpu_stopped(struct kvm_vcpu *vcpu) 65static inline int is_vcpu_stopped(struct kvm_vcpu *vcpu)
51{ 66{
52 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED; 67 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_STOPPED);
53} 68}
54 69
55static inline int is_vcpu_idle(struct kvm_vcpu *vcpu) 70static inline int is_vcpu_idle(struct kvm_vcpu *vcpu)
@@ -367,6 +382,9 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu,
367 void __user *buf, int len); 382 void __user *buf, int len);
368int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, 383int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu,
369 __u8 __user *buf, int len); 384 __u8 __user *buf, int len);
385void kvm_s390_gisa_init(struct kvm *kvm);
386void kvm_s390_gisa_clear(struct kvm *kvm);
387void kvm_s390_gisa_destroy(struct kvm *kvm);
370 388
371/* implemented in guestdbg.c */ 389/* implemented in guestdbg.c */
372void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu); 390void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu);
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index 572496c688cc..125a7ff98e2a 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -2,7 +2,7 @@
2/* 2/*
3 * handling privileged instructions 3 * handling privileged instructions
4 * 4 *
5 * Copyright IBM Corp. 2008, 2013 5 * Copyright IBM Corp. 2008, 2018
6 * 6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com> 7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com>
@@ -34,6 +34,8 @@
34 34
35static int handle_ri(struct kvm_vcpu *vcpu) 35static int handle_ri(struct kvm_vcpu *vcpu)
36{ 36{
37 vcpu->stat.instruction_ri++;
38
37 if (test_kvm_facility(vcpu->kvm, 64)) { 39 if (test_kvm_facility(vcpu->kvm, 64)) {
38 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)"); 40 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)");
39 vcpu->arch.sie_block->ecb3 |= ECB3_RI; 41 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
@@ -53,6 +55,8 @@ int kvm_s390_handle_aa(struct kvm_vcpu *vcpu)
53 55
54static int handle_gs(struct kvm_vcpu *vcpu) 56static int handle_gs(struct kvm_vcpu *vcpu)
55{ 57{
58 vcpu->stat.instruction_gs++;
59
56 if (test_kvm_facility(vcpu->kvm, 133)) { 60 if (test_kvm_facility(vcpu->kvm, 133)) {
57 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); 61 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)");
58 preempt_disable(); 62 preempt_disable();
@@ -85,6 +89,8 @@ static int handle_set_clock(struct kvm_vcpu *vcpu)
85 u8 ar; 89 u8 ar;
86 u64 op2, val; 90 u64 op2, val;
87 91
92 vcpu->stat.instruction_sck++;
93
88 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 94 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
89 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 95 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
90 96
@@ -203,14 +209,14 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
203 209
204 trace_kvm_s390_skey_related_inst(vcpu); 210 trace_kvm_s390_skey_related_inst(vcpu);
205 if (!(sie_block->ictl & (ICTL_ISKE | ICTL_SSKE | ICTL_RRBE)) && 211 if (!(sie_block->ictl & (ICTL_ISKE | ICTL_SSKE | ICTL_RRBE)) &&
206 !(atomic_read(&sie_block->cpuflags) & CPUSTAT_KSS)) 212 !kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
207 return rc; 213 return rc;
208 214
209 rc = s390_enable_skey(); 215 rc = s390_enable_skey();
210 VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc); 216 VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc);
211 if (!rc) { 217 if (!rc) {
212 if (atomic_read(&sie_block->cpuflags) & CPUSTAT_KSS) 218 if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
213 atomic_andnot(CPUSTAT_KSS, &sie_block->cpuflags); 219 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
214 else 220 else
215 sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | 221 sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE |
216 ICTL_RRBE); 222 ICTL_RRBE);
@@ -222,7 +228,6 @@ static int try_handle_skey(struct kvm_vcpu *vcpu)
222{ 228{
223 int rc; 229 int rc;
224 230
225 vcpu->stat.instruction_storage_key++;
226 rc = kvm_s390_skey_check_enable(vcpu); 231 rc = kvm_s390_skey_check_enable(vcpu);
227 if (rc) 232 if (rc)
228 return rc; 233 return rc;
@@ -242,6 +247,8 @@ static int handle_iske(struct kvm_vcpu *vcpu)
242 int reg1, reg2; 247 int reg1, reg2;
243 int rc; 248 int rc;
244 249
250 vcpu->stat.instruction_iske++;
251
245 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 252 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
246 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 253 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
247 254
@@ -274,6 +281,8 @@ static int handle_rrbe(struct kvm_vcpu *vcpu)
274 int reg1, reg2; 281 int reg1, reg2;
275 int rc; 282 int rc;
276 283
284 vcpu->stat.instruction_rrbe++;
285
277 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 286 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
278 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 287 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
279 288
@@ -312,6 +321,8 @@ static int handle_sske(struct kvm_vcpu *vcpu)
312 int reg1, reg2; 321 int reg1, reg2;
313 int rc; 322 int rc;
314 323
324 vcpu->stat.instruction_sske++;
325
315 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 326 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
316 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 327 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
317 328
@@ -392,6 +403,8 @@ static int handle_test_block(struct kvm_vcpu *vcpu)
392 gpa_t addr; 403 gpa_t addr;
393 int reg2; 404 int reg2;
394 405
406 vcpu->stat.instruction_tb++;
407
395 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 408 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
396 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 409 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
397 410
@@ -424,6 +437,8 @@ static int handle_tpi(struct kvm_vcpu *vcpu)
424 u64 addr; 437 u64 addr;
425 u8 ar; 438 u8 ar;
426 439
440 vcpu->stat.instruction_tpi++;
441
427 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 442 addr = kvm_s390_get_base_disp_s(vcpu, &ar);
428 if (addr & 3) 443 if (addr & 3)
429 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 444 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -484,6 +499,8 @@ static int handle_tsch(struct kvm_vcpu *vcpu)
484 struct kvm_s390_interrupt_info *inti = NULL; 499 struct kvm_s390_interrupt_info *inti = NULL;
485 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 500 const u64 isc_mask = 0xffUL << 24; /* all iscs set */
486 501
502 vcpu->stat.instruction_tsch++;
503
487 /* a valid schid has at least one bit set */ 504 /* a valid schid has at least one bit set */
488 if (vcpu->run->s.regs.gprs[1]) 505 if (vcpu->run->s.regs.gprs[1])
489 inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask, 506 inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask,
@@ -527,6 +544,7 @@ static int handle_io_inst(struct kvm_vcpu *vcpu)
527 if (vcpu->arch.sie_block->ipa == 0xb235) 544 if (vcpu->arch.sie_block->ipa == 0xb235)
528 return handle_tsch(vcpu); 545 return handle_tsch(vcpu);
529 /* Handle in userspace. */ 546 /* Handle in userspace. */
547 vcpu->stat.instruction_io_other++;
530 return -EOPNOTSUPP; 548 return -EOPNOTSUPP;
531 } else { 549 } else {
532 /* 550 /*
@@ -592,6 +610,8 @@ int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu)
592 int rc; 610 int rc;
593 u8 ar; 611 u8 ar;
594 612
613 vcpu->stat.instruction_lpsw++;
614
595 if (gpsw->mask & PSW_MASK_PSTATE) 615 if (gpsw->mask & PSW_MASK_PSTATE)
596 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 616 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
597 617
@@ -619,6 +639,8 @@ static int handle_lpswe(struct kvm_vcpu *vcpu)
619 int rc; 639 int rc;
620 u8 ar; 640 u8 ar;
621 641
642 vcpu->stat.instruction_lpswe++;
643
622 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 644 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
623 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 645 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
624 646
@@ -828,6 +850,8 @@ static int handle_epsw(struct kvm_vcpu *vcpu)
828{ 850{
829 int reg1, reg2; 851 int reg1, reg2;
830 852
853 vcpu->stat.instruction_epsw++;
854
831 kvm_s390_get_regs_rre(vcpu, &reg1, &reg2); 855 kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
832 856
833 /* This basically extracts the mask half of the psw. */ 857 /* This basically extracts the mask half of the psw. */
@@ -1332,6 +1356,8 @@ static int handle_sckpf(struct kvm_vcpu *vcpu)
1332{ 1356{
1333 u32 value; 1357 u32 value;
1334 1358
1359 vcpu->stat.instruction_sckpf++;
1360
1335 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1361 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1336 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1362 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1337 1363
@@ -1347,6 +1373,8 @@ static int handle_sckpf(struct kvm_vcpu *vcpu)
1347 1373
1348static int handle_ptff(struct kvm_vcpu *vcpu) 1374static int handle_ptff(struct kvm_vcpu *vcpu)
1349{ 1375{
1376 vcpu->stat.instruction_ptff++;
1377
1350 /* we don't emulate any control instructions yet */ 1378 /* we don't emulate any control instructions yet */
1351 kvm_s390_set_psw_cc(vcpu, 3); 1379 kvm_s390_set_psw_cc(vcpu, 3);
1352 return 0; 1380 return 0;
diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c
index 5cafd1e2651b..683036c1c92a 100644
--- a/arch/s390/kvm/sigp.c
+++ b/arch/s390/kvm/sigp.c
@@ -20,19 +20,18 @@
20static int __sigp_sense(struct kvm_vcpu *vcpu, struct kvm_vcpu *dst_vcpu, 20static int __sigp_sense(struct kvm_vcpu *vcpu, struct kvm_vcpu *dst_vcpu,
21 u64 *reg) 21 u64 *reg)
22{ 22{
23 int cpuflags; 23 const bool stopped = kvm_s390_test_cpuflags(dst_vcpu, CPUSTAT_STOPPED);
24 int rc; 24 int rc;
25 int ext_call_pending; 25 int ext_call_pending;
26 26
27 cpuflags = atomic_read(&dst_vcpu->arch.sie_block->cpuflags);
28 ext_call_pending = kvm_s390_ext_call_pending(dst_vcpu); 27 ext_call_pending = kvm_s390_ext_call_pending(dst_vcpu);
29 if (!(cpuflags & CPUSTAT_STOPPED) && !ext_call_pending) 28 if (!stopped && !ext_call_pending)
30 rc = SIGP_CC_ORDER_CODE_ACCEPTED; 29 rc = SIGP_CC_ORDER_CODE_ACCEPTED;
31 else { 30 else {
32 *reg &= 0xffffffff00000000UL; 31 *reg &= 0xffffffff00000000UL;
33 if (ext_call_pending) 32 if (ext_call_pending)
34 *reg |= SIGP_STATUS_EXT_CALL_PENDING; 33 *reg |= SIGP_STATUS_EXT_CALL_PENDING;
35 if (cpuflags & CPUSTAT_STOPPED) 34 if (stopped)
36 *reg |= SIGP_STATUS_STOPPED; 35 *reg |= SIGP_STATUS_STOPPED;
37 rc = SIGP_CC_STATUS_STORED; 36 rc = SIGP_CC_STATUS_STORED;
38 } 37 }
@@ -205,11 +204,9 @@ static int __sigp_store_status_at_addr(struct kvm_vcpu *vcpu,
205 struct kvm_vcpu *dst_vcpu, 204 struct kvm_vcpu *dst_vcpu,
206 u32 addr, u64 *reg) 205 u32 addr, u64 *reg)
207{ 206{
208 int flags;
209 int rc; 207 int rc;
210 208
211 flags = atomic_read(&dst_vcpu->arch.sie_block->cpuflags); 209 if (!kvm_s390_test_cpuflags(dst_vcpu, CPUSTAT_STOPPED)) {
212 if (!(flags & CPUSTAT_STOPPED)) {
213 *reg &= 0xffffffff00000000UL; 210 *reg &= 0xffffffff00000000UL;
214 *reg |= SIGP_STATUS_INCORRECT_STATE; 211 *reg |= SIGP_STATUS_INCORRECT_STATE;
215 return SIGP_CC_STATUS_STORED; 212 return SIGP_CC_STATUS_STORED;
@@ -236,8 +233,7 @@ static int __sigp_sense_running(struct kvm_vcpu *vcpu,
236 return SIGP_CC_STATUS_STORED; 233 return SIGP_CC_STATUS_STORED;
237 } 234 }
238 235
239 if (atomic_read(&dst_vcpu->arch.sie_block->cpuflags) & 236 if (kvm_s390_test_cpuflags(dst_vcpu, CPUSTAT_RUNNING)) {
240 CPUSTAT_RUNNING) {
241 /* running */ 237 /* running */
242 rc = SIGP_CC_ORDER_CODE_ACCEPTED; 238 rc = SIGP_CC_ORDER_CODE_ACCEPTED;
243 } else { 239 } else {
diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index 5d6ae0326d9e..6d494ed5907e 100644
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -28,13 +28,23 @@ struct vsie_page {
28 * the same offset as that in struct sie_page! 28 * the same offset as that in struct sie_page!
29 */ 29 */
30 struct mcck_volatile_info mcck_info; /* 0x0200 */ 30 struct mcck_volatile_info mcck_info; /* 0x0200 */
31 /* the pinned originial scb */ 31 /*
32 * The pinned original scb. Be aware that other VCPUs can modify
33 * it while we read from it. Values that are used for conditions or
34 * are reused conditionally, should be accessed via READ_ONCE.
35 */
32 struct kvm_s390_sie_block *scb_o; /* 0x0218 */ 36 struct kvm_s390_sie_block *scb_o; /* 0x0218 */
33 /* the shadow gmap in use by the vsie_page */ 37 /* the shadow gmap in use by the vsie_page */
34 struct gmap *gmap; /* 0x0220 */ 38 struct gmap *gmap; /* 0x0220 */
35 /* address of the last reported fault to guest2 */ 39 /* address of the last reported fault to guest2 */
36 unsigned long fault_addr; /* 0x0228 */ 40 unsigned long fault_addr; /* 0x0228 */
37 __u8 reserved[0x0700 - 0x0230]; /* 0x0230 */ 41 /* calculated guest addresses of satellite control blocks */
42 gpa_t sca_gpa; /* 0x0230 */
43 gpa_t itdba_gpa; /* 0x0238 */
44 gpa_t gvrd_gpa; /* 0x0240 */
45 gpa_t riccbd_gpa; /* 0x0248 */
46 gpa_t sdnx_gpa; /* 0x0250 */
47 __u8 reserved[0x0700 - 0x0258]; /* 0x0258 */
38 struct kvm_s390_crypto_cb crycb; /* 0x0700 */ 48 struct kvm_s390_crypto_cb crycb; /* 0x0700 */
39 __u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE]; /* 0x0800 */ 49 __u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE]; /* 0x0800 */
40}; 50};
@@ -140,12 +150,13 @@ static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
140{ 150{
141 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; 151 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
142 struct kvm_s390_sie_block *scb_o = vsie_page->scb_o; 152 struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
143 u32 crycb_addr = scb_o->crycbd & 0x7ffffff8U; 153 const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
154 const u32 crycb_addr = crycbd_o & 0x7ffffff8U;
144 unsigned long *b1, *b2; 155 unsigned long *b1, *b2;
145 u8 ecb3_flags; 156 u8 ecb3_flags;
146 157
147 scb_s->crycbd = 0; 158 scb_s->crycbd = 0;
148 if (!(scb_o->crycbd & vcpu->arch.sie_block->crycbd & CRYCB_FORMAT1)) 159 if (!(crycbd_o & vcpu->arch.sie_block->crycbd & CRYCB_FORMAT1))
149 return 0; 160 return 0;
150 /* format-1 is supported with message-security-assist extension 3 */ 161 /* format-1 is supported with message-security-assist extension 3 */
151 if (!test_kvm_facility(vcpu->kvm, 76)) 162 if (!test_kvm_facility(vcpu->kvm, 76))
@@ -183,12 +194,15 @@ static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
183{ 194{
184 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; 195 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
185 struct kvm_s390_sie_block *scb_o = vsie_page->scb_o; 196 struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
197 /* READ_ONCE does not work on bitfields - use a temporary variable */
198 const uint32_t __new_ibc = scb_o->ibc;
199 const uint32_t new_ibc = READ_ONCE(__new_ibc) & 0x0fffU;
186 __u64 min_ibc = (sclp.ibc >> 16) & 0x0fffU; 200 __u64 min_ibc = (sclp.ibc >> 16) & 0x0fffU;
187 201
188 scb_s->ibc = 0; 202 scb_s->ibc = 0;
189 /* ibc installed in g2 and requested for g3 */ 203 /* ibc installed in g2 and requested for g3 */
190 if (vcpu->kvm->arch.model.ibc && (scb_o->ibc & 0x0fffU)) { 204 if (vcpu->kvm->arch.model.ibc && new_ibc) {
191 scb_s->ibc = scb_o->ibc & 0x0fffU; 205 scb_s->ibc = new_ibc;
192 /* takte care of the minimum ibc level of the machine */ 206 /* takte care of the minimum ibc level of the machine */
193 if (scb_s->ibc < min_ibc) 207 if (scb_s->ibc < min_ibc)
194 scb_s->ibc = min_ibc; 208 scb_s->ibc = min_ibc;
@@ -253,6 +267,10 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
253{ 267{
254 struct kvm_s390_sie_block *scb_o = vsie_page->scb_o; 268 struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
255 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; 269 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
270 /* READ_ONCE does not work on bitfields - use a temporary variable */
271 const uint32_t __new_prefix = scb_o->prefix;
272 const uint32_t new_prefix = READ_ONCE(__new_prefix);
273 const bool wants_tx = READ_ONCE(scb_o->ecb) & ECB_TE;
256 bool had_tx = scb_s->ecb & ECB_TE; 274 bool had_tx = scb_s->ecb & ECB_TE;
257 unsigned long new_mso = 0; 275 unsigned long new_mso = 0;
258 int rc; 276 int rc;
@@ -299,14 +317,14 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
299 scb_s->icpua = scb_o->icpua; 317 scb_s->icpua = scb_o->icpua;
300 318
301 if (!(atomic_read(&scb_s->cpuflags) & CPUSTAT_SM)) 319 if (!(atomic_read(&scb_s->cpuflags) & CPUSTAT_SM))
302 new_mso = scb_o->mso & 0xfffffffffff00000UL; 320 new_mso = READ_ONCE(scb_o->mso) & 0xfffffffffff00000UL;
303 /* if the hva of the prefix changes, we have to remap the prefix */ 321 /* if the hva of the prefix changes, we have to remap the prefix */
304 if (scb_s->mso != new_mso || scb_s->prefix != scb_o->prefix) 322 if (scb_s->mso != new_mso || scb_s->prefix != new_prefix)
305 prefix_unmapped(vsie_page); 323 prefix_unmapped(vsie_page);
306 /* SIE will do mso/msl validity and exception checks for us */ 324 /* SIE will do mso/msl validity and exception checks for us */
307 scb_s->msl = scb_o->msl & 0xfffffffffff00000UL; 325 scb_s->msl = scb_o->msl & 0xfffffffffff00000UL;
308 scb_s->mso = new_mso; 326 scb_s->mso = new_mso;
309 scb_s->prefix = scb_o->prefix; 327 scb_s->prefix = new_prefix;
310 328
311 /* We have to definetly flush the tlb if this scb never ran */ 329 /* We have to definetly flush the tlb if this scb never ran */
312 if (scb_s->ihcpu != 0xffffU) 330 if (scb_s->ihcpu != 0xffffU)
@@ -318,11 +336,11 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
318 if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ESOP)) 336 if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ESOP))
319 scb_s->ecb |= scb_o->ecb & ECB_HOSTPROTINT; 337 scb_s->ecb |= scb_o->ecb & ECB_HOSTPROTINT;
320 /* transactional execution */ 338 /* transactional execution */
321 if (test_kvm_facility(vcpu->kvm, 73)) { 339 if (test_kvm_facility(vcpu->kvm, 73) && wants_tx) {
322 /* remap the prefix is tx is toggled on */ 340 /* remap the prefix is tx is toggled on */
323 if ((scb_o->ecb & ECB_TE) && !had_tx) 341 if (!had_tx)
324 prefix_unmapped(vsie_page); 342 prefix_unmapped(vsie_page);
325 scb_s->ecb |= scb_o->ecb & ECB_TE; 343 scb_s->ecb |= ECB_TE;
326 } 344 }
327 /* SIMD */ 345 /* SIMD */
328 if (test_kvm_facility(vcpu->kvm, 129)) { 346 if (test_kvm_facility(vcpu->kvm, 129)) {
@@ -463,46 +481,42 @@ static void unpin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t hpa)
463/* unpin all blocks previously pinned by pin_blocks(), marking them dirty */ 481/* unpin all blocks previously pinned by pin_blocks(), marking them dirty */
464static void unpin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) 482static void unpin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
465{ 483{
466 struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
467 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; 484 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
468 hpa_t hpa; 485 hpa_t hpa;
469 gpa_t gpa;
470 486
471 hpa = (u64) scb_s->scaoh << 32 | scb_s->scaol; 487 hpa = (u64) scb_s->scaoh << 32 | scb_s->scaol;
472 if (hpa) { 488 if (hpa) {
473 gpa = scb_o->scaol & ~0xfUL; 489 unpin_guest_page(vcpu->kvm, vsie_page->sca_gpa, hpa);
474 if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO)) 490 vsie_page->sca_gpa = 0;
475 gpa |= (u64) scb_o->scaoh << 32;
476 unpin_guest_page(vcpu->kvm, gpa, hpa);
477 scb_s->scaol = 0; 491 scb_s->scaol = 0;
478 scb_s->scaoh = 0; 492 scb_s->scaoh = 0;
479 } 493 }
480 494
481 hpa = scb_s->itdba; 495 hpa = scb_s->itdba;
482 if (hpa) { 496 if (hpa) {
483 gpa = scb_o->itdba & ~0xffUL; 497 unpin_guest_page(vcpu->kvm, vsie_page->itdba_gpa, hpa);
484 unpin_guest_page(vcpu->kvm, gpa, hpa); 498 vsie_page->itdba_gpa = 0;
485 scb_s->itdba = 0; 499 scb_s->itdba = 0;
486 } 500 }
487 501
488 hpa = scb_s->gvrd; 502 hpa = scb_s->gvrd;
489 if (hpa) { 503 if (hpa) {
490 gpa = scb_o->gvrd & ~0x1ffUL; 504 unpin_guest_page(vcpu->kvm, vsie_page->gvrd_gpa, hpa);
491 unpin_guest_page(vcpu->kvm, gpa, hpa); 505 vsie_page->gvrd_gpa = 0;
492 scb_s->gvrd = 0; 506 scb_s->gvrd = 0;
493 } 507 }
494 508
495 hpa = scb_s->riccbd; 509 hpa = scb_s->riccbd;
496 if (hpa) { 510 if (hpa) {
497 gpa = scb_o->riccbd & ~0x3fUL; 511 unpin_guest_page(vcpu->kvm, vsie_page->riccbd_gpa, hpa);
498 unpin_guest_page(vcpu->kvm, gpa, hpa); 512 vsie_page->riccbd_gpa = 0;
499 scb_s->riccbd = 0; 513 scb_s->riccbd = 0;
500 } 514 }
501 515
502 hpa = scb_s->sdnxo; 516 hpa = scb_s->sdnxo;
503 if (hpa) { 517 if (hpa) {
504 gpa = scb_o->sdnxo; 518 unpin_guest_page(vcpu->kvm, vsie_page->sdnx_gpa, hpa);
505 unpin_guest_page(vcpu->kvm, gpa, hpa); 519 vsie_page->sdnx_gpa = 0;
506 scb_s->sdnxo = 0; 520 scb_s->sdnxo = 0;
507 } 521 }
508} 522}
@@ -529,9 +543,9 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
529 gpa_t gpa; 543 gpa_t gpa;
530 int rc = 0; 544 int rc = 0;
531 545
532 gpa = scb_o->scaol & ~0xfUL; 546 gpa = READ_ONCE(scb_o->scaol) & ~0xfUL;
533 if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO)) 547 if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO))
534 gpa |= (u64) scb_o->scaoh << 32; 548 gpa |= (u64) READ_ONCE(scb_o->scaoh) << 32;
535 if (gpa) { 549 if (gpa) {
536 if (!(gpa & ~0x1fffUL)) 550 if (!(gpa & ~0x1fffUL))
537 rc = set_validity_icpt(scb_s, 0x0038U); 551 rc = set_validity_icpt(scb_s, 0x0038U);
@@ -547,11 +561,12 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
547 } 561 }
548 if (rc) 562 if (rc)
549 goto unpin; 563 goto unpin;
564 vsie_page->sca_gpa = gpa;
550 scb_s->scaoh = (u32)((u64)hpa >> 32); 565 scb_s->scaoh = (u32)((u64)hpa >> 32);
551 scb_s->scaol = (u32)(u64)hpa; 566 scb_s->scaol = (u32)(u64)hpa;
552 } 567 }
553 568
554 gpa = scb_o->itdba & ~0xffUL; 569 gpa = READ_ONCE(scb_o->itdba) & ~0xffUL;
555 if (gpa && (scb_s->ecb & ECB_TE)) { 570 if (gpa && (scb_s->ecb & ECB_TE)) {
556 if (!(gpa & ~0x1fffU)) { 571 if (!(gpa & ~0x1fffU)) {
557 rc = set_validity_icpt(scb_s, 0x0080U); 572 rc = set_validity_icpt(scb_s, 0x0080U);
@@ -563,10 +578,11 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
563 rc = set_validity_icpt(scb_s, 0x0080U); 578 rc = set_validity_icpt(scb_s, 0x0080U);
564 goto unpin; 579 goto unpin;
565 } 580 }
581 vsie_page->itdba_gpa = gpa;
566 scb_s->itdba = hpa; 582 scb_s->itdba = hpa;
567 } 583 }
568 584
569 gpa = scb_o->gvrd & ~0x1ffUL; 585 gpa = READ_ONCE(scb_o->gvrd) & ~0x1ffUL;
570 if (gpa && (scb_s->eca & ECA_VX) && !(scb_s->ecd & ECD_HOSTREGMGMT)) { 586 if (gpa && (scb_s->eca & ECA_VX) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
571 if (!(gpa & ~0x1fffUL)) { 587 if (!(gpa & ~0x1fffUL)) {
572 rc = set_validity_icpt(scb_s, 0x1310U); 588 rc = set_validity_icpt(scb_s, 0x1310U);
@@ -581,10 +597,11 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
581 rc = set_validity_icpt(scb_s, 0x1310U); 597 rc = set_validity_icpt(scb_s, 0x1310U);
582 goto unpin; 598 goto unpin;
583 } 599 }
600 vsie_page->gvrd_gpa = gpa;
584 scb_s->gvrd = hpa; 601 scb_s->gvrd = hpa;
585 } 602 }
586 603
587 gpa = scb_o->riccbd & ~0x3fUL; 604 gpa = READ_ONCE(scb_o->riccbd) & ~0x3fUL;
588 if (gpa && (scb_s->ecb3 & ECB3_RI)) { 605 if (gpa && (scb_s->ecb3 & ECB3_RI)) {
589 if (!(gpa & ~0x1fffUL)) { 606 if (!(gpa & ~0x1fffUL)) {
590 rc = set_validity_icpt(scb_s, 0x0043U); 607 rc = set_validity_icpt(scb_s, 0x0043U);
@@ -597,13 +614,14 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
597 goto unpin; 614 goto unpin;
598 } 615 }
599 /* Validity 0x0044 will be checked by SIE */ 616 /* Validity 0x0044 will be checked by SIE */
617 vsie_page->riccbd_gpa = gpa;
600 scb_s->riccbd = hpa; 618 scb_s->riccbd = hpa;
601 } 619 }
602 if ((scb_s->ecb & ECB_GS) && !(scb_s->ecd & ECD_HOSTREGMGMT)) { 620 if ((scb_s->ecb & ECB_GS) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
603 unsigned long sdnxc; 621 unsigned long sdnxc;
604 622
605 gpa = scb_o->sdnxo & ~0xfUL; 623 gpa = READ_ONCE(scb_o->sdnxo) & ~0xfUL;
606 sdnxc = scb_o->sdnxo & 0xfUL; 624 sdnxc = READ_ONCE(scb_o->sdnxo) & 0xfUL;
607 if (!gpa || !(gpa & ~0x1fffUL)) { 625 if (!gpa || !(gpa & ~0x1fffUL)) {
608 rc = set_validity_icpt(scb_s, 0x10b0U); 626 rc = set_validity_icpt(scb_s, 0x10b0U);
609 goto unpin; 627 goto unpin;
@@ -624,6 +642,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
624 rc = set_validity_icpt(scb_s, 0x10b0U); 642 rc = set_validity_icpt(scb_s, 0x10b0U);
625 goto unpin; 643 goto unpin;
626 } 644 }
645 vsie_page->sdnx_gpa = gpa;
627 scb_s->sdnxo = hpa | sdnxc; 646 scb_s->sdnxo = hpa | sdnxc;
628 } 647 }
629 return 0; 648 return 0;
@@ -768,7 +787,7 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page)
768static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) 787static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
769{ 788{
770 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; 789 struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
771 __u32 fac = vsie_page->scb_o->fac & 0x7ffffff8U; 790 __u32 fac = READ_ONCE(vsie_page->scb_o->fac) & 0x7ffffff8U;
772 791
773 if (fac && test_kvm_facility(vcpu->kvm, 7)) { 792 if (fac && test_kvm_facility(vcpu->kvm, 7)) {
774 retry_vsie_icpt(vsie_page); 793 retry_vsie_icpt(vsie_page);
@@ -894,7 +913,7 @@ static void register_shadow_scb(struct kvm_vcpu *vcpu,
894 * External calls have to lead to a kick of the vcpu and 913 * External calls have to lead to a kick of the vcpu and
895 * therefore the vsie -> Simulate Wait state. 914 * therefore the vsie -> Simulate Wait state.
896 */ 915 */
897 atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 916 kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
898 /* 917 /*
899 * We have to adjust the g3 epoch by the g2 epoch. The epoch will 918 * We have to adjust the g3 epoch by the g2 epoch. The epoch will
900 * automatically be adjusted on tod clock changes via kvm_sync_clock. 919 * automatically be adjusted on tod clock changes via kvm_sync_clock.
@@ -916,7 +935,7 @@ static void register_shadow_scb(struct kvm_vcpu *vcpu,
916 */ 935 */
917static void unregister_shadow_scb(struct kvm_vcpu *vcpu) 936static void unregister_shadow_scb(struct kvm_vcpu *vcpu)
918{ 937{
919 atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 938 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
920 WRITE_ONCE(vcpu->arch.vsie_block, NULL); 939 WRITE_ONCE(vcpu->arch.vsie_block, NULL);
921} 940}
922 941
diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c
index 54cfd51a5a27..2c55a2b9d6c6 100644
--- a/arch/s390/mm/gmap.c
+++ b/arch/s390/mm/gmap.c
@@ -1021,18 +1021,17 @@ static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,
1021} 1021}
1022 1022
1023/** 1023/**
1024 * gmap_protect_rmap - modify access rights to memory and create an rmap 1024 * gmap_protect_rmap - restrict access rights to memory (RO) and create an rmap
1025 * @sg: pointer to the shadow guest address space structure 1025 * @sg: pointer to the shadow guest address space structure
1026 * @raddr: rmap address in the shadow gmap 1026 * @raddr: rmap address in the shadow gmap
1027 * @paddr: address in the parent guest address space 1027 * @paddr: address in the parent guest address space
1028 * @len: length of the memory area to protect 1028 * @len: length of the memory area to protect
1029 * @prot: indicates access rights: none, read-only or read-write
1030 * 1029 *
1031 * Returns 0 if successfully protected and the rmap was created, -ENOMEM 1030 * Returns 0 if successfully protected and the rmap was created, -ENOMEM
1032 * if out of memory and -EFAULT if paddr is invalid. 1031 * if out of memory and -EFAULT if paddr is invalid.
1033 */ 1032 */
1034static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr, 1033static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
1035 unsigned long paddr, unsigned long len, int prot) 1034 unsigned long paddr, unsigned long len)
1036{ 1035{
1037 struct gmap *parent; 1036 struct gmap *parent;
1038 struct gmap_rmap *rmap; 1037 struct gmap_rmap *rmap;
@@ -1060,7 +1059,7 @@ static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
1060 ptep = gmap_pte_op_walk(parent, paddr, &ptl); 1059 ptep = gmap_pte_op_walk(parent, paddr, &ptl);
1061 if (ptep) { 1060 if (ptep) {
1062 spin_lock(&sg->guest_table_lock); 1061 spin_lock(&sg->guest_table_lock);
1063 rc = ptep_force_prot(parent->mm, paddr, ptep, prot, 1062 rc = ptep_force_prot(parent->mm, paddr, ptep, PROT_READ,
1064 PGSTE_VSIE_BIT); 1063 PGSTE_VSIE_BIT);
1065 if (!rc) 1064 if (!rc)
1066 gmap_insert_rmap(sg, vmaddr, rmap); 1065 gmap_insert_rmap(sg, vmaddr, rmap);
@@ -1070,7 +1069,7 @@ static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
1070 radix_tree_preload_end(); 1069 radix_tree_preload_end();
1071 if (rc) { 1070 if (rc) {
1072 kfree(rmap); 1071 kfree(rmap);
1073 rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot); 1072 rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ);
1074 if (rc) 1073 if (rc)
1075 return rc; 1074 return rc;
1076 continue; 1075 continue;
@@ -1609,7 +1608,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
1609 origin = r2t & _REGION_ENTRY_ORIGIN; 1608 origin = r2t & _REGION_ENTRY_ORIGIN;
1610 offset = ((r2t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE; 1609 offset = ((r2t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
1611 len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; 1610 len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
1612 rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ); 1611 rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
1613 spin_lock(&sg->guest_table_lock); 1612 spin_lock(&sg->guest_table_lock);
1614 if (!rc) { 1613 if (!rc) {
1615 table = gmap_table_walk(sg, saddr, 4); 1614 table = gmap_table_walk(sg, saddr, 4);
@@ -1692,7 +1691,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
1692 origin = r3t & _REGION_ENTRY_ORIGIN; 1691 origin = r3t & _REGION_ENTRY_ORIGIN;
1693 offset = ((r3t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE; 1692 offset = ((r3t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
1694 len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; 1693 len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
1695 rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ); 1694 rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
1696 spin_lock(&sg->guest_table_lock); 1695 spin_lock(&sg->guest_table_lock);
1697 if (!rc) { 1696 if (!rc) {
1698 table = gmap_table_walk(sg, saddr, 3); 1697 table = gmap_table_walk(sg, saddr, 3);
@@ -1776,7 +1775,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
1776 origin = sgt & _REGION_ENTRY_ORIGIN; 1775 origin = sgt & _REGION_ENTRY_ORIGIN;
1777 offset = ((sgt & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE; 1776 offset = ((sgt & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
1778 len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; 1777 len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
1779 rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ); 1778 rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
1780 spin_lock(&sg->guest_table_lock); 1779 spin_lock(&sg->guest_table_lock);
1781 if (!rc) { 1780 if (!rc) {
1782 table = gmap_table_walk(sg, saddr, 2); 1781 table = gmap_table_walk(sg, saddr, 2);
@@ -1895,7 +1894,7 @@ int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
1895 /* Make pgt read-only in parent gmap page table (not the pgste) */ 1894 /* Make pgt read-only in parent gmap page table (not the pgste) */
1896 raddr = (saddr & _SEGMENT_MASK) | _SHADOW_RMAP_SEGMENT; 1895 raddr = (saddr & _SEGMENT_MASK) | _SHADOW_RMAP_SEGMENT;
1897 origin = pgt & _SEGMENT_ENTRY_ORIGIN & PAGE_MASK; 1896 origin = pgt & _SEGMENT_ENTRY_ORIGIN & PAGE_MASK;
1898 rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE, PROT_READ); 1897 rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE);
1899 spin_lock(&sg->guest_table_lock); 1898 spin_lock(&sg->guest_table_lock);
1900 if (!rc) { 1899 if (!rc) {
1901 table = gmap_table_walk(sg, saddr, 1); 1900 table = gmap_table_walk(sg, saddr, 1);
@@ -1998,7 +1997,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_page);
1998 * Called with sg->parent->shadow_lock. 1997 * Called with sg->parent->shadow_lock.
1999 */ 1998 */
2000static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, 1999static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,
2001 unsigned long gaddr, pte_t *pte) 2000 unsigned long gaddr)
2002{ 2001{
2003 struct gmap_rmap *rmap, *rnext, *head; 2002 struct gmap_rmap *rmap, *rnext, *head;
2004 unsigned long start, end, bits, raddr; 2003 unsigned long start, end, bits, raddr;
@@ -2083,7 +2082,7 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr,
2083 spin_lock(&gmap->shadow_lock); 2082 spin_lock(&gmap->shadow_lock);
2084 list_for_each_entry_safe(sg, next, 2083 list_for_each_entry_safe(sg, next,
2085 &gmap->children, list) 2084 &gmap->children, list)
2086 gmap_shadow_notify(sg, vmaddr, gaddr, pte); 2085 gmap_shadow_notify(sg, vmaddr, gaddr);
2087 spin_unlock(&gmap->shadow_lock); 2086 spin_unlock(&gmap->shadow_lock);
2088 } 2087 }
2089 if (bits & PGSTE_IN_BIT) 2088 if (bits & PGSTE_IN_BIT)
diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c
index d06bc5674e5f..6b1891539c84 100644
--- a/drivers/s390/char/sclp_early.c
+++ b/drivers/s390/char/sclp_early.c
@@ -49,7 +49,7 @@ struct read_info_sccb {
49 u8 _pad_112[116 - 112]; /* 112-115 */ 49 u8 _pad_112[116 - 112]; /* 112-115 */
50 u8 fac116; /* 116 */ 50 u8 fac116; /* 116 */
51 u8 fac117; /* 117 */ 51 u8 fac117; /* 117 */
52 u8 _pad_118; /* 118 */ 52 u8 fac118; /* 118 */
53 u8 fac119; /* 119 */ 53 u8 fac119; /* 119 */
54 u16 hcpua; /* 120-121 */ 54 u16 hcpua; /* 120-121 */
55 u8 _pad_122[124 - 122]; /* 122-123 */ 55 u8 _pad_122[124 - 122]; /* 122-123 */
@@ -100,6 +100,7 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb)
100 sclp.has_esca = !!(sccb->fac116 & 0x08); 100 sclp.has_esca = !!(sccb->fac116 & 0x08);
101 sclp.has_pfmfi = !!(sccb->fac117 & 0x40); 101 sclp.has_pfmfi = !!(sccb->fac117 & 0x40);
102 sclp.has_ibs = !!(sccb->fac117 & 0x20); 102 sclp.has_ibs = !!(sccb->fac117 & 0x20);
103 sclp.has_gisaf = !!(sccb->fac118 & 0x08);
103 sclp.has_hvs = !!(sccb->fac119 & 0x80); 104 sclp.has_hvs = !!(sccb->fac119 & 0x80);
104 sclp.has_kss = !!(sccb->fac98 & 0x01); 105 sclp.has_kss = !!(sccb->fac98 & 0x01);
105 if (sccb->fac85 & 0x02) 106 if (sccb->fac85 & 0x02)
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-29 03:10:16 -0400