3 files changed, 53 insertions, 49 deletions

diff --git a/Documentation/virtual/kvm/devices/vm.txt b/Documentation/virtual/kvm/devices/vm.txt
index 0d16f96c0eac..d426fc87fe93 100644
--- a/Documentation/virtual/kvm/devices/vm.txt
+++ b/Documentation/virtual/kvm/devices/vm.txt
@@ -12,14 +12,14 @@ specific.
 1. GROUP: KVM_S390_VM_MEM_CTRL
 Architectures: s390
 
-1.1. ATTRIBUTE: KVM_S390_VM_MEM_CTRL
+1.1. ATTRIBUTE: KVM_S390_VM_MEM_ENABLE_CMMA
 Parameters: none
-Returns: -EBUSY if already a vcpus is defined, otherwise 0
+Returns: -EBUSY if a vcpu is already defined, otherwise 0
 
-Enables CMMA for the virtual machine
+Enables Collaborative Memory Management Assist (CMMA) for the virtual machine.
 
-1.2. ATTRIBUTE: KVM_S390_VM_CLR_CMMA
-Parameteres: none
+1.2. ATTRIBUTE: KVM_S390_VM_MEM_CLR_CMMA
+Parameters: none
 Returns: 0
 
 Clear the CMMA status for all guest pages, so any pages the guest marked
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index c1424e8b7f09..8b9ccf02a2c5 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -227,10 +227,12 @@ static void ipte_lock_simple(struct kvm_vcpu *vcpu)
                 goto out;
         ic = &vcpu->kvm->arch.sca->ipte_control;
         do {
-                old = ACCESS_ONCE(*ic);
+                old = *ic;
+                barrier();
                 while (old.k) {
                         cond_resched();
-                        old = ACCESS_ONCE(*ic);
+                        old = *ic;
+                        barrier();
                 }
                 new = old;
                 new.k = 1;
@@ -249,7 +251,9 @@ static void ipte_unlock_simple(struct kvm_vcpu *vcpu)
                 goto out;
         ic = &vcpu->kvm->arch.sca->ipte_control;
         do {
-                new = old = ACCESS_ONCE(*ic);
+                old = *ic;
+                barrier();
+                new = old;
                 new.k = 0;
         } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
         wake_up(&vcpu->kvm->arch.ipte_wq);
@@ -263,10 +267,12 @@ static void ipte_lock_siif(struct kvm_vcpu *vcpu)
 
         ic = &vcpu->kvm->arch.sca->ipte_control;
         do {
-                old = ACCESS_ONCE(*ic);
+                old = *ic;
+                barrier();
                 while (old.kg) {
                         cond_resched();
-                        old = ACCESS_ONCE(*ic);
+                        old = *ic;
+                        barrier();
                 }
                 new = old;
                 new.k = 1;
@@ -280,7 +286,9 @@ static void ipte_unlock_siif(struct kvm_vcpu *vcpu)
 
         ic = &vcpu->kvm->arch.sca->ipte_control;
         do {
-                new = old = ACCESS_ONCE(*ic);
+                old = *ic;
+                barrier();
+                new = old;
                 new.kh--;
                 if (!new.kh)
                         new.k = 0;
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index 72bb2dd8b9cd..9bde32f66eb5 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -762,8 +762,8 @@ int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
 {
         int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
         int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
-        u32 val = 0;
-        int reg, rc;
+        int reg, rc, nr_regs;
+        u32 ctl_array[16];
         u64 ga;
 
         vcpu->stat.instruction_lctl++;
@@ -779,19 +779,20 @@ int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
         VCPU_EVENT(vcpu, 5, "lctl r1:%x, r3:%x, addr:%llx", reg1, reg3, ga);
         trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga);
 
+        nr_regs = ((reg3 - reg1) & 0xf) + 1;
+        rc = read_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u32));
+        if (rc)
+                return kvm_s390_inject_prog_cond(vcpu, rc);
         reg = reg1;
+        nr_regs = 0;
         do {
-                rc = read_guest(vcpu, ga, &val, sizeof(val));
-                if (rc)
-                        return kvm_s390_inject_prog_cond(vcpu, rc);
                 vcpu->arch.sie_block->gcr[reg] &= 0xffffffff00000000ul;
-                vcpu->arch.sie_block->gcr[reg] |= val;
-                ga += 4;
+                vcpu->arch.sie_block->gcr[reg] |= ctl_array[nr_regs++];
                 if (reg == reg3)
                         break;
                 reg = (reg + 1) % 16;
         } while (1);
-
+        kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
         return 0;
 }
 
@@ -799,9 +800,9 @@ int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu)
 {
         int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
         int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
+        int reg, rc, nr_regs;
+        u32 ctl_array[16];
         u64 ga;
-        u32 val;
-        int reg, rc;
 
         vcpu->stat.instruction_stctl++;
 
@@ -817,26 +818,24 @@ int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu)
         trace_kvm_s390_handle_stctl(vcpu, 0, reg1, reg3, ga);
 
         reg = reg1;
+        nr_regs = 0;
         do {
-                val = vcpu->arch.sie_block->gcr[reg] &  0x00000000fffffffful;
-                rc = write_guest(vcpu, ga, &val, sizeof(val));
-                if (rc)
-                        return kvm_s390_inject_prog_cond(vcpu, rc);
-                ga += 4;
+                ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg];
                 if (reg == reg3)
                         break;
                 reg = (reg + 1) % 16;
         } while (1);
-
-        return 0;
+        rc = write_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u32));
+        return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
 }
 
 static int handle_lctlg(struct kvm_vcpu *vcpu)
 {
         int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
         int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
-        u64 ga, val;
-        int reg, rc;
+        int reg, rc, nr_regs;
+        u64 ctl_array[16];
+        u64 ga;
 
         vcpu->stat.instruction_lctlg++;
 
@@ -848,22 +847,22 @@ static int handle_lctlg(struct kvm_vcpu *vcpu)
         if (ga & 7)
                 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 
-        reg = reg1;
-
         VCPU_EVENT(vcpu, 5, "lctlg r1:%x, r3:%x, addr:%llx", reg1, reg3, ga);
         trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga);
 
+        nr_regs = ((reg3 - reg1) & 0xf) + 1;
+        rc = read_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u64));
+        if (rc)
+                return kvm_s390_inject_prog_cond(vcpu, rc);
+        reg = reg1;
+        nr_regs = 0;
         do {
-                rc = read_guest(vcpu, ga, &val, sizeof(val));
-                if (rc)
-                        return kvm_s390_inject_prog_cond(vcpu, rc);
-                vcpu->arch.sie_block->gcr[reg] = val;
-                ga += 8;
+                vcpu->arch.sie_block->gcr[reg] = ctl_array[nr_regs++];
                 if (reg == reg3)
                         break;
                 reg = (reg + 1) % 16;
         } while (1);
-
+        kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
         return 0;
 }
 
@@ -871,8 +870,9 @@ static int handle_stctg(struct kvm_vcpu *vcpu)
 {
         int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
         int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
-        u64 ga, val;
-        int reg, rc;
+        int reg, rc, nr_regs;
+        u64 ctl_array[16];
+        u64 ga;
 
         vcpu->stat.instruction_stctg++;
 
@@ -884,23 +884,19 @@ static int handle_stctg(struct kvm_vcpu *vcpu)
         if (ga & 7)
                 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 
-        reg = reg1;
-
         VCPU_EVENT(vcpu, 5, "stctg r1:%x, r3:%x, addr:%llx", reg1, reg3, ga);
         trace_kvm_s390_handle_stctl(vcpu, 1, reg1, reg3, ga);
 
+        reg = reg1;
+        nr_regs = 0;
         do {
-                val = vcpu->arch.sie_block->gcr[reg];
-                rc = write_guest(vcpu, ga, &val, sizeof(val));
-                if (rc)
-                        return kvm_s390_inject_prog_cond(vcpu, rc);
-                ga += 8;
+                ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg];
                 if (reg == reg3)
                         break;
                 reg = (reg + 1) % 16;
         } while (1);
-
-        return 0;
+        rc = write_guest(vcpu, ga, ctl_array, nr_regs * sizeof(u64));
+        return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
 }
 
 static const intercept_handler_t eb_handlers[256] = {