KVM: async_pf: Async page fault support on s390

This patch enables async page faults for s390 kvm guests.
It provides the userspace API to enable and disable_wait this feature.
The disable_wait will enforce that the feature is off by waiting on it.
Also it includes the diagnose code, called by the guest to enable async page faults.

The async page faults will use an already existing guest interface for this
purpose, as described in "CP Programming Services (SC24-6084)".

Signed-off-by: Dominik Dingel <dingel@linux.vnet.ibm.com>
Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>

8 files changed, 290 insertions, 9 deletions

diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig
index 70b46eacf8e1..c8bacbcd2e5b 100644
--- a/arch/s390/kvm/Kconfig
+++ b/arch/s390/kvm/Kconfig
@@ -23,6 +23,8 @@ config KVM
         select ANON_INODES
         select HAVE_KVM_CPU_RELAX_INTERCEPT
         select HAVE_KVM_EVENTFD
+        select KVM_ASYNC_PF
+        select KVM_ASYNC_PF_SYNC
         ---help---
           Support hosting paravirtualized guest machines using the SIE
           virtualization capability on the mainframe. This should work
diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile
index 40b4c6470f88..a47d2c355f68 100644
--- a/arch/s390/kvm/Makefile
+++ b/arch/s390/kvm/Makefile
@@ -7,7 +7,7 @@
 # as published by the Free Software Foundation.
 
 KVM := ../../../virt/kvm
-common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o
+common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o  $(KVM)/async_pf.o
 
 ccflags-y := -Ivirt/kvm -Iarch/s390/kvm
 
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c
index 8216c0e0b2e2..bf9ed34c2bcd 100644
--- a/arch/s390/kvm/diag.c
+++ b/arch/s390/kvm/diag.c
@@ -17,6 +17,7 @@
 #include "kvm-s390.h"
 #include "trace.h"
 #include "trace-s390.h"
+#include "gaccess.h"
 
 static int diag_release_pages(struct kvm_vcpu *vcpu)
 {
@@ -46,6 +47,87 @@ static int diag_release_pages(struct kvm_vcpu *vcpu)
         return 0;
 }
 
+static int __diag_page_ref_service(struct kvm_vcpu *vcpu)
+{
+        struct prs_parm {
+                u16 code;
+                u16 subcode;
+                u16 parm_len;
+                u16 parm_version;
+                u64 token_addr;
+                u64 select_mask;
+                u64 compare_mask;
+                u64 zarch;
+        };
+        struct prs_parm parm;
+        int rc;
+        u16 rx = (vcpu->arch.sie_block->ipa & 0xf0) >> 4;
+        u16 ry = (vcpu->arch.sie_block->ipa & 0x0f);
+        unsigned long hva_token = KVM_HVA_ERR_BAD;
+
+        if (vcpu->run->s.regs.gprs[rx] & 7)
+                return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
+        if (copy_from_guest(vcpu, &parm, vcpu->run->s.regs.gprs[rx], sizeof(parm)))
+                return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
+        if (parm.parm_version != 2 || parm.parm_len < 5 || parm.code != 0x258)
+                return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
+
+        switch (parm.subcode) {
+        case 0: /* TOKEN */
+                if (vcpu->arch.pfault_token != KVM_S390_PFAULT_TOKEN_INVALID) {
+                        /*
+                         * If the pagefault handshake is already activated,
+                         * the token must not be changed.  We have to return
+                         * decimal 8 instead, as mandated in SC24-6084.
+                         */
+                        vcpu->run->s.regs.gprs[ry] = 8;
+                        return 0;
+                }
+
+                if ((parm.compare_mask & parm.select_mask) != parm.compare_mask ||
+                    parm.token_addr & 7 || parm.zarch != 0x8000000000000000ULL)
+                        return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
+
+                hva_token = gfn_to_hva(vcpu->kvm, gpa_to_gfn(parm.token_addr));
+                if (kvm_is_error_hva(hva_token))
+                        return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
+
+                vcpu->arch.pfault_token = parm.token_addr;
+                vcpu->arch.pfault_select = parm.select_mask;
+                vcpu->arch.pfault_compare = parm.compare_mask;
+                vcpu->run->s.regs.gprs[ry] = 0;
+                rc = 0;
+                break;
+        case 1: /*
+                 * CANCEL
+                 * Specification allows to let already pending tokens survive
+                 * the cancel, therefore to reduce code complexity, we assume
+                 * all outstanding tokens are already pending.
+                 */
+                if (parm.token_addr || parm.select_mask ||
+                    parm.compare_mask || parm.zarch)
+                        return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
+
+                vcpu->run->s.regs.gprs[ry] = 0;
+                /*
+                 * If the pfault handling was not established or is already
+                 * canceled SC24-6084 requests to return decimal 4.
+                 */
+                if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
+                        vcpu->run->s.regs.gprs[ry] = 4;
+                else
+                        vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
+
+                rc = 0;
+                break;
+        default:
+                rc = -EOPNOTSUPP;
+                break;
+        }
+
+        return rc;
+}
+
 static int __diag_time_slice_end(struct kvm_vcpu *vcpu)
 {
         VCPU_EVENT(vcpu, 5, "%s", "diag time slice end");
@@ -150,6 +232,8 @@ int kvm_s390_handle_diag(struct kvm_vcpu *vcpu)
                 return __diag_time_slice_end(vcpu);
         case 0x9c:
                 return __diag_time_slice_end_directed(vcpu);
+        case 0x258:
+                return __diag_page_ref_service(vcpu);
         case 0x308:
                 return __diag_ipl_functions(vcpu);
         case 0x500:
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 9c9192b5e339..1848080c3f34 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -31,7 +31,7 @@ static int is_ioint(u64 type)
         return ((type & 0xfffe0000u) != 0xfffe0000u);
 }
 
-static int psw_extint_disabled(struct kvm_vcpu *vcpu)
+int psw_extint_disabled(struct kvm_vcpu *vcpu)
 {
         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
 }
@@ -78,11 +78,8 @@ static int __interrupt_is_deliverable(struct kvm_vcpu *vcpu,
                         return 1;
                 return 0;
         case KVM_S390_INT_SERVICE:
-                if (psw_extint_disabled(vcpu))
-                        return 0;
-                if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
-                        return 1;
-                return 0;
+        case KVM_S390_INT_PFAULT_INIT:
+        case KVM_S390_INT_PFAULT_DONE:
         case KVM_S390_INT_VIRTIO:
                 if (psw_extint_disabled(vcpu))
                         return 0;
@@ -150,6 +147,8 @@ static void __set_intercept_indicator(struct kvm_vcpu *vcpu,
         case KVM_S390_INT_EXTERNAL_CALL:
         case KVM_S390_INT_EMERGENCY:
         case KVM_S390_INT_SERVICE:
+        case KVM_S390_INT_PFAULT_INIT:
+        case KVM_S390_INT_PFAULT_DONE:
         case KVM_S390_INT_VIRTIO:
                 if (psw_extint_disabled(vcpu))
                         __set_cpuflag(vcpu, CPUSTAT_EXT_INT);
@@ -223,6 +222,30 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
                 rc |= put_guest(vcpu, inti->ext.ext_params,
                                 (u32 __user *)__LC_EXT_PARAMS);
                 break;
+        case KVM_S390_INT_PFAULT_INIT:
+                trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, 0,
+                                                 inti->ext.ext_params2);
+                rc  = put_guest(vcpu, 0x2603, (u16 __user *) __LC_EXT_INT_CODE);
+                rc |= put_guest(vcpu, 0x0600, (u16 __user *) __LC_EXT_CPU_ADDR);
+                rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
+                                    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
+                rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
+                                      __LC_EXT_NEW_PSW, sizeof(psw_t));
+                rc |= put_guest(vcpu, inti->ext.ext_params2,
+                                (u64 __user *) __LC_EXT_PARAMS2);
+                break;
+        case KVM_S390_INT_PFAULT_DONE:
+                trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, 0,
+                                                 inti->ext.ext_params2);
+                rc  = put_guest(vcpu, 0x2603, (u16 __user *) __LC_EXT_INT_CODE);
+                rc |= put_guest(vcpu, 0x0680, (u16 __user *) __LC_EXT_CPU_ADDR);
+                rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
+                                    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
+                rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
+                                      __LC_EXT_NEW_PSW, sizeof(psw_t));
+                rc |= put_guest(vcpu, inti->ext.ext_params2,
+                                (u64 __user *) __LC_EXT_PARAMS2);
+                break;
         case KVM_S390_INT_VIRTIO:
                 VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx",
                            inti->ext.ext_params, inti->ext.ext_params2);
@@ -357,7 +380,7 @@ static int __try_deliver_ckc_interrupt(struct kvm_vcpu *vcpu)
         return 1;
 }
 
-static int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu)
+int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu)
 {
         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
         struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
@@ -737,6 +760,10 @@ int kvm_s390_inject_vm(struct kvm *kvm,
                 VM_EVENT(kvm, 5, "inject: sclp parm:%x", s390int->parm);
                 inti->ext.ext_params = s390int->parm;
                 break;
+        case KVM_S390_INT_PFAULT_DONE:
+                inti->type = s390int->type;
+                inti->ext.ext_params2 = s390int->parm64;
+                break;
         case KVM_S390_MCHK:
                 VM_EVENT(kvm, 5, "inject: machine check parm64:%llx",
                          s390int->parm64);
@@ -823,6 +850,10 @@ int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu,
                 inti->type = s390int->type;
                 inti->mchk.mcic = s390int->parm64;
                 break;
+        case KVM_S390_INT_PFAULT_INIT:
+                inti->type = s390int->type;
+                inti->ext.ext_params2 = s390int->parm64;
+                break;
         case KVM_S390_INT_VIRTIO:
         case KVM_S390_INT_SERVICE:
         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
@@ -877,6 +908,8 @@ static inline int copy_irq_to_user(struct kvm_s390_interrupt_info *inti,
 
         irq.type = inti->type;
         switch (inti->type) {
+        case KVM_S390_INT_PFAULT_INIT:
+        case KVM_S390_INT_PFAULT_DONE:
         case KVM_S390_INT_VIRTIO:
         case KVM_S390_INT_SERVICE:
                 irq.u.ext = inti->ext;
@@ -956,6 +989,8 @@ static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
                 return -EFAULT;
 
         switch (inti->type) {
+        case KVM_S390_INT_PFAULT_INIT:
+        case KVM_S390_INT_PFAULT_DONE:
         case KVM_S390_INT_VIRTIO:
         case KVM_S390_INT_SERVICE:
                 target = (void *) &inti->ext;
@@ -1019,6 +1054,8 @@ static int enqueue_floating_irq(struct kvm_device *dev,
 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
 {
         int r = 0;
+        unsigned int i;
+        struct kvm_vcpu *vcpu;
 
         switch (attr->group) {
         case KVM_DEV_FLIC_ENQUEUE:
@@ -1028,6 +1065,20 @@ static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
                 r = 0;
                 clear_floating_interrupts(dev->kvm);
                 break;
+        case KVM_DEV_FLIC_APF_ENABLE:
+                dev->kvm->arch.gmap->pfault_enabled = 1;
+                break;
+        case KVM_DEV_FLIC_APF_DISABLE_WAIT:
+                dev->kvm->arch.gmap->pfault_enabled = 0;
+                /*
+                 * Make sure no async faults are in transition when
+                 * clearing the queues. So we don't need to worry
+                 * about late coming workers.
+                 */
+                synchronize_srcu(&dev->kvm->srcu);
+                kvm_for_each_vcpu(i, vcpu, dev->kvm)
+                        kvm_clear_async_pf_completion_queue(vcpu);
+                break;
         default:
                 r = -EINVAL;
         }
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 9eec794caa7f..d8e9f04977db 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -152,6 +152,7 @@ int kvm_dev_ioctl_check_extension(long ext)
 #ifdef CONFIG_KVM_S390_UCONTROL
         case KVM_CAP_S390_UCONTROL:
 #endif
+        case KVM_CAP_ASYNC_PF:
         case KVM_CAP_SYNC_REGS:
         case KVM_CAP_ONE_REG:
         case KVM_CAP_ENABLE_CAP:
@@ -273,6 +274,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
+        kvm_clear_async_pf_completion_queue(vcpu);
         if (!kvm_is_ucontrol(vcpu->kvm)) {
                 clear_bit(63 - vcpu->vcpu_id,
                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
@@ -322,6 +324,8 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 /* Section: vcpu related */
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 {
+        vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
+        kvm_clear_async_pf_completion_queue(vcpu);
         if (kvm_is_ucontrol(vcpu->kvm)) {
                 vcpu->arch.gmap = gmap_alloc(current->mm);
                 if (!vcpu->arch.gmap)
@@ -382,6 +386,8 @@ static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
         vcpu->arch.guest_fpregs.fpc = 0;
         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
         vcpu->arch.sie_block->gbea = 1;
+        vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
+        kvm_clear_async_pf_completion_queue(vcpu);
         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
 }
 
@@ -713,10 +719,89 @@ static long kvm_arch_fault_in_sync(struct kvm_vcpu *vcpu)
         return rc;
 }
 
+static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
+                                      unsigned long token)
+{
+        struct kvm_s390_interrupt inti;
+        inti.parm64 = token;
+
+        if (start_token) {
+                inti.type = KVM_S390_INT_PFAULT_INIT;
+                WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
+        } else {
+                inti.type = KVM_S390_INT_PFAULT_DONE;
+                WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
+        }
+}
+
+void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
+                                     struct kvm_async_pf *work)
+{
+        trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
+        __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
+}
+
+void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
+                                 struct kvm_async_pf *work)
+{
+        trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
+        __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
+}
+
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
+                               struct kvm_async_pf *work)
+{
+        /* s390 will always inject the page directly */
+}
+
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
+{
+        /*
+         * s390 will always inject the page directly,
+         * but we still want check_async_completion to cleanup
+         */
+        return true;
+}
+
+static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
+{
+        hva_t hva;
+        struct kvm_arch_async_pf arch;
+        int rc;
+
+        if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
+                return 0;
+        if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
+            vcpu->arch.pfault_compare)
+                return 0;
+        if (psw_extint_disabled(vcpu))
+                return 0;
+        if (kvm_cpu_has_interrupt(vcpu))
+                return 0;
+        if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
+                return 0;
+        if (!vcpu->arch.gmap->pfault_enabled)
+                return 0;
+
+        hva = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
+        if (copy_from_guest(vcpu, &arch.pfault_token, vcpu->arch.pfault_token, 8))
+                return 0;
+
+        rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
+        return rc;
+}
+
 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
 {
         int rc, cpuflags;
 
+        /*
+         * On s390 notifications for arriving pages will be delivered directly
+         * to the guest but the house keeping for completed pfaults is
+         * handled outside the worker.
+         */
+        kvm_check_async_pf_completion(vcpu);
+
         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
 
         if (need_resched())
@@ -758,8 +843,10 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
                 rc = -EREMOTE;
 
         } else if (current->thread.gmap_pfault) {
+                trace_kvm_s390_major_guest_pfault(vcpu);
                 current->thread.gmap_pfault = 0;
-                if (kvm_arch_fault_in_sync(vcpu) >= 0)
+                if (kvm_arch_setup_async_pf(vcpu) ||
+                    (kvm_arch_fault_in_sync(vcpu) >= 0))
                         rc = 0;
         }
 
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index f9559b0bd620..ed4750a5bc3c 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -159,4 +159,8 @@ void exit_sie_sync(struct kvm_vcpu *vcpu);
 /* implemented in diag.c */
 int kvm_s390_handle_diag(struct kvm_vcpu *vcpu);
 
+/* implemented in interrupt.c */
+int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
+int psw_extint_disabled(struct kvm_vcpu *vcpu);
+
 #endif
diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c
index 87c2b3a3bd3e..fe9442d39f0e 100644
--- a/arch/s390/kvm/sigp.c
+++ b/arch/s390/kvm/sigp.c
@@ -224,6 +224,8 @@ unlock:
 static int __sigp_set_arch(struct kvm_vcpu *vcpu, u32 parameter)
 {
         int rc;
+        unsigned int i;
+        struct kvm_vcpu *v;
 
         switch (parameter & 0xff) {
         case 0:
@@ -231,6 +233,11 @@ static int __sigp_set_arch(struct kvm_vcpu *vcpu, u32 parameter)
                 break;
         case 1:
         case 2:
+                kvm_for_each_vcpu(i, v, vcpu->kvm) {
+                        v->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
+                        kvm_clear_async_pf_completion_queue(v);
+                }
+
                 rc = SIGP_CC_ORDER_CODE_ACCEPTED;
                 break;
         default:
diff --git a/arch/s390/kvm/trace.h b/arch/s390/kvm/trace.h
index 3db76b2daed7..e8e7213d4cc5 100644
--- a/arch/s390/kvm/trace.h
+++ b/arch/s390/kvm/trace.h
@@ -30,6 +30,52 @@
         TP_printk("%02d[%016lx-%016lx]: " p_str, __entry->id,           \
                   __entry->pswmask, __entry->pswaddr, p_args)
 
+TRACE_EVENT(kvm_s390_major_guest_pfault,
+            TP_PROTO(VCPU_PROTO_COMMON),
+            TP_ARGS(VCPU_ARGS_COMMON),
+
+            TP_STRUCT__entry(
+                    VCPU_FIELD_COMMON
+                    ),
+
+            TP_fast_assign(
+                    VCPU_ASSIGN_COMMON
+                    ),
+            VCPU_TP_PRINTK("%s", "major fault, maybe applicable for pfault")
+        );
+
+TRACE_EVENT(kvm_s390_pfault_init,
+            TP_PROTO(VCPU_PROTO_COMMON, long pfault_token),
+            TP_ARGS(VCPU_ARGS_COMMON, pfault_token),
+
+            TP_STRUCT__entry(
+                    VCPU_FIELD_COMMON
+                    __field(long, pfault_token)
+                    ),
+
+            TP_fast_assign(
+                    VCPU_ASSIGN_COMMON
+                    __entry->pfault_token = pfault_token;
+                    ),
+            VCPU_TP_PRINTK("init pfault token %ld", __entry->pfault_token)
+        );
+
+TRACE_EVENT(kvm_s390_pfault_done,
+            TP_PROTO(VCPU_PROTO_COMMON, long pfault_token),
+            TP_ARGS(VCPU_ARGS_COMMON, pfault_token),
+
+            TP_STRUCT__entry(
+                    VCPU_FIELD_COMMON
+                    __field(long, pfault_token)
+                    ),
+
+            TP_fast_assign(
+                    VCPU_ASSIGN_COMMON
+                    __entry->pfault_token = pfault_token;
+                    ),
+            VCPU_TP_PRINTK("done pfault token %ld", __entry->pfault_token)
+        );
+
 /*
  * Tracepoints for SIE entry and exit.
  */