xen/privcmd: improve performance of MMAPBATCH_V2

Make the IOCTL_PRIVCMD_MMAPBATCH_V2 (and older V1 version) map
multiple frames at a time rather than one at a time, despite the pages
being non-consecutive GFNs.

xen_remap_foreign_mfn_array() is added which maps an array of GFNs
(instead of a consecutive range of GFNs).

Since per-frame errors are returned in an array, privcmd must set the
MMAPBATCH_V1 error bits as part of the "report errors" phase, after
all the frames are mapped.

Migrate times are significantly improved (when using a PV toolstack
domain).  For example, for an idle 12 GiB PV guest:

        Before     After
  real  0m38.179s  0m26.868s
  user  0m15.096s  0m13.652s
  sys   0m28.988s  0m18.732s

Signed-off-by: David Vrabel <david.vrabel@citrix.com>
Reviewed-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>

1 files changed, 82 insertions, 19 deletions

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 3d536a56ddf1..29b3be230ede 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -2439,7 +2439,8 @@ void __init xen_hvm_init_mmu_ops(void)
 #define REMAP_BATCH_SIZE 16
 
 struct remap_data {
-        unsigned long mfn;
+        xen_pfn_t *mfn;
+        bool contiguous;
         pgprot_t prot;
         struct mmu_update *mmu_update;
 };
@@ -2448,7 +2449,14 @@ static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
                                  unsigned long addr, void *data)
 {
         struct remap_data *rmd = data;
-        pte_t pte = pte_mkspecial(mfn_pte(rmd->mfn++, rmd->prot));
+        pte_t pte = pte_mkspecial(mfn_pte(*rmd->mfn, rmd->prot));
+
+        /* If we have a contigious range, just update the mfn itself,
+           else update pointer to be "next mfn". */
+        if (rmd->contiguous)
+                (*rmd->mfn)++;
+        else
+                rmd->mfn++;
 
         rmd->mmu_update->ptr = virt_to_machine(ptep).maddr;
         rmd->mmu_update->val = pte_val_ma(pte);
@@ -2457,26 +2465,26 @@ static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
         return 0;
 }
 
-int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
-                               unsigned long addr,
-                               xen_pfn_t mfn, int nr,
-                               pgprot_t prot, unsigned domid,
-                               struct page **pages)
-
+static int do_remap_mfn(struct vm_area_struct *vma,
+                        unsigned long addr,
+                        xen_pfn_t *mfn, int nr,
+                        int *err_ptr, pgprot_t prot,
+                        unsigned domid,
+                        struct page **pages)
 {
+        int err = 0;
         struct remap_data rmd;
         struct mmu_update mmu_update[REMAP_BATCH_SIZE];
-        int batch;
         unsigned long range;
-        int err = 0;
+        int mapped = 0;
 
         BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
 
         if (xen_feature(XENFEAT_auto_translated_physmap)) {
 #ifdef CONFIG_XEN_PVH
                 /* We need to update the local page tables and the xen HAP */
-                return xen_xlate_remap_gfn_range(vma, addr, mfn, nr, prot,
-                                                 domid, pages);
+                return xen_xlate_remap_gfn_array(vma, addr, mfn, nr, err_ptr,
+                                                 prot, domid, pages);
 #else
                 return -EINVAL;
 #endif
@@ -2484,9 +2492,15 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
 
         rmd.mfn = mfn;
         rmd.prot = prot;
+        /* We use the err_ptr to indicate if there we are doing a contigious
+         * mapping or a discontigious mapping. */
+        rmd.contiguous = !err_ptr;
 
         while (nr) {
-                batch = min(REMAP_BATCH_SIZE, nr);
+                int index = 0;
+                int done = 0;
+                int batch = min(REMAP_BATCH_SIZE, nr);
+                int batch_left = batch;
                 range = (unsigned long)batch << PAGE_SHIFT;
 
                 rmd.mmu_update = mmu_update;
@@ -2495,23 +2509,72 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
                 if (err)
                         goto out;
 
-                err = HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid);
-                if (err < 0)
-                        goto out;
+                /* We record the error for each page that gives an error, but
+                 * continue mapping until the whole set is done */
+                do {
+                        int i;
+
+                        err = HYPERVISOR_mmu_update(&mmu_update[index],
+                                                    batch_left, &done, domid);
+
+                        /*
+                         * @err_ptr may be the same buffer as @mfn, so
+                         * only clear it after each chunk of @mfn is
+                         * used.
+                         */
+                        if (err_ptr) {
+                                for (i = index; i < index + done; i++)
+                                        err_ptr[i] = 0;
+                        }
+                        if (err < 0) {
+                                if (!err_ptr)
+                                        goto out;
+                                err_ptr[i] = err;
+                                done++; /* Skip failed frame. */
+                        } else
+                                mapped += done;
+                        batch_left -= done;
+                        index += done;
+                } while (batch_left);
 
                 nr -= batch;
                 addr += range;
+                if (err_ptr)
+                        err_ptr += batch;
         }
-
-        err = 0;
 out:
 
         xen_flush_tlb_all();
 
-        return err;
+        return err < 0 ? err : mapped;
+}
+
+int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
+                               unsigned long addr,
+                               xen_pfn_t mfn, int nr,
+                               pgprot_t prot, unsigned domid,
+                               struct page **pages)
+{
+        return do_remap_mfn(vma, addr, &mfn, nr, NULL, prot, domid, pages);
 }
 EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range);
 
+int xen_remap_domain_mfn_array(struct vm_area_struct *vma,
+                               unsigned long addr,
+                               xen_pfn_t *mfn, int nr,
+                               int *err_ptr, pgprot_t prot,
+                               unsigned domid, struct page **pages)
+{
+        /* We BUG_ON because it's a programmer error to pass a NULL err_ptr,
+         * and the consequences later is quite hard to detect what the actual
+         * cause of "wrong memory was mapped in".
+         */
+        BUG_ON(err_ptr == NULL);
+        return do_remap_mfn(vma, addr, mfn, nr, err_ptr, prot, domid, pages);
+}
+EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array);
+
+
 /* Returns: 0 success */
 int xen_unmap_domain_mfn_range(struct vm_area_struct *vma,
                                int numpgs, struct page **pages)