vfio: hugepage support for vfio_iommu_type1

We currently send all mappings to the iommu in PAGE_SIZE chunks,
which prevents the iommu from enabling support for larger page sizes.
We still need to pin pages, which means we step through them in
PAGE_SIZE chunks, but we can batch up contiguous physical memory
chunks to allow the iommu the opportunity to use larger pages.  The
approach here is a bit different that the one currently used for
legacy KVM device assignment.  Rather than looking at the vma page
size and using that as the maximum size to pass to the iommu, we
instead simply look at whether the next page is physically
contiguous.  This means we might ask the iommu to map a 4MB region,
while legacy KVM might limit itself to a maximum of 2MB.

Splitting our mapping path also allows us to be smarter about locked
memory because we can more easily unwind if the user attempts to
exceed the limit.  Therefore, rather than assuming that a mapping
will result in locked memory, we test each page as it is pinned to
determine whether it locks RAM vs an mmap'd MMIO region.  This should
result in better locking granularity and less locked page fudge
factors in userspace.

The unmap path uses the same algorithm as legacy KVM.  We don't want
to track the pfn for each mapping ourselves, but we need the pfn in
order to unpin pages.  We therefore ask the iommu for the iova to
physical address translation, ask it to unpin a page, and see how many
pages were actually unpinned.  iommus supporting large pages will
often return something bigger than a page here, which we know will be
physically contiguous and we can unpin a batch of pfns.  iommus not
supporting large mappings won't see an improvement in batching here as
they only unmap a page at a time.

With this change, we also make a clarification to the API for mapping
and unmapping DMA.  We can only guarantee unmaps at the same
granularity as used for the original mapping.  In other words,
unmapping a subregion of a previous mapping is not guaranteed and may
result in a larger or smaller unmapping than requested.  The size
field in the unmapping structure is updated to reflect this.
Previously this was unmodified on mapping, always returning the the
requested unmap size.  This is now updated to return the actual unmap
size on success, allowing userspace to appropriately track mappings.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>

2 files changed, 344 insertions, 187 deletions

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 0e863b3ddcab..6654a7eb42d3 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -60,7 +60,7 @@ struct vfio_dma {
         struct rb_node          node;
         dma_addr_t              iova;           /* Device address */
         unsigned long           vaddr;          /* Process virtual addr */
-        long                    npage;          /* Number of pages */
+        size_t                  size;           /* Map size (bytes) */
         int                     prot;           /* IOMMU_READ/WRITE */
 };
 
@@ -74,8 +74,6 @@ struct vfio_group {
  * into DMA'ble space using the IOMMU
  */
 
-#define NPAGE_TO_SIZE(npage)    ((size_t)(npage) << PAGE_SHIFT)
-
 static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,
                                       dma_addr_t start, size_t size)
 {
@@ -86,7 +84,7 @@ static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,
 
                 if (start + size <= dma->iova)
                         node = node->rb_left;
-                else if (start >= dma->iova + NPAGE_TO_SIZE(dma->npage))
+                else if (start >= dma->iova + dma->size)
                         node = node->rb_right;
                 else
                         return dma;
@@ -104,7 +102,7 @@ static void vfio_insert_dma(struct vfio_iommu *iommu, struct vfio_dma *new)
                 parent = *link;
                 dma = rb_entry(parent, struct vfio_dma, node);
 
-                if (new->iova + NPAGE_TO_SIZE(new->npage) <= dma->iova)
+                if (new->iova + new->size <= dma->iova)
                         link = &(*link)->rb_left;
                 else
                         link = &(*link)->rb_right;
@@ -144,8 +142,8 @@ static void vfio_lock_acct(long npage)
         struct vwork *vwork;
         struct mm_struct *mm;
 
-        if (!current->mm)
-                return; /* process exited */
+        if (!current->mm || !npage)
+                return; /* process exited or nothing to do */
 
         if (down_write_trylock(&current->mm->mmap_sem)) {
                 current->mm->locked_vm += npage;
@@ -217,33 +215,6 @@ static int put_pfn(unsigned long pfn, int prot)
         return 0;
 }
 
-/* Unmap DMA region */
-static long __vfio_dma_do_unmap(struct vfio_iommu *iommu, dma_addr_t iova,
-                             long npage, int prot)
-{
-        long i, unlocked = 0;
-
-        for (i = 0; i < npage; i++, iova += PAGE_SIZE) {
-                unsigned long pfn;
-
-                pfn = iommu_iova_to_phys(iommu->domain, iova) >> PAGE_SHIFT;
-                if (pfn) {
-                        iommu_unmap(iommu->domain, iova, PAGE_SIZE);
-                        unlocked += put_pfn(pfn, prot);
-                }
-        }
-        return unlocked;
-}
-
-static void vfio_dma_unmap(struct vfio_iommu *iommu, dma_addr_t iova,
-                           long npage, int prot)
-{
-        long unlocked;
-
-        unlocked = __vfio_dma_do_unmap(iommu, iova, npage, prot);
-        vfio_lock_acct(-unlocked);
-}
-
 static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
 {
         struct page *page[1];
@@ -270,79 +241,142 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
         return ret;
 }
 
-/* Map DMA region */
-static int __vfio_dma_map(struct vfio_iommu *iommu, dma_addr_t iova,
-                          unsigned long vaddr, long npage, int prot)
+/*
+ * Attempt to pin pages.  We really don't want to track all the pfns and
+ * the iommu can only map chunks of consecutive pfns anyway, so get the
+ * first page and all consecutive pages with the same locking.
+ */
+static long vfio_pin_pages(unsigned long vaddr, long npage,
+                           int prot, unsigned long *pfn_base)
 {
-        dma_addr_t start = iova;
-        long i, locked = 0;
-        int ret;
+        unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+        bool lock_cap = capable(CAP_IPC_LOCK);
+        long ret, i;
 
-        /* Verify that pages are not already mapped */
-        for (i = 0; i < npage; i++, iova += PAGE_SIZE)
-                if (iommu_iova_to_phys(iommu->domain, iova))
-                        return -EBUSY;
+        if (!current->mm)
+                return -ENODEV;
 
-        iova = start;
+        ret = vaddr_get_pfn(vaddr, prot, pfn_base);
+        if (ret)
+                return ret;
 
-        if (iommu->cache)
-                prot |= IOMMU_CACHE;
+        if (is_invalid_reserved_pfn(*pfn_base))
+                return 1;
 
-        /*
-         * XXX We break mappings into pages and use get_user_pages_fast to
-         * pin the pages in memory.  It's been suggested that mlock might
-         * provide a more efficient mechanism, but nothing prevents the
-         * user from munlocking the pages, which could then allow the user
-         * access to random host memory.  We also have no guarantee from the
-         * IOMMU API that the iommu driver can unmap sub-pages of previous
-         * mappings.  This means we might lose an entire range if a single
-         * page within it is unmapped.  Single page mappings are inefficient,
-         * but provide the most flexibility for now.
-         */
-        for (i = 0; i < npage; i++, iova += PAGE_SIZE, vaddr += PAGE_SIZE) {
+        if (!lock_cap && current->mm->locked_vm + 1 > limit) {
+                put_pfn(*pfn_base, prot);
+                pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
+                        limit << PAGE_SHIFT);
+                return -ENOMEM;
+        }
+
+        /* Lock all the consecutive pages from pfn_base */
+        for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
                 unsigned long pfn = 0;
 
                 ret = vaddr_get_pfn(vaddr, prot, &pfn);
-                if (ret) {
-                        __vfio_dma_do_unmap(iommu, start, i, prot);
-                        return ret;
+                if (ret)
+                        break;
+
+                if (pfn != *pfn_base + i || is_invalid_reserved_pfn(pfn)) {
+                        put_pfn(pfn, prot);
+                        break;
                 }
 
+                if (!lock_cap && current->mm->locked_vm + i + 1 > limit) {
+                        put_pfn(pfn, prot);
+                        pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
+                                __func__, limit << PAGE_SHIFT);
+                        break;
+                }
+        }
+