vfio/iommu_type1: Multi-IOMMU domain support

We currently have a problem that we cannot support advanced features
of an IOMMU domain (ex. IOMMU_CACHE), because we have no guarantee
that those features will be supported by all of the hardware units
involved with the domain over its lifetime.  For instance, the Intel
VT-d architecture does not require that all DRHDs support snoop
control.  If we create a domain based on a device behind a DRHD that
does support snoop control and enable SNP support via the IOMMU_CACHE
mapping option, we cannot then add a device behind a DRHD which does
not support snoop control or we'll get reserved bit faults from the
SNP bit in the pagetables.  To add to the complexity, we can't know
the properties of a domain until a device is attached.

We could pass this problem off to userspace and require that a
separate vfio container be used, but we don't know how to handle page
accounting in that case.  How do we know that a page pinned in one
container is the same page as a different container and avoid double
billing the user for the page.

The solution is therefore to support multiple IOMMU domains per
container.  In the majority of cases, only one domain will be required
since hardware is typically consistent within a system.  However, this
provides us the ability to validate compatibility of domains and
support mixed environments where page table flags can be different
between domains.

To do this, our DMA tracking needs to change.  We currently try to
coalesce user mappings into as few tracking entries as possible.  The
problem then becomes that we lose granularity of user mappings.  We've
never guaranteed that a user is able to unmap at a finer granularity
than the original mapping, but we must honor the granularity of the
original mapping.  This coalescing code is therefore removed, allowing
only unmaps covering complete maps.  The change in accounting is
fairly small here, a typical QEMU VM will start out with roughly a
dozen entries, so it's arguable if this coalescing was ever needed.

We also move IOMMU domain creation to the point where a group is
attached to the container.  An interesting side-effect of this is that
we now have access to the device at the time of domain creation and
can probe the devices within the group to determine the bus_type.
This finally makes vfio_iommu_type1 completely device/bus agnostic.
In fact, each IOMMU domain can host devices on different buses managed
by different physical IOMMUs, and present a single DMA mapping
interface to the user.  When a new domain is created, mappings are
replayed to bring the IOMMU pagetables up to the state of the current
container.  And of course, DMA mapping and unmapping automatically
traverse all of the configured IOMMU domains.

Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
Cc: Varun Sethi <Varun.Sethi@freescale.com>

1 files changed, 335 insertions, 302 deletions

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 4fb7a8f83c8a..8c7bb9befdab 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -30,7 +30,6 @@
 #include <linux/iommu.h>
 #include <linux/module.h>
 #include <linux/mm.h>
-#include <linux/pci.h>          /* pci_bus_type */
 #include <linux/rbtree.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
@@ -55,11 +54,17 @@ MODULE_PARM_DESC(disable_hugepages,
                  "Disable VFIO IOMMU support for IOMMU hugepages.");
 
 struct vfio_iommu {
-        struct iommu_domain     *domain;
+        struct list_head        domain_list;
         struct mutex            lock;
         struct rb_root          dma_list;
+        bool v2;
+};
+
+struct vfio_domain {
+        struct iommu_domain     *domain;
+        struct list_head        next;
         struct list_head        group_list;
-        bool                    cache;
+        int                     prot;           /* IOMMU_CACHE */
 };
 
 struct vfio_dma {
@@ -99,7 +104,7 @@ static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,
         return NULL;
 }
 
-static void vfio_insert_dma(struct vfio_iommu *iommu, struct vfio_dma *new)
+static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new)
 {
         struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL;
         struct vfio_dma *dma;
@@ -118,7 +123,7 @@ static void vfio_insert_dma(struct vfio_iommu *iommu, struct vfio_dma *new)
         rb_insert_color(&new->node, &iommu->dma_list);
 }
 
-static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
+static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
 {
         rb_erase(&old->node, &iommu->dma_list);
 }
@@ -322,32 +327,39 @@ static long vfio_unpin_pages(unsigned long pfn, long npage,
         return unlocked;
 }
 
-static int vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
-                            dma_addr_t iova, size_t *size)
+static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
 {
-        dma_addr_t start = iova, end = iova + *size;
+        dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
+        struct vfio_domain *domain, *d;
         long unlocked = 0;
 
+        if (!dma->size)
+                return;
+        /*
+         * We use the IOMMU to track the physical addresses, otherwise we'd
+         * need a much more complicated tracking system.  Unfortunately that
+         * means we need to use one of the iommu domains to figure out the
+         * pfns to unpin.  The rest need to be unmapped in advance so we have
+         * no iommu translations remaining when the pages are unpinned.
+         */
+        domain = d = list_first_entry(&iommu->domain_list,
+                                      struct vfio_domain, next);
+
+        list_for_each_entry_continue(d, &iommu->domain_list, next)
+                iommu_unmap(d->domain, dma->iova, dma->size);
+
         while (iova < end) {
                 size_t unmapped;
                 phys_addr_t phys;
 
-                /*
-                 * We use the IOMMU to track the physical address.  This
-                 * saves us from having a lot more entries in our mapping
-                 * tree.  The downside is that we don't track the size
-                 * used to do the mapping.  We request unmap of a single
-                 * page, but expect IOMMUs that support large pages to
-                 * unmap a larger chunk.
-                 */
-                phys = iommu_iova_to_phys(iommu->domain, iova);
+                phys = iommu_iova_to_phys(domain->domain, iova);
                 if (WARN_ON(!phys)) {
                         iova += PAGE_SIZE;
                         continue;
                 }
 
-                unmapped = iommu_unmap(iommu->domain, iova, PAGE_SIZE);
-                if (!unmapped)
+                unmapped = iommu_unmap(domain->domain, iova, PAGE_SIZE);
+                if (WARN_ON(!unmapped))
                         break;
 
                 unlocked += vfio_unpin_pages(phys >> PAGE_SHIFT,
@@ -357,119 +369,26 @@ static int vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
         }
 
         vfio_lock_acct(-unlocked);
-
-        *size = iova - start;
-
-        return 0;
 }
 
-static int vfio_remove_dma_overlap(struct vfio_iommu *iommu, dma_addr_t start,
-                                   size_t *size, struct vfio_dma *dma)
+static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
 {
-        size_t offset, overlap, tmp;
-        struct vfio_dma *split;
-        int ret;
-
-        if (!*size)
-                return 0;
-
-        /*
-         * Existing dma region is completely covered, unmap all.  This is
-         * the likely case since userspace tends to map and unmap buffers
-         * in one shot rather than multiple mappings within a buffer.
-         */
-        if (likely(start <= dma->iova &&
-                   start + *size >= dma->iova + dma->size)) {
-                *size = dma->size;
-                ret = vfio_unmap_unpin(iommu, dma, dma->iova, size);
-                if (ret)
-                        return ret;
-
-                /*
-                 * Did we remove more than we have?  Should never happen
-                 * since a vfio_dma is contiguous in iova and vaddr.
-                 */
-                WARN_ON(*size != dma->size);
-
-                vfio_remove_dma(iommu, dma);
-                kfree(dma);
-                return 0;
-        }
-
-        /* Overlap low address of existing range */
-        if (start <= dma->iova) {
-                overlap = start + *size - dma->iova;
-                ret = vfio_unmap_unpin(iommu, dma, dma->iova, &overlap);
-                if (ret)
-                        return ret;
-
-                vfio_remove_dma(iommu, dma);
-
-                /*
+        vfio_unmap_unpin(iommu, dma);
+        vfio_unlink_dma(iommu, dma);
+        kfree(dma);
+}