5 files changed, 359 insertions, 118 deletions

diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index 779262f59e25..fbe1a48bd629 100644
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -27,3 +27,6 @@ config HAVE_KVM_MSI
 
 config HAVE_KVM_CPU_RELAX_INTERCEPT
        bool
+
+config KVM_VFIO
+       bool
diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c
index 8a39dda7a325..8631d9c14320 100644
--- a/virt/kvm/async_pf.c
+++ b/virt/kvm/async_pf.c
@@ -56,7 +56,6 @@ void kvm_async_pf_vcpu_init(struct kvm_vcpu *vcpu)
 
 static void async_pf_execute(struct work_struct *work)
 {
-        struct page *page = NULL;
         struct kvm_async_pf *apf =
                 container_of(work, struct kvm_async_pf, work);
         struct mm_struct *mm = apf->mm;
@@ -68,14 +67,12 @@ static void async_pf_execute(struct work_struct *work)
 
         use_mm(mm);
         down_read(&mm->mmap_sem);
-        get_user_pages(current, mm, addr, 1, 1, 0, &page, NULL);
+        get_user_pages(current, mm, addr, 1, 1, 0, NULL, NULL);
         up_read(&mm->mmap_sem);
         unuse_mm(mm);
 
         spin_lock(&vcpu->async_pf.lock);
         list_add_tail(&apf->link, &vcpu->async_pf.done);
-        apf->page = page;
-        apf->done = true;
         spin_unlock(&vcpu->async_pf.lock);
 
         /*
@@ -83,7 +80,7 @@ static void async_pf_execute(struct work_struct *work)
          * this point
          */
 
-        trace_kvm_async_pf_completed(addr, page, gva);
+        trace_kvm_async_pf_completed(addr, gva);
 
         if (waitqueue_active(&vcpu->wq))
                 wake_up_interruptible(&vcpu->wq);
@@ -99,9 +96,8 @@ void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu)
                 struct kvm_async_pf *work =
                         list_entry(vcpu->async_pf.queue.next,
                                    typeof(*work), queue);
-                cancel_work_sync(&work->work);
                 list_del(&work->queue);
-                if (!work->done) { /* work was canceled */
+                if (cancel_work_sync(&work->work)) {
                         mmdrop(work->mm);
                         kvm_put_kvm(vcpu->kvm); /* == work->vcpu->kvm */
                         kmem_cache_free(async_pf_cache, work);
@@ -114,8 +110,6 @@ void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu)
                         list_entry(vcpu->async_pf.done.next,
                                    typeof(*work), link);
                 list_del(&work->link);
-                if (!is_error_page(work->page))
-                        kvm_release_page_clean(work->page);
                 kmem_cache_free(async_pf_cache, work);
         }
         spin_unlock(&vcpu->async_pf.lock);
@@ -135,14 +129,11 @@ void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu)
                 list_del(&work->link);
                 spin_unlock(&vcpu->async_pf.lock);
 
-                if (work->page)
-                        kvm_arch_async_page_ready(vcpu, work);
+                kvm_arch_async_page_ready(vcpu, work);
                 kvm_arch_async_page_present(vcpu, work);
 
                 list_del(&work->queue);
                 vcpu->async_pf.queued--;
-                if (!is_error_page(work->page))
-                        kvm_release_page_clean(work->page);
                 kmem_cache_free(async_pf_cache, work);
         }
 }
@@ -165,8 +156,7 @@ int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
         if (!work)
                 return 0;
 
-        work->page = NULL;
-        work->done = false;
+        work->wakeup_all = false;
         work->vcpu = vcpu;
         work->gva = gva;
         work->addr = gfn_to_hva(vcpu->kvm, gfn);
@@ -206,7 +196,7 @@ int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu)
         if (!work)
                 return -ENOMEM;
 
-        work->page = KVM_ERR_PTR_BAD_PAGE;
+        work->wakeup_all = true;
         INIT_LIST_HEAD(&work->queue); /* for list_del to work */
 
         spin_lock(&vcpu->async_pf.lock);
diff --git a/virt/kvm/iommu.c b/virt/kvm/iommu.c
index 72a130bc448a..0df7d4b34dfe 100644
--- a/virt/kvm/iommu.c
+++ b/virt/kvm/iommu.c
@@ -79,7 +79,7 @@ int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
         flags = IOMMU_READ;
         if (!(slot->flags & KVM_MEM_READONLY))
                 flags |= IOMMU_WRITE;
-        if (kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY)
+        if (!kvm->arch.iommu_noncoherent)
                 flags |= IOMMU_CACHE;
 
 
@@ -103,6 +103,10 @@ int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
                 while ((gfn << PAGE_SHIFT) & (page_size - 1))
                         page_size >>= 1;
 
+                /* Make sure hva is aligned to the page size we want to map */
+                while (__gfn_to_hva_memslot(slot, gfn) & (page_size - 1))
+                        page_size >>= 1;
+
                 /*
                  * Pin all pages we are about to map in memory. This is
                  * important because we unmap and unpin in 4kb steps later.
@@ -140,6 +144,9 @@ static int kvm_iommu_map_memslots(struct kvm *kvm)
         struct kvm_memslots *slots;
         struct kvm_memory_slot *memslot;
 
+        if (kvm->arch.iommu_noncoherent)
+                kvm_arch_register_noncoherent_dma(kvm);
+
         idx = srcu_read_lock(&kvm->srcu);
         slots = kvm_memslots(kvm);
 
@@ -158,7 +165,8 @@ int kvm_assign_device(struct kvm *kvm,
 {
         struct pci_dev *pdev = NULL;
         struct iommu_domain *domain = kvm->arch.iommu_domain;
-        int r, last_flags;
+        int r;
+        bool noncoherent;
 
         /* check if iommu exists and in use */
         if (!domain)
@@ -174,15 +182,13 @@ int kvm_assign_device(struct kvm *kvm,
                 return r;
         }
 
-        last_flags = kvm->arch.iommu_flags;
-        if (iommu_domain_has_cap(kvm->arch.iommu_domain,
-                                 IOMMU_CAP_CACHE_COHERENCY))
-                kvm->arch.iommu_flags |= KVM_IOMMU_CACHE_COHERENCY;
+        noncoherent = !iommu_domain_has_cap(kvm->arch.iommu_domain,
+                                            IOMMU_CAP_CACHE_COHERENCY);
 
         /* Check if need to update IOMMU page table for guest memory */
-        if ((last_flags ^ kvm->arch.iommu_flags) ==
-                        KVM_IOMMU_CACHE_COHERENCY) {
+        if (noncoherent != kvm->arch.iommu_noncoherent) {
                 kvm_iommu_unmap_memslots(kvm);
+                kvm->arch.iommu_noncoherent = noncoherent;
                 r = kvm_iommu_map_memslots(kvm);
                 if (r)
                         goto out_unmap;
@@ -190,11 +196,7 @@ int kvm_assign_device(struct kvm *kvm,
 
         pdev->dev_flags |= PCI_DEV_FLAGS_ASSIGNED;
 
-        printk(KERN_DEBUG "assign device %x:%x:%x.%x\n",
-                assigned_dev->host_segnr,
-                assigned_dev->host_busnr,
-                PCI_SLOT(assigned_dev->host_devfn),
-                PCI_FUNC(assigned_dev->host_devfn));
+        dev_info(&pdev->dev, "kvm assign device\n");
 
         return 0;
 out_unmap:
@@ -220,11 +222,7 @@ int kvm_deassign_device(struct kvm *kvm,
 
         pdev->dev_flags &= ~PCI_DEV_FLAGS_ASSIGNED;
 
-        printk(KERN_DEBUG "deassign device %x:%x:%x.%x\n",
-                assigned_dev->host_segnr,
-                assigned_dev->host_busnr,
-                PCI_SLOT(assigned_dev->host_devfn),
-                PCI_FUNC(assigned_dev->host_devfn));
+        dev_info(&pdev->dev, "kvm deassign device\n");
 
         return 0;
 }
@@ -336,6 +334,9 @@ static int kvm_iommu_unmap_memslots(struct kvm *kvm)
 
         srcu_read_unlock(&kvm->srcu, idx);
 
+        if (kvm->arch.iommu_noncoherent)
+                kvm_arch_unregister_noncoherent_dma(kvm);
+
         return 0;
 }
 
@@ -350,6 +351,7 @@ int kvm_iommu_unmap_guest(struct kvm *kvm)
         mutex_lock(&kvm->slots_lock);
         kvm_iommu_unmap_memslots(kvm);
         kvm->arch.iommu_domain = NULL;
+        kvm->arch.iommu_noncoherent = false;
         mutex_unlock(&kvm->slots_lock);
 
         iommu_domain_free(domain);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 979bff485fb0..4f588bc94186 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -70,7 +70,8 @@ MODULE_LICENSE("GPL");
  *              kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  */
 
-DEFINE_RAW_SPINLOCK(kvm_lock);
+DEFINE_SPINLOCK(kvm_lock);
+static DEFINE_RAW_SPINLOCK(kvm_count_lock);
 LIST_HEAD(vm_list);
 
 static cpumask_var_t cpus_hardware_enabled;
@@ -186,6 +187,7 @@ void kvm_flush_remote_tlbs(struct kvm *kvm)
                 ++kvm->stat.remote_tlb_flush;
         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
 }
+EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
 
 void kvm_reload_remote_mmus(struct kvm *kvm)
 {
@@ -490,9 +492,9 @@ static struct kvm *kvm_create_vm(unsigned long type)
         if (r)
                 goto out_err;
 
-        raw_spin_lock(&kvm_lock);
+        spin_lock(&kvm_lock);
         list_add(&kvm->vm_list, &vm_list);
-        raw_spin_unlock(&kvm_lock);
+        spin_unlock(&kvm_lock);
 
         return kvm;
 
@@ -540,13 +542,13 @@ static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 /*
  * Free any memory in @free but not in @dont.
  */
-static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
+static void kvm_free_physmem_slot(struct kvm *kvm, struct kvm_memory_slot *free,
                                   struct kvm_memory_slot *dont)
 {
         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
                 kvm_destroy_dirty_bitmap(free);
 
-        kvm_arch_free_memslot(free, dont);
+        kvm_arch_free_memslot(kvm, free, dont);
 
         free->npages = 0;
 }
@@ -557,7 +559,7 @@ void kvm_free_physmem(struct kvm *kvm)
         struct kvm_memory_slot *memslot;
 
         kvm_for_each_memslot(memslot, slots)
-                kvm_free_physmem_slot(memslot, NULL);
+                kvm_free_physmem_slot(kvm, memslot, NULL);
 
         kfree(kvm->memslots);
 }
@@ -581,9 +583,9 @@ static void kvm_destroy_vm(struct kvm *kvm)
         struct mm_struct *mm = kvm->mm;
 
         kvm_arch_sync_events(kvm);
-        raw_spin_lock(&kvm_lock);
+        spin_lock(&kvm_lock);
         list_del(&kvm->vm_list);
-        raw_spin_unlock(&kvm_lock);
+        spin_unlock(&kvm_lock);
         kvm_free_irq_routing(kvm);
         for (i = 0; i < KVM_NR_BUSES; i++)
                 kvm_io_bus_destroy(kvm->buses[i]);
@@ -821,7 +823,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
         if (change == KVM_MR_CREATE) {
                 new.userspace_addr = mem->userspace_addr;
 
-                if (kvm_arch_create_memslot(&new, npages))
+                if (kvm_arch_create_memslot(kvm, &new, npages))
                         goto out_free;
         }
 
@@ -872,6 +874,19 @@ int __kvm_set_memory_region(struct kvm *kvm,
                         goto out_free;
         }
 
+        /* actual memory is freed via old in kvm_free_physmem_slot below */
+        if (change == KVM_MR_DELETE) {
+                new.dirty_bitmap = NULL;
+                memset(&new.arch, 0, sizeof(new.arch));
+        }
+
+        old_memslots = install_new_memslots(kvm, slots, &new);
+
+        kvm_arch_commit_memory_region(kvm, mem, &old, change);
+
+        kvm_free_physmem_slot(kvm, &old, &new);
+        kfree(old_memslots);
+
         /*
          * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
          * un-mapped and re-mapped if their base changes.  Since base change
@@ -883,29 +898,15 @@ int __kvm_set_memory_region(struct kvm *kvm,
          */
         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
                 r = kvm_iommu_map_pages(kvm, &new);
-                if (r)
-                        goto out_slots;
-        }
-
-        /* actual memory is freed via old in kvm_free_physmem_slot below */
-        if (change == KVM_MR_DELETE) {
-                new.dirty_bitmap = NULL;
-                memset(&new.arch, 0, sizeof(new.arch));
+                return r;
         }
 
-        old_memslots = install_new_memslots(kvm, slots, &new);
-
-        kvm_arch_commit_memory_region(kvm, mem, &old, change);
-
-        kvm_free_physmem_slot(&old, &new);
-        kfree(old_memslots);
-
         return 0;
 
 out_slots:
         kfree(slots);
 out_free:
-        kvm_free_physmem_slot(&new, &old);
+        kvm_free_physmem_slot(kvm, &new, &old);
 out:
         return r;
 }
@@ -964,6 +965,7 @@ int kvm_get_dirty_log(struct kvm *kvm,
 out:
         return r;
 }
+EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
 
 bool kvm_largepages_enabled(void)
 {
@@ -1064,10 +1066,12 @@ EXPORT_SYMBOL_GPL(gfn_to_hva);
 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
 {
         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
-        if (writable)
+        unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
+
+        if (!kvm_is_error_hva(hva) && writable)
                 *writable = !memslot_is_readonly(slot);
 
-        return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
+        return hva;
 }
 
 static int kvm_read_hva(void *data, void __user *hva, int len)
@@ -1611,8 +1615,9 @@ EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
 
 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
 {
-        return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
-                                    offset, len);
+        const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
+
+        return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
 }
 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
 
@@ -1652,6 +1657,7 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
         memslot = gfn_to_memslot(kvm, gfn);
         mark_page_dirty_in_slot(kvm, memslot, gfn);
 }
+EXPORT_SYMBOL_GPL(mark_page_dirty);
 
 /*
  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
@@ -1677,6 +1683,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 
         finish_wait(&vcpu->wq, &wait);
 }
+EXPORT_SYMBOL_GPL(kvm_vcpu_block);
 
 #ifndef CONFIG_S390
 /*
@@ -1891,6 +1898,9 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
         int r;
         struct kvm_vcpu *vcpu, *v;
 
+        if (id >= KVM_MAX_VCPUS)
+                return -EINVAL;
+
         vcpu = kvm_arch_vcpu_create(kvm, id);
         if (IS_ERR(vcpu))
                 return PTR_ERR(vcpu);
@@ -2269,6 +2279,11 @@ static int kvm_ioctl_create_device(struct kvm *kvm,
                 ops = &kvm_xics_ops;
                 break;
 #endif
+#ifdef CONFIG_KVM_VFIO
+        case KVM_DEV_TYPE_VFIO:
+                ops = &kvm_vfio_ops;
+                break;
+#endif
         default:
                 return -ENODEV;
         }
@@ -2517,44 +2532,12 @@ out:
 }
 #endif
 
-static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
-{
-        struct page *page[1];
-        unsigned long addr;
-        int npages;
-        gfn_t gfn = vmf->pgoff;
-        struct kvm *kvm = vma->vm_file->private_data;
-
-        addr = gfn_to_hva(kvm, gfn);
-        if (kvm_is_error_hva(addr))
-                return VM_FAULT_SIGBUS;
-
-        npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
-                                NULL);
-        if (unlikely(npages != 1))
-                return VM_FAULT_SIGBUS;
-
-        vmf->page = page[0];
-        return 0;
-}
-
-static const struct vm_operations_struct kvm_vm_vm_ops = {
-        .fault = kvm_vm_fault,
-};
-
-static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
-{
-        vma->vm_ops = &kvm_vm_vm_ops;
-        return 0;
-}
-
 static struct file_operations kvm_vm_fops = {
         .release        = kvm_vm_release,
         .unlocked_ioctl = kvm_vm_ioctl,
 #ifdef CONFIG_COMPAT
         .compat_ioctl   = kvm_vm_compat_ioctl,
 #endif
-        .mmap           = kvm_vm_mmap,
         .llseek         = noop_llseek,
 };
 
@@ -2681,11 +2664,12 @@ static void hardware_enable_nolock(void *junk)
         }
 }
 
-static void hardware_enable(void *junk)
+static void hardware_enable(void)
 {
-        raw_spin_lock(&kvm_lock);
-        hardware_enable_nolock(junk);
-        raw_spin_unlock(&kvm_lock);
+        raw_spin_lock(&kvm_count_lock);
+        if (kvm_usage_count)
+                hardware_enable_nolock(NULL);
+        raw_spin_unlock(&kvm_count_lock);
 }
 
 static void hardware_disable_nolock(void *junk)
@@ -2698,11 +2682,12 @@ static void hardware_disable_nolock(void *junk)
         kvm_arch_hardware_disable(NULL);
 }
 
-static void hardware_disable(void *junk)
+static void hardware_disable(void)
 {
-        raw_spin_lock(&kvm_lock);
-        hardware_disable_nolock(junk);
-        raw_spin_unlock(&kvm_lock);
+        raw_spin_lock(&kvm_count_lock);
+        if (kvm_usage_count)
+                hardware_disable_nolock(NULL);
+        raw_spin_unlock(&kvm_count_lock);
 }
 
 static void hardware_disable_all_nolock(void)
@@ -2716,16 +2701,16 @@ static void hardware_disable_all_nolock(void)
 
 static void hardware_disable_all(void)
 {
-        raw_spin_lock(&kvm_lock);
+        raw_spin_lock(&kvm_count_lock);
         hardware_disable_all_nolock();
-        raw_spin_unlock(&kvm_lock);
+        raw_spin_unlock(&kvm_count_lock);
 }
 
 static int hardware_enable_all(void)
 {
         int r = 0;
 
-        raw_spin_lock(&kvm_lock);
+        raw_spin_lock(&kvm_count_lock);
 
         kvm_usage_count++;
         if (kvm_usage_count == 1) {
@@ -2738,7 +2723,7 @@ static int hardware_enable_all(void)
                 }
         }
 
-        raw_spin_unlock(&kvm_lock);
+        raw_spin_unlock(&kvm_count_lock);
 
         return r;
 }
@@ -2748,20 +2733,17 @@ static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
 {
         int cpu = (long)v;
 
-        if (!kvm_usage_count)
-                return NOTIFY_OK;
-
         val &= ~CPU_TASKS_FROZEN;
         switch (val) {
         case CPU_DYING:
                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
                        cpu);
-                hardware_disable(NULL);
+                hardware_disable();
                 break;
         case CPU_STARTING:
                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
                        cpu);
-                hardware_enable(NULL);
+                hardware_enable();
                 break;
         }
         return NOTIFY_OK;
@@ -3054,10 +3036,10 @@ static int vm_stat_get(void *_offset, u64 *val)
         struct kvm *kvm;
 
         *val = 0;
-        raw_spin_lock(&kvm_lock);
+        spin_lock(&kvm_lock);
         list_for_each_entry(kvm, &vm_list, vm_list)
                 *val += *(u32 *)((void *)kvm + offset);
-        raw_spin_unlock(&kvm_lock);
+        spin_unlock(&kvm_lock);
         return 0;
 }
 
@@ -3071,12 +3053,12 @@ static int vcpu_stat_get(void *_offset, u64 *val)
         int i;
 
         *val = 0;
-        raw_spin_lock(&kvm_lock);
+        spin_lock(&kvm_lock);
         list_for_each_entry(kvm, &vm_list, vm_list)
                 kvm_for_each_vcpu(i, vcpu, kvm)
                         *val += *(u32 *)((void *)vcpu + offset);
 
-        raw_spin_unlock(&kvm_lock);
+        spin_unlock(&kvm_lock);
         return 0;
 }
 
@@ -3089,7 +3071,7 @@ static const struct file_operations *stat_fops[] = {
 
 static int kvm_init_debug(void)
 {
-        int r = -EFAULT;
+        int r = -EEXIST;
         struct kvm_stats_debugfs_item *p;
 
         kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
@@ -3131,7 +3113,7 @@ static int kvm_suspend(void)
 static void kvm_resume(void)
 {
         if (kvm_usage_count) {
-                WARN_ON(raw_spin_is_locked(&kvm_lock));
+                WARN_ON(raw_spin_is_locked(&kvm_count_lock));
                 hardware_enable_nolock(NULL);
         }
 }
diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c
new file mode 100644
index 000000000000..ca4260e35037
--- /dev/null
+++ b/virt/kvm/vfio.c
@@ -0,0 +1,264 @@
+/*
+ * VFIO-KVM bridge pseudo device
+ *
+ * Copyright (C) 2013 Red Hat, Inc.  All rights reserved.
+ *     Author: Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/errno.h>
+#include <linux/file.h>
+#include <linux/kvm_host.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/vfio.h>
+
+struct kvm_vfio_group {
+        struct list_head node;
+        struct vfio_group *vfio_group;
+};
+
+struct kvm_vfio {
+        struct list_head group_list;
+        struct mutex lock;
+        bool noncoherent;
+};
+
+static struct vfio_group *kvm_vfio_group_get_external_user(struct file *filep)
+{
+        struct vfio_group *vfio_group;
+        struct vfio_group *(*fn)(struct file *);
+
+        fn = symbol_get(vfio_group_get_external_user);
+        if (!fn)
+                return ERR_PTR(-EINVAL);
+
+        vfio_group = fn(filep);
+
+        symbol_put(vfio_group_get_external_user);
+
+        return vfio_group;
+}
+
+static void kvm_vfio_group_put_external_user(struct vfio_group *vfio_group)
+{
+        void (*fn)(struct vfio_group *);
+
+        fn = symbol_get(vfio_group_put_external_user);
+        if (!fn)
+                return;
+
+        fn(vfio_group);
+
+        symbol_put(vfio_group_put_external_user);
+}
+
+/*
+ * Groups can use the same or different IOMMU domains.  If the same then
+ * adding a new group may change the coherency of groups we've previously
+ * been told about.  We don't want to care about any of that so we retest
+ * each group and bail as soon as we find one that's noncoherent.  This
+ * means we only ever [un]register_noncoherent_dma once for the whole device.
+ */
+static void kvm_vfio_update_coherency(struct kvm_device *dev)
+{
+        struct kvm_vfio *kv = dev->private;
+        bool noncoherent = false;
+        struct kvm_vfio_group *kvg;
+
+        mutex_lock(&kv->lock);
+
+        list_for_each_entry(kvg, &kv->group_list, node) {
+                /*
+                 * TODO: We need an interface to check the coherency of
+                 * the IOMMU domain this group is using.  For now, assume
+                 * it's always noncoherent.
+                 */
+                noncoherent = true;
+                break;
+        }
+
+        if (noncoherent != kv->noncoherent) {
+                kv->noncoherent = noncoherent;
+
+                if (kv->noncoherent)
+                        kvm_arch_register_noncoherent_dma(dev->kvm);
+                else
+                        kvm_arch_unregister_noncoherent_dma(dev->kvm);
+        }
+
+        mutex_unlock(&kv->lock);
+}
+
+static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)
+{
+        struct kvm_vfio *kv = dev->private;
+        struct vfio_group *vfio_group;
+        struct kvm_vfio_group *kvg;
+        void __user *argp = (void __user *)arg;
+        struct fd f;
+        int32_t fd;
+        int ret;
+
+        switch (attr) {
+        case KVM_DEV_VFIO_GROUP_ADD:
+                if (get_user(fd, (int32_t __user *)argp))
+                        return -EFAULT;
+
+                f = fdget(fd);
+                if (!f.file)
+                        return -EBADF;
+
+                vfio_group = kvm_vfio_group_get_external_user(f.file);
+                fdput(f);
+
+                if (IS_ERR(vfio_group))
+                        return PTR_ERR(vfio_group);
+
+                mutex_lock(&kv->lock);
+
+                list_for_each_entry(kvg, &kv->group_list, node) {
+                        if (kvg->vfio_group == vfio_group) {
+                                mutex_unlock(&kv->lock);
+                                kvm_vfio_group_put_external_user(vfio_group);
+                                return -EEXIST;
+                        }
+                }
+
+                kvg = kzalloc(sizeof(*kvg), GFP_KERNEL);
+                if (!kvg) {
+                        mutex_unlock(&kv->lock);
+                        kvm_vfio_group_put_external_user(vfio_group);
+                        return -ENOMEM;
+                }
+
+                list_add_tail(&kvg->node, &kv->group_list);
+                kvg->vfio_group = vfio_group;
+
+                mutex_unlock(&kv->lock);
+
+                kvm_vfio_update_coherency(dev);
+
+                return 0;
+
+        case KVM_DEV_VFIO_GROUP_DEL:
+                if (get_user(fd, (int32_t __user *)argp))
+                        return -EFAULT;
+
+                f = fdget(fd);
+                if (!f.file)
+                        return -EBADF;
+
+                vfio_group = kvm_vfio_group_get_external_user(f.file);
+                fdput(f);
+
+                if (IS_ERR(vfio_group))
+                        return PTR_ERR(vfio_group);
+
+                ret = -ENOENT;
+
+                mutex_lock(&kv->lock);
+
+                list_for_each_entry(kvg, &kv->group_list, node) {
+                        if (kvg->vfio_group != vfio_group)
+                                continue;
+
+                        list_del(&kvg->node);
+                        kvm_vfio_group_put_external_user(kvg->vfio_group);
+                        kfree(kvg);
+                        ret = 0;
+                        break;
+                }
+
+                mutex_unlock(&kv->lock);
+
+                kvm_vfio_group_put_external_user(vfio_group);
+
+                kvm_vfio_update_coherency(dev);
+
+                return ret;
+        }
+
+        return -ENXIO;
+}
+
+static int kvm_vfio_set_attr(struct kvm_device *dev,
+                             struct kvm_device_attr *attr)
+{
+        switch (attr->group) {
+        case KVM_DEV_VFIO_GROUP:
+                return kvm_vfio_set_group(dev, attr->attr, attr->addr);
+        }
+
+        return -ENXIO;
+}
+
+static int kvm_vfio_has_attr(struct kvm_device *dev,
+                             struct kvm_device_attr *attr)
+{
+        switch (attr->group) {
+        case KVM_DEV_VFIO_GROUP:
+                switch (attr->attr) {
+                case KVM_DEV_VFIO_GROUP_ADD:
+                case KVM_DEV_VFIO_GROUP_DEL:
+                        return 0;
+                }
+
+                break;
+        }
+
+        return -ENXIO;
+}
+
+static void kvm_vfio_destroy(struct kvm_device *dev)
+{
+        struct kvm_vfio *kv = dev->private;
+        struct kvm_vfio_group *kvg, *tmp;
+
+        list_for_each_entry_safe(kvg, tmp, &kv->group_list, node) {
+                kvm_vfio_group_put_external_user(kvg->vfio_group);
+                list_del(&kvg->node);
+                kfree(kvg);
+        }
+
+        kvm_vfio_update_coherency(dev);
+
+        kfree(kv);
+        kfree(dev); /* alloc by kvm_ioctl_create_device, free by .destroy */
+}
+
+static int kvm_vfio_create(struct kvm_device *dev, u32 type)
+{
+        struct kvm_device *tmp;
+        struct kvm_vfio *kv;
+
+        /* Only one VFIO "device" per VM */
+        list_for_each_entry(tmp, &dev->kvm->devices, vm_node)
+                if (tmp->ops == &kvm_vfio_ops)
+                        return -EBUSY;
+
+        kv = kzalloc(sizeof(*kv), GFP_KERNEL);
+        if (!kv)
+                return -ENOMEM;
+
+        INIT_LIST_HEAD(&kv->group_list);
+        mutex_init(&kv->lock);
+
+        dev->private = kv;
+
+        return 0;
+}
+
+struct kvm_device_ops kvm_vfio_ops = {
+        .name = "kvm-vfio",
+        .create = kvm_vfio_create,
+        .destroy = kvm_vfio_destroy,
+        .set_attr = kvm_vfio_set_attr,
+        .has_attr = kvm_vfio_has_attr,
+};