5 files changed, 95 insertions, 50 deletions

diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c
index 06e6401d6ef4..d6a3d0993d88 100644
--- a/virt/kvm/async_pf.c
+++ b/virt/kvm/async_pf.c
@@ -80,12 +80,10 @@ static void async_pf_execute(struct work_struct *work)
 
         might_sleep();
 
-        use_mm(mm);
         down_read(&mm->mmap_sem);
-        get_user_pages(current, mm, addr, 1, 1, 0, NULL, NULL);
+        get_user_pages(NULL, mm, addr, 1, 1, 0, NULL, NULL);
         up_read(&mm->mmap_sem);
         kvm_async_page_present_sync(vcpu, apf);
-        unuse_mm(mm);
 
         spin_lock(&vcpu->async_pf.lock);
         list_add_tail(&apf->link, &vcpu->async_pf.done);
diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c
index 29c2a04e036e..20c3af7692c5 100644
--- a/virt/kvm/eventfd.c
+++ b/virt/kvm/eventfd.c
@@ -31,6 +31,7 @@
 #include <linux/list.h>
 #include <linux/eventfd.h>
 #include <linux/kernel.h>
+#include <linux/srcu.h>
 #include <linux/slab.h>
 
 #include "iodev.h"
@@ -118,19 +119,22 @@ static void
 irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian)
 {
         struct _irqfd_resampler *resampler;
+        struct kvm *kvm;
         struct _irqfd *irqfd;
+        int idx;
 
         resampler = container_of(kian, struct _irqfd_resampler, notifier);
+        kvm = resampler->kvm;
 
-        kvm_set_irq(resampler->kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
+        kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
                     resampler->notifier.gsi, 0, false);
 
-        rcu_read_lock();
+        idx = srcu_read_lock(&kvm->irq_srcu);
 
         list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link)
                 eventfd_signal(irqfd->resamplefd, 1);
 
-        rcu_read_unlock();
+        srcu_read_unlock(&kvm->irq_srcu, idx);
 }
 
 static void
@@ -142,7 +146,7 @@ irqfd_resampler_shutdown(struct _irqfd *irqfd)
         mutex_lock(&kvm->irqfds.resampler_lock);
 
         list_del_rcu(&irqfd->resampler_link);
-        synchronize_rcu();
+        synchronize_srcu(&kvm->irq_srcu);
 
         if (list_empty(&resampler->list)) {
                 list_del(&resampler->link);
@@ -221,17 +225,18 @@ irqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key)
         unsigned long flags = (unsigned long)key;
         struct kvm_kernel_irq_routing_entry *irq;
         struct kvm *kvm = irqfd->kvm;
+        int idx;
 
         if (flags & POLLIN) {
-                rcu_read_lock();
-                irq = rcu_dereference(irqfd->irq_entry);
+                idx = srcu_read_lock(&kvm->irq_srcu);
+                irq = srcu_dereference(irqfd->irq_entry, &kvm->irq_srcu);
                 /* An event has been signaled, inject an interrupt */
                 if (irq)
                         kvm_set_msi(irq, kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1,
                                         false);
                 else
                         schedule_work(&irqfd->inject);
-                rcu_read_unlock();
+                srcu_read_unlock(&kvm->irq_srcu, idx);
         }
 
         if (flags & POLLHUP) {
@@ -363,7 +368,7 @@ kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
                 }
 
                 list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);
-                synchronize_rcu();
+                synchronize_srcu(&kvm->irq_srcu);
 
                 mutex_unlock(&kvm->irqfds.resampler_lock);
         }
@@ -465,7 +470,7 @@ kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args)
                          * another thread calls kvm_irq_routing_update before
                          * we flush workqueue below (we synchronize with
                          * kvm_irq_routing_update using irqfds.lock).
-                         * It is paired with synchronize_rcu done by caller
+                         * It is paired with synchronize_srcu done by caller
                          * of that function.
                          */
                         rcu_assign_pointer(irqfd->irq_entry, NULL);
@@ -524,7 +529,7 @@ kvm_irqfd_release(struct kvm *kvm)
 
 /*
  * Change irq_routing and irqfd.
- * Caller must invoke synchronize_rcu afterwards.
+ * Caller must invoke synchronize_srcu(&kvm->irq_srcu) afterwards.
  */
 void kvm_irq_routing_update(struct kvm *kvm,
                             struct kvm_irq_routing_table *irq_rt)
@@ -600,7 +605,15 @@ ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val)
 {
         u64 _val;
 
-        if (!(addr == p->addr && len == p->length))
+        if (addr != p->addr)
+                /* address must be precise for a hit */
+                return false;
+
+        if (!p->length)
+                /* length = 0 means only look at the address, so always a hit */
+                return true;
+
+        if (len != p->length)
                 /* address-range must be precise for a hit */
                 return false;
 
@@ -671,9 +684,11 @@ ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p)
 
         list_for_each_entry(_p, &kvm->ioeventfds, list)
                 if (_p->bus_idx == p->bus_idx &&
-                    _p->addr == p->addr && _p->length == p->length &&
-                    (_p->wildcard || p->wildcard ||
-                     _p->datamatch == p->datamatch))
+                    _p->addr == p->addr &&
+                    (!_p->length || !p->length ||
+                     (_p->length == p->length &&
+                      (_p->wildcard || p->wildcard ||
+                       _p->datamatch == p->datamatch))))
                         return true;
 
         return false;
@@ -697,8 +712,9 @@ kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
         int                       ret;
 
         bus_idx = ioeventfd_bus_from_flags(args->flags);
-        /* must be natural-word sized */
+        /* must be natural-word sized, or 0 to ignore length */
         switch (args->len) {
+        case 0:
         case 1:
         case 2:
         case 4:
@@ -716,6 +732,12 @@ kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
         if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK)
                 return -EINVAL;
 
+        /* ioeventfd with no length can't be combined with DATAMATCH */
+        if (!args->len &&
+            args->flags & (KVM_IOEVENTFD_FLAG_PIO |
+                           KVM_IOEVENTFD_FLAG_DATAMATCH))
+                return -EINVAL;
+
         eventfd = eventfd_ctx_fdget(args->fd);
         if (IS_ERR(eventfd))
                 return PTR_ERR(eventfd);
@@ -753,6 +775,16 @@ kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
         if (ret < 0)
                 goto unlock_fail;
 
+        /* When length is ignored, MMIO is also put on a separate bus, for
+         * faster lookups.
+         */
+        if (!args->len && !(args->flags & KVM_IOEVENTFD_FLAG_PIO)) {
+                ret = kvm_io_bus_register_dev(kvm, KVM_FAST_MMIO_BUS,
+                                              p->addr, 0, &p->dev);
+                if (ret < 0)
+                        goto register_fail;
+        }
+
         kvm->buses[bus_idx]->ioeventfd_count++;
         list_add_tail(&p->list, &kvm->ioeventfds);
 
@@ -760,6 +792,8 @@ kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
 
         return 0;
 
+register_fail:
+        kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
 unlock_fail:
         mutex_unlock(&kvm->slots_lock);
 
@@ -799,6 +833,10 @@ kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
                         continue;
 
                 kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
+                if (!p->length) {
+                        kvm_io_bus_unregister_dev(kvm, KVM_FAST_MMIO_BUS,
+                                                  &p->dev);
+                }
                 kvm->buses[bus_idx]->ioeventfd_count--;
                 ioeventfd_release(p);
                 ret = 0;
diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c
index e2e6b4473a96..ced4a542a031 100644
--- a/virt/kvm/irq_comm.c
+++ b/virt/kvm/irq_comm.c
@@ -163,6 +163,7 @@ int kvm_set_irq_inatomic(struct kvm *kvm, int irq_source_id, u32 irq, int level)
         struct kvm_kernel_irq_routing_entry *e;
         int ret = -EINVAL;
         struct kvm_irq_routing_table *irq_rt;
+        int idx;
 
         trace_kvm_set_irq(irq, level, irq_source_id);
 
@@ -174,8 +175,8 @@ int kvm_set_irq_inatomic(struct kvm *kvm, int irq_source_id, u32 irq, int level)
          * Since there's no easy way to do this, we only support injecting MSI
          * which is limited to 1:1 GSI mapping.
          */
-        rcu_read_lock();
-        irq_rt = rcu_dereference(kvm->irq_routing);
+        idx = srcu_read_lock(&kvm->irq_srcu);
+        irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
         if (irq < irq_rt->nr_rt_entries)
                 hlist_for_each_entry(e, &irq_rt->map[irq], link) {
                         if (likely(e->type == KVM_IRQ_ROUTING_MSI))
@@ -184,7 +185,7 @@ int kvm_set_irq_inatomic(struct kvm *kvm, int irq_source_id, u32 irq, int level)
                                 ret = -EWOULDBLOCK;
                         break;
                 }
-        rcu_read_unlock();
+        srcu_read_unlock(&kvm->irq_srcu, idx);
         return ret;
 }
 
@@ -253,22 +254,22 @@ void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
         mutex_lock(&kvm->irq_lock);
         hlist_del_rcu(&kimn->link);
         mutex_unlock(&kvm->irq_lock);
-        synchronize_rcu();
+        synchronize_srcu(&kvm->irq_srcu);
 }
 
 void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
                              bool mask)
 {
         struct kvm_irq_mask_notifier *kimn;
-        int gsi;
+        int idx, gsi;
 
-        rcu_read_lock();
-        gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin];
+        idx = srcu_read_lock(&kvm->irq_srcu);
+        gsi = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu)->chip[irqchip][pin];
         if (gsi != -1)
                 hlist_for_each_entry_rcu(kimn, &kvm->mask_notifier_list, link)
                         if (kimn->irq == gsi)
                                 kimn->func(kimn, mask);
-        rcu_read_unlock();
+        srcu_read_unlock(&kvm->irq_srcu, idx);
 }
 
 int kvm_set_routing_entry(struct kvm_irq_routing_table *rt,
diff --git a/virt/kvm/irqchip.c b/virt/kvm/irqchip.c
index 20dc9e4a8f6c..b43c275775cd 100644
--- a/virt/kvm/irqchip.c
+++ b/virt/kvm/irqchip.c
@@ -26,6 +26,7 @@
 
 #include <linux/kvm_host.h>
 #include <linux/slab.h>
+#include <linux/srcu.h>
 #include <linux/export.h>
 #include <trace/events/kvm.h>
 #include "irq.h"
@@ -33,19 +34,19 @@
 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)
 {
         struct kvm_irq_ack_notifier *kian;
-        int gsi;
+        int gsi, idx;
 
-        rcu_read_lock();
-        gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin];
+        idx = srcu_read_lock(&kvm->irq_srcu);
+        gsi = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu)->chip[irqchip][pin];
         if (gsi != -1)
                 hlist_for_each_entry_rcu(kian, &kvm->irq_ack_notifier_list,
                                          link)
                         if (kian->gsi == gsi) {
-                                rcu_read_unlock();
+                                srcu_read_unlock(&kvm->irq_srcu, idx);
                                 return true;
                         }
 
-        rcu_read_unlock();
+        srcu_read_unlock(&kvm->irq_srcu, idx);
 
         return false;
 }
@@ -54,18 +55,18 @@ EXPORT_SYMBOL_GPL(kvm_irq_has_notifier);
 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)
 {
         struct kvm_irq_ack_notifier *kian;
-        int gsi;
+        int gsi, idx;
 
         trace_kvm_ack_irq(irqchip, pin);
 
-        rcu_read_lock();
-        gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin];
+        idx = srcu_read_lock(&kvm->irq_srcu);
+        gsi = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu)->chip[irqchip][pin];
         if (gsi != -1)
                 hlist_for_each_entry_rcu(kian, &kvm->irq_ack_notifier_list,
                                          link)
                         if (kian->gsi == gsi)
                                 kian->irq_acked(kian);
-        rcu_read_unlock();
+        srcu_read_unlock(&kvm->irq_srcu, idx);
 }
 
 void kvm_register_irq_ack_notifier(struct kvm *kvm,
@@ -85,7 +86,7 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
         mutex_lock(&kvm->irq_lock);
         hlist_del_init_rcu(&kian->link);
         mutex_unlock(&kvm->irq_lock);
-        synchronize_rcu();
+        synchronize_srcu(&kvm->irq_srcu);
 #ifdef __KVM_HAVE_IOAPIC
         kvm_vcpu_request_scan_ioapic(kvm);
 #endif
@@ -115,7 +116,7 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
                 bool line_status)
 {
         struct kvm_kernel_irq_routing_entry *e, irq_set[KVM_NR_IRQCHIPS];
-        int ret = -1, i = 0;
+        int ret = -1, i = 0, idx;
         struct kvm_irq_routing_table *irq_rt;
 
         trace_kvm_set_irq(irq, level, irq_source_id);
@@ -124,12 +125,12 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
          * IOAPIC.  So set the bit in both. The guest will ignore
          * writes to the unused one.
          */
-        rcu_read_lock();
-        irq_rt = rcu_dereference(kvm->irq_routing);
+        idx = srcu_read_lock(&kvm->irq_srcu);
+        irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
         if (irq < irq_rt->nr_rt_entries)
                 hlist_for_each_entry(e, &irq_rt->map[irq], link)
                         irq_set[i++] = *e;
-        rcu_read_unlock();
+        srcu_read_unlock(&kvm->irq_srcu, idx);
 
         while(i--) {
                 int r;
@@ -226,7 +227,7 @@ int kvm_set_irq_routing(struct kvm *kvm,
         kvm_irq_routing_update(kvm, new);
         mutex_unlock(&kvm->irq_lock);
 
-        synchronize_rcu();
+        synchronize_srcu_expedited(&kvm->irq_srcu);
 
         new = old;
         r = 0;
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 86d1c457458d..4b6c01b477f9 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -186,9 +186,12 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
 
 void kvm_flush_remote_tlbs(struct kvm *kvm)
 {
+        long dirty_count = kvm->tlbs_dirty;
+
+        smp_mb();
         if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
                 ++kvm->stat.remote_tlb_flush;
-        kvm->tlbs_dirty = false;
+        cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
 
@@ -454,11 +457,11 @@ static struct kvm *kvm_create_vm(unsigned long type)
 
         r = kvm_arch_init_vm(kvm, type);
         if (r)
-                goto out_err_nodisable;
+                goto out_err_no_disable;
 
         r = hardware_enable_all();
         if (r)
-                goto out_err_nodisable;
+                goto out_err_no_disable;
 
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
         INIT_HLIST_HEAD(&kvm->mask_notifier_list);
@@ -470,10 +473,12 @@ static struct kvm *kvm_create_vm(unsigned long type)
         r = -ENOMEM;
         kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
         if (!kvm->memslots)
-                goto out_err_nosrcu;
+                goto out_err_no_srcu;
         kvm_init_memslots_id(kvm);
         if (init_srcu_struct(&kvm->srcu))
-                goto out_err_nosrcu;
+                goto out_err_no_srcu;
+        if (init_srcu_struct(&kvm->irq_srcu))
+                goto out_err_no_irq_srcu;
         for (i = 0; i < KVM_NR_BUSES; i++) {
                 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
                                         GFP_KERNEL);
@@ -502,10 +507,12 @@ static struct kvm *kvm_create_vm(unsigned long type)
         return kvm;
 
 out_err:
+        cleanup_srcu_struct(&kvm->irq_srcu);
+out_err_no_irq_srcu:
         cleanup_srcu_struct(&kvm->srcu);
-out_err_nosrcu:
+out_err_no_srcu:
         hardware_disable_all();
-out_err_nodisable:
+out_err_no_disable:
         for (i = 0; i < KVM_NR_BUSES; i++)
                 kfree(kvm->buses[i]);
         kfree(kvm->memslots);
@@ -601,6 +608,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
         kvm_arch_destroy_vm(kvm);
         kvm_destroy_devices(kvm);
         kvm_free_physmem(kvm);
+        cleanup_srcu_struct(&kvm->irq_srcu);
         cleanup_srcu_struct(&kvm->srcu);
         kvm_arch_free_vm(kvm);
         hardware_disable_all();
@@ -637,14 +645,12 @@ static int kvm_vm_release(struct inode *inode, struct file *filp)
  */
 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
 {
-#ifndef CONFIG_S390
         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
 
         memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
         if (!memslot->dirty_bitmap)
                 return -ENOMEM;
 
-#endif /* !CONFIG_S390 */
         return 0;
 }
 
@@ -2922,6 +2928,7 @@ static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range,
 
         return -EOPNOTSUPP;
 }
+EXPORT_SYMBOL_GPL(kvm_io_bus_write);
 
 /* kvm_io_bus_read - called under kvm->slots_lock */
 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,