1 files changed, 136 insertions, 95 deletions

diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c
index 001663ff401a..9b077342ab54 100644
--- a/virt/kvm/irq_comm.c
+++ b/virt/kvm/irq_comm.c
@@ -31,20 +31,39 @@
 
 #include "ioapic.h"
 
+static inline int kvm_irq_line_state(unsigned long *irq_state,
+                                     int irq_source_id, int level)
+{
+        /* Logical OR for level trig interrupt */
+        if (level)
+                set_bit(irq_source_id, irq_state);
+        else
+                clear_bit(irq_source_id, irq_state);
+
+        return !!(*irq_state);
+}
+
 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e,
-                           struct kvm *kvm, int level)
+                           struct kvm *kvm, int irq_source_id, int level)
 {
 #ifdef CONFIG_X86
-        return kvm_pic_set_irq(pic_irqchip(kvm), e->irqchip.pin, level);
+        struct kvm_pic *pic = pic_irqchip(kvm);
+        level = kvm_irq_line_state(&pic->irq_states[e->irqchip.pin],
+                                   irq_source_id, level);
+        return kvm_pic_set_irq(pic, e->irqchip.pin, level);
 #else
         return -1;
 #endif
 }
 
 static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e,
-                              struct kvm *kvm, int level)
+                              struct kvm *kvm, int irq_source_id, int level)
 {
-        return kvm_ioapic_set_irq(kvm->arch.vioapic, e->irqchip.pin, level);
+        struct kvm_ioapic *ioapic = kvm->arch.vioapic;
+        level = kvm_irq_line_state(&ioapic->irq_states[e->irqchip.pin],
+                                   irq_source_id, level);
+
+        return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, level);
 }
 
 inline static bool kvm_is_dm_lowest_prio(struct kvm_lapic_irq *irq)
@@ -63,8 +82,6 @@ int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
         int i, r = -1;
         struct kvm_vcpu *vcpu, *lowest = NULL;
 
-        WARN_ON(!mutex_is_locked(&kvm->irq_lock));
-
         if (irq->dest_mode == 0 && irq->dest_id == 0xff &&
                         kvm_is_dm_lowest_prio(irq))
                 printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n");
@@ -96,10 +113,13 @@ int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
 }
 
 static int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
-                       struct kvm *kvm, int level)
+                       struct kvm *kvm, int irq_source_id, int level)
 {
         struct kvm_lapic_irq irq;
 
+        if (!level)
+                return -1;
+
         trace_kvm_msi_set_irq(e->msi.address_lo, e->msi.data);
 
         irq.dest_id = (e->msi.address_lo &
@@ -116,78 +136,67 @@ static int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
         return kvm_irq_delivery_to_apic(kvm, NULL, &irq);
 }
 
-/* This should be called with the kvm->irq_lock mutex held
+/*
  * Return value:
  *  < 0   Interrupt was ignored (masked or not delivered for other reasons)
  *  = 0   Interrupt was coalesced (previous irq is still pending)
  *  > 0   Number of CPUs interrupt was delivered to
  */
-int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level)
+int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level)
 {
-        struct kvm_kernel_irq_routing_entry *e;
-        unsigned long *irq_state, sig_level;
-        int ret = -1;
+        struct kvm_kernel_irq_routing_entry *e, irq_set[KVM_NR_IRQCHIPS];
+        int ret = -1, i = 0;
+        struct kvm_irq_routing_table *irq_rt;
+        struct hlist_node *n;
 
         trace_kvm_set_irq(irq, level, irq_source_id);
 
-        WARN_ON(!mutex_is_locked(&kvm->irq_lock));
-
-        if (irq < KVM_IOAPIC_NUM_PINS) {
-                irq_state = (unsigned long *)&kvm->arch.irq_states[irq];
-
-                /* Logical OR for level trig interrupt */
-                if (level)
-                        set_bit(irq_source_id, irq_state);
-                else
-                        clear_bit(irq_source_id, irq_state);
-                sig_level = !!(*irq_state);
-        } else if (!level)
-                return ret;
-        else /* Deal with MSI/MSI-X */
-                sig_level = 1;
-
         /* Not possible to detect if the guest uses the PIC or the
          * IOAPIC.  So set the bit in both. The guest will ignore
          * writes to the unused one.
          */
-        list_for_each_entry(e, &kvm->irq_routing, link)
-                if (e->gsi == irq) {
-                        int r = e->set(e, kvm, sig_level);
-                        if (r < 0)
-                                continue;
+        rcu_read_lock();
+        irq_rt = rcu_dereference(kvm->irq_routing);
+        if (irq < irq_rt->nr_rt_entries)
+                hlist_for_each_entry(e, n, &irq_rt->map[irq], link)
+                        irq_set[i++] = *e;
+        rcu_read_unlock();
+
+        while(i--) {
+                int r;
+                r = irq_set[i].set(&irq_set[i], kvm, irq_source_id, level);
+                if (r < 0)
+                        continue;
+
+                ret = r + ((ret < 0) ? 0 : ret);
+        }
 
-                        ret = r + ((ret < 0) ? 0 : ret);
-                }
         return ret;
 }
 
 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)
 {
-        struct kvm_kernel_irq_routing_entry *e;
         struct kvm_irq_ack_notifier *kian;
         struct hlist_node *n;
-        unsigned gsi = pin;
+        int gsi;
 
         trace_kvm_ack_irq(irqchip, pin);
 
-        list_for_each_entry(e, &kvm->irq_routing, link)
-                if (e->type == KVM_IRQ_ROUTING_IRQCHIP &&
-                    e->irqchip.irqchip == irqchip &&
-                    e->irqchip.pin == pin) {
-                        gsi = e->gsi;
-                        break;
-                }
-
-        hlist_for_each_entry(kian, n, &kvm->arch.irq_ack_notifier_list, link)
-                if (kian->gsi == gsi)
-                        kian->irq_acked(kian);
+        rcu_read_lock();
+        gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin];
+        if (gsi != -1)
+                hlist_for_each_entry_rcu(kian, n, &kvm->irq_ack_notifier_list,
+                                         link)
+                        if (kian->gsi == gsi)
+                                kian->irq_acked(kian);
+        rcu_read_unlock();
 }
 
 void kvm_register_irq_ack_notifier(struct kvm *kvm,
                                    struct kvm_irq_ack_notifier *kian)
 {
         mutex_lock(&kvm->irq_lock);
-        hlist_add_head(&kian->link, &kvm->arch.irq_ack_notifier_list);
+        hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);
         mutex_unlock(&kvm->irq_lock);
 }
 
@@ -195,8 +204,9 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
                                     struct kvm_irq_ack_notifier *kian)
 {
         mutex_lock(&kvm->irq_lock);
-        hlist_del_init(&kian->link);
+        hlist_del_init_rcu(&kian->link);
         mutex_unlock(&kvm->irq_lock);
+        synchronize_rcu();
 }
 
 int kvm_request_irq_source_id(struct kvm *kvm)
@@ -205,16 +215,17 @@ int kvm_request_irq_source_id(struct kvm *kvm)
         int irq_source_id;
 
         mutex_lock(&kvm->irq_lock);
-        irq_source_id = find_first_zero_bit(bitmap,
-                                sizeof(kvm->arch.irq_sources_bitmap));
+        irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG);
 
-        if (irq_source_id >= sizeof(kvm->arch.irq_sources_bitmap)) {
+        if (irq_source_id >= BITS_PER_LONG) {
                 printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n");
-                return -EFAULT;
+                irq_source_id = -EFAULT;
+                goto unlock;
         }
 
         ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
         set_bit(irq_source_id, bitmap);
+unlock:
         mutex_unlock(&kvm->irq_lock);
 
         return irq_source_id;
@@ -228,13 +239,23 @@ void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id)
 
         mutex_lock(&kvm->irq_lock);
         if (irq_source_id < 0 ||
-            irq_source_id >= sizeof(kvm->arch.irq_sources_bitmap)) {
+            irq_source_id >= BITS_PER_LONG) {
                 printk(KERN_ERR "kvm: IRQ source ID out of range!\n");
-                return;
+                goto unlock;
         }
-        for (i = 0; i < KVM_IOAPIC_NUM_PINS; i++)
-                clear_bit(irq_source_id, &kvm->arch.irq_states[i]);
         clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap);
+        if (!irqchip_in_kernel(kvm))
+                goto unlock;
+
+        for (i = 0; i < KVM_IOAPIC_NUM_PINS; i++) {
+                clear_bit(irq_source_id, &kvm->arch.vioapic->irq_states[i]);
+                if (i >= 16)
+                        continue;
+#ifdef CONFIG_X86
+                clear_bit(irq_source_id, &pic_irqchip(kvm)->irq_states[i]);
+#endif
+        }
+unlock:
         mutex_unlock(&kvm->irq_lock);
 }
 
@@ -243,7 +264,7 @@ void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
 {
         mutex_lock(&kvm->irq_lock);
         kimn->irq = irq;
-        hlist_add_head(&kimn->link, &kvm->mask_notifier_list);
+        hlist_add_head_rcu(&kimn->link, &kvm->mask_notifier_list);
         mutex_unlock(&kvm->irq_lock);
 }
 
@@ -251,8 +272,9 @@ void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
                                       struct kvm_irq_mask_notifier *kimn)
 {
         mutex_lock(&kvm->irq_lock);
-        hlist_del(&kimn->link);
+        hlist_del_rcu(&kimn->link);
         mutex_unlock(&kvm->irq_lock);
+        synchronize_rcu();
 }
 
 void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask)
@@ -260,33 +282,37 @@ void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask)
         struct kvm_irq_mask_notifier *kimn;
         struct hlist_node *n;
 
-        WARN_ON(!mutex_is_locked(&kvm->irq_lock));
-
-        hlist_for_each_entry(kimn, n, &kvm->mask_notifier_list, link)
+        rcu_read_lock();
+        hlist_for_each_entry_rcu(kimn, n, &kvm->mask_notifier_list, link)
                 if (kimn->irq == irq)
                         kimn->func(kimn, mask);
-}
-
-static void __kvm_free_irq_routing(struct list_head *irq_routing)
-{
-        struct kvm_kernel_irq_routing_entry *e, *n;
-
-        list_for_each_entry_safe(e, n, irq_routing, link)
-                kfree(e);
+        rcu_read_unlock();
 }
 
 void kvm_free_irq_routing(struct kvm *kvm)
 {
-        mutex_lock(&kvm->irq_lock);
-        __kvm_free_irq_routing(&kvm->irq_routing);
-        mutex_unlock(&kvm->irq_lock);
+        /* Called only during vm destruction. Nobody can use the pointer
+           at this stage */
+        kfree(kvm->irq_routing);
 }
 
-static int setup_routing_entry(struct kvm_kernel_irq_routing_entry *e,
+static int setup_routing_entry(struct kvm_irq_routing_table *rt,
+                               struct kvm_kernel_irq_routing_entry *e,
                                const struct kvm_irq_routing_entry *ue)
 {
         int r = -EINVAL;
         int delta;
+        struct kvm_kernel_irq_routing_entry *ei;
+        struct hlist_node *n;
+
+        /*
+         * Do not allow GSI to be mapped to the same irqchip more than once.
+         * Allow only one to one mapping between GSI and MSI.
+         */
+        hlist_for_each_entry(ei, n, &rt->map[ue->gsi], link)
+                if (ei->type == KVM_IRQ_ROUTING_MSI ||
+                    ue->u.irqchip.irqchip == ei->irqchip.irqchip)
+                        return r;
 
         e->gsi = ue->gsi;
         e->type = ue->type;
@@ -309,6 +335,9 @@ static int setup_routing_entry(struct kvm_kernel_irq_routing_entry *e,
                 }
                 e->irqchip.irqchip = ue->u.irqchip.irqchip;
                 e->irqchip.pin = ue->u.irqchip.pin + delta;
+                if (e->irqchip.pin >= KVM_IOAPIC_NUM_PINS)
+                        goto out;
+                rt->chip[ue->u.irqchip.irqchip][e->irqchip.pin] = ue->gsi;
                 break;
         case KVM_IRQ_ROUTING_MSI:
                 e->set = kvm_set_msi;
@@ -319,6 +348,8 @@ static int setup_routing_entry(struct kvm_kernel_irq_routing_entry *e,
         default:
                 goto out;
         }
+
+        hlist_add_head(&e->link, &rt->map[e->gsi]);
         r = 0;
 out:
         return r;
@@ -330,43 +361,53 @@ int kvm_set_irq_routing(struct kvm *kvm,
                         unsigned nr,
                         unsigned flags)
 {
-        struct list_head irq_list = LIST_HEAD_INIT(irq_list);
-        struct list_head tmp = LIST_HEAD_INIT(tmp);
-        struct kvm_kernel_irq_routing_entry *e = NULL;
-        unsigned i;
+        struct kvm_irq_routing_table *new, *old;
+        u32 i, j, nr_rt_entries = 0;
         int r;
 
         for (i = 0; i < nr; ++i) {
+                if (ue[i].gsi >= KVM_MAX_IRQ_ROUTES)
+                        return -EINVAL;
+                nr_rt_entries = max(nr_rt_entries, ue[i].gsi);
+        }
+
+        nr_rt_entries += 1;
+
+        new = kzalloc(sizeof(*new) + (nr_rt_entries * sizeof(struct hlist_head))
+                      + (nr * sizeof(struct kvm_kernel_irq_routing_entry)),
+                      GFP_KERNEL);
+
+        if (!new)
+                return -ENOMEM;
+
+        new->rt_entries = (void *)&new->map[nr_rt_entries];
+
+        new->nr_rt_entries = nr_rt_entries;
+        for (i = 0; i < 3; i++)
+                for (j = 0; j < KVM_IOAPIC_NUM_PINS; j++)
+                        new->chip[i][j] = -1;
+
+        for (i = 0; i < nr; ++i) {
                 r = -EINVAL;
-                if (ue->gsi >= KVM_MAX_IRQ_ROUTES)
-                        goto out;
                 if (ue->flags)
                         goto out;
-                r = -ENOMEM;
-                e = kzalloc(sizeof(*e), GFP_KERNEL);
-                if (!e)
-                        goto out;
-                r = setup_routing_entry(e, ue);
+                r = setup_routing_entry(new, &new->rt_entries[i], ue);
                 if (r)
                         goto out;
                 ++ue;
-                list_add(&e->link, &irq_list);
-                e = NULL;
         }
 
         mutex_lock(&kvm->irq_lock);
-        list_splice(&kvm->irq_routing, &tmp);
-        INIT_LIST_HEAD(&kvm->irq_routing);
-        list_splice(&irq_list, &kvm->irq_routing);
-        INIT_LIST_HEAD(&irq_list);
-        list_splice(&tmp, &irq_list);
+        old = kvm->irq_routing;
+        rcu_assign_pointer(kvm->irq_routing, new);
         mutex_unlock(&kvm->irq_lock);
+        synchronize_rcu();
 
+        new = old;
         r = 0;
 
 out:
-        kfree(e);
-        __kvm_free_irq_routing(&irq_list);
+        kfree(new);
         return r;
 }