sparse irq_desc[] array: core kernel and x86 changes

Impact: new feature

Problem on distro kernels: irq_desc[NR_IRQS] takes megabytes of RAM with
NR_CPUS set to large values. The goal is to be able to scale up to much
larger NR_IRQS value without impacting the (important) common case.

To solve this, we generalize irq_desc[NR_IRQS] to an (optional) array of
irq_desc pointers.

When CONFIG_SPARSE_IRQ=y is used, we use kzalloc_node to get irq_desc,
this also makes the IRQ descriptors NUMA-local (to the site that calls
request_irq()).

This gets rid of the irq_cfg[] static array on x86 as well: irq_cfg now
uses desc->chip_data for x86 to store irq_cfg.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

3 files changed, 87 insertions, 23 deletions

diff --git a/drivers/char/random.c b/drivers/char/random.c
index 675076f5fca8..d26891bfcd41 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -558,23 +558,9 @@ struct timer_rand_state {
         unsigned dont_count_entropy:1;
 };
 
-static struct timer_rand_state *irq_timer_state[NR_IRQS];
-
-static struct timer_rand_state *get_timer_rand_state(unsigned int irq)
-{
-        if (irq >= nr_irqs)
-                return NULL;
-
-        return irq_timer_state[irq];
-}
-
-static void set_timer_rand_state(unsigned int irq, struct timer_rand_state *state)
-{
-        if (irq >= nr_irqs)
-                return;
-
-        irq_timer_state[irq] = state;
-}
+#ifndef CONFIG_SPARSE_IRQ
+struct timer_rand_state *irq_timer_state[NR_IRQS];
+#endif
 
 static struct timer_rand_state input_timer_state;
 
@@ -933,8 +919,10 @@ void rand_initialize_irq(int irq)
 {
         struct timer_rand_state *state;
 
+#ifndef CONFIG_SPARSE_IRQ
         if (irq >= nr_irqs)
                 return;
+#endif
 
         state = get_timer_rand_state(irq);
 
diff --git a/drivers/pci/intr_remapping.c b/drivers/pci/intr_remapping.c
index 2de5a3238c94..c9958ec5e25e 100644
--- a/drivers/pci/intr_remapping.c
+++ b/drivers/pci/intr_remapping.c
@@ -19,17 +19,75 @@ struct irq_2_iommu {
         u8  irte_mask;
 };
 
-static struct irq_2_iommu irq_2_iommuX[NR_IRQS];
+#ifdef CONFIG_SPARSE_IRQ
+static struct irq_2_iommu *get_one_free_irq_2_iommu(int cpu)
+{
+        struct irq_2_iommu *iommu;
+        int node;
+
+        node = cpu_to_node(cpu);
+
+        iommu = kzalloc_node(sizeof(*iommu), GFP_ATOMIC, node);
+        printk(KERN_DEBUG "alloc irq_2_iommu on cpu %d node %d\n", cpu, node);
+
+        return iommu;
+}
 
 static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
 {
-        return (irq < nr_irqs) ? irq_2_iommuX + irq : NULL;
+        struct irq_desc *desc;
+
+        desc = irq_to_desc(irq);
+
+        if (WARN_ON_ONCE(!desc))
+                return NULL;
+
+        return desc->irq_2_iommu;
+}
+
+static struct irq_2_iommu *irq_2_iommu_alloc_cpu(unsigned int irq, int cpu)
+{
+        struct irq_desc *desc;
+        struct irq_2_iommu *irq_iommu;
+
+        /*
+         * alloc irq desc if not allocated already.
+         */
+        desc = irq_to_desc_alloc_cpu(irq, cpu);
+        if (!desc) {
+                printk(KERN_INFO "can not get irq_desc for %d\n", irq);
+                return NULL;
+        }
+
+        irq_iommu = desc->irq_2_iommu;
+
+        if (!irq_iommu)
+                desc->irq_2_iommu = get_one_free_irq_2_iommu(cpu);
+
+        return desc->irq_2_iommu;
 }
 
 static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq)
 {
+        return irq_2_iommu_alloc_cpu(irq, boot_cpu_id);
+}
+
+#else /* !CONFIG_SPARSE_IRQ */
+
+static struct irq_2_iommu irq_2_iommuX[NR_IRQS];
+
+static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
+{
+        if (irq < nr_irqs)
+                return &irq_2_iommuX[irq];
+
+        return NULL;
+}
+static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq)
+{
         return irq_2_iommu(irq);
 }
+#endif
 
 static DEFINE_SPINLOCK(irq_2_ir_lock);
 
@@ -86,9 +144,11 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
         if (!count)
                 return -1;
 
+#ifndef CONFIG_SPARSE_IRQ
         /* protect irq_2_iommu_alloc later */
         if (irq >= nr_irqs)
                 return -1;
+#endif
 
         /*
          * start the IRTE search from index 0.
@@ -130,6 +190,12 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
                 table->base[i].present = 1;
 
         irq_iommu = irq_2_iommu_alloc(irq);
+        if (!irq_iommu) {
+                spin_unlock(&irq_2_ir_lock);
+                printk(KERN_ERR "can't allocate irq_2_iommu\n");
+                return -1;
+        }
+
         irq_iommu->iommu = iommu;
         irq_iommu->irte_index =  index;
         irq_iommu->sub_handle = 0;
@@ -177,6 +243,12 @@ int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
 
         irq_iommu = irq_2_iommu_alloc(irq);
 
+        if (!irq_iommu) {
+                spin_unlock(&irq_2_ir_lock);
+                printk(KERN_ERR "can't allocate irq_2_iommu\n");
+                return -1;
+        }
+
         irq_iommu->iommu = iommu;
         irq_iommu->irte_index = index;
         irq_iommu->sub_handle = subhandle;
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 1e3b934a4cf7..2924faa7f6c4 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -141,8 +141,12 @@ static void init_evtchn_cpu_bindings(void)
         int i;
 
         /* By default all event channels notify CPU#0. */
-        for_each_irq_desc(i, desc)
+        for_each_irq_desc(i, desc) {
+                if (!desc)
+                        continue;
+
                 desc->affinity = cpumask_of_cpu(0);
+        }
 #endif
 
         memset(cpu_evtchn, 0, sizeof(cpu_evtchn));
@@ -231,7 +235,7 @@ static int find_unbound_irq(void)
         int irq;
 
         /* Only allocate from dynirq range */
-        for_each_irq_nr(irq)
+        for (irq = 0; irq < nr_irqs; irq++)
                 if (irq_bindcount[irq] == 0)
                         break;
 
@@ -792,7 +796,7 @@ void xen_irq_resume(void)
                 mask_evtchn(evtchn);
 
         /* No IRQ <-> event-channel mappings. */
-        for_each_irq_nr(irq)
+        for (irq = 0; irq < nr_irqs; irq++)
                 irq_info[irq].evtchn = 0; /* zap event-channel binding */
 
         for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
@@ -824,7 +828,7 @@ void __init xen_init_IRQ(void)
                 mask_evtchn(i);
 
         /* Dynamic IRQ space is currently unbound. Zero the refcnts. */
-        for_each_irq_nr(i)
+        for (i = 0; i < nr_irqs; i++)
                 irq_bindcount[i] = 0;
 
         irq_ctx_init(smp_processor_id());