x86/irq: Store irq descriptor in vector array

We can spare the irq_desc lookup in the interrupt entry code if we
store the descriptor pointer in the vector array instead the interrupt
number.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jiang Liu <jiang.liu@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Link: http://lkml.kernel.org/r/20150802203609.717724106@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

8 files changed, 58 insertions, 66 deletions

diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index 62bb8d23b826..1e3408e88604 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -182,10 +182,10 @@ extern char irq_entries_start[];
 #define trace_irq_entries_start irq_entries_start
 #endif
 
-#define VECTOR_UNUSED           (-1)
-#define VECTOR_RETRIGGERED      (-2)
+#define VECTOR_UNUSED           NULL
+#define VECTOR_RETRIGGERED      ((void *)~0UL)
 
-typedef int vector_irq_t[NR_VECTORS];
+typedef struct irq_desc* vector_irq_t[NR_VECTORS];
 DECLARE_PER_CPU(vector_irq_t, vector_irq);
 
 #endif /* !ASSEMBLY_ */
diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h
index 8008d06581c7..881b4768644a 100644
--- a/arch/x86/include/asm/irq.h
+++ b/arch/x86/include/asm/irq.h
@@ -36,7 +36,9 @@ extern void kvm_set_posted_intr_wakeup_handler(void (*handler)(void));
 
 extern void (*x86_platform_ipi_callback)(void);
 extern void native_init_IRQ(void);
-extern bool handle_irq(unsigned irq, struct pt_regs *regs);
+
+struct irq_desc;
+extern bool handle_irq(struct irq_desc *desc, struct pt_regs *regs);
 
 extern __visible unsigned int do_IRQ(struct pt_regs *regs);
 
diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c
index 9a6d11258684..200b5a5d6b79 100644
--- a/arch/x86/kernel/apic/vector.c
+++ b/arch/x86/kernel/apic/vector.c
@@ -169,7 +169,7 @@ next:
                         goto next;
 
                 for_each_cpu_and(new_cpu, vector_cpumask, cpu_online_mask) {
-                        if (per_cpu(vector_irq, new_cpu)[vector] > VECTOR_UNUSED)
+                        if (!IS_ERR_OR_NULL(per_cpu(vector_irq, new_cpu)[vector]))
                                 goto next;
                 }
                 /* Found one! */
@@ -181,7 +181,7 @@ next:
                            cpumask_intersects(d->old_domain, cpu_online_mask);
                 }
                 for_each_cpu_and(new_cpu, vector_cpumask, cpu_online_mask)
-                        per_cpu(vector_irq, new_cpu)[vector] = irq;
+                        per_cpu(vector_irq, new_cpu)[vector] = irq_to_desc(irq);
                 d->cfg.vector = vector;
                 cpumask_copy(d->domain, vector_cpumask);
                 err = 0;
@@ -223,8 +223,9 @@ static int assign_irq_vector_policy(int irq, int node,
 
 static void clear_irq_vector(int irq, struct apic_chip_data *data)
 {
-        int cpu, vector;
+        struct irq_desc *desc;
         unsigned long flags;
+        int cpu, vector;
 
         raw_spin_lock_irqsave(&vector_lock, flags);
         BUG_ON(!data->cfg.vector);
@@ -241,10 +242,11 @@ static void clear_irq_vector(int irq, struct apic_chip_data *data)
                 return;
         }
 
+        desc = irq_to_desc(irq);
         for_each_cpu_and(cpu, data->old_domain, cpu_online_mask) {
                 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
                      vector++) {
-                        if (per_cpu(vector_irq, cpu)[vector] != irq)
+                        if (per_cpu(vector_irq, cpu)[vector] != desc)
                                 continue;
                         per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED;
                         break;
@@ -402,30 +404,30 @@ int __init arch_early_irq_init(void)
         return arch_early_ioapic_init();
 }
 
+/* Initialize vector_irq on a new cpu */
 static void __setup_vector_irq(int cpu)
 {
-        /* Initialize vector_irq on a new cpu */
-        int irq, vector;
         struct apic_chip_data *data;
+        struct irq_desc *desc;
+        int irq, vector;
 
         /* Mark the inuse vectors */
-        for_each_active_irq(irq) {
-                data = apic_chip_data(irq_get_irq_data(irq));
-                if (!data)
-                        continue;
+        for_each_irq_desc(irq, desc) {
+                struct irq_data *idata = irq_desc_get_irq_data(desc);
 
-                if (!cpumask_test_cpu(cpu, data->domain))
+                data = apic_chip_data(idata);
+                if (!data || !cpumask_test_cpu(cpu, data->domain))
                         continue;
                 vector = data->cfg.vector;
-                per_cpu(vector_irq, cpu)[vector] = irq;
+                per_cpu(vector_irq, cpu)[vector] = desc;
         }
         /* Mark the free vectors */
         for (vector = 0; vector < NR_VECTORS; ++vector) {
-                irq = per_cpu(vector_irq, cpu)[vector];
-                if (irq <= VECTOR_UNUSED)
+                desc = per_cpu(vector_irq, cpu)[vector];
+                if (IS_ERR_OR_NULL(desc))
                         continue;
 
-                data = apic_chip_data(irq_get_irq_data(irq));
+                data = apic_chip_data(irq_desc_get_irq_data(desc));
                 if (!cpumask_test_cpu(cpu, data->domain))
                         per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED;
         }
@@ -447,7 +449,7 @@ void setup_vector_irq(int cpu)
          * legacy vector to irq mapping:
          */
         for (irq = 0; irq < nr_legacy_irqs(); irq++)
-                per_cpu(vector_irq, cpu)[ISA_IRQ_VECTOR(irq)] = irq;
+                per_cpu(vector_irq, cpu)[ISA_IRQ_VECTOR(irq)] = irq_to_desc(irq);
 
         __setup_vector_irq(cpu);
 }
@@ -543,19 +545,13 @@ asmlinkage __visible void smp_irq_move_cleanup_interrupt(void)
 
         me = smp_processor_id();
         for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
-                int irq;
-                unsigned int irr;
-                struct irq_desc *desc;
                 struct apic_chip_data *data;
+                struct irq_desc *desc;
+                unsigned int irr;
 
         retry:
-                irq = __this_cpu_read(vector_irq[vector]);
-
-                if (irq <= VECTOR_UNUSED)
-                        continue;
-
-                desc = irq_to_desc(irq);
-                if (!desc)
+                desc = __this_cpu_read(vector_irq[vector]);
+                if (IS_ERR_OR_NULL(desc))
                         continue;
 
                 if (!raw_spin_trylock(&desc->lock)) {
@@ -565,9 +561,10 @@ asmlinkage __visible void smp_irq_move_cleanup_interrupt(void)
                         goto retry;
                 }
 
-                data = apic_chip_data(&desc->irq_data);
+                data = apic_chip_data(irq_desc_get_irq_data(desc));
                 if (!data)
                         goto unlock;
+
                 /*
                  * Check if the irq migration is in progress. If so, we
                  * haven't received the cleanup request yet for this irq.
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 140950fb9902..e010847583d7 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -211,22 +211,21 @@ u64 arch_irq_stat(void)
 __visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
 {
         struct pt_regs *old_regs = set_irq_regs(regs);
-
+        struct irq_desc * desc;
         /* high bit used in ret_from_ code  */
         unsigned vector = ~regs->orig_ax;
-        unsigned irq;
 
         entering_irq();
 
-        irq = __this_cpu_read(vector_irq[vector]);
+        desc = __this_cpu_read(vector_irq[vector]);
 
-        if (!handle_irq(irq, regs)) {
+        if (!handle_irq(desc, regs)) {
                 ack_APIC_irq();
 
-                if (irq != VECTOR_RETRIGGERED) {
-                        pr_emerg_ratelimited("%s: %d.%d No irq handler for vector (irq %d)\n",
+                if (desc != VECTOR_RETRIGGERED) {
+                        pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
                                              __func__, smp_processor_id(),
-                                             vector, irq);
+                                             vector);
                 } else {
                         __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
                 }
@@ -330,10 +329,10 @@ static struct cpumask affinity_new, online_new;
  */
 int check_irq_vectors_for_cpu_disable(void)
 {
-        int irq, cpu;
         unsigned int this_cpu, vector, this_count, count;
         struct irq_desc *desc;
         struct irq_data *data;
+        int cpu;
 
         this_cpu = smp_processor_id();
         cpumask_copy(&online_new, cpu_online_mask);
@@ -341,24 +340,21 @@ int check_irq_vectors_for_cpu_disable(void)
 
         this_count = 0;
         for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
-                irq = __this_cpu_read(vector_irq[vector]);
-                if (irq < 0)
-                        continue;
-                desc = irq_to_desc(irq);
-                if (!desc)
+                desc = __this_cpu_read(vector_irq[vector]);
+                if (IS_ERR_OR_NULL(desc))
                         continue;
-
                 /*
                  * Protect against concurrent action removal, affinity
                  * changes etc.
                  */
                 raw_spin_lock(&desc->lock);
                 data = irq_desc_get_irq_data(desc);
-                cpumask_copy(&affinity_new, irq_data_get_affinity_mask(data));
+                cpumask_copy(&affinity_new,
+                             irq_data_get_affinity_mask(data));
                 cpumask_clear_cpu(this_cpu, &affinity_new);
 
                 /* Do not count inactive or per-cpu irqs. */
-                if (!irq_has_action(irq) || irqd_is_per_cpu(data)) {
+                if (!irq_desc_has_action(desc) || irqd_is_per_cpu(data)) {
                         raw_spin_unlock(&desc->lock);
                         continue;
                 }
@@ -399,8 +395,8 @@ int check_irq_vectors_for_cpu_disable(void)
                 for (vector = FIRST_EXTERNAL_VECTOR;
                      vector < first_system_vector; vector++) {
                         if (!test_bit(vector, used_vectors) &&
-                            per_cpu(vector_irq, cpu)[vector] <= VECTOR_UNUSED)
-                                count++;
+                            IS_ERR_OR_NULL(per_cpu(vector_irq, cpu)[vector]))
+                            count++;
                 }
         }
 
@@ -504,14 +500,13 @@ void fixup_irqs(void)
         for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
                 unsigned int irr;
 
-                if (__this_cpu_read(vector_irq[vector]) <= VECTOR_UNUSED)
+                if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
                         continue;
 
                 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
                 if (irr  & (1 << (vector % 32))) {
-                        irq = __this_cpu_read(vector_irq[vector]);
+                        desc = __this_cpu_read(vector_irq[vector]);
 
-                        desc = irq_to_desc(irq);
                         raw_spin_lock(&desc->lock);
                         data = irq_desc_get_irq_data(desc);
                         chip = irq_data_get_irq_chip(data);
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index cd74f5978ab9..217b01388038 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -148,21 +148,20 @@ void do_softirq_own_stack(void)
         call_on_stack(__do_softirq, isp);
 }
 
-bool handle_irq(unsigned irq, struct pt_regs *regs)
+bool handle_irq(struct irq_desc *desc, struct pt_regs *regs)
 {
-        struct irq_desc *desc;
+        unsigned int irq = irq_desc_get_irq(desc);
         int overflow;
 
         overflow = check_stack_overflow();
 
-        desc = irq_to_desc(irq);
-        if (unlikely(!desc))
+        if (IS_ERR_OR_NULL(desc))
                 return false;
 
         if (user_mode(regs) || !execute_on_irq_stack(overflow, desc, irq)) {
                 if (unlikely(overflow))
                         print_stack_overflow();
-                desc->handle_irq(irq, desc);
+                generic_handle_irq_desc(irq, desc);
         }
 
         return true;
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index bc4604e500a3..ff16ccb918f2 100644
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -68,16 +68,13 @@ static inline void stack_overflow_check(struct pt_regs *regs)
 #endif
 }
 
-bool handle_irq(unsigned irq, struct pt_regs *regs)
+bool handle_irq(struct irq_desc *desc, struct pt_regs *regs)
 {
-        struct irq_desc *desc;
-
         stack_overflow_check(regs);
 
-        desc = irq_to_desc(irq);
-        if (unlikely(!desc))
+        if (unlikely(IS_ERR_OR_NULL(desc)))
                 return false;
 
-        generic_handle_irq_desc(irq, desc);
+        generic_handle_irq_desc(irq_desc_get_irq(desc), desc);
         return true;
 }
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 32a637ffdf98..1423ab1b0312 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -60,7 +60,7 @@ int vector_used_by_percpu_irq(unsigned int vector)
         int cpu;
 
         for_each_online_cpu(cpu) {
-                if (per_cpu(vector_irq, cpu)[vector] > VECTOR_UNUSED)
+                if (!IS_ERR_OR_NULL(per_cpu(vector_irq, cpu)[vector]))
                         return 1;
         }
 
@@ -94,7 +94,7 @@ void __init init_IRQ(void)
          * irq's migrate etc.
          */
         for (i = 0; i < nr_legacy_irqs(); i++)
-                per_cpu(vector_irq, 0)[ISA_IRQ_VECTOR(i)] = i;
+                per_cpu(vector_irq, 0)[ISA_IRQ_VECTOR(i)] = irq_to_desc(i);
 
         x86_init.irqs.intr_init();
 }
diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c
index 2165f45befff..47071a08bfa6 100644
--- a/arch/x86/lguest/boot.c
+++ b/arch/x86/lguest/boot.c
@@ -843,6 +843,7 @@ static struct irq_chip lguest_irq_controller = {
  */
 static int lguest_setup_irq(unsigned int irq)
 {
+        struct irq_desc *desc;
         int err;
 
         /* Returns -ve error or vector number. */
@@ -858,7 +859,8 @@ static int lguest_setup_irq(unsigned int irq)
                                       handle_level_irq, "level");
 
         /* Some systems map "vectors" to interrupts weirdly.  Not us! */
-        __this_cpu_write(vector_irq[FIRST_EXTERNAL_VECTOR + irq], irq);
+        desc = irq_to_desc(irq);
+        __this_cpu_write(vector_irq[FIRST_EXTERNAL_VECTOR + irq], desc);
         return 0;
 }