[PATCH] x86-64: Make remote TLB flush more scalable

Instead of using a global spinlock to protect the state
of the remote TLB flush use a lock and state for each sending CPU.

To tell the receiver where to look for the state use 8 different
call vectors.  Each CPU uses a specific vector to trigger flushes on other
CPUs. Depending on the received vector the target CPUs look into
the right per cpu variable for the flush data.

When the system has more than 8 CPUs they are hashed to the 8 available
vectors. The limited global vector space forces us to this right now.
In future when interrupts are split into per CPU domains this could be
fixed, at the cost of needing more IPIs in flat mode.

Also some minor cleanup in the smp flush code and remove some outdated
debug code.

Requires patch to move cpu_possible_map setup earlier.

Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

4 files changed, 117 insertions, 58 deletions

diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 3620508c8bd9..873c39d8f818 100644
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -536,8 +536,19 @@ ENTRY(thermal_interrupt)
 ENTRY(reschedule_interrupt)
         apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
 
-ENTRY(invalidate_interrupt)
-        apicinterrupt INVALIDATE_TLB_VECTOR,smp_invalidate_interrupt
+        .macro INVALIDATE_ENTRY num
+ENTRY(invalidate_interrupt\num)
+        apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt 
+        .endm
+
+        INVALIDATE_ENTRY 0
+        INVALIDATE_ENTRY 1
+        INVALIDATE_ENTRY 2
+        INVALIDATE_ENTRY 3
+        INVALIDATE_ENTRY 4
+        INVALIDATE_ENTRY 5
+        INVALIDATE_ENTRY 6
+        INVALIDATE_ENTRY 7
 
 ENTRY(call_function_interrupt)
         apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
diff --git a/arch/x86_64/kernel/i8259.c b/arch/x86_64/kernel/i8259.c
index a89169095129..909fc63c03b6 100644
--- a/arch/x86_64/kernel/i8259.c
+++ b/arch/x86_64/kernel/i8259.c
@@ -486,7 +486,14 @@ void spurious_interrupt(void);
 void error_interrupt(void);
 void reschedule_interrupt(void);
 void call_function_interrupt(void);
-void invalidate_interrupt(void);
+void invalidate_interrupt0(void);
+void invalidate_interrupt1(void);
+void invalidate_interrupt2(void);
+void invalidate_interrupt3(void);
+void invalidate_interrupt4(void);
+void invalidate_interrupt5(void);
+void invalidate_interrupt6(void);
+void invalidate_interrupt7(void);
 void thermal_interrupt(void);
 void i8254_timer_resume(void);
 
@@ -562,8 +569,15 @@ void __init init_IRQ(void)
          */
         set_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
 
-        /* IPI for invalidation */
-        set_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt);
+        /* IPIs for invalidation */
+        set_intr_gate(INVALIDATE_TLB_VECTOR_START+0, invalidate_interrupt0);
+        set_intr_gate(INVALIDATE_TLB_VECTOR_START+1, invalidate_interrupt1);
+        set_intr_gate(INVALIDATE_TLB_VECTOR_START+2, invalidate_interrupt2);
+        set_intr_gate(INVALIDATE_TLB_VECTOR_START+3, invalidate_interrupt3);
+        set_intr_gate(INVALIDATE_TLB_VECTOR_START+4, invalidate_interrupt4);
+        set_intr_gate(INVALIDATE_TLB_VECTOR_START+5, invalidate_interrupt5);
+        set_intr_gate(INVALIDATE_TLB_VECTOR_START+6, invalidate_interrupt6);
+        set_intr_gate(INVALIDATE_TLB_VECTOR_START+7, invalidate_interrupt7);
 
         /* IPI for generic function call */
         set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
diff --git a/arch/x86_64/kernel/smp.c b/arch/x86_64/kernel/smp.c
index 82d38f145b43..801db885955c 100644
--- a/arch/x86_64/kernel/smp.c
+++ b/arch/x86_64/kernel/smp.c
@@ -29,6 +29,8 @@
 #include <asm/proto.h>
 #include <asm/apicdef.h>
 
+#define __cpuinit __init
+
 /*
  *      Smarter SMP flushing macros. 
  *              c/o Linus Torvalds.
@@ -37,19 +39,41 @@
  *      writing to user space from interrupts. (Its not allowed anyway).
  *
  *      Optimizations Manfred Spraul <manfred@colorfullife.com>
+ *
+ *      More scalable flush, from Andi Kleen
+ *
+ *      To avoid global state use 8 different call vectors.
+ *      Each CPU uses a specific vector to trigger flushes on other
+ *      CPUs. Depending on the received vector the target CPUs look into
+ *      the right per cpu variable for the flush data.
+ *
+ *      With more than 8 CPUs they are hashed to the 8 available
+ *      vectors. The limited global vector space forces us to this right now.
+ *      In future when interrupts are split into per CPU domains this could be
+ *      fixed, at the cost of triggering multiple IPIs in some cases.
  */
 
-static cpumask_t flush_cpumask;
-static struct mm_struct * flush_mm;
-static unsigned long flush_va;
-static DEFINE_SPINLOCK(tlbstate_lock);
+union smp_flush_state {
+        struct {
+                cpumask_t flush_cpumask;
+                struct mm_struct *flush_mm;
+                unsigned long flush_va;
 #define FLUSH_ALL       -1ULL
+                spinlock_t tlbstate_lock;
+        };
+        char pad[SMP_CACHE_BYTES];
+} ____cacheline_aligned;
+
+/* State is put into the per CPU data section, but padded
+   to a full cache line because other CPUs can access it and we don't
+   want false sharing in the per cpu data segment. */
+static DEFINE_PER_CPU(union smp_flush_state, flush_state);
 
 /*
  * We cannot call mmdrop() because we are in interrupt context, 
  * instead update mm->cpu_vm_mask.
  */
-static inline void leave_mm (unsigned long cpu)
+static inline void leave_mm(int cpu)
 {
         if (read_pda(mmu_state) == TLBSTATE_OK)
                 BUG();
@@ -101,15 +125,25 @@ static inline void leave_mm (unsigned long cpu)
  *
  * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
  * 2) Leave the mm if we are in the lazy tlb mode.
+ *
+ * Interrupts are disabled.
  */
 
-asmlinkage void smp_invalidate_interrupt (void)
+asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
 {
-        unsigned long cpu;
+        int cpu;
+        int sender;
+        union smp_flush_state *f;
 
-        cpu = get_cpu();
+        cpu = smp_processor_id();
+        /*
+         * orig_rax contains the interrupt vector - 256.
+         * Use that to determine where the sender put the data.
+         */
+        sender = regs->orig_rax + 256 - INVALIDATE_TLB_VECTOR_START;
+        f = &per_cpu(flush_state, sender);
 
-        if (!cpu_isset(cpu, flush_cpumask))
+        if (!cpu_isset(cpu, f->flush_cpumask))
                 goto out;
                 /* 
                  * This was a BUG() but until someone can quote me the
@@ -120,64 +154,63 @@ asmlinkage void smp_invalidate_interrupt (void)
                  * BUG();
                  */
                  
-        if (flush_mm == read_pda(active_mm)) {
+        if (f->flush_mm == read_pda(active_mm)) {
                 if (read_pda(mmu_state) == TLBSTATE_OK) {
-                        if (flush_va == FLUSH_ALL)
+                        if (f->flush_va == FLUSH_ALL)
                                 local_flush_tlb();
                         else
-                                __flush_tlb_one(flush_va);
+                                __flush_tlb_one(f->flush_va);
                 } else
                         leave_mm(cpu);
         }
 out:
         ack_APIC_irq();
-        cpu_clear(cpu, flush_cpumask);
-        put_cpu_no_resched();
+        cpu_clear(cpu, f->flush_cpumask);
 }
 
 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
                                                 unsigned long va)
 {
-        cpumask_t tmp;
-        /*
-         * A couple of (to be removed) sanity checks:
-         *
-         * - we do not send IPIs to not-yet booted CPUs.
-         * - current CPU must not be in mask
-         * - mask must exist :)
-         */
-        BUG_ON(cpus_empty(cpumask));
-        cpus_and(tmp, cpumask, cpu_online_map);
-        BUG_ON(!cpus_equal(tmp, cpumask));
-        BUG_ON(cpu_isset(smp_processor_id(), cpumask));
-        if (!mm)
-                BUG();
+        int sender;
+        union smp_flush_state *f;
 
-        /*
-         * I'm not happy about this global shared spinlock in the
-         * MM hot path, but we'll see how contended it is.
-         * Temporarily this turns IRQs off, so that lockups are
-         * detected by the NMI watchdog.
-         */
-        spin_lock(&tlbstate_lock);
-        
-        flush_mm = mm;
-        flush_va = va;
-        cpus_or(flush_cpumask, cpumask, flush_cpumask);
+        /* Caller has disabled preemption */
+        sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
+        f = &per_cpu(flush_state, sender);
+
+        /* Could avoid this lock when
+           num_online_cpus() <= NUM_INVALIDATE_TLB_VECTORS, but it is
+           probably not worth checking this for a cache-hot lock. */
+        spin_lock(&f->tlbstate_lock);
+
+        f->flush_mm = mm;
+        f->flush_va = va;
+        cpus_or(f->flush_cpumask, cpumask, f->flush_cpumask);
 
         /*
          * We have to send the IPI only to
          * CPUs affected.
          */
-        send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR);
+        send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR_START + sender);
 
-        while (!cpus_empty(flush_cpumask))
-                mb();   /* nothing. lockup detection does not belong here */;
+        while (!cpus_empty(f->flush_cpumask))
+                cpu_relax();
 
-        flush_mm = NULL;
-        flush_va = 0;
-        spin_unlock(&tlbstate_lock);
+        f->flush_mm = NULL;
+        f->flush_va = 0;
+        spin_unlock(&f->tlbstate_lock);
 }
+
+int __cpuinit init_smp_flush(void)
+{
+        int i;
+        for_each_cpu_mask(i, cpu_possible_map) {
+                spin_lock_init(&per_cpu(flush_state.tlbstate_lock, i));
+        }
+        return 0;
+}
+
+core_initcall(init_smp_flush);
         
 void flush_tlb_current_task(void)
 {
diff --git a/include/asm-x86_64/hw_irq.h b/include/asm-x86_64/hw_irq.h
index 2b5cb2865d21..d9212eb4e894 100644
--- a/include/asm-x86_64/hw_irq.h
+++ b/include/asm-x86_64/hw_irq.h
@@ -50,14 +50,15 @@ struct hw_interrupt_type;
  */
 #define SPURIOUS_APIC_VECTOR    0xff
 #define ERROR_APIC_VECTOR       0xfe
-#define INVALIDATE_TLB_VECTOR   0xfd
-#define RESCHEDULE_VECTOR       0xfc
-#define TASK_MIGRATION_VECTOR   0xfb
-#define CALL_FUNCTION_VECTOR    0xfa
-#define KDB_VECTOR      0xf9
-
-#define THERMAL_APIC_VECTOR     0xf0
-
+#define RESCHEDULE_VECTOR       0xfd
+#define CALL_FUNCTION_VECTOR    0xfc
+#define KDB_VECTOR              0xfb    /* reserved for KDB */
+#define THERMAL_APIC_VECTOR     0xfa
+/* 0xf9 free */
+#define INVALIDATE_TLB_VECTOR_END       0xf8
+#define INVALIDATE_TLB_VECTOR_START     0xf0    /* f0-f8 used for TLB flush */
+
+#define NUM_INVALIDATE_TLB_VECTORS      8
 
 /*
  * Local APIC timer IRQ vector is on a different priority level,