1 files changed, 166 insertions, 138 deletions

diff --git a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c
index 0e8977871b1f..89a45a9ddcd4 100644
--- a/arch/i386/kernel/smp.c
+++ b/arch/i386/kernel/smp.c
@@ -165,20 +165,20 @@ void fastcall send_IPI_self(int vector)
 }
 
 /*
- * This is only used on smaller machines.
+ * This is used to send an IPI with no shorthand notation (the destination is
+ * specified in bits 56 to 63 of the ICR).
  */
-void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
+static inline void __send_IPI_dest_field(unsigned long mask, int vector)
 {
-        unsigned long mask = cpus_addr(cpumask)[0];
         unsigned long cfg;
-        unsigned long flags;
 
-        local_irq_save(flags);
-        WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]);
         /*
          * Wait for idle.
          */
-        apic_wait_icr_idle();
+        if (unlikely(vector == NMI_VECTOR))
+                safe_apic_wait_icr_idle();
+        else
+                apic_wait_icr_idle();
                 
         /*
          * prepare target chip field
@@ -195,13 +195,25 @@ void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
          * Send the IPI. The write to APIC_ICR fires this off.
          */
         apic_write_around(APIC_ICR, cfg);
+}
+
+/*
+ * This is only used on smaller machines.
+ */
+void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
+{
+        unsigned long mask = cpus_addr(cpumask)[0];
+        unsigned long flags;
 
+        local_irq_save(flags);
+        WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]);
+        __send_IPI_dest_field(mask, vector);
         local_irq_restore(flags);
 }
 
 void send_IPI_mask_sequence(cpumask_t mask, int vector)
 {
-        unsigned long cfg, flags;
+        unsigned long flags;
         unsigned int query_cpu;
 
         /*
@@ -211,30 +223,10 @@ void send_IPI_mask_sequence(cpumask_t mask, int vector)
          */ 
 
         local_irq_save(flags);
-
         for (query_cpu = 0; query_cpu < NR_CPUS; ++query_cpu) {
                 if (cpu_isset(query_cpu, mask)) {
-                
-                        /*
-                         * Wait for idle.
-                         */
-                        apic_wait_icr_idle();
-                
-                        /*
-                         * prepare target chip field
-                         */
-                        cfg = __prepare_ICR2(cpu_to_logical_apicid(query_cpu));
-                        apic_write_around(APIC_ICR2, cfg);
-                
-                        /*
-                         * program the ICR 
-                         */
-                        cfg = __prepare_ICR(0, vector);
-                        
-                        /*
-                         * Send the IPI. The write to APIC_ICR fires this off.
-                         */
-                        apic_write_around(APIC_ICR, cfg);
+                        __send_IPI_dest_field(cpu_to_logical_apicid(query_cpu),
+                                              vector);
                 }
         }
         local_irq_restore(flags);
@@ -256,7 +248,6 @@ static cpumask_t flush_cpumask;
 static struct mm_struct * flush_mm;
 static unsigned long flush_va;
 static DEFINE_SPINLOCK(tlbstate_lock);
-#define FLUSH_ALL       0xffffffff
 
 /*
  * We cannot call mmdrop() because we are in interrupt context, 
@@ -338,7 +329,7 @@ fastcall void smp_invalidate_interrupt(struct pt_regs *regs)
                  
         if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) {
                 if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) {
-                        if (flush_va == FLUSH_ALL)
+                        if (flush_va == TLB_FLUSH_ALL)
                                 local_flush_tlb();
                         else
                                 __flush_tlb_one(flush_va);
@@ -353,9 +344,11 @@ out:
         put_cpu_no_resched();
 }
 
-static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
-                                                unsigned long va)
+void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
+                             unsigned long va)
 {
+        cpumask_t cpumask = *cpumaskp;
+
         /*
          * A couple of (to be removed) sanity checks:
          *
@@ -366,10 +359,12 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
         BUG_ON(cpu_isset(smp_processor_id(), cpumask));
         BUG_ON(!mm);
 
+#ifdef CONFIG_HOTPLUG_CPU
         /* If a CPU which we ran on has gone down, OK. */
         cpus_and(cpumask, cpumask, cpu_online_map);
-        if (cpus_empty(cpumask))
+        if (unlikely(cpus_empty(cpumask)))
                 return;
+#endif
 
         /*
          * i'm not happy about this global shared spinlock in the
@@ -380,17 +375,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
         
         flush_mm = mm;
         flush_va = va;
-#if NR_CPUS <= BITS_PER_LONG
-        atomic_set_mask(cpumask, &flush_cpumask);
-#else
-        {
-                int k;
-                unsigned long *flush_mask = (unsigned long *)&flush_cpumask;
-                unsigned long *cpu_mask = (unsigned long *)&cpumask;
-                for (k = 0; k < BITS_TO_LONGS(NR_CPUS); ++k)
-                        atomic_set_mask(cpu_mask[k], &flush_mask[k]);
-        }
-#endif
+        cpus_or(flush_cpumask, cpumask, flush_cpumask);
         /*
          * We have to send the IPI only to
          * CPUs affected.
@@ -417,7 +402,7 @@ void flush_tlb_current_task(void)
 
         local_flush_tlb();
         if (!cpus_empty(cpu_mask))
-                flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
+                flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
         preempt_enable();
 }
 
@@ -436,7 +421,7 @@ void flush_tlb_mm (struct mm_struct * mm)
                         leave_mm(smp_processor_id());
         }
         if (!cpus_empty(cpu_mask))
-                flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
+                flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
 
         preempt_enable();
 }
@@ -483,7 +468,7 @@ void flush_tlb_all(void)
  * it goes straight through and wastes no time serializing
  * anything. Worst case is that we lose a reschedule ...
  */
-void smp_send_reschedule(int cpu)
+void native_smp_send_reschedule(int cpu)
 {
         WARN_ON(cpu_is_offline(cpu));
         send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
@@ -515,36 +500,78 @@ void unlock_ipi_call_lock(void)
 
 static struct call_data_struct *call_data;
 
+static void __smp_call_function(void (*func) (void *info), void *info,
+                                int nonatomic, int wait)
+{
+        struct call_data_struct data;
+        int cpus = num_online_cpus() - 1;
+
+        if (!cpus)
+                return;
+
+        data.func = func;
+        data.info = info;
+        atomic_set(&data.started, 0);
+        data.wait = wait;
+        if (wait)
+                atomic_set(&data.finished, 0);
+
+        call_data = &data;
+        mb();
+        
+        /* Send a message to all other CPUs and wait for them to respond */
+        send_IPI_allbutself(CALL_FUNCTION_VECTOR);
+
+        /* Wait for response */
+        while (atomic_read(&data.started) != cpus)
+                cpu_relax();
+
+        if (wait)
+                while (atomic_read(&data.finished) != cpus)
+                        cpu_relax();
+}
+
+
 /**
- * smp_call_function(): Run a function on all other CPUs.
+ * smp_call_function_mask(): Run a function on a set of other CPUs.
+ * @mask: The set of cpus to run on.  Must not include the current cpu.
  * @func: The function to run. This must be fast and non-blocking.
  * @info: An arbitrary pointer to pass to the function.
- * @nonatomic: currently unused.
  * @wait: If true, wait (atomically) until function has completed on other CPUs.
  *
- * Returns 0 on success, else a negative status code. Does not return until
- * remote CPUs are nearly ready to execute <<func>> or are or have executed.
+  * Returns 0 on success, else a negative status code.
+ *
+ * If @wait is true, then returns once @func has returned; otherwise
+ * it returns just before the target cpu calls @func.
  *
  * You must not call this function with disabled interrupts or from a
  * hardware interrupt handler or from a bottom half handler.
  */
-int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
-                        int wait)
+int native_smp_call_function_mask(cpumask_t mask,
+                                  void (*func)(void *), void *info,
+                                  int wait)
 {
         struct call_data_struct data;
+        cpumask_t allbutself;
         int cpus;
 
+        /* Can deadlock when called with interrupts disabled */
+        WARN_ON(irqs_disabled());
+
         /* Holding any lock stops cpus from going down. */
         spin_lock(&call_lock);
-        cpus = num_online_cpus() - 1;
+
+        allbutself = cpu_online_map;
+        cpu_clear(smp_processor_id(), allbutself);
+
+        cpus_and(mask, mask, allbutself);
+        cpus = cpus_weight(mask);
+
         if (!cpus) {
                 spin_unlock(&call_lock);
                 return 0;
         }
 
-        /* Can deadlock when called with interrupts disabled */
-        WARN_ON(irqs_disabled());
-
         data.func = func;
         data.info = info;
         atomic_set(&data.started, 0);
@@ -554,9 +581,12 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
 
         call_data = &data;
         mb();
-        
-        /* Send a message to all other CPUs and wait for them to respond */
-        send_IPI_allbutself(CALL_FUNCTION_VECTOR);
+
+        /* Send a message to other CPUs */
+        if (cpus_equal(mask, allbutself))
+                send_IPI_allbutself(CALL_FUNCTION_VECTOR);
+        else
+                send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
 
         /* Wait for response */
         while (atomic_read(&data.started) != cpus)
@@ -569,15 +599,68 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
 
         return 0;
 }
+
+/**
+ * smp_call_function(): Run a function on all other CPUs.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @nonatomic: Unused.
+ * @wait: If true, wait (atomically) until function has completed on other CPUs.
+ *
+ * Returns 0 on success, else a negative status code.
+ *
+ * If @wait is true, then returns once @func has returned; otherwise
+ * it returns just before the target cpu calls @func.
+ *
+ * You must not call this function with disabled interrupts or from a
+ * hardware interrupt handler or from a bottom half handler.
+ */
+int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
+                      int wait)
+{
+        return smp_call_function_mask(cpu_online_map, func, info, wait);
+}
 EXPORT_SYMBOL(smp_call_function);
 
+/**
+ * smp_call_function_single - Run a function on another CPU
+ * @cpu: The target CPU.  Cannot be the calling CPU.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @nonatomic: Unused.
+ * @wait: If true, wait until function has completed on other CPUs.
+ *
+ * Returns 0 on success, else a negative status code.
+ *
+ * If @wait is true, then returns once @func has returned; otherwise
+ * it returns just before the target cpu calls @func.
+ */
+int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
+                             int nonatomic, int wait)
+{
+        /* prevent preemption and reschedule on another processor */
+        int ret;
+        int me = get_cpu();
+        if (cpu == me) {
+                WARN_ON(1);
+                put_cpu();
+                return -EBUSY;
+        }
+
+        ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
+
+        put_cpu();
+        return ret;
+}
+EXPORT_SYMBOL(smp_call_function_single);
+
 static void stop_this_cpu (void * dummy)
 {
+        local_irq_disable();
         /*
          * Remove this CPU:
          */
         cpu_clear(smp_processor_id(), cpu_online_map);
-        local_irq_disable();
         disable_local_APIC();
         if (cpu_data[smp_processor_id()].hlt_works_ok)
                 for(;;) halt();
@@ -588,13 +671,18 @@ static void stop_this_cpu (void * dummy)
  * this function calls the 'stop' function on all other CPUs in the system.
  */
 
-void smp_send_stop(void)
+void native_smp_send_stop(void)
 {
-        smp_call_function(stop_this_cpu, NULL, 1, 0);
+        /* Don't deadlock on the call lock in panic */
+        int nolock = !spin_trylock(&call_lock);
+        unsigned long flags;
 
-        local_irq_disable();
+        local_irq_save(flags);
+        __smp_call_function(stop_this_cpu, NULL, 0, 0);
+        if (!nolock)
+                spin_unlock(&call_lock);
         disable_local_APIC();
-        local_irq_enable();
+        local_irq_restore(flags);
 }
 
 /*
@@ -633,77 +721,6 @@ fastcall void smp_call_function_interrupt(struct pt_regs *regs)
         }
 }
 
-/*
- * this function sends a 'generic call function' IPI to one other CPU
- * in the system.
- *
- * cpu is a standard Linux logical CPU number.
- */
-static void
-__smp_call_function_single(int cpu, void (*func) (void *info), void *info,
-                                int nonatomic, int wait)
-{
-        struct call_data_struct data;
-        int cpus = 1;
-
-        data.func = func;
-        data.info = info;
-        atomic_set(&data.started, 0);
-        data.wait = wait;
-        if (wait)
-                atomic_set(&data.finished, 0);
-
-        call_data = &data;
-        wmb();
-        /* Send a message to all other CPUs and wait for them to respond */
-        send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNCTION_VECTOR);
-
-        /* Wait for response */
-        while (atomic_read(&data.started) != cpus)
-                cpu_relax();
-
-        if (!wait)
-                return;
-
-        while (atomic_read(&data.finished) != cpus)
-                cpu_relax();
-}
-
-/*
- * smp_call_function_single - Run a function on another CPU
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @nonatomic: Currently unused.
- * @wait: If true, wait until function has completed on other CPUs.
- *
- * Retrurns 0 on success, else a negative status code.
- *
- * Does not return until the remote CPU is nearly ready to execute <func>
- * or is or has executed.
- */
-
-int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
-                        int nonatomic, int wait)
-{
-        /* prevent preemption and reschedule on another processor */
-        int me = get_cpu();
-        if (cpu == me) {
-                WARN_ON(1);
-                put_cpu();
-                return -EBUSY;
-        }
-
-        /* Can deadlock when called with interrupts disabled */
-        WARN_ON(irqs_disabled());
-
-        spin_lock_bh(&call_lock);
-        __smp_call_function_single(cpu, func, info, nonatomic, wait);
-        spin_unlock_bh(&call_lock);
-        put_cpu();
-        return 0;
-}
-EXPORT_SYMBOL(smp_call_function_single);
-
 static int convert_apicid_to_cpu(int apic_id)
 {
         int i;
@@ -730,3 +747,14 @@ int safe_smp_processor_id(void)
 
         return cpuid >= 0 ? cpuid : 0;
 }
+
+struct smp_ops smp_ops = {
+        .smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
+        .smp_prepare_cpus = native_smp_prepare_cpus,
+        .cpu_up = native_cpu_up,
+        .smp_cpus_done = native_smp_cpus_done,
+
+        .smp_send_stop = native_smp_send_stop,
+        .smp_send_reschedule = native_smp_send_reschedule,
+        .smp_call_function_mask = native_smp_call_function_mask,
+};