1 files changed, 176 insertions, 149 deletions

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 8b3bfc4dd708..9fd3137230d4 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -62,8 +62,9 @@
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 #include <asm/mtrr.h>
-#include <asm/vmi.h>
+#include <asm/mwait.h>
 #include <asm/apic.h>
+#include <asm/io_apic.h>
 #include <asm/setup.h>
 #include <asm/uv/uv.h>
 #include <linux/mc146818rtc.h>
@@ -71,10 +72,6 @@
 #include <asm/smpboot_hooks.h>
 #include <asm/i8259.h>
 
-#ifdef CONFIG_X86_32
-u8 apicid_2_node[MAX_APICID];
-#endif
-
 /* State of each CPU */
 DEFINE_PER_CPU(int, cpu_state) = { 0 };
 
@@ -97,12 +94,12 @@ static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
  */
 static DEFINE_MUTEX(x86_cpu_hotplug_driver_mutex);
 
-void cpu_hotplug_driver_lock()
+void cpu_hotplug_driver_lock(void)
 {
         mutex_lock(&x86_cpu_hotplug_driver_mutex);
 }
 
-void cpu_hotplug_driver_unlock()
+void cpu_hotplug_driver_unlock(void)
 {
         mutex_unlock(&x86_cpu_hotplug_driver_mutex);
 }
@@ -130,68 +127,14 @@ EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
 DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
 EXPORT_PER_CPU_SYMBOL(cpu_core_map);
 
+DEFINE_PER_CPU(cpumask_var_t, cpu_llc_shared_map);
+
 /* Per CPU bogomips and other parameters */
 DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
 EXPORT_PER_CPU_SYMBOL(cpu_info);
 
 atomic_t init_deasserted;
 
-#if defined(CONFIG_NUMA) && defined(CONFIG_X86_32)
-/* which node each logical CPU is on */
-int cpu_to_node_map[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = 0 };
-EXPORT_SYMBOL(cpu_to_node_map);
-
-/* set up a mapping between cpu and node. */
-static void map_cpu_to_node(int cpu, int node)
-{
-        printk(KERN_INFO "Mapping cpu %d to node %d\n", cpu, node);
-        cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
-        cpu_to_node_map[cpu] = node;
-}
-
-/* undo a mapping between cpu and node. */
-static void unmap_cpu_to_node(int cpu)
-{
-        int node;
-
-        printk(KERN_INFO "Unmapping cpu %d from all nodes\n", cpu);
-        for (node = 0; node < MAX_NUMNODES; node++)
-                cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
-        cpu_to_node_map[cpu] = 0;
-}
-#else /* !(CONFIG_NUMA && CONFIG_X86_32) */
-#define map_cpu_to_node(cpu, node)      ({})
-#define unmap_cpu_to_node(cpu)  ({})
-#endif
-
-#ifdef CONFIG_X86_32
-static int boot_cpu_logical_apicid;
-
-u8 cpu_2_logical_apicid[NR_CPUS] __read_mostly =
-                                        { [0 ... NR_CPUS-1] = BAD_APICID };
-
-static void map_cpu_to_logical_apicid(void)
-{
-        int cpu = smp_processor_id();
-        int apicid = logical_smp_processor_id();
-        int node = apic->apicid_to_node(apicid);
-
-        if (!node_online(node))
-                node = first_online_node;
-
-        cpu_2_logical_apicid[cpu] = apicid;
-        map_cpu_to_node(cpu, node);
-}
-
-void numa_remove_cpu(int cpu)
-{
-        cpu_2_logical_apicid[cpu] = BAD_APICID;
-        unmap_cpu_to_node(cpu);
-}
-#else
-#define map_cpu_to_logical_apicid()  do {} while (0)
-#endif
-
 /*
  * Report back to the Boot Processor.
  * Running on AP.
@@ -259,7 +202,6 @@ static void __cpuinit smp_callin(void)
                 apic->smp_callin_clear_local_apic();
         setup_local_APIC();
         end_local_APIC_setup();
-        map_cpu_to_logical_apicid();
 
         /*
          * Need to setup vector mappings before we enable interrupts.
@@ -281,6 +223,13 @@ static void __cpuinit smp_callin(void)
          */
         smp_store_cpu_info(cpuid);
 
+        /*
+         * This must be done before setting cpu_online_mask
+         * or calling notify_cpu_starting.
+         */
+        set_cpu_sibling_map(raw_smp_processor_id());
+        wmb();
+
         notify_cpu_starting(cpuid);
 
         /*
@@ -299,23 +248,16 @@ notrace static void __cpuinit start_secondary(void *unused)
          * fragile that we want to limit the things done here to the
          * most necessary things.
          */
+        cpu_init();
+        preempt_disable();
+        smp_callin();
 
 #ifdef CONFIG_X86_32
-        /*
-         * Switch away from the trampoline page-table
-         *
-         * Do this before cpu_init() because it needs to access per-cpu
-         * data which may not be mapped in the trampoline page-table.
-         */
+        /* switch away from the initial page table */
         load_cr3(swapper_pg_dir);
         __flush_tlb_all();
 #endif
 
-        vmi_bringup();
-        cpu_init();
-        preempt_disable();
-        smp_callin();
-
         /* otherwise gcc will move up smp_processor_id before the cpu_init */
         barrier();
         /*
@@ -323,16 +265,6 @@ notrace static void __cpuinit start_secondary(void *unused)
          */
         check_tsc_sync_target();
 
-        if (nmi_watchdog == NMI_IO_APIC) {
-                legacy_pic->chip->mask(0);
-                enable_NMI_through_LVT0();
-                legacy_pic->chip->unmask(0);
-        }
-
-        /* This must be done before setting cpu_online_mask */
-        set_cpu_sibling_map(raw_smp_processor_id());
-        wmb();
-
         /*
          * We need to hold call_lock, so there is no inconsistency
          * between the time smp_call_function() determines number of
@@ -353,6 +285,19 @@ notrace static void __cpuinit start_secondary(void *unused)
         per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
         x86_platform.nmi_init();
 
+        /*
+         * Wait until the cpu which brought this one up marked it
+         * online before enabling interrupts. If we don't do that then
+         * we can end up waking up the softirq thread before this cpu
+         * reached the active state, which makes the scheduler unhappy
+         * and schedule the softirq thread on the wrong cpu. This is
+         * only observable with forced threaded interrupts, but in
+         * theory it could also happen w/o them. It's just way harder
+         * to achieve.
+         */
+        while (!cpumask_test_cpu(smp_processor_id(), cpu_active_mask))
+                cpu_relax();
+
         /* enable local interrupts */
         local_irq_enable();
 
@@ -365,23 +310,6 @@ notrace static void __cpuinit start_secondary(void *unused)
         cpu_idle();
 }
 
-#ifdef CONFIG_CPUMASK_OFFSTACK
-/* In this case, llc_shared_map is a pointer to a cpumask. */
-static inline void copy_cpuinfo_x86(struct cpuinfo_x86 *dst,
-                                    const struct cpuinfo_x86 *src)
-{
-        struct cpumask *llc = dst->llc_shared_map;
-        *dst = *src;
-        dst->llc_shared_map = llc;
-}
-#else
-static inline void copy_cpuinfo_x86(struct cpuinfo_x86 *dst,
-                                    const struct cpuinfo_x86 *src)
-{
-        *dst = *src;
-}
-#endif /* CONFIG_CPUMASK_OFFSTACK */
-
 /*
  * The bootstrap kernel entry code has set these up. Save them for
  * a given CPU
@@ -391,12 +319,22 @@ void __cpuinit smp_store_cpu_info(int id)
 {
         struct cpuinfo_x86 *c = &cpu_data(id);
 
-        copy_cpuinfo_x86(c, &boot_cpu_data);
+        *c = boot_cpu_data;
         c->cpu_index = id;
         if (id != 0)
                 identify_secondary_cpu(c);
 }
 
+static void __cpuinit link_thread_siblings(int cpu1, int cpu2)
+{
+        cpumask_set_cpu(cpu1, cpu_sibling_mask(cpu2));
+        cpumask_set_cpu(cpu2, cpu_sibling_mask(cpu1));
+        cpumask_set_cpu(cpu1, cpu_core_mask(cpu2));
+        cpumask_set_cpu(cpu2, cpu_core_mask(cpu1));
+        cpumask_set_cpu(cpu1, cpu_llc_shared_mask(cpu2));
+        cpumask_set_cpu(cpu2, cpu_llc_shared_mask(cpu1));
+}
+
 
 void __cpuinit set_cpu_sibling_map(int cpu)
 {
@@ -409,23 +347,23 @@ void __cpuinit set_cpu_sibling_map(int cpu)
                 for_each_cpu(i, cpu_sibling_setup_mask) {
                         struct cpuinfo_x86 *o = &cpu_data(i);
 
-                        if (c->phys_proc_id == o->phys_proc_id &&
-                            c->cpu_core_id == o->cpu_core_id) {
-                                cpumask_set_cpu(i, cpu_sibling_mask(cpu));
-                                cpumask_set_cpu(cpu, cpu_sibling_mask(i));
-                                cpumask_set_cpu(i, cpu_core_mask(cpu));
-                                cpumask_set_cpu(cpu, cpu_core_mask(i));
-                                cpumask_set_cpu(i, c->llc_shared_map);
-                                cpumask_set_cpu(cpu, o->llc_shared_map);
+                        if (cpu_has(c, X86_FEATURE_TOPOEXT)) {
+                                if (c->phys_proc_id == o->phys_proc_id &&
+                                    per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i) &&
+                                    c->compute_unit_id == o->compute_unit_id)
+                                        link_thread_siblings(cpu, i);
+                        } else if (c->phys_proc_id == o->phys_proc_id &&
+                                   c->cpu_core_id == o->cpu_core_id) {
+                                link_thread_siblings(cpu, i);
                         }
                 }
         } else {
                 cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
         }
 
-        cpumask_set_cpu(cpu, c->llc_shared_map);
+        cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
 
-        if (current_cpu_data.x86_max_cores == 1) {
+        if (__this_cpu_read(cpu_info.x86_max_cores) == 1) {
                 cpumask_copy(cpu_core_mask(cpu), cpu_sibling_mask(cpu));
                 c->booted_cores = 1;
                 return;
@@ -434,8 +372,8 @@ void __cpuinit set_cpu_sibling_map(int cpu)
         for_each_cpu(i, cpu_sibling_setup_mask) {
                 if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
                     per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
-                        cpumask_set_cpu(i, c->llc_shared_map);
-                        cpumask_set_cpu(cpu, cpu_data(i).llc_shared_map);
+                        cpumask_set_cpu(i, cpu_llc_shared_mask(cpu));
+                        cpumask_set_cpu(cpu, cpu_llc_shared_mask(i));
                 }
                 if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
                         cpumask_set_cpu(i, cpu_core_mask(cpu));
@@ -474,7 +412,7 @@ const struct cpumask *cpu_coregroup_mask(int cpu)
             !(cpu_has(c, X86_FEATURE_AMD_DCM)))
                 return cpu_core_mask(cpu);
         else
-                return c->llc_shared_map;
+                return cpu_llc_shared_mask(cpu);
 }
 
 static void impress_friends(void)
@@ -636,7 +574,7 @@ wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
          * target processor state.
          */
         startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
-                         (unsigned long)stack_start.sp);
+                         stack_start);
 
         /*
          * Run STARTUP IPI loop.
@@ -742,7 +680,7 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
                 .done   = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
         };
 
-        INIT_WORK_ON_STACK(&c_idle.work, do_fork_idle);
+        INIT_WORK_ONSTACK(&c_idle.work, do_fork_idle);
 
         alternatives_smp_switch(1);
 
@@ -774,7 +712,6 @@ do_rest:
 #ifdef CONFIG_X86_32
         /* Stack for startup_32 can be just as for start_secondary onwards */
         irq_ctx_init(cpu);
-        initial_page_table = __pa(&trampoline_pg_dir);
 #else
         clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
         initial_gs = per_cpu_offset(cpu);
@@ -784,10 +721,10 @@ do_rest:
 #endif
         early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
         initial_code = (unsigned long)start_secondary;
-        stack_start.sp = (void *) c_idle.idle->thread.sp;
+        stack_start  = c_idle.idle->thread.sp;
 
         /* start_ip had better be page-aligned! */
-        start_ip = setup_trampoline();
+        start_ip = trampoline_address();
 
         /* So we see what's up */
         announce_cpu(cpu, apicid);
@@ -797,6 +734,8 @@ do_rest:
          * the targeted processor.
          */
 
+        printk(KERN_DEBUG "smpboot cpu %d: start_ip = %lx\n", cpu, start_ip);
+
         atomic_set(&init_deasserted, 0);
 
         if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
@@ -850,8 +789,8 @@ do_rest:
                         pr_debug("CPU%d: has booted.\n", cpu);
                 else {
                         boot_error = 1;
-                        if (*((volatile unsigned char *)trampoline_base)
-                                        == 0xA5)
+                        if (*(volatile u32 *)TRAMPOLINE_SYM(trampoline_status)
+                            == 0xA5A5A5A5)
                                 /* trampoline started but...? */
                                 pr_err("CPU%d: Stuck ??\n", cpu);
                         else
@@ -877,7 +816,7 @@ do_rest:
         }
 
         /* mark "stuck" area as not stuck */
-        *((volatile unsigned long *)trampoline_base) = 0;
+        *(volatile u32 *)TRAMPOLINE_SYM(trampoline_status) = 0;
 
         if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
                 /*
@@ -923,7 +862,6 @@ int __cpuinit native_cpu_up(unsigned int cpu)
         per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 
         err = do_boot_cpu(apicid, cpu);
-
         if (err) {
                 pr_debug("do_boot_cpu failed %d\n", err);
                 return -EIO;
@@ -945,6 +883,14 @@ int __cpuinit native_cpu_up(unsigned int cpu)
         return 0;
 }
 
+/**
+ * arch_disable_smp_support() - disables SMP support for x86 at runtime
+ */
+void arch_disable_smp_support(void)
+{
+        disable_ioapic_support();
+}
+
 /*
  * Fall back to non SMP mode after errors.
  *
@@ -960,7 +906,6 @@ static __init void disable_smp(void)
                 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
         else
                 physid_set_mask_of_physid(0, &phys_cpu_present_map);
-        map_cpu_to_logical_apicid();
         cpumask_set_cpu(0, cpu_sibling_mask(0));
         cpumask_set_cpu(0, cpu_core_mask(0));
 }
@@ -1045,7 +990,7 @@ static int __init smp_sanity_check(unsigned max_cpus)
                                 "(tell your hw vendor)\n");
                 }
                 smpboot_clear_io_apic();
-                arch_disable_smp_support();
+                disable_ioapic_support();
                 return -1;
         }
 
@@ -1058,11 +1003,9 @@ static int __init smp_sanity_check(unsigned max_cpus)
                 printk(KERN_INFO "SMP mode deactivated.\n");
                 smpboot_clear_io_apic();
 
-                localise_nmi_watchdog();
-
                 connect_bsp_APIC();
                 setup_local_APIC();
-                end_local_APIC_setup();
+                bsp_end_local_APIC_setup();
                 return -1;
         }
 
@@ -1091,26 +1034,22 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 
         preempt_disable();
         smp_cpu_index_default();
-        current_cpu_data = boot_cpu_data;
-        cpumask_copy(cpu_callin_mask, cpumask_of(0));
-        mb();
+
         /*
          * Setup boot CPU information
          */
         smp_store_cpu_info(0); /* Final full version of the data */
-#ifdef CONFIG_X86_32
-        boot_cpu_logical_apicid = logical_smp_processor_id();
-#endif
+        cpumask_copy(cpu_callin_mask, cpumask_of(0));
+        mb();
+
         current_thread_info()->cpu = 0;  /* needed? */
         for_each_possible_cpu(i) {
                 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
                 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
-                zalloc_cpumask_var(&cpu_data(i).llc_shared_map, GFP_KERNEL);
+                zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
         }
         set_cpu_sibling_map(0);
 
-        enable_IR_x2apic();
-        default_setup_apic_routing();
 
         if (smp_sanity_check(max_cpus) < 0) {
                 printk(KERN_INFO "SMP disabled\n");
@@ -1118,6 +1057,8 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
                 goto out;
         }
 
+        default_setup_apic_routing();
+
         preempt_disable();
         if (read_apic_id() != boot_cpu_physical_apicid) {
                 panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
@@ -1139,9 +1080,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
         if (!skip_ioapic_setup && nr_ioapics)
                 enable_IO_APIC();
 
-        end_local_APIC_setup();
-
-        map_cpu_to_logical_apicid();
+        bsp_end_local_APIC_setup();
 
         if (apic->setup_portio_remap)
                 apic->setup_portio_remap();
@@ -1163,6 +1102,20 @@ out:
         preempt_enable();
 }
 
+void arch_disable_nonboot_cpus_begin(void)
+{
+        /*
+         * Avoid the smp alternatives switch during the disable_nonboot_cpus().
+         * In the suspend path, we will be back in the SMP mode shortly anyways.
+         */
+        skip_smp_alternatives = true;
+}
+
+void arch_disable_nonboot_cpus_end(void)
+{
+        skip_smp_alternatives = false;
+}
+
 void arch_enable_nonboot_cpus_begin(void)
 {
         set_mtrr_aps_delayed_init();
@@ -1193,7 +1146,6 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
 #ifdef CONFIG_X86_IO_APIC
         setup_ioapic_dest();
 #endif
-        check_nmi_watchdog();
         mtrr_aps_init();
 }
 
@@ -1338,8 +1290,6 @@ int native_cpu_disable(void)
         if (cpu == 0)
                 return -EBUSY;
 
-        if (nmi_watchdog == NMI_LOCAL_APIC)
-                stop_apic_nmi_watchdog(NULL);
         clear_local_APIC();
 
         cpu_disable_common();
@@ -1370,12 +1320,11 @@ void play_dead_common(void)
 {
         idle_task_exit();
         reset_lazy_tlbstate();
-        irq_ctx_exit(raw_smp_processor_id());
-        c1e_remove_cpu(raw_smp_processor_id());
+        amd_e400_remove_cpu(raw_smp_processor_id());
 
         mb();
         /* Ack it */
-        __get_cpu_var(cpu_state) = CPU_DEAD;
+        __this_cpu_write(cpu_state, CPU_DEAD);
 
         /*
          * With physical CPU hotplug, we should halt the cpu
@@ -1383,11 +1332,89 @@ void play_dead_common(void)
         local_irq_disable();
 }
 
+/*
+ * We need to flush the caches before going to sleep, lest we have
+ * dirty data in our caches when we come back up.
+ */
+static inline void mwait_play_dead(void)
+{
+        unsigned int eax, ebx, ecx, edx;
+        unsigned int highest_cstate = 0;
+        unsigned int highest_subcstate = 0;
+        int i;
+        void *mwait_ptr;
+        struct cpuinfo_x86 *c = __this_cpu_ptr(&cpu_info);
+
+        if (!(this_cpu_has(X86_FEATURE_MWAIT) && mwait_usable(c)))
+                return;
+        if (!this_cpu_has(X86_FEATURE_CLFLSH))
+                return;
+        if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
+                return;
+
+        eax = CPUID_MWAIT_LEAF;
+        ecx = 0;
+        native_cpuid(&eax, &ebx, &ecx, &edx);
+
+        /*
+         * eax will be 0 if EDX enumeration is not valid.
+         * Initialized below to cstate, sub_cstate value when EDX is valid.
+         */
+        if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
+                eax = 0;
+        } else {
+                edx >>= MWAIT_SUBSTATE_SIZE;
+                for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
+                        if (edx & MWAIT_SUBSTATE_MASK) {
+                                highest_cstate = i;
+                                highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
+                        }
+                }
+                eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
+                        (highest_subcstate - 1);
+        }
+
+        /*
+         * This should be a memory location in a cache line which is
+         * unlikely to be touched by other processors.  The actual
+         * content is immaterial as it is not actually modified in any way.
+         */
+        mwait_ptr = &current_thread_info()->flags;
+
+        wbinvd();
+
+        while (1) {
+                /*
+                 * The CLFLUSH is a workaround for erratum AAI65 for
+                 * the Xeon 7400 series.  It's not clear it is actually
+                 * needed, but it should be harmless in either case.
+                 * The WBINVD is insufficient due to the spurious-wakeup
+                 * case where we return around the loop.
+                 */
+                clflush(mwait_ptr);
+                __monitor(mwait_ptr, 0, 0);
+                mb();
+                __mwait(eax, 0);
+        }
+}
+
+static inline void hlt_play_dead(void)
+{
+        if (__this_cpu_read(cpu_info.x86) >= 4)
+                wbinvd();
+
+        while (1) {
+                native_halt();
+        }
+}
+
 void native_play_dead(void)
 {
         play_dead_common();
         tboot_shutdown(TB_SHUTDOWN_WFS);
-        wbinvd_halt();
+
+        mwait_play_dead();      /* Only returns on failure */
+        hlt_play_dead();
 }
 
 #else /* ... !CONFIG_HOTPLUG_CPU */