1 files changed, 81 insertions, 204 deletions

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index bb1a3b1fc87..58d24ef917d 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -2,7 +2,7 @@
  *      x86 SMP booting functions
  *
  *      (c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
- *      (c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
+ *      (c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
  *      Copyright 2001 Andi Kleen, SuSE Labs.
  *
  *      Much of the core SMP work is based on previous work by Thomas Radke, to
@@ -53,7 +53,6 @@
 #include <asm/nmi.h>
 #include <asm/irq.h>
 #include <asm/idle.h>
-#include <asm/smp.h>
 #include <asm/trampoline.h>
 #include <asm/cpu.h>
 #include <asm/numa.h>
@@ -61,13 +60,12 @@
 #include <asm/tlbflush.h>
 #include <asm/mtrr.h>
 #include <asm/vmi.h>
-#include <asm/genapic.h>
+#include <asm/apic.h>
 #include <asm/setup.h>
+#include <asm/uv/uv.h>
 #include <linux/mc146818rtc.h>
 
-#include <mach_apic.h>
-#include <mach_wakecpu.h>
-#include <smpboot_hooks.h>
+#include <asm/smpboot_hooks.h>
 
 #ifdef CONFIG_X86_32
 u8 apicid_2_node[MAX_APICID];
@@ -103,29 +101,20 @@ EXPORT_SYMBOL(smp_num_siblings);
 DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
 
 /* representing HT siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
+DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
 
 /* representing HT and core siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_t, cpu_core_map);
+DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
 EXPORT_PER_CPU_SYMBOL(cpu_core_map);
 
 /* Per CPU bogomips and other parameters */
 DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
 EXPORT_PER_CPU_SYMBOL(cpu_info);
 
-static atomic_t init_deasserted;
-
-
-/* Set if we find a B stepping CPU */
-static int __cpuinitdata smp_b_stepping;
+atomic_t init_deasserted;
 
 #if defined(CONFIG_NUMA) && defined(CONFIG_X86_32)
-
-/* which logical CPUs are on which nodes */
-cpumask_t node_to_cpumask_map[MAX_NUMNODES] __read_mostly =
-                                { [0 ... MAX_NUMNODES-1] = CPU_MASK_NONE };
-EXPORT_SYMBOL(node_to_cpumask_map);
 /* which node each logical CPU is on */
 int cpu_to_node_map[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = 0 };
 EXPORT_SYMBOL(cpu_to_node_map);
@@ -134,7 +123,7 @@ EXPORT_SYMBOL(cpu_to_node_map);
 static void map_cpu_to_node(int cpu, int node)
 {
         printk(KERN_INFO "Mapping cpu %d to node %d\n", cpu, node);
-        cpumask_set_cpu(cpu, &node_to_cpumask_map[node]);
+        cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
         cpu_to_node_map[cpu] = node;
 }
 
@@ -145,7 +134,7 @@ static void unmap_cpu_to_node(int cpu)
 
         printk(KERN_INFO "Unmapping cpu %d from all nodes\n", cpu);
         for (node = 0; node < MAX_NUMNODES; node++)
-                cpumask_clear_cpu(cpu, &node_to_cpumask_map[node]);
+                cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
         cpu_to_node_map[cpu] = 0;
 }
 #else /* !(CONFIG_NUMA && CONFIG_X86_32) */
@@ -163,7 +152,7 @@ static void map_cpu_to_logical_apicid(void)
 {
         int cpu = smp_processor_id();
         int apicid = logical_smp_processor_id();
-        int node = apicid_to_node(apicid);
+        int node = apic->apicid_to_node(apicid);
 
         if (!node_online(node))
                 node = first_online_node;
@@ -196,7 +185,8 @@ static void __cpuinit smp_callin(void)
          * our local APIC.  We have to wait for the IPI or we'll
          * lock up on an APIC access.
          */
-        wait_for_init_deassert(&init_deasserted);
+        if (apic->wait_for_init_deassert)
+                apic->wait_for_init_deassert(&init_deasserted);
 
         /*
          * (This works even if the APIC is not enabled.)
@@ -243,7 +233,8 @@ static void __cpuinit smp_callin(void)
          */
 
         pr_debug("CALLIN, before setup_local_APIC().\n");
-        smp_callin_clear_local_apic();
+        if (apic->smp_callin_clear_local_apic)
+                apic->smp_callin_clear_local_apic();
         setup_local_APIC();
         end_local_APIC_setup();
         map_cpu_to_logical_apicid();
@@ -271,8 +262,6 @@ static void __cpuinit smp_callin(void)
         cpumask_set_cpu(cpuid, cpu_callin_mask);
 }
 
-static int __cpuinitdata unsafe_smp;
-
 /*
  * Activate a secondary processor.
  */
@@ -307,7 +296,7 @@ notrace static void __cpuinit start_secondary(void *unused)
         __flush_tlb_all();
 #endif
 
-        /* This must be done before setting cpu_online_map */
+        /* This must be done before setting cpu_online_mask */
         set_cpu_sibling_map(raw_smp_processor_id());
         wmb();
 
@@ -340,75 +329,22 @@ notrace static void __cpuinit start_secondary(void *unused)
         cpu_idle();
 }
 
-static void __cpuinit smp_apply_quirks(struct cpuinfo_x86 *c)
+#ifdef CONFIG_CPUMASK_OFFSTACK
+/* In this case, llc_shared_map is a pointer to a cpumask. */
+static inline void copy_cpuinfo_x86(struct cpuinfo_x86 *dst,
+                                    const struct cpuinfo_x86 *src)
 {
-        /*
-         * Mask B, Pentium, but not Pentium MMX
-         */
-        if (c->x86_vendor == X86_VENDOR_INTEL &&
-            c->x86 == 5 &&
-            c->x86_mask >= 1 && c->x86_mask <= 4 &&
-            c->x86_model <= 3)
-                /*
-                 * Remember we have B step Pentia with bugs
-                 */
-                smp_b_stepping = 1;
-
-        /*
-         * Certain Athlons might work (for various values of 'work') in SMP
-         * but they are not certified as MP capable.
-         */
-        if ((c->x86_vendor == X86_VENDOR_AMD) && (c->x86 == 6)) {
-
-                if (num_possible_cpus() == 1)
-                        goto valid_k7;
-
-                /* Athlon 660/661 is valid. */
-                if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
-                    (c->x86_mask == 1)))
-                        goto valid_k7;
-
-                /* Duron 670 is valid */
-                if ((c->x86_model == 7) && (c->x86_mask == 0))
-                        goto valid_k7;
-
-                /*
-                 * Athlon 662, Duron 671, and Athlon >model 7 have capability
-                 * bit. It's worth noting that the A5 stepping (662) of some
-                 * Athlon XP's have the MP bit set.
-                 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
-                 * more.
-                 */
-                if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
-                    ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
-                     (c->x86_model > 7))
-                        if (cpu_has_mp)
-                                goto valid_k7;
-
-                /* If we get here, not a certified SMP capable AMD system. */
-                unsafe_smp = 1;
-        }
-
-valid_k7:
-        ;
+        struct cpumask *llc = dst->llc_shared_map;
+        *dst = *src;
+        dst->llc_shared_map = llc;
 }
-
-static void __cpuinit smp_checks(void)
+#else
+static inline void copy_cpuinfo_x86(struct cpuinfo_x86 *dst,
+                                    const struct cpuinfo_x86 *src)
 {
-        if (smp_b_stepping)
-                printk(KERN_WARNING "WARNING: SMP operation may be unreliable"
-                                    "with B stepping processors.\n");
-
-        /*
-         * Don't taint if we are running SMP kernel on a single non-MP
-         * approved Athlon
-         */
-        if (unsafe_smp && num_online_cpus() > 1) {
-                printk(KERN_INFO "WARNING: This combination of AMD"
-                        "processors is not suitable for SMP.\n");
-                add_taint(TAINT_UNSAFE_SMP);
-        }
+        *dst = *src;
 }
+#endif /* CONFIG_CPUMASK_OFFSTACK */
 
 /*
  * The bootstrap kernel entry code has set these up. Save them for
@@ -419,11 +355,10 @@ void __cpuinit smp_store_cpu_info(int id)
 {
         struct cpuinfo_x86 *c = &cpu_data(id);
 
-        *c = boot_cpu_data;
+        copy_cpuinfo_x86(c, &boot_cpu_data);
         c->cpu_index = id;
         if (id != 0)
                 identify_secondary_cpu(c);
-        smp_apply_quirks(c);
 }
 
 
@@ -444,15 +379,15 @@ void __cpuinit set_cpu_sibling_map(int cpu)
                                 cpumask_set_cpu(cpu, cpu_sibling_mask(i));
                                 cpumask_set_cpu(i, cpu_core_mask(cpu));
                                 cpumask_set_cpu(cpu, cpu_core_mask(i));
-                                cpumask_set_cpu(i, &c->llc_shared_map);
-                                cpumask_set_cpu(cpu, &o->llc_shared_map);
+                                cpumask_set_cpu(i, c->llc_shared_map);
+                                cpumask_set_cpu(cpu, o->llc_shared_map);
                         }
                 }
         } else {
                 cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
         }
 
-        cpumask_set_cpu(cpu, &c->llc_shared_map);
+        cpumask_set_cpu(cpu, c->llc_shared_map);
 
         if (current_cpu_data.x86_max_cores == 1) {
                 cpumask_copy(cpu_core_mask(cpu), cpu_sibling_mask(cpu));
@@ -463,8 +398,8 @@ void __cpuinit set_cpu_sibling_map(int cpu)
         for_each_cpu(i, cpu_sibling_setup_mask) {
                 if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
                     per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
-                        cpumask_set_cpu(i, &c->llc_shared_map);
-                        cpumask_set_cpu(cpu, &cpu_data(i).llc_shared_map);
+                        cpumask_set_cpu(i, c->llc_shared_map);
+                        cpumask_set_cpu(cpu, cpu_data(i).llc_shared_map);
                 }
                 if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
                         cpumask_set_cpu(i, cpu_core_mask(cpu));
@@ -502,12 +437,7 @@ const struct cpumask *cpu_coregroup_mask(int cpu)
         if (sched_mc_power_savings || sched_smt_power_savings)
                 return cpu_core_mask(cpu);
         else
-                return &c->llc_shared_map;
-}
-
-cpumask_t cpu_coregroup_map(int cpu)
-{
-        return *cpu_coregroup_mask(cpu);
+                return c->llc_shared_map;
 }
 
 static void impress_friends(void)
@@ -583,7 +513,7 @@ wakeup_secondary_cpu_via_nmi(int logical_apicid, unsigned long start_eip)
         /* Target chip */
         /* Boot on the stack */
         /* Kick the second */
-        apic_icr_write(APIC_DM_NMI | APIC_DEST_LOGICAL, logical_apicid);
+        apic_icr_write(APIC_DM_NMI | apic->dest_logical, logical_apicid);
 
         pr_debug("Waiting for send to finish...\n");
         send_status = safe_apic_wait_icr_idle();
@@ -614,12 +544,6 @@ wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
         unsigned long send_status, accept_status = 0;
         int maxlvt, num_starts, j;
 
-        if (get_uv_system_type() == UV_NON_UNIQUE_APIC) {
-                send_status = uv_wakeup_secondary(phys_apicid, start_eip);
-                atomic_set(&init_deasserted, 1);
-                return send_status;
-        }
-
         maxlvt = lapic_get_maxlvt();
 
         /*
@@ -745,78 +669,23 @@ static void __cpuinit do_fork_idle(struct work_struct *work)
         complete(&c_idle->done);
 }
 
-#ifdef CONFIG_X86_64
-
-/* __ref because it's safe to call free_bootmem when after_bootmem == 0. */
-static void __ref free_bootmem_pda(struct x8664_pda *oldpda)
-{
-        if (!after_bootmem)
-                free_bootmem((unsigned long)oldpda, sizeof(*oldpda));
-}
-
-/*
- * Allocate node local memory for the AP pda.
- *
- * Must be called after the _cpu_pda pointer table is initialized.
- */
-int __cpuinit get_local_pda(int cpu)
-{
-        struct x8664_pda *oldpda, *newpda;
-        unsigned long size = sizeof(struct x8664_pda);
-        int node = cpu_to_node(cpu);
-
-        if (cpu_pda(cpu) && !cpu_pda(cpu)->in_bootmem)
-                return 0;
-
-        oldpda = cpu_pda(cpu);
-        newpda = kmalloc_node(size, GFP_ATOMIC, node);
-        if (!newpda) {
-                printk(KERN_ERR "Could not allocate node local PDA "
-                        "for CPU %d on node %d\n", cpu, node);
-
-                if (oldpda)
-                        return 0;       /* have a usable pda */
-                else
-                        return -1;
-        }
-
-        if (oldpda) {
-                memcpy(newpda, oldpda, size);
-                free_bootmem_pda(oldpda);
-        }
-
-        newpda->in_bootmem = 0;
-        cpu_pda(cpu) = newpda;
-        return 0;
-}
-#endif /* CONFIG_X86_64 */
-
-static int __cpuinit do_boot_cpu(int apicid, int cpu)
 /*
  * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
  * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
- * Returns zero if CPU booted OK, else error code from wakeup_secondary_cpu.
+ * Returns zero if CPU booted OK, else error code from
+ * ->wakeup_secondary_cpu.
  */
+static int __cpuinit do_boot_cpu(int apicid, int cpu)
 {
         unsigned long boot_error = 0;
-        int timeout;
         unsigned long start_ip;
-        unsigned short nmi_high = 0, nmi_low = 0;
+        int timeout;
         struct create_idle c_idle = {
-                .cpu = cpu,
-                .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
+                .cpu    = cpu,
+                .done   = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
         };
-        INIT_WORK(&c_idle.work, do_fork_idle);
 
-#ifdef CONFIG_X86_64
-        /* Allocate node local memory for AP pdas */
-        if (cpu > 0) {
-                boot_error = get_local_pda(cpu);
-                if (boot_error)
-                        goto restore_state;
-                        /* if can't get pda memory, can't start cpu */
-        }
-#endif
+        INIT_WORK(&c_idle.work, do_fork_idle);
 
         alternatives_smp_switch(1);
 
@@ -847,14 +716,16 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
 
         set_idle_for_cpu(cpu, c_idle.idle);
 do_rest:
-#ifdef CONFIG_X86_32
         per_cpu(current_task, cpu) = c_idle.idle;
-        init_gdt(cpu);
+#ifdef CONFIG_X86_32
         /* Stack for startup_32 can be just as for start_secondary onwards */
         irq_ctx_init(cpu);
 #else
-        cpu_pda(cpu)->pcurrent = c_idle.idle;
         clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
+        initial_gs = per_cpu_offset(cpu);
+        per_cpu(kernel_stack, cpu) =
+                (unsigned long)task_stack_page(c_idle.idle) -
+                KERNEL_STACK_OFFSET + THREAD_SIZE;
 #endif
         early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
         initial_code = (unsigned long)start_secondary;
@@ -878,8 +749,6 @@ do_rest:
 
                 pr_debug("Setting warm reset code and vector.\n");
 
-                store_NMI_vector(&nmi_high, &nmi_low);
-
                 smpboot_setup_warm_reset_vector(start_ip);
                 /*
                  * Be paranoid about clearing APIC errors.
@@ -891,9 +760,13 @@ do_rest:
         }
 
         /*
-         * Starting actual IPI sequence...
+         * Kick the secondary CPU. Use the method in the APIC driver
+         * if it's defined - or use an INIT boot APIC message otherwise:
          */
-        boot_error = wakeup_secondary_cpu(apicid, start_ip);
+        if (apic->wakeup_secondary_cpu)
+                boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
+        else
+                boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
 
         if (!boot_error) {
                 /*
@@ -927,13 +800,11 @@ do_rest:
                         else
                                 /* trampoline code not run */
                                 printk(KERN_ERR "Not responding.\n");
-                        if (get_uv_system_type() != UV_NON_UNIQUE_APIC)
-                                inquire_remote_apic(apicid);
+                        if (apic->inquire_remote_apic)
+                                apic->inquire_remote_apic(apicid);
                 }
         }
-#ifdef CONFIG_X86_64
-restore_state:
-#endif
+
         if (boot_error) {
                 /* Try to put things back the way they were before ... */
                 numa_remove_cpu(cpu); /* was set by numa_add_cpu */
@@ -961,7 +832,7 @@ restore_state:
 
 int __cpuinit native_cpu_up(unsigned int cpu)
 {
-        int apicid = cpu_present_to_apicid(cpu);
+        int apicid = apic->cpu_present_to_apicid(cpu);
         unsigned long flags;
         int err;
 
@@ -1033,9 +904,8 @@ int __cpuinit native_cpu_up(unsigned int cpu)
  */
 static __init void disable_smp(void)
 {
-        /* use the read/write pointers to the present and possible maps */
-        cpumask_copy(&cpu_present_map, cpumask_of(0));
-        cpumask_copy(&cpu_possible_map, cpumask_of(0));
+        init_cpu_present(cpumask_of(0));
+        init_cpu_possible(cpumask_of(0));
         smpboot_clear_io_apic_irqs();
 
         if (smp_found_config)
@@ -1054,14 +924,14 @@ static int __init smp_sanity_check(unsigned max_cpus)
 {
         preempt_disable();
 
-#if defined(CONFIG_X86_PC) && defined(CONFIG_X86_32)
+#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
         if (def_to_bigsmp && nr_cpu_ids > 8) {
                 unsigned int cpu;
                 unsigned nr;
 
                 printk(KERN_WARNING
                        "More than 8 CPUs detected - skipping them.\n"
-                       "Use CONFIG_X86_GENERICARCH and CONFIG_X86_BIGSMP.\n");
+                       "Use CONFIG_X86_BIGSMP.\n");
 
                 nr = 0;
                 for_each_present_cpu(cpu) {
@@ -1107,7 +977,7 @@ static int __init smp_sanity_check(unsigned max_cpus)
          * Should not be necessary because the MP table should list the boot
          * CPU too, but we do it for the sake of robustness anyway.
          */
-        if (!check_phys_apicid_present(boot_cpu_physical_apicid)) {
+        if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
                 printk(KERN_NOTICE
                         "weird, boot CPU (#%d) not listed by the BIOS.\n",
                         boot_cpu_physical_apicid);
@@ -1125,6 +995,7 @@ static int __init smp_sanity_check(unsigned max_cpus)
                 printk(KERN_ERR "... forcing use of dummy APIC emulation."
                                 "(tell your hw vendor)\n");
                 smpboot_clear_io_apic();
+                arch_disable_smp_support();
                 return -1;
         }
 
@@ -1166,6 +1037,8 @@ static void __init smp_cpu_index_default(void)
  */
 void __init native_smp_prepare_cpus(unsigned int max_cpus)
 {
+        unsigned int i;
+
         preempt_disable();
         smp_cpu_index_default();
         current_cpu_data = boot_cpu_data;
@@ -1179,11 +1052,19 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
         boot_cpu_logical_apicid = logical_smp_processor_id();
 #endif
         current_thread_info()->cpu = 0;  /* needed? */
+        for_each_possible_cpu(i) {
+                alloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
+                alloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
+                alloc_cpumask_var(&cpu_data(i).llc_shared_map, GFP_KERNEL);
+                cpumask_clear(per_cpu(cpu_core_map, i));
+                cpumask_clear(per_cpu(cpu_sibling_map, i));
+                cpumask_clear(cpu_data(i).llc_shared_map);
+        }
         set_cpu_sibling_map(0);
 
-#ifdef CONFIG_X86_64
         enable_IR_x2apic();
-        setup_apic_routing();
+#ifdef CONFIG_X86_64
+        default_setup_apic_routing();
 #endif
 
         if (smp_sanity_check(max_cpus) < 0) {
@@ -1207,18 +1088,18 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
          */
         setup_local_APIC();
 
-#ifdef CONFIG_X86_64
         /*
          * Enable IO APIC before setting up error vector
          */
         if (!skip_ioapic_setup && nr_ioapics)
                 enable_IO_APIC();
-#endif
+
         end_local_APIC_setup();
 
         map_cpu_to_logical_apicid();
 
-        setup_portio_remap();
+        if (apic->setup_portio_remap)
+                apic->setup_portio_remap();
 
         smpboot_setup_io_apic();
         /*
@@ -1240,10 +1121,7 @@ out:
 void __init native_smp_prepare_boot_cpu(void)
 {
         int me = smp_processor_id();
-#ifdef CONFIG_X86_32
-        init_gdt(me);
-#endif
-        switch_to_new_gdt();
+        switch_to_new_gdt(me);
         /* already set me in cpu_online_mask in boot_cpu_init() */
         cpumask_set_cpu(me, cpu_callout_mask);
         per_cpu(cpu_state, me) = CPU_ONLINE;
@@ -1254,7 +1132,6 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
         pr_debug("Boot done.\n");
 
         impress_friends();
-        smp_checks();
 #ifdef CONFIG_X86_IO_APIC
         setup_ioapic_dest();
 #endif
@@ -1271,11 +1148,11 @@ early_param("possible_cpus", _setup_possible_cpus);
 
 
 /*
- * cpu_possible_map should be static, it cannot change as cpu's
+ * cpu_possible_mask should be static, it cannot change as cpu's
  * are onlined, or offlined. The reason is per-cpu data-structures
  * are allocated by some modules at init time, and dont expect to
  * do this dynamically on cpu arrival/departure.
- * cpu_present_map on the other hand can change dynamically.
+ * cpu_present_mask on the other hand can change dynamically.
  * In case when cpu_hotplug is not compiled, then we resort to current
  * behaviour, which is cpu_possible == cpu_present.
  * - Ashok Raj