1 files changed, 179 insertions, 70 deletions

diff --git a/arch/i386/kernel/cpu/common.c b/arch/i386/kernel/cpu/common.c
index d9f3e3c31f05..1b34c56f8123 100644
--- a/arch/i386/kernel/cpu/common.c
+++ b/arch/i386/kernel/cpu/common.c
@@ -18,14 +18,15 @@
 #include <asm/apic.h>
 #include <mach_apic.h>
 #endif
+#include <asm/pda.h>
 
 #include "cpu.h"
 
 DEFINE_PER_CPU(struct Xgt_desc_struct, cpu_gdt_descr);
 EXPORT_PER_CPU_SYMBOL(cpu_gdt_descr);
 
-DEFINE_PER_CPU(unsigned char, cpu_16bit_stack[CPU_16BIT_STACK_SIZE]);
-EXPORT_PER_CPU_SYMBOL(cpu_16bit_stack);
+struct i386_pda *_cpu_pda[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(_cpu_pda);
 
 static int cachesize_override __cpuinitdata = -1;
 static int disable_x86_fxsr __cpuinitdata;
@@ -235,29 +236,14 @@ static int __cpuinit have_cpuid_p(void)
         return flag_is_changeable_p(X86_EFLAGS_ID);
 }
 
-/* Do minimum CPU detection early.
-   Fields really needed: vendor, cpuid_level, family, model, mask, cache alignment.
-   The others are not touched to avoid unwanted side effects.
-
-   WARNING: this function is only called on the BP.  Don't add code here
-   that is supposed to run on all CPUs. */
-static void __init early_cpu_detect(void)
+void __init cpu_detect(struct cpuinfo_x86 *c)
 {
-        struct cpuinfo_x86 *c = &boot_cpu_data;
-
-        c->x86_cache_alignment = 32;
-
-        if (!have_cpuid_p())
-                return;
-
         /* Get vendor name */
         cpuid(0x00000000, &c->cpuid_level,
               (int *)&c->x86_vendor_id[0],
               (int *)&c->x86_vendor_id[8],
               (int *)&c->x86_vendor_id[4]);
 
-        get_cpu_vendor(c, 1);
-
         c->x86 = 4;
         if (c->cpuid_level >= 0x00000001) {
                 u32 junk, tfms, cap0, misc;
@@ -274,6 +260,26 @@ static void __init early_cpu_detect(void)
         }
 }
 
+/* Do minimum CPU detection early.
+   Fields really needed: vendor, cpuid_level, family, model, mask, cache alignment.
+   The others are not touched to avoid unwanted side effects.
+
+   WARNING: this function is only called on the BP.  Don't add code here
+   that is supposed to run on all CPUs. */
+static void __init early_cpu_detect(void)
+{
+        struct cpuinfo_x86 *c = &boot_cpu_data;
+
+        c->x86_cache_alignment = 32;
+
+        if (!have_cpuid_p())
+                return;
+
+        cpu_detect(c);
+
+        get_cpu_vendor(c, 1);
+}
+
 static void __cpuinit generic_identify(struct cpuinfo_x86 * c)
 {
         u32 tfms, xlvl;
@@ -308,6 +314,8 @@ static void __cpuinit generic_identify(struct cpuinfo_x86 * c)
 #else
                         c->apicid = (ebx >> 24) & 0xFF;
 #endif
+                        if (c->x86_capability[0] & (1<<19))
+                                c->x86_clflush_size = ((ebx >> 8) & 0xff) * 8;
                 } else {
                         /* Have CPUID level 0 only - unheard of */
                         c->x86 = 4;
@@ -372,6 +380,7 @@ void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
         c->x86_vendor_id[0] = '\0'; /* Unset */
         c->x86_model_id[0] = '\0';  /* Unset */
         c->x86_max_cores = 1;
+        c->x86_clflush_size = 32;
         memset(&c->x86_capability, 0, sizeof c->x86_capability);
 
         if (!have_cpuid_p()) {
@@ -591,42 +600,24 @@ void __init early_cpu_init(void)
         disable_pse = 1;
 #endif
 }
-/*
- * cpu_init() initializes state that is per-CPU. Some data is already
- * initialized (naturally) in the bootstrap process, such as the GDT
- * and IDT. We reload them nevertheless, this function acts as a
- * 'CPU state barrier', nothing should get across.
- */
-void __cpuinit cpu_init(void)
+
+/* Make sure %gs is initialized properly in idle threads */
+struct pt_regs * __devinit idle_regs(struct pt_regs *regs)
 {
-        int cpu = smp_processor_id();
-        struct tss_struct * t = &per_cpu(init_tss, cpu);
-        struct thread_struct *thread = &current->thread;
-        struct desc_struct *gdt;
-        __u32 stk16_off = (__u32)&per_cpu(cpu_16bit_stack, cpu);
-        struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
+        memset(regs, 0, sizeof(struct pt_regs));
+        regs->xgs = __KERNEL_PDA;
+        return regs;
+}
 
-        if (cpu_test_and_set(cpu, cpu_initialized)) {
-                printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
-                for (;;) local_irq_enable();
-        }
-        printk(KERN_INFO "Initializing CPU#%d\n", cpu);
+static __cpuinit int alloc_gdt(int cpu)
+{
+        struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
+        struct desc_struct *gdt;
+        struct i386_pda *pda;
 
-        if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
-                clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
-        if (tsc_disable && cpu_has_tsc) {
-                printk(KERN_NOTICE "Disabling TSC...\n");
-                /**** FIX-HPA: DOES THIS REALLY BELONG HERE? ****/
-                clear_bit(X86_FEATURE_TSC, boot_cpu_data.x86_capability);
-                set_in_cr4(X86_CR4_TSD);
-        }
+        gdt = (struct desc_struct *)cpu_gdt_descr->address;
+        pda = cpu_pda(cpu);
 
-        /* The CPU hotplug case */
-        if (cpu_gdt_descr->address) {
-                gdt = (struct desc_struct *)cpu_gdt_descr->address;
-                memset(gdt, 0, PAGE_SIZE);
-                goto old_gdt;
-        }
         /*
          * This is a horrible hack to allocate the GDT.  The problem
          * is that cpu_init() is called really early for the boot CPU
@@ -634,43 +625,130 @@ void __cpuinit cpu_init(void)
          * CPUs, when bootmem will have gone away
          */
         if (NODE_DATA(0)->bdata->node_bootmem_map) {
-                gdt = (struct desc_struct *)alloc_bootmem_pages(PAGE_SIZE);
-                /* alloc_bootmem_pages panics on failure, so no check */
+                BUG_ON(gdt != NULL || pda != NULL);
+
+                gdt = alloc_bootmem_pages(PAGE_SIZE);
+                pda = alloc_bootmem(sizeof(*pda));
+                /* alloc_bootmem(_pages) panics on failure, so no check */
+
                 memset(gdt, 0, PAGE_SIZE);
+                memset(pda, 0, sizeof(*pda));
         } else {
-                gdt = (struct desc_struct *)get_zeroed_page(GFP_KERNEL);
-                if (unlikely(!gdt)) {
-                        printk(KERN_CRIT "CPU%d failed to allocate GDT\n", cpu);
-                        for (;;)
-                                local_irq_enable();
+                /* GDT and PDA might already have been allocated if
+                   this is a CPU hotplug re-insertion. */
+                if (gdt == NULL)
+                        gdt = (struct desc_struct *)get_zeroed_page(GFP_KERNEL);
+
+                if (pda == NULL)
+                        pda = kmalloc_node(sizeof(*pda), GFP_KERNEL, cpu_to_node(cpu));
+
+                if (unlikely(!gdt || !pda)) {
+                        free_pages((unsigned long)gdt, 0);
+                        kfree(pda);
+                        return 0;
                 }
         }
-old_gdt:
+
+        cpu_gdt_descr->address = (unsigned long)gdt;
+        cpu_pda(cpu) = pda;
+
+        return 1;
+}
+
+/* Initial PDA used by boot CPU */
+struct i386_pda boot_pda = {
+        ._pda = &boot_pda,
+        .cpu_number = 0,
+        .pcurrent = &init_task,
+};
+
+static inline void set_kernel_gs(void)
+{
+        /* Set %gs for this CPU's PDA.  Memory clobber is to create a
+           barrier with respect to any PDA operations, so the compiler
+           doesn't move any before here. */
+        asm volatile ("mov %0, %%gs" : : "r" (__KERNEL_PDA) : "memory");
+}
+
+/* Initialize the CPU's GDT and PDA.  The boot CPU does this for
+   itself, but secondaries find this done for them. */
+__cpuinit int init_gdt(int cpu, struct task_struct *idle)
+{
+        struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
+        struct desc_struct *gdt;
+        struct i386_pda *pda;
+
+        /* For non-boot CPUs, the GDT and PDA should already have been
+           allocated. */
+        if (!alloc_gdt(cpu)) {
+                printk(KERN_CRIT "CPU%d failed to allocate GDT or PDA\n", cpu);
+                return 0;
+        }
+
+        gdt = (struct desc_struct *)cpu_gdt_descr->address;
+        pda = cpu_pda(cpu);
+
+        BUG_ON(gdt == NULL || pda == NULL);
+
         /*
          * Initialize the per-CPU GDT with the boot GDT,
          * and set up the GDT descriptor:
          */
         memcpy(gdt, cpu_gdt_table, GDT_SIZE);
+        cpu_gdt_descr->size = GDT_SIZE - 1;
 
-        /* Set up GDT entry for 16bit stack */
-        *(__u64 *)(&gdt[GDT_ENTRY_ESPFIX_SS]) |=
-                ((((__u64)stk16_off) << 16) & 0x000000ffffff0000ULL) |
-                ((((__u64)stk16_off) << 32) & 0xff00000000000000ULL) |
-                (CPU_16BIT_STACK_SIZE - 1);
+        pack_descriptor((u32 *)&gdt[GDT_ENTRY_PDA].a,
+                        (u32 *)&gdt[GDT_ENTRY_PDA].b,
+                        (unsigned long)pda, sizeof(*pda) - 1,
+                        0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */
 
-        cpu_gdt_descr->size = GDT_SIZE - 1;
-        cpu_gdt_descr->address = (unsigned long)gdt;
+        memset(pda, 0, sizeof(*pda));
+        pda->_pda = pda;
+        pda->cpu_number = cpu;
+        pda->pcurrent = idle;
+
+        return 1;
+}
+
+/* Common CPU init for both boot and secondary CPUs */
+static void __cpuinit _cpu_init(int cpu, struct task_struct *curr)
+{
+        struct tss_struct * t = &per_cpu(init_tss, cpu);
+        struct thread_struct *thread = &curr->thread;
+        struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
 
+        /* Reinit these anyway, even if they've already been done (on
+           the boot CPU, this will transition from the boot gdt+pda to
+           the real ones). */
         load_gdt(cpu_gdt_descr);
+        set_kernel_gs();
+
+        if (cpu_test_and_set(cpu, cpu_initialized)) {
+                printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
+                for (;;) local_irq_enable();
+        }
+
+        printk(KERN_INFO "Initializing CPU#%d\n", cpu);
+
+        if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
+                clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
+        if (tsc_disable && cpu_has_tsc) {
+                printk(KERN_NOTICE "Disabling TSC...\n");
+                /**** FIX-HPA: DOES THIS REALLY BELONG HERE? ****/
+                clear_bit(X86_FEATURE_TSC, boot_cpu_data.x86_capability);
+                set_in_cr4(X86_CR4_TSD);
+        }
+
         load_idt(&idt_descr);
 
         /*
          * Set up and load the per-CPU TSS and LDT
          */
         atomic_inc(&init_mm.mm_count);
-        current->active_mm = &init_mm;
-        BUG_ON(current->mm);
-        enter_lazy_tlb(&init_mm, current);
+        curr->active_mm = &init_mm;
+        if (curr->mm)
+                BUG();
+        enter_lazy_tlb(&init_mm, curr);
 
         load_esp0(t, thread);
         set_tss_desc(cpu,t);
@@ -682,8 +760,8 @@ old_gdt:
         __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
 #endif
 
-        /* Clear %fs and %gs. */
-        asm volatile ("movl %0, %%fs; movl %0, %%gs" : : "r" (0));
+        /* Clear %fs. */
+        asm volatile ("mov %0, %%fs" : : "r" (0));
 
         /* Clear all 6 debug registers: */
         set_debugreg(0, 0);
@@ -701,6 +779,37 @@ old_gdt:
         mxcsr_feature_mask_init();
 }
 
+/* Entrypoint to initialize secondary CPU */
+void __cpuinit secondary_cpu_init(void)
+{
+        int cpu = smp_processor_id();
+        struct task_struct *curr = current;
+
+        _cpu_init(cpu, curr);
+}
+
+/*
+ * cpu_init() initializes state that is per-CPU. Some data is already
+ * initialized (naturally) in the bootstrap process, such as the GDT
+ * and IDT. We reload them nevertheless, this function acts as a
+ * 'CPU state barrier', nothing should get across.
+ */
+void __cpuinit cpu_init(void)
+{
+        int cpu = smp_processor_id();
+        struct task_struct *curr = current;
+
+        /* Set up the real GDT and PDA, so we can transition from the
+           boot versions. */
+        if (!init_gdt(cpu, curr)) {
+                /* failed to allocate something; not much we can do... */
+                for (;;)
+                        local_irq_enable();
+        }
+
+        _cpu_init(cpu, curr);
+}
+
 #ifdef CONFIG_HOTPLUG_CPU
 void __cpuinit cpu_uninit(void)
 {