[PATCH] i386: Initialize the per-CPU data area

When a CPU is brought up, a PDA and GDT are allocated for it.  The GDT's
__KERNEL_PDA entry is pointed to the allocated PDA memory, so that all
references using this segment descriptor will refer to the PDA.

This patch rearranges CPU initialization a bit, so that the GDT/PDA are set up
as early as possible in cpu_init().  Also for secondary CPUs, GDT+PDA are
preallocated and initialized so all the secondary CPU needs to do is set up
the ldt and load %gs.  This will be important once smp_processor_id() and
current use the PDA.

In all cases, the PDA is set up in head.S, before a CPU starts running C code,
so the PDA is always available.

Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Cc: Chuck Ebbert <76306.1226@compuserve.com>
Cc: Zachary Amsden <zach@vmware.com>
Cc: Jan Beulich <jbeulich@novell.com>
Cc: Andi Kleen <ak@suse.de>
Cc: James Bottomley <James.Bottomley@SteelEye.com>
Cc: Matt Tolentino <matthew.e.tolentino@intel.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>

4 files changed, 172 insertions, 50 deletions

diff --git a/arch/i386/kernel/cpu/common.c b/arch/i386/kernel/cpu/common.c
index 5532fc4e1bf0..2534e25ed745 100644
--- a/arch/i386/kernel/cpu/common.c
+++ b/arch/i386/kernel/cpu/common.c
@@ -18,12 +18,16 @@
 #include <asm/apic.h>
 #include <mach_apic.h>
 #endif
+#include <asm/pda.h>
 
 #include "cpu.h"
 
 DEFINE_PER_CPU(struct Xgt_desc_struct, cpu_gdt_descr);
 EXPORT_PER_CPU_SYMBOL(cpu_gdt_descr);
 
+struct i386_pda *_cpu_pda[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(_cpu_pda);
+
 static int cachesize_override __cpuinitdata = -1;
 static int disable_x86_fxsr __cpuinitdata;
 static int disable_x86_serial_nr __cpuinitdata = 1;
@@ -588,41 +592,16 @@ void __init early_cpu_init(void)
         disable_pse = 1;
 #endif
 }
-/*
- * cpu_init() initializes state that is per-CPU. Some data is already
- * initialized (naturally) in the bootstrap process, such as the GDT
- * and IDT. We reload them nevertheless, this function acts as a
- * 'CPU state barrier', nothing should get across.
- */
-void __cpuinit cpu_init(void)
+
+__cpuinit int alloc_gdt(int cpu)
 {
-        int cpu = smp_processor_id();
-        struct tss_struct * t = &per_cpu(init_tss, cpu);
-        struct thread_struct *thread = &current->thread;
-        struct desc_struct *gdt;
         struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
+        struct desc_struct *gdt;
+        struct i386_pda *pda;
 
-        if (cpu_test_and_set(cpu, cpu_initialized)) {
-                printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
-                for (;;) local_irq_enable();
-        }
-        printk(KERN_INFO "Initializing CPU#%d\n", cpu);
+        gdt = (struct desc_struct *)cpu_gdt_descr->address;
+        pda = cpu_pda(cpu);
 
-        if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
-                clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
-        if (tsc_disable && cpu_has_tsc) {
-                printk(KERN_NOTICE "Disabling TSC...\n");
-                /**** FIX-HPA: DOES THIS REALLY BELONG HERE? ****/
-                clear_bit(X86_FEATURE_TSC, boot_cpu_data.x86_capability);
-                set_in_cr4(X86_CR4_TSD);
-        }
-
-        /* The CPU hotplug case */
-        if (cpu_gdt_descr->address) {
-                gdt = (struct desc_struct *)cpu_gdt_descr->address;
-                memset(gdt, 0, PAGE_SIZE);
-                goto old_gdt;
-        }
         /*
          * This is a horrible hack to allocate the GDT.  The problem
          * is that cpu_init() is called really early for the boot CPU
@@ -630,36 +609,117 @@ void __cpuinit cpu_init(void)
          * CPUs, when bootmem will have gone away
          */
         if (NODE_DATA(0)->bdata->node_bootmem_map) {
-                gdt = (struct desc_struct *)alloc_bootmem_pages(PAGE_SIZE);
-                /* alloc_bootmem_pages panics on failure, so no check */
+                BUG_ON(gdt != NULL || pda != NULL);
+
+                gdt = alloc_bootmem_pages(PAGE_SIZE);
+                pda = alloc_bootmem(sizeof(*pda));
+                /* alloc_bootmem(_pages) panics on failure, so no check */
+
                 memset(gdt, 0, PAGE_SIZE);
+                memset(pda, 0, sizeof(*pda));
         } else {
-                gdt = (struct desc_struct *)get_zeroed_page(GFP_KERNEL);
-                if (unlikely(!gdt)) {
-                        printk(KERN_CRIT "CPU%d failed to allocate GDT\n", cpu);
-                        for (;;)
-                                local_irq_enable();
+                /* GDT and PDA might already have been allocated if
+                   this is a CPU hotplug re-insertion. */
+                if (gdt == NULL)
+                        gdt = (struct desc_struct *)get_zeroed_page(GFP_KERNEL);
+
+                if (pda == NULL)
+                        pda = kmalloc_node(sizeof(*pda), GFP_KERNEL, cpu_to_node(cpu));
+
+                if (unlikely(!gdt || !pda)) {
+                        free_pages((unsigned long)gdt, 0);
+                        kfree(pda);
+                        return 0;
                 }
         }
-old_gdt:
+
+        cpu_gdt_descr->address = (unsigned long)gdt;
+        cpu_pda(cpu) = pda;
+
+        return 1;
+}
+
+/* Initial PDA used by boot CPU */
+struct i386_pda boot_pda = {
+        ._pda = &boot_pda,
+};
+
+/* Initialize the CPU's GDT and PDA.  The boot CPU does this for
+   itself, but secondaries find this done for them. */
+__cpuinit int init_gdt(int cpu, struct task_struct *idle)
+{
+        struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
+        struct desc_struct *gdt;
+        struct i386_pda *pda;
+
+        /* For non-boot CPUs, the GDT and PDA should already have been
+           allocated. */
+        if (!alloc_gdt(cpu)) {
+                printk(KERN_CRIT "CPU%d failed to allocate GDT or PDA\n", cpu);
+                return 0;
+        }
+
+        gdt = (struct desc_struct *)cpu_gdt_descr->address;
+        pda = cpu_pda(cpu);
+
+        BUG_ON(gdt == NULL || pda == NULL);
+
         /*
          * Initialize the per-CPU GDT with the boot GDT,
          * and set up the GDT descriptor:
          */
         memcpy(gdt, cpu_gdt_table, GDT_SIZE);
         cpu_gdt_descr->size = GDT_SIZE - 1;
-        cpu_gdt_descr->address = (unsigned long)gdt;
 
+        pack_descriptor((u32 *)&gdt[GDT_ENTRY_PDA].a,
+                        (u32 *)&gdt[GDT_ENTRY_PDA].b,
+                        (unsigned long)pda, sizeof(*pda) - 1,
+                        0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */
+
+        memset(pda, 0, sizeof(*pda));
+        pda->_pda = pda;
+
+        return 1;
+}
+
+/* Common CPU init for both boot and secondary CPUs */
+static void __cpuinit _cpu_init(int cpu, struct task_struct *curr)
+{
+        struct tss_struct * t = &per_cpu(init_tss, cpu);
+        struct thread_struct *thread = &curr->thread;
+        struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
+
+        /* Reinit these anyway, even if they've already been done (on
+           the boot CPU, this will transition from the boot gdt+pda to
+           the real ones). */
         load_gdt(cpu_gdt_descr);
+
+        if (cpu_test_and_set(cpu, cpu_initialized)) {
+                printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
+                for (;;) local_irq_enable();
+        }
+
+        printk(KERN_INFO "Initializing CPU#%d\n", cpu);
+
+        if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
+                clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
+        if (tsc_disable && cpu_has_tsc) {
+                printk(KERN_NOTICE "Disabling TSC...\n");
+                /**** FIX-HPA: DOES THIS REALLY BELONG HERE? ****/
+                clear_bit(X86_FEATURE_TSC, boot_cpu_data.x86_capability);
+                set_in_cr4(X86_CR4_TSD);
+        }
+
         load_idt(&idt_descr);
 
         /*
          * Set up and load the per-CPU TSS and LDT
          */
         atomic_inc(&init_mm.mm_count);
-        current->active_mm = &init_mm;
-        BUG_ON(current->mm);
-        enter_lazy_tlb(&init_mm, current);
+        curr->active_mm = &init_mm;
+        if (curr->mm)
+                BUG();
+        enter_lazy_tlb(&init_mm, curr);
 
         load_esp0(t, thread);
         set_tss_desc(cpu,t);
@@ -690,6 +750,37 @@ old_gdt:
         mxcsr_feature_mask_init();
 }
 
+/* Entrypoint to initialize secondary CPU */
+void __cpuinit secondary_cpu_init(void)
+{
+        int cpu = smp_processor_id();
+        struct task_struct *curr = current;
+
+        _cpu_init(cpu, curr);
+}
+
+/*
+ * cpu_init() initializes state that is per-CPU. Some data is already
+ * initialized (naturally) in the bootstrap process, such as the GDT
+ * and IDT. We reload them nevertheless, this function acts as a
+ * 'CPU state barrier', nothing should get across.
+ */
+void __cpuinit cpu_init(void)
+{
+        int cpu = smp_processor_id();
+        struct task_struct *curr = current;
+
+        /* Set up the real GDT and PDA, so we can transition from the
+           boot versions. */
+        if (!init_gdt(cpu, curr)) {
+                /* failed to allocate something; not much we can do... */
+                for (;;)
+                        local_irq_enable();
+        }
+
+        _cpu_init(cpu, curr);
+}
+
 #ifdef CONFIG_HOTPLUG_CPU
 void __cpuinit cpu_uninit(void)
 {
diff --git a/arch/i386/kernel/smpboot.c b/arch/i386/kernel/smpboot.c
index 4bb8b77cd65b..095636620fa2 100644
--- a/arch/i386/kernel/smpboot.c
+++ b/arch/i386/kernel/smpboot.c
@@ -52,6 +52,7 @@
 #include <asm/desc.h>
 #include <asm/arch_hooks.h>
 #include <asm/nmi.h>
+#include <asm/pda.h>
 
 #include <mach_apic.h>
 #include <mach_wakecpu.h>
@@ -536,11 +537,11 @@ set_cpu_sibling_map(int cpu)
 static void __devinit start_secondary(void *unused)
 {
         /*
-         * Dont put anything before smp_callin(), SMP
+         * Don't put *anything* before secondary_cpu_init(), SMP
          * booting is too fragile that we want to limit the
          * things done here to the most necessary things.
          */
-        cpu_init();
+        secondary_cpu_init();
         preempt_disable();
         smp_callin();
         while (!cpu_isset(smp_processor_id(), smp_commenced_mask))
@@ -599,13 +600,16 @@ void __devinit initialize_secondary(void)
                 "movl %0,%%esp\n\t"
                 "jmp *%1"
                 :
-                :"r" (current->thread.esp),"r" (current->thread.eip));
+                :"m" (current->thread.esp),"m" (current->thread.eip));
 }
 
+/* Static state in head.S used to set up a CPU */
 extern struct {
         void * esp;
         unsigned short ss;
 } stack_start;
+extern struct i386_pda *start_pda;
+extern struct Xgt_desc_struct cpu_gdt_descr;
 
 #ifdef CONFIG_NUMA
 
@@ -936,9 +940,6 @@ static int __devinit do_boot_cpu(int apicid, int cpu)
         unsigned long start_eip;
         unsigned short nmi_high = 0, nmi_low = 0;
 
-        ++cpucount;
-        alternatives_smp_switch(1);
-
         /*
          * We can't use kernel_thread since we must avoid to
          * reschedule the child.
@@ -946,15 +947,30 @@ static int __devinit do_boot_cpu(int apicid, int cpu)
         idle = alloc_idle_task(cpu);
         if (IS_ERR(idle))
                 panic("failed fork for CPU %d", cpu);
+
+        /* Pre-allocate and initialize the CPU's GDT and PDA so it
+           doesn't have to do any memory allocation during the
+           delicate CPU-bringup phase. */
+        if (!init_gdt(cpu, idle)) {
+                printk(KERN_INFO "Couldn't allocate GDT/PDA for CPU %d\n", cpu);
+                return -1;      /* ? */
+        }
+
         idle->thread.eip = (unsigned long) start_secondary;
         /* start_eip had better be page-aligned! */
         start_eip = setup_trampoline();
 
+        ++cpucount;
+        alternatives_smp_switch(1);
+
         /* So we see what's up   */
         printk("Booting processor %d/%d eip %lx\n", cpu, apicid, start_eip);
         /* Stack for startup_32 can be just as for start_secondary onwards */
         stack_start.esp = (void *) idle->thread.esp;
 
+        start_pda = cpu_pda(cpu);
+        cpu_gdt_descr = per_cpu(cpu_gdt_descr, cpu);
+
         irq_ctx_init(cpu);
 
         x86_cpu_to_apicid[cpu] = apicid;
diff --git a/arch/i386/mach-voyager/voyager_smp.c b/arch/i386/mach-voyager/voyager_smp.c
index f3fea2ad50fe..55428e656a3f 100644
--- a/arch/i386/mach-voyager/voyager_smp.c
+++ b/arch/i386/mach-voyager/voyager_smp.c
@@ -28,6 +28,7 @@
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 #include <asm/arch_hooks.h>
+#include <asm/pda.h>
 
 /* TLB state -- visible externally, indexed physically */
 DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0 };
@@ -422,6 +423,7 @@ find_smp_config(void)
              VOYAGER_SUS_IN_CONTROL_PORT);
 
         current_thread_info()->cpu = boot_cpu_id;
+        write_pda(cpu_number, boot_cpu_id);
 }
 
 /*
@@ -458,7 +460,7 @@ start_secondary(void *unused)
         /* external functions not defined in the headers */
         extern void calibrate_delay(void);
 
-        cpu_init();
+        secondary_cpu_init();
 
         /* OK, we're in the routine */
         ack_CPI(VIC_CPU_BOOT_CPI);
@@ -578,6 +580,15 @@ do_boot_cpu(__u8 cpu)
         /* init_tasks (in sched.c) is indexed logically */
         stack_start.esp = (void *) idle->thread.esp;
 
+        /* Pre-allocate and initialize the CPU's GDT and PDA so it
+           doesn't have to do any memory allocation during the
+           delicate CPU-bringup phase. */
+        if (!init_gdt(cpu, idle)) {
+                printk(KERN_INFO "Couldn't allocate GDT/PDA for CPU %d\n", cpu);
+                cpucount--;
+                return;
+        }
+
         irq_ctx_init(cpu);
 
         /* Note: Don't modify initial ss override */
@@ -1963,4 +1974,5 @@ void __init
 smp_setup_processor_id(void)
 {
         current_thread_info()->cpu = hard_smp_processor_id();
+        write_pda(cpu_number, hard_smp_processor_id());
 }
diff --git a/include/asm-i386/processor.h b/include/asm-i386/processor.h
index e0ddca94d50c..a9f2041c7c87 100644
--- a/include/asm-i386/processor.h
+++ b/include/asm-i386/processor.h
@@ -727,4 +727,7 @@ extern unsigned long boot_option_idle_override;
 extern void enable_sep_cpu(void);
 extern int sysenter_setup(void);
 
+extern int init_gdt(int cpu, struct task_struct *idle);
+extern void secondary_cpu_init(void);
+
 #endif /* __ASM_I386_PROCESSOR_H */