aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorRusty Russell <rusty@rustcorp.com.au>2007-05-02 13:27:10 -0400
committerAndi Kleen <andi@basil.nowhere.org>2007-05-02 13:27:10 -0400
commitbf50467204b435421d8de33ad080fa46c6f3d50b (patch)
tree87a27c6f23f28d0e7dacee4ac45986e897d244a8 /arch
parentae1ee11be77f51cedb6c569887dddc70c163ab6d (diff)
[PATCH] i386: Use per-cpu GDT immediately upon boot
Now we are no longer dynamically allocating the GDT, we don't need the "cpu_gdt_table" at all: we can switch straight from "boot_gdt_table" to the per-cpu GDT. This means initializing the cpu_gdt array in C. The boot CPU uses the per-cpu var directly, then in smp_prepare_cpus() it switches to the per-cpu copy just allocated. For secondary CPUs, the early_gdt_descr is set to point directly to their per-cpu copy. For UP the code is very simple: it keeps using the "per-cpu" GDT as per SMP, but we never have to move. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> Signed-off-by: Andi Kleen <ak@suse.de> Cc: Andi Kleen <ak@suse.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Diffstat (limited to 'arch')
-rw-r--r--arch/i386/kernel/cpu/common.c72
-rw-r--r--arch/i386/kernel/head.S55
-rw-r--r--arch/i386/kernel/smpboot.c59
-rw-r--r--arch/i386/mach-voyager/voyager_smp.c6
4 files changed, 75 insertions, 117 deletions
diff --git a/arch/i386/kernel/cpu/common.c b/arch/i386/kernel/cpu/common.c
index 2335f4464ead..fd6b079f7609 100644
--- a/arch/i386/kernel/cpu/common.c
+++ b/arch/i386/kernel/cpu/common.c
@@ -25,7 +25,33 @@
25DEFINE_PER_CPU(struct Xgt_desc_struct, cpu_gdt_descr); 25DEFINE_PER_CPU(struct Xgt_desc_struct, cpu_gdt_descr);
26EXPORT_PER_CPU_SYMBOL(cpu_gdt_descr); 26EXPORT_PER_CPU_SYMBOL(cpu_gdt_descr);
27 27
28DEFINE_PER_CPU(struct desc_struct, cpu_gdt[GDT_ENTRIES]); 28DEFINE_PER_CPU(struct desc_struct, cpu_gdt[GDT_ENTRIES]) = {
29 [GDT_ENTRY_KERNEL_CS] = { 0x0000ffff, 0x00cf9a00 },
30 [GDT_ENTRY_KERNEL_DS] = { 0x0000ffff, 0x00cf9200 },
31 [GDT_ENTRY_DEFAULT_USER_CS] = { 0x0000ffff, 0x00cffa00 },
32 [GDT_ENTRY_DEFAULT_USER_DS] = { 0x0000ffff, 0x00cff200 },
33 /*
34 * Segments used for calling PnP BIOS have byte granularity.
35 * They code segments and data segments have fixed 64k limits,
36 * the transfer segment sizes are set at run time.
37 */
38 [GDT_ENTRY_PNPBIOS_CS32] = { 0x0000ffff, 0x00409a00 },/* 32-bit code */
39 [GDT_ENTRY_PNPBIOS_CS16] = { 0x0000ffff, 0x00009a00 },/* 16-bit code */
40 [GDT_ENTRY_PNPBIOS_DS] = { 0x0000ffff, 0x00009200 }, /* 16-bit data */
41 [GDT_ENTRY_PNPBIOS_TS1] = { 0x00000000, 0x00009200 },/* 16-bit data */
42 [GDT_ENTRY_PNPBIOS_TS2] = { 0x00000000, 0x00009200 },/* 16-bit data */
43 /*
44 * The APM segments have byte granularity and their bases
45 * are set at run time. All have 64k limits.
46 */
47 [GDT_ENTRY_APMBIOS_BASE] = { 0x0000ffff, 0x00409a00 },/* 32-bit code */
48 /* 16-bit code */
49 [GDT_ENTRY_APMBIOS_BASE+1] = { 0x0000ffff, 0x00009a00 },
50 [GDT_ENTRY_APMBIOS_BASE+2] = { 0x0000ffff, 0x00409200 }, /* data */
51
52 [GDT_ENTRY_ESPFIX_SS] = { 0x00000000, 0x00c09200 },
53 [GDT_ENTRY_PDA] = { 0x00000000, 0x00c09200 }, /* set in setup_pda */
54};
29 55
30DEFINE_PER_CPU(struct i386_pda, _cpu_pda); 56DEFINE_PER_CPU(struct i386_pda, _cpu_pda);
31EXPORT_PER_CPU_SYMBOL(_cpu_pda); 57EXPORT_PER_CPU_SYMBOL(_cpu_pda);
@@ -618,46 +644,6 @@ struct i386_pda boot_pda = {
618 .pcurrent = &init_task, 644 .pcurrent = &init_task,
619}; 645};
620 646
621static inline void set_kernel_fs(void)
622{
623 /* Set %fs for this CPU's PDA. Memory clobber is to create a
624 barrier with respect to any PDA operations, so the compiler
625 doesn't move any before here. */
626 asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_PDA) : "memory");
627}
628
629/* Initialize the CPU's GDT and PDA. This is either the boot CPU doing itself
630 (still using cpu_gdt_table), or a CPU doing it for a secondary which
631 will soon come up. */
632__cpuinit void init_gdt(int cpu, struct task_struct *idle)
633{
634 struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
635 struct desc_struct *gdt = per_cpu(cpu_gdt, cpu);
636 struct i386_pda *pda = &per_cpu(_cpu_pda, cpu);
637
638 memcpy(gdt, cpu_gdt_table, GDT_SIZE);
639 cpu_gdt_descr->address = (unsigned long)gdt;
640 cpu_gdt_descr->size = GDT_SIZE - 1;
641
642 pack_descriptor((u32 *)&gdt[GDT_ENTRY_PDA].a,
643 (u32 *)&gdt[GDT_ENTRY_PDA].b,
644 (unsigned long)pda, sizeof(*pda) - 1,
645 0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */
646
647 memset(pda, 0, sizeof(*pda));
648 pda->_pda = pda;
649 pda->cpu_number = cpu;
650 pda->pcurrent = idle;
651}
652
653void __cpuinit cpu_set_gdt(int cpu)
654{
655 struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
656
657 load_gdt(cpu_gdt_descr);
658 set_kernel_fs();
659}
660
661/* Common CPU init for both boot and secondary CPUs */ 647/* Common CPU init for both boot and secondary CPUs */
662static void __cpuinit _cpu_init(int cpu, struct task_struct *curr) 648static void __cpuinit _cpu_init(int cpu, struct task_struct *curr)
663{ 649{
@@ -740,10 +726,6 @@ void __cpuinit cpu_init(void)
740 int cpu = smp_processor_id(); 726 int cpu = smp_processor_id();
741 struct task_struct *curr = current; 727 struct task_struct *curr = current;
742 728
743 /* Set up the real GDT and PDA, so we can transition from the
744 boot_gdt_table & boot_pda. */
745 init_gdt(cpu, curr);
746 cpu_set_gdt(cpu);
747 _cpu_init(cpu, curr); 729 _cpu_init(cpu, curr);
748} 730}
749 731
diff --git a/arch/i386/kernel/head.S b/arch/i386/kernel/head.S
index cb185f40c282..633fd2f47429 100644
--- a/arch/i386/kernel/head.S
+++ b/arch/i386/kernel/head.S
@@ -599,7 +599,7 @@ idt_descr:
599 .word 0 # 32 bit align gdt_desc.address 599 .word 0 # 32 bit align gdt_desc.address
600ENTRY(early_gdt_descr) 600ENTRY(early_gdt_descr)
601 .word GDT_ENTRIES*8-1 601 .word GDT_ENTRIES*8-1
602 .long cpu_gdt_table 602 .long per_cpu__cpu_gdt /* Overwritten for secondary CPUs */
603 603
604/* 604/*
605 * The boot_gdt_table must mirror the equivalent in setup.S and is 605 * The boot_gdt_table must mirror the equivalent in setup.S and is
@@ -610,56 +610,3 @@ ENTRY(boot_gdt_table)
610 .fill GDT_ENTRY_BOOT_CS,8,0 610 .fill GDT_ENTRY_BOOT_CS,8,0
611 .quad 0x00cf9a000000ffff /* kernel 4GB code at 0x00000000 */ 611 .quad 0x00cf9a000000ffff /* kernel 4GB code at 0x00000000 */
612 .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */ 612 .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */
613
614/*
615 * The Global Descriptor Table contains 32 quadwords, per-CPU.
616 */
617 .align L1_CACHE_BYTES
618ENTRY(cpu_gdt_table)
619 .quad 0x0000000000000000 /* NULL descriptor */
620 .quad 0x0000000000000000 /* 0x0b reserved */
621 .quad 0x0000000000000000 /* 0x13 reserved */
622 .quad 0x0000000000000000 /* 0x1b reserved */
623 .quad 0x0000000000000000 /* 0x20 unused */
624 .quad 0x0000000000000000 /* 0x28 unused */
625 .quad 0x0000000000000000 /* 0x33 TLS entry 1 */
626 .quad 0x0000000000000000 /* 0x3b TLS entry 2 */
627 .quad 0x0000000000000000 /* 0x43 TLS entry 3 */
628 .quad 0x0000000000000000 /* 0x4b reserved */
629 .quad 0x0000000000000000 /* 0x53 reserved */
630 .quad 0x0000000000000000 /* 0x5b reserved */
631
632 .quad 0x00cf9a000000ffff /* 0x60 kernel 4GB code at 0x00000000 */
633 .quad 0x00cf92000000ffff /* 0x68 kernel 4GB data at 0x00000000 */
634 .quad 0x00cffa000000ffff /* 0x73 user 4GB code at 0x00000000 */
635 .quad 0x00cff2000000ffff /* 0x7b user 4GB data at 0x00000000 */
636
637 .quad 0x0000000000000000 /* 0x80 TSS descriptor */
638 .quad 0x0000000000000000 /* 0x88 LDT descriptor */
639
640 /*
641 * Segments used for calling PnP BIOS have byte granularity.
642 * The code segments and data segments have fixed 64k limits,
643 * the transfer segment sizes are set at run time.
644 */
645 .quad 0x00409a000000ffff /* 0x90 32-bit code */
646 .quad 0x00009a000000ffff /* 0x98 16-bit code */
647 .quad 0x000092000000ffff /* 0xa0 16-bit data */
648 .quad 0x0000920000000000 /* 0xa8 16-bit data */
649 .quad 0x0000920000000000 /* 0xb0 16-bit data */
650
651 /*
652 * The APM segments have byte granularity and their bases
653 * are set at run time. All have 64k limits.
654 */
655 .quad 0x00409a000000ffff /* 0xb8 APM CS code */
656 .quad 0x00009a000000ffff /* 0xc0 APM CS 16 code (16 bit) */
657 .quad 0x004092000000ffff /* 0xc8 APM DS data */
658
659 .quad 0x00c0920000000000 /* 0xd0 - ESPFIX SS */
660 .quad 0x00cf92000000ffff /* 0xd8 - PDA */
661 .quad 0x0000000000000000 /* 0xe0 - unused */
662 .quad 0x0000000000000000 /* 0xe8 - unused */
663 .quad 0x0000000000000000 /* 0xf0 - unused */
664 .quad 0x0000000000000000 /* 0xf8 - GDT entry 31: double-fault TSS */
665
diff --git a/arch/i386/kernel/smpboot.c b/arch/i386/kernel/smpboot.c
index b36a5f174cc9..a9447c3e86dd 100644
--- a/arch/i386/kernel/smpboot.c
+++ b/arch/i386/kernel/smpboot.c
@@ -440,12 +440,6 @@ static void __cpuinit start_secondary(void *unused)
440void __devinit initialize_secondary(void) 440void __devinit initialize_secondary(void)
441{ 441{
442 /* 442 /*
443 * switch to the per CPU GDT we already set up
444 * in do_boot_cpu()
445 */
446 cpu_set_gdt(current_thread_info()->cpu);
447
448 /*
449 * We don't actually need to load the full TSS, 443 * We don't actually need to load the full TSS,
450 * basically just the stack pointer and the eip. 444 * basically just the stack pointer and the eip.
451 */ 445 */
@@ -787,6 +781,32 @@ static inline struct task_struct * alloc_idle_task(int cpu)
787#define alloc_idle_task(cpu) fork_idle(cpu) 781#define alloc_idle_task(cpu) fork_idle(cpu)
788#endif 782#endif
789 783
784/* Initialize the CPU's GDT. This is either the boot CPU doing itself
785 (still using the master per-cpu area), or a CPU doing it for a
786 secondary which will soon come up. */
787static __cpuinit void init_gdt(int cpu, struct task_struct *idle)
788{
789 struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
790 struct desc_struct *gdt = per_cpu(cpu_gdt, cpu);
791 struct i386_pda *pda = &per_cpu(_cpu_pda, cpu);
792
793 cpu_gdt_descr->address = (unsigned long)gdt;
794 cpu_gdt_descr->size = GDT_SIZE - 1;
795
796 pack_descriptor((u32 *)&gdt[GDT_ENTRY_PDA].a,
797 (u32 *)&gdt[GDT_ENTRY_PDA].b,
798 (unsigned long)pda, sizeof(*pda) - 1,
799 0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */
800
801 memset(pda, 0, sizeof(*pda));
802 pda->_pda = pda;
803 pda->cpu_number = cpu;
804 pda->pcurrent = idle;
805}
806
807/* Defined in head.S */
808extern struct Xgt_desc_struct early_gdt_descr;
809
790static int __cpuinit do_boot_cpu(int apicid, int cpu) 810static int __cpuinit do_boot_cpu(int apicid, int cpu)
791/* 811/*
792 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad 812 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
@@ -809,6 +829,8 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
809 panic("failed fork for CPU %d", cpu); 829 panic("failed fork for CPU %d", cpu);
810 830
811 init_gdt(cpu, idle); 831 init_gdt(cpu, idle);
832 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
833 start_pda = cpu_pda(cpu);
812 834
813 idle->thread.eip = (unsigned long) start_secondary; 835 idle->thread.eip = (unsigned long) start_secondary;
814 /* start_eip had better be page-aligned! */ 836 /* start_eip had better be page-aligned! */
@@ -1161,13 +1183,26 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
1161 smp_boot_cpus(max_cpus); 1183 smp_boot_cpus(max_cpus);
1162} 1184}
1163 1185
1164void __devinit smp_prepare_boot_cpu(void) 1186/* Current gdt points %fs at the "master" per-cpu area: after this,
1187 * it's on the real one. */
1188static inline void switch_to_new_gdt(void)
1165{ 1189{
1166 cpu_set(smp_processor_id(), cpu_online_map); 1190 load_gdt(&per_cpu(cpu_gdt_descr, smp_processor_id()));
1167 cpu_set(smp_processor_id(), cpu_callout_map); 1191 asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_PDA) : "memory");
1168 cpu_set(smp_processor_id(), cpu_present_map); 1192}
1169 cpu_set(smp_processor_id(), cpu_possible_map); 1193
1170 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 1194void __init smp_prepare_boot_cpu(void)
1195{
1196 unsigned int cpu = smp_processor_id();
1197
1198 init_gdt(cpu, current);
1199 switch_to_new_gdt();
1200
1201 cpu_set(cpu, cpu_online_map);
1202 cpu_set(cpu, cpu_callout_map);
1203 cpu_set(cpu, cpu_present_map);
1204 cpu_set(cpu, cpu_possible_map);
1205 __get_cpu_var(cpu_state) = CPU_ONLINE;
1171} 1206}
1172 1207
1173#ifdef CONFIG_HOTPLUG_CPU 1208#ifdef CONFIG_HOTPLUG_CPU
diff --git a/arch/i386/mach-voyager/voyager_smp.c b/arch/i386/mach-voyager/voyager_smp.c
index 15d8132d4f5e..b9ce33c0c202 100644
--- a/arch/i386/mach-voyager/voyager_smp.c
+++ b/arch/i386/mach-voyager/voyager_smp.c
@@ -765,12 +765,6 @@ initialize_secondary(void)
765#endif 765#endif
766 766
767 /* 767 /*
768 * switch to the per CPU GDT we already set up
769 * in do_boot_cpu()
770 */
771 cpu_set_gdt(current_thread_info()->cpu);
772
773 /*
774 * We don't actually need to load the full TSS, 768 * We don't actually need to load the full TSS,
775 * basically just the stack pointer and the eip. 769 * basically just the stack pointer and the eip.
776 */ 770 */