diff options
author | Rusty Russell <rusty@rustcorp.com.au> | 2007-05-02 13:27:10 -0400 |
---|---|---|
committer | Andi Kleen <andi@basil.nowhere.org> | 2007-05-02 13:27:10 -0400 |
commit | bf50467204b435421d8de33ad080fa46c6f3d50b (patch) | |
tree | 87a27c6f23f28d0e7dacee4ac45986e897d244a8 | |
parent | ae1ee11be77f51cedb6c569887dddc70c163ab6d (diff) |
[PATCH] i386: Use per-cpu GDT immediately upon boot
Now we are no longer dynamically allocating the GDT, we don't need the
"cpu_gdt_table" at all: we can switch straight from "boot_gdt_table" to the
per-cpu GDT. This means initializing the cpu_gdt array in C.
The boot CPU uses the per-cpu var directly, then in smp_prepare_cpus() it
switches to the per-cpu copy just allocated. For secondary CPUs, the
early_gdt_descr is set to point directly to their per-cpu copy.
For UP the code is very simple: it keeps using the "per-cpu" GDT as per SMP,
but we never have to move.
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Andi Kleen <ak@suse.de>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
-rw-r--r-- | arch/i386/kernel/cpu/common.c | 72 | ||||
-rw-r--r-- | arch/i386/kernel/head.S | 55 | ||||
-rw-r--r-- | arch/i386/kernel/smpboot.c | 59 | ||||
-rw-r--r-- | arch/i386/mach-voyager/voyager_smp.c | 6 | ||||
-rw-r--r-- | include/asm-i386/desc.h | 2 | ||||
-rw-r--r-- | include/asm-i386/processor.h | 1 |
6 files changed, 75 insertions, 120 deletions
diff --git a/arch/i386/kernel/cpu/common.c b/arch/i386/kernel/cpu/common.c index 2335f4464ead..fd6b079f7609 100644 --- a/arch/i386/kernel/cpu/common.c +++ b/arch/i386/kernel/cpu/common.c | |||
@@ -25,7 +25,33 @@ | |||
25 | DEFINE_PER_CPU(struct Xgt_desc_struct, cpu_gdt_descr); | 25 | DEFINE_PER_CPU(struct Xgt_desc_struct, cpu_gdt_descr); |
26 | EXPORT_PER_CPU_SYMBOL(cpu_gdt_descr); | 26 | EXPORT_PER_CPU_SYMBOL(cpu_gdt_descr); |
27 | 27 | ||
28 | DEFINE_PER_CPU(struct desc_struct, cpu_gdt[GDT_ENTRIES]); | 28 | DEFINE_PER_CPU(struct desc_struct, cpu_gdt[GDT_ENTRIES]) = { |
29 | [GDT_ENTRY_KERNEL_CS] = { 0x0000ffff, 0x00cf9a00 }, | ||
30 | [GDT_ENTRY_KERNEL_DS] = { 0x0000ffff, 0x00cf9200 }, | ||
31 | [GDT_ENTRY_DEFAULT_USER_CS] = { 0x0000ffff, 0x00cffa00 }, | ||
32 | [GDT_ENTRY_DEFAULT_USER_DS] = { 0x0000ffff, 0x00cff200 }, | ||
33 | /* | ||
34 | * Segments used for calling PnP BIOS have byte granularity. | ||
35 | * They code segments and data segments have fixed 64k limits, | ||
36 | * the transfer segment sizes are set at run time. | ||
37 | */ | ||
38 | [GDT_ENTRY_PNPBIOS_CS32] = { 0x0000ffff, 0x00409a00 },/* 32-bit code */ | ||
39 | [GDT_ENTRY_PNPBIOS_CS16] = { 0x0000ffff, 0x00009a00 },/* 16-bit code */ | ||
40 | [GDT_ENTRY_PNPBIOS_DS] = { 0x0000ffff, 0x00009200 }, /* 16-bit data */ | ||
41 | [GDT_ENTRY_PNPBIOS_TS1] = { 0x00000000, 0x00009200 },/* 16-bit data */ | ||
42 | [GDT_ENTRY_PNPBIOS_TS2] = { 0x00000000, 0x00009200 },/* 16-bit data */ | ||
43 | /* | ||
44 | * The APM segments have byte granularity and their bases | ||
45 | * are set at run time. All have 64k limits. | ||
46 | */ | ||
47 | [GDT_ENTRY_APMBIOS_BASE] = { 0x0000ffff, 0x00409a00 },/* 32-bit code */ | ||
48 | /* 16-bit code */ | ||
49 | [GDT_ENTRY_APMBIOS_BASE+1] = { 0x0000ffff, 0x00009a00 }, | ||
50 | [GDT_ENTRY_APMBIOS_BASE+2] = { 0x0000ffff, 0x00409200 }, /* data */ | ||
51 | |||
52 | [GDT_ENTRY_ESPFIX_SS] = { 0x00000000, 0x00c09200 }, | ||
53 | [GDT_ENTRY_PDA] = { 0x00000000, 0x00c09200 }, /* set in setup_pda */ | ||
54 | }; | ||
29 | 55 | ||
30 | DEFINE_PER_CPU(struct i386_pda, _cpu_pda); | 56 | DEFINE_PER_CPU(struct i386_pda, _cpu_pda); |
31 | EXPORT_PER_CPU_SYMBOL(_cpu_pda); | 57 | EXPORT_PER_CPU_SYMBOL(_cpu_pda); |
@@ -618,46 +644,6 @@ struct i386_pda boot_pda = { | |||
618 | .pcurrent = &init_task, | 644 | .pcurrent = &init_task, |
619 | }; | 645 | }; |
620 | 646 | ||
621 | static inline void set_kernel_fs(void) | ||
622 | { | ||
623 | /* Set %fs for this CPU's PDA. Memory clobber is to create a | ||
624 | barrier with respect to any PDA operations, so the compiler | ||
625 | doesn't move any before here. */ | ||
626 | asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_PDA) : "memory"); | ||
627 | } | ||
628 | |||
629 | /* Initialize the CPU's GDT and PDA. This is either the boot CPU doing itself | ||
630 | (still using cpu_gdt_table), or a CPU doing it for a secondary which | ||
631 | will soon come up. */ | ||
632 | __cpuinit void init_gdt(int cpu, struct task_struct *idle) | ||
633 | { | ||
634 | struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu); | ||
635 | struct desc_struct *gdt = per_cpu(cpu_gdt, cpu); | ||
636 | struct i386_pda *pda = &per_cpu(_cpu_pda, cpu); | ||
637 | |||
638 | memcpy(gdt, cpu_gdt_table, GDT_SIZE); | ||
639 | cpu_gdt_descr->address = (unsigned long)gdt; | ||
640 | cpu_gdt_descr->size = GDT_SIZE - 1; | ||
641 | |||
642 | pack_descriptor((u32 *)&gdt[GDT_ENTRY_PDA].a, | ||
643 | (u32 *)&gdt[GDT_ENTRY_PDA].b, | ||
644 | (unsigned long)pda, sizeof(*pda) - 1, | ||
645 | 0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */ | ||
646 | |||
647 | memset(pda, 0, sizeof(*pda)); | ||
648 | pda->_pda = pda; | ||
649 | pda->cpu_number = cpu; | ||
650 | pda->pcurrent = idle; | ||
651 | } | ||
652 | |||
653 | void __cpuinit cpu_set_gdt(int cpu) | ||
654 | { | ||
655 | struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu); | ||
656 | |||
657 | load_gdt(cpu_gdt_descr); | ||
658 | set_kernel_fs(); | ||
659 | } | ||
660 | |||
661 | /* Common CPU init for both boot and secondary CPUs */ | 647 | /* Common CPU init for both boot and secondary CPUs */ |
662 | static void __cpuinit _cpu_init(int cpu, struct task_struct *curr) | 648 | static void __cpuinit _cpu_init(int cpu, struct task_struct *curr) |
663 | { | 649 | { |
@@ -740,10 +726,6 @@ void __cpuinit cpu_init(void) | |||
740 | int cpu = smp_processor_id(); | 726 | int cpu = smp_processor_id(); |
741 | struct task_struct *curr = current; | 727 | struct task_struct *curr = current; |
742 | 728 | ||
743 | /* Set up the real GDT and PDA, so we can transition from the | ||
744 | boot_gdt_table & boot_pda. */ | ||
745 | init_gdt(cpu, curr); | ||
746 | cpu_set_gdt(cpu); | ||
747 | _cpu_init(cpu, curr); | 729 | _cpu_init(cpu, curr); |
748 | } | 730 | } |
749 | 731 | ||
diff --git a/arch/i386/kernel/head.S b/arch/i386/kernel/head.S index cb185f40c282..633fd2f47429 100644 --- a/arch/i386/kernel/head.S +++ b/arch/i386/kernel/head.S | |||
@@ -599,7 +599,7 @@ idt_descr: | |||
599 | .word 0 # 32 bit align gdt_desc.address | 599 | .word 0 # 32 bit align gdt_desc.address |
600 | ENTRY(early_gdt_descr) | 600 | ENTRY(early_gdt_descr) |
601 | .word GDT_ENTRIES*8-1 | 601 | .word GDT_ENTRIES*8-1 |
602 | .long cpu_gdt_table | 602 | .long per_cpu__cpu_gdt /* Overwritten for secondary CPUs */ |
603 | 603 | ||
604 | /* | 604 | /* |
605 | * The boot_gdt_table must mirror the equivalent in setup.S and is | 605 | * The boot_gdt_table must mirror the equivalent in setup.S and is |
@@ -610,56 +610,3 @@ ENTRY(boot_gdt_table) | |||
610 | .fill GDT_ENTRY_BOOT_CS,8,0 | 610 | .fill GDT_ENTRY_BOOT_CS,8,0 |
611 | .quad 0x00cf9a000000ffff /* kernel 4GB code at 0x00000000 */ | 611 | .quad 0x00cf9a000000ffff /* kernel 4GB code at 0x00000000 */ |
612 | .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */ | 612 | .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */ |
613 | |||
614 | /* | ||
615 | * The Global Descriptor Table contains 32 quadwords, per-CPU. | ||
616 | */ | ||
617 | .align L1_CACHE_BYTES | ||
618 | ENTRY(cpu_gdt_table) | ||
619 | .quad 0x0000000000000000 /* NULL descriptor */ | ||
620 | .quad 0x0000000000000000 /* 0x0b reserved */ | ||
621 | .quad 0x0000000000000000 /* 0x13 reserved */ | ||
622 | .quad 0x0000000000000000 /* 0x1b reserved */ | ||
623 | .quad 0x0000000000000000 /* 0x20 unused */ | ||
624 | .quad 0x0000000000000000 /* 0x28 unused */ | ||
625 | .quad 0x0000000000000000 /* 0x33 TLS entry 1 */ | ||
626 | .quad 0x0000000000000000 /* 0x3b TLS entry 2 */ | ||
627 | .quad 0x0000000000000000 /* 0x43 TLS entry 3 */ | ||
628 | .quad 0x0000000000000000 /* 0x4b reserved */ | ||
629 | .quad 0x0000000000000000 /* 0x53 reserved */ | ||
630 | .quad 0x0000000000000000 /* 0x5b reserved */ | ||
631 | |||
632 | .quad 0x00cf9a000000ffff /* 0x60 kernel 4GB code at 0x00000000 */ | ||
633 | .quad 0x00cf92000000ffff /* 0x68 kernel 4GB data at 0x00000000 */ | ||
634 | .quad 0x00cffa000000ffff /* 0x73 user 4GB code at 0x00000000 */ | ||
635 | .quad 0x00cff2000000ffff /* 0x7b user 4GB data at 0x00000000 */ | ||
636 | |||
637 | .quad 0x0000000000000000 /* 0x80 TSS descriptor */ | ||
638 | .quad 0x0000000000000000 /* 0x88 LDT descriptor */ | ||
639 | |||
640 | /* | ||
641 | * Segments used for calling PnP BIOS have byte granularity. | ||
642 | * The code segments and data segments have fixed 64k limits, | ||
643 | * the transfer segment sizes are set at run time. | ||
644 | */ | ||
645 | .quad 0x00409a000000ffff /* 0x90 32-bit code */ | ||
646 | .quad 0x00009a000000ffff /* 0x98 16-bit code */ | ||
647 | .quad 0x000092000000ffff /* 0xa0 16-bit data */ | ||
648 | .quad 0x0000920000000000 /* 0xa8 16-bit data */ | ||
649 | .quad 0x0000920000000000 /* 0xb0 16-bit data */ | ||
650 | |||
651 | /* | ||
652 | * The APM segments have byte granularity and their bases | ||
653 | * are set at run time. All have 64k limits. | ||
654 | */ | ||
655 | .quad 0x00409a000000ffff /* 0xb8 APM CS code */ | ||
656 | .quad 0x00009a000000ffff /* 0xc0 APM CS 16 code (16 bit) */ | ||
657 | .quad 0x004092000000ffff /* 0xc8 APM DS data */ | ||
658 | |||
659 | .quad 0x00c0920000000000 /* 0xd0 - ESPFIX SS */ | ||
660 | .quad 0x00cf92000000ffff /* 0xd8 - PDA */ | ||
661 | .quad 0x0000000000000000 /* 0xe0 - unused */ | ||
662 | .quad 0x0000000000000000 /* 0xe8 - unused */ | ||
663 | .quad 0x0000000000000000 /* 0xf0 - unused */ | ||
664 | .quad 0x0000000000000000 /* 0xf8 - GDT entry 31: double-fault TSS */ | ||
665 | |||
diff --git a/arch/i386/kernel/smpboot.c b/arch/i386/kernel/smpboot.c index b36a5f174cc9..a9447c3e86dd 100644 --- a/arch/i386/kernel/smpboot.c +++ b/arch/i386/kernel/smpboot.c | |||
@@ -440,12 +440,6 @@ static void __cpuinit start_secondary(void *unused) | |||
440 | void __devinit initialize_secondary(void) | 440 | void __devinit initialize_secondary(void) |
441 | { | 441 | { |
442 | /* | 442 | /* |
443 | * switch to the per CPU GDT we already set up | ||
444 | * in do_boot_cpu() | ||
445 | */ | ||
446 | cpu_set_gdt(current_thread_info()->cpu); | ||
447 | |||
448 | /* | ||
449 | * We don't actually need to load the full TSS, | 443 | * We don't actually need to load the full TSS, |
450 | * basically just the stack pointer and the eip. | 444 | * basically just the stack pointer and the eip. |
451 | */ | 445 | */ |
@@ -787,6 +781,32 @@ static inline struct task_struct * alloc_idle_task(int cpu) | |||
787 | #define alloc_idle_task(cpu) fork_idle(cpu) | 781 | #define alloc_idle_task(cpu) fork_idle(cpu) |
788 | #endif | 782 | #endif |
789 | 783 | ||
784 | /* Initialize the CPU's GDT. This is either the boot CPU doing itself | ||
785 | (still using the master per-cpu area), or a CPU doing it for a | ||
786 | secondary which will soon come up. */ | ||
787 | static __cpuinit void init_gdt(int cpu, struct task_struct *idle) | ||
788 | { | ||
789 | struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu); | ||
790 | struct desc_struct *gdt = per_cpu(cpu_gdt, cpu); | ||
791 | struct i386_pda *pda = &per_cpu(_cpu_pda, cpu); | ||
792 | |||
793 | cpu_gdt_descr->address = (unsigned long)gdt; | ||
794 | cpu_gdt_descr->size = GDT_SIZE - 1; | ||
795 | |||
796 | pack_descriptor((u32 *)&gdt[GDT_ENTRY_PDA].a, | ||
797 | (u32 *)&gdt[GDT_ENTRY_PDA].b, | ||
798 | (unsigned long)pda, sizeof(*pda) - 1, | ||
799 | 0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */ | ||
800 | |||
801 | memset(pda, 0, sizeof(*pda)); | ||
802 | pda->_pda = pda; | ||
803 | pda->cpu_number = cpu; | ||
804 | pda->pcurrent = idle; | ||
805 | } | ||
806 | |||
807 | /* Defined in head.S */ | ||
808 | extern struct Xgt_desc_struct early_gdt_descr; | ||
809 | |||
790 | static int __cpuinit do_boot_cpu(int apicid, int cpu) | 810 | static int __cpuinit do_boot_cpu(int apicid, int cpu) |
791 | /* | 811 | /* |
792 | * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad | 812 | * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad |
@@ -809,6 +829,8 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu) | |||
809 | panic("failed fork for CPU %d", cpu); | 829 | panic("failed fork for CPU %d", cpu); |
810 | 830 | ||
811 | init_gdt(cpu, idle); | 831 | init_gdt(cpu, idle); |
832 | early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu); | ||
833 | start_pda = cpu_pda(cpu); | ||
812 | 834 | ||
813 | idle->thread.eip = (unsigned long) start_secondary; | 835 | idle->thread.eip = (unsigned long) start_secondary; |
814 | /* start_eip had better be page-aligned! */ | 836 | /* start_eip had better be page-aligned! */ |
@@ -1161,13 +1183,26 @@ void __init smp_prepare_cpus(unsigned int max_cpus) | |||
1161 | smp_boot_cpus(max_cpus); | 1183 | smp_boot_cpus(max_cpus); |
1162 | } | 1184 | } |
1163 | 1185 | ||
1164 | void __devinit smp_prepare_boot_cpu(void) | 1186 | /* Current gdt points %fs at the "master" per-cpu area: after this, |
1187 | * it's on the real one. */ | ||
1188 | static inline void switch_to_new_gdt(void) | ||
1165 | { | 1189 | { |
1166 | cpu_set(smp_processor_id(), cpu_online_map); | 1190 | load_gdt(&per_cpu(cpu_gdt_descr, smp_processor_id())); |
1167 | cpu_set(smp_processor_id(), cpu_callout_map); | 1191 | asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_PDA) : "memory"); |
1168 | cpu_set(smp_processor_id(), cpu_present_map); | 1192 | } |
1169 | cpu_set(smp_processor_id(), cpu_possible_map); | 1193 | |
1170 | per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; | 1194 | void __init smp_prepare_boot_cpu(void) |
1195 | { | ||
1196 | unsigned int cpu = smp_processor_id(); | ||
1197 | |||
1198 | init_gdt(cpu, current); | ||
1199 | switch_to_new_gdt(); | ||
1200 | |||
1201 | cpu_set(cpu, cpu_online_map); | ||
1202 | cpu_set(cpu, cpu_callout_map); | ||
1203 | cpu_set(cpu, cpu_present_map); | ||
1204 | cpu_set(cpu, cpu_possible_map); | ||
1205 | __get_cpu_var(cpu_state) = CPU_ONLINE; | ||
1171 | } | 1206 | } |
1172 | 1207 | ||
1173 | #ifdef CONFIG_HOTPLUG_CPU | 1208 | #ifdef CONFIG_HOTPLUG_CPU |
diff --git a/arch/i386/mach-voyager/voyager_smp.c b/arch/i386/mach-voyager/voyager_smp.c index 15d8132d4f5e..b9ce33c0c202 100644 --- a/arch/i386/mach-voyager/voyager_smp.c +++ b/arch/i386/mach-voyager/voyager_smp.c | |||
@@ -765,12 +765,6 @@ initialize_secondary(void) | |||
765 | #endif | 765 | #endif |
766 | 766 | ||
767 | /* | 767 | /* |
768 | * switch to the per CPU GDT we already set up | ||
769 | * in do_boot_cpu() | ||
770 | */ | ||
771 | cpu_set_gdt(current_thread_info()->cpu); | ||
772 | |||
773 | /* | ||
774 | * We don't actually need to load the full TSS, | 768 | * We don't actually need to load the full TSS, |
775 | * basically just the stack pointer and the eip. | 769 | * basically just the stack pointer and the eip. |
776 | */ | 770 | */ |
diff --git a/include/asm-i386/desc.h b/include/asm-i386/desc.h index 53c5916687b6..a75ae6b97860 100644 --- a/include/asm-i386/desc.h +++ b/include/asm-i386/desc.h | |||
@@ -12,8 +12,6 @@ | |||
12 | 12 | ||
13 | #include <asm/mmu.h> | 13 | #include <asm/mmu.h> |
14 | 14 | ||
15 | extern struct desc_struct cpu_gdt_table[GDT_ENTRIES]; | ||
16 | |||
17 | struct Xgt_desc_struct { | 15 | struct Xgt_desc_struct { |
18 | unsigned short size; | 16 | unsigned short size; |
19 | unsigned long address __attribute__((packed)); | 17 | unsigned long address __attribute__((packed)); |
diff --git a/include/asm-i386/processor.h b/include/asm-i386/processor.h index cd940befef53..b25a2f5b5375 100644 --- a/include/asm-i386/processor.h +++ b/include/asm-i386/processor.h | |||
@@ -743,7 +743,6 @@ extern unsigned long boot_option_idle_override; | |||
743 | extern void enable_sep_cpu(void); | 743 | extern void enable_sep_cpu(void); |
744 | extern int sysenter_setup(void); | 744 | extern int sysenter_setup(void); |
745 | 745 | ||
746 | extern void init_gdt(int cpu, struct task_struct *idle); | ||
747 | extern void cpu_set_gdt(int); | 746 | extern void cpu_set_gdt(int); |
748 | extern void secondary_cpu_init(void); | 747 | extern void secondary_cpu_init(void); |
749 | 748 | ||