aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJeremy Fitzhardinge <jeremy@goop.org>2007-05-02 13:27:16 -0400
committerAndi Kleen <andi@basil.nowhere.org>2007-05-02 13:27:16 -0400
commit7c3576d261ce046789a7db14f43303f8120910c7 (patch)
treead27a8459bbcdb183fe2411aec3b840942992ad5
parent7a61d35d4b4056e7711031202da7605e052f4137 (diff)
[PATCH] i386: Convert PDA into the percpu section
Currently x86 (similar to x84-64) has a special per-cpu structure called "i386_pda" which can be easily and efficiently referenced via the %fs register. An ELF section is more flexible than a structure, allowing any piece of code to use this area. Indeed, such a section already exists: the per-cpu area. So this patch: (1) Removes the PDA and uses per-cpu variables for each current member. (2) Replaces the __KERNEL_PDA segment with __KERNEL_PERCPU. (3) Creates a per-cpu mirror of __per_cpu_offset called this_cpu_off, which can be used to calculate addresses for this CPU's variables. (4) Simplifies startup, because %fs doesn't need to be loaded with a special segment at early boot; it can be deferred until the first percpu area is allocated (or never for UP). The result is less code and one less x86-specific concept. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com> Signed-off-by: Andi Kleen <ak@suse.de> Cc: Andi Kleen <ak@suse.de>
Diffstat
-rw-r--r--arch/i386/kernel/asm-offsets.c5
-rw-r--r--arch/i386/kernel/cpu/common.c17
-rw-r--r--arch/i386/kernel/entry.S5
-rw-r--r--arch/i386/kernel/head.S31
-rw-r--r--arch/i386/kernel/i386_ksyms.c2
-rw-r--r--arch/i386/kernel/irq.c3
-rw-r--r--arch/i386/kernel/process.c12
-rw-r--r--arch/i386/kernel/smpboot.c30
-rw-r--r--arch/i386/kernel/vmi.c6
-rw-r--r--arch/i386/kernel/vmlinux.lds.S1
-rw-r--r--include/asm-i386/current.h5
-rw-r--r--include/asm-i386/irq_regs.h12
-rw-r--r--include/asm-i386/pda.h99
-rw-r--r--include/asm-i386/percpu.h132
-rw-r--r--include/asm-i386/processor.h2
-rw-r--r--include/asm-i386/segment.h6
-rw-r--r--include/asm-i386/smp.h4
17 files changed, 177 insertions, 195 deletions
diff --git a/arch/i386/kernel/asm-offsets.c b/arch/i386/kernel/asm-offsets.c
index d558adfc293c..b05e85fd1c1e 100644
--- a/arch/i386/kernel/asm-offsets.c
+++ b/arch/i386/kernel/asm-offsets.c
@@ -15,7 +15,6 @@
15#include <asm/processor.h> 15#include <asm/processor.h>
16#include <asm/thread_info.h> 16#include <asm/thread_info.h>
17#include <asm/elf.h> 17#include <asm/elf.h>
18#include <asm/pda.h>
19 18
20#define DEFINE(sym, val) \ 19#define DEFINE(sym, val) \
21 asm volatile("\n->" #sym " %0 " #val : : "i" (val)) 20 asm volatile("\n->" #sym " %0 " #val : : "i" (val))
@@ -101,10 +100,6 @@ void foo(void)
101 100
102 OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx); 101 OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
103 102
104 BLANK();
105 OFFSET(PDA_cpu, i386_pda, cpu_number);
106 OFFSET(PDA_pcurrent, i386_pda, pcurrent);
107
108#ifdef CONFIG_PARAVIRT 103#ifdef CONFIG_PARAVIRT
109 BLANK(); 104 BLANK();
110 OFFSET(PARAVIRT_enabled, paravirt_ops, paravirt_enabled); 105 OFFSET(PARAVIRT_enabled, paravirt_ops, paravirt_enabled);
diff --git a/arch/i386/kernel/cpu/common.c b/arch/i386/kernel/cpu/common.c
index 7a4c036d93c8..27e00565f5e4 100644
--- a/arch/i386/kernel/cpu/common.c
+++ b/arch/i386/kernel/cpu/common.c
@@ -18,7 +18,6 @@
18#include <asm/apic.h> 18#include <asm/apic.h>
19#include <mach_apic.h> 19#include <mach_apic.h>
20#endif 20#endif
21#include <asm/pda.h>
22 21
23#include "cpu.h" 22#include "cpu.h"
24 23
@@ -47,13 +46,10 @@ DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = {
47 [GDT_ENTRY_APMBIOS_BASE+2] = { 0x0000ffff, 0x00409200 }, /* data */ 46 [GDT_ENTRY_APMBIOS_BASE+2] = { 0x0000ffff, 0x00409200 }, /* data */
48 47
49 [GDT_ENTRY_ESPFIX_SS] = { 0x00000000, 0x00c09200 }, 48 [GDT_ENTRY_ESPFIX_SS] = { 0x00000000, 0x00c09200 },
50 [GDT_ENTRY_PDA] = { 0x00000000, 0x00c09200 }, /* set in setup_pda */ 49 [GDT_ENTRY_PERCPU] = { 0x00000000, 0x00000000 },
51} }; 50} };
52EXPORT_PER_CPU_SYMBOL_GPL(gdt_page); 51EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
53 52
54DEFINE_PER_CPU(struct i386_pda, _cpu_pda);
55EXPORT_PER_CPU_SYMBOL(_cpu_pda);
56
57static int cachesize_override __cpuinitdata = -1; 53static int cachesize_override __cpuinitdata = -1;
58static int disable_x86_fxsr __cpuinitdata; 54static int disable_x86_fxsr __cpuinitdata;
59static int disable_x86_serial_nr __cpuinitdata = 1; 55static int disable_x86_serial_nr __cpuinitdata = 1;
@@ -634,21 +630,14 @@ void __init early_cpu_init(void)
634#endif 630#endif
635} 631}
636 632
637/* Make sure %gs is initialized properly in idle threads */ 633/* Make sure %fs is initialized properly in idle threads */
638struct pt_regs * __devinit idle_regs(struct pt_regs *regs) 634struct pt_regs * __devinit idle_regs(struct pt_regs *regs)
639{ 635{
640 memset(regs, 0, sizeof(struct pt_regs)); 636 memset(regs, 0, sizeof(struct pt_regs));
641 regs->xfs = __KERNEL_PDA; 637 regs->xfs = __KERNEL_PERCPU;
642 return regs; 638 return regs;
643} 639}
644 640
645/* Initial PDA used by boot CPU */
646struct i386_pda boot_pda = {
647 ._pda = &boot_pda,
648 .cpu_number = 0,
649 .pcurrent = &init_task,
650};
651
652/* 641/*
653 * cpu_init() initializes state that is per-CPU. Some data is already 642 * cpu_init() initializes state that is per-CPU. Some data is already
654 * initialized (naturally) in the bootstrap process, such as the GDT 643 * initialized (naturally) in the bootstrap process, such as the GDT
diff --git a/arch/i386/kernel/entry.S b/arch/i386/kernel/entry.S
index 3e4aa1fd33e2..7f92ceb428ad 100644
--- a/arch/i386/kernel/entry.S
+++ b/arch/i386/kernel/entry.S
@@ -132,7 +132,7 @@ VM_MASK = 0x00020000
132 movl $(__USER_DS), %edx; \ 132 movl $(__USER_DS), %edx; \
133 movl %edx, %ds; \ 133 movl %edx, %ds; \
134 movl %edx, %es; \ 134 movl %edx, %es; \
135 movl $(__KERNEL_PDA), %edx; \ 135 movl $(__KERNEL_PERCPU), %edx; \
136 movl %edx, %fs 136 movl %edx, %fs
137 137
138#define RESTORE_INT_REGS \ 138#define RESTORE_INT_REGS \
@@ -556,7 +556,6 @@ END(syscall_badsys)
556 556
557#define FIXUP_ESPFIX_STACK \ 557#define FIXUP_ESPFIX_STACK \
558 /* since we are on a wrong stack, we cant make it a C code :( */ \ 558 /* since we are on a wrong stack, we cant make it a C code :( */ \
559 movl %fs:PDA_cpu, %ebx; \
560 PER_CPU(gdt_page, %ebx); \ 559 PER_CPU(gdt_page, %ebx); \
561 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \ 560 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
562 addl %esp, %eax; \ 561 addl %esp, %eax; \
@@ -681,7 +680,7 @@ error_code:
681 pushl %fs 680 pushl %fs
682 CFI_ADJUST_CFA_OFFSET 4 681 CFI_ADJUST_CFA_OFFSET 4
683 /*CFI_REL_OFFSET fs, 0*/ 682 /*CFI_REL_OFFSET fs, 0*/
684 movl $(__KERNEL_PDA), %ecx 683 movl $(__KERNEL_PERCPU), %ecx
685 movl %ecx, %fs 684 movl %ecx, %fs
686 UNWIND_ESPFIX_STACK 685 UNWIND_ESPFIX_STACK
687 popl %ecx 686 popl %ecx
diff --git a/arch/i386/kernel/head.S b/arch/i386/kernel/head.S
index bb36c24311b4..12277d8938df 100644
--- a/arch/i386/kernel/head.S
+++ b/arch/i386/kernel/head.S
@@ -317,12 +317,12 @@ is386: movl $2,%ecx # set MP
317 movl %eax,%cr0 317 movl %eax,%cr0
318 318
319 call check_x87 319 call check_x87
320 call setup_pda
321 lgdt early_gdt_descr 320 lgdt early_gdt_descr
322 lidt idt_descr 321 lidt idt_descr
323 ljmp $(__KERNEL_CS),$1f 322 ljmp $(__KERNEL_CS),$1f
3241: movl $(__KERNEL_DS),%eax # reload all the segment registers 3231: movl $(__KERNEL_DS),%eax # reload all the segment registers
325 movl %eax,%ss # after changing gdt. 324 movl %eax,%ss # after changing gdt.
325 movl %eax,%fs # gets reset once there's real percpu
326 326
327 movl $(__USER_DS),%eax # DS/ES contains default USER segment 327 movl $(__USER_DS),%eax # DS/ES contains default USER segment
328 movl %eax,%ds 328 movl %eax,%ds
@@ -332,16 +332,17 @@ is386: movl $2,%ecx # set MP
332 movl %eax,%gs 332 movl %eax,%gs
333 lldt %ax 333 lldt %ax
334 334
335 movl $(__KERNEL_PDA),%eax
336 mov %eax,%fs
337
338 cld # gcc2 wants the direction flag cleared at all times 335 cld # gcc2 wants the direction flag cleared at all times
339 pushl $0 # fake return address for unwinder 336 pushl $0 # fake return address for unwinder
340#ifdef CONFIG_SMP 337#ifdef CONFIG_SMP
341 movb ready, %cl 338 movb ready, %cl
342 movb $1, ready 339 movb $1, ready
343 cmpb $0,%cl # the first CPU calls start_kernel 340 cmpb $0,%cl # the first CPU calls start_kernel
344 jne initialize_secondary # all other CPUs call initialize_secondary 341 je 1f
342 movl $(__KERNEL_PERCPU), %eax
343 movl %eax,%fs # set this cpu's percpu
344 jmp initialize_secondary # all other CPUs call initialize_secondary
3451:
345#endif /* CONFIG_SMP */ 346#endif /* CONFIG_SMP */
346 jmp start_kernel 347 jmp start_kernel
347 348
@@ -365,23 +366,6 @@ check_x87:
365 ret 366 ret
366 367
367/* 368/*
368 * Point the GDT at this CPU's PDA. On boot this will be
369 * cpu_gdt_table and boot_pda; for secondary CPUs, these will be
370 * that CPU's GDT and PDA.
371 */
372ENTRY(setup_pda)
373 /* get the PDA pointer */
374 movl start_pda, %eax
375
376 /* slot the PDA address into the GDT */
377 mov early_gdt_descr+2, %ecx
378 mov %ax, (__KERNEL_PDA+0+2)(%ecx) /* base & 0x0000ffff */
379 shr $16, %eax
380 mov %al, (__KERNEL_PDA+4+0)(%ecx) /* base & 0x00ff0000 */
381 mov %ah, (__KERNEL_PDA+4+3)(%ecx) /* base & 0xff000000 */
382 ret
383
384/*
385 * setup_idt 369 * setup_idt
386 * 370 *
387 * sets up a idt with 256 entries pointing to 371 * sets up a idt with 256 entries pointing to
@@ -553,9 +537,6 @@ ENTRY(empty_zero_page)
553 * This starts the data section. 537 * This starts the data section.
554 */ 538 */
555.data 539.data
556ENTRY(start_pda)
557 .long boot_pda
558
559ENTRY(stack_start) 540ENTRY(stack_start)
560 .long init_thread_union+THREAD_SIZE 541 .long init_thread_union+THREAD_SIZE
561 .long __BOOT_DS 542 .long __BOOT_DS
diff --git a/arch/i386/kernel/i386_ksyms.c b/arch/i386/kernel/i386_ksyms.c
index 4afe26e86260..e3d4b73bfdb0 100644
--- a/arch/i386/kernel/i386_ksyms.c
+++ b/arch/i386/kernel/i386_ksyms.c
@@ -28,5 +28,3 @@ EXPORT_SYMBOL(__read_lock_failed);
28#endif 28#endif
29 29
30EXPORT_SYMBOL(csum_partial); 30EXPORT_SYMBOL(csum_partial);
31
32EXPORT_SYMBOL(_proxy_pda);
diff --git a/arch/i386/kernel/irq.c b/arch/i386/kernel/irq.c
index 8db8d514c9c0..d2daf672f4a2 100644
--- a/arch/i386/kernel/irq.c
+++ b/arch/i386/kernel/irq.c
@@ -24,6 +24,9 @@
24DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp; 24DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp;
25EXPORT_PER_CPU_SYMBOL(irq_stat); 25EXPORT_PER_CPU_SYMBOL(irq_stat);
26 26
27DEFINE_PER_CPU(struct pt_regs *, irq_regs);
28EXPORT_PER_CPU_SYMBOL(irq_regs);
29
27/* 30/*
28 * 'what should we do if we get a hw irq event on an illegal vector'. 31 * 'what should we do if we get a hw irq event on an illegal vector'.
29 * each architecture has to answer this themselves. 32 * each architecture has to answer this themselves.
diff --git a/arch/i386/kernel/process.c b/arch/i386/kernel/process.c
index 5fb9524c6f4b..61999479b7a4 100644
--- a/arch/i386/kernel/process.c
+++ b/arch/i386/kernel/process.c
@@ -39,6 +39,7 @@
39#include <linux/random.h> 39#include <linux/random.h>
40#include <linux/personality.h> 40#include <linux/personality.h>
41#include <linux/tick.h> 41#include <linux/tick.h>
42#include <linux/percpu.h>
42 43
43#include <asm/uaccess.h> 44#include <asm/uaccess.h>
44#include <asm/pgtable.h> 45#include <asm/pgtable.h>
@@ -57,7 +58,6 @@
57 58
58#include <asm/tlbflush.h> 59#include <asm/tlbflush.h>
59#include <asm/cpu.h> 60#include <asm/cpu.h>
60#include <asm/pda.h>
61 61
62asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); 62asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
63 63
@@ -66,6 +66,12 @@ static int hlt_counter;
66unsigned long boot_option_idle_override = 0; 66unsigned long boot_option_idle_override = 0;
67EXPORT_SYMBOL(boot_option_idle_override); 67EXPORT_SYMBOL(boot_option_idle_override);
68 68
69DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
70EXPORT_PER_CPU_SYMBOL(current_task);
71
72DEFINE_PER_CPU(int, cpu_number);
73EXPORT_PER_CPU_SYMBOL(cpu_number);
74
69/* 75/*
70 * Return saved PC of a blocked thread. 76 * Return saved PC of a blocked thread.
71 */ 77 */
@@ -342,7 +348,7 @@ int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
342 348
343 regs.xds = __USER_DS; 349 regs.xds = __USER_DS;
344 regs.xes = __USER_DS; 350 regs.xes = __USER_DS;
345 regs.xfs = __KERNEL_PDA; 351 regs.xfs = __KERNEL_PERCPU;
346 regs.orig_eax = -1; 352 regs.orig_eax = -1;
347 regs.eip = (unsigned long) kernel_thread_helper; 353 regs.eip = (unsigned long) kernel_thread_helper;
348 regs.xcs = __KERNEL_CS | get_kernel_rpl(); 354 regs.xcs = __KERNEL_CS | get_kernel_rpl();
@@ -711,7 +717,7 @@ struct task_struct fastcall * __switch_to(struct task_struct *prev_p, struct tas
711 if (prev->gs | next->gs) 717 if (prev->gs | next->gs)
712 loadsegment(gs, next->gs); 718 loadsegment(gs, next->gs);
713 719
714 write_pda(pcurrent, next_p); 720 x86_write_percpu(current_task, next_p);
715 721
716 return prev_p; 722 return prev_p;
717} 723}
diff --git a/arch/i386/kernel/smpboot.c b/arch/i386/kernel/smpboot.c
index 61e2842add36..f79b6233db78 100644
--- a/arch/i386/kernel/smpboot.c
+++ b/arch/i386/kernel/smpboot.c
@@ -53,7 +53,6 @@
53#include <asm/desc.h> 53#include <asm/desc.h>
54#include <asm/arch_hooks.h> 54#include <asm/arch_hooks.h>
55#include <asm/nmi.h> 55#include <asm/nmi.h>
56#include <asm/pda.h>
57 56
58#include <mach_apic.h> 57#include <mach_apic.h>
59#include <mach_wakecpu.h> 58#include <mach_wakecpu.h>
@@ -99,6 +98,9 @@ EXPORT_SYMBOL(x86_cpu_to_apicid);
99 98
100u8 apicid_2_node[MAX_APICID]; 99u8 apicid_2_node[MAX_APICID];
101 100
101DEFINE_PER_CPU(unsigned long, this_cpu_off);
102EXPORT_PER_CPU_SYMBOL(this_cpu_off);
103
102/* 104/*
103 * Trampoline 80x86 program as an array. 105 * Trampoline 80x86 program as an array.
104 */ 106 */
@@ -456,7 +458,6 @@ extern struct {
456 void * esp; 458 void * esp;
457 unsigned short ss; 459 unsigned short ss;
458} stack_start; 460} stack_start;
459extern struct i386_pda *start_pda;
460 461
461#ifdef CONFIG_NUMA 462#ifdef CONFIG_NUMA
462 463
@@ -784,20 +785,17 @@ static inline struct task_struct * alloc_idle_task(int cpu)
784/* Initialize the CPU's GDT. This is either the boot CPU doing itself 785/* Initialize the CPU's GDT. This is either the boot CPU doing itself
785 (still using the master per-cpu area), or a CPU doing it for a 786 (still using the master per-cpu area), or a CPU doing it for a
786 secondary which will soon come up. */ 787 secondary which will soon come up. */
787static __cpuinit void init_gdt(int cpu, struct task_struct *idle) 788static __cpuinit void init_gdt(int cpu)
788{ 789{
789 struct desc_struct *gdt = get_cpu_gdt_table(cpu); 790 struct desc_struct *gdt = get_cpu_gdt_table(cpu);
790 struct i386_pda *pda = &per_cpu(_cpu_pda, cpu);
791 791
792 pack_descriptor((u32 *)&gdt[GDT_ENTRY_PDA].a, 792 pack_descriptor((u32 *)&gdt[GDT_ENTRY_PERCPU].a,
793 (u32 *)&gdt[GDT_ENTRY_PDA].b, 793 (u32 *)&gdt[GDT_ENTRY_PERCPU].b,
794 (unsigned long)pda, sizeof(*pda) - 1, 794 __per_cpu_offset[cpu], 0xFFFFF,
795 0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */ 795 0x80 | DESCTYPE_S | 0x2, 0x8);
796 796
797 memset(pda, 0, sizeof(*pda)); 797 per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
798 pda->_pda = pda; 798 per_cpu(cpu_number, cpu) = cpu;
799 pda->cpu_number = cpu;
800 pda->pcurrent = idle;
801} 799}
802 800
803/* Defined in head.S */ 801/* Defined in head.S */
@@ -824,9 +822,9 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
824 if (IS_ERR(idle)) 822 if (IS_ERR(idle))
825 panic("failed fork for CPU %d", cpu); 823 panic("failed fork for CPU %d", cpu);
826 824
827 init_gdt(cpu, idle); 825 init_gdt(cpu);
826 per_cpu(current_task, cpu) = idle;
828 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu); 827 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
829 start_pda = cpu_pda(cpu);
830 828
831 idle->thread.eip = (unsigned long) start_secondary; 829 idle->thread.eip = (unsigned long) start_secondary;
832 /* start_eip had better be page-aligned! */ 830 /* start_eip had better be page-aligned! */
@@ -1188,14 +1186,14 @@ static inline void switch_to_new_gdt(void)
1188 gdt_descr.address = (long)get_cpu_gdt_table(smp_processor_id()); 1186 gdt_descr.address = (long)get_cpu_gdt_table(smp_processor_id());
1189 gdt_descr.size = GDT_SIZE - 1; 1187 gdt_descr.size = GDT_SIZE - 1;
1190 load_gdt(&gdt_descr); 1188 load_gdt(&gdt_descr);
1191 asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_PDA) : "memory"); 1189 asm("mov %0, %%fs" : : "r" (__KERNEL_PERCPU) : "memory");
1192} 1190}
1193 1191
1194void __init native_smp_prepare_boot_cpu(void) 1192void __init native_smp_prepare_boot_cpu(void)
1195{ 1193{
1196 unsigned int cpu = smp_processor_id(); 1194 unsigned int cpu = smp_processor_id();
1197 1195
1198 init_gdt(cpu, current); 1196 init_gdt(cpu);
1199 switch_to_new_gdt(); 1197 switch_to_new_gdt();
1200 1198
1201 cpu_set(cpu, cpu_online_map); 1199 cpu_set(cpu, cpu_online_map);
diff --git a/arch/i386/kernel/vmi.c b/arch/i386/kernel/vmi.c
index ccad7ee960aa..12312988c626 100644
--- a/arch/i386/kernel/vmi.c
+++ b/arch/i386/kernel/vmi.c
@@ -504,8 +504,6 @@ static void vmi_pmd_clear(pmd_t *pmd)
504#endif 504#endif
505 505
506#ifdef CONFIG_SMP 506#ifdef CONFIG_SMP
507extern void setup_pda(void);
508
509static void __devinit 507static void __devinit
510vmi_startup_ipi_hook(int phys_apicid, unsigned long start_eip, 508vmi_startup_ipi_hook(int phys_apicid, unsigned long start_eip,
511 unsigned long start_esp) 509 unsigned long start_esp)
@@ -530,13 +528,11 @@ vmi_startup_ipi_hook(int phys_apicid, unsigned long start_eip,
530 528
531 ap.ds = __USER_DS; 529 ap.ds = __USER_DS;
532 ap.es = __USER_DS; 530 ap.es = __USER_DS;
533 ap.fs = __KERNEL_PDA; 531 ap.fs = __KERNEL_PERCPU;
534 ap.gs = 0; 532 ap.gs = 0;
535 533
536 ap.eflags = 0; 534 ap.eflags = 0;
537 535
538 setup_pda();
539
540#ifdef CONFIG_X86_PAE 536#ifdef CONFIG_X86_PAE
541 /* efer should match BSP efer. */ 537 /* efer should match BSP efer. */
542 if (cpu_has_nx) { 538 if (cpu_has_nx) {
diff --git a/arch/i386/kernel/vmlinux.lds.S b/arch/i386/kernel/vmlinux.lds.S
index 2ce4aa185fc8..d125784ddf5e 100644
--- a/arch/i386/kernel/vmlinux.lds.S
+++ b/arch/i386/kernel/vmlinux.lds.S
@@ -26,7 +26,6 @@ OUTPUT_FORMAT("elf32-i386", "elf32-i386", "elf32-i386")
26OUTPUT_ARCH(i386) 26OUTPUT_ARCH(i386)
27ENTRY(phys_startup_32) 27ENTRY(phys_startup_32)
28jiffies = jiffies_64; 28jiffies = jiffies_64;
29_proxy_pda = 1;
30 29
31PHDRS { 30PHDRS {
32 text PT_LOAD FLAGS(5); /* R_E */ 31 text PT_LOAD FLAGS(5); /* R_E */
diff --git a/include/asm-i386/current.h b/include/asm-i386/current.h
index 5252ee0f6d7a..d35248539912 100644
--- a/include/asm-i386/current.h
+++ b/include/asm-i386/current.h
@@ -1,14 +1,15 @@
1#ifndef _I386_CURRENT_H 1#ifndef _I386_CURRENT_H
2#define _I386_CURRENT_H 2#define _I386_CURRENT_H
3 3
4#include <asm/pda.h>
5#include <linux/compiler.h> 4#include <linux/compiler.h>
5#include <asm/percpu.h>
6 6
7struct task_struct; 7struct task_struct;
8 8
9DECLARE_PER_CPU(struct task_struct *, current_task);
9static __always_inline struct task_struct *get_current(void) 10static __always_inline struct task_struct *get_current(void)
10{ 11{
11 return read_pda(pcurrent); 12 return x86_read_percpu(current_task);
12} 13}
13 14
14#define current get_current() 15#define current get_current()
diff --git a/include/asm-i386/irq_regs.h b/include/asm-i386/irq_regs.h
index a1b3f7f594a2..3368b20c0b48 100644
--- a/include/asm-i386/irq_regs.h
+++ b/include/asm-i386/irq_regs.h
@@ -1,25 +1,27 @@
1/* 1/*
2 * Per-cpu current frame pointer - the location of the last exception frame on 2 * Per-cpu current frame pointer - the location of the last exception frame on
3 * the stack, stored in the PDA. 3 * the stack, stored in the per-cpu area.
4 * 4 *
5 * Jeremy Fitzhardinge <jeremy@goop.org> 5 * Jeremy Fitzhardinge <jeremy@goop.org>
6 */ 6 */
7#ifndef _ASM_I386_IRQ_REGS_H 7#ifndef _ASM_I386_IRQ_REGS_H
8#define _ASM_I386_IRQ_REGS_H 8#define _ASM_I386_IRQ_REGS_H
9 9
10#include <asm/pda.h> 10#include <asm/percpu.h>
11
12DECLARE_PER_CPU(struct pt_regs *, irq_regs);
11 13
12static inline struct pt_regs *get_irq_regs(void) 14static inline struct pt_regs *get_irq_regs(void)
13{ 15{
14 return read_pda(irq_regs); 16 return x86_read_percpu(irq_regs);
15} 17}
16 18
17static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs) 19static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
18{ 20{
19 struct pt_regs *old_regs; 21 struct pt_regs *old_regs;
20 22
21 old_regs = read_pda(irq_regs); 23 old_regs = get_irq_regs();
22 write_pda(irq_regs, new_regs); 24 x86_write_percpu(irq_regs, new_regs);
23 25
24 return old_regs; 26 return old_regs;
25} 27}
diff --git a/include/asm-i386/pda.h b/include/asm-i386/pda.h
deleted file mode 100644
index aef7f732f77e..000000000000
--- a/include/asm-i386/pda.h
+++ /dev/null
@@ -1,99 +0,0 @@
1/*
2 Per-processor Data Areas
3 Jeremy Fitzhardinge <jeremy@goop.org> 2006
4 Based on asm-x86_64/pda.h by Andi Kleen.
5 */
6#ifndef _I386_PDA_H
7#define _I386_PDA_H
8
9#include <linux/stddef.h>
10#include <linux/types.h>
11#include <asm/percpu.h>
12
13struct i386_pda
14{
15 struct i386_pda *_pda; /* pointer to self */
16
17 int cpu_number;
18 struct task_struct *pcurrent; /* current process */
19 struct pt_regs *irq_regs;
20};
21
22DECLARE_PER_CPU(struct i386_pda, _cpu_pda);
23#define cpu_pda(i) (&per_cpu(_cpu_pda, (i)))
24#define pda_offset(field) offsetof(struct i386_pda, field)
25
26extern void __bad_pda_field(void);
27
28/* This variable is never instantiated. It is only used as a stand-in
29 for the real per-cpu PDA memory, so that gcc can understand what
30 memory operations the inline asms() below are performing. This
31 eliminates the need to make the asms volatile or have memory
32 clobbers, so gcc can readily analyse them. */
33extern struct i386_pda _proxy_pda;
34
35#define pda_to_op(op,field,val) \
36 do { \
37 typedef typeof(_proxy_pda.field) T__; \
38 if (0) { T__ tmp__; tmp__ = (val); } \
39 switch (sizeof(_proxy_pda.field)) { \
40 case 1: \
41 asm(op "b %1,%%fs:%c2" \
42 : "+m" (_proxy_pda.field) \
43 :"ri" ((T__)val), \
44 "i"(pda_offset(field))); \
45 break; \
46 case 2: \
47 asm(op "w %1,%%fs:%c2" \
48 : "+m" (_proxy_pda.field) \
49 :"ri" ((T__)val), \
50 "i"(pda_offset(field))); \
51 break; \
52 case 4: \
53 asm(op "l %1,%%fs:%c2" \
54 : "+m" (_proxy_pda.field) \
55 :"ri" ((T__)val), \
56 "i"(pda_offset(field))); \
57 break; \
58 default: __bad_pda_field(); \
59 } \
60 } while (0)
61
62#define pda_from_op(op,field) \
63 ({ \
64 typeof(_proxy_pda.field) ret__; \
65 switch (sizeof(_proxy_pda.field)) { \
66 case 1: \
67 asm(op "b %%fs:%c1,%0" \
68 : "=r" (ret__) \
69 : "i" (pda_offset(field)), \
70 "m" (_proxy_pda.field)); \
71 break; \
72 case 2: \
73 asm(op "w %%fs:%c1,%0" \
74 : "=r" (ret__) \
75 : "i" (pda_offset(field)), \
76 "m" (_proxy_pda.field)); \
77 break; \
78 case 4: \
79 asm(op "l %%fs:%c1,%0" \
80 : "=r" (ret__) \
81 : "i" (pda_offset(field)), \
82 "m" (_proxy_pda.field)); \
83 break; \
84 default: __bad_pda_field(); \
85 } \
86 ret__; })
87
88/* Return a pointer to a pda field */
89#define pda_addr(field) \
90 ((typeof(_proxy_pda.field) *)((unsigned char *)read_pda(_pda) + \
91 pda_offset(field)))
92
93#define read_pda(field) pda_from_op("mov",field)
94#define write_pda(field,val) pda_to_op("mov",field,val)
95#define add_pda(field,val) pda_to_op("add",field,val)
96#define sub_pda(field,val) pda_to_op("sub",field,val)
97#define or_pda(field,val) pda_to_op("or",field,val)
98
99#endif /* _I386_PDA_H */
diff --git a/include/asm-i386/percpu.h b/include/asm-i386/percpu.h
index a10e7c68ae9d..c5f12f0d9c23 100644
--- a/include/asm-i386/percpu.h
+++ b/include/asm-i386/percpu.h
@@ -1,9 +1,30 @@
1#ifndef __ARCH_I386_PERCPU__ 1#ifndef __ARCH_I386_PERCPU__
2#define __ARCH_I386_PERCPU__ 2#define __ARCH_I386_PERCPU__
3 3
4#ifndef __ASSEMBLY__ 4#ifdef __ASSEMBLY__
5#include <asm-generic/percpu.h> 5
6#else 6/*
7 * PER_CPU finds an address of a per-cpu variable.
8 *
9 * Args:
10 * var - variable name
11 * reg - 32bit register
12 *
13 * The resulting address is stored in the "reg" argument.
14 *
15 * Example:
16 * PER_CPU(cpu_gdt_descr, %ebx)
17 */
18#ifdef CONFIG_SMP
19#define PER_CPU(var, reg) \
20 movl %fs:per_cpu__this_cpu_off, reg; \
21 addl $per_cpu__##var, reg
22#else /* ! SMP */
23#define PER_CPU(var, reg) \
24 movl $per_cpu__##var, reg;
25#endif /* SMP */
26
27#else /* ...!ASSEMBLY */
7 28
8/* 29/*
9 * PER_CPU finds an address of a per-cpu variable. 30 * PER_CPU finds an address of a per-cpu variable.
@@ -18,14 +39,107 @@
18 * PER_CPU(cpu_gdt_descr, %ebx) 39 * PER_CPU(cpu_gdt_descr, %ebx)
19 */ 40 */
20#ifdef CONFIG_SMP 41#ifdef CONFIG_SMP
21#define PER_CPU(var, cpu) \ 42/* Same as generic implementation except for optimized local access. */
22 movl __per_cpu_offset(,cpu,4), cpu; \ 43#define __GENERIC_PER_CPU
23 addl $per_cpu__##var, cpu; 44
24#else /* ! SMP */ 45/* This is used for other cpus to find our section. */
25#define PER_CPU(var, cpu) \ 46extern unsigned long __per_cpu_offset[];
26 movl $per_cpu__##var, cpu; 47
48/* Separate out the type, so (int[3], foo) works. */
49#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
50#define DEFINE_PER_CPU(type, name) \
51 __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name
52
53/* We can use this directly for local CPU (faster). */
54DECLARE_PER_CPU(unsigned long, this_cpu_off);
55
56/* var is in discarded region: offset to particular copy we want */
57#define per_cpu(var, cpu) (*({ \
58 extern int simple_indentifier_##var(void); \
59 RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]); }))
60
61#define __raw_get_cpu_var(var) (*({ \
62 extern int simple_indentifier_##var(void); \
63 RELOC_HIDE(&per_cpu__##var, x86_read_percpu(this_cpu_off)); \
64}))
65
66#define __get_cpu_var(var) __raw_get_cpu_var(var)
67
68/* A macro to avoid #include hell... */
69#define percpu_modcopy(pcpudst, src, size) \
70do { \
71 unsigned int __i; \
72 for_each_possible_cpu(__i) \
73 memcpy((pcpudst)+__per_cpu_offset[__i], \
74 (src), (size)); \
75} while (0)
76
77#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
78#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
79
80/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
81#define __percpu_seg "%%fs:"
82#else /* !SMP */
83#include <asm-generic/percpu.h>
84#define __percpu_seg ""
27#endif /* SMP */ 85#endif /* SMP */
28 86
87/* For arch-specific code, we can use direct single-insn ops (they
88 * don't give an lvalue though). */
89extern void __bad_percpu_size(void);
90
91#define percpu_to_op(op,var,val) \
92 do { \
93 typedef typeof(var) T__; \
94 if (0) { T__ tmp__; tmp__ = (val); } \
95 switch (sizeof(var)) { \
96 case 1: \
97 asm(op "b %1,"__percpu_seg"%0" \
98 : "+m" (var) \
99 :"ri" ((T__)val)); \
100 break; \
101 case 2: \
102 asm(op "w %1,"__percpu_seg"%0" \
103 : "+m" (var) \
104 :"ri" ((T__)val)); \
105 break; \
106 case 4: \
107 asm(op "l %1,"__percpu_seg"%0" \
108 : "+m" (var) \
109 :"ri" ((T__)val)); \
110 break; \
111 default: __bad_percpu_size(); \
112 } \
113 } while (0)
114
115#define percpu_from_op(op,var) \
116 ({ \
117 typeof(var) ret__; \
118 switch (sizeof(var)) { \
119 case 1: \
120 asm(op "b "__percpu_seg"%1,%0" \
121 : "=r" (ret__) \
122 : "m" (var)); \
123 break; \
124 case 2: \
125 asm(op "w "__percpu_seg"%1,%0" \
126 : "=r" (ret__) \
127 : "m" (var)); \
128 break; \
129 case 4: \
130 asm(op "l "__percpu_seg"%1,%0" \
131 : "=r" (ret__) \
132 : "m" (var)); \
133 break; \
134 default: __bad_percpu_size(); \
135 } \
136 ret__; })
137
138#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
139#define x86_write_percpu(var,val) percpu_to_op("mov", per_cpu__##var, val)
140#define x86_add_percpu(var,val) percpu_to_op("add", per_cpu__##var, val)
141#define x86_sub_percpu(var,val) percpu_to_op("sub", per_cpu__##var, val)
142#define x86_or_percpu(var,val) percpu_to_op("or", per_cpu__##var, val)
29#endif /* !__ASSEMBLY__ */ 143#endif /* !__ASSEMBLY__ */
30 144
31#endif /* __ARCH_I386_PERCPU__ */ 145#endif /* __ARCH_I386_PERCPU__ */
diff --git a/include/asm-i386/processor.h b/include/asm-i386/processor.h
index 922260474646..ced2da8a0d65 100644
--- a/include/asm-i386/processor.h
+++ b/include/asm-i386/processor.h
@@ -377,7 +377,7 @@ struct thread_struct {
377 .vm86_info = NULL, \ 377 .vm86_info = NULL, \
378 .sysenter_cs = __KERNEL_CS, \ 378 .sysenter_cs = __KERNEL_CS, \
379 .io_bitmap_ptr = NULL, \ 379 .io_bitmap_ptr = NULL, \
380 .fs = __KERNEL_PDA, \ 380 .fs = __KERNEL_PERCPU, \
381} 381}
382 382
383/* 383/*
diff --git a/include/asm-i386/segment.h b/include/asm-i386/segment.h
index 065f10bfa487..07e70624d87c 100644
--- a/include/asm-i386/segment.h
+++ b/include/asm-i386/segment.h
@@ -39,7 +39,7 @@
39 * 25 - APM BIOS support 39 * 25 - APM BIOS support
40 * 40 *
41 * 26 - ESPFIX small SS 41 * 26 - ESPFIX small SS
42 * 27 - PDA [ per-cpu private data area ] 42 * 27 - per-cpu [ offset to per-cpu data area ]
43 * 28 - unused 43 * 28 - unused
44 * 29 - unused 44 * 29 - unused
45 * 30 - unused 45 * 30 - unused
@@ -74,8 +74,8 @@
74#define GDT_ENTRY_ESPFIX_SS (GDT_ENTRY_KERNEL_BASE + 14) 74#define GDT_ENTRY_ESPFIX_SS (GDT_ENTRY_KERNEL_BASE + 14)
75#define __ESPFIX_SS (GDT_ENTRY_ESPFIX_SS * 8) 75#define __ESPFIX_SS (GDT_ENTRY_ESPFIX_SS * 8)
76 76
77#define GDT_ENTRY_PDA (GDT_ENTRY_KERNEL_BASE + 15) 77#define GDT_ENTRY_PERCPU (GDT_ENTRY_KERNEL_BASE + 15)
78#define __KERNEL_PDA (GDT_ENTRY_PDA * 8) 78#define __KERNEL_PERCPU (GDT_ENTRY_PERCPU * 8)
79 79
80#define GDT_ENTRY_DOUBLEFAULT_TSS 31 80#define GDT_ENTRY_DOUBLEFAULT_TSS 31
81 81
diff --git a/include/asm-i386/smp.h b/include/asm-i386/smp.h
index 2d083cb4ca93..090abc1da32a 100644
--- a/include/asm-i386/smp.h
+++ b/include/asm-i386/smp.h
@@ -8,7 +8,6 @@
8#include <linux/kernel.h> 8#include <linux/kernel.h>
9#include <linux/threads.h> 9#include <linux/threads.h>
10#include <linux/cpumask.h> 10#include <linux/cpumask.h>
11#include <asm/pda.h>
12#endif 11#endif
13 12
14#if defined(CONFIG_X86_LOCAL_APIC) && !defined(__ASSEMBLY__) 13#if defined(CONFIG_X86_LOCAL_APIC) && !defined(__ASSEMBLY__)
@@ -112,7 +111,8 @@ do { } while (0)
112 * from the initial startup. We map APIC_BASE very early in page_setup(), 111 * from the initial startup. We map APIC_BASE very early in page_setup(),
113 * so this is correct in the x86 case. 112 * so this is correct in the x86 case.
114 */ 113 */
115#define raw_smp_processor_id() (read_pda(cpu_number)) 114DECLARE_PER_CPU(int, cpu_number);
115#define raw_smp_processor_id() (x86_read_percpu(cpu_number))
116 116
117extern cpumask_t cpu_callout_map; 117extern cpumask_t cpu_callout_map;
118extern cpumask_t cpu_callin_map; 118extern cpumask_t cpu_callin_map;
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-08 18:31:14 -0400