aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorJeremy Fitzhardinge <jeremy@goop.org>2007-05-02 13:27:16 -0400
committerAndi Kleen <andi@basil.nowhere.org>2007-05-02 13:27:16 -0400
commit7c3576d261ce046789a7db14f43303f8120910c7 (patch)
treead27a8459bbcdb183fe2411aec3b840942992ad5 /include
parent7a61d35d4b4056e7711031202da7605e052f4137 (diff)
[PATCH] i386: Convert PDA into the percpu section
Currently x86 (similar to x84-64) has a special per-cpu structure called "i386_pda" which can be easily and efficiently referenced via the %fs register. An ELF section is more flexible than a structure, allowing any piece of code to use this area. Indeed, such a section already exists: the per-cpu area. So this patch: (1) Removes the PDA and uses per-cpu variables for each current member. (2) Replaces the __KERNEL_PDA segment with __KERNEL_PERCPU. (3) Creates a per-cpu mirror of __per_cpu_offset called this_cpu_off, which can be used to calculate addresses for this CPU's variables. (4) Simplifies startup, because %fs doesn't need to be loaded with a special segment at early boot; it can be deferred until the first percpu area is allocated (or never for UP). The result is less code and one less x86-specific concept. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com> Signed-off-by: Andi Kleen <ak@suse.de> Cc: Andi Kleen <ak@suse.de>
Diffstat (limited to 'include')
-rw-r--r--include/asm-i386/current.h5
-rw-r--r--include/asm-i386/irq_regs.h12
-rw-r--r--include/asm-i386/pda.h99
-rw-r--r--include/asm-i386/percpu.h132
-rw-r--r--include/asm-i386/processor.h2
-rw-r--r--include/asm-i386/segment.h6
-rw-r--r--include/asm-i386/smp.h4
7 files changed, 139 insertions, 121 deletions
diff --git a/include/asm-i386/current.h b/include/asm-i386/current.h
index 5252ee0f6d7a..d35248539912 100644
--- a/include/asm-i386/current.h
+++ b/include/asm-i386/current.h
@@ -1,14 +1,15 @@
1#ifndef _I386_CURRENT_H 1#ifndef _I386_CURRENT_H
2#define _I386_CURRENT_H 2#define _I386_CURRENT_H
3 3
4#include <asm/pda.h>
5#include <linux/compiler.h> 4#include <linux/compiler.h>
5#include <asm/percpu.h>
6 6
7struct task_struct; 7struct task_struct;
8 8
9DECLARE_PER_CPU(struct task_struct *, current_task);
9static __always_inline struct task_struct *get_current(void) 10static __always_inline struct task_struct *get_current(void)
10{ 11{
11 return read_pda(pcurrent); 12 return x86_read_percpu(current_task);
12} 13}
13 14
14#define current get_current() 15#define current get_current()
diff --git a/include/asm-i386/irq_regs.h b/include/asm-i386/irq_regs.h
index a1b3f7f594a2..3368b20c0b48 100644
--- a/include/asm-i386/irq_regs.h
+++ b/include/asm-i386/irq_regs.h
@@ -1,25 +1,27 @@
1/* 1/*
2 * Per-cpu current frame pointer - the location of the last exception frame on 2 * Per-cpu current frame pointer - the location of the last exception frame on
3 * the stack, stored in the PDA. 3 * the stack, stored in the per-cpu area.
4 * 4 *
5 * Jeremy Fitzhardinge <jeremy@goop.org> 5 * Jeremy Fitzhardinge <jeremy@goop.org>
6 */ 6 */
7#ifndef _ASM_I386_IRQ_REGS_H 7#ifndef _ASM_I386_IRQ_REGS_H
8#define _ASM_I386_IRQ_REGS_H 8#define _ASM_I386_IRQ_REGS_H
9 9
10#include <asm/pda.h> 10#include <asm/percpu.h>
11
12DECLARE_PER_CPU(struct pt_regs *, irq_regs);
11 13
12static inline struct pt_regs *get_irq_regs(void) 14static inline struct pt_regs *get_irq_regs(void)
13{ 15{
14 return read_pda(irq_regs); 16 return x86_read_percpu(irq_regs);
15} 17}
16 18
17static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs) 19static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
18{ 20{
19 struct pt_regs *old_regs; 21 struct pt_regs *old_regs;
20 22
21 old_regs = read_pda(irq_regs); 23 old_regs = get_irq_regs();
22 write_pda(irq_regs, new_regs); 24 x86_write_percpu(irq_regs, new_regs);
23 25
24 return old_regs; 26 return old_regs;
25} 27}
diff --git a/include/asm-i386/pda.h b/include/asm-i386/pda.h
deleted file mode 100644
index aef7f732f77e..000000000000
--- a/include/asm-i386/pda.h
+++ /dev/null
@@ -1,99 +0,0 @@
1/*
2 Per-processor Data Areas
3 Jeremy Fitzhardinge <jeremy@goop.org> 2006
4 Based on asm-x86_64/pda.h by Andi Kleen.
5 */
6#ifndef _I386_PDA_H
7#define _I386_PDA_H
8
9#include <linux/stddef.h>
10#include <linux/types.h>
11#include <asm/percpu.h>
12
13struct i386_pda
14{
15 struct i386_pda *_pda; /* pointer to self */
16
17 int cpu_number;
18 struct task_struct *pcurrent; /* current process */
19 struct pt_regs *irq_regs;
20};
21
22DECLARE_PER_CPU(struct i386_pda, _cpu_pda);
23#define cpu_pda(i) (&per_cpu(_cpu_pda, (i)))
24#define pda_offset(field) offsetof(struct i386_pda, field)
25
26extern void __bad_pda_field(void);
27
28/* This variable is never instantiated. It is only used as a stand-in
29 for the real per-cpu PDA memory, so that gcc can understand what
30 memory operations the inline asms() below are performing. This
31 eliminates the need to make the asms volatile or have memory
32 clobbers, so gcc can readily analyse them. */
33extern struct i386_pda _proxy_pda;
34
35#define pda_to_op(op,field,val) \
36 do { \
37 typedef typeof(_proxy_pda.field) T__; \
38 if (0) { T__ tmp__; tmp__ = (val); } \
39 switch (sizeof(_proxy_pda.field)) { \
40 case 1: \
41 asm(op "b %1,%%fs:%c2" \
42 : "+m" (_proxy_pda.field) \
43 :"ri" ((T__)val), \
44 "i"(pda_offset(field))); \
45 break; \
46 case 2: \
47 asm(op "w %1,%%fs:%c2" \
48 : "+m" (_proxy_pda.field) \
49 :"ri" ((T__)val), \
50 "i"(pda_offset(field))); \
51 break; \
52 case 4: \
53 asm(op "l %1,%%fs:%c2" \
54 : "+m" (_proxy_pda.field) \
55 :"ri" ((T__)val), \
56 "i"(pda_offset(field))); \
57 break; \
58 default: __bad_pda_field(); \
59 } \
60 } while (0)
61
62#define pda_from_op(op,field) \
63 ({ \
64 typeof(_proxy_pda.field) ret__; \
65 switch (sizeof(_proxy_pda.field)) { \
66 case 1: \
67 asm(op "b %%fs:%c1,%0" \
68 : "=r" (ret__) \
69 : "i" (pda_offset(field)), \
70 "m" (_proxy_pda.field)); \
71 break; \
72 case 2: \
73 asm(op "w %%fs:%c1,%0" \
74 : "=r" (ret__) \
75 : "i" (pda_offset(field)), \
76 "m" (_proxy_pda.field)); \
77 break; \
78 case 4: \
79 asm(op "l %%fs:%c1,%0" \
80 : "=r" (ret__) \
81 : "i" (pda_offset(field)), \
82 "m" (_proxy_pda.field)); \
83 break; \
84 default: __bad_pda_field(); \
85 } \
86 ret__; })
87
88/* Return a pointer to a pda field */
89#define pda_addr(field) \
90 ((typeof(_proxy_pda.field) *)((unsigned char *)read_pda(_pda) + \
91 pda_offset(field)))
92
93#define read_pda(field) pda_from_op("mov",field)
94#define write_pda(field,val) pda_to_op("mov",field,val)
95#define add_pda(field,val) pda_to_op("add",field,val)
96#define sub_pda(field,val) pda_to_op("sub",field,val)
97#define or_pda(field,val) pda_to_op("or",field,val)
98
99#endif /* _I386_PDA_H */
diff --git a/include/asm-i386/percpu.h b/include/asm-i386/percpu.h
index a10e7c68ae9d..c5f12f0d9c23 100644
--- a/include/asm-i386/percpu.h
+++ b/include/asm-i386/percpu.h
@@ -1,9 +1,30 @@
1#ifndef __ARCH_I386_PERCPU__ 1#ifndef __ARCH_I386_PERCPU__
2#define __ARCH_I386_PERCPU__ 2#define __ARCH_I386_PERCPU__
3 3
4#ifndef __ASSEMBLY__ 4#ifdef __ASSEMBLY__
5#include <asm-generic/percpu.h> 5
6#else 6/*
7 * PER_CPU finds an address of a per-cpu variable.
8 *
9 * Args:
10 * var - variable name
11 * reg - 32bit register
12 *
13 * The resulting address is stored in the "reg" argument.
14 *
15 * Example:
16 * PER_CPU(cpu_gdt_descr, %ebx)
17 */
18#ifdef CONFIG_SMP
19#define PER_CPU(var, reg) \
20 movl %fs:per_cpu__this_cpu_off, reg; \
21 addl $per_cpu__##var, reg
22#else /* ! SMP */
23#define PER_CPU(var, reg) \
24 movl $per_cpu__##var, reg;
25#endif /* SMP */
26
27#else /* ...!ASSEMBLY */
7 28
8/* 29/*
9 * PER_CPU finds an address of a per-cpu variable. 30 * PER_CPU finds an address of a per-cpu variable.
@@ -18,14 +39,107 @@
18 * PER_CPU(cpu_gdt_descr, %ebx) 39 * PER_CPU(cpu_gdt_descr, %ebx)
19 */ 40 */
20#ifdef CONFIG_SMP 41#ifdef CONFIG_SMP
21#define PER_CPU(var, cpu) \ 42/* Same as generic implementation except for optimized local access. */
22 movl __per_cpu_offset(,cpu,4), cpu; \ 43#define __GENERIC_PER_CPU
23 addl $per_cpu__##var, cpu; 44
24#else /* ! SMP */ 45/* This is used for other cpus to find our section. */
25#define PER_CPU(var, cpu) \ 46extern unsigned long __per_cpu_offset[];
26 movl $per_cpu__##var, cpu; 47
48/* Separate out the type, so (int[3], foo) works. */
49#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
50#define DEFINE_PER_CPU(type, name) \
51 __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name
52
53/* We can use this directly for local CPU (faster). */
54DECLARE_PER_CPU(unsigned long, this_cpu_off);
55
56/* var is in discarded region: offset to particular copy we want */
57#define per_cpu(var, cpu) (*({ \
58 extern int simple_indentifier_##var(void); \
59 RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]); }))
60
61#define __raw_get_cpu_var(var) (*({ \
62 extern int simple_indentifier_##var(void); \
63 RELOC_HIDE(&per_cpu__##var, x86_read_percpu(this_cpu_off)); \
64}))
65
66#define __get_cpu_var(var) __raw_get_cpu_var(var)
67
68/* A macro to avoid #include hell... */
69#define percpu_modcopy(pcpudst, src, size) \
70do { \
71 unsigned int __i; \
72 for_each_possible_cpu(__i) \
73 memcpy((pcpudst)+__per_cpu_offset[__i], \
74 (src), (size)); \
75} while (0)
76
77#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
78#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
79
80/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
81#define __percpu_seg "%%fs:"
82#else /* !SMP */
83#include <asm-generic/percpu.h>
84#define __percpu_seg ""
27#endif /* SMP */ 85#endif /* SMP */
28 86
87/* For arch-specific code, we can use direct single-insn ops (they
88 * don't give an lvalue though). */
89extern void __bad_percpu_size(void);
90
91#define percpu_to_op(op,var,val) \
92 do { \
93 typedef typeof(var) T__; \
94 if (0) { T__ tmp__; tmp__ = (val); } \
95 switch (sizeof(var)) { \
96 case 1: \
97 asm(op "b %1,"__percpu_seg"%0" \
98 : "+m" (var) \
99 :"ri" ((T__)val)); \
100 break; \
101 case 2: \
102 asm(op "w %1,"__percpu_seg"%0" \
103 : "+m" (var) \
104 :"ri" ((T__)val)); \
105 break; \
106 case 4: \
107 asm(op "l %1,"__percpu_seg"%0" \
108 : "+m" (var) \
109 :"ri" ((T__)val)); \
110 break; \
111 default: __bad_percpu_size(); \
112 } \
113 } while (0)
114
115#define percpu_from_op(op,var) \
116 ({ \
117 typeof(var) ret__; \
118 switch (sizeof(var)) { \
119 case 1: \
120 asm(op "b "__percpu_seg"%1,%0" \
121 : "=r" (ret__) \
122 : "m" (var)); \
123 break; \
124 case 2: \
125 asm(op "w "__percpu_seg"%1,%0" \
126 : "=r" (ret__) \
127 : "m" (var)); \
128 break; \
129 case 4: \
130 asm(op "l "__percpu_seg"%1,%0" \
131 : "=r" (ret__) \
132 : "m" (var)); \
133 break; \
134 default: __bad_percpu_size(); \
135 } \
136 ret__; })
137
138#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
139#define x86_write_percpu(var,val) percpu_to_op("mov", per_cpu__##var, val)
140#define x86_add_percpu(var,val) percpu_to_op("add", per_cpu__##var, val)
141#define x86_sub_percpu(var,val) percpu_to_op("sub", per_cpu__##var, val)
142#define x86_or_percpu(var,val) percpu_to_op("or", per_cpu__##var, val)
29#endif /* !__ASSEMBLY__ */ 143#endif /* !__ASSEMBLY__ */
30 144
31#endif /* __ARCH_I386_PERCPU__ */ 145#endif /* __ARCH_I386_PERCPU__ */
diff --git a/include/asm-i386/processor.h b/include/asm-i386/processor.h
index 922260474646..ced2da8a0d65 100644
--- a/include/asm-i386/processor.h
+++ b/include/asm-i386/processor.h
@@ -377,7 +377,7 @@ struct thread_struct {
377 .vm86_info = NULL, \ 377 .vm86_info = NULL, \
378 .sysenter_cs = __KERNEL_CS, \ 378 .sysenter_cs = __KERNEL_CS, \
379 .io_bitmap_ptr = NULL, \ 379 .io_bitmap_ptr = NULL, \
380 .fs = __KERNEL_PDA, \ 380 .fs = __KERNEL_PERCPU, \
381} 381}
382 382
383/* 383/*
diff --git a/include/asm-i386/segment.h b/include/asm-i386/segment.h
index 065f10bfa487..07e70624d87c 100644
--- a/include/asm-i386/segment.h
+++ b/include/asm-i386/segment.h
@@ -39,7 +39,7 @@
39 * 25 - APM BIOS support 39 * 25 - APM BIOS support
40 * 40 *
41 * 26 - ESPFIX small SS 41 * 26 - ESPFIX small SS
42 * 27 - PDA [ per-cpu private data area ] 42 * 27 - per-cpu [ offset to per-cpu data area ]
43 * 28 - unused 43 * 28 - unused
44 * 29 - unused 44 * 29 - unused
45 * 30 - unused 45 * 30 - unused
@@ -74,8 +74,8 @@
74#define GDT_ENTRY_ESPFIX_SS (GDT_ENTRY_KERNEL_BASE + 14) 74#define GDT_ENTRY_ESPFIX_SS (GDT_ENTRY_KERNEL_BASE + 14)
75#define __ESPFIX_SS (GDT_ENTRY_ESPFIX_SS * 8) 75#define __ESPFIX_SS (GDT_ENTRY_ESPFIX_SS * 8)
76 76
77#define GDT_ENTRY_PDA (GDT_ENTRY_KERNEL_BASE + 15) 77#define GDT_ENTRY_PERCPU (GDT_ENTRY_KERNEL_BASE + 15)
78#define __KERNEL_PDA (GDT_ENTRY_PDA * 8) 78#define __KERNEL_PERCPU (GDT_ENTRY_PERCPU * 8)
79 79
80#define GDT_ENTRY_DOUBLEFAULT_TSS 31 80#define GDT_ENTRY_DOUBLEFAULT_TSS 31
81 81
diff --git a/include/asm-i386/smp.h b/include/asm-i386/smp.h
index 2d083cb4ca93..090abc1da32a 100644
--- a/include/asm-i386/smp.h
+++ b/include/asm-i386/smp.h
@@ -8,7 +8,6 @@
8#include <linux/kernel.h> 8#include <linux/kernel.h>
9#include <linux/threads.h> 9#include <linux/threads.h>
10#include <linux/cpumask.h> 10#include <linux/cpumask.h>
11#include <asm/pda.h>
12#endif 11#endif
13 12
14#if defined(CONFIG_X86_LOCAL_APIC) && !defined(__ASSEMBLY__) 13#if defined(CONFIG_X86_LOCAL_APIC) && !defined(__ASSEMBLY__)
@@ -112,7 +111,8 @@ do { } while (0)
112 * from the initial startup. We map APIC_BASE very early in page_setup(), 111 * from the initial startup. We map APIC_BASE very early in page_setup(),
113 * so this is correct in the x86 case. 112 * so this is correct in the x86 case.
114 */ 113 */
115#define raw_smp_processor_id() (read_pda(cpu_number)) 114DECLARE_PER_CPU(int, cpu_number);
115#define raw_smp_processor_id() (x86_read_percpu(cpu_number))
116 116
117extern cpumask_t cpu_callout_map; 117extern cpumask_t cpu_callout_map;
118extern cpumask_t cpu_callin_map; 118extern cpumask_t cpu_callin_map;
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-02 01:51:52 -0500