aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/include/asm/percpu.h
diff options
context:
space:
mode:
authorTejun Heo <tj@kernel.org>2009-01-13 06:41:35 -0500
committerIngo Molnar <mingo@elte.hu>2009-01-16 08:19:58 -0500
commit9939ddaff52787b2a7c1adf1b2afc95421aa0884 (patch)
tree6e7266d065914e19c3c3f4b4e475f09b9669fa51 /arch/x86/include/asm/percpu.h
parent1a51e3a0aed18767cf2762e95456ecfeb0bca5e6 (diff)
x86: merge 64 and 32 SMP percpu handling
Now that pda is allocated as part of percpu, percpu doesn't need to be accessed through pda. Unify x86_64 SMP percpu access with x86_32 SMP one. Other than the segment register, operand size and the base of percpu symbols, they behave identical now. This patch replaces now unnecessary pda->data_offset with a dummy field which is necessary to keep stack_canary at its place. This patch also moves per_cpu_offset initialization out of init_gdt() into setup_per_cpu_areas(). Note that this change also necessitates explicit per_cpu_offset initializations in voyager_smp.c. With this change, x86_OP_percpu()'s are as efficient on x86_64 as on x86_32 and also x86_64 can use assembly PER_CPU macros. Signed-off-by: Tejun Heo <tj@kernel.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'arch/x86/include/asm/percpu.h')
-rw-r--r--arch/x86/include/asm/percpu.h127
1 files changed, 39 insertions, 88 deletions
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 0ed77cf33f76..556f84b9ea96 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -1,62 +1,13 @@
1#ifndef _ASM_X86_PERCPU_H 1#ifndef _ASM_X86_PERCPU_H
2#define _ASM_X86_PERCPU_H 2#define _ASM_X86_PERCPU_H
3 3
4#ifndef __ASSEMBLY__
5#ifdef CONFIG_X86_64 4#ifdef CONFIG_X86_64
6extern void load_pda_offset(int cpu); 5#define __percpu_seg gs
6#define __percpu_mov_op movq
7#else 7#else
8static inline void load_pda_offset(int cpu) { } 8#define __percpu_seg fs
9#endif 9#define __percpu_mov_op movl
10#endif
11
12#ifdef CONFIG_X86_64
13#include <linux/compiler.h>
14
15/* Same as asm-generic/percpu.h, except that we store the per cpu offset
16 in the PDA. Longer term the PDA and every per cpu variable
17 should be just put into a single section and referenced directly
18 from %gs */
19
20#ifdef CONFIG_SMP
21#include <asm/pda.h>
22
23#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
24#define __my_cpu_offset read_pda(data_offset)
25
26#define per_cpu_offset(x) (__per_cpu_offset(x))
27
28#endif 10#endif
29#include <asm-generic/percpu.h>
30
31DECLARE_PER_CPU(struct x8664_pda, pda);
32
33/*
34 * These are supposed to be implemented as a single instruction which
35 * operates on the per-cpu data base segment. x86-64 doesn't have
36 * that yet, so this is a fairly inefficient workaround for the
37 * meantime. The single instruction is atomic with respect to
38 * preemption and interrupts, so we need to explicitly disable
39 * interrupts here to achieve the same effect. However, because it
40 * can be used from within interrupt-disable/enable, we can't actually
41 * disable interrupts; disabling preemption is enough.
42 */
43#define x86_read_percpu(var) \
44 ({ \
45 typeof(per_cpu_var(var)) __tmp; \
46 preempt_disable(); \
47 __tmp = __get_cpu_var(var); \
48 preempt_enable(); \
49 __tmp; \
50 })
51
52#define x86_write_percpu(var, val) \
53 do { \
54 preempt_disable(); \
55 __get_cpu_var(var) = (val); \
56 preempt_enable(); \
57 } while(0)
58
59#else /* CONFIG_X86_64 */
60 11
61#ifdef __ASSEMBLY__ 12#ifdef __ASSEMBLY__
62 13
@@ -73,42 +24,26 @@ DECLARE_PER_CPU(struct x8664_pda, pda);
73 * PER_CPU(cpu_gdt_descr, %ebx) 24 * PER_CPU(cpu_gdt_descr, %ebx)
74 */ 25 */
75#ifdef CONFIG_SMP 26#ifdef CONFIG_SMP
76#define PER_CPU(var, reg) \ 27#define PER_CPU(var, reg) \
77 movl %fs:per_cpu__##this_cpu_off, reg; \ 28 __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg; \
78 lea per_cpu__##var(reg), reg 29 lea per_cpu__##var(reg), reg
79#define PER_CPU_VAR(var) %fs:per_cpu__##var 30#define PER_CPU_VAR(var) %__percpu_seg:per_cpu__##var
80#else /* ! SMP */ 31#else /* ! SMP */
81#define PER_CPU(var, reg) \ 32#define PER_CPU(var, reg) \
82 movl $per_cpu__##var, reg 33 __percpu_mov_op $per_cpu__##var, reg
83#define PER_CPU_VAR(var) per_cpu__##var 34#define PER_CPU_VAR(var) per_cpu__##var
84#endif /* SMP */ 35#endif /* SMP */
85 36
86#else /* ...!ASSEMBLY */ 37#else /* ...!ASSEMBLY */
87 38
88/* 39#include <linux/stringify.h>
89 * PER_CPU finds an address of a per-cpu variable.
90 *
91 * Args:
92 * var - variable name
93 * cpu - 32bit register containing the current CPU number
94 *
95 * The resulting address is stored in the "cpu" argument.
96 *
97 * Example:
98 * PER_CPU(cpu_gdt_descr, %ebx)
99 */
100#ifdef CONFIG_SMP
101
102#define __my_cpu_offset x86_read_percpu(this_cpu_off)
103
104/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
105#define __percpu_seg "%%fs:"
106 40
107#else /* !SMP */ 41#ifdef CONFIG_SMP
108 42#define __percpu_seg_str "%%"__stringify(__percpu_seg)":"
109#define __percpu_seg "" 43#define __my_cpu_offset x86_read_percpu(this_cpu_off)
110 44#else
111#endif /* SMP */ 45#define __percpu_seg_str
46#endif
112 47
113#include <asm-generic/percpu.h> 48#include <asm-generic/percpu.h>
114 49
@@ -128,20 +63,25 @@ do { \
128 } \ 63 } \
129 switch (sizeof(var)) { \ 64 switch (sizeof(var)) { \
130 case 1: \ 65 case 1: \
131 asm(op "b %1,"__percpu_seg"%0" \ 66 asm(op "b %1,"__percpu_seg_str"%0" \
132 : "+m" (var) \ 67 : "+m" (var) \
133 : "ri" ((T__)val)); \ 68 : "ri" ((T__)val)); \
134 break; \ 69 break; \
135 case 2: \ 70 case 2: \
136 asm(op "w %1,"__percpu_seg"%0" \ 71 asm(op "w %1,"__percpu_seg_str"%0" \
137 : "+m" (var) \ 72 : "+m" (var) \
138 : "ri" ((T__)val)); \ 73 : "ri" ((T__)val)); \
139 break; \ 74 break; \
140 case 4: \ 75 case 4: \
141 asm(op "l %1,"__percpu_seg"%0" \ 76 asm(op "l %1,"__percpu_seg_str"%0" \
142 : "+m" (var) \ 77 : "+m" (var) \
143 : "ri" ((T__)val)); \ 78 : "ri" ((T__)val)); \
144 break; \ 79 break; \
80 case 8: \
81 asm(op "q %1,"__percpu_seg_str"%0" \
82 : "+m" (var) \
83 : "r" ((T__)val)); \
84 break; \
145 default: __bad_percpu_size(); \ 85 default: __bad_percpu_size(); \
146 } \ 86 } \
147} while (0) 87} while (0)
@@ -151,17 +91,22 @@ do { \
151 typeof(var) ret__; \ 91 typeof(var) ret__; \
152 switch (sizeof(var)) { \ 92 switch (sizeof(var)) { \
153 case 1: \ 93 case 1: \
154 asm(op "b "__percpu_seg"%1,%0" \ 94 asm(op "b "__percpu_seg_str"%1,%0" \
155 : "=r" (ret__) \ 95 : "=r" (ret__) \
156 : "m" (var)); \ 96 : "m" (var)); \
157 break; \ 97 break; \
158 case 2: \ 98 case 2: \
159 asm(op "w "__percpu_seg"%1,%0" \ 99 asm(op "w "__percpu_seg_str"%1,%0" \
160 : "=r" (ret__) \ 100 : "=r" (ret__) \
161 : "m" (var)); \ 101 : "m" (var)); \
162 break; \ 102 break; \
163 case 4: \ 103 case 4: \
164 asm(op "l "__percpu_seg"%1,%0" \ 104 asm(op "l "__percpu_seg_str"%1,%0" \
105 : "=r" (ret__) \
106 : "m" (var)); \
107 break; \
108 case 8: \
109 asm(op "q "__percpu_seg_str"%1,%0" \
165 : "=r" (ret__) \ 110 : "=r" (ret__) \
166 : "m" (var)); \ 111 : "m" (var)); \
167 break; \ 112 break; \
@@ -175,8 +120,14 @@ do { \
175#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val) 120#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val)
176#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val) 121#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val)
177#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val) 122#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val)
123
124#ifdef CONFIG_X86_64
125extern void load_pda_offset(int cpu);
126#else
127static inline void load_pda_offset(int cpu) { }
128#endif
129
178#endif /* !__ASSEMBLY__ */ 130#endif /* !__ASSEMBLY__ */
179#endif /* !CONFIG_X86_64 */
180 131
181#ifdef CONFIG_SMP 132#ifdef CONFIG_SMP
182 133
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-16 00:19:06 -0500