aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/include/asm/percpu.h
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/include/asm/percpu.h')
-rw-r--r--arch/x86/include/asm/percpu.h169
1 files changed, 78 insertions, 91 deletions
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index ece72053ba6..aee103b26d0 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -2,53 +2,12 @@
2#define _ASM_X86_PERCPU_H 2#define _ASM_X86_PERCPU_H
3 3
4#ifdef CONFIG_X86_64 4#ifdef CONFIG_X86_64
5#include <linux/compiler.h> 5#define __percpu_seg gs
6 6#define __percpu_mov_op movq
7/* Same as asm-generic/percpu.h, except that we store the per cpu offset 7#else
8 in the PDA. Longer term the PDA and every per cpu variable 8#define __percpu_seg fs
9 should be just put into a single section and referenced directly 9#define __percpu_mov_op movl
10 from %gs */
11
12#ifdef CONFIG_SMP
13#include <asm/pda.h>
14
15#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
16#define __my_cpu_offset read_pda(data_offset)
17
18#define per_cpu_offset(x) (__per_cpu_offset(x))
19
20#endif 10#endif
21#include <asm-generic/percpu.h>
22
23DECLARE_PER_CPU(struct x8664_pda, pda);
24
25/*
26 * These are supposed to be implemented as a single instruction which
27 * operates on the per-cpu data base segment. x86-64 doesn't have
28 * that yet, so this is a fairly inefficient workaround for the
29 * meantime. The single instruction is atomic with respect to
30 * preemption and interrupts, so we need to explicitly disable
31 * interrupts here to achieve the same effect. However, because it
32 * can be used from within interrupt-disable/enable, we can't actually
33 * disable interrupts; disabling preemption is enough.
34 */
35#define x86_read_percpu(var) \
36 ({ \
37 typeof(per_cpu_var(var)) __tmp; \
38 preempt_disable(); \
39 __tmp = __get_cpu_var(var); \
40 preempt_enable(); \
41 __tmp; \
42 })
43
44#define x86_write_percpu(var, val) \
45 do { \
46 preempt_disable(); \
47 __get_cpu_var(var) = (val); \
48 preempt_enable(); \
49 } while(0)
50
51#else /* CONFIG_X86_64 */
52 11
53#ifdef __ASSEMBLY__ 12#ifdef __ASSEMBLY__
54 13
@@ -65,47 +24,48 @@ DECLARE_PER_CPU(struct x8664_pda, pda);
65 * PER_CPU(cpu_gdt_descr, %ebx) 24 * PER_CPU(cpu_gdt_descr, %ebx)
66 */ 25 */
67#ifdef CONFIG_SMP 26#ifdef CONFIG_SMP
68#define PER_CPU(var, reg) \ 27#define PER_CPU(var, reg) \
69 movl %fs:per_cpu__##this_cpu_off, reg; \ 28 __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg; \
70 lea per_cpu__##var(reg), reg 29 lea per_cpu__##var(reg), reg
71#define PER_CPU_VAR(var) %fs:per_cpu__##var 30#define PER_CPU_VAR(var) %__percpu_seg:per_cpu__##var
72#else /* ! SMP */ 31#else /* ! SMP */
73#define PER_CPU(var, reg) \ 32#define PER_CPU(var, reg) \
74 movl $per_cpu__##var, reg 33 __percpu_mov_op $per_cpu__##var, reg
75#define PER_CPU_VAR(var) per_cpu__##var 34#define PER_CPU_VAR(var) per_cpu__##var
76#endif /* SMP */ 35#endif /* SMP */
77 36
37#ifdef CONFIG_X86_64_SMP
38#define INIT_PER_CPU_VAR(var) init_per_cpu__##var
39#else
40#define INIT_PER_CPU_VAR(var) per_cpu__##var
41#endif
42
78#else /* ...!ASSEMBLY */ 43#else /* ...!ASSEMBLY */
79 44
45#include <linux/stringify.h>
46
47#ifdef CONFIG_SMP
48#define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x
49#define __my_cpu_offset percpu_read(this_cpu_off)
50#else
51#define __percpu_arg(x) "%" #x
52#endif
53
80/* 54/*
81 * PER_CPU finds an address of a per-cpu variable. 55 * Initialized pointers to per-cpu variables needed for the boot
56 * processor need to use these macros to get the proper address
57 * offset from __per_cpu_load on SMP.
82 * 58 *
83 * Args: 59 * There also must be an entry in vmlinux_64.lds.S
84 * var - variable name
85 * cpu - 32bit register containing the current CPU number
86 *
87 * The resulting address is stored in the "cpu" argument.
88 *
89 * Example:
90 * PER_CPU(cpu_gdt_descr, %ebx)
91 */ 60 */
92#ifdef CONFIG_SMP 61#define DECLARE_INIT_PER_CPU(var) \
93 62 extern typeof(per_cpu_var(var)) init_per_cpu_var(var)
94#define __my_cpu_offset x86_read_percpu(this_cpu_off)
95
96/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
97#define __percpu_seg "%%fs:"
98
99#else /* !SMP */
100
101#define __percpu_seg ""
102
103#endif /* SMP */
104
105#include <asm-generic/percpu.h>
106 63
107/* We can use this directly for local CPU (faster). */ 64#ifdef CONFIG_X86_64_SMP
108DECLARE_PER_CPU(unsigned long, this_cpu_off); 65#define init_per_cpu_var(var) init_per_cpu__##var
66#else
67#define init_per_cpu_var(var) per_cpu_var(var)
68#endif
109 69
110/* For arch-specific code, we can use direct single-insn ops (they 70/* For arch-specific code, we can use direct single-insn ops (they
111 * don't give an lvalue though). */ 71 * don't give an lvalue though). */
@@ -120,20 +80,25 @@ do { \
120 } \ 80 } \
121 switch (sizeof(var)) { \ 81 switch (sizeof(var)) { \
122 case 1: \ 82 case 1: \
123 asm(op "b %1,"__percpu_seg"%0" \ 83 asm(op "b %1,"__percpu_arg(0) \
124 : "+m" (var) \ 84 : "+m" (var) \
125 : "ri" ((T__)val)); \ 85 : "ri" ((T__)val)); \
126 break; \ 86 break; \
127 case 2: \ 87 case 2: \
128 asm(op "w %1,"__percpu_seg"%0" \ 88 asm(op "w %1,"__percpu_arg(0) \
129 : "+m" (var) \ 89 : "+m" (var) \
130 : "ri" ((T__)val)); \ 90 : "ri" ((T__)val)); \
131 break; \ 91 break; \
132 case 4: \ 92 case 4: \
133 asm(op "l %1,"__percpu_seg"%0" \ 93 asm(op "l %1,"__percpu_arg(0) \
134 : "+m" (var) \ 94 : "+m" (var) \
135 : "ri" ((T__)val)); \ 95 : "ri" ((T__)val)); \
136 break; \ 96 break; \
97 case 8: \
98 asm(op "q %1,"__percpu_arg(0) \
99 : "+m" (var) \
100 : "re" ((T__)val)); \
101 break; \
137 default: __bad_percpu_size(); \ 102 default: __bad_percpu_size(); \
138 } \ 103 } \
139} while (0) 104} while (0)
@@ -143,17 +108,22 @@ do { \
143 typeof(var) ret__; \ 108 typeof(var) ret__; \
144 switch (sizeof(var)) { \ 109 switch (sizeof(var)) { \
145 case 1: \ 110 case 1: \
146 asm(op "b "__percpu_seg"%1,%0" \ 111 asm(op "b "__percpu_arg(1)",%0" \
147 : "=r" (ret__) \ 112 : "=r" (ret__) \
148 : "m" (var)); \ 113 : "m" (var)); \
149 break; \ 114 break; \
150 case 2: \ 115 case 2: \
151 asm(op "w "__percpu_seg"%1,%0" \ 116 asm(op "w "__percpu_arg(1)",%0" \
152 : "=r" (ret__) \ 117 : "=r" (ret__) \
153 : "m" (var)); \ 118 : "m" (var)); \
154 break; \ 119 break; \
155 case 4: \ 120 case 4: \
156 asm(op "l "__percpu_seg"%1,%0" \ 121 asm(op "l "__percpu_arg(1)",%0" \
122 : "=r" (ret__) \
123 : "m" (var)); \
124 break; \
125 case 8: \
126 asm(op "q "__percpu_arg(1)",%0" \
157 : "=r" (ret__) \ 127 : "=r" (ret__) \
158 : "m" (var)); \ 128 : "m" (var)); \
159 break; \ 129 break; \
@@ -162,13 +132,30 @@ do { \
162 ret__; \ 132 ret__; \
163}) 133})
164 134
165#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var) 135#define percpu_read(var) percpu_from_op("mov", per_cpu__##var)
166#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val) 136#define percpu_write(var, val) percpu_to_op("mov", per_cpu__##var, val)
167#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val) 137#define percpu_add(var, val) percpu_to_op("add", per_cpu__##var, val)
168#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val) 138#define percpu_sub(var, val) percpu_to_op("sub", per_cpu__##var, val)
169#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val) 139#define percpu_and(var, val) percpu_to_op("and", per_cpu__##var, val)
140#define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val)
141#define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val)
142
143/* This is not atomic against other CPUs -- CPU preemption needs to be off */
144#define x86_test_and_clear_bit_percpu(bit, var) \
145({ \
146 int old__; \
147 asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \
148 : "=r" (old__), "+m" (per_cpu__##var) \
149 : "dIr" (bit)); \
150 old__; \
151})
152
153#include <asm-generic/percpu.h>
154
155/* We can use this directly for local CPU (faster). */
156DECLARE_PER_CPU(unsigned long, this_cpu_off);
157
170#endif /* !__ASSEMBLY__ */ 158#endif /* !__ASSEMBLY__ */
171#endif /* !CONFIG_X86_64 */
172 159
173#ifdef CONFIG_SMP 160#ifdef CONFIG_SMP
174 161
@@ -195,9 +182,9 @@ do { \
195#define early_per_cpu_ptr(_name) (_name##_early_ptr) 182#define early_per_cpu_ptr(_name) (_name##_early_ptr)
196#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) 183#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
197#define early_per_cpu(_name, _cpu) \ 184#define early_per_cpu(_name, _cpu) \
198 (early_per_cpu_ptr(_name) ? \ 185 *(early_per_cpu_ptr(_name) ? \
199 early_per_cpu_ptr(_name)[_cpu] : \ 186 &early_per_cpu_ptr(_name)[_cpu] : \
200 per_cpu(_name, _cpu)) 187 &per_cpu(_name, _cpu))
201 188
202#else /* !CONFIG_SMP */ 189#else /* !CONFIG_SMP */
203#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ 190#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-19 10:15:24 -0400