diff options
-rw-r--r-- | include/asm-x86_64/local.h | 196 | ||||
-rw-r--r-- | include/asm-x86_64/system.h | 36 |
2 files changed, 201 insertions, 31 deletions
diff --git a/include/asm-x86_64/local.h b/include/asm-x86_64/local.h index e769e6200225..e87492bb0693 100644 --- a/include/asm-x86_64/local.h +++ b/include/asm-x86_64/local.h | |||
@@ -2,49 +2,183 @@ | |||
2 | #define _ARCH_X8664_LOCAL_H | 2 | #define _ARCH_X8664_LOCAL_H |
3 | 3 | ||
4 | #include <linux/percpu.h> | 4 | #include <linux/percpu.h> |
5 | #include <asm/atomic.h> | ||
5 | 6 | ||
6 | typedef struct | 7 | typedef struct |
7 | { | 8 | { |
8 | volatile long counter; | 9 | atomic_long_t a; |
9 | } local_t; | 10 | } local_t; |
10 | 11 | ||
11 | #define LOCAL_INIT(i) { (i) } | 12 | #define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) } |
12 | 13 | ||
13 | #define local_read(v) ((v)->counter) | 14 | #define local_read(l) atomic_long_read(&(l)->a) |
14 | #define local_set(v,i) (((v)->counter) = (i)) | 15 | #define local_set(l,i) atomic_long_set(&(l)->a, (i)) |
15 | 16 | ||
16 | static inline void local_inc(local_t *v) | 17 | static inline void local_inc(local_t *l) |
17 | { | 18 | { |
18 | __asm__ __volatile__( | 19 | __asm__ __volatile__( |
19 | "incq %0" | 20 | "incq %0" |
20 | :"=m" (v->counter) | 21 | :"=m" (l->a.counter) |
21 | :"m" (v->counter)); | 22 | :"m" (l->a.counter)); |
22 | } | 23 | } |
23 | 24 | ||
24 | static inline void local_dec(local_t *v) | 25 | static inline void local_dec(local_t *l) |
25 | { | 26 | { |
26 | __asm__ __volatile__( | 27 | __asm__ __volatile__( |
27 | "decq %0" | 28 | "decq %0" |
28 | :"=m" (v->counter) | 29 | :"=m" (l->a.counter) |
29 | :"m" (v->counter)); | 30 | :"m" (l->a.counter)); |
30 | } | 31 | } |
31 | 32 | ||
32 | static inline void local_add(long i, local_t *v) | 33 | static inline void local_add(long i, local_t *l) |
33 | { | 34 | { |
34 | __asm__ __volatile__( | 35 | __asm__ __volatile__( |
35 | "addq %1,%0" | 36 | "addq %1,%0" |
36 | :"=m" (v->counter) | 37 | :"=m" (l->a.counter) |
37 | :"ir" (i), "m" (v->counter)); | 38 | :"ir" (i), "m" (l->a.counter)); |
38 | } | 39 | } |
39 | 40 | ||
40 | static inline void local_sub(long i, local_t *v) | 41 | static inline void local_sub(long i, local_t *l) |
41 | { | 42 | { |
42 | __asm__ __volatile__( | 43 | __asm__ __volatile__( |
43 | "subq %1,%0" | 44 | "subq %1,%0" |
44 | :"=m" (v->counter) | 45 | :"=m" (l->a.counter) |
45 | :"ir" (i), "m" (v->counter)); | 46 | :"ir" (i), "m" (l->a.counter)); |
46 | } | 47 | } |
47 | 48 | ||
49 | /** | ||
50 | * local_sub_and_test - subtract value from variable and test result | ||
51 | * @i: integer value to subtract | ||
52 | * @l: pointer to type local_t | ||
53 | * | ||
54 | * Atomically subtracts @i from @l and returns | ||
55 | * true if the result is zero, or false for all | ||
56 | * other cases. | ||
57 | */ | ||
58 | static __inline__ int local_sub_and_test(long i, local_t *l) | ||
59 | { | ||
60 | unsigned char c; | ||
61 | |||
62 | __asm__ __volatile__( | ||
63 | "subq %2,%0; sete %1" | ||
64 | :"=m" (l->a.counter), "=qm" (c) | ||
65 | :"ir" (i), "m" (l->a.counter) : "memory"); | ||
66 | return c; | ||
67 | } | ||
68 | |||
69 | /** | ||
70 | * local_dec_and_test - decrement and test | ||
71 | * @l: pointer to type local_t | ||
72 | * | ||
73 | * Atomically decrements @l by 1 and | ||
74 | * returns true if the result is 0, or false for all other | ||
75 | * cases. | ||
76 | */ | ||
77 | static __inline__ int local_dec_and_test(local_t *l) | ||
78 | { | ||
79 | unsigned char c; | ||
80 | |||
81 | __asm__ __volatile__( | ||
82 | "decq %0; sete %1" | ||
83 | :"=m" (l->a.counter), "=qm" (c) | ||
84 | :"m" (l->a.counter) : "memory"); | ||
85 | return c != 0; | ||
86 | } | ||
87 | |||
88 | /** | ||
89 | * local_inc_and_test - increment and test | ||
90 | * @l: pointer to type local_t | ||
91 | * | ||
92 | * Atomically increments @l by 1 | ||
93 | * and returns true if the result is zero, or false for all | ||
94 | * other cases. | ||
95 | */ | ||
96 | static __inline__ int local_inc_and_test(local_t *l) | ||
97 | { | ||
98 | unsigned char c; | ||
99 | |||
100 | __asm__ __volatile__( | ||
101 | "incq %0; sete %1" | ||
102 | :"=m" (l->a.counter), "=qm" (c) | ||
103 | :"m" (l->a.counter) : "memory"); | ||
104 | return c != 0; | ||
105 | } | ||
106 | |||
107 | /** | ||
108 | * local_add_negative - add and test if negative | ||
109 | * @i: integer value to add | ||
110 | * @l: pointer to type local_t | ||
111 | * | ||
112 | * Atomically adds @i to @l and returns true | ||
113 | * if the result is negative, or false when | ||
114 | * result is greater than or equal to zero. | ||
115 | */ | ||
116 | static __inline__ int local_add_negative(long i, local_t *l) | ||
117 | { | ||
118 | unsigned char c; | ||
119 | |||
120 | __asm__ __volatile__( | ||
121 | "addq %2,%0; sets %1" | ||
122 | :"=m" (l->a.counter), "=qm" (c) | ||
123 | :"ir" (i), "m" (l->a.counter) : "memory"); | ||
124 | return c; | ||
125 | } | ||
126 | |||
127 | /** | ||
128 | * local_add_return - add and return | ||
129 | * @i: integer value to add | ||
130 | * @l: pointer to type local_t | ||
131 | * | ||
132 | * Atomically adds @i to @l and returns @i + @l | ||
133 | */ | ||
134 | static __inline__ long local_add_return(long i, local_t *l) | ||
135 | { | ||
136 | long __i = i; | ||
137 | __asm__ __volatile__( | ||
138 | "xaddq %0, %1;" | ||
139 | :"+r" (i), "+m" (l->a.counter) | ||
140 | : : "memory"); | ||
141 | return i + __i; | ||
142 | } | ||
143 | |||
144 | static __inline__ long local_sub_return(long i, local_t *l) | ||
145 | { | ||
146 | return local_add_return(-i,l); | ||
147 | } | ||
148 | |||
149 | #define local_inc_return(l) (local_add_return(1,l)) | ||
150 | #define local_dec_return(l) (local_sub_return(1,l)) | ||
151 | |||
152 | #define local_cmpxchg(l, o, n) \ | ||
153 | (cmpxchg_local(&((l)->a.counter), (o), (n))) | ||
154 | /* Always has a lock prefix */ | ||
155 | #define local_xchg(l, n) (xchg(&((l)->a.counter), (n))) | ||
156 | |||
157 | /** | ||
158 | * atomic_up_add_unless - add unless the number is a given value | ||
159 | * @l: pointer of type local_t | ||
160 | * @a: the amount to add to l... | ||
161 | * @u: ...unless l is equal to u. | ||
162 | * | ||
163 | * Atomically adds @a to @l, so long as it was not @u. | ||
164 | * Returns non-zero if @l was not @u, and zero otherwise. | ||
165 | */ | ||
166 | #define local_add_unless(l, a, u) \ | ||
167 | ({ \ | ||
168 | long c, old; \ | ||
169 | c = local_read(l); \ | ||
170 | for (;;) { \ | ||
171 | if (unlikely(c == (u))) \ | ||
172 | break; \ | ||
173 | old = local_cmpxchg((l), c, c + (a)); \ | ||
174 | if (likely(old == c)) \ | ||
175 | break; \ | ||
176 | c = old; \ | ||
177 | } \ | ||
178 | c != (u); \ | ||
179 | }) | ||
180 | #define local_inc_not_zero(l) local_add_unless((l), 1, 0) | ||
181 | |||
48 | /* On x86-64 these are better than the atomic variants on SMP kernels | 182 | /* On x86-64 these are better than the atomic variants on SMP kernels |
49 | because they dont use a lock prefix. */ | 183 | because they dont use a lock prefix. */ |
50 | #define __local_inc(l) local_inc(l) | 184 | #define __local_inc(l) local_inc(l) |
@@ -62,27 +196,27 @@ static inline void local_sub(long i, local_t *v) | |||
62 | 196 | ||
63 | /* Need to disable preemption for the cpu local counters otherwise we could | 197 | /* Need to disable preemption for the cpu local counters otherwise we could |
64 | still access a variable of a previous CPU in a non atomic way. */ | 198 | still access a variable of a previous CPU in a non atomic way. */ |
65 | #define cpu_local_wrap_v(v) \ | 199 | #define cpu_local_wrap_v(l) \ |
66 | ({ local_t res__; \ | 200 | ({ local_t res__; \ |
67 | preempt_disable(); \ | 201 | preempt_disable(); \ |
68 | res__ = (v); \ | 202 | res__ = (l); \ |
69 | preempt_enable(); \ | 203 | preempt_enable(); \ |
70 | res__; }) | 204 | res__; }) |
71 | #define cpu_local_wrap(v) \ | 205 | #define cpu_local_wrap(l) \ |
72 | ({ preempt_disable(); \ | 206 | ({ preempt_disable(); \ |
73 | v; \ | 207 | l; \ |
74 | preempt_enable(); }) \ | 208 | preempt_enable(); }) \ |
75 | 209 | ||
76 | #define cpu_local_read(v) cpu_local_wrap_v(local_read(&__get_cpu_var(v))) | 210 | #define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var(l))) |
77 | #define cpu_local_set(v, i) cpu_local_wrap(local_set(&__get_cpu_var(v), (i))) | 211 | #define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i))) |
78 | #define cpu_local_inc(v) cpu_local_wrap(local_inc(&__get_cpu_var(v))) | 212 | #define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var(l))) |
79 | #define cpu_local_dec(v) cpu_local_wrap(local_dec(&__get_cpu_var(v))) | 213 | #define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var(l))) |
80 | #define cpu_local_add(i, v) cpu_local_wrap(local_add((i), &__get_cpu_var(v))) | 214 | #define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l))) |
81 | #define cpu_local_sub(i, v) cpu_local_wrap(local_sub((i), &__get_cpu_var(v))) | 215 | #define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l))) |
82 | 216 | ||
83 | #define __cpu_local_inc(v) cpu_local_inc(v) | 217 | #define __cpu_local_inc(l) cpu_local_inc(l) |
84 | #define __cpu_local_dec(v) cpu_local_dec(v) | 218 | #define __cpu_local_dec(l) cpu_local_dec(l) |
85 | #define __cpu_local_add(i, v) cpu_local_add((i), (v)) | 219 | #define __cpu_local_add(i, l) cpu_local_add((i), (l)) |
86 | #define __cpu_local_sub(i, v) cpu_local_sub((i), (v)) | 220 | #define __cpu_local_sub(i, l) cpu_local_sub((i), (l)) |
87 | 221 | ||
88 | #endif /* _ARCH_I386_LOCAL_H */ | 222 | #endif /* _ARCH_X8664_LOCAL_H */ |
diff --git a/include/asm-x86_64/system.h b/include/asm-x86_64/system.h index 213b7fe5d998..1f1c0bf4a5df 100644 --- a/include/asm-x86_64/system.h +++ b/include/asm-x86_64/system.h | |||
@@ -214,9 +214,45 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, | |||
214 | return old; | 214 | return old; |
215 | } | 215 | } |
216 | 216 | ||
217 | static inline unsigned long __cmpxchg_local(volatile void *ptr, | ||
218 | unsigned long old, unsigned long new, int size) | ||
219 | { | ||
220 | unsigned long prev; | ||
221 | switch (size) { | ||
222 | case 1: | ||
223 | __asm__ __volatile__("cmpxchgb %b1,%2" | ||
224 | : "=a"(prev) | ||
225 | : "q"(new), "m"(*__xg(ptr)), "0"(old) | ||
226 | : "memory"); | ||
227 | return prev; | ||
228 | case 2: | ||
229 | __asm__ __volatile__("cmpxchgw %w1,%2" | ||
230 | : "=a"(prev) | ||
231 | : "r"(new), "m"(*__xg(ptr)), "0"(old) | ||
232 | : "memory"); | ||
233 | return prev; | ||
234 | case 4: | ||
235 | __asm__ __volatile__("cmpxchgl %k1,%2" | ||
236 | : "=a"(prev) | ||
237 | : "r"(new), "m"(*__xg(ptr)), "0"(old) | ||
238 | : "memory"); | ||
239 | return prev; | ||
240 | case 8: | ||
241 | __asm__ __volatile__("cmpxchgq %1,%2" | ||
242 | : "=a"(prev) | ||
243 | : "r"(new), "m"(*__xg(ptr)), "0"(old) | ||
244 | : "memory"); | ||
245 | return prev; | ||
246 | } | ||
247 | return old; | ||
248 | } | ||
249 | |||
217 | #define cmpxchg(ptr,o,n)\ | 250 | #define cmpxchg(ptr,o,n)\ |
218 | ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ | 251 | ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ |
219 | (unsigned long)(n),sizeof(*(ptr)))) | 252 | (unsigned long)(n),sizeof(*(ptr)))) |
253 | #define cmpxchg_local(ptr,o,n)\ | ||
254 | ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ | ||
255 | (unsigned long)(n),sizeof(*(ptr)))) | ||
220 | 256 | ||
221 | #ifdef CONFIG_SMP | 257 | #ifdef CONFIG_SMP |
222 | #define smp_mb() mb() | 258 | #define smp_mb() mb() |