aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/asm-i386/atomic.h1
-rw-r--r--include/asm-i386/cmpxchg.h293
-rw-r--r--include/asm-i386/system.h289
-rw-r--r--include/asm-um/cmpxchg.h6
-rw-r--r--include/asm-x86_64/atomic.h2
-rw-r--r--include/asm-x86_64/cmpxchg.h134
-rw-r--r--include/asm-x86_64/system.h130
7 files changed, 437 insertions, 418 deletions
diff --git a/include/asm-i386/atomic.h b/include/asm-i386/atomic.h
index ff90c6e3fcb4..b9fd03cef51e 100644
--- a/include/asm-i386/atomic.h
+++ b/include/asm-i386/atomic.h
@@ -3,6 +3,7 @@
3 3
4#include <linux/compiler.h> 4#include <linux/compiler.h>
5#include <asm/processor.h> 5#include <asm/processor.h>
6#include <asm/cmpxchg.h>
6 7
7/* 8/*
8 * Atomic operations that C can't guarantee us. Useful for 9 * Atomic operations that C can't guarantee us. Useful for
diff --git a/include/asm-i386/cmpxchg.h b/include/asm-i386/cmpxchg.h
new file mode 100644
index 000000000000..7adcef0cd53b
--- /dev/null
+++ b/include/asm-i386/cmpxchg.h
@@ -0,0 +1,293 @@
1#ifndef __ASM_CMPXCHG_H
2#define __ASM_CMPXCHG_H
3
4#include <linux/bitops.h> /* for LOCK_PREFIX */
5
6#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr))))
7
8struct __xchg_dummy { unsigned long a[100]; };
9#define __xg(x) ((struct __xchg_dummy *)(x))
10
11
12#ifdef CONFIG_X86_CMPXCHG64
13
14/*
15 * The semantics of XCHGCMP8B are a bit strange, this is why
16 * there is a loop and the loading of %%eax and %%edx has to
17 * be inside. This inlines well in most cases, the cached
18 * cost is around ~38 cycles. (in the future we might want
19 * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
20 * might have an implicit FPU-save as a cost, so it's not
21 * clear which path to go.)
22 *
23 * cmpxchg8b must be used with the lock prefix here to allow
24 * the instruction to be executed atomically, see page 3-102
25 * of the instruction set reference 24319102.pdf. We need
26 * the reader side to see the coherent 64bit value.
27 */
28static inline void __set_64bit (unsigned long long * ptr,
29 unsigned int low, unsigned int high)
30{
31 __asm__ __volatile__ (
32 "\n1:\t"
33 "movl (%0), %%eax\n\t"
34 "movl 4(%0), %%edx\n\t"
35 "lock cmpxchg8b (%0)\n\t"
36 "jnz 1b"
37 : /* no outputs */
38 : "D"(ptr),
39 "b"(low),
40 "c"(high)
41 : "ax","dx","memory");
42}
43
44static inline void __set_64bit_constant (unsigned long long *ptr,
45 unsigned long long value)
46{
47 __set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL));
48}
49#define ll_low(x) *(((unsigned int*)&(x))+0)
50#define ll_high(x) *(((unsigned int*)&(x))+1)
51
52static inline void __set_64bit_var (unsigned long long *ptr,
53 unsigned long long value)
54{
55 __set_64bit(ptr,ll_low(value), ll_high(value));
56}
57
58#define set_64bit(ptr,value) \
59(__builtin_constant_p(value) ? \
60 __set_64bit_constant(ptr, value) : \
61 __set_64bit_var(ptr, value) )
62
63#define _set_64bit(ptr,value) \
64(__builtin_constant_p(value) ? \
65 __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \
66 __set_64bit(ptr, ll_low(value), ll_high(value)) )
67
68#endif
69
70/*
71 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
72 * Note 2: xchg has side effect, so that attribute volatile is necessary,
73 * but generally the primitive is invalid, *ptr is output argument. --ANK
74 */
75static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
76{
77 switch (size) {
78 case 1:
79 __asm__ __volatile__("xchgb %b0,%1"
80 :"=q" (x)
81 :"m" (*__xg(ptr)), "0" (x)
82 :"memory");
83 break;
84 case 2:
85 __asm__ __volatile__("xchgw %w0,%1"
86 :"=r" (x)
87 :"m" (*__xg(ptr)), "0" (x)
88 :"memory");
89 break;
90 case 4:
91 __asm__ __volatile__("xchgl %0,%1"
92 :"=r" (x)
93 :"m" (*__xg(ptr)), "0" (x)
94 :"memory");
95 break;
96 }
97 return x;
98}
99
100/*
101 * Atomic compare and exchange. Compare OLD with MEM, if identical,
102 * store NEW in MEM. Return the initial value in MEM. Success is
103 * indicated by comparing RETURN with OLD.
104 */
105
106#ifdef CONFIG_X86_CMPXCHG
107#define __HAVE_ARCH_CMPXCHG 1
108#define cmpxchg(ptr,o,n)\
109 ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
110 (unsigned long)(n),sizeof(*(ptr))))
111#define sync_cmpxchg(ptr,o,n)\
112 ((__typeof__(*(ptr)))__sync_cmpxchg((ptr),(unsigned long)(o),\
113 (unsigned long)(n),sizeof(*(ptr))))
114#define cmpxchg_local(ptr,o,n)\
115 ((__typeof__(*(ptr)))__cmpxchg_local((ptr),(unsigned long)(o),\
116 (unsigned long)(n),sizeof(*(ptr))))
117#endif
118
119static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
120 unsigned long new, int size)
121{
122 unsigned long prev;
123 switch (size) {
124 case 1:
125 __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
126 : "=a"(prev)
127 : "q"(new), "m"(*__xg(ptr)), "0"(old)
128 : "memory");
129 return prev;
130 case 2:
131 __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
132 : "=a"(prev)
133 : "r"(new), "m"(*__xg(ptr)), "0"(old)
134 : "memory");
135 return prev;
136 case 4:
137 __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
138 : "=a"(prev)
139 : "r"(new), "m"(*__xg(ptr)), "0"(old)
140 : "memory");
141 return prev;
142 }
143 return old;
144}
145
146/*
147 * Always use locked operations when touching memory shared with a
148 * hypervisor, since the system may be SMP even if the guest kernel
149 * isn't.
150 */
151static inline unsigned long __sync_cmpxchg(volatile void *ptr,
152 unsigned long old,
153 unsigned long new, int size)
154{
155 unsigned long prev;
156 switch (size) {
157 case 1:
158 __asm__ __volatile__("lock; cmpxchgb %b1,%2"
159 : "=a"(prev)
160 : "q"(new), "m"(*__xg(ptr)), "0"(old)
161 : "memory");
162 return prev;
163 case 2:
164 __asm__ __volatile__("lock; cmpxchgw %w1,%2"
165 : "=a"(prev)
166 : "r"(new), "m"(*__xg(ptr)), "0"(old)
167 : "memory");
168 return prev;
169 case 4:
170 __asm__ __volatile__("lock; cmpxchgl %1,%2"
171 : "=a"(prev)
172 : "r"(new), "m"(*__xg(ptr)), "0"(old)
173 : "memory");
174 return prev;
175 }
176 return old;
177}
178
179static inline unsigned long __cmpxchg_local(volatile void *ptr,
180 unsigned long old, unsigned long new, int size)
181{
182 unsigned long prev;
183 switch (size) {
184 case 1:
185 __asm__ __volatile__("cmpxchgb %b1,%2"
186 : "=a"(prev)
187 : "q"(new), "m"(*__xg(ptr)), "0"(old)
188 : "memory");
189 return prev;
190 case 2:
191 __asm__ __volatile__("cmpxchgw %w1,%2"
192 : "=a"(prev)
193 : "r"(new), "m"(*__xg(ptr)), "0"(old)
194 : "memory");
195 return prev;
196 case 4:
197 __asm__ __volatile__("cmpxchgl %1,%2"
198 : "=a"(prev)
199 : "r"(new), "m"(*__xg(ptr)), "0"(old)
200 : "memory");
201 return prev;
202 }
203 return old;
204}
205
206#ifndef CONFIG_X86_CMPXCHG
207/*
208 * Building a kernel capable running on 80386. It may be necessary to
209 * simulate the cmpxchg on the 80386 CPU. For that purpose we define
210 * a function for each of the sizes we support.
211 */
212
213extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
214extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
215extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
216
217static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
218 unsigned long new, int size)
219{
220 switch (size) {
221 case 1:
222 return cmpxchg_386_u8(ptr, old, new);
223 case 2:
224 return cmpxchg_386_u16(ptr, old, new);
225 case 4:
226 return cmpxchg_386_u32(ptr, old, new);
227 }
228 return old;
229}
230
231#define cmpxchg(ptr,o,n) \
232({ \
233 __typeof__(*(ptr)) __ret; \
234 if (likely(boot_cpu_data.x86 > 3)) \
235 __ret = __cmpxchg((ptr), (unsigned long)(o), \
236 (unsigned long)(n), sizeof(*(ptr))); \
237 else \
238 __ret = cmpxchg_386((ptr), (unsigned long)(o), \
239 (unsigned long)(n), sizeof(*(ptr))); \
240 __ret; \
241})
242#define cmpxchg_local(ptr,o,n) \
243({ \
244 __typeof__(*(ptr)) __ret; \
245 if (likely(boot_cpu_data.x86 > 3)) \
246 __ret = __cmpxchg_local((ptr), (unsigned long)(o), \
247 (unsigned long)(n), sizeof(*(ptr))); \
248 else \
249 __ret = cmpxchg_386((ptr), (unsigned long)(o), \
250 (unsigned long)(n), sizeof(*(ptr))); \
251 __ret; \
252})
253#endif
254
255#ifdef CONFIG_X86_CMPXCHG64
256
257static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long long old,
258 unsigned long long new)
259{
260 unsigned long long prev;
261 __asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
262 : "=A"(prev)
263 : "b"((unsigned long)new),
264 "c"((unsigned long)(new >> 32)),
265 "m"(*__xg(ptr)),
266 "0"(old)
267 : "memory");
268 return prev;
269}
270
271static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
272 unsigned long long old, unsigned long long new)
273{
274 unsigned long long prev;
275 __asm__ __volatile__("cmpxchg8b %3"
276 : "=A"(prev)
277 : "b"((unsigned long)new),
278 "c"((unsigned long)(new >> 32)),
279 "m"(*__xg(ptr)),
280 "0"(old)
281 : "memory");
282 return prev;
283}
284
285#define cmpxchg64(ptr,o,n)\
286 ((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\
287 (unsigned long long)(n)))
288#define cmpxchg64_local(ptr,o,n)\
289 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr),(unsigned long long)(o),\
290 (unsigned long long)(n)))
291#endif
292
293#endif
diff --git a/include/asm-i386/system.h b/include/asm-i386/system.h
index a4ed087ac0ae..94ed3686a5f3 100644
--- a/include/asm-i386/system.h
+++ b/include/asm-i386/system.h
@@ -4,7 +4,7 @@
4#include <linux/kernel.h> 4#include <linux/kernel.h>
5#include <asm/segment.h> 5#include <asm/segment.h>
6#include <asm/cpufeature.h> 6#include <asm/cpufeature.h>
7#include <linux/bitops.h> /* for LOCK_PREFIX */ 7#include <asm/cmpxchg.h>
8 8
9#ifdef __KERNEL__ 9#ifdef __KERNEL__
10 10
@@ -195,293 +195,6 @@ static inline unsigned long get_limit(unsigned long segment)
195 195
196#define nop() __asm__ __volatile__ ("nop") 196#define nop() __asm__ __volatile__ ("nop")
197 197
198#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr))))
199
200struct __xchg_dummy { unsigned long a[100]; };
201#define __xg(x) ((struct __xchg_dummy *)(x))
202
203
204#ifdef CONFIG_X86_CMPXCHG64
205
206/*
207 * The semantics of XCHGCMP8B are a bit strange, this is why
208 * there is a loop and the loading of %%eax and %%edx has to
209 * be inside. This inlines well in most cases, the cached
210 * cost is around ~38 cycles. (in the future we might want
211 * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
212 * might have an implicit FPU-save as a cost, so it's not
213 * clear which path to go.)
214 *
215 * cmpxchg8b must be used with the lock prefix here to allow
216 * the instruction to be executed atomically, see page 3-102
217 * of the instruction set reference 24319102.pdf. We need
218 * the reader side to see the coherent 64bit value.
219 */
220static inline void __set_64bit (unsigned long long * ptr,
221 unsigned int low, unsigned int high)
222{
223 __asm__ __volatile__ (
224 "\n1:\t"
225 "movl (%0), %%eax\n\t"
226 "movl 4(%0), %%edx\n\t"
227 "lock cmpxchg8b (%0)\n\t"
228 "jnz 1b"
229 : /* no outputs */
230 : "D"(ptr),
231 "b"(low),
232 "c"(high)
233 : "ax","dx","memory");
234}
235
236static inline void __set_64bit_constant (unsigned long long *ptr,
237 unsigned long long value)
238{
239 __set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL));
240}
241#define ll_low(x) *(((unsigned int*)&(x))+0)
242#define ll_high(x) *(((unsigned int*)&(x))+1)
243
244static inline void __set_64bit_var (unsigned long long *ptr,
245 unsigned long long value)
246{
247 __set_64bit(ptr,ll_low(value), ll_high(value));
248}
249
250#define set_64bit(ptr,value) \
251(__builtin_constant_p(value) ? \
252 __set_64bit_constant(ptr, value) : \
253 __set_64bit_var(ptr, value) )
254
255#define _set_64bit(ptr,value) \
256(__builtin_constant_p(value) ? \
257 __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \
258 __set_64bit(ptr, ll_low(value), ll_high(value)) )
259
260#endif
261
262/*
263 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
264 * Note 2: xchg has side effect, so that attribute volatile is necessary,
265 * but generally the primitive is invalid, *ptr is output argument. --ANK
266 */
267static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
268{
269 switch (size) {
270 case 1:
271 __asm__ __volatile__("xchgb %b0,%1"
272 :"=q" (x)
273 :"m" (*__xg(ptr)), "0" (x)
274 :"memory");
275 break;
276 case 2:
277 __asm__ __volatile__("xchgw %w0,%1"
278 :"=r" (x)
279 :"m" (*__xg(ptr)), "0" (x)
280 :"memory");
281 break;
282 case 4:
283 __asm__ __volatile__("xchgl %0,%1"
284 :"=r" (x)
285 :"m" (*__xg(ptr)), "0" (x)
286 :"memory");
287 break;
288 }
289 return x;
290}
291
292/*
293 * Atomic compare and exchange. Compare OLD with MEM, if identical,
294 * store NEW in MEM. Return the initial value in MEM. Success is
295 * indicated by comparing RETURN with OLD.
296 */
297
298#ifdef CONFIG_X86_CMPXCHG
299#define __HAVE_ARCH_CMPXCHG 1
300#define cmpxchg(ptr,o,n)\
301 ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
302 (unsigned long)(n),sizeof(*(ptr))))
303#define sync_cmpxchg(ptr,o,n)\
304 ((__typeof__(*(ptr)))__sync_cmpxchg((ptr),(unsigned long)(o),\
305 (unsigned long)(n),sizeof(*(ptr))))
306#define cmpxchg_local(ptr,o,n)\
307 ((__typeof__(*(ptr)))__cmpxchg_local((ptr),(unsigned long)(o),\
308 (unsigned long)(n),sizeof(*(ptr))))
309#endif
310
311static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
312 unsigned long new, int size)
313{
314 unsigned long prev;
315 switch (size) {
316 case 1:
317 __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
318 : "=a"(prev)
319 : "q"(new), "m"(*__xg(ptr)), "0"(old)
320 : "memory");
321 return prev;
322 case 2:
323 __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
324 : "=a"(prev)
325 : "r"(new), "m"(*__xg(ptr)), "0"(old)
326 : "memory");
327 return prev;
328 case 4:
329 __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
330 : "=a"(prev)
331 : "r"(new), "m"(*__xg(ptr)), "0"(old)
332 : "memory");
333 return prev;
334 }
335 return old;
336}
337
338/*
339 * Always use locked operations when touching memory shared with a
340 * hypervisor, since the system may be SMP even if the guest kernel
341 * isn't.
342 */
343static inline unsigned long __sync_cmpxchg(volatile void *ptr,
344 unsigned long old,
345 unsigned long new, int size)
346{
347 unsigned long prev;
348 switch (size) {
349 case 1:
350 __asm__ __volatile__("lock; cmpxchgb %b1,%2"
351 : "=a"(prev)
352 : "q"(new), "m"(*__xg(ptr)), "0"(old)
353 : "memory");
354 return prev;
355 case 2:
356 __asm__ __volatile__("lock; cmpxchgw %w1,%2"
357 : "=a"(prev)
358 : "r"(new), "m"(*__xg(ptr)), "0"(old)
359 : "memory");
360 return prev;
361 case 4:
362 __asm__ __volatile__("lock; cmpxchgl %1,%2"
363 : "=a"(prev)
364 : "r"(new), "m"(*__xg(ptr)), "0"(old)
365 : "memory");
366 return prev;
367 }
368 return old;
369}
370
371static inline unsigned long __cmpxchg_local(volatile void *ptr,
372 unsigned long old, unsigned long new, int size)
373{
374 unsigned long prev;
375 switch (size) {
376 case 1:
377 __asm__ __volatile__("cmpxchgb %b1,%2"
378 : "=a"(prev)
379 : "q"(new), "m"(*__xg(ptr)), "0"(old)
380 : "memory");
381 return prev;
382 case 2:
383 __asm__ __volatile__("cmpxchgw %w1,%2"
384 : "=a"(prev)
385 : "r"(new), "m"(*__xg(ptr)), "0"(old)
386 : "memory");
387 return prev;
388 case 4:
389 __asm__ __volatile__("cmpxchgl %1,%2"
390 : "=a"(prev)
391 : "r"(new), "m"(*__xg(ptr)), "0"(old)
392 : "memory");
393 return prev;
394 }
395 return old;
396}
397
398#ifndef CONFIG_X86_CMPXCHG
399/*
400 * Building a kernel capable running on 80386. It may be necessary to
401 * simulate the cmpxchg on the 80386 CPU. For that purpose we define
402 * a function for each of the sizes we support.
403 */
404
405extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
406extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
407extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
408
409static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
410 unsigned long new, int size)
411{
412 switch (size) {
413 case 1:
414 return cmpxchg_386_u8(ptr, old, new);
415 case 2:
416 return cmpxchg_386_u16(ptr, old, new);
417 case 4:
418 return cmpxchg_386_u32(ptr, old, new);
419 }
420 return old;
421}
422
423#define cmpxchg(ptr,o,n) \
424({ \
425 __typeof__(*(ptr)) __ret; \
426 if (likely(boot_cpu_data.x86 > 3)) \
427 __ret = __cmpxchg((ptr), (unsigned long)(o), \
428 (unsigned long)(n), sizeof(*(ptr))); \
429 else \
430 __ret = cmpxchg_386((ptr), (unsigned long)(o), \
431 (unsigned long)(n), sizeof(*(ptr))); \
432 __ret; \
433})
434#define cmpxchg_local(ptr,o,n) \
435({ \
436 __typeof__(*(ptr)) __ret; \
437 if (likely(boot_cpu_data.x86 > 3)) \
438 __ret = __cmpxchg_local((ptr), (unsigned long)(o), \
439 (unsigned long)(n), sizeof(*(ptr))); \
440 else \
441 __ret = cmpxchg_386((ptr), (unsigned long)(o), \
442 (unsigned long)(n), sizeof(*(ptr))); \
443 __ret; \
444})
445#endif
446
447#ifdef CONFIG_X86_CMPXCHG64
448
449static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long long old,
450 unsigned long long new)
451{
452 unsigned long long prev;
453 __asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
454 : "=A"(prev)
455 : "b"((unsigned long)new),
456 "c"((unsigned long)(new >> 32)),
457 "m"(*__xg(ptr)),
458 "0"(old)
459 : "memory");
460 return prev;
461}
462
463static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
464 unsigned long long old, unsigned long long new)
465{
466 unsigned long long prev;
467 __asm__ __volatile__("cmpxchg8b %3"
468 : "=A"(prev)
469 : "b"((unsigned long)new),
470 "c"((unsigned long)(new >> 32)),
471 "m"(*__xg(ptr)),
472 "0"(old)
473 : "memory");
474 return prev;
475}
476
477#define cmpxchg64(ptr,o,n)\
478 ((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\
479 (unsigned long long)(n)))
480#define cmpxchg64_local(ptr,o,n)\
481 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr),(unsigned long long)(o),\
482 (unsigned long long)(n)))
483#endif
484
485/* 198/*
486 * Force strict CPU ordering. 199 * Force strict CPU ordering.
487 * And yes, this is required on UP too when we're talking 200 * And yes, this is required on UP too when we're talking
diff --git a/include/asm-um/cmpxchg.h b/include/asm-um/cmpxchg.h
new file mode 100644
index 000000000000..529376a99885
--- /dev/null
+++ b/include/asm-um/cmpxchg.h
@@ -0,0 +1,6 @@
1#ifndef __UM_CMPXCHG_H
2#define __UM_CMPXCHG_H
3
4#include "asm/arch/cmpxchg.h"
5
6#endif
diff --git a/include/asm-x86_64/atomic.h b/include/asm-x86_64/atomic.h
index 19e0c607b568..f2e64634fa48 100644
--- a/include/asm-x86_64/atomic.h
+++ b/include/asm-x86_64/atomic.h
@@ -2,7 +2,7 @@
2#define __ARCH_X86_64_ATOMIC__ 2#define __ARCH_X86_64_ATOMIC__
3 3
4#include <asm/alternative.h> 4#include <asm/alternative.h>
5#include <asm/system.h> 5#include <asm/cmpxchg.h>
6 6
7/* atomic_t should be 32 bit signed type */ 7/* atomic_t should be 32 bit signed type */
8 8
diff --git a/include/asm-x86_64/cmpxchg.h b/include/asm-x86_64/cmpxchg.h
new file mode 100644
index 000000000000..09a6b6b6b74d
--- /dev/null
+++ b/include/asm-x86_64/cmpxchg.h
@@ -0,0 +1,134 @@
1#ifndef __ASM_CMPXCHG_H
2#define __ASM_CMPXCHG_H
3
4#include <asm/alternative.h> /* Provides LOCK_PREFIX */
5
6#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr))))
7
8#define __xg(x) ((volatile long *)(x))
9
10static inline void set_64bit(volatile unsigned long *ptr, unsigned long val)
11{
12 *ptr = val;
13}
14
15#define _set_64bit set_64bit
16
17/*
18 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
19 * Note 2: xchg has side effect, so that attribute volatile is necessary,
20 * but generally the primitive is invalid, *ptr is output argument. --ANK
21 */
22static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
23{
24 switch (size) {
25 case 1:
26 __asm__ __volatile__("xchgb %b0,%1"
27 :"=q" (x)
28 :"m" (*__xg(ptr)), "0" (x)
29 :"memory");
30 break;
31 case 2:
32 __asm__ __volatile__("xchgw %w0,%1"
33 :"=r" (x)
34 :"m" (*__xg(ptr)), "0" (x)
35 :"memory");
36 break;
37 case 4:
38 __asm__ __volatile__("xchgl %k0,%1"
39 :"=r" (x)
40 :"m" (*__xg(ptr)), "0" (x)
41 :"memory");
42 break;
43 case 8:
44 __asm__ __volatile__("xchgq %0,%1"
45 :"=r" (x)
46 :"m" (*__xg(ptr)), "0" (x)
47 :"memory");
48 break;
49 }
50 return x;
51}
52
53/*
54 * Atomic compare and exchange. Compare OLD with MEM, if identical,
55 * store NEW in MEM. Return the initial value in MEM. Success is
56 * indicated by comparing RETURN with OLD.
57 */
58
59#define __HAVE_ARCH_CMPXCHG 1
60
61static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
62 unsigned long new, int size)
63{
64 unsigned long prev;
65 switch (size) {
66 case 1:
67 __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
68 : "=a"(prev)
69 : "q"(new), "m"(*__xg(ptr)), "0"(old)
70 : "memory");
71 return prev;
72 case 2:
73 __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
74 : "=a"(prev)
75 : "r"(new), "m"(*__xg(ptr)), "0"(old)
76 : "memory");
77 return prev;
78 case 4:
79 __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %k1,%2"
80 : "=a"(prev)
81 : "r"(new), "m"(*__xg(ptr)), "0"(old)
82 : "memory");
83 return prev;
84 case 8:
85 __asm__ __volatile__(LOCK_PREFIX "cmpxchgq %1,%2"
86 : "=a"(prev)
87 : "r"(new), "m"(*__xg(ptr)), "0"(old)
88 : "memory");
89 return prev;
90 }
91 return old;
92}
93
94static inline unsigned long __cmpxchg_local(volatile void *ptr,
95 unsigned long old, unsigned long new, int size)
96{
97 unsigned long prev;
98 switch (size) {
99 case 1:
100 __asm__ __volatile__("cmpxchgb %b1,%2"
101 : "=a"(prev)
102 : "q"(new), "m"(*__xg(ptr)), "0"(old)
103 : "memory");
104 return prev;
105 case 2:
106 __asm__ __volatile__("cmpxchgw %w1,%2"
107 : "=a"(prev)
108 : "r"(new), "m"(*__xg(ptr)), "0"(old)
109 : "memory");
110 return prev;
111 case 4:
112 __asm__ __volatile__("cmpxchgl %k1,%2"
113 : "=a"(prev)
114 : "r"(new), "m"(*__xg(ptr)), "0"(old)
115 : "memory");
116 return prev;
117 case 8:
118 __asm__ __volatile__("cmpxchgq %1,%2"
119 : "=a"(prev)
120 : "r"(new), "m"(*__xg(ptr)), "0"(old)
121 : "memory");
122 return prev;
123 }
124 return old;
125}
126
127#define cmpxchg(ptr,o,n)\
128 ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
129 (unsigned long)(n),sizeof(*(ptr))))
130#define cmpxchg_local(ptr,o,n)\
131 ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
132 (unsigned long)(n),sizeof(*(ptr))))
133
134#endif
diff --git a/include/asm-x86_64/system.h b/include/asm-x86_64/system.h
index 5316f3cac230..b7b8021e8c43 100644
--- a/include/asm-x86_64/system.h
+++ b/include/asm-x86_64/system.h
@@ -3,7 +3,7 @@
3 3
4#include <linux/kernel.h> 4#include <linux/kernel.h>
5#include <asm/segment.h> 5#include <asm/segment.h>
6#include <asm/alternative.h> 6#include <asm/cmpxchg.h>
7 7
8#ifdef __KERNEL__ 8#ifdef __KERNEL__
9 9
@@ -124,134 +124,6 @@ static inline void sched_cacheflush(void)
124 124
125#define nop() __asm__ __volatile__ ("nop") 125#define nop() __asm__ __volatile__ ("nop")
126 126
127#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr))))
128
129#define __xg(x) ((volatile long *)(x))
130
131static inline void set_64bit(volatile unsigned long *ptr, unsigned long val)
132{
133 *ptr = val;
134}
135
136#define _set_64bit set_64bit
137
138/*
139 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
140 * Note 2: xchg has side effect, so that attribute volatile is necessary,
141 * but generally the primitive is invalid, *ptr is output argument. --ANK
142 */
143static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
144{
145 switch (size) {
146 case 1:
147 __asm__ __volatile__("xchgb %b0,%1"
148 :"=q" (x)
149 :"m" (*__xg(ptr)), "0" (x)
150 :"memory");
151 break;
152 case 2:
153 __asm__ __volatile__("xchgw %w0,%1"
154 :"=r" (x)
155 :"m" (*__xg(ptr)), "0" (x)
156 :"memory");
157 break;
158 case 4:
159 __asm__ __volatile__("xchgl %k0,%1"
160 :"=r" (x)
161 :"m" (*__xg(ptr)), "0" (x)
162 :"memory");
163 break;
164 case 8:
165 __asm__ __volatile__("xchgq %0,%1"
166 :"=r" (x)
167 :"m" (*__xg(ptr)), "0" (x)
168 :"memory");
169 break;
170 }
171 return x;
172}
173
174/*
175 * Atomic compare and exchange. Compare OLD with MEM, if identical,
176 * store NEW in MEM. Return the initial value in MEM. Success is
177 * indicated by comparing RETURN with OLD.
178 */
179
180#define __HAVE_ARCH_CMPXCHG 1
181
182static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
183 unsigned long new, int size)
184{
185 unsigned long prev;
186 switch (size) {
187 case 1:
188 __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
189 : "=a"(prev)
190 : "q"(new), "m"(*__xg(ptr)), "0"(old)
191 : "memory");
192 return prev;
193 case 2:
194 __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
195 : "=a"(prev)
196 : "r"(new), "m"(*__xg(ptr)), "0"(old)
197 : "memory");
198 return prev;
199 case 4:
200 __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %k1,%2"
201 : "=a"(prev)
202 : "r"(new), "m"(*__xg(ptr)), "0"(old)
203 : "memory");
204 return prev;
205 case 8:
206 __asm__ __volatile__(LOCK_PREFIX "cmpxchgq %1,%2"
207 : "=a"(prev)
208 : "r"(new), "m"(*__xg(ptr)), "0"(old)
209 : "memory");
210 return prev;
211 }
212 return old;
213}
214
215static inline unsigned long __cmpxchg_local(volatile void *ptr,
216 unsigned long old, unsigned long new, int size)
217{
218 unsigned long prev;
219 switch (size) {
220 case 1:
221 __asm__ __volatile__("cmpxchgb %b1,%2"
222 : "=a"(prev)
223 : "q"(new), "m"(*__xg(ptr)), "0"(old)
224 : "memory");
225 return prev;
226 case 2:
227 __asm__ __volatile__("cmpxchgw %w1,%2"
228 : "=a"(prev)
229 : "r"(new), "m"(*__xg(ptr)), "0"(old)
230 : "memory");
231 return prev;
232 case 4:
233 __asm__ __volatile__("cmpxchgl %k1,%2"
234 : "=a"(prev)
235 : "r"(new), "m"(*__xg(ptr)), "0"(old)
236 : "memory");
237 return prev;
238 case 8:
239 __asm__ __volatile__("cmpxchgq %1,%2"
240 : "=a"(prev)
241 : "r"(new), "m"(*__xg(ptr)), "0"(old)
242 : "memory");
243 return prev;
244 }
245 return old;
246}
247
248#define cmpxchg(ptr,o,n)\
249 ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
250 (unsigned long)(n),sizeof(*(ptr))))
251#define cmpxchg_local(ptr,o,n)\
252 ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
253 (unsigned long)(n),sizeof(*(ptr))))
254
255#ifdef CONFIG_SMP 127#ifdef CONFIG_SMP
256#define smp_mb() mb() 128#define smp_mb() mb()
257#define smp_rmb() rmb() 129#define smp_rmb() rmb()
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-11 14:17:00 -0500