aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/include/asm/cmpxchg_64.h
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/include/asm/cmpxchg_64.h')
-rw-r--r--arch/x86/include/asm/cmpxchg_64.h174
1 files changed, 44 insertions, 130 deletions
diff --git a/arch/x86/include/asm/cmpxchg_64.h b/arch/x86/include/asm/cmpxchg_64.h
index 423ae58aa020..285da02c38fa 100644
--- a/arch/x86/include/asm/cmpxchg_64.h
+++ b/arch/x86/include/asm/cmpxchg_64.h
@@ -1,154 +1,68 @@
1#ifndef _ASM_X86_CMPXCHG_64_H 1#ifndef _ASM_X86_CMPXCHG_64_H
2#define _ASM_X86_CMPXCHG_64_H 2#define _ASM_X86_CMPXCHG_64_H
3 3
4#include <asm/alternative.h> /* Provides LOCK_PREFIX */
5
6static inline void set_64bit(volatile u64 *ptr, u64 val) 4static inline void set_64bit(volatile u64 *ptr, u64 val)
7{ 5{
8 *ptr = val; 6 *ptr = val;
9} 7}
10 8
11extern void __xchg_wrong_size(void); 9#define __HAVE_ARCH_CMPXCHG 1
12extern void __cmpxchg_wrong_size(void);
13 10
14/* 11#define cmpxchg64(ptr, o, n) \
15 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
16 * Since this is generally used to protect other memory information, we
17 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
18 * information around.
19 */
20#define __xchg(x, ptr, size) \
21({ \ 12({ \
22 __typeof(*(ptr)) __x = (x); \ 13 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
23 switch (size) { \ 14 cmpxchg((ptr), (o), (n)); \
24 case 1: \
25 { \
26 volatile u8 *__ptr = (volatile u8 *)(ptr); \
27 asm volatile("xchgb %0,%1" \
28 : "=q" (__x), "+m" (*__ptr) \
29 : "0" (__x) \
30 : "memory"); \
31 break; \
32 } \
33 case 2: \
34 { \
35 volatile u16 *__ptr = (volatile u16 *)(ptr); \
36 asm volatile("xchgw %0,%1" \
37 : "=r" (__x), "+m" (*__ptr) \
38 : "0" (__x) \
39 : "memory"); \
40 break; \
41 } \
42 case 4: \
43 { \
44 volatile u32 *__ptr = (volatile u32 *)(ptr); \
45 asm volatile("xchgl %0,%1" \
46 : "=r" (__x), "+m" (*__ptr) \
47 : "0" (__x) \
48 : "memory"); \
49 break; \
50 } \
51 case 8: \
52 { \
53 volatile u64 *__ptr = (volatile u64 *)(ptr); \
54 asm volatile("xchgq %0,%1" \
55 : "=r" (__x), "+m" (*__ptr) \
56 : "0" (__x) \
57 : "memory"); \
58 break; \
59 } \
60 default: \
61 __xchg_wrong_size(); \
62 } \
63 __x; \
64}) 15})
65 16
66#define xchg(ptr, v) \ 17#define cmpxchg64_local(ptr, o, n) \
67 __xchg((v), (ptr), sizeof(*ptr))
68
69#define __HAVE_ARCH_CMPXCHG 1
70
71/*
72 * Atomic compare and exchange. Compare OLD with MEM, if identical,
73 * store NEW in MEM. Return the initial value in MEM. Success is
74 * indicated by comparing RETURN with OLD.
75 */
76#define __raw_cmpxchg(ptr, old, new, size, lock) \
77({ \ 18({ \
78 __typeof__(*(ptr)) __ret; \ 19 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
79 __typeof__(*(ptr)) __old = (old); \ 20 cmpxchg_local((ptr), (o), (n)); \
80 __typeof__(*(ptr)) __new = (new); \
81 switch (size) { \
82 case 1: \
83 { \
84 volatile u8 *__ptr = (volatile u8 *)(ptr); \
85 asm volatile(lock "cmpxchgb %2,%1" \
86 : "=a" (__ret), "+m" (*__ptr) \
87 : "q" (__new), "0" (__old) \
88 : "memory"); \
89 break; \
90 } \
91 case 2: \
92 { \
93 volatile u16 *__ptr = (volatile u16 *)(ptr); \
94 asm volatile(lock "cmpxchgw %2,%1" \
95 : "=a" (__ret), "+m" (*__ptr) \
96 : "r" (__new), "0" (__old) \
97 : "memory"); \
98 break; \
99 } \
100 case 4: \
101 { \
102 volatile u32 *__ptr = (volatile u32 *)(ptr); \
103 asm volatile(lock "cmpxchgl %2,%1" \
104 : "=a" (__ret), "+m" (*__ptr) \
105 : "r" (__new), "0" (__old) \
106 : "memory"); \
107 break; \
108 } \
109 case 8: \
110 { \
111 volatile u64 *__ptr = (volatile u64 *)(ptr); \
112 asm volatile(lock "cmpxchgq %2,%1" \
113 : "=a" (__ret), "+m" (*__ptr) \
114 : "r" (__new), "0" (__old) \
115 : "memory"); \
116 break; \
117 } \
118 default: \
119 __cmpxchg_wrong_size(); \
120 } \
121 __ret; \
122}) 21})
123 22
124#define __cmpxchg(ptr, old, new, size) \ 23#define cmpxchg16b(ptr, o1, o2, n1, n2) \
125 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX) 24({ \
126 25 char __ret; \
127#define __sync_cmpxchg(ptr, old, new, size) \ 26 __typeof__(o2) __junk; \
128 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ") 27 __typeof__(*(ptr)) __old1 = (o1); \
129 28 __typeof__(o2) __old2 = (o2); \
130#define __cmpxchg_local(ptr, old, new, size) \ 29 __typeof__(*(ptr)) __new1 = (n1); \
131 __raw_cmpxchg((ptr), (old), (new), (size), "") 30 __typeof__(o2) __new2 = (n2); \
132 31 asm volatile(LOCK_PREFIX "cmpxchg16b %2;setz %1" \
133#define cmpxchg(ptr, old, new) \ 32 : "=d"(__junk), "=a"(__ret), "+m" (*ptr) \
134 __cmpxchg((ptr), (old), (new), sizeof(*ptr)) 33 : "b"(__new1), "c"(__new2), \
135 34 "a"(__old1), "d"(__old2)); \
136#define sync_cmpxchg(ptr, old, new) \ 35 __ret; })
137 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr)) 36
138 37
139#define cmpxchg_local(ptr, old, new) \ 38#define cmpxchg16b_local(ptr, o1, o2, n1, n2) \
140 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr)) 39({ \
141 40 char __ret; \
142#define cmpxchg64(ptr, o, n) \ 41 __typeof__(o2) __junk; \
42 __typeof__(*(ptr)) __old1 = (o1); \
43 __typeof__(o2) __old2 = (o2); \
44 __typeof__(*(ptr)) __new1 = (n1); \
45 __typeof__(o2) __new2 = (n2); \
46 asm volatile("cmpxchg16b %2;setz %1" \
47 : "=d"(__junk), "=a"(__ret), "+m" (*ptr) \
48 : "b"(__new1), "c"(__new2), \
49 "a"(__old1), "d"(__old2)); \
50 __ret; })
51
52#define cmpxchg_double(ptr, o1, o2, n1, n2) \
143({ \ 53({ \
144 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 54 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
145 cmpxchg((ptr), (o), (n)); \ 55 VM_BUG_ON((unsigned long)(ptr) % 16); \
56 cmpxchg16b((ptr), (o1), (o2), (n1), (n2)); \
146}) 57})
147 58
148#define cmpxchg64_local(ptr, o, n) \ 59#define cmpxchg_double_local(ptr, o1, o2, n1, n2) \
149({ \ 60({ \
150 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 61 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
151 cmpxchg_local((ptr), (o), (n)); \ 62 VM_BUG_ON((unsigned long)(ptr) % 16); \
63 cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2)); \
152}) 64})
153 65
66#define system_has_cmpxchg_double() cpu_has_cx16
67
154#endif /* _ASM_X86_CMPXCHG_64_H */ 68#endif /* _ASM_X86_CMPXCHG_64_H */
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-18 15:03:32 -0500