aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBoqun Feng <boqun.feng@gmail.com>2016-01-05 21:08:25 -0500
committerMichael Ellerman <mpe@ellerman.id.au>2016-02-17 08:11:21 -0500
commitdc53617c4a3f6ca35641dfd4279720365ce9f4da (patch)
tree925cc4b32384a52b9b10ed2e5136829fde079b80
parente1ab7f39d7e0dbfbdefe148be3ae4ee121e47ecc (diff)
powerpc: atomic: Implement atomic{, 64}_*_return_* variants
On powerpc, acquire and release semantics can be achieved with lightweight barriers("lwsync" and "ctrl+isync"), which can be used to implement __atomic_op_{acquire,release}. For release semantics, since we only need to ensure all memory accesses that issue before must take effects before the -store- part of the atomics, "lwsync" is what we only need. On the platform without "lwsync", "sync" should be used. Therefore in __atomic_op_release() we use PPC_RELEASE_BARRIER. For acquire semantics, "lwsync" is what we only need for the similar reason. However on the platform without "lwsync", we can use "isync" rather than "sync" as an acquire barrier. Therefore in __atomic_op_acquire() we use PPC_ACQUIRE_BARRIER, which is barrier() on UP, "lwsync" if available and "isync" otherwise. Implement atomic{,64}_{add,sub,inc,dec}_return_relaxed, and build other variants with these helpers. Signed-off-by: Boqun Feng <boqun.feng@gmail.com> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Diffstat
-rw-r--r--arch/powerpc/include/asm/atomic.h147
1 files changed, 85 insertions, 62 deletions
diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h
index 55f106ed12bf..a35c27709e05 100644
--- a/arch/powerpc/include/asm/atomic.h
+++ b/arch/powerpc/include/asm/atomic.h
@@ -12,6 +12,24 @@
12 12
13#define ATOMIC_INIT(i) { (i) } 13#define ATOMIC_INIT(i) { (i) }
14 14
15/*
16 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with
17 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier
18 * on the platform without lwsync.
19 */
20#define __atomic_op_acquire(op, args...) \
21({ \
22 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
23 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory"); \
24 __ret; \
25})
26
27#define __atomic_op_release(op, args...) \
28({ \
29 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory"); \
30 op##_relaxed(args); \
31})
32
15static __inline__ int atomic_read(const atomic_t *v) 33static __inline__ int atomic_read(const atomic_t *v)
16{ 34{
17 int t; 35 int t;
@@ -42,27 +60,27 @@ static __inline__ void atomic_##op(int a, atomic_t *v) \
42 : "cc"); \ 60 : "cc"); \
43} \ 61} \
44 62
45#define ATOMIC_OP_RETURN(op, asm_op) \ 63#define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \
46static __inline__ int atomic_##op##_return(int a, atomic_t *v) \ 64static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \
47{ \ 65{ \
48 int t; \ 66 int t; \
49 \ 67 \
50 __asm__ __volatile__( \ 68 __asm__ __volatile__( \
51 PPC_ATOMIC_ENTRY_BARRIER \ 69"1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \
52"1: lwarx %0,0,%2 # atomic_" #op "_return\n" \ 70 #asm_op " %0,%2,%0\n" \
53 #asm_op " %0,%1,%0\n" \ 71 PPC405_ERR77(0, %3) \
54 PPC405_ERR77(0,%2) \ 72" stwcx. %0,0,%3\n" \
55" stwcx. %0,0,%2 \n" \
56" bne- 1b\n" \ 73" bne- 1b\n" \
57 PPC_ATOMIC_EXIT_BARRIER \ 74 : "=&r" (t), "+m" (v->counter) \
58 : "=&r" (t) \
59 : "r" (a), "r" (&v->counter) \ 75 : "r" (a), "r" (&v->counter) \
60 : "cc", "memory"); \ 76 : "cc"); \
61 \ 77 \
62 return t; \ 78 return t; \
63} 79}
64 80
65#define ATOMIC_OPS(op, asm_op) ATOMIC_OP(op, asm_op) ATOMIC_OP_RETURN(op, asm_op) 81#define ATOMIC_OPS(op, asm_op) \
82 ATOMIC_OP(op, asm_op) \
83 ATOMIC_OP_RETURN_RELAXED(op, asm_op)
66 84
67ATOMIC_OPS(add, add) 85ATOMIC_OPS(add, add)
68ATOMIC_OPS(sub, subf) 86ATOMIC_OPS(sub, subf)
@@ -71,8 +89,11 @@ ATOMIC_OP(and, and)
71ATOMIC_OP(or, or) 89ATOMIC_OP(or, or)
72ATOMIC_OP(xor, xor) 90ATOMIC_OP(xor, xor)
73 91
92#define atomic_add_return_relaxed atomic_add_return_relaxed
93#define atomic_sub_return_relaxed atomic_sub_return_relaxed
94
74#undef ATOMIC_OPS 95#undef ATOMIC_OPS
75#undef ATOMIC_OP_RETURN 96#undef ATOMIC_OP_RETURN_RELAXED
76#undef ATOMIC_OP 97#undef ATOMIC_OP
77 98
78#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) 99#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0)
@@ -92,21 +113,19 @@ static __inline__ void atomic_inc(atomic_t *v)
92 : "cc", "xer"); 113 : "cc", "xer");
93} 114}
94 115
95static __inline__ int atomic_inc_return(atomic_t *v) 116static __inline__ int atomic_inc_return_relaxed(atomic_t *v)
96{ 117{
97 int t; 118 int t;
98 119
99 __asm__ __volatile__( 120 __asm__ __volatile__(
100 PPC_ATOMIC_ENTRY_BARRIER 121"1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n"
101"1: lwarx %0,0,%1 # atomic_inc_return\n\ 122" addic %0,%0,1\n"
102 addic %0,%0,1\n" 123 PPC405_ERR77(0, %2)
103 PPC405_ERR77(0,%1) 124" stwcx. %0,0,%2\n"
104" stwcx. %0,0,%1 \n\ 125" bne- 1b"
105 bne- 1b" 126 : "=&r" (t), "+m" (v->counter)
106 PPC_ATOMIC_EXIT_BARRIER
107 : "=&r" (t)
108 : "r" (&v->counter) 127 : "r" (&v->counter)
109 : "cc", "xer", "memory"); 128 : "cc", "xer");
110 129
111 return t; 130 return t;
112} 131}
@@ -136,25 +155,26 @@ static __inline__ void atomic_dec(atomic_t *v)
136 : "cc", "xer"); 155 : "cc", "xer");
137} 156}
138 157
139static __inline__ int atomic_dec_return(atomic_t *v) 158static __inline__ int atomic_dec_return_relaxed(atomic_t *v)
140{ 159{
141 int t; 160 int t;
142 161
143 __asm__ __volatile__( 162 __asm__ __volatile__(
144 PPC_ATOMIC_ENTRY_BARRIER 163"1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n"
145"1: lwarx %0,0,%1 # atomic_dec_return\n\ 164" addic %0,%0,-1\n"
146 addic %0,%0,-1\n" 165 PPC405_ERR77(0, %2)
147 PPC405_ERR77(0,%1) 166" stwcx. %0,0,%2\n"
148" stwcx. %0,0,%1\n\ 167" bne- 1b"
149 bne- 1b" 168 : "=&r" (t), "+m" (v->counter)
150 PPC_ATOMIC_EXIT_BARRIER
151 : "=&r" (t)
152 : "r" (&v->counter) 169 : "r" (&v->counter)
153 : "cc", "xer", "memory"); 170 : "cc", "xer");
154 171
155 return t; 172 return t;
156} 173}
157 174
175#define atomic_inc_return_relaxed atomic_inc_return_relaxed
176#define atomic_dec_return_relaxed atomic_dec_return_relaxed
177
158#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 178#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
159#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 179#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
160 180
@@ -285,26 +305,27 @@ static __inline__ void atomic64_##op(long a, atomic64_t *v) \
285 : "cc"); \ 305 : "cc"); \
286} 306}
287 307
288#define ATOMIC64_OP_RETURN(op, asm_op) \ 308#define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \
289static __inline__ long atomic64_##op##_return(long a, atomic64_t *v) \ 309static inline long \
310atomic64_##op##_return_relaxed(long a, atomic64_t *v) \
290{ \ 311{ \
291 long t; \ 312 long t; \
292 \ 313 \
293 __asm__ __volatile__( \ 314 __asm__ __volatile__( \
294 PPC_ATOMIC_ENTRY_BARRIER \ 315"1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \
295"1: ldarx %0,0,%2 # atomic64_" #op "_return\n" \ 316 #asm_op " %0,%2,%0\n" \
296 #asm_op " %0,%1,%0\n" \ 317" stdcx. %0,0,%3\n" \
297" stdcx. %0,0,%2 \n" \
298" bne- 1b\n" \ 318" bne- 1b\n" \
299 PPC_ATOMIC_EXIT_BARRIER \ 319 : "=&r" (t), "+m" (v->counter) \
300 : "=&r" (t) \
301 : "r" (a), "r" (&v->counter) \ 320 : "r" (a), "r" (&v->counter) \
302 : "cc", "memory"); \ 321 : "cc"); \
303 \ 322 \
304 return t; \ 323 return t; \
305} 324}
306 325
307#define ATOMIC64_OPS(op, asm_op) ATOMIC64_OP(op, asm_op) ATOMIC64_OP_RETURN(op, asm_op) 326#define ATOMIC64_OPS(op, asm_op) \
327 ATOMIC64_OP(op, asm_op) \
328 ATOMIC64_OP_RETURN_RELAXED(op, asm_op)
308 329
309ATOMIC64_OPS(add, add) 330ATOMIC64_OPS(add, add)
310ATOMIC64_OPS(sub, subf) 331ATOMIC64_OPS(sub, subf)
@@ -312,8 +333,11 @@ ATOMIC64_OP(and, and)
312ATOMIC64_OP(or, or) 333ATOMIC64_OP(or, or)
313ATOMIC64_OP(xor, xor) 334ATOMIC64_OP(xor, xor)
314 335
315#undef ATOMIC64_OPS 336#define atomic64_add_return_relaxed atomic64_add_return_relaxed
316#undef ATOMIC64_OP_RETURN 337#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
338
339#undef ATOPIC64_OPS
340#undef ATOMIC64_OP_RETURN_RELAXED
317#undef ATOMIC64_OP 341#undef ATOMIC64_OP
318 342
319#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) 343#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
@@ -332,20 +356,18 @@ static __inline__ void atomic64_inc(atomic64_t *v)
332 : "cc", "xer"); 356 : "cc", "xer");
333} 357}
334 358
335static __inline__ long atomic64_inc_return(atomic64_t *v) 359static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v)
336{ 360{
337 long t; 361 long t;
338 362
339 __asm__ __volatile__( 363 __asm__ __volatile__(
340 PPC_ATOMIC_ENTRY_BARRIER 364"1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n"
341"1: ldarx %0,0,%1 # atomic64_inc_return\n\ 365" addic %0,%0,1\n"
342 addic %0,%0,1\n\ 366" stdcx. %0,0,%2\n"
343 stdcx. %0,0,%1 \n\ 367" bne- 1b"
344 bne- 1b" 368 : "=&r" (t), "+m" (v->counter)
345 PPC_ATOMIC_EXIT_BARRIER
346 : "=&r" (t)
347 : "r" (&v->counter) 369 : "r" (&v->counter)
348 : "cc", "xer", "memory"); 370 : "cc", "xer");
349 371
350 return t; 372 return t;
351} 373}
@@ -374,24 +396,25 @@ static __inline__ void atomic64_dec(atomic64_t *v)
374 : "cc", "xer"); 396 : "cc", "xer");
375} 397}
376 398
377static __inline__ long atomic64_dec_return(atomic64_t *v) 399static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v)
378{ 400{
379 long t; 401 long t;
380 402
381 __asm__ __volatile__( 403 __asm__ __volatile__(
382 PPC_ATOMIC_ENTRY_BARRIER 404"1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n"
383"1: ldarx %0,0,%1 # atomic64_dec_return\n\ 405" addic %0,%0,-1\n"
384 addic %0,%0,-1\n\ 406" stdcx. %0,0,%2\n"
385 stdcx. %0,0,%1\n\ 407" bne- 1b"
386 bne- 1b" 408 : "=&r" (t), "+m" (v->counter)
387 PPC_ATOMIC_EXIT_BARRIER
388 : "=&r" (t)
389 : "r" (&v->counter) 409 : "r" (&v->counter)
390 : "cc", "xer", "memory"); 410 : "cc", "xer");
391 411
392 return t; 412 return t;
393} 413}
394 414
415#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
416#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
417
395#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) 418#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
396#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) 419#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
397 420
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-14 08:29:23 -0400