summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJoshua Kinard <kumba@gentoo.org>2017-11-18 23:00:51 -0500
committerPaul Burton <paul.burton@mips.com>2018-07-12 14:25:25 -0400
commit4936084c2ee227524c242d790a9fbad7650320c7 (patch)
tree153449026d2682524d9e43d03aa05e3caf2ff3ac
parenta0a5ac3ce8fe6bf26694f49f9ba42ed859487424 (diff)
MIPS: Cleanup R10000_LLSC_WAR logic in atomic.h
This patch reduces down the conditionals in MIPS atomic code that deal with a silicon bug in early R10000 cpus that required a workaround of a branch-likely instruction following a store-conditional in order to to guarantee the whole ll/sc sequence is atomic. As the only real difference is a branch-likely instruction (beqzl) over a standard branch (beqz), the conditional is reduced down to a single preprocessor check at the top to pick the required instruction. This requires writing the uses in assembler, thus we discard the non-R10000 case that uses a mixture of a C do...while loop with embedded assembler that was added back in commit 7837314d141c ("MIPS: Get rid of branches to .subsections."). A note found in the git log for commit 5999eca25c1f ("[MIPS] Improve branch prediction in ll/sc atomic operations.") is also addressed. The macro definition for the branch instruction and the code comment derives from a patch sent in earlier by Paul Burton for various cmpxchg cleanups. [paul.burton@mips.com: - Minor whitespace fix for checkpatch.] Signed-off-by: Joshua Kinard <kumba@gentoo.org> Signed-off-by: Paul Burton <paul.burton@mips.com> Patchwork: https://patchwork.linux-mips.org/patch/17736/ Cc: Ralf Baechle <ralf@linux-mips.org> Cc: James Hogan <james.hogan@mips.com> Cc: "Maciej W. Rozycki" <macro@mips.com> Cc: linux-mips@linux-mips.org
Diffstat
-rw-r--r--arch/mips/include/asm/atomic.h179
1 files changed, 32 insertions, 147 deletions
diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h
index 0babf2775d8e..3ccea238be2d 100644
--- a/arch/mips/include/asm/atomic.h
+++ b/arch/mips/include/asm/atomic.h
@@ -22,6 +22,17 @@
22#include <asm/cmpxchg.h> 22#include <asm/cmpxchg.h>
23#include <asm/war.h> 23#include <asm/war.h>
24 24
25/*
26 * Using a branch-likely instruction to check the result of an sc instruction
27 * works around a bug present in R10000 CPUs prior to revision 3.0 that could
28 * cause ll-sc sequences to execute non-atomically.
29 */
30#if R10000_LLSC_WAR
31# define __scbeqz "beqzl"
32#else
33# define __scbeqz "beqz"
34#endif
35
25#define ATOMIC_INIT(i) { (i) } 36#define ATOMIC_INIT(i) { (i) }
26 37
27/* 38/*
@@ -44,31 +55,18 @@
44#define ATOMIC_OP(op, c_op, asm_op) \ 55#define ATOMIC_OP(op, c_op, asm_op) \
45static __inline__ void atomic_##op(int i, atomic_t * v) \ 56static __inline__ void atomic_##op(int i, atomic_t * v) \
46{ \ 57{ \
47 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 58 if (kernel_uses_llsc) { \
48 int temp; \ 59 int temp; \
49 \ 60 \
50 __asm__ __volatile__( \ 61 __asm__ __volatile__( \
51 " .set arch=r4000 \n" \ 62 " .set "MIPS_ISA_LEVEL" \n" \
52 "1: ll %0, %1 # atomic_" #op " \n" \ 63 "1: ll %0, %1 # atomic_" #op " \n" \
53 " " #asm_op " %0, %2 \n" \ 64 " " #asm_op " %0, %2 \n" \
54 " sc %0, %1 \n" \ 65 " sc %0, %1 \n" \
55 " beqzl %0, 1b \n" \ 66 "\t" __scbeqz " %0, 1b \n" \
56 " .set mips0 \n" \ 67 " .set mips0 \n" \
57 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ 68 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
58 : "Ir" (i)); \ 69 : "Ir" (i)); \
59 } else if (kernel_uses_llsc) { \
60 int temp; \
61 \
62 do { \
63 __asm__ __volatile__( \
64 " .set "MIPS_ISA_LEVEL" \n" \
65 " ll %0, %1 # atomic_" #op "\n" \
66 " " #asm_op " %0, %2 \n" \
67 " sc %0, %1 \n" \
68 " .set mips0 \n" \
69 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
70 : "Ir" (i)); \
71 } while (unlikely(!temp)); \
72 } else { \ 70 } else { \
73 unsigned long flags; \ 71 unsigned long flags; \
74 \ 72 \
@@ -83,36 +81,20 @@ static __inline__ int atomic_##op##_return_relaxed(int i, atomic_t * v) \
83{ \ 81{ \
84 int result; \ 82 int result; \
85 \ 83 \
86 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 84 if (kernel_uses_llsc) { \
87 int temp; \ 85 int temp; \
88 \ 86 \
89 __asm__ __volatile__( \ 87 __asm__ __volatile__( \
90 " .set arch=r4000 \n" \ 88 " .set "MIPS_ISA_LEVEL" \n" \
91 "1: ll %1, %2 # atomic_" #op "_return \n" \ 89 "1: ll %1, %2 # atomic_" #op "_return \n" \
92 " " #asm_op " %0, %1, %3 \n" \ 90 " " #asm_op " %0, %1, %3 \n" \
93 " sc %0, %2 \n" \ 91 " sc %0, %2 \n" \
94 " beqzl %0, 1b \n" \ 92 "\t" __scbeqz " %0, 1b \n" \
95 " " #asm_op " %0, %1, %3 \n" \ 93 " " #asm_op " %0, %1, %3 \n" \
96 " .set mips0 \n" \ 94 " .set mips0 \n" \
97 : "=&r" (result), "=&r" (temp), \ 95 : "=&r" (result), "=&r" (temp), \
98 "+" GCC_OFF_SMALL_ASM() (v->counter) \ 96 "+" GCC_OFF_SMALL_ASM() (v->counter) \
99 : "Ir" (i)); \ 97 : "Ir" (i)); \
100 } else if (kernel_uses_llsc) { \
101 int temp; \
102 \
103 do { \
104 __asm__ __volatile__( \
105 " .set "MIPS_ISA_LEVEL" \n" \
106 " ll %1, %2 # atomic_" #op "_return \n" \
107 " " #asm_op " %0, %1, %3 \n" \
108 " sc %0, %2 \n" \
109 " .set mips0 \n" \
110 : "=&r" (result), "=&r" (temp), \
111 "+" GCC_OFF_SMALL_ASM() (v->counter) \
112 : "Ir" (i)); \
113 } while (unlikely(!result)); \
114 \
115 result = temp; result c_op i; \
116 } else { \ 98 } else { \
117 unsigned long flags; \ 99 unsigned long flags; \
118 \ 100 \
@@ -131,36 +113,20 @@ static __inline__ int atomic_fetch_##op##_relaxed(int i, atomic_t * v) \
131{ \ 113{ \
132 int result; \ 114 int result; \
133 \ 115 \
134 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 116 if (kernel_uses_llsc) { \
135 int temp; \ 117 int temp; \
136 \ 118 \
137 __asm__ __volatile__( \ 119 __asm__ __volatile__( \
138 " .set arch=r4000 \n" \ 120 " .set "MIPS_ISA_LEVEL" \n" \
139 "1: ll %1, %2 # atomic_fetch_" #op " \n" \ 121 "1: ll %1, %2 # atomic_fetch_" #op " \n" \
140 " " #asm_op " %0, %1, %3 \n" \ 122 " " #asm_op " %0, %1, %3 \n" \
141 " sc %0, %2 \n" \ 123 " sc %0, %2 \n" \
142 " beqzl %0, 1b \n" \ 124 "\t" __scbeqz " %0, 1b \n" \
143 " move %0, %1 \n" \ 125 " move %0, %1 \n" \
144 " .set mips0 \n" \ 126 " .set mips0 \n" \
145 : "=&r" (result), "=&r" (temp), \ 127 : "=&r" (result), "=&r" (temp), \
146 "+" GCC_OFF_SMALL_ASM() (v->counter) \ 128 "+" GCC_OFF_SMALL_ASM() (v->counter) \
147 : "Ir" (i)); \ 129 : "Ir" (i)); \
148 } else if (kernel_uses_llsc) { \
149 int temp; \
150 \
151 do { \
152 __asm__ __volatile__( \
153 " .set "MIPS_ISA_LEVEL" \n" \
154 " ll %1, %2 # atomic_fetch_" #op " \n" \
155 " " #asm_op " %0, %1, %3 \n" \
156 " sc %0, %2 \n" \
157 " .set mips0 \n" \
158 : "=&r" (result), "=&r" (temp), \
159 "+" GCC_OFF_SMALL_ASM() (v->counter) \
160 : "Ir" (i)); \
161 } while (unlikely(!result)); \
162 \
163 result = temp; \
164 } else { \ 130 } else { \
165 unsigned long flags; \ 131 unsigned long flags; \
166 \ 132 \
@@ -218,24 +184,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
218 184
219 smp_mb__before_llsc(); 185 smp_mb__before_llsc();
220 186
221 if (kernel_uses_llsc && R10000_LLSC_WAR) { 187 if (kernel_uses_llsc) {
222 int temp;
223
224 __asm__ __volatile__(
225 " .set arch=r4000 \n"
226 "1: ll %1, %2 # atomic_sub_if_positive\n"
227 " subu %0, %1, %3 \n"
228 " move %1, %0 \n"
229 " bltz %0, 1f \n"
230 " sc %1, %2 \n"
231 " beqzl %1, 1b \n"
232 "1: \n"
233 " .set mips0 \n"
234 : "=&r" (result), "=&r" (temp),
235 "+" GCC_OFF_SMALL_ASM() (v->counter)
236 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter)
237 : "memory");
238 } else if (kernel_uses_llsc) {
239 int temp; 188 int temp;
240 189
241 __asm__ __volatile__( 190 __asm__ __volatile__(
@@ -245,7 +194,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
245 " move %1, %0 \n" 194 " move %1, %0 \n"
246 " bltz %0, 1f \n" 195 " bltz %0, 1f \n"
247 " sc %1, %2 \n" 196 " sc %1, %2 \n"
248 " beqz %1, 1b \n" 197 "\t" __scbeqz " %1, 1b \n"
249 "1: \n" 198 "1: \n"
250 " .set mips0 \n" 199 " .set mips0 \n"
251 : "=&r" (result), "=&r" (temp), 200 : "=&r" (result), "=&r" (temp),
@@ -382,31 +331,18 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
382#define ATOMIC64_OP(op, c_op, asm_op) \ 331#define ATOMIC64_OP(op, c_op, asm_op) \
383static __inline__ void atomic64_##op(long i, atomic64_t * v) \ 332static __inline__ void atomic64_##op(long i, atomic64_t * v) \
384{ \ 333{ \
385 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 334 if (kernel_uses_llsc) { \
386 long temp; \ 335 long temp; \
387 \ 336 \
388 __asm__ __volatile__( \ 337 __asm__ __volatile__( \
389 " .set arch=r4000 \n" \ 338 " .set "MIPS_ISA_LEVEL" \n" \
390 "1: lld %0, %1 # atomic64_" #op " \n" \ 339 "1: lld %0, %1 # atomic64_" #op " \n" \
391 " " #asm_op " %0, %2 \n" \ 340 " " #asm_op " %0, %2 \n" \
392 " scd %0, %1 \n" \ 341 " scd %0, %1 \n" \
393 " beqzl %0, 1b \n" \ 342 "\t" __scbeqz " %0, 1b \n" \
394 " .set mips0 \n" \ 343 " .set mips0 \n" \
395 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ 344 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
396 : "Ir" (i)); \ 345 : "Ir" (i)); \
397 } else if (kernel_uses_llsc) { \
398 long temp; \
399 \
400 do { \
401 __asm__ __volatile__( \
402 " .set "MIPS_ISA_LEVEL" \n" \
403 " lld %0, %1 # atomic64_" #op "\n" \
404 " " #asm_op " %0, %2 \n" \
405 " scd %0, %1 \n" \
406 " .set mips0 \n" \
407 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
408 : "Ir" (i)); \
409 } while (unlikely(!temp)); \
410 } else { \ 346 } else { \
411 unsigned long flags; \ 347 unsigned long flags; \
412 \ 348 \
@@ -421,37 +357,20 @@ static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v) \
421{ \ 357{ \
422 long result; \ 358 long result; \
423 \ 359 \
424 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 360 if (kernel_uses_llsc) { \
425 long temp; \ 361 long temp; \
426 \ 362 \
427 __asm__ __volatile__( \ 363 __asm__ __volatile__( \
428 " .set arch=r4000 \n" \ 364 " .set "MIPS_ISA_LEVEL" \n" \
429 "1: lld %1, %2 # atomic64_" #op "_return\n" \ 365 "1: lld %1, %2 # atomic64_" #op "_return\n" \
430 " " #asm_op " %0, %1, %3 \n" \ 366 " " #asm_op " %0, %1, %3 \n" \
431 " scd %0, %2 \n" \ 367 " scd %0, %2 \n" \
432 " beqzl %0, 1b \n" \ 368 "\t" __scbeqz " %0, 1b \n" \
433 " " #asm_op " %0, %1, %3 \n" \ 369 " " #asm_op " %0, %1, %3 \n" \
434 " .set mips0 \n" \ 370 " .set mips0 \n" \
435 : "=&r" (result), "=&r" (temp), \ 371 : "=&r" (result), "=&r" (temp), \
436 "+" GCC_OFF_SMALL_ASM() (v->counter) \ 372 "+" GCC_OFF_SMALL_ASM() (v->counter) \
437 : "Ir" (i)); \ 373 : "Ir" (i)); \
438 } else if (kernel_uses_llsc) { \
439 long temp; \
440 \
441 do { \
442 __asm__ __volatile__( \
443 " .set "MIPS_ISA_LEVEL" \n" \
444 " lld %1, %2 # atomic64_" #op "_return\n" \
445 " " #asm_op " %0, %1, %3 \n" \
446 " scd %0, %2 \n" \
447 " .set mips0 \n" \
448 : "=&r" (result), "=&r" (temp), \
449 "=" GCC_OFF_SMALL_ASM() (v->counter) \
450 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) \
451 : "memory"); \
452 } while (unlikely(!result)); \
453 \
454 result = temp; result c_op i; \
455 } else { \ 374 } else { \
456 unsigned long flags; \ 375 unsigned long flags; \
457 \ 376 \
@@ -474,33 +393,16 @@ static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \
474 long temp; \ 393 long temp; \
475 \ 394 \
476 __asm__ __volatile__( \ 395 __asm__ __volatile__( \
477 " .set arch=r4000 \n" \ 396 " .set "MIPS_ISA_LEVEL" \n" \
478 "1: lld %1, %2 # atomic64_fetch_" #op "\n" \ 397 "1: lld %1, %2 # atomic64_fetch_" #op "\n" \
479 " " #asm_op " %0, %1, %3 \n" \ 398 " " #asm_op " %0, %1, %3 \n" \
480 " scd %0, %2 \n" \ 399 " scd %0, %2 \n" \
481 " beqzl %0, 1b \n" \ 400 "\t" __scbeqz " %0, 1b \n" \
482 " move %0, %1 \n" \ 401 " move %0, %1 \n" \
483 " .set mips0 \n" \ 402 " .set mips0 \n" \
484 : "=&r" (result), "=&r" (temp), \ 403 : "=&r" (result), "=&r" (temp), \
485 "+" GCC_OFF_SMALL_ASM() (v->counter) \ 404 "+" GCC_OFF_SMALL_ASM() (v->counter) \
486 : "Ir" (i)); \ 405 : "Ir" (i)); \
487 } else if (kernel_uses_llsc) { \
488 long temp; \
489 \
490 do { \
491 __asm__ __volatile__( \
492 " .set "MIPS_ISA_LEVEL" \n" \
493 " lld %1, %2 # atomic64_fetch_" #op "\n" \
494 " " #asm_op " %0, %1, %3 \n" \
495 " scd %0, %2 \n" \
496 " .set mips0 \n" \
497 : "=&r" (result), "=&r" (temp), \
498 "=" GCC_OFF_SMALL_ASM() (v->counter) \
499 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) \
500 : "memory"); \
501 } while (unlikely(!result)); \
502 \
503 result = temp; \
504 } else { \ 406 } else { \
505 unsigned long flags; \ 407 unsigned long flags; \
506 \ 408 \
@@ -559,24 +461,7 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
559 461
560 smp_mb__before_llsc(); 462 smp_mb__before_llsc();
561 463
562 if (kernel_uses_llsc && R10000_LLSC_WAR) { 464 if (kernel_uses_llsc) {
563 long temp;
564
565 __asm__ __volatile__(
566 " .set arch=r4000 \n"
567 "1: lld %1, %2 # atomic64_sub_if_positive\n"
568 " dsubu %0, %1, %3 \n"
569 " move %1, %0 \n"
570 " bltz %0, 1f \n"
571 " scd %1, %2 \n"
572 " beqzl %1, 1b \n"
573 "1: \n"
574 " .set mips0 \n"
575 : "=&r" (result), "=&r" (temp),
576 "=" GCC_OFF_SMALL_ASM() (v->counter)
577 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter)
578 : "memory");
579 } else if (kernel_uses_llsc) {
580 long temp; 465 long temp;
581 466
582 __asm__ __volatile__( 467 __asm__ __volatile__(
@@ -586,7 +471,7 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
586 " move %1, %0 \n" 471 " move %1, %0 \n"
587 " bltz %0, 1f \n" 472 " bltz %0, 1f \n"
588 " scd %1, %2 \n" 473 " scd %1, %2 \n"
589 " beqz %1, 1b \n" 474 "\t" __scbeqz " %1, 1b \n"
590 "1: \n" 475 "1: \n"
591 " .set mips0 \n" 476 " .set mips0 \n"
592 : "=&r" (result), "=&r" (temp), 477 : "=&r" (result), "=&r" (temp),
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-25 13:47:17 -0400