aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/mips/mm/pg-r4k.c2
-rw-r--r--include/asm-mips/cmpxchg.h107
-rw-r--r--include/asm-mips/local.h69
-rw-r--r--include/asm-mips/system.h261
4 files changed, 110 insertions, 329 deletions
diff --git a/arch/mips/mm/pg-r4k.c b/arch/mips/mm/pg-r4k.c
index dc795be62807..e47e9e9486bf 100644
--- a/arch/mips/mm/pg-r4k.c
+++ b/arch/mips/mm/pg-r4k.c
@@ -209,7 +209,7 @@ static inline void build_cdex_p(void)
209 } 209 }
210 210
211 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) 211 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
212 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */ 212 build_insn_word(0x8c200000); /* lw $zero, ($at) */
213 213
214 mi.c_format.opcode = cache_op; 214 mi.c_format.opcode = cache_op;
215 mi.c_format.rs = 4; /* $a0 */ 215 mi.c_format.rs = 4; /* $a0 */
diff --git a/include/asm-mips/cmpxchg.h b/include/asm-mips/cmpxchg.h
new file mode 100644
index 000000000000..c5b4708e003b
--- /dev/null
+++ b/include/asm-mips/cmpxchg.h
@@ -0,0 +1,107 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
7 */
8#ifndef __ASM_CMPXCHG_H
9#define __ASM_CMPXCHG_H
10
11#include <linux/irqflags.h>
12
13#define __HAVE_ARCH_CMPXCHG 1
14
15#define __cmpxchg_asm(ld, st, m, old, new) \
16({ \
17 __typeof(*(m)) __ret; \
18 \
19 if (cpu_has_llsc && R10000_LLSC_WAR) { \
20 __asm__ __volatile__( \
21 " .set push \n" \
22 " .set noat \n" \
23 " .set mips3 \n" \
24 "1: " ld " %0, %2 # __cmpxchg_asm \n" \
25 " bne %0, %z3, 2f \n" \
26 " .set mips0 \n" \
27 " move $1, %z4 \n" \
28 " .set mips3 \n" \
29 " " st " $1, %1 \n" \
30 " beqzl $1, 1b \n" \
31 "2: \n" \
32 " .set pop \n" \
33 : "=&r" (__ret), "=R" (*m) \
34 : "R" (*m), "Jr" (old), "Jr" (new) \
35 : "memory"); \
36 } else if (cpu_has_llsc) { \
37 __asm__ __volatile__( \
38 " .set push \n" \
39 " .set noat \n" \
40 " .set mips3 \n" \
41 "1: " ld " %0, %2 # __cmpxchg_asm \n" \
42 " bne %0, %z3, 2f \n" \
43 " .set mips0 \n" \
44 " move $1, %z4 \n" \
45 " .set mips3 \n" \
46 " " st " $1, %1 \n" \
47 " beqz $1, 3f \n" \
48 "2: \n" \
49 " .subsection 2 \n" \
50 "3: b 1b \n" \
51 " .previous \n" \
52 " .set pop \n" \
53 : "=&r" (__ret), "=R" (*m) \
54 : "R" (*m), "Jr" (old), "Jr" (new) \
55 : "memory"); \
56 } else { \
57 unsigned long __flags; \
58 \
59 raw_local_irq_save(__flags); \
60 __ret = *m; \
61 if (__ret == old) \
62 *m = new; \
63 raw_local_irq_restore(__flags); \
64 } \
65 \
66 __ret; \
67})
68
69/*
70 * This function doesn't exist, so you'll get a linker error
71 * if something tries to do an invalid cmpxchg().
72 */
73extern void __cmpxchg_called_with_bad_pointer(void);
74
75#define __cmpxchg(ptr,old,new,barrier) \
76({ \
77 __typeof__(ptr) __ptr = (ptr); \
78 __typeof__(*(ptr)) __old = (old); \
79 __typeof__(*(ptr)) __new = (new); \
80 __typeof__(*(ptr)) __res = 0; \
81 \
82 barrier; \
83 \
84 switch (sizeof(*(__ptr))) { \
85 case 4: \
86 __res = __cmpxchg_asm("ll", "sc", __ptr, __old, __new); \
87 break; \
88 case 8: \
89 if (sizeof(long) == 8) { \
90 __res = __cmpxchg_asm("lld", "scd", __ptr, \
91 __old, __new); \
92 break; \
93 } \
94 default: \
95 __cmpxchg_called_with_bad_pointer(); \
96 break; \
97 } \
98 \
99 barrier; \
100 \
101 __res; \
102})
103
104#define cmpxchg(ptr, old, new) __cmpxchg(ptr, old, new, smp_llsc_mb())
105#define cmpxchg_local(ptr, old, new) __cmpxchg(ptr, old, new,)
106
107#endif /* __ASM_CMPXCHG_H */
diff --git a/include/asm-mips/local.h b/include/asm-mips/local.h
index ed882c88e0ca..f9a5ce5c9af1 100644
--- a/include/asm-mips/local.h
+++ b/include/asm-mips/local.h
@@ -4,6 +4,7 @@
4#include <linux/percpu.h> 4#include <linux/percpu.h>
5#include <linux/bitops.h> 5#include <linux/bitops.h>
6#include <asm/atomic.h> 6#include <asm/atomic.h>
7#include <asm/cmpxchg.h>
7#include <asm/war.h> 8#include <asm/war.h>
8 9
9typedef struct 10typedef struct
@@ -114,68 +115,6 @@ static __inline__ long local_sub_return(long i, local_t * l)
114 return result; 115 return result;
115} 116}
116 117
117/*
118 * local_sub_if_positive - conditionally subtract integer from atomic variable
119 * @i: integer value to subtract
120 * @l: pointer of type local_t
121 *
122 * Atomically test @l and subtract @i if @l is greater or equal than @i.
123 * The function returns the old value of @l minus @i.
124 */
125static __inline__ long local_sub_if_positive(long i, local_t * l)
126{
127 unsigned long result;
128
129 if (cpu_has_llsc && R10000_LLSC_WAR) {
130 unsigned long temp;
131
132 __asm__ __volatile__(
133 " .set mips3 \n"
134 "1:" __LL "%1, %2 # local_sub_if_positive\n"
135 " dsubu %0, %1, %3 \n"
136 " bltz %0, 1f \n"
137 __SC "%0, %2 \n"
138 " .set noreorder \n"
139 " beqzl %0, 1b \n"
140 " dsubu %0, %1, %3 \n"
141 " .set reorder \n"
142 "1: \n"
143 " .set mips0 \n"
144 : "=&r" (result), "=&r" (temp), "=m" (l->a.counter)
145 : "Ir" (i), "m" (l->a.counter)
146 : "memory");
147 } else if (cpu_has_llsc) {
148 unsigned long temp;
149
150 __asm__ __volatile__(
151 " .set mips3 \n"
152 "1:" __LL "%1, %2 # local_sub_if_positive\n"
153 " dsubu %0, %1, %3 \n"
154 " bltz %0, 1f \n"
155 __SC "%0, %2 \n"
156 " .set noreorder \n"
157 " beqz %0, 1b \n"
158 " dsubu %0, %1, %3 \n"
159 " .set reorder \n"
160 "1: \n"
161 " .set mips0 \n"
162 : "=&r" (result), "=&r" (temp), "=m" (l->a.counter)
163 : "Ir" (i), "m" (l->a.counter)
164 : "memory");
165 } else {
166 unsigned long flags;
167
168 local_irq_save(flags);
169 result = l->a.counter;
170 result -= i;
171 if (result >= 0)
172 l->a.counter = result;
173 local_irq_restore(flags);
174 }
175
176 return result;
177}
178
179#define local_cmpxchg(l, o, n) \ 118#define local_cmpxchg(l, o, n) \
180 ((long)cmpxchg_local(&((l)->a.counter), (o), (n))) 119 ((long)cmpxchg_local(&((l)->a.counter), (o), (n)))
181#define local_xchg(l, n) (xchg_local(&((l)->a.counter),(n))) 120#define local_xchg(l, n) (xchg_local(&((l)->a.counter),(n)))
@@ -234,12 +173,6 @@ static __inline__ long local_sub_if_positive(long i, local_t * l)
234#define local_dec_and_test(l) (local_sub_return(1, (l)) == 0) 173#define local_dec_and_test(l) (local_sub_return(1, (l)) == 0)
235 174
236/* 175/*
237 * local_dec_if_positive - decrement by 1 if old value positive
238 * @l: pointer of type local_t
239 */
240#define local_dec_if_positive(l) local_sub_if_positive(1, l)
241
242/*
243 * local_add_negative - add and test if negative 176 * local_add_negative - add and test if negative
244 * @l: pointer of type local_t 177 * @l: pointer of type local_t
245 * @i: integer value to add 178 * @i: integer value to add
diff --git a/include/asm-mips/system.h b/include/asm-mips/system.h
index 357251f42518..480b574e2483 100644
--- a/include/asm-mips/system.h
+++ b/include/asm-mips/system.h
@@ -17,6 +17,7 @@
17 17
18#include <asm/addrspace.h> 18#include <asm/addrspace.h>
19#include <asm/barrier.h> 19#include <asm/barrier.h>
20#include <asm/cmpxchg.h>
20#include <asm/cpu-features.h> 21#include <asm/cpu-features.h>
21#include <asm/dsp.h> 22#include <asm/dsp.h>
22#include <asm/war.h> 23#include <asm/war.h>
@@ -194,266 +195,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz
194 195
195#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) 196#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
196 197
197#define __HAVE_ARCH_CMPXCHG 1
198
199static inline unsigned long __cmpxchg_u32(volatile int * m, unsigned long old,
200 unsigned long new)
201{
202 __u32 retval;
203
204 if (cpu_has_llsc && R10000_LLSC_WAR) {
205 __asm__ __volatile__(
206 " .set push \n"
207 " .set noat \n"
208 " .set mips3 \n"
209 "1: ll %0, %2 # __cmpxchg_u32 \n"
210 " bne %0, %z3, 2f \n"
211 " .set mips0 \n"
212 " move $1, %z4 \n"
213 " .set mips3 \n"
214 " sc $1, %1 \n"
215 " beqzl $1, 1b \n"
216 "2: \n"
217 " .set pop \n"
218 : "=&r" (retval), "=R" (*m)
219 : "R" (*m), "Jr" (old), "Jr" (new)
220 : "memory");
221 } else if (cpu_has_llsc) {
222 __asm__ __volatile__(
223 " .set push \n"
224 " .set noat \n"
225 " .set mips3 \n"
226 "1: ll %0, %2 # __cmpxchg_u32 \n"
227 " bne %0, %z3, 2f \n"
228 " .set mips0 \n"
229 " move $1, %z4 \n"
230 " .set mips3 \n"
231 " sc $1, %1 \n"
232 " beqz $1, 3f \n"
233 "2: \n"
234 " .subsection 2 \n"
235 "3: b 1b \n"
236 " .previous \n"
237 " .set pop \n"
238 : "=&r" (retval), "=R" (*m)
239 : "R" (*m), "Jr" (old), "Jr" (new)
240 : "memory");
241 } else {
242 unsigned long flags;
243
244 raw_local_irq_save(flags);
245 retval = *m;
246 if (retval == old)
247 *m = new;
248 raw_local_irq_restore(flags); /* implies memory barrier */
249 }
250
251 smp_llsc_mb();
252
253 return retval;
254}
255
256static inline unsigned long __cmpxchg_u32_local(volatile int * m,
257 unsigned long old, unsigned long new)
258{
259 __u32 retval;
260
261 if (cpu_has_llsc && R10000_LLSC_WAR) {
262 __asm__ __volatile__(
263 " .set push \n"
264 " .set noat \n"
265 " .set mips3 \n"
266 "1: ll %0, %2 # __cmpxchg_u32 \n"
267 " bne %0, %z3, 2f \n"
268 " .set mips0 \n"
269 " move $1, %z4 \n"
270 " .set mips3 \n"
271 " sc $1, %1 \n"
272 " beqzl $1, 1b \n"
273 "2: \n"
274 " .set pop \n"
275 : "=&r" (retval), "=R" (*m)
276 : "R" (*m), "Jr" (old), "Jr" (new)
277 : "memory");
278 } else if (cpu_has_llsc) {
279 __asm__ __volatile__(
280 " .set push \n"
281 " .set noat \n"
282 " .set mips3 \n"
283 "1: ll %0, %2 # __cmpxchg_u32 \n"
284 " bne %0, %z3, 2f \n"
285 " .set mips0 \n"
286 " move $1, %z4 \n"
287 " .set mips3 \n"
288 " sc $1, %1 \n"
289 " beqz $1, 1b \n"
290 "2: \n"
291 " .set pop \n"
292 : "=&r" (retval), "=R" (*m)
293 : "R" (*m), "Jr" (old), "Jr" (new)
294 : "memory");
295 } else {
296 unsigned long flags;
297
298 local_irq_save(flags);
299 retval = *m;
300 if (retval == old)
301 *m = new;
302 local_irq_restore(flags); /* implies memory barrier */
303 }
304
305 return retval;
306}
307
308#ifdef CONFIG_64BIT
309static inline unsigned long __cmpxchg_u64(volatile int * m, unsigned long old,
310 unsigned long new)
311{
312 __u64 retval;
313
314 if (cpu_has_llsc && R10000_LLSC_WAR) {
315 __asm__ __volatile__(
316 " .set push \n"
317 " .set noat \n"
318 " .set mips3 \n"
319 "1: lld %0, %2 # __cmpxchg_u64 \n"
320 " bne %0, %z3, 2f \n"
321 " move $1, %z4 \n"
322 " scd $1, %1 \n"
323 " beqzl $1, 1b \n"
324 "2: \n"
325 " .set pop \n"
326 : "=&r" (retval), "=R" (*m)
327 : "R" (*m), "Jr" (old), "Jr" (new)
328 : "memory");
329 } else if (cpu_has_llsc) {
330 __asm__ __volatile__(
331 " .set push \n"
332 " .set noat \n"
333 " .set mips3 \n"
334 "1: lld %0, %2 # __cmpxchg_u64 \n"
335 " bne %0, %z3, 2f \n"
336 " move $1, %z4 \n"
337 " scd $1, %1 \n"
338 " beqz $1, 3f \n"
339 "2: \n"
340 " .subsection 2 \n"
341 "3: b 1b \n"
342 " .previous \n"
343 " .set pop \n"
344 : "=&r" (retval), "=R" (*m)
345 : "R" (*m), "Jr" (old), "Jr" (new)
346 : "memory");
347 } else {
348 unsigned long flags;
349
350 raw_local_irq_save(flags);
351 retval = *m;
352 if (retval == old)
353 *m = new;
354 raw_local_irq_restore(flags); /* implies memory barrier */
355 }
356
357 smp_llsc_mb();
358
359 return retval;
360}
361
362static inline unsigned long __cmpxchg_u64_local(volatile int * m,
363 unsigned long old, unsigned long new)
364{
365 __u64 retval;
366
367 if (cpu_has_llsc && R10000_LLSC_WAR) {
368 __asm__ __volatile__(
369 " .set push \n"
370 " .set noat \n"
371 " .set mips3 \n"
372 "1: lld %0, %2 # __cmpxchg_u64 \n"
373 " bne %0, %z3, 2f \n"
374 " move $1, %z4 \n"
375 " scd $1, %1 \n"
376 " beqzl $1, 1b \n"
377 "2: \n"
378 " .set pop \n"
379 : "=&r" (retval), "=R" (*m)
380 : "R" (*m), "Jr" (old), "Jr" (new)
381 : "memory");
382 } else if (cpu_has_llsc) {
383 __asm__ __volatile__(
384 " .set push \n"
385 " .set noat \n"
386 " .set mips3 \n"
387 "1: lld %0, %2 # __cmpxchg_u64 \n"
388 " bne %0, %z3, 2f \n"
389 " move $1, %z4 \n"
390 " scd $1, %1 \n"
391 " beqz $1, 1b \n"
392 "2: \n"
393 " .set pop \n"
394 : "=&r" (retval), "=R" (*m)
395 : "R" (*m), "Jr" (old), "Jr" (new)
396 : "memory");
397 } else {
398 unsigned long flags;
399
400 local_irq_save(flags);
401 retval = *m;
402 if (retval == old)
403 *m = new;
404 local_irq_restore(flags); /* implies memory barrier */
405 }
406
407 return retval;
408}
409
410#else
411extern unsigned long __cmpxchg_u64_unsupported_on_32bit_kernels(
412 volatile int * m, unsigned long old, unsigned long new);
413#define __cmpxchg_u64 __cmpxchg_u64_unsupported_on_32bit_kernels
414extern unsigned long __cmpxchg_u64_local_unsupported_on_32bit_kernels(
415 volatile int * m, unsigned long old, unsigned long new);
416#define __cmpxchg_u64_local __cmpxchg_u64_local_unsupported_on_32bit_kernels
417#endif
418
419/* This function doesn't exist, so you'll get a linker error
420 if something tries to do an invalid cmpxchg(). */
421extern void __cmpxchg_called_with_bad_pointer(void);
422
423static inline unsigned long __cmpxchg(volatile void * ptr, unsigned long old,
424 unsigned long new, int size)
425{
426 switch (size) {
427 case 4:
428 return __cmpxchg_u32(ptr, old, new);
429 case 8:
430 return __cmpxchg_u64(ptr, old, new);
431 }
432 __cmpxchg_called_with_bad_pointer();
433 return old;
434}
435
436static inline unsigned long __cmpxchg_local(volatile void * ptr,
437 unsigned long old, unsigned long new, int size)
438{
439 switch (size) {
440 case 4:
441 return __cmpxchg_u32_local(ptr, old, new);
442 case 8:
443 return __cmpxchg_u64_local(ptr, old, new);
444 }
445 __cmpxchg_called_with_bad_pointer();
446 return old;
447}
448
449#define cmpxchg(ptr,old,new) \
450 ((__typeof__(*(ptr)))__cmpxchg((ptr), \
451 (unsigned long)(old), (unsigned long)(new),sizeof(*(ptr))))
452
453#define cmpxchg_local(ptr,old,new) \
454 ((__typeof__(*(ptr)))__cmpxchg_local((ptr), \
455 (unsigned long)(old), (unsigned long)(new),sizeof(*(ptr))))
456
457extern void set_handler (unsigned long offset, void *addr, unsigned long len); 198extern void set_handler (unsigned long offset, void *addr, unsigned long len);
458extern void set_uncached_handler (unsigned long offset, void *addr, unsigned long len); 199extern void set_uncached_handler (unsigned long offset, void *addr, unsigned long len);
459 200
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-30 15:06:29 -0500