aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips
diff options
context:
space:
mode:
authorMarkos Chandras <markos.chandras@imgtec.com>2014-01-07 07:57:04 -0500
committerRalf Baechle <ralf@linux-mips.org>2014-03-26 18:09:14 -0400
commit5bc05971d353cd10bce5044a1ef60b9d5b504810 (patch)
tree4368ca5ba8281a2c23692d456bacbd14f5888e67 /arch/mips
parentb3c3025b2c5536c2b243f4947cffe58628758020 (diff)
MIPS: lib: memcpy: Merge EXC and load/store macros
Each load/store macro always adds an entry to the __ex_table using the EXC macro. Therefore, these load/store macros are now merged with the EXC one. The argument list is also expanded in order to make the macro more flexible. The extra 'type' argument is not used by this commit, but it will be used when the EVA support is added to the memcpy. Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Diffstat (limited to 'arch/mips')
-rw-r--r--arch/mips/lib/memcpy.S158
1 files changed, 89 insertions, 69 deletions
diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S
index c5c40dad0bbf..ab9f04641640 100644
--- a/arch/mips/lib/memcpy.S
+++ b/arch/mips/lib/memcpy.S
@@ -10,6 +10,7 @@
10 * Copyright (C) 2002 Broadcom, Inc. 10 * Copyright (C) 2002 Broadcom, Inc.
11 * memcpy/copy_user author: Mark Vandevoorde 11 * memcpy/copy_user author: Mark Vandevoorde
12 * Copyright (C) 2007 Maciej W. Rozycki 12 * Copyright (C) 2007 Maciej W. Rozycki
13 * Copyright (C) 2014 Imagination Technologies Ltd.
13 * 14 *
14 * Mnemonic names for arguments to memcpy/__copy_user 15 * Mnemonic names for arguments to memcpy/__copy_user
15 */ 16 */
@@ -85,8 +86,22 @@
85 * they're not protected. 86 * they're not protected.
86 */ 87 */
87 88
88#define EXC(inst_reg,addr,handler) \ 89/* Instruction type */
899: inst_reg, addr; \ 90#define LD_INSN 1
91#define ST_INSN 2
92
93/*
94 * Wrapper to add an entry in the exception table
95 * in case the insn causes a memory exception.
96 * Arguments:
97 * insn : Load/store instruction
98 * type : Instruction type
99 * reg : Register
100 * addr : Address
101 * handler : Exception handler
102 */
103#define EXC(insn, type, reg, addr, handler) \
1049: insn reg, addr; \
90 .section __ex_table,"a"; \ 105 .section __ex_table,"a"; \
91 PTR 9b, handler; \ 106 PTR 9b, handler; \
92 .previous 107 .previous
@@ -100,12 +115,13 @@
100 115
101#ifdef USE_DOUBLE 116#ifdef USE_DOUBLE
102 117
103#define LOAD ld 118#define LOADK ld /* No exception */
104#define LOADL ldl 119#define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
105#define LOADR ldr 120#define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
106#define STOREL sdl 121#define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
107#define STORER sdr 122#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
108#define STORE sd 123#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
124#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
109#define ADD daddu 125#define ADD daddu
110#define SUB dsubu 126#define SUB dsubu
111#define SRL dsrl 127#define SRL dsrl
@@ -136,12 +152,13 @@
136 152
137#else 153#else
138 154
139#define LOAD lw 155#define LOADK lw /* No exception */
140#define LOADL lwl 156#define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
141#define LOADR lwr 157#define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
142#define STOREL swl 158#define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
143#define STORER swr 159#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
144#define STORE sw 160#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
161#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
145#define ADD addu 162#define ADD addu
146#define SUB subu 163#define SUB subu
147#define SRL srl 164#define SRL srl
@@ -154,6 +171,9 @@
154 171
155#endif /* USE_DOUBLE */ 172#endif /* USE_DOUBLE */
156 173
174#define LOADB(reg, addr, handler) EXC(lb, LD_INSN, reg, addr, handler)
175#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
176
157#ifdef CONFIG_CPU_LITTLE_ENDIAN 177#ifdef CONFIG_CPU_LITTLE_ENDIAN
158#define LDFIRST LOADR 178#define LDFIRST LOADR
159#define LDREST LOADL 179#define LDREST LOADL
@@ -243,25 +263,25 @@ __copy_user_common:
243 .align 4 263 .align 4
2441: 2641:
245 R10KCBARRIER(0(ra)) 265 R10KCBARRIER(0(ra))
246EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 266 LOAD(t0, UNIT(0)(src), .Ll_exc)
247EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 267 LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
248EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 268 LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
249EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 269 LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
250 SUB len, len, 8*NBYTES 270 SUB len, len, 8*NBYTES
251EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy) 271 LOAD(t4, UNIT(4)(src), .Ll_exc_copy)
252EXC( LOAD t7, UNIT(5)(src), .Ll_exc_copy) 272 LOAD(t7, UNIT(5)(src), .Ll_exc_copy)
253EXC( STORE t0, UNIT(0)(dst), .Ls_exc_p8u) 273 STORE(t0, UNIT(0)(dst), .Ls_exc_p8u)
254EXC( STORE t1, UNIT(1)(dst), .Ls_exc_p7u) 274 STORE(t1, UNIT(1)(dst), .Ls_exc_p7u)
255EXC( LOAD t0, UNIT(6)(src), .Ll_exc_copy) 275 LOAD(t0, UNIT(6)(src), .Ll_exc_copy)
256EXC( LOAD t1, UNIT(7)(src), .Ll_exc_copy) 276 LOAD(t1, UNIT(7)(src), .Ll_exc_copy)
257 ADD src, src, 8*NBYTES 277 ADD src, src, 8*NBYTES
258 ADD dst, dst, 8*NBYTES 278 ADD dst, dst, 8*NBYTES
259EXC( STORE t2, UNIT(-6)(dst), .Ls_exc_p6u) 279 STORE(t2, UNIT(-6)(dst), .Ls_exc_p6u)
260EXC( STORE t3, UNIT(-5)(dst), .Ls_exc_p5u) 280 STORE(t3, UNIT(-5)(dst), .Ls_exc_p5u)
261EXC( STORE t4, UNIT(-4)(dst), .Ls_exc_p4u) 281 STORE(t4, UNIT(-4)(dst), .Ls_exc_p4u)
262EXC( STORE t7, UNIT(-3)(dst), .Ls_exc_p3u) 282 STORE(t7, UNIT(-3)(dst), .Ls_exc_p3u)
263EXC( STORE t0, UNIT(-2)(dst), .Ls_exc_p2u) 283 STORE(t0, UNIT(-2)(dst), .Ls_exc_p2u)
264EXC( STORE t1, UNIT(-1)(dst), .Ls_exc_p1u) 284 STORE(t1, UNIT(-1)(dst), .Ls_exc_p1u)
265 PREF( 0, 8*32(src) ) 285 PREF( 0, 8*32(src) )
266 PREF( 1, 8*32(dst) ) 286 PREF( 1, 8*32(dst) )
267 bne len, rem, 1b 287 bne len, rem, 1b
@@ -278,17 +298,17 @@ EXC( STORE t1, UNIT(-1)(dst), .Ls_exc_p1u)
278 /* 298 /*
279 * len >= 4*NBYTES 299 * len >= 4*NBYTES
280 */ 300 */
281EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 301 LOAD( t0, UNIT(0)(src), .Ll_exc)
282EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 302 LOAD( t1, UNIT(1)(src), .Ll_exc_copy)
283EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 303 LOAD( t2, UNIT(2)(src), .Ll_exc_copy)
284EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 304 LOAD( t3, UNIT(3)(src), .Ll_exc_copy)
285 SUB len, len, 4*NBYTES 305 SUB len, len, 4*NBYTES
286 ADD src, src, 4*NBYTES 306 ADD src, src, 4*NBYTES
287 R10KCBARRIER(0(ra)) 307 R10KCBARRIER(0(ra))
288EXC( STORE t0, UNIT(0)(dst), .Ls_exc_p4u) 308 STORE(t0, UNIT(0)(dst), .Ls_exc_p4u)
289EXC( STORE t1, UNIT(1)(dst), .Ls_exc_p3u) 309 STORE(t1, UNIT(1)(dst), .Ls_exc_p3u)
290EXC( STORE t2, UNIT(2)(dst), .Ls_exc_p2u) 310 STORE(t2, UNIT(2)(dst), .Ls_exc_p2u)
291EXC( STORE t3, UNIT(3)(dst), .Ls_exc_p1u) 311 STORE(t3, UNIT(3)(dst), .Ls_exc_p1u)
292 .set reorder /* DADDI_WAR */ 312 .set reorder /* DADDI_WAR */
293 ADD dst, dst, 4*NBYTES 313 ADD dst, dst, 4*NBYTES
294 beqz len, .Ldone 314 beqz len, .Ldone
@@ -301,10 +321,10 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc_p1u)
301 nop 321 nop
3021: 3221:
303 R10KCBARRIER(0(ra)) 323 R10KCBARRIER(0(ra))
304EXC( LOAD t0, 0(src), .Ll_exc) 324 LOAD(t0, 0(src), .Ll_exc)
305 ADD src, src, NBYTES 325 ADD src, src, NBYTES
306 SUB len, len, NBYTES 326 SUB len, len, NBYTES
307EXC( STORE t0, 0(dst), .Ls_exc_p1u) 327 STORE(t0, 0(dst), .Ls_exc_p1u)
308 .set reorder /* DADDI_WAR */ 328 .set reorder /* DADDI_WAR */
309 ADD dst, dst, NBYTES 329 ADD dst, dst, NBYTES
310 bne rem, len, 1b 330 bne rem, len, 1b
@@ -326,10 +346,10 @@ EXC( STORE t0, 0(dst), .Ls_exc_p1u)
326 ADD t1, dst, len # t1 is just past last byte of dst 346 ADD t1, dst, len # t1 is just past last byte of dst
327 li bits, 8*NBYTES 347 li bits, 8*NBYTES
328 SLL rem, len, 3 # rem = number of bits to keep 348 SLL rem, len, 3 # rem = number of bits to keep
329EXC( LOAD t0, 0(src), .Ll_exc) 349 LOAD(t0, 0(src), .Ll_exc)
330 SUB bits, bits, rem # bits = number of bits to discard 350 SUB bits, bits, rem # bits = number of bits to discard
331 SHIFT_DISCARD t0, t0, bits 351 SHIFT_DISCARD t0, t0, bits
332EXC( STREST t0, -1(t1), .Ls_exc) 352 STREST(t0, -1(t1), .Ls_exc)
333 jr ra 353 jr ra
334 move len, zero 354 move len, zero
335.Ldst_unaligned: 355.Ldst_unaligned:
@@ -343,13 +363,13 @@ EXC( STREST t0, -1(t1), .Ls_exc)
343 * Set match = (src and dst have same alignment) 363 * Set match = (src and dst have same alignment)
344 */ 364 */
345#define match rem 365#define match rem
346EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc) 366 LDFIRST(t3, FIRST(0)(src), .Ll_exc)
347 ADD t2, zero, NBYTES 367 ADD t2, zero, NBYTES
348EXC( LDREST t3, REST(0)(src), .Ll_exc_copy) 368 LDREST(t3, REST(0)(src), .Ll_exc_copy)
349 SUB t2, t2, t1 # t2 = number of bytes copied 369 SUB t2, t2, t1 # t2 = number of bytes copied
350 xor match, t0, t1 370 xor match, t0, t1
351 R10KCBARRIER(0(ra)) 371 R10KCBARRIER(0(ra))
352EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) 372 STFIRST(t3, FIRST(0)(dst), .Ls_exc)
353 beq len, t2, .Ldone 373 beq len, t2, .Ldone
354 SUB len, len, t2 374 SUB len, len, t2
355 ADD dst, dst, t2 375 ADD dst, dst, t2
@@ -370,24 +390,24 @@ EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc)
370 * are to the same unit (unless src is aligned, but it's not). 390 * are to the same unit (unless src is aligned, but it's not).
371 */ 391 */
372 R10KCBARRIER(0(ra)) 392 R10KCBARRIER(0(ra))
373EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 393 LDFIRST(t0, FIRST(0)(src), .Ll_exc)
374EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy) 394 LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)
375 SUB len, len, 4*NBYTES 395 SUB len, len, 4*NBYTES
376EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 396 LDREST(t0, REST(0)(src), .Ll_exc_copy)
377EXC( LDREST t1, REST(1)(src), .Ll_exc_copy) 397 LDREST(t1, REST(1)(src), .Ll_exc_copy)
378EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy) 398 LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)
379EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy) 399 LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)
380EXC( LDREST t2, REST(2)(src), .Ll_exc_copy) 400 LDREST(t2, REST(2)(src), .Ll_exc_copy)
381EXC( LDREST t3, REST(3)(src), .Ll_exc_copy) 401 LDREST(t3, REST(3)(src), .Ll_exc_copy)
382 PREF( 0, 9*32(src) ) # 0 is PREF_LOAD (not streamed) 402 PREF( 0, 9*32(src) ) # 0 is PREF_LOAD (not streamed)
383 ADD src, src, 4*NBYTES 403 ADD src, src, 4*NBYTES
384#ifdef CONFIG_CPU_SB1 404#ifdef CONFIG_CPU_SB1
385 nop # improves slotting 405 nop # improves slotting
386#endif 406#endif
387EXC( STORE t0, UNIT(0)(dst), .Ls_exc_p4u) 407 STORE(t0, UNIT(0)(dst), .Ls_exc_p4u)
388EXC( STORE t1, UNIT(1)(dst), .Ls_exc_p3u) 408 STORE(t1, UNIT(1)(dst), .Ls_exc_p3u)
389EXC( STORE t2, UNIT(2)(dst), .Ls_exc_p2u) 409 STORE(t2, UNIT(2)(dst), .Ls_exc_p2u)
390EXC( STORE t3, UNIT(3)(dst), .Ls_exc_p1u) 410 STORE(t3, UNIT(3)(dst), .Ls_exc_p1u)
391 PREF( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed) 411 PREF( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed)
392 .set reorder /* DADDI_WAR */ 412 .set reorder /* DADDI_WAR */
393 ADD dst, dst, 4*NBYTES 413 ADD dst, dst, 4*NBYTES
@@ -401,11 +421,11 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc_p1u)
401 nop 421 nop
4021: 4221:
403 R10KCBARRIER(0(ra)) 423 R10KCBARRIER(0(ra))
404EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 424 LDFIRST(t0, FIRST(0)(src), .Ll_exc)
405EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 425 LDREST(t0, REST(0)(src), .Ll_exc_copy)
406 ADD src, src, NBYTES 426 ADD src, src, NBYTES
407 SUB len, len, NBYTES 427 SUB len, len, NBYTES
408EXC( STORE t0, 0(dst), .Ls_exc_p1u) 428 STORE(t0, 0(dst), .Ls_exc_p1u)
409 .set reorder /* DADDI_WAR */ 429 .set reorder /* DADDI_WAR */
410 ADD dst, dst, NBYTES 430 ADD dst, dst, NBYTES
411 bne len, rem, 1b 431 bne len, rem, 1b
@@ -418,10 +438,10 @@ EXC( STORE t0, 0(dst), .Ls_exc_p1u)
418 /* 0 < len < NBYTES */ 438 /* 0 < len < NBYTES */
419 R10KCBARRIER(0(ra)) 439 R10KCBARRIER(0(ra))
420#define COPY_BYTE(N) \ 440#define COPY_BYTE(N) \
421EXC( lb t0, N(src), .Ll_exc); \ 441 LOADB(t0, N(src), .Ll_exc); \
422 SUB len, len, 1; \ 442 SUB len, len, 1; \
423 beqz len, .Ldone; \ 443 beqz len, .Ldone; \
424EXC( sb t0, N(dst), .Ls_exc_p1) 444 STOREB(t0, N(dst), .Ls_exc_p1)
425 445
426 COPY_BYTE(0) 446 COPY_BYTE(0)
427 COPY_BYTE(1) 447 COPY_BYTE(1)
@@ -431,10 +451,10 @@ EXC( sb t0, N(dst), .Ls_exc_p1)
431 COPY_BYTE(4) 451 COPY_BYTE(4)
432 COPY_BYTE(5) 452 COPY_BYTE(5)
433#endif 453#endif
434EXC( lb t0, NBYTES-2(src), .Ll_exc) 454 LOADB(t0, NBYTES-2(src), .Ll_exc)
435 SUB len, len, 1 455 SUB len, len, 1
436 jr ra 456 jr ra
437EXC( sb t0, NBYTES-2(dst), .Ls_exc_p1) 457 STOREB(t0, NBYTES-2(dst), .Ls_exc_p1)
438.Ldone: 458.Ldone:
439 jr ra 459 jr ra
440 nop 460 nop
@@ -451,11 +471,11 @@ EXC( sb t0, NBYTES-2(dst), .Ls_exc_p1)
451 * 471 *
452 * Assumes src < THREAD_BUADDR($28) 472 * Assumes src < THREAD_BUADDR($28)
453 */ 473 */
454 LOAD t0, TI_TASK($28) 474 LOADK t0, TI_TASK($28)
455 nop 475 nop
456 LOAD t0, THREAD_BUADDR(t0) 476 LOADK t0, THREAD_BUADDR(t0)
4571: 4771:
458EXC( lb t1, 0(src), .Ll_exc) 478 LOADB(t1, 0(src), .Ll_exc)
459 ADD src, src, 1 479 ADD src, src, 1
460 sb t1, 0(dst) # can't fault -- we're copy_from_user 480 sb t1, 0(dst) # can't fault -- we're copy_from_user
461 .set reorder /* DADDI_WAR */ 481 .set reorder /* DADDI_WAR */
@@ -463,9 +483,9 @@ EXC( lb t1, 0(src), .Ll_exc)
463 bne src, t0, 1b 483 bne src, t0, 1b
464 .set noreorder 484 .set noreorder
465.Ll_exc: 485.Ll_exc:
466 LOAD t0, TI_TASK($28) 486 LOADK t0, TI_TASK($28)
467 nop 487 nop
468 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address 488 LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
469 nop 489 nop
470 SUB len, AT, t0 # len number of uncopied bytes 490 SUB len, AT, t0 # len number of uncopied bytes
471 bnez t6, .Ldone /* Skip the zeroing part if inatomic */ 491 bnez t6, .Ldone /* Skip the zeroing part if inatomic */
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-22 23:57:30 -0500