aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/mips/lib/memcpy.S158
1 files changed, 89 insertions, 69 deletions
diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S
index c5c40dad0bbf..ab9f04641640 100644
--- a/arch/mips/lib/memcpy.S
+++ b/arch/mips/lib/memcpy.S
@@ -10,6 +10,7 @@
10 * Copyright (C) 2002 Broadcom, Inc. 10 * Copyright (C) 2002 Broadcom, Inc.
11 * memcpy/copy_user author: Mark Vandevoorde 11 * memcpy/copy_user author: Mark Vandevoorde
12 * Copyright (C) 2007 Maciej W. Rozycki 12 * Copyright (C) 2007 Maciej W. Rozycki
13 * Copyright (C) 2014 Imagination Technologies Ltd.
13 * 14 *
14 * Mnemonic names for arguments to memcpy/__copy_user 15 * Mnemonic names for arguments to memcpy/__copy_user
15 */ 16 */
@@ -85,8 +86,22 @@
85 * they're not protected. 86 * they're not protected.
86 */ 87 */
87 88
88#define EXC(inst_reg,addr,handler) \ 89/* Instruction type */
899: inst_reg, addr; \ 90#define LD_INSN 1
91#define ST_INSN 2
92
93/*
94 * Wrapper to add an entry in the exception table
95 * in case the insn causes a memory exception.
96 * Arguments:
97 * insn : Load/store instruction
98 * type : Instruction type
99 * reg : Register
100 * addr : Address
101 * handler : Exception handler
102 */
103#define EXC(insn, type, reg, addr, handler) \
1049: insn reg, addr; \
90 .section __ex_table,"a"; \ 105 .section __ex_table,"a"; \
91 PTR 9b, handler; \ 106 PTR 9b, handler; \
92 .previous 107 .previous
@@ -100,12 +115,13 @@
100 115
101#ifdef USE_DOUBLE 116#ifdef USE_DOUBLE
102 117
103#define LOAD ld 118#define LOADK ld /* No exception */
104#define LOADL ldl 119#define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
105#define LOADR ldr 120#define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
106#define STOREL sdl 121#define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
107#define STORER sdr 122#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
108#define STORE sd 123#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
124#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
109#define ADD daddu 125#define ADD daddu
110#define SUB dsubu 126#define SUB dsubu
111#define SRL dsrl 127#define SRL dsrl
@@ -136,12 +152,13 @@
136 152
137#else 153#else
138 154
139#define LOAD lw 155#define LOADK lw /* No exception */
140#define LOADL lwl 156#define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
141#define LOADR lwr 157#define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
142#define STOREL swl 158#define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
143#define STORER swr 159#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
144#define STORE sw 160#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
161#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
145#define ADD addu 162#define ADD addu
146#define SUB subu 163#define SUB subu
147#define SRL srl 164#define SRL srl
@@ -154,6 +171,9 @@
154 171
155#endif /* USE_DOUBLE */ 172#endif /* USE_DOUBLE */
156 173
174#define LOADB(reg, addr, handler) EXC(lb, LD_INSN, reg, addr, handler)
175#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
176
157#ifdef CONFIG_CPU_LITTLE_ENDIAN 177#ifdef CONFIG_CPU_LITTLE_ENDIAN
158#define LDFIRST LOADR 178#define LDFIRST LOADR
159#define LDREST LOADL 179#define LDREST LOADL
@@ -243,25 +263,25 @@ __copy_user_common:
243 .align 4 263 .align 4
2441: 2641:
245 R10KCBARRIER(0(ra)) 265 R10KCBARRIER(0(ra))
246EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 266 LOAD(t0, UNIT(0)(src), .Ll_exc)
247EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 267 LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
248EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 268 LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
249EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 269 LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
250 SUB len, len, 8*NBYTES 270 SUB len, len, 8*NBYTES
251EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy) 271 LOAD(t4, UNIT(4)(src), .Ll_exc_copy)
252EXC( LOAD t7, UNIT(5)(src), .Ll_exc_copy) 272 LOAD(t7, UNIT(5)(src), .Ll_exc_copy)
253EXC( STORE t0, UNIT(0)(dst), .Ls_exc_p8u) 273 STORE(t0, UNIT(0)(dst), .Ls_exc_p8u)
254EXC( STORE t1, UNIT(1)(dst), .Ls_exc_p7u) 274 STORE(t1, UNIT(1)(dst), .Ls_exc_p7u)
255EXC( LOAD t0, UNIT(6)(src), .Ll_exc_copy) 275 LOAD(t0, UNIT(6)(src), .Ll_exc_copy)
256EXC( LOAD t1, UNIT(7)(src), .Ll_exc_copy) 276 LOAD(t1, UNIT(7)(src), .Ll_exc_copy)
257 ADD src, src, 8*NBYTES 277 ADD src, src, 8*NBYTES
258 ADD dst, dst, 8*NBYTES 278 ADD dst, dst, 8*NBYTES
259EXC( STORE t2, UNIT(-6)(dst), .Ls_exc_p6u) 279 STORE(t2, UNIT(-6)(dst), .Ls_exc_p6u)
260EXC( STORE t3, UNIT(-5)(dst), .Ls_exc_p5u) 280 STORE(t3, UNIT(-5)(dst), .Ls_exc_p5u)
261EXC( STORE t4, UNIT(-4)(dst), .Ls_exc_p4u) 281 STORE(t4, UNIT(-4)(dst), .Ls_exc_p4u)
262EXC( STORE t7, UNIT(-3)(dst), .Ls_exc_p3u) 282 STORE(t7, UNIT(-3)(dst), .Ls_exc_p3u)
263EXC( STORE t0, UNIT(-2)(dst), .Ls_exc_p2u) 283 STORE(t0, UNIT(-2)(dst), .Ls_exc_p2u)
264EXC( STORE t1, UNIT(-1)(dst), .Ls_exc_p1u) 284 STORE(t1, UNIT(-1)(dst), .Ls_exc_p1u)
265 PREF( 0, 8*32(src) ) 285 PREF( 0, 8*32(src) )
266 PREF( 1, 8*32(dst) ) 286 PREF( 1, 8*32(dst) )
267 bne len, rem, 1b 287 bne len, rem, 1b
@@ -278,17 +298,17 @@ EXC( STORE t1, UNIT(-1)(dst), .Ls_exc_p1u)
278 /* 298 /*
279 * len >= 4*NBYTES 299 * len >= 4*NBYTES
280 */ 300 */
281EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 301 LOAD( t0, UNIT(0)(src), .Ll_exc)
282EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 302 LOAD( t1, UNIT(1)(src), .Ll_exc_copy)
283EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 303 LOAD( t2, UNIT(2)(src), .Ll_exc_copy)
284EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 304 LOAD( t3, UNIT(3)(src), .Ll_exc_copy)
285 SUB len, len, 4*NBYTES 305 SUB len, len, 4*NBYTES
286 ADD src, src, 4*NBYTES 306 ADD src, src, 4*NBYTES
287 R10KCBARRIER(0(ra)) 307 R10KCBARRIER(0(ra))
288EXC( STORE t0, UNIT(0)(dst), .Ls_exc_p4u) 308 STORE(t0, UNIT(0)(dst), .Ls_exc_p4u)
289EXC( STORE t1, UNIT(1)(dst), .Ls_exc_p3u) 309 STORE(t1, UNIT(1)(dst), .Ls_exc_p3u)
290EXC( STORE t2, UNIT(2)(dst), .Ls_exc_p2u) 310 STORE(t2, UNIT(2)(dst), .Ls_exc_p2u)
291EXC( STORE t3, UNIT(3)(dst), .Ls_exc_p1u) 311 STORE(t3, UNIT(3)(dst), .Ls_exc_p1u)
292 .set reorder /* DADDI_WAR */ 312 .set reorder /* DADDI_WAR */
293 ADD dst, dst, 4*NBYTES 313 ADD dst, dst, 4*NBYTES
294 beqz len, .Ldone 314 beqz len, .Ldone
@@ -301,10 +321,10 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc_p1u)
301 nop 321 nop
3021: 3221:
303 R10KCBARRIER(0(ra)) 323 R10KCBARRIER(0(ra))
304EXC( LOAD t0, 0(src), .Ll_exc) 324 LOAD(t0, 0(src), .Ll_exc)
305 ADD src, src, NBYTES 325 ADD src, src, NBYTES
306 SUB len, len, NBYTES 326 SUB len, len, NBYTES
307EXC( STORE t0, 0(dst), .Ls_exc_p1u) 327 STORE(t0, 0(dst), .Ls_exc_p1u)
308 .set reorder /* DADDI_WAR */ 328 .set reorder /* DADDI_WAR */
309 ADD dst, dst, NBYTES 329 ADD dst, dst, NBYTES
310 bne rem, len, 1b 330 bne rem, len, 1b
@@ -326,10 +346,10 @@ EXC( STORE t0, 0(dst), .Ls_exc_p1u)
326 ADD t1, dst, len # t1 is just past last byte of dst 346 ADD t1, dst, len # t1 is just past last byte of dst
327 li bits, 8*NBYTES 347 li bits, 8*NBYTES
328 SLL rem, len, 3 # rem = number of bits to keep 348 SLL rem, len, 3 # rem = number of bits to keep
329EXC( LOAD t0, 0(src), .Ll_exc) 349 LOAD(t0, 0(src), .Ll_exc)
330 SUB bits, bits, rem # bits = number of bits to discard 350 SUB bits, bits, rem # bits = number of bits to discard
331 SHIFT_DISCARD t0, t0, bits 351 SHIFT_DISCARD t0, t0, bits
332EXC( STREST t0, -1(t1), .Ls_exc) 352 STREST(t0, -1(t1), .Ls_exc)
333 jr ra 353 jr ra
334 move len, zero 354 move len, zero
335.Ldst_unaligned: 355.Ldst_unaligned:
@@ -343,13 +363,13 @@ EXC( STREST t0, -1(t1), .Ls_exc)
343 * Set match = (src and dst have same alignment) 363 * Set match = (src and dst have same alignment)
344 */ 364 */
345#define match rem 365#define match rem
346EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc) 366 LDFIRST(t3, FIRST(0)(src), .Ll_exc)
347 ADD t2, zero, NBYTES 367 ADD t2, zero, NBYTES
348EXC( LDREST t3, REST(0)(src), .Ll_exc_copy) 368 LDREST(t3, REST(0)(src), .Ll_exc_copy)
349 SUB t2, t2, t1 # t2 = number of bytes copied 369 SUB t2, t2, t1 # t2 = number of bytes copied
350 xor match, t0, t1 370 xor match, t0, t1
351 R10KCBARRIER(0(ra)) 371 R10KCBARRIER(0(ra))
352EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) 372 STFIRST(t3, FIRST(0)(dst), .Ls_exc)
353 beq len, t2, .Ldone 373 beq len, t2, .Ldone
354 SUB len, len, t2 374 SUB len, len, t2
355 ADD dst, dst, t2 375 ADD dst, dst, t2
@@ -370,24 +390,24 @@ EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc)
370 * are to the same unit (unless src is aligned, but it's not). 390 * are to the same unit (unless src is aligned, but it's not).
371 */ 391 */
372 R10KCBARRIER(0(ra)) 392 R10KCBARRIER(0(ra))
373EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 393 LDFIRST(t0, FIRST(0)(src), .Ll_exc)
374EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy) 394 LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)
375 SUB len, len, 4*NBYTES 395 SUB len, len, 4*NBYTES
376EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 396 LDREST(t0, REST(0)(src), .Ll_exc_copy)
377EXC( LDREST t1, REST(1)(src), .Ll_exc_copy) 397 LDREST(t1, REST(1)(src), .Ll_exc_copy)
378EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy) 398 LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)
379EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy) 399 LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)
380EXC( LDREST t2, REST(2)(src), .Ll_exc_copy) 400 LDREST(t2, REST(2)(src), .Ll_exc_copy)
381EXC( LDREST t3, REST(3)(src), .Ll_exc_copy) 401 LDREST(t3, REST(3)(src), .Ll_exc_copy)
382 PREF( 0, 9*32(src) ) # 0 is PREF_LOAD (not streamed) 402 PREF( 0, 9*32(src) ) # 0 is PREF_LOAD (not streamed)
383 ADD src, src, 4*NBYTES 403 ADD src, src, 4*NBYTES
384#ifdef CONFIG_CPU_SB1 404#ifdef CONFIG_CPU_SB1
385 nop # improves slotting 405 nop # improves slotting
386#endif 406#endif
387EXC( STORE t0, UNIT(0)(dst), .Ls_exc_p4u) 407 STORE(t0, UNIT(0)(dst), .Ls_exc_p4u)
388EXC( STORE t1, UNIT(1)(dst), .Ls_exc_p3u) 408 STORE(t1, UNIT(1)(dst), .Ls_exc_p3u)
389EXC( STORE t2, UNIT(2)(dst), .Ls_exc_p2u) 409 STORE(t2, UNIT(2)(dst), .Ls_exc_p2u)
390EXC( STORE t3, UNIT(3)(dst), .Ls_exc_p1u) 410 STORE(t3, UNIT(3)(dst), .Ls_exc_p1u)
391 PREF( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed) 411 PREF( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed)
392 .set reorder /* DADDI_WAR */ 412 .set reorder /* DADDI_WAR */
393 ADD dst, dst, 4*NBYTES 413 ADD dst, dst, 4*NBYTES
@@ -401,11 +421,11 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc_p1u)
401 nop 421 nop
4021: 4221:
403 R10KCBARRIER(0(ra)) 423 R10KCBARRIER(0(ra))
404EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 424 LDFIRST(t0, FIRST(0)(src), .Ll_exc)
405EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 425 LDREST(t0, REST(0)(src), .Ll_exc_copy)
406 ADD src, src, NBYTES 426 ADD src, src, NBYTES
407 SUB len, len, NBYTES 427 SUB len, len, NBYTES
408EXC( STORE t0, 0(dst), .Ls_exc_p1u) 428 STORE(t0, 0(dst), .Ls_exc_p1u)
409 .set reorder /* DADDI_WAR */ 429 .set reorder /* DADDI_WAR */
410 ADD dst, dst, NBYTES 430 ADD dst, dst, NBYTES
411 bne len, rem, 1b 431 bne len, rem, 1b
@@ -418,10 +438,10 @@ EXC( STORE t0, 0(dst), .Ls_exc_p1u)
418 /* 0 < len < NBYTES */ 438 /* 0 < len < NBYTES */
419 R10KCBARRIER(0(ra)) 439 R10KCBARRIER(0(ra))
420#define COPY_BYTE(N) \ 440#define COPY_BYTE(N) \
421EXC( lb t0, N(src), .Ll_exc); \ 441 LOADB(t0, N(src), .Ll_exc); \
422 SUB len, len, 1; \ 442 SUB len, len, 1; \
423 beqz len, .Ldone; \ 443 beqz len, .Ldone; \
424EXC( sb t0, N(dst), .Ls_exc_p1) 444 STOREB(t0, N(dst), .Ls_exc_p1)
425 445
426 COPY_BYTE(0) 446 COPY_BYTE(0)
427 COPY_BYTE(1) 447 COPY_BYTE(1)
@@ -431,10 +451,10 @@ EXC( sb t0, N(dst), .Ls_exc_p1)
431 COPY_BYTE(4) 451 COPY_BYTE(4)
432 COPY_BYTE(5) 452 COPY_BYTE(5)
433#endif 453#endif
434EXC( lb t0, NBYTES-2(src), .Ll_exc) 454 LOADB(t0, NBYTES-2(src), .Ll_exc)
435 SUB len, len, 1 455 SUB len, len, 1
436 jr ra 456 jr ra
437EXC( sb t0, NBYTES-2(dst), .Ls_exc_p1) 457 STOREB(t0, NBYTES-2(dst), .Ls_exc_p1)
438.Ldone: 458.Ldone:
439 jr ra 459 jr ra
440 nop 460 nop
@@ -451,11 +471,11 @@ EXC( sb t0, NBYTES-2(dst), .Ls_exc_p1)
451 * 471 *
452 * Assumes src < THREAD_BUADDR($28) 472 * Assumes src < THREAD_BUADDR($28)
453 */ 473 */
454 LOAD t0, TI_TASK($28) 474 LOADK t0, TI_TASK($28)
455 nop 475 nop
456 LOAD t0, THREAD_BUADDR(t0) 476 LOADK t0, THREAD_BUADDR(t0)
4571: 4771:
458EXC( lb t1, 0(src), .Ll_exc) 478 LOADB(t1, 0(src), .Ll_exc)
459 ADD src, src, 1 479 ADD src, src, 1
460 sb t1, 0(dst) # can't fault -- we're copy_from_user 480 sb t1, 0(dst) # can't fault -- we're copy_from_user
461 .set reorder /* DADDI_WAR */ 481 .set reorder /* DADDI_WAR */
@@ -463,9 +483,9 @@ EXC( lb t1, 0(src), .Ll_exc)
463 bne src, t0, 1b 483 bne src, t0, 1b
464 .set noreorder 484 .set noreorder
465.Ll_exc: 485.Ll_exc:
466 LOAD t0, TI_TASK($28) 486 LOADK t0, TI_TASK($28)
467 nop 487 nop
468 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address 488 LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
469 nop 489 nop
470 SUB len, AT, t0 # len number of uncopied bytes 490 SUB len, AT, t0 # len number of uncopied bytes
471 bnez t6, .Ldone /* Skip the zeroing part if inatomic */ 491 bnez t6, .Ldone /* Skip the zeroing part if inatomic */
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-16 15:30:03 -0500