aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips
diff options
context:
space:
mode:
authorMarkos Chandras <markos.chandras@imgtec.com>2014-01-16 12:02:13 -0500
committerRalf Baechle <ralf@linux-mips.org>2014-03-26 18:09:17 -0400
commit2ab82e66483798670e129c48c05d7fc8a39ea996 (patch)
treeaa750e8cbb02aca8f55cd4eee2f10af777a0d5f3 /arch/mips
parentac85227f7637cfb0d811519b8253c454d0d0a159 (diff)
MIPS: lib: csum_partial: Merge EXC and load/store macros
Each load/store macro always adds an entry to the __ex_table using the EXC macro. There are cases where a load instruction may never fail such as when we are sure the load happens in the kernel address space. Therefore, we merge these the EXC and LOADX/STOREX macros into a single one. We also expand the argument list in the EXC macro to make the macro more flexible. The extra 'type' argument is not used by this commit, but it will be used when EVA support is added to memcpy. Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Diffstat (limited to 'arch/mips')
-rw-r--r--arch/mips/lib/csum_partial.S160
1 files changed, 91 insertions, 69 deletions
diff --git a/arch/mips/lib/csum_partial.S b/arch/mips/lib/csum_partial.S
index 5d73d0d704a3..bff5167b59a6 100644
--- a/arch/mips/lib/csum_partial.S
+++ b/arch/mips/lib/csum_partial.S
@@ -328,20 +328,39 @@ LEAF(csum_partial)
328 * These handlers do not need to overwrite any data. 328 * These handlers do not need to overwrite any data.
329 */ 329 */
330 330
331#define EXC(inst_reg,addr,handler) \ 331/* Instruction type */
3329: inst_reg, addr; \ 332#define LD_INSN 1
333#define ST_INSN 2
334
335/*
336 * Wrapper to add an entry in the exception table
337 * in case the insn causes a memory exception.
338 * Arguments:
339 * insn : Load/store instruction
340 * type : Instruction type
341 * reg : Register
342 * addr : Address
343 * handler : Exception handler
344 */
345#define EXC(insn, type, reg, addr, handler) \
3469: insn reg, addr; \
333 .section __ex_table,"a"; \ 347 .section __ex_table,"a"; \
334 PTR 9b, handler; \ 348 PTR 9b, handler; \
335 .previous 349 .previous
336 350
351#undef LOAD
352
337#ifdef USE_DOUBLE 353#ifdef USE_DOUBLE
338 354
339#define LOAD ld 355#define LOADK ld /* No exception */
340#define LOADL ldl 356#define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
341#define LOADR ldr 357#define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
342#define STOREL sdl 358#define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
343#define STORER sdr 359#define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
344#define STORE sd 360#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
361#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
362#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
363#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
345#define ADD daddu 364#define ADD daddu
346#define SUB dsubu 365#define SUB dsubu
347#define SRL dsrl 366#define SRL dsrl
@@ -353,12 +372,15 @@ LEAF(csum_partial)
353 372
354#else 373#else
355 374
356#define LOAD lw 375#define LOADK lw /* No exception */
357#define LOADL lwl 376#define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
358#define LOADR lwr 377#define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
359#define STOREL swl 378#define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
360#define STORER swr 379#define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
361#define STORE sw 380#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
381#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
382#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
383#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
362#define ADD addu 384#define ADD addu
363#define SUB subu 385#define SUB subu
364#define SRL srl 386#define SRL srl
@@ -439,31 +461,31 @@ FEXPORT(csum_partial_copy_nocheck)
439 SUB len, 8*NBYTES # subtract here for bgez loop 461 SUB len, 8*NBYTES # subtract here for bgez loop
440 .align 4 462 .align 4
4411: 4631:
442EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 464 LOAD(t0, UNIT(0)(src), .Ll_exc)
443EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 465 LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
444EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 466 LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
445EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 467 LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
446EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy) 468 LOAD(t4, UNIT(4)(src), .Ll_exc_copy)
447EXC( LOAD t5, UNIT(5)(src), .Ll_exc_copy) 469 LOAD(t5, UNIT(5)(src), .Ll_exc_copy)
448EXC( LOAD t6, UNIT(6)(src), .Ll_exc_copy) 470 LOAD(t6, UNIT(6)(src), .Ll_exc_copy)
449EXC( LOAD t7, UNIT(7)(src), .Ll_exc_copy) 471 LOAD(t7, UNIT(7)(src), .Ll_exc_copy)
450 SUB len, len, 8*NBYTES 472 SUB len, len, 8*NBYTES
451 ADD src, src, 8*NBYTES 473 ADD src, src, 8*NBYTES
452EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 474 STORE(t0, UNIT(0)(dst), .Ls_exc)
453 ADDC(sum, t0) 475 ADDC(sum, t0)
454EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 476 STORE(t1, UNIT(1)(dst), .Ls_exc)
455 ADDC(sum, t1) 477 ADDC(sum, t1)
456EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 478 STORE(t2, UNIT(2)(dst), .Ls_exc)
457 ADDC(sum, t2) 479 ADDC(sum, t2)
458EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 480 STORE(t3, UNIT(3)(dst), .Ls_exc)
459 ADDC(sum, t3) 481 ADDC(sum, t3)
460EXC( STORE t4, UNIT(4)(dst), .Ls_exc) 482 STORE(t4, UNIT(4)(dst), .Ls_exc)
461 ADDC(sum, t4) 483 ADDC(sum, t4)
462EXC( STORE t5, UNIT(5)(dst), .Ls_exc) 484 STORE(t5, UNIT(5)(dst), .Ls_exc)
463 ADDC(sum, t5) 485 ADDC(sum, t5)
464EXC( STORE t6, UNIT(6)(dst), .Ls_exc) 486 STORE(t6, UNIT(6)(dst), .Ls_exc)
465 ADDC(sum, t6) 487 ADDC(sum, t6)
466EXC( STORE t7, UNIT(7)(dst), .Ls_exc) 488 STORE(t7, UNIT(7)(dst), .Ls_exc)
467 ADDC(sum, t7) 489 ADDC(sum, t7)
468 .set reorder /* DADDI_WAR */ 490 .set reorder /* DADDI_WAR */
469 ADD dst, dst, 8*NBYTES 491 ADD dst, dst, 8*NBYTES
@@ -483,19 +505,19 @@ EXC( STORE t7, UNIT(7)(dst), .Ls_exc)
483 /* 505 /*
484 * len >= 4*NBYTES 506 * len >= 4*NBYTES
485 */ 507 */
486EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 508 LOAD(t0, UNIT(0)(src), .Ll_exc)
487EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 509 LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
488EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 510 LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
489EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 511 LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
490 SUB len, len, 4*NBYTES 512 SUB len, len, 4*NBYTES
491 ADD src, src, 4*NBYTES 513 ADD src, src, 4*NBYTES
492EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 514 STORE(t0, UNIT(0)(dst), .Ls_exc)
493 ADDC(sum, t0) 515 ADDC(sum, t0)
494EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 516 STORE(t1, UNIT(1)(dst), .Ls_exc)
495 ADDC(sum, t1) 517 ADDC(sum, t1)
496EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 518 STORE(t2, UNIT(2)(dst), .Ls_exc)
497 ADDC(sum, t2) 519 ADDC(sum, t2)
498EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 520 STORE(t3, UNIT(3)(dst), .Ls_exc)
499 ADDC(sum, t3) 521 ADDC(sum, t3)
500 .set reorder /* DADDI_WAR */ 522 .set reorder /* DADDI_WAR */
501 ADD dst, dst, 4*NBYTES 523 ADD dst, dst, 4*NBYTES
@@ -508,10 +530,10 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
508 beq rem, len, .Lcopy_bytes 530 beq rem, len, .Lcopy_bytes
509 nop 531 nop
5101: 5321:
511EXC( LOAD t0, 0(src), .Ll_exc) 533 LOAD(t0, 0(src), .Ll_exc)
512 ADD src, src, NBYTES 534 ADD src, src, NBYTES
513 SUB len, len, NBYTES 535 SUB len, len, NBYTES
514EXC( STORE t0, 0(dst), .Ls_exc) 536 STORE(t0, 0(dst), .Ls_exc)
515 ADDC(sum, t0) 537 ADDC(sum, t0)
516 .set reorder /* DADDI_WAR */ 538 .set reorder /* DADDI_WAR */
517 ADD dst, dst, NBYTES 539 ADD dst, dst, NBYTES
@@ -534,10 +556,10 @@ EXC( STORE t0, 0(dst), .Ls_exc)
534 ADD t1, dst, len # t1 is just past last byte of dst 556 ADD t1, dst, len # t1 is just past last byte of dst
535 li bits, 8*NBYTES 557 li bits, 8*NBYTES
536 SLL rem, len, 3 # rem = number of bits to keep 558 SLL rem, len, 3 # rem = number of bits to keep
537EXC( LOAD t0, 0(src), .Ll_exc) 559 LOAD(t0, 0(src), .Ll_exc)
538 SUB bits, bits, rem # bits = number of bits to discard 560 SUB bits, bits, rem # bits = number of bits to discard
539 SHIFT_DISCARD t0, t0, bits 561 SHIFT_DISCARD t0, t0, bits
540EXC( STREST t0, -1(t1), .Ls_exc) 562 STREST(t0, -1(t1), .Ls_exc)
541 SHIFT_DISCARD_REVERT t0, t0, bits 563 SHIFT_DISCARD_REVERT t0, t0, bits
542 .set reorder 564 .set reorder
543 ADDC(sum, t0) 565 ADDC(sum, t0)
@@ -554,12 +576,12 @@ EXC( STREST t0, -1(t1), .Ls_exc)
554 * Set match = (src and dst have same alignment) 576 * Set match = (src and dst have same alignment)
555 */ 577 */
556#define match rem 578#define match rem
557EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc) 579 LDFIRST(t3, FIRST(0)(src), .Ll_exc)
558 ADD t2, zero, NBYTES 580 ADD t2, zero, NBYTES
559EXC( LDREST t3, REST(0)(src), .Ll_exc_copy) 581 LDREST(t3, REST(0)(src), .Ll_exc_copy)
560 SUB t2, t2, t1 # t2 = number of bytes copied 582 SUB t2, t2, t1 # t2 = number of bytes copied
561 xor match, t0, t1 583 xor match, t0, t1
562EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) 584 STFIRST(t3, FIRST(0)(dst), .Ls_exc)
563 SLL t4, t1, 3 # t4 = number of bits to discard 585 SLL t4, t1, 3 # t4 = number of bits to discard
564 SHIFT_DISCARD t3, t3, t4 586 SHIFT_DISCARD t3, t3, t4
565 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */ 587 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
@@ -581,26 +603,26 @@ EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc)
581 * It's OK to load FIRST(N+1) before REST(N) because the two addresses 603 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
582 * are to the same unit (unless src is aligned, but it's not). 604 * are to the same unit (unless src is aligned, but it's not).
583 */ 605 */
584EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 606 LDFIRST(t0, FIRST(0)(src), .Ll_exc)
585EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy) 607 LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)
586 SUB len, len, 4*NBYTES 608 SUB len, len, 4*NBYTES
587EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 609 LDREST(t0, REST(0)(src), .Ll_exc_copy)
588EXC( LDREST t1, REST(1)(src), .Ll_exc_copy) 610 LDREST(t1, REST(1)(src), .Ll_exc_copy)
589EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy) 611 LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)
590EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy) 612 LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)
591EXC( LDREST t2, REST(2)(src), .Ll_exc_copy) 613 LDREST(t2, REST(2)(src), .Ll_exc_copy)
592EXC( LDREST t3, REST(3)(src), .Ll_exc_copy) 614 LDREST(t3, REST(3)(src), .Ll_exc_copy)
593 ADD src, src, 4*NBYTES 615 ADD src, src, 4*NBYTES
594#ifdef CONFIG_CPU_SB1 616#ifdef CONFIG_CPU_SB1
595 nop # improves slotting 617 nop # improves slotting
596#endif 618#endif
597EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 619 STORE(t0, UNIT(0)(dst), .Ls_exc)
598 ADDC(sum, t0) 620 ADDC(sum, t0)
599EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 621 STORE(t1, UNIT(1)(dst), .Ls_exc)
600 ADDC(sum, t1) 622 ADDC(sum, t1)
601EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 623 STORE(t2, UNIT(2)(dst), .Ls_exc)
602 ADDC(sum, t2) 624 ADDC(sum, t2)
603EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 625 STORE(t3, UNIT(3)(dst), .Ls_exc)
604 ADDC(sum, t3) 626 ADDC(sum, t3)
605 .set reorder /* DADDI_WAR */ 627 .set reorder /* DADDI_WAR */
606 ADD dst, dst, 4*NBYTES 628 ADD dst, dst, 4*NBYTES
@@ -613,11 +635,11 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
613 beq rem, len, .Lcopy_bytes 635 beq rem, len, .Lcopy_bytes
614 nop 636 nop
6151: 6371:
616EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 638 LDFIRST(t0, FIRST(0)(src), .Ll_exc)
617EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 639 LDREST(t0, REST(0)(src), .Ll_exc_copy)
618 ADD src, src, NBYTES 640 ADD src, src, NBYTES
619 SUB len, len, NBYTES 641 SUB len, len, NBYTES
620EXC( STORE t0, 0(dst), .Ls_exc) 642 STORE(t0, 0(dst), .Ls_exc)
621 ADDC(sum, t0) 643 ADDC(sum, t0)
622 .set reorder /* DADDI_WAR */ 644 .set reorder /* DADDI_WAR */
623 ADD dst, dst, NBYTES 645 ADD dst, dst, NBYTES
@@ -640,9 +662,9 @@ EXC( STORE t0, 0(dst), .Ls_exc)
640 li t3, SHIFT_START # shift 662 li t3, SHIFT_START # shift
641/* use .Ll_exc_copy here to return correct sum on fault */ 663/* use .Ll_exc_copy here to return correct sum on fault */
642#define COPY_BYTE(N) \ 664#define COPY_BYTE(N) \
643EXC( lbu t0, N(src), .Ll_exc_copy); \ 665 LOADBU(t0, N(src), .Ll_exc_copy); \
644 SUB len, len, 1; \ 666 SUB len, len, 1; \
645EXC( sb t0, N(dst), .Ls_exc); \ 667 STOREB(t0, N(dst), .Ls_exc); \
646 SLLV t0, t0, t3; \ 668 SLLV t0, t0, t3; \
647 addu t3, SHIFT_INC; \ 669 addu t3, SHIFT_INC; \
648 beqz len, .Lcopy_bytes_done; \ 670 beqz len, .Lcopy_bytes_done; \
@@ -656,9 +678,9 @@ EXC( sb t0, N(dst), .Ls_exc); \
656 COPY_BYTE(4) 678 COPY_BYTE(4)
657 COPY_BYTE(5) 679 COPY_BYTE(5)
658#endif 680#endif
659EXC( lbu t0, NBYTES-2(src), .Ll_exc_copy) 681 LOADBU(t0, NBYTES-2(src), .Ll_exc_copy)
660 SUB len, len, 1 682 SUB len, len, 1
661EXC( sb t0, NBYTES-2(dst), .Ls_exc) 683 STOREB(t0, NBYTES-2(dst), .Ls_exc)
662 SLLV t0, t0, t3 684 SLLV t0, t0, t3
663 or t2, t0 685 or t2, t0
664.Lcopy_bytes_done: 686.Lcopy_bytes_done:
@@ -703,11 +725,11 @@ EXC( sb t0, NBYTES-2(dst), .Ls_exc)
703 * 725 *
704 * Assumes src < THREAD_BUADDR($28) 726 * Assumes src < THREAD_BUADDR($28)
705 */ 727 */
706 LOAD t0, TI_TASK($28) 728 LOADK t0, TI_TASK($28)
707 li t2, SHIFT_START 729 li t2, SHIFT_START
708 LOAD t0, THREAD_BUADDR(t0) 730 LOADK t0, THREAD_BUADDR(t0)
7091: 7311:
710EXC( lbu t1, 0(src), .Ll_exc) 732 LOADBU(t1, 0(src), .Ll_exc)
711 ADD src, src, 1 733 ADD src, src, 1
712 sb t1, 0(dst) # can't fault -- we're copy_from_user 734 sb t1, 0(dst) # can't fault -- we're copy_from_user
713 SLLV t1, t1, t2 735 SLLV t1, t1, t2
@@ -718,9 +740,9 @@ EXC( lbu t1, 0(src), .Ll_exc)
718 bne src, t0, 1b 740 bne src, t0, 1b
719 .set noreorder 741 .set noreorder
720.Ll_exc: 742.Ll_exc:
721 LOAD t0, TI_TASK($28) 743 LOADK t0, TI_TASK($28)
722 nop 744 nop
723 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address 745 LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
724 nop 746 nop
725 SUB len, AT, t0 # len number of uncopied bytes 747 SUB len, AT, t0 # len number of uncopied bytes
726 /* 748 /*
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-22 23:21:58 -0500