aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips/lib/csum_partial.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/lib/csum_partial.S')
-rw-r--r--arch/mips/lib/csum_partial.S282
1 files changed, 174 insertions, 108 deletions
diff --git a/arch/mips/lib/csum_partial.S b/arch/mips/lib/csum_partial.S
index a6adffbb4e5f..2e4825e48388 100644
--- a/arch/mips/lib/csum_partial.S
+++ b/arch/mips/lib/csum_partial.S
@@ -8,6 +8,7 @@
8 * Copyright (C) 1998, 1999 Ralf Baechle 8 * Copyright (C) 1998, 1999 Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc. 9 * Copyright (C) 1999 Silicon Graphics, Inc.
10 * Copyright (C) 2007 Maciej W. Rozycki 10 * Copyright (C) 2007 Maciej W. Rozycki
11 * Copyright (C) 2014 Imagination Technologies Ltd.
11 */ 12 */
12#include <linux/errno.h> 13#include <linux/errno.h>
13#include <asm/asm.h> 14#include <asm/asm.h>
@@ -296,7 +297,7 @@ LEAF(csum_partial)
296 * checksum and copy routines based on memcpy.S 297 * checksum and copy routines based on memcpy.S
297 * 298 *
298 * csum_partial_copy_nocheck(src, dst, len, sum) 299 * csum_partial_copy_nocheck(src, dst, len, sum)
299 * __csum_partial_copy_user(src, dst, len, sum, errp) 300 * __csum_partial_copy_kernel(src, dst, len, sum, errp)
300 * 301 *
301 * See "Spec" in memcpy.S for details. Unlike __copy_user, all 302 * See "Spec" in memcpy.S for details. Unlike __copy_user, all
302 * function in this file use the standard calling convention. 303 * function in this file use the standard calling convention.
@@ -327,20 +328,58 @@ LEAF(csum_partial)
327 * These handlers do not need to overwrite any data. 328 * These handlers do not need to overwrite any data.
328 */ 329 */
329 330
330#define EXC(inst_reg,addr,handler) \ 331/* Instruction type */
3319: inst_reg, addr; \ 332#define LD_INSN 1
332 .section __ex_table,"a"; \ 333#define ST_INSN 2
333 PTR 9b, handler; \ 334#define LEGACY_MODE 1
334 .previous 335#define EVA_MODE 2
336#define USEROP 1
337#define KERNELOP 2
338
339/*
340 * Wrapper to add an entry in the exception table
341 * in case the insn causes a memory exception.
342 * Arguments:
343 * insn : Load/store instruction
344 * type : Instruction type
345 * reg : Register
346 * addr : Address
347 * handler : Exception handler
348 */
349#define EXC(insn, type, reg, addr, handler) \
350 .if \mode == LEGACY_MODE; \
3519: insn reg, addr; \
352 .section __ex_table,"a"; \
353 PTR 9b, handler; \
354 .previous; \
355 /* This is enabled in EVA mode */ \
356 .else; \
357 /* If loading from user or storing to user */ \
358 .if ((\from == USEROP) && (type == LD_INSN)) || \
359 ((\to == USEROP) && (type == ST_INSN)); \
3609: __BUILD_EVA_INSN(insn##e, reg, addr); \
361 .section __ex_table,"a"; \
362 PTR 9b, handler; \
363 .previous; \
364 .else; \
365 /* EVA without exception */ \
366 insn reg, addr; \
367 .endif; \
368 .endif
369
370#undef LOAD
335 371
336#ifdef USE_DOUBLE 372#ifdef USE_DOUBLE
337 373
338#define LOAD ld 374#define LOADK ld /* No exception */
339#define LOADL ldl 375#define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
340#define LOADR ldr 376#define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
341#define STOREL sdl 377#define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
342#define STORER sdr 378#define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
343#define STORE sd 379#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
380#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
381#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
382#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
344#define ADD daddu 383#define ADD daddu
345#define SUB dsubu 384#define SUB dsubu
346#define SRL dsrl 385#define SRL dsrl
@@ -352,12 +391,15 @@ LEAF(csum_partial)
352 391
353#else 392#else
354 393
355#define LOAD lw 394#define LOADK lw /* No exception */
356#define LOADL lwl 395#define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
357#define LOADR lwr 396#define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
358#define STOREL swl 397#define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
359#define STORER swr 398#define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
360#define STORE sw 399#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
400#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
401#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
402#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
361#define ADD addu 403#define ADD addu
362#define SUB subu 404#define SUB subu
363#define SRL srl 405#define SRL srl
@@ -396,14 +438,20 @@ LEAF(csum_partial)
396 .set at=v1 438 .set at=v1
397#endif 439#endif
398 440
399LEAF(__csum_partial_copy_user) 441 .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to, __nocheck
442
400 PTR_ADDU AT, src, len /* See (1) above. */ 443 PTR_ADDU AT, src, len /* See (1) above. */
444 /* initialize __nocheck if this the first time we execute this
445 * macro
446 */
401#ifdef CONFIG_64BIT 447#ifdef CONFIG_64BIT
402 move errptr, a4 448 move errptr, a4
403#else 449#else
404 lw errptr, 16(sp) 450 lw errptr, 16(sp)
405#endif 451#endif
406FEXPORT(csum_partial_copy_nocheck) 452 .if \__nocheck == 1
453 FEXPORT(csum_partial_copy_nocheck)
454 .endif
407 move sum, zero 455 move sum, zero
408 move odd, zero 456 move odd, zero
409 /* 457 /*
@@ -419,48 +467,48 @@ FEXPORT(csum_partial_copy_nocheck)
419 */ 467 */
420 sltu t2, len, NBYTES 468 sltu t2, len, NBYTES
421 and t1, dst, ADDRMASK 469 and t1, dst, ADDRMASK
422 bnez t2, .Lcopy_bytes_checklen 470 bnez t2, .Lcopy_bytes_checklen\@
423 and t0, src, ADDRMASK 471 and t0, src, ADDRMASK
424 andi odd, dst, 0x1 /* odd buffer? */ 472 andi odd, dst, 0x1 /* odd buffer? */
425 bnez t1, .Ldst_unaligned 473 bnez t1, .Ldst_unaligned\@
426 nop 474 nop
427 bnez t0, .Lsrc_unaligned_dst_aligned 475 bnez t0, .Lsrc_unaligned_dst_aligned\@
428 /* 476 /*
429 * use delay slot for fall-through 477 * use delay slot for fall-through
430 * src and dst are aligned; need to compute rem 478 * src and dst are aligned; need to compute rem
431 */ 479 */
432.Lboth_aligned: 480.Lboth_aligned\@:
433 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter 481 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
434 beqz t0, .Lcleanup_both_aligned # len < 8*NBYTES 482 beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
435 nop 483 nop
436 SUB len, 8*NBYTES # subtract here for bgez loop 484 SUB len, 8*NBYTES # subtract here for bgez loop
437 .align 4 485 .align 4
4381: 4861:
439EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 487 LOAD(t0, UNIT(0)(src), .Ll_exc\@)
440EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 488 LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
441EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 489 LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
442EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 490 LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
443EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy) 491 LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
444EXC( LOAD t5, UNIT(5)(src), .Ll_exc_copy) 492 LOAD(t5, UNIT(5)(src), .Ll_exc_copy\@)
445EXC( LOAD t6, UNIT(6)(src), .Ll_exc_copy) 493 LOAD(t6, UNIT(6)(src), .Ll_exc_copy\@)
446EXC( LOAD t7, UNIT(7)(src), .Ll_exc_copy) 494 LOAD(t7, UNIT(7)(src), .Ll_exc_copy\@)
447 SUB len, len, 8*NBYTES 495 SUB len, len, 8*NBYTES
448 ADD src, src, 8*NBYTES 496 ADD src, src, 8*NBYTES
449EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 497 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
450 ADDC(sum, t0) 498 ADDC(sum, t0)
451EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 499 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
452 ADDC(sum, t1) 500 ADDC(sum, t1)
453EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 501 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
454 ADDC(sum, t2) 502 ADDC(sum, t2)
455EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 503 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
456 ADDC(sum, t3) 504 ADDC(sum, t3)
457EXC( STORE t4, UNIT(4)(dst), .Ls_exc) 505 STORE(t4, UNIT(4)(dst), .Ls_exc\@)
458 ADDC(sum, t4) 506 ADDC(sum, t4)
459EXC( STORE t5, UNIT(5)(dst), .Ls_exc) 507 STORE(t5, UNIT(5)(dst), .Ls_exc\@)
460 ADDC(sum, t5) 508 ADDC(sum, t5)
461EXC( STORE t6, UNIT(6)(dst), .Ls_exc) 509 STORE(t6, UNIT(6)(dst), .Ls_exc\@)
462 ADDC(sum, t6) 510 ADDC(sum, t6)
463EXC( STORE t7, UNIT(7)(dst), .Ls_exc) 511 STORE(t7, UNIT(7)(dst), .Ls_exc\@)
464 ADDC(sum, t7) 512 ADDC(sum, t7)
465 .set reorder /* DADDI_WAR */ 513 .set reorder /* DADDI_WAR */
466 ADD dst, dst, 8*NBYTES 514 ADD dst, dst, 8*NBYTES
@@ -471,44 +519,44 @@ EXC( STORE t7, UNIT(7)(dst), .Ls_exc)
471 /* 519 /*
472 * len == the number of bytes left to copy < 8*NBYTES 520 * len == the number of bytes left to copy < 8*NBYTES
473 */ 521 */
474.Lcleanup_both_aligned: 522.Lcleanup_both_aligned\@:
475#define rem t7 523#define rem t7
476 beqz len, .Ldone 524 beqz len, .Ldone\@
477 sltu t0, len, 4*NBYTES 525 sltu t0, len, 4*NBYTES
478 bnez t0, .Lless_than_4units 526 bnez t0, .Lless_than_4units\@
479 and rem, len, (NBYTES-1) # rem = len % NBYTES 527 and rem, len, (NBYTES-1) # rem = len % NBYTES
480 /* 528 /*
481 * len >= 4*NBYTES 529 * len >= 4*NBYTES
482 */ 530 */
483EXC( LOAD t0, UNIT(0)(src), .Ll_exc) 531 LOAD(t0, UNIT(0)(src), .Ll_exc\@)
484EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) 532 LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
485EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) 533 LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
486EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) 534 LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
487 SUB len, len, 4*NBYTES 535 SUB len, len, 4*NBYTES
488 ADD src, src, 4*NBYTES 536 ADD src, src, 4*NBYTES
489EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 537 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
490 ADDC(sum, t0) 538 ADDC(sum, t0)
491EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 539 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
492 ADDC(sum, t1) 540 ADDC(sum, t1)
493EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 541 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
494 ADDC(sum, t2) 542 ADDC(sum, t2)
495EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 543 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
496 ADDC(sum, t3) 544 ADDC(sum, t3)
497 .set reorder /* DADDI_WAR */ 545 .set reorder /* DADDI_WAR */
498 ADD dst, dst, 4*NBYTES 546 ADD dst, dst, 4*NBYTES
499 beqz len, .Ldone 547 beqz len, .Ldone\@
500 .set noreorder 548 .set noreorder
501.Lless_than_4units: 549.Lless_than_4units\@:
502 /* 550 /*
503 * rem = len % NBYTES 551 * rem = len % NBYTES
504 */ 552 */
505 beq rem, len, .Lcopy_bytes 553 beq rem, len, .Lcopy_bytes\@
506 nop 554 nop
5071: 5551:
508EXC( LOAD t0, 0(src), .Ll_exc) 556 LOAD(t0, 0(src), .Ll_exc\@)
509 ADD src, src, NBYTES 557 ADD src, src, NBYTES
510 SUB len, len, NBYTES 558 SUB len, len, NBYTES
511EXC( STORE t0, 0(dst), .Ls_exc) 559 STORE(t0, 0(dst), .Ls_exc\@)
512 ADDC(sum, t0) 560 ADDC(sum, t0)
513 .set reorder /* DADDI_WAR */ 561 .set reorder /* DADDI_WAR */
514 ADD dst, dst, NBYTES 562 ADD dst, dst, NBYTES
@@ -527,20 +575,20 @@ EXC( STORE t0, 0(dst), .Ls_exc)
527 * more instruction-level parallelism. 575 * more instruction-level parallelism.
528 */ 576 */
529#define bits t2 577#define bits t2
530 beqz len, .Ldone 578 beqz len, .Ldone\@
531 ADD t1, dst, len # t1 is just past last byte of dst 579 ADD t1, dst, len # t1 is just past last byte of dst
532 li bits, 8*NBYTES 580 li bits, 8*NBYTES
533 SLL rem, len, 3 # rem = number of bits to keep 581 SLL rem, len, 3 # rem = number of bits to keep
534EXC( LOAD t0, 0(src), .Ll_exc) 582 LOAD(t0, 0(src), .Ll_exc\@)
535 SUB bits, bits, rem # bits = number of bits to discard 583 SUB bits, bits, rem # bits = number of bits to discard
536 SHIFT_DISCARD t0, t0, bits 584 SHIFT_DISCARD t0, t0, bits
537EXC( STREST t0, -1(t1), .Ls_exc) 585 STREST(t0, -1(t1), .Ls_exc\@)
538 SHIFT_DISCARD_REVERT t0, t0, bits 586 SHIFT_DISCARD_REVERT t0, t0, bits
539 .set reorder 587 .set reorder
540 ADDC(sum, t0) 588 ADDC(sum, t0)
541 b .Ldone 589 b .Ldone\@
542 .set noreorder 590 .set noreorder
543.Ldst_unaligned: 591.Ldst_unaligned\@:
544 /* 592 /*
545 * dst is unaligned 593 * dst is unaligned
546 * t0 = src & ADDRMASK 594 * t0 = src & ADDRMASK
@@ -551,25 +599,25 @@ EXC( STREST t0, -1(t1), .Ls_exc)
551 * Set match = (src and dst have same alignment) 599 * Set match = (src and dst have same alignment)
552 */ 600 */
553#define match rem 601#define match rem
554EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc) 602 LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
555 ADD t2, zero, NBYTES 603 ADD t2, zero, NBYTES
556EXC( LDREST t3, REST(0)(src), .Ll_exc_copy) 604 LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
557 SUB t2, t2, t1 # t2 = number of bytes copied 605 SUB t2, t2, t1 # t2 = number of bytes copied
558 xor match, t0, t1 606 xor match, t0, t1
559EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) 607 STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
560 SLL t4, t1, 3 # t4 = number of bits to discard 608 SLL t4, t1, 3 # t4 = number of bits to discard
561 SHIFT_DISCARD t3, t3, t4 609 SHIFT_DISCARD t3, t3, t4
562 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */ 610 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
563 ADDC(sum, t3) 611 ADDC(sum, t3)
564 beq len, t2, .Ldone 612 beq len, t2, .Ldone\@
565 SUB len, len, t2 613 SUB len, len, t2
566 ADD dst, dst, t2 614 ADD dst, dst, t2
567 beqz match, .Lboth_aligned 615 beqz match, .Lboth_aligned\@
568 ADD src, src, t2 616 ADD src, src, t2
569 617
570.Lsrc_unaligned_dst_aligned: 618.Lsrc_unaligned_dst_aligned\@:
571 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter 619 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
572 beqz t0, .Lcleanup_src_unaligned 620 beqz t0, .Lcleanup_src_unaligned\@
573 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES 621 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
5741: 6221:
575/* 623/*
@@ -578,53 +626,53 @@ EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc)
578 * It's OK to load FIRST(N+1) before REST(N) because the two addresses 626 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
579 * are to the same unit (unless src is aligned, but it's not). 627 * are to the same unit (unless src is aligned, but it's not).
580 */ 628 */
581EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 629 LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
582EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy) 630 LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
583 SUB len, len, 4*NBYTES 631 SUB len, len, 4*NBYTES
584EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 632 LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
585EXC( LDREST t1, REST(1)(src), .Ll_exc_copy) 633 LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
586EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy) 634 LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
587EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy) 635 LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
588EXC( LDREST t2, REST(2)(src), .Ll_exc_copy) 636 LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
589EXC( LDREST t3, REST(3)(src), .Ll_exc_copy) 637 LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
590 ADD src, src, 4*NBYTES 638 ADD src, src, 4*NBYTES
591#ifdef CONFIG_CPU_SB1 639#ifdef CONFIG_CPU_SB1
592 nop # improves slotting 640 nop # improves slotting
593#endif 641#endif
594EXC( STORE t0, UNIT(0)(dst), .Ls_exc) 642 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
595 ADDC(sum, t0) 643 ADDC(sum, t0)
596EXC( STORE t1, UNIT(1)(dst), .Ls_exc) 644 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
597 ADDC(sum, t1) 645 ADDC(sum, t1)
598EXC( STORE t2, UNIT(2)(dst), .Ls_exc) 646 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
599 ADDC(sum, t2) 647 ADDC(sum, t2)
600EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 648 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
601 ADDC(sum, t3) 649 ADDC(sum, t3)
602 .set reorder /* DADDI_WAR */ 650 .set reorder /* DADDI_WAR */
603 ADD dst, dst, 4*NBYTES 651 ADD dst, dst, 4*NBYTES
604 bne len, rem, 1b 652 bne len, rem, 1b
605 .set noreorder 653 .set noreorder
606 654
607.Lcleanup_src_unaligned: 655.Lcleanup_src_unaligned\@:
608 beqz len, .Ldone 656 beqz len, .Ldone\@
609 and rem, len, NBYTES-1 # rem = len % NBYTES 657 and rem, len, NBYTES-1 # rem = len % NBYTES
610 beq rem, len, .Lcopy_bytes 658 beq rem, len, .Lcopy_bytes\@
611 nop 659 nop
6121: 6601:
613EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) 661 LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
614EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) 662 LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
615 ADD src, src, NBYTES 663 ADD src, src, NBYTES
616 SUB len, len, NBYTES 664 SUB len, len, NBYTES
617EXC( STORE t0, 0(dst), .Ls_exc) 665 STORE(t0, 0(dst), .Ls_exc\@)
618 ADDC(sum, t0) 666 ADDC(sum, t0)
619 .set reorder /* DADDI_WAR */ 667 .set reorder /* DADDI_WAR */
620 ADD dst, dst, NBYTES 668 ADD dst, dst, NBYTES
621 bne len, rem, 1b 669 bne len, rem, 1b
622 .set noreorder 670 .set noreorder
623 671
624.Lcopy_bytes_checklen: 672.Lcopy_bytes_checklen\@:
625 beqz len, .Ldone 673 beqz len, .Ldone\@
626 nop 674 nop
627.Lcopy_bytes: 675.Lcopy_bytes\@:
628 /* 0 < len < NBYTES */ 676 /* 0 < len < NBYTES */
629#ifdef CONFIG_CPU_LITTLE_ENDIAN 677#ifdef CONFIG_CPU_LITTLE_ENDIAN
630#define SHIFT_START 0 678#define SHIFT_START 0
@@ -637,12 +685,12 @@ EXC( STORE t0, 0(dst), .Ls_exc)
637 li t3, SHIFT_START # shift 685 li t3, SHIFT_START # shift
638/* use .Ll_exc_copy here to return correct sum on fault */ 686/* use .Ll_exc_copy here to return correct sum on fault */
639#define COPY_BYTE(N) \ 687#define COPY_BYTE(N) \
640EXC( lbu t0, N(src), .Ll_exc_copy); \ 688 LOADBU(t0, N(src), .Ll_exc_copy\@); \
641 SUB len, len, 1; \ 689 SUB len, len, 1; \
642EXC( sb t0, N(dst), .Ls_exc); \ 690 STOREB(t0, N(dst), .Ls_exc\@); \
643 SLLV t0, t0, t3; \ 691 SLLV t0, t0, t3; \
644 addu t3, SHIFT_INC; \ 692 addu t3, SHIFT_INC; \
645 beqz len, .Lcopy_bytes_done; \ 693 beqz len, .Lcopy_bytes_done\@; \
646 or t2, t0 694 or t2, t0
647 695
648 COPY_BYTE(0) 696 COPY_BYTE(0)
@@ -653,14 +701,14 @@ EXC( sb t0, N(dst), .Ls_exc); \
653 COPY_BYTE(4) 701 COPY_BYTE(4)
654 COPY_BYTE(5) 702 COPY_BYTE(5)
655#endif 703#endif
656EXC( lbu t0, NBYTES-2(src), .Ll_exc_copy) 704 LOADBU(t0, NBYTES-2(src), .Ll_exc_copy\@)
657 SUB len, len, 1 705 SUB len, len, 1
658EXC( sb t0, NBYTES-2(dst), .Ls_exc) 706 STOREB(t0, NBYTES-2(dst), .Ls_exc\@)
659 SLLV t0, t0, t3 707 SLLV t0, t0, t3
660 or t2, t0 708 or t2, t0
661.Lcopy_bytes_done: 709.Lcopy_bytes_done\@:
662 ADDC(sum, t2) 710 ADDC(sum, t2)
663.Ldone: 711.Ldone\@:
664 /* fold checksum */ 712 /* fold checksum */
665#ifdef USE_DOUBLE 713#ifdef USE_DOUBLE
666 dsll32 v1, sum, 0 714 dsll32 v1, sum, 0
@@ -689,7 +737,7 @@ EXC( sb t0, NBYTES-2(dst), .Ls_exc)
689 jr ra 737 jr ra
690 .set noreorder 738 .set noreorder
691 739
692.Ll_exc_copy: 740.Ll_exc_copy\@:
693 /* 741 /*
694 * Copy bytes from src until faulting load address (or until a 742 * Copy bytes from src until faulting load address (or until a
695 * lb faults) 743 * lb faults)
@@ -700,11 +748,11 @@ EXC( sb t0, NBYTES-2(dst), .Ls_exc)
700 * 748 *
701 * Assumes src < THREAD_BUADDR($28) 749 * Assumes src < THREAD_BUADDR($28)
702 */ 750 */
703 LOAD t0, TI_TASK($28) 751 LOADK t0, TI_TASK($28)
704 li t2, SHIFT_START 752 li t2, SHIFT_START
705 LOAD t0, THREAD_BUADDR(t0) 753 LOADK t0, THREAD_BUADDR(t0)
7061: 7541:
707EXC( lbu t1, 0(src), .Ll_exc) 755 LOADBU(t1, 0(src), .Ll_exc\@)
708 ADD src, src, 1 756 ADD src, src, 1
709 sb t1, 0(dst) # can't fault -- we're copy_from_user 757 sb t1, 0(dst) # can't fault -- we're copy_from_user
710 SLLV t1, t1, t2 758 SLLV t1, t1, t2
@@ -714,10 +762,10 @@ EXC( lbu t1, 0(src), .Ll_exc)
714 ADD dst, dst, 1 762 ADD dst, dst, 1
715 bne src, t0, 1b 763 bne src, t0, 1b
716 .set noreorder 764 .set noreorder
717.Ll_exc: 765.Ll_exc\@:
718 LOAD t0, TI_TASK($28) 766 LOADK t0, TI_TASK($28)
719 nop 767 nop
720 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address 768 LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
721 nop 769 nop
722 SUB len, AT, t0 # len number of uncopied bytes 770 SUB len, AT, t0 # len number of uncopied bytes
723 /* 771 /*
@@ -733,7 +781,7 @@ EXC( lbu t1, 0(src), .Ll_exc)
733 */ 781 */
734 .set reorder /* DADDI_WAR */ 782 .set reorder /* DADDI_WAR */
735 SUB src, len, 1 783 SUB src, len, 1
736 beqz len, .Ldone 784 beqz len, .Ldone\@
737 .set noreorder 785 .set noreorder
7381: sb zero, 0(dst) 7861: sb zero, 0(dst)
739 ADD dst, dst, 1 787 ADD dst, dst, 1
@@ -748,13 +796,31 @@ EXC( lbu t1, 0(src), .Ll_exc)
748 SUB src, src, v1 796 SUB src, src, v1
749#endif 797#endif
750 li v1, -EFAULT 798 li v1, -EFAULT
751 b .Ldone 799 b .Ldone\@
752 sw v1, (errptr) 800 sw v1, (errptr)
753 801
754.Ls_exc: 802.Ls_exc\@:
755 li v0, -1 /* invalid checksum */ 803 li v0, -1 /* invalid checksum */
756 li v1, -EFAULT 804 li v1, -EFAULT
757 jr ra 805 jr ra
758 sw v1, (errptr) 806 sw v1, (errptr)
759 .set pop 807 .set pop
760 END(__csum_partial_copy_user) 808 .endm
809
810LEAF(__csum_partial_copy_kernel)
811#ifndef CONFIG_EVA
812FEXPORT(__csum_partial_copy_to_user)
813FEXPORT(__csum_partial_copy_from_user)
814#endif
815__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 1
816END(__csum_partial_copy_kernel)
817
818#ifdef CONFIG_EVA
819LEAF(__csum_partial_copy_to_user)
820__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP 0
821END(__csum_partial_copy_to_user)
822
823LEAF(__csum_partial_copy_from_user)
824__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP 0
825END(__csum_partial_copy_from_user)
826#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-06 15:43:03 -0400