MIPS: lib: csum_partial: Add macro to build csum_partial symbols

In preparation for EVA support, we use a macro to build the
__csum_partial_copy_user main code so it can be shared across
multiple implementations. EVA uses the same code but it replaces
the load/store/prefetch instructions with the EVA specific ones
therefore using a macro avoids unnecessary code duplications.

Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>

1 files changed, 108 insertions, 92 deletions

diff --git a/arch/mips/lib/csum_partial.S b/arch/mips/lib/csum_partial.S
index bff5167b59a6..62c8768a59ce 100644
--- a/arch/mips/lib/csum_partial.S
+++ b/arch/mips/lib/csum_partial.S
@@ -331,6 +331,10 @@ LEAF(csum_partial)
 /* Instruction type */
 #define LD_INSN 1
 #define ST_INSN 2
+#define LEGACY_MODE 1
+#define EVA_MODE    2
+#define USEROP   1
+#define KERNELOP 2
 
 /*
  * Wrapper to add an entry in the exception table
@@ -343,10 +347,12 @@ LEAF(csum_partial)
  * handler : Exception handler
  */
 #define EXC(insn, type, reg, addr, handler)     \
-9:      insn reg, addr;                         \
-        .section __ex_table,"a";                \
-        PTR     9b, handler;                    \
-        .previous
+        .if \mode == LEGACY_MODE;               \
+9:              insn reg, addr;                 \
+                .section __ex_table,"a";        \
+                PTR     9b, handler;            \
+                .previous;                      \
+        .endif
 
 #undef LOAD
 
@@ -419,16 +425,20 @@ LEAF(csum_partial)
         .set    at=v1
 #endif
 
-LEAF(__csum_partial_copy_kernel)
-FEXPORT(__csum_partial_copy_to_user)
-FEXPORT(__csum_partial_copy_from_user)
+        .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to, __nocheck
+
         PTR_ADDU        AT, src, len    /* See (1) above. */
+        /* initialize __nocheck if this the first time we execute this
+         * macro
+         */
 #ifdef CONFIG_64BIT
         move    errptr, a4
 #else
         lw      errptr, 16(sp)
 #endif
-FEXPORT(csum_partial_copy_nocheck)
+        .if \__nocheck == 1
+        FEXPORT(csum_partial_copy_nocheck)
+        .endif
         move    sum, zero
         move    odd, zero
         /*
@@ -444,48 +454,48 @@ FEXPORT(csum_partial_copy_nocheck)
          */
         sltu    t2, len, NBYTES
         and     t1, dst, ADDRMASK
-        bnez    t2, .Lcopy_bytes_checklen
+        bnez    t2, .Lcopy_bytes_checklen\@
          and    t0, src, ADDRMASK
         andi    odd, dst, 0x1                   /* odd buffer? */
-        bnez    t1, .Ldst_unaligned
+        bnez    t1, .Ldst_unaligned\@
          nop
-        bnez    t0, .Lsrc_unaligned_dst_aligned
+        bnez    t0, .Lsrc_unaligned_dst_aligned\@
         /*
          * use delay slot for fall-through
          * src and dst are aligned; need to compute rem
          */
-.Lboth_aligned:
+.Lboth_aligned\@:
          SRL    t0, len, LOG_NBYTES+3    # +3 for 8 units/iter
-        beqz    t0, .Lcleanup_both_aligned # len < 8*NBYTES
+        beqz    t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
          nop
         SUB     len, 8*NBYTES           # subtract here for bgez loop
         .align  4
 1:
-        LOAD(t0, UNIT(0)(src), .Ll_exc)
-        LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
-        LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
-        LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
-        LOAD(t4, UNIT(4)(src), .Ll_exc_copy)
-        LOAD(t5, UNIT(5)(src), .Ll_exc_copy)
-        LOAD(t6, UNIT(6)(src), .Ll_exc_copy)
-        LOAD(t7, UNIT(7)(src), .Ll_exc_copy)
+        LOAD(t0, UNIT(0)(src), .Ll_exc\@)
+        LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
+        LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
+        LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
+        LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
+        LOAD(t5, UNIT(5)(src), .Ll_exc_copy\@)
+        LOAD(t6, UNIT(6)(src), .Ll_exc_copy\@)
+        LOAD(t7, UNIT(7)(src), .Ll_exc_copy\@)
         SUB     len, len, 8*NBYTES
         ADD     src, src, 8*NBYTES
-        STORE(t0, UNIT(0)(dst), .Ls_exc)
+        STORE(t0, UNIT(0)(dst), .Ls_exc\@)
         ADDC(sum, t0)
-        STORE(t1, UNIT(1)(dst), .Ls_exc)
+        STORE(t1, UNIT(1)(dst), .Ls_exc\@)
         ADDC(sum, t1)
-        STORE(t2, UNIT(2)(dst), .Ls_exc)
+        STORE(t2, UNIT(2)(dst), .Ls_exc\@)
         ADDC(sum, t2)
-        STORE(t3, UNIT(3)(dst), .Ls_exc)
+        STORE(t3, UNIT(3)(dst), .Ls_exc\@)
         ADDC(sum, t3)
-        STORE(t4, UNIT(4)(dst), .Ls_exc)
+        STORE(t4, UNIT(4)(dst), .Ls_exc\@)
         ADDC(sum, t4)
-        STORE(t5, UNIT(5)(dst), .Ls_exc)
+        STORE(t5, UNIT(5)(dst), .Ls_exc\@)
         ADDC(sum, t5)
-        STORE(t6, UNIT(6)(dst), .Ls_exc)
+        STORE(t6, UNIT(6)(dst), .Ls_exc\@)
         ADDC(sum, t6)
-        STORE(t7, UNIT(7)(dst), .Ls_exc)
+        STORE(t7, UNIT(7)(dst), .Ls_exc\@)
         ADDC(sum, t7)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, 8*NBYTES
@@ -496,44 +506,44 @@ FEXPORT(csum_partial_copy_nocheck)
         /*
          * len == the number of bytes left to copy < 8*NBYTES
          */
-.Lcleanup_both_aligned:
+.Lcleanup_both_aligned\@:
 #define rem t7
-        beqz    len, .Ldone
+        beqz    len, .Ldone\@
          sltu   t0, len, 4*NBYTES
-        bnez    t0, .Lless_than_4units
+        bnez    t0, .Lless_than_4units\@
          and    rem, len, (NBYTES-1)    # rem = len % NBYTES
         /*
          * len >= 4*NBYTES
          */
-        LOAD(t0, UNIT(0)(src), .Ll_exc)
-        LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
-        LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
-        LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
+        LOAD(t0, UNIT(0)(src), .Ll_exc\@)
+        LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
+        LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
+        LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
         SUB     len, len, 4*NBYTES
         ADD     src, src, 4*NBYTES
-        STORE(t0, UNIT(0)(dst), .Ls_exc)
+        STORE(t0, UNIT(0)(dst), .Ls_exc\@)
         ADDC(sum, t0)
-        STORE(t1, UNIT(1)(dst), .Ls_exc)
+        STORE(t1, UNIT(1)(dst), .Ls_exc\@)
         ADDC(sum, t1)
-        STORE(t2, UNIT(2)(dst), .Ls_exc)
+        STORE(t2, UNIT(2)(dst), .Ls_exc\@)
         ADDC(sum, t2)
-        STORE(t3, UNIT(3)(dst), .Ls_exc)
+        STORE(t3, UNIT(3)(dst), .Ls_exc\@)
         ADDC(sum, t3)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, 4*NBYTES
-        beqz    len, .Ldone
+        beqz    len, .Ldone\@
         .set    noreorder
-.Lless_than_4units:
+.Lless_than_4units\@:
         /*
          * rem = len % NBYTES
          */
-        beq     rem, len, .Lcopy_bytes
+        beq     rem, len, .Lcopy_bytes\@
          nop
 1:
-        LOAD(t0, 0(src), .Ll_exc)
+        LOAD(t0, 0(src), .Ll_exc\@)
         ADD     src, src, NBYTES
         SUB     len, len, NBYTES
-        STORE(t0, 0(dst), .Ls_exc)
+        STORE(t0, 0(dst), .Ls_exc\@)
         ADDC(sum, t0)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, NBYTES
@@ -552,20 +562,20 @@ FEXPORT(csum_partial_copy_nocheck)
          * more instruction-level parallelism.
          */
 #define bits t2
-        beqz    len, .Ldone
+        beqz    len, .Ldone\@
          ADD    t1, dst, len    # t1 is just past last byte of dst
         li      bits, 8*NBYTES
         SLL     rem, len, 3     # rem = number of bits to keep
-        LOAD(t0, 0(src), .Ll_exc)
+        LOAD(t0, 0(src), .Ll_exc\@)
         SUB     bits, bits, rem # bits = number of bits to discard
         SHIFT_DISCARD t0, t0, bits
-        STREST(t0, -1(t1), .Ls_exc)
+        STREST(t0, -1(t1), .Ls_exc\@)
         SHIFT_DISCARD_REVERT t0, t0, bits
         .set reorder
         ADDC(sum, t0)
-        b       .Ldone
+        b       .Ldone\@
         .set noreorder
-.Ldst_unaligned:
+.Ldst_unaligned\@:
         /*
          * dst is unaligned
          * t0 = src & ADDRMASK
@@ -576,25 +586,25 @@ FEXPORT(csum_partial_copy_nocheck)
          * Set match = (src and dst have same alignment)
          */
 #define match rem
-        LDFIRST(t3, FIRST(0)(src), .Ll_exc)
+        LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
         ADD     t2, zero, NBYTES
-        LDREST(t3, REST(0)(src), .Ll_exc_copy)
+        LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
         SUB     t2, t2, t1      # t2 = number of bytes copied
         xor     match, t0, t1
-        STFIRST(t3, FIRST(0)(dst), .Ls_exc)
+        STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
         SLL     t4, t1, 3               # t4 = number of bits to discard
         SHIFT_DISCARD t3, t3, t4
         /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
         ADDC(sum, t3)
-        beq     len, t2, .Ldone
+        beq     len, t2, .Ldone\@
          SUB    len, len, t2
         ADD     dst, dst, t2
-        beqz    match, .Lboth_aligned
+        beqz    match, .Lboth_aligned\@
          ADD    src, src, t2
 
-.Lsrc_unaligned_dst_aligned:
+.Lsrc_unaligned_dst_aligned\@:
         SRL     t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
-        beqz    t0, .Lcleanup_src_unaligned
+        beqz    t0, .Lcleanup_src_unaligned\@
          and    rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
 1:
 /*
@@ -603,53 +613,53 @@ FEXPORT(csum_partial_copy_nocheck)
  * It's OK to load FIRST(N+1) before REST(N) because the two addresses
  * are to the same unit (unless src is aligned, but it's not).
  */
-        LDFIRST(t0, FIRST(0)(src), .Ll_exc)
-        LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)
+        LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
+        LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
         SUB     len, len, 4*NBYTES
-        LDREST(t0, REST(0)(src), .Ll_exc_copy)
-        LDREST(t1, REST(1)(src), .Ll_exc_copy)
-        LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)
-        LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)
-        LDREST(t2, REST(2)(src), .Ll_exc_copy)
-        LDREST(t3, REST(3)(src), .Ll_exc_copy)
+        LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
+        LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
+        LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
+        LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
+        LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
+        LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
         ADD     src, src, 4*NBYTES
 #ifdef CONFIG_CPU_SB1
         nop                             # improves slotting
 #endif
-        STORE(t0, UNIT(0)(dst), .Ls_exc)
+        STORE(t0, UNIT(0)(dst), .Ls_exc\@)
         ADDC(sum, t0)
-        STORE(t1, UNIT(1)(dst), .Ls_exc)
+        STORE(t1, UNIT(1)(dst), .Ls_exc\@)
         ADDC(sum, t1)
-        STORE(t2, UNIT(2)(dst), .Ls_exc)
+        STORE(t2, UNIT(2)(dst), .Ls_exc\@)
         ADDC(sum, t2)
-        STORE(t3, UNIT(3)(dst), .Ls_exc)
+        STORE(t3, UNIT(3)(dst), .Ls_exc\@)
         ADDC(sum, t3)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, 4*NBYTES
         bne     len, rem, 1b
         .set    noreorder
 
-.Lcleanup_src_unaligned:
-        beqz    len, .Ldone
+.Lcleanup_src_unaligned\@:
+        beqz    len, .Ldone\@
          and    rem, len, NBYTES-1  # rem = len % NBYTES
-        beq     rem, len, .Lcopy_bytes
+        beq     rem, len, .Lcopy_bytes\@
          nop
 1:
-        LDFIRST(t0, FIRST(0)(src), .Ll_exc)
-        LDREST(t0, REST(0)(src), .Ll_exc_copy)
+        LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
+        LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
         ADD     src, src, NBYTES
         SUB     len, len, NBYTES
-        STORE(t0, 0(dst), .Ls_exc)
+        STORE(t0, 0(dst), .Ls_exc\@)
         ADDC(sum, t0)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, NBYTES
         bne     len, rem, 1b
         .set    noreorder
 
-.Lcopy_bytes_checklen:
-        beqz    len, .Ldone
+.Lcopy_bytes_checklen\@:
+        beqz    len, .Ldone\@
          nop
-.Lcopy_bytes:
+.Lcopy_bytes\@:
         /* 0 < len < NBYTES  */
 #ifdef CONFIG_CPU_LITTLE_ENDIAN
 #define SHIFT_START 0
@@ -662,12 +672,12 @@ FEXPORT(csum_partial_copy_nocheck)
         li      t3, SHIFT_START # shift
 /* use .Ll_exc_copy here to return correct sum on fault */
 #define COPY_BYTE(N)                    \
-        LOADBU(t0, N(src), .Ll_exc_copy);       \
+        LOADBU(t0, N(src), .Ll_exc_copy\@);     \
         SUB     len, len, 1;            \
-        STOREB(t0, N(dst), .Ls_exc);    \
+        STOREB(t0, N(dst), .Ls_exc\@);  \
         SLLV    t0, t0, t3;             \
         addu    t3, SHIFT_INC;          \
-        beqz    len, .Lcopy_bytes_done; \
+        beqz    len, .Lcopy_bytes_done\@; \
          or     t2, t0
 
         COPY_BYTE(0)
@@ -678,14 +688,14 @@ FEXPORT(csum_partial_copy_nocheck)
         COPY_BYTE(4)
         COPY_BYTE(5)
 #endif
-        LOADBU(t0, NBYTES-2(src), .Ll_exc_copy)
+        LOADBU(t0, NBYTES-2(src), .Ll_exc_copy\@)
         SUB     len, len, 1
-        STOREB(t0, NBYTES-2(dst), .Ls_exc)
+        STOREB(t0, NBYTES-2(dst), .Ls_exc\@)
         SLLV    t0, t0, t3
         or      t2, t0
-.Lcopy_bytes_done:
+.Lcopy_bytes_done\@:
         ADDC(sum, t2)
-.Ldone:
+.Ldone\@:
         /* fold checksum */
 #ifdef USE_DOUBLE
         dsll32  v1, sum, 0
@@ -714,7 +724,7 @@ FEXPORT(csum_partial_copy_nocheck)
         jr      ra
         .set noreorder
 
-.Ll_exc_copy:
+.Ll_exc_copy\@:
         /*
          * Copy bytes from src until faulting load address (or until a
          * lb faults)
@@ -729,7 +739,7 @@ FEXPORT(csum_partial_copy_nocheck)
          li     t2, SHIFT_START
         LOADK   t0, THREAD_BUADDR(t0)
 1:
-        LOADBU(t1, 0(src), .Ll_exc)
+        LOADBU(t1, 0(src), .Ll_exc\@)
         ADD     src, src, 1
         sb      t1, 0(dst)      # can't fault -- we're copy_from_user
         SLLV    t1, t1, t2
@@ -739,7 +749,7 @@ FEXPORT(csum_partial_copy_nocheck)
         ADD     dst, dst, 1
         bne     src, t0, 1b
         .set    noreorder
-.Ll_exc:
+.Ll_exc\@:
         LOADK   t0, TI_TASK($28)
          nop
         LOADK   t0, THREAD_BUADDR(t0)   # t0 is just past last good address
@@ -758,7 +768,7 @@ FEXPORT(csum_partial_copy_nocheck)
          */
         .set    reorder                         /* DADDI_WAR */
         SUB     src, len, 1
-        beqz    len, .Ldone
+        beqz    len, .Ldone\@
         .set    noreorder
 1:      sb      zero, 0(dst)
         ADD     dst, dst, 1
@@ -773,13 +783,19 @@ FEXPORT(csum_partial_copy_nocheck)
          SUB    src, src, v1
 #endif
         li      v1, -EFAULT
-        b       .Ldone
+        b       .Ldone\@
          sw     v1, (errptr)
 
-.Ls_exc:
+.Ls_exc\@:
         li      v0, -1 /* invalid checksum */
         li      v1, -EFAULT
         jr      ra
          sw     v1, (errptr)
         .set    pop
-        END(__csum_partial_copy_kernel)
+        .endm
+
+LEAF(__csum_partial_copy_kernel)
+FEXPORT(__csum_partial_copy_to_user)
+FEXPORT(__csum_partial_copy_from_user)
+__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 1
+END(__csum_partial_copy_kernel)