MIPS: lib: memcpy: Use macro to build the copy_user code

The code can be shared between EVA and non-EVA configurations,
therefore use a macro to build it to avoid code duplications.

Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>

1 files changed, 143 insertions, 110 deletions

diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S
index eed6e07bf222..d630a28a118b 100644
--- a/arch/mips/lib/memcpy.S
+++ b/arch/mips/lib/memcpy.S
@@ -92,6 +92,10 @@
 /* Pretech type */
 #define SRC_PREFETCH 1
 #define DST_PREFETCH 2
+#define LEGACY_MODE 1
+#define EVA_MODE    2
+#define USEROP   1
+#define KERNELOP 2
 
 /*
  * Wrapper to add an entry in the exception table
@@ -103,12 +107,14 @@
  * addr    : Address
  * handler : Exception handler
  */
-#define EXC(insn, type, reg, addr, handler)     \
-9:      insn reg, addr;                         \
-        .section __ex_table,"a";                \
-        PTR     9b, handler;                    \
-        .previous
 
+#define EXC(insn, type, reg, addr, handler)                     \
+        .if \mode == LEGACY_MODE;                               \
+9:              insn reg, addr;                                 \
+                .section __ex_table,"a";                        \
+                PTR     9b, handler;                            \
+                .previous;                                      \
+        .endif
 /*
  * Only on the 64-bit kernel we can made use of 64-bit registers.
  */
@@ -177,7 +183,10 @@
 #define LOADB(reg, addr, handler)       EXC(lb, LD_INSN, reg, addr, handler)
 #define STOREB(reg, addr, handler)      EXC(sb, ST_INSN, reg, addr, handler)
 
-#define _PREF(hint, addr, type)         PREF(hint, addr)
+#define _PREF(hint, addr, type)                                         \
+        .if \mode == LEGACY_MODE;                                       \
+                PREF(hint, addr);                                       \
+        .endif
 
 #define PREFS(hint, addr) _PREF(hint, addr, SRC_PREFETCH)
 #define PREFD(hint, addr) _PREF(hint, addr, DST_PREFETCH)
@@ -210,27 +219,23 @@
         .set    at=v1
 #endif
 
-/*
- * t6 is used as a flag to note inatomic mode.
- */
-LEAF(__copy_user_inatomic)
-        b       __copy_user_common
-         li     t6, 1
-        END(__copy_user_inatomic)
-
-/*
- * A combined memcpy/__copy_user
- * __copy_user sets len to 0 for success; else to an upper bound of
- * the number of uncopied bytes.
- * memcpy sets v0 to dst.
- */
         .align  5
-LEAF(memcpy)                                    /* a0=dst a1=src a2=len */
-        move    v0, dst                         /* return value */
-.L__memcpy:
-FEXPORT(__copy_user)
-        li      t6, 0   /* not inatomic */
-__copy_user_common:
+
+        /*
+         * Macro to build the __copy_user common code
+         * Arguements:
+         * mode : LEGACY_MODE or EVA_MODE
+         * from : Source operand. USEROP or KERNELOP
+         * to   : Destination operand. USEROP or KERNELOP
+         */
+        .macro __BUILD_COPY_USER mode, from, to
+
+        /* initialize __memcpy if this the first time we execute this macro */
+        .ifnotdef __memcpy
+        .set __memcpy, 1
+        .hidden __memcpy /* make sure it does not leak */
+        .endif
+
         /*
          * Note: dst & src may be unaligned, len may be 0
          * Temps
@@ -251,45 +256,45 @@ __copy_user_common:
         and     t1, dst, ADDRMASK
         PREFS(  0, 1*32(src) )
         PREFD(  1, 1*32(dst) )
-        bnez    t2, .Lcopy_bytes_checklen
+        bnez    t2, .Lcopy_bytes_checklen\@
          and    t0, src, ADDRMASK
         PREFS(  0, 2*32(src) )
         PREFD(  1, 2*32(dst) )
-        bnez    t1, .Ldst_unaligned
+        bnez    t1, .Ldst_unaligned\@
          nop
-        bnez    t0, .Lsrc_unaligned_dst_aligned
+        bnez    t0, .Lsrc_unaligned_dst_aligned\@
         /*
          * use delay slot for fall-through
          * src and dst are aligned; need to compute rem
          */
-.Lboth_aligned:
+.Lboth_aligned\@:
          SRL    t0, len, LOG_NBYTES+3    # +3 for 8 units/iter
-        beqz    t0, .Lcleanup_both_aligned # len < 8*NBYTES
+        beqz    t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
          and    rem, len, (8*NBYTES-1)   # rem = len % (8*NBYTES)
         PREFS(  0, 3*32(src) )
         PREFD(  1, 3*32(dst) )
         .align  4
 1:
         R10KCBARRIER(0(ra))
-        LOAD(t0, UNIT(0)(src), .Ll_exc)
-        LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
-        LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
-        LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
+        LOAD(t0, UNIT(0)(src), .Ll_exc\@)
+        LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
+        LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
+        LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
         SUB     len, len, 8*NBYTES
-        LOAD(t4, UNIT(4)(src), .Ll_exc_copy)
-        LOAD(t7, UNIT(5)(src), .Ll_exc_copy)
-        STORE(t0, UNIT(0)(dst), .Ls_exc_p8u)
-        STORE(t1, UNIT(1)(dst), .Ls_exc_p7u)
-        LOAD(t0, UNIT(6)(src), .Ll_exc_copy)
-        LOAD(t1, UNIT(7)(src), .Ll_exc_copy)
+        LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
+        LOAD(t7, UNIT(5)(src), .Ll_exc_copy\@)
+        STORE(t0, UNIT(0)(dst), .Ls_exc_p8u\@)
+        STORE(t1, UNIT(1)(dst), .Ls_exc_p7u\@)
+        LOAD(t0, UNIT(6)(src), .Ll_exc_copy\@)
+        LOAD(t1, UNIT(7)(src), .Ll_exc_copy\@)
         ADD     src, src, 8*NBYTES
         ADD     dst, dst, 8*NBYTES
-        STORE(t2, UNIT(-6)(dst), .Ls_exc_p6u)
-        STORE(t3, UNIT(-5)(dst), .Ls_exc_p5u)
-        STORE(t4, UNIT(-4)(dst), .Ls_exc_p4u)
-        STORE(t7, UNIT(-3)(dst), .Ls_exc_p3u)
-        STORE(t0, UNIT(-2)(dst), .Ls_exc_p2u)
-        STORE(t1, UNIT(-1)(dst), .Ls_exc_p1u)
+        STORE(t2, UNIT(-6)(dst), .Ls_exc_p6u\@)
+        STORE(t3, UNIT(-5)(dst), .Ls_exc_p5u\@)
+        STORE(t4, UNIT(-4)(dst), .Ls_exc_p4u\@)
+        STORE(t7, UNIT(-3)(dst), .Ls_exc_p3u\@)
+        STORE(t0, UNIT(-2)(dst), .Ls_exc_p2u\@)
+        STORE(t1, UNIT(-1)(dst), .Ls_exc_p1u\@)
         PREFS(  0, 8*32(src) )
         PREFD(  1, 8*32(dst) )
         bne     len, rem, 1b
@@ -298,41 +303,41 @@ __copy_user_common:
         /*
          * len == rem == the number of bytes left to copy < 8*NBYTES
          */
-.Lcleanup_both_aligned:
-        beqz    len, .Ldone
+.Lcleanup_both_aligned\@:
+        beqz    len, .Ldone\@
          sltu   t0, len, 4*NBYTES
-        bnez    t0, .Lless_than_4units
+        bnez    t0, .Lless_than_4units\@
          and    rem, len, (NBYTES-1)    # rem = len % NBYTES
         /*
          * len >= 4*NBYTES
          */
-        LOAD( t0, UNIT(0)(src), .Ll_exc)
-        LOAD( t1, UNIT(1)(src), .Ll_exc_copy)
-        LOAD( t2, UNIT(2)(src), .Ll_exc_copy)
-        LOAD( t3, UNIT(3)(src), .Ll_exc_copy)
+        LOAD( t0, UNIT(0)(src), .Ll_exc\@)
+        LOAD( t1, UNIT(1)(src), .Ll_exc_copy\@)
+        LOAD( t2, UNIT(2)(src), .Ll_exc_copy\@)
+        LOAD( t3, UNIT(3)(src), .Ll_exc_copy\@)
         SUB     len, len, 4*NBYTES
         ADD     src, src, 4*NBYTES
         R10KCBARRIER(0(ra))
-        STORE(t0, UNIT(0)(dst), .Ls_exc_p4u)
-        STORE(t1, UNIT(1)(dst), .Ls_exc_p3u)
-        STORE(t2, UNIT(2)(dst), .Ls_exc_p2u)
-        STORE(t3, UNIT(3)(dst), .Ls_exc_p1u)
+        STORE(t0, UNIT(0)(dst), .Ls_exc_p4u\@)
+        STORE(t1, UNIT(1)(dst), .Ls_exc_p3u\@)
+        STORE(t2, UNIT(2)(dst), .Ls_exc_p2u\@)
+        STORE(t3, UNIT(3)(dst), .Ls_exc_p1u\@)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, 4*NBYTES
-        beqz    len, .Ldone
+        beqz    len, .Ldone\@
         .set    noreorder
-.Lless_than_4units:
+.Lless_than_4units\@:
         /*
          * rem = len % NBYTES
          */
-        beq     rem, len, .Lcopy_bytes
+        beq     rem, len, .Lcopy_bytes\@
          nop
 1:
         R10KCBARRIER(0(ra))
-        LOAD(t0, 0(src), .Ll_exc)
+        LOAD(t0, 0(src), .Ll_exc\@)
         ADD     src, src, NBYTES
         SUB     len, len, NBYTES
-        STORE(t0, 0(dst), .Ls_exc_p1u)
+        STORE(t0, 0(dst), .Ls_exc_p1u\@)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, NBYTES
         bne     rem, len, 1b
@@ -350,17 +355,17 @@ __copy_user_common:
          * more instruction-level parallelism.
          */
 #define bits t2
-        beqz    len, .Ldone
+        beqz    len, .Ldone\@
          ADD    t1, dst, len    # t1 is just past last byte of dst
         li      bits, 8*NBYTES
         SLL     rem, len, 3     # rem = number of bits to keep
-        LOAD(t0, 0(src), .Ll_exc)
+        LOAD(t0, 0(src), .Ll_exc\@)
         SUB     bits, bits, rem # bits = number of bits to discard
         SHIFT_DISCARD t0, t0, bits
-        STREST(t0, -1(t1), .Ls_exc)
+        STREST(t0, -1(t1), .Ls_exc\@)
         jr      ra
          move   len, zero
-.Ldst_unaligned:
+.Ldst_unaligned\@:
         /*
          * dst is unaligned
          * t0 = src & ADDRMASK
@@ -371,23 +376,23 @@ __copy_user_common:
          * Set match = (src and dst have same alignment)
          */
 #define match rem
-        LDFIRST(t3, FIRST(0)(src), .Ll_exc)
+        LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
         ADD     t2, zero, NBYTES
-        LDREST(t3, REST(0)(src), .Ll_exc_copy)
+        LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
         SUB     t2, t2, t1      # t2 = number of bytes copied
         xor     match, t0, t1
         R10KCBARRIER(0(ra))
-        STFIRST(t3, FIRST(0)(dst), .Ls_exc)
-        beq     len, t2, .Ldone
+        STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
+        beq     len, t2, .Ldone\@
          SUB    len, len, t2
         ADD     dst, dst, t2
-        beqz    match, .Lboth_aligned
+        beqz    match, .Lboth_aligned\@
          ADD    src, src, t2
 
-.Lsrc_unaligned_dst_aligned:
+.Lsrc_unaligned_dst_aligned\@:
         SRL     t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
         PREFS(  0, 3*32(src) )
-        beqz    t0, .Lcleanup_src_unaligned
+        beqz    t0, .Lcleanup_src_unaligned\@
          and    rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
         PREFD(  1, 3*32(dst) )
 1:
@@ -398,58 +403,58 @@ __copy_user_common:
  * are to the same unit (unless src is aligned, but it's not).
  */
         R10KCBARRIER(0(ra))
-        LDFIRST(t0, FIRST(0)(src), .Ll_exc)
-        LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)
+        LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
+        LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
         SUB     len, len, 4*NBYTES
-        LDREST(t0, REST(0)(src), .Ll_exc_copy)
-        LDREST(t1, REST(1)(src), .Ll_exc_copy)
-        LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)
-        LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)
-        LDREST(t2, REST(2)(src), .Ll_exc_copy)
-        LDREST(t3, REST(3)(src), .Ll_exc_copy)
+        LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
+        LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
+        LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
+        LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
+        LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
+        LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
         PREFS(  0, 9*32(src) )          # 0 is PREF_LOAD  (not streamed)
         ADD     src, src, 4*NBYTES
 #ifdef CONFIG_CPU_SB1
         nop                             # improves slotting
 #endif
-        STORE(t0, UNIT(0)(dst), .Ls_exc_p4u)
-        STORE(t1, UNIT(1)(dst), .Ls_exc_p3u)
-        STORE(t2, UNIT(2)(dst), .Ls_exc_p2u)
-        STORE(t3, UNIT(3)(dst), .Ls_exc_p1u)
+        STORE(t0, UNIT(0)(dst), .Ls_exc_p4u\@)
+        STORE(t1, UNIT(1)(dst), .Ls_exc_p3u\@)
+        STORE(t2, UNIT(2)(dst), .Ls_exc_p2u\@)
+        STORE(t3, UNIT(3)(dst), .Ls_exc_p1u\@)
         PREFD(  1, 9*32(dst) )          # 1 is PREF_STORE (not streamed)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, 4*NBYTES
         bne     len, rem, 1b
         .set    noreorder
 
-.Lcleanup_src_unaligned:
-        beqz    len, .Ldone
+.Lcleanup_src_unaligned\@:
+        beqz    len, .Ldone\@
          and    rem, len, NBYTES-1  # rem = len % NBYTES
-        beq     rem, len, .Lcopy_bytes
+        beq     rem, len, .Lcopy_bytes\@
          nop
 1:
         R10KCBARRIER(0(ra))
-        LDFIRST(t0, FIRST(0)(src), .Ll_exc)
-        LDREST(t0, REST(0)(src), .Ll_exc_copy)
+        LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
+        LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
         ADD     src, src, NBYTES
         SUB     len, len, NBYTES
-        STORE(t0, 0(dst), .Ls_exc_p1u)
+        STORE(t0, 0(dst), .Ls_exc_p1u\@)
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, NBYTES
         bne     len, rem, 1b
         .set    noreorder
 
-.Lcopy_bytes_checklen:
-        beqz    len, .Ldone
+.Lcopy_bytes_checklen\@:
+        beqz    len, .Ldone\@
          nop
-.Lcopy_bytes:
+.Lcopy_bytes\@:
         /* 0 < len < NBYTES  */
         R10KCBARRIER(0(ra))
 #define COPY_BYTE(N)                    \
-        LOADB(t0, N(src), .Ll_exc);     \
+        LOADB(t0, N(src), .Ll_exc\@);   \
         SUB     len, len, 1;            \
-        beqz    len, .Ldone;            \
-        STOREB(t0, N(dst), .Ls_exc_p1)
+        beqz    len, .Ldone\@;          \
+        STOREB(t0, N(dst), .Ls_exc_p1\@)
 
         COPY_BYTE(0)
         COPY_BYTE(1)
@@ -459,16 +464,19 @@ __copy_user_common:
         COPY_BYTE(4)
         COPY_BYTE(5)
 #endif
-        LOADB(t0, NBYTES-2(src), .Ll_exc)
+        LOADB(t0, NBYTES-2(src), .Ll_exc\@)
         SUB     len, len, 1
         jr      ra
-        STOREB(t0, NBYTES-2(dst), .Ls_exc_p1)
-.Ldone:
+        STOREB(t0, NBYTES-2(dst), .Ls_exc_p1\@)
+.Ldone\@:
         jr      ra
-         nop
+        .if __memcpy == 1
         END(memcpy)
+        .set __memcpy, 0
+        .hidden __memcpy
+        .endif
 
-.Ll_exc_copy:
+.Ll_exc_copy\@:
         /*
          * Copy bytes from src until faulting load address (or until a
          * lb faults)
@@ -483,20 +491,20 @@ __copy_user_common:
          nop
         LOADK   t0, THREAD_BUADDR(t0)
 1:
-        LOADB(t1, 0(src), .Ll_exc)
+        LOADB(t1, 0(src), .Ll_exc\@)
         ADD     src, src, 1
         sb      t1, 0(dst)      # can't fault -- we're copy_from_user
         .set    reorder                         /* DADDI_WAR */
         ADD     dst, dst, 1
         bne     src, t0, 1b
         .set    noreorder
-.Ll_exc:
+.Ll_exc\@:
         LOADK   t0, TI_TASK($28)
          nop
         LOADK   t0, THREAD_BUADDR(t0)   # t0 is just past last good address
          nop
         SUB     len, AT, t0             # len number of uncopied bytes
-        bnez    t6, .Ldone      /* Skip the zeroing part if inatomic */
+        bnez    t6, .Ldone\@    /* Skip the zeroing part if inatomic */
         /*
          * Here's where we rely on src and dst being incremented in tandem,
          *   See (3) above.
@@ -510,7 +518,7 @@ __copy_user_common:
          */
         .set    reorder                         /* DADDI_WAR */
         SUB     src, len, 1
-        beqz    len, .Ldone
+        beqz    len, .Ldone\@
         .set    noreorder
 1:      sb      zero, 0(dst)
         ADD     dst, dst, 1
@@ -531,7 +539,7 @@ __copy_user_common:
 
 #define SEXC(n)                                                 \
         .set    reorder;                        /* DADDI_WAR */ \
-.Ls_exc_p ## n ## u:                                            \
+.Ls_exc_p ## n ## u\@:                                          \
         ADD     len, len, n*NBYTES;                             \
         jr      ra;                                             \
         .set    noreorder
@@ -545,14 +553,15 @@ SEXC(3)
 SEXC(2)
 SEXC(1)
 
-.Ls_exc_p1:
+.Ls_exc_p1\@:
         .set    reorder                         /* DADDI_WAR */
         ADD     len, len, 1
         jr      ra
         .set    noreorder
-.Ls_exc:
+.Ls_exc\@:
         jr      ra
          nop
+        .endm
 
         .align  5
 LEAF(memmove)
@@ -603,3 +612,27 @@ LEAF(__rmemcpy)					/* a0=dst a1=src a2=len */
         jr      ra
          move   a2, zero
         END(__rmemcpy)
+
+/*
+ * t6 is used as a flag to note inatomic mode.
+ */
+LEAF(__copy_user_inatomic)
+        b       __copy_user_common
+        li      t6, 1
+        END(__copy_user_inatomic)
+
+/*
+ * A combined memcpy/__copy_user
+ * __copy_user sets len to 0 for success; else to an upper bound of
+ * the number of uncopied bytes.
+ * memcpy sets v0 to dst.
+ */
+        .align  5
+LEAF(memcpy)                                    /* a0=dst a1=src a2=len */
+        move    v0, dst                         /* return value */
+.L__memcpy:
+FEXPORT(__copy_user)
+        li      t6, 0   /* not inatomic */
+__copy_user_common:
+        /* Legacy Mode, user <-> user */
+        __BUILD_COPY_USER LEGACY_MODE USEROP USEROP