8 files changed, 375 insertions, 592 deletions
diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h
index edfbf9d6a6dd..8442727f28d2 100644
--- a/arch/parisc/include/asm/uaccess.h
+++ b/arch/parisc/include/asm/uaccess.h
@@ -65,6 +65,15 @@ struct exception_table_entry {
        ".previous\n"
 /*
+ * ASM_EXCEPTIONTABLE_ENTRY_EFAULT() creates a special exception table entry
+ * (with lowest bit set) for which the fault handler in fixup_exception() will
+ * load -EFAULT into %r8 for a read or write fault, and zeroes the target
+ * register in case of a read fault in get_user().
+ */
+#define ASM_EXCEPTIONTABLE_ENTRY_EFAULT( fault_addr, except_addr )\
+        ASM_EXCEPTIONTABLE_ENTRY( fault_addr, except_addr + 1)
+/*
 * The page fault handler stores, in a per-cpu area, the following information
 * if a fixup routine is available.
 */
@@ -91,7 +100,7 @@ struct exception_data {
 #define __get_user(x, ptr)                               \
 ({                                                       \
        register long __gu_err __asm__ ("r8") = 0;       \
-        register long __gu_val __asm__ ("r9") = 0;       \
+        register long __gu_val;                          \
                                                         \
        load_sr2();                                      \
        switch (sizeof(*(ptr))) {                        \
@@ -107,22 +116,23 @@ struct exception_data {
 })
 #define __get_user_asm(ldx, ptr)                        \
-        __asm__("\n1:\t" ldx "\t0(%%sr2,%2),%0\n\t"     \
+        __asm__("1: " ldx " 0(%%sr2,%2),%0\n"           \
-                ASM_EXCEPTIONTABLE_ENTRY(1b, fixup_get_user_skip_1)\
+                "9:\n"                                  \
+                ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \
                : "=r"(__gu_val), "=r"(__gu_err)        \
-                : "r"(ptr), "1"(__gu_err)               \
+                : "r"(ptr), "1"(__gu_err));
-                : "r1");
 #if !defined(CONFIG_64BIT)
 #define __get_user_asm64(ptr)                           \
-        __asm__("\n1:\tldw 0(%%sr2,%2),%0"              \
+        __asm__("   copy %%r0,%R0\n"                    \
-                "\n2:\tldw 4(%%sr2,%2),%R0\n\t"         \
+                "1: ldw 0(%%sr2,%2),%0\n"               \
-                ASM_EXCEPTIONTABLE_ENTRY(1b, fixup_get_user_skip_2)\
+                "2: ldw 4(%%sr2,%2),%R0\n"              \
-                ASM_EXCEPTIONTABLE_ENTRY(2b, fixup_get_user_skip_1)\
+                "9:\n"                                  \
+                ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \
+                ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 9b) \
                : "=r"(__gu_val), "=r"(__gu_err)        \
-                : "r"(ptr), "1"(__gu_err)               \
+                : "r"(ptr), "1"(__gu_err));
-                : "r1");
 #endif /* !defined(CONFIG_64BIT) */
@@ -148,32 +158,31 @@ struct exception_data {
 * The "__put_user/kernel_asm()" macros tell gcc they read from memory
 * instead of writing. This is because they do not write to any memory
 * gcc knows about, so there are no aliasing issues. These macros must
- * also be aware that "fixup_put_user_skip_[12]" are executed in the
+ * also be aware that fixups are executed in the context of the fault,
- * context of the fault, and any registers used there must be listed
+ * and any registers used there must be listed as clobbers.
- * as clobbers. In this case only "r1" is used by the current routines.
+ * r8 is already listed as err.
- * r8/r9 are already listed as err/val.
 */
 #define __put_user_asm(stx, x, ptr)                         \
        __asm__ __volatile__ (                              \
-                "\n1:\t" stx "\t%2,0(%%sr2,%1)\n\t"         \
+                "1: " stx " %2,0(%%sr2,%1)\n"               \
-                ASM_EXCEPTIONTABLE_ENTRY(1b, fixup_put_user_skip_1)\
+                "9:\n"                                      \
+                ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b)     \
                : "=r"(__pu_err)                            \
-                : "r"(ptr), "r"(x), "0"(__pu_err)           \
+                : "r"(ptr), "r"(x), "0"(__pu_err))
-                : "r1")
 #if !defined(CONFIG_64BIT)
 #define __put_user_asm64(__val, ptr) do {                   \
        __asm__ __volatile__ (                              \
-                "\n1:\tstw %2,0(%%sr2,%1)"                  \
+                "1: stw %2,0(%%sr2,%1)\n"                   \
-                "\n2:\tstw %R2,4(%%sr2,%1)\n\t"             \
+                "2: stw %R2,4(%%sr2,%1)\n"                  \
-                ASM_EXCEPTIONTABLE_ENTRY(1b, fixup_put_user_skip_2)\
+                "9:\n"                                      \
-                ASM_EXCEPTIONTABLE_ENTRY(2b, fixup_put_user_skip_1)\
+                ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b)     \
+                ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 9b)     \
                : "=r"(__pu_err)                            \
-                : "r"(ptr), "r"(__val), "0"(__pu_err) \
+                : "r"(ptr), "r"(__val), "0"(__pu_err));     \
-                : "r1");                                    \
 } while (0)
 #endif /* !defined(CONFIG_64BIT) */
diff --git a/arch/parisc/kernel/parisc_ksyms.c b/arch/parisc/kernel/parisc_ksyms.c
index 7484b3d11e0d..c6d6272a934f 100644
--- a/arch/parisc/kernel/parisc_ksyms.c
+++ b/arch/parisc/kernel/parisc_ksyms.c
@@ -47,16 +47,6 @@ EXPORT_SYMBOL(__cmpxchg_u64);
 EXPORT_SYMBOL(lclear_user);
 EXPORT_SYMBOL(lstrnlen_user);
-/* Global fixups - defined as int to avoid creation of function pointers */
-extern int fixup_get_user_skip_1;
-extern int fixup_get_user_skip_2;
-extern int fixup_put_user_skip_1;
-extern int fixup_put_user_skip_2;
-EXPORT_SYMBOL(fixup_get_user_skip_1);
-EXPORT_SYMBOL(fixup_get_user_skip_2);
-EXPORT_SYMBOL(fixup_put_user_skip_1);
-EXPORT_SYMBOL(fixup_put_user_skip_2);
 #ifndef CONFIG_64BIT
 /* Needed so insmod can set dp value */
 extern int $global$;
diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c
index b76f503eee4a..4516a5b53f38 100644
--- a/arch/parisc/kernel/process.c
+++ b/arch/parisc/kernel/process.c
@@ -143,6 +143,8 @@ void machine_power_off(void)
        printk(KERN_EMERG "System shut down completed.\n"
               "Please power this system off now.");
+        /* prevent soft lockup/stalled CPU messages for endless loop. */
+        rcu_sysrq_start();
        for (;;);
 }
diff --git a/arch/parisc/lib/Makefile b/arch/parisc/lib/Makefile
index 8fa92b8d839a..f2dac4d73b1b 100644
--- a/arch/parisc/lib/Makefile
+++ b/arch/parisc/lib/Makefile
@@ -2,7 +2,7 @@
 # Makefile for parisc-specific library files
 #
-lib-y   := lusercopy.o bitops.o checksum.o io.o memset.o fixup.o memcpy.o \
+lib-y   := lusercopy.o bitops.o checksum.o io.o memset.o memcpy.o \
           ucmpdi2.o delay.o
 obj-y   := iomap.o
diff --git a/arch/parisc/lib/fixup.S b/arch/parisc/lib/fixup.S
deleted file mode 100644
index a5b72f22c7a6..000000000000
--- a/arch/parisc/lib/fixup.S
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Linux/PA-RISC Project (http://www.parisc-linux.org/)
- *
- *  Copyright (C) 2004  Randolph Chung <tausq@debian.org>
- *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; either version 2, or (at your option)
- *    any later version.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *    GNU General Public License for more details.
- *
- *    You should have received a copy of the GNU General Public License
- *    along with this program; if not, write to the Free Software
- *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- * 
- * Fixup routines for kernel exception handling.
- */
-#include <asm/asm-offsets.h>
-#include <asm/assembly.h>
-#include <asm/errno.h>
-#include <linux/linkage.h>
-#ifdef CONFIG_SMP
-        .macro  get_fault_ip t1 t2
-        loadgp
-        addil LT%__per_cpu_offset,%r27
-        LDREG RT%__per_cpu_offset(%r1),\t1
-        /* t2 = smp_processor_id() */
-        mfctl 30,\t2
-        ldw TI_CPU(\t2),\t2
-#ifdef CONFIG_64BIT
-        extrd,u \t2,63,32,\t2
-#endif
-        /* t2 = &__per_cpu_offset[smp_processor_id()]; */
-        LDREGX \t2(\t1),\t2 
-        addil LT%exception_data,%r27
-        LDREG RT%exception_data(%r1),\t1
-        /* t1 = this_cpu_ptr(&exception_data) */
-        add,l \t1,\t2,\t1
-        /* %r27 = t1->fault_gp - restore gp */
-        LDREG EXCDATA_GP(\t1), %r27
-        /* t1 = t1->fault_ip */
-        LDREG EXCDATA_IP(\t1), \t1
-        .endm
-#else
-        .macro  get_fault_ip t1 t2
-        loadgp
-        /* t1 = this_cpu_ptr(&exception_data) */
-        addil LT%exception_data,%r27
-        LDREG RT%exception_data(%r1),\t2
-        /* %r27 = t2->fault_gp - restore gp */
-        LDREG EXCDATA_GP(\t2), %r27
-        /* t1 = t2->fault_ip */
-        LDREG EXCDATA_IP(\t2), \t1
-        .endm
-#endif
-        .level LEVEL
-        .text
-        .section .fixup, "ax"
-        /* get_user() fixups, store -EFAULT in r8, and 0 in r9 */
-ENTRY_CFI(fixup_get_user_skip_1)
-        get_fault_ip %r1,%r8
-        ldo 4(%r1), %r1
-        ldi -EFAULT, %r8
-        bv %r0(%r1)
-        copy %r0, %r9
-ENDPROC_CFI(fixup_get_user_skip_1)
-ENTRY_CFI(fixup_get_user_skip_2)
-        get_fault_ip %r1,%r8
-        ldo 8(%r1), %r1
-        ldi -EFAULT, %r8
-        bv %r0(%r1)
-        copy %r0, %r9
-ENDPROC_CFI(fixup_get_user_skip_2)
-        /* put_user() fixups, store -EFAULT in r8 */
-ENTRY_CFI(fixup_put_user_skip_1)
-        get_fault_ip %r1,%r8
-        ldo 4(%r1), %r1
-        bv %r0(%r1)
-        ldi -EFAULT, %r8
-ENDPROC_CFI(fixup_put_user_skip_1)
-ENTRY_CFI(fixup_put_user_skip_2)
-        get_fault_ip %r1,%r8
-        ldo 8(%r1), %r1
-        bv %r0(%r1)
-        ldi -EFAULT, %r8
-ENDPROC_CFI(fixup_put_user_skip_2)
diff --git a/arch/parisc/lib/lusercopy.S b/arch/parisc/lib/lusercopy.S
index 56845de6b5df..f01188c044ee 100644
--- a/arch/parisc/lib/lusercopy.S
+++ b/arch/parisc/lib/lusercopy.S
@@ -5,6 +5,8 @@
 *    Copyright (C) 2000 Richard Hirst <rhirst with parisc-linux.org>
 *    Copyright (C) 2001 Matthieu Delahaye <delahaym at esiee.fr>
 *    Copyright (C) 2003 Randolph Chung <tausq with parisc-linux.org>
+ *    Copyright (C) 2017 Helge Deller <deller@gmx.de>
+ *    Copyright (C) 2017 John David Anglin <dave.anglin@bell.net>
 *
 *
 *    This program is free software; you can redistribute it and/or modify
@@ -132,4 +134,320 @@ ENDPROC_CFI(lstrnlen_user)
        .procend
+/*
+ * unsigned long pa_memcpy(void *dstp, const void *srcp, unsigned long len)
+ *
+ * Inputs:
+ * - sr1 already contains space of source region
+ * - sr2 already contains space of destination region
+ *
+ * Returns:
+ * - number of bytes that could not be copied.
+ *   On success, this will be zero.
+ *
+ * This code is based on a C-implementation of a copy routine written by
+ * Randolph Chung, which in turn was derived from the glibc.
+ *
+ * Several strategies are tried to try to get the best performance for various
+ * conditions. In the optimal case, we copy by loops that copy 32- or 16-bytes
+ * at a time using general registers.  Unaligned copies are handled either by
+ * aligning the destination and then using shift-and-write method, or in a few
+ * cases by falling back to a byte-at-a-time copy.
+ *
+ * Testing with various alignments and buffer sizes shows that this code is
+ * often >10x faster than a simple byte-at-a-time copy, even for strangely
+ * aligned operands. It is interesting to note that the glibc version of memcpy
+ * (written in C) is actually quite fast already. This routine is able to beat
+ * it by 30-40% for aligned copies because of the loop unrolling, but in some
+ * cases the glibc version is still slightly faster. This lends more
+ * credibility that gcc can generate very good code as long as we are careful.
+ *
+ * Possible optimizations:
+ * - add cache prefetching
+ * - try not to use the post-increment address modifiers; they may create
+ *   additional interlocks. Assumption is that those were only efficient on old
+ *   machines (pre PA8000 processors)
+ */
+        dst = arg0
+        src = arg1
+        len = arg2
+        end = arg3
+        t1  = r19
+        t2  = r20
+        t3  = r21
+        t4  = r22
+        srcspc = sr1
+        dstspc = sr2
+        t0 = r1
+        a1 = t1
+        a2 = t2
+        a3 = t3
+        a0 = t4
+        save_src = ret0
+        save_dst = ret1
+        save_len = r31
+ENTRY_CFI(pa_memcpy)
+        .proc
+        .callinfo NO_CALLS
+        .entry
+        /* Last destination address */
+        add     dst,len,end
+        /* short copy with less than 16 bytes? */
+        cmpib,>>=,n 15,len,.Lbyte_loop
+        /* same alignment? */
+        xor     src,dst,t0
+        extru   t0,31,2,t1
+        cmpib,<>,n  0,t1,.Lunaligned_copy
+#ifdef CONFIG_64BIT
+        /* only do 64-bit copies if we can get aligned. */
+        extru   t0,31,3,t1
+        cmpib,<>,n  0,t1,.Lalign_loop32
+        /* loop until we are 64-bit aligned */
+.Lalign_loop64:
+        extru   dst,31,3,t1
+        cmpib,=,n       0,t1,.Lcopy_loop_16
+20:     ldb,ma  1(srcspc,src),t1
+21:     stb,ma  t1,1(dstspc,dst)
+        b       .Lalign_loop64
+        ldo     -1(len),len
+        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
+        ldi     31,t0
+.Lcopy_loop_16:
+        cmpb,COND(>>=),n t0,len,.Lword_loop
+10:     ldd     0(srcspc,src),t1
+11:     ldd     8(srcspc,src),t2
+        ldo     16(src),src
+12:     std,ma  t1,8(dstspc,dst)
+13:     std,ma  t2,8(dstspc,dst)
+14:     ldd     0(srcspc,src),t1
+15:     ldd     8(srcspc,src),t2
+        ldo     16(src),src
+16:     std,ma  t1,8(dstspc,dst)
+17:     std,ma  t2,8(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy16_fault)
+        ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy16_fault)
+        ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_done)
+        b       .Lcopy_loop_16
+        ldo     -32(len),len
+.Lword_loop:
+        cmpib,COND(>>=),n 3,len,.Lbyte_loop
+20:     ldw,ma  4(srcspc,src),t1
+21:     stw,ma  t1,4(dstspc,dst)
+        b       .Lword_loop
+        ldo     -4(len),len
+        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
+#endif /* CONFIG_64BIT */
+        /* loop until we are 32-bit aligned */
+.Lalign_loop32:
+        extru   dst,31,2,t1
+        cmpib,=,n       0,t1,.Lcopy_loop_4
+20:     ldb,ma  1(srcspc,src),t1
+21:     stb,ma  t1,1(dstspc,dst)
+        b       .Lalign_loop32
+        ldo     -1(len),len
+        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
+.Lcopy_loop_4:
+        cmpib,COND(>>=),n 15,len,.Lbyte_loop
+10:     ldw     0(srcspc,src),t1
+11:     ldw     4(srcspc,src),t2
+12:     stw,ma  t1,4(dstspc,dst)
+13:     stw,ma  t2,4(dstspc,dst)
+14:     ldw     8(srcspc,src),t1
+15:     ldw     12(srcspc,src),t2
+        ldo     16(src),src
+16:     stw,ma  t1,4(dstspc,dst)
+17:     stw,ma  t2,4(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy8_fault)
+        ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy8_fault)
+        ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_done)
+        b       .Lcopy_loop_4
+        ldo     -16(len),len
+.Lbyte_loop:
+        cmpclr,COND(<>) len,%r0,%r0
+        b,n     .Lcopy_done
+20:     ldb     0(srcspc,src),t1
+        ldo     1(src),src
+21:     stb,ma  t1,1(dstspc,dst)
+        b       .Lbyte_loop
+        ldo     -1(len),len
+        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
+.Lcopy_done:
+        bv      %r0(%r2)
+        sub     end,dst,ret0
+        /* src and dst are not aligned the same way. */
+        /* need to go the hard way */
+.Lunaligned_copy:
+        /* align until dst is 32bit-word-aligned */
+        extru   dst,31,2,t1
+        cmpib,COND(=),n 0,t1,.Lcopy_dstaligned
+20:     ldb     0(srcspc,src),t1
+        ldo     1(src),src
+21:     stb,ma  t1,1(dstspc,dst)
+        b       .Lunaligned_copy
+        ldo     -1(len),len
+        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
+        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
+.Lcopy_dstaligned:
+        /* store src, dst and len in safe place */
+        copy    src,save_src
+        copy    dst,save_dst
+        copy    len,save_len
+        /* len now needs give number of words to copy */
+        SHRREG  len,2,len
+        /*
+         * Copy from a not-aligned src to an aligned dst using shifts.
+         * Handles 4 words per loop.
+         */
+        depw,z src,28,2,t0
+        subi 32,t0,t0
+        mtsar t0
+        extru len,31,2,t0
+        cmpib,= 2,t0,.Lcase2
+        /* Make src aligned by rounding it down.  */
+        depi 0,31,2,src
+        cmpiclr,<> 3,t0,%r0
+        b,n .Lcase3
+        cmpiclr,<> 1,t0,%r0
+        b,n .Lcase1
+.Lcase0:
+        cmpb,= %r0,len,.Lcda_finish
+        nop
+1:      ldw,ma 4(srcspc,src), a3
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+1:      ldw,ma 4(srcspc,src), a0
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+        b,n .Ldo3
+.Lcase1:
+1:      ldw,ma 4(srcspc,src), a2
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+1:      ldw,ma 4(srcspc,src), a3
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+        ldo -1(len),len
+        cmpb,=,n %r0,len,.Ldo0
+.Ldo4:
+1:      ldw,ma 4(srcspc,src), a0
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+        shrpw a2, a3, %sar, t0
+1:      stw,ma t0, 4(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
+.Ldo3:
+1:      ldw,ma 4(srcspc,src), a1
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+        shrpw a3, a0, %sar, t0
+1:      stw,ma t0, 4(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
+.Ldo2:
+1:      ldw,ma 4(srcspc,src), a2
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+        shrpw a0, a1, %sar, t0
+1:      stw,ma t0, 4(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
+.Ldo1:
+1:      ldw,ma 4(srcspc,src), a3
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+        shrpw a1, a2, %sar, t0
+1:      stw,ma t0, 4(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
+        ldo -4(len),len
+        cmpb,<> %r0,len,.Ldo4
+        nop
+.Ldo0:
+        shrpw a2, a3, %sar, t0
+1:      stw,ma t0, 4(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
+.Lcda_rdfault:
+.Lcda_finish:
+        /* calculate new src, dst and len and jump to byte-copy loop */
+        sub     dst,save_dst,t0
+        add     save_src,t0,src
+        b       .Lbyte_loop
+        sub     save_len,t0,len
+.Lcase3:
+1:      ldw,ma 4(srcspc,src), a0
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+1:      ldw,ma 4(srcspc,src), a1
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+        b .Ldo2
+        ldo 1(len),len
+.Lcase2:
+1:      ldw,ma 4(srcspc,src), a1
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+1:      ldw,ma 4(srcspc,src), a2
+        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
+        b .Ldo1
+        ldo 2(len),len
+        /* fault exception fixup handlers: */
+#ifdef CONFIG_64BIT
+.Lcopy16_fault:
+10:     b       .Lcopy_done
+        std,ma  t1,8(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
+#endif
+.Lcopy8_fault:
+10:     b       .Lcopy_done
+        stw,ma  t1,4(dstspc,dst)
+        ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
+        .exit
+ENDPROC_CFI(pa_memcpy)
+        .procend
        .end
diff --git a/arch/parisc/lib/memcpy.c b/arch/parisc/lib/memcpy.c
index f82ff10ed974..b3d47ec1d80a 100644
--- a/arch/parisc/lib/memcpy.c
+++ b/arch/parisc/lib/memcpy.c
@@ -2,7 +2,7 @@
 *    Optimized memory copy routines.
 *
 *    Copyright (C) 2004 Randolph Chung <tausq@debian.org>
- *    Copyright (C) 2013 Helge Deller <deller@gmx.de>
+ *    Copyright (C) 2013-2017 Helge Deller <deller@gmx.de>
 *
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
@@ -21,474 +21,21 @@
 *    Portions derived from the GNU C Library
 *    Copyright (C) 1991, 1997, 2003 Free Software Foundation, Inc.
 *
- * Several strategies are tried to try to get the best performance for various
- * conditions. In the optimal case, we copy 64-bytes in an unrolled loop using 
- * fp regs. This is followed by loops that copy 32- or 16-bytes at a time using
- * general registers.  Unaligned copies are handled either by aligning the 
- * destination and then using shift-and-write method, or in a few cases by 
- * falling back to a byte-at-a-time copy.
- *
- * I chose to implement this in C because it is easier to maintain and debug,
- * and in my experiments it appears that the C code generated by gcc (3.3/3.4
- * at the time of writing) is fairly optimal. Unfortunately some of the 
- * semantics of the copy routine (exception handling) is difficult to express
- * in C, so we have to play some tricks to get it to work.
- *
- * All the loads and stores are done via explicit asm() code in order to use
- * the right space registers. 
- * 
- * Testing with various alignments and buffer sizes shows that this code is 
- * often >10x faster than a simple byte-at-a-time copy, even for strangely
- * aligned operands. It is interesting to note that the glibc version
- * of memcpy (written in C) is actually quite fast already. This routine is 
- * able to beat it by 30-40% for aligned copies because of the loop unrolling, 
- * but in some cases the glibc version is still slightly faster. This lends 
- * more credibility that gcc can generate very good code as long as we are 
- * careful.
- *
- * TODO:
- * - cache prefetching needs more experimentation to get optimal settings
- * - try not to use the post-increment address modifiers; they create additional
- *   interlocks
- * - replace byte-copy loops with stybs sequences
 */
-#ifdef __KERNEL__
 #include <linux/module.h>
 #include <linux/compiler.h>
 #include <linux/uaccess.h>
-#define s_space "%%sr1"
-#define d_space "%%sr2"
-#else
-#include "memcpy.h"
-#define s_space "%%sr0"
-#define d_space "%%sr0"
-#define pa_memcpy new2_copy
-#endif
 DECLARE_PER_CPU(struct exception_data, exception_data);
-#define preserve_branch(label)  do {                                    \
-        volatile int dummy = 0;                                         \
-        /* The following branch is never taken, it's just here to  */   \
-        /* prevent gcc from optimizing away our exception code. */      \
-        if (unlikely(dummy != dummy))                                   \
-                goto label;                                             \
-} while (0)
 #define get_user_space() (segment_eq(get_fs(), KERNEL_DS) ? 0 : mfsp(3))
 #define get_kernel_space() (0)
-#define MERGE(w0, sh_1, w1, sh_2)  ({                                   \
-        unsigned int _r;                                                \
-        asm volatile (                                                  \
-        "mtsar %3\n"                                                    \
-        "shrpw %1, %2, %%sar, %0\n"                                     \
-        : "=r"(_r)                                                      \
-        : "r"(w0), "r"(w1), "r"(sh_2)                                   \
-        );                                                              \
-        _r;                                                             \
-})
-#define THRESHOLD       16
-#ifdef DEBUG_MEMCPY
-#define DPRINTF(fmt, args...) do { printk(KERN_DEBUG "%s:%d:%s ", __FILE__, __LINE__, __func__ ); printk(KERN_DEBUG fmt, ##args ); } while (0)
-#else
-#define DPRINTF(fmt, args...)
-#endif
-#define def_load_ai_insn(_insn,_sz,_tt,_s,_a,_t,_e)     \
-        __asm__ __volatile__ (                          \
-        "1:\t" #_insn ",ma " #_sz "(" _s ",%1), %0\n\t" \
-        ASM_EXCEPTIONTABLE_ENTRY(1b,_e)                 \
-        : _tt(_t), "+r"(_a)                             \
-        :                                               \
-        : "r8")
-#define def_store_ai_insn(_insn,_sz,_tt,_s,_a,_t,_e)    \
-        __asm__ __volatile__ (                          \
-        "1:\t" #_insn ",ma %1, " #_sz "(" _s ",%0)\n\t" \
-        ASM_EXCEPTIONTABLE_ENTRY(1b,_e)                 \
-        : "+r"(_a)                                      \
-        : _tt(_t)                                       \
-        : "r8")
-#define ldbma(_s, _a, _t, _e) def_load_ai_insn(ldbs,1,"=r",_s,_a,_t,_e)
-#define stbma(_s, _t, _a, _e) def_store_ai_insn(stbs,1,"r",_s,_a,_t,_e)
-#define ldwma(_s, _a, _t, _e) def_load_ai_insn(ldw,4,"=r",_s,_a,_t,_e)
-#define stwma(_s, _t, _a, _e) def_store_ai_insn(stw,4,"r",_s,_a,_t,_e)
-#define flddma(_s, _a, _t, _e) def_load_ai_insn(fldd,8,"=f",_s,_a,_t,_e)
-#define fstdma(_s, _t, _a, _e) def_store_ai_insn(fstd,8,"f",_s,_a,_t,_e)
-#define def_load_insn(_insn,_tt,_s,_o,_a,_t,_e)         \
-        __asm__ __volatile__ (                          \
-        "1:\t" #_insn " " #_o "(" _s ",%1), %0\n\t"     \
-        ASM_EXCEPTIONTABLE_ENTRY(1b,_e)                 \
-        : _tt(_t)                                       \
-        : "r"(_a)                                       \
-        : "r8")
-#define def_store_insn(_insn,_tt,_s,_t,_o,_a,_e)        \
-        __asm__ __volatile__ (                          \
-        "1:\t" #_insn " %0, " #_o "(" _s ",%1)\n\t"     \
-        ASM_EXCEPTIONTABLE_ENTRY(1b,_e)                 \
-        :                                               \
-        : _tt(_t), "r"(_a)                              \
-        : "r8")
-#define ldw(_s,_o,_a,_t,_e)     def_load_insn(ldw,"=r",_s,_o,_a,_t,_e)
-#define stw(_s,_t,_o,_a,_e)     def_store_insn(stw,"r",_s,_t,_o,_a,_e)
-#ifdef  CONFIG_PREFETCH
-static inline void prefetch_src(const void *addr)
-{
-        __asm__("ldw 0(" s_space ",%0), %%r0" : : "r" (addr));
-}
-static inline void prefetch_dst(const void *addr)
-{
-        __asm__("ldd 0(" d_space ",%0), %%r0" : : "r" (addr));
-}
-#else
-#define prefetch_src(addr) do { } while(0)
-#define prefetch_dst(addr) do { } while(0)
-#endif
-#define PA_MEMCPY_OK            0
-#define PA_MEMCPY_LOAD_ERROR    1
-#define PA_MEMCPY_STORE_ERROR   2
-/* Copy from a not-aligned src to an aligned dst, using shifts. Handles 4 words
- * per loop.  This code is derived from glibc. 
- */
-static noinline unsigned long copy_dstaligned(unsigned long dst,
-                                        unsigned long src, unsigned long len)
-{
-        /* gcc complains that a2 and a3 may be uninitialized, but actually
-         * they cannot be.  Initialize a2/a3 to shut gcc up.
-         */
-        register unsigned int a0, a1, a2 = 0, a3 = 0;
-        int sh_1, sh_2;
-        /* prefetch_src((const void *)src); */
-        /* Calculate how to shift a word read at the memory operation
-           aligned srcp to make it aligned for copy.  */
-        sh_1 = 8 * (src % sizeof(unsigned int));
-        sh_2 = 8 * sizeof(unsigned int) - sh_1;
-        /* Make src aligned by rounding it down.  */
-        src &= -sizeof(unsigned int);
-        switch (len % 4)
-        {
-                case 2:
-                        /* a1 = ((unsigned int *) src)[0];
-                           a2 = ((unsigned int *) src)[1]; */
-                        ldw(s_space, 0, src, a1, cda_ldw_exc);
-                        ldw(s_space, 4, src, a2, cda_ldw_exc);
-                        src -= 1 * sizeof(unsigned int);
-                        dst -= 3 * sizeof(unsigned int);
-                        len += 2;
-                        goto do1;
-                case 3:
-                        /* a0 = ((unsigned int *) src)[0];
-                           a1 = ((unsigned int *) src)[1]; */
-                        ldw(s_space, 0, src, a0, cda_ldw_exc);
-                        ldw(s_space, 4, src, a1, cda_ldw_exc);
-                        src -= 0 * sizeof(unsigned int);
-                        dst -= 2 * sizeof(unsigned int);
-                        len += 1;
-                        goto do2;
-                case 0:
-                        if (len == 0)
-                                return PA_MEMCPY_OK;
-                        /* a3 = ((unsigned int *) src)[0];
-                           a0 = ((unsigned int *) src)[1]; */
-                        ldw(s_space, 0, src, a3, cda_ldw_exc);
-                        ldw(s_space, 4, src, a0, cda_ldw_exc);
-                        src -=-1 * sizeof(unsigned int);
-                        dst -= 1 * sizeof(unsigned int);
-                        len += 0;
-                        goto do3;
-                case 1:
-                        /* a2 = ((unsigned int *) src)[0];
-                           a3 = ((unsigned int *) src)[1]; */
-                        ldw(s_space, 0, src, a2, cda_ldw_exc);
-                        ldw(s_space, 4, src, a3, cda_ldw_exc);
-                        src -=-2 * sizeof(unsigned int);
-                        dst -= 0 * sizeof(unsigned int);
-                        len -= 1;
-                        if (len == 0)
-                                goto do0;
-                        goto do4;                       /* No-op.  */
-        }
-        do
-        {
-                /* prefetch_src((const void *)(src + 4 * sizeof(unsigned int))); */
-do4:
-                /* a0 = ((unsigned int *) src)[0]; */
-                ldw(s_space, 0, src, a0, cda_ldw_exc);
-                /* ((unsigned int *) dst)[0] = MERGE (a2, sh_1, a3, sh_2); */
-                stw(d_space, MERGE (a2, sh_1, a3, sh_2), 0, dst, cda_stw_exc);
-do3:
-                /* a1 = ((unsigned int *) src)[1]; */
-                ldw(s_space, 4, src, a1, cda_ldw_exc);
-                /* ((unsigned int *) dst)[1] = MERGE (a3, sh_1, a0, sh_2); */
-                stw(d_space, MERGE (a3, sh_1, a0, sh_2), 4, dst, cda_stw_exc);
-do2:
-                /* a2 = ((unsigned int *) src)[2]; */
-                ldw(s_space, 8, src, a2, cda_ldw_exc);
-                /* ((unsigned int *) dst)[2] = MERGE (a0, sh_1, a1, sh_2); */
-                stw(d_space, MERGE (a0, sh_1, a1, sh_2), 8, dst, cda_stw_exc);
-do1:
-                /* a3 = ((unsigned int *) src)[3]; */
-                ldw(s_space, 12, src, a3, cda_ldw_exc);
-                /* ((unsigned int *) dst)[3] = MERGE (a1, sh_1, a2, sh_2); */
-                stw(d_space, MERGE (a1, sh_1, a2, sh_2), 12, dst, cda_stw_exc);
-                src += 4 * sizeof(unsigned int);
-                dst += 4 * sizeof(unsigned int);
-                len -= 4;
-        }
-        while (len != 0);
-do0:
-        /* ((unsigned int *) dst)[0] = MERGE (a2, sh_1, a3, sh_2); */
-        stw(d_space, MERGE (a2, sh_1, a3, sh_2), 0, dst, cda_stw_exc);
-        preserve_branch(handle_load_error);
-        preserve_branch(handle_store_error);
-        return PA_MEMCPY_OK;
-handle_load_error:
-        __asm__ __volatile__ ("cda_ldw_exc:\n");
-        return PA_MEMCPY_LOAD_ERROR;
-handle_store_error:
-        __asm__ __volatile__ ("cda_stw_exc:\n");
-        return PA_MEMCPY_STORE_ERROR;
-}
-/* Returns PA_MEMCPY_OK, PA_MEMCPY_LOAD_ERROR or PA_MEMCPY_STORE_ERROR.
- * In case of an access fault the faulty address can be read from the per_cpu
- * exception data struct. */
-static noinline unsigned long pa_memcpy_internal(void *dstp, const void *srcp,
-                                        unsigned long len)
-{
-        register unsigned long src, dst, t1, t2, t3;
-        register unsigned char *pcs, *pcd;
-        register unsigned int *pws, *pwd;
-        register double *pds, *pdd;
-        unsigned long ret;
-        src = (unsigned long)srcp;
-        dst = (unsigned long)dstp;
-        pcs = (unsigned char *)srcp;
-        pcd = (unsigned char *)dstp;
-        /* prefetch_src((const void *)srcp); */
-        if (len < THRESHOLD)
-                goto byte_copy;
-        /* Check alignment */
-        t1 = (src ^ dst);
-        if (unlikely(t1 & (sizeof(double)-1)))
-                goto unaligned_copy;
-        /* src and dst have same alignment. */
-        /* Copy bytes till we are double-aligned. */
-        t2 = src & (sizeof(double) - 1);
-        if (unlikely(t2 != 0)) {
-                t2 = sizeof(double) - t2;
-                while (t2 && len) {
-                        /* *pcd++ = *pcs++; */
-                        ldbma(s_space, pcs, t3, pmc_load_exc);
-                        len--;
-                        stbma(d_space, t3, pcd, pmc_store_exc);
-                        t2--;
-                }
-        }
-        pds = (double *)pcs;
-        pdd = (double *)pcd;
-#if 0
-        /* Copy 8 doubles at a time */
-        while (len >= 8*sizeof(double)) {
-                register double r1, r2, r3, r4, r5, r6, r7, r8;
-                /* prefetch_src((char *)pds + L1_CACHE_BYTES); */
-                flddma(s_space, pds, r1, pmc_load_exc);
-                flddma(s_space, pds, r2, pmc_load_exc);
-                flddma(s_space, pds, r3, pmc_load_exc);
-                flddma(s_space, pds, r4, pmc_load_exc);
-                fstdma(d_space, r1, pdd, pmc_store_exc);
-                fstdma(d_space, r2, pdd, pmc_store_exc);
-                fstdma(d_space, r3, pdd, pmc_store_exc);
-                fstdma(d_space, r4, pdd, pmc_store_exc);
-#if 0
-                if (L1_CACHE_BYTES <= 32)
-                        prefetch_src((char *)pds + L1_CACHE_BYTES);
-#endif
-                flddma(s_space, pds, r5, pmc_load_exc);
-                flddma(s_space, pds, r6, pmc_load_exc);
-                flddma(s_space, pds, r7, pmc_load_exc);
-                flddma(s_space, pds, r8, pmc_load_exc);
-                fstdma(d_space, r5, pdd, pmc_store_exc);
-                fstdma(d_space, r6, pdd, pmc_store_exc);
-                fstdma(d_space, r7, pdd, pmc_store_exc);
-                fstdma(d_space, r8, pdd, pmc_store_exc);
-                len -= 8*sizeof(double);
-        }
-#endif
-        pws = (unsigned int *)pds;
-        pwd = (unsigned int *)pdd;
-word_copy:
-        while (len >= 8*sizeof(unsigned int)) {
-                register unsigned int r1,r2,r3,r4,r5,r6,r7,r8;
-                /* prefetch_src((char *)pws + L1_CACHE_BYTES); */
-                ldwma(s_space, pws, r1, pmc_load_exc);
-                ldwma(s_space, pws, r2, pmc_load_exc);
-                ldwma(s_space, pws, r3, pmc_load_exc);
-                ldwma(s_space, pws, r4, pmc_load_exc);
-                stwma(d_space, r1, pwd, pmc_store_exc);
-                stwma(d_space, r2, pwd, pmc_store_exc);
-                stwma(d_space, r3, pwd, pmc_store_exc);
-                stwma(d_space, r4, pwd, pmc_store_exc);
-                ldwma(s_space, pws, r5, pmc_load_exc);
-                ldwma(s_space, pws, r6, pmc_load_exc);
-                ldwma(s_space, pws, r7, pmc_load_exc);
-                ldwma(s_space, pws, r8, pmc_load_exc);
-                stwma(d_space, r5, pwd, pmc_store_exc);
-                stwma(d_space, r6, pwd, pmc_store_exc);
-                stwma(d_space, r7, pwd, pmc_store_exc);
-                stwma(d_space, r8, pwd, pmc_store_exc);
-                len -= 8*sizeof(unsigned int);
-        }
-        while (len >= 4*sizeof(unsigned int)) {
-                register unsigned int r1,r2,r3,r4;
-                ldwma(s_space, pws, r1, pmc_load_exc);
-                ldwma(s_space, pws, r2, pmc_load_exc);
-                ldwma(s_space, pws, r3, pmc_load_exc);
-                ldwma(s_space, pws, r4, pmc_load_exc);
-                stwma(d_space, r1, pwd, pmc_store_exc);
-                stwma(d_space, r2, pwd, pmc_store_exc);
-                stwma(d_space, r3, pwd, pmc_store_exc);
-                stwma(d_space, r4, pwd, pmc_store_exc);
-                len -= 4*sizeof(unsigned int);
-        }
-        pcs = (unsigned char *)pws;
-        pcd = (unsigned char *)pwd;
-byte_copy:
-        while (len) {
-                /* *pcd++ = *pcs++; */
-                ldbma(s_space, pcs, t3, pmc_load_exc);
-                stbma(d_space, t3, pcd, pmc_store_exc);
-                len--;
-        }
-        return PA_MEMCPY_OK;
-unaligned_copy:
-        /* possibly we are aligned on a word, but not on a double... */
-        if (likely((t1 & (sizeof(unsigned int)-1)) == 0)) {
-                t2 = src & (sizeof(unsigned int) - 1);
-                if (unlikely(t2 != 0)) {
-                        t2 = sizeof(unsigned int) - t2;
-                        while (t2) {
-                                /* *pcd++ = *pcs++; */
-                                ldbma(s_space, pcs, t3, pmc_load_exc);
-                                stbma(d_space, t3, pcd, pmc_store_exc);
-                                len--;
-                                t2--;
-                        }
-                }
-                pws = (unsigned int *)pcs;
-                pwd = (unsigned int *)pcd;
-                goto word_copy;
-        }
-        /* Align the destination.  */
-        if (unlikely((dst & (sizeof(unsigned int) - 1)) != 0)) {
-                t2 = sizeof(unsigned int) - (dst & (sizeof(unsigned int) - 1));
-                while (t2) {
-                        /* *pcd++ = *pcs++; */
-                        ldbma(s_space, pcs, t3, pmc_load_exc);
-                        stbma(d_space, t3, pcd, pmc_store_exc);
-                        len--;
-                        t2--;
-                }
-                dst = (unsigned long)pcd;
-                src = (unsigned long)pcs;
-        }
-        ret = copy_dstaligned(dst, src, len / sizeof(unsigned int));
-        if (ret)
-                return ret;
-        pcs += (len & -sizeof(unsigned int));
-        pcd += (len & -sizeof(unsigned int));
-        len %= sizeof(unsigned int);
-        preserve_branch(handle_load_error);
-        preserve_branch(handle_store_error);
-        goto byte_copy;
-handle_load_error:
-        __asm__ __volatile__ ("pmc_load_exc:\n");
-        return PA_MEMCPY_LOAD_ERROR;
-handle_store_error:
-        __asm__ __volatile__ ("pmc_store_exc:\n");
-        return PA_MEMCPY_STORE_ERROR;
-}
 /* Returns 0 for success, otherwise, returns number of bytes not transferred. */
-static unsigned long pa_memcpy(void *dstp, const void *srcp, unsigned long len)
+extern unsigned long pa_memcpy(void *dst, const void *src,
-{
+                                unsigned long len);
-        unsigned long ret, fault_addr, reference;
-        struct exception_data *d;
-        ret = pa_memcpy_internal(dstp, srcp, len);
-        if (likely(ret == PA_MEMCPY_OK))
-                return 0;
-        /* if a load or store fault occured we can get the faulty addr */
-        d = this_cpu_ptr(&exception_data);
-        fault_addr = d->fault_addr;
-        /* error in load or store? */
-        if (ret == PA_MEMCPY_LOAD_ERROR)
-                reference = (unsigned long) srcp;
-        else
-                reference = (unsigned long) dstp;
-        DPRINTF("pa_memcpy: fault type = %lu, len=%lu fault_addr=%lu ref=%lu\n",
-                ret, len, fault_addr, reference);
-        if (fault_addr >= reference)
-                return len - (fault_addr - reference);
-        else
-                return len;
-}
-#ifdef __KERNEL__
 unsigned long __copy_to_user(void __user *dst, const void *src,
                             unsigned long len)
 {
@@ -537,5 +84,3 @@ long probe_kernel_read(void *dst, const void *src, size_t size)
        return __probe_kernel_read(dst, src, size);
 }
-#endif
diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c
index deab89a8915a..32ec22146141 100644
--- a/arch/parisc/mm/fault.c
+++ b/arch/parisc/mm/fault.c
@@ -150,6 +150,23 @@ int fixup_exception(struct pt_regs *regs)
                d->fault_space = regs->isr;
                d->fault_addr = regs->ior;
+                /*
+                 * Fix up get_user() and put_user().
+                 * ASM_EXCEPTIONTABLE_ENTRY_EFAULT() sets the least-significant
+                 * bit in the relative address of the fixup routine to indicate
+                 * that %r8 should be loaded with -EFAULT to report a userspace
+                 * access error.
+                 */
+                if (fix->fixup & 1) {
+                        regs->gr[8] = -EFAULT;
+                        /* zero target register for get_user() */
+                        if (parisc_acctyp(0, regs->iir) == VM_READ) {
+                                int treg = regs->iir & 0x1f;
+                                regs->gr[treg] = 0;
+                        }
+                }
                regs->iaoq[0] = (unsigned long)&fix->fixup + fix->fixup;
                regs->iaoq[0] &= ~3;
                /*