3 files changed, 124 insertions, 79 deletions

diff --git a/arch/s390/lib/uaccess_mvcos.c b/arch/s390/lib/uaccess_mvcos.c
index 2443ae476e33..1829742bf479 100644
--- a/arch/s390/lib/uaccess_mvcos.c
+++ b/arch/s390/lib/uaccess_mvcos.c
@@ -162,19 +162,19 @@ static size_t clear_user_mvcos(size_t size, void __user *to)
 
 static size_t strnlen_user_mvcos(size_t count, const char __user *src)
 {
+        size_t done, len, offset, len_str;
         char buf[256];
-        int rc;
-        size_t done, len, len_str;
 
         done = 0;
         do {
-                len = min(count - done, (size_t) 256);
-                rc = uaccess.copy_from_user(len, src + done, buf);
-                if (unlikely(rc == len))
+                offset = (size_t)src & ~PAGE_MASK;
+                len = min(256UL, PAGE_SIZE - offset);
+                len = min(count - done, len);
+                if (copy_from_user_mvcos(len, src, buf))
                         return 0;
-                len -= rc;
                 len_str = strnlen(buf, len);
                 done += len_str;
+                src += len_str;
         } while ((len_str == len) && (done < count));
         return done + 1;
 }
@@ -182,18 +182,20 @@ static size_t strnlen_user_mvcos(size_t count, const char __user *src)
 static size_t strncpy_from_user_mvcos(size_t count, const char __user *src,
                                       char *dst)
 {
-        int rc;
-        size_t done, len, len_str;
+        size_t done, len, offset, len_str;
 
+        if (unlikely(!count))
+                return 0;
         done = 0;
         do {
-                len = min(count - done, (size_t) 4096);
-                rc = uaccess.copy_from_user(len, src + done, dst);
-                if (unlikely(rc == len))
+                offset = (size_t)src & ~PAGE_MASK;
+                len = min(count - done, PAGE_SIZE - offset);
+                if (copy_from_user_mvcos(len, src, dst))
                         return -EFAULT;
-                len -= rc;
                 len_str = strnlen(dst, len);
                 done += len_str;
+                src += len_str;
+                dst += len_str;
         } while ((len_str == len) && (done < count));
         return done;
 }
diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c
index a70ee84c0241..dff631d34b45 100644
--- a/arch/s390/lib/uaccess_pt.c
+++ b/arch/s390/lib/uaccess_pt.c
@@ -14,6 +14,63 @@
 #include <asm/futex.h>
 #include "uaccess.h"
 
+#ifndef CONFIG_64BIT
+#define AHI     "ahi"
+#define SLR     "slr"
+#else
+#define AHI     "aghi"
+#define SLR     "slgr"
+#endif
+
+static size_t strnlen_kernel(size_t count, const char __user *src)
+{
+        register unsigned long reg0 asm("0") = 0UL;
+        unsigned long tmp1, tmp2;
+
+        asm volatile(
+                "   la    %2,0(%1)\n"
+                "   la    %3,0(%0,%1)\n"
+                "  "SLR"  %0,%0\n"
+                "0: srst  %3,%2\n"
+                "   jo    0b\n"
+                "   la    %0,1(%3)\n"   /* strnlen_kernel results includes \0 */
+                "  "SLR"  %0,%1\n"
+                "1:\n"
+                EX_TABLE(0b,1b)
+                : "+a" (count), "+a" (src), "=a" (tmp1), "=a" (tmp2)
+                : "d" (reg0) : "cc", "memory");
+        return count;
+}
+
+static size_t copy_in_kernel(size_t count, void __user *to,
+                             const void __user *from)
+{
+        unsigned long tmp1;
+
+        asm volatile(
+                "  "AHI"  %0,-1\n"
+                "   jo    5f\n"
+                "   bras  %3,3f\n"
+                "0:"AHI"  %0,257\n"
+                "1: mvc   0(1,%1),0(%2)\n"
+                "   la    %1,1(%1)\n"
+                "   la    %2,1(%2)\n"
+                "  "AHI"  %0,-1\n"
+                "   jnz   1b\n"
+                "   j     5f\n"
+                "2: mvc   0(256,%1),0(%2)\n"
+                "   la    %1,256(%1)\n"
+                "   la    %2,256(%2)\n"
+                "3:"AHI"  %0,-256\n"
+                "   jnm   2b\n"
+                "4: ex    %0,1b-0b(%3)\n"
+                "5:"SLR"  %0,%0\n"
+                "6:\n"
+                EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
+                : "+a" (count), "+a" (to), "+a" (from), "=a" (tmp1)
+                : : "cc", "memory");
+        return count;
+}
 
 /*
  * Returns kernel address for user virtual address. If the returned address is
@@ -123,10 +180,8 @@ size_t copy_from_user_pt(size_t n, const void __user *from, void *to)
 {
         size_t rc;
 
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memcpy(to, (void __kernel __force *) from, n);
-                return 0;
-        }
+        if (segment_eq(get_fs(), KERNEL_DS))
+                return copy_in_kernel(n, (void __user *) to, from);
         rc = __user_copy_pt((unsigned long) from, to, n, 0);
         if (unlikely(rc))
                 memset(to + n - rc, 0, rc);
@@ -135,30 +190,28 @@ size_t copy_from_user_pt(size_t n, const void __user *from, void *to)
 
 size_t copy_to_user_pt(size_t n, void __user *to, const void *from)
 {
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memcpy((void __kernel __force *) to, from, n);
-                return 0;
-        }
+        if (segment_eq(get_fs(), KERNEL_DS))
+                return copy_in_kernel(n, to, (void __user *) from);
         return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
 }
 
 static size_t clear_user_pt(size_t n, void __user *to)
 {
+        void *zpage = &empty_zero_page;
         long done, size, ret;
 
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memset((void __kernel __force *) to, 0, n);
-                return 0;
-        }
         done = 0;
         do {
                 if (n - done > PAGE_SIZE)
                         size = PAGE_SIZE;
                 else
                         size = n - done;
-                ret = __user_copy_pt((unsigned long) to + done,
-                                      &empty_zero_page, size, 1);
+                if (segment_eq(get_fs(), KERNEL_DS))
+                        ret = copy_in_kernel(n, to, (void __user *) zpage);
+                else
+                        ret = __user_copy_pt((unsigned long) to, zpage, size, 1);
                 done += size;
+                to += size;
                 if (ret)
                         return ret + n - done;
         } while (done < n);
@@ -172,8 +225,10 @@ static size_t strnlen_user_pt(size_t count, const char __user *src)
         unsigned long offset, done, len, kaddr;
         size_t len_str;
 
+        if (unlikely(!count))
+                return 0;
         if (segment_eq(get_fs(), KERNEL_DS))
-                return strnlen((const char __kernel __force *) src, count) + 1;
+                return strnlen_kernel(count, src);
         done = 0;
 retry:
         spin_lock(&mm->page_table_lock);
@@ -200,25 +255,27 @@ fault:
 static size_t strncpy_from_user_pt(size_t count, const char __user *src,
                                    char *dst)
 {
-        size_t n = strnlen_user_pt(count, src);
+        size_t done, len, offset, len_str;
 
-        if (!n)
-                return -EFAULT;
-        if (n > count)
-                n = count;
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memcpy(dst, (const char __kernel __force *) src, n);
-                if (dst[n-1] == '\0')
-                        return n-1;
-                else
-                        return n;
-        }
-        if (__user_copy_pt((unsigned long) src, dst, n, 0))
-                return -EFAULT;
-        if (dst[n-1] == '\0')
-                return n-1;
-        else
-                return n;
+        if (unlikely(!count))
+                return 0;
+        done = 0;
+        do {
+                offset = (size_t)src & ~PAGE_MASK;
+                len = min(count - done, PAGE_SIZE - offset);
+                if (segment_eq(get_fs(), KERNEL_DS)) {
+                        if (copy_in_kernel(len, (void __user *) dst, src))
+                                return -EFAULT;
+                } else {
+                        if (__user_copy_pt((unsigned long) src, dst, len, 0))
+                                return -EFAULT;
+                }
+                len_str = strnlen(dst, len);
+                done += len_str;
+                src += len_str;
+                dst += len_str;
+        } while ((len_str == len) && (done < count));
+        return done;
 }
 
 static size_t copy_in_user_pt(size_t n, void __user *to,
@@ -231,10 +288,8 @@ static size_t copy_in_user_pt(size_t n, void __user *to,
         unsigned long kaddr_to, kaddr_from;
         int write_user;
 
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memcpy((void __force *) to, (void __force *) from, n);
-                return 0;
-        }
+        if (segment_eq(get_fs(), KERNEL_DS))
+                return copy_in_kernel(n, to, from);
         done = 0;
 retry:
         spin_lock(&mm->page_table_lock);
diff --git a/arch/s390/lib/uaccess_std.c b/arch/s390/lib/uaccess_std.c
index 6fbd06338270..4a75d475b06a 100644
--- a/arch/s390/lib/uaccess_std.c
+++ b/arch/s390/lib/uaccess_std.c
@@ -188,6 +188,8 @@ size_t strnlen_user_std(size_t size, const char __user *src)
         register unsigned long reg0 asm("0") = 0UL;
         unsigned long tmp1, tmp2;
 
+        if (unlikely(!size))
+                return 0;
         asm volatile(
                 "   la    %2,0(%1)\n"
                 "   la    %3,0(%0,%1)\n"
@@ -204,38 +206,24 @@ size_t strnlen_user_std(size_t size, const char __user *src)
         return size;
 }
 
-size_t strncpy_from_user_std(size_t size, const char __user *src, char *dst)
+size_t strncpy_from_user_std(size_t count, const char __user *src, char *dst)
 {
-        register unsigned long reg0 asm("0") = 0UL;
-        unsigned long tmp1, tmp2;
+        size_t done, len, offset, len_str;
 
-        asm volatile(
-                "   la    %3,0(%1)\n"
-                "   la    %4,0(%0,%1)\n"
-                "   sacf  256\n"
-                "0: srst  %4,%3\n"
-                "   jo    0b\n"
-                "   sacf  0\n"
-                "   la    %0,0(%4)\n"
-                "   jh    1f\n"         /* found \0 in string ? */
-                "  "AHI"  %4,1\n"       /* include \0 in copy */
-                "1:"SLR"  %0,%1\n"      /* %0 = return length (without \0) */
-                "  "SLR"  %4,%1\n"      /* %4 = copy length (including \0) */
-                "2: mvcp  0(%4,%2),0(%1),%5\n"
-                "   jz    9f\n"
-                "3:"AHI"  %4,-256\n"
-                "   la    %1,256(%1)\n"
-                "   la    %2,256(%2)\n"
-                "4: mvcp  0(%4,%2),0(%1),%5\n"
-                "   jnz   3b\n"
-                "   j     9f\n"
-                "7: sacf  0\n"
-                "8:"LHI"  %0,%6\n"
-                "9:\n"
-                EX_TABLE(0b,7b) EX_TABLE(2b,8b) EX_TABLE(4b,8b)
-                : "+a" (size), "+a" (src), "+d" (dst), "=a" (tmp1), "=a" (tmp2)
-                : "d" (reg0), "K" (-EFAULT) : "cc", "memory");
-        return size;
+        if (unlikely(!count))
+                return 0;
+        done = 0;
+        do {
+                offset = (size_t)src & ~PAGE_MASK;
+                len = min(count - done, PAGE_SIZE - offset);
+                if (copy_from_user_std(len, src, dst))
+                        return -EFAULT;
+                len_str = strnlen(dst, len);
+                done += len_str;
+                src += len_str;
+                dst += len_str;
+        } while ((len_str == len) && (done < count));
+        return done;
 }
 
 #define __futex_atomic_op(insn, ret, oldval, newval, uaddr, oparg)      \