1 files changed, 77 insertions, 28 deletions

diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c
index 304e07086ab3..dff631d34b45 100644
--- a/arch/s390/lib/uaccess_pt.c
+++ b/arch/s390/lib/uaccess_pt.c
@@ -14,6 +14,63 @@
 #include <asm/futex.h>
 #include "uaccess.h"
 
+#ifndef CONFIG_64BIT
+#define AHI     "ahi"
+#define SLR     "slr"
+#else
+#define AHI     "aghi"
+#define SLR     "slgr"
+#endif
+
+static size_t strnlen_kernel(size_t count, const char __user *src)
+{
+        register unsigned long reg0 asm("0") = 0UL;
+        unsigned long tmp1, tmp2;
+
+        asm volatile(
+                "   la    %2,0(%1)\n"
+                "   la    %3,0(%0,%1)\n"
+                "  "SLR"  %0,%0\n"
+                "0: srst  %3,%2\n"
+                "   jo    0b\n"
+                "   la    %0,1(%3)\n"   /* strnlen_kernel results includes \0 */
+                "  "SLR"  %0,%1\n"
+                "1:\n"
+                EX_TABLE(0b,1b)
+                : "+a" (count), "+a" (src), "=a" (tmp1), "=a" (tmp2)
+                : "d" (reg0) : "cc", "memory");
+        return count;
+}
+
+static size_t copy_in_kernel(size_t count, void __user *to,
+                             const void __user *from)
+{
+        unsigned long tmp1;
+
+        asm volatile(
+                "  "AHI"  %0,-1\n"
+                "   jo    5f\n"
+                "   bras  %3,3f\n"
+                "0:"AHI"  %0,257\n"
+                "1: mvc   0(1,%1),0(%2)\n"
+                "   la    %1,1(%1)\n"
+                "   la    %2,1(%2)\n"
+                "  "AHI"  %0,-1\n"
+                "   jnz   1b\n"
+                "   j     5f\n"
+                "2: mvc   0(256,%1),0(%2)\n"
+                "   la    %1,256(%1)\n"
+                "   la    %2,256(%2)\n"
+                "3:"AHI"  %0,-256\n"
+                "   jnm   2b\n"
+                "4: ex    %0,1b-0b(%3)\n"
+                "5:"SLR"  %0,%0\n"
+                "6:\n"
+                EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
+                : "+a" (count), "+a" (to), "+a" (from), "=a" (tmp1)
+                : : "cc", "memory");
+        return count;
+}
 
 /*
  * Returns kernel address for user virtual address. If the returned address is
@@ -123,10 +180,8 @@ size_t copy_from_user_pt(size_t n, const void __user *from, void *to)
 {
         size_t rc;
 
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memcpy(to, (void __kernel __force *) from, n);
-                return 0;
-        }
+        if (segment_eq(get_fs(), KERNEL_DS))
+                return copy_in_kernel(n, (void __user *) to, from);
         rc = __user_copy_pt((unsigned long) from, to, n, 0);
         if (unlikely(rc))
                 memset(to + n - rc, 0, rc);
@@ -135,30 +190,28 @@ size_t copy_from_user_pt(size_t n, const void __user *from, void *to)
 
 size_t copy_to_user_pt(size_t n, void __user *to, const void *from)
 {
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memcpy((void __kernel __force *) to, from, n);
-                return 0;
-        }
+        if (segment_eq(get_fs(), KERNEL_DS))
+                return copy_in_kernel(n, to, (void __user *) from);
         return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
 }
 
 static size_t clear_user_pt(size_t n, void __user *to)
 {
+        void *zpage = &empty_zero_page;
         long done, size, ret;
 
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memset((void __kernel __force *) to, 0, n);
-                return 0;
-        }
         done = 0;
         do {
                 if (n - done > PAGE_SIZE)
                         size = PAGE_SIZE;
                 else
                         size = n - done;
-                ret = __user_copy_pt((unsigned long) to + done,
-                                      &empty_zero_page, size, 1);
+                if (segment_eq(get_fs(), KERNEL_DS))
+                        ret = copy_in_kernel(n, to, (void __user *) zpage);
+                else
+                        ret = __user_copy_pt((unsigned long) to, zpage, size, 1);
                 done += size;
+                to += size;
                 if (ret)
                         return ret + n - done;
         } while (done < n);
@@ -175,7 +228,7 @@ static size_t strnlen_user_pt(size_t count, const char __user *src)
         if (unlikely(!count))
                 return 0;
         if (segment_eq(get_fs(), KERNEL_DS))
-                return strnlen((const char __kernel __force *) src, count) + 1;
+                return strnlen_kernel(count, src);
         done = 0;
 retry:
         spin_lock(&mm->page_table_lock);
@@ -206,19 +259,17 @@ static size_t strncpy_from_user_pt(size_t count, const char __user *src,
 
         if (unlikely(!count))
                 return 0;
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                len = strnlen((const char __kernel __force *) src, count) + 1;
-                if (len > count)
-                        len = count;
-                memcpy(dst, (const char __kernel __force *) src, len);
-                return (dst[len - 1] == '\0') ? len - 1 : len;
-        }
         done = 0;
         do {
                 offset = (size_t)src & ~PAGE_MASK;
                 len = min(count - done, PAGE_SIZE - offset);
-                if (__user_copy_pt((unsigned long) src, dst, len, 0))
-                        return -EFAULT;
+                if (segment_eq(get_fs(), KERNEL_DS)) {
+                        if (copy_in_kernel(len, (void __user *) dst, src))
+                                return -EFAULT;
+                } else {
+                        if (__user_copy_pt((unsigned long) src, dst, len, 0))
+                                return -EFAULT;
+                }
                 len_str = strnlen(dst, len);
                 done += len_str;
                 src += len_str;
@@ -237,10 +288,8 @@ static size_t copy_in_user_pt(size_t n, void __user *to,
         unsigned long kaddr_to, kaddr_from;
         int write_user;
 
-        if (segment_eq(get_fs(), KERNEL_DS)) {
-                memcpy((void __force *) to, (void __force *) from, n);
-                return 0;
-        }
+        if (segment_eq(get_fs(), KERNEL_DS))
+                return copy_in_kernel(n, to, from);
         done = 0;
 retry:
         spin_lock(&mm->page_table_lock);