2 files changed, 35 insertions, 33 deletions

diff --git a/arch/s390/lib/uaccess.S b/arch/s390/lib/uaccess.S
index 5d59e2625048..837275284d9f 100644
--- a/arch/s390/lib/uaccess.S
+++ b/arch/s390/lib/uaccess.S
@@ -88,30 +88,31 @@ __copy_to_user_asm:
         .globl __copy_in_user_asm
         # %r2 = from, %r3 = n, %r4 = to
 __copy_in_user_asm:
+        ahi     %r3,-1
+        jo      6f
         sacf    256
-        bras    1,1f
-        mvc     0(1,%r4),0(%r2)
-0:      mvc     0(256,%r4),0(%r2)
-        la      %r2,256(%r2)
-        la      %r4,256(%r4)
-1:      ahi     %r3,-256
-        jnm     0b
-2:      ex      %r3,0(%r1)
-        sacf    0
-        slr     %r2,%r2
-        br      14
-3:      mvc     0(1,%r4),0(%r2)
+        bras    %r1,4f
+0:      ahi     %r3,257
+1:      mvc     0(1,%r4),0(%r2)
         la      %r2,1(%r2)
         la      %r4,1(%r4)
         ahi     %r3,-1
+        jnz     1b
+2:      lr      %r2,%r3
+        br      %r14
+3:      mvc     0(256,%r4),0(%r2)
+        la      %r2,256(%r2)
+        la      %r4,256(%r4)
+4:      ahi     %r3,-256
         jnm     3b
-4:      lr      %r2,%r3
+5:      ex      %r3,4(%r1)
         sacf    0
+6:      slr     %r2,%r2
         br      %r14
         .section __ex_table,"a"
-        .long   0b,3b
-        .long   2b,3b
-        .long   3b,4b
+        .long   1b,2b
+        .long   3b,0b
+        .long   5b,0b
         .previous
 
         .align 4
diff --git a/arch/s390/lib/uaccess64.S b/arch/s390/lib/uaccess64.S
index 19b41a33c230..1f755be22f92 100644
--- a/arch/s390/lib/uaccess64.S
+++ b/arch/s390/lib/uaccess64.S
@@ -88,30 +88,31 @@ __copy_to_user_asm:
         .globl __copy_in_user_asm
         # %r2 = from, %r3 = n, %r4 = to
 __copy_in_user_asm:
+        aghi    %r3,-1
+        jo      6f
         sacf    256
-        bras    1,1f
-        mvc     0(1,%r4),0(%r2)
-0:      mvc     0(256,%r4),0(%r2)
-        la      %r2,256(%r2)
-        la      %r4,256(%r4)
-1:      aghi    %r3,-256
-        jnm     0b
-2:      ex      %r3,0(%r1)
-        sacf    0
-        slgr    %r2,%r2
-        br      14
-3:      mvc     0(1,%r4),0(%r2)
+        bras    %r1,4f
+0:      aghi    %r3,257
+1:      mvc     0(1,%r4),0(%r2)
         la      %r2,1(%r2)
         la      %r4,1(%r4)
         aghi    %r3,-1
+        jnz     1b
+2:      lgr     %r2,%r3
+        br      %r14
+3:      mvc     0(256,%r4),0(%r2)
+        la      %r2,256(%r2)
+        la      %r4,256(%r4)
+4:      aghi    %r3,-256
         jnm     3b
-4:      lgr     %r2,%r3
+5:      ex      %r3,4(%r1)
         sacf    0
-        br      %r14
+6:      slgr    %r2,%r2
+        br      14
         .section __ex_table,"a"
-        .quad   0b,3b
-        .quad   2b,3b
-        .quad   3b,4b
+        .quad   1b,2b
+        .quad   3b,0b
+        .quad   5b,0b
         .previous
 
         .align 4