[PATCH] x86_64: Clean up copy_*_user

- Remove optimization for old B stepping Opteron
- Make the fast path for copies with a multiple of eight length faster.
- Minor instruction rearrangement to hopefully avoid a pipeline
stall or two.
- Add comment about errata to consider.

Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

1 files changed, 23 insertions, 221 deletions

diff --git a/arch/x86_64/lib/copy_user.S b/arch/x86_64/lib/copy_user.S
index dfa358b05c89..79422b6559c3 100644
--- a/arch/x86_64/lib/copy_user.S
+++ b/arch/x86_64/lib/copy_user.S
@@ -4,12 +4,9 @@
  * Functions to copy from and to user space.            
  */              
 
-#define FIX_ALIGNMENT 1
-                
         #include <asm/current.h>
         #include <asm/asm-offsets.h>
         #include <asm/thread_info.h>
-        #include <asm/cpufeature.h>
 
 /* Standard copy_to_user with segment limit checking */         
         .globl copy_to_user
@@ -21,23 +18,7 @@ copy_to_user:
         jc  bad_to_user
         cmpq threadinfo_addr_limit(%rax),%rcx
         jae bad_to_user
-2:      
-        .byte 0xe9      /* 32bit jump */
-        .long .Lcug-1f
-1:
-
-        .section .altinstr_replacement,"ax"
-3:      .byte 0xe9                      /* replacement jmp with 8 bit immediate */
-        .long copy_user_generic_c-1b    /* offset */
-        .previous
-        .section .altinstructions,"a"
-        .align 8
-        .quad  2b
-        .quad  3b
-        .byte  X86_FEATURE_K8_C
-        .byte  5
-        .byte  5
-        .previous
+        jmp copy_user_generic
 
 /* Standard copy_from_user with segment limit checking */       
         .globl copy_from_user
@@ -72,223 +53,44 @@ bad_to_user:
  * rsi source
  * rdx count
  *
+ * Only 4GB of copy is supported. This shouldn't be a problem
+ * because the kernel normally only writes from/to page sized chunks
+ * even if user space passed a longer buffer.
+ * And more would be dangerous because both Intel and AMD have
+ * errata with rep movsq > 4GB. If someone feels the need to fix
+ * this please consider this.
+ *
  * Output:              
  * eax uncopied bytes or 0 if successful.
  */
-        .globl copy_user_generic        
-        .p2align 4
-copy_user_generic:      
-        .byte 0x66,0x66,0x90    /* 5 byte nop for replacement jump */   
-        .byte 0x66,0x90
-1:              
-        .section .altinstr_replacement,"ax"
-2:      .byte 0xe9                   /* near jump with 32bit immediate */
-        .long copy_user_generic_c-1b /* offset */
-        .previous
-        .section .altinstructions,"a"
-        .align 8
-        .quad  copy_user_generic
-        .quad  2b
-        .byte  X86_FEATURE_K8_C
-        .byte  5
-        .byte  5
-        .previous
-.Lcug:  
-        pushq %rbx
-        xorl %eax,%eax          /*zero for the exception handler */
-
-#ifdef FIX_ALIGNMENT
-        /* check for bad alignment of destination */
-        movl %edi,%ecx
-        andl $7,%ecx
-        jnz  .Lbad_alignment
-.Lafter_bad_alignment:
-#endif
 
-        movq %rdx,%rcx
-
-        movl $64,%ebx   
-        shrq $6,%rdx
-        decq %rdx
-        js   .Lhandle_tail
-        
-        .p2align 4
-.Lloop:
-.Ls1:   movq (%rsi),%r11
-.Ls2:   movq 1*8(%rsi),%r8
-.Ls3:   movq 2*8(%rsi),%r9
-.Ls4:   movq 3*8(%rsi),%r10
-.Ld1:   movq %r11,(%rdi)
-.Ld2:   movq %r8,1*8(%rdi)
-.Ld3:   movq %r9,2*8(%rdi)
-.Ld4:   movq %r10,3*8(%rdi)
-                
-.Ls5:   movq 4*8(%rsi),%r11
-.Ls6:   movq 5*8(%rsi),%r8
-.Ls7:   movq 6*8(%rsi),%r9
-.Ls8:   movq 7*8(%rsi),%r10
-.Ld5:   movq %r11,4*8(%rdi)
-.Ld6:   movq %r8,5*8(%rdi)
-.Ld7:   movq %r9,6*8(%rdi)
-.Ld8:   movq %r10,7*8(%rdi)
-        
-        decq %rdx
-
-        leaq 64(%rsi),%rsi
-        leaq 64(%rdi),%rdi
-        
-        jns  .Lloop
-
-        .p2align 4
-.Lhandle_tail:
-        movl %ecx,%edx
-        andl $63,%ecx
-        shrl $3,%ecx
-        jz   .Lhandle_7
-        movl $8,%ebx
-        .p2align 4
-.Lloop_8:
-.Ls9:   movq (%rsi),%r8
-.Ld9:   movq %r8,(%rdi)
-        decl %ecx
-        leaq 8(%rdi),%rdi
-        leaq 8(%rsi),%rsi
-        jnz .Lloop_8
-        
-.Lhandle_7:             
-        movl %edx,%ecx  
-        andl $7,%ecx
-        jz   .Lende
-        .p2align 4
-.Lloop_1:
-.Ls10:  movb (%rsi),%bl
-.Ld10:  movb %bl,(%rdi)
-        incq %rdi
-        incq %rsi
-        decl %ecx
-        jnz .Lloop_1
-                        
-.Lende:
-        popq %rbx
-        ret     
-
-#ifdef FIX_ALIGNMENT                            
-        /* align destination */
-        .p2align 4
-.Lbad_alignment:
-        movl $8,%r9d
-        subl %ecx,%r9d
-        movl %r9d,%ecx
-        cmpq %r9,%rdx
-        jz   .Lhandle_7
-        js   .Lhandle_7
-.Lalign_1:              
-.Ls11:  movb (%rsi),%bl
-.Ld11:  movb %bl,(%rdi)
-        incq %rsi
-        incq %rdi
-        decl %ecx
-        jnz .Lalign_1
-        subq %r9,%rdx
-        jmp .Lafter_bad_alignment
-#endif
-        
-        /* table sorted by exception address */ 
-        .section __ex_table,"a"
-        .align 8
-        .quad .Ls1,.Ls1e
-        .quad .Ls2,.Ls2e
-        .quad .Ls3,.Ls3e
-        .quad .Ls4,.Ls4e        
-        .quad .Ld1,.Ls1e
-        .quad .Ld2,.Ls2e
-        .quad .Ld3,.Ls3e
-        .quad .Ld4,.Ls4e
-        .quad .Ls5,.Ls5e
-        .quad .Ls6,.Ls6e
-        .quad .Ls7,.Ls7e
-        .quad .Ls8,.Ls8e        
-        .quad .Ld5,.Ls5e
-        .quad .Ld6,.Ls6e
-        .quad .Ld7,.Ls7e
-        .quad .Ld8,.Ls8e
-        .quad .Ls9,.Le_quad
-        .quad .Ld9,.Le_quad
-        .quad .Ls10,.Le_byte
-        .quad .Ld10,.Le_byte
-#ifdef FIX_ALIGNMENT    
-        .quad .Ls11,.Lzero_rest
-        .quad .Ld11,.Lzero_rest
-#endif
-        .quad .Le5,.Le_zero
-        .previous
-
-        /* compute 64-offset for main loop. 8 bytes accuracy with error on the 
-           pessimistic side. this is gross. it would be better to fix the 
-           interface. */        
-        /* eax: zero, ebx: 64 */
-.Ls1e:  addl $8,%eax
-.Ls2e:  addl $8,%eax
-.Ls3e:  addl $8,%eax
-.Ls4e:  addl $8,%eax
-.Ls5e:  addl $8,%eax
-.Ls6e:  addl $8,%eax
-.Ls7e:  addl $8,%eax
-.Ls8e:  addl $8,%eax
-        addq %rbx,%rdi  /* +64 */
-        subq %rax,%rdi  /* correct destination with computed offset */
-
-        shlq $6,%rdx    /* loop counter * 64 (stride length) */
-        addq %rax,%rdx  /* add offset to loopcnt */
-        andl $63,%ecx   /* remaining bytes */
-        addq %rcx,%rdx  /* add them */
-        jmp .Lzero_rest
-
-        /* exception on quad word loop in tail handling */
-        /* ecx: loopcnt/8, %edx: length, rdi: correct */
-.Le_quad:
-        shll $3,%ecx
-        andl $7,%edx
-        addl %ecx,%edx
-        /* edx: bytes to zero, rdi: dest, eax:zero */
-.Lzero_rest:
-        movq %rdx,%rcx
-.Le_byte:
-        xorl %eax,%eax
-.Le5:   rep 
-        stosb
-        /* when there is another exception while zeroing the rest just return */
-.Le_zero:               
-        movq %rdx,%rax
-        jmp .Lende
-
-        /* C stepping K8 run faster using the string copy instructions.
-           This is also a lot simpler. Use them when possible.
-           Patch in jmps to this code instead of copying it fully
-           to avoid unwanted aliasing in the exception tables. */
-                
- /* rdi destination
-  * rsi source
-  * rdx count
-  *
-  * Output:             
-  * eax uncopied bytes or 0 if successfull.
-  */                    
-copy_user_generic_c:
+        .globl copy_user_generic
+copy_user_generic:
         movl %edx,%ecx
         shrl $3,%ecx
         andl $7,%edx    
+        jz   5f
 1:      rep 
         movsq 
         movl %edx,%ecx
+        xor %eax,%eax
 2:      rep
         movsb
-4:      movl %ecx,%eax
         ret
+        /* align here? */
+5:      xorl %eax,%eax
+6:      rep movsq
+        ret
+
+        .section .fixup,"ax"
 3:      lea (%rdx,%rcx,8),%rax
         ret
-        
+4:      movl %ecx,%eax
+        ret
+        .previous
+
         .section __ex_table,"a"
         .quad 1b,3b
         .quad 2b,4b
+        .quad 6b,4b
         .previous