powerpc: 64bit optimised __clear_user

I noticed __clear_user high up in a profile of one of my RAID stress
tests. The testcase was doing a dd from /dev/zero which ends up
calling __clear_user.

__clear_user is basically a loop with a single 4 byte store which
is horribly slow. We can do much better by aligning the desination
and doing 32 bytes of 8 byte stores in a loop.

The following testcase was used to verify the patch:

http://ozlabs.org/~anton/junkcode/stress_clear_user.c

To show the improvement in performance I ran a dd from /dev/zero
to /dev/null on a POWER7 box:

Before:

# dd if=/dev/zero of=/dev/null bs=1M count=10000
10485760000 bytes (10 GB) copied, 3.72379 s, 2.8 GB/s

After:

# time dd if=/dev/zero of=/dev/null bs=1M count=10000
10485760000 bytes (10 GB) copied, 0.728318 s, 14.4 GB/s

Over 5x faster.

Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

3 files changed, 144 insertions, 1 deletions

diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index 7735a2c2e6d9..f049e339e456 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -17,7 +17,7 @@ obj-$(CONFIG_HAS_IOMEM)	+= devres.o
 obj-$(CONFIG_PPC64)     += copypage_64.o copyuser_64.o \
                            memcpy_64.o usercopy_64.o mem_64.o string.o \
                            checksum_wrappers_64.o hweight_64.o \
-                           copyuser_power7.o
+                           copyuser_power7.o string_64.o
 obj-$(CONFIG_XMON)      += sstep.o ldstfp.o
 obj-$(CONFIG_KPROBES)   += sstep.o ldstfp.o
 obj-$(CONFIG_HAVE_HW_BREAKPOINT)        += sstep.o ldstfp.o
diff --git a/arch/powerpc/lib/string.S b/arch/powerpc/lib/string.S
index 093d6316435c..1b5a0a09d609 100644
--- a/arch/powerpc/lib/string.S
+++ b/arch/powerpc/lib/string.S
@@ -119,6 +119,7 @@ _GLOBAL(memchr)
 2:      li      r3,0
         blr
 
+#ifdef CONFIG_PPC32
 _GLOBAL(__clear_user)
         addi    r6,r3,-4
         li      r3,0
@@ -160,3 +161,4 @@ _GLOBAL(__clear_user)
         PPC_LONG        1b,91b
         PPC_LONG        8b,92b
         .text
+#endif
diff --git a/arch/powerpc/lib/string_64.S b/arch/powerpc/lib/string_64.S
new file mode 100644
index 000000000000..6613b9047005
--- /dev/null
+++ b/arch/powerpc/lib/string_64.S
@@ -0,0 +1,141 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) IBM Corporation, 2012
+ *
+ * Author: Anton Blanchard <anton@au.ibm.com>
+ */
+
+#include <asm/ppc_asm.h>
+
+/**
+ * __clear_user: - Zero a block of memory in user space, with less checking.
+ * @to:   Destination address, in user space.
+ * @n:    Number of bytes to zero.
+ *
+ * Zero a block of memory in user space.  Caller must check
+ * the specified block with access_ok() before calling this function.
+ *
+ * Returns number of bytes that could not be cleared.
+ * On success, this will be zero.
+ */
+
+        .macro err1
+100:
+        .section __ex_table,"a"
+        .align 3
+        .llong 100b,.Ldo_err1
+        .previous
+        .endm
+
+        .macro err2
+200:
+        .section __ex_table,"a"
+        .align 3
+        .llong 200b,.Ldo_err2
+        .previous
+        .endm
+
+        .macro err3
+300:
+        .section __ex_table,"a"
+        .align 3
+        .llong 300b,.Ldo_err3
+        .previous
+        .endm
+
+.Ldo_err1:
+        mr      r3,r8
+
+.Ldo_err2:
+        mtctr   r4
+1:
+err3;   stb     r0,0(r3)
+        addi    r3,r3,1
+        addi    r4,r4,-1
+        bdnz    1b
+
+.Ldo_err3:
+        mr      r3,r4
+        blr
+
+_GLOBAL(__clear_user)
+        cmpdi   r4,32
+        neg     r6,r3
+        li      r0,0
+        blt     .Lshort_clear
+        mr      r8,r3
+        mtocrf  0x01,r6
+        clrldi  r6,r6,(64-3)
+
+        /* Get the destination 8 byte aligned */
+        bf      cr7*4+3,1f
+err1;   stb     r0,0(r3)
+        addi    r3,r3,1
+
+1:      bf      cr7*4+2,2f
+err1;   sth     r0,0(r3)
+        addi    r3,r3,2
+
+2:      bf      cr7*4+1,3f
+err1;   stw     r0,0(r3)
+        addi    r3,r3,4
+
+3:      sub     r4,r4,r6
+        srdi    r6,r4,5
+        cmpdi   r4,32
+        blt     .Lshort_clear
+        mtctr   r6
+
+        /* Do 32 byte chunks */
+4:
+err2;   std     r0,0(r3)
+err2;   std     r0,8(r3)
+err2;   std     r0,16(r3)
+err2;   std     r0,24(r3)
+        addi    r3,r3,32
+        addi    r4,r4,-32
+        bdnz    4b
+
+.Lshort_clear:
+        /* up to 31 bytes to go */
+        cmpdi   r4,16
+        blt     6f
+err2;   std     r0,0(r3)
+err2;   std     r0,8(r3)
+        addi    r3,r3,16
+        addi    r4,r4,-16
+
+        /* Up to 15 bytes to go */
+6:      mr      r8,r3
+        clrldi  r4,r4,(64-4)
+        mtocrf  0x01,r4
+        bf      cr7*4+0,7f
+err1;   std     r0,0(r3)
+        addi    r3,r3,8
+
+7:      bf      cr7*4+1,8f
+err1;   stw     r0,0(r3)
+        addi    r3,r3,4
+
+8:      bf      cr7*4+2,9f
+err1;   sth     r0,0(r3)
+        addi    r3,r3,2
+
+9:      bf      cr7*4+3,10f
+err1;   stb     r0,0(r3)
+
+10:     li      r3,0
+        blr