aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/lib/memcpy_64.S
diff options
context:
space:
mode:
authorMark Nelson <markn@au1.ibm.com>2008-10-26 20:46:51 -0400
committerPaul Mackerras <paulus@samba.org>2008-11-05 06:08:29 -0500
commit25d6e2d7c58ddc4a3b614fc5381591c0cfe66556 (patch)
tree0d6739ffe7c9a87504f00198b207ecff40b884ea /arch/powerpc/lib/memcpy_64.S
parent4ec577a28980a0790df3c3dfe9c81f6e2222acfb (diff)
powerpc: Update 64bit memcpy() using CPU_FTR_UNALIGNED_LD_STD
Update memcpy() to add two new feature sections: one for aligning the destination before copying and one for copying using aligned load and store doubles. These new feature sections will only affect Power6 and Cell because the CPU feature bit was only added to these two processors. Power6 gets its best performance in memcpy() when aligning neither the source nor the destination, while Cell gets its best performance when just the destination is aligned. But in order to save on CPU feature bits we can use the previously added CPU_FTR_CP_USE_DCBTZ feature bit to differentiate between Power6 and Cell (because CPU_FTR_CP_USE_DCBTZ was added to Cell but not Power6). The first feature section acts to nop out the branch that takes us to the code that aligns us to an eight byte boundary for the destination. We only want to nop out this branch on Power6. So the ALT_FTR_SECTION_END() for this feature section creates a test mask of the two feature bits ORed together and provides an expected result of just CPU_FTR_UNALIGNED_LD_STD, thus we nop out the branch if we're on a CPU that has CPU_FTR_UNALIGNED_LD_STD set and CPU_FTR_CP_USE_DCBTZ unset. For the second feature section added, if we're on a CPU that has the CPU_FTR_UNALIGNED_LD_STD bit set then we don't want to do the copy with aligned loads and stores (and the appropriate shifting left and right instructions), so we want to nop out the branch to .Lsrc_unaligned. The andi. used for this branch is moved to just above the branch because this allows us to nop out both instructions with just one feature section which gives us better performance and doesn't hurt readability which two separate feature sections did. Moving the andi. to just above the branch doesn't have any noticeable negative effect on the remaining 64bit processors (the ones that didn't have this feature bit added). On Cell this simple modification results in an improvement to measured memcpy() bandwidth of up to 50% in the hot cache case and up to 15% in the cold cache case. On Power6 we get memory bandwidth results that are up to three times faster in the hot cache case and up to 50% faster in the cold cache case. Commit 2a9294369bd020db89bfdf78b84c3615b39a5c84 ("powerpc: Add new CPU feature: CPU_FTR_CP_USE_DCBTZ") was where CPU_FTR_CP_USE_DCBTZ was added. To say that Cell gets its best performance in memcpy() with just the destination aligned is true but only for the reason that the indirect shift and rotate instructions, sld and srd, are microcoded on Cell. This means that either the destination or the source can be aligned, but not both, and seeing as we get better performance with the destination aligned we choose this option. While we're at it make a one line change from cmpldi r1,... to cmpldi cr1,... for consistency. Signed-off-by: Mark Nelson <markn@au1.ibm.com> Signed-off-by: Paul Mackerras <paulus@samba.org>
Diffstat (limited to 'arch/powerpc/lib/memcpy_64.S')
-rw-r--r--arch/powerpc/lib/memcpy_64.S16
1 files changed, 14 insertions, 2 deletions
diff --git a/arch/powerpc/lib/memcpy_64.S b/arch/powerpc/lib/memcpy_64.S
index 3f131129d1c1..fe2d34e5332d 100644
--- a/arch/powerpc/lib/memcpy_64.S
+++ b/arch/powerpc/lib/memcpy_64.S
@@ -18,11 +18,23 @@ _GLOBAL(memcpy)
18 andi. r6,r6,7 18 andi. r6,r6,7
19 dcbt 0,r4 19 dcbt 0,r4
20 blt cr1,.Lshort_copy 20 blt cr1,.Lshort_copy
21/* Below we want to nop out the bne if we're on a CPU that has the
22 CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
23 cleared.
24 At the time of writing the only CPU that has this combination of bits
25 set is Power6. */
26BEGIN_FTR_SECTION
27 nop
28FTR_SECTION_ELSE
21 bne .Ldst_unaligned 29 bne .Ldst_unaligned
30ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \
31 CPU_FTR_UNALIGNED_LD_STD)
22.Ldst_aligned: 32.Ldst_aligned:
23 andi. r0,r4,7
24 addi r3,r3,-16 33 addi r3,r3,-16
34BEGIN_FTR_SECTION
35 andi. r0,r4,7
25 bne .Lsrc_unaligned 36 bne .Lsrc_unaligned
37END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
26 srdi r7,r5,4 38 srdi r7,r5,4
27 ld r9,0(r4) 39 ld r9,0(r4)
28 addi r4,r4,-8 40 addi r4,r4,-8
@@ -131,7 +143,7 @@ _GLOBAL(memcpy)
131 PPC_MTOCRF 0x01,r6 # put #bytes to 8B bdry into cr7 143 PPC_MTOCRF 0x01,r6 # put #bytes to 8B bdry into cr7
132 subf r5,r6,r5 144 subf r5,r6,r5
133 li r7,0 145 li r7,0
134 cmpldi r1,r5,16 146 cmpldi cr1,r5,16
135 bf cr7*4+3,1f 147 bf cr7*4+3,1f
136 lbz r0,0(r4) 148 lbz r0,0(r4)
137 stb r0,0(r3) 149 stb r0,0(r3)