aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/lib
diff options
context:
space:
mode:
authorMa Ling <ling.ma@intel.com>2010-09-16 15:12:40 -0400
committerH. Peter Anvin <hpa@zytor.com>2010-09-24 21:57:11 -0400
commit3b4b682becdfa9f42321aa024d5cc84f71f06d8c (patch)
tree807cac0bfbc45d45f156dae742b31f9a66aaa559 /arch/x86/lib
parent59daa706fbec745684702741b9f5373142dd9fdc (diff)
x86, mem: Optimize memmove for small size and unaligned cases
movs instruction will combine data to accelerate moving data, however we need to concern two cases about it. 1. movs instruction need long lantency to startup, so here we use general mov instruction to copy data. 2. movs instruction is not good for unaligned case, even if src offset is 0x10, dest offset is 0x0, we avoid and handle the case by general mov instruction. Signed-off-by: Ma Ling <ling.ma@intel.com> LKML-Reference: <1284664360-6138-1-git-send-email-ling.ma@intel.com> Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Diffstat (limited to 'arch/x86/lib')
-rw-r--r--arch/x86/lib/memcpy_32.c213
-rw-r--r--arch/x86/lib/memmove_64.c225
2 files changed, 362 insertions, 76 deletions
diff --git a/arch/x86/lib/memcpy_32.c b/arch/x86/lib/memcpy_32.c
index 81130d477ee2..b908a59eccf5 100644
--- a/arch/x86/lib/memcpy_32.c
+++ b/arch/x86/lib/memcpy_32.c
@@ -22,36 +22,187 @@ EXPORT_SYMBOL(memset);
22 22
23void *memmove(void *dest, const void *src, size_t n) 23void *memmove(void *dest, const void *src, size_t n)
24{ 24{
25 int d0, d1, d2; 25 int d0,d1,d2,d3,d4,d5;
26 26 char *ret = dest;
27 if (dest < src) { 27
28 if ((dest + n) < src) 28 __asm__ __volatile__(
29 return memcpy(dest, src, n); 29 /* Handle more 16bytes in loop */
30 else 30 "cmp $0x10, %0\n\t"
31 __asm__ __volatile__( 31 "jb 1f\n\t"
32 "rep\n\t" 32
33 "movsb\n\t" 33 /* Decide forward/backward copy mode */
34 : "=&c" (d0), "=&S" (d1), "=&D" (d2) 34 "cmp %2, %1\n\t"
35 :"0" (n), 35 "jb 2f\n\t"
36 "1" (src), 36
37 "2" (dest) 37 /*
38 :"memory"); 38 * movs instruction have many startup latency
39 } else { 39 * so we handle small size by general register.
40 if((src + n) < dest) 40 */
41 return memcpy(dest, src, n); 41 "cmp $680, %0\n\t"
42 else 42 "jb 3f\n\t"
43 __asm__ __volatile__( 43 /*
44 "std\n\t" 44 * movs instruction is only good for aligned case.
45 "rep\n\t" 45 */
46 "movsb\n\t" 46 "mov %1, %3\n\t"
47 "cld" 47 "xor %2, %3\n\t"
48 : "=&c" (d0), "=&S" (d1), "=&D" (d2) 48 "and $0xff, %3\n\t"
49 :"0" (n), 49 "jz 4f\n\t"
50 "1" (n-1+src), 50 "3:\n\t"
51 "2" (n-1+dest) 51 "sub $0x10, %0\n\t"
52 :"memory"); 52
53 } 53 /*
54 54 * We gobble 16byts forward in each loop.
55 return dest; 55 */
56 "3:\n\t"
57 "sub $0x10, %0\n\t"
58 "mov 0*4(%1), %3\n\t"
59 "mov 1*4(%1), %4\n\t"
60 "mov %3, 0*4(%2)\n\t"
61 "mov %4, 1*4(%2)\n\t"
62 "mov 2*4(%1), %3\n\t"
63 "mov 3*4(%1), %4\n\t"
64 "mov %3, 2*4(%2)\n\t"
65 "mov %4, 3*4(%2)\n\t"
66 "lea 0x10(%1), %1\n\t"
67 "lea 0x10(%2), %2\n\t"
68 "jae 3b\n\t"
69 "add $0x10, %0\n\t"
70 "jmp 1f\n\t"
71
72 /*
73 * Handle data forward by movs.
74 */
75 ".p2align 4\n\t"
76 "4:\n\t"
77 "mov -4(%1, %0), %3\n\t"
78 "lea -4(%2, %0), %4\n\t"
79 "shr $2, %0\n\t"
80 "rep movsl\n\t"
81 "mov %3, (%4)\n\t"
82 "jmp 11f\n\t"
83 /*
84 * Handle data backward by movs.
85 */
86 ".p2align 4\n\t"
87 "6:\n\t"
88 "mov (%1), %3\n\t"
89 "mov %2, %4\n\t"
90 "lea -4(%1, %0), %1\n\t"
91 "lea -4(%2, %0), %2\n\t"
92 "shr $2, %0\n\t"
93 "std\n\t"
94 "rep movsl\n\t"
95 "mov %3,(%4)\n\t"
96 "cld\n\t"
97 "jmp 11f\n\t"
98
99 /*
100 * Start to prepare for backward copy.
101 */
102 ".p2align 4\n\t"
103 "2:\n\t"
104 "cmp $680, %0\n\t"
105 "jb 5f\n\t"
106 "mov %1, %3\n\t"
107 "xor %2, %3\n\t"
108 "and $0xff, %3\n\t"
109 "jz 6b\n\t"
110
111 /*
112 * Calculate copy position to tail.
113 */
114 "5:\n\t"
115 "add %0, %1\n\t"
116 "add %0, %2\n\t"
117 "sub $0x10, %0\n\t"
118
119 /*
120 * We gobble 16byts backward in each loop.
121 */
122 "7:\n\t"
123 "sub $0x10, %0\n\t"
124
125 "mov -1*4(%1), %3\n\t"
126 "mov -2*4(%1), %4\n\t"
127 "mov %3, -1*4(%2)\n\t"
128 "mov %4, -2*4(%2)\n\t"
129 "mov -3*4(%1), %3\n\t"
130 "mov -4*4(%1), %4\n\t"
131 "mov %3, -3*4(%2)\n\t"
132 "mov %4, -4*4(%2)\n\t"
133 "lea -0x10(%1), %1\n\t"
134 "lea -0x10(%2), %2\n\t"
135 "jae 7b\n\t"
136 /*
137 * Calculate copy position to head.
138 */
139 "add $0x10, %0\n\t"
140 "sub %0, %1\n\t"
141 "sub %0, %2\n\t"
142
143 /*
144 * Move data from 8 bytes to 15 bytes.
145 */
146 ".p2align 4\n\t"
147 "1:\n\t"
148 "cmp $8, %0\n\t"
149 "jb 8f\n\t"
150 "mov 0*4(%1), %3\n\t"
151 "mov 1*4(%1), %4\n\t"
152 "mov -2*4(%1, %0), %5\n\t"
153 "mov -1*4(%1, %0), %1\n\t"
154
155 "mov %3, 0*4(%2)\n\t"
156 "mov %4, 1*4(%2)\n\t"
157 "mov %5, -2*4(%2, %0)\n\t"
158 "mov %1, -1*4(%2, %0)\n\t"
159 "jmp 11f\n\t"
160
161 /*
162 * Move data from 4 bytes to 7 bytes.
163 */
164 ".p2align 4\n\t"
165 "8:\n\t"
166 "cmp $4, %0\n\t"
167 "jb 9f\n\t"
168 "mov 0*4(%1), %3\n\t"
169 "mov -1*4(%1, %0), %4\n\t"
170 "mov %3, 0*4(%2)\n\t"
171 "mov %4, -1*4(%2, %0)\n\t"
172 "jmp 11f\n\t"
173
174 /*
175 * Move data from 2 bytes to 3 bytes.
176 */
177 ".p2align 4\n\t"
178 "9:\n\t"
179 "cmp $2, %0\n\t"
180 "jb 10f\n\t"
181 "movw 0*2(%1), %%dx\n\t"
182 "movw -1*2(%1, %0), %%bx\n\t"
183 "movw %%dx, 0*2(%2)\n\t"
184 "movw %%bx, -1*2(%2, %0)\n\t"
185 "jmp 11f\n\t"
186
187 /*
188 * Move data for 1 byte.
189 */
190 ".p2align 4\n\t"
191 "10:\n\t"
192 "cmp $1, %0\n\t"
193 "jb 11f\n\t"
194 "movb (%1), %%cl\n\t"
195 "movb %%cl, (%2)\n\t"
196 ".p2align 4\n\t"
197 "11:"
198 : "=&c" (d0), "=&S" (d1), "=&D" (d2),
199 "=r" (d3),"=r" (d4), "=r"(d5)
200 :"0" (n),
201 "1" (src),
202 "2" (dest)
203 :"memory");
204
205 return ret;
206
56} 207}
57EXPORT_SYMBOL(memmove); 208EXPORT_SYMBOL(memmove);
diff --git a/arch/x86/lib/memmove_64.c b/arch/x86/lib/memmove_64.c
index ecacc4b3d9e5..6d0f0ec41b34 100644
--- a/arch/x86/lib/memmove_64.c
+++ b/arch/x86/lib/memmove_64.c
@@ -8,50 +8,185 @@
8#undef memmove 8#undef memmove
9void *memmove(void *dest, const void *src, size_t count) 9void *memmove(void *dest, const void *src, size_t count)
10{ 10{
11 unsigned long d0, d1, d2, d3; 11 unsigned long d0,d1,d2,d3,d4,d5,d6,d7;
12 if (dest < src) { 12 char *ret;
13 if ((dest + count) < src) 13
14 return memcpy(dest, src, count); 14 __asm__ __volatile__(
15 else 15 /* Handle more 32bytes in loop */
16 __asm__ __volatile__( 16 "mov %2, %3\n\t"
17 "movq %0, %3\n\t" 17 "cmp $0x20, %0\n\t"
18 "shr $3, %0\n\t" 18 "jb 1f\n\t"
19 "andq $7, %3\n\t" 19
20 "rep\n\t" 20 /* Decide forward/backward copy mode */
21 "movsq\n\t" 21 "cmp %2, %1\n\t"
22 "movq %3, %0\n\t" 22 "jb 2f\n\t"
23 "rep\n\t" 23
24 "movsb" 24 /*
25 : "=&c" (d0), "=&S" (d1), "=&D" (d2), "=r" (d3) 25 * movsq instruction have many startup latency
26 :"0" (count), 26 * so we handle small size by general register.
27 "1" (src), 27 */
28 "2" (dest) 28 "cmp $680, %0\n\t"
29 :"memory"); 29 "jb 3f\n\t"
30 } else { 30 /*
31 if((src + count) < dest) 31 * movsq instruction is only good for aligned case.
32 return memcpy(dest, src, count); 32 */
33 else 33 "cmpb %%dil, %%sil\n\t"
34 __asm__ __volatile__( 34 "je 4f\n\t"
35 "movq %0, %3\n\t" 35 "3:\n\t"
36 "lea -8(%1, %0), %1\n\t" 36 "sub $0x20, %0\n\t"
37 "lea -8(%2, %0), %2\n\t" 37 /*
38 "shr $3, %0\n\t" 38 * We gobble 32byts forward in each loop.
39 "andq $7, %3\n\t" 39 */
40 "std\n\t" 40 "5:\n\t"
41 "rep\n\t" 41 "sub $0x20, %0\n\t"
42 "movsq\n\t" 42 "movq 0*8(%1), %4\n\t"
43 "lea 7(%1), %1\n\t" 43 "movq 1*8(%1), %5\n\t"
44 "lea 7(%2), %2\n\t" 44 "movq 2*8(%1), %6\n\t"
45 "movq %3, %0\n\t" 45 "movq 3*8(%1), %7\n\t"
46 "rep\n\t" 46 "leaq 4*8(%1), %1\n\t"
47 "movsb\n\t" 47
48 "cld" 48 "movq %4, 0*8(%2)\n\t"
49 : "=&c" (d0), "=&S" (d1), "=&D" (d2), "=r" (d3) 49 "movq %5, 1*8(%2)\n\t"
50 :"0" (count), 50 "movq %6, 2*8(%2)\n\t"
51 "1" (src), 51 "movq %7, 3*8(%2)\n\t"
52 "2" (dest) 52 "leaq 4*8(%2), %2\n\t"
53 :"memory"); 53 "jae 5b\n\t"
54 } 54 "addq $0x20, %0\n\t"
55 return dest; 55 "jmp 1f\n\t"
56 /*
57 * Handle data forward by movsq.
58 */
59 ".p2align 4\n\t"
60 "4:\n\t"
61 "movq %0, %8\n\t"
62 "movq -8(%1, %0), %4\n\t"
63 "lea -8(%2, %0), %5\n\t"
64 "shrq $3, %8\n\t"
65 "rep movsq\n\t"
66 "movq %4, (%5)\n\t"
67 "jmp 13f\n\t"
68 /*
69 * Handle data backward by movsq.
70 */
71 ".p2align 4\n\t"
72 "7:\n\t"
73 "movq %0, %8\n\t"
74 "movq (%1), %4\n\t"
75 "movq %2, %5\n\t"
76 "leaq -8(%1, %0), %1\n\t"
77 "leaq -8(%2, %0), %2\n\t"
78 "shrq $3, %8\n\t"
79 "std\n\t"
80 "rep movsq\n\t"
81 "cld\n\t"
82 "movq %4, (%5)\n\t"
83 "jmp 13f\n\t"
84
85 /*
86 * Start to prepare for backward copy.
87 */
88 ".p2align 4\n\t"
89 "2:\n\t"
90 "cmp $680, %0\n\t"
91 "jb 6f \n\t"
92 "cmp %%dil, %%sil\n\t"
93 "je 7b \n\t"
94 "6:\n\t"
95 /*
96 * Calculate copy position to tail.
97 */
98 "addq %0, %1\n\t"
99 "addq %0, %2\n\t"
100 "subq $0x20, %0\n\t"
101 /*
102 * We gobble 32byts backward in each loop.
103 */
104 "8:\n\t"
105 "subq $0x20, %0\n\t"
106 "movq -1*8(%1), %4\n\t"
107 "movq -2*8(%1), %5\n\t"
108 "movq -3*8(%1), %6\n\t"
109 "movq -4*8(%1), %7\n\t"
110 "leaq -4*8(%1), %1\n\t"
111
112 "movq %4, -1*8(%2)\n\t"
113 "movq %5, -2*8(%2)\n\t"
114 "movq %6, -3*8(%2)\n\t"
115 "movq %7, -4*8(%2)\n\t"
116 "leaq -4*8(%2), %2\n\t"
117 "jae 8b\n\t"
118 /*
119 * Calculate copy position to head.
120 */
121 "addq $0x20, %0\n\t"
122 "subq %0, %1\n\t"
123 "subq %0, %2\n\t"
124 "1:\n\t"
125 "cmpq $16, %0\n\t"
126 "jb 9f\n\t"
127 /*
128 * Move data from 16 bytes to 31 bytes.
129 */
130 "movq 0*8(%1), %4\n\t"
131 "movq 1*8(%1), %5\n\t"
132 "movq -2*8(%1, %0), %6\n\t"
133 "movq -1*8(%1, %0), %7\n\t"
134 "movq %4, 0*8(%2)\n\t"
135 "movq %5, 1*8(%2)\n\t"
136 "movq %6, -2*8(%2, %0)\n\t"
137 "movq %7, -1*8(%2, %0)\n\t"
138 "jmp 13f\n\t"
139 ".p2align 4\n\t"
140 "9:\n\t"
141 "cmpq $8, %0\n\t"
142 "jb 10f\n\t"
143 /*
144 * Move data from 8 bytes to 15 bytes.
145 */
146 "movq 0*8(%1), %4\n\t"
147 "movq -1*8(%1, %0), %5\n\t"
148 "movq %4, 0*8(%2)\n\t"
149 "movq %5, -1*8(%2, %0)\n\t"
150 "jmp 13f\n\t"
151 "10:\n\t"
152 "cmpq $4, %0\n\t"
153 "jb 11f\n\t"
154 /*
155 * Move data from 4 bytes to 7 bytes.
156 */
157 "movl (%1), %4d\n\t"
158 "movl -4(%1, %0), %5d\n\t"
159 "movl %4d, (%2)\n\t"
160 "movl %5d, -4(%2, %0)\n\t"
161 "jmp 13f\n\t"
162 "11:\n\t"
163 "cmp $2, %0\n\t"
164 "jb 12f\n\t"
165 /*
166 * Move data from 2 bytes to 3 bytes.
167 */
168 "movw (%1), %4w\n\t"
169 "movw -2(%1, %0), %5w\n\t"
170 "movw %4w, (%2)\n\t"
171 "movw %5w, -2(%2, %0)\n\t"
172 "jmp 13f\n\t"
173 "12:\n\t"
174 "cmp $1, %0\n\t"
175 "jb 13f\n\t"
176 /*
177 * Move data for 1 byte.
178 */
179 "movb (%1), %4b\n\t"
180 "movb %4b, (%2)\n\t"
181 "13:\n\t"
182 : "=&d" (d0), "=&S" (d1), "=&D" (d2), "=&a" (ret) ,
183 "=r"(d3), "=r"(d4), "=r"(d5), "=r"(d6), "=&c" (d7)
184 :"0" (count),
185 "1" (src),
186 "2" (dest)
187 :"memory");
188
189 return ret;
190
56} 191}
57EXPORT_SYMBOL(memmove); 192EXPORT_SYMBOL(memmove);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-18 04:15:42 -0500