1 files changed, 180 insertions, 9 deletions
diff --git a/arch/x86/lib/memmove_64.c b/arch/x86/lib/memmove_64.c
index 0a33909bf122..6d0f0ec41b34 100644
--- a/arch/x86/lib/memmove_64.c
+++ b/arch/x86/lib/memmove_64.c
@@ -8,14 +8,185 @@
 #undef memmove
 void *memmove(void *dest, const void *src, size_t count)
 {
-        if (dest < src) {
+        unsigned long d0,d1,d2,d3,d4,d5,d6,d7;
-                return memcpy(dest, src, count);
+        char *ret;
-        } else {
-                char *p = dest + count;
+        __asm__ __volatile__(
-                const char *s = src + count;
+                /* Handle more 32bytes in loop */
-                while (count--)
+                "mov %2, %3\n\t"
-                        *--p = *--s;
+                "cmp $0x20, %0\n\t"
-        }
+                "jb     1f\n\t"
-        return dest;
+                /* Decide forward/backward copy mode */
+                "cmp %2, %1\n\t"
+                "jb     2f\n\t"
+                /*
+                 * movsq instruction have many startup latency
+                 * so we handle small size by general register.
+                 */
+                "cmp  $680, %0\n\t"
+                "jb 3f\n\t"
+                /*
+                 * movsq instruction is only good for aligned case.
+                 */
+                "cmpb %%dil, %%sil\n\t"
+                "je 4f\n\t"
+                "3:\n\t"
+                "sub $0x20, %0\n\t"
+                /*
+                 * We gobble 32byts forward in each loop.
+                 */
+                "5:\n\t"
+                "sub $0x20, %0\n\t"
+                "movq 0*8(%1), %4\n\t"
+                "movq 1*8(%1), %5\n\t"
+                "movq 2*8(%1), %6\n\t"
+                "movq 3*8(%1), %7\n\t"
+                "leaq 4*8(%1), %1\n\t"
+                "movq %4, 0*8(%2)\n\t"
+                "movq %5, 1*8(%2)\n\t"
+                "movq %6, 2*8(%2)\n\t"
+                "movq %7, 3*8(%2)\n\t"
+                "leaq 4*8(%2), %2\n\t"
+                "jae 5b\n\t"
+                "addq $0x20, %0\n\t"
+                "jmp 1f\n\t"
+                /*
+                 * Handle data forward by movsq.
+                 */
+                ".p2align 4\n\t"
+                "4:\n\t"
+                "movq %0, %8\n\t"
+                "movq -8(%1, %0), %4\n\t"
+                "lea -8(%2, %0), %5\n\t"
+                "shrq $3, %8\n\t"
+                "rep movsq\n\t"
+                "movq %4, (%5)\n\t"
+                "jmp 13f\n\t"
+                /*
+                 * Handle data backward by movsq.
+                 */
+                ".p2align 4\n\t"
+                "7:\n\t"
+                "movq %0, %8\n\t"
+                "movq (%1), %4\n\t"
+                "movq %2, %5\n\t"
+                "leaq -8(%1, %0), %1\n\t"
+                "leaq -8(%2, %0), %2\n\t"
+                "shrq $3, %8\n\t"
+                "std\n\t"
+                "rep movsq\n\t"
+                "cld\n\t"
+                "movq %4, (%5)\n\t"
+                "jmp 13f\n\t"
+                /*
+                 * Start to prepare for backward copy.
+                 */
+                ".p2align 4\n\t"
+                "2:\n\t"
+                "cmp $680, %0\n\t"
+                "jb 6f \n\t"
+                "cmp %%dil, %%sil\n\t"
+                "je 7b \n\t"
+                "6:\n\t"
+                /*
+                 * Calculate copy position to tail.
+                 */
+                "addq %0, %1\n\t"
+                "addq %0, %2\n\t"
+                "subq $0x20, %0\n\t"
+                /*
+                 * We gobble 32byts backward in each loop.
+                 */
+                "8:\n\t"
+                "subq $0x20, %0\n\t"
+                "movq -1*8(%1), %4\n\t"
+                "movq -2*8(%1), %5\n\t"
+                "movq -3*8(%1), %6\n\t"
+                "movq -4*8(%1), %7\n\t"
+                "leaq -4*8(%1), %1\n\t"
+                "movq %4, -1*8(%2)\n\t"
+                "movq %5, -2*8(%2)\n\t"
+                "movq %6, -3*8(%2)\n\t"
+                "movq %7, -4*8(%2)\n\t"
+                "leaq -4*8(%2), %2\n\t"
+                "jae 8b\n\t"
+                /*
+                 * Calculate copy position to head.
+                 */
+                "addq $0x20, %0\n\t"
+                "subq %0, %1\n\t"
+                "subq %0, %2\n\t"
+                "1:\n\t"
+                "cmpq $16, %0\n\t"
+                "jb 9f\n\t"
+                /*
+                 * Move data from 16 bytes to 31 bytes.
+                 */
+                "movq 0*8(%1), %4\n\t"
+                "movq 1*8(%1), %5\n\t"
+                "movq -2*8(%1, %0), %6\n\t"
+                "movq -1*8(%1, %0), %7\n\t"
+                "movq %4, 0*8(%2)\n\t"
+                "movq %5, 1*8(%2)\n\t"
+                "movq %6, -2*8(%2, %0)\n\t"
+                "movq %7, -1*8(%2, %0)\n\t"
+                "jmp 13f\n\t"
+                ".p2align 4\n\t"
+                "9:\n\t"
+                "cmpq $8, %0\n\t"
+                "jb 10f\n\t"
+                /*
+                 * Move data from 8 bytes to 15 bytes.
+                 */
+                "movq 0*8(%1), %4\n\t"
+                "movq -1*8(%1, %0), %5\n\t"
+                "movq %4, 0*8(%2)\n\t"
+                "movq %5, -1*8(%2, %0)\n\t"
+                "jmp 13f\n\t"
+                "10:\n\t"
+                "cmpq $4, %0\n\t"
+                "jb 11f\n\t"
+                /*
+                 * Move data from 4 bytes to 7 bytes.
+                 */
+                "movl (%1), %4d\n\t"
+                "movl -4(%1, %0), %5d\n\t"
+                "movl %4d, (%2)\n\t"
+                "movl %5d, -4(%2, %0)\n\t"
+                "jmp 13f\n\t"
+                "11:\n\t"
+                "cmp $2, %0\n\t"
+                "jb 12f\n\t"
+                /*
+                 * Move data from 2 bytes to 3 bytes.
+                 */
+                "movw (%1), %4w\n\t"
+                "movw -2(%1, %0), %5w\n\t"
+                "movw %4w, (%2)\n\t"
+                "movw %5w, -2(%2, %0)\n\t"
+                "jmp 13f\n\t"
+                "12:\n\t"
+                "cmp $1, %0\n\t"
+                "jb 13f\n\t"
+                /*
+                 * Move data for 1 byte.
+                 */
+                "movb (%1), %4b\n\t"
+                "movb %4b, (%2)\n\t"
+                "13:\n\t"
+                : "=&d" (d0), "=&S" (d1), "=&D" (d2), "=&a" (ret) ,
+                  "=r"(d3), "=r"(d4), "=r"(d5), "=r"(d6), "=&c" (d7)
+                :"0" (count),
+                 "1" (src),
+                 "2" (dest)
+                :"memory");
+                return ret;
 }
 EXPORT_SYMBOL(memmove);

diff --git a/arch/x86/lib/memmove_64.c b/arch/x86/lib/memmove_64.c index 0a33909bf122..6d0f0ec41b34 100644 --- a/arch/x86/lib/memmove_64.c +++ b/arch/x86/lib/memmove_64.c
@@ -8,14 +8,185 @@
8	#undef memmove	8	#undef memmove
9	void memmove(void dest, const void *src, size_t count)	9	void memmove(void dest, const void *src, size_t count)
10	{	10	{
11	if (dest < src) {	11	unsigned long d0,d1,d2,d3,d4,d5,d6,d7;
12	return memcpy(dest, src, count);	12	char *ret;
13	} else {	13
14	char *p = dest + count;	14	__asm__ __volatile__(
15	const char *s = src + count;	15	/* Handle more 32bytes in loop */
16	while (count--)	16	"mov %2, %3\n\t"
17	--p = --s;	17	"cmp $0x20, %0\n\t"
18	}	18	"jb 1f\n\t"
19	return dest;	19
		20	/* Decide forward/backward copy mode */
		21	"cmp %2, %1\n\t"
		22	"jb 2f\n\t"
		23
		24	/*
		25	* movsq instruction have many startup latency
		26	* so we handle small size by general register.
		27	*/
		28	"cmp $680, %0\n\t"
		29	"jb 3f\n\t"
		30	/*
		31	* movsq instruction is only good for aligned case.
		32	*/
		33	"cmpb %%dil, %%sil\n\t"
		34	"je 4f\n\t"
		35	"3:\n\t"
		36	"sub $0x20, %0\n\t"
		37	/*
		38	* We gobble 32byts forward in each loop.
		39	*/
		40	"5:\n\t"
		41	"sub $0x20, %0\n\t"
		42	"movq 0*8(%1), %4\n\t"
		43	"movq 1*8(%1), %5\n\t"
		44	"movq 2*8(%1), %6\n\t"
		45	"movq 3*8(%1), %7\n\t"
		46	"leaq 4*8(%1), %1\n\t"
		47
		48	"movq %4, 0*8(%2)\n\t"
		49	"movq %5, 1*8(%2)\n\t"
		50	"movq %6, 2*8(%2)\n\t"
		51	"movq %7, 3*8(%2)\n\t"
		52	"leaq 4*8(%2), %2\n\t"
		53	"jae 5b\n\t"
		54	"addq $0x20, %0\n\t"
		55	"jmp 1f\n\t"
		56	/*
		57	* Handle data forward by movsq.
		58	*/
		59	".p2align 4\n\t"
		60	"4:\n\t"
		61	"movq %0, %8\n\t"
		62	"movq -8(%1, %0), %4\n\t"
		63	"lea -8(%2, %0), %5\n\t"
		64	"shrq $3, %8\n\t"
		65	"rep movsq\n\t"
		66	"movq %4, (%5)\n\t"
		67	"jmp 13f\n\t"
		68	/*
		69	* Handle data backward by movsq.
		70	*/
		71	".p2align 4\n\t"
		72	"7:\n\t"
		73	"movq %0, %8\n\t"
		74	"movq (%1), %4\n\t"
		75	"movq %2, %5\n\t"
		76	"leaq -8(%1, %0), %1\n\t"
		77	"leaq -8(%2, %0), %2\n\t"
		78	"shrq $3, %8\n\t"
		79	"std\n\t"
		80	"rep movsq\n\t"
		81	"cld\n\t"
		82	"movq %4, (%5)\n\t"
		83	"jmp 13f\n\t"
		84
		85	/*
		86	* Start to prepare for backward copy.
		87	*/
		88	".p2align 4\n\t"
		89	"2:\n\t"
		90	"cmp $680, %0\n\t"
		91	"jb 6f \n\t"
		92	"cmp %%dil, %%sil\n\t"
		93	"je 7b \n\t"
		94	"6:\n\t"
		95	/*
		96	* Calculate copy position to tail.
		97	*/
		98	"addq %0, %1\n\t"
		99	"addq %0, %2\n\t"
		100	"subq $0x20, %0\n\t"
		101	/*
		102	* We gobble 32byts backward in each loop.
		103	*/
		104	"8:\n\t"
		105	"subq $0x20, %0\n\t"
		106	"movq -1*8(%1), %4\n\t"
		107	"movq -2*8(%1), %5\n\t"
		108	"movq -3*8(%1), %6\n\t"
		109	"movq -4*8(%1), %7\n\t"
		110	"leaq -4*8(%1), %1\n\t"
		111
		112	"movq %4, -1*8(%2)\n\t"
		113	"movq %5, -2*8(%2)\n\t"
		114	"movq %6, -3*8(%2)\n\t"
		115	"movq %7, -4*8(%2)\n\t"
		116	"leaq -4*8(%2), %2\n\t"
		117	"jae 8b\n\t"
		118	/*
		119	* Calculate copy position to head.
		120	*/
		121	"addq $0x20, %0\n\t"
		122	"subq %0, %1\n\t"
		123	"subq %0, %2\n\t"
		124	"1:\n\t"
		125	"cmpq $16, %0\n\t"
		126	"jb 9f\n\t"
		127	/*
		128	* Move data from 16 bytes to 31 bytes.
		129	*/
		130	"movq 0*8(%1), %4\n\t"
		131	"movq 1*8(%1), %5\n\t"
		132	"movq -2*8(%1, %0), %6\n\t"
		133	"movq -1*8(%1, %0), %7\n\t"
		134	"movq %4, 0*8(%2)\n\t"
		135	"movq %5, 1*8(%2)\n\t"
		136	"movq %6, -2*8(%2, %0)\n\t"
		137	"movq %7, -1*8(%2, %0)\n\t"
		138	"jmp 13f\n\t"
		139	".p2align 4\n\t"
		140	"9:\n\t"
		141	"cmpq $8, %0\n\t"
		142	"jb 10f\n\t"
		143	/*
		144	* Move data from 8 bytes to 15 bytes.
		145	*/
		146	"movq 0*8(%1), %4\n\t"
		147	"movq -1*8(%1, %0), %5\n\t"
		148	"movq %4, 0*8(%2)\n\t"
		149	"movq %5, -1*8(%2, %0)\n\t"
		150	"jmp 13f\n\t"
		151	"10:\n\t"
		152	"cmpq $4, %0\n\t"
		153	"jb 11f\n\t"
		154	/*
		155	* Move data from 4 bytes to 7 bytes.
		156	*/
		157	"movl (%1), %4d\n\t"
		158	"movl -4(%1, %0), %5d\n\t"
		159	"movl %4d, (%2)\n\t"
		160	"movl %5d, -4(%2, %0)\n\t"
		161	"jmp 13f\n\t"
		162	"11:\n\t"
		163	"cmp $2, %0\n\t"
		164	"jb 12f\n\t"
		165	/*
		166	* Move data from 2 bytes to 3 bytes.
		167	*/
		168	"movw (%1), %4w\n\t"
		169	"movw -2(%1, %0), %5w\n\t"
		170	"movw %4w, (%2)\n\t"
		171	"movw %5w, -2(%2, %0)\n\t"
		172	"jmp 13f\n\t"
		173	"12:\n\t"
		174	"cmp $1, %0\n\t"
		175	"jb 13f\n\t"
		176	/*
		177	* Move data for 1 byte.
		178	*/
		179	"movb (%1), %4b\n\t"
		180	"movb %4b, (%2)\n\t"
		181	"13:\n\t"
		182	: "=&d" (d0), "=&S" (d1), "=&D" (d2), "=&a" (ret) ,
		183	"=r"(d3), "=r"(d4), "=r"(d5), "=r"(d6), "=&c" (d7)
		184	:"0" (count),
		185	"1" (src),
		186	"2" (dest)
		187	:"memory");
		188
		189	return ret;
		190
20	}	191	}
21	EXPORT_SYMBOL(memmove);	192	EXPORT_SYMBOL(memmove);