sh: Partially unroll the SH-4 __flush_xxx_region() flushers.

This does a bit of unrolling for the SH-4 region flushers. Based on an earlier patch by SUGIOKA Toshinobu. Signed-off-by: Paul Mundt <lethal@linux-sh.org>
author: Paul Mundt <lethal@linux-sh.org> 2009-08-04 05:09:54 -0400
committer: Paul Mundt <lethal@linux-sh.org> 2009-08-04 05:09:54 -0400
commit: 0837f52463583f76670ab2350e0f1541cb0351f5 (patch)
tree: b604059bcc41dba456f33223f78616853a3d7ad9
parent: 817425275271f2514f0dc6952182aa057ce80973 (diff)
1 files changed, 83 insertions, 21 deletions
diff --git a/arch/sh/mm/flush-sh4.c b/arch/sh/mm/flush-sh4.c
index e6d918f6ec0b..edefc53891a8 100644
--- a/arch/sh/mm/flush-sh4.c
+++ b/arch/sh/mm/flush-sh4.c
@@ -10,16 +10,37 @@
 */
 void __weak __flush_wback_region(void *start, int size)
 {
-        unsigned long v;
+        unsigned long v, cnt, end;
-        unsigned long begin, end;
-        begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
+        v = (unsigned long)start & ~(L1_CACHE_BYTES-1);
        end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
                & ~(L1_CACHE_BYTES-1);
-        for (v = begin; v < end; v+=L1_CACHE_BYTES) {
+        cnt = (end - v) / L1_CACHE_BYTES;
-                asm volatile("ocbwb     %0"
-                             : /* no output */
+        while (cnt >= 8) {
-                             : "m" (__m(v)));
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                cnt -= 8;
+        }
+        while (cnt) {
+                asm volatile("ocbwb     @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                cnt--;
        }
 }
@@ -31,16 +52,36 @@ void __weak __flush_wback_region(void *start, int size)
 */
 void __weak __flush_purge_region(void *start, int size)
 {
-        unsigned long v;
+        unsigned long v, cnt, end;
-        unsigned long begin, end;
-        begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
+        v = (unsigned long)start & ~(L1_CACHE_BYTES-1);
        end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
                & ~(L1_CACHE_BYTES-1);
-        for (v = begin; v < end; v+=L1_CACHE_BYTES) {
+        cnt = (end - v) / L1_CACHE_BYTES;
-                asm volatile("ocbp      %0"
-                             : /* no output */
+        while (cnt >= 8) {
-                             : "m" (__m(v)));
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                cnt -= 8;
+        }
+        while (cnt) {
+                asm volatile("ocbp      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                cnt--;
        }
 }
@@ -49,15 +90,36 @@ void __weak __flush_purge_region(void *start, int size)
 */
 void __weak __flush_invalidate_region(void *start, int size)
 {
-        unsigned long v;
+        unsigned long v, cnt, end;
-        unsigned long begin, end;
-        begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
+        v = (unsigned long)start & ~(L1_CACHE_BYTES-1);
        end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
                & ~(L1_CACHE_BYTES-1);
-        for (v = begin; v < end; v+=L1_CACHE_BYTES) {
+        cnt = (end - v) / L1_CACHE_BYTES;
-                asm volatile("ocbi      %0"
-                             : /* no output */
+        while (cnt >= 8) {
-                             : "m" (__m(v)));
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                cnt -= 8;
+        }
+        while (cnt) {
+                asm volatile("ocbi      @%0" : : "r" (v));
+                v += L1_CACHE_BYTES;
+                cnt--;
        }
 }
author	Paul Mundt <lethal@linux-sh.org>	2009-08-04 05:09:54 -0400
committer	Paul Mundt <lethal@linux-sh.org>	2009-08-04 05:09:54 -0400
commit	0837f52463583f76670ab2350e0f1541cb0351f5 (patch)
tree	b604059bcc41dba456f33223f78616853a3d7ad9
parent	817425275271f2514f0dc6952182aa057ce80973 (diff)

diff --git a/arch/sh/mm/flush-sh4.c b/arch/sh/mm/flush-sh4.c index e6d918f6ec0b..edefc53891a8 100644 --- a/arch/sh/mm/flush-sh4.c +++ b/arch/sh/mm/flush-sh4.c
@@ -10,16 +10,37 @@
10	*/	10	*/
11	void __weak __flush_wback_region(void *start, int size)	11	void __weak __flush_wback_region(void *start, int size)
12	{	12	{
13	unsigned long v;	13	unsigned long v, cnt, end;
14	unsigned long begin, end;
15		14
16	begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);	15	v = (unsigned long)start & ~(L1_CACHE_BYTES-1);
17	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)	16	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
18	& ~(L1_CACHE_BYTES-1);	17	& ~(L1_CACHE_BYTES-1);
19	for (v = begin; v < end; v+=L1_CACHE_BYTES) {	18	cnt = (end - v) / L1_CACHE_BYTES;
20	asm volatile("ocbwb %0"	19
21	: /* no output */	20	while (cnt >= 8) {
22	: "m" (__m(v)));	21	asm volatile("ocbwb @%0" : : "r" (v));
		22	v += L1_CACHE_BYTES;
		23	asm volatile("ocbwb @%0" : : "r" (v));
		24	v += L1_CACHE_BYTES;
		25	asm volatile("ocbwb @%0" : : "r" (v));
		26	v += L1_CACHE_BYTES;
		27	asm volatile("ocbwb @%0" : : "r" (v));
		28	v += L1_CACHE_BYTES;
		29	asm volatile("ocbwb @%0" : : "r" (v));
		30	v += L1_CACHE_BYTES;
		31	asm volatile("ocbwb @%0" : : "r" (v));
		32	v += L1_CACHE_BYTES;
		33	asm volatile("ocbwb @%0" : : "r" (v));
		34	v += L1_CACHE_BYTES;
		35	asm volatile("ocbwb @%0" : : "r" (v));
		36	v += L1_CACHE_BYTES;
		37	cnt -= 8;
		38	}
		39
		40	while (cnt) {
		41	asm volatile("ocbwb @%0" : : "r" (v));
		42	v += L1_CACHE_BYTES;
		43	cnt--;
23	}	44	}
24	}	45	}
25		46
@@ -31,16 +52,36 @@ void __weak __flush_wback_region(void *start, int size)
31	*/	52	*/
32	void __weak __flush_purge_region(void *start, int size)	53	void __weak __flush_purge_region(void *start, int size)
33	{	54	{
34	unsigned long v;	55	unsigned long v, cnt, end;
35	unsigned long begin, end;
36		56
37	begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);	57	v = (unsigned long)start & ~(L1_CACHE_BYTES-1);
38	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)	58	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
39	& ~(L1_CACHE_BYTES-1);	59	& ~(L1_CACHE_BYTES-1);
40	for (v = begin; v < end; v+=L1_CACHE_BYTES) {	60	cnt = (end - v) / L1_CACHE_BYTES;
41	asm volatile("ocbp %0"	61
42	: /* no output */	62	while (cnt >= 8) {
43	: "m" (__m(v)));	63	asm volatile("ocbp @%0" : : "r" (v));
		64	v += L1_CACHE_BYTES;
		65	asm volatile("ocbp @%0" : : "r" (v));
		66	v += L1_CACHE_BYTES;
		67	asm volatile("ocbp @%0" : : "r" (v));
		68	v += L1_CACHE_BYTES;
		69	asm volatile("ocbp @%0" : : "r" (v));
		70	v += L1_CACHE_BYTES;
		71	asm volatile("ocbp @%0" : : "r" (v));
		72	v += L1_CACHE_BYTES;
		73	asm volatile("ocbp @%0" : : "r" (v));
		74	v += L1_CACHE_BYTES;
		75	asm volatile("ocbp @%0" : : "r" (v));
		76	v += L1_CACHE_BYTES;
		77	asm volatile("ocbp @%0" : : "r" (v));
		78	v += L1_CACHE_BYTES;
		79	cnt -= 8;
		80	}
		81	while (cnt) {
		82	asm volatile("ocbp @%0" : : "r" (v));
		83	v += L1_CACHE_BYTES;
		84	cnt--;
44	}	85	}
45	}	86	}
46		87
@@ -49,15 +90,36 @@ void __weak __flush_purge_region(void *start, int size)
49	*/	90	*/
50	void __weak __flush_invalidate_region(void *start, int size)	91	void __weak __flush_invalidate_region(void *start, int size)
51	{	92	{
52	unsigned long v;	93	unsigned long v, cnt, end;
53	unsigned long begin, end;
54		94
55	begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);	95	v = (unsigned long)start & ~(L1_CACHE_BYTES-1);
56	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)	96	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
57	& ~(L1_CACHE_BYTES-1);	97	& ~(L1_CACHE_BYTES-1);
58	for (v = begin; v < end; v+=L1_CACHE_BYTES) {	98	cnt = (end - v) / L1_CACHE_BYTES;
59	asm volatile("ocbi %0"	99
60	: /* no output */	100	while (cnt >= 8) {
61	: "m" (__m(v)));	101	asm volatile("ocbi @%0" : : "r" (v));
		102	v += L1_CACHE_BYTES;
		103	asm volatile("ocbi @%0" : : "r" (v));
		104	v += L1_CACHE_BYTES;
		105	asm volatile("ocbi @%0" : : "r" (v));
		106	v += L1_CACHE_BYTES;
		107	asm volatile("ocbi @%0" : : "r" (v));
		108	v += L1_CACHE_BYTES;
		109	asm volatile("ocbi @%0" : : "r" (v));
		110	v += L1_CACHE_BYTES;
		111	asm volatile("ocbi @%0" : : "r" (v));
		112	v += L1_CACHE_BYTES;
		113	asm volatile("ocbi @%0" : : "r" (v));
		114	v += L1_CACHE_BYTES;
		115	asm volatile("ocbi @%0" : : "r" (v));
		116	v += L1_CACHE_BYTES;
		117	cnt -= 8;
		118	}
		119
		120	while (cnt) {
		121	asm volatile("ocbi @%0" : : "r" (v));
		122	v += L1_CACHE_BYTES;
		123	cnt--;
62	}	124	}
63	}	125	}