aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc64/lib
diff options
context:
space:
mode:
authorDavid S. Miller <davem@sunset.davemloft.net>2007-08-08 20:11:39 -0400
committerDavid S. Miller <davem@sunset.davemloft.net>2007-08-08 20:33:45 -0400
commit6c70b6fc7b6fc321636a014082d9e32333da1f80 (patch)
treeff0e52bb7ba43b058b7cbb88b952fd268fbe3ad2 /arch/sparc64/lib
parent68c9f9fd336dc7e793cecad25f8ac40ccaa7a256 (diff)
[SPARC64]: Do not assume sun4v chips have load-twin/store-init support.
Check the cpu type in the OBP device tree before committing to using the optimized Niagara memcpy and memset implementation. If we don't recognize the cpu type, use a completely generic version. Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'arch/sparc64/lib')
-rw-r--r--arch/sparc64/lib/GENbzero.S160
-rw-r--r--arch/sparc64/lib/GENcopy_from_user.S34
-rw-r--r--arch/sparc64/lib/GENcopy_to_user.S38
-rw-r--r--arch/sparc64/lib/GENmemcpy.S121
-rw-r--r--arch/sparc64/lib/GENpage.S77
-rw-r--r--arch/sparc64/lib/GENpatch.S33
-rw-r--r--arch/sparc64/lib/Makefile4
7 files changed, 466 insertions, 1 deletions
diff --git a/arch/sparc64/lib/GENbzero.S b/arch/sparc64/lib/GENbzero.S
new file mode 100644
index 000000000000..f9c71d64eba1
--- /dev/null
+++ b/arch/sparc64/lib/GENbzero.S
@@ -0,0 +1,160 @@
1/* GENbzero.S: Generic sparc64 memset/clear_user.
2 *
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
4 */
5#include <asm/asi.h>
6
7#define EX_ST(x,y) \
898: x,y; \
9 .section .fixup; \
10 .align 4; \
1199: retl; \
12 mov %o1, %o0; \
13 .section __ex_table; \
14 .align 4; \
15 .word 98b, 99b; \
16 .text; \
17 .align 4;
18
19 .align 32
20 .text
21
22 .globl GENmemset
23 .type GENmemset, #function
24GENmemset: /* %o0=buf, %o1=pat, %o2=len */
25 and %o1, 0xff, %o3
26 mov %o2, %o1
27 sllx %o3, 8, %g1
28 or %g1, %o3, %o2
29 sllx %o2, 16, %g1
30 or %g1, %o2, %o2
31 sllx %o2, 32, %g1
32 ba,pt %xcc, 1f
33 or %g1, %o2, %o2
34
35 .globl GENbzero
36 .type GENbzero, #function
37GENbzero:
38 clr %o2
391: brz,pn %o1, GENbzero_return
40 mov %o0, %o3
41
42 /* %o5: saved %asi, restored at GENbzero_done
43 * %o4: store %asi to use
44 */
45 rd %asi, %o5
46 mov ASI_P, %o4
47 wr %o4, 0x0, %asi
48
49GENbzero_from_clear_user:
50 cmp %o1, 15
51 bl,pn %icc, GENbzero_tiny
52 andcc %o0, 0x7, %g1
53 be,pt %xcc, 2f
54 mov 8, %g2
55 sub %g2, %g1, %g1
56 sub %o1, %g1, %o1
571: EX_ST(stba %o2, [%o0 + 0x00] %asi)
58 subcc %g1, 1, %g1
59 bne,pt %xcc, 1b
60 add %o0, 1, %o0
612: cmp %o1, 128
62 bl,pn %icc, GENbzero_medium
63 andcc %o0, (64 - 1), %g1
64 be,pt %xcc, GENbzero_pre_loop
65 mov 64, %g2
66 sub %g2, %g1, %g1
67 sub %o1, %g1, %o1
681: EX_ST(stxa %o2, [%o0 + 0x00] %asi)
69 subcc %g1, 8, %g1
70 bne,pt %xcc, 1b
71 add %o0, 8, %o0
72
73GENbzero_pre_loop:
74 andn %o1, (64 - 1), %g1
75 sub %o1, %g1, %o1
76GENbzero_loop:
77 EX_ST(stxa %o2, [%o0 + 0x00] %asi)
78 EX_ST(stxa %o2, [%o0 + 0x08] %asi)
79 EX_ST(stxa %o2, [%o0 + 0x10] %asi)
80 EX_ST(stxa %o2, [%o0 + 0x18] %asi)
81 EX_ST(stxa %o2, [%o0 + 0x20] %asi)
82 EX_ST(stxa %o2, [%o0 + 0x28] %asi)
83 EX_ST(stxa %o2, [%o0 + 0x30] %asi)
84 EX_ST(stxa %o2, [%o0 + 0x38] %asi)
85 subcc %g1, 64, %g1
86 bne,pt %xcc, GENbzero_loop
87 add %o0, 64, %o0
88
89 membar #Sync
90 wr %o4, 0x0, %asi
91 brz,pn %o1, GENbzero_done
92GENbzero_medium:
93 andncc %o1, 0x7, %g1
94 be,pn %xcc, 2f
95 sub %o1, %g1, %o1
961: EX_ST(stxa %o2, [%o0 + 0x00] %asi)
97 subcc %g1, 8, %g1
98 bne,pt %xcc, 1b
99 add %o0, 8, %o0
1002: brz,pt %o1, GENbzero_done
101 nop
102
103GENbzero_tiny:
1041: EX_ST(stba %o2, [%o0 + 0x00] %asi)
105 subcc %o1, 1, %o1
106 bne,pt %icc, 1b
107 add %o0, 1, %o0
108
109 /* fallthrough */
110
111GENbzero_done:
112 wr %o5, 0x0, %asi
113
114GENbzero_return:
115 retl
116 mov %o3, %o0
117 .size GENbzero, .-GENbzero
118 .size GENmemset, .-GENmemset
119
120 .globl GENclear_user
121 .type GENclear_user, #function
122GENclear_user: /* %o0=buf, %o1=len */
123 rd %asi, %o5
124 brz,pn %o1, GENbzero_done
125 clr %o3
126 cmp %o5, ASI_AIUS
127 bne,pn %icc, GENbzero
128 clr %o2
129 ba,pt %xcc, GENbzero_from_clear_user
130 mov ASI_AIUS, %o4
131 .size GENclear_user, .-GENclear_user
132
133#define BRANCH_ALWAYS 0x10680000
134#define NOP 0x01000000
135#define GEN_DO_PATCH(OLD, NEW) \
136 sethi %hi(NEW), %g1; \
137 or %g1, %lo(NEW), %g1; \
138 sethi %hi(OLD), %g2; \
139 or %g2, %lo(OLD), %g2; \
140 sub %g1, %g2, %g1; \
141 sethi %hi(BRANCH_ALWAYS), %g3; \
142 sll %g1, 11, %g1; \
143 srl %g1, 11 + 2, %g1; \
144 or %g3, %lo(BRANCH_ALWAYS), %g3; \
145 or %g3, %g1, %g3; \
146 stw %g3, [%g2]; \
147 sethi %hi(NOP), %g3; \
148 or %g3, %lo(NOP), %g3; \
149 stw %g3, [%g2 + 0x4]; \
150 flush %g2;
151
152 .globl generic_patch_bzero
153 .type generic_patch_bzero,#function
154generic_patch_bzero:
155 GEN_DO_PATCH(memset, GENmemset)
156 GEN_DO_PATCH(__bzero, GENbzero)
157 GEN_DO_PATCH(__clear_user, GENclear_user)
158 retl
159 nop
160 .size generic_patch_bzero,.-generic_patch_bzero
diff --git a/arch/sparc64/lib/GENcopy_from_user.S b/arch/sparc64/lib/GENcopy_from_user.S
new file mode 100644
index 000000000000..2b9df99e87f9
--- /dev/null
+++ b/arch/sparc64/lib/GENcopy_from_user.S
@@ -0,0 +1,34 @@
1/* GENcopy_from_user.S: Generic sparc64 copy from userspace.
2 *
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
4 */
5
6#define EX_LD(x) \
798: x; \
8 .section .fixup; \
9 .align 4; \
1099: retl; \
11 mov 1, %o0; \
12 .section __ex_table,"a";\
13 .align 4; \
14 .word 98b, 99b; \
15 .text; \
16 .align 4;
17
18#ifndef ASI_AIUS
19#define ASI_AIUS 0x11
20#endif
21
22#define FUNC_NAME GENcopy_from_user
23#define LOAD(type,addr,dest) type##a [addr] ASI_AIUS, dest
24#define EX_RETVAL(x) 0
25
26#ifdef __KERNEL__
27#define PREAMBLE \
28 rd %asi, %g1; \
29 cmp %g1, ASI_AIUS; \
30 bne,pn %icc, memcpy_user_stub; \
31 nop
32#endif
33
34#include "GENmemcpy.S"
diff --git a/arch/sparc64/lib/GENcopy_to_user.S b/arch/sparc64/lib/GENcopy_to_user.S
new file mode 100644
index 000000000000..bb3f7084daf9
--- /dev/null
+++ b/arch/sparc64/lib/GENcopy_to_user.S
@@ -0,0 +1,38 @@
1/* GENcopy_to_user.S: Generic sparc64 copy to userspace.
2 *
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
4 */
5
6#define EX_ST(x) \
798: x; \
8 .section .fixup; \
9 .align 4; \
1099: retl; \
11 mov 1, %o0; \
12 .section __ex_table,"a";\
13 .align 4; \
14 .word 98b, 99b; \
15 .text; \
16 .align 4;
17
18#ifndef ASI_AIUS
19#define ASI_AIUS 0x11
20#endif
21
22#define FUNC_NAME GENcopy_to_user
23#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS
24#define EX_RETVAL(x) 0
25
26#ifdef __KERNEL__
27 /* Writing to %asi is _expensive_ so we hardcode it.
28 * Reading %asi to check for KERNEL_DS is comparatively
29 * cheap.
30 */
31#define PREAMBLE \
32 rd %asi, %g1; \
33 cmp %g1, ASI_AIUS; \
34 bne,pn %icc, memcpy_user_stub; \
35 nop
36#endif
37
38#include "GENmemcpy.S"
diff --git a/arch/sparc64/lib/GENmemcpy.S b/arch/sparc64/lib/GENmemcpy.S
new file mode 100644
index 000000000000..89358ee94851
--- /dev/null
+++ b/arch/sparc64/lib/GENmemcpy.S
@@ -0,0 +1,121 @@
1/* GENmemcpy.S: Generic sparc64 memcpy.
2 *
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
4 */
5
6#ifdef __KERNEL__
7#define GLOBAL_SPARE %g7
8#else
9#define GLOBAL_SPARE %g5
10#endif
11
12#ifndef EX_LD
13#define EX_LD(x) x
14#endif
15
16#ifndef EX_ST
17#define EX_ST(x) x
18#endif
19
20#ifndef EX_RETVAL
21#define EX_RETVAL(x) x
22#endif
23
24#ifndef LOAD
25#define LOAD(type,addr,dest) type [addr], dest
26#endif
27
28#ifndef STORE
29#define STORE(type,src,addr) type src, [addr]
30#endif
31
32#ifndef FUNC_NAME
33#define FUNC_NAME GENmemcpy
34#endif
35
36#ifndef PREAMBLE
37#define PREAMBLE
38#endif
39
40#ifndef XCC
41#define XCC xcc
42#endif
43
44 .register %g2,#scratch
45 .register %g3,#scratch
46
47 .text
48 .align 64
49
50 .globl FUNC_NAME
51 .type FUNC_NAME,#function
52FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
53 srlx %o2, 31, %g2
54 cmp %g2, 0
55 tne %XCC, 5
56 PREAMBLE
57 mov %o0, GLOBAL_SPARE
58
59 cmp %o2, 0
60 be,pn %XCC, 85f
61 or %o0, %o1, %o3
62 cmp %o2, 16
63 blu,a,pn %XCC, 80f
64 or %o3, %o2, %o3
65
66 xor %o0, %o1, %o4
67 andcc %o4, 0x7, %g0
68 bne,a,pn %XCC, 90f
69 sub %o0, %o1, %o3
70
71 and %o0, 0x7, %o4
72 sub %o4, 0x8, %o4
73 sub %g0, %o4, %o4
74 sub %o2, %o4, %o2
751: subcc %o4, 1, %o4
76 EX_LD(LOAD(ldub, %o1, %g1))
77 EX_ST(STORE(stb, %g1, %o0))
78 add %o1, 1, %o1
79 bne,pt %XCC, 1b
80 add %o0, 1, %o0
81
82 andn %o2, 0x7, %g1
83 sub %o2, %g1, %o2
841: subcc %g1, 0x8, %g1
85 EX_LD(LOAD(ldx, %o1, %g2))
86 EX_ST(STORE(stx, %g2, %o0))
87 add %o1, 0x8, %o1
88 bne,pt %XCC, 1b
89 add %o0, 0x8, %o0
90
91 brz,pt %o2, 85f
92 sub %o0, %o1, %o3
93 ba,a,pt %XCC, 90f
94
95 .align 64
9680: /* 0 < len <= 16 */
97 andcc %o3, 0x3, %g0
98 bne,pn %XCC, 90f
99 sub %o0, %o1, %o3
100
1011:
102 subcc %o2, 4, %o2
103 EX_LD(LOAD(lduw, %o1, %g1))
104 EX_ST(STORE(stw, %g1, %o1 + %o3))
105 bgu,pt %XCC, 1b
106 add %o1, 4, %o1
107
10885: retl
109 mov EX_RETVAL(GLOBAL_SPARE), %o0
110
111 .align 32
11290:
113 subcc %o2, 1, %o2
114 EX_LD(LOAD(ldub, %o1, %g1))
115 EX_ST(STORE(stb, %g1, %o1 + %o3))
116 bgu,pt %XCC, 90b
117 add %o1, 1, %o1
118 retl
119 mov EX_RETVAL(GLOBAL_SPARE), %o0
120
121 .size FUNC_NAME, .-FUNC_NAME
diff --git a/arch/sparc64/lib/GENpage.S b/arch/sparc64/lib/GENpage.S
new file mode 100644
index 000000000000..2ef9d05f21bc
--- /dev/null
+++ b/arch/sparc64/lib/GENpage.S
@@ -0,0 +1,77 @@
1/* GENpage.S: Generic clear and copy page.
2 *
3 * Copyright (C) 2007 (davem@davemloft.net)
4 */
5#include <asm/page.h>
6
7 .text
8 .align 32
9
10GENcopy_user_page:
11 set PAGE_SIZE, %g7
121: ldx [%o1 + 0x00], %o2
13 ldx [%o1 + 0x08], %o3
14 ldx [%o1 + 0x10], %o4
15 ldx [%o1 + 0x18], %o5
16 stx %o2, [%o0 + 0x00]
17 stx %o3, [%o0 + 0x08]
18 stx %o4, [%o0 + 0x10]
19 stx %o5, [%o0 + 0x18]
20 ldx [%o1 + 0x20], %o2
21 ldx [%o1 + 0x28], %o3
22 ldx [%o1 + 0x30], %o4
23 ldx [%o1 + 0x38], %o5
24 stx %o2, [%o0 + 0x20]
25 stx %o3, [%o0 + 0x28]
26 stx %o4, [%o0 + 0x30]
27 stx %o5, [%o0 + 0x38]
28 subcc %g7, 64, %g7
29 add %o1, 64, %o1
30 bne,pt %xcc, 1b
31 add %o0, 64, %o0
32 retl
33 nop
34
35GENclear_page:
36GENclear_user_page:
37 set PAGE_SIZE, %g7
381: stx %g0, [%o0 + 0x00]
39 stx %g0, [%o0 + 0x08]
40 stx %g0, [%o0 + 0x10]
41 stx %g0, [%o0 + 0x18]
42 stx %g0, [%o0 + 0x20]
43 stx %g0, [%o0 + 0x28]
44 stx %g0, [%o0 + 0x30]
45 stx %g0, [%o0 + 0x38]
46 subcc %g7, 64, %g7
47 bne,pt %xcc, 1b
48 add %o0, 64, %o0
49
50#define BRANCH_ALWAYS 0x10680000
51#define NOP 0x01000000
52#define GEN_DO_PATCH(OLD, NEW) \
53 sethi %hi(NEW), %g1; \
54 or %g1, %lo(NEW), %g1; \
55 sethi %hi(OLD), %g2; \
56 or %g2, %lo(OLD), %g2; \
57 sub %g1, %g2, %g1; \
58 sethi %hi(BRANCH_ALWAYS), %g3; \
59 sll %g1, 11, %g1; \
60 srl %g1, 11 + 2, %g1; \
61 or %g3, %lo(BRANCH_ALWAYS), %g3; \
62 or %g3, %g1, %g3; \
63 stw %g3, [%g2]; \
64 sethi %hi(NOP), %g3; \
65 or %g3, %lo(NOP), %g3; \
66 stw %g3, [%g2 + 0x4]; \
67 flush %g2;
68
69 .globl generic_patch_pageops
70 .type generic_patch_pageops,#function
71generic_patch_pageops:
72 GEN_DO_PATCH(copy_user_page, GENcopy_user_page)
73 GEN_DO_PATCH(_clear_page, GENclear_page)
74 GEN_DO_PATCH(clear_user_page, GENclear_user_page)
75 retl
76 nop
77 .size generic_patch_pageops,.-generic_patch_pageops
diff --git a/arch/sparc64/lib/GENpatch.S b/arch/sparc64/lib/GENpatch.S
new file mode 100644
index 000000000000..fab9e89f16bd
--- /dev/null
+++ b/arch/sparc64/lib/GENpatch.S
@@ -0,0 +1,33 @@
1/* GENpatch.S: Patch Ultra-I routines with generic variant.
2 *
3 * Copyright (C) 2007 David S. Miller <davem@davemloft.net>
4 */
5
6#define BRANCH_ALWAYS 0x10680000
7#define NOP 0x01000000
8#define GEN_DO_PATCH(OLD, NEW) \
9 sethi %hi(NEW), %g1; \
10 or %g1, %lo(NEW), %g1; \
11 sethi %hi(OLD), %g2; \
12 or %g2, %lo(OLD), %g2; \
13 sub %g1, %g2, %g1; \
14 sethi %hi(BRANCH_ALWAYS), %g3; \
15 sll %g1, 11, %g1; \
16 srl %g1, 11 + 2, %g1; \
17 or %g3, %lo(BRANCH_ALWAYS), %g3; \
18 or %g3, %g1, %g3; \
19 stw %g3, [%g2]; \
20 sethi %hi(NOP), %g3; \
21 or %g3, %lo(NOP), %g3; \
22 stw %g3, [%g2 + 0x4]; \
23 flush %g2;
24
25 .globl generic_patch_copyops
26 .type generic_patch_copyops,#function
27generic_patch_copyops:
28 GEN_DO_PATCH(memcpy, GENmemcpy)
29 GEN_DO_PATCH(___copy_from_user, GENcopy_from_user)
30 GEN_DO_PATCH(___copy_to_user, GENcopy_to_user)
31 retl
32 nop
33 .size generic_patch_copyops,.-generic_patch_copyops
diff --git a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
index c4a6d6e7d03c..f95fbfa3eeb8 100644
--- a/arch/sparc64/lib/Makefile
+++ b/arch/sparc64/lib/Makefile
@@ -1,4 +1,4 @@
1# $Id: Makefile,v 1.25 2000/12/14 22:57:25 davem Exp $ 1#
2# Makefile for Sparc64 library files.. 2# Makefile for Sparc64 library files..
3# 3#
4 4
@@ -13,6 +13,8 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
13 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \ 13 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
14 NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o NGpatch.o \ 14 NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o NGpatch.o \
15 NGpage.o NGbzero.o \ 15 NGpage.o NGbzero.o \
16 GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o GENpatch.o \
17 GENpage.o GENbzero.o \
16 copy_in_user.o user_fixup.o memmove.o \ 18 copy_in_user.o user_fixup.o memmove.o \
17 mcount.o ipcsum.o rwsem.o xor.o 19 mcount.o ipcsum.o rwsem.o xor.o
18 20
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-27 15:35:59 -0400