aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/sparc/include/asm/asi.h19
-rw-r--r--arch/sparc/kernel/head_64.S2
-rw-r--r--arch/sparc/lib/Makefile2
-rw-r--r--arch/sparc/lib/NG4clear_page.S29
-rw-r--r--arch/sparc/lib/NG4copy_page.S16
-rw-r--r--arch/sparc/lib/NG4memset.S105
-rw-r--r--arch/sparc/lib/NG4patch.S15
7 files changed, 176 insertions, 12 deletions
diff --git a/arch/sparc/include/asm/asi.h b/arch/sparc/include/asm/asi.h
index cc0006dc5d4a..aace6f313716 100644
--- a/arch/sparc/include/asm/asi.h
+++ b/arch/sparc/include/asm/asi.h
@@ -270,9 +270,28 @@
270#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 /* (NG) init-store, twin load, 270#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 /* (NG) init-store, twin load,
271 * primary, implicit 271 * primary, implicit
272 */ 272 */
273#define ASI_BLK_INIT_QUAD_LDD_S 0xe3 /* (NG) init-store, twin load,
274 * secondary, implicit
275 */
273#define ASI_BLK_P 0xf0 /* Primary, blk ld/st */ 276#define ASI_BLK_P 0xf0 /* Primary, blk ld/st */
274#define ASI_BLK_S 0xf1 /* Secondary, blk ld/st */ 277#define ASI_BLK_S 0xf1 /* Secondary, blk ld/st */
278#define ASI_ST_BLKINIT_MRU_P 0xf2 /* (NG4) init-store, twin load,
279 * Most-Recently-Used, primary,
280 * implicit
281 */
282#define ASI_ST_BLKINIT_MRU_S 0xf2 /* (NG4) init-store, twin load,
283 * Most-Recently-Used, secondary,
284 * implicit
285 */
275#define ASI_BLK_PL 0xf8 /* Primary, blk ld/st, little */ 286#define ASI_BLK_PL 0xf8 /* Primary, blk ld/st, little */
276#define ASI_BLK_SL 0xf9 /* Secondary, blk ld/st, little */ 287#define ASI_BLK_SL 0xf9 /* Secondary, blk ld/st, little */
288#define ASI_ST_BLKINIT_MRU_PL 0xfa /* (NG4) init-store, twin load,
289 * Most-Recently-Used, primary,
290 * implicit, little-endian
291 */
292#define ASI_ST_BLKINIT_MRU_SL 0xfb /* (NG4) init-store, twin load,
293 * Most-Recently-Used, secondary,
294 * implicit, little-endian
295 */
277 296
278#endif /* _SPARC_ASI_H */ 297#endif /* _SPARC_ASI_H */
diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
index ee5dcced2499..2feb15c35d9e 100644
--- a/arch/sparc/kernel/head_64.S
+++ b/arch/sparc/kernel/head_64.S
@@ -576,7 +576,7 @@ niagara_tlb_fixup:
576niagara4_patch: 576niagara4_patch:
577 call niagara4_patch_copyops 577 call niagara4_patch_copyops
578 nop 578 nop
579 call niagara_patch_bzero 579 call niagara4_patch_bzero
580 nop 580 nop
581 call niagara4_patch_pageops 581 call niagara4_patch_pageops
582 nop 582 nop
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index 30f6ab51c551..8410065f2862 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -33,7 +33,7 @@ lib-$(CONFIG_SPARC64) += NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o
33lib-$(CONFIG_SPARC64) += NG2patch.o 33lib-$(CONFIG_SPARC64) += NG2patch.o
34 34
35lib-$(CONFIG_SPARC64) += NG4memcpy.o NG4copy_from_user.o NG4copy_to_user.o 35lib-$(CONFIG_SPARC64) += NG4memcpy.o NG4copy_from_user.o NG4copy_to_user.o
36lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o 36lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o
37 37
38lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o 38lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
39lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o 39lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
diff --git a/arch/sparc/lib/NG4clear_page.S b/arch/sparc/lib/NG4clear_page.S
new file mode 100644
index 000000000000..e16c88204a42
--- /dev/null
+++ b/arch/sparc/lib/NG4clear_page.S
@@ -0,0 +1,29 @@
1/* NG4copy_page.S: Niagara-4 optimized clear page.
2 *
3 * Copyright (C) 2012 (davem@davemloft.net)
4 */
5
6#include <asm/asi.h>
7#include <asm/page.h>
8
9 .text
10
11 .register %g3, #scratch
12
13 .align 32
14 .globl NG4clear_page
15 .globl NG4clear_user_page
16NG4clear_page: /* %o0=dest */
17NG4clear_user_page: /* %o0=dest, %o1=vaddr */
18 set PAGE_SIZE, %g7
19 mov 0x20, %g3
201: stxa %g0, [%o0 + %g0] ASI_ST_BLKINIT_MRU_P
21 subcc %g7, 0x40, %g7
22 stxa %g0, [%o0 + %g3] ASI_ST_BLKINIT_MRU_P
23 bne,pt %xcc, 1b
24 add %o0, 0x40, %o0
25 membar #StoreLoad|#StoreStore
26 retl
27 nop
28 .size NG4clear_page,.-NG4clear_page
29 .size NG4clear_user_page,.-NG4clear_user_page \ No newline at end of file
diff --git a/arch/sparc/lib/NG4copy_page.S b/arch/sparc/lib/NG4copy_page.S
index f30ec10bbcac..28504e88c535 100644
--- a/arch/sparc/lib/NG4copy_page.S
+++ b/arch/sparc/lib/NG4copy_page.S
@@ -30,25 +30,25 @@ NG4copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
30 ldx [%o1 + 0x10], %o4 30 ldx [%o1 + 0x10], %o4
31 ldx [%o1 + 0x18], %o5 31 ldx [%o1 + 0x18], %o5
32 ldx [%o1 + 0x20], %g1 32 ldx [%o1 + 0x20], %g1
33 stxa %o2, [%o0] ASI_BLK_INIT_QUAD_LDD_P 33 stxa %o2, [%o0] ASI_ST_BLKINIT_MRU_P
34 add %o0, 0x08, %o0 34 add %o0, 0x08, %o0
35 ldx [%o1 + 0x28], %g2 35 ldx [%o1 + 0x28], %g2
36 stxa %o3, [%o0] ASI_BLK_INIT_QUAD_LDD_P 36 stxa %o3, [%o0] ASI_ST_BLKINIT_MRU_P
37 add %o0, 0x08, %o0 37 add %o0, 0x08, %o0
38 ldx [%o1 + 0x30], %g3 38 ldx [%o1 + 0x30], %g3
39 stxa %o4, [%o0] ASI_BLK_INIT_QUAD_LDD_P 39 stxa %o4, [%o0] ASI_ST_BLKINIT_MRU_P
40 add %o0, 0x08, %o0 40 add %o0, 0x08, %o0
41 ldx [%o1 + 0x38], %o2 41 ldx [%o1 + 0x38], %o2
42 add %o1, 0x40, %o1 42 add %o1, 0x40, %o1
43 stxa %o5, [%o0] ASI_BLK_INIT_QUAD_LDD_P 43 stxa %o5, [%o0] ASI_ST_BLKINIT_MRU_P
44 add %o0, 0x08, %o0 44 add %o0, 0x08, %o0
45 stxa %g1, [%o0] ASI_BLK_INIT_QUAD_LDD_P 45 stxa %g1, [%o0] ASI_ST_BLKINIT_MRU_P
46 add %o0, 0x08, %o0 46 add %o0, 0x08, %o0
47 stxa %g2, [%o0] ASI_BLK_INIT_QUAD_LDD_P 47 stxa %g2, [%o0] ASI_ST_BLKINIT_MRU_P
48 add %o0, 0x08, %o0 48 add %o0, 0x08, %o0
49 stxa %g3, [%o0] ASI_BLK_INIT_QUAD_LDD_P 49 stxa %g3, [%o0] ASI_ST_BLKINIT_MRU_P
50 add %o0, 0x08, %o0 50 add %o0, 0x08, %o0
51 stxa %o2, [%o0] ASI_BLK_INIT_QUAD_LDD_P 51 stxa %o2, [%o0] ASI_ST_BLKINIT_MRU_P
52 add %o0, 0x08, %o0 52 add %o0, 0x08, %o0
53 bne,pt %icc, 1b 53 bne,pt %icc, 1b
54 prefetch [%o1 + 0x200], #n_reads_strong 54 prefetch [%o1 + 0x200], #n_reads_strong
diff --git a/arch/sparc/lib/NG4memset.S b/arch/sparc/lib/NG4memset.S
new file mode 100644
index 000000000000..41da4bdd95cb
--- /dev/null
+++ b/arch/sparc/lib/NG4memset.S
@@ -0,0 +1,105 @@
1/* NG4memset.S: Niagara-4 optimized memset/bzero.
2 *
3 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
4 */
5
6#include <asm/asi.h>
7
8 .register %g2, #scratch
9 .register %g3, #scratch
10
11 .text
12 .align 32
13 .globl NG4memset
14NG4memset:
15 andcc %o1, 0xff, %o4
16 be,pt %icc, 1f
17 mov %o2, %o1
18 sllx %o4, 8, %g1
19 or %g1, %o4, %o2
20 sllx %o2, 16, %g1
21 or %g1, %o2, %o2
22 sllx %o2, 32, %g1
23 ba,pt %icc, 1f
24 or %g1, %o2, %o4
25 .size NG4memset,.-NG4memset
26
27 .align 32
28 .globl NG4bzero
29NG4bzero:
30 clr %o4
311: cmp %o1, 16
32 ble %icc, .Ltiny
33 mov %o0, %o3
34 sub %g0, %o0, %g1
35 and %g1, 0x7, %g1
36 brz,pt %g1, .Laligned8
37 sub %o1, %g1, %o1
381: stb %o4, [%o0 + 0x00]
39 subcc %g1, 1, %g1
40 bne,pt %icc, 1b
41 add %o0, 1, %o0
42.Laligned8:
43 cmp %o1, 64 + (64 - 8)
44 ble .Lmedium
45 sub %g0, %o0, %g1
46 andcc %g1, (64 - 1), %g1
47 brz,pn %g1, .Laligned64
48 sub %o1, %g1, %o1
491: stx %o4, [%o0 + 0x00]
50 subcc %g1, 8, %g1
51 bne,pt %icc, 1b
52 add %o0, 0x8, %o0
53.Laligned64:
54 andn %o1, 64 - 1, %g1
55 sub %o1, %g1, %o1
56 brnz,pn %o4, .Lnon_bzero_loop
57 mov 0x20, %g2
581: stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P
59 subcc %g1, 0x40, %g1
60 stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P
61 bne,pt %icc, 1b
62 add %o0, 0x40, %o0
63.Lpostloop:
64 cmp %o1, 8
65 bl,pn %icc, .Ltiny
66 membar #StoreStore|#StoreLoad
67.Lmedium:
68 andn %o1, 0x7, %g1
69 sub %o1, %g1, %o1
701: stx %o4, [%o0 + 0x00]
71 subcc %g1, 0x8, %g1
72 bne,pt %icc, 1b
73 add %o0, 0x08, %o0
74 andcc %o1, 0x4, %g1
75 be,pt %icc, .Ltiny
76 sub %o1, %g1, %o1
77 stw %o4, [%o0 + 0x00]
78 add %o0, 0x4, %o0
79.Ltiny:
80 cmp %o1, 0
81 be,pn %icc, .Lexit
821: subcc %o1, 1, %o1
83 stb %o4, [%o0 + 0x00]
84 bne,pt %icc, 1b
85 add %o0, 1, %o0
86.Lexit:
87 retl
88 mov %o3, %o0
89.Lnon_bzero_loop:
90 mov 0x08, %g3
91 mov 0x28, %o5
921: stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P
93 subcc %g1, 0x40, %g1
94 stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P
95 stxa %o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P
96 stxa %o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P
97 add %o0, 0x10, %o0
98 stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P
99 stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P
100 stxa %o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P
101 stxa %o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P
102 bne,pt %icc, 1b
103 add %o0, 0x30, %o0
104 ba,a,pt %icc, .Lpostloop
105 .size NG4bzero,.-NG4bzero
diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S
index c21c34c61dda..a114cbcf2a48 100644
--- a/arch/sparc/lib/NG4patch.S
+++ b/arch/sparc/lib/NG4patch.S
@@ -32,12 +32,23 @@ niagara4_patch_copyops:
32 nop 32 nop
33 .size niagara4_patch_copyops,.-niagara4_patch_copyops 33 .size niagara4_patch_copyops,.-niagara4_patch_copyops
34 34
35 .globl niagara4_patch_bzero
36 .type niagara4_patch_bzero,#function
37niagara4_patch_bzero:
38 NG_DO_PATCH(memset, NG4memset)
39 NG_DO_PATCH(__bzero, NG4bzero)
40 NG_DO_PATCH(__clear_user, NGclear_user)
41 NG_DO_PATCH(tsb_init, NGtsb_init)
42 retl
43 nop
44 .size niagara4_patch_bzero,.-niagara4_patch_bzero
45
35 .globl niagara4_patch_pageops 46 .globl niagara4_patch_pageops
36 .type niagara4_patch_pageops,#function 47 .type niagara4_patch_pageops,#function
37niagara4_patch_pageops: 48niagara4_patch_pageops:
38 NG_DO_PATCH(copy_user_page, NG4copy_user_page) 49 NG_DO_PATCH(copy_user_page, NG4copy_user_page)
39 NG_DO_PATCH(_clear_page, NGclear_page) 50 NG_DO_PATCH(_clear_page, NG4clear_page)
40 NG_DO_PATCH(clear_user_page, NGclear_user_page) 51 NG_DO_PATCH(clear_user_page, NG4clear_user_page)
41 retl 52 retl
42 nop 53 nop
43 .size niagara4_patch_pageops,.-niagara4_patch_pageops 54 .size niagara4_patch_pageops,.-niagara4_patch_pageops
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-28 07:06:41 -0500