aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2012-10-08 17:39:30 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2012-10-08 17:39:30 -0400
commit3c5af8d1aad6f193c0a89702c87292a0ed81add0 (patch)
treec44aaa89c45dda5d023601ab0ca4d6f1d4631698 /arch
parente9eca4de957ac33744fb994ccacd4a5102e445a8 (diff)
parent2863bc54ec03df7a6e2c48cff0477d7e2384efc9 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc
Pull sparc changes from David S Miller: "There is an attempt to fix a bad interaction between syscall tracing and force_successful_syscall() from Al Viro, but it needs to be redone as it introduced regressions and thus had to be reverted for now. Al is working on an updated version. But what we do have here are some significant bzero/memset improvements for Niagara-4. An 8K page can be cleared in around 600 cycles, because we essentially have a store that behaves like powerpc's dcbz that we can actually make real use of." * git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc: Revert strace hiccups fix. sparc64: Niagara-4 bzero/memset, plus use MRU stores in page copy. sparc64: Fix strace hiccups when force_successful_syscall() triggers. sparc64: Rearrange thread info to cheaply clear syscall noerror state.
Diffstat (limited to 'arch')
-rw-r--r--arch/sparc/include/asm/asi.h19
-rw-r--r--arch/sparc/kernel/head_64.S2
-rw-r--r--arch/sparc/lib/Makefile2
-rw-r--r--arch/sparc/lib/NG4clear_page.S29
-rw-r--r--arch/sparc/lib/NG4copy_page.S16
-rw-r--r--arch/sparc/lib/NG4memset.S105
-rw-r--r--arch/sparc/lib/NG4patch.S15
7 files changed, 176 insertions, 12 deletions
diff --git a/arch/sparc/include/asm/asi.h b/arch/sparc/include/asm/asi.h
index cc0006dc5d4a..aace6f313716 100644
--- a/arch/sparc/include/asm/asi.h
+++ b/arch/sparc/include/asm/asi.h
@@ -270,9 +270,28 @@
270#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 /* (NG) init-store, twin load, 270#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 /* (NG) init-store, twin load,
271 * primary, implicit 271 * primary, implicit
272 */ 272 */
273#define ASI_BLK_INIT_QUAD_LDD_S 0xe3 /* (NG) init-store, twin load,
274 * secondary, implicit
275 */
273#define ASI_BLK_P 0xf0 /* Primary, blk ld/st */ 276#define ASI_BLK_P 0xf0 /* Primary, blk ld/st */
274#define ASI_BLK_S 0xf1 /* Secondary, blk ld/st */ 277#define ASI_BLK_S 0xf1 /* Secondary, blk ld/st */
278#define ASI_ST_BLKINIT_MRU_P 0xf2 /* (NG4) init-store, twin load,
279 * Most-Recently-Used, primary,
280 * implicit
281 */
282#define ASI_ST_BLKINIT_MRU_S 0xf2 /* (NG4) init-store, twin load,
283 * Most-Recently-Used, secondary,
284 * implicit
285 */
275#define ASI_BLK_PL 0xf8 /* Primary, blk ld/st, little */ 286#define ASI_BLK_PL 0xf8 /* Primary, blk ld/st, little */
276#define ASI_BLK_SL 0xf9 /* Secondary, blk ld/st, little */ 287#define ASI_BLK_SL 0xf9 /* Secondary, blk ld/st, little */
288#define ASI_ST_BLKINIT_MRU_PL 0xfa /* (NG4) init-store, twin load,
289 * Most-Recently-Used, primary,
290 * implicit, little-endian
291 */
292#define ASI_ST_BLKINIT_MRU_SL 0xfb /* (NG4) init-store, twin load,
293 * Most-Recently-Used, secondary,
294 * implicit, little-endian
295 */
277 296
278#endif /* _SPARC_ASI_H */ 297#endif /* _SPARC_ASI_H */
diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
index ee5dcced2499..2feb15c35d9e 100644
--- a/arch/sparc/kernel/head_64.S
+++ b/arch/sparc/kernel/head_64.S
@@ -576,7 +576,7 @@ niagara_tlb_fixup:
576niagara4_patch: 576niagara4_patch:
577 call niagara4_patch_copyops 577 call niagara4_patch_copyops
578 nop 578 nop
579 call niagara_patch_bzero 579 call niagara4_patch_bzero
580 nop 580 nop
581 call niagara4_patch_pageops 581 call niagara4_patch_pageops
582 nop 582 nop
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index 30f6ab51c551..8410065f2862 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -33,7 +33,7 @@ lib-$(CONFIG_SPARC64) += NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o
33lib-$(CONFIG_SPARC64) += NG2patch.o 33lib-$(CONFIG_SPARC64) += NG2patch.o
34 34
35lib-$(CONFIG_SPARC64) += NG4memcpy.o NG4copy_from_user.o NG4copy_to_user.o 35lib-$(CONFIG_SPARC64) += NG4memcpy.o NG4copy_from_user.o NG4copy_to_user.o
36lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o 36lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o
37 37
38lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o 38lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
39lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o 39lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
diff --git a/arch/sparc/lib/NG4clear_page.S b/arch/sparc/lib/NG4clear_page.S
new file mode 100644
index 000000000000..e16c88204a42
--- /dev/null
+++ b/arch/sparc/lib/NG4clear_page.S
@@ -0,0 +1,29 @@
1/* NG4copy_page.S: Niagara-4 optimized clear page.
2 *
3 * Copyright (C) 2012 (davem@davemloft.net)
4 */
5
6#include <asm/asi.h>
7#include <asm/page.h>
8
9 .text
10
11 .register %g3, #scratch
12
13 .align 32
14 .globl NG4clear_page
15 .globl NG4clear_user_page
16NG4clear_page: /* %o0=dest */
17NG4clear_user_page: /* %o0=dest, %o1=vaddr */
18 set PAGE_SIZE, %g7
19 mov 0x20, %g3
201: stxa %g0, [%o0 + %g0] ASI_ST_BLKINIT_MRU_P
21 subcc %g7, 0x40, %g7
22 stxa %g0, [%o0 + %g3] ASI_ST_BLKINIT_MRU_P
23 bne,pt %xcc, 1b
24 add %o0, 0x40, %o0
25 membar #StoreLoad|#StoreStore
26 retl
27 nop
28 .size NG4clear_page,.-NG4clear_page
29 .size NG4clear_user_page,.-NG4clear_user_page \ No newline at end of file
diff --git a/arch/sparc/lib/NG4copy_page.S b/arch/sparc/lib/NG4copy_page.S
index f30ec10bbcac..28504e88c535 100644
--- a/arch/sparc/lib/NG4copy_page.S
+++ b/arch/sparc/lib/NG4copy_page.S
@@ -30,25 +30,25 @@ NG4copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
30 ldx [%o1 + 0x10], %o4 30 ldx [%o1 + 0x10], %o4
31 ldx [%o1 + 0x18], %o5 31 ldx [%o1 + 0x18], %o5
32 ldx [%o1 + 0x20], %g1 32 ldx [%o1 + 0x20], %g1
33 stxa %o2, [%o0] ASI_BLK_INIT_QUAD_LDD_P 33 stxa %o2, [%o0] ASI_ST_BLKINIT_MRU_P
34 add %o0, 0x08, %o0 34 add %o0, 0x08, %o0
35 ldx [%o1 + 0x28], %g2 35 ldx [%o1 + 0x28], %g2
36 stxa %o3, [%o0] ASI_BLK_INIT_QUAD_LDD_P 36 stxa %o3, [%o0] ASI_ST_BLKINIT_MRU_P
37 add %o0, 0x08, %o0 37 add %o0, 0x08, %o0
38 ldx [%o1 + 0x30], %g3 38 ldx [%o1 + 0x30], %g3
39 stxa %o4, [%o0] ASI_BLK_INIT_QUAD_LDD_P 39 stxa %o4, [%o0] ASI_ST_BLKINIT_MRU_P
40 add %o0, 0x08, %o0 40 add %o0, 0x08, %o0
41 ldx [%o1 + 0x38], %o2 41 ldx [%o1 + 0x38], %o2
42 add %o1, 0x40, %o1 42 add %o1, 0x40, %o1
43 stxa %o5, [%o0] ASI_BLK_INIT_QUAD_LDD_P 43 stxa %o5, [%o0] ASI_ST_BLKINIT_MRU_P
44 add %o0, 0x08, %o0 44 add %o0, 0x08, %o0
45 stxa %g1, [%o0] ASI_BLK_INIT_QUAD_LDD_P 45 stxa %g1, [%o0] ASI_ST_BLKINIT_MRU_P
46 add %o0, 0x08, %o0 46 add %o0, 0x08, %o0
47 stxa %g2, [%o0] ASI_BLK_INIT_QUAD_LDD_P 47 stxa %g2, [%o0] ASI_ST_BLKINIT_MRU_P
48 add %o0, 0x08, %o0 48 add %o0, 0x08, %o0
49 stxa %g3, [%o0] ASI_BLK_INIT_QUAD_LDD_P 49 stxa %g3, [%o0] ASI_ST_BLKINIT_MRU_P
50 add %o0, 0x08, %o0 50 add %o0, 0x08, %o0
51 stxa %o2, [%o0] ASI_BLK_INIT_QUAD_LDD_P 51 stxa %o2, [%o0] ASI_ST_BLKINIT_MRU_P
52 add %o0, 0x08, %o0 52 add %o0, 0x08, %o0
53 bne,pt %icc, 1b 53 bne,pt %icc, 1b
54 prefetch [%o1 + 0x200], #n_reads_strong 54 prefetch [%o1 + 0x200], #n_reads_strong
diff --git a/arch/sparc/lib/NG4memset.S b/arch/sparc/lib/NG4memset.S
new file mode 100644
index 000000000000..41da4bdd95cb
--- /dev/null
+++ b/arch/sparc/lib/NG4memset.S
@@ -0,0 +1,105 @@
1/* NG4memset.S: Niagara-4 optimized memset/bzero.
2 *
3 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
4 */
5
6#include <asm/asi.h>
7
8 .register %g2, #scratch
9 .register %g3, #scratch
10
11 .text
12 .align 32
13 .globl NG4memset
14NG4memset:
15 andcc %o1, 0xff, %o4
16 be,pt %icc, 1f
17 mov %o2, %o1
18 sllx %o4, 8, %g1
19 or %g1, %o4, %o2
20 sllx %o2, 16, %g1
21 or %g1, %o2, %o2
22 sllx %o2, 32, %g1
23 ba,pt %icc, 1f
24 or %g1, %o2, %o4
25 .size NG4memset,.-NG4memset
26
27 .align 32
28 .globl NG4bzero
29NG4bzero:
30 clr %o4
311: cmp %o1, 16
32 ble %icc, .Ltiny
33 mov %o0, %o3
34 sub %g0, %o0, %g1
35 and %g1, 0x7, %g1
36 brz,pt %g1, .Laligned8
37 sub %o1, %g1, %o1
381: stb %o4, [%o0 + 0x00]
39 subcc %g1, 1, %g1
40 bne,pt %icc, 1b
41 add %o0, 1, %o0
42.Laligned8:
43 cmp %o1, 64 + (64 - 8)
44 ble .Lmedium
45 sub %g0, %o0, %g1
46 andcc %g1, (64 - 1), %g1
47 brz,pn %g1, .Laligned64
48 sub %o1, %g1, %o1
491: stx %o4, [%o0 + 0x00]
50 subcc %g1, 8, %g1
51 bne,pt %icc, 1b
52 add %o0, 0x8, %o0
53.Laligned64:
54 andn %o1, 64 - 1, %g1
55 sub %o1, %g1, %o1
56 brnz,pn %o4, .Lnon_bzero_loop
57 mov 0x20, %g2
581: stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P
59 subcc %g1, 0x40, %g1
60 stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P
61 bne,pt %icc, 1b
62 add %o0, 0x40, %o0
63.Lpostloop:
64 cmp %o1, 8
65 bl,pn %icc, .Ltiny
66 membar #StoreStore|#StoreLoad
67.Lmedium:
68 andn %o1, 0x7, %g1
69 sub %o1, %g1, %o1
701: stx %o4, [%o0 + 0x00]
71 subcc %g1, 0x8, %g1
72 bne,pt %icc, 1b
73 add %o0, 0x08, %o0
74 andcc %o1, 0x4, %g1
75 be,pt %icc, .Ltiny
76 sub %o1, %g1, %o1
77 stw %o4, [%o0 + 0x00]
78 add %o0, 0x4, %o0
79.Ltiny:
80 cmp %o1, 0
81 be,pn %icc, .Lexit
821: subcc %o1, 1, %o1
83 stb %o4, [%o0 + 0x00]
84 bne,pt %icc, 1b
85 add %o0, 1, %o0
86.Lexit:
87 retl
88 mov %o3, %o0
89.Lnon_bzero_loop:
90 mov 0x08, %g3
91 mov 0x28, %o5
921: stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P
93 subcc %g1, 0x40, %g1
94 stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P
95 stxa %o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P
96 stxa %o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P
97 add %o0, 0x10, %o0
98 stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P
99 stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P
100 stxa %o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P
101 stxa %o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P
102 bne,pt %icc, 1b
103 add %o0, 0x30, %o0
104 ba,a,pt %icc, .Lpostloop
105 .size NG4bzero,.-NG4bzero
diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S
index c21c34c61dda..a114cbcf2a48 100644
--- a/arch/sparc/lib/NG4patch.S
+++ b/arch/sparc/lib/NG4patch.S
@@ -32,12 +32,23 @@ niagara4_patch_copyops:
32 nop 32 nop
33 .size niagara4_patch_copyops,.-niagara4_patch_copyops 33 .size niagara4_patch_copyops,.-niagara4_patch_copyops
34 34
35 .globl niagara4_patch_bzero
36 .type niagara4_patch_bzero,#function
37niagara4_patch_bzero:
38 NG_DO_PATCH(memset, NG4memset)
39 NG_DO_PATCH(__bzero, NG4bzero)
40 NG_DO_PATCH(__clear_user, NGclear_user)
41 NG_DO_PATCH(tsb_init, NGtsb_init)
42 retl
43 nop
44 .size niagara4_patch_bzero,.-niagara4_patch_bzero
45
35 .globl niagara4_patch_pageops 46 .globl niagara4_patch_pageops
36 .type niagara4_patch_pageops,#function 47 .type niagara4_patch_pageops,#function
37niagara4_patch_pageops: 48niagara4_patch_pageops:
38 NG_DO_PATCH(copy_user_page, NG4copy_user_page) 49 NG_DO_PATCH(copy_user_page, NG4copy_user_page)
39 NG_DO_PATCH(_clear_page, NGclear_page) 50 NG_DO_PATCH(_clear_page, NG4clear_page)
40 NG_DO_PATCH(clear_user_page, NGclear_user_page) 51 NG_DO_PATCH(clear_user_page, NG4clear_user_page)
41 retl 52 retl
42 nop 53 nop
43 .size niagara4_patch_pageops,.-niagara4_patch_pageops 54 .size niagara4_patch_pageops,.-niagara4_patch_pageops
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-21 03:52:10 -0500