aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc/lib
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc/lib')
-rw-r--r--arch/sparc/lib/Makefile4
-rw-r--r--arch/sparc/lib/NG2page.S61
-rw-r--r--arch/sparc/lib/NGpage.S114
-rw-r--r--arch/sparc/lib/atomic32.c8
-rw-r--r--arch/sparc/lib/ffs.S84
-rw-r--r--arch/sparc/lib/hweight.S51
-rw-r--r--arch/sparc/lib/memcpy.S804
7 files changed, 315 insertions, 811 deletions
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index 7f01b8fce8b..a3fc4375a15 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -31,13 +31,13 @@ lib-$(CONFIG_SPARC64) += NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o
31lib-$(CONFIG_SPARC64) += NGpatch.o NGpage.o NGbzero.o 31lib-$(CONFIG_SPARC64) += NGpatch.o NGpage.o NGbzero.o
32 32
33lib-$(CONFIG_SPARC64) += NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o 33lib-$(CONFIG_SPARC64) += NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o
34lib-$(CONFIG_SPARC64) += NG2patch.o NG2page.o 34lib-$(CONFIG_SPARC64) += NG2patch.o
35 35
36lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o 36lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
37lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o 37lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
38 38
39lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o 39lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o
40lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o 40lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
41 41
42obj-y += iomap.o 42obj-y += iomap.o
43obj-$(CONFIG_SPARC32) += atomic32.o 43obj-$(CONFIG_SPARC32) += atomic32.o
diff --git a/arch/sparc/lib/NG2page.S b/arch/sparc/lib/NG2page.S
deleted file mode 100644
index 73b6b7c72cb..00000000000
--- a/arch/sparc/lib/NG2page.S
+++ /dev/null
@@ -1,61 +0,0 @@
1/* NG2page.S: Niagara-2 optimized clear and copy page.
2 *
3 * Copyright (C) 2007 (davem@davemloft.net)
4 */
5
6#include <asm/asi.h>
7#include <asm/page.h>
8#include <asm/visasm.h>
9
10 .text
11 .align 32
12
13 /* This is heavily simplified from the sun4u variants
14 * because Niagara-2 does not have any D-cache aliasing issues.
15 */
16NG2copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
17 prefetch [%o1 + 0x00], #one_read
18 prefetch [%o1 + 0x40], #one_read
19 VISEntryHalf
20 set PAGE_SIZE, %g7
21 sub %o0, %o1, %g3
221: stxa %g0, [%o1 + %g3] ASI_BLK_INIT_QUAD_LDD_P
23 subcc %g7, 64, %g7
24 ldda [%o1] ASI_BLK_P, %f0
25 stda %f0, [%o1 + %g3] ASI_BLK_P
26 add %o1, 64, %o1
27 bne,pt %xcc, 1b
28 prefetch [%o1 + 0x40], #one_read
29 membar #Sync
30 VISExitHalf
31 retl
32 nop
33
34#define BRANCH_ALWAYS 0x10680000
35#define NOP 0x01000000
36#define NG_DO_PATCH(OLD, NEW) \
37 sethi %hi(NEW), %g1; \
38 or %g1, %lo(NEW), %g1; \
39 sethi %hi(OLD), %g2; \
40 or %g2, %lo(OLD), %g2; \
41 sub %g1, %g2, %g1; \
42 sethi %hi(BRANCH_ALWAYS), %g3; \
43 sll %g1, 11, %g1; \
44 srl %g1, 11 + 2, %g1; \
45 or %g3, %lo(BRANCH_ALWAYS), %g3; \
46 or %g3, %g1, %g3; \
47 stw %g3, [%g2]; \
48 sethi %hi(NOP), %g3; \
49 or %g3, %lo(NOP), %g3; \
50 stw %g3, [%g2 + 0x4]; \
51 flush %g2;
52
53 .globl niagara2_patch_pageops
54 .type niagara2_patch_pageops,#function
55niagara2_patch_pageops:
56 NG_DO_PATCH(copy_user_page, NG2copy_user_page)
57 NG_DO_PATCH(_clear_page, NGclear_page)
58 NG_DO_PATCH(clear_user_page, NGclear_user_page)
59 retl
60 nop
61 .size niagara2_patch_pageops,.-niagara2_patch_pageops
diff --git a/arch/sparc/lib/NGpage.S b/arch/sparc/lib/NGpage.S
index 428920de05b..b9e790b9c6b 100644
--- a/arch/sparc/lib/NGpage.S
+++ b/arch/sparc/lib/NGpage.S
@@ -16,55 +16,91 @@
16 */ 16 */
17 17
18NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ 18NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
19 prefetch [%o1 + 0x00], #one_read 19 save %sp, -192, %sp
20 mov 8, %g1 20 rd %asi, %g3
21 mov 16, %g2 21 wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
22 mov 24, %g3
23 set PAGE_SIZE, %g7 22 set PAGE_SIZE, %g7
23 prefetch [%i1 + 0x00], #one_read
24 prefetch [%i1 + 0x40], #one_read
24 25
251: ldda [%o1 + %g0] ASI_BLK_INIT_QUAD_LDD_P, %o2 261: prefetch [%i1 + 0x80], #one_read
26 ldda [%o1 + %g2] ASI_BLK_INIT_QUAD_LDD_P, %o4 27 prefetch [%i1 + 0xc0], #one_read
27 prefetch [%o1 + 0x40], #one_read 28 ldda [%i1 + 0x00] %asi, %o2
28 add %o1, 32, %o1 29 ldda [%i1 + 0x10] %asi, %o4
29 stxa %o2, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P 30 ldda [%i1 + 0x20] %asi, %l2
30 stxa %o3, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P 31 ldda [%i1 + 0x30] %asi, %l4
31 ldda [%o1 + %g0] ASI_BLK_INIT_QUAD_LDD_P, %o2 32 stxa %o2, [%i0 + 0x00] %asi
32 stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P 33 stxa %o3, [%i0 + 0x08] %asi
33 stxa %o5, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P 34 stxa %o4, [%i0 + 0x10] %asi
34 ldda [%o1 + %g2] ASI_BLK_INIT_QUAD_LDD_P, %o4 35 stxa %o5, [%i0 + 0x18] %asi
35 add %o1, 32, %o1 36 stxa %l2, [%i0 + 0x20] %asi
36 add %o0, 32, %o0 37 stxa %l3, [%i0 + 0x28] %asi
37 stxa %o2, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P 38 stxa %l4, [%i0 + 0x30] %asi
38 stxa %o3, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P 39 stxa %l5, [%i0 + 0x38] %asi
39 stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P 40 ldda [%i1 + 0x40] %asi, %o2
40 stxa %o5, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P 41 ldda [%i1 + 0x50] %asi, %o4
41 subcc %g7, 64, %g7 42 ldda [%i1 + 0x60] %asi, %l2
43 ldda [%i1 + 0x70] %asi, %l4
44 stxa %o2, [%i0 + 0x40] %asi
45 stxa %o3, [%i0 + 0x48] %asi
46 stxa %o4, [%i0 + 0x50] %asi
47 stxa %o5, [%i0 + 0x58] %asi
48 stxa %l2, [%i0 + 0x60] %asi
49 stxa %l3, [%i0 + 0x68] %asi
50 stxa %l4, [%i0 + 0x70] %asi
51 stxa %l5, [%i0 + 0x78] %asi
52 add %i1, 128, %i1
53 subcc %g7, 128, %g7
42 bne,pt %xcc, 1b 54 bne,pt %xcc, 1b
43 add %o0, 32, %o0 55 add %i0, 128, %i0
56 wr %g3, 0x0, %asi
44 membar #Sync 57 membar #Sync
45 retl 58 ret
46 nop 59 restore
47 60
48 .globl NGclear_page, NGclear_user_page 61 .align 32
49NGclear_page: /* %o0=dest */ 62NGclear_page: /* %o0=dest */
50NGclear_user_page: /* %o0=dest, %o1=vaddr */ 63NGclear_user_page: /* %o0=dest, %o1=vaddr */
51 mov 8, %g1 64 rd %asi, %g3
52 mov 16, %g2 65 wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
53 mov 24, %g3
54 set PAGE_SIZE, %g7 66 set PAGE_SIZE, %g7
55 67
561: stxa %g0, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P 681: stxa %g0, [%o0 + 0x00] %asi
57 stxa %g0, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P 69 stxa %g0, [%o0 + 0x08] %asi
58 stxa %g0, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P 70 stxa %g0, [%o0 + 0x10] %asi
59 stxa %g0, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P 71 stxa %g0, [%o0 + 0x18] %asi
60 add %o0, 32, %o0 72 stxa %g0, [%o0 + 0x20] %asi
61 stxa %g0, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P 73 stxa %g0, [%o0 + 0x28] %asi
62 stxa %g0, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P 74 stxa %g0, [%o0 + 0x30] %asi
63 stxa %g0, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P 75 stxa %g0, [%o0 + 0x38] %asi
64 stxa %g0, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P 76 stxa %g0, [%o0 + 0x40] %asi
65 subcc %g7, 64, %g7 77 stxa %g0, [%o0 + 0x48] %asi
78 stxa %g0, [%o0 + 0x50] %asi
79 stxa %g0, [%o0 + 0x58] %asi
80 stxa %g0, [%o0 + 0x60] %asi
81 stxa %g0, [%o0 + 0x68] %asi
82 stxa %g0, [%o0 + 0x70] %asi
83 stxa %g0, [%o0 + 0x78] %asi
84 stxa %g0, [%o0 + 0x80] %asi
85 stxa %g0, [%o0 + 0x88] %asi
86 stxa %g0, [%o0 + 0x90] %asi
87 stxa %g0, [%o0 + 0x98] %asi
88 stxa %g0, [%o0 + 0xa0] %asi
89 stxa %g0, [%o0 + 0xa8] %asi
90 stxa %g0, [%o0 + 0xb0] %asi
91 stxa %g0, [%o0 + 0xb8] %asi
92 stxa %g0, [%o0 + 0xc0] %asi
93 stxa %g0, [%o0 + 0xc8] %asi
94 stxa %g0, [%o0 + 0xd0] %asi
95 stxa %g0, [%o0 + 0xd8] %asi
96 stxa %g0, [%o0 + 0xe0] %asi
97 stxa %g0, [%o0 + 0xe8] %asi
98 stxa %g0, [%o0 + 0xf0] %asi
99 stxa %g0, [%o0 + 0xf8] %asi
100 subcc %g7, 256, %g7
66 bne,pt %xcc, 1b 101 bne,pt %xcc, 1b
67 add %o0, 32, %o0 102 add %o0, 256, %o0
103 wr %g3, 0x0, %asi
68 membar #Sync 104 membar #Sync
69 retl 105 retl
70 nop 106 nop
diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c
index d3c7a12ad87..1d32b54089a 100644
--- a/arch/sparc/lib/atomic32.c
+++ b/arch/sparc/lib/atomic32.c
@@ -7,7 +7,7 @@
7 * Based on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf 7 * Based on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf
8 */ 8 */
9 9
10#include <asm/atomic.h> 10#include <linux/atomic.h>
11#include <linux/spinlock.h> 11#include <linux/spinlock.h>
12#include <linux/module.h> 12#include <linux/module.h>
13 13
@@ -55,7 +55,7 @@ int atomic_cmpxchg(atomic_t *v, int old, int new)
55} 55}
56EXPORT_SYMBOL(atomic_cmpxchg); 56EXPORT_SYMBOL(atomic_cmpxchg);
57 57
58int atomic_add_unless(atomic_t *v, int a, int u) 58int __atomic_add_unless(atomic_t *v, int a, int u)
59{ 59{
60 int ret; 60 int ret;
61 unsigned long flags; 61 unsigned long flags;
@@ -65,9 +65,9 @@ int atomic_add_unless(atomic_t *v, int a, int u)
65 if (ret != u) 65 if (ret != u)
66 v->counter += a; 66 v->counter += a;
67 spin_unlock_irqrestore(ATOMIC_HASH(v), flags); 67 spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
68 return ret != u; 68 return ret;
69} 69}
70EXPORT_SYMBOL(atomic_add_unless); 70EXPORT_SYMBOL(__atomic_add_unless);
71 71
72/* Atomic operations are already serializing */ 72/* Atomic operations are already serializing */
73void atomic_set(atomic_t *v, int i) 73void atomic_set(atomic_t *v, int i)
diff --git a/arch/sparc/lib/ffs.S b/arch/sparc/lib/ffs.S
new file mode 100644
index 00000000000..b39389f6989
--- /dev/null
+++ b/arch/sparc/lib/ffs.S
@@ -0,0 +1,84 @@
1#include <linux/linkage.h>
2
3 .register %g2,#scratch
4
5 .text
6 .align 32
7
8ENTRY(ffs)
9 brnz,pt %o0, 1f
10 mov 1, %o1
11 retl
12 clr %o0
13 nop
14 nop
15ENTRY(__ffs)
16 sllx %o0, 32, %g1 /* 1 */
17 srlx %o0, 32, %g2
18
19 clr %o1 /* 2 */
20 movrz %g1, %g2, %o0
21
22 movrz %g1, 32, %o1 /* 3 */
231: clr %o2
24
25 sllx %o0, (64 - 16), %g1 /* 4 */
26 srlx %o0, 16, %g2
27
28 movrz %g1, %g2, %o0 /* 5 */
29 clr %o3
30
31 movrz %g1, 16, %o2 /* 6 */
32 clr %o4
33
34 and %o0, 0xff, %g1 /* 7 */
35 srlx %o0, 8, %g2
36
37 movrz %g1, %g2, %o0 /* 8 */
38 clr %o5
39
40 movrz %g1, 8, %o3 /* 9 */
41 add %o2, %o1, %o2
42
43 and %o0, 0xf, %g1 /* 10 */
44 srlx %o0, 4, %g2
45
46 movrz %g1, %g2, %o0 /* 11 */
47 add %o2, %o3, %o2
48
49 movrz %g1, 4, %o4 /* 12 */
50
51 and %o0, 0x3, %g1 /* 13 */
52 srlx %o0, 2, %g2
53
54 movrz %g1, %g2, %o0 /* 14 */
55 add %o2, %o4, %o2
56
57 movrz %g1, 2, %o5 /* 15 */
58
59 and %o0, 0x1, %g1 /* 16 */
60
61 add %o2, %o5, %o2 /* 17 */
62 xor %g1, 0x1, %g1
63
64 retl /* 18 */
65 add %o2, %g1, %o0
66ENDPROC(ffs)
67ENDPROC(__ffs)
68
69 .section .popc_6insn_patch, "ax"
70 .word ffs
71 brz,pn %o0, 98f
72 neg %o0, %g1
73 xnor %o0, %g1, %o1
74 popc %o1, %o0
7598: retl
76 nop
77 .word __ffs
78 neg %o0, %g1
79 xnor %o0, %g1, %o1
80 popc %o1, %o0
81 retl
82 sub %o0, 1, %o0
83 nop
84 .previous
diff --git a/arch/sparc/lib/hweight.S b/arch/sparc/lib/hweight.S
new file mode 100644
index 00000000000..95414e0a680
--- /dev/null
+++ b/arch/sparc/lib/hweight.S
@@ -0,0 +1,51 @@
1#include <linux/linkage.h>
2
3 .text
4 .align 32
5ENTRY(__arch_hweight8)
6 ba,pt %xcc, __sw_hweight8
7 nop
8 nop
9ENDPROC(__arch_hweight8)
10 .section .popc_3insn_patch, "ax"
11 .word __arch_hweight8
12 sllx %o0, 64-8, %g1
13 retl
14 popc %g1, %o0
15 .previous
16
17ENTRY(__arch_hweight16)
18 ba,pt %xcc, __sw_hweight16
19 nop
20 nop
21ENDPROC(__arch_hweight16)
22 .section .popc_3insn_patch, "ax"
23 .word __arch_hweight16
24 sllx %o0, 64-16, %g1
25 retl
26 popc %g1, %o0
27 .previous
28
29ENTRY(__arch_hweight32)
30 ba,pt %xcc, __sw_hweight32
31 nop
32 nop
33ENDPROC(__arch_hweight32)
34 .section .popc_3insn_patch, "ax"
35 .word __arch_hweight32
36 sllx %o0, 64-32, %g1
37 retl
38 popc %g1, %o0
39 .previous
40
41ENTRY(__arch_hweight64)
42 ba,pt %xcc, __sw_hweight64
43 nop
44 nop
45ENDPROC(__arch_hweight64)
46 .section .popc_3insn_patch, "ax"
47 .word __arch_hweight64
48 retl
49 popc %o0, %o0
50 nop
51 .previous
diff --git a/arch/sparc/lib/memcpy.S b/arch/sparc/lib/memcpy.S
index 34fe6575173..4d8c497517b 100644
--- a/arch/sparc/lib/memcpy.S
+++ b/arch/sparc/lib/memcpy.S
@@ -7,40 +7,12 @@
7 * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 7 * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 */ 8 */
9 9
10#ifdef __KERNEL__ 10#define FUNC(x) \
11
12#define FUNC(x) \
13 .globl x; \ 11 .globl x; \
14 .type x,@function; \ 12 .type x,@function; \
15 .align 4; \ 13 .align 4; \
16x: 14x:
17 15
18#undef FASTER_REVERSE
19#undef FASTER_NONALIGNED
20#define FASTER_ALIGNED
21
22/* In kernel these functions don't return a value.
23 * One should use macros in asm/string.h for that purpose.
24 * We return 0, so that bugs are more apparent.
25 */
26#define SETUP_RETL
27#define RETL_INSN clr %o0
28
29#else
30
31/* libc */
32
33#include "DEFS.h"
34
35#define FASTER_REVERSE
36#define FASTER_NONALIGNED
37#define FASTER_ALIGNED
38
39#define SETUP_RETL mov %o0, %g6
40#define RETL_INSN mov %g6, %o0
41
42#endif
43
44/* Both these macros have to start with exactly the same insn */ 16/* Both these macros have to start with exactly the same insn */
45#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ 17#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
46 ldd [%src + (offset) + 0x00], %t0; \ 18 ldd [%src + (offset) + 0x00], %t0; \
@@ -164,30 +136,6 @@ x:
164 .text 136 .text
165 .align 4 137 .align 4
166 138
167#ifdef FASTER_REVERSE
168
16970: /* rdword_align */
170
171 andcc %o1, 1, %g0
172 be 4f
173 andcc %o1, 2, %g0
174
175 ldub [%o1 - 1], %g2
176 sub %o1, 1, %o1
177 stb %g2, [%o0 - 1]
178 sub %o2, 1, %o2
179 be 3f
180 sub %o0, 1, %o0
1814:
182 lduh [%o1 - 2], %g2
183 sub %o1, 2, %o1
184 sth %g2, [%o0 - 2]
185 sub %o2, 2, %o2
186 b 3f
187 sub %o0, 2, %o0
188
189#endif /* FASTER_REVERSE */
190
1910: 1390:
192 retl 140 retl
193 nop ! Only bcopy returns here and it retuns void... 141 nop ! Only bcopy returns here and it retuns void...
@@ -198,7 +146,7 @@ FUNC(__memmove)
198#endif 146#endif
199FUNC(memmove) 147FUNC(memmove)
200 cmp %o0, %o1 148 cmp %o0, %o1
201 SETUP_RETL 149 mov %o0, %g7
202 bleu 9f 150 bleu 9f
203 sub %o0, %o1, %o4 151 sub %o0, %o1, %o4
204 152
@@ -207,8 +155,6 @@ FUNC(memmove)
207 bleu 0f 155 bleu 0f
208 andcc %o4, 3, %o5 156 andcc %o4, 3, %o5
209 157
210#ifndef FASTER_REVERSE
211
212 add %o1, %o2, %o1 158 add %o1, %o2, %o1
213 add %o0, %o2, %o0 159 add %o0, %o2, %o0
214 sub %o1, 1, %o1 160 sub %o1, 1, %o1
@@ -224,295 +170,7 @@ FUNC(memmove)
224 sub %o0, 1, %o0 170 sub %o0, 1, %o0
225 171
226 retl 172 retl
227 RETL_INSN 173 mov %g7, %o0
228
229#else /* FASTER_REVERSE */
230
231 add %o1, %o2, %o1
232 add %o0, %o2, %o0
233 bne 77f
234 cmp %o2, 15
235 bleu 91f
236 andcc %o1, 3, %g0
237 bne 70b
2383:
239 andcc %o1, 4, %g0
240
241 be 2f
242 mov %o2, %g1
243
244 ld [%o1 - 4], %o4
245 sub %g1, 4, %g1
246 st %o4, [%o0 - 4]
247 sub %o1, 4, %o1
248 sub %o0, 4, %o0
2492:
250 andcc %g1, 0xffffff80, %g7
251 be 3f
252 andcc %o0, 4, %g0
253
254 be 74f + 4
2555:
256 RMOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
257 RMOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
258 RMOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
259 RMOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
260 subcc %g7, 128, %g7
261 sub %o1, 128, %o1
262 bne 5b
263 sub %o0, 128, %o0
2643:
265 andcc %g1, 0x70, %g7
266 be 72f
267 andcc %g1, 8, %g0
268
269 sethi %hi(72f), %o5
270 srl %g7, 1, %o4
271 add %g7, %o4, %o4
272 sub %o1, %g7, %o1
273 sub %o5, %o4, %o5
274 jmpl %o5 + %lo(72f), %g0
275 sub %o0, %g7, %o0
276
27771: /* rmemcpy_table */
278 RMOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
279 RMOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
280 RMOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
281 RMOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
282 RMOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
283 RMOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
284 RMOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
285
28672: /* rmemcpy_table_end */
287
288 be 73f
289 andcc %g1, 4, %g0
290
291 ldd [%o1 - 0x08], %g2
292 sub %o0, 8, %o0
293 sub %o1, 8, %o1
294 st %g2, [%o0]
295 st %g3, [%o0 + 0x04]
296
29773: /* rmemcpy_last7 */
298
299 be 1f
300 andcc %g1, 2, %g0
301
302 ld [%o1 - 4], %g2
303 sub %o1, 4, %o1
304 st %g2, [%o0 - 4]
305 sub %o0, 4, %o0
3061:
307 be 1f
308 andcc %g1, 1, %g0
309
310 lduh [%o1 - 2], %g2
311 sub %o1, 2, %o1
312 sth %g2, [%o0 - 2]
313 sub %o0, 2, %o0
3141:
315 be 1f
316 nop
317
318 ldub [%o1 - 1], %g2
319 stb %g2, [%o0 - 1]
3201:
321 retl
322 RETL_INSN
323
32474: /* rldd_std */
325 RMOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
326 RMOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
327 RMOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
328 RMOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
329 subcc %g7, 128, %g7
330 sub %o1, 128, %o1
331 bne 74b
332 sub %o0, 128, %o0
333
334 andcc %g1, 0x70, %g7
335 be 72b
336 andcc %g1, 8, %g0
337
338 sethi %hi(72b), %o5
339 srl %g7, 1, %o4
340 add %g7, %o4, %o4
341 sub %o1, %g7, %o1
342 sub %o5, %o4, %o5
343 jmpl %o5 + %lo(72b), %g0
344 sub %o0, %g7, %o0
345
34675: /* rshort_end */
347
348 and %o2, 0xe, %o3
3492:
350 sethi %hi(76f), %o5
351 sll %o3, 3, %o4
352 sub %o0, %o3, %o0
353 sub %o5, %o4, %o5
354 sub %o1, %o3, %o1
355 jmpl %o5 + %lo(76f), %g0
356 andcc %o2, 1, %g0
357
358 RMOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3)
359 RMOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3)
360 RMOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3)
361 RMOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3)
362 RMOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3)
363 RMOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3)
364 RMOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3)
365
36676: /* rshort_table_end */
367
368 be 1f
369 nop
370 ldub [%o1 - 1], %g2
371 stb %g2, [%o0 - 1]
3721:
373 retl
374 RETL_INSN
375
37691: /* rshort_aligned_end */
377
378 bne 75b
379 andcc %o2, 8, %g0
380
381 be 1f
382 andcc %o2, 4, %g0
383
384 ld [%o1 - 0x08], %g2
385 ld [%o1 - 0x04], %g3
386 sub %o1, 8, %o1
387 st %g2, [%o0 - 0x08]
388 st %g3, [%o0 - 0x04]
389 sub %o0, 8, %o0
3901:
391 b 73b
392 mov %o2, %g1
393
39477: /* rnon_aligned */
395 cmp %o2, 15
396 bleu 75b
397 andcc %o0, 3, %g0
398 be 64f
399 andcc %o0, 1, %g0
400 be 63f
401 andcc %o0, 2, %g0
402 ldub [%o1 - 1], %g5
403 sub %o1, 1, %o1
404 stb %g5, [%o0 - 1]
405 sub %o0, 1, %o0
406 be 64f
407 sub %o2, 1, %o2
40863:
409 ldub [%o1 - 1], %g5
410 sub %o1, 2, %o1
411 stb %g5, [%o0 - 1]
412 sub %o0, 2, %o0
413 ldub [%o1], %g5
414 sub %o2, 2, %o2
415 stb %g5, [%o0]
41664:
417 and %o1, 3, %g2
418 and %o1, -4, %o1
419 and %o2, 0xc, %g3
420 add %o1, 4, %o1
421 cmp %g3, 4
422 sll %g2, 3, %g4
423 mov 32, %g2
424 be 4f
425 sub %g2, %g4, %g7
426
427 blu 3f
428 cmp %g3, 8
429
430 be 2f
431 srl %o2, 2, %g3
432
433 ld [%o1 - 4], %o3
434 add %o0, -8, %o0
435 ld [%o1 - 8], %o4
436 add %o1, -16, %o1
437 b 7f
438 add %g3, 1, %g3
4392:
440 ld [%o1 - 4], %o4
441 add %o0, -4, %o0
442 ld [%o1 - 8], %g1
443 add %o1, -12, %o1
444 b 8f
445 add %g3, 2, %g3
4463:
447 ld [%o1 - 4], %o5
448 add %o0, -12, %o0
449 ld [%o1 - 8], %o3
450 add %o1, -20, %o1
451 b 6f
452 srl %o2, 2, %g3
4534:
454 ld [%o1 - 4], %g1
455 srl %o2, 2, %g3
456 ld [%o1 - 8], %o5
457 add %o1, -24, %o1
458 add %o0, -16, %o0
459 add %g3, -1, %g3
460
461 ld [%o1 + 12], %o3
4625:
463 sll %o5, %g4, %g2
464 srl %g1, %g7, %g5
465 or %g2, %g5, %g2
466 st %g2, [%o0 + 12]
4676:
468 ld [%o1 + 8], %o4
469 sll %o3, %g4, %g2
470 srl %o5, %g7, %g5
471 or %g2, %g5, %g2
472 st %g2, [%o0 + 8]
4737:
474 ld [%o1 + 4], %g1
475 sll %o4, %g4, %g2
476 srl %o3, %g7, %g5
477 or %g2, %g5, %g2
478 st %g2, [%o0 + 4]
4798:
480 ld [%o1], %o5
481 sll %g1, %g4, %g2
482 srl %o4, %g7, %g5
483 addcc %g3, -4, %g3
484 or %g2, %g5, %g2
485 add %o1, -16, %o1
486 st %g2, [%o0]
487 add %o0, -16, %o0
488 bne,a 5b
489 ld [%o1 + 12], %o3
490 sll %o5, %g4, %g2
491 srl %g1, %g7, %g5
492 srl %g4, 3, %g3
493 or %g2, %g5, %g2
494 add %o1, %g3, %o1
495 andcc %o2, 2, %g0
496 st %g2, [%o0 + 12]
497 be 1f
498 andcc %o2, 1, %g0
499
500 ldub [%o1 + 15], %g5
501 add %o1, -2, %o1
502 stb %g5, [%o0 + 11]
503 add %o0, -2, %o0
504 ldub [%o1 + 16], %g5
505 stb %g5, [%o0 + 12]
5061:
507 be 1f
508 nop
509 ldub [%o1 + 15], %g5
510 stb %g5, [%o0 + 11]
5111:
512 retl
513 RETL_INSN
514
515#endif /* FASTER_REVERSE */
516 174
517/* NOTE: This code is executed just for the cases, 175/* NOTE: This code is executed just for the cases,
518 where %src (=%o1) & 3 is != 0. 176 where %src (=%o1) & 3 is != 0.
@@ -546,7 +204,7 @@ FUNC(memmove)
546FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ 204FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
547 205
548 sub %o0, %o1, %o4 206 sub %o0, %o1, %o4
549 SETUP_RETL 207 mov %o0, %g7
5509: 2089:
551 andcc %o4, 3, %o5 209 andcc %o4, 3, %o5
5520: 2100:
@@ -569,7 +227,7 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
569 add %o1, 4, %o1 227 add %o1, 4, %o1
570 add %o0, 4, %o0 228 add %o0, 4, %o0
5712: 2292:
572 andcc %g1, 0xffffff80, %g7 230 andcc %g1, 0xffffff80, %g0
573 be 3f 231 be 3f
574 andcc %o0, 4, %g0 232 andcc %o0, 4, %g0
575 233
@@ -579,22 +237,23 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
579 MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) 237 MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
580 MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) 238 MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
581 MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) 239 MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
582 subcc %g7, 128, %g7 240 sub %g1, 128, %g1
583 add %o1, 128, %o1 241 add %o1, 128, %o1
584 bne 5b 242 cmp %g1, 128
243 bge 5b
585 add %o0, 128, %o0 244 add %o0, 128, %o0
5863: 2453:
587 andcc %g1, 0x70, %g7 246 andcc %g1, 0x70, %g4
588 be 80f 247 be 80f
589 andcc %g1, 8, %g0 248 andcc %g1, 8, %g0
590 249
591 sethi %hi(80f), %o5 250 sethi %hi(80f), %o5
592 srl %g7, 1, %o4 251 srl %g4, 1, %o4
593 add %g7, %o4, %o4 252 add %g4, %o4, %o4
594 add %o1, %g7, %o1 253 add %o1, %g4, %o1
595 sub %o5, %o4, %o5 254 sub %o5, %o4, %o5
596 jmpl %o5 + %lo(80f), %g0 255 jmpl %o5 + %lo(80f), %g0
597 add %o0, %g7, %o0 256 add %o0, %g4, %o0
598 257
59979: /* memcpy_table */ 25879: /* memcpy_table */
600 259
@@ -641,43 +300,28 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
641 stb %g2, [%o0] 300 stb %g2, [%o0]
6421: 3011:
643 retl 302 retl
644 RETL_INSN 303 mov %g7, %o0
645 304
64682: /* ldd_std */ 30582: /* ldd_std */
647 MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5) 306 MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
648 MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) 307 MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
649 MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) 308 MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
650 MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) 309 MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
651 subcc %g7, 128, %g7 310 subcc %g1, 128, %g1
652 add %o1, 128, %o1 311 add %o1, 128, %o1
653 bne 82b 312 cmp %g1, 128
313 bge 82b
654 add %o0, 128, %o0 314 add %o0, 128, %o0
655 315
656#ifndef FASTER_ALIGNED 316 andcc %g1, 0x70, %g4
657
658 andcc %g1, 0x70, %g7
659 be 80b
660 andcc %g1, 8, %g0
661
662 sethi %hi(80b), %o5
663 srl %g7, 1, %o4
664 add %g7, %o4, %o4
665 add %o1, %g7, %o1
666 sub %o5, %o4, %o5
667 jmpl %o5 + %lo(80b), %g0
668 add %o0, %g7, %o0
669
670#else /* FASTER_ALIGNED */
671
672 andcc %g1, 0x70, %g7
673 be 84f 317 be 84f
674 andcc %g1, 8, %g0 318 andcc %g1, 8, %g0
675 319
676 sethi %hi(84f), %o5 320 sethi %hi(84f), %o5
677 add %o1, %g7, %o1 321 add %o1, %g4, %o1
678 sub %o5, %g7, %o5 322 sub %o5, %g4, %o5
679 jmpl %o5 + %lo(84f), %g0 323 jmpl %o5 + %lo(84f), %g0
680 add %o0, %g7, %o0 324 add %o0, %g4, %o0
681 325
68283: /* amemcpy_table */ 32683: /* amemcpy_table */
683 327
@@ -721,382 +365,132 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
721 stb %g2, [%o0] 365 stb %g2, [%o0]
7221: 3661:
723 retl 367 retl
724 RETL_INSN 368 mov %g7, %o0
725
726#endif /* FASTER_ALIGNED */
727 369
72886: /* non_aligned */ 37086: /* non_aligned */
729 cmp %o2, 6 371 cmp %o2, 6
730 bleu 88f 372 bleu 88f
373 nop
731 374
732#ifdef FASTER_NONALIGNED 375 save %sp, -96, %sp
733 376 andcc %i0, 3, %g0
734 cmp %o2, 256
735 bcc 87f
736
737#endif /* FASTER_NONALIGNED */
738
739 andcc %o0, 3, %g0
740 be 61f 377 be 61f
741 andcc %o0, 1, %g0 378 andcc %i0, 1, %g0
742 be 60f 379 be 60f
743 andcc %o0, 2, %g0 380 andcc %i0, 2, %g0
744 381
745 ldub [%o1], %g5 382 ldub [%i1], %g5
746 add %o1, 1, %o1 383 add %i1, 1, %i1
747 stb %g5, [%o0] 384 stb %g5, [%i0]
748 sub %o2, 1, %o2 385 sub %i2, 1, %i2
749 bne 61f 386 bne 61f
750 add %o0, 1, %o0 387 add %i0, 1, %i0
75160: 38860:
752 ldub [%o1], %g3 389 ldub [%i1], %g3
753 add %o1, 2, %o1 390 add %i1, 2, %i1
754 stb %g3, [%o0] 391 stb %g3, [%i0]
755 sub %o2, 2, %o2 392 sub %i2, 2, %i2
756 ldub [%o1 - 1], %g3 393 ldub [%i1 - 1], %g3
757 add %o0, 2, %o0 394 add %i0, 2, %i0
758 stb %g3, [%o0 - 1] 395 stb %g3, [%i0 - 1]
75961: 39661:
760 and %o1, 3, %g2 397 and %i1, 3, %g2
761 and %o2, 0xc, %g3 398 and %i2, 0xc, %g3
762 and %o1, -4, %o1 399 and %i1, -4, %i1
763 cmp %g3, 4 400 cmp %g3, 4
764 sll %g2, 3, %g4 401 sll %g2, 3, %g4
765 mov 32, %g2 402 mov 32, %g2
766 be 4f 403 be 4f
767 sub %g2, %g4, %g7 404 sub %g2, %g4, %l0
768 405
769 blu 3f 406 blu 3f
770 cmp %g3, 0x8 407 cmp %g3, 0x8
771 408
772 be 2f 409 be 2f
773 srl %o2, 2, %g3 410 srl %i2, 2, %g3
774 411
775 ld [%o1], %o3 412 ld [%i1], %i3
776 add %o0, -8, %o0 413 add %i0, -8, %i0
777 ld [%o1 + 4], %o4 414 ld [%i1 + 4], %i4
778 b 8f 415 b 8f
779 add %g3, 1, %g3 416 add %g3, 1, %g3
7802: 4172:
781 ld [%o1], %o4 418 ld [%i1], %i4
782 add %o0, -12, %o0 419 add %i0, -12, %i0
783 ld [%o1 + 4], %o5 420 ld [%i1 + 4], %i5
784 add %g3, 2, %g3 421 add %g3, 2, %g3
785 b 9f 422 b 9f
786 add %o1, -4, %o1 423 add %i1, -4, %i1
7873: 4243:
788 ld [%o1], %g1 425 ld [%i1], %g1
789 add %o0, -4, %o0 426 add %i0, -4, %i0
790 ld [%o1 + 4], %o3 427 ld [%i1 + 4], %i3
791 srl %o2, 2, %g3 428 srl %i2, 2, %g3
792 b 7f 429 b 7f
793 add %o1, 4, %o1 430 add %i1, 4, %i1
7944: 4314:
795 ld [%o1], %o5 432 ld [%i1], %i5
796 cmp %o2, 7 433 cmp %i2, 7
797 ld [%o1 + 4], %g1 434 ld [%i1 + 4], %g1
798 srl %o2, 2, %g3 435 srl %i2, 2, %g3
799 bleu 10f 436 bleu 10f
800 add %o1, 8, %o1 437 add %i1, 8, %i1
801 438
802 ld [%o1], %o3 439 ld [%i1], %i3
803 add %g3, -1, %g3 440 add %g3, -1, %g3
8045: 4415:
805 sll %o5, %g4, %g2 442 sll %i5, %g4, %g2
806 srl %g1, %g7, %g5 443 srl %g1, %l0, %g5
807 or %g2, %g5, %g2 444 or %g2, %g5, %g2
808 st %g2, [%o0] 445 st %g2, [%i0]
8097: 4467:
810 ld [%o1 + 4], %o4 447 ld [%i1 + 4], %i4
811 sll %g1, %g4, %g2 448 sll %g1, %g4, %g2
812 srl %o3, %g7, %g5 449 srl %i3, %l0, %g5
813 or %g2, %g5, %g2 450 or %g2, %g5, %g2
814 st %g2, [%o0 + 4] 451 st %g2, [%i0 + 4]
8158: 4528:
816 ld [%o1 + 8], %o5 453 ld [%i1 + 8], %i5
817 sll %o3, %g4, %g2 454 sll %i3, %g4, %g2
818 srl %o4, %g7, %g5 455 srl %i4, %l0, %g5
819 or %g2, %g5, %g2 456 or %g2, %g5, %g2
820 st %g2, [%o0 + 8] 457 st %g2, [%i0 + 8]
8219: 4589:
822 ld [%o1 + 12], %g1 459 ld [%i1 + 12], %g1
823 sll %o4, %g4, %g2 460 sll %i4, %g4, %g2
824 srl %o5, %g7, %g5 461 srl %i5, %l0, %g5
825 addcc %g3, -4, %g3 462 addcc %g3, -4, %g3
826 or %g2, %g5, %g2 463 or %g2, %g5, %g2
827 add %o1, 16, %o1 464 add %i1, 16, %i1
828 st %g2, [%o0 + 12] 465 st %g2, [%i0 + 12]
829 add %o0, 16, %o0 466 add %i0, 16, %i0
830 bne,a 5b 467 bne,a 5b
831 ld [%o1], %o3 468 ld [%i1], %i3
83210: 46910:
833 sll %o5, %g4, %g2 470 sll %i5, %g4, %g2
834 srl %g1, %g7, %g5 471 srl %g1, %l0, %g5
835 srl %g7, 3, %g3 472 srl %l0, 3, %g3
836 or %g2, %g5, %g2 473 or %g2, %g5, %g2
837 sub %o1, %g3, %o1 474 sub %i1, %g3, %i1
838 andcc %o2, 2, %g0 475 andcc %i2, 2, %g0
839 st %g2, [%o0] 476 st %g2, [%i0]
840 be 1f 477 be 1f
841 andcc %o2, 1, %g0 478 andcc %i2, 1, %g0
842 479
843 ldub [%o1], %g2 480 ldub [%i1], %g2
844 add %o1, 2, %o1 481 add %i1, 2, %i1
845 stb %g2, [%o0 + 4] 482 stb %g2, [%i0 + 4]
846 add %o0, 2, %o0 483 add %i0, 2, %i0
847 ldub [%o1 - 1], %g2 484 ldub [%i1 - 1], %g2
848 stb %g2, [%o0 + 3] 485 stb %g2, [%i0 + 3]
8491: 4861:
850 be 1f 487 be 1f
851 nop 488 nop
852 ldub [%o1], %g2 489 ldub [%i1], %g2
853 stb %g2, [%o0 + 4] 490 stb %g2, [%i0 + 4]
8541:
855 retl
856 RETL_INSN
857
858#ifdef FASTER_NONALIGNED
859
86087: /* faster_nonaligned */
861
862 andcc %o1, 3, %g0
863 be 3f
864 andcc %o1, 1, %g0
865
866 be 4f
867 andcc %o1, 2, %g0
868
869 ldub [%o1], %g2
870 add %o1, 1, %o1
871 stb %g2, [%o0]
872 sub %o2, 1, %o2
873 bne 3f
874 add %o0, 1, %o0
8754:
876 lduh [%o1], %g2
877 add %o1, 2, %o1
878 srl %g2, 8, %g3
879 sub %o2, 2, %o2
880 stb %g3, [%o0]
881 add %o0, 2, %o0
882 stb %g2, [%o0 - 1]
8833:
884 andcc %o1, 4, %g0
885
886 bne 2f
887 cmp %o5, 1
888
889 ld [%o1], %o4
890 srl %o4, 24, %g2
891 stb %g2, [%o0]
892 srl %o4, 16, %g3
893 stb %g3, [%o0 + 1]
894 srl %o4, 8, %g2
895 stb %g2, [%o0 + 2]
896 sub %o2, 4, %o2
897 stb %o4, [%o0 + 3]
898 add %o1, 4, %o1
899 add %o0, 4, %o0
9002:
901 be 33f
902 cmp %o5, 2
903 be 32f
904 sub %o2, 4, %o2
90531:
906 ld [%o1], %g2
907 add %o1, 4, %o1
908 srl %g2, 24, %g3
909 and %o0, 7, %g5
910 stb %g3, [%o0]
911 cmp %g5, 7
912 sll %g2, 8, %g1
913 add %o0, 4, %o0
914 be 41f
915 and %o2, 0xffffffc0, %o3
916 ld [%o0 - 7], %o4
9174:
918 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
919 SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
920 SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
921 SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
922 subcc %o3, 64, %o3
923 add %o1, 64, %o1
924 bne 4b
925 add %o0, 64, %o0
926
927 andcc %o2, 0x30, %o3
928 be,a 1f
929 srl %g1, 16, %g2
9304:
931 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
932 subcc %o3, 16, %o3
933 add %o1, 16, %o1
934 bne 4b
935 add %o0, 16, %o0
936
937 srl %g1, 16, %g2
9381:
939 st %o4, [%o0 - 7]
940 sth %g2, [%o0 - 3]
941 srl %g1, 8, %g4
942 b 88f
943 stb %g4, [%o0 - 1]
94432:
945 ld [%o1], %g2
946 add %o1, 4, %o1
947 srl %g2, 16, %g3
948 and %o0, 7, %g5
949 sth %g3, [%o0]
950 cmp %g5, 6
951 sll %g2, 16, %g1
952 add %o0, 4, %o0
953 be 42f
954 and %o2, 0xffffffc0, %o3
955 ld [%o0 - 6], %o4
9564:
957 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
958 SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
959 SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
960 SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
961 subcc %o3, 64, %o3
962 add %o1, 64, %o1
963 bne 4b
964 add %o0, 64, %o0
965
966 andcc %o2, 0x30, %o3
967 be,a 1f
968 srl %g1, 16, %g2
9694:
970 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
971 subcc %o3, 16, %o3
972 add %o1, 16, %o1
973 bne 4b
974 add %o0, 16, %o0
975
976 srl %g1, 16, %g2
9771:
978 st %o4, [%o0 - 6]
979 b 88f
980 sth %g2, [%o0 - 2]
98133:
982 ld [%o1], %g2
983 sub %o2, 4, %o2
984 srl %g2, 24, %g3
985 and %o0, 7, %g5
986 stb %g3, [%o0]
987 cmp %g5, 5
988 srl %g2, 8, %g4
989 sll %g2, 24, %g1
990 sth %g4, [%o0 + 1]
991 add %o1, 4, %o1
992 be 43f
993 and %o2, 0xffffffc0, %o3
994
995 ld [%o0 - 1], %o4
996 add %o0, 4, %o0
9974:
998 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
999 SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
1000 SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
1001 SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
1002 subcc %o3, 64, %o3
1003 add %o1, 64, %o1
1004 bne 4b
1005 add %o0, 64, %o0
1006
1007 andcc %o2, 0x30, %o3
1008 be,a 1f
1009 srl %g1, 24, %g2
10104:
1011 SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1)
1012 subcc %o3, 16, %o3
1013 add %o1, 16, %o1
1014 bne 4b
1015 add %o0, 16, %o0
1016
1017 srl %g1, 24, %g2
10181:
1019 st %o4, [%o0 - 5]
1020 b 88f
1021 stb %g2, [%o0 - 1]
102241:
1023 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
1024 SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
1025 SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
1026 SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
1027 subcc %o3, 64, %o3
1028 add %o1, 64, %o1
1029 bne 41b
1030 add %o0, 64, %o0
1031
1032 andcc %o2, 0x30, %o3
1033 be,a 1f
1034 srl %g1, 16, %g2
10354:
1036 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3)
1037 subcc %o3, 16, %o3
1038 add %o1, 16, %o1
1039 bne 4b
1040 add %o0, 16, %o0
1041
1042 srl %g1, 16, %g2
10431: 4911:
1044 sth %g2, [%o0 - 3] 492 ret
1045 srl %g1, 8, %g4 493 restore %g7, %g0, %o0
1046 b 88f
1047 stb %g4, [%o0 - 1]
104843:
1049 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
1050 SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
1051 SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
1052 SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
1053 subcc %o3, 64, %o3
1054 add %o1, 64, %o1
1055 bne 43b
1056 add %o0, 64, %o0
1057
1058 andcc %o2, 0x30, %o3
1059 be,a 1f
1060 srl %g1, 24, %g2
10614:
1062 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3)
1063 subcc %o3, 16, %o3
1064 add %o1, 16, %o1
1065 bne 4b
1066 add %o0, 16, %o0
1067
1068 srl %g1, 24, %g2
10691:
1070 stb %g2, [%o0 + 3]
1071 b 88f
1072 add %o0, 4, %o0
107342:
1074 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
1075 SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
1076 SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
1077 SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
1078 subcc %o3, 64, %o3
1079 add %o1, 64, %o1
1080 bne 42b
1081 add %o0, 64, %o0
1082
1083 andcc %o2, 0x30, %o3
1084 be,a 1f
1085 srl %g1, 16, %g2
10864:
1087 SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2)
1088 subcc %o3, 16, %o3
1089 add %o1, 16, %o1
1090 bne 4b
1091 add %o0, 16, %o0
1092
1093 srl %g1, 16, %g2
10941:
1095 sth %g2, [%o0 - 2]
1096
1097 /* Fall through */
1098
1099#endif /* FASTER_NONALIGNED */
1100 494
110188: /* short_end */ 49588: /* short_end */
1102 496
@@ -1127,7 +521,7 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
1127 stb %g2, [%o0] 521 stb %g2, [%o0]
11281: 5221:
1129 retl 523 retl
1130 RETL_INSN 524 mov %g7, %o0
1131 525
113290: /* short_aligned_end */ 52690: /* short_aligned_end */
1133 bne 88b 527 bne 88b
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-01 19:30:42 -0400