aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm/lib/Makefile23
-rw-r--r--arch/arm/lib/io-readsw-armv3.S106
-rw-r--r--arch/arm/lib/io-writesw-armv3.S126
-rw-r--r--arch/arm/lib/uaccess.S564
4 files changed, 816 insertions, 3 deletions
diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
index 2473fd1fd51c..af72969820b4 100644
--- a/arch/arm/lib/Makefile
+++ b/arch/arm/lib/Makefile
@@ -16,13 +16,30 @@ lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \
16 call_with_stack.o 16 call_with_stack.o
17 17
18mmu-y := clear_user.o copy_page.o getuser.o putuser.o 18mmu-y := clear_user.o copy_page.o getuser.o putuser.o
19mmu-y += copy_from_user.o copy_to_user.o 19
20# the code in uaccess.S is not preemption safe and
21# probably faster on ARMv3 only
22ifeq ($(CONFIG_PREEMPT),y)
23 mmu-y += copy_from_user.o copy_to_user.o
24else
25ifneq ($(CONFIG_CPU_32v3),y)
26 mmu-y += copy_from_user.o copy_to_user.o
27else
28 mmu-y += uaccess.o
29endif
30endif
20 31
21# using lib_ here won't override already available weak symbols 32# using lib_ here won't override already available weak symbols
22obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o 33obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o
23 34
24lib-$(CONFIG_MMU) += $(mmu-y) 35lib-$(CONFIG_MMU) += $(mmu-y)
25lib-y += io-readsw-armv4.o io-writesw-armv4.o 36
37ifeq ($(CONFIG_CPU_32v3),y)
38 lib-y += io-readsw-armv3.o io-writesw-armv3.o
39else
40 lib-y += io-readsw-armv4.o io-writesw-armv4.o
41endif
42
26lib-$(CONFIG_ARCH_RPC) += ecard.o io-acorn.o floppydma.o 43lib-$(CONFIG_ARCH_RPC) += ecard.o io-acorn.o floppydma.o
27lib-$(CONFIG_ARCH_SHARK) += io-shark.o 44lib-$(CONFIG_ARCH_SHARK) += io-shark.o
28 45
diff --git a/arch/arm/lib/io-readsw-armv3.S b/arch/arm/lib/io-readsw-armv3.S
new file mode 100644
index 000000000000..88487c8c4f23
--- /dev/null
+++ b/arch/arm/lib/io-readsw-armv3.S
@@ -0,0 +1,106 @@
1/*
2 * linux/arch/arm/lib/io-readsw-armv3.S
3 *
4 * Copyright (C) 1995-2000 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10#include <linux/linkage.h>
11#include <asm/assembler.h>
12
13.Linsw_bad_alignment:
14 adr r0, .Linsw_bad_align_msg
15 mov r2, lr
16 b panic
17.Linsw_bad_align_msg:
18 .asciz "insw: bad buffer alignment (0x%p, lr=0x%08lX)\n"
19 .align
20
21.Linsw_align: tst r1, #1
22 bne .Linsw_bad_alignment
23
24 ldr r3, [r0]
25 strb r3, [r1], #1
26 mov r3, r3, lsr #8
27 strb r3, [r1], #1
28
29 subs r2, r2, #1
30 moveq pc, lr
31
32ENTRY(__raw_readsw)
33 teq r2, #0 @ do we have to check for the zero len?
34 moveq pc, lr
35 tst r1, #3
36 bne .Linsw_align
37
38.Linsw_aligned: mov ip, #0xff
39 orr ip, ip, ip, lsl #8
40 stmfd sp!, {r4, r5, r6, lr}
41
42 subs r2, r2, #8
43 bmi .Lno_insw_8
44
45.Linsw_8_lp: ldr r3, [r0]
46 and r3, r3, ip
47 ldr r4, [r0]
48 orr r3, r3, r4, lsl #16
49
50 ldr r4, [r0]
51 and r4, r4, ip
52 ldr r5, [r0]
53 orr r4, r4, r5, lsl #16
54
55 ldr r5, [r0]
56 and r5, r5, ip
57 ldr r6, [r0]
58 orr r5, r5, r6, lsl #16
59
60 ldr r6, [r0]
61 and r6, r6, ip
62 ldr lr, [r0]
63 orr r6, r6, lr, lsl #16
64
65 stmia r1!, {r3 - r6}
66
67 subs r2, r2, #8
68 bpl .Linsw_8_lp
69
70 tst r2, #7
71 ldmeqfd sp!, {r4, r5, r6, pc}
72
73.Lno_insw_8: tst r2, #4
74 beq .Lno_insw_4
75
76 ldr r3, [r0]
77 and r3, r3, ip
78 ldr r4, [r0]
79 orr r3, r3, r4, lsl #16
80
81 ldr r4, [r0]
82 and r4, r4, ip
83 ldr r5, [r0]
84 orr r4, r4, r5, lsl #16
85
86 stmia r1!, {r3, r4}
87
88.Lno_insw_4: tst r2, #2
89 beq .Lno_insw_2
90
91 ldr r3, [r0]
92 and r3, r3, ip
93 ldr r4, [r0]
94 orr r3, r3, r4, lsl #16
95
96 str r3, [r1], #4
97
98.Lno_insw_2: tst r2, #1
99 ldrne r3, [r0]
100 strneb r3, [r1], #1
101 movne r3, r3, lsr #8
102 strneb r3, [r1]
103
104 ldmfd sp!, {r4, r5, r6, pc}
105
106
diff --git a/arch/arm/lib/io-writesw-armv3.S b/arch/arm/lib/io-writesw-armv3.S
new file mode 100644
index 000000000000..49b800419e32
--- /dev/null
+++ b/arch/arm/lib/io-writesw-armv3.S
@@ -0,0 +1,126 @@
1/*
2 * linux/arch/arm/lib/io-writesw-armv3.S
3 *
4 * Copyright (C) 1995-2000 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10#include <linux/linkage.h>
11#include <asm/assembler.h>
12
13.Loutsw_bad_alignment:
14 adr r0, .Loutsw_bad_align_msg
15 mov r2, lr
16 b panic
17.Loutsw_bad_align_msg:
18 .asciz "outsw: bad buffer alignment (0x%p, lr=0x%08lX)\n"
19 .align
20
21.Loutsw_align: tst r1, #1
22 bne .Loutsw_bad_alignment
23
24 add r1, r1, #2
25
26 ldr r3, [r1, #-4]
27 mov r3, r3, lsr #16
28 orr r3, r3, r3, lsl #16
29 str r3, [r0]
30 subs r2, r2, #1
31 moveq pc, lr
32
33ENTRY(__raw_writesw)
34 teq r2, #0 @ do we have to check for the zero len?
35 moveq pc, lr
36 tst r1, #3
37 bne .Loutsw_align
38
39 stmfd sp!, {r4, r5, r6, lr}
40
41 subs r2, r2, #8
42 bmi .Lno_outsw_8
43
44.Loutsw_8_lp: ldmia r1!, {r3, r4, r5, r6}
45
46 mov ip, r3, lsl #16
47 orr ip, ip, ip, lsr #16
48 str ip, [r0]
49
50 mov ip, r3, lsr #16
51 orr ip, ip, ip, lsl #16
52 str ip, [r0]
53
54 mov ip, r4, lsl #16
55 orr ip, ip, ip, lsr #16
56 str ip, [r0]
57
58 mov ip, r4, lsr #16
59 orr ip, ip, ip, lsl #16
60 str ip, [r0]
61
62 mov ip, r5, lsl #16
63 orr ip, ip, ip, lsr #16
64 str ip, [r0]
65
66 mov ip, r5, lsr #16
67 orr ip, ip, ip, lsl #16
68 str ip, [r0]
69
70 mov ip, r6, lsl #16
71 orr ip, ip, ip, lsr #16
72 str ip, [r0]
73
74 mov ip, r6, lsr #16
75 orr ip, ip, ip, lsl #16
76 str ip, [r0]
77
78 subs r2, r2, #8
79 bpl .Loutsw_8_lp
80
81 tst r2, #7
82 ldmeqfd sp!, {r4, r5, r6, pc}
83
84.Lno_outsw_8: tst r2, #4
85 beq .Lno_outsw_4
86
87 ldmia r1!, {r3, r4}
88
89 mov ip, r3, lsl #16
90 orr ip, ip, ip, lsr #16
91 str ip, [r0]
92
93 mov ip, r3, lsr #16
94 orr ip, ip, ip, lsl #16
95 str ip, [r0]
96
97 mov ip, r4, lsl #16
98 orr ip, ip, ip, lsr #16
99 str ip, [r0]
100
101 mov ip, r4, lsr #16
102 orr ip, ip, ip, lsl #16
103 str ip, [r0]
104
105.Lno_outsw_4: tst r2, #2
106 beq .Lno_outsw_2
107
108 ldr r3, [r1], #4
109
110 mov ip, r3, lsl #16
111 orr ip, ip, ip, lsr #16
112 str ip, [r0]
113
114 mov ip, r3, lsr #16
115 orr ip, ip, ip, lsl #16
116 str ip, [r0]
117
118.Lno_outsw_2: tst r2, #1
119
120 ldrne r3, [r1]
121
122 movne ip, r3, lsl #16
123 orrne ip, ip, ip, lsr #16
124 strne ip, [r0]
125
126 ldmfd sp!, {r4, r5, r6, pc}
diff --git a/arch/arm/lib/uaccess.S b/arch/arm/lib/uaccess.S
new file mode 100644
index 000000000000..5c908b1cb8ed
--- /dev/null
+++ b/arch/arm/lib/uaccess.S
@@ -0,0 +1,564 @@
1/*
2 * linux/arch/arm/lib/uaccess.S
3 *
4 * Copyright (C) 1995, 1996,1997,1998 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Routines to block copy data to/from user memory
11 * These are highly optimised both for the 4k page size
12 * and for various alignments.
13 */
14#include <linux/linkage.h>
15#include <asm/assembler.h>
16#include <asm/errno.h>
17#include <asm/domain.h>
18
19 .text
20
21#define PAGE_SHIFT 12
22
23/* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
24 * Purpose : copy a block to user memory from kernel memory
25 * Params : to - user memory
26 * : from - kernel memory
27 * : n - number of bytes to copy
28 * Returns : Number of bytes NOT copied.
29 */
30
31.Lc2u_dest_not_aligned:
32 rsb ip, ip, #4
33 cmp ip, #2
34 ldrb r3, [r1], #1
35USER( TUSER( strb) r3, [r0], #1) @ May fault
36 ldrgeb r3, [r1], #1
37USER( TUSER( strgeb) r3, [r0], #1) @ May fault
38 ldrgtb r3, [r1], #1
39USER( TUSER( strgtb) r3, [r0], #1) @ May fault
40 sub r2, r2, ip
41 b .Lc2u_dest_aligned
42
43ENTRY(__copy_to_user)
44 stmfd sp!, {r2, r4 - r7, lr}
45 cmp r2, #4
46 blt .Lc2u_not_enough
47 ands ip, r0, #3
48 bne .Lc2u_dest_not_aligned
49.Lc2u_dest_aligned:
50
51 ands ip, r1, #3
52 bne .Lc2u_src_not_aligned
53/*
54 * Seeing as there has to be at least 8 bytes to copy, we can
55 * copy one word, and force a user-mode page fault...
56 */
57
58.Lc2u_0fupi: subs r2, r2, #4
59 addmi ip, r2, #4
60 bmi .Lc2u_0nowords
61 ldr r3, [r1], #4
62USER( TUSER( str) r3, [r0], #4) @ May fault
63 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
64 rsb ip, ip, #0
65 movs ip, ip, lsr #32 - PAGE_SHIFT
66 beq .Lc2u_0fupi
67/*
68 * ip = max no. of bytes to copy before needing another "strt" insn
69 */
70 cmp r2, ip
71 movlt ip, r2
72 sub r2, r2, ip
73 subs ip, ip, #32
74 blt .Lc2u_0rem8lp
75
76.Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
77 stmia r0!, {r3 - r6} @ Shouldnt fault
78 ldmia r1!, {r3 - r6}
79 subs ip, ip, #32
80 stmia r0!, {r3 - r6} @ Shouldnt fault
81 bpl .Lc2u_0cpy8lp
82
83.Lc2u_0rem8lp: cmn ip, #16
84 ldmgeia r1!, {r3 - r6}
85 stmgeia r0!, {r3 - r6} @ Shouldnt fault
86 tst ip, #8
87 ldmneia r1!, {r3 - r4}
88 stmneia r0!, {r3 - r4} @ Shouldnt fault
89 tst ip, #4
90 ldrne r3, [r1], #4
91 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
92 ands ip, ip, #3
93 beq .Lc2u_0fupi
94.Lc2u_0nowords: teq ip, #0
95 beq .Lc2u_finished
96.Lc2u_nowords: cmp ip, #2
97 ldrb r3, [r1], #1
98USER( TUSER( strb) r3, [r0], #1) @ May fault
99 ldrgeb r3, [r1], #1
100USER( TUSER( strgeb) r3, [r0], #1) @ May fault
101 ldrgtb r3, [r1], #1
102USER( TUSER( strgtb) r3, [r0], #1) @ May fault
103 b .Lc2u_finished
104
105.Lc2u_not_enough:
106 movs ip, r2
107 bne .Lc2u_nowords
108.Lc2u_finished: mov r0, #0
109 ldmfd sp!, {r2, r4 - r7, pc}
110
111.Lc2u_src_not_aligned:
112 bic r1, r1, #3
113 ldr r7, [r1], #4
114 cmp ip, #2
115 bgt .Lc2u_3fupi
116 beq .Lc2u_2fupi
117.Lc2u_1fupi: subs r2, r2, #4
118 addmi ip, r2, #4
119 bmi .Lc2u_1nowords
120 mov r3, r7, pull #8
121 ldr r7, [r1], #4
122 orr r3, r3, r7, push #24
123USER( TUSER( str) r3, [r0], #4) @ May fault
124 mov ip, r0, lsl #32 - PAGE_SHIFT
125 rsb ip, ip, #0
126 movs ip, ip, lsr #32 - PAGE_SHIFT
127 beq .Lc2u_1fupi
128 cmp r2, ip
129 movlt ip, r2
130 sub r2, r2, ip
131 subs ip, ip, #16
132 blt .Lc2u_1rem8lp
133
134.Lc2u_1cpy8lp: mov r3, r7, pull #8
135 ldmia r1!, {r4 - r7}
136 subs ip, ip, #16
137 orr r3, r3, r4, push #24
138 mov r4, r4, pull #8
139 orr r4, r4, r5, push #24
140 mov r5, r5, pull #8
141 orr r5, r5, r6, push #24
142 mov r6, r6, pull #8
143 orr r6, r6, r7, push #24
144 stmia r0!, {r3 - r6} @ Shouldnt fault
145 bpl .Lc2u_1cpy8lp
146
147.Lc2u_1rem8lp: tst ip, #8
148 movne r3, r7, pull #8
149 ldmneia r1!, {r4, r7}
150 orrne r3, r3, r4, push #24
151 movne r4, r4, pull #8
152 orrne r4, r4, r7, push #24
153 stmneia r0!, {r3 - r4} @ Shouldnt fault
154 tst ip, #4
155 movne r3, r7, pull #8
156 ldrne r7, [r1], #4
157 orrne r3, r3, r7, push #24
158 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
159 ands ip, ip, #3
160 beq .Lc2u_1fupi
161.Lc2u_1nowords: mov r3, r7, get_byte_1
162 teq ip, #0
163 beq .Lc2u_finished
164 cmp ip, #2
165USER( TUSER( strb) r3, [r0], #1) @ May fault
166 movge r3, r7, get_byte_2
167USER( TUSER( strgeb) r3, [r0], #1) @ May fault
168 movgt r3, r7, get_byte_3
169USER( TUSER( strgtb) r3, [r0], #1) @ May fault
170 b .Lc2u_finished
171
172.Lc2u_2fupi: subs r2, r2, #4
173 addmi ip, r2, #4
174 bmi .Lc2u_2nowords
175 mov r3, r7, pull #16
176 ldr r7, [r1], #4
177 orr r3, r3, r7, push #16
178USER( TUSER( str) r3, [r0], #4) @ May fault
179 mov ip, r0, lsl #32 - PAGE_SHIFT
180 rsb ip, ip, #0
181 movs ip, ip, lsr #32 - PAGE_SHIFT
182 beq .Lc2u_2fupi
183 cmp r2, ip
184 movlt ip, r2
185 sub r2, r2, ip
186 subs ip, ip, #16
187 blt .Lc2u_2rem8lp
188
189.Lc2u_2cpy8lp: mov r3, r7, pull #16
190 ldmia r1!, {r4 - r7}
191 subs ip, ip, #16
192 orr r3, r3, r4, push #16
193 mov r4, r4, pull #16
194 orr r4, r4, r5, push #16
195 mov r5, r5, pull #16
196 orr r5, r5, r6, push #16
197 mov r6, r6, pull #16
198 orr r6, r6, r7, push #16
199 stmia r0!, {r3 - r6} @ Shouldnt fault
200 bpl .Lc2u_2cpy8lp
201
202.Lc2u_2rem8lp: tst ip, #8
203 movne r3, r7, pull #16
204 ldmneia r1!, {r4, r7}
205 orrne r3, r3, r4, push #16
206 movne r4, r4, pull #16
207 orrne r4, r4, r7, push #16
208 stmneia r0!, {r3 - r4} @ Shouldnt fault
209 tst ip, #4
210 movne r3, r7, pull #16
211 ldrne r7, [r1], #4
212 orrne r3, r3, r7, push #16
213 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
214 ands ip, ip, #3
215 beq .Lc2u_2fupi
216.Lc2u_2nowords: mov r3, r7, get_byte_2
217 teq ip, #0
218 beq .Lc2u_finished
219 cmp ip, #2
220USER( TUSER( strb) r3, [r0], #1) @ May fault
221 movge r3, r7, get_byte_3
222USER( TUSER( strgeb) r3, [r0], #1) @ May fault
223 ldrgtb r3, [r1], #0
224USER( TUSER( strgtb) r3, [r0], #1) @ May fault
225 b .Lc2u_finished
226
227.Lc2u_3fupi: subs r2, r2, #4
228 addmi ip, r2, #4
229 bmi .Lc2u_3nowords
230 mov r3, r7, pull #24
231 ldr r7, [r1], #4
232 orr r3, r3, r7, push #8
233USER( TUSER( str) r3, [r0], #4) @ May fault
234 mov ip, r0, lsl #32 - PAGE_SHIFT
235 rsb ip, ip, #0
236 movs ip, ip, lsr #32 - PAGE_SHIFT
237 beq .Lc2u_3fupi
238 cmp r2, ip
239 movlt ip, r2
240 sub r2, r2, ip
241 subs ip, ip, #16
242 blt .Lc2u_3rem8lp
243
244.Lc2u_3cpy8lp: mov r3, r7, pull #24
245 ldmia r1!, {r4 - r7}
246 subs ip, ip, #16
247 orr r3, r3, r4, push #8
248 mov r4, r4, pull #24
249 orr r4, r4, r5, push #8
250 mov r5, r5, pull #24
251 orr r5, r5, r6, push #8
252 mov r6, r6, pull #24
253 orr r6, r6, r7, push #8
254 stmia r0!, {r3 - r6} @ Shouldnt fault
255 bpl .Lc2u_3cpy8lp
256
257.Lc2u_3rem8lp: tst ip, #8
258 movne r3, r7, pull #24
259 ldmneia r1!, {r4, r7}
260 orrne r3, r3, r4, push #8
261 movne r4, r4, pull #24
262 orrne r4, r4, r7, push #8
263 stmneia r0!, {r3 - r4} @ Shouldnt fault
264 tst ip, #4
265 movne r3, r7, pull #24
266 ldrne r7, [r1], #4
267 orrne r3, r3, r7, push #8
268 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
269 ands ip, ip, #3
270 beq .Lc2u_3fupi
271.Lc2u_3nowords: mov r3, r7, get_byte_3
272 teq ip, #0
273 beq .Lc2u_finished
274 cmp ip, #2
275USER( TUSER( strb) r3, [r0], #1) @ May fault
276 ldrgeb r3, [r1], #1
277USER( TUSER( strgeb) r3, [r0], #1) @ May fault
278 ldrgtb r3, [r1], #0
279USER( TUSER( strgtb) r3, [r0], #1) @ May fault
280 b .Lc2u_finished
281ENDPROC(__copy_to_user)
282
283 .pushsection .fixup,"ax"
284 .align 0
2859001: ldmfd sp!, {r0, r4 - r7, pc}
286 .popsection
287
288/* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
289 * Purpose : copy a block from user memory to kernel memory
290 * Params : to - kernel memory
291 * : from - user memory
292 * : n - number of bytes to copy
293 * Returns : Number of bytes NOT copied.
294 */
295.Lcfu_dest_not_aligned:
296 rsb ip, ip, #4
297 cmp ip, #2
298USER( TUSER( ldrb) r3, [r1], #1) @ May fault
299 strb r3, [r0], #1
300USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
301 strgeb r3, [r0], #1
302USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
303 strgtb r3, [r0], #1
304 sub r2, r2, ip
305 b .Lcfu_dest_aligned
306
307ENTRY(__copy_from_user)
308 stmfd sp!, {r0, r2, r4 - r7, lr}
309 cmp r2, #4
310 blt .Lcfu_not_enough
311 ands ip, r0, #3
312 bne .Lcfu_dest_not_aligned
313.Lcfu_dest_aligned:
314 ands ip, r1, #3
315 bne .Lcfu_src_not_aligned
316
317/*
318 * Seeing as there has to be at least 8 bytes to copy, we can
319 * copy one word, and force a user-mode page fault...
320 */
321
322.Lcfu_0fupi: subs r2, r2, #4
323 addmi ip, r2, #4
324 bmi .Lcfu_0nowords
325USER( TUSER( ldr) r3, [r1], #4)
326 str r3, [r0], #4
327 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
328 rsb ip, ip, #0
329 movs ip, ip, lsr #32 - PAGE_SHIFT
330 beq .Lcfu_0fupi
331/*
332 * ip = max no. of bytes to copy before needing another "strt" insn
333 */
334 cmp r2, ip
335 movlt ip, r2
336 sub r2, r2, ip
337 subs ip, ip, #32
338 blt .Lcfu_0rem8lp
339
340.Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
341 stmia r0!, {r3 - r6}
342 ldmia r1!, {r3 - r6} @ Shouldnt fault
343 subs ip, ip, #32
344 stmia r0!, {r3 - r6}
345 bpl .Lcfu_0cpy8lp
346
347.Lcfu_0rem8lp: cmn ip, #16
348 ldmgeia r1!, {r3 - r6} @ Shouldnt fault
349 stmgeia r0!, {r3 - r6}
350 tst ip, #8
351 ldmneia r1!, {r3 - r4} @ Shouldnt fault
352 stmneia r0!, {r3 - r4}
353 tst ip, #4
354 TUSER( ldrne) r3, [r1], #4 @ Shouldnt fault
355 strne r3, [r0], #4
356 ands ip, ip, #3
357 beq .Lcfu_0fupi
358.Lcfu_0nowords: teq ip, #0
359 beq .Lcfu_finished
360.Lcfu_nowords: cmp ip, #2
361USER( TUSER( ldrb) r3, [r1], #1) @ May fault
362 strb r3, [r0], #1
363USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
364 strgeb r3, [r0], #1
365USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
366 strgtb r3, [r0], #1
367 b .Lcfu_finished
368
369.Lcfu_not_enough:
370 movs ip, r2
371 bne .Lcfu_nowords
372.Lcfu_finished: mov r0, #0
373 add sp, sp, #8
374 ldmfd sp!, {r4 - r7, pc}
375
376.Lcfu_src_not_aligned:
377 bic r1, r1, #3
378USER( TUSER( ldr) r7, [r1], #4) @ May fault
379 cmp ip, #2
380 bgt .Lcfu_3fupi
381 beq .Lcfu_2fupi
382.Lcfu_1fupi: subs r2, r2, #4
383 addmi ip, r2, #4
384 bmi .Lcfu_1nowords
385 mov r3, r7, pull #8
386USER( TUSER( ldr) r7, [r1], #4) @ May fault
387 orr r3, r3, r7, push #24
388 str r3, [r0], #4
389 mov ip, r1, lsl #32 - PAGE_SHIFT
390 rsb ip, ip, #0
391 movs ip, ip, lsr #32 - PAGE_SHIFT
392 beq .Lcfu_1fupi
393 cmp r2, ip
394 movlt ip, r2
395 sub r2, r2, ip
396 subs ip, ip, #16
397 blt .Lcfu_1rem8lp
398
399.Lcfu_1cpy8lp: mov r3, r7, pull #8
400 ldmia r1!, {r4 - r7} @ Shouldnt fault
401 subs ip, ip, #16
402 orr r3, r3, r4, push #24
403 mov r4, r4, pull #8
404 orr r4, r4, r5, push #24
405 mov r5, r5, pull #8
406 orr r5, r5, r6, push #24
407 mov r6, r6, pull #8
408 orr r6, r6, r7, push #24
409 stmia r0!, {r3 - r6}
410 bpl .Lcfu_1cpy8lp
411
412.Lcfu_1rem8lp: tst ip, #8
413 movne r3, r7, pull #8
414 ldmneia r1!, {r4, r7} @ Shouldnt fault
415 orrne r3, r3, r4, push #24
416 movne r4, r4, pull #8
417 orrne r4, r4, r7, push #24
418 stmneia r0!, {r3 - r4}
419 tst ip, #4
420 movne r3, r7, pull #8
421USER( TUSER( ldrne) r7, [r1], #4) @ May fault
422 orrne r3, r3, r7, push #24
423 strne r3, [r0], #4
424 ands ip, ip, #3
425 beq .Lcfu_1fupi
426.Lcfu_1nowords: mov r3, r7, get_byte_1
427 teq ip, #0
428 beq .Lcfu_finished
429 cmp ip, #2
430 strb r3, [r0], #1
431 movge r3, r7, get_byte_2
432 strgeb r3, [r0], #1
433 movgt r3, r7, get_byte_3
434 strgtb r3, [r0], #1
435 b .Lcfu_finished
436
437.Lcfu_2fupi: subs r2, r2, #4
438 addmi ip, r2, #4
439 bmi .Lcfu_2nowords
440 mov r3, r7, pull #16
441USER( TUSER( ldr) r7, [r1], #4) @ May fault
442 orr r3, r3, r7, push #16
443 str r3, [r0], #4
444 mov ip, r1, lsl #32 - PAGE_SHIFT
445 rsb ip, ip, #0
446 movs ip, ip, lsr #32 - PAGE_SHIFT
447 beq .Lcfu_2fupi
448 cmp r2, ip
449 movlt ip, r2
450 sub r2, r2, ip
451 subs ip, ip, #16
452 blt .Lcfu_2rem8lp
453
454
455.Lcfu_2cpy8lp: mov r3, r7, pull #16
456 ldmia r1!, {r4 - r7} @ Shouldnt fault
457 subs ip, ip, #16
458 orr r3, r3, r4, push #16
459 mov r4, r4, pull #16
460 orr r4, r4, r5, push #16
461 mov r5, r5, pull #16
462 orr r5, r5, r6, push #16
463 mov r6, r6, pull #16
464 orr r6, r6, r7, push #16
465 stmia r0!, {r3 - r6}
466 bpl .Lcfu_2cpy8lp
467
468.Lcfu_2rem8lp: tst ip, #8
469 movne r3, r7, pull #16
470 ldmneia r1!, {r4, r7} @ Shouldnt fault
471 orrne r3, r3, r4, push #16
472 movne r4, r4, pull #16
473 orrne r4, r4, r7, push #16
474 stmneia r0!, {r3 - r4}
475 tst ip, #4
476 movne r3, r7, pull #16
477USER( TUSER( ldrne) r7, [r1], #4) @ May fault
478 orrne r3, r3, r7, push #16
479 strne r3, [r0], #4
480 ands ip, ip, #3
481 beq .Lcfu_2fupi
482.Lcfu_2nowords: mov r3, r7, get_byte_2
483 teq ip, #0
484 beq .Lcfu_finished
485 cmp ip, #2
486 strb r3, [r0], #1
487 movge r3, r7, get_byte_3
488 strgeb r3, [r0], #1
489USER( TUSER( ldrgtb) r3, [r1], #0) @ May fault
490 strgtb r3, [r0], #1
491 b .Lcfu_finished
492
493.Lcfu_3fupi: subs r2, r2, #4
494 addmi ip, r2, #4
495 bmi .Lcfu_3nowords
496 mov r3, r7, pull #24
497USER( TUSER( ldr) r7, [r1], #4) @ May fault
498 orr r3, r3, r7, push #8
499 str r3, [r0], #4
500 mov ip, r1, lsl #32 - PAGE_SHIFT
501 rsb ip, ip, #0
502 movs ip, ip, lsr #32 - PAGE_SHIFT
503 beq .Lcfu_3fupi
504 cmp r2, ip
505 movlt ip, r2
506 sub r2, r2, ip
507 subs ip, ip, #16
508 blt .Lcfu_3rem8lp
509
510.Lcfu_3cpy8lp: mov r3, r7, pull #24
511 ldmia r1!, {r4 - r7} @ Shouldnt fault
512 orr r3, r3, r4, push #8
513 mov r4, r4, pull #24
514 orr r4, r4, r5, push #8
515 mov r5, r5, pull #24
516 orr r5, r5, r6, push #8
517 mov r6, r6, pull #24
518 orr r6, r6, r7, push #8
519 stmia r0!, {r3 - r6}
520 subs ip, ip, #16
521 bpl .Lcfu_3cpy8lp
522
523.Lcfu_3rem8lp: tst ip, #8
524 movne r3, r7, pull #24
525 ldmneia r1!, {r4, r7} @ Shouldnt fault
526 orrne r3, r3, r4, push #8
527 movne r4, r4, pull #24
528 orrne r4, r4, r7, push #8
529 stmneia r0!, {r3 - r4}
530 tst ip, #4
531 movne r3, r7, pull #24
532USER( TUSER( ldrne) r7, [r1], #4) @ May fault
533 orrne r3, r3, r7, push #8
534 strne r3, [r0], #4
535 ands ip, ip, #3
536 beq .Lcfu_3fupi
537.Lcfu_3nowords: mov r3, r7, get_byte_3
538 teq ip, #0
539 beq .Lcfu_finished
540 cmp ip, #2
541 strb r3, [r0], #1
542USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
543 strgeb r3, [r0], #1
544USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
545 strgtb r3, [r0], #1
546 b .Lcfu_finished
547ENDPROC(__copy_from_user)
548
549 .pushsection .fixup,"ax"
550 .align 0
551 /*
552 * We took an exception. r0 contains a pointer to
553 * the byte not copied.
554 */
5559001: ldr r2, [sp], #4 @ void *to
556 sub r2, r0, r2 @ bytes copied
557 ldr r1, [sp], #4 @ unsigned long count
558 subs r4, r1, r2 @ bytes left to copy
559 movne r1, r4
560 blne __memzero
561 mov r0, r4
562 ldmfd sp!, {r4 - r7, pc}
563 .popsection
564
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-07 01:42:35 -0500