aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm/lib
diff options
context:
space:
mode:
authorGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
committerGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
commitc71c03bda1e86c9d5198c5d83f712e695c4f2a1e (patch)
treeecb166cb3e2b7e2adb3b5e292245fefd23381ac8 /arch/arm/lib
parentea53c912f8a86a8567697115b6a0d8152beee5c8 (diff)
parent6a00f206debf8a5c8899055726ad127dbeeed098 (diff)
Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp
Conflicts: litmus/sched_cedf.c
Diffstat (limited to 'arch/arm/lib')
-rw-r--r--arch/arm/lib/bitops.h50
-rw-r--r--arch/arm/lib/changebit.S10
-rw-r--r--arch/arm/lib/clearbit.S11
-rw-r--r--arch/arm/lib/delay.S4
-rw-r--r--arch/arm/lib/findbit.S6
-rw-r--r--arch/arm/lib/getuser.S13
-rw-r--r--arch/arm/lib/lib1funcs.S25
-rw-r--r--arch/arm/lib/putuser.S29
-rw-r--r--arch/arm/lib/setbit.S11
-rw-r--r--arch/arm/lib/testchangebit.S9
-rw-r--r--arch/arm/lib/testclearbit.S9
-rw-r--r--arch/arm/lib/testsetbit.S9
-rw-r--r--arch/arm/lib/uaccess.S83
-rw-r--r--arch/arm/lib/uaccess_with_memcpy.c7
14 files changed, 143 insertions, 133 deletions
diff --git a/arch/arm/lib/bitops.h b/arch/arm/lib/bitops.h
index d42252918bfb..10d868a5a481 100644
--- a/arch/arm/lib/bitops.h
+++ b/arch/arm/lib/bitops.h
@@ -1,44 +1,52 @@
1 1#if __LINUX_ARM_ARCH__ >= 6
2#if __LINUX_ARM_ARCH__ >= 6 && defined(CONFIG_CPU_32v6K)
3 .macro bitop, instr 2 .macro bitop, instr
3 ands ip, r1, #3
4 strneb r1, [ip] @ assert word-aligned
4 mov r2, #1 5 mov r2, #1
5 and r3, r0, #7 @ Get bit offset 6 and r3, r0, #31 @ Get bit offset
6 add r1, r1, r0, lsr #3 @ Get byte offset 7 mov r0, r0, lsr #5
8 add r1, r1, r0, lsl #2 @ Get word offset
7 mov r3, r2, lsl r3 9 mov r3, r2, lsl r3
81: ldrexb r2, [r1] 101: ldrex r2, [r1]
9 \instr r2, r2, r3 11 \instr r2, r2, r3
10 strexb r0, r2, [r1] 12 strex r0, r2, [r1]
11 cmp r0, #0 13 cmp r0, #0
12 bne 1b 14 bne 1b
13 mov pc, lr 15 bx lr
14 .endm 16 .endm
15 17
16 .macro testop, instr, store 18 .macro testop, instr, store
17 and r3, r0, #7 @ Get bit offset 19 ands ip, r1, #3
20 strneb r1, [ip] @ assert word-aligned
18 mov r2, #1 21 mov r2, #1
19 add r1, r1, r0, lsr #3 @ Get byte offset 22 and r3, r0, #31 @ Get bit offset
23 mov r0, r0, lsr #5
24 add r1, r1, r0, lsl #2 @ Get word offset
20 mov r3, r2, lsl r3 @ create mask 25 mov r3, r2, lsl r3 @ create mask
21 smp_dmb 26 smp_dmb
221: ldrexb r2, [r1] 271: ldrex r2, [r1]
23 ands r0, r2, r3 @ save old value of bit 28 ands r0, r2, r3 @ save old value of bit
24 \instr r2, r2, r3 @ toggle bit 29 \instr r2, r2, r3 @ toggle bit
25 strexb ip, r2, [r1] 30 strex ip, r2, [r1]
26 cmp ip, #0 31 cmp ip, #0
27 bne 1b 32 bne 1b
28 smp_dmb 33 smp_dmb
29 cmp r0, #0 34 cmp r0, #0
30 movne r0, #1 35 movne r0, #1
312: mov pc, lr 362: bx lr
32 .endm 37 .endm
33#else 38#else
34 .macro bitop, instr 39 .macro bitop, instr
35 and r2, r0, #7 40 ands ip, r1, #3
41 strneb r1, [ip] @ assert word-aligned
42 and r2, r0, #31
43 mov r0, r0, lsr #5
36 mov r3, #1 44 mov r3, #1
37 mov r3, r3, lsl r2 45 mov r3, r3, lsl r2
38 save_and_disable_irqs ip 46 save_and_disable_irqs ip
39 ldrb r2, [r1, r0, lsr #3] 47 ldr r2, [r1, r0, lsl #2]
40 \instr r2, r2, r3 48 \instr r2, r2, r3
41 strb r2, [r1, r0, lsr #3] 49 str r2, [r1, r0, lsl #2]
42 restore_irqs ip 50 restore_irqs ip
43 mov pc, lr 51 mov pc, lr
44 .endm 52 .endm
@@ -52,11 +60,13 @@
52 * to avoid dirtying the data cache. 60 * to avoid dirtying the data cache.
53 */ 61 */
54 .macro testop, instr, store 62 .macro testop, instr, store
55 add r1, r1, r0, lsr #3 63 ands ip, r1, #3
56 and r3, r0, #7 64 strneb r1, [ip] @ assert word-aligned
57 mov r0, #1 65 and r3, r0, #31
66 mov r0, r0, lsr #5
58 save_and_disable_irqs ip 67 save_and_disable_irqs ip
59 ldrb r2, [r1] 68 ldr r2, [r1, r0, lsl #2]!
69 mov r0, #1
60 tst r2, r0, lsl r3 70 tst r2, r0, lsl r3
61 \instr r2, r2, r0, lsl r3 71 \instr r2, r2, r0, lsl r3
62 \store r2, [r1] 72 \store r2, [r1]
diff --git a/arch/arm/lib/changebit.S b/arch/arm/lib/changebit.S
index 80f3115cbee2..68ed5b62e839 100644
--- a/arch/arm/lib/changebit.S
+++ b/arch/arm/lib/changebit.S
@@ -12,12 +12,6 @@
12#include "bitops.h" 12#include "bitops.h"
13 .text 13 .text
14 14
15/* Purpose : Function to change a bit 15ENTRY(_change_bit)
16 * Prototype: int change_bit(int bit, void *addr)
17 */
18ENTRY(_change_bit_be)
19 eor r0, r0, #0x18 @ big endian byte ordering
20ENTRY(_change_bit_le)
21 bitop eor 16 bitop eor
22ENDPROC(_change_bit_be) 17ENDPROC(_change_bit)
23ENDPROC(_change_bit_le)
diff --git a/arch/arm/lib/clearbit.S b/arch/arm/lib/clearbit.S
index 1a63e43a1df0..4c04c3b51eeb 100644
--- a/arch/arm/lib/clearbit.S
+++ b/arch/arm/lib/clearbit.S
@@ -12,13 +12,6 @@
12#include "bitops.h" 12#include "bitops.h"
13 .text 13 .text
14 14
15/* 15ENTRY(_clear_bit)
16 * Purpose : Function to clear a bit
17 * Prototype: int clear_bit(int bit, void *addr)
18 */
19ENTRY(_clear_bit_be)
20 eor r0, r0, #0x18 @ big endian byte ordering
21ENTRY(_clear_bit_le)
22 bitop bic 16 bitop bic
23ENDPROC(_clear_bit_be) 17ENDPROC(_clear_bit)
24ENDPROC(_clear_bit_le)
diff --git a/arch/arm/lib/delay.S b/arch/arm/lib/delay.S
index 8d6a8762ab88..3c9a05c8d20b 100644
--- a/arch/arm/lib/delay.S
+++ b/arch/arm/lib/delay.S
@@ -25,11 +25,15 @@ ENTRY(__udelay)
25 ldr r2, .LC1 25 ldr r2, .LC1
26 mul r0, r2, r0 26 mul r0, r2, r0
27ENTRY(__const_udelay) @ 0 <= r0 <= 0x7fffff06 27ENTRY(__const_udelay) @ 0 <= r0 <= 0x7fffff06
28 mov r1, #-1
28 ldr r2, .LC0 29 ldr r2, .LC0
29 ldr r2, [r2] @ max = 0x01ffffff 30 ldr r2, [r2] @ max = 0x01ffffff
31 add r0, r0, r1, lsr #32-14
30 mov r0, r0, lsr #14 @ max = 0x0001ffff 32 mov r0, r0, lsr #14 @ max = 0x0001ffff
33 add r2, r2, r1, lsr #32-10
31 mov r2, r2, lsr #10 @ max = 0x00007fff 34 mov r2, r2, lsr #10 @ max = 0x00007fff
32 mul r0, r2, r0 @ max = 2^32-1 35 mul r0, r2, r0 @ max = 2^32-1
36 add r0, r0, r1, lsr #32-6
33 movs r0, r0, lsr #6 37 movs r0, r0, lsr #6
34 moveq pc, lr 38 moveq pc, lr
35 39
diff --git a/arch/arm/lib/findbit.S b/arch/arm/lib/findbit.S
index 1e4cbd4e7be9..64f6bc1a9132 100644
--- a/arch/arm/lib/findbit.S
+++ b/arch/arm/lib/findbit.S
@@ -174,8 +174,8 @@ ENDPROC(_find_next_bit_be)
174 */ 174 */
175.L_found: 175.L_found:
176#if __LINUX_ARM_ARCH__ >= 5 176#if __LINUX_ARM_ARCH__ >= 5
177 rsb r1, r3, #0 177 rsb r0, r3, #0
178 and r3, r3, r1 178 and r3, r3, r0
179 clz r3, r3 179 clz r3, r3
180 rsb r3, r3, #31 180 rsb r3, r3, #31
181 add r0, r2, r3 181 add r0, r2, r3
@@ -190,5 +190,7 @@ ENDPROC(_find_next_bit_be)
190 addeq r2, r2, #1 190 addeq r2, r2, #1
191 mov r0, r2 191 mov r0, r2
192#endif 192#endif
193 cmp r1, r0 @ Clamp to maxbit
194 movlo r0, r1
193 mov pc, lr 195 mov pc, lr
194 196
diff --git a/arch/arm/lib/getuser.S b/arch/arm/lib/getuser.S
index b1631a7dbe75..1b049cd7a49a 100644
--- a/arch/arm/lib/getuser.S
+++ b/arch/arm/lib/getuser.S
@@ -28,20 +28,21 @@
28 */ 28 */
29#include <linux/linkage.h> 29#include <linux/linkage.h>
30#include <asm/errno.h> 30#include <asm/errno.h>
31#include <asm/domain.h>
31 32
32ENTRY(__get_user_1) 33ENTRY(__get_user_1)
331: ldrbt r2, [r0] 341: T(ldrb) r2, [r0]
34 mov r0, #0 35 mov r0, #0
35 mov pc, lr 36 mov pc, lr
36ENDPROC(__get_user_1) 37ENDPROC(__get_user_1)
37 38
38ENTRY(__get_user_2) 39ENTRY(__get_user_2)
39#ifdef CONFIG_THUMB2_KERNEL 40#ifdef CONFIG_THUMB2_KERNEL
402: ldrbt r2, [r0] 412: T(ldrb) r2, [r0]
413: ldrbt r3, [r0, #1] 423: T(ldrb) r3, [r0, #1]
42#else 43#else
432: ldrbt r2, [r0], #1 442: T(ldrb) r2, [r0], #1
443: ldrbt r3, [r0] 453: T(ldrb) r3, [r0]
45#endif 46#endif
46#ifndef __ARMEB__ 47#ifndef __ARMEB__
47 orr r2, r2, r3, lsl #8 48 orr r2, r2, r3, lsl #8
@@ -53,7 +54,7 @@ ENTRY(__get_user_2)
53ENDPROC(__get_user_2) 54ENDPROC(__get_user_2)
54 55
55ENTRY(__get_user_4) 56ENTRY(__get_user_4)
564: ldrt r2, [r0] 574: T(ldr) r2, [r0]
57 mov r0, #0 58 mov r0, #0
58 mov pc, lr 59 mov pc, lr
59ENDPROC(__get_user_4) 60ENDPROC(__get_user_4)
diff --git a/arch/arm/lib/lib1funcs.S b/arch/arm/lib/lib1funcs.S
index 6dc06487f3c3..c562f649734c 100644
--- a/arch/arm/lib/lib1funcs.S
+++ b/arch/arm/lib/lib1funcs.S
@@ -35,7 +35,7 @@ Boston, MA 02111-1307, USA. */
35 35
36#include <linux/linkage.h> 36#include <linux/linkage.h>
37#include <asm/assembler.h> 37#include <asm/assembler.h>
38 38#include <asm/unwind.h>
39 39
40.macro ARM_DIV_BODY dividend, divisor, result, curbit 40.macro ARM_DIV_BODY dividend, divisor, result, curbit
41 41
@@ -207,6 +207,7 @@ Boston, MA 02111-1307, USA. */
207 207
208ENTRY(__udivsi3) 208ENTRY(__udivsi3)
209ENTRY(__aeabi_uidiv) 209ENTRY(__aeabi_uidiv)
210UNWIND(.fnstart)
210 211
211 subs r2, r1, #1 212 subs r2, r1, #1
212 moveq pc, lr 213 moveq pc, lr
@@ -230,10 +231,12 @@ ENTRY(__aeabi_uidiv)
230 mov r0, r0, lsr r2 231 mov r0, r0, lsr r2
231 mov pc, lr 232 mov pc, lr
232 233
234UNWIND(.fnend)
233ENDPROC(__udivsi3) 235ENDPROC(__udivsi3)
234ENDPROC(__aeabi_uidiv) 236ENDPROC(__aeabi_uidiv)
235 237
236ENTRY(__umodsi3) 238ENTRY(__umodsi3)
239UNWIND(.fnstart)
237 240
238 subs r2, r1, #1 @ compare divisor with 1 241 subs r2, r1, #1 @ compare divisor with 1
239 bcc Ldiv0 242 bcc Ldiv0
@@ -247,10 +250,12 @@ ENTRY(__umodsi3)
247 250
248 mov pc, lr 251 mov pc, lr
249 252
253UNWIND(.fnend)
250ENDPROC(__umodsi3) 254ENDPROC(__umodsi3)
251 255
252ENTRY(__divsi3) 256ENTRY(__divsi3)
253ENTRY(__aeabi_idiv) 257ENTRY(__aeabi_idiv)
258UNWIND(.fnstart)
254 259
255 cmp r1, #0 260 cmp r1, #0
256 eor ip, r0, r1 @ save the sign of the result. 261 eor ip, r0, r1 @ save the sign of the result.
@@ -287,10 +292,12 @@ ENTRY(__aeabi_idiv)
287 rsbmi r0, r0, #0 292 rsbmi r0, r0, #0
288 mov pc, lr 293 mov pc, lr
289 294
295UNWIND(.fnend)
290ENDPROC(__divsi3) 296ENDPROC(__divsi3)
291ENDPROC(__aeabi_idiv) 297ENDPROC(__aeabi_idiv)
292 298
293ENTRY(__modsi3) 299ENTRY(__modsi3)
300UNWIND(.fnstart)
294 301
295 cmp r1, #0 302 cmp r1, #0
296 beq Ldiv0 303 beq Ldiv0
@@ -310,11 +317,14 @@ ENTRY(__modsi3)
310 rsbmi r0, r0, #0 317 rsbmi r0, r0, #0
311 mov pc, lr 318 mov pc, lr
312 319
320UNWIND(.fnend)
313ENDPROC(__modsi3) 321ENDPROC(__modsi3)
314 322
315#ifdef CONFIG_AEABI 323#ifdef CONFIG_AEABI
316 324
317ENTRY(__aeabi_uidivmod) 325ENTRY(__aeabi_uidivmod)
326UNWIND(.fnstart)
327UNWIND(.save {r0, r1, ip, lr} )
318 328
319 stmfd sp!, {r0, r1, ip, lr} 329 stmfd sp!, {r0, r1, ip, lr}
320 bl __aeabi_uidiv 330 bl __aeabi_uidiv
@@ -323,10 +333,12 @@ ENTRY(__aeabi_uidivmod)
323 sub r1, r1, r3 333 sub r1, r1, r3
324 mov pc, lr 334 mov pc, lr
325 335
336UNWIND(.fnend)
326ENDPROC(__aeabi_uidivmod) 337ENDPROC(__aeabi_uidivmod)
327 338
328ENTRY(__aeabi_idivmod) 339ENTRY(__aeabi_idivmod)
329 340UNWIND(.fnstart)
341UNWIND(.save {r0, r1, ip, lr} )
330 stmfd sp!, {r0, r1, ip, lr} 342 stmfd sp!, {r0, r1, ip, lr}
331 bl __aeabi_idiv 343 bl __aeabi_idiv
332 ldmfd sp!, {r1, r2, ip, lr} 344 ldmfd sp!, {r1, r2, ip, lr}
@@ -334,15 +346,18 @@ ENTRY(__aeabi_idivmod)
334 sub r1, r1, r3 346 sub r1, r1, r3
335 mov pc, lr 347 mov pc, lr
336 348
349UNWIND(.fnend)
337ENDPROC(__aeabi_idivmod) 350ENDPROC(__aeabi_idivmod)
338 351
339#endif 352#endif
340 353
341Ldiv0: 354Ldiv0:
342 355UNWIND(.fnstart)
356UNWIND(.pad #4)
357UNWIND(.save {lr})
343 str lr, [sp, #-8]! 358 str lr, [sp, #-8]!
344 bl __div0 359 bl __div0
345 mov r0, #0 @ About as wrong as it could be. 360 mov r0, #0 @ About as wrong as it could be.
346 ldr pc, [sp], #8 361 ldr pc, [sp], #8
347 362UNWIND(.fnend)
348 363ENDPROC(Ldiv0)
diff --git a/arch/arm/lib/putuser.S b/arch/arm/lib/putuser.S
index 5a01a23c6c06..c023fc11e86c 100644
--- a/arch/arm/lib/putuser.S
+++ b/arch/arm/lib/putuser.S
@@ -28,9 +28,10 @@
28 */ 28 */
29#include <linux/linkage.h> 29#include <linux/linkage.h>
30#include <asm/errno.h> 30#include <asm/errno.h>
31#include <asm/domain.h>
31 32
32ENTRY(__put_user_1) 33ENTRY(__put_user_1)
331: strbt r2, [r0] 341: T(strb) r2, [r0]
34 mov r0, #0 35 mov r0, #0
35 mov pc, lr 36 mov pc, lr
36ENDPROC(__put_user_1) 37ENDPROC(__put_user_1)
@@ -39,19 +40,19 @@ ENTRY(__put_user_2)
39 mov ip, r2, lsr #8 40 mov ip, r2, lsr #8
40#ifdef CONFIG_THUMB2_KERNEL 41#ifdef CONFIG_THUMB2_KERNEL
41#ifndef __ARMEB__ 42#ifndef __ARMEB__
422: strbt r2, [r0] 432: T(strb) r2, [r0]
433: strbt ip, [r0, #1] 443: T(strb) ip, [r0, #1]
44#else 45#else
452: strbt ip, [r0] 462: T(strb) ip, [r0]
463: strbt r2, [r0, #1] 473: T(strb) r2, [r0, #1]
47#endif 48#endif
48#else /* !CONFIG_THUMB2_KERNEL */ 49#else /* !CONFIG_THUMB2_KERNEL */
49#ifndef __ARMEB__ 50#ifndef __ARMEB__
502: strbt r2, [r0], #1 512: T(strb) r2, [r0], #1
513: strbt ip, [r0] 523: T(strb) ip, [r0]
52#else 53#else
532: strbt ip, [r0], #1 542: T(strb) ip, [r0], #1
543: strbt r2, [r0] 553: T(strb) r2, [r0]
55#endif 56#endif
56#endif /* CONFIG_THUMB2_KERNEL */ 57#endif /* CONFIG_THUMB2_KERNEL */
57 mov r0, #0 58 mov r0, #0
@@ -59,18 +60,18 @@ ENTRY(__put_user_2)
59ENDPROC(__put_user_2) 60ENDPROC(__put_user_2)
60 61
61ENTRY(__put_user_4) 62ENTRY(__put_user_4)
624: strt r2, [r0] 634: T(str) r2, [r0]
63 mov r0, #0 64 mov r0, #0
64 mov pc, lr 65 mov pc, lr
65ENDPROC(__put_user_4) 66ENDPROC(__put_user_4)
66 67
67ENTRY(__put_user_8) 68ENTRY(__put_user_8)
68#ifdef CONFIG_THUMB2_KERNEL 69#ifdef CONFIG_THUMB2_KERNEL
695: strt r2, [r0] 705: T(str) r2, [r0]
706: strt r3, [r0, #4] 716: T(str) r3, [r0, #4]
71#else 72#else
725: strt r2, [r0], #4 735: T(str) r2, [r0], #4
736: strt r3, [r0] 746: T(str) r3, [r0]
74#endif 75#endif
75 mov r0, #0 76 mov r0, #0
76 mov pc, lr 77 mov pc, lr
diff --git a/arch/arm/lib/setbit.S b/arch/arm/lib/setbit.S
index 1dd7176c4b2b..bbee5c66a23e 100644
--- a/arch/arm/lib/setbit.S
+++ b/arch/arm/lib/setbit.S
@@ -12,13 +12,6 @@
12#include "bitops.h" 12#include "bitops.h"
13 .text 13 .text
14 14
15/* 15ENTRY(_set_bit)
16 * Purpose : Function to set a bit
17 * Prototype: int set_bit(int bit, void *addr)
18 */
19ENTRY(_set_bit_be)
20 eor r0, r0, #0x18 @ big endian byte ordering
21ENTRY(_set_bit_le)
22 bitop orr 16 bitop orr
23ENDPROC(_set_bit_be) 17ENDPROC(_set_bit)
24ENDPROC(_set_bit_le)
diff --git a/arch/arm/lib/testchangebit.S b/arch/arm/lib/testchangebit.S
index 5c98dc567f0f..15a4d431f229 100644
--- a/arch/arm/lib/testchangebit.S
+++ b/arch/arm/lib/testchangebit.S
@@ -12,9 +12,6 @@
12#include "bitops.h" 12#include "bitops.h"
13 .text 13 .text
14 14
15ENTRY(_test_and_change_bit_be) 15ENTRY(_test_and_change_bit)
16 eor r0, r0, #0x18 @ big endian byte ordering 16 testop eor, str
17ENTRY(_test_and_change_bit_le) 17ENDPROC(_test_and_change_bit)
18 testop eor, strb
19ENDPROC(_test_and_change_bit_be)
20ENDPROC(_test_and_change_bit_le)
diff --git a/arch/arm/lib/testclearbit.S b/arch/arm/lib/testclearbit.S
index 543d7094d18e..521b66b5b95d 100644
--- a/arch/arm/lib/testclearbit.S
+++ b/arch/arm/lib/testclearbit.S
@@ -12,9 +12,6 @@
12#include "bitops.h" 12#include "bitops.h"
13 .text 13 .text
14 14
15ENTRY(_test_and_clear_bit_be) 15ENTRY(_test_and_clear_bit)
16 eor r0, r0, #0x18 @ big endian byte ordering 16 testop bicne, strne
17ENTRY(_test_and_clear_bit_le) 17ENDPROC(_test_and_clear_bit)
18 testop bicne, strneb
19ENDPROC(_test_and_clear_bit_be)
20ENDPROC(_test_and_clear_bit_le)
diff --git a/arch/arm/lib/testsetbit.S b/arch/arm/lib/testsetbit.S
index 0b3f390401ce..1c98cc2185bb 100644
--- a/arch/arm/lib/testsetbit.S
+++ b/arch/arm/lib/testsetbit.S
@@ -12,9 +12,6 @@
12#include "bitops.h" 12#include "bitops.h"
13 .text 13 .text
14 14
15ENTRY(_test_and_set_bit_be) 15ENTRY(_test_and_set_bit)
16 eor r0, r0, #0x18 @ big endian byte ordering 16 testop orreq, streq
17ENTRY(_test_and_set_bit_le) 17ENDPROC(_test_and_set_bit)
18 testop orreq, streqb
19ENDPROC(_test_and_set_bit_be)
20ENDPROC(_test_and_set_bit_le)
diff --git a/arch/arm/lib/uaccess.S b/arch/arm/lib/uaccess.S
index fee9f6f88adb..d0ece2aeb70d 100644
--- a/arch/arm/lib/uaccess.S
+++ b/arch/arm/lib/uaccess.S
@@ -14,6 +14,7 @@
14#include <linux/linkage.h> 14#include <linux/linkage.h>
15#include <asm/assembler.h> 15#include <asm/assembler.h>
16#include <asm/errno.h> 16#include <asm/errno.h>
17#include <asm/domain.h>
17 18
18 .text 19 .text
19 20
@@ -31,11 +32,11 @@
31 rsb ip, ip, #4 32 rsb ip, ip, #4
32 cmp ip, #2 33 cmp ip, #2
33 ldrb r3, [r1], #1 34 ldrb r3, [r1], #1
34USER( strbt r3, [r0], #1) @ May fault 35USER( T(strb) r3, [r0], #1) @ May fault
35 ldrgeb r3, [r1], #1 36 ldrgeb r3, [r1], #1
36USER( strgebt r3, [r0], #1) @ May fault 37USER( T(strgeb) r3, [r0], #1) @ May fault
37 ldrgtb r3, [r1], #1 38 ldrgtb r3, [r1], #1
38USER( strgtbt r3, [r0], #1) @ May fault 39USER( T(strgtb) r3, [r0], #1) @ May fault
39 sub r2, r2, ip 40 sub r2, r2, ip
40 b .Lc2u_dest_aligned 41 b .Lc2u_dest_aligned
41 42
@@ -58,7 +59,7 @@ ENTRY(__copy_to_user)
58 addmi ip, r2, #4 59 addmi ip, r2, #4
59 bmi .Lc2u_0nowords 60 bmi .Lc2u_0nowords
60 ldr r3, [r1], #4 61 ldr r3, [r1], #4
61USER( strt r3, [r0], #4) @ May fault 62USER( T(str) r3, [r0], #4) @ May fault
62 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction 63 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
63 rsb ip, ip, #0 64 rsb ip, ip, #0
64 movs ip, ip, lsr #32 - PAGE_SHIFT 65 movs ip, ip, lsr #32 - PAGE_SHIFT
@@ -87,18 +88,18 @@ USER( strt r3, [r0], #4) @ May fault
87 stmneia r0!, {r3 - r4} @ Shouldnt fault 88 stmneia r0!, {r3 - r4} @ Shouldnt fault
88 tst ip, #4 89 tst ip, #4
89 ldrne r3, [r1], #4 90 ldrne r3, [r1], #4
90 strnet r3, [r0], #4 @ Shouldnt fault 91 T(strne) r3, [r0], #4 @ Shouldnt fault
91 ands ip, ip, #3 92 ands ip, ip, #3
92 beq .Lc2u_0fupi 93 beq .Lc2u_0fupi
93.Lc2u_0nowords: teq ip, #0 94.Lc2u_0nowords: teq ip, #0
94 beq .Lc2u_finished 95 beq .Lc2u_finished
95.Lc2u_nowords: cmp ip, #2 96.Lc2u_nowords: cmp ip, #2
96 ldrb r3, [r1], #1 97 ldrb r3, [r1], #1
97USER( strbt r3, [r0], #1) @ May fault 98USER( T(strb) r3, [r0], #1) @ May fault
98 ldrgeb r3, [r1], #1 99 ldrgeb r3, [r1], #1
99USER( strgebt r3, [r0], #1) @ May fault 100USER( T(strgeb) r3, [r0], #1) @ May fault
100 ldrgtb r3, [r1], #1 101 ldrgtb r3, [r1], #1
101USER( strgtbt r3, [r0], #1) @ May fault 102USER( T(strgtb) r3, [r0], #1) @ May fault
102 b .Lc2u_finished 103 b .Lc2u_finished
103 104
104.Lc2u_not_enough: 105.Lc2u_not_enough:
@@ -119,7 +120,7 @@ USER( strgtbt r3, [r0], #1) @ May fault
119 mov r3, r7, pull #8 120 mov r3, r7, pull #8
120 ldr r7, [r1], #4 121 ldr r7, [r1], #4
121 orr r3, r3, r7, push #24 122 orr r3, r3, r7, push #24
122USER( strt r3, [r0], #4) @ May fault 123USER( T(str) r3, [r0], #4) @ May fault
123 mov ip, r0, lsl #32 - PAGE_SHIFT 124 mov ip, r0, lsl #32 - PAGE_SHIFT
124 rsb ip, ip, #0 125 rsb ip, ip, #0
125 movs ip, ip, lsr #32 - PAGE_SHIFT 126 movs ip, ip, lsr #32 - PAGE_SHIFT
@@ -154,18 +155,18 @@ USER( strt r3, [r0], #4) @ May fault
154 movne r3, r7, pull #8 155 movne r3, r7, pull #8
155 ldrne r7, [r1], #4 156 ldrne r7, [r1], #4
156 orrne r3, r3, r7, push #24 157 orrne r3, r3, r7, push #24
157 strnet r3, [r0], #4 @ Shouldnt fault 158 T(strne) r3, [r0], #4 @ Shouldnt fault
158 ands ip, ip, #3 159 ands ip, ip, #3
159 beq .Lc2u_1fupi 160 beq .Lc2u_1fupi
160.Lc2u_1nowords: mov r3, r7, get_byte_1 161.Lc2u_1nowords: mov r3, r7, get_byte_1
161 teq ip, #0 162 teq ip, #0
162 beq .Lc2u_finished 163 beq .Lc2u_finished
163 cmp ip, #2 164 cmp ip, #2
164USER( strbt r3, [r0], #1) @ May fault 165USER( T(strb) r3, [r0], #1) @ May fault
165 movge r3, r7, get_byte_2 166 movge r3, r7, get_byte_2
166USER( strgebt r3, [r0], #1) @ May fault 167USER( T(strgeb) r3, [r0], #1) @ May fault
167 movgt r3, r7, get_byte_3 168 movgt r3, r7, get_byte_3
168USER( strgtbt r3, [r0], #1) @ May fault 169USER( T(strgtb) r3, [r0], #1) @ May fault
169 b .Lc2u_finished 170 b .Lc2u_finished
170 171
171.Lc2u_2fupi: subs r2, r2, #4 172.Lc2u_2fupi: subs r2, r2, #4
@@ -174,7 +175,7 @@ USER( strgtbt r3, [r0], #1) @ May fault
174 mov r3, r7, pull #16 175 mov r3, r7, pull #16
175 ldr r7, [r1], #4 176 ldr r7, [r1], #4
176 orr r3, r3, r7, push #16 177 orr r3, r3, r7, push #16
177USER( strt r3, [r0], #4) @ May fault 178USER( T(str) r3, [r0], #4) @ May fault
178 mov ip, r0, lsl #32 - PAGE_SHIFT 179 mov ip, r0, lsl #32 - PAGE_SHIFT
179 rsb ip, ip, #0 180 rsb ip, ip, #0
180 movs ip, ip, lsr #32 - PAGE_SHIFT 181 movs ip, ip, lsr #32 - PAGE_SHIFT
@@ -209,18 +210,18 @@ USER( strt r3, [r0], #4) @ May fault
209 movne r3, r7, pull #16 210 movne r3, r7, pull #16
210 ldrne r7, [r1], #4 211 ldrne r7, [r1], #4
211 orrne r3, r3, r7, push #16 212 orrne r3, r3, r7, push #16
212 strnet r3, [r0], #4 @ Shouldnt fault 213 T(strne) r3, [r0], #4 @ Shouldnt fault
213 ands ip, ip, #3 214 ands ip, ip, #3
214 beq .Lc2u_2fupi 215 beq .Lc2u_2fupi
215.Lc2u_2nowords: mov r3, r7, get_byte_2 216.Lc2u_2nowords: mov r3, r7, get_byte_2
216 teq ip, #0 217 teq ip, #0
217 beq .Lc2u_finished 218 beq .Lc2u_finished
218 cmp ip, #2 219 cmp ip, #2
219USER( strbt r3, [r0], #1) @ May fault 220USER( T(strb) r3, [r0], #1) @ May fault
220 movge r3, r7, get_byte_3 221 movge r3, r7, get_byte_3
221USER( strgebt r3, [r0], #1) @ May fault 222USER( T(strgeb) r3, [r0], #1) @ May fault
222 ldrgtb r3, [r1], #0 223 ldrgtb r3, [r1], #0
223USER( strgtbt r3, [r0], #1) @ May fault 224USER( T(strgtb) r3, [r0], #1) @ May fault
224 b .Lc2u_finished 225 b .Lc2u_finished
225 226
226.Lc2u_3fupi: subs r2, r2, #4 227.Lc2u_3fupi: subs r2, r2, #4
@@ -229,7 +230,7 @@ USER( strgtbt r3, [r0], #1) @ May fault
229 mov r3, r7, pull #24 230 mov r3, r7, pull #24
230 ldr r7, [r1], #4 231 ldr r7, [r1], #4
231 orr r3, r3, r7, push #8 232 orr r3, r3, r7, push #8
232USER( strt r3, [r0], #4) @ May fault 233USER( T(str) r3, [r0], #4) @ May fault
233 mov ip, r0, lsl #32 - PAGE_SHIFT 234 mov ip, r0, lsl #32 - PAGE_SHIFT
234 rsb ip, ip, #0 235 rsb ip, ip, #0
235 movs ip, ip, lsr #32 - PAGE_SHIFT 236 movs ip, ip, lsr #32 - PAGE_SHIFT
@@ -264,18 +265,18 @@ USER( strt r3, [r0], #4) @ May fault
264 movne r3, r7, pull #24 265 movne r3, r7, pull #24
265 ldrne r7, [r1], #4 266 ldrne r7, [r1], #4
266 orrne r3, r3, r7, push #8 267 orrne r3, r3, r7, push #8
267 strnet r3, [r0], #4 @ Shouldnt fault 268 T(strne) r3, [r0], #4 @ Shouldnt fault
268 ands ip, ip, #3 269 ands ip, ip, #3
269 beq .Lc2u_3fupi 270 beq .Lc2u_3fupi
270.Lc2u_3nowords: mov r3, r7, get_byte_3 271.Lc2u_3nowords: mov r3, r7, get_byte_3
271 teq ip, #0 272 teq ip, #0
272 beq .Lc2u_finished 273 beq .Lc2u_finished
273 cmp ip, #2 274 cmp ip, #2
274USER( strbt r3, [r0], #1) @ May fault 275USER( T(strb) r3, [r0], #1) @ May fault
275 ldrgeb r3, [r1], #1 276 ldrgeb r3, [r1], #1
276USER( strgebt r3, [r0], #1) @ May fault 277USER( T(strgeb) r3, [r0], #1) @ May fault
277 ldrgtb r3, [r1], #0 278 ldrgtb r3, [r1], #0
278USER( strgtbt r3, [r0], #1) @ May fault 279USER( T(strgtb) r3, [r0], #1) @ May fault
279 b .Lc2u_finished 280 b .Lc2u_finished
280ENDPROC(__copy_to_user) 281ENDPROC(__copy_to_user)
281 282
@@ -294,11 +295,11 @@ ENDPROC(__copy_to_user)
294.Lcfu_dest_not_aligned: 295.Lcfu_dest_not_aligned:
295 rsb ip, ip, #4 296 rsb ip, ip, #4
296 cmp ip, #2 297 cmp ip, #2
297USER( ldrbt r3, [r1], #1) @ May fault 298USER( T(ldrb) r3, [r1], #1) @ May fault
298 strb r3, [r0], #1 299 strb r3, [r0], #1
299USER( ldrgebt r3, [r1], #1) @ May fault 300USER( T(ldrgeb) r3, [r1], #1) @ May fault
300 strgeb r3, [r0], #1 301 strgeb r3, [r0], #1
301USER( ldrgtbt r3, [r1], #1) @ May fault 302USER( T(ldrgtb) r3, [r1], #1) @ May fault
302 strgtb r3, [r0], #1 303 strgtb r3, [r0], #1
303 sub r2, r2, ip 304 sub r2, r2, ip
304 b .Lcfu_dest_aligned 305 b .Lcfu_dest_aligned
@@ -321,7 +322,7 @@ ENTRY(__copy_from_user)
321.Lcfu_0fupi: subs r2, r2, #4 322.Lcfu_0fupi: subs r2, r2, #4
322 addmi ip, r2, #4 323 addmi ip, r2, #4
323 bmi .Lcfu_0nowords 324 bmi .Lcfu_0nowords
324USER( ldrt r3, [r1], #4) 325USER( T(ldr) r3, [r1], #4)
325 str r3, [r0], #4 326 str r3, [r0], #4
326 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction 327 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
327 rsb ip, ip, #0 328 rsb ip, ip, #0
@@ -350,18 +351,18 @@ USER( ldrt r3, [r1], #4)
350 ldmneia r1!, {r3 - r4} @ Shouldnt fault 351 ldmneia r1!, {r3 - r4} @ Shouldnt fault
351 stmneia r0!, {r3 - r4} 352 stmneia r0!, {r3 - r4}
352 tst ip, #4 353 tst ip, #4
353 ldrnet r3, [r1], #4 @ Shouldnt fault 354 T(ldrne) r3, [r1], #4 @ Shouldnt fault
354 strne r3, [r0], #4 355 strne r3, [r0], #4
355 ands ip, ip, #3 356 ands ip, ip, #3
356 beq .Lcfu_0fupi 357 beq .Lcfu_0fupi
357.Lcfu_0nowords: teq ip, #0 358.Lcfu_0nowords: teq ip, #0
358 beq .Lcfu_finished 359 beq .Lcfu_finished
359.Lcfu_nowords: cmp ip, #2 360.Lcfu_nowords: cmp ip, #2
360USER( ldrbt r3, [r1], #1) @ May fault 361USER( T(ldrb) r3, [r1], #1) @ May fault
361 strb r3, [r0], #1 362 strb r3, [r0], #1
362USER( ldrgebt r3, [r1], #1) @ May fault 363USER( T(ldrgeb) r3, [r1], #1) @ May fault
363 strgeb r3, [r0], #1 364 strgeb r3, [r0], #1
364USER( ldrgtbt r3, [r1], #1) @ May fault 365USER( T(ldrgtb) r3, [r1], #1) @ May fault
365 strgtb r3, [r0], #1 366 strgtb r3, [r0], #1
366 b .Lcfu_finished 367 b .Lcfu_finished
367 368
@@ -374,7 +375,7 @@ USER( ldrgtbt r3, [r1], #1) @ May fault
374 375
375.Lcfu_src_not_aligned: 376.Lcfu_src_not_aligned:
376 bic r1, r1, #3 377 bic r1, r1, #3
377USER( ldrt r7, [r1], #4) @ May fault 378USER( T(ldr) r7, [r1], #4) @ May fault
378 cmp ip, #2 379 cmp ip, #2
379 bgt .Lcfu_3fupi 380 bgt .Lcfu_3fupi
380 beq .Lcfu_2fupi 381 beq .Lcfu_2fupi
@@ -382,7 +383,7 @@ USER( ldrt r7, [r1], #4) @ May fault
382 addmi ip, r2, #4 383 addmi ip, r2, #4
383 bmi .Lcfu_1nowords 384 bmi .Lcfu_1nowords
384 mov r3, r7, pull #8 385 mov r3, r7, pull #8
385USER( ldrt r7, [r1], #4) @ May fault 386USER( T(ldr) r7, [r1], #4) @ May fault
386 orr r3, r3, r7, push #24 387 orr r3, r3, r7, push #24
387 str r3, [r0], #4 388 str r3, [r0], #4
388 mov ip, r1, lsl #32 - PAGE_SHIFT 389 mov ip, r1, lsl #32 - PAGE_SHIFT
@@ -417,7 +418,7 @@ USER( ldrt r7, [r1], #4) @ May fault
417 stmneia r0!, {r3 - r4} 418 stmneia r0!, {r3 - r4}
418 tst ip, #4 419 tst ip, #4
419 movne r3, r7, pull #8 420 movne r3, r7, pull #8
420USER( ldrnet r7, [r1], #4) @ May fault 421USER( T(ldrne) r7, [r1], #4) @ May fault
421 orrne r3, r3, r7, push #24 422 orrne r3, r3, r7, push #24
422 strne r3, [r0], #4 423 strne r3, [r0], #4
423 ands ip, ip, #3 424 ands ip, ip, #3
@@ -437,7 +438,7 @@ USER( ldrnet r7, [r1], #4) @ May fault
437 addmi ip, r2, #4 438 addmi ip, r2, #4
438 bmi .Lcfu_2nowords 439 bmi .Lcfu_2nowords
439 mov r3, r7, pull #16 440 mov r3, r7, pull #16
440USER( ldrt r7, [r1], #4) @ May fault 441USER( T(ldr) r7, [r1], #4) @ May fault
441 orr r3, r3, r7, push #16 442 orr r3, r3, r7, push #16
442 str r3, [r0], #4 443 str r3, [r0], #4
443 mov ip, r1, lsl #32 - PAGE_SHIFT 444 mov ip, r1, lsl #32 - PAGE_SHIFT
@@ -473,7 +474,7 @@ USER( ldrt r7, [r1], #4) @ May fault
473 stmneia r0!, {r3 - r4} 474 stmneia r0!, {r3 - r4}
474 tst ip, #4 475 tst ip, #4
475 movne r3, r7, pull #16 476 movne r3, r7, pull #16
476USER( ldrnet r7, [r1], #4) @ May fault 477USER( T(ldrne) r7, [r1], #4) @ May fault
477 orrne r3, r3, r7, push #16 478 orrne r3, r3, r7, push #16
478 strne r3, [r0], #4 479 strne r3, [r0], #4
479 ands ip, ip, #3 480 ands ip, ip, #3
@@ -485,7 +486,7 @@ USER( ldrnet r7, [r1], #4) @ May fault
485 strb r3, [r0], #1 486 strb r3, [r0], #1
486 movge r3, r7, get_byte_3 487 movge r3, r7, get_byte_3
487 strgeb r3, [r0], #1 488 strgeb r3, [r0], #1
488USER( ldrgtbt r3, [r1], #0) @ May fault 489USER( T(ldrgtb) r3, [r1], #0) @ May fault
489 strgtb r3, [r0], #1 490 strgtb r3, [r0], #1
490 b .Lcfu_finished 491 b .Lcfu_finished
491 492
@@ -493,7 +494,7 @@ USER( ldrgtbt r3, [r1], #0) @ May fault
493 addmi ip, r2, #4 494 addmi ip, r2, #4
494 bmi .Lcfu_3nowords 495 bmi .Lcfu_3nowords
495 mov r3, r7, pull #24 496 mov r3, r7, pull #24
496USER( ldrt r7, [r1], #4) @ May fault 497USER( T(ldr) r7, [r1], #4) @ May fault
497 orr r3, r3, r7, push #8 498 orr r3, r3, r7, push #8
498 str r3, [r0], #4 499 str r3, [r0], #4
499 mov ip, r1, lsl #32 - PAGE_SHIFT 500 mov ip, r1, lsl #32 - PAGE_SHIFT
@@ -528,7 +529,7 @@ USER( ldrt r7, [r1], #4) @ May fault
528 stmneia r0!, {r3 - r4} 529 stmneia r0!, {r3 - r4}
529 tst ip, #4 530 tst ip, #4
530 movne r3, r7, pull #24 531 movne r3, r7, pull #24
531USER( ldrnet r7, [r1], #4) @ May fault 532USER( T(ldrne) r7, [r1], #4) @ May fault
532 orrne r3, r3, r7, push #8 533 orrne r3, r3, r7, push #8
533 strne r3, [r0], #4 534 strne r3, [r0], #4
534 ands ip, ip, #3 535 ands ip, ip, #3
@@ -538,9 +539,9 @@ USER( ldrnet r7, [r1], #4) @ May fault
538 beq .Lcfu_finished 539 beq .Lcfu_finished
539 cmp ip, #2 540 cmp ip, #2
540 strb r3, [r0], #1 541 strb r3, [r0], #1
541USER( ldrgebt r3, [r1], #1) @ May fault 542USER( T(ldrgeb) r3, [r1], #1) @ May fault
542 strgeb r3, [r0], #1 543 strgeb r3, [r0], #1
543USER( ldrgtbt r3, [r1], #1) @ May fault 544USER( T(ldrgtb) r3, [r1], #1) @ May fault
544 strgtb r3, [r0], #1 545 strgtb r3, [r0], #1
545 b .Lcfu_finished 546 b .Lcfu_finished
546ENDPROC(__copy_from_user) 547ENDPROC(__copy_from_user)
diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c
index e2d2f2cd0c4f..8b9b13649f81 100644
--- a/arch/arm/lib/uaccess_with_memcpy.c
+++ b/arch/arm/lib/uaccess_with_memcpy.c
@@ -27,13 +27,18 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
27 pgd_t *pgd; 27 pgd_t *pgd;
28 pmd_t *pmd; 28 pmd_t *pmd;
29 pte_t *pte; 29 pte_t *pte;
30 pud_t *pud;
30 spinlock_t *ptl; 31 spinlock_t *ptl;
31 32
32 pgd = pgd_offset(current->mm, addr); 33 pgd = pgd_offset(current->mm, addr);
33 if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd))) 34 if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
34 return 0; 35 return 0;
35 36
36 pmd = pmd_offset(pgd, addr); 37 pud = pud_offset(pgd, addr);
38 if (unlikely(pud_none(*pud) || pud_bad(*pud)))
39 return 0;
40
41 pmd = pmd_offset(pud, addr);
37 if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd))) 42 if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
38 return 0; 43 return 0;
39 44
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-03 09:24:40 -0400