10 files changed, 25 insertions, 48 deletions
diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
index 9b0e58f9f970..b7d0bd6b1406 100644
--- a/arch/sparc/lib/GENcopy_from_user.S
+++ b/arch/sparc/lib/GENcopy_from_user.S
@@ -23,7 +23,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
index ce79b495c65d..780550e1afc7 100644
--- a/arch/sparc/lib/GENcopy_to_user.S
+++ b/arch/sparc/lib/GENcopy_to_user.S
@@ -27,7 +27,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
index 01ac546a86f4..119ccb9a54f4 100644
--- a/arch/sparc/lib/NG2copy_from_user.S
+++ b/arch/sparc/lib/NG2copy_from_user.S
@@ -28,7 +28,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
index c477e1594a72..7fe1ccefd9d0 100644
--- a/arch/sparc/lib/NG2copy_to_user.S
+++ b/arch/sparc/lib/NG2copy_to_user.S
@@ -37,7 +37,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
index 39bb8912c6e7..5d1e4d1ac21e 100644
--- a/arch/sparc/lib/NGcopy_from_user.S
+++ b/arch/sparc/lib/NGcopy_from_user.S
@@ -25,7 +25,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
index bf62fc1a26c7..ff630dcb273c 100644
--- a/arch/sparc/lib/NGcopy_to_user.S
+++ b/arch/sparc/lib/NGcopy_to_user.S
@@ -28,7 +28,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
index f14efdd6d4a9..a6ae2ea04bf5 100644
--- a/arch/sparc/lib/U1copy_from_user.S
+++ b/arch/sparc/lib/U1copy_from_user.S
@@ -23,7 +23,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop;                                           \
 #include "U1memcpy.S"
diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
index 59d531b4a1a5..f4b970eeb485 100644
--- a/arch/sparc/lib/U1copy_to_user.S
+++ b/arch/sparc/lib/U1copy_to_user.S
@@ -23,7 +23,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop;                                           \
 #include "U1memcpy.S"
diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
index b4cbfe5a97ba..ef1e493afdfa 100644
--- a/arch/sparc/lib/U3copy_to_user.S
+++ b/arch/sparc/lib/U3copy_to_user.S
@@ -23,7 +23,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop;                                           \
 #include "U3memcpy.S"
diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
index 884f46499d4e..302c0e60dc2c 100644
--- a/arch/sparc/lib/copy_in_user.S
+++ b/arch/sparc/lib/copy_in_user.S
@@ -3,6 +3,7 @@
 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
 */
+#include <linux/linkage.h>
 #include <asm/asi.h>
 #define XCC xcc
@@ -27,18 +28,7 @@
         * to copy register windows around during thread cloning.
         */
-        .globl          ___copy_in_user
+ENTRY(___copy_in_user)  /* %o0=dst, %o1=src, %o2=len */
-        .type           ___copy_in_user,#function
-___copy_in_user:        /* %o0=dst, %o1=src, %o2=len */
-        /* Writing to %asi is _expensive_ so we hardcode it.
-         * Reading %asi to check for KERNEL_DS is comparatively
-         * cheap.
-         */
-        rd              %asi, %g1
-        cmp             %g1, ASI_AIUS
-        bne,pn          %icc, memcpy_user_stub
-         nop
        cmp             %o2, 0
        be,pn           %XCC, 85f
         or             %o0, %o1, %o3
@@ -49,22 +39,24 @@ ___copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
        /* 16 < len <= 64 */
        andcc           %o3, 0x7, %g0
        bne,pn          %XCC, 90f
-         sub            %o0, %o1, %o3
+         nop
        andn            %o2, 0x7, %o4
        and             %o2, 0x7, %o2
 1:      subcc           %o4, 0x8, %o4
        EX(ldxa [%o1] %asi, %o5)
-        EX(stxa %o5, [%o1 + %o3] ASI_AIUS)
+        EX(stxa %o5, [%o0] %asi)
+        add             %o1, 0x8, %o1
        bgu,pt          %XCC, 1b
-         add            %o1, 0x8, %o1
+         add            %o0, 0x8, %o0
        andcc           %o2, 0x4, %g0
        be,pt           %XCC, 1f
         nop
        sub             %o2, 0x4, %o2
        EX(lduwa [%o1] %asi, %o5)
-        EX(stwa %o5, [%o1 + %o3] ASI_AIUS)
+        EX(stwa %o5, [%o0] %asi)
        add             %o1, 0x4, %o1
+        add             %o0, 0x4, %o0
 1:      cmp             %o2, 0
        be,pt           %XCC, 85f
         nop
@@ -74,14 +66,15 @@ ___copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
 80:     /* 0 < len <= 16 */
        andcc           %o3, 0x3, %g0
        bne,pn          %XCC, 90f
-         sub            %o0, %o1, %o3
+         nop
 82:
        subcc           %o2, 4, %o2
        EX(lduwa [%o1] %asi, %g1)
-        EX(stwa %g1, [%o1 + %o3] ASI_AIUS)
+        EX(stwa %g1, [%o0] %asi)
+        add             %o1, 4, %o1
        bgu,pt          %XCC, 82b
-         add            %o1, 4, %o1
+         add            %o0, 4, %o0
 85:     retl
         clr            %o0
@@ -90,26 +83,10 @@ ___copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
 90:
        subcc           %o2, 1, %o2
        EX(lduba [%o1] %asi, %g1)
-        EX(stba %g1, [%o1 + %o3] ASI_AIUS)
+        EX(stba %g1, [%o0] %asi)
+        add             %o1, 1, %o1
        bgu,pt          %XCC, 90b
-         add            %o1, 1, %o1
+         add            %o0, 1, %o0
        retl
         clr            %o0
+ENDPROC(___copy_in_user)
-        .size           ___copy_in_user, .-___copy_in_user
-        /* Act like copy_{to,in}_user(), ie. return zero instead
-         * of original destination pointer.  This is invoked when
-         * copy_{to,in}_user() finds that %asi is kernel space.
-         */
-        .globl          memcpy_user_stub
-        .type           memcpy_user_stub,#function
-memcpy_user_stub:
-        save            %sp, -192, %sp
-        mov             %i0, %o0
-        mov             %i1, %o1
-        call            memcpy
-         mov            %i2, %o2
-        ret
-         restore        %g0, %g0, %o0
-        .size           memcpy_user_stub, .-memcpy_user_stub

diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S index 9b0e58f9f970..b7d0bd6b1406 100644 --- a/arch/sparc/lib/GENcopy_from_user.S +++ b/arch/sparc/lib/GENcopy_from_user.S
@@ -23,7 +23,7 @@
23	#define PREAMBLE \	23	#define PREAMBLE \
24	rd %asi, %g1; \	24	rd %asi, %g1; \
25	cmp %g1, ASI_AIUS; \	25	cmp %g1, ASI_AIUS; \
26	bne,pn %icc, memcpy_user_stub; \	26	bne,pn %icc, ___copy_in_user; \
27	nop	27	nop
28	#endif	28	#endif
29		29


diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S index ce79b495c65d..780550e1afc7 100644 --- a/arch/sparc/lib/GENcopy_to_user.S +++ b/arch/sparc/lib/GENcopy_to_user.S
@@ -27,7 +27,7 @@
27	#define PREAMBLE \	27	#define PREAMBLE \
28	rd %asi, %g1; \	28	rd %asi, %g1; \
29	cmp %g1, ASI_AIUS; \	29	cmp %g1, ASI_AIUS; \
30	bne,pn %icc, memcpy_user_stub; \	30	bne,pn %icc, ___copy_in_user; \
31	nop	31	nop
32	#endif	32	#endif
33		33


diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S index 01ac546a86f4..119ccb9a54f4 100644 --- a/arch/sparc/lib/NG2copy_from_user.S +++ b/arch/sparc/lib/NG2copy_from_user.S
@@ -28,7 +28,7 @@
28	#define PREAMBLE \	28	#define PREAMBLE \
29	rd %asi, %g1; \	29	rd %asi, %g1; \
30	cmp %g1, ASI_AIUS; \	30	cmp %g1, ASI_AIUS; \
31	bne,pn %icc, memcpy_user_stub; \	31	bne,pn %icc, ___copy_in_user; \
32	nop	32	nop
33	#endif	33	#endif
34		34


diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S index c477e1594a72..7fe1ccefd9d0 100644 --- a/arch/sparc/lib/NG2copy_to_user.S +++ b/arch/sparc/lib/NG2copy_to_user.S
@@ -37,7 +37,7 @@
37	#define PREAMBLE \	37	#define PREAMBLE \
38	rd %asi, %g1; \	38	rd %asi, %g1; \
39	cmp %g1, ASI_AIUS; \	39	cmp %g1, ASI_AIUS; \
40	bne,pn %icc, memcpy_user_stub; \	40	bne,pn %icc, ___copy_in_user; \
41	nop	41	nop
42	#endif	42	#endif
43		43


diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S index 39bb8912c6e7..5d1e4d1ac21e 100644 --- a/arch/sparc/lib/NGcopy_from_user.S +++ b/arch/sparc/lib/NGcopy_from_user.S
@@ -25,7 +25,7 @@
25	#define PREAMBLE \	25	#define PREAMBLE \
26	rd %asi, %g1; \	26	rd %asi, %g1; \
27	cmp %g1, ASI_AIUS; \	27	cmp %g1, ASI_AIUS; \
28	bne,pn %icc, memcpy_user_stub; \	28	bne,pn %icc, ___copy_in_user; \
29	nop	29	nop
30	#endif	30	#endif
31		31


diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S index bf62fc1a26c7..ff630dcb273c 100644 --- a/arch/sparc/lib/NGcopy_to_user.S +++ b/arch/sparc/lib/NGcopy_to_user.S
@@ -28,7 +28,7 @@
28	#define PREAMBLE \	28	#define PREAMBLE \
29	rd %asi, %g1; \	29	rd %asi, %g1; \
30	cmp %g1, ASI_AIUS; \	30	cmp %g1, ASI_AIUS; \
31	bne,pn %icc, memcpy_user_stub; \	31	bne,pn %icc, ___copy_in_user; \
32	nop	32	nop
33	#endif	33	#endif
34		34


diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S index f14efdd6d4a9..a6ae2ea04bf5 100644 --- a/arch/sparc/lib/U1copy_from_user.S +++ b/arch/sparc/lib/U1copy_from_user.S
@@ -23,7 +23,7 @@
23	#define PREAMBLE \	23	#define PREAMBLE \
24	rd %asi, %g1; \	24	rd %asi, %g1; \
25	cmp %g1, ASI_AIUS; \	25	cmp %g1, ASI_AIUS; \
26	bne,pn %icc, memcpy_user_stub; \	26	bne,pn %icc, ___copy_in_user; \
27	nop; \	27	nop; \
28		28
29	#include "U1memcpy.S"	29	#include "U1memcpy.S"


diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S index 59d531b4a1a5..f4b970eeb485 100644 --- a/arch/sparc/lib/U1copy_to_user.S +++ b/arch/sparc/lib/U1copy_to_user.S
@@ -23,7 +23,7 @@
23	#define PREAMBLE \	23	#define PREAMBLE \
24	rd %asi, %g1; \	24	rd %asi, %g1; \
25	cmp %g1, ASI_AIUS; \	25	cmp %g1, ASI_AIUS; \
26	bne,pn %icc, memcpy_user_stub; \	26	bne,pn %icc, ___copy_in_user; \
27	nop; \	27	nop; \
28		28
29	#include "U1memcpy.S"	29	#include "U1memcpy.S"


diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S index b4cbfe5a97ba..ef1e493afdfa 100644 --- a/arch/sparc/lib/U3copy_to_user.S +++ b/arch/sparc/lib/U3copy_to_user.S
@@ -23,7 +23,7 @@
23	#define PREAMBLE \	23	#define PREAMBLE \
24	rd %asi, %g1; \	24	rd %asi, %g1; \
25	cmp %g1, ASI_AIUS; \	25	cmp %g1, ASI_AIUS; \
26	bne,pn %icc, memcpy_user_stub; \	26	bne,pn %icc, ___copy_in_user; \
27	nop; \	27	nop; \
28		28
29	#include "U3memcpy.S"	29	#include "U3memcpy.S"


diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S index 884f46499d4e..302c0e60dc2c 100644 --- a/arch/sparc/lib/copy_in_user.S +++ b/arch/sparc/lib/copy_in_user.S
@@ -3,6 +3,7 @@
3	* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)	3	* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
4	*/	4	*/
5		5
		6	#include <linux/linkage.h>
6	#include <asm/asi.h>	7	#include <asm/asi.h>
7		8
8	#define XCC xcc	9	#define XCC xcc
@@ -27,18 +28,7 @@
27	* to copy register windows around during thread cloning.	28	* to copy register windows around during thread cloning.
28	*/	29	*/
29		30
30	.globl ___copy_in_user	31	ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
31	.type ___copy_in_user,#function
32	___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
33	/* Writing to %asi is _expensive_ so we hardcode it.
34	* Reading %asi to check for KERNEL_DS is comparatively
35	* cheap.
36	*/
37	rd %asi, %g1
38	cmp %g1, ASI_AIUS
39	bne,pn %icc, memcpy_user_stub
40	nop
41
42	cmp %o2, 0	32	cmp %o2, 0
43	be,pn %XCC, 85f	33	be,pn %XCC, 85f
44	or %o0, %o1, %o3	34	or %o0, %o1, %o3
@@ -49,22 +39,24 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
49	/* 16 < len <= 64 */	39	/* 16 < len <= 64 */
50	andcc %o3, 0x7, %g0	40	andcc %o3, 0x7, %g0
51	bne,pn %XCC, 90f	41	bne,pn %XCC, 90f
52	sub %o0, %o1, %o3	42	nop
53		43
54	andn %o2, 0x7, %o4	44	andn %o2, 0x7, %o4
55	and %o2, 0x7, %o2	45	and %o2, 0x7, %o2
56	1: subcc %o4, 0x8, %o4	46	1: subcc %o4, 0x8, %o4
57	EX(ldxa [%o1] %asi, %o5)	47	EX(ldxa [%o1] %asi, %o5)
58	EX(stxa %o5, [%o1 + %o3] ASI_AIUS)	48	EX(stxa %o5, [%o0] %asi)
		49	add %o1, 0x8, %o1
59	bgu,pt %XCC, 1b	50	bgu,pt %XCC, 1b
60	add %o1, 0x8, %o1	51	add %o0, 0x8, %o0
61	andcc %o2, 0x4, %g0	52	andcc %o2, 0x4, %g0
62	be,pt %XCC, 1f	53	be,pt %XCC, 1f
63	nop	54	nop
64	sub %o2, 0x4, %o2	55	sub %o2, 0x4, %o2
65	EX(lduwa [%o1] %asi, %o5)	56	EX(lduwa [%o1] %asi, %o5)
66	EX(stwa %o5, [%o1 + %o3] ASI_AIUS)	57	EX(stwa %o5, [%o0] %asi)
67	add %o1, 0x4, %o1	58	add %o1, 0x4, %o1
		59	add %o0, 0x4, %o0
68	1: cmp %o2, 0	60	1: cmp %o2, 0
69	be,pt %XCC, 85f	61	be,pt %XCC, 85f
70	nop	62	nop
@@ -74,14 +66,15 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
74	80: /* 0 < len <= 16 */	66	80: /* 0 < len <= 16 */
75	andcc %o3, 0x3, %g0	67	andcc %o3, 0x3, %g0
76	bne,pn %XCC, 90f	68	bne,pn %XCC, 90f
77	sub %o0, %o1, %o3	69	nop
78		70
79	82:	71	82:
80	subcc %o2, 4, %o2	72	subcc %o2, 4, %o2
81	EX(lduwa [%o1] %asi, %g1)	73	EX(lduwa [%o1] %asi, %g1)
82	EX(stwa %g1, [%o1 + %o3] ASI_AIUS)	74	EX(stwa %g1, [%o0] %asi)
		75	add %o1, 4, %o1
83	bgu,pt %XCC, 82b	76	bgu,pt %XCC, 82b
84	add %o1, 4, %o1	77	add %o0, 4, %o0
85		78
86	85: retl	79	85: retl
87	clr %o0	80	clr %o0
@@ -90,26 +83,10 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
90	90:	83	90:
91	subcc %o2, 1, %o2	84	subcc %o2, 1, %o2
92	EX(lduba [%o1] %asi, %g1)	85	EX(lduba [%o1] %asi, %g1)
93	EX(stba %g1, [%o1 + %o3] ASI_AIUS)	86	EX(stba %g1, [%o0] %asi)
		87	add %o1, 1, %o1
94	bgu,pt %XCC, 90b	88	bgu,pt %XCC, 90b
95	add %o1, 1, %o1	89	add %o0, 1, %o0
96	retl	90	retl
97	clr %o0	91	clr %o0
98		92	ENDPROC(___copy_in_user)
99	.size ___copy_in_user, .-___copy_in_user
100
101	/* Act like copy_{to,in}_user(), ie. return zero instead
102	* of original destination pointer. This is invoked when
103	* copy_{to,in}_user() finds that %asi is kernel space.
104	*/
105	.globl memcpy_user_stub
106	.type memcpy_user_stub,#function
107	memcpy_user_stub:
108	save %sp, -192, %sp
109	mov %i0, %o0
110	mov %i1, %o1
111	call memcpy
112	mov %i2, %o2
113	ret
114	restore %g0, %g0, %o0
115	.size memcpy_user_stub, .-memcpy_user_stub