14 files changed, 39 insertions, 122 deletions
diff --git a/arch/sparc/lib/GENbzero.S b/arch/sparc/lib/GENbzero.S
index 6a4f956a2f7..8e7a843ddd8 100644
--- a/arch/sparc/lib/GENbzero.S
+++ b/arch/sparc/lib/GENbzero.S
@@ -6,13 +6,9 @@
 #define EX_ST(x,y)              \
 98:     x,y;                    \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    %o1, %o0;       \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_o1;   \
        .text;                  \
        .align 4;
diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
index 2b9df99e87f..b7d0bd6b140 100644
--- a/arch/sparc/lib/GENcopy_from_user.S
+++ b/arch/sparc/lib/GENcopy_from_user.S
@@ -5,13 +5,9 @@
 #define EX_LD(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    1, %o0;         \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one;  \
        .text;                  \
        .align 4;
@@ -27,7 +23,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
index bb3f7084daf..780550e1afc 100644
--- a/arch/sparc/lib/GENcopy_to_user.S
+++ b/arch/sparc/lib/GENcopy_to_user.S
@@ -5,13 +5,9 @@
 #define EX_ST(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    1, %o0;         \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one;  \
        .text;                  \
        .align 4;
@@ -31,7 +27,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
index c77ef5f2210..119ccb9a54f 100644
--- a/arch/sparc/lib/NG2copy_from_user.S
+++ b/arch/sparc/lib/NG2copy_from_user.S
@@ -5,14 +5,9 @@
 #define EX_LD(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     wr      %g0, ASI_AIUS, %asi;\
-        retl;                   \
-         mov    1, %o0;         \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one_asi;\
        .text;                  \
        .align 4;
@@ -33,7 +28,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
index 4bd4093acbb..7fe1ccefd9d 100644
--- a/arch/sparc/lib/NG2copy_to_user.S
+++ b/arch/sparc/lib/NG2copy_to_user.S
@@ -5,14 +5,9 @@
 #define EX_ST(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     wr      %g0, ASI_AIUS, %asi;\
-        retl;                   \
-         mov    1, %o0;         \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one_asi;\
        .text;                  \
        .align 4;
@@ -42,7 +37,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/NGbzero.S b/arch/sparc/lib/NGbzero.S
index 814d5f7a45e..beab29bf419 100644
--- a/arch/sparc/lib/NGbzero.S
+++ b/arch/sparc/lib/NGbzero.S
@@ -6,13 +6,9 @@
 #define EX_ST(x,y)              \
 98:     x,y;                    \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    %o1, %o0;       \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_o1;   \
        .text;                  \
        .align 4;
diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
index e7f433f71b4..5d1e4d1ac21 100644
--- a/arch/sparc/lib/NGcopy_from_user.S
+++ b/arch/sparc/lib/NGcopy_from_user.S
@@ -5,14 +5,9 @@
 #define EX_LD(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     wr      %g0, ASI_AIUS, %asi;\
-        ret;                    \
-         restore %g0, 1, %o0;   \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __ret_one_asi;\
        .text;                  \
        .align 4;
@@ -30,7 +25,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
index 6ea01c5532a..ff630dcb273 100644
--- a/arch/sparc/lib/NGcopy_to_user.S
+++ b/arch/sparc/lib/NGcopy_to_user.S
@@ -5,14 +5,9 @@
 #define EX_ST(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     wr      %g0, ASI_AIUS, %asi;\
-        ret;                    \
-         restore %g0, 1, %o0;   \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __ret_one_asi;\
        .text;                  \
        .align 4;
@@ -33,7 +28,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop
 #endif
diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
index 3192b0bf4fa..a6ae2ea04bf 100644
--- a/arch/sparc/lib/U1copy_from_user.S
+++ b/arch/sparc/lib/U1copy_from_user.S
@@ -5,13 +5,9 @@
 #define EX_LD(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    1, %o0;         \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one;  \
        .text;                  \
        .align 4;
@@ -27,7 +23,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop;                                           \
 #include "U1memcpy.S"
diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
index d1210ffb0b8..f4b970eeb48 100644
--- a/arch/sparc/lib/U1copy_to_user.S
+++ b/arch/sparc/lib/U1copy_to_user.S
@@ -5,13 +5,9 @@
 #define EX_ST(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    1, %o0;         \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one;  \
        .text;                  \
        .align 4;
@@ -27,7 +23,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop;                                           \
 #include "U1memcpy.S"
diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S
index f5bfc8d9d21..b1acd1331c3 100644
--- a/arch/sparc/lib/U3copy_from_user.S
+++ b/arch/sparc/lib/U3copy_from_user.S
@@ -5,13 +5,9 @@
 #define EX_LD(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    1, %o0;         \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one;  \
        .text;                  \
        .align 4;
diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
index 2334f111bb0..ef1e493afdf 100644
--- a/arch/sparc/lib/U3copy_to_user.S
+++ b/arch/sparc/lib/U3copy_to_user.S
@@ -5,13 +5,9 @@
 #define EX_ST(x)                \
 98:     x;                      \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    1, %o0;         \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one;  \
        .text;                  \
        .align 4;
@@ -27,7 +23,7 @@
 #define PREAMBLE                                        \
        rd              %asi, %g1;                      \
        cmp             %g1, ASI_AIUS;                  \
-        bne,pn          %icc, memcpy_user_stub;         \
+        bne,pn          %icc, ___copy_in_user;          \
         nop;                                           \
 #include "U3memcpy.S"
diff --git a/arch/sparc/lib/bzero.S b/arch/sparc/lib/bzero.S
index c7bbae8c590..b6557297440 100644
--- a/arch/sparc/lib/bzero.S
+++ b/arch/sparc/lib/bzero.S
@@ -88,13 +88,9 @@ __bzero_done:
 #define EX_ST(x,y)              \
 98:     x,y;                    \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov    %o1, %o0;       \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_o1;   \
        .text;                  \
        .align 4;
diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
index 650af3f21f7..302c0e60dc2 100644
--- a/arch/sparc/lib/copy_in_user.S
+++ b/arch/sparc/lib/copy_in_user.S
@@ -3,19 +3,16 @@
 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
 */
+#include <linux/linkage.h>
 #include <asm/asi.h>
 #define XCC xcc
 #define EX(x,y)                 \
 98:     x,y;                    \
-        .section .fixup;        \
-        .align 4;               \
-99:     retl;                   \
-         mov 1, %o0;            \
        .section __ex_table,"a";\
        .align 4;               \
-        .word 98b, 99b;         \
+        .word 98b, __retl_one;  \
        .text;                  \
        .align 4;
@@ -31,18 +28,7 @@
         * to copy register windows around during thread cloning.
         */
-        .globl          ___copy_in_user
+ENTRY(___copy_in_user)  /* %o0=dst, %o1=src, %o2=len */
-        .type           ___copy_in_user,#function
-___copy_in_user:        /* %o0=dst, %o1=src, %o2=len */
-        /* Writing to %asi is _expensive_ so we hardcode it.
-         * Reading %asi to check for KERNEL_DS is comparatively
-         * cheap.
-         */
-        rd              %asi, %g1
-        cmp             %g1, ASI_AIUS
-        bne,pn          %icc, memcpy_user_stub
-         nop
        cmp             %o2, 0
        be,pn           %XCC, 85f
         or             %o0, %o1, %o3
@@ -53,22 +39,24 @@ ___copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
        /* 16 < len <= 64 */
        andcc           %o3, 0x7, %g0
        bne,pn          %XCC, 90f
-         sub            %o0, %o1, %o3
+         nop
        andn            %o2, 0x7, %o4
        and             %o2, 0x7, %o2
 1:      subcc           %o4, 0x8, %o4
        EX(ldxa [%o1] %asi, %o5)
-        EX(stxa %o5, [%o1 + %o3] ASI_AIUS)
+        EX(stxa %o5, [%o0] %asi)
+        add             %o1, 0x8, %o1
        bgu,pt          %XCC, 1b
-         add            %o1, 0x8, %o1
+         add            %o0, 0x8, %o0
        andcc           %o2, 0x4, %g0
        be,pt           %XCC, 1f
         nop
        sub             %o2, 0x4, %o2
        EX(lduwa [%o1] %asi, %o5)
-        EX(stwa %o5, [%o1 + %o3] ASI_AIUS)
+        EX(stwa %o5, [%o0] %asi)
        add             %o1, 0x4, %o1
+        add             %o0, 0x4, %o0
 1:      cmp             %o2, 0
        be,pt           %XCC, 85f
         nop
@@ -78,14 +66,15 @@ ___copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
 80:     /* 0 < len <= 16 */
        andcc           %o3, 0x3, %g0
        bne,pn          %XCC, 90f
-         sub            %o0, %o1, %o3
+         nop
 82:
        subcc           %o2, 4, %o2
        EX(lduwa [%o1] %asi, %g1)
-        EX(stwa %g1, [%o1 + %o3] ASI_AIUS)
+        EX(stwa %g1, [%o0] %asi)
+        add             %o1, 4, %o1
        bgu,pt          %XCC, 82b
-         add            %o1, 4, %o1
+         add            %o0, 4, %o0
 85:     retl
         clr            %o0
@@ -94,26 +83,10 @@ ___copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
 90:
        subcc           %o2, 1, %o2
        EX(lduba [%o1] %asi, %g1)
-        EX(stba %g1, [%o1 + %o3] ASI_AIUS)
+        EX(stba %g1, [%o0] %asi)
+        add             %o1, 1, %o1
        bgu,pt          %XCC, 90b
-         add            %o1, 1, %o1
+         add            %o0, 1, %o0
        retl
         clr            %o0
+ENDPROC(___copy_in_user)
-        .size           ___copy_in_user, .-___copy_in_user
-        /* Act like copy_{to,in}_user(), ie. return zero instead
-         * of original destination pointer.  This is invoked when
-         * copy_{to,in}_user() finds that %asi is kernel space.
-         */
-        .globl          memcpy_user_stub
-        .type           memcpy_user_stub,#function
-memcpy_user_stub:
-        save            %sp, -192, %sp
-        mov             %i0, %o0
-        mov             %i1, %o1
-        call            memcpy
-         mov            %i2, %o2
-        ret
-         restore        %g0, %g0, %o0
-        .size           memcpy_user_stub, .-memcpy_user_stub

diff --git a/arch/sparc/lib/GENbzero.S b/arch/sparc/lib/GENbzero.S index 6a4f956a2f7..8e7a843ddd8 100644 --- a/arch/sparc/lib/GENbzero.S +++ b/arch/sparc/lib/GENbzero.S
@@ -6,13 +6,9 @@
6		6
7	#define EX_ST(x,y) \	7	#define EX_ST(x,y) \
8	98: x,y; \	8	98: x,y; \
9	.section .fixup; \
10	.align 4; \
11	99: retl; \
12	mov %o1, %o0; \
13	.section __ex_table,"a";\	9	.section __ex_table,"a";\
14	.align 4; \	10	.align 4; \
15	.word 98b, 99b; \	11	.word 98b, __retl_o1; \
16	.text; \	12	.text; \
17	.align 4;	13	.align 4;
18		14


diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S index 2b9df99e87f..b7d0bd6b140 100644 --- a/arch/sparc/lib/GENcopy_from_user.S +++ b/arch/sparc/lib/GENcopy_from_user.S
@@ -5,13 +5,9 @@
5		5
6	#define EX_LD(x) \	6	#define EX_LD(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: retl; \
11	mov 1, %o0; \
12	.section __ex_table,"a";\	8	.section __ex_table,"a";\
13	.align 4; \	9	.align 4; \
14	.word 98b, 99b; \	10	.word 98b, __retl_one; \
15	.text; \	11	.text; \
16	.align 4;	12	.align 4;
17		13
@@ -27,7 +23,7 @@
27	#define PREAMBLE \	23	#define PREAMBLE \
28	rd %asi, %g1; \	24	rd %asi, %g1; \
29	cmp %g1, ASI_AIUS; \	25	cmp %g1, ASI_AIUS; \
30	bne,pn %icc, memcpy_user_stub; \	26	bne,pn %icc, ___copy_in_user; \
31	nop	27	nop
32	#endif	28	#endif
33		29


diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S index bb3f7084daf..780550e1afc 100644 --- a/arch/sparc/lib/GENcopy_to_user.S +++ b/arch/sparc/lib/GENcopy_to_user.S
@@ -5,13 +5,9 @@
5		5
6	#define EX_ST(x) \	6	#define EX_ST(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: retl; \
11	mov 1, %o0; \
12	.section __ex_table,"a";\	8	.section __ex_table,"a";\
13	.align 4; \	9	.align 4; \
14	.word 98b, 99b; \	10	.word 98b, __retl_one; \
15	.text; \	11	.text; \
16	.align 4;	12	.align 4;
17		13
@@ -31,7 +27,7 @@
31	#define PREAMBLE \	27	#define PREAMBLE \
32	rd %asi, %g1; \	28	rd %asi, %g1; \
33	cmp %g1, ASI_AIUS; \	29	cmp %g1, ASI_AIUS; \
34	bne,pn %icc, memcpy_user_stub; \	30	bne,pn %icc, ___copy_in_user; \
35	nop	31	nop
36	#endif	32	#endif
37		33


diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S index c77ef5f2210..119ccb9a54f 100644 --- a/arch/sparc/lib/NG2copy_from_user.S +++ b/arch/sparc/lib/NG2copy_from_user.S
@@ -5,14 +5,9 @@
5		5
6	#define EX_LD(x) \	6	#define EX_LD(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: wr %g0, ASI_AIUS, %asi;\
11	retl; \
12	mov 1, %o0; \
13	.section __ex_table,"a";\	8	.section __ex_table,"a";\
14	.align 4; \	9	.align 4; \
15	.word 98b, 99b; \	10	.word 98b, __retl_one_asi;\
16	.text; \	11	.text; \
17	.align 4;	12	.align 4;
18		13
@@ -33,7 +28,7 @@
33	#define PREAMBLE \	28	#define PREAMBLE \
34	rd %asi, %g1; \	29	rd %asi, %g1; \
35	cmp %g1, ASI_AIUS; \	30	cmp %g1, ASI_AIUS; \
36	bne,pn %icc, memcpy_user_stub; \	31	bne,pn %icc, ___copy_in_user; \
37	nop	32	nop
38	#endif	33	#endif
39		34


diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S index 4bd4093acbb..7fe1ccefd9d 100644 --- a/arch/sparc/lib/NG2copy_to_user.S +++ b/arch/sparc/lib/NG2copy_to_user.S
@@ -5,14 +5,9 @@
5		5
6	#define EX_ST(x) \	6	#define EX_ST(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: wr %g0, ASI_AIUS, %asi;\
11	retl; \
12	mov 1, %o0; \
13	.section __ex_table,"a";\	8	.section __ex_table,"a";\
14	.align 4; \	9	.align 4; \
15	.word 98b, 99b; \	10	.word 98b, __retl_one_asi;\
16	.text; \	11	.text; \
17	.align 4;	12	.align 4;
18		13
@@ -42,7 +37,7 @@
42	#define PREAMBLE \	37	#define PREAMBLE \
43	rd %asi, %g1; \	38	rd %asi, %g1; \
44	cmp %g1, ASI_AIUS; \	39	cmp %g1, ASI_AIUS; \
45	bne,pn %icc, memcpy_user_stub; \	40	bne,pn %icc, ___copy_in_user; \
46	nop	41	nop
47	#endif	42	#endif
48		43


diff --git a/arch/sparc/lib/NGbzero.S b/arch/sparc/lib/NGbzero.S index 814d5f7a45e..beab29bf419 100644 --- a/arch/sparc/lib/NGbzero.S +++ b/arch/sparc/lib/NGbzero.S
@@ -6,13 +6,9 @@
6		6
7	#define EX_ST(x,y) \	7	#define EX_ST(x,y) \
8	98: x,y; \	8	98: x,y; \
9	.section .fixup; \
10	.align 4; \
11	99: retl; \
12	mov %o1, %o0; \
13	.section __ex_table,"a";\	9	.section __ex_table,"a";\
14	.align 4; \	10	.align 4; \
15	.word 98b, 99b; \	11	.word 98b, __retl_o1; \
16	.text; \	12	.text; \
17	.align 4;	13	.align 4;
18		14


diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S index e7f433f71b4..5d1e4d1ac21 100644 --- a/arch/sparc/lib/NGcopy_from_user.S +++ b/arch/sparc/lib/NGcopy_from_user.S
@@ -5,14 +5,9 @@
5		5
6	#define EX_LD(x) \	6	#define EX_LD(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: wr %g0, ASI_AIUS, %asi;\
11	ret; \
12	restore %g0, 1, %o0; \
13	.section __ex_table,"a";\	8	.section __ex_table,"a";\
14	.align 4; \	9	.align 4; \
15	.word 98b, 99b; \	10	.word 98b, __ret_one_asi;\
16	.text; \	11	.text; \
17	.align 4;	12	.align 4;
18		13
@@ -30,7 +25,7 @@
30	#define PREAMBLE \	25	#define PREAMBLE \
31	rd %asi, %g1; \	26	rd %asi, %g1; \
32	cmp %g1, ASI_AIUS; \	27	cmp %g1, ASI_AIUS; \
33	bne,pn %icc, memcpy_user_stub; \	28	bne,pn %icc, ___copy_in_user; \
34	nop	29	nop
35	#endif	30	#endif
36		31


diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S index 6ea01c5532a..ff630dcb273 100644 --- a/arch/sparc/lib/NGcopy_to_user.S +++ b/arch/sparc/lib/NGcopy_to_user.S
@@ -5,14 +5,9 @@
5		5
6	#define EX_ST(x) \	6	#define EX_ST(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: wr %g0, ASI_AIUS, %asi;\
11	ret; \
12	restore %g0, 1, %o0; \
13	.section __ex_table,"a";\	8	.section __ex_table,"a";\
14	.align 4; \	9	.align 4; \
15	.word 98b, 99b; \	10	.word 98b, __ret_one_asi;\
16	.text; \	11	.text; \
17	.align 4;	12	.align 4;
18		13
@@ -33,7 +28,7 @@
33	#define PREAMBLE \	28	#define PREAMBLE \
34	rd %asi, %g1; \	29	rd %asi, %g1; \
35	cmp %g1, ASI_AIUS; \	30	cmp %g1, ASI_AIUS; \
36	bne,pn %icc, memcpy_user_stub; \	31	bne,pn %icc, ___copy_in_user; \
37	nop	32	nop
38	#endif	33	#endif
39		34


diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S index 3192b0bf4fa..a6ae2ea04bf 100644 --- a/arch/sparc/lib/U1copy_from_user.S +++ b/arch/sparc/lib/U1copy_from_user.S
@@ -5,13 +5,9 @@
5		5
6	#define EX_LD(x) \	6	#define EX_LD(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: retl; \
11	mov 1, %o0; \
12	.section __ex_table,"a";\	8	.section __ex_table,"a";\
13	.align 4; \	9	.align 4; \
14	.word 98b, 99b; \	10	.word 98b, __retl_one; \
15	.text; \	11	.text; \
16	.align 4;	12	.align 4;
17		13
@@ -27,7 +23,7 @@
27	#define PREAMBLE \	23	#define PREAMBLE \
28	rd %asi, %g1; \	24	rd %asi, %g1; \
29	cmp %g1, ASI_AIUS; \	25	cmp %g1, ASI_AIUS; \
30	bne,pn %icc, memcpy_user_stub; \	26	bne,pn %icc, ___copy_in_user; \
31	nop; \	27	nop; \
32		28
33	#include "U1memcpy.S"	29	#include "U1memcpy.S"


diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S index d1210ffb0b8..f4b970eeb48 100644 --- a/arch/sparc/lib/U1copy_to_user.S +++ b/arch/sparc/lib/U1copy_to_user.S
@@ -5,13 +5,9 @@
5		5
6	#define EX_ST(x) \	6	#define EX_ST(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: retl; \
11	mov 1, %o0; \
12	.section __ex_table,"a";\	8	.section __ex_table,"a";\
13	.align 4; \	9	.align 4; \
14	.word 98b, 99b; \	10	.word 98b, __retl_one; \
15	.text; \	11	.text; \
16	.align 4;	12	.align 4;
17		13
@@ -27,7 +23,7 @@
27	#define PREAMBLE \	23	#define PREAMBLE \
28	rd %asi, %g1; \	24	rd %asi, %g1; \
29	cmp %g1, ASI_AIUS; \	25	cmp %g1, ASI_AIUS; \
30	bne,pn %icc, memcpy_user_stub; \	26	bne,pn %icc, ___copy_in_user; \
31	nop; \	27	nop; \
32		28
33	#include "U1memcpy.S"	29	#include "U1memcpy.S"


diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S index f5bfc8d9d21..b1acd1331c3 100644 --- a/arch/sparc/lib/U3copy_from_user.S +++ b/arch/sparc/lib/U3copy_from_user.S
@@ -5,13 +5,9 @@
5		5
6	#define EX_LD(x) \	6	#define EX_LD(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: retl; \
11	mov 1, %o0; \
12	.section __ex_table,"a";\	8	.section __ex_table,"a";\
13	.align 4; \	9	.align 4; \
14	.word 98b, 99b; \	10	.word 98b, __retl_one; \
15	.text; \	11	.text; \
16	.align 4;	12	.align 4;
17		13


diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S index 2334f111bb0..ef1e493afdf 100644 --- a/arch/sparc/lib/U3copy_to_user.S +++ b/arch/sparc/lib/U3copy_to_user.S
@@ -5,13 +5,9 @@
5		5
6	#define EX_ST(x) \	6	#define EX_ST(x) \
7	98: x; \	7	98: x; \
8	.section .fixup; \
9	.align 4; \
10	99: retl; \
11	mov 1, %o0; \
12	.section __ex_table,"a";\	8	.section __ex_table,"a";\
13	.align 4; \	9	.align 4; \
14	.word 98b, 99b; \	10	.word 98b, __retl_one; \
15	.text; \	11	.text; \
16	.align 4;	12	.align 4;
17		13
@@ -27,7 +23,7 @@
27	#define PREAMBLE \	23	#define PREAMBLE \
28	rd %asi, %g1; \	24	rd %asi, %g1; \
29	cmp %g1, ASI_AIUS; \	25	cmp %g1, ASI_AIUS; \
30	bne,pn %icc, memcpy_user_stub; \	26	bne,pn %icc, ___copy_in_user; \
31	nop; \	27	nop; \
32		28
33	#include "U3memcpy.S"	29	#include "U3memcpy.S"


diff --git a/arch/sparc/lib/bzero.S b/arch/sparc/lib/bzero.S index c7bbae8c590..b6557297440 100644 --- a/arch/sparc/lib/bzero.S +++ b/arch/sparc/lib/bzero.S
@@ -88,13 +88,9 @@ __bzero_done:
88		88
89	#define EX_ST(x,y) \	89	#define EX_ST(x,y) \
90	98: x,y; \	90	98: x,y; \
91	.section .fixup; \
92	.align 4; \
93	99: retl; \
94	mov %o1, %o0; \
95	.section __ex_table,"a";\	91	.section __ex_table,"a";\
96	.align 4; \	92	.align 4; \
97	.word 98b, 99b; \	93	.word 98b, __retl_o1; \
98	.text; \	94	.text; \
99	.align 4;	95	.align 4;
100		96


diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S index 650af3f21f7..302c0e60dc2 100644 --- a/arch/sparc/lib/copy_in_user.S +++ b/arch/sparc/lib/copy_in_user.S
@@ -3,19 +3,16 @@
3	* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)	3	* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
4	*/	4	*/
5		5
		6	#include <linux/linkage.h>
6	#include <asm/asi.h>	7	#include <asm/asi.h>
7		8
8	#define XCC xcc	9	#define XCC xcc
9		10
10	#define EX(x,y) \	11	#define EX(x,y) \
11	98: x,y; \	12	98: x,y; \
12	.section .fixup; \
13	.align 4; \
14	99: retl; \
15	mov 1, %o0; \
16	.section __ex_table,"a";\	13	.section __ex_table,"a";\
17	.align 4; \	14	.align 4; \
18	.word 98b, 99b; \	15	.word 98b, __retl_one; \
19	.text; \	16	.text; \
20	.align 4;	17	.align 4;
21		18
@@ -31,18 +28,7 @@
31	* to copy register windows around during thread cloning.	28	* to copy register windows around during thread cloning.
32	*/	29	*/
33		30
34	.globl ___copy_in_user	31	ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
35	.type ___copy_in_user,#function
36	___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
37	/* Writing to %asi is _expensive_ so we hardcode it.
38	* Reading %asi to check for KERNEL_DS is comparatively
39	* cheap.
40	*/
41	rd %asi, %g1
42	cmp %g1, ASI_AIUS
43	bne,pn %icc, memcpy_user_stub
44	nop
45
46	cmp %o2, 0	32	cmp %o2, 0
47	be,pn %XCC, 85f	33	be,pn %XCC, 85f
48	or %o0, %o1, %o3	34	or %o0, %o1, %o3
@@ -53,22 +39,24 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
53	/* 16 < len <= 64 */	39	/* 16 < len <= 64 */
54	andcc %o3, 0x7, %g0	40	andcc %o3, 0x7, %g0
55	bne,pn %XCC, 90f	41	bne,pn %XCC, 90f
56	sub %o0, %o1, %o3	42	nop
57		43
58	andn %o2, 0x7, %o4	44	andn %o2, 0x7, %o4
59	and %o2, 0x7, %o2	45	and %o2, 0x7, %o2
60	1: subcc %o4, 0x8, %o4	46	1: subcc %o4, 0x8, %o4
61	EX(ldxa [%o1] %asi, %o5)	47	EX(ldxa [%o1] %asi, %o5)
62	EX(stxa %o5, [%o1 + %o3] ASI_AIUS)	48	EX(stxa %o5, [%o0] %asi)
		49	add %o1, 0x8, %o1
63	bgu,pt %XCC, 1b	50	bgu,pt %XCC, 1b
64	add %o1, 0x8, %o1	51	add %o0, 0x8, %o0
65	andcc %o2, 0x4, %g0	52	andcc %o2, 0x4, %g0
66	be,pt %XCC, 1f	53	be,pt %XCC, 1f
67	nop	54	nop
68	sub %o2, 0x4, %o2	55	sub %o2, 0x4, %o2
69	EX(lduwa [%o1] %asi, %o5)	56	EX(lduwa [%o1] %asi, %o5)
70	EX(stwa %o5, [%o1 + %o3] ASI_AIUS)	57	EX(stwa %o5, [%o0] %asi)
71	add %o1, 0x4, %o1	58	add %o1, 0x4, %o1
		59	add %o0, 0x4, %o0
72	1: cmp %o2, 0	60	1: cmp %o2, 0
73	be,pt %XCC, 85f	61	be,pt %XCC, 85f
74	nop	62	nop
@@ -78,14 +66,15 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
78	80: /* 0 < len <= 16 */	66	80: /* 0 < len <= 16 */
79	andcc %o3, 0x3, %g0	67	andcc %o3, 0x3, %g0
80	bne,pn %XCC, 90f	68	bne,pn %XCC, 90f
81	sub %o0, %o1, %o3	69	nop
82		70
83	82:	71	82:
84	subcc %o2, 4, %o2	72	subcc %o2, 4, %o2
85	EX(lduwa [%o1] %asi, %g1)	73	EX(lduwa [%o1] %asi, %g1)
86	EX(stwa %g1, [%o1 + %o3] ASI_AIUS)	74	EX(stwa %g1, [%o0] %asi)
		75	add %o1, 4, %o1
87	bgu,pt %XCC, 82b	76	bgu,pt %XCC, 82b
88	add %o1, 4, %o1	77	add %o0, 4, %o0
89		78
90	85: retl	79	85: retl
91	clr %o0	80	clr %o0
@@ -94,26 +83,10 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
94	90:	83	90:
95	subcc %o2, 1, %o2	84	subcc %o2, 1, %o2
96	EX(lduba [%o1] %asi, %g1)	85	EX(lduba [%o1] %asi, %g1)
97	EX(stba %g1, [%o1 + %o3] ASI_AIUS)	86	EX(stba %g1, [%o0] %asi)
		87	add %o1, 1, %o1
98	bgu,pt %XCC, 90b	88	bgu,pt %XCC, 90b
99	add %o1, 1, %o1	89	add %o0, 1, %o0
100	retl	90	retl
101	clr %o0	91	clr %o0
102		92	ENDPROC(___copy_in_user)
103	.size ___copy_in_user, .-___copy_in_user
104
105	/* Act like copy_{to,in}_user(), ie. return zero instead
106	* of original destination pointer. This is invoked when
107	* copy_{to,in}_user() finds that %asi is kernel space.
108	*/
109	.globl memcpy_user_stub
110	.type memcpy_user_stub,#function
111	memcpy_user_stub:
112	save %sp, -192, %sp
113	mov %i0, %o0
114	mov %i1, %o1
115	call memcpy
116	mov %i2, %o2
117	ret
118	restore %g0, %g0, %o0
119	.size memcpy_user_stub, .-memcpy_user_stub