aboutsummaryrefslogtreecommitdiffstats
path: root/arch/tile
diff options
context:
space:
mode:
authorChris Metcalf <cmetcalf@tilera.com>2012-03-29 13:39:51 -0400
committerChris Metcalf <cmetcalf@tilera.com>2012-05-25 12:48:23 -0400
commit47d632f9f8f3ed62b21f725e98b726d65769b6d7 (patch)
tree9599e3a0106ee320b293be1dbc2d4dbb93b6e1ff /arch/tile
parent1efea40d4172a2a475ccb29b59d6221e9d0c174b (diff)
arch/tile: optimize get_user/put_user and friends
Use direct load/store for the get_user/put_user. Previously, we would call out to a helper routine that would do the appropriate thing and then return, handling the possible exception internally. Now we inline the load or store, along with a "we succeeded" indication in a register; if the load or store faults, we write a "we failed" indication into the same register and then return to the following instruction. This is more efficient and gives us more compact code, as well as being more in line with what other architectures do. The special futex assembly source file for TILE-Gx also disappears in this change; we just use the same inlining idiom there as well, putting the appropriate atomic operations directly into futex_atomic_op_inuser() (and thus into the FUTEX_WAIT function). The underlying atomic copy_from_user, copy_to_user functions were renamed using the (cryptic) x86 convention as copy_from_user_ll and copy_to_user_ll. Signed-off-by: Chris Metcalf <cmetcalf@tilera.com>
Diffstat (limited to 'arch/tile')
-rw-r--r--arch/tile/include/asm/atomic_32.h10
-rw-r--r--arch/tile/include/asm/futex.h143
-rw-r--r--arch/tile/include/asm/uaccess.h222
-rw-r--r--arch/tile/kernel/Makefile1
-rw-r--r--arch/tile/lib/atomic_32.c47
-rw-r--r--arch/tile/lib/exports.c8
-rw-r--r--arch/tile/lib/usercopy_32.S76
-rw-r--r--arch/tile/lib/usercopy_64.S49
8 files changed, 241 insertions, 315 deletions
diff --git a/arch/tile/include/asm/atomic_32.h b/arch/tile/include/asm/atomic_32.h
index 54d1da826f93..e7fb5cfb9597 100644
--- a/arch/tile/include/asm/atomic_32.h
+++ b/arch/tile/include/asm/atomic_32.h
@@ -303,7 +303,14 @@ void __init_atomic_per_cpu(void);
303void __atomic_fault_unlock(int *lock_ptr); 303void __atomic_fault_unlock(int *lock_ptr);
304#endif 304#endif
305 305
306/* Return a pointer to the lock for the given address. */
307int *__atomic_hashed_lock(volatile void *v);
308
306/* Private helper routines in lib/atomic_asm_32.S */ 309/* Private helper routines in lib/atomic_asm_32.S */
310struct __get_user {
311 unsigned long val;
312 int err;
313};
307extern struct __get_user __atomic_cmpxchg(volatile int *p, 314extern struct __get_user __atomic_cmpxchg(volatile int *p,
308 int *lock, int o, int n); 315 int *lock, int o, int n);
309extern struct __get_user __atomic_xchg(volatile int *p, int *lock, int n); 316extern struct __get_user __atomic_xchg(volatile int *p, int *lock, int n);
@@ -319,6 +326,9 @@ extern u64 __atomic64_xchg_add(volatile u64 *p, int *lock, u64 n);
319extern u64 __atomic64_xchg_add_unless(volatile u64 *p, 326extern u64 __atomic64_xchg_add_unless(volatile u64 *p,
320 int *lock, u64 o, u64 n); 327 int *lock, u64 o, u64 n);
321 328
329/* Return failure from the atomic wrappers. */
330struct __get_user __atomic_bad_address(int __user *addr);
331
322#endif /* !__ASSEMBLY__ */ 332#endif /* !__ASSEMBLY__ */
323 333
324#endif /* _ASM_TILE_ATOMIC_32_H */ 334#endif /* _ASM_TILE_ATOMIC_32_H */
diff --git a/arch/tile/include/asm/futex.h b/arch/tile/include/asm/futex.h
index d03ec124a598..5909ac3d7218 100644
--- a/arch/tile/include/asm/futex.h
+++ b/arch/tile/include/asm/futex.h
@@ -28,29 +28,81 @@
28#include <linux/futex.h> 28#include <linux/futex.h>
29#include <linux/uaccess.h> 29#include <linux/uaccess.h>
30#include <linux/errno.h> 30#include <linux/errno.h>
31#include <asm/atomic.h>
31 32
32extern struct __get_user futex_set(u32 __user *v, int i); 33/*
33extern struct __get_user futex_add(u32 __user *v, int n); 34 * Support macros for futex operations. Do not use these macros directly.
34extern struct __get_user futex_or(u32 __user *v, int n); 35 * They assume "ret", "val", "oparg", and "uaddr" in the lexical context.
35extern struct __get_user futex_andn(u32 __user *v, int n); 36 * __futex_cmpxchg() additionally assumes "oldval".
36extern struct __get_user futex_cmpxchg(u32 __user *v, int o, int n); 37 */
38
39#ifdef __tilegx__
40
41#define __futex_asm(OP) \
42 asm("1: {" #OP " %1, %3, %4; movei %0, 0 }\n" \
43 ".pushsection .fixup,\"ax\"\n" \
44 "0: { movei %0, %5; j 9f }\n" \
45 ".section __ex_table,\"a\"\n" \
46 ".quad 1b, 0b\n" \
47 ".popsection\n" \
48 "9:" \
49 : "=r" (ret), "=r" (val), "+m" (*(uaddr)) \
50 : "r" (uaddr), "r" (oparg), "i" (-EFAULT))
51
52#define __futex_set() __futex_asm(exch4)
53#define __futex_add() __futex_asm(fetchadd4)
54#define __futex_or() __futex_asm(fetchor4)
55#define __futex_andn() ({ oparg = ~oparg; __futex_asm(fetchand4); })
56#define __futex_cmpxchg() \
57 ({ __insn_mtspr(SPR_CMPEXCH_VALUE, oldval); __futex_asm(cmpexch4); })
58
59#define __futex_xor() \
60 ({ \
61 u32 oldval, n = oparg; \
62 if ((ret = __get_user(oldval, uaddr)) == 0) { \
63 do { \
64 oparg = oldval ^ n; \
65 __futex_cmpxchg(); \
66 } while (ret == 0 && oldval != val); \
67 } \
68 })
69
70/* No need to prefetch, since the atomic ops go to the home cache anyway. */
71#define __futex_prolog()
37 72
38#ifndef __tilegx__
39extern struct __get_user futex_xor(u32 __user *v, int n);
40#else 73#else
41static inline struct __get_user futex_xor(u32 __user *uaddr, int n) 74
42{ 75#define __futex_call(FN) \
43 struct __get_user asm_ret = __get_user_4(uaddr); 76 { \
44 if (!asm_ret.err) { 77 struct __get_user gu = FN((u32 __force *)uaddr, lock, oparg); \
45 int oldval, newval; 78 val = gu.val; \
46 do { 79 ret = gu.err; \
47 oldval = asm_ret.val;
48 newval = oldval ^ n;
49 asm_ret = futex_cmpxchg(uaddr, oldval, newval);
50 } while (asm_ret.err == 0 && oldval != asm_ret.val);
51 } 80 }
52 return asm_ret; 81
53} 82#define __futex_set() __futex_call(__atomic_xchg)
83#define __futex_add() __futex_call(__atomic_xchg_add)
84#define __futex_or() __futex_call(__atomic_or)
85#define __futex_andn() __futex_call(__atomic_andn)
86#define __futex_xor() __futex_call(__atomic_xor)
87
88#define __futex_cmpxchg() \
89 { \
90 struct __get_user gu = __atomic_cmpxchg((u32 __force *)uaddr, \
91 lock, oldval, oparg); \
92 val = gu.val; \
93 ret = gu.err; \
94 }
95
96/*
97 * Find the lock pointer for the atomic calls to use, and issue a
98 * prefetch to the user address to bring it into cache. Similar to
99 * __atomic_setup(), but we can't do a read into the L1 since it might
100 * fault; instead we do a prefetch into the L2.
101 */
102#define __futex_prolog() \
103 int *lock; \
104 __insn_prefetch(uaddr); \
105 lock = __atomic_hashed_lock((int __force *)uaddr)
54#endif 106#endif
55 107
56static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr) 108static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
@@ -59,8 +111,12 @@ static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
59 int cmp = (encoded_op >> 24) & 15; 111 int cmp = (encoded_op >> 24) & 15;
60 int oparg = (encoded_op << 8) >> 20; 112 int oparg = (encoded_op << 8) >> 20;
61 int cmparg = (encoded_op << 20) >> 20; 113 int cmparg = (encoded_op << 20) >> 20;
62 int ret; 114 int uninitialized_var(val), ret;
63 struct __get_user asm_ret; 115
116 __futex_prolog();
117
118 /* The 32-bit futex code makes this assumption, so validate it here. */
119 BUILD_BUG_ON(sizeof(atomic_t) != sizeof(int));
64 120
65 if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) 121 if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
66 oparg = 1 << oparg; 122 oparg = 1 << oparg;
@@ -71,46 +127,45 @@ static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
71 pagefault_disable(); 127 pagefault_disable();
72 switch (op) { 128 switch (op) {
73 case FUTEX_OP_SET: 129 case FUTEX_OP_SET:
74 asm_ret = futex_set(uaddr, oparg); 130 __futex_set();
75 break; 131 break;
76 case FUTEX_OP_ADD: 132 case FUTEX_OP_ADD:
77 asm_ret = futex_add(uaddr, oparg); 133 __futex_add();
78 break; 134 break;
79 case FUTEX_OP_OR: 135 case FUTEX_OP_OR:
80 asm_ret = futex_or(uaddr, oparg); 136 __futex_or();
81 break; 137 break;
82 case FUTEX_OP_ANDN: 138 case FUTEX_OP_ANDN:
83 asm_ret = futex_andn(uaddr, oparg); 139 __futex_andn();
84 break; 140 break;
85 case FUTEX_OP_XOR: 141 case FUTEX_OP_XOR:
86 asm_ret = futex_xor(uaddr, oparg); 142 __futex_xor();
87 break; 143 break;
88 default: 144 default:
89 asm_ret.err = -ENOSYS; 145 ret = -ENOSYS;
146 break;
90 } 147 }
91 pagefault_enable(); 148 pagefault_enable();
92 149
93 ret = asm_ret.err;
94
95 if (!ret) { 150 if (!ret) {
96 switch (cmp) { 151 switch (cmp) {
97 case FUTEX_OP_CMP_EQ: 152 case FUTEX_OP_CMP_EQ:
98 ret = (asm_ret.val == cmparg); 153 ret = (val == cmparg);
99 break; 154 break;
100 case FUTEX_OP_CMP_NE: 155 case FUTEX_OP_CMP_NE:
101 ret = (asm_ret.val != cmparg); 156 ret = (val != cmparg);
102 break; 157 break;
103 case FUTEX_OP_CMP_LT: 158 case FUTEX_OP_CMP_LT:
104 ret = (asm_ret.val < cmparg); 159 ret = (val < cmparg);
105 break; 160 break;
106 case FUTEX_OP_CMP_GE: 161 case FUTEX_OP_CMP_GE:
107 ret = (asm_ret.val >= cmparg); 162 ret = (val >= cmparg);
108 break; 163 break;
109 case FUTEX_OP_CMP_LE: 164 case FUTEX_OP_CMP_LE:
110 ret = (asm_ret.val <= cmparg); 165 ret = (val <= cmparg);
111 break; 166 break;
112 case FUTEX_OP_CMP_GT: 167 case FUTEX_OP_CMP_GT:
113 ret = (asm_ret.val > cmparg); 168 ret = (val > cmparg);
114 break; 169 break;
115 default: 170 default:
116 ret = -ENOSYS; 171 ret = -ENOSYS;
@@ -120,22 +175,20 @@ static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
120} 175}
121 176
122static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, 177static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
123 u32 oldval, u32 newval) 178 u32 oldval, u32 oparg)
124{ 179{
125 struct __get_user asm_ret; 180 int ret, val;
181
182 __futex_prolog();
126 183
127 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) 184 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
128 return -EFAULT; 185 return -EFAULT;
129 186
130 asm_ret = futex_cmpxchg(uaddr, oldval, newval); 187 __futex_cmpxchg();
131 *uval = asm_ret.val;
132 return asm_ret.err;
133}
134 188
135#ifndef __tilegx__ 189 *uval = val;
136/* Return failure from the atomic wrappers. */ 190 return ret;
137struct __get_user __atomic_bad_address(int __user *addr); 191}
138#endif
139 192
140#endif /* !__ASSEMBLY__ */ 193#endif /* !__ASSEMBLY__ */
141 194
diff --git a/arch/tile/include/asm/uaccess.h b/arch/tile/include/asm/uaccess.h
index ef34d2caa5b1..c3dd275f25e2 100644
--- a/arch/tile/include/asm/uaccess.h
+++ b/arch/tile/include/asm/uaccess.h
@@ -114,45 +114,75 @@ struct exception_table_entry {
114extern int fixup_exception(struct pt_regs *regs); 114extern int fixup_exception(struct pt_regs *regs);
115 115
116/* 116/*
117 * We return the __get_user_N function results in a structure, 117 * Support macros for __get_user().
118 * thus in r0 and r1. If "err" is zero, "val" is the result 118 *
119 * of the read; otherwise, "err" is -EFAULT. 119 * Implementation note: The "case 8" logic of casting to the type of
120 * 120 * the result of subtracting the value from itself is basically a way
121 * We rarely need 8-byte values on a 32-bit architecture, but 121 * of keeping all integer types the same, but casting any pointers to
122 * we size the structure to accommodate. In practice, for the 122 * ptrdiff_t, i.e. also an integer type. This way there are no
123 * the smaller reads, we can zero the high word for free, and 123 * questionable casts seen by the compiler on an ILP32 platform.
124 * the caller will ignore it by virtue of casting anyway. 124 *
125 * Note that __get_user() and __put_user() assume proper alignment.
125 */ 126 */
126struct __get_user {
127 unsigned long long val;
128 int err;
129};
130 127
131/* 128#ifdef __LP64__
132 * FIXME: we should express these as inline extended assembler, since 129#define _ASM_PTR ".quad"
133 * they're fundamentally just a variable dereference and some 130#else
134 * supporting exception_table gunk. Note that (a la i386) we can 131#define _ASM_PTR ".long"
135 * extend the copy_to_user and copy_from_user routines to call into 132#endif
136 * such extended assembler routines, though we will have to use a 133
137 * different return code in that case (1, 2, or 4, rather than -EFAULT). 134#define __get_user_asm(OP, x, ptr, ret) \
138 */ 135 asm volatile("1: {" #OP " %1, %2; movei %0, 0 }\n" \
139extern struct __get_user __get_user_1(const void __user *); 136 ".pushsection .fixup,\"ax\"\n" \
140extern struct __get_user __get_user_2(const void __user *); 137 "0: { movei %1, 0; movei %0, %3 }\n" \
141extern struct __get_user __get_user_4(const void __user *); 138 "j 9f\n" \
142extern struct __get_user __get_user_8(const void __user *); 139 ".section __ex_table,\"a\"\n" \
143extern int __put_user_1(long, void __user *); 140 _ASM_PTR " 1b, 0b\n" \
144extern int __put_user_2(long, void __user *); 141 ".popsection\n" \
145extern int __put_user_4(long, void __user *); 142 "9:" \
146extern int __put_user_8(long long, void __user *); 143 : "=r" (ret), "=r" (x) \
147 144 : "r" (ptr), "i" (-EFAULT))
148/* Unimplemented routines to cause linker failures */ 145
149extern struct __get_user __get_user_bad(void); 146#ifdef __tilegx__
150extern int __put_user_bad(void); 147#define __get_user_1(x, ptr, ret) __get_user_asm(ld1u, x, ptr, ret)
148#define __get_user_2(x, ptr, ret) __get_user_asm(ld2u, x, ptr, ret)
149#define __get_user_4(x, ptr, ret) __get_user_asm(ld4u, x, ptr, ret)
150#define __get_user_8(x, ptr, ret) __get_user_asm(ld, x, ptr, ret)
151#else
152#define __get_user_1(x, ptr, ret) __get_user_asm(lb_u, x, ptr, ret)
153#define __get_user_2(x, ptr, ret) __get_user_asm(lh_u, x, ptr, ret)
154#define __get_user_4(x, ptr, ret) __get_user_asm(lw, x, ptr, ret)
155#ifdef __LITTLE_ENDIAN
156#define __lo32(a, b) a
157#define __hi32(a, b) b
158#else
159#define __lo32(a, b) b
160#define __hi32(a, b) a
161#endif
162#define __get_user_8(x, ptr, ret) \
163 ({ \
164 unsigned int __a, __b; \
165 asm volatile("1: { lw %1, %3; addi %2, %3, 4 }\n" \
166 "2: { lw %2, %2; movei %0, 0 }\n" \
167 ".pushsection .fixup,\"ax\"\n" \
168 "0: { movei %1, 0; movei %2, 0 }\n" \
169 "{ movei %0, %4; j 9f }\n" \
170 ".section __ex_table,\"a\"\n" \
171 ".word 1b, 0b\n" \
172 ".word 2b, 0b\n" \
173 ".popsection\n" \
174 "9:" \
175 : "=r" (ret), "=r" (__a), "=&r" (__b) \
176 : "r" (ptr), "i" (-EFAULT)); \
177 (x) = (__typeof(x))(__typeof((x)-(x))) \
178 (((u64)__hi32(__a, __b) << 32) | \
179 __lo32(__a, __b)); \
180 })
181#endif
182
183extern int __get_user_bad(void)
184 __attribute__((warning("sizeof __get_user argument not 1, 2, 4 or 8")));
151 185
152/*
153 * Careful: we have to cast the result to the type of the pointer
154 * for sign reasons.
155 */
156/** 186/**
157 * __get_user: - Get a simple variable from user space, with less checking. 187 * __get_user: - Get a simple variable from user space, with less checking.
158 * @x: Variable to store result. 188 * @x: Variable to store result.
@@ -174,30 +204,62 @@ extern int __put_user_bad(void);
174 * function. 204 * function.
175 */ 205 */
176#define __get_user(x, ptr) \ 206#define __get_user(x, ptr) \
177({ struct __get_user __ret; \ 207 ({ \
178 __typeof__(*(ptr)) const __user *__gu_addr = (ptr); \ 208 int __ret; \
179 __chk_user_ptr(__gu_addr); \ 209 __chk_user_ptr(ptr); \
180 switch (sizeof(*(__gu_addr))) { \ 210 switch (sizeof(*(ptr))) { \
181 case 1: \ 211 case 1: __get_user_1(x, ptr, __ret); break; \
182 __ret = __get_user_1(__gu_addr); \ 212 case 2: __get_user_2(x, ptr, __ret); break; \
183 break; \ 213 case 4: __get_user_4(x, ptr, __ret); break; \
184 case 2: \ 214 case 8: __get_user_8(x, ptr, __ret); break; \
185 __ret = __get_user_2(__gu_addr); \ 215 default: __ret = __get_user_bad(); break; \
186 break; \ 216 } \
187 case 4: \ 217 __ret; \
188 __ret = __get_user_4(__gu_addr); \ 218 })
189 break; \ 219
190 case 8: \ 220/* Support macros for __put_user(). */
191 __ret = __get_user_8(__gu_addr); \ 221
192 break; \ 222#define __put_user_asm(OP, x, ptr, ret) \
193 default: \ 223 asm volatile("1: {" #OP " %1, %2; movei %0, 0 }\n" \
194 __ret = __get_user_bad(); \ 224 ".pushsection .fixup,\"ax\"\n" \
195 break; \ 225 "0: { movei %0, %3; j 9f }\n" \
196 } \ 226 ".section __ex_table,\"a\"\n" \
197 (x) = (__typeof__(*__gu_addr)) (__typeof__(*__gu_addr - *__gu_addr)) \ 227 _ASM_PTR " 1b, 0b\n" \
198 __ret.val; \ 228 ".popsection\n" \
199 __ret.err; \ 229 "9:" \
200}) 230 : "=r" (ret) \
231 : "r" (ptr), "r" (x), "i" (-EFAULT))
232
233#ifdef __tilegx__
234#define __put_user_1(x, ptr, ret) __put_user_asm(st1, x, ptr, ret)
235#define __put_user_2(x, ptr, ret) __put_user_asm(st2, x, ptr, ret)
236#define __put_user_4(x, ptr, ret) __put_user_asm(st4, x, ptr, ret)
237#define __put_user_8(x, ptr, ret) __put_user_asm(st, x, ptr, ret)
238#else
239#define __put_user_1(x, ptr, ret) __put_user_asm(sb, x, ptr, ret)
240#define __put_user_2(x, ptr, ret) __put_user_asm(sh, x, ptr, ret)
241#define __put_user_4(x, ptr, ret) __put_user_asm(sw, x, ptr, ret)
242#define __put_user_8(x, ptr, ret) \
243 ({ \
244 u64 __x = (__typeof((x)-(x)))(x); \
245 int __lo = (int) __x, __hi = (int) (__x >> 32); \
246 asm volatile("1: { sw %1, %2; addi %0, %1, 4 }\n" \
247 "2: { sw %0, %3; movei %0, 0 }\n" \
248 ".pushsection .fixup,\"ax\"\n" \
249 "0: { movei %0, %4; j 9f }\n" \
250 ".section __ex_table,\"a\"\n" \
251 ".word 1b, 0b\n" \
252 ".word 2b, 0b\n" \
253 ".popsection\n" \
254 "9:" \
255 : "=&r" (ret) \
256 : "r" (ptr), "r" (__lo32(__lo, __hi)), \
257 "r" (__hi32(__lo, __hi)), "i" (-EFAULT)); \
258 })
259#endif
260
261extern int __put_user_bad(void)
262 __attribute__((warning("sizeof __put_user argument not 1, 2, 4 or 8")));
201 263
202/** 264/**
203 * __put_user: - Write a simple value into user space, with less checking. 265 * __put_user: - Write a simple value into user space, with less checking.
@@ -217,39 +279,19 @@ extern int __put_user_bad(void);
217 * function. 279 * function.
218 * 280 *
219 * Returns zero on success, or -EFAULT on error. 281 * Returns zero on success, or -EFAULT on error.
220 *
221 * Implementation note: The "case 8" logic of casting to the type of
222 * the result of subtracting the value from itself is basically a way
223 * of keeping all integer types the same, but casting any pointers to
224 * ptrdiff_t, i.e. also an integer type. This way there are no
225 * questionable casts seen by the compiler on an ILP32 platform.
226 */ 282 */
227#define __put_user(x, ptr) \ 283#define __put_user(x, ptr) \
228({ \ 284({ \
229 int __pu_err = 0; \ 285 int __ret; \
230 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 286 __chk_user_ptr(ptr); \
231 typeof(*__pu_addr) __pu_val = (x); \ 287 switch (sizeof(*(ptr))) { \
232 __chk_user_ptr(__pu_addr); \ 288 case 1: __put_user_1(x, ptr, __ret); break; \
233 switch (sizeof(__pu_val)) { \ 289 case 2: __put_user_2(x, ptr, __ret); break; \
234 case 1: \ 290 case 4: __put_user_4(x, ptr, __ret); break; \
235 __pu_err = __put_user_1((long)__pu_val, __pu_addr); \ 291 case 8: __put_user_8(x, ptr, __ret); break; \
236 break; \ 292 default: __ret = __put_user_bad(); break; \
237 case 2: \
238 __pu_err = __put_user_2((long)__pu_val, __pu_addr); \
239 break; \
240 case 4: \
241 __pu_err = __put_user_4((long)__pu_val, __pu_addr); \
242 break; \
243 case 8: \
244 __pu_err = \
245 __put_user_8((__typeof__(__pu_val - __pu_val))__pu_val,\
246 __pu_addr); \
247 break; \
248 default: \
249 __pu_err = __put_user_bad(); \
250 break; \
251 } \ 293 } \
252 __pu_err; \ 294 __ret; \
253}) 295})
254 296
255/* 297/*
@@ -378,7 +420,7 @@ static inline unsigned long __must_check copy_from_user(void *to,
378/** 420/**
379 * __copy_in_user() - copy data within user space, with less checking. 421 * __copy_in_user() - copy data within user space, with less checking.
380 * @to: Destination address, in user space. 422 * @to: Destination address, in user space.
381 * @from: Source address, in kernel space. 423 * @from: Source address, in user space.
382 * @n: Number of bytes to copy. 424 * @n: Number of bytes to copy.
383 * 425 *
384 * Context: User context only. This function may sleep. 426 * Context: User context only. This function may sleep.
diff --git a/arch/tile/kernel/Makefile b/arch/tile/kernel/Makefile
index b4dbc057baad..d6261e4670fe 100644
--- a/arch/tile/kernel/Makefile
+++ b/arch/tile/kernel/Makefile
@@ -9,7 +9,6 @@ obj-y := backtrace.o entry.o init_task.o irq.o messaging.o \
9 intvec_$(BITS).o regs_$(BITS).o tile-desc_$(BITS).o 9 intvec_$(BITS).o regs_$(BITS).o tile-desc_$(BITS).o
10 10
11obj-$(CONFIG_HARDWALL) += hardwall.o 11obj-$(CONFIG_HARDWALL) += hardwall.o
12obj-$(CONFIG_TILEGX) += futex_64.o
13obj-$(CONFIG_COMPAT) += compat.o compat_signal.o 12obj-$(CONFIG_COMPAT) += compat.o compat_signal.o
14obj-$(CONFIG_SMP) += smpboot.o smp.o tlb.o 13obj-$(CONFIG_SMP) += smpboot.o smp.o tlb.o
15obj-$(CONFIG_MODULES) += module.o 14obj-$(CONFIG_MODULES) += module.o
diff --git a/arch/tile/lib/atomic_32.c b/arch/tile/lib/atomic_32.c
index 771b251b409d..f5cada70c3c8 100644
--- a/arch/tile/lib/atomic_32.c
+++ b/arch/tile/lib/atomic_32.c
@@ -18,7 +18,6 @@
18#include <linux/module.h> 18#include <linux/module.h>
19#include <linux/mm.h> 19#include <linux/mm.h>
20#include <linux/atomic.h> 20#include <linux/atomic.h>
21#include <asm/futex.h>
22#include <arch/chip.h> 21#include <arch/chip.h>
23 22
24/* See <asm/atomic_32.h> */ 23/* See <asm/atomic_32.h> */
@@ -50,7 +49,7 @@ int atomic_locks[PAGE_SIZE / sizeof(int)] __page_aligned_bss;
50 49
51#endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */ 50#endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
52 51
53static inline int *__atomic_hashed_lock(volatile void *v) 52int *__atomic_hashed_lock(volatile void *v)
54{ 53{
55 /* NOTE: this code must match "sys_cmpxchg" in kernel/intvec_32.S */ 54 /* NOTE: this code must match "sys_cmpxchg" in kernel/intvec_32.S */
56#if ATOMIC_LOCKS_FOUND_VIA_TABLE() 55#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
@@ -191,47 +190,6 @@ u64 _atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n)
191EXPORT_SYMBOL(_atomic64_cmpxchg); 190EXPORT_SYMBOL(_atomic64_cmpxchg);
192 191
193 192
194static inline int *__futex_setup(int __user *v)
195{
196 /*
197 * Issue a prefetch to the counter to bring it into cache.
198 * As for __atomic_setup, but we can't do a read into the L1
199 * since it might fault; instead we do a prefetch into the L2.
200 */
201 __insn_prefetch(v);
202 return __atomic_hashed_lock((int __force *)v);
203}
204
205struct __get_user futex_set(u32 __user *v, int i)
206{
207 return __atomic_xchg((int __force *)v, __futex_setup(v), i);
208}
209
210struct __get_user futex_add(u32 __user *v, int n)
211{
212 return __atomic_xchg_add((int __force *)v, __futex_setup(v), n);
213}
214
215struct __get_user futex_or(u32 __user *v, int n)
216{
217 return __atomic_or((int __force *)v, __futex_setup(v), n);
218}
219
220struct __get_user futex_andn(u32 __user *v, int n)
221{
222 return __atomic_andn((int __force *)v, __futex_setup(v), n);
223}
224
225struct __get_user futex_xor(u32 __user *v, int n)
226{
227 return __atomic_xor((int __force *)v, __futex_setup(v), n);
228}
229
230struct __get_user futex_cmpxchg(u32 __user *v, int o, int n)
231{
232 return __atomic_cmpxchg((int __force *)v, __futex_setup(v), o, n);
233}
234
235/* 193/*
236 * If any of the atomic or futex routines hit a bad address (not in 194 * If any of the atomic or futex routines hit a bad address (not in
237 * the page tables at kernel PL) this routine is called. The futex 195 * the page tables at kernel PL) this routine is called. The futex
@@ -323,7 +281,4 @@ void __init __init_atomic_per_cpu(void)
323 BUILD_BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE); 281 BUILD_BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE);
324 282
325#endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */ 283#endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
326
327 /* The futex code makes this assumption, so we validate it here. */
328 BUILD_BUG_ON(sizeof(atomic_t) != sizeof(int));
329} 284}
diff --git a/arch/tile/lib/exports.c b/arch/tile/lib/exports.c
index 2a81d32de0da..dd5f0a33fdaf 100644
--- a/arch/tile/lib/exports.c
+++ b/arch/tile/lib/exports.c
@@ -18,14 +18,6 @@
18 18
19/* arch/tile/lib/usercopy.S */ 19/* arch/tile/lib/usercopy.S */
20#include <linux/uaccess.h> 20#include <linux/uaccess.h>
21EXPORT_SYMBOL(__get_user_1);
22EXPORT_SYMBOL(__get_user_2);
23EXPORT_SYMBOL(__get_user_4);
24EXPORT_SYMBOL(__get_user_8);
25EXPORT_SYMBOL(__put_user_1);
26EXPORT_SYMBOL(__put_user_2);
27EXPORT_SYMBOL(__put_user_4);
28EXPORT_SYMBOL(__put_user_8);
29EXPORT_SYMBOL(strnlen_user_asm); 21EXPORT_SYMBOL(strnlen_user_asm);
30EXPORT_SYMBOL(strncpy_from_user_asm); 22EXPORT_SYMBOL(strncpy_from_user_asm);
31EXPORT_SYMBOL(clear_user_asm); 23EXPORT_SYMBOL(clear_user_asm);
diff --git a/arch/tile/lib/usercopy_32.S b/arch/tile/lib/usercopy_32.S
index 979f76d83746..b62d002af009 100644
--- a/arch/tile/lib/usercopy_32.S
+++ b/arch/tile/lib/usercopy_32.S
@@ -19,82 +19,6 @@
19 19
20/* Access user memory, but use MMU to avoid propagating kernel exceptions. */ 20/* Access user memory, but use MMU to avoid propagating kernel exceptions. */
21 21
22 .pushsection .fixup,"ax"
23
24get_user_fault:
25 { move r0, zero; move r1, zero }
26 { movei r2, -EFAULT; jrp lr }
27 ENDPROC(get_user_fault)
28
29put_user_fault:
30 { movei r0, -EFAULT; jrp lr }
31 ENDPROC(put_user_fault)
32
33 .popsection
34
35/*
36 * __get_user_N functions take a pointer in r0, and return 0 in r2
37 * on success, with the value in r0; or else -EFAULT in r2.
38 */
39#define __get_user_N(bytes, LOAD) \
40 STD_ENTRY(__get_user_##bytes); \
411: { LOAD r0, r0; move r1, zero; move r2, zero }; \
42 jrp lr; \
43 STD_ENDPROC(__get_user_##bytes); \
44 .pushsection __ex_table,"a"; \
45 .word 1b, get_user_fault; \
46 .popsection
47
48__get_user_N(1, lb_u)
49__get_user_N(2, lh_u)
50__get_user_N(4, lw)
51
52/*
53 * __get_user_8 takes a pointer in r0, and returns 0 in r2
54 * on success, with the value in r0/r1; or else -EFAULT in r2.
55 */
56 STD_ENTRY(__get_user_8);
571: { lw r0, r0; addi r1, r0, 4 };
582: { lw r1, r1; move r2, zero };
59 jrp lr;
60 STD_ENDPROC(__get_user_8);
61 .pushsection __ex_table,"a";
62 .word 1b, get_user_fault;
63 .word 2b, get_user_fault;
64 .popsection
65
66/*
67 * __put_user_N functions take a value in r0 and a pointer in r1,
68 * and return 0 in r0 on success or -EFAULT on failure.
69 */
70#define __put_user_N(bytes, STORE) \
71 STD_ENTRY(__put_user_##bytes); \
721: { STORE r1, r0; move r0, zero }; \
73 jrp lr; \
74 STD_ENDPROC(__put_user_##bytes); \
75 .pushsection __ex_table,"a"; \
76 .word 1b, put_user_fault; \
77 .popsection
78
79__put_user_N(1, sb)
80__put_user_N(2, sh)
81__put_user_N(4, sw)
82
83/*
84 * __put_user_8 takes a value in r0/r1 and a pointer in r2,
85 * and returns 0 in r0 on success or -EFAULT on failure.
86 */
87STD_ENTRY(__put_user_8)
881: { sw r2, r0; addi r2, r2, 4 }
892: { sw r2, r1; move r0, zero }
90 jrp lr
91 STD_ENDPROC(__put_user_8)
92 .pushsection __ex_table,"a"
93 .word 1b, put_user_fault
94 .word 2b, put_user_fault
95 .popsection
96
97
98/* 22/*
99 * strnlen_user_asm takes the pointer in r0, and the length bound in r1. 23 * strnlen_user_asm takes the pointer in r0, and the length bound in r1.
100 * It returns the length, including the terminating NUL, or zero on exception. 24 * It returns the length, including the terminating NUL, or zero on exception.
diff --git a/arch/tile/lib/usercopy_64.S b/arch/tile/lib/usercopy_64.S
index 2ff44f87b78e..adb2dbbc70cd 100644
--- a/arch/tile/lib/usercopy_64.S
+++ b/arch/tile/lib/usercopy_64.S
@@ -19,55 +19,6 @@
19 19
20/* Access user memory, but use MMU to avoid propagating kernel exceptions. */ 20/* Access user memory, but use MMU to avoid propagating kernel exceptions. */
21 21
22 .pushsection .fixup,"ax"
23
24get_user_fault:
25 { movei r1, -EFAULT; move r0, zero }
26 jrp lr
27 ENDPROC(get_user_fault)
28
29put_user_fault:
30 { movei r0, -EFAULT; jrp lr }
31 ENDPROC(put_user_fault)
32
33 .popsection
34
35/*
36 * __get_user_N functions take a pointer in r0, and return 0 in r1
37 * on success, with the value in r0; or else -EFAULT in r1.
38 */
39#define __get_user_N(bytes, LOAD) \
40 STD_ENTRY(__get_user_##bytes); \
411: { LOAD r0, r0; move r1, zero }; \
42 jrp lr; \
43 STD_ENDPROC(__get_user_##bytes); \
44 .pushsection __ex_table,"a"; \
45 .quad 1b, get_user_fault; \
46 .popsection
47
48__get_user_N(1, ld1u)
49__get_user_N(2, ld2u)
50__get_user_N(4, ld4u)
51__get_user_N(8, ld)
52
53/*
54 * __put_user_N functions take a value in r0 and a pointer in r1,
55 * and return 0 in r0 on success or -EFAULT on failure.
56 */
57#define __put_user_N(bytes, STORE) \
58 STD_ENTRY(__put_user_##bytes); \
591: { STORE r1, r0; move r0, zero }; \
60 jrp lr; \
61 STD_ENDPROC(__put_user_##bytes); \
62 .pushsection __ex_table,"a"; \
63 .quad 1b, put_user_fault; \
64 .popsection
65
66__put_user_N(1, st1)
67__put_user_N(2, st2)
68__put_user_N(4, st4)
69__put_user_N(8, st)
70
71/* 22/*
72 * strnlen_user_asm takes the pointer in r0, and the length bound in r1. 23 * strnlen_user_asm takes the pointer in r0, and the length bound in r1.
73 * It returns the length, including the terminating NUL, or zero on exception. 24 * It returns the length, including the terminating NUL, or zero on exception.
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-25 01:55:42 -0500