aboutsummaryrefslogtreecommitdiffstats
path: root/include/asm-x86/uaccess_32.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/asm-x86/uaccess_32.h')
-rw-r--r--include/asm-x86/uaccess_32.h316
1 files changed, 187 insertions, 129 deletions
diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h
index fcc570ec4fee..8e7595c1f34e 100644
--- a/include/asm-x86/uaccess_32.h
+++ b/include/asm-x86/uaccess_32.h
@@ -32,7 +32,7 @@
32#define get_fs() (current_thread_info()->addr_limit) 32#define get_fs() (current_thread_info()->addr_limit)
33#define set_fs(x) (current_thread_info()->addr_limit = (x)) 33#define set_fs(x) (current_thread_info()->addr_limit = (x))
34 34
35#define segment_eq(a,b) ((a).seg == (b).seg) 35#define segment_eq(a, b) ((a).seg == (b).seg)
36 36
37/* 37/*
38 * movsl can be slow when source and dest are not both 8-byte aligned 38 * movsl can be slow when source and dest are not both 8-byte aligned
@@ -43,7 +43,9 @@ extern struct movsl_mask {
43} ____cacheline_aligned_in_smp movsl_mask; 43} ____cacheline_aligned_in_smp movsl_mask;
44#endif 44#endif
45 45
46#define __addr_ok(addr) ((unsigned long __force)(addr) < (current_thread_info()->addr_limit.seg)) 46#define __addr_ok(addr) \
47 ((unsigned long __force)(addr) < \
48 (current_thread_info()->addr_limit.seg))
47 49
48/* 50/*
49 * Test whether a block of memory is a valid user space address. 51 * Test whether a block of memory is a valid user space address.
@@ -54,13 +56,16 @@ extern struct movsl_mask {
54 * 56 *
55 * This needs 33-bit arithmetic. We have a carry... 57 * This needs 33-bit arithmetic. We have a carry...
56 */ 58 */
57#define __range_ok(addr,size) ({ \ 59#define __range_ok(addr, size) \
58 unsigned long flag,roksum; \ 60({ \
59 __chk_user_ptr(addr); \ 61 unsigned long flag, roksum; \
60 asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \ 62 __chk_user_ptr(addr); \
61 :"=&r" (flag), "=r" (roksum) \ 63 asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \
62 :"1" (addr),"g" ((int)(size)),"rm" (current_thread_info()->addr_limit.seg)); \ 64 :"=&r" (flag), "=r" (roksum) \
63 flag; }) 65 :"1" (addr), "g" ((int)(size)), \
66 "rm" (current_thread_info()->addr_limit.seg)); \
67 flag; \
68})
64 69
65/** 70/**
66 * access_ok: - Checks if a user space pointer is valid 71 * access_ok: - Checks if a user space pointer is valid
@@ -81,7 +86,7 @@ extern struct movsl_mask {
81 * checks that the pointer is in the user space range - after calling 86 * checks that the pointer is in the user space range - after calling
82 * this function, memory access functions may still return -EFAULT. 87 * this function, memory access functions may still return -EFAULT.
83 */ 88 */
84#define access_ok(type,addr,size) (likely(__range_ok(addr,size) == 0)) 89#define access_ok(type, addr, size) (likely(__range_ok(addr, size) == 0))
85 90
86/* 91/*
87 * The exception table consists of pairs of addresses: the first is the 92 * The exception table consists of pairs of addresses: the first is the
@@ -96,8 +101,7 @@ extern struct movsl_mask {
96 * on our cache or tlb entries. 101 * on our cache or tlb entries.
97 */ 102 */
98 103
99struct exception_table_entry 104struct exception_table_entry {
100{
101 unsigned long insn, fixup; 105 unsigned long insn, fixup;
102}; 106};
103 107
@@ -122,13 +126,15 @@ extern void __get_user_1(void);
122extern void __get_user_2(void); 126extern void __get_user_2(void);
123extern void __get_user_4(void); 127extern void __get_user_4(void);
124 128
125#define __get_user_x(size,ret,x,ptr) \ 129#define __get_user_x(size, ret, x, ptr) \
126 __asm__ __volatile__("call __get_user_" #size \ 130 asm volatile("call __get_user_" #size \
127 :"=a" (ret),"=d" (x) \ 131 :"=a" (ret),"=d" (x) \
128 :"0" (ptr)) 132 :"0" (ptr))
133
129 134
135/* Careful: we have to cast the result to the type of the pointer
136 * for sign reasons */
130 137
131/* Careful: we have to cast the result to the type of the pointer for sign reasons */
132/** 138/**
133 * get_user: - Get a simple variable from user space. 139 * get_user: - Get a simple variable from user space.
134 * @x: Variable to store result. 140 * @x: Variable to store result.
@@ -146,15 +152,24 @@ extern void __get_user_4(void);
146 * Returns zero on success, or -EFAULT on error. 152 * Returns zero on success, or -EFAULT on error.
147 * On error, the variable @x is set to zero. 153 * On error, the variable @x is set to zero.
148 */ 154 */
149#define get_user(x,ptr) \ 155#define get_user(x, ptr) \
150({ int __ret_gu; \ 156({ \
157 int __ret_gu; \
151 unsigned long __val_gu; \ 158 unsigned long __val_gu; \
152 __chk_user_ptr(ptr); \ 159 __chk_user_ptr(ptr); \
153 switch(sizeof (*(ptr))) { \ 160 switch (sizeof(*(ptr))) { \
154 case 1: __get_user_x(1,__ret_gu,__val_gu,ptr); break; \ 161 case 1: \
155 case 2: __get_user_x(2,__ret_gu,__val_gu,ptr); break; \ 162 __get_user_x(1, __ret_gu, __val_gu, ptr); \
156 case 4: __get_user_x(4,__ret_gu,__val_gu,ptr); break; \ 163 break; \
157 default: __get_user_x(X,__ret_gu,__val_gu,ptr); break; \ 164 case 2: \
165 __get_user_x(2, __ret_gu, __val_gu, ptr); \
166 break; \
167 case 4: \
168 __get_user_x(4, __ret_gu, __val_gu, ptr); \
169 break; \
170 default: \
171 __get_user_x(X, __ret_gu, __val_gu, ptr); \
172 break; \
158 } \ 173 } \
159 (x) = (__typeof__(*(ptr)))__val_gu; \ 174 (x) = (__typeof__(*(ptr)))__val_gu; \
160 __ret_gu; \ 175 __ret_gu; \
@@ -171,11 +186,25 @@ extern void __put_user_2(void);
171extern void __put_user_4(void); 186extern void __put_user_4(void);
172extern void __put_user_8(void); 187extern void __put_user_8(void);
173 188
174#define __put_user_1(x, ptr) __asm__ __volatile__("call __put_user_1":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr)) 189#define __put_user_1(x, ptr) \
175#define __put_user_2(x, ptr) __asm__ __volatile__("call __put_user_2":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr)) 190 asm volatile("call __put_user_1" : "=a" (__ret_pu) \
176#define __put_user_4(x, ptr) __asm__ __volatile__("call __put_user_4":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr)) 191 : "0" ((typeof(*(ptr)))(x)), "c" (ptr))
177#define __put_user_8(x, ptr) __asm__ __volatile__("call __put_user_8":"=a" (__ret_pu):"A" ((typeof(*(ptr)))(x)), "c" (ptr)) 192
178#define __put_user_X(x, ptr) __asm__ __volatile__("call __put_user_X":"=a" (__ret_pu):"c" (ptr)) 193#define __put_user_2(x, ptr) \
194 asm volatile("call __put_user_2" : "=a" (__ret_pu) \
195 : "0" ((typeof(*(ptr)))(x)), "c" (ptr))
196
197#define __put_user_4(x, ptr) \
198 asm volatile("call __put_user_4" : "=a" (__ret_pu) \
199 : "0" ((typeof(*(ptr)))(x)), "c" (ptr))
200
201#define __put_user_8(x, ptr) \
202 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
203 : "A" ((typeof(*(ptr)))(x)), "c" (ptr))
204
205#define __put_user_X(x, ptr) \
206 asm volatile("call __put_user_X" : "=a" (__ret_pu) \
207 : "c" (ptr))
179 208
180/** 209/**
181 * put_user: - Write a simple value into user space. 210 * put_user: - Write a simple value into user space.
@@ -195,32 +224,43 @@ extern void __put_user_8(void);
195 */ 224 */
196#ifdef CONFIG_X86_WP_WORKS_OK 225#ifdef CONFIG_X86_WP_WORKS_OK
197 226
198#define put_user(x,ptr) \ 227#define put_user(x, ptr) \
199({ int __ret_pu; \ 228({ \
229 int __ret_pu; \
200 __typeof__(*(ptr)) __pu_val; \ 230 __typeof__(*(ptr)) __pu_val; \
201 __chk_user_ptr(ptr); \ 231 __chk_user_ptr(ptr); \
202 __pu_val = x; \ 232 __pu_val = x; \
203 switch(sizeof(*(ptr))) { \ 233 switch (sizeof(*(ptr))) { \
204 case 1: __put_user_1(__pu_val, ptr); break; \ 234 case 1: \
205 case 2: __put_user_2(__pu_val, ptr); break; \ 235 __put_user_1(__pu_val, ptr); \
206 case 4: __put_user_4(__pu_val, ptr); break; \ 236 break; \
207 case 8: __put_user_8(__pu_val, ptr); break; \ 237 case 2: \
208 default:__put_user_X(__pu_val, ptr); break; \ 238 __put_user_2(__pu_val, ptr); \
239 break; \
240 case 4: \
241 __put_user_4(__pu_val, ptr); \
242 break; \
243 case 8: \
244 __put_user_8(__pu_val, ptr); \
245 break; \
246 default: \
247 __put_user_X(__pu_val, ptr); \
248 break; \
209 } \ 249 } \
210 __ret_pu; \ 250 __ret_pu; \
211}) 251})
212 252
213#else 253#else
214#define put_user(x,ptr) \ 254#define put_user(x, ptr) \
215({ \ 255({ \
216 int __ret_pu; \ 256 int __ret_pu; \
217 __typeof__(*(ptr)) __pus_tmp = x; \ 257 __typeof__(*(ptr))__pus_tmp = x; \
218 __ret_pu=0; \ 258 __ret_pu = 0; \
219 if(unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \ 259 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \
220 sizeof(*(ptr))) != 0)) \ 260 sizeof(*(ptr))) != 0)) \
221 __ret_pu=-EFAULT; \ 261 __ret_pu = -EFAULT; \
222 __ret_pu; \ 262 __ret_pu; \
223 }) 263})
224 264
225 265
226#endif 266#endif
@@ -245,8 +285,8 @@ extern void __put_user_8(void);
245 * Returns zero on success, or -EFAULT on error. 285 * Returns zero on success, or -EFAULT on error.
246 * On error, the variable @x is set to zero. 286 * On error, the variable @x is set to zero.
247 */ 287 */
248#define __get_user(x,ptr) \ 288#define __get_user(x, ptr) \
249 __get_user_nocheck((x),(ptr),sizeof(*(ptr))) 289 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
250 290
251 291
252/** 292/**
@@ -268,54 +308,62 @@ extern void __put_user_8(void);
268 * 308 *
269 * Returns zero on success, or -EFAULT on error. 309 * Returns zero on success, or -EFAULT on error.
270 */ 310 */
271#define __put_user(x,ptr) \ 311#define __put_user(x, ptr) \
272 __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) 312 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
273 313
274#define __put_user_nocheck(x,ptr,size) \ 314#define __put_user_nocheck(x, ptr, size) \
275({ \ 315({ \
276 long __pu_err; \ 316 long __pu_err; \
277 __put_user_size((x),(ptr),(size),__pu_err,-EFAULT); \ 317 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
278 __pu_err; \ 318 __pu_err; \
279}) 319})
280 320
281 321
282#define __put_user_u64(x, addr, err) \ 322#define __put_user_u64(x, addr, err) \
283 __asm__ __volatile__( \ 323 asm volatile("1: movl %%eax,0(%2)\n" \
284 "1: movl %%eax,0(%2)\n" \ 324 "2: movl %%edx,4(%2)\n" \
285 "2: movl %%edx,4(%2)\n" \ 325 "3:\n" \
286 "3:\n" \ 326 ".section .fixup,\"ax\"\n" \
287 ".section .fixup,\"ax\"\n" \ 327 "4: movl %3,%0\n" \
288 "4: movl %3,%0\n" \ 328 " jmp 3b\n" \
289 " jmp 3b\n" \ 329 ".previous\n" \
290 ".previous\n" \ 330 _ASM_EXTABLE(1b, 4b) \
291 _ASM_EXTABLE(1b,4b) \ 331 _ASM_EXTABLE(2b, 4b) \
292 _ASM_EXTABLE(2b,4b) \ 332 : "=r" (err) \
293 : "=r"(err) \ 333 : "A" (x), "r" (addr), "i" (-EFAULT), "0" (err))
294 : "A" (x), "r" (addr), "i"(-EFAULT), "0"(err))
295 334
296#ifdef CONFIG_X86_WP_WORKS_OK 335#ifdef CONFIG_X86_WP_WORKS_OK
297 336
298#define __put_user_size(x,ptr,size,retval,errret) \ 337#define __put_user_size(x, ptr, size, retval, errret) \
299do { \ 338do { \
300 retval = 0; \ 339 retval = 0; \
301 __chk_user_ptr(ptr); \ 340 __chk_user_ptr(ptr); \
302 switch (size) { \ 341 switch (size) { \
303 case 1: __put_user_asm(x,ptr,retval,"b","b","iq",errret);break; \ 342 case 1: \
304 case 2: __put_user_asm(x,ptr,retval,"w","w","ir",errret);break; \ 343 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
305 case 4: __put_user_asm(x,ptr,retval,"l","","ir",errret); break; \ 344 break; \
306 case 8: __put_user_u64((__typeof__(*ptr))(x),ptr,retval); break;\ 345 case 2: \
307 default: __put_user_bad(); \ 346 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
347 break; \
348 case 4: \
349 __put_user_asm(x, ptr, retval, "l", "", "ir", errret); \
350 break; \
351 case 8: \
352 __put_user_u64((__typeof__(*ptr))(x), ptr, retval); \
353 break; \
354 default: \
355 __put_user_bad(); \
308 } \ 356 } \
309} while (0) 357} while (0)
310 358
311#else 359#else
312 360
313#define __put_user_size(x,ptr,size,retval,errret) \ 361#define __put_user_size(x, ptr, size, retval, errret) \
314do { \ 362do { \
315 __typeof__(*(ptr)) __pus_tmp = x; \ 363 __typeof__(*(ptr))__pus_tmp = x; \
316 retval = 0; \ 364 retval = 0; \
317 \ 365 \
318 if(unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \ 366 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \
319 retval = errret; \ 367 retval = errret; \
320} while (0) 368} while (0)
321 369
@@ -329,65 +377,70 @@ struct __large_struct { unsigned long buf[100]; };
329 * aliasing issues. 377 * aliasing issues.
330 */ 378 */
331#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ 379#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
332 __asm__ __volatile__( \ 380 asm volatile("1: mov"itype" %"rtype"1,%2\n" \
333 "1: mov"itype" %"rtype"1,%2\n" \ 381 "2:\n" \
334 "2:\n" \ 382 ".section .fixup,\"ax\"\n" \
335 ".section .fixup,\"ax\"\n" \ 383 "3: movl %3,%0\n" \
336 "3: movl %3,%0\n" \ 384 " jmp 2b\n" \
337 " jmp 2b\n" \ 385 ".previous\n" \
338 ".previous\n" \ 386 _ASM_EXTABLE(1b, 3b) \
339 _ASM_EXTABLE(1b,3b) \ 387 : "=r"(err) \
340 : "=r"(err) \ 388 : ltype (x), "m" (__m(addr)), "i" (errret), "0" (err))
341 : ltype (x), "m"(__m(addr)), "i"(errret), "0"(err)) 389
342 390
343 391#define __get_user_nocheck(x, ptr, size) \
344#define __get_user_nocheck(x,ptr,size) \ 392({ \
345({ \ 393 long __gu_err; \
346 long __gu_err; \ 394 unsigned long __gu_val; \
347 unsigned long __gu_val; \ 395 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
348 __get_user_size(__gu_val,(ptr),(size),__gu_err,-EFAULT);\ 396 (x) = (__typeof__(*(ptr)))__gu_val; \
349 (x) = (__typeof__(*(ptr)))__gu_val; \ 397 __gu_err; \
350 __gu_err; \
351}) 398})
352 399
353extern long __get_user_bad(void); 400extern long __get_user_bad(void);
354 401
355#define __get_user_size(x,ptr,size,retval,errret) \ 402#define __get_user_size(x, ptr, size, retval, errret) \
356do { \ 403do { \
357 retval = 0; \ 404 retval = 0; \
358 __chk_user_ptr(ptr); \ 405 __chk_user_ptr(ptr); \
359 switch (size) { \ 406 switch (size) { \
360 case 1: __get_user_asm(x,ptr,retval,"b","b","=q",errret);break; \ 407 case 1: \
361 case 2: __get_user_asm(x,ptr,retval,"w","w","=r",errret);break; \ 408 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
362 case 4: __get_user_asm(x,ptr,retval,"l","","=r",errret);break; \ 409 break; \
363 default: (x) = __get_user_bad(); \ 410 case 2: \
411 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
412 break; \
413 case 4: \
414 __get_user_asm(x, ptr, retval, "l", "", "=r", errret); \
415 break; \
416 default: \
417 (x) = __get_user_bad(); \
364 } \ 418 } \
365} while (0) 419} while (0)
366 420
367#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ 421#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
368 __asm__ __volatile__( \ 422 asm volatile("1: mov"itype" %2,%"rtype"1\n" \
369 "1: mov"itype" %2,%"rtype"1\n" \ 423 "2:\n" \
370 "2:\n" \ 424 ".section .fixup,\"ax\"\n" \
371 ".section .fixup,\"ax\"\n" \ 425 "3: movl %3,%0\n" \
372 "3: movl %3,%0\n" \ 426 " xor"itype" %"rtype"1,%"rtype"1\n" \
373 " xor"itype" %"rtype"1,%"rtype"1\n" \ 427 " jmp 2b\n" \
374 " jmp 2b\n" \ 428 ".previous\n" \
375 ".previous\n" \ 429 _ASM_EXTABLE(1b, 3b) \
376 _ASM_EXTABLE(1b,3b) \ 430 : "=r" (err), ltype (x) \
377 : "=r"(err), ltype (x) \ 431 : "m" (__m(addr)), "i" (errret), "0" (err))
378 : "m"(__m(addr)), "i"(errret), "0"(err)) 432
379 433
380 434unsigned long __must_check __copy_to_user_ll
381unsigned long __must_check __copy_to_user_ll(void __user *to, 435 (void __user *to, const void *from, unsigned long n);
382 const void *from, unsigned long n); 436unsigned long __must_check __copy_from_user_ll
383unsigned long __must_check __copy_from_user_ll(void *to, 437 (void *to, const void __user *from, unsigned long n);
384 const void __user *from, unsigned long n); 438unsigned long __must_check __copy_from_user_ll_nozero
385unsigned long __must_check __copy_from_user_ll_nozero(void *to, 439 (void *to, const void __user *from, unsigned long n);
386 const void __user *from, unsigned long n); 440unsigned long __must_check __copy_from_user_ll_nocache
387unsigned long __must_check __copy_from_user_ll_nocache(void *to, 441 (void *to, const void __user *from, unsigned long n);
388 const void __user *from, unsigned long n); 442unsigned long __must_check __copy_from_user_ll_nocache_nozero
389unsigned long __must_check __copy_from_user_ll_nocache_nozero(void *to, 443 (void *to, const void __user *from, unsigned long n);
390 const void __user *from, unsigned long n);
391 444
392/** 445/**
393 * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking. 446 * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking.
@@ -416,13 +469,16 @@ __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
416 469
417 switch (n) { 470 switch (n) {
418 case 1: 471 case 1:
419 __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1); 472 __put_user_size(*(u8 *)from, (u8 __user *)to,
473 1, ret, 1);
420 return ret; 474 return ret;
421 case 2: 475 case 2:
422 __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2); 476 __put_user_size(*(u16 *)from, (u16 __user *)to,
477 2, ret, 2);
423 return ret; 478 return ret;
424 case 4: 479 case 4:
425 __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4); 480 __put_user_size(*(u32 *)from, (u32 __user *)to,
481 4, ret, 4);
426 return ret; 482 return ret;
427 } 483 }
428 } 484 }
@@ -545,19 +601,21 @@ static __always_inline unsigned long __copy_from_user_nocache(void *to,
545} 601}
546 602
547static __always_inline unsigned long 603static __always_inline unsigned long
548__copy_from_user_inatomic_nocache(void *to, const void __user *from, unsigned long n) 604__copy_from_user_inatomic_nocache(void *to, const void __user *from,
605 unsigned long n)
549{ 606{
550 return __copy_from_user_ll_nocache_nozero(to, from, n); 607 return __copy_from_user_ll_nocache_nozero(to, from, n);
551} 608}
552 609
553unsigned long __must_check copy_to_user(void __user *to, 610unsigned long __must_check copy_to_user(void __user *to,
554 const void *from, unsigned long n); 611 const void *from, unsigned long n);
555unsigned long __must_check copy_from_user(void *to, 612unsigned long __must_check copy_from_user(void *to,
556 const void __user *from, unsigned long n); 613 const void __user *from,
614 unsigned long n);
557long __must_check strncpy_from_user(char *dst, const char __user *src, 615long __must_check strncpy_from_user(char *dst, const char __user *src,
558 long count); 616 long count);
559long __must_check __strncpy_from_user(char *dst, 617long __must_check __strncpy_from_user(char *dst,
560 const char __user *src, long count); 618 const char __user *src, long count);
561 619
562/** 620/**
563 * strlen_user: - Get the size of a string in user space. 621 * strlen_user: - Get the size of a string in user space.
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-02 05:08:19 -0500