aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2012-05-26 19:57:16 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2012-05-26 19:57:16 -0400
commit1e2aec873ad6d16538512dbb96853caa1fa076af (patch)
treed792b19ac47be44debd24610ae27f1330fa490e4
parentae32adc1e06d096399f195eeda12d443d53539c4 (diff)
parent2c66f623631709aa5f2e4c14c7e089682e7394a3 (diff)
Merge branch 'generic-string-functions'
This makes <asm/word-at-a-time.h> actually live up to its promise of allowing architectures to help tune the string functions that do their work a word at a time. David had already taken the x86 strncpy_from_user() function, modified it to work on sparc, and then done the extra work to make it generically useful. This then expands on that work by making x86 use that generic version, completing the circle. But more importantly, it fixes up the word-at-a-time interfaces so that it's now easy to also support things like strnlen_user(), and pretty much most random string functions. David reports that it all works fine on sparc, and Jonas Bonn reported that an earlier version of this worked on OpenRISC too. It's pretty easy for architectures to add support for this and just replace their private versions with the generic code. * generic-string-functions: sparc: use the new generic strnlen_user() function x86: use the new generic strnlen_user() function lib: add generic strnlen_user() function word-at-a-time: make the interfaces truly generic x86: use generic strncpy_from_user routine
Diffstat
-rw-r--r--arch/openrisc/include/asm/Kbuild1
-rw-r--r--arch/sparc/Kconfig1
-rw-r--r--arch/sparc/include/asm/Kbuild1
-rw-r--r--arch/sparc/include/asm/uaccess_32.h22
-rw-r--r--arch/sparc/include/asm/uaccess_64.h8
-rw-r--r--arch/sparc/lib/Makefile1
-rw-r--r--arch/sparc/lib/ksyms.c2
-rw-r--r--arch/sparc/lib/strlen_user_32.S109
-rw-r--r--arch/sparc/lib/strlen_user_64.S97
-rw-r--r--arch/x86/Kconfig2
-rw-r--r--arch/x86/include/asm/uaccess.h4
-rw-r--r--arch/x86/include/asm/uaccess_32.h17
-rw-r--r--arch/x86/include/asm/uaccess_64.h3
-rw-r--r--arch/x86/include/asm/word-at-a-time.h32
-rw-r--r--arch/x86/lib/usercopy.c97
-rw-r--r--arch/x86/lib/usercopy_32.c41
-rw-r--r--arch/x86/lib/usercopy_64.c48
-rw-r--r--fs/namei.c22
-rw-r--r--include/asm-generic/word-at-a-time.h52
-rw-r--r--lib/Kconfig3
-rw-r--r--lib/Makefile1
-rw-r--r--lib/strncpy_from_user.c47
-rw-r--r--lib/strnlen_user.c138
23 files changed, 259 insertions, 490 deletions
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild
index c936483bc8e2..3f35c38d7b64 100644
--- a/arch/openrisc/include/asm/Kbuild
+++ b/arch/openrisc/include/asm/Kbuild
@@ -66,3 +66,4 @@ generic-y += topology.h
66generic-y += types.h 66generic-y += types.h
67generic-y += ucontext.h 67generic-y += ucontext.h
68generic-y += user.h 68generic-y += user.h
69generic-y += word-at-a-time.h
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 15e9e05740da..83bd051754e1 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -35,6 +35,7 @@ config SPARC
35 select GENERIC_CMOS_UPDATE 35 select GENERIC_CMOS_UPDATE
36 select GENERIC_CLOCKEVENTS 36 select GENERIC_CLOCKEVENTS
37 select GENERIC_STRNCPY_FROM_USER 37 select GENERIC_STRNCPY_FROM_USER
38 select GENERIC_STRNLEN_USER
38 39
39config SPARC32 40config SPARC32
40 def_bool !64BIT 41 def_bool !64BIT
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild
index 2c2e38821f60..67f83e0a0d68 100644
--- a/arch/sparc/include/asm/Kbuild
+++ b/arch/sparc/include/asm/Kbuild
@@ -21,3 +21,4 @@ generic-y += div64.h
21generic-y += local64.h 21generic-y += local64.h
22generic-y += irq_regs.h 22generic-y += irq_regs.h
23generic-y += local.h 23generic-y += local.h
24generic-y += word-at-a-time.h
diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h
index 59586b57ef1a..53a28dd59f59 100644
--- a/arch/sparc/include/asm/uaccess_32.h
+++ b/arch/sparc/include/asm/uaccess_32.h
@@ -16,6 +16,8 @@
16 16
17#ifndef __ASSEMBLY__ 17#ifndef __ASSEMBLY__
18 18
19#include <asm/processor.h>
20
19#define ARCH_HAS_SORT_EXTABLE 21#define ARCH_HAS_SORT_EXTABLE
20#define ARCH_HAS_SEARCH_EXTABLE 22#define ARCH_HAS_SEARCH_EXTABLE
21 23
@@ -304,24 +306,8 @@ static inline unsigned long clear_user(void __user *addr, unsigned long n)
304 return n; 306 return n;
305} 307}
306 308
307extern long __strlen_user(const char __user *); 309extern __must_check long strlen_user(const char __user *str);
308extern long __strnlen_user(const char __user *, long len); 310extern __must_check long strnlen_user(const char __user *str, long n);
309
310static inline long strlen_user(const char __user *str)
311{
312 if (!access_ok(VERIFY_READ, str, 0))
313 return 0;
314 else
315 return __strlen_user(str);
316}
317
318static inline long strnlen_user(const char __user *str, long len)
319{
320 if (!access_ok(VERIFY_READ, str, 0))
321 return 0;
322 else
323 return __strnlen_user(str, len);
324}
325 311
326#endif /* __ASSEMBLY__ */ 312#endif /* __ASSEMBLY__ */
327 313
diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
index dcdfb89cbf3f..7c831d848b4e 100644
--- a/arch/sparc/include/asm/uaccess_64.h
+++ b/arch/sparc/include/asm/uaccess_64.h
@@ -17,6 +17,8 @@
17 17
18#ifndef __ASSEMBLY__ 18#ifndef __ASSEMBLY__
19 19
20#include <asm/processor.h>
21
20/* 22/*
21 * Sparc64 is segmented, though more like the M68K than the I386. 23 * Sparc64 is segmented, though more like the M68K than the I386.
22 * We use the secondary ASI to address user memory, which references a 24 * We use the secondary ASI to address user memory, which references a
@@ -257,11 +259,9 @@ extern unsigned long __must_check __clear_user(void __user *, unsigned long);
257 259
258#define clear_user __clear_user 260#define clear_user __clear_user
259 261
260extern long __strlen_user(const char __user *); 262extern __must_check long strlen_user(const char __user *str);
261extern long __strnlen_user(const char __user *, long len); 263extern __must_check long strnlen_user(const char __user *str, long n);
262 264
263#define strlen_user __strlen_user
264#define strnlen_user __strnlen_user
265#define __copy_to_user_inatomic ___copy_to_user 265#define __copy_to_user_inatomic ___copy_to_user
266#define __copy_from_user_inatomic ___copy_from_user 266#define __copy_from_user_inatomic ___copy_from_user
267 267
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index 943d98dc4cdb..dff4096f3dec 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -10,7 +10,6 @@ lib-y += strlen.o
10lib-y += checksum_$(BITS).o 10lib-y += checksum_$(BITS).o
11lib-$(CONFIG_SPARC32) += blockops.o 11lib-$(CONFIG_SPARC32) += blockops.o
12lib-y += memscan_$(BITS).o memcmp.o strncmp_$(BITS).o 12lib-y += memscan_$(BITS).o memcmp.o strncmp_$(BITS).o
13lib-y += strlen_user_$(BITS).o
14lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o 13lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o
15lib-$(CONFIG_SPARC32) += copy_user.o locks.o 14lib-$(CONFIG_SPARC32) += copy_user.o locks.o
16lib-$(CONFIG_SPARC64) += atomic_64.o 15lib-$(CONFIG_SPARC64) += atomic_64.o
diff --git a/arch/sparc/lib/ksyms.c b/arch/sparc/lib/ksyms.c
index 6b278abdb63d..3b31218cafc6 100644
--- a/arch/sparc/lib/ksyms.c
+++ b/arch/sparc/lib/ksyms.c
@@ -15,8 +15,6 @@
15 15
16/* string functions */ 16/* string functions */
17EXPORT_SYMBOL(strlen); 17EXPORT_SYMBOL(strlen);
18EXPORT_SYMBOL(__strlen_user);
19EXPORT_SYMBOL(__strnlen_user);
20EXPORT_SYMBOL(strncmp); 18EXPORT_SYMBOL(strncmp);
21 19
22/* mem* functions */ 20/* mem* functions */
diff --git a/arch/sparc/lib/strlen_user_32.S b/arch/sparc/lib/strlen_user_32.S
deleted file mode 100644
index 8c8a371df3c9..000000000000
--- a/arch/sparc/lib/strlen_user_32.S
+++ /dev/null
@@ -1,109 +0,0 @@
1/* strlen_user.S: Sparc optimized strlen_user code
2 *
3 * Return length of string in userspace including terminating 0
4 * or 0 for error
5 *
6 * Copyright (C) 1991,1996 Free Software Foundation
7 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
8 * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
9 */
10
11#define LO_MAGIC 0x01010101
12#define HI_MAGIC 0x80808080
13
1410:
15 ldub [%o0], %o5
16 cmp %o5, 0
17 be 1f
18 add %o0, 1, %o0
19 andcc %o0, 3, %g0
20 be 4f
21 or %o4, %lo(HI_MAGIC), %o3
2211:
23 ldub [%o0], %o5
24 cmp %o5, 0
25 be 2f
26 add %o0, 1, %o0
27 andcc %o0, 3, %g0
28 be 5f
29 sethi %hi(LO_MAGIC), %o4
3012:
31 ldub [%o0], %o5
32 cmp %o5, 0
33 be 3f
34 add %o0, 1, %o0
35 b 13f
36 or %o4, %lo(LO_MAGIC), %o2
371:
38 retl
39 mov 1, %o0
402:
41 retl
42 mov 2, %o0
433:
44 retl
45 mov 3, %o0
46
47 .align 4
48 .global __strlen_user, __strnlen_user
49__strlen_user:
50 sethi %hi(32768), %o1
51__strnlen_user:
52 mov %o1, %g1
53 mov %o0, %o1
54 andcc %o0, 3, %g0
55 bne 10b
56 sethi %hi(HI_MAGIC), %o4
57 or %o4, %lo(HI_MAGIC), %o3
584:
59 sethi %hi(LO_MAGIC), %o4
605:
61 or %o4, %lo(LO_MAGIC), %o2
6213:
63 ld [%o0], %o5
642:
65 sub %o5, %o2, %o4
66 andcc %o4, %o3, %g0
67 bne 82f
68 add %o0, 4, %o0
69 sub %o0, %o1, %g2
7081: cmp %g2, %g1
71 blu 13b
72 mov %o0, %o4
73 ba,a 1f
74
75 /* Check every byte. */
7682: srl %o5, 24, %g5
77 andcc %g5, 0xff, %g0
78 be 1f
79 add %o0, -3, %o4
80 srl %o5, 16, %g5
81 andcc %g5, 0xff, %g0
82 be 1f
83 add %o4, 1, %o4
84 srl %o5, 8, %g5
85 andcc %g5, 0xff, %g0
86 be 1f
87 add %o4, 1, %o4
88 andcc %o5, 0xff, %g0
89 bne 81b
90 sub %o0, %o1, %g2
91
92 add %o4, 1, %o4
931:
94 retl
95 sub %o4, %o1, %o0
96
97 .section .fixup,#alloc,#execinstr
98 .align 4
999:
100 retl
101 clr %o0
102
103 .section __ex_table,#alloc
104 .align 4
105
106 .word 10b, 9b
107 .word 11b, 9b
108 .word 12b, 9b
109 .word 13b, 9b
diff --git a/arch/sparc/lib/strlen_user_64.S b/arch/sparc/lib/strlen_user_64.S
deleted file mode 100644
index c3df71fa4928..000000000000
--- a/arch/sparc/lib/strlen_user_64.S
+++ /dev/null
@@ -1,97 +0,0 @@
1/* strlen_user.S: Sparc64 optimized strlen_user code
2 *
3 * Return length of string in userspace including terminating 0
4 * or 0 for error
5 *
6 * Copyright (C) 1991,1996 Free Software Foundation
7 * Copyright (C) 1996,1999 David S. Miller (davem@redhat.com)
8 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
9 */
10
11#include <linux/linkage.h>
12#include <asm/asi.h>
13
14#define LO_MAGIC 0x01010101
15#define HI_MAGIC 0x80808080
16
17 .align 4
18ENTRY(__strlen_user)
19 sethi %hi(32768), %o1
20ENTRY(__strnlen_user)
21 mov %o1, %g1
22 mov %o0, %o1
23 andcc %o0, 3, %g0
24 be,pt %icc, 9f
25 sethi %hi(HI_MAGIC), %o4
2610: lduba [%o0] %asi, %o5
27 brz,pn %o5, 21f
28 add %o0, 1, %o0
29 andcc %o0, 3, %g0
30 be,pn %icc, 4f
31 or %o4, %lo(HI_MAGIC), %o3
3211: lduba [%o0] %asi, %o5
33 brz,pn %o5, 22f
34 add %o0, 1, %o0
35 andcc %o0, 3, %g0
36 be,pt %icc, 13f
37 srl %o3, 7, %o2
3812: lduba [%o0] %asi, %o5
39 brz,pn %o5, 23f
40 add %o0, 1, %o0
41 ba,pt %icc, 2f
4215: lda [%o0] %asi, %o5
439: or %o4, %lo(HI_MAGIC), %o3
444: srl %o3, 7, %o2
4513: lda [%o0] %asi, %o5
462: sub %o5, %o2, %o4
47 andcc %o4, %o3, %g0
48 bne,pn %icc, 82f
49 add %o0, 4, %o0
50 sub %o0, %o1, %g2
5181: cmp %g2, %g1
52 blu,pt %icc, 13b
53 mov %o0, %o4
54 ba,a,pt %xcc, 1f
55
56 /* Check every byte. */
5782: srl %o5, 24, %g7
58 andcc %g7, 0xff, %g0
59 be,pn %icc, 1f
60 add %o0, -3, %o4
61 srl %o5, 16, %g7
62 andcc %g7, 0xff, %g0
63 be,pn %icc, 1f
64 add %o4, 1, %o4
65 srl %o5, 8, %g7
66 andcc %g7, 0xff, %g0
67 be,pn %icc, 1f
68 add %o4, 1, %o4
69 andcc %o5, 0xff, %g0
70 bne,pt %icc, 81b
71 sub %o0, %o1, %g2
72 add %o4, 1, %o4
731: retl
74 sub %o4, %o1, %o0
7521: retl
76 mov 1, %o0
7722: retl
78 mov 2, %o0
7923: retl
80 mov 3, %o0
81ENDPROC(__strlen_user)
82ENDPROC(__strnlen_user)
83
84 .section .fixup,#alloc,#execinstr
85 .align 4
8630:
87 retl
88 clr %o0
89
90 .section __ex_table,"a"
91 .align 4
92
93 .word 10b, 30b
94 .word 11b, 30b
95 .word 12b, 30b
96 .word 15b, 30b
97 .word 13b, 30b
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 81c3e8be789a..d700811785ea 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -93,6 +93,8 @@ config X86
93 select GENERIC_CLOCKEVENTS_BROADCAST if X86_64 || (X86_32 && X86_LOCAL_APIC) 93 select GENERIC_CLOCKEVENTS_BROADCAST if X86_64 || (X86_32 && X86_LOCAL_APIC)
94 select GENERIC_TIME_VSYSCALL if X86_64 94 select GENERIC_TIME_VSYSCALL if X86_64
95 select KTIME_SCALAR if X86_32 95 select KTIME_SCALAR if X86_32
96 select GENERIC_STRNCPY_FROM_USER
97 select GENERIC_STRNLEN_USER
96 98
97config INSTRUCTION_DECODER 99config INSTRUCTION_DECODER
98 def_bool (KPROBES || PERF_EVENTS || UPROBES) 100 def_bool (KPROBES || PERF_EVENTS || UPROBES)
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 851fe0dc13bc..04cd6882308e 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -32,6 +32,7 @@
32 32
33#define segment_eq(a, b) ((a).seg == (b).seg) 33#define segment_eq(a, b) ((a).seg == (b).seg)
34 34
35#define user_addr_max() (current_thread_info()->addr_limit.seg)
35#define __addr_ok(addr) \ 36#define __addr_ok(addr) \
36 ((unsigned long __force)(addr) < \ 37 ((unsigned long __force)(addr) < \
37 (current_thread_info()->addr_limit.seg)) 38 (current_thread_info()->addr_limit.seg))
@@ -565,6 +566,9 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
565extern __must_check long 566extern __must_check long
566strncpy_from_user(char *dst, const char __user *src, long count); 567strncpy_from_user(char *dst, const char __user *src, long count);
567 568
569extern __must_check long strlen_user(const char __user *str);
570extern __must_check long strnlen_user(const char __user *str, long n);
571
568/* 572/*
569 * movsl can be slow when source and dest are not both 8-byte aligned 573 * movsl can be slow when source and dest are not both 8-byte aligned
570 */ 574 */
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index 8084bc73b18c..576e39bca6ad 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -213,23 +213,6 @@ static inline unsigned long __must_check copy_from_user(void *to,
213 return n; 213 return n;
214} 214}
215 215
216/**
217 * strlen_user: - Get the size of a string in user space.
218 * @str: The string to measure.
219 *
220 * Context: User context only. This function may sleep.
221 *
222 * Get the size of a NUL-terminated string in user space.
223 *
224 * Returns the size of the string INCLUDING the terminating NUL.
225 * On exception, returns 0.
226 *
227 * If there is a limit on the length of a valid string, you may wish to
228 * consider using strnlen_user() instead.
229 */
230#define strlen_user(str) strnlen_user(str, LONG_MAX)
231
232long strnlen_user(const char __user *str, long n);
233unsigned long __must_check clear_user(void __user *mem, unsigned long len); 216unsigned long __must_check clear_user(void __user *mem, unsigned long len);
234unsigned long __must_check __clear_user(void __user *mem, unsigned long len); 217unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
235 218
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index fcd4b6f3ef02..8e796fbbf9c6 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -208,9 +208,6 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
208 } 208 }
209} 209}
210 210
211__must_check long strnlen_user(const char __user *str, long n);
212__must_check long __strnlen_user(const char __user *str, long n);
213__must_check long strlen_user(const char __user *str);
214__must_check unsigned long clear_user(void __user *mem, unsigned long len); 211__must_check unsigned long clear_user(void __user *mem, unsigned long len);
215__must_check unsigned long __clear_user(void __user *mem, unsigned long len); 212__must_check unsigned long __clear_user(void __user *mem, unsigned long len);
216 213
diff --git a/arch/x86/include/asm/word-at-a-time.h b/arch/x86/include/asm/word-at-a-time.h
index ae03facfadd6..5b238981542a 100644
--- a/arch/x86/include/asm/word-at-a-time.h
+++ b/arch/x86/include/asm/word-at-a-time.h
@@ -10,6 +10,11 @@
10 * bit count instruction, that might be better than the multiply 10 * bit count instruction, that might be better than the multiply
11 * and shift, for example. 11 * and shift, for example.
12 */ 12 */
13struct word_at_a_time {
14 const unsigned long one_bits, high_bits;
15};
16
17#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
13 18
14#ifdef CONFIG_64BIT 19#ifdef CONFIG_64BIT
15 20
@@ -37,10 +42,31 @@ static inline long count_masked_bytes(long mask)
37 42
38#endif 43#endif
39 44
40/* Return the high bit set in the first byte that is a zero */ 45/* Return nonzero if it has a zero */
41static inline unsigned long has_zero(unsigned long a) 46static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c)
47{
48 unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
49 *bits = mask;
50 return mask;
51}
52
53static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c)
54{
55 return bits;
56}
57
58static inline unsigned long create_zero_mask(unsigned long bits)
59{
60 bits = (bits - 1) & ~bits;
61 return bits >> 7;
62}
63
64/* The mask we created is directly usable as a bytemask */
65#define zero_bytemask(mask) (mask)
66
67static inline unsigned long find_zero(unsigned long mask)
42{ 68{
43 return ((a - REPEAT_BYTE(0x01)) & ~a) & REPEAT_BYTE(0x80); 69 return count_masked_bytes(mask);
44} 70}
45 71
46/* 72/*
diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c
index 2e4e4b02c37a..f61ee67ec00f 100644
--- a/arch/x86/lib/usercopy.c
+++ b/arch/x86/lib/usercopy.c
@@ -43,100 +43,3 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
43 return len; 43 return len;
44} 44}
45EXPORT_SYMBOL_GPL(copy_from_user_nmi); 45EXPORT_SYMBOL_GPL(copy_from_user_nmi);
46
47/*
48 * Do a strncpy, return length of string without final '\0'.
49 * 'count' is the user-supplied count (return 'count' if we
50 * hit it), 'max' is the address space maximum (and we return
51 * -EFAULT if we hit it).
52 */
53static inline long do_strncpy_from_user(char *dst, const char __user *src, long count, unsigned long max)
54{
55 long res = 0;
56
57 /*
58 * Truncate 'max' to the user-specified limit, so that
59 * we only have one limit we need to check in the loop
60 */
61 if (max > count)
62 max = count;
63
64 while (max >= sizeof(unsigned long)) {
65 unsigned long c, mask;
66
67 /* Fall back to byte-at-a-time if we get a page fault */
68 if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
69 break;
70 mask = has_zero(c);
71 if (mask) {
72 mask = (mask - 1) & ~mask;
73 mask >>= 7;
74 *(unsigned long *)(dst+res) = c & mask;
75 return res + count_masked_bytes(mask);
76 }
77 *(unsigned long *)(dst+res) = c;
78 res += sizeof(unsigned long);
79 max -= sizeof(unsigned long);
80 }
81
82 while (max) {
83 char c;
84
85 if (unlikely(__get_user(c,src+res)))
86 return -EFAULT;
87 dst[res] = c;
88 if (!c)
89 return res;
90 res++;
91 max--;
92 }
93
94 /*
95 * Uhhuh. We hit 'max'. But was that the user-specified maximum
96 * too? If so, that's ok - we got as much as the user asked for.
97 */
98 if (res >= count)
99 return res;
100
101 /*
102 * Nope: we hit the address space limit, and we still had more
103 * characters the caller would have wanted. That's an EFAULT.
104 */
105 return -EFAULT;
106}
107
108/**
109 * strncpy_from_user: - Copy a NUL terminated string from userspace.
110 * @dst: Destination address, in kernel space. This buffer must be at
111 * least @count bytes long.
112 * @src: Source address, in user space.
113 * @count: Maximum number of bytes to copy, including the trailing NUL.
114 *
115 * Copies a NUL-terminated string from userspace to kernel space.
116 *
117 * On success, returns the length of the string (not including the trailing
118 * NUL).
119 *
120 * If access to userspace fails, returns -EFAULT (some data may have been
121 * copied).
122 *
123 * If @count is smaller than the length of the string, copies @count bytes
124 * and returns @count.
125 */
126long
127strncpy_from_user(char *dst, const char __user *src, long count)
128{
129 unsigned long max_addr, src_addr;
130
131 if (unlikely(count <= 0))
132 return 0;
133
134 max_addr = current_thread_info()->addr_limit.seg;
135 src_addr = (unsigned long)src;
136 if (likely(src_addr < max_addr)) {
137 unsigned long max = max_addr - src_addr;
138 return do_strncpy_from_user(dst, src, count, max);
139 }
140 return -EFAULT;
141}
142EXPORT_SYMBOL(strncpy_from_user);
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 883b216c60b2..1781b2f950e2 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -95,47 +95,6 @@ __clear_user(void __user *to, unsigned long n)
95} 95}
96EXPORT_SYMBOL(__clear_user); 96EXPORT_SYMBOL(__clear_user);
97 97
98/**
99 * strnlen_user: - Get the size of a string in user space.
100 * @s: The string to measure.
101 * @n: The maximum valid length
102 *
103 * Get the size of a NUL-terminated string in user space.
104 *
105 * Returns the size of the string INCLUDING the terminating NUL.
106 * On exception, returns 0.
107 * If the string is too long, returns a value greater than @n.
108 */
109long strnlen_user(const char __user *s, long n)
110{
111 unsigned long mask = -__addr_ok(s);
112 unsigned long res, tmp;
113
114 might_fault();
115
116 __asm__ __volatile__(
117 " testl %0, %0\n"
118 " jz 3f\n"
119 " andl %0,%%ecx\n"
120 "0: repne; scasb\n"
121 " setne %%al\n"
122 " subl %%ecx,%0\n"
123 " addl %0,%%eax\n"
124 "1:\n"
125 ".section .fixup,\"ax\"\n"
126 "2: xorl %%eax,%%eax\n"
127 " jmp 1b\n"
128 "3: movb $1,%%al\n"
129 " jmp 1b\n"
130 ".previous\n"
131 _ASM_EXTABLE(0b,2b)
132 :"=&r" (n), "=&D" (s), "=&a" (res), "=&c" (tmp)
133 :"0" (n), "1" (s), "2" (0), "3" (mask)
134 :"cc");
135 return res & mask;
136}
137EXPORT_SYMBOL(strnlen_user);
138
139#ifdef CONFIG_X86_INTEL_USERCOPY 98#ifdef CONFIG_X86_INTEL_USERCOPY
140static unsigned long 99static unsigned long
141__copy_user_intel(void __user *to, const void *from, unsigned long size) 100__copy_user_intel(void __user *to, const void *from, unsigned long size)
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 0d0326f388c0..e5b130bc2d0e 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -52,54 +52,6 @@ unsigned long clear_user(void __user *to, unsigned long n)
52} 52}
53EXPORT_SYMBOL(clear_user); 53EXPORT_SYMBOL(clear_user);
54 54
55/*
56 * Return the size of a string (including the ending 0)
57 *
58 * Return 0 on exception, a value greater than N if too long
59 */
60
61long __strnlen_user(const char __user *s, long n)
62{
63 long res = 0;
64 char c;
65
66 while (1) {
67 if (res>n)
68 return n+1;
69 if (__get_user(c, s))
70 return 0;
71 if (!c)
72 return res+1;
73 res++;
74 s++;
75 }
76}
77EXPORT_SYMBOL(__strnlen_user);
78
79long strnlen_user(const char __user *s, long n)
80{
81 if (!access_ok(VERIFY_READ, s, 1))
82 return 0;
83 return __strnlen_user(s, n);
84}
85EXPORT_SYMBOL(strnlen_user);
86
87long strlen_user(const char __user *s)
88{
89 long res = 0;
90 char c;
91
92 for (;;) {
93 if (get_user(c, s))
94 return 0;
95 if (!c)
96 return res+1;
97 res++;
98 s++;
99 }
100}
101EXPORT_SYMBOL(strlen_user);
102
103unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len) 55unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len)
104{ 56{
105 if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len)) { 57 if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len)) {
diff --git a/fs/namei.c b/fs/namei.c
index 93ff12b1a1de..c651f02c9fec 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -1452,7 +1452,8 @@ EXPORT_SYMBOL(full_name_hash);
1452 */ 1452 */
1453static inline unsigned long hash_name(const char *name, unsigned int *hashp) 1453static inline unsigned long hash_name(const char *name, unsigned int *hashp)
1454{ 1454{
1455 unsigned long a, mask, hash, len; 1455 unsigned long a, b, adata, bdata, mask, hash, len;
1456 const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
1456 1457
1457 hash = a = 0; 1458 hash = a = 0;
1458 len = -sizeof(unsigned long); 1459 len = -sizeof(unsigned long);
@@ -1460,17 +1461,18 @@ static inline unsigned long hash_name(const char *name, unsigned int *hashp)
1460 hash = (hash + a) * 9; 1461 hash = (hash + a) * 9;
1461 len += sizeof(unsigned long); 1462 len += sizeof(unsigned long);
1462 a = load_unaligned_zeropad(name+len); 1463 a = load_unaligned_zeropad(name+len);
1463 /* Do we have any NUL or '/' bytes in this word? */ 1464 b = a ^ REPEAT_BYTE('/');
1464 mask = has_zero(a) | has_zero(a ^ REPEAT_BYTE('/')); 1465 } while (!(has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants)));
1465 } while (!mask); 1466
1466 1467 adata = prep_zero_mask(a, adata, &constants);
1467 /* The mask *below* the first high bit set */ 1468 bdata = prep_zero_mask(b, bdata, &constants);
1468 mask = (mask - 1) & ~mask; 1469
1469 mask >>= 7; 1470 mask = create_zero_mask(adata | bdata);
1470 hash += a & mask; 1471
1472 hash += a & zero_bytemask(mask);
1471 *hashp = fold_hash(hash); 1473 *hashp = fold_hash(hash);
1472 1474
1473 return len + count_masked_bytes(mask); 1475 return len + find_zero(mask);
1474} 1476}
1475 1477
1476#else 1478#else
diff --git a/include/asm-generic/word-at-a-time.h b/include/asm-generic/word-at-a-time.h
new file mode 100644
index 000000000000..3f21f1b72e45
--- /dev/null
+++ b/include/asm-generic/word-at-a-time.h
@@ -0,0 +1,52 @@
1#ifndef _ASM_WORD_AT_A_TIME_H
2#define _ASM_WORD_AT_A_TIME_H
3
4/*
5 * This says "generic", but it's actually big-endian only.
6 * Little-endian can use more efficient versions of these
7 * interfaces, see for example
8 * arch/x86/include/asm/word-at-a-time.h
9 * for those.
10 */
11
12#include <linux/kernel.h>
13
14struct word_at_a_time {
15 const unsigned long high_bits, low_bits;
16};
17
18#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0xfe) + 1, REPEAT_BYTE(0x7f) }
19
20/* Bit set in the bytes that have a zero */
21static inline long prep_zero_mask(unsigned long val, unsigned long rhs, const struct word_at_a_time *c)
22{
23 unsigned long mask = (val & c->low_bits) + c->low_bits;
24 return ~(mask | rhs);
25}
26
27#define create_zero_mask(mask) (mask)
28
29static inline long find_zero(unsigned long mask)
30{
31 long byte = 0;
32#ifdef CONFIG_64BIT
33 if (mask >> 32)
34 mask >>= 32;
35 else
36 byte = 4;
37#endif
38 if (mask >> 16)
39 mask >>= 16;
40 else
41 byte += 2;
42 return (mask >> 8) ? byte : byte + 1;
43}
44
45static inline bool has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c)
46{
47 unsigned long rhs = val | c->low_bits;
48 *data = rhs;
49 return (val + c->high_bits) & ~rhs;
50}
51
52#endif /* _ASM_WORD_AT_A_TIME_H */
diff --git a/lib/Kconfig b/lib/Kconfig
index 3e63af089082..a9e15403434e 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -19,6 +19,9 @@ config RATIONAL
19config GENERIC_STRNCPY_FROM_USER 19config GENERIC_STRNCPY_FROM_USER
20 bool 20 bool
21 21
22config GENERIC_STRNLEN_USER
23 bool
24
22config GENERIC_FIND_FIRST_BIT 25config GENERIC_FIND_FIRST_BIT
23 bool 26 bool
24 27
diff --git a/lib/Makefile b/lib/Makefile
index 2a1d7f9d39a2..8c31a0cb75e9 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -126,6 +126,7 @@ obj-$(CONFIG_CLZ_TAB) += clz_tab.o
126obj-$(CONFIG_DDR) += jedec_ddr_data.o 126obj-$(CONFIG_DDR) += jedec_ddr_data.o
127 127
128obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o 128obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o
129obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o
129 130
130obj-$(CONFIG_STMP_DEVICE) += stmp_device.o 131obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
131 132
diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c
index c4c09b0e96ba..bb2b201d6ad0 100644
--- a/lib/strncpy_from_user.c
+++ b/lib/strncpy_from_user.c
@@ -4,37 +4,7 @@
4#include <linux/errno.h> 4#include <linux/errno.h>
5 5
6#include <asm/byteorder.h> 6#include <asm/byteorder.h>
7 7#include <asm/word-at-a-time.h>
8static inline long find_zero(unsigned long mask)
9{
10 long byte = 0;
11
12#ifdef __BIG_ENDIAN
13#ifdef CONFIG_64BIT
14 if (mask >> 32)
15 mask >>= 32;
16 else
17 byte = 4;
18#endif
19 if (mask >> 16)
20 mask >>= 16;
21 else
22 byte += 2;
23 return (mask >> 8) ? byte : byte + 1;
24#else
25#ifdef CONFIG_64BIT
26 if (!((unsigned int) mask)) {
27 mask >>= 32;
28 byte = 4;
29 }
30#endif
31 if (!(mask & 0xffff)) {
32 mask >>= 16;
33 byte += 2;
34 }
35 return (mask & 0xff) ? byte : byte + 1;
36#endif
37}
38 8
39#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 9#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
40#define IS_UNALIGNED(src, dst) 0 10#define IS_UNALIGNED(src, dst) 0
@@ -51,8 +21,7 @@ static inline long find_zero(unsigned long mask)
51 */ 21 */
52static inline long do_strncpy_from_user(char *dst, const char __user *src, long count, unsigned long max) 22static inline long do_strncpy_from_user(char *dst, const char __user *src, long count, unsigned long max)
53{ 23{
54 const unsigned long high_bits = REPEAT_BYTE(0xfe) + 1; 24 const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
55 const unsigned long low_bits = REPEAT_BYTE(0x7f);
56 long res = 0; 25 long res = 0;
57 26
58 /* 27 /*
@@ -66,18 +35,16 @@ static inline long do_strncpy_from_user(char *dst, const char __user *src, long
66 goto byte_at_a_time; 35 goto byte_at_a_time;
67 36
68 while (max >= sizeof(unsigned long)) { 37 while (max >= sizeof(unsigned long)) {
69 unsigned long c, v, rhs; 38 unsigned long c, data;
70 39
71 /* Fall back to byte-at-a-time if we get a page fault */ 40 /* Fall back to byte-at-a-time if we get a page fault */
72 if (unlikely(__get_user(c,(unsigned long __user *)(src+res)))) 41 if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
73 break; 42 break;
74 rhs = c | low_bits;
75 v = (c + high_bits) & ~rhs;
76 *(unsigned long *)(dst+res) = c; 43 *(unsigned long *)(dst+res) = c;
77 if (v) { 44 if (has_zero(c, &data, &constants)) {
78 v = (c & low_bits) + low_bits; 45 data = prep_zero_mask(c, data, &constants);
79 v = ~(v | rhs); 46 data = create_zero_mask(data);
80 return res + find_zero(v); 47 return res + find_zero(data);
81 } 48 }
82 res += sizeof(unsigned long); 49 res += sizeof(unsigned long);
83 max -= sizeof(unsigned long); 50 max -= sizeof(unsigned long);
diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c
new file mode 100644
index 000000000000..90900ecfeb54
--- /dev/null
+++ b/lib/strnlen_user.c
@@ -0,0 +1,138 @@
1#include <linux/kernel.h>
2#include <linux/export.h>
3#include <linux/uaccess.h>
4
5#include <asm/word-at-a-time.h>
6
7/* Set bits in the first 'n' bytes when loaded from memory */
8#ifdef __LITTLE_ENDIAN
9# define aligned_byte_mask(n) ((1ul << 8*(n))-1)
10#else
11# define aligned_byte_mask(n) (~0xfful << 8*(7-(n)))
12#endif
13
14/*
15 * Do a strnlen, return length of string *with* final '\0'.
16 * 'count' is the user-supplied count, while 'max' is the
17 * address space maximum.
18 *
19 * Return 0 for exceptions (which includes hitting the address
20 * space maximum), or 'count+1' if hitting the user-supplied
21 * maximum count.
22 *
23 * NOTE! We can sometimes overshoot the user-supplied maximum
24 * if it fits in a aligned 'long'. The caller needs to check
25 * the return value against "> max".
26 */
27static inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max)
28{
29 const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
30 long align, res = 0;
31 unsigned long c;
32
33 /*
34 * Truncate 'max' to the user-specified limit, so that
35 * we only have one limit we need to check in the loop
36 */
37 if (max > count)
38 max = count;
39
40 /*
41 * Do everything aligned. But that means that we
42 * need to also expand the maximum..
43 */
44 align = (sizeof(long) - 1) & (unsigned long)src;
45 src -= align;
46 max += align;
47
48 if (unlikely(__get_user(c,(unsigned long __user *)src)))
49 return 0;
50 c |= aligned_byte_mask(align);
51
52 for (;;) {
53 unsigned long data;
54 if (has_zero(c, &data, &constants)) {
55 data = prep_zero_mask(c, data, &constants);
56 data = create_zero_mask(data);
57 return res + find_zero(data) + 1 - align;
58 }
59 res += sizeof(unsigned long);
60 if (unlikely(max < sizeof(unsigned long)))
61 break;
62 max -= sizeof(unsigned long);
63 if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
64 return 0;
65 }
66 res -= align;
67
68 /*
69 * Uhhuh. We hit 'max'. But was that the user-specified maximum
70 * too? If so, return the marker for "too long".
71 */
72 if (res >= count)
73 return count+1;
74
75 /*
76 * Nope: we hit the address space limit, and we still had more
77 * characters the caller would have wanted. That's 0.
78 */
79 return 0;
80}
81
82/**
83 * strnlen_user: - Get the size of a user string INCLUDING final NUL.
84 * @str: The string to measure.
85 * @count: Maximum count (including NUL character)
86 *
87 * Context: User context only. This function may sleep.
88 *
89 * Get the size of a NUL-terminated string in user space.
90 *
91 * Returns the size of the string INCLUDING the terminating NUL.
92 * If the string is too long, returns 'count+1'.
93 * On exception (or invalid count), returns 0.
94 */
95long strnlen_user(const char __user *str, long count)
96{
97 unsigned long max_addr, src_addr;
98
99 if (unlikely(count <= 0))
100 return 0;
101
102 max_addr = user_addr_max();
103 src_addr = (unsigned long)str;
104 if (likely(src_addr < max_addr)) {
105 unsigned long max = max_addr - src_addr;
106 return do_strnlen_user(str, count, max);
107 }
108 return 0;
109}
110EXPORT_SYMBOL(strnlen_user);
111
112/**
113 * strlen_user: - Get the size of a user string INCLUDING final NUL.
114 * @str: The string to measure.
115 *
116 * Context: User context only. This function may sleep.
117 *
118 * Get the size of a NUL-terminated string in user space.
119 *
120 * Returns the size of the string INCLUDING the terminating NUL.
121 * On exception, returns 0.
122 *
123 * If there is a limit on the length of a valid string, you may wish to
124 * consider using strnlen_user() instead.
125 */
126long strlen_user(const char __user *str)
127{
128 unsigned long max_addr, src_addr;
129
130 max_addr = user_addr_max();
131 src_addr = (unsigned long)str;
132 if (likely(src_addr < max_addr)) {
133 unsigned long max = max_addr - src_addr;
134 return do_strnlen_user(str, ~0ul, max);
135 }
136 return 0;
137}
138EXPORT_SYMBOL(strlen_user);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-23 06:18:41 -0500