aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-12-17 13:05:19 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2015-12-17 13:05:19 -0500
commit9fd4470ff4974c41b1db43c3b355b9085af9c12a (patch)
tree68a9c479852bf8e1603262f32bfda48e2f6409a9
parent5b24a7a2aa2040c8c50c3b71122901d01661ff78 (diff)
Use the new batched user accesses in generic user string handling
This converts the generic user string functions to use the batched user access functions. It makes a big difference on Skylake, which is the first x86 microarchitecture to implement SMAP. The STAC/CLAC instructions are not very fast, and doing them for each access inside the loop that copies strings from user space (which is what the pathname handling does for every pathname the kernel uses, for example) is very inefficient. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r--lib/strncpy_from_user.c11
-rw-r--r--lib/strnlen_user.c18
2 files changed, 22 insertions, 7 deletions
diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c
index e0af6ff73d14..33840324138c 100644
--- a/lib/strncpy_from_user.c
+++ b/lib/strncpy_from_user.c
@@ -39,7 +39,7 @@ static inline long do_strncpy_from_user(char *dst, const char __user *src, long
39 unsigned long c, data; 39 unsigned long c, data;
40 40
41 /* Fall back to byte-at-a-time if we get a page fault */ 41 /* Fall back to byte-at-a-time if we get a page fault */
42 if (unlikely(__get_user(c,(unsigned long __user *)(src+res)))) 42 if (unlikely(unsafe_get_user(c,(unsigned long __user *)(src+res))))
43 break; 43 break;
44 *(unsigned long *)(dst+res) = c; 44 *(unsigned long *)(dst+res) = c;
45 if (has_zero(c, &data, &constants)) { 45 if (has_zero(c, &data, &constants)) {
@@ -55,7 +55,7 @@ byte_at_a_time:
55 while (max) { 55 while (max) {
56 char c; 56 char c;
57 57
58 if (unlikely(__get_user(c,src+res))) 58 if (unlikely(unsafe_get_user(c,src+res)))
59 return -EFAULT; 59 return -EFAULT;
60 dst[res] = c; 60 dst[res] = c;
61 if (!c) 61 if (!c)
@@ -107,7 +107,12 @@ long strncpy_from_user(char *dst, const char __user *src, long count)
107 src_addr = (unsigned long)src; 107 src_addr = (unsigned long)src;
108 if (likely(src_addr < max_addr)) { 108 if (likely(src_addr < max_addr)) {
109 unsigned long max = max_addr - src_addr; 109 unsigned long max = max_addr - src_addr;
110 return do_strncpy_from_user(dst, src, count, max); 110 long retval;
111
112 user_access_begin();
113 retval = do_strncpy_from_user(dst, src, count, max);
114 user_access_end();
115 return retval;
111 } 116 }
112 return -EFAULT; 117 return -EFAULT;
113} 118}
diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c
index 3a5f2b366d84..2625943625d7 100644
--- a/lib/strnlen_user.c
+++ b/lib/strnlen_user.c
@@ -45,7 +45,7 @@ static inline long do_strnlen_user(const char __user *src, unsigned long count,
45 src -= align; 45 src -= align;
46 max += align; 46 max += align;
47 47
48 if (unlikely(__get_user(c,(unsigned long __user *)src))) 48 if (unlikely(unsafe_get_user(c,(unsigned long __user *)src)))
49 return 0; 49 return 0;
50 c |= aligned_byte_mask(align); 50 c |= aligned_byte_mask(align);
51 51
@@ -61,7 +61,7 @@ static inline long do_strnlen_user(const char __user *src, unsigned long count,
61 if (unlikely(max <= sizeof(unsigned long))) 61 if (unlikely(max <= sizeof(unsigned long)))
62 break; 62 break;
63 max -= sizeof(unsigned long); 63 max -= sizeof(unsigned long);
64 if (unlikely(__get_user(c,(unsigned long __user *)(src+res)))) 64 if (unlikely(unsafe_get_user(c,(unsigned long __user *)(src+res))))
65 return 0; 65 return 0;
66 } 66 }
67 res -= align; 67 res -= align;
@@ -112,7 +112,12 @@ long strnlen_user(const char __user *str, long count)
112 src_addr = (unsigned long)str; 112 src_addr = (unsigned long)str;
113 if (likely(src_addr < max_addr)) { 113 if (likely(src_addr < max_addr)) {
114 unsigned long max = max_addr - src_addr; 114 unsigned long max = max_addr - src_addr;
115 return do_strnlen_user(str, count, max); 115 long retval;
116
117 user_access_begin();
118 retval = do_strnlen_user(str, count, max);
119 user_access_end();
120 return retval;
116 } 121 }
117 return 0; 122 return 0;
118} 123}
@@ -141,7 +146,12 @@ long strlen_user(const char __user *str)
141 src_addr = (unsigned long)str; 146 src_addr = (unsigned long)str;
142 if (likely(src_addr < max_addr)) { 147 if (likely(src_addr < max_addr)) {
143 unsigned long max = max_addr - src_addr; 148 unsigned long max = max_addr - src_addr;
144 return do_strnlen_user(str, ~0ul, max); 149 long retval;
150
151 user_access_begin();
152 retval = do_strnlen_user(str, ~0ul, max);
153 user_access_end();
154 return retval;
145 } 155 }
146 return 0; 156 return 0;
147} 157}