diff options
-rw-r--r-- | arch/x86/include/asm/uaccess_32.h | 4 | ||||
-rw-r--r-- | arch/x86/include/asm/uaccess_64.h | 25 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_gem.c | 2 | ||||
-rw-r--r-- | include/linux/uaccess.h | 4 | ||||
-rw-r--r-- | mm/filemap.c | 11 | ||||
-rw-r--r-- | mm/filemap_xip.c | 2 |
6 files changed, 17 insertions, 31 deletions
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h index a0ba61386972..5e06259e90e5 100644 --- a/arch/x86/include/asm/uaccess_32.h +++ b/arch/x86/include/asm/uaccess_32.h | |||
@@ -157,7 +157,7 @@ __copy_from_user(void *to, const void __user *from, unsigned long n) | |||
157 | } | 157 | } |
158 | 158 | ||
159 | static __always_inline unsigned long __copy_from_user_nocache(void *to, | 159 | static __always_inline unsigned long __copy_from_user_nocache(void *to, |
160 | const void __user *from, unsigned long n, unsigned long total) | 160 | const void __user *from, unsigned long n) |
161 | { | 161 | { |
162 | might_fault(); | 162 | might_fault(); |
163 | if (__builtin_constant_p(n)) { | 163 | if (__builtin_constant_p(n)) { |
@@ -180,7 +180,7 @@ static __always_inline unsigned long __copy_from_user_nocache(void *to, | |||
180 | 180 | ||
181 | static __always_inline unsigned long | 181 | static __always_inline unsigned long |
182 | __copy_from_user_inatomic_nocache(void *to, const void __user *from, | 182 | __copy_from_user_inatomic_nocache(void *to, const void __user *from, |
183 | unsigned long n, unsigned long total) | 183 | unsigned long n) |
184 | { | 184 | { |
185 | return __copy_from_user_ll_nocache_nozero(to, from, n); | 185 | return __copy_from_user_ll_nocache_nozero(to, from, n); |
186 | } | 186 | } |
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index dcaa0404cf7b..8cc687326eb8 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h | |||
@@ -188,29 +188,18 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) | |||
188 | extern long __copy_user_nocache(void *dst, const void __user *src, | 188 | extern long __copy_user_nocache(void *dst, const void __user *src, |
189 | unsigned size, int zerorest); | 189 | unsigned size, int zerorest); |
190 | 190 | ||
191 | static inline int __copy_from_user_nocache(void *dst, const void __user *src, | 191 | static inline int |
192 | unsigned size, unsigned long total) | 192 | __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) |
193 | { | 193 | { |
194 | might_sleep(); | 194 | might_sleep(); |
195 | /* | 195 | return __copy_user_nocache(dst, src, size, 1); |
196 | * In practice this limit means that large file write()s | ||
197 | * which get chunked to 4K copies get handled via | ||
198 | * non-temporal stores here. Smaller writes get handled | ||
199 | * via regular __copy_from_user(): | ||
200 | */ | ||
201 | if (likely(total >= PAGE_SIZE)) | ||
202 | return __copy_user_nocache(dst, src, size, 1); | ||
203 | else | ||
204 | return __copy_from_user(dst, src, size); | ||
205 | } | 196 | } |
206 | 197 | ||
207 | static inline int __copy_from_user_inatomic_nocache(void *dst, | 198 | static inline int |
208 | const void __user *src, unsigned size, unsigned total) | 199 | __copy_from_user_inatomic_nocache(void *dst, const void __user *src, |
200 | unsigned size) | ||
209 | { | 201 | { |
210 | if (likely(total >= PAGE_SIZE)) | 202 | return __copy_user_nocache(dst, src, size, 0); |
211 | return __copy_user_nocache(dst, src, size, 0); | ||
212 | else | ||
213 | return __copy_from_user_inatomic(dst, src, size); | ||
214 | } | 203 | } |
215 | 204 | ||
216 | unsigned long | 205 | unsigned long |
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 6b209db8370d..818576654092 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c | |||
@@ -215,7 +215,7 @@ fast_user_write(struct io_mapping *mapping, | |||
215 | 215 | ||
216 | vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); | 216 | vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); |
217 | unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset, | 217 | unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset, |
218 | user_data, length, length); | 218 | user_data, length); |
219 | io_mapping_unmap_atomic(vaddr_atomic); | 219 | io_mapping_unmap_atomic(vaddr_atomic); |
220 | if (unwritten) | 220 | if (unwritten) |
221 | return -EFAULT; | 221 | return -EFAULT; |
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 6f3c603b0d67..6b58367d145e 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h | |||
@@ -41,13 +41,13 @@ static inline void pagefault_enable(void) | |||
41 | #ifndef ARCH_HAS_NOCACHE_UACCESS | 41 | #ifndef ARCH_HAS_NOCACHE_UACCESS |
42 | 42 | ||
43 | static inline unsigned long __copy_from_user_inatomic_nocache(void *to, | 43 | static inline unsigned long __copy_from_user_inatomic_nocache(void *to, |
44 | const void __user *from, unsigned long n, unsigned long total) | 44 | const void __user *from, unsigned long n) |
45 | { | 45 | { |
46 | return __copy_from_user_inatomic(to, from, n); | 46 | return __copy_from_user_inatomic(to, from, n); |
47 | } | 47 | } |
48 | 48 | ||
49 | static inline unsigned long __copy_from_user_nocache(void *to, | 49 | static inline unsigned long __copy_from_user_nocache(void *to, |
50 | const void __user *from, unsigned long n, unsigned long total) | 50 | const void __user *from, unsigned long n) |
51 | { | 51 | { |
52 | return __copy_from_user(to, from, n); | 52 | return __copy_from_user(to, from, n); |
53 | } | 53 | } |
diff --git a/mm/filemap.c b/mm/filemap.c index 60fd56772cc6..126d3973b3d1 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
@@ -1816,14 +1816,14 @@ EXPORT_SYMBOL(file_remove_suid); | |||
1816 | static size_t __iovec_copy_from_user_inatomic(char *vaddr, | 1816 | static size_t __iovec_copy_from_user_inatomic(char *vaddr, |
1817 | const struct iovec *iov, size_t base, size_t bytes) | 1817 | const struct iovec *iov, size_t base, size_t bytes) |
1818 | { | 1818 | { |
1819 | size_t copied = 0, left = 0, total = bytes; | 1819 | size_t copied = 0, left = 0; |
1820 | 1820 | ||
1821 | while (bytes) { | 1821 | while (bytes) { |
1822 | char __user *buf = iov->iov_base + base; | 1822 | char __user *buf = iov->iov_base + base; |
1823 | int copy = min(bytes, iov->iov_len - base); | 1823 | int copy = min(bytes, iov->iov_len - base); |
1824 | 1824 | ||
1825 | base = 0; | 1825 | base = 0; |
1826 | left = __copy_from_user_inatomic_nocache(vaddr, buf, copy, total); | 1826 | left = __copy_from_user_inatomic(vaddr, buf, copy); |
1827 | copied += copy; | 1827 | copied += copy; |
1828 | bytes -= copy; | 1828 | bytes -= copy; |
1829 | vaddr += copy; | 1829 | vaddr += copy; |
@@ -1851,9 +1851,7 @@ size_t iov_iter_copy_from_user_atomic(struct page *page, | |||
1851 | if (likely(i->nr_segs == 1)) { | 1851 | if (likely(i->nr_segs == 1)) { |
1852 | int left; | 1852 | int left; |
1853 | char __user *buf = i->iov->iov_base + i->iov_offset; | 1853 | char __user *buf = i->iov->iov_base + i->iov_offset; |
1854 | 1854 | left = __copy_from_user_inatomic(kaddr + offset, buf, bytes); | |
1855 | left = __copy_from_user_inatomic_nocache(kaddr + offset, | ||
1856 | buf, bytes, bytes); | ||
1857 | copied = bytes - left; | 1855 | copied = bytes - left; |
1858 | } else { | 1856 | } else { |
1859 | copied = __iovec_copy_from_user_inatomic(kaddr + offset, | 1857 | copied = __iovec_copy_from_user_inatomic(kaddr + offset, |
@@ -1881,8 +1879,7 @@ size_t iov_iter_copy_from_user(struct page *page, | |||
1881 | if (likely(i->nr_segs == 1)) { | 1879 | if (likely(i->nr_segs == 1)) { |
1882 | int left; | 1880 | int left; |
1883 | char __user *buf = i->iov->iov_base + i->iov_offset; | 1881 | char __user *buf = i->iov->iov_base + i->iov_offset; |
1884 | 1882 | left = __copy_from_user(kaddr + offset, buf, bytes); | |
1885 | left = __copy_from_user_nocache(kaddr + offset, buf, bytes, bytes); | ||
1886 | copied = bytes - left; | 1883 | copied = bytes - left; |
1887 | } else { | 1884 | } else { |
1888 | copied = __iovec_copy_from_user_inatomic(kaddr + offset, | 1885 | copied = __iovec_copy_from_user_inatomic(kaddr + offset, |
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c index bf54f8a2cf1d..0c04615651b7 100644 --- a/mm/filemap_xip.c +++ b/mm/filemap_xip.c | |||
@@ -354,7 +354,7 @@ __xip_file_write(struct file *filp, const char __user *buf, | |||
354 | break; | 354 | break; |
355 | 355 | ||
356 | copied = bytes - | 356 | copied = bytes - |
357 | __copy_from_user_nocache(xip_mem + offset, buf, bytes, bytes); | 357 | __copy_from_user_nocache(xip_mem + offset, buf, bytes); |
358 | 358 | ||
359 | if (likely(copied > 0)) { | 359 | if (likely(copied > 0)) { |
360 | status = copied; | 360 | status = copied; |