aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/x86/include/asm/uaccess_32.h4
-rw-r--r--arch/x86/include/asm/uaccess_64.h25
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c2
-rw-r--r--include/linux/uaccess.h4
-rw-r--r--mm/filemap.c11
-rw-r--r--mm/filemap_xip.c2
6 files changed, 17 insertions, 31 deletions
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index a0ba61386972..5e06259e90e5 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -157,7 +157,7 @@ __copy_from_user(void *to, const void __user *from, unsigned long n)
157} 157}
158 158
159static __always_inline unsigned long __copy_from_user_nocache(void *to, 159static __always_inline unsigned long __copy_from_user_nocache(void *to,
160 const void __user *from, unsigned long n, unsigned long total) 160 const void __user *from, unsigned long n)
161{ 161{
162 might_fault(); 162 might_fault();
163 if (__builtin_constant_p(n)) { 163 if (__builtin_constant_p(n)) {
@@ -180,7 +180,7 @@ static __always_inline unsigned long __copy_from_user_nocache(void *to,
180 180
181static __always_inline unsigned long 181static __always_inline unsigned long
182__copy_from_user_inatomic_nocache(void *to, const void __user *from, 182__copy_from_user_inatomic_nocache(void *to, const void __user *from,
183 unsigned long n, unsigned long total) 183 unsigned long n)
184{ 184{
185 return __copy_from_user_ll_nocache_nozero(to, from, n); 185 return __copy_from_user_ll_nocache_nozero(to, from, n);
186} 186}
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index dcaa0404cf7b..8cc687326eb8 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -188,29 +188,18 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
188extern long __copy_user_nocache(void *dst, const void __user *src, 188extern long __copy_user_nocache(void *dst, const void __user *src,
189 unsigned size, int zerorest); 189 unsigned size, int zerorest);
190 190
191static inline int __copy_from_user_nocache(void *dst, const void __user *src, 191static inline int
192 unsigned size, unsigned long total) 192__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
193{ 193{
194 might_sleep(); 194 might_sleep();
195 /* 195 return __copy_user_nocache(dst, src, size, 1);
196 * In practice this limit means that large file write()s
197 * which get chunked to 4K copies get handled via
198 * non-temporal stores here. Smaller writes get handled
199 * via regular __copy_from_user():
200 */
201 if (likely(total >= PAGE_SIZE))
202 return __copy_user_nocache(dst, src, size, 1);
203 else
204 return __copy_from_user(dst, src, size);
205} 196}
206 197
207static inline int __copy_from_user_inatomic_nocache(void *dst, 198static inline int
208 const void __user *src, unsigned size, unsigned total) 199__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
200 unsigned size)
209{ 201{
210 if (likely(total >= PAGE_SIZE)) 202 return __copy_user_nocache(dst, src, size, 0);
211 return __copy_user_nocache(dst, src, size, 0);
212 else
213 return __copy_from_user_inatomic(dst, src, size);
214} 203}
215 204
216unsigned long 205unsigned long
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 6b209db8370d..818576654092 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -215,7 +215,7 @@ fast_user_write(struct io_mapping *mapping,
215 215
216 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); 216 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
217 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset, 217 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
218 user_data, length, length); 218 user_data, length);
219 io_mapping_unmap_atomic(vaddr_atomic); 219 io_mapping_unmap_atomic(vaddr_atomic);
220 if (unwritten) 220 if (unwritten)
221 return -EFAULT; 221 return -EFAULT;
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 6f3c603b0d67..6b58367d145e 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -41,13 +41,13 @@ static inline void pagefault_enable(void)
41#ifndef ARCH_HAS_NOCACHE_UACCESS 41#ifndef ARCH_HAS_NOCACHE_UACCESS
42 42
43static inline unsigned long __copy_from_user_inatomic_nocache(void *to, 43static inline unsigned long __copy_from_user_inatomic_nocache(void *to,
44 const void __user *from, unsigned long n, unsigned long total) 44 const void __user *from, unsigned long n)
45{ 45{
46 return __copy_from_user_inatomic(to, from, n); 46 return __copy_from_user_inatomic(to, from, n);
47} 47}
48 48
49static inline unsigned long __copy_from_user_nocache(void *to, 49static inline unsigned long __copy_from_user_nocache(void *to,
50 const void __user *from, unsigned long n, unsigned long total) 50 const void __user *from, unsigned long n)
51{ 51{
52 return __copy_from_user(to, from, n); 52 return __copy_from_user(to, from, n);
53} 53}
diff --git a/mm/filemap.c b/mm/filemap.c
index 60fd56772cc6..126d3973b3d1 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1816,14 +1816,14 @@ EXPORT_SYMBOL(file_remove_suid);
1816static size_t __iovec_copy_from_user_inatomic(char *vaddr, 1816static size_t __iovec_copy_from_user_inatomic(char *vaddr,
1817 const struct iovec *iov, size_t base, size_t bytes) 1817 const struct iovec *iov, size_t base, size_t bytes)
1818{ 1818{
1819 size_t copied = 0, left = 0, total = bytes; 1819 size_t copied = 0, left = 0;
1820 1820
1821 while (bytes) { 1821 while (bytes) {
1822 char __user *buf = iov->iov_base + base; 1822 char __user *buf = iov->iov_base + base;
1823 int copy = min(bytes, iov->iov_len - base); 1823 int copy = min(bytes, iov->iov_len - base);
1824 1824
1825 base = 0; 1825 base = 0;
1826 left = __copy_from_user_inatomic_nocache(vaddr, buf, copy, total); 1826 left = __copy_from_user_inatomic(vaddr, buf, copy);
1827 copied += copy; 1827 copied += copy;
1828 bytes -= copy; 1828 bytes -= copy;
1829 vaddr += copy; 1829 vaddr += copy;
@@ -1851,9 +1851,7 @@ size_t iov_iter_copy_from_user_atomic(struct page *page,
1851 if (likely(i->nr_segs == 1)) { 1851 if (likely(i->nr_segs == 1)) {
1852 int left; 1852 int left;
1853 char __user *buf = i->iov->iov_base + i->iov_offset; 1853 char __user *buf = i->iov->iov_base + i->iov_offset;
1854 1854 left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
1855 left = __copy_from_user_inatomic_nocache(kaddr + offset,
1856 buf, bytes, bytes);
1857 copied = bytes - left; 1855 copied = bytes - left;
1858 } else { 1856 } else {
1859 copied = __iovec_copy_from_user_inatomic(kaddr + offset, 1857 copied = __iovec_copy_from_user_inatomic(kaddr + offset,
@@ -1881,8 +1879,7 @@ size_t iov_iter_copy_from_user(struct page *page,
1881 if (likely(i->nr_segs == 1)) { 1879 if (likely(i->nr_segs == 1)) {
1882 int left; 1880 int left;
1883 char __user *buf = i->iov->iov_base + i->iov_offset; 1881 char __user *buf = i->iov->iov_base + i->iov_offset;
1884 1882 left = __copy_from_user(kaddr + offset, buf, bytes);
1885 left = __copy_from_user_nocache(kaddr + offset, buf, bytes, bytes);
1886 copied = bytes - left; 1883 copied = bytes - left;
1887 } else { 1884 } else {
1888 copied = __iovec_copy_from_user_inatomic(kaddr + offset, 1885 copied = __iovec_copy_from_user_inatomic(kaddr + offset,
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c
index bf54f8a2cf1d..0c04615651b7 100644
--- a/mm/filemap_xip.c
+++ b/mm/filemap_xip.c
@@ -354,7 +354,7 @@ __xip_file_write(struct file *filp, const char __user *buf,
354 break; 354 break;
355 355
356 copied = bytes - 356 copied = bytes -
357 __copy_from_user_nocache(xip_mem + offset, buf, bytes, bytes); 357 __copy_from_user_nocache(xip_mem + offset, buf, bytes);
358 358
359 if (likely(copied > 0)) { 359 if (likely(copied > 0)) {
360 status = copied; 360 status = copied;