diff options
-rw-r--r-- | mm/filemap.c | 36 | ||||
-rw-r--r-- | mm/filemap.h | 100 | ||||
-rw-r--r-- | mm/filemap_xip.c | 17 |
3 files changed, 67 insertions, 86 deletions
diff --git a/mm/filemap.c b/mm/filemap.c index fb4c1c0792e7..c59d5b3cd99a 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
@@ -1823,12 +1823,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, | |||
1823 | /* | 1823 | /* |
1824 | * handle partial DIO write. Adjust cur_iov if needed. | 1824 | * handle partial DIO write. Adjust cur_iov if needed. |
1825 | */ | 1825 | */ |
1826 | if (likely(nr_segs == 1)) | 1826 | filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, written); |
1827 | buf = iov->iov_base + written; | ||
1828 | else { | ||
1829 | filemap_set_next_iovec(&cur_iov, &iov_offset, written); | ||
1830 | buf = cur_iov->iov_base + iov_offset; | ||
1831 | } | ||
1832 | 1827 | ||
1833 | do { | 1828 | do { |
1834 | struct page *page; | 1829 | struct page *page; |
@@ -1838,6 +1833,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, | |||
1838 | size_t bytes; /* Bytes to write to page */ | 1833 | size_t bytes; /* Bytes to write to page */ |
1839 | size_t copied; /* Bytes copied from user */ | 1834 | size_t copied; /* Bytes copied from user */ |
1840 | 1835 | ||
1836 | buf = cur_iov->iov_base + iov_offset; | ||
1841 | offset = (pos & (PAGE_CACHE_SIZE - 1)); | 1837 | offset = (pos & (PAGE_CACHE_SIZE - 1)); |
1842 | index = pos >> PAGE_CACHE_SHIFT; | 1838 | index = pos >> PAGE_CACHE_SHIFT; |
1843 | bytes = PAGE_CACHE_SIZE - offset; | 1839 | bytes = PAGE_CACHE_SIZE - offset; |
@@ -1869,13 +1865,10 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, | |||
1869 | if (unlikely(status)) | 1865 | if (unlikely(status)) |
1870 | goto fs_write_aop_error; | 1866 | goto fs_write_aop_error; |
1871 | 1867 | ||
1872 | if (likely(nr_segs == 1)) | 1868 | copied = filemap_copy_from_user(page, offset, |
1873 | copied = filemap_copy_from_user(page, offset, | 1869 | cur_iov, nr_segs, iov_offset, bytes); |
1874 | buf, bytes); | ||
1875 | else | ||
1876 | copied = filemap_copy_from_user_iovec(page, offset, | ||
1877 | cur_iov, iov_offset, bytes); | ||
1878 | flush_dcache_page(page); | 1870 | flush_dcache_page(page); |
1871 | |||
1879 | status = a_ops->commit_write(file, page, offset, offset+bytes); | 1872 | status = a_ops->commit_write(file, page, offset, offset+bytes); |
1880 | if (unlikely(status < 0 || status == AOP_TRUNCATED_PAGE)) | 1873 | if (unlikely(status < 0 || status == AOP_TRUNCATED_PAGE)) |
1881 | goto fs_write_aop_error; | 1874 | goto fs_write_aop_error; |
@@ -1886,20 +1879,11 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, | |||
1886 | if (unlikely(status > 0)) /* filesystem did partial write */ | 1879 | if (unlikely(status > 0)) /* filesystem did partial write */ |
1887 | copied = status; | 1880 | copied = status; |
1888 | 1881 | ||
1889 | if (likely(copied > 0)) { | 1882 | written += copied; |
1890 | written += copied; | 1883 | count -= copied; |
1891 | count -= copied; | 1884 | pos += copied; |
1892 | pos += copied; | 1885 | filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, copied); |
1893 | buf += copied; | 1886 | |
1894 | if (unlikely(nr_segs > 1)) { | ||
1895 | filemap_set_next_iovec(&cur_iov, | ||
1896 | &iov_offset, copied); | ||
1897 | if (count) | ||
1898 | buf = cur_iov->iov_base + iov_offset; | ||
1899 | } else { | ||
1900 | iov_offset += copied; | ||
1901 | } | ||
1902 | } | ||
1903 | unlock_page(page); | 1887 | unlock_page(page); |
1904 | mark_page_accessed(page); | 1888 | mark_page_accessed(page); |
1905 | page_cache_release(page); | 1889 | page_cache_release(page); |
diff --git a/mm/filemap.h b/mm/filemap.h index a1e10a232e92..b500d936cec5 100644 --- a/mm/filemap.h +++ b/mm/filemap.h | |||
@@ -22,82 +22,82 @@ __filemap_copy_from_user_iovec_inatomic(char *vaddr, | |||
22 | 22 | ||
23 | /* | 23 | /* |
24 | * Copy as much as we can into the page and return the number of bytes which | 24 | * Copy as much as we can into the page and return the number of bytes which |
25 | * were sucessfully copied. If a fault is encountered then clear the page | 25 | * were sucessfully copied. If a fault is encountered then return the number of |
26 | * out to (offset+bytes) and return the number of bytes which were copied. | 26 | * bytes which were copied. |
27 | * | ||
28 | * NOTE: For this to work reliably we really want copy_from_user_inatomic_nocache | ||
29 | * to *NOT* zero any tail of the buffer that it failed to copy. If it does, | ||
30 | * and if the following non-atomic copy succeeds, then there is a small window | ||
31 | * where the target page contains neither the data before the write, nor the | ||
32 | * data after the write (it contains zero). A read at this time will see | ||
33 | * data that is inconsistent with any ordering of the read and the write. | ||
34 | * (This has been detected in practice). | ||
35 | */ | 27 | */ |
36 | static inline size_t | 28 | static inline size_t |
37 | filemap_copy_from_user(struct page *page, unsigned long offset, | 29 | filemap_copy_from_user_atomic(struct page *page, unsigned long offset, |
38 | const char __user *buf, unsigned bytes) | 30 | const struct iovec *iov, unsigned long nr_segs, |
31 | size_t base, size_t bytes) | ||
39 | { | 32 | { |
40 | char *kaddr; | 33 | char *kaddr; |
41 | int left; | 34 | size_t copied; |
42 | 35 | ||
43 | kaddr = kmap_atomic(page, KM_USER0); | 36 | kaddr = kmap_atomic(page, KM_USER0); |
44 | left = __copy_from_user_inatomic_nocache(kaddr + offset, buf, bytes); | 37 | if (likely(nr_segs == 1)) { |
38 | int left; | ||
39 | char __user *buf = iov->iov_base + base; | ||
40 | left = __copy_from_user_inatomic_nocache(kaddr + offset, | ||
41 | buf, bytes); | ||
42 | copied = bytes - left; | ||
43 | } else { | ||
44 | copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, | ||
45 | iov, base, bytes); | ||
46 | } | ||
45 | kunmap_atomic(kaddr, KM_USER0); | 47 | kunmap_atomic(kaddr, KM_USER0); |
46 | 48 | ||
47 | if (left != 0) { | 49 | return copied; |
48 | /* Do it the slow way */ | ||
49 | kaddr = kmap(page); | ||
50 | left = __copy_from_user_nocache(kaddr + offset, buf, bytes); | ||
51 | kunmap(page); | ||
52 | } | ||
53 | return bytes - left; | ||
54 | } | 50 | } |
55 | 51 | ||
56 | /* | 52 | /* |
57 | * This has the same sideeffects and return value as filemap_copy_from_user(). | 53 | * This has the same sideeffects and return value as |
58 | * The difference is that on a fault we need to memset the remainder of the | 54 | * filemap_copy_from_user_atomic(). |
59 | * page (out to offset+bytes), to emulate filemap_copy_from_user()'s | 55 | * The difference is that it attempts to resolve faults. |
60 | * single-segment behaviour. | ||
61 | */ | 56 | */ |
62 | static inline size_t | 57 | static inline size_t |
63 | filemap_copy_from_user_iovec(struct page *page, unsigned long offset, | 58 | filemap_copy_from_user(struct page *page, unsigned long offset, |
64 | const struct iovec *iov, size_t base, size_t bytes) | 59 | const struct iovec *iov, unsigned long nr_segs, |
60 | size_t base, size_t bytes) | ||
65 | { | 61 | { |
66 | char *kaddr; | 62 | char *kaddr; |
67 | size_t copied; | 63 | size_t copied; |
68 | 64 | ||
69 | kaddr = kmap_atomic(page, KM_USER0); | 65 | kaddr = kmap(page); |
70 | copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, iov, | 66 | if (likely(nr_segs == 1)) { |
71 | base, bytes); | 67 | int left; |
72 | kunmap_atomic(kaddr, KM_USER0); | 68 | char __user *buf = iov->iov_base + base; |
73 | if (copied != bytes) { | 69 | left = __copy_from_user_nocache(kaddr + offset, buf, bytes); |
74 | kaddr = kmap(page); | 70 | copied = bytes - left; |
75 | copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, iov, | 71 | } else { |
76 | base, bytes); | 72 | copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, |
77 | if (bytes - copied) | 73 | iov, base, bytes); |
78 | memset(kaddr + offset + copied, 0, bytes - copied); | ||
79 | kunmap(page); | ||
80 | } | 74 | } |
75 | kunmap(page); | ||
81 | return copied; | 76 | return copied; |
82 | } | 77 | } |
83 | 78 | ||
84 | static inline void | 79 | static inline void |
85 | filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes) | 80 | filemap_set_next_iovec(const struct iovec **iovp, unsigned long nr_segs, |
81 | size_t *basep, size_t bytes) | ||
86 | { | 82 | { |
87 | const struct iovec *iov = *iovp; | 83 | if (likely(nr_segs == 1)) { |
88 | size_t base = *basep; | 84 | *basep += bytes; |
85 | } else { | ||
86 | const struct iovec *iov = *iovp; | ||
87 | size_t base = *basep; | ||
89 | 88 | ||
90 | while (bytes) { | 89 | while (bytes) { |
91 | int copy = min(bytes, iov->iov_len - base); | 90 | int copy = min(bytes, iov->iov_len - base); |
92 | 91 | ||
93 | bytes -= copy; | 92 | bytes -= copy; |
94 | base += copy; | 93 | base += copy; |
95 | if (iov->iov_len == base) { | 94 | if (iov->iov_len == base) { |
96 | iov++; | 95 | iov++; |
97 | base = 0; | 96 | base = 0; |
97 | } | ||
98 | } | 98 | } |
99 | *iovp = iov; | ||
100 | *basep = base; | ||
99 | } | 101 | } |
100 | *iovp = iov; | ||
101 | *basep = base; | ||
102 | } | 102 | } |
103 | #endif | 103 | #endif |
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c index 53ee6a299635..32132f3cd641 100644 --- a/mm/filemap_xip.c +++ b/mm/filemap_xip.c | |||
@@ -15,7 +15,6 @@ | |||
15 | #include <linux/rmap.h> | 15 | #include <linux/rmap.h> |
16 | #include <linux/sched.h> | 16 | #include <linux/sched.h> |
17 | #include <asm/tlbflush.h> | 17 | #include <asm/tlbflush.h> |
18 | #include "filemap.h" | ||
19 | 18 | ||
20 | /* | 19 | /* |
21 | * We do use our own empty page to avoid interference with other users | 20 | * We do use our own empty page to avoid interference with other users |
@@ -288,6 +287,7 @@ __xip_file_write(struct file *filp, const char __user *buf, | |||
288 | unsigned long index; | 287 | unsigned long index; |
289 | unsigned long offset; | 288 | unsigned long offset; |
290 | size_t copied; | 289 | size_t copied; |
290 | char *kaddr; | ||
291 | 291 | ||
292 | offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */ | 292 | offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */ |
293 | index = pos >> PAGE_CACHE_SHIFT; | 293 | index = pos >> PAGE_CACHE_SHIFT; |
@@ -295,14 +295,6 @@ __xip_file_write(struct file *filp, const char __user *buf, | |||
295 | if (bytes > count) | 295 | if (bytes > count) |
296 | bytes = count; | 296 | bytes = count; |
297 | 297 | ||
298 | /* | ||
299 | * Bring in the user page that we will copy from _first_. | ||
300 | * Otherwise there's a nasty deadlock on copying from the | ||
301 | * same page as we're writing to, without it being marked | ||
302 | * up-to-date. | ||
303 | */ | ||
304 | fault_in_pages_readable(buf, bytes); | ||
305 | |||
306 | page = a_ops->get_xip_page(mapping, | 298 | page = a_ops->get_xip_page(mapping, |
307 | index*(PAGE_SIZE/512), 0); | 299 | index*(PAGE_SIZE/512), 0); |
308 | if (IS_ERR(page) && (PTR_ERR(page) == -ENODATA)) { | 300 | if (IS_ERR(page) && (PTR_ERR(page) == -ENODATA)) { |
@@ -319,8 +311,13 @@ __xip_file_write(struct file *filp, const char __user *buf, | |||
319 | break; | 311 | break; |
320 | } | 312 | } |
321 | 313 | ||
322 | copied = filemap_copy_from_user(page, offset, buf, bytes); | 314 | fault_in_pages_readable(buf, bytes); |
315 | kaddr = kmap_atomic(page, KM_USER0); | ||
316 | copied = bytes - | ||
317 | __copy_from_user_inatomic_nocache(kaddr, buf, bytes); | ||
318 | kunmap_atomic(kaddr, KM_USER0); | ||
323 | flush_dcache_page(page); | 319 | flush_dcache_page(page); |
320 | |||
324 | if (likely(copied > 0)) { | 321 | if (likely(copied > 0)) { |
325 | status = copied; | 322 | status = copied; |
326 | 323 | ||