Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs

Pull vfs updates from Al Viro:
 "This the bunch that sat in -next + lock_parent() fix.  This is the
  minimal set; there's more pending stuff.

  In particular, I really hope to get acct.c fixes merged this cycle -
  we need that to deal sanely with delayed-mntput stuff.  In the next
  pile, hopefully - that series is fairly short and localized
  (kernel/acct.c, fs/super.c and fs/namespace.c).  In this pile: more
  iov_iter work.  Most of prereqs for ->splice_write with sane locking
  order are there and Kent's dio rewrite would also fit nicely on top of
  this pile"

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs: (70 commits)
  lock_parent: don't step on stale ->d_parent of all-but-freed one
  kill generic_file_splice_write()
  ceph: switch to iter_file_splice_write()
  shmem: switch to iter_file_splice_write()
  nfs: switch to iter_splice_write_file()
  fs/splice.c: remove unneeded exports
  ocfs2: switch to iter_file_splice_write()
  ->splice_write() via ->write_iter()
  bio_vec-backed iov_iter
  optimize copy_page_{to,from}_iter()
  bury generic_file_aio_{read,write}
  lustre: get rid of messing with iovecs
  ceph: switch to ->write_iter()
  ceph_sync_direct_write: stop poking into iov_iter guts
  ceph_sync_read: stop poking into iov_iter guts
  new helper: copy_page_from_iter()
  fuse: switch to ->write_iter()
  btrfs: switch to ->write_iter()
  ocfs2: switch to ->write_iter()
  xfs: switch to ->write_iter()
  ...

6 files changed, 626 insertions, 186 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index 7fadf1c62838..dafb06f70a09 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1665,96 +1665,42 @@ out:
         return written ? written : error;
 }
 
-/*
- * Performs necessary checks before doing a write
- * @iov:        io vector request
- * @nr_segs:    number of segments in the iovec
- * @count:      number of bytes to write
- * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE
- *
- * Adjust number of segments and amount of bytes to write (nr_segs should be
- * properly initialized first). Returns appropriate error code that caller
- * should return or zero in case that write should be allowed.
- */
-int generic_segment_checks(const struct iovec *iov,
-                        unsigned long *nr_segs, size_t *count, int access_flags)
-{
-        unsigned long   seg;
-        size_t cnt = 0;
-        for (seg = 0; seg < *nr_segs; seg++) {
-                const struct iovec *iv = &iov[seg];
-
-                /*
-                 * If any segment has a negative length, or the cumulative
-                 * length ever wraps negative then return -EINVAL.
-                 */
-                cnt += iv->iov_len;
-                if (unlikely((ssize_t)(cnt|iv->iov_len) < 0))
-                        return -EINVAL;
-                if (access_ok(access_flags, iv->iov_base, iv->iov_len))
-                        continue;
-                if (seg == 0)
-                        return -EFAULT;
-                *nr_segs = seg;
-                cnt -= iv->iov_len;     /* This segment is no good */
-                break;
-        }
-        *count = cnt;
-        return 0;
-}
-EXPORT_SYMBOL(generic_segment_checks);
-
 /**
- * generic_file_aio_read - generic filesystem read routine
+ * generic_file_read_iter - generic filesystem read routine
  * @iocb:       kernel I/O control block
- * @iov:        io vector request
- * @nr_segs:    number of segments in the iovec
- * @pos:        current file position
+ * @iter:       destination for the data read
  *
- * This is the "read()" routine for all filesystems
+ * This is the "read_iter()" routine for all filesystems
  * that can use the page cache directly.
  */
 ssize_t
-generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
-                unsigned long nr_segs, loff_t pos)
+generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
 {
-        struct file *filp = iocb->ki_filp;
-        ssize_t retval;
-        size_t count;
+        struct file *file = iocb->ki_filp;
+        ssize_t retval = 0;
         loff_t *ppos = &iocb->ki_pos;
-        struct iov_iter i;
-
-        count = 0;
-        retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
-        if (retval)
-                return retval;
-        iov_iter_init(&i, iov, nr_segs, count, 0);
+        loff_t pos = *ppos;
 
         /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
-        if (filp->f_flags & O_DIRECT) {
+        if (file->f_flags & O_DIRECT) {
+                struct address_space *mapping = file->f_mapping;
+                struct inode *inode = mapping->host;
+                size_t count = iov_iter_count(iter);
                 loff_t size;
-                struct address_space *mapping;
-                struct inode *inode;
 
-                mapping = filp->f_mapping;
-                inode = mapping->host;
                 if (!count)
                         goto out; /* skip atime */
                 size = i_size_read(inode);
                 retval = filemap_write_and_wait_range(mapping, pos,
-                                        pos + iov_length(iov, nr_segs) - 1);
+                                        pos + count - 1);
                 if (!retval) {
-                        retval = mapping->a_ops->direct_IO(READ, iocb,
-                                                           iov, pos, nr_segs);
+                        struct iov_iter data = *iter;
+                        retval = mapping->a_ops->direct_IO(READ, iocb, &data, pos);
                 }
+
                 if (retval > 0) {
                         *ppos = pos + retval;
-                        count -= retval;
-                        /*
-                         * If we did a short DIO read we need to skip the
-                         * section of the iov that we've already read data into.
-                         */
-                        iov_iter_advance(&i, retval);
+                        iov_iter_advance(iter, retval);
                 }
 
                 /*
@@ -1765,17 +1711,17 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
                  * and return.  Otherwise fallthrough to buffered io for
                  * the rest of the read.
                  */
-                if (retval < 0 || !count || *ppos >= size) {
-                        file_accessed(filp);
+                if (retval < 0 || !iov_iter_count(iter) || *ppos >= size) {
+                        file_accessed(file);
                         goto out;
                 }
         }
 
-        retval = do_generic_file_read(filp, ppos, &i, retval);
+        retval = do_generic_file_read(file, ppos, iter, retval);
 out:
         return retval;
 }
-EXPORT_SYMBOL(generic_file_aio_read);
+EXPORT_SYMBOL(generic_file_read_iter);
 
 #ifdef CONFIG_MMU
 /**
@@ -2386,9 +2332,7 @@ int pagecache_write_end(struct file *file, struct address_space *mapping,
 EXPORT_SYMBOL(pagecache_write_end);
 
 ssize_t
-generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
-                unsigned long *nr_segs, loff_t pos,
-                size_t count, size_t ocount)
+generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos)
 {
         struct file     *file = iocb->ki_filp;
         struct address_space *mapping = file->f_mapping;
@@ -2396,11 +2340,9 @@ generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
         ssize_t         written;
         size_t          write_len;
         pgoff_t         end;
+        struct iov_iter data;
 
-        if (count != ocount)
-                *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count);
-
-        write_len = iov_length(iov, *nr_segs);
+        write_len = iov_iter_count(from);
         end = (pos + write_len - 1) >> PAGE_CACHE_SHIFT;
 
         written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);
@@ -2427,7 +2369,8 @@ generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
                 }
         }
 
-        written = mapping->a_ops->direct_IO(WRITE, iocb, iov, pos, *nr_segs);
+        data = *from;
+        written = mapping->a_ops->direct_IO(WRITE, iocb, &data, pos);
 
         /*
          * Finally, try again to invalidate clean pages which might have been
@@ -2444,6 +2387,7 @@ generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
 
         if (written > 0) {
                 pos += written;
+                iov_iter_advance(from, written);
                 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
                         i_size_write(inode, pos);
                         mark_inode_dirty(inode);
@@ -2568,10 +2512,9 @@ again:
 EXPORT_SYMBOL(generic_perform_write);
 
 /**
- * __generic_file_aio_write - write data to a file
+ * __generic_file_write_iter - write data to a file
  * @iocb:       IO state structure (file, offset, etc.)
- * @iov:        vector with data to write
- * @nr_segs:    number of segments in the vector
+ * @from:       iov_iter with data to write
  *
  * This function does all the work needed for actually writing data to a
  * file. It does all basic checks, removes SUID from the file, updates
@@ -2585,26 +2528,16 @@ EXPORT_SYMBOL(generic_perform_write);
  * A caller has to handle it. This is mainly due to the fact that we want to
  * avoid syncing under i_mutex.
  */
-ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
-                                 unsigned long nr_segs)
+ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
         struct file *file = iocb->ki_filp;
         struct address_space * mapping = file->f_mapping;
-        size_t ocount;          /* original count */
-        size_t count;           /* after file limit checks */
         struct inode    *inode = mapping->host;
         loff_t          pos = iocb->ki_pos;
         ssize_t         written = 0;
         ssize_t         err;
         ssize_t         status;
-        struct iov_iter from;
-
-        ocount = 0;
-        err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
-        if (err)
-                return err;
-
-        count = ocount;
+        size_t          count = iov_iter_count(from);
 
         /* We can write back this queue in page reclaim */
         current->backing_dev_info = mapping->backing_dev_info;
@@ -2615,6 +2548,8 @@ ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
         if (count == 0)
                 goto out;
 
+        iov_iter_truncate(from, count);
+
         err = file_remove_suid(file);
         if (err)
                 goto out;
@@ -2623,17 +2558,13 @@ ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
         if (err)
                 goto out;
 
-        iov_iter_init(&from, iov, nr_segs, count, 0);
-
         /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
         if (unlikely(file->f_flags & O_DIRECT)) {
                 loff_t endbyte;
 
-                written = generic_file_direct_write(iocb, iov, &from.nr_segs, pos,
-                                                        count, ocount);
+                written = generic_file_direct_write(iocb, from, pos);
                 if (written < 0 || written == count)
                         goto out;
-                iov_iter_advance(&from, written);
 
                 /*
                  * direct-io write to a hole: fall through to buffered I/O
@@ -2642,7 +2573,7 @@ ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
                 pos += written;
                 count -= written;
 
-                status = generic_perform_write(file, &from, pos);
+                status = generic_perform_write(file, from, pos);
                 /*
                  * If generic_perform_write() returned a synchronous error
                  * then we want to return the number of bytes which were
@@ -2674,7 +2605,7 @@ ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
                          */
                 }
         } else {
-                written = generic_perform_write(file, &from, pos);
+                written = generic_perform_write(file, from, pos);
                 if (likely(written >= 0))
                         iocb->ki_pos = pos + written;
         }
@@ -2682,30 +2613,25 @@ out:
         current->backing_dev_info = NULL;
         return written ? written : err;
 }
-EXPORT_SYMBOL(__generic_file_aio_write);
+EXPORT_SYMBOL(__generic_file_write_iter);
 
 /**
- * generic_file_aio_write - write data to a file
+ * generic_file_write_iter - write data to a file
  * @iocb:       IO state structure
- * @iov:        vector with data to write
- * @nr_segs:    number of segments in the vector
- * @pos:        position in file where to write
+ * @from:       iov_iter with data to write
  *
- * This is a wrapper around __generic_file_aio_write() to be used by most
+ * This is a wrapper around __generic_file_write_iter() to be used by most
  * filesystems. It takes care of syncing the file in case of O_SYNC file
  * and acquires i_mutex as needed.
  */
-ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
-                unsigned long nr_segs, loff_t pos)
+ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
         struct file *file = iocb->ki_filp;
         struct inode *inode = file->f_mapping->host;
         ssize_t ret;
 
-        BUG_ON(iocb->ki_pos != pos);
-
         mutex_lock(&inode->i_mutex);
-        ret = __generic_file_aio_write(iocb, iov, nr_segs);
+        ret = __generic_file_write_iter(iocb, from);
         mutex_unlock(&inode->i_mutex);
 
         if (ret > 0) {
@@ -2717,7 +2643,7 @@ ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
         }
         return ret;
 }
-EXPORT_SYMBOL(generic_file_aio_write);
+EXPORT_SYMBOL(generic_file_write_iter);
 
 /**
  * try_to_release_page() - release old fs-specific metadata on a page
diff --git a/mm/iov_iter.c b/mm/iov_iter.c
index 10e46cd721de..7b5dbd1517b5 100644
--- a/mm/iov_iter.c
+++ b/mm/iov_iter.c
@@ -1,8 +1,10 @@
 #include <linux/export.h>
 #include <linux/uio.h>
 #include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
 
-size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
+static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
                          struct iov_iter *i)
 {
         size_t skip, copy, left, wanted;
@@ -72,13 +74,97 @@ size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
         }
         kunmap(page);
 done:
+        if (skip == iov->iov_len) {
+                iov++;
+                skip = 0;
+        }
+        i->count -= wanted - bytes;
+        i->nr_segs -= iov - i->iov;
+        i->iov = iov;
+        i->iov_offset = skip;
+        return wanted - bytes;
+}
+
+static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
+                         struct iov_iter *i)
+{
+        size_t skip, copy, left, wanted;
+        const struct iovec *iov;
+        char __user *buf;
+        void *kaddr, *to;
+
+        if (unlikely(bytes > i->count))
+                bytes = i->count;
+
+        if (unlikely(!bytes))
+                return 0;
+
+        wanted = bytes;
+        iov = i->iov;
+        skip = i->iov_offset;
+        buf = iov->iov_base + skip;
+        copy = min(bytes, iov->iov_len - skip);
+
+        if (!fault_in_pages_readable(buf, copy)) {
+                kaddr = kmap_atomic(page);
+                to = kaddr + offset;
+
+                /* first chunk, usually the only one */
+                left = __copy_from_user_inatomic(to, buf, copy);
+                copy -= left;
+                skip += copy;
+                to += copy;
+                bytes -= copy;
+
+                while (unlikely(!left && bytes)) {
+                        iov++;
+                        buf = iov->iov_base;
+                        copy = min(bytes, iov->iov_len);
+                        left = __copy_from_user_inatomic(to, buf, copy);
+                        copy -= left;
+                        skip = copy;
+                        to += copy;
+                        bytes -= copy;
+                }
+                if (likely(!bytes)) {
+                        kunmap_atomic(kaddr);
+                        goto done;
+                }
+                offset = to - kaddr;
+                buf += copy;
+                kunmap_atomic(kaddr);
+                copy = min(bytes, iov->iov_len - skip);
+        }
+        /* Too bad - revert to non-atomic kmap */
+        kaddr = kmap(page);
+        to = kaddr + offset;
+        left = __copy_from_user(to, buf, copy);
+        copy -= left;
+        skip += copy;
+        to += copy;
+        bytes -= copy;
+        while (unlikely(!left && bytes)) {
+                iov++;
+                buf = iov->iov_base;
+                copy = min(bytes, iov->iov_len);
+                left = __copy_from_user(to, buf, copy);
+                copy -= left;
+                skip = copy;
+                to += copy;
+                bytes -= copy;
+        }
+        kunmap(page);
+done:
+        if (skip == iov->iov_len) {
+                iov++;
+                skip = 0;
+        }
         i->count -= wanted - bytes;
         i->nr_segs -= iov - i->iov;
         i->iov = iov;
         i->iov_offset = skip;
         return wanted - bytes;
 }
-EXPORT_SYMBOL(copy_page_to_iter);
 
 static size_t __iovec_copy_from_user_inatomic(char *vaddr,
                         const struct iovec *iov, size_t base, size_t bytes)
@@ -107,7 +193,7 @@ static size_t __iovec_copy_from_user_inatomic(char *vaddr,
  * were successfully copied.  If a fault is encountered then return the number of
  * bytes which were copied.
  */
-size_t iov_iter_copy_from_user_atomic(struct page *page,
+static size_t copy_from_user_atomic_iovec(struct page *page,
                 struct iov_iter *i, unsigned long offset, size_t bytes)
 {
         char *kaddr;
@@ -127,36 +213,8 @@ size_t iov_iter_copy_from_user_atomic(struct page *page,
 
         return copied;
 }
-EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
-
-/*
- * This has the same sideeffects and return value as
- * iov_iter_copy_from_user_atomic().
- * The difference is that it attempts to resolve faults.
- * Page must not be locked.
- */
-size_t iov_iter_copy_from_user(struct page *page,
-                struct iov_iter *i, unsigned long offset, size_t bytes)
-{
-        char *kaddr;
-        size_t copied;
-
-        kaddr = kmap(page);
-        if (likely(i->nr_segs == 1)) {
-                int left;
-                char __user *buf = i->iov->iov_base + i->iov_offset;
-                left = __copy_from_user(kaddr + offset, buf, bytes);
-                copied = bytes - left;
-        } else {
-                copied = __iovec_copy_from_user_inatomic(kaddr + offset,
-                                                i->iov, i->iov_offset, bytes);
-        }
-        kunmap(page);
-        return copied;
-}
-EXPORT_SYMBOL(iov_iter_copy_from_user);
 
-void iov_iter_advance(struct iov_iter *i, size_t bytes)
+static void advance_iovec(struct iov_iter *i, size_t bytes)
 {
         BUG_ON(i->count < bytes);
 
@@ -191,7 +249,6 @@ void iov_iter_advance(struct iov_iter *i, size_t bytes)
                 i->nr_segs = nr_segs;
         }
 }
-EXPORT_SYMBOL(iov_iter_advance);
 
 /*
  * Fault in the first iovec of the given iov_iter, to a maximum length
@@ -204,21 +261,483 @@ EXPORT_SYMBOL(iov_iter_advance);
  */
 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
 {
-        char __user *buf = i->iov->iov_base + i->iov_offset;
-        bytes = min(bytes, i->iov->iov_len - i->iov_offset);
-        return fault_in_pages_readable(buf, bytes);
+        if (!(i->type & ITER_BVEC)) {
+                char __user *buf = i->iov->iov_base + i->iov_offset;
+                bytes = min(bytes, i->iov->iov_len - i->iov_offset);
+                return fault_in_pages_readable(buf, bytes);
+        }
+        return 0;
 }
 EXPORT_SYMBOL(iov_iter_fault_in_readable);
 
+static unsigned long alignment_iovec(const struct iov_iter *i)
+{
+        const struct iovec *iov = i->iov;
+        unsigned long res;
+        size_t size = i->count;
+        size_t n;
+
+        if (!size)
+                return 0;
+
+        res = (unsigned long)iov->iov_base + i->iov_offset;
+        n = iov->iov_len - i->iov_offset;
+        if (n >= size)
+                return res | size;
+        size -= n;
+        res |= n;
+        while (size > (++iov)->iov_len) {
+                res |= (unsigned long)iov->iov_base | iov->iov_len;
+                size -= iov->iov_len;
+        }
+        res |= (unsigned long)iov->iov_base | size;
+        return res;
+}
+
+void iov_iter_init(struct iov_iter *i, int direction,
+                        const struct iovec *iov, unsigned long nr_segs,
+                        size_t count)
+{
+        /* It will get better.  Eventually... */
+        if (segment_eq(get_fs(), KERNEL_DS))
+                direction |= ITER_KVEC;
+        i->type = direction;
+        i->iov = iov;
+        i->nr_segs = nr_segs;
+        i->iov_offset = 0;
+        i->count = count;
+}
+EXPORT_SYMBOL(iov_iter_init);
+
+static ssize_t get_pages_iovec(struct iov_iter *i,
+                   struct page **pages, size_t maxsize,
+                   size_t *start)
+{
+        size_t offset = i->iov_offset;
+        const struct iovec *iov = i->iov;
+        size_t len;
+        unsigned long addr;
+        int n;
+        int res;
+
+        len = iov->iov_len - offset;
+        if (len > i->count)
+                len = i->count;
+        if (len > maxsize)
+                len = maxsize;
+        addr = (unsigned long)iov->iov_base + offset;
+        len += *start = addr & (PAGE_SIZE - 1);
+        addr &= ~(PAGE_SIZE - 1);
+        n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
+        res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
+        if (unlikely(res < 0))
+                return res;
+        return (res == n ? len : res * PAGE_SIZE) - *start;
+}
+
+static ssize_t get_pages_alloc_iovec(struct iov_iter *i,
+                   struct page ***pages, size_t maxsize,
+                   size_t *start)
+{
+        size_t offset = i->iov_offset;
+        const struct iovec *iov = i->iov;
+        size_t len;
+        unsigned long addr;
+        void *p;
+        int n;
+        int res;
+
+        len = iov->iov_len - offset;
+        if (len > i->count)
+                len = i->count;
+        if (len > maxsize)
+                len = maxsize;
+        addr = (unsigned long)iov->iov_base + offset;
+        len += *start = addr & (PAGE_SIZE - 1);
+        addr &= ~(PAGE_SIZE - 1);
+        n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
+        
+        p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
+        if (!p)
+                p = vmalloc(n * sizeof(struct page *));
+        if (!p)
+                return -ENOMEM;
+
+        res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
+        if (unlikely(res < 0)) {
+                kvfree(p);
+                return res;
+        }
+        *pages = p;
+        return (res == n ? len : res * PAGE_SIZE) - *start;
+}
+
+static int iov_iter_npages_iovec(const struct iov_iter *i, int maxpages)
+{
+        size_t offset = i->iov_offset;
+        size_t size = i->count;
+        const struct iovec *iov = i->iov;
+        int npages = 0;
+        int n;
+
+        for (n = 0; size && n < i->nr_segs; n++, iov++) {
+                unsigned long addr = (unsigned long)iov->iov_base + offset;
+                size_t len = iov->iov_len - offset;
+                offset = 0;
+                if (unlikely(!len))     /* empty segment */
+                        continue;
+                if (len > size)
+                        len = size;
+                npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE
+                          - addr / PAGE_SIZE;
+                if (npages >= maxpages) /* don't bother going further */
+                        return maxpages;
+                size -= len;
+                offset = 0;
+        }
+        return min(npages, maxpages);
+}
+
+static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
+{
+        char *from = kmap_atomic(page);
+        memcpy(to, from + offset, len);
+        kunmap_atomic(from);
+}
+
+static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len)
+{
+        char *to = kmap_atomic(page);
+        memcpy(to + offset, from, len);
+        kunmap_atomic(to);
+}
+
+static size_t copy_page_to_iter_bvec(struct page *page, size_t offset, size_t bytes,
+                         struct iov_iter *i)
+{
+        size_t skip, copy, wanted;
+        const struct bio_vec *bvec;
+        void *kaddr, *from;
+
+        if (unlikely(bytes > i->count))
+                bytes = i->count;
+
+        if (unlikely(!bytes))
+                return 0;
+
+        wanted = bytes;
+        bvec = i->bvec;
+        skip = i->iov_offset;
+        copy = min_t(size_t, bytes, bvec->bv_len - skip);
+
+        kaddr = kmap_atomic(page);
+        from = kaddr + offset;
+        memcpy_to_page(bvec->bv_page, skip + bvec->bv_offset, from, copy);
+        skip += copy;
+        from += copy;
+        bytes -= copy;
+        while (bytes) {
+                bvec++;
+                copy = min(bytes, (size_t)bvec->bv_len);
+                memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, copy);
+                skip = copy;
+                from += copy;
+                bytes -= copy;
+        }
+        kunmap_atomic(kaddr);
+        if (skip == bvec->bv_len) {
+                bvec++;
+                skip = 0;
+        }
+        i->count -= wanted - bytes;
+        i->nr_segs -= bvec - i->bvec;
+        i->bvec = bvec;
+        i->iov_offset = skip;
+        return wanted - bytes;
+}
+
+static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, size_t bytes,
+                         struct iov_iter *i)
+{
+        size_t skip, copy, wanted;
+        const struct bio_vec *bvec;
+        void *kaddr, *to;
+
+        if (unlikely(bytes > i->count))
+                bytes = i->count;
+
+        if (unlikely(!bytes))
+                return 0;
+
+        wanted = bytes;
+        bvec = i->bvec;
+        skip = i->iov_offset;
+
+        kaddr = kmap_atomic(page);
+
+        to = kaddr + offset;
+
+        copy = min(bytes, bvec->bv_len - skip);
+
+        memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy);
+
+        to += copy;
+        skip += copy;
+        bytes -= copy;
+
+        while (bytes) {
+                bvec++;
+                copy = min(bytes, (size_t)bvec->bv_len);
+                memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, copy);
+                skip = copy;
+                to += copy;
+                bytes -= copy;
+        }
+        kunmap_atomic(kaddr);
+        if (skip == bvec->bv_len) {
+                bvec++;
+                skip = 0;
+        }
+        i->count -= wanted;
+        i->nr_segs -= bvec - i->bvec;
+        i->bvec = bvec;
+        i->iov_offset = skip;
+        return wanted;
+}
+
+static size_t copy_from_user_bvec(struct page *page,
+                struct iov_iter *i, unsigned long offset, size_t bytes)
+{
+        char *kaddr;
+        size_t left;
+        const struct bio_vec *bvec;
+        size_t base = i->iov_offset;
+
+        kaddr = kmap_atomic(page);
+        for (left = bytes, bvec = i->bvec; left; bvec++, base = 0) {
+                size_t copy = min(left, bvec->bv_len - base);
+                if (!bvec->bv_len)
+                        continue;
+                memcpy_from_page(kaddr + offset, bvec->bv_page,
+                                 bvec->bv_offset + base, copy);
+                offset += copy;
+                left -= copy;
+        }
+        kunmap_atomic(kaddr);
+        return bytes;
+}
+
+static void advance_bvec(struct iov_iter *i, size_t bytes)
+{
+        BUG_ON(i->count < bytes);
+
+        if (likely(i->nr_segs == 1)) {
+                i->iov_offset += bytes;
+                i->count -= bytes;
+        } else {
+                const struct bio_vec *bvec = i->bvec;
+                size_t base = i->iov_offset;
+                unsigned long nr_segs = i->nr_segs;
+
+                /*
+                 * The !iov->iov_len check ensures we skip over unlikely
+                 * zero-length segments (without overruning the iovec).
+                 */
+                while (bytes || unlikely(i->count && !bvec->bv_len)) {
+                        int copy;
+
+                        copy = min(bytes, bvec->bv_len - base);
+                        BUG_ON(!i->count || i->count < copy);
+                        i->count -= copy;
+                        bytes -= copy;
+                        base += copy;
+                        if (bvec->bv_len == base) {
+                                bvec++;
+                                nr_segs--;
+                                base = 0;
+                        }
+                }
+                i->bvec = bvec;
+                i->iov_offset = base;
+                i->nr_segs = nr_segs;
+        }
+}
+
+static unsigned long alignment_bvec(const struct iov_iter *i)
+{
+        const struct bio_vec *bvec = i->bvec;
+        unsigned long res;
+        size_t size = i->count;
+        size_t n;
+
+        if (!size)
+                return 0;
+
+        res = bvec->bv_offset + i->iov_offset;
+        n = bvec->bv_len - i->iov_offset;
+        if (n >= size)
+                return res | size;
+        size -= n;
+        res |= n;
+        while (size > (++bvec)->bv_len) {
+                res |= bvec->bv_offset | bvec->bv_len;
+                size -= bvec->bv_len;
+        }
+        res |= bvec->bv_offset | size;
+        return res;
+}
+
+static ssize_t get_pages_bvec(struct iov_iter *i,
+                   struct page **pages, size_t maxsize,
+                   size_t *start)
+{
+        const struct bio_vec *bvec = i->bvec;
+        size_t len = bvec->bv_len - i->iov_offset;
+        if (len > i->count)
+                len = i->count;
+        if (len > maxsize)
+                len = maxsize;
+        *start = bvec->bv_offset + i->iov_offset;
+
+        get_page(*pages = bvec->bv_page);
+
+        return len;
+}
+
+static ssize_t get_pages_alloc_bvec(struct iov_iter *i,
+                   struct page ***pages, size_t maxsize,
+                   size_t *start)
+{
+        const struct bio_vec *bvec = i->bvec;
+        size_t len = bvec->bv_len - i->iov_offset;
+        if (len > i->count)
+                len = i->count;
+        if (len > maxsize)
+                len = maxsize;
+        *start = bvec->bv_offset + i->iov_offset;
+
+        *pages = kmalloc(sizeof(struct page *), GFP_KERNEL);
+        if (!*pages)
+                return -ENOMEM;
+
+        get_page(**pages = bvec->bv_page);
+
+        return len;
+}
+
+static int iov_iter_npages_bvec(const struct iov_iter *i, int maxpages)
+{
+        size_t offset = i->iov_offset;
+        size_t size = i->count;
+        const struct bio_vec *bvec = i->bvec;
+        int npages = 0;
+        int n;
+
+        for (n = 0; size && n < i->nr_segs; n++, bvec++) {
+                size_t len = bvec->bv_len - offset;
+                offset = 0;
+                if (unlikely(!len))     /* empty segment */
+                        continue;
+                if (len > size)
+                        len = size;
+                npages++;
+                if (npages >= maxpages) /* don't bother going further */
+                        return maxpages;
+                size -= len;
+                offset = 0;
+        }
+        return min(npages, maxpages);
+}
+
+size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
+                         struct iov_iter *i)
+{
+        if (i->type & ITER_BVEC)
+                return copy_page_to_iter_bvec(page, offset, bytes, i);
+        else
+                return copy_page_to_iter_iovec(page, offset, bytes, i);
+}
+EXPORT_SYMBOL(copy_page_to_iter);
+
+size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
+                         struct iov_iter *i)
+{
+        if (i->type & ITER_BVEC)
+                return copy_page_from_iter_bvec(page, offset, bytes, i);
+        else
+                return copy_page_from_iter_iovec(page, offset, bytes, i);
+}
+EXPORT_SYMBOL(copy_page_from_iter);
+
+size_t iov_iter_copy_from_user_atomic(struct page *page,
+                struct iov_iter *i, unsigned long offset, size_t bytes)
+{
+        if (i->type & ITER_BVEC)
+                return copy_from_user_bvec(page, i, offset, bytes);
+        else
+                return copy_from_user_atomic_iovec(page, i, offset, bytes);
+}
+EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
+
+void iov_iter_advance(struct iov_iter *i, size_t size)
+{
+        if (i->type & ITER_BVEC)
+                advance_bvec(i, size);
+        else
+                advance_iovec(i, size);
+}
+EXPORT_SYMBOL(iov_iter_advance);
+
 /*
  * Return the count of just the current iov_iter segment.
  */
 size_t iov_iter_single_seg_count(const struct iov_iter *i)
 {
-        const struct iovec *iov = i->iov;
         if (i->nr_segs == 1)
                 return i->count;
+        else if (i->type & ITER_BVEC)
+                return min(i->count, i->iov->iov_len - i->iov_offset);
         else
-                return min(i->count, iov->iov_len - i->iov_offset);
+                return min(i->count, i->bvec->bv_len - i->iov_offset);
 }
 EXPORT_SYMBOL(iov_iter_single_seg_count);
+
+unsigned long iov_iter_alignment(const struct iov_iter *i)
+{
+        if (i->type & ITER_BVEC)
+                return alignment_bvec(i);
+        else
+                return alignment_iovec(i);
+}
+EXPORT_SYMBOL(iov_iter_alignment);
+
+ssize_t iov_iter_get_pages(struct iov_iter *i,
+                   struct page **pages, size_t maxsize,
+                   size_t *start)
+{
+        if (i->type & ITER_BVEC)
+                return get_pages_bvec(i, pages, maxsize, start);
+        else
+                return get_pages_iovec(i, pages, maxsize, start);
+}
+EXPORT_SYMBOL(iov_iter_get_pages);
+
+ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
+                   struct page ***pages, size_t maxsize,
+                   size_t *start)
+{
+        if (i->type & ITER_BVEC)
+                return get_pages_alloc_bvec(i, pages, maxsize, start);
+        else
+                return get_pages_alloc_iovec(i, pages, maxsize, start);
+}
+EXPORT_SYMBOL(iov_iter_get_pages_alloc);
+
+int iov_iter_npages(const struct iov_iter *i, int maxpages)
+{
+        if (i->type & ITER_BVEC)
+                return iov_iter_npages_bvec(i, maxpages);
+        else
+                return iov_iter_npages_iovec(i, maxpages);
+}
+EXPORT_SYMBOL(iov_iter_npages);
diff --git a/mm/page_io.c b/mm/page_io.c
index 58b50d2901fe..243a9b76e5ce 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -264,9 +264,17 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc,
                 struct kiocb kiocb;
                 struct file *swap_file = sis->swap_file;
                 struct address_space *mapping = swap_file->f_mapping;
-                struct iovec iov = {
-                        .iov_base = kmap(page),
-                        .iov_len  = PAGE_SIZE,
+                struct bio_vec bv = {
+                        .bv_page = page,
+                        .bv_len  = PAGE_SIZE,
+                        .bv_offset = 0
+                };
+                struct iov_iter from = {
+                        .type = ITER_BVEC | WRITE,
+                        .count = PAGE_SIZE,
+                        .iov_offset = 0,
+                        .nr_segs = 1,
+                        .bvec = &bv
                 };
 
                 init_sync_kiocb(&kiocb, swap_file);
@@ -275,10 +283,9 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc,
 
                 set_page_writeback(page);
                 unlock_page(page);
-                ret = mapping->a_ops->direct_IO(KERNEL_WRITE,
-                                                &kiocb, &iov,
-                                                kiocb.ki_pos, 1);
-                kunmap(page);
+                ret = mapping->a_ops->direct_IO(ITER_BVEC | WRITE,
+                                                &kiocb, &from,
+                                                kiocb.ki_pos);
                 if (ret == PAGE_SIZE) {
                         count_vm_event(PSWPOUT);
                         ret = 0;
diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c
index 8505c9262b35..5077afcd9e11 100644
--- a/mm/process_vm_access.c
+++ b/mm/process_vm_access.c
@@ -46,11 +46,7 @@ static int process_vm_rw_pages(struct page **pages,
                         copy = len;
 
                 if (vm_write) {
-                        if (copy > iov_iter_count(iter))
-                                copy = iov_iter_count(iter);
-                        copied = iov_iter_copy_from_user(page, iter,
-                                        offset, copy);
-                        iov_iter_advance(iter, copied);
+                        copied = copy_page_from_iter(page, offset, copy, iter);
                         set_page_dirty_lock(page);
                 } else {
                         copied = copy_page_to_iter(page, offset, copy, iter);
@@ -278,7 +274,7 @@ static ssize_t process_vm_rw(pid_t pid,
         if (rc <= 0)
                 goto free_iovecs;
 
-        iov_iter_init(&iter, iov_l, liovcnt, rc, 0);
+        iov_iter_init(&iter, vm_write ? WRITE : READ, iov_l, liovcnt, rc);
 
         rc = rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt, UIO_FASTIOV,
                                    iovstack_r, &iov_r);
@@ -341,7 +337,7 @@ compat_process_vm_rw(compat_pid_t pid,
                                                   &iov_l);
         if (rc <= 0)
                 goto free_iovecs;
-        iov_iter_init(&iter, iov_l, liovcnt, rc, 0);
+        iov_iter_init(&iter, vm_write ? WRITE : READ, iov_l, liovcnt, rc);
         rc = compat_rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt,
                                           UIO_FASTIOV, iovstack_r,
                                           &iov_r);
diff --git a/mm/shmem.c b/mm/shmem.c
index 5402481c28d1..f484c276e994 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1406,8 +1406,7 @@ shmem_write_end(struct file *file, struct address_space *mapping,
         return copied;
 }
 
-static ssize_t shmem_file_aio_read(struct kiocb *iocb,
-                const struct iovec *iov, unsigned long nr_segs, loff_t pos)
+static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
         struct file *file = iocb->ki_filp;
         struct inode *inode = file_inode(file);
@@ -1416,15 +1415,8 @@ static ssize_t shmem_file_aio_read(struct kiocb *iocb,
         unsigned long offset;
         enum sgp_type sgp = SGP_READ;
         int error = 0;
-        ssize_t retval;
-        size_t count;
+        ssize_t retval = 0;
         loff_t *ppos = &iocb->ki_pos;
-        struct iov_iter iter;
-
-        retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
-        if (retval)
-                return retval;
-        iov_iter_init(&iter, iov, nr_segs, count, 0);
 
         /*
          * Might this read be for a stacking filesystem?  Then when reading
@@ -1500,14 +1492,14 @@ static ssize_t shmem_file_aio_read(struct kiocb *iocb,
                  * Ok, we have the page, and it's up-to-date, so
                  * now we can copy it to user space...
                  */
-                ret = copy_page_to_iter(page, offset, nr, &iter);
+                ret = copy_page_to_iter(page, offset, nr, to);
                 retval += ret;
                 offset += ret;
                 index += offset >> PAGE_CACHE_SHIFT;
                 offset &= ~PAGE_CACHE_MASK;
 
                 page_cache_release(page);
-                if (!iov_iter_count(&iter))
+                if (!iov_iter_count(to))
                         break;
                 if (ret < nr) {
                         error = -EFAULT;
@@ -2629,13 +2621,13 @@ static const struct file_operations shmem_file_operations = {
         .mmap           = shmem_mmap,
 #ifdef CONFIG_TMPFS
         .llseek         = shmem_file_llseek,
-        .read           = do_sync_read,
-        .write          = do_sync_write,
-        .aio_read       = shmem_file_aio_read,
-        .aio_write      = generic_file_aio_write,
+        .read           = new_sync_read,
+        .write          = new_sync_write,
+        .read_iter      = shmem_file_read_iter,
+        .write_iter     = generic_file_write_iter,
         .fsync          = noop_fsync,
         .splice_read    = shmem_file_splice_read,
-        .splice_write   = generic_file_splice_write,
+        .splice_write   = iter_file_splice_write,
         .fallocate      = shmem_fallocate,
 #endif
 };
diff --git a/mm/vmscan.c b/mm/vmscan.c
index e01ded365440..0f16ffe8eb67 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -464,7 +464,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping,
          * stalls if we need to run get_block().  We could test
          * PagePrivate for that.
          *
-         * If this process is currently in __generic_file_aio_write() against
+         * If this process is currently in __generic_file_write_iter() against
          * this page's queue, we can perform writeback even if that
          * will block.
          *