1 files changed, 159 insertions, 43 deletions

diff --git a/fs/splice.c b/fs/splice.c
index 7c2bbf18d7a7..bfa42a277bb8 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -22,7 +22,10 @@
 #include <linux/pipe_fs_i.h>
 #include <linux/mm_inline.h>
 #include <linux/swap.h>
+#include <linux/writeback.h>
+#include <linux/buffer_head.h>
 #include <linux/module.h>
+#include <linux/syscalls.h>
 
 /*
  * Passed to the actors
@@ -34,28 +37,37 @@ struct splice_desc {
         loff_t pos;                     /* file position */
 };
 
+/*
+ * Attempt to steal a page from a pipe buffer. This should perhaps go into
+ * a vm helper function, it's already simplified quite a bit by the
+ * addition of remove_mapping(). If success is returned, the caller may
+ * attempt to reuse this page for another destination.
+ */
 static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
                                      struct pipe_buffer *buf)
 {
         struct page *page = buf->page;
+        struct address_space *mapping = page_mapping(page);
 
         WARN_ON(!PageLocked(page));
         WARN_ON(!PageUptodate(page));
 
-        if (!remove_mapping(page_mapping(page), page))
-                return 1;
+        /*
+         * At least for ext2 with nobh option, we need to wait on writeback
+         * completing on this page, since we'll remove it from the pagecache.
+         * Otherwise truncate wont wait on the page, allowing the disk
+         * blocks to be reused by someone else before we actually wrote our
+         * data to them. fs corruption ensues.
+         */
+        wait_on_page_writeback(page);
 
-        if (PageLRU(page)) {
-                struct zone *zone = page_zone(page);
+        if (PagePrivate(page))
+                try_to_release_page(page, mapping_gfp_mask(mapping));
 
-                spin_lock_irq(&zone->lru_lock);
-                BUG_ON(!PageLRU(page));
-                __ClearPageLRU(page);
-                del_page_from_lru(zone, page);
-                spin_unlock_irq(&zone->lru_lock);
-        }
+        if (!remove_mapping(mapping, page))
+                return 1;
 
-        buf->stolen = 1;
+        buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
         return 0;
 }
 
@@ -64,7 +76,7 @@ static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
 {
         page_cache_release(buf->page);
         buf->page = NULL;
-        buf->stolen = 0;
+        buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
 }
 
 static void *page_cache_pipe_buf_map(struct file *file,
@@ -91,8 +103,7 @@ static void *page_cache_pipe_buf_map(struct file *file,
 static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
                                       struct pipe_buffer *buf)
 {
-        if (!buf->stolen)
-                unlock_page(buf->page);
+        unlock_page(buf->page);
         kunmap(buf->page);
 }
 
@@ -104,9 +115,13 @@ static struct pipe_buf_operations page_cache_pipe_buf_ops = {
         .steal = page_cache_pipe_buf_steal,
 };
 
+/*
+ * Pipe output worker. This sets up our pipe format with the page cache
+ * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
+ */
 static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
                             int nr_pages, unsigned long offset,
-                            unsigned long len)
+                            unsigned long len, unsigned int flags)
 {
         struct pipe_inode_info *info;
         int ret, do_wakeup, i;
@@ -159,6 +174,12 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
                         break;
                 }
 
+                if (flags & SPLICE_F_NONBLOCK) {
+                        if (!ret)
+                                ret = -EAGAIN;
+                        break;
+                }
+
                 if (signal_pending(current)) {
                         if (!ret)
                                 ret = -ERESTARTSYS;
@@ -191,7 +212,7 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
 }
 
 static int __generic_file_splice_read(struct file *in, struct inode *pipe,
-                                      size_t len)
+                                      size_t len, unsigned int flags)
 {
         struct address_space *mapping = in->f_mapping;
         unsigned int offset, nr_pages;
@@ -231,9 +252,9 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe,
          * fill shadow[] with pages at the right locations, so we only
          * have to fill holes
          */
-        memset(shadow, 0, i * sizeof(struct page *));
-        for (j = 0, pidx = index; j < i; pidx++, j++)
-                shadow[pages[j]->index - pidx] = pages[j];
+        memset(shadow, 0, nr_pages * sizeof(struct page *));
+        for (j = 0; j < i; j++)
+                shadow[pages[j]->index - index] = pages[j];
 
         /*
          * now fill in the holes
@@ -279,9 +300,19 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe,
          * Now we splice them into the pipe..
          */
 splice_them:
-        return move_to_pipe(pipe, pages, i, offset, len);
+        return move_to_pipe(pipe, pages, i, offset, len, flags);
 }
 
+/**
+ * generic_file_splice_read - splice data from file to a pipe
+ * @in:         file to splice from
+ * @pipe:       pipe to splice to
+ * @len:        number of bytes to splice
+ * @flags:      splice modifier flags
+ *
+ * Will read pages from given file and fill them into a pipe.
+ *
+ */
 ssize_t generic_file_splice_read(struct file *in, struct inode *pipe,
                                  size_t len, unsigned int flags)
 {
@@ -291,7 +322,7 @@ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe,
         ret = 0;
         spliced = 0;
         while (len) {
-                ret = __generic_file_splice_read(in, pipe, len);
+                ret = __generic_file_splice_read(in, pipe, len, flags);
 
                 if (ret <= 0)
                         break;
@@ -299,6 +330,11 @@ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe,
                 in->f_pos += ret;
                 len -= ret;
                 spliced += ret;
+
+                if (!(flags & SPLICE_F_NONBLOCK))
+                        continue;
+                ret = -EAGAIN;
+                break;
         }
 
         if (spliced)
@@ -307,8 +343,11 @@ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe,
         return ret;
 }
 
+EXPORT_SYMBOL(generic_file_splice_read);
+
 /*
- * Send 'len' bytes to socket from 'file' at position 'pos' using sendpage().
+ * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
+ * using sendpage().
  */
 static int pipe_to_sendpage(struct pipe_inode_info *info,
                             struct pipe_buffer *buf, struct splice_desc *sd)
@@ -318,6 +357,7 @@ static int pipe_to_sendpage(struct pipe_inode_info *info,
         unsigned int offset;
         ssize_t ret;
         void *ptr;
+        int more;
 
         /*
          * sub-optimal, but we are limited by the pipe ->map. we don't
@@ -330,9 +370,9 @@ static int pipe_to_sendpage(struct pipe_inode_info *info,
                 return PTR_ERR(ptr);
 
         offset = pos & ~PAGE_CACHE_MASK;
+        more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
 
-        ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,
-                                        sd->len < sd->total_len);
+        ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more);
 
         buf->ops->unmap(info, buf);
         if (ret == sd->len)
@@ -354,16 +394,19 @@ static int pipe_to_sendpage(struct pipe_inode_info *info,
  *      - Destination page does not exist, we can add the pipe page to
  *        the page cache and avoid the copy.
  *
- * For now we just do the slower thing and always copy pages over, it's
- * easier than migrating pages from the pipe to the target file. For the
- * case of doing file | file splicing, the migrate approach had some LRU
- * nastiness...
+ * If asked to move pages to the output file (SPLICE_F_MOVE is set in
+ * sd->flags), we attempt to migrate pages from the pipe to the output
+ * file address space page cache. This is possible if no one else has
+ * the pipe page referenced outside of the pipe and page cache. If
+ * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
+ * a new page in the output file page cache and fill/dirty that.
  */
 static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
                         struct splice_desc *sd)
 {
         struct file *file = sd->file;
         struct address_space *mapping = file->f_mapping;
+        gfp_t gfp_mask = mapping_gfp_mask(mapping);
         unsigned int offset;
         struct page *page;
         pgoff_t index;
@@ -384,18 +427,23 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
          * reuse buf page, if SPLICE_F_MOVE is set
          */
         if (sd->flags & SPLICE_F_MOVE) {
+                /*
+                 * If steal succeeds, buf->page is now pruned from the vm
+                 * side (LRU and page cache) and we can reuse it.
+                 */
                 if (buf->ops->steal(info, buf))
                         goto find_page;
 
                 page = buf->page;
-                if (add_to_page_cache_lru(page, mapping, index,
-                                                mapping_gfp_mask(mapping)))
+                if (add_to_page_cache(page, mapping, index, gfp_mask))
                         goto find_page;
+
+                if (!(buf->flags & PIPE_BUF_FLAG_LRU))
+                        lru_cache_add(page);
         } else {
 find_page:
                 ret = -ENOMEM;
-                page = find_or_create_page(mapping, index,
-                                                mapping_gfp_mask(mapping));
+                page = find_or_create_page(mapping, index, gfp_mask);
                 if (!page)
                         goto out;
 
@@ -432,10 +480,13 @@ find_page:
         }
 
         ret = mapping->a_ops->prepare_write(file, page, 0, sd->len);
-        if (ret)
+        if (ret == AOP_TRUNCATED_PAGE) {
+                page_cache_release(page);
+                goto find_page;
+        } else if (ret)
                 goto out;
 
-        if (!buf->stolen) {
+        if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
                 char *dst = kmap_atomic(page, KM_USER0);
 
                 memcpy(dst + offset, src + buf->offset, sd->len);
@@ -444,16 +495,18 @@ find_page:
         }
 
         ret = mapping->a_ops->commit_write(file, page, 0, sd->len);
-        if (ret < 0)
+        if (ret == AOP_TRUNCATED_PAGE) {
+                page_cache_release(page);
+                goto find_page;
+        } else if (ret)
                 goto out;
 
-        set_page_dirty(page);
-        ret = write_one_page(page, 0);
+        balance_dirty_pages_ratelimited(mapping);
 out:
-        if (ret < 0)
-                unlock_page(page);
-        if (!buf->stolen)
+        if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
                 page_cache_release(page);
+                unlock_page(page);
+        }
         buf->ops->unmap(info, buf);
         return ret;
 }
@@ -461,6 +514,11 @@ out:
 typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
                            struct splice_desc *);
 
+/*
+ * Pipe input worker. Most of this logic works like a regular pipe, the
+ * key here is the 'actor' worker passed in that actually moves the data
+ * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
+ */
 static ssize_t move_from_pipe(struct inode *inode, struct file *out,
                               size_t len, unsigned int flags,
                               splice_actor *actor)
@@ -527,6 +585,12 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out,
                                 break;
                 }
 
+                if (flags & SPLICE_F_NONBLOCK) {
+                        if (!ret)
+                                ret = -EAGAIN;
+                        break;
+                }
+
                 if (signal_pending(current)) {
                         if (!ret)
                                 ret = -ERESTARTSYS;
@@ -556,21 +620,67 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out,
 
 }
 
+/**
+ * generic_file_splice_write - splice data from a pipe to a file
+ * @inode:      pipe inode
+ * @out:        file to write to
+ * @len:        number of bytes to splice
+ * @flags:      splice modifier flags
+ *
+ * Will either move or copy pages (determined by @flags options) from
+ * the given pipe inode to the given file.
+ *
+ */
 ssize_t generic_file_splice_write(struct inode *inode, struct file *out,
                                   size_t len, unsigned int flags)
 {
-        return move_from_pipe(inode, out, len, flags, pipe_to_file);
+        struct address_space *mapping = out->f_mapping;
+        ssize_t ret = move_from_pipe(inode, out, len, flags, pipe_to_file);
+
+        /*
+         * if file or inode is SYNC and we actually wrote some data, sync it
+         */
+        if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host))
+            && ret > 0) {
+                struct inode *inode = mapping->host;
+                int err;
+
+                mutex_lock(&inode->i_mutex);
+                err = generic_osync_inode(mapping->host, mapping,
+                                                OSYNC_METADATA|OSYNC_DATA);
+                mutex_unlock(&inode->i_mutex);
+
+                if (err)
+                        ret = err;
+        }
+
+        return ret;
 }
 
+EXPORT_SYMBOL(generic_file_splice_write);
+
+/**
+ * generic_splice_sendpage - splice data from a pipe to a socket
+ * @inode:      pipe inode
+ * @out:        socket to write to
+ * @len:        number of bytes to splice
+ * @flags:      splice modifier flags
+ *
+ * Will send @len bytes from the pipe to a network socket. No data copying
+ * is involved.
+ *
+ */
 ssize_t generic_splice_sendpage(struct inode *inode, struct file *out,
                                 size_t len, unsigned int flags)
 {
         return move_from_pipe(inode, out, len, flags, pipe_to_sendpage);
 }
 
-EXPORT_SYMBOL(generic_file_splice_write);
-EXPORT_SYMBOL(generic_file_splice_read);
+EXPORT_SYMBOL(generic_splice_sendpage);
 
+/*
+ * Attempt to initiate a splice from pipe to file.
+ */
 static long do_splice_from(struct inode *pipe, struct file *out, size_t len,
                            unsigned int flags)
 {
@@ -591,6 +701,9 @@ static long do_splice_from(struct inode *pipe, struct file *out, size_t len,
         return out->f_op->splice_write(pipe, out, len, flags);
 }
 
+/*
+ * Attempt to initiate a splice from a file to a pipe.
+ */
 static long do_splice_to(struct file *in, struct inode *pipe, size_t len,
                          unsigned int flags)
 {
@@ -619,6 +732,9 @@ static long do_splice_to(struct file *in, struct inode *pipe, size_t len,
         return in->f_op->splice_read(in, pipe, len, flags);
 }
 
+/*
+ * Determine where to splice to/from.
+ */
 static long do_splice(struct file *in, struct file *out, size_t len,
                       unsigned int flags)
 {