1 files changed, 338 insertions, 154 deletions

diff --git a/fs/splice.c b/fs/splice.c
index bfa42a277bb8..e50a460239dd 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -9,11 +9,12 @@
  * that transfers data buffers to or from a pipe buffer.
  *
  * Named by Larry McVoy, original implementation from Linus, extended by
- * Jens to support splicing to files and fixing the initial implementation
- * bugs.
+ * Jens to support splicing to files, network, direct splicing, etc and
+ * fixing lots of bugs.
  *
- * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
- * Copyright (C) 2005 Linus Torvalds <torvalds@osdl.org>
+ * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de>
+ * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
+ * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
  *
  */
 #include <linux/fs.h>
@@ -84,26 +85,43 @@ static void *page_cache_pipe_buf_map(struct file *file,
                                      struct pipe_buffer *buf)
 {
         struct page *page = buf->page;
-
-        lock_page(page);
+        int err;
 
         if (!PageUptodate(page)) {
-                unlock_page(page);
-                return ERR_PTR(-EIO);
-        }
+                lock_page(page);
+
+                /*
+                 * Page got truncated/unhashed. This will cause a 0-byte
+                 * splice, if this is the first page.
+                 */
+                if (!page->mapping) {
+                        err = -ENODATA;
+                        goto error;
+                }
 
-        if (!page->mapping) {
+                /*
+                 * Uh oh, read-error from disk.
+                 */
+                if (!PageUptodate(page)) {
+                        err = -EIO;
+                        goto error;
+                }
+
+                /*
+                 * Page is ok afterall, fall through to mapping.
+                 */
                 unlock_page(page);
-                return ERR_PTR(-ENODATA);
         }
 
-        return kmap(buf->page);
+        return kmap(page);
+error:
+        unlock_page(page);
+        return ERR_PTR(err);
 }
 
 static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
                                       struct pipe_buffer *buf)
 {
-        unlock_page(buf->page);
         kunmap(buf->page);
 }
 
@@ -119,34 +137,30 @@ static struct pipe_buf_operations page_cache_pipe_buf_ops = {
  * Pipe output worker. This sets up our pipe format with the page cache
  * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
  */
-static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
+static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
                             int nr_pages, unsigned long offset,
                             unsigned long len, unsigned int flags)
 {
-        struct pipe_inode_info *info;
         int ret, do_wakeup, i;
 
         ret = 0;
         do_wakeup = 0;
         i = 0;
 
-        mutex_lock(PIPE_MUTEX(*inode));
+        if (pipe->inode)
+                mutex_lock(&pipe->inode->i_mutex);
 
-        info = inode->i_pipe;
         for (;;) {
-                int bufs;
-
-                if (!PIPE_READERS(*inode)) {
+                if (!pipe->readers) {
                         send_sig(SIGPIPE, current, 0);
                         if (!ret)
                                 ret = -EPIPE;
                         break;
                 }
 
-                bufs = info->nrbufs;
-                if (bufs < PIPE_BUFFERS) {
-                        int newbuf = (info->curbuf + bufs) & (PIPE_BUFFERS - 1);
-                        struct pipe_buffer *buf = info->bufs + newbuf;
+                if (pipe->nrbufs < PIPE_BUFFERS) {
+                        int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
+                        struct pipe_buffer *buf = pipe->bufs + newbuf;
                         struct page *page = pages[i++];
                         unsigned long this_len;
 
@@ -158,8 +172,9 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
                         buf->offset = offset;
                         buf->len = this_len;
                         buf->ops = &page_cache_pipe_buf_ops;
-                        info->nrbufs = ++bufs;
-                        do_wakeup = 1;
+                        pipe->nrbufs++;
+                        if (pipe->inode)
+                                do_wakeup = 1;
 
                         ret += this_len;
                         len -= this_len;
@@ -168,7 +183,7 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
                                 break;
                         if (!len)
                                 break;
-                        if (bufs < PIPE_BUFFERS)
+                        if (pipe->nrbufs < PIPE_BUFFERS)
                                 continue;
 
                         break;
@@ -187,22 +202,26 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
                 }
 
                 if (do_wakeup) {
-                        wake_up_interruptible_sync(PIPE_WAIT(*inode));
-                        kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO,
-                                    POLL_IN);
+                        smp_mb();
+                        if (waitqueue_active(&pipe->wait))
+                                wake_up_interruptible_sync(&pipe->wait);
+                        kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
                         do_wakeup = 0;
                 }
 
-                PIPE_WAITING_WRITERS(*inode)++;
-                pipe_wait(inode);
-                PIPE_WAITING_WRITERS(*inode)--;
+                pipe->waiting_writers++;
+                pipe_wait(pipe);
+                pipe->waiting_writers--;
         }
 
-        mutex_unlock(PIPE_MUTEX(*inode));
+        if (pipe->inode)
+                mutex_unlock(&pipe->inode->i_mutex);
 
         if (do_wakeup) {
-                wake_up_interruptible(PIPE_WAIT(*inode));
-                kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN);
+                smp_mb();
+                if (waitqueue_active(&pipe->wait))
+                        wake_up_interruptible(&pipe->wait);
+                kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
         }
 
         while (i < nr_pages)
@@ -211,15 +230,16 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
         return ret;
 }
 
-static int __generic_file_splice_read(struct file *in, struct inode *pipe,
-                                      size_t len, unsigned int flags)
+static int
+__generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
+                           size_t len, unsigned int flags)
 {
         struct address_space *mapping = in->f_mapping;
         unsigned int offset, nr_pages;
-        struct page *pages[PIPE_BUFFERS], *shadow[PIPE_BUFFERS];
+        struct page *pages[PIPE_BUFFERS];
         struct page *page;
-        pgoff_t index, pidx;
-        int i, j;
+        pgoff_t index;
+        int i, error;
 
         index = in->f_pos >> PAGE_CACHE_SHIFT;
         offset = in->f_pos & ~PAGE_CACHE_MASK;
@@ -229,78 +249,94 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe,
                 nr_pages = PIPE_BUFFERS;
 
         /*
-         * initiate read-ahead on this page range
-         */
-        do_page_cache_readahead(mapping, in, index, nr_pages);
-
-        /*
-         * Get as many pages from the page cache as possible..
-         * Start IO on the page cache entries we create (we
-         * can assume that any pre-existing ones we find have
-         * already had IO started on them).
+         * Initiate read-ahead on this page range. however, don't call into
+         * read-ahead if this is a non-zero offset (we are likely doing small
+         * chunk splice and the page is already there) for a single page.
          */
-        i = find_get_pages(mapping, index, nr_pages, pages);
+        if (!offset || nr_pages > 1)
+                do_page_cache_readahead(mapping, in, index, nr_pages);
 
         /*
-         * common case - we found all pages and they are contiguous,
-         * kick them off
+         * Now fill in the holes:
          */
-        if (i && (pages[i - 1]->index == index + i - 1))
-                goto splice_them;
+        error = 0;
+        for (i = 0; i < nr_pages; i++, index++) {
+find_page:
+                /*
+                 * lookup the page for this index
+                 */
+                page = find_get_page(mapping, index);
+                if (!page) {
+                        /*
+                         * If in nonblock mode then dont block on
+                         * readpage (we've kicked readahead so there
+                         * will be asynchronous progress):
+                         */
+                        if (flags & SPLICE_F_NONBLOCK)
+                                break;
 
-        /*
-         * fill shadow[] with pages at the right locations, so we only
-         * have to fill holes
-         */
-        memset(shadow, 0, nr_pages * sizeof(struct page *));
-        for (j = 0; j < i; j++)
-                shadow[pages[j]->index - index] = pages[j];
+                        /*
+                         * page didn't exist, allocate one
+                         */
+                        page = page_cache_alloc_cold(mapping);
+                        if (!page)
+                                break;
 
-        /*
-         * now fill in the holes
-         */
-        for (i = 0, pidx = index; i < nr_pages; pidx++, i++) {
-                int error;
+                        error = add_to_page_cache_lru(page, mapping, index,
+                                                mapping_gfp_mask(mapping));
+                        if (unlikely(error)) {
+                                page_cache_release(page);
+                                break;
+                        }
 
-                if (shadow[i])
-                        continue;
+                        goto readpage;
+                }
 
                 /*
-                 * no page there, look one up / create it
+                 * If the page isn't uptodate, we may need to start io on it
                  */
-                page = find_or_create_page(mapping, pidx,
-                                                   mapping_gfp_mask(mapping));
-                if (!page)
-                        break;
+                if (!PageUptodate(page)) {
+                        lock_page(page);
+
+                        /*
+                         * page was truncated, stop here. if this isn't the
+                         * first page, we'll just complete what we already
+                         * added
+                         */
+                        if (!page->mapping) {
+                                unlock_page(page);
+                                page_cache_release(page);
+                                break;
+                        }
+                        /*
+                         * page was already under io and is now done, great
+                         */
+                        if (PageUptodate(page)) {
+                                unlock_page(page);
+                                goto fill_it;
+                        }
 
-                if (PageUptodate(page))
-                        unlock_page(page);
-                else {
+readpage:
+                        /*
+                         * need to read in the page
+                         */
                         error = mapping->a_ops->readpage(in, page);
 
                         if (unlikely(error)) {
                                 page_cache_release(page);
+                                if (error == AOP_TRUNCATED_PAGE)
+                                        goto find_page;
                                 break;
                         }
                 }
-                shadow[i] = page;
+fill_it:
+                pages[i] = page;
         }
 
-        if (!i) {
-                for (i = 0; i < nr_pages; i++) {
-                         if (shadow[i])
-                                page_cache_release(shadow[i]);
-                }
-                return 0;
-        }
+        if (i)
+                return move_to_pipe(pipe, pages, i, offset, len, flags);
 
-        memcpy(pages, shadow, i * sizeof(struct page *));
-
-        /*
-         * Now we splice them into the pipe..
-         */
-splice_them:
-        return move_to_pipe(pipe, pages, i, offset, len, flags);
+        return error;
 }
 
 /**
@@ -311,9 +347,8 @@ splice_them:
  * @flags:      splice modifier flags
  *
  * Will read pages from given file and fill them into a pipe.
- *
  */
-ssize_t generic_file_splice_read(struct file *in, struct inode *pipe,
+ssize_t generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
                                  size_t len, unsigned int flags)
 {
         ssize_t spliced;
@@ -321,6 +356,7 @@ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe,
 
         ret = 0;
         spliced = 0;
+
         while (len) {
                 ret = __generic_file_splice_read(in, pipe, len, flags);
 
@@ -360,10 +396,10 @@ static int pipe_to_sendpage(struct pipe_inode_info *info,
         int more;
 
         /*
-         * sub-optimal, but we are limited by the pipe ->map. we don't
+         * Sub-optimal, but we are limited by the pipe ->map. We don't
          * need a kmap'ed buffer here, we just want to make sure we
          * have the page pinned if the pipe page originates from the
-         * page cache
+         * page cache.
          */
         ptr = buf->ops->map(file, info, buf);
         if (IS_ERR(ptr))
@@ -414,7 +450,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
         int ret;
 
         /*
-         * after this, page will be locked and unmapped
+         * make sure the data in this buffer is uptodate
          */
         src = buf->ops->map(file, info, buf);
         if (IS_ERR(src))
@@ -424,7 +460,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
         offset = sd->pos & ~PAGE_CACHE_MASK;
 
         /*
-         * reuse buf page, if SPLICE_F_MOVE is set
+         * Reuse buf page, if SPLICE_F_MOVE is set.
          */
         if (sd->flags & SPLICE_F_MOVE) {
                 /*
@@ -434,6 +470,9 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
                 if (buf->ops->steal(info, buf))
                         goto find_page;
 
+                /*
+                 * this will also set the page locked
+                 */
                 page = buf->page;
                 if (add_to_page_cache(page, mapping, index, gfp_mask))
                         goto find_page;
@@ -445,7 +484,7 @@ find_page:
                 ret = -ENOMEM;
                 page = find_or_create_page(mapping, index, gfp_mask);
                 if (!page)
-                        goto out;
+                        goto out_nomem;
 
                 /*
                  * If the page is uptodate, it is also locked. If it isn't
@@ -462,7 +501,7 @@ find_page:
 
                                 if (!PageUptodate(page)) {
                                         /*
-                                         * page got invalidated, repeat
+                                         * Page got invalidated, repeat.
                                          */
                                         if (!page->mapping) {
                                                 unlock_page(page);
@@ -501,12 +540,14 @@ find_page:
         } else if (ret)
                 goto out;
 
+        mark_page_accessed(page);
         balance_dirty_pages_ratelimited(mapping);
 out:
         if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
                 page_cache_release(page);
                 unlock_page(page);
         }
+out_nomem:
         buf->ops->unmap(info, buf);
         return ret;
 }
@@ -519,11 +560,10 @@ typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
  * key here is the 'actor' worker passed in that actually moves the data
  * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
  */
-static ssize_t move_from_pipe(struct inode *inode, struct file *out,
+static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
                               size_t len, unsigned int flags,
                               splice_actor *actor)
 {
-        struct pipe_inode_info *info;
         int ret, do_wakeup, err;
         struct splice_desc sd;
 
@@ -535,22 +575,19 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out,
         sd.file = out;
         sd.pos = out->f_pos;
 
-        mutex_lock(PIPE_MUTEX(*inode));
+        if (pipe->inode)
+                mutex_lock(&pipe->inode->i_mutex);
 
-        info = inode->i_pipe;
         for (;;) {
-                int bufs = info->nrbufs;
-
-                if (bufs) {
-                        int curbuf = info->curbuf;
-                        struct pipe_buffer *buf = info->bufs + curbuf;
+                if (pipe->nrbufs) {
+                        struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
                         struct pipe_buf_operations *ops = buf->ops;
 
                         sd.len = buf->len;
                         if (sd.len > sd.total_len)
                                 sd.len = sd.total_len;
 
-                        err = actor(info, buf, &sd);
+                        err = actor(pipe, buf, &sd);
                         if (err) {
                                 if (!ret && err != -ENODATA)
                                         ret = err;
@@ -561,13 +598,14 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out,
                         ret += sd.len;
                         buf->offset += sd.len;
                         buf->len -= sd.len;
+
                         if (!buf->len) {
                                 buf->ops = NULL;
-                                ops->release(info, buf);
-                                curbuf = (curbuf + 1) & (PIPE_BUFFERS - 1);
-                                info->curbuf = curbuf;
-                                info->nrbufs = --bufs;
-                                do_wakeup = 1;
+                                ops->release(pipe, buf);
+                                pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
+                                pipe->nrbufs--;
+                                if (pipe->inode)
+                                        do_wakeup = 1;
                         }
 
                         sd.pos += sd.len;
@@ -576,11 +614,11 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out,
                                 break;
                 }
 
-                if (bufs)
+                if (pipe->nrbufs)
                         continue;
-                if (!PIPE_WRITERS(*inode))
+                if (!pipe->writers)
                         break;
-                if (!PIPE_WAITING_WRITERS(*inode)) {
+                if (!pipe->waiting_writers) {
                         if (ret)
                                 break;
                 }
@@ -598,31 +636,34 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out,
                 }
 
                 if (do_wakeup) {
-                        wake_up_interruptible_sync(PIPE_WAIT(*inode));
-                        kill_fasync(PIPE_FASYNC_WRITERS(*inode),SIGIO,POLL_OUT);
+                        smp_mb();
+                        if (waitqueue_active(&pipe->wait))
+                                wake_up_interruptible_sync(&pipe->wait);
+                        kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
                         do_wakeup = 0;
                 }
 
-                pipe_wait(inode);
+                pipe_wait(pipe);
         }
 
-        mutex_unlock(PIPE_MUTEX(*inode));
+        if (pipe->inode)
+                mutex_unlock(&pipe->inode->i_mutex);
 
         if (do_wakeup) {
-                wake_up_interruptible(PIPE_WAIT(*inode));
-                kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT);
+                smp_mb();
+                if (waitqueue_active(&pipe->wait))
+                        wake_up_interruptible(&pipe->wait);
+                kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
         }
 
-        mutex_lock(&out->f_mapping->host->i_mutex);
         out->f_pos = sd.pos;
-        mutex_unlock(&out->f_mapping->host->i_mutex);
         return ret;
 
 }
 
 /**
  * generic_file_splice_write - splice data from a pipe to a file
- * @inode:      pipe inode
+ * @pipe:       pipe info
  * @out:        file to write to
  * @len:        number of bytes to splice
  * @flags:      splice modifier flags
@@ -631,14 +672,17 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out,
  * the given pipe inode to the given file.
  *
  */
-ssize_t generic_file_splice_write(struct inode *inode, struct file *out,
-                                  size_t len, unsigned int flags)
+ssize_t
+generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
+                          size_t len, unsigned int flags)
 {
         struct address_space *mapping = out->f_mapping;
-        ssize_t ret = move_from_pipe(inode, out, len, flags, pipe_to_file);
+        ssize_t ret;
+
+        ret = move_from_pipe(pipe, out, len, flags, pipe_to_file);
 
         /*
-         * if file or inode is SYNC and we actually wrote some data, sync it
+         * If file or inode is SYNC and we actually wrote some data, sync it.
          */
         if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host))
             && ret > 0) {
@@ -647,7 +691,7 @@ ssize_t generic_file_splice_write(struct inode *inode, struct file *out,
 
                 mutex_lock(&inode->i_mutex);
                 err = generic_osync_inode(mapping->host, mapping,
-                                                OSYNC_METADATA|OSYNC_DATA);
+                                          OSYNC_METADATA|OSYNC_DATA);
                 mutex_unlock(&inode->i_mutex);
 
                 if (err)
@@ -670,10 +714,10 @@ EXPORT_SYMBOL(generic_file_splice_write);
  * is involved.
  *
  */
-ssize_t generic_splice_sendpage(struct inode *inode, struct file *out,
+ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
                                 size_t len, unsigned int flags)
 {
-        return move_from_pipe(inode, out, len, flags, pipe_to_sendpage);
+        return move_from_pipe(pipe, out, len, flags, pipe_to_sendpage);
 }
 
 EXPORT_SYMBOL(generic_splice_sendpage);
@@ -681,19 +725,20 @@ EXPORT_SYMBOL(generic_splice_sendpage);
 /*
  * Attempt to initiate a splice from pipe to file.
  */
-static long do_splice_from(struct inode *pipe, struct file *out, size_t len,
-                           unsigned int flags)
+static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
+                           size_t len, unsigned int flags)
 {
         loff_t pos;
         int ret;
 
-        if (!out->f_op || !out->f_op->splice_write)
+        if (unlikely(!out->f_op || !out->f_op->splice_write))
                 return -EINVAL;
 
-        if (!(out->f_mode & FMODE_WRITE))
+        if (unlikely(!(out->f_mode & FMODE_WRITE)))
                 return -EBADF;
 
         pos = out->f_pos;
+
         ret = rw_verify_area(WRITE, out, &pos, len);
         if (unlikely(ret < 0))
                 return ret;
@@ -704,19 +749,20 @@ static long do_splice_from(struct inode *pipe, struct file *out, size_t len,
 /*
  * Attempt to initiate a splice from a file to a pipe.
  */
-static long do_splice_to(struct file *in, struct inode *pipe, size_t len,
-                         unsigned int flags)
+static long do_splice_to(struct file *in, struct pipe_inode_info *pipe,
+                         size_t len, unsigned int flags)
 {
         loff_t pos, isize, left;
         int ret;
 
-        if (!in->f_op || !in->f_op->splice_read)
+        if (unlikely(!in->f_op || !in->f_op->splice_read))
                 return -EINVAL;
 
-        if (!(in->f_mode & FMODE_READ))
+        if (unlikely(!(in->f_mode & FMODE_READ)))
                 return -EBADF;
 
         pos = in->f_pos;
+
         ret = rw_verify_area(READ, in, &pos, len);
         if (unlikely(ret < 0))
                 return ret;
@@ -726,32 +772,168 @@ static long do_splice_to(struct file *in, struct inode *pipe, size_t len,
                 return 0;
         
         left = isize - in->f_pos;
-        if (left < len)
+        if (unlikely(left < len))
                 len = left;
 
         return in->f_op->splice_read(in, pipe, len, flags);
 }
 
+long do_splice_direct(struct file *in, struct file *out, size_t len,
+                      unsigned int flags)
+{
+        struct pipe_inode_info *pipe;
+        long ret, bytes;
+        umode_t i_mode;
+        int i;
+
+        /*
+         * We require the input being a regular file, as we don't want to
+         * randomly drop data for eg socket -> socket splicing. Use the
+         * piped splicing for that!
+         */
+        i_mode = in->f_dentry->d_inode->i_mode;
+        if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
+                return -EINVAL;
+
+        /*
+         * neither in nor out is a pipe, setup an internal pipe attached to
+         * 'out' and transfer the wanted data from 'in' to 'out' through that
+         */
+        pipe = current->splice_pipe;
+        if (unlikely(!pipe)) {
+                pipe = alloc_pipe_info(NULL);
+                if (!pipe)
+                        return -ENOMEM;
+
+                /*
+                 * We don't have an immediate reader, but we'll read the stuff
+                 * out of the pipe right after the move_to_pipe(). So set
+                 * PIPE_READERS appropriately.
+                 */
+                pipe->readers = 1;
+
+                current->splice_pipe = pipe;
+        }
+
+        /*
+         * Do the splice.
+         */
+        ret = 0;
+        bytes = 0;
+
+        while (len) {
+                size_t read_len, max_read_len;
+
+                /*
+                 * Do at most PIPE_BUFFERS pages worth of transfer:
+                 */
+                max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
+
+                ret = do_splice_to(in, pipe, max_read_len, flags);
+                if (unlikely(ret < 0))
+                        goto out_release;
+
+                read_len = ret;
+
+                /*
+                 * NOTE: nonblocking mode only applies to the input. We
+                 * must not do the output in nonblocking mode as then we
+                 * could get stuck data in the internal pipe:
+                 */
+                ret = do_splice_from(pipe, out, read_len,
+                                     flags & ~SPLICE_F_NONBLOCK);
+                if (unlikely(ret < 0))
+                        goto out_release;
+
+                bytes += ret;
+                len -= ret;
+
+                /*
+                 * In nonblocking mode, if we got back a short read then
+                 * that was due to either an IO error or due to the
+                 * pagecache entry not being there. In the IO error case
+                 * the _next_ splice attempt will produce a clean IO error
+                 * return value (not a short read), so in both cases it's
+                 * correct to break out of the loop here:
+                 */
+                if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
+                        break;
+        }
+
+        pipe->nrbufs = pipe->curbuf = 0;
+
+        return bytes;
+
+out_release:
+        /*
+         * If we did an incomplete transfer we must release
+         * the pipe buffers in question:
+         */
+        for (i = 0; i < PIPE_BUFFERS; i++) {
+                struct pipe_buffer *buf = pipe->bufs + i;
+
+                if (buf->ops) {
+                        buf->ops->release(pipe, buf);
+                        buf->ops = NULL;
+                }
+        }
+        pipe->nrbufs = pipe->curbuf = 0;
+
+        /*
+         * If we transferred some data, return the number of bytes:
+         */
+        if (bytes > 0)
+                return bytes;
+
+        return ret;
+}
+
+EXPORT_SYMBOL(do_splice_direct);
+
 /*
  * Determine where to splice to/from.
  */
-static long do_splice(struct file *in, struct file *out, size_t len,
-                      unsigned int flags)
+static long do_splice(struct file *in, loff_t __user *off_in,
+                      struct file *out, loff_t __user *off_out,
+                      size_t len, unsigned int flags)
 {
-        struct inode *pipe;
+        struct pipe_inode_info *pipe;
+
+        pipe = in->f_dentry->d_inode->i_pipe;
+        if (pipe) {
+                if (off_in)
+                        return -ESPIPE;
+                if (off_out) {
+                        if (out->f_op->llseek == no_llseek)
+                                return -EINVAL;
+                        if (copy_from_user(&out->f_pos, off_out,
+                                           sizeof(loff_t)))
+                                return -EFAULT;
+                }
 
-        pipe = in->f_dentry->d_inode;
-        if (pipe->i_pipe)
                 return do_splice_from(pipe, out, len, flags);
+        }
+
+        pipe = out->f_dentry->d_inode->i_pipe;
+        if (pipe) {
+                if (off_out)
+                        return -ESPIPE;
+                if (off_in) {
+                        if (in->f_op->llseek == no_llseek)
+                                return -EINVAL;
+                        if (copy_from_user(&in->f_pos, off_in, sizeof(loff_t)))
+                                return -EFAULT;
+                }
 
-        pipe = out->f_dentry->d_inode;
-        if (pipe->i_pipe)
                 return do_splice_to(in, pipe, len, flags);
+        }
 
         return -EINVAL;
 }
 
-asmlinkage long sys_splice(int fdin, int fdout, size_t len, unsigned int flags)
+asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
+                           int fd_out, loff_t __user *off_out,
+                           size_t len, unsigned int flags)
 {
         long error;
         struct file *in, *out;
@@ -761,13 +943,15 @@ asmlinkage long sys_splice(int fdin, int fdout, size_t len, unsigned int flags)
                 return 0;
 
         error = -EBADF;
-        in = fget_light(fdin, &fput_in);
+        in = fget_light(fd_in, &fput_in);
         if (in) {
                 if (in->f_mode & FMODE_READ) {
-                        out = fget_light(fdout, &fput_out);
+                        out = fget_light(fd_out, &fput_out);
                         if (out) {
                                 if (out->f_mode & FMODE_WRITE)
-                                        error = do_splice(in, out, len, flags);
+                                        error = do_splice(in, off_in,
+                                                          out, off_out,
+                                                          len, flags);
                                 fput_light(out, fput_out);
                         }
                 }