1 files changed, 178 insertions, 83 deletions

diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index afcab007a22b..16844f98f50e 100644
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -193,7 +193,7 @@ static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
         if (dreq->iocb)
                 goto out;
 
-        result = wait_for_completion_interruptible(&dreq->completion);
+        result = wait_for_completion_killable(&dreq->completion);
 
         if (!result)
                 result = dreq->error;
@@ -263,17 +263,29 @@ static const struct rpc_call_ops nfs_read_direct_ops = {
  * handled automatically by nfs_direct_read_result().  Otherwise, if
  * no requests have been sent, just return an error.
  */
-static ssize_t nfs_direct_read_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t pos)
+static ssize_t nfs_direct_read_schedule_segment(struct nfs_direct_req *dreq,
+                                                const struct iovec *iov,
+                                                loff_t pos)
 {
         struct nfs_open_context *ctx = dreq->ctx;
         struct inode *inode = ctx->path.dentry->d_inode;
+        unsigned long user_addr = (unsigned long)iov->iov_base;
+        size_t count = iov->iov_len;
         size_t rsize = NFS_SERVER(inode)->rsize;
+        struct rpc_task *task;
+        struct rpc_message msg = {
+                .rpc_cred = ctx->cred,
+        };
+        struct rpc_task_setup task_setup_data = {
+                .rpc_client = NFS_CLIENT(inode),
+                .rpc_message = &msg,
+                .callback_ops = &nfs_read_direct_ops,
+                .flags = RPC_TASK_ASYNC,
+        };
         unsigned int pgbase;
         int result;
         ssize_t started = 0;
 
-        get_dreq(dreq);
-
         do {
                 struct nfs_read_data *data;
                 size_t bytes;
@@ -309,7 +321,7 @@ static ssize_t nfs_direct_read_schedule(struct nfs_direct_req *dreq, unsigned lo
 
                 data->req = (struct nfs_page *) dreq;
                 data->inode = inode;
-                data->cred = ctx->cred;
+                data->cred = msg.rpc_cred;
                 data->args.fh = NFS_FH(inode);
                 data->args.context = ctx;
                 data->args.offset = pos;
@@ -319,14 +331,16 @@ static ssize_t nfs_direct_read_schedule(struct nfs_direct_req *dreq, unsigned lo
                 data->res.fattr = &data->fattr;
                 data->res.eof = 0;
                 data->res.count = bytes;
+                msg.rpc_argp = &data->args;
+                msg.rpc_resp = &data->res;
 
-                rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
-                                &nfs_read_direct_ops, data);
-                NFS_PROTO(inode)->read_setup(data);
+                task_setup_data.task = &data->task;
+                task_setup_data.callback_data = data;
+                NFS_PROTO(inode)->read_setup(data, &msg);
 
-                data->task.tk_cookie = (unsigned long) inode;
-
-                rpc_execute(&data->task);
+                task = rpc_run_task(&task_setup_data);
+                if (!IS_ERR(task))
+                        rpc_put_task(task);
 
                 dprintk("NFS: %5u initiated direct read call "
                         "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
@@ -347,20 +361,49 @@ static ssize_t nfs_direct_read_schedule(struct nfs_direct_req *dreq, unsigned lo
                 count -= bytes;
         } while (count != 0);
 
+        if (started)
+                return started;
+        return result < 0 ? (ssize_t) result : -EFAULT;
+}
+
+static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
+                                              const struct iovec *iov,
+                                              unsigned long nr_segs,
+                                              loff_t pos)
+{
+        ssize_t result = -EINVAL;
+        size_t requested_bytes = 0;
+        unsigned long seg;
+
+        get_dreq(dreq);
+
+        for (seg = 0; seg < nr_segs; seg++) {
+                const struct iovec *vec = &iov[seg];
+                result = nfs_direct_read_schedule_segment(dreq, vec, pos);
+                if (result < 0)
+                        break;
+                requested_bytes += result;
+                if ((size_t)result < vec->iov_len)
+                        break;
+                pos += vec->iov_len;
+        }
+
         if (put_dreq(dreq))
                 nfs_direct_complete(dreq);
 
-        if (started)
+        if (requested_bytes != 0)
                 return 0;
-        return result < 0 ? (ssize_t) result : -EFAULT;
+
+        if (result < 0)
+                return result;
+        return -EIO;
 }
 
-static ssize_t nfs_direct_read(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t pos)
+static ssize_t nfs_direct_read(struct kiocb *iocb, const struct iovec *iov,
+                               unsigned long nr_segs, loff_t pos)
 {
         ssize_t result = 0;
-        sigset_t oldset;
         struct inode *inode = iocb->ki_filp->f_mapping->host;
-        struct rpc_clnt *clnt = NFS_CLIENT(inode);
         struct nfs_direct_req *dreq;
 
         dreq = nfs_direct_req_alloc();
@@ -372,12 +415,9 @@ static ssize_t nfs_direct_read(struct kiocb *iocb, unsigned long user_addr, size
         if (!is_sync_kiocb(iocb))
                 dreq->iocb = iocb;
 
-        nfs_add_stats(inode, NFSIOS_DIRECTREADBYTES, count);
-        rpc_clnt_sigmask(clnt, &oldset);
-        result = nfs_direct_read_schedule(dreq, user_addr, count, pos);
+        result = nfs_direct_read_schedule_iovec(dreq, iov, nr_segs, pos);
         if (!result)
                 result = nfs_direct_wait(dreq);
-        rpc_clnt_sigunmask(clnt, &oldset);
         nfs_direct_req_release(dreq);
 
         return result;
@@ -399,6 +439,15 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
         struct inode *inode = dreq->inode;
         struct list_head *p;
         struct nfs_write_data *data;
+        struct rpc_task *task;
+        struct rpc_message msg = {
+                .rpc_cred = dreq->ctx->cred,
+        };
+        struct rpc_task_setup task_setup_data = {
+                .rpc_client = NFS_CLIENT(inode),
+                .callback_ops = &nfs_write_direct_ops,
+                .flags = RPC_TASK_ASYNC,
+        };
 
         dreq->count = 0;
         get_dreq(dreq);
@@ -408,6 +457,9 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
 
                 get_dreq(dreq);
 
+                /* Use stable writes */
+                data->args.stable = NFS_FILE_SYNC;
+
                 /*
                  * Reset data->res.
                  */
@@ -419,17 +471,18 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
                  * Reuse data->task; data->args should not have changed
                  * since the original request was sent.
                  */
-                rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
-                                &nfs_write_direct_ops, data);
-                NFS_PROTO(inode)->write_setup(data, FLUSH_STABLE);
-
-                data->task.tk_priority = RPC_PRIORITY_NORMAL;
-                data->task.tk_cookie = (unsigned long) inode;
+                task_setup_data.task = &data->task;
+                task_setup_data.callback_data = data;
+                msg.rpc_argp = &data->args;
+                msg.rpc_resp = &data->res;
+                NFS_PROTO(inode)->write_setup(data, &msg);
 
                 /*
                  * We're called via an RPC callback, so BKL is already held.
                  */
-                rpc_execute(&data->task);
+                task = rpc_run_task(&task_setup_data);
+                if (!IS_ERR(task))
+                        rpc_put_task(task);
 
                 dprintk("NFS: %5u rescheduled direct write call (req %s/%Ld, %u bytes @ offset %Lu)\n",
                                 data->task.tk_pid,
@@ -472,9 +525,23 @@ static const struct rpc_call_ops nfs_commit_direct_ops = {
 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
 {
         struct nfs_write_data *data = dreq->commit_data;
+        struct rpc_task *task;
+        struct rpc_message msg = {
+                .rpc_argp = &data->args,
+                .rpc_resp = &data->res,
+                .rpc_cred = dreq->ctx->cred,
+        };
+        struct rpc_task_setup task_setup_data = {
+                .task = &data->task,
+                .rpc_client = NFS_CLIENT(dreq->inode),
+                .rpc_message = &msg,
+                .callback_ops = &nfs_commit_direct_ops,
+                .callback_data = data,
+                .flags = RPC_TASK_ASYNC,
+        };
 
         data->inode = dreq->inode;
-        data->cred = dreq->ctx->cred;
+        data->cred = msg.rpc_cred;
 
         data->args.fh = NFS_FH(data->inode);
         data->args.offset = 0;
@@ -483,18 +550,16 @@ static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
         data->res.fattr = &data->fattr;
         data->res.verf = &data->verf;
 
-        rpc_init_task(&data->task, NFS_CLIENT(dreq->inode), RPC_TASK_ASYNC,
-                                &nfs_commit_direct_ops, data);
-        NFS_PROTO(data->inode)->commit_setup(data, 0);
+        NFS_PROTO(data->inode)->commit_setup(data, &msg);
 
-        data->task.tk_priority = RPC_PRIORITY_NORMAL;
-        data->task.tk_cookie = (unsigned long)data->inode;
         /* Note: task.tk_ops->rpc_release will free dreq->commit_data */
         dreq->commit_data = NULL;
 
         dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
 
-        rpc_execute(&data->task);
+        task = rpc_run_task(&task_setup_data);
+        if (!IS_ERR(task))
+                rpc_put_task(task);
 }
 
 static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
@@ -601,17 +666,29 @@ static const struct rpc_call_ops nfs_write_direct_ops = {
  * handled automatically by nfs_direct_write_result().  Otherwise, if
  * no requests have been sent, just return an error.
  */
-static ssize_t nfs_direct_write_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t pos, int sync)
+static ssize_t nfs_direct_write_schedule_segment(struct nfs_direct_req *dreq,
+                                                 const struct iovec *iov,
+                                                 loff_t pos, int sync)
 {
         struct nfs_open_context *ctx = dreq->ctx;
         struct inode *inode = ctx->path.dentry->d_inode;
+        unsigned long user_addr = (unsigned long)iov->iov_base;
+        size_t count = iov->iov_len;
+        struct rpc_task *task;
+        struct rpc_message msg = {
+                .rpc_cred = ctx->cred,
+        };
+        struct rpc_task_setup task_setup_data = {
+                .rpc_client = NFS_CLIENT(inode),
+                .rpc_message = &msg,
+                .callback_ops = &nfs_write_direct_ops,
+                .flags = RPC_TASK_ASYNC,
+        };
         size_t wsize = NFS_SERVER(inode)->wsize;
         unsigned int pgbase;
         int result;
         ssize_t started = 0;
 
-        get_dreq(dreq);
-
         do {
                 struct nfs_write_data *data;
                 size_t bytes;
@@ -649,25 +726,27 @@ static ssize_t nfs_direct_write_schedule(struct nfs_direct_req *dreq, unsigned l
 
                 data->req = (struct nfs_page *) dreq;
                 data->inode = inode;
-                data->cred = ctx->cred;
+                data->cred = msg.rpc_cred;
                 data->args.fh = NFS_FH(inode);
                 data->args.context = ctx;
                 data->args.offset = pos;
                 data->args.pgbase = pgbase;
                 data->args.pages = data->pagevec;
                 data->args.count = bytes;
+                data->args.stable = sync;
                 data->res.fattr = &data->fattr;
                 data->res.count = bytes;
                 data->res.verf = &data->verf;
 
-                rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
-                                &nfs_write_direct_ops, data);
-                NFS_PROTO(inode)->write_setup(data, sync);
+                task_setup_data.task = &data->task;
+                task_setup_data.callback_data = data;
+                msg.rpc_argp = &data->args;
+                msg.rpc_resp = &data->res;
+                NFS_PROTO(inode)->write_setup(data, &msg);
 
-                data->task.tk_priority = RPC_PRIORITY_NORMAL;
-                data->task.tk_cookie = (unsigned long) inode;
-
-                rpc_execute(&data->task);
+                task = rpc_run_task(&task_setup_data);
+                if (!IS_ERR(task))
+                        rpc_put_task(task);
 
                 dprintk("NFS: %5u initiated direct write call "
                         "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
@@ -689,23 +768,54 @@ static ssize_t nfs_direct_write_schedule(struct nfs_direct_req *dreq, unsigned l
                 count -= bytes;
         } while (count != 0);
 
+        if (started)
+                return started;
+        return result < 0 ? (ssize_t) result : -EFAULT;
+}
+
+static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
+                                               const struct iovec *iov,
+                                               unsigned long nr_segs,
+                                               loff_t pos, int sync)
+{
+        ssize_t result = 0;
+        size_t requested_bytes = 0;
+        unsigned long seg;
+
+        get_dreq(dreq);
+
+        for (seg = 0; seg < nr_segs; seg++) {
+                const struct iovec *vec = &iov[seg];
+                result = nfs_direct_write_schedule_segment(dreq, vec,
+                                                           pos, sync);
+                if (result < 0)
+                        break;
+                requested_bytes += result;
+                if ((size_t)result < vec->iov_len)
+                        break;
+                pos += vec->iov_len;
+        }
+
         if (put_dreq(dreq))
-                nfs_direct_write_complete(dreq, inode);
+                nfs_direct_write_complete(dreq, dreq->inode);
 
-        if (started)
+        if (requested_bytes != 0)
                 return 0;
-        return result < 0 ? (ssize_t) result : -EFAULT;
+
+        if (result < 0)
+                return result;
+        return -EIO;
 }
 
-static ssize_t nfs_direct_write(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t pos)
+static ssize_t nfs_direct_write(struct kiocb *iocb, const struct iovec *iov,
+                                unsigned long nr_segs, loff_t pos,
+                                size_t count)
 {
         ssize_t result = 0;
-        sigset_t oldset;
         struct inode *inode = iocb->ki_filp->f_mapping->host;
-        struct rpc_clnt *clnt = NFS_CLIENT(inode);
         struct nfs_direct_req *dreq;
         size_t wsize = NFS_SERVER(inode)->wsize;
-        int sync = 0;
+        int sync = NFS_UNSTABLE;
 
         dreq = nfs_direct_req_alloc();
         if (!dreq)
@@ -713,20 +823,16 @@ static ssize_t nfs_direct_write(struct kiocb *iocb, unsigned long user_addr, siz
         nfs_alloc_commit_data(dreq);
 
         if (dreq->commit_data == NULL || count < wsize)
-                sync = FLUSH_STABLE;
+                sync = NFS_FILE_SYNC;
 
         dreq->inode = inode;
         dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
         if (!is_sync_kiocb(iocb))
                 dreq->iocb = iocb;
 
-        nfs_add_stats(inode, NFSIOS_DIRECTWRITTENBYTES, count);
-
-        rpc_clnt_sigmask(clnt, &oldset);
-        result = nfs_direct_write_schedule(dreq, user_addr, count, pos, sync);
+        result = nfs_direct_write_schedule_iovec(dreq, iov, nr_segs, pos, sync);
         if (!result)
                 result = nfs_direct_wait(dreq);
-        rpc_clnt_sigunmask(clnt, &oldset);
         nfs_direct_req_release(dreq);
 
         return result;
@@ -759,21 +865,16 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov,
         ssize_t retval = -EINVAL;
         struct file *file = iocb->ki_filp;
         struct address_space *mapping = file->f_mapping;
-        /* XXX: temporary */
-        const char __user *buf = iov[0].iov_base;
-        size_t count = iov[0].iov_len;
+        size_t count;
 
-        dprintk("nfs: direct read(%s/%s, %lu@%Ld)\n",
+        count = iov_length(iov, nr_segs);
+        nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
+
+        dprintk("nfs: direct read(%s/%s, %zd@%Ld)\n",
                 file->f_path.dentry->d_parent->d_name.name,
                 file->f_path.dentry->d_name.name,
-                (unsigned long) count, (long long) pos);
-
-        if (nr_segs != 1)
-                goto out;
+                count, (long long) pos);
 
-        retval = -EFAULT;
-        if (!access_ok(VERIFY_WRITE, buf, count))
-                goto out;
         retval = 0;
         if (!count)
                 goto out;
@@ -782,7 +883,7 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov,
         if (retval)
                 goto out;
 
-        retval = nfs_direct_read(iocb, (unsigned long) buf, count, pos);
+        retval = nfs_direct_read(iocb, iov, nr_segs, pos);
         if (retval > 0)
                 iocb->ki_pos = pos + retval;
 
@@ -821,17 +922,15 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
         ssize_t retval = -EINVAL;
         struct file *file = iocb->ki_filp;
         struct address_space *mapping = file->f_mapping;
-        /* XXX: temporary */
-        const char __user *buf = iov[0].iov_base;
-        size_t count = iov[0].iov_len;
+        size_t count;
+
+        count = iov_length(iov, nr_segs);
+        nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
 
-        dprintk("nfs: direct write(%s/%s, %lu@%Ld)\n",
+        dfprintk(VFS, "nfs: direct write(%s/%s, %zd@%Ld)\n",
                 file->f_path.dentry->d_parent->d_name.name,
                 file->f_path.dentry->d_name.name,
-                (unsigned long) count, (long long) pos);
-
-        if (nr_segs != 1)
-                goto out;
+                count, (long long) pos);
 
         retval = generic_write_checks(file, &pos, &count, 0);
         if (retval)
@@ -844,15 +943,11 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
         if (!count)
                 goto out;
 
-        retval = -EFAULT;
-        if (!access_ok(VERIFY_READ, buf, count))
-                goto out;
-
         retval = nfs_sync_mapping(mapping);
         if (retval)
                 goto out;
 
-        retval = nfs_direct_write(iocb, (unsigned long) buf, count, pos);
+        retval = nfs_direct_write(iocb, iov, nr_segs, pos, count);
 
         if (retval > 0)
                 iocb->ki_pos = pos + retval;