Merge tag 'nfs-for-3.20-1' of git://git.linux-nfs.org/projects/trondmy/linux-nfs

Pull NFS client updates from Trond Myklebust:
 "Highlights incluse:

  Features:
   - Removing the forced serialisation of open()/close() calls in
     NFSv4.x (x>0) makes for a significant performance improvement in
     metadata intensive workloads.
   - Full support for the pNFS "flexible files" layout type
   - Further RPC/RDMA client improvements from Chuck

  Bugfixes:
   - Stable fix: NFSv4.1 backchannel calls blocking operations with !TASK_RUNNING
   - Stable fix: pnfs_generic_pg_init_read/write can be called with lseg == NULL
   - Stable fix: Fix an Oopsable condition when nsm_mon_unmon is called
     as part of the namespace cleanup,
   - Stable fix: Ensure we reference the inode for return-on-close in
     delegreturn
   - Use SO_REUSEPORT to ensure that NFSv3 TCP connections can rebind to
     the same source address/port combination during a disconnect/
     reconnect event.  This is a requirement imposed by most NFSv3
     server duplicate reply cache implementations.

  Optimisations:
   - Ask for no NFSv4.1 delegations on OPEN if using O_DIRECT

  Other:
   - Add Anna Schumaker as co-maintainer for the NFS client"

* tag 'nfs-for-3.20-1' of git://git.linux-nfs.org/projects/trondmy/linux-nfs: (119 commits)
  SUNRPC: Cleanup to remove xs_tcp_close()
  pnfs: delete an unintended goto
  pnfs/flexfiles: Do not dprintk after the free
  SUNRPC: Fix stupid typo in xs_sock_set_reuseport
  SUNRPC: Define xs_tcp_fin_timeout only if CONFIG_SUNRPC_DEBUG
  SUNRPC: Handle connection reset more efficiently.
  SUNRPC: Remove the redundant XPRT_CONNECTION_CLOSE flag
  SUNRPC: Make xs_tcp_close() do a socket shutdown rather than a sock_release
  SUNRPC: Ensure xs_tcp_shutdown() requests a full close of the connection
  SUNRPC: Cleanup to remove remaining uses of XPRT_CONNECTION_ABORT
  SUNRPC: Remove TCP socket linger code
  SUNRPC: Remove TCP client connection reset hack
  SUNRPC: TCP/UDP always close the old socket before reconnecting
  SUNRPC: Add helpers to prevent socket create from racing
  SUNRPC: Ensure xs_reset_transport() resets the close connection flags
  SUNRPC: Do not clear the source port in xs_reset_transport
  SUNRPC: Handle EADDRINUSE on connect
  SUNRPC: Set SO_REUSEPORT socket option for TCP connections
  NFSv4.1: Fix pnfs_put_lseg races
  NFSv4.1: pnfs_send_layoutreturn should use GFP_NOFS
  ...

60 files changed, 5236 insertions, 1729 deletions

diff --git a/Documentation/filesystems/nfs/pnfs.txt b/Documentation/filesystems/nfs/pnfs.txt
index adc81a35fe2d..44a9f2493a88 100644
--- a/Documentation/filesystems/nfs/pnfs.txt
+++ b/Documentation/filesystems/nfs/pnfs.txt
@@ -57,15 +57,16 @@ bit is set, preventing any new lsegs from being added.
 layout drivers
 --------------
 
-PNFS utilizes what is called layout drivers. The STD defines 3 basic
-layout types: "files" "objects" and "blocks". For each of these types
-there is a layout-driver with a common function-vectors table which
-are called by the nfs-client pnfs-core to implement the different layout
-types.
+PNFS utilizes what is called layout drivers. The STD defines 4 basic
+layout types: "files", "objects", "blocks", and "flexfiles". For each
+of these types there is a layout-driver with a common function-vectors
+table which are called by the nfs-client pnfs-core to implement the
+different layout types.
 
-Files-layout-driver code is in: fs/nfs/nfs4filelayout.c && nfs4filelayoutdev.c
+Files-layout-driver code is in: fs/nfs/filelayout/.. directory
 Objects-layout-deriver code is in: fs/nfs/objlayout/.. directory
 Blocks-layout-deriver code is in: fs/nfs/blocklayout/.. directory
+Flexfiles-layout-driver code is in: fs/nfs/flexfilelayout/.. directory
 
 objects-layout setup
 --------------------
diff --git a/MAINTAINERS b/MAINTAINERS
index cc66549dd761..37c10098a627 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6777,6 +6777,7 @@ F:	Documentation/devicetree/bindings/net/nfc/
 
 NFS, SUNRPC, AND LOCKD CLIENTS
 M:      Trond Myklebust <trond.myklebust@primarydata.com>
+M:      Anna Schumaker <anna.schumaker@netapp.com>
 L:      linux-nfs@vger.kernel.org
 W:      http://client.linux-nfs.org
 T:      git git://git.linux-nfs.org/projects/trondmy/linux-nfs.git
diff --git a/fs/lockd/mon.c b/fs/lockd/mon.c
index 1cc6ec51e6b1..47a32b6d9b90 100644
--- a/fs/lockd/mon.c
+++ b/fs/lockd/mon.c
@@ -65,7 +65,7 @@ static inline struct sockaddr *nsm_addr(const struct nsm_handle *nsm)
         return (struct sockaddr *)&nsm->sm_addr;
 }
 
-static struct rpc_clnt *nsm_create(struct net *net)
+static struct rpc_clnt *nsm_create(struct net *net, const char *nodename)
 {
         struct sockaddr_in sin = {
                 .sin_family             = AF_INET,
@@ -77,6 +77,7 @@ static struct rpc_clnt *nsm_create(struct net *net)
                 .address                = (struct sockaddr *)&sin,
                 .addrsize               = sizeof(sin),
                 .servername             = "rpc.statd",
+                .nodename               = nodename,
                 .program                = &nsm_program,
                 .version                = NSM_VERSION,
                 .authflavor             = RPC_AUTH_NULL,
@@ -102,7 +103,7 @@ out:
         return clnt;
 }
 
-static struct rpc_clnt *nsm_client_get(struct net *net)
+static struct rpc_clnt *nsm_client_get(struct net *net, const char *nodename)
 {
         struct rpc_clnt *clnt, *new;
         struct lockd_net *ln = net_generic(net, lockd_net_id);
@@ -111,7 +112,7 @@ static struct rpc_clnt *nsm_client_get(struct net *net)
         if (clnt != NULL)
                 goto out;
 
-        clnt = new = nsm_create(net);
+        clnt = new = nsm_create(net, nodename);
         if (IS_ERR(clnt))
                 goto out;
 
@@ -190,19 +191,23 @@ int nsm_monitor(const struct nlm_host *host)
         struct nsm_res  res;
         int             status;
         struct rpc_clnt *clnt;
+        const char *nodename = NULL;
 
         dprintk("lockd: nsm_monitor(%s)\n", nsm->sm_name);
 
         if (nsm->sm_monitored)
                 return 0;
 
+        if (host->h_rpcclnt)
+                nodename = host->h_rpcclnt->cl_nodename;
+
         /*
          * Choose whether to record the caller_name or IP address of
          * this peer in the local rpc.statd's database.
          */
         nsm->sm_mon_name = nsm_use_hostnames ? nsm->sm_name : nsm->sm_addrbuf;
 
-        clnt = nsm_client_get(host->net);
+        clnt = nsm_client_get(host->net, nodename);
         if (IS_ERR(clnt)) {
                 status = PTR_ERR(clnt);
                 dprintk("lockd: failed to create NSM upcall transport, "
diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig
index 3dece03f2fc8..c7abc10279af 100644
--- a/fs/nfs/Kconfig
+++ b/fs/nfs/Kconfig
@@ -128,6 +128,11 @@ config PNFS_OBJLAYOUT
         depends on NFS_V4_1 && SCSI_OSD_ULD
         default NFS_V4
 
+config PNFS_FLEXFILE_LAYOUT
+        tristate
+        depends on NFS_V4_1 && NFS_V3
+        default m
+
 config NFS_V4_1_IMPLEMENTATION_ID_DOMAIN
         string "NFSv4.1 Implementation ID Domain"
         depends on NFS_V4_1
diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile
index 04cb830fa09f..1e987acf20c9 100644
--- a/fs/nfs/Makefile
+++ b/fs/nfs/Makefile
@@ -27,9 +27,10 @@ nfsv4-y := nfs4proc.o nfs4xdr.o nfs4state.o nfs4renewd.o nfs4super.o nfs4file.o
           dns_resolve.o nfs4trace.o
 nfsv4-$(CONFIG_NFS_USE_LEGACY_DNS) += cache_lib.o
 nfsv4-$(CONFIG_SYSCTL)  += nfs4sysctl.o
-nfsv4-$(CONFIG_NFS_V4_1)        += pnfs.o pnfs_dev.o
+nfsv4-$(CONFIG_NFS_V4_1)        += pnfs.o pnfs_dev.o pnfs_nfs.o
 nfsv4-$(CONFIG_NFS_V4_2)        += nfs42proc.o
 
 obj-$(CONFIG_PNFS_FILE_LAYOUT) += filelayout/
 obj-$(CONFIG_PNFS_OBJLAYOUT) += objlayout/
 obj-$(CONFIG_PNFS_BLOCK) += blocklayout/
+obj-$(CONFIG_PNFS_FLEXFILE_LAYOUT) += flexfilelayout/
diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c
index 77fec6a55f57..1cac3c175d18 100644
--- a/fs/nfs/blocklayout/blocklayout.c
+++ b/fs/nfs/blocklayout/blocklayout.c
@@ -860,12 +860,14 @@ static const struct nfs_pageio_ops bl_pg_read_ops = {
         .pg_init = bl_pg_init_read,
         .pg_test = bl_pg_test_read,
         .pg_doio = pnfs_generic_pg_readpages,
+        .pg_cleanup = pnfs_generic_pg_cleanup,
 };
 
 static const struct nfs_pageio_ops bl_pg_write_ops = {
         .pg_init = bl_pg_init_write,
         .pg_test = bl_pg_test_write,
         .pg_doio = pnfs_generic_pg_writepages,
+        .pg_cleanup = pnfs_generic_pg_cleanup,
 };
 
 static struct pnfs_layoutdriver_type blocklayout_type = {
diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index b8fb3a4ef649..351be9205bf8 100644
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -128,22 +128,24 @@ nfs41_callback_svc(void *vrqstp)
                 if (try_to_freeze())
                         continue;
 
-                prepare_to_wait(&serv->sv_cb_waitq, &wq, TASK_INTERRUPTIBLE);
+                prepare_to_wait(&serv->sv_cb_waitq, &wq, TASK_UNINTERRUPTIBLE);
                 spin_lock_bh(&serv->sv_cb_lock);
                 if (!list_empty(&serv->sv_cb_list)) {
                         req = list_first_entry(&serv->sv_cb_list,
                                         struct rpc_rqst, rq_bc_list);
                         list_del(&req->rq_bc_list);
                         spin_unlock_bh(&serv->sv_cb_lock);
+                        finish_wait(&serv->sv_cb_waitq, &wq);
                         dprintk("Invoking bc_svc_process()\n");
                         error = bc_svc_process(serv, req, rqstp);
                         dprintk("bc_svc_process() returned w/ error code= %d\n",
                                 error);
                 } else {
                         spin_unlock_bh(&serv->sv_cb_lock);
-                        schedule();
+                        /* schedule_timeout to game the hung task watchdog */
+                        schedule_timeout(60 * HZ);
+                        finish_wait(&serv->sv_cb_waitq, &wq);
                 }
-                finish_wait(&serv->sv_cb_waitq, &wq);
         }
         return 0;
 }
diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c
index 8cdb2b28a104..da5433230bb1 100644
--- a/fs/nfs/delegation.c
+++ b/fs/nfs/delegation.c
@@ -306,6 +306,17 @@ nfs_inode_detach_delegation(struct inode *inode)
         return nfs_detach_delegation(nfsi, delegation, server);
 }
 
+static void
+nfs_update_inplace_delegation(struct nfs_delegation *delegation,
+                const struct nfs_delegation *update)
+{
+        if (nfs4_stateid_is_newer(&update->stateid, &delegation->stateid)) {
+                delegation->stateid.seqid = update->stateid.seqid;
+                smp_wmb();
+                delegation->type = update->type;
+        }
+}
+
 /**
  * nfs_inode_set_delegation - set up a delegation on an inode
  * @inode: inode to which delegation applies
@@ -339,9 +350,12 @@ int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred, struct
         old_delegation = rcu_dereference_protected(nfsi->delegation,
                                         lockdep_is_held(&clp->cl_lock));
         if (old_delegation != NULL) {
-                if (nfs4_stateid_match(&delegation->stateid,
-                                        &old_delegation->stateid) &&
-                                delegation->type == old_delegation->type) {
+                /* Is this an update of the existing delegation? */
+                if (nfs4_stateid_match_other(&old_delegation->stateid,
+                                        &delegation->stateid)) {
+                        nfs_update_inplace_delegation(old_delegation,
+                                        delegation);
+                        nfsi->delegation_state = old_delegation->type;
                         goto out;
                 }
                 /*
diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index 294692ff83b1..7077521acdf4 100644
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -66,6 +66,10 @@ static struct kmem_cache *nfs_direct_cachep;
 /*
  * This represents a set of asynchronous requests that we're waiting on
  */
+struct nfs_direct_mirror {
+        ssize_t count;
+};
+
 struct nfs_direct_req {
         struct kref             kref;           /* release manager */
 
@@ -78,8 +82,13 @@ struct nfs_direct_req {
         /* completion state */
         atomic_t                io_count;       /* i/os we're waiting for */
         spinlock_t              lock;           /* protect completion state */
+
+        struct nfs_direct_mirror mirrors[NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX];
+        int                     mirror_count;
+
         ssize_t                 count,          /* bytes actually processed */
                                 bytes_left,     /* bytes left to be sent */
+                                io_start,       /* start of IO */
                                 error;          /* any reported error */
         struct completion       completion;     /* wait for i/o completion */
 
@@ -108,26 +117,56 @@ static inline int put_dreq(struct nfs_direct_req *dreq)
         return atomic_dec_and_test(&dreq->io_count);
 }
 
+void nfs_direct_set_resched_writes(struct nfs_direct_req *dreq)
+{
+        dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
+}
+EXPORT_SYMBOL_GPL(nfs_direct_set_resched_writes);
+
+static void
+nfs_direct_good_bytes(struct nfs_direct_req *dreq, struct nfs_pgio_header *hdr)
+{
+        int i;
+        ssize_t count;
+
+        WARN_ON_ONCE(hdr->pgio_mirror_idx >= dreq->mirror_count);
+
+        count = dreq->mirrors[hdr->pgio_mirror_idx].count;
+        if (count + dreq->io_start < hdr->io_start + hdr->good_bytes) {
+                count = hdr->io_start + hdr->good_bytes - dreq->io_start;
+                dreq->mirrors[hdr->pgio_mirror_idx].count = count;
+        }
+
+        /* update the dreq->count by finding the minimum agreed count from all
+         * mirrors */
+        count = dreq->mirrors[0].count;
+
+        for (i = 1; i < dreq->mirror_count; i++)
+                count = min(count, dreq->mirrors[i].count);
+
+        dreq->count = count;
+}
+
 /*
  * nfs_direct_select_verf - select the right verifier
  * @dreq - direct request possibly spanning multiple servers
  * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs
- * @ds_idx - index of data server in data server list, only valid if ds_clp set
+ * @commit_idx - commit bucket index for the DS
  *
  * returns the correct verifier to use given the role of the server
  */
 static struct nfs_writeverf *
 nfs_direct_select_verf(struct nfs_direct_req *dreq,
                        struct nfs_client *ds_clp,
-                       int ds_idx)
+                       int commit_idx)
 {
         struct nfs_writeverf *verfp = &dreq->verf;
 
 #ifdef CONFIG_NFS_V4_1
         if (ds_clp) {
                 /* pNFS is in use, use the DS verf */
-                if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets)
-                        verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf;
+                if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)
+                        verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;
                 else
                         WARN_ON_ONCE(1);
         }
@@ -148,8 +187,7 @@ static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,
 {
         struct nfs_writeverf *verfp;
 
-        verfp = nfs_direct_select_verf(dreq, hdr->ds_clp,
-                                      hdr->ds_idx);
+        verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
         WARN_ON_ONCE(verfp->committed >= 0);
         memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf));
         WARN_ON_ONCE(verfp->committed < 0);
@@ -169,8 +207,7 @@ static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,
 {
         struct nfs_writeverf *verfp;
 
-        verfp = nfs_direct_select_verf(dreq, hdr->ds_clp,
-                                         hdr->ds_idx);
+        verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
         if (verfp->committed < 0) {
                 nfs_direct_set_hdr_verf(dreq, hdr);
                 return 0;
@@ -193,7 +230,11 @@ static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,
 
         verfp = nfs_direct_select_verf(dreq, data->ds_clp,
                                          data->ds_commit_index);
-        WARN_ON_ONCE(verfp->committed < 0);
+
+        /* verifier not set so always fail */
+        if (verfp->committed < 0)
+                return 1;
+
         return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf));
 }
 
@@ -249,6 +290,18 @@ void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
         cinfo->completion_ops = &nfs_direct_commit_completion_ops;
 }
 
+static inline void nfs_direct_setup_mirroring(struct nfs_direct_req *dreq,
+                                             struct nfs_pageio_descriptor *pgio,
+                                             struct nfs_page *req)
+{
+        int mirror_count = 1;
+
+        if (pgio->pg_ops->pg_get_mirror_count)
+                mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
+
+        dreq->mirror_count = mirror_count;
+}
+
 static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
 {
         struct nfs_direct_req *dreq;
@@ -263,6 +316,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
         INIT_LIST_HEAD(&dreq->mds_cinfo.list);
         dreq->verf.committed = NFS_INVALID_STABLE_HOW;  /* not set yet */
         INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
+        dreq->mirror_count = 1;
         spin_lock_init(&dreq->lock);
 
         return dreq;
@@ -369,7 +423,8 @@ static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
         if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))
                 dreq->error = hdr->error;
         else
-                dreq->count += hdr->good_bytes;
+                nfs_direct_good_bytes(dreq, hdr);
+
         spin_unlock(&dreq->lock);
 
         while (!list_empty(&hdr->pages)) {
@@ -547,6 +602,7 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter,
 
         dreq->inode = inode;
         dreq->bytes_left = count;
+        dreq->io_start = pos;
         dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
         l_ctx = nfs_get_lock_context(dreq->ctx);
         if (IS_ERR(l_ctx)) {
@@ -579,6 +635,20 @@ out:
         return result;
 }
 
+static void
+nfs_direct_write_scan_commit_list(struct inode *inode,
+                                  struct list_head *list,
+                                  struct nfs_commit_info *cinfo)
+{
+        spin_lock(cinfo->lock);
+#ifdef CONFIG_NFS_V4_1
+        if (cinfo->ds != NULL && cinfo->ds->nwritten != 0)
+                NFS_SERVER(inode)->pnfs_curr_ld->recover_commit_reqs(list, cinfo);
+#endif
+        nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
+        spin_unlock(cinfo->lock);
+}
+
 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
 {
         struct nfs_pageio_descriptor desc;
@@ -586,20 +656,23 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
         LIST_HEAD(reqs);
         struct nfs_commit_info cinfo;
         LIST_HEAD(failed);
+        int i;
 
         nfs_init_cinfo_from_dreq(&cinfo, dreq);
-        pnfs_recover_commit_reqs(dreq->inode, &reqs, &cinfo);
-        spin_lock(cinfo.lock);
-        nfs_scan_commit_list(&cinfo.mds->list, &reqs, &cinfo, 0);
-        spin_unlock(cinfo.lock);
+        nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
 
         dreq->count = 0;
+        for (i = 0; i < dreq->mirror_count; i++)
+                dreq->mirrors[i].count = 0;
         get_dreq(dreq);
 
         nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
                               &nfs_direct_write_completion_ops);
         desc.pg_dreq = dreq;
 
+        req = nfs_list_entry(reqs.next);
+        nfs_direct_setup_mirroring(dreq, &desc, req);
+
         list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
                 if (!nfs_pageio_add_request(&desc, req)) {
                         nfs_list_remove_request(req);
@@ -646,7 +719,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data)
                 nfs_list_remove_request(req);
                 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
                         /* Note the rewrite will go through mds */
-                        nfs_mark_request_commit(req, NULL, &cinfo);
+                        nfs_mark_request_commit(req, NULL, &cinfo, 0);
                 } else
                         nfs_release_request(req);
                 nfs_unlock_and_release_request(req);
@@ -721,7 +794,7 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
                 dreq->error = hdr->error;
         }
         if (dreq->error == 0) {
-                dreq->count += hdr->good_bytes;
+                nfs_direct_good_bytes(dreq, hdr);
                 if (nfs_write_need_commit(hdr)) {
                         if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)
                                 request_commit = true;
@@ -745,7 +818,8 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
                 nfs_list_remove_request(req);
                 if (request_commit) {
                         kref_get(&req->wb_kref);
-                        nfs_mark_request_commit(req, hdr->lseg, &cinfo);
+                        nfs_mark_request_commit(req, hdr->lseg, &cinfo,
+                                hdr->ds_commit_idx);
                 }
                 nfs_unlock_and_release_request(req);
         }
@@ -826,6 +900,9 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
                                 result = PTR_ERR(req);
                                 break;
                         }
+
+                        nfs_direct_setup_mirroring(dreq, &desc, req);
+
                         nfs_lock_request(req);
                         req->wb_index = pos >> PAGE_SHIFT;
                         req->wb_offset = pos & ~PAGE_MASK;
@@ -934,6 +1011,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter,
 
         dreq->inode = inode;
         dreq->bytes_left = count;
+        dreq->io_start = pos;
         dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
         l_ctx = nfs_get_lock_context(dreq->ctx);
         if (IS_ERR(l_ctx)) {
diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c
index 7afb52f6a25a..3c9769441f36 100644
--- a/fs/nfs/filelayout/filelayout.c
+++ b/fs/nfs/filelayout/filelayout.c
@@ -118,13 +118,6 @@ static void filelayout_reset_read(struct nfs_pgio_header *hdr)
         }
 }
 
-static void filelayout_fenceme(struct inode *inode, struct pnfs_layout_hdr *lo)
-{
-        if (!test_and_clear_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
-                return;
-        pnfs_return_layout(inode);
-}
-
 static int filelayout_async_handle_error(struct rpc_task *task,
                                          struct nfs4_state *state,
                                          struct nfs_client *clp,
@@ -207,7 +200,7 @@ static int filelayout_async_handle_error(struct rpc_task *task,
                 dprintk("%s DS connection error %d\n", __func__,
                         task->tk_status);
                 nfs4_mark_deviceid_unavailable(devid);
-                set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
+                pnfs_error_mark_layout_for_return(inode, lseg);
                 rpc_wake_up(&tbl->slot_tbl_waitq);
                 /* fall through */
         default:
@@ -339,16 +332,6 @@ static void filelayout_read_count_stats(struct rpc_task *task, void *data)
         rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics);
 }
 
-static void filelayout_read_release(void *data)
-{
-        struct nfs_pgio_header *hdr = data;
-        struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
-
-        filelayout_fenceme(lo->plh_inode, lo);
-        nfs_put_client(hdr->ds_clp);
-        hdr->mds_ops->rpc_release(data);
-}
-
 static int filelayout_write_done_cb(struct rpc_task *task,
                                 struct nfs_pgio_header *hdr)
 {
@@ -371,17 +354,6 @@ static int filelayout_write_done_cb(struct rpc_task *task,
         return 0;
 }
 
-/* Fake up some data that will cause nfs_commit_release to retry the writes. */
-static void prepare_to_resend_writes(struct nfs_commit_data *data)
-{
-        struct nfs_page *first = nfs_list_entry(data->pages.next);
-
-        data->task.tk_status = 0;
-        memcpy(&data->verf.verifier, &first->wb_verf,
-               sizeof(data->verf.verifier));
-        data->verf.verifier.data[0]++; /* ensure verifier mismatch */
-}
-
 static int filelayout_commit_done_cb(struct rpc_task *task,
                                      struct nfs_commit_data *data)
 {
@@ -393,7 +365,7 @@ static int filelayout_commit_done_cb(struct rpc_task *task,
 
         switch (err) {
         case -NFS4ERR_RESET_TO_MDS:
-                prepare_to_resend_writes(data);
+                pnfs_generic_prepare_to_resend_writes(data);
                 return -EAGAIN;
         case -EAGAIN:
                 rpc_restart_call_prepare(task);
@@ -451,16 +423,6 @@ static void filelayout_write_count_stats(struct rpc_task *task, void *data)
         rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics);
 }
 
-static void filelayout_write_release(void *data)
-{
-        struct nfs_pgio_header *hdr = data;
-        struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
-
-        filelayout_fenceme(lo->plh_inode, lo);
-        nfs_put_client(hdr->ds_clp);
-        hdr->mds_ops->rpc_release(data);
-}
-
 static void filelayout_commit_prepare(struct rpc_task *task, void *data)
 {
         struct nfs_commit_data *wdata = data;
@@ -471,14 +433,6 @@ static void filelayout_commit_prepare(struct rpc_task *task, void *data)
                         task);
 }
 
-static void filelayout_write_commit_done(struct rpc_task *task, void *data)
-{
-        struct nfs_commit_data *wdata = data;
-
-        /* Note this may cause RPC to be resent */
-        wdata->mds_ops->rpc_call_done(task, data);
-}
-
 static void filelayout_commit_count_stats(struct rpc_task *task, void *data)
 {
         struct nfs_commit_data *cdata = data;
@@ -486,35 +440,25 @@ static void filelayout_commit_count_stats(struct rpc_task *task, void *data)
         rpc_count_iostats(task, NFS_SERVER(cdata->inode)->client->cl_metrics);
 }
 
-static void filelayout_commit_release(void *calldata)
-{
-        struct nfs_commit_data *data = calldata;
-
-        data->completion_ops->completion(data);
-        pnfs_put_lseg(data->lseg);
-        nfs_put_client(data->ds_clp);
-        nfs_commitdata_release(data);
-}
-
 static const struct rpc_call_ops filelayout_read_call_ops = {
         .rpc_call_prepare = filelayout_read_prepare,
         .rpc_call_done = filelayout_read_call_done,
         .rpc_count_stats = filelayout_read_count_stats,
-        .rpc_release = filelayout_read_release,
+        .rpc_release = pnfs_generic_rw_release,
 };
 
 static const struct rpc_call_ops filelayout_write_call_ops = {
         .rpc_call_prepare = filelayout_write_prepare,
         .rpc_call_done = filelayout_write_call_done,
         .rpc_count_stats = filelayout_write_count_stats,
-        .rpc_release = filelayout_write_release,
+        .rpc_release = pnfs_generic_rw_release,
 };
 
 static const struct rpc_call_ops filelayout_commit_call_ops = {
         .rpc_call_prepare = filelayout_commit_prepare,
-        .rpc_call_done = filelayout_write_commit_done,
+        .rpc_call_done = pnfs_generic_write_commit_done,
         .rpc_count_stats = filelayout_commit_count_stats,
-        .rpc_release = filelayout_commit_release,
+        .rpc_release = pnfs_generic_commit_release,
 };
 
 static enum pnfs_try_status
@@ -548,7 +492,7 @@ filelayout_read_pagelist(struct nfs_pgio_header *hdr)
         /* No multipath support. Use first DS */
         atomic_inc(&ds->ds_clp->cl_count);
         hdr->ds_clp = ds->ds_clp;
-        hdr->ds_idx = idx;
+        hdr->ds_commit_idx = idx;
         fh = nfs4_fl_select_ds_fh(lseg, j);
         if (fh)
                 hdr->args.fh = fh;
@@ -557,8 +501,9 @@ filelayout_read_pagelist(struct nfs_pgio_header *hdr)
         hdr->mds_offset = offset;
 
         /* Perform an asynchronous read to ds */
-        nfs_initiate_pgio(ds_clnt, hdr,
-                            &filelayout_read_call_ops, 0, RPC_TASK_SOFTCONN);
+        nfs_initiate_pgio(ds_clnt, hdr, hdr->cred,
+                          NFS_PROTO(hdr->inode), &filelayout_read_call_ops,
+                          0, RPC_TASK_SOFTCONN);
         return PNFS_ATTEMPTED;
 }
 
@@ -591,16 +536,16 @@ filelayout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
         hdr->pgio_done_cb = filelayout_write_done_cb;
         atomic_inc(&ds->ds_clp->cl_count);
         hdr->ds_clp = ds->ds_clp;
-        hdr->ds_idx = idx;
+        hdr->ds_commit_idx = idx;
         fh = nfs4_fl_select_ds_fh(lseg, j);
         if (fh)
                 hdr->args.fh = fh;
         hdr->args.offset = filelayout_get_dserver_offset(lseg, offset);
 
         /* Perform an asynchronous write */
-        nfs_initiate_pgio(ds_clnt, hdr,
-                                    &filelayout_write_call_ops, sync,
-                                    RPC_TASK_SOFTCONN);
+        nfs_initiate_pgio(ds_clnt, hdr, hdr->cred,
+                          NFS_PROTO(hdr->inode), &filelayout_write_call_ops,
+                          sync, RPC_TASK_SOFTCONN);
         return PNFS_ATTEMPTED;
 }
 
@@ -988,12 +933,14 @@ static const struct nfs_pageio_ops filelayout_pg_read_ops = {
         .pg_init = filelayout_pg_init_read,
         .pg_test = filelayout_pg_test,
         .pg_doio = pnfs_generic_pg_readpages,
+        .pg_cleanup = pnfs_generic_pg_cleanup,
 };
 
 static const struct nfs_pageio_ops filelayout_pg_write_ops = {
         .pg_init = filelayout_pg_init_write,
         .pg_test = filelayout_pg_test,
         .pg_doio = pnfs_generic_pg_writepages,
+        .pg_cleanup = pnfs_generic_pg_cleanup,
 };
 
 static u32 select_bucket_index(struct nfs4_filelayout_segment *fl, u32 j)
@@ -1004,37 +951,11 @@ static u32 select_bucket_index(struct nfs4_filelayout_segment *fl, u32 j)
                 return j;
 }
 
-/* The generic layer is about to remove the req from the commit list.
- * If this will make the bucket empty, it will need to put the lseg reference.
- * Note this is must be called holding the inode (/cinfo) lock
- */
-static void
-filelayout_clear_request_commit(struct nfs_page *req,
-                                struct nfs_commit_info *cinfo)
-{
-        struct pnfs_layout_segment *freeme = NULL;
-
-        if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
-                goto out;
-        cinfo->ds->nwritten--;
-        if (list_is_singular(&req->wb_list)) {
-                struct pnfs_commit_bucket *bucket;
-
-                bucket = list_first_entry(&req->wb_list,
-                                          struct pnfs_commit_bucket,
-                                          written);
-                freeme = bucket->wlseg;
-                bucket->wlseg = NULL;
-        }
-out:
-        nfs_request_remove_commit_list(req, cinfo);
-        pnfs_put_lseg_locked(freeme);
-}
-
 static void
 filelayout_mark_request_commit(struct nfs_page *req,
                                struct pnfs_layout_segment *lseg,
-                               struct nfs_commit_info *cinfo)
+                               struct nfs_commit_info *cinfo,
+                               u32 ds_commit_idx)
 
 {
         struct nfs4_filelayout_segment *fl = FILELAYOUT_LSEG(lseg);
@@ -1064,7 +985,7 @@ filelayout_mark_request_commit(struct nfs_page *req,
                  * is normally transferred to the COMMIT call and released
                  * there.  It could also be released if the last req is pulled
                  * off due to a rewrite, in which case it will be done in
-                 * filelayout_clear_request_commit
+                 * pnfs_generic_clear_request_commit
                  */
                 buckets[i].wlseg = pnfs_get_lseg(lseg);
         }
@@ -1138,101 +1059,15 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
         fh = select_ds_fh_from_commit(lseg, data->ds_commit_index);
         if (fh)
                 data->args.fh = fh;
-        return nfs_initiate_commit(ds_clnt, data,
+        return nfs_initiate_commit(ds_clnt, data, NFS_PROTO(data->inode),
                                    &filelayout_commit_call_ops, how,
                                    RPC_TASK_SOFTCONN);
 out_err:
-        prepare_to_resend_writes(data);
-        filelayout_commit_release(data);
+        pnfs_generic_prepare_to_resend_writes(data);
+        pnfs_generic_commit_release(data);
         return -EAGAIN;
 }
 
-static int
-transfer_commit_list(struct list_head *src, struct list_head *dst,
-                     struct nfs_commit_info *cinfo, int max)
-{
-        struct nfs_page *req, *tmp;
-        int ret = 0;
-
-        list_for_each_entry_safe(req, tmp, src, wb_list) {
-                if (!nfs_lock_request(req))
-                        continue;
-                kref_get(&req->wb_kref);
-                if (cond_resched_lock(cinfo->lock))
-                        list_safe_reset_next(req, tmp, wb_list);
-                nfs_request_remove_commit_list(req, cinfo);
-                clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
-                nfs_list_add_request(req, dst);
-                ret++;
-                if ((ret == max) && !cinfo->dreq)
-                        break;
-        }
-        return ret;
-}
-
-/* Note called with cinfo->lock held. */
-static int
-filelayout_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
-                               struct nfs_commit_info *cinfo,
-                               int max)
-{
-        struct list_head *src = &bucket->written;
-        struct list_head *dst = &bucket->committing;
-        int ret;
-
-        ret = transfer_commit_list(src, dst, cinfo, max);
-        if (ret) {
-                cinfo->ds->nwritten -= ret;
-                cinfo->ds->ncommitting += ret;
-                bucket->clseg = bucket->wlseg;
-                if (list_empty(src))
-                        bucket->wlseg = NULL;
-                else
-                        pnfs_get_lseg(bucket->clseg);
-        }
-        return ret;
-}
-
-/* Move reqs from written to committing lists, returning count of number moved.
- * Note called with cinfo->lock held.
- */
-static int filelayout_scan_commit_lists(struct nfs_commit_info *cinfo,
-                                        int max)
-{
-        int i, rv = 0, cnt;
-
-        for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) {
-                cnt = filelayout_scan_ds_commit_list(&cinfo->ds->buckets[i],
-                                                     cinfo, max);
-                max -= cnt;
-                rv += cnt;
-        }
-        return rv;
-}
-
-/* Pull everything off the committing lists and dump into @dst */
-static void filelayout_recover_commit_reqs(struct list_head *dst,
-                                           struct nfs_commit_info *cinfo)
-{
-        struct pnfs_commit_bucket *b;
-        struct pnfs_layout_segment *freeme;
-        int i;
-
-restart:
-        spin_lock(cinfo->lock);
-        for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
-                if (transfer_commit_list(&b->written, dst, cinfo, 0)) {
-                        freeme = b->wlseg;
-                        b->wlseg = NULL;
-                        spin_unlock(cinfo->lock);
-                        pnfs_put_lseg(freeme);
-                        goto restart;
-                }
-        }
-        cinfo->ds->nwritten = 0;
-        spin_unlock(cinfo->lock);
-}
-
 /* filelayout_search_commit_reqs - Search lists in @cinfo for the head reqest
  *                                 for @page
  * @cinfo - commit info for current inode
@@ -1263,108 +1098,14 @@ filelayout_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page)
         return NULL;
 }
 
-static void filelayout_retry_commit(struct nfs_commit_info *cinfo, int idx)
-{
-        struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
-        struct pnfs_commit_bucket *bucket;
-        struct pnfs_layout_segment *freeme;
-        int i;
-
-        for (i = idx; i < fl_cinfo->nbuckets; i++) {
-                bucket = &fl_cinfo->buckets[i];
-                if (list_empty(&bucket->committing))
-                        continue;
-                nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo);
-                spin_lock(cinfo->lock);
-                freeme = bucket->clseg;
-                bucket->clseg = NULL;
-                spin_unlock(cinfo->lock);
-                pnfs_put_lseg(freeme);
-        }
-}
-
-static unsigned int
-alloc_ds_commits(struct nfs_commit_info *cinfo, struct list_head *list)
-{
-        struct pnfs_ds_commit_info *fl_cinfo;
-        struct pnfs_commit_bucket *bucket;
-        struct nfs_commit_data *data;
-        int i;
-        unsigned int nreq = 0;
-
-        fl_cinfo = cinfo->ds;
-        bucket = fl_cinfo->buckets;
-        for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) {
-                if (list_empty(&bucket->committing))
-                        continue;
-                data = nfs_commitdata_alloc();
-                if (!data)
-                        break;
-                data->ds_commit_index = i;
-                spin_lock(cinfo->lock);
-                data->lseg = bucket->clseg;
-                bucket->clseg = NULL;
-                spin_unlock(cinfo->lock);
-                list_add(&data->pages, list);
-                nreq++;
-        }
-
-        /* Clean up on error */
-        filelayout_retry_commit(cinfo, i);
-        /* Caller will clean up entries put on list */
-        return nreq;
-}
-
-/* This follows nfs_commit_list pretty closely */
 static int
 filelayout_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
                            int how, struct nfs_commit_info *cinfo)
 {
-        struct nfs_commit_data *data, *tmp;
-        LIST_HEAD(list);
-        unsigned int nreq = 0;
-
-        if (!list_empty(mds_pages)) {
-                data = nfs_commitdata_alloc();
-                if (data != NULL) {
-                        data->lseg = NULL;
-                        list_add(&data->pages, &list);
-                        nreq++;
-                } else {
-                        nfs_retry_commit(mds_pages, NULL, cinfo);
-                        filelayout_retry_commit(cinfo, 0);
-                        cinfo->completion_ops->error_cleanup(NFS_I(inode));
-                        return -ENOMEM;
-                }
-        }
-
-        nreq += alloc_ds_commits(cinfo, &list);
-
-        if (nreq == 0) {
-                cinfo->completion_ops->error_cleanup(NFS_I(inode));
-                goto out;
-        }
-
-        atomic_add(nreq, &cinfo->mds->rpcs_out);
-
-        list_for_each_entry_safe(data, tmp, &list, pages) {
-                list_del_init(&data->pages);
-                if (!data->lseg) {
-                        nfs_init_commit(data, mds_pages, NULL, cinfo);
-                        nfs_initiate_commit(NFS_CLIENT(inode), data,
-                                            data->mds_ops, how, 0);
-                } else {
-                        struct pnfs_commit_bucket *buckets;
-
-                        buckets = cinfo->ds->buckets;
-                        nfs_init_commit(data, &buckets[data->ds_commit_index].committing, data->lseg, cinfo);
-                        filelayout_initiate_commit(data, how);
-                }
-        }
-out:
-        cinfo->ds->ncommitting = 0;
-        return PNFS_ATTEMPTED;
+        return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo,
+                                            filelayout_initiate_commit);
 }
+
 static struct nfs4_deviceid_node *
 filelayout_alloc_deviceid_node(struct nfs_server *server,
                 struct pnfs_device *pdev, gfp_t gfp_flags)
@@ -1421,9 +1162,9 @@ static struct pnfs_layoutdriver_type filelayout_type = {
         .pg_write_ops           = &filelayout_pg_write_ops,
         .get_ds_info            = &filelayout_get_ds_info,
         .mark_request_commit    = filelayout_mark_request_commit,
-        .clear_request_commit   = filelayout_clear_request_commit,
-        .scan_commit_lists      = filelayout_scan_commit_lists,
-        .recover_commit_reqs    = filelayout_recover_commit_reqs,
+        .clear_request_commit   = pnfs_generic_clear_request_commit,
+        .scan_commit_lists      = pnfs_generic_scan_commit_lists,
+        .recover_commit_reqs    = pnfs_generic_recover_commit_reqs,
         .search_commit_reqs     = filelayout_search_commit_reqs,
         .commit_pagelist        = filelayout_commit_pagelist,
         .read_pagelist          = filelayout_read_pagelist,
diff --git a/fs/nfs/filelayout/filelayout.h b/fs/nfs/filelayout/filelayout.h
index 7c9f800c49d7..2896cb833a11 100644
--- a/fs/nfs/filelayout/filelayout.h
+++ b/fs/nfs/filelayout/filelayout.h
@@ -33,13 +33,6 @@
 #include "../pnfs.h"
 
 /*
- * Default data server connection timeout and retrans vaules.
- * Set by module paramters dataserver_timeo and dataserver_retrans.
- */
-#define NFS4_DEF_DS_TIMEO   600 /* in tenths of a second */
-#define NFS4_DEF_DS_RETRANS 5
-
-/*
  * Field testing shows we need to support up to 4096 stripe indices.
  * We store each index as a u8 (u32 on the wire) to keep the memory footprint
  * reasonable. This in turn means we support a maximum of 256
@@ -48,32 +41,11 @@
 #define NFS4_PNFS_MAX_STRIPE_CNT 4096
 #define NFS4_PNFS_MAX_MULTI_CNT  256 /* 256 fit into a u8 stripe_index */
 
-/* error codes for internal use */
-#define NFS4ERR_RESET_TO_MDS   12001
-
 enum stripetype4 {
         STRIPE_SPARSE = 1,
         STRIPE_DENSE = 2
 };
 
-/* Individual ip address */
-struct nfs4_pnfs_ds_addr {
-        struct sockaddr_storage da_addr;
-        size_t                  da_addrlen;
-        struct list_head        da_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
-        char                    *da_remotestr;  /* human readable addr+port */
-};
-
-struct nfs4_pnfs_ds {
-        struct list_head        ds_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
-        char                    *ds_remotestr;  /* comma sep list of addrs */
-        struct list_head        ds_addrs;
-        struct nfs_client       *ds_clp;
-        atomic_t                ds_count;
-        unsigned long           ds_state;
-#define NFS4DS_CONNECTING       0       /* ds is establishing connection */
-};
-
 struct nfs4_file_layout_dsaddr {
         struct nfs4_deviceid_node       id_node;
         u32                             stripe_count;
@@ -119,17 +91,6 @@ FILELAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg)
         return &FILELAYOUT_LSEG(lseg)->dsaddr->id_node;
 }
 
-static inline void
-filelayout_mark_devid_invalid(struct nfs4_deviceid_node *node)
-{
-        u32 *p = (u32 *)&node->deviceid;
-
-        printk(KERN_WARNING "NFS: Deviceid [%x%x%x%x] marked out of use.\n",
-                p[0], p[1], p[2], p[3]);
-
-        set_bit(NFS_DEVICEID_INVALID, &node->flags);
-}
-
 static inline bool
 filelayout_test_devid_invalid(struct nfs4_deviceid_node *node)
 {
@@ -142,7 +103,6 @@ filelayout_test_devid_unavailable(struct nfs4_deviceid_node *node);
 extern struct nfs_fh *
 nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j);
 
-extern void print_ds(struct nfs4_pnfs_ds *ds);
 u32 nfs4_fl_calc_j_index(struct pnfs_layout_segment *lseg, loff_t offset);
 u32 nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, u32 j);
 struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg,
diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c
index bfecac781f19..4f372e224603 100644
--- a/fs/nfs/filelayout/filelayoutdev.c
+++ b/fs/nfs/filelayout/filelayoutdev.c
@@ -31,7 +31,6 @@
 #include <linux/nfs_fs.h>
 #include <linux/vmalloc.h>
 #include <linux/module.h>
-#include <linux/sunrpc/addr.h>
 
 #include "../internal.h"
 #include "../nfs4session.h"
@@ -42,183 +41,6 @@
 static unsigned int dataserver_timeo = NFS4_DEF_DS_TIMEO;
 static unsigned int dataserver_retrans = NFS4_DEF_DS_RETRANS;
 
-/*
- * Data server cache
- *
- * Data servers can be mapped to different device ids.
- * nfs4_pnfs_ds reference counting
- *   - set to 1 on allocation
- *   - incremented when a device id maps a data server already in the cache.
- *   - decremented when deviceid is removed from the cache.
- */
-static DEFINE_SPINLOCK(nfs4_ds_cache_lock);
-static LIST_HEAD(nfs4_data_server_cache);
-
-/* Debug routines */
-void
-print_ds(struct nfs4_pnfs_ds *ds)
-{
-        if (ds == NULL) {
-                printk("%s NULL device\n", __func__);
-                return;
-        }
-        printk("        ds %s\n"
-                "        ref count %d\n"
-                "        client %p\n"
-                "        cl_exchange_flags %x\n",
-                ds->ds_remotestr,
-                atomic_read(&ds->ds_count), ds->ds_clp,
-                ds->ds_clp ? ds->ds_clp->cl_exchange_flags : 0);
-}
-
-static bool
-same_sockaddr(struct sockaddr *addr1, struct sockaddr *addr2)
-{
-        struct sockaddr_in *a, *b;
-        struct sockaddr_in6 *a6, *b6;
-
-        if (addr1->sa_family != addr2->sa_family)
-                return false;
-
-        switch (addr1->sa_family) {
-        case AF_INET:
-                a = (struct sockaddr_in *)addr1;
-                b = (struct sockaddr_in *)addr2;
-
-                if (a->sin_addr.s_addr == b->sin_addr.s_addr &&
-                    a->sin_port == b->sin_port)
-                        return true;
-                break;
-
-        case AF_INET6:
-                a6 = (struct sockaddr_in6 *)addr1;
-                b6 = (struct sockaddr_in6 *)addr2;
-
-                /* LINKLOCAL addresses must have matching scope_id */
-                if (ipv6_addr_src_scope(&a6->sin6_addr) ==
-                    IPV6_ADDR_SCOPE_LINKLOCAL &&
-                    a6->sin6_scope_id != b6->sin6_scope_id)
-                        return false;
-
-                if (ipv6_addr_equal(&a6->sin6_addr, &b6->sin6_addr) &&
-                    a6->sin6_port == b6->sin6_port)
-                        return true;
-                break;
-
-        default:
-                dprintk("%s: unhandled address family: %u\n",
-                        __func__, addr1->sa_family);
-                return false;
-        }
-
-        return false;
-}
-
-static bool
-_same_data_server_addrs_locked(const struct list_head *dsaddrs1,
-                               const struct list_head *dsaddrs2)
-{
-        struct nfs4_pnfs_ds_addr *da1, *da2;
-
-        /* step through both lists, comparing as we go */
-        for (da1 = list_first_entry(dsaddrs1, typeof(*da1), da_node),
-             da2 = list_first_entry(dsaddrs2, typeof(*da2), da_node);
-             da1 != NULL && da2 != NULL;
-             da1 = list_entry(da1->da_node.next, typeof(*da1), da_node),
-             da2 = list_entry(da2->da_node.next, typeof(*da2), da_node)) {
-                if (!same_sockaddr((struct sockaddr *)&da1->da_addr,
-                                   (struct sockaddr *)&da2->da_addr))
-                        return false;
-        }
-        if (da1 == NULL && da2 == NULL)
-                return true;
-
-        return false;
-}
-
-/*
- * Lookup DS by addresses.  nfs4_ds_cache_lock is held
- */
-static struct nfs4_pnfs_ds *
-_data_server_lookup_locked(const struct list_head *dsaddrs)
-{
-        struct nfs4_pnfs_ds *ds;
-
-        list_for_each_entry(ds, &nfs4_data_server_cache, ds_node)
-                if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs))
-                        return ds;
-        return NULL;
-}
-
-/*
- * Create an rpc connection to the nfs4_pnfs_ds data server
- * Currently only supports IPv4 and IPv6 addresses
- */
-static int
-nfs4_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds)
-{
-        struct nfs_client *clp = ERR_PTR(-EIO);
-        struct nfs4_pnfs_ds_addr *da;
-        int status = 0;
-
-        dprintk("--> %s DS %s au_flavor %d\n", __func__, ds->ds_remotestr,
-                mds_srv->nfs_client->cl_rpcclient->cl_auth->au_flavor);
-
-        list_for_each_entry(da, &ds->ds_addrs, da_node) {
-                dprintk("%s: DS %s: trying address %s\n",
-                        __func__, ds->ds_remotestr, da->da_remotestr);
-
-                clp = nfs4_set_ds_client(mds_srv->nfs_client,
-                                        (struct sockaddr *)&da->da_addr,
-                                        da->da_addrlen, IPPROTO_TCP,
-                                        dataserver_timeo, dataserver_retrans);
-                if (!IS_ERR(clp))
-                        break;
-        }
-
-        if (IS_ERR(clp)) {
-                status = PTR_ERR(clp);
-                goto out;
-        }
-
-        status = nfs4_init_ds_session(clp, mds_srv->nfs_client->cl_lease_time);
-        if (status)
-                goto out_put;
-
-        smp_wmb();
-        ds->ds_clp = clp;
-        dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr);
-out:
-        return status;
-out_put:
-        nfs_put_client(clp);
-        goto out;
-}
-
-static void
-destroy_ds(struct nfs4_pnfs_ds *ds)
-{
-        struct nfs4_pnfs_ds_addr *da;
-
-        dprintk("--> %s\n", __func__);
-        ifdebug(FACILITY)
-                print_ds(ds);
-
-        nfs_put_client(ds->ds_clp);
-
-        while (!list_empty(&ds->ds_addrs)) {
-                da = list_first_entry(&ds->ds_addrs,
-                                      struct nfs4_pnfs_ds_addr,
-                                      da_node);
-                list_del_init(&da->da_node);
-                kfree(da->da_remotestr);
-                kfree(da);
-        }
-
-        kfree(ds->ds_remotestr);
-        kfree(ds);
-}
-
 void
 nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
 {
@@ -229,259 +51,13 @@ nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
 
         for (i = 0; i < dsaddr->ds_num; i++) {
                 ds = dsaddr->ds_list[i];
-                if (ds != NULL) {
-                        if (atomic_dec_and_lock(&ds->ds_count,
-                                                &nfs4_ds_cache_lock)) {
-                                list_del_init(&ds->ds_node);
-                                spin_unlock(&nfs4_ds_cache_lock);
-                                destroy_ds(ds);
-                        }
-                }
+                if (ds != NULL)
+                        nfs4_pnfs_ds_put(ds);
         }
         kfree(dsaddr->stripe_indices);
         kfree(dsaddr);
 }
 
-/*
- * Create a string with a human readable address and port to avoid
- * complicated setup around many dprinks.
- */
-static char *
-nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags)
-{
-        struct nfs4_pnfs_ds_addr *da;
-        char *remotestr;
-        size_t len;
-        char *p;
-
-        len = 3;        /* '{', '}' and eol */
-        list_for_each_entry(da, dsaddrs, da_node) {
-                len += strlen(da->da_remotestr) + 1;    /* string plus comma */
-        }
-
-        remotestr = kzalloc(len, gfp_flags);
-        if (!remotestr)
-                return NULL;
-
-        p = remotestr;
-        *(p++) = '{';
-        len--;
-        list_for_each_entry(da, dsaddrs, da_node) {
-                size_t ll = strlen(da->da_remotestr);
-
-                if (ll > len)
-                        goto out_err;
-
-                memcpy(p, da->da_remotestr, ll);
-                p += ll;
-                len -= ll;
-
-                if (len < 1)
-                        goto out_err;
-                (*p++) = ',';
-                len--;
-        }
-        if (len < 2)
-                goto out_err;
-        *(p++) = '}';
-        *p = '\0';
-        return remotestr;
-out_err:
-        kfree(remotestr);
-        return NULL;
-}
-
-static struct nfs4_pnfs_ds *
-nfs4_pnfs_ds_add(struct list_head *dsaddrs, gfp_t gfp_flags)
-{
-        struct nfs4_pnfs_ds *tmp_ds, *ds = NULL;
-        char *remotestr;
-
-        if (list_empty(dsaddrs)) {
-                dprintk("%s: no addresses defined\n", __func__);
-                goto out;
-        }
-
-        ds = kzalloc(sizeof(*ds), gfp_flags);
-        if (!ds)
-                goto out;
-
-        /* this is only used for debugging, so it's ok if its NULL */
-        remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags);
-
-        spin_lock(&nfs4_ds_cache_lock);
-        tmp_ds = _data_server_lookup_locked(dsaddrs);
-        if (tmp_ds == NULL) {
-                INIT_LIST_HEAD(&ds->ds_addrs);
-                list_splice_init(dsaddrs, &ds->ds_addrs);
-                ds->ds_remotestr = remotestr;
-                atomic_set(&ds->ds_count, 1);
-                INIT_LIST_HEAD(&ds->ds_node);
-                ds->ds_clp = NULL;
-                list_add(&ds->ds_node, &nfs4_data_server_cache);
-                dprintk("%s add new data server %s\n", __func__,
-                        ds->ds_remotestr);
-        } else {
-                kfree(remotestr);
-                kfree(ds);
-                atomic_inc(&tmp_ds->ds_count);
-                dprintk("%s data server %s found, inc'ed ds_count to %d\n",
-                        __func__, tmp_ds->ds_remotestr,
-                        atomic_read(&tmp_ds->ds_count));
-                ds = tmp_ds;
-        }
-        spin_unlock(&nfs4_ds_cache_lock);
-out:
-        return ds;
-}
-
-/*
- * Currently only supports ipv4, ipv6 and one multi-path address.
- */
-static struct nfs4_pnfs_ds_addr *
-decode_ds_addr(struct net *net, struct xdr_stream *streamp, gfp_t gfp_flags)
-{
-        struct nfs4_pnfs_ds_addr *da = NULL;
-        char *buf, *portstr;
-        __be16 port;
-        int nlen, rlen;
-        int tmp[2];
-        __be32 *p;
-        char *netid, *match_netid;
-        size_t len, match_netid_len;
-        char *startsep = "";
-        char *endsep = "";
-
-
-        /* r_netid */
-        p = xdr_inline_decode(streamp, 4);
-        if (unlikely(!p))
-                goto out_err;
-        nlen = be32_to_cpup(p++);
-
-        p = xdr_inline_decode(streamp, nlen);
-        if (unlikely(!p))
-                goto out_err;
-
-        netid = kmalloc(nlen+1, gfp_flags);
-        if (unlikely(!netid))
-                goto out_err;
-
-        netid[nlen] = '\0';
-        memcpy(netid, p, nlen);
-
-        /* r_addr: ip/ip6addr with port in dec octets - see RFC 5665 */
-        p = xdr_inline_decode(streamp, 4);
-        if (unlikely(!p))
-                goto out_free_netid;
-        rlen = be32_to_cpup(p);
-
-        p = xdr_inline_decode(streamp, rlen);
-        if (unlikely(!p))
-                goto out_free_netid;
-
-        /* port is ".ABC.DEF", 8 chars max */
-        if (rlen > INET6_ADDRSTRLEN + IPV6_SCOPE_ID_LEN + 8) {
-                dprintk("%s: Invalid address, length %d\n", __func__,
-                        rlen);
-                goto out_free_netid;
-        }
-        buf = kmalloc(rlen + 1, gfp_flags);
-        if (!buf) {
-                dprintk("%s: Not enough memory\n", __func__);
-                goto out_free_netid;
-        }
-        buf[rlen] = '\0';
-        memcpy(buf, p, rlen);
-
-        /* replace port '.' with '-' */
-        portstr = strrchr(buf, '.');
-        if (!portstr) {
-                dprintk("%s: Failed finding expected dot in port\n",
-                        __func__);
-                goto out_free_buf;
-        }
-        *portstr = '-';
-
-        /* find '.' between address and port */
-        portstr = strrchr(buf, '.');
-        if (!portstr) {
-                dprintk("%s: Failed finding expected dot between address and "
-                        "port\n", __func__);
-                goto out_free_buf;
-        }
-        *portstr = '\0';
-
-        da = kzalloc(sizeof(*da), gfp_flags);
-        if (unlikely(!da))
-                goto out_free_buf;
-
-        INIT_LIST_HEAD(&da->da_node);
-
-        if (!rpc_pton(net, buf, portstr-buf, (struct sockaddr *)&da->da_addr,
-                      sizeof(da->da_addr))) {
-                dprintk("%s: error parsing address %s\n", __func__, buf);
-                goto out_free_da;
-        }
-
-        portstr++;
-        sscanf(portstr, "%d-%d", &tmp[0], &tmp[1]);
-        port = htons((tmp[0] << 8) | (tmp[1]));
-
-        switch (da->da_addr.ss_family) {
-        case AF_INET:
-                ((struct sockaddr_in *)&da->da_addr)->sin_port = port;
-                da->da_addrlen = sizeof(struct sockaddr_in);
-                match_netid = "tcp";
-                match_netid_len = 3;
-                break;
-
-        case AF_INET6:
-                ((struct sockaddr_in6 *)&da->da_addr)->sin6_port = port;
-                da->da_addrlen = sizeof(struct sockaddr_in6);
-                match_netid = "tcp6";
-                match_netid_len = 4;
-                startsep = "[";
-                endsep = "]";
-                break;
-
-        default:
-                dprintk("%s: unsupported address family: %u\n",
-                        __func__, da->da_addr.ss_family);
-                goto out_free_da;
-        }
-
-        if (nlen != match_netid_len || strncmp(netid, match_netid, nlen)) {
-                dprintk("%s: ERROR: r_netid \"%s\" != \"%s\"\n",
-                        __func__, netid, match_netid);
-                goto out_free_da;
-        }
-
-        /* save human readable address */
-        len = strlen(startsep) + strlen(buf) + strlen(endsep) + 7;
-        da->da_remotestr = kzalloc(len, gfp_flags);
-
-        /* NULL is ok, only used for dprintk */
-        if (da->da_remotestr)
-                snprintf(da->da_remotestr, len, "%s%s%s:%u", startsep,
-                         buf, endsep, ntohs(port));
-
-        dprintk("%s: Parsed DS addr %s\n", __func__, da->da_remotestr);
-        kfree(buf);
-        kfree(netid);
-        return da;
-
-out_free_da:
-        kfree(da);
-out_free_buf:
-        dprintk("%s: Error parsing DS addr: %s\n", __func__, buf);
-        kfree(buf);
-out_free_netid:
-        kfree(netid);
-out_err:
-        return NULL;
-}
-
 /* Decode opaque device data and return the result */
 struct nfs4_file_layout_dsaddr *
 nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
@@ -584,8 +160,8 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
 
                 mp_count = be32_to_cpup(p); /* multipath count */
                 for (j = 0; j < mp_count; j++) {
-                        da = decode_ds_addr(server->nfs_client->cl_net,
-                                            &stream, gfp_flags);
+                        da = nfs4_decode_mp_ds_addr(server->nfs_client->cl_net,
+                                                    &stream, gfp_flags);
                         if (da)
                                 list_add_tail(&da->da_node, &dsaddrs);
                 }
@@ -681,22 +257,7 @@ nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j)
         return flseg->fh_array[i];
 }
 
-static void nfs4_wait_ds_connect(struct nfs4_pnfs_ds *ds)
-{
-        might_sleep();
-        wait_on_bit_action(&ds->ds_state, NFS4DS_CONNECTING,
-                           nfs_wait_bit_killable, TASK_KILLABLE);
-}
-
-static void nfs4_clear_ds_conn_bit(struct nfs4_pnfs_ds *ds)
-{
-        smp_mb__before_atomic();
-        clear_bit(NFS4DS_CONNECTING, &ds->ds_state);
-        smp_mb__after_atomic();
-        wake_up_bit(&ds->ds_state, NFS4DS_CONNECTING);
-}
-
-
+/* Upon return, either ds is connected, or ds is NULL */
 struct nfs4_pnfs_ds *
 nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
 {
@@ -704,29 +265,23 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
         struct nfs4_pnfs_ds *ds = dsaddr->ds_list[ds_idx];
         struct nfs4_deviceid_node *devid = FILELAYOUT_DEVID_NODE(lseg);
         struct nfs4_pnfs_ds *ret = ds;
+        struct nfs_server *s = NFS_SERVER(lseg->pls_layout->plh_inode);
 
         if (ds == NULL) {
                 printk(KERN_ERR "NFS: %s: No data server for offset index %d\n",
                         __func__, ds_idx);
-                filelayout_mark_devid_invalid(devid);
+                pnfs_generic_mark_devid_invalid(devid);
                 goto out;
         }
         smp_rmb();
         if (ds->ds_clp)
                 goto out_test_devid;
 
-        if (test_and_set_bit(NFS4DS_CONNECTING, &ds->ds_state) == 0) {
-                struct nfs_server *s = NFS_SERVER(lseg->pls_layout->plh_inode);
-                int err;
-
-                err = nfs4_ds_connect(s, ds);
-                if (err)
-                        nfs4_mark_deviceid_unavailable(devid);
-                nfs4_clear_ds_conn_bit(ds);
-        } else {
-                /* Either ds is connected, or ds is NULL */
-                nfs4_wait_ds_connect(ds);
-        }
+        nfs4_pnfs_ds_connect(s, ds, devid, dataserver_timeo,
+                             dataserver_retrans, 4,
+                             s->nfs_client->cl_minorversion,
+                             s->nfs_client->cl_rpcclient->cl_auth->au_flavor);
+
 out_test_devid:
         if (filelayout_test_devid_unavailable(devid))
                 ret = NULL;
diff --git a/fs/nfs/flexfilelayout/Makefile b/fs/nfs/flexfilelayout/Makefile
new file mode 100644
index 000000000000..1d2c9f6bbcd4
--- /dev/null
+++ b/fs/nfs/flexfilelayout/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for the pNFS Flexfile Layout Driver kernel module
+#
+obj-$(CONFIG_PNFS_FLEXFILE_LAYOUT) += nfs_layout_flexfiles.o
+nfs_layout_flexfiles-y := flexfilelayout.o flexfilelayoutdev.o
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
new file mode 100644
index 000000000000..f29fb7d7e8f8
--- /dev/null
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -0,0 +1,1574 @@
+/*
+ * Module for pnfs flexfile layout driver.
+ *
+ * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
+ *
+ * Tao Peng <bergwolf@primarydata.com>
+ */
+
+#include <linux/nfs_fs.h>
+#include <linux/nfs_page.h>
+#include <linux/module.h>
+
+#include <linux/sunrpc/metrics.h>
+#include <linux/nfs_idmap.h>
+
+#include "flexfilelayout.h"
+#include "../nfs4session.h"
+#include "../internal.h"
+#include "../delegation.h"
+#include "../nfs4trace.h"
+#include "../iostat.h"
+#include "../nfs.h"
+
+#define NFSDBG_FACILITY         NFSDBG_PNFS_LD
+
+#define FF_LAYOUT_POLL_RETRY_MAX     (15*HZ)
+
+static struct pnfs_layout_hdr *
+ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags)
+{
+        struct nfs4_flexfile_layout *ffl;
+
+        ffl = kzalloc(sizeof(*ffl), gfp_flags);
+        if (ffl) {
+                INIT_LIST_HEAD(&ffl->error_list);
+                return &ffl->generic_hdr;
+        } else
+                return NULL;
+}
+
+static void
+ff_layout_free_layout_hdr(struct pnfs_layout_hdr *lo)
+{
+        struct nfs4_ff_layout_ds_err *err, *n;
+
+        list_for_each_entry_safe(err, n, &FF_LAYOUT_FROM_HDR(lo)->error_list,
+                                 list) {
+                list_del(&err->list);
+                kfree(err);
+        }
+        kfree(FF_LAYOUT_FROM_HDR(lo));
+}
+
+static int decode_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid)
+{
+        __be32 *p;
+
+        p = xdr_inline_decode(xdr, NFS4_STATEID_SIZE);
+        if (unlikely(p == NULL))
+                return -ENOBUFS;
+        memcpy(stateid, p, NFS4_STATEID_SIZE);
+        dprintk("%s: stateid id= [%x%x%x%x]\n", __func__,
+                p[0], p[1], p[2], p[3]);
+        return 0;
+}
+
+static int decode_deviceid(struct xdr_stream *xdr, struct nfs4_deviceid *devid)
+{
+        __be32 *p;
+
+        p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE);
+        if (unlikely(!p))
+                return -ENOBUFS;
+        memcpy(devid, p, NFS4_DEVICEID4_SIZE);
+        nfs4_print_deviceid(devid);
+        return 0;
+}
+
+static int decode_nfs_fh(struct xdr_stream *xdr, struct nfs_fh *fh)
+{
+        __be32 *p;
+
+        p = xdr_inline_decode(xdr, 4);
+        if (unlikely(!p))
+                return -ENOBUFS;
+        fh->size = be32_to_cpup(p++);
+        if (fh->size > sizeof(struct nfs_fh)) {
+                printk(KERN_ERR "NFS flexfiles: Too big fh received %d\n",
+                       fh->size);
+                return -EOVERFLOW;
+        }
+        /* fh.data */
+        p = xdr_inline_decode(xdr, fh->size);
+        if (unlikely(!p))
+                return -ENOBUFS;
+        memcpy(&fh->data, p, fh->size);
+        dprintk("%s: fh len %d\n", __func__, fh->size);
+
+        return 0;
+}
+
+/*
+ * Currently only stringified uids and gids are accepted.
+ * I.e., kerberos is not supported to the DSes, so no pricipals.
+ *
+ * That means that one common function will suffice, but when
+ * principals are added, this should be split to accomodate
+ * calls to both nfs_map_name_to_uid() and nfs_map_group_to_gid().
+ */
+static int
+decode_name(struct xdr_stream *xdr, u32 *id)
+{
+        __be32 *p;
+        int len;
+
+        /* opaque_length(4)*/
+        p = xdr_inline_decode(xdr, 4);
+        if (unlikely(!p))
+                return -ENOBUFS;
+        len = be32_to_cpup(p++);
+        if (len < 0)
+                return -EINVAL;
+
+        dprintk("%s: len %u\n", __func__, len);
+
+        /* opaque body */
+        p = xdr_inline_decode(xdr, len);
+        if (unlikely(!p))
+                return -ENOBUFS;
+
+        if (!nfs_map_string_to_numeric((char *)p, len, id))
+                return -EINVAL;
+
+        return 0;
+}
+
+static void ff_layout_free_mirror_array(struct nfs4_ff_layout_segment *fls)
+{
+        int i;
+
+        if (fls->mirror_array) {
+                for (i = 0; i < fls->mirror_array_cnt; i++) {
+                        /* normally mirror_ds is freed in
+                         * .free_deviceid_node but we still do it here
+                         * for .alloc_lseg error path */
+                        if (fls->mirror_array[i]) {
+                                kfree(fls->mirror_array[i]->fh_versions);
+                                nfs4_ff_layout_put_deviceid(fls->mirror_array[i]->mirror_ds);
+                                kfree(fls->mirror_array[i]);
+                        }
+                }
+                kfree(fls->mirror_array);
+                fls->mirror_array = NULL;
+        }
+}
+
+static int ff_layout_check_layout(struct nfs4_layoutget_res *lgr)
+{
+        int ret = 0;
+
+        dprintk("--> %s\n", __func__);
+
+        /* FIXME: remove this check when layout segment support is added */
+        if (lgr->range.offset != 0 ||
+            lgr->range.length != NFS4_MAX_UINT64) {
+                dprintk("%s Only whole file layouts supported. Use MDS i/o\n",
+                        __func__);
+                ret = -EINVAL;
+        }
+
+        dprintk("--> %s returns %d\n", __func__, ret);
+        return ret;
+}
+
+static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls)
+{
+        if (fls) {
+                ff_layout_free_mirror_array(fls);
+                kfree(fls);
+        }
+}
+
+static void ff_layout_sort_mirrors(struct nfs4_ff_layout_segment *fls)
+{
+        struct nfs4_ff_layout_mirror *tmp;
+        int i, j;
+
+        for (i = 0; i < fls->mirror_array_cnt - 1; i++) {
+                for (j = i + 1; j < fls->mirror_array_cnt; j++)
+                        if (fls->mirror_array[i]->efficiency <
+                            fls->mirror_array[j]->efficiency) {
+                                tmp = fls->mirror_array[i];
+                                fls->mirror_array[i] = fls->mirror_array[j];
+                                fls->mirror_array[j] = tmp;
+                        }
+        }
+}
+
+static struct pnfs_layout_segment *
+ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
+                     struct nfs4_layoutget_res *lgr,
+                     gfp_t gfp_flags)
+{
+        struct pnfs_layout_segment *ret;
+        struct nfs4_ff_layout_segment *fls = NULL;
+        struct xdr_stream stream;
+        struct xdr_buf buf;
+        struct page *scratch;
+        u64 stripe_unit;
+        u32 mirror_array_cnt;
+        __be32 *p;
+        int i, rc;
+
+        dprintk("--> %s\n", __func__);
+        scratch = alloc_page(gfp_flags);
+        if (!scratch)
+                return ERR_PTR(-ENOMEM);
+
+        xdr_init_decode_pages(&stream, &buf, lgr->layoutp->pages,
+                              lgr->layoutp->len);
+        xdr_set_scratch_buffer(&stream, page_address(scratch), PAGE_SIZE);
+
+        /* stripe unit and mirror_array_cnt */
+        rc = -EIO;
+        p = xdr_inline_decode(&stream, 8 + 4);
+        if (!p)
+                goto out_err_free;
+
+        p = xdr_decode_hyper(p, &stripe_unit);
+        mirror_array_cnt = be32_to_cpup(p++);
+        dprintk("%s: stripe_unit=%llu mirror_array_cnt=%u\n", __func__,
+                stripe_unit, mirror_array_cnt);
+
+        if (mirror_array_cnt > NFS4_FLEXFILE_LAYOUT_MAX_MIRROR_CNT ||
+            mirror_array_cnt == 0)
+                goto out_err_free;
+
+        rc = -ENOMEM;
+        fls = kzalloc(sizeof(*fls), gfp_flags);
+        if (!fls)
+                goto out_err_free;
+
+        fls->mirror_array_cnt = mirror_array_cnt;
+        fls->stripe_unit = stripe_unit;
+        fls->mirror_array = kcalloc(fls->mirror_array_cnt,
+                                    sizeof(fls->mirror_array[0]), gfp_flags);
+        if (fls->mirror_array == NULL)
+                goto out_err_free;
+
+        for (i = 0; i < fls->mirror_array_cnt; i++) {
+                struct nfs4_deviceid devid;
+                struct nfs4_deviceid_node *idnode;
+                u32 ds_count;
+                u32 fh_count;
+                int j;
+
+                rc = -EIO;
+                p = xdr_inline_decode(&stream, 4);
+                if (!p)
+                        goto out_err_free;
+                ds_count = be32_to_cpup(p);
+
+                /* FIXME: allow for striping? */
+                if (ds_count != 1)
+                        goto out_err_free;
+
+                fls->mirror_array[i] =
+                        kzalloc(sizeof(struct nfs4_ff_layout_mirror),
+                                gfp_flags);
+                if (fls->mirror_array[i] == NULL) {
+                        rc = -ENOMEM;
+                        goto out_err_free;
+                }
+
+                spin_lock_init(&fls->mirror_array[i]->lock);
+                fls->mirror_array[i]->ds_count = ds_count;
+
+                /* deviceid */
+                rc = decode_deviceid(&stream, &devid);
+                if (rc)
+                        goto out_err_free;
+
+                idnode = nfs4_find_get_deviceid(NFS_SERVER(lh->plh_inode),
+                                                &devid, lh->plh_lc_cred,
+                                                gfp_flags);
+                /*
+                 * upon success, mirror_ds is allocated by previous
+                 * getdeviceinfo, or newly by .alloc_deviceid_node
+                 * nfs4_find_get_deviceid failure is indeed getdeviceinfo falure
+                 */
+                if (idnode)
+                        fls->mirror_array[i]->mirror_ds =
+                                FF_LAYOUT_MIRROR_DS(idnode);
+                else
+                        goto out_err_free;
+
+                /* efficiency */
+                rc = -EIO;
+                p = xdr_inline_decode(&stream, 4);
+                if (!p)
+                        goto out_err_free;
+                fls->mirror_array[i]->efficiency = be32_to_cpup(p);
+
+                /* stateid */
+                rc = decode_stateid(&stream, &fls->mirror_array[i]->stateid);
+                if (rc)
+                        goto out_err_free;
+
+                /* fh */
+                p = xdr_inline_decode(&stream, 4);
+                if (!p)
+                        goto out_err_free;
+                fh_count = be32_to_cpup(p);
+
+                fls->mirror_array[i]->fh_versions =
+                        kzalloc(fh_count * sizeof(struct nfs_fh),
+                                gfp_flags);
+                if (fls->mirror_array[i]->fh_versions == NULL) {
+                        rc = -ENOMEM;
+                        goto out_err_free;
+                }
+
+                for (j = 0; j < fh_count; j++) {
+                        rc = decode_nfs_fh(&stream,
+                                           &fls->mirror_array[i]->fh_versions[j]);
+                        if (rc)
+                                goto out_err_free;
+                }
+
+                fls->mirror_array[i]->fh_versions_cnt = fh_count;
+
+                /* user */
+                rc = decode_name(&stream, &fls->mirror_array[i]->uid);
+                if (rc)
+                        goto out_err_free;
+
+                /* group */
+                rc = decode_name(&stream, &fls->mirror_array[i]->gid);
+                if (rc)
+                        goto out_err_free;
+
+                dprintk("%s: uid %d gid %d\n", __func__,
+                        fls->mirror_array[i]->uid,
+                        fls->mirror_array[i]->gid);
+        }
+
+        ff_layout_sort_mirrors(fls);
+        rc = ff_layout_check_layout(lgr);
+        if (rc)
+                goto out_err_free;
+
+        ret = &fls->generic_hdr;
+        dprintk("<-- %s (success)\n", __func__);
+out_free_page:
+        __free_page(scratch);
+        return ret;
+out_err_free:
+        _ff_layout_free_lseg(fls);
+        ret = ERR_PTR(rc);
+        dprintk("<-- %s (%d)\n", __func__, rc);
+        goto out_free_page;
+}
+
+static bool ff_layout_has_rw_segments(struct pnfs_layout_hdr *layout)
+{
+        struct pnfs_layout_segment *lseg;
+
+        list_for_each_entry(lseg, &layout->plh_segs, pls_list)
+                if (lseg->pls_range.iomode == IOMODE_RW)
+                        return true;
+
+        return false;
+}
+
+static void
+ff_layout_free_lseg(struct pnfs_layout_segment *lseg)
+{
+        struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg);
+        int i;
+
+        dprintk("--> %s\n", __func__);
+
+        for (i = 0; i < fls->mirror_array_cnt; i++) {
+                if (fls->mirror_array[i]) {
+                        nfs4_ff_layout_put_deviceid(fls->mirror_array[i]->mirror_ds);
+                        fls->mirror_array[i]->mirror_ds = NULL;
+                        if (fls->mirror_array[i]->cred) {
+                                put_rpccred(fls->mirror_array[i]->cred);
+                                fls->mirror_array[i]->cred = NULL;
+                        }
+                }
+        }
+
+        if (lseg->pls_range.iomode == IOMODE_RW) {
+                struct nfs4_flexfile_layout *ffl;
+                struct inode *inode;
+
+                ffl = FF_LAYOUT_FROM_HDR(lseg->pls_layout);
+                inode = ffl->generic_hdr.plh_inode;
+                spin_lock(&inode->i_lock);
+                if (!ff_layout_has_rw_segments(lseg->pls_layout)) {
+                        ffl->commit_info.nbuckets = 0;
+                        kfree(ffl->commit_info.buckets);
+                        ffl->commit_info.buckets = NULL;
+                }
+                spin_unlock(&inode->i_lock);
+        }
+        _ff_layout_free_lseg(fls);
+}
+
+/* Return 1 until we have multiple lsegs support */
+static int
+ff_layout_get_lseg_count(struct nfs4_ff_layout_segment *fls)
+{
+        return 1;
+}
+
+static int
+ff_layout_alloc_commit_info(struct pnfs_layout_segment *lseg,
+                            struct nfs_commit_info *cinfo,
+                            gfp_t gfp_flags)
+{
+        struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg);
+        struct pnfs_commit_bucket *buckets;
+        int size;
+
+        if (cinfo->ds->nbuckets != 0) {
+                /* This assumes there is only one RW lseg per file.
+                 * To support multiple lseg per file, we need to
+                 * change struct pnfs_commit_bucket to allow dynamic
+                 * increasing nbuckets.
+                 */
+                return 0;
+        }
+
+        size = ff_layout_get_lseg_count(fls) * FF_LAYOUT_MIRROR_COUNT(lseg);
+
+        buckets = kcalloc(size, sizeof(struct pnfs_commit_bucket),
+                          gfp_flags);
+        if (!buckets)
+                return -ENOMEM;
+        else {
+                int i;
+
+                spin_lock(cinfo->lock);
+                if (cinfo->ds->nbuckets != 0)
+                        kfree(buckets);
+                else {
+                        cinfo->ds->buckets = buckets;
+                        cinfo->ds->nbuckets = size;
+                        for (i = 0; i < size; i++) {
+                                INIT_LIST_HEAD(&buckets[i].written);
+                                INIT_LIST_HEAD(&buckets[i].committing);
+                                /* mark direct verifier as unset */
+                                buckets[i].direct_verf.committed =
+                                        NFS_INVALID_STABLE_HOW;
+                        }
+                }
+                spin_unlock(cinfo->lock);
+                return 0;
+        }
+}
+
+static struct nfs4_pnfs_ds *
+ff_layout_choose_best_ds_for_read(struct nfs_pageio_descriptor *pgio,
+                                  int *best_idx)
+{
+        struct nfs4_ff_layout_segment *fls;
+        struct nfs4_pnfs_ds *ds;
+        int idx;
+
+        fls = FF_LAYOUT_LSEG(pgio->pg_lseg);
+        /* mirrors are sorted by efficiency */
+        for (idx = 0; idx < fls->mirror_array_cnt; idx++) {
+                ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, idx, false);
+                if (ds) {
+                        *best_idx = idx;
+                        return ds;
+                }
+        }
+
+        return NULL;
+}
+
+static void
+ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio,
+                        struct nfs_page *req)
+{
+        struct nfs_pgio_mirror *pgm;
+        struct nfs4_ff_layout_mirror *mirror;
+        struct nfs4_pnfs_ds *ds;
+        int ds_idx;
+
+        /* Use full layout for now */
+        if (!pgio->pg_lseg)
+                pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
+                                                   req->wb_context,
+                                                   0,
+                                                   NFS4_MAX_UINT64,
+                                                   IOMODE_READ,
+                                                   GFP_KERNEL);
+        /* If no lseg, fall back to read through mds */
+        if (pgio->pg_lseg == NULL)
+                goto out_mds;
+
+        ds = ff_layout_choose_best_ds_for_read(pgio, &ds_idx);
+        if (!ds)
+                goto out_mds;
+        mirror = FF_LAYOUT_COMP(pgio->pg_lseg, ds_idx);
+
+        pgio->pg_mirror_idx = ds_idx;
+
+        /* read always uses only one mirror - idx 0 for pgio layer */
+        pgm = &pgio->pg_mirrors[0];
+        pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].rsize;
+
+        return;
+out_mds:
+        pnfs_put_lseg(pgio->pg_lseg);
+        pgio->pg_lseg = NULL;
+        nfs_pageio_reset_read_mds(pgio);
+}
+
+static void
+ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio,
+                        struct nfs_page *req)
+{
+        struct nfs4_ff_layout_mirror *mirror;
+        struct nfs_pgio_mirror *pgm;
+        struct nfs_commit_info cinfo;
+        struct nfs4_pnfs_ds *ds;
+        int i;
+        int status;
+
+        if (!pgio->pg_lseg)
+                pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
+                                                   req->wb_context,
+                                                   0,
+                                                   NFS4_MAX_UINT64,
+                                                   IOMODE_RW,
+                                                   GFP_NOFS);
+        /* If no lseg, fall back to write through mds */
+        if (pgio->pg_lseg == NULL)
+                goto out_mds;
+
+        nfs_init_cinfo(&cinfo, pgio->pg_inode, pgio->pg_dreq);
+        status = ff_layout_alloc_commit_info(pgio->pg_lseg, &cinfo, GFP_NOFS);
+        if (status < 0)
+                goto out_mds;
+
+        /* Use a direct mapping of ds_idx to pgio mirror_idx */
+        if (WARN_ON_ONCE(pgio->pg_mirror_count !=
+            FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg)))
+                goto out_mds;
+
+        for (i = 0; i < pgio->pg_mirror_count; i++) {
+                ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, i, true);
+                if (!ds)
+                        goto out_mds;
+                pgm = &pgio->pg_mirrors[i];
+                mirror = FF_LAYOUT_COMP(pgio->pg_lseg, i);
+                pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].wsize;
+        }
+
+        return;
+
+out_mds:
+        pnfs_put_lseg(pgio->pg_lseg);
+        pgio->pg_lseg = NULL;
+        nfs_pageio_reset_write_mds(pgio);
+}
+
+static unsigned int
+ff_layout_pg_get_mirror_count_write(struct nfs_pageio_descriptor *pgio,
+                                    struct nfs_page *req)
+{
+        if (!pgio->pg_lseg)
+                pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
+                                                   req->wb_context,
+                                                   0,
+                                                   NFS4_MAX_UINT64,
+                                                   IOMODE_RW,
+                                                   GFP_NOFS);
+        if (pgio->pg_lseg)
+                return FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg);
+
+        /* no lseg means that pnfs is not in use, so no mirroring here */
+        pnfs_put_lseg(pgio->pg_lseg);
+        pgio->pg_lseg = NULL;
+        nfs_pageio_reset_write_mds(pgio);
+        return 1;
+}
+
+static const struct nfs_pageio_ops ff_layout_pg_read_ops = {
+        .pg_init = ff_layout_pg_init_read,
+        .pg_test = pnfs_generic_pg_test,
+        .pg_doio = pnfs_generic_pg_readpages,
+        .pg_cleanup = pnfs_generic_pg_cleanup,
+};
+
+static const struct nfs_pageio_ops ff_layout_pg_write_ops = {
+        .pg_init = ff_layout_pg_init_write,
+        .pg_test = pnfs_generic_pg_test,
+        .pg_doio = pnfs_generic_pg_writepages,
+        .pg_get_mirror_count = ff_layout_pg_get_mirror_count_write,
+        .pg_cleanup = pnfs_generic_pg_cleanup,
+};
+
+static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs)
+{
+        struct rpc_task *task = &hdr->task;
+
+        pnfs_layoutcommit_inode(hdr->inode, false);
+
+        if (retry_pnfs) {
+                dprintk("%s Reset task %5u for i/o through pNFS "
+                        "(req %s/%llu, %u bytes @ offset %llu)\n", __func__,
+                        hdr->task.tk_pid,
+                        hdr->inode->i_sb->s_id,
+                        (unsigned long long)NFS_FILEID(hdr->inode),
+                        hdr->args.count,
+                        (unsigned long long)hdr->args.offset);
+
+                if (!hdr->dreq) {
+                        struct nfs_open_context *ctx;
+
+                        ctx = nfs_list_entry(hdr->pages.next)->wb_context;
+                        set_bit(NFS_CONTEXT_RESEND_WRITES, &ctx->flags);
+                        hdr->completion_ops->error_cleanup(&hdr->pages);
+                } else {
+                        nfs_direct_set_resched_writes(hdr->dreq);
+                        /* fake unstable write to let common nfs resend pages */
+                        hdr->verf.committed = NFS_UNSTABLE;
+                        hdr->good_bytes = 0;
+                }
+                return;
+        }
+
+        if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
+                dprintk("%s Reset task %5u for i/o through MDS "
+                        "(req %s/%llu, %u bytes @ offset %llu)\n", __func__,
+                        hdr->task.tk_pid,
+                        hdr->inode->i_sb->s_id,
+                        (unsigned long long)NFS_FILEID(hdr->inode),
+                        hdr->args.count,
+                        (unsigned long long)hdr->args.offset);
+
+                task->tk_status = pnfs_write_done_resend_to_mds(hdr);
+        }
+}
+
+static void ff_layout_reset_read(struct nfs_pgio_header *hdr)
+{
+        struct rpc_task *task = &hdr->task;
+
+        pnfs_layoutcommit_inode(hdr->inode, false);
+
+        if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
+                dprintk("%s Reset task %5u for i/o through MDS "
+                        "(req %s/%llu, %u bytes @ offset %llu)\n", __func__,
+                        hdr->task.tk_pid,
+                        hdr->inode->i_sb->s_id,
+                        (unsigned long long)NFS_FILEID(hdr->inode),
+                        hdr->args.count,
+                        (unsigned long long)hdr->args.offset);
+
+                task->tk_status = pnfs_read_done_resend_to_mds(hdr);
+        }
+}
+
+static int ff_layout_async_handle_error_v4(struct rpc_task *task,
+                                           struct nfs4_state *state,
+                                           struct nfs_client *clp,
+                                           struct pnfs_layout_segment *lseg,
+                                           int idx)
+{
+        struct pnfs_layout_hdr *lo = lseg->pls_layout;
+        struct inode *inode = lo->plh_inode;
+        struct nfs_server *mds_server = NFS_SERVER(inode);
+
+        struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
+        struct nfs_client *mds_client = mds_server->nfs_client;
+        struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table;
+
+        if (task->tk_status >= 0)
+                return 0;
+
+        switch (task->tk_status) {
+        /* MDS state errors */
+        case -NFS4ERR_DELEG_REVOKED:
+        case -NFS4ERR_ADMIN_REVOKED:
+        case -NFS4ERR_BAD_STATEID:
+                if (state == NULL)
+                        break;
+                nfs_remove_bad_delegation(state->inode);
+        case -NFS4ERR_OPENMODE:
+                if (state == NULL)
+                        break;
+                if (nfs4_schedule_stateid_recovery(mds_server, state) < 0)
+                        goto out_bad_stateid;
+                goto wait_on_recovery;
+        case -NFS4ERR_EXPIRED:
+                if (state != NULL) {
+                        if (nfs4_schedule_stateid_recovery(mds_server, state) < 0)
+                                goto out_bad_stateid;
+                }
+                nfs4_schedule_lease_recovery(mds_client);
+                goto wait_on_recovery;
+        /* DS session errors */
+        case -NFS4ERR_BADSESSION:
+        case -NFS4ERR_BADSLOT:
+        case -NFS4ERR_BAD_HIGH_SLOT:
+        case -NFS4ERR_DEADSESSION:
+        case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
+        case -NFS4ERR_SEQ_FALSE_RETRY:
+        case -NFS4ERR_SEQ_MISORDERED:
+                dprintk("%s ERROR %d, Reset session. Exchangeid "
+                        "flags 0x%x\n", __func__, task->tk_status,
+                        clp->cl_exchange_flags);
+                nfs4_schedule_session_recovery(clp->cl_session, task->tk_status);
+                break;
+        case -NFS4ERR_DELAY:
+        case -NFS4ERR_GRACE:
+                rpc_delay(task, FF_LAYOUT_POLL_RETRY_MAX);
+                break;
+        case -NFS4ERR_RETRY_UNCACHED_REP:
+                break;
+        /* Invalidate Layout errors */
+        case -NFS4ERR_PNFS_NO_LAYOUT:
+        case -ESTALE:           /* mapped NFS4ERR_STALE */
+        case -EBADHANDLE:       /* mapped NFS4ERR_BADHANDLE */
+        case -EISDIR:           /* mapped NFS4ERR_ISDIR */
+        case -NFS4ERR_FHEXPIRED:
+        case -NFS4ERR_WRONG_TYPE:
+                dprintk("%s Invalid layout error %d\n", __func__,
+                        task->tk_status);
+                /*
+                 * Destroy layout so new i/o will get a new layout.
+                 * Layout will not be destroyed until all current lseg
+                 * references are put. Mark layout as invalid to resend failed
+                 * i/o and all i/o waiting on the slot table to the MDS until
+                 * layout is destroyed and a new valid layout is obtained.
+                 */
+                pnfs_destroy_layout(NFS_I(inode));
+                rpc_wake_up(&tbl->slot_tbl_waitq);
+                goto reset;
+        /* RPC connection errors */
+        case -ECONNREFUSED:
+        case -EHOSTDOWN:
+        case -EHOSTUNREACH:
+        case -ENETUNREACH:
+        case -EIO:
+        case -ETIMEDOUT:
+        case -EPIPE:
+                dprintk("%s DS connection error %d\n", __func__,
+                        task->tk_status);
+                nfs4_mark_deviceid_unavailable(devid);
+                rpc_wake_up(&tbl->slot_tbl_waitq);
+                /* fall through */
+        default:
+                if (ff_layout_has_available_ds(lseg))
+                        return -NFS4ERR_RESET_TO_PNFS;
+reset:
+                dprintk("%s Retry through MDS. Error %d\n", __func__,
+                        task->tk_status);
+                return -NFS4ERR_RESET_TO_MDS;
+        }
+out:
+        task->tk_status = 0;
+        return -EAGAIN;
+out_bad_stateid:
+        task->tk_status = -EIO;
+        return 0;
+wait_on_recovery:
+        rpc_sleep_on(&mds_client->cl_rpcwaitq, task, NULL);
+        if (test_bit(NFS4CLNT_MANAGER_RUNNING, &mds_client->cl_state) == 0)
+                rpc_wake_up_queued_task(&mds_client->cl_rpcwaitq, task);
+        goto out;
+}
+
+/* Retry all errors through either pNFS or MDS except for -EJUKEBOX */
+static int ff_layout_async_handle_error_v3(struct rpc_task *task,
+                                           struct pnfs_layout_segment *lseg,
+                                           int idx)
+{
+        struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
+
+        if (task->tk_status >= 0)
+                return 0;
+
+        if (task->tk_status != -EJUKEBOX) {
+                dprintk("%s DS connection error %d\n", __func__,
+                        task->tk_status);
+                nfs4_mark_deviceid_unavailable(devid);
+                if (ff_layout_has_available_ds(lseg))
+                        return -NFS4ERR_RESET_TO_PNFS;
+                else
+                        return -NFS4ERR_RESET_TO_MDS;
+        }
+
+        if (task->tk_status == -EJUKEBOX)
+                nfs_inc_stats(lseg->pls_layout->plh_inode, NFSIOS_DELAY);
+        task->tk_status = 0;
+        rpc_restart_call(task);
+        rpc_delay(task, NFS_JUKEBOX_RETRY_TIME);
+        return -EAGAIN;
+}
+
+static int ff_layout_async_handle_error(struct rpc_task *task,
+                                        struct nfs4_state *state,
+                                        struct nfs_client *clp,
+                                        struct pnfs_layout_segment *lseg,
+                                        int idx)
+{
+        int vers = clp->cl_nfs_mod->rpc_vers->number;
+
+        switch (vers) {
+        case 3:
+                return ff_layout_async_handle_error_v3(task, lseg, idx);
+        case 4:
+                return ff_layout_async_handle_error_v4(task, state, clp,
+                                                       lseg, idx);
+        default:
+                /* should never happen */
+                WARN_ON_ONCE(1);
+                return 0;
+        }
+}
+
+static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg,
+                                        int idx, u64 offset, u64 length,
+                                        u32 status, int opnum)
+{
+        struct nfs4_ff_layout_mirror *mirror;
+        int err;
+
+        mirror = FF_LAYOUT_COMP(lseg, idx);
+        err = ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout),
+                                       mirror, offset, length, status, opnum,
+                                       GFP_NOIO);
+        dprintk("%s: err %d op %d status %u\n", __func__, err, opnum, status);
+}
+
+/* NFS_PROTO call done callback routines */
+
+static int ff_layout_read_done_cb(struct rpc_task *task,
+                                struct nfs_pgio_header *hdr)
+{
+        struct inode *inode;
+        int err;
+
+        trace_nfs4_pnfs_read(hdr, task->tk_status);
+        if (task->tk_status == -ETIMEDOUT && !hdr->res.op_status)
+                hdr->res.op_status = NFS4ERR_NXIO;
+        if (task->tk_status < 0 && hdr->res.op_status)
+                ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx,
+                                            hdr->args.offset, hdr->args.count,
+                                            hdr->res.op_status, OP_READ);
+        err = ff_layout_async_handle_error(task, hdr->args.context->state,
+                                           hdr->ds_clp, hdr->lseg,
+                                           hdr->pgio_mirror_idx);
+
+        switch (err) {
+        case -NFS4ERR_RESET_TO_PNFS:
+                set_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
+                        &hdr->lseg->pls_layout->plh_flags);
+                pnfs_read_resend_pnfs(hdr);
+                return task->tk_status;
+        case -NFS4ERR_RESET_TO_MDS:
+                inode = hdr->lseg->pls_layout->plh_inode;
+                pnfs_error_mark_layout_for_return(inode, hdr->lseg);
+                ff_layout_reset_read(hdr);
+                return task->tk_status;
+        case -EAGAIN:
+                rpc_restart_call_prepare(task);
+                return -EAGAIN;
+        }
+
+        return 0;
+}
+
+/*
+ * We reference the rpc_cred of the first WRITE that triggers the need for
+ * a LAYOUTCOMMIT, and use it to send the layoutcommit compound.
+ * rfc5661 is not clear about which credential should be used.
+ *
+ * Flexlayout client should treat DS replied FILE_SYNC as DATA_SYNC, so
+ * to follow http://www.rfc-editor.org/errata_search.php?rfc=5661&eid=2751
+ * we always send layoutcommit after DS writes.
+ */
+static void
+ff_layout_set_layoutcommit(struct nfs_pgio_header *hdr)
+{
+        pnfs_set_layoutcommit(hdr);
+        dprintk("%s inode %lu pls_end_pos %lu\n", __func__, hdr->inode->i_ino,
+                (unsigned long) NFS_I(hdr->inode)->layout->plh_lwb);
+}
+
+static bool
+ff_layout_reset_to_mds(struct pnfs_layout_segment *lseg, int idx)
+{
+        /* No mirroring for now */
+        struct nfs4_deviceid_node *node = FF_LAYOUT_DEVID_NODE(lseg, idx);
+
+        return ff_layout_test_devid_unavailable(node);
+}
+
+static int ff_layout_read_prepare_common(struct rpc_task *task,
+                                         struct nfs_pgio_header *hdr)
+{
+        if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) {
+                rpc_exit(task, -EIO);
+                return -EIO;
+        }
+        if (ff_layout_reset_to_mds(hdr->lseg, hdr->pgio_mirror_idx)) {
+                dprintk("%s task %u reset io to MDS\n", __func__, task->tk_pid);
+                if (ff_layout_has_available_ds(hdr->lseg))
+                        pnfs_read_resend_pnfs(hdr);
+                else
+                        ff_layout_reset_read(hdr);
+                rpc_exit(task, 0);
+                return -EAGAIN;
+        }
+        hdr->pgio_done_cb = ff_layout_read_done_cb;
+
+        return 0;
+}
+
+/*
+ * Call ops for the async read/write cases
+ * In the case of dense layouts, the offset needs to be reset to its
+ * original value.
+ */
+static void ff_layout_read_prepare_v3(struct rpc_task *task, void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        if (ff_layout_read_prepare_common(task, hdr))
+                return;
+
+        rpc_call_start(task);
+}
+
+static int ff_layout_setup_sequence(struct nfs_client *ds_clp,
+                                    struct nfs4_sequence_args *args,
+                                    struct nfs4_sequence_res *res,
+                                    struct rpc_task *task)
+{
+        if (ds_clp->cl_session)
+                return nfs41_setup_sequence(ds_clp->cl_session,
+                                           args,
+                                           res,
+                                           task);
+        return nfs40_setup_sequence(ds_clp->cl_slot_tbl,
+                                   args,
+                                   res,
+                                   task);
+}
+
+static void ff_layout_read_prepare_v4(struct rpc_task *task, void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        if (ff_layout_read_prepare_common(task, hdr))
+                return;
+
+        if (ff_layout_setup_sequence(hdr->ds_clp,
+                                     &hdr->args.seq_args,
+                                     &hdr->res.seq_res,
+                                     task))
+                return;
+
+        if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context,
+                        hdr->args.lock_context, FMODE_READ) == -EIO)
+                rpc_exit(task, -EIO); /* lost lock, terminate I/O */
+}
+
+static void ff_layout_read_call_done(struct rpc_task *task, void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        dprintk("--> %s task->tk_status %d\n", __func__, task->tk_status);
+
+        if (test_bit(NFS_IOHDR_REDO, &hdr->flags) &&
+            task->tk_status == 0) {
+                nfs4_sequence_done(task, &hdr->res.seq_res);
+                return;
+        }
+
+        /* Note this may cause RPC to be resent */
+        hdr->mds_ops->rpc_call_done(task, hdr);
+}
+
+static void ff_layout_read_count_stats(struct rpc_task *task, void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        rpc_count_iostats_metrics(task,
+            &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_READ]);
+}
+
+static int ff_layout_write_done_cb(struct rpc_task *task,
+                                struct nfs_pgio_header *hdr)
+{
+        struct inode *inode;
+        int err;
+
+        trace_nfs4_pnfs_write(hdr, task->tk_status);
+        if (task->tk_status == -ETIMEDOUT && !hdr->res.op_status)
+                hdr->res.op_status = NFS4ERR_NXIO;
+        if (task->tk_status < 0 && hdr->res.op_status)
+                ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx,
+                                            hdr->args.offset, hdr->args.count,
+                                            hdr->res.op_status, OP_WRITE);
+        err = ff_layout_async_handle_error(task, hdr->args.context->state,
+                                           hdr->ds_clp, hdr->lseg,
+                                           hdr->pgio_mirror_idx);
+
+        switch (err) {
+        case -NFS4ERR_RESET_TO_PNFS:
+        case -NFS4ERR_RESET_TO_MDS:
+                inode = hdr->lseg->pls_layout->plh_inode;
+                pnfs_error_mark_layout_for_return(inode, hdr->lseg);
+                if (err == -NFS4ERR_RESET_TO_PNFS) {
+                        pnfs_set_retry_layoutget(hdr->lseg->pls_layout);
+                        ff_layout_reset_write(hdr, true);
+                } else {
+                        pnfs_clear_retry_layoutget(hdr->lseg->pls_layout);
+                        ff_layout_reset_write(hdr, false);
+                }
+                return task->tk_status;
+        case -EAGAIN:
+                rpc_restart_call_prepare(task);
+                return -EAGAIN;
+        }
+
+        if (hdr->res.verf->committed == NFS_FILE_SYNC ||
+            hdr->res.verf->committed == NFS_DATA_SYNC)
+                ff_layout_set_layoutcommit(hdr);
+
+        return 0;
+}
+
+static int ff_layout_commit_done_cb(struct rpc_task *task,
+                                     struct nfs_commit_data *data)
+{
+        struct inode *inode;
+        int err;
+
+        trace_nfs4_pnfs_commit_ds(data, task->tk_status);
+        if (task->tk_status == -ETIMEDOUT && !data->res.op_status)
+                data->res.op_status = NFS4ERR_NXIO;
+        if (task->tk_status < 0 && data->res.op_status)
+                ff_layout_io_track_ds_error(data->lseg, data->ds_commit_index,
+                                            data->args.offset, data->args.count,
+                                            data->res.op_status, OP_COMMIT);
+        err = ff_layout_async_handle_error(task, NULL, data->ds_clp,
+                                           data->lseg, data->ds_commit_index);
+
+        switch (err) {
+        case -NFS4ERR_RESET_TO_PNFS:
+        case -NFS4ERR_RESET_TO_MDS:
+                inode = data->lseg->pls_layout->plh_inode;
+                pnfs_error_mark_layout_for_return(inode, data->lseg);
+                if (err == -NFS4ERR_RESET_TO_PNFS)
+                        pnfs_set_retry_layoutget(data->lseg->pls_layout);
+                else
+                        pnfs_clear_retry_layoutget(data->lseg->pls_layout);
+                pnfs_generic_prepare_to_resend_writes(data);
+                return -EAGAIN;
+        case -EAGAIN:
+                rpc_restart_call_prepare(task);
+                return -EAGAIN;
+        }
+
+        if (data->verf.committed == NFS_UNSTABLE)
+                pnfs_commit_set_layoutcommit(data);
+
+        return 0;
+}
+
+static int ff_layout_write_prepare_common(struct rpc_task *task,
+                                          struct nfs_pgio_header *hdr)
+{
+        if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) {
+                rpc_exit(task, -EIO);
+                return -EIO;
+        }
+
+        if (ff_layout_reset_to_mds(hdr->lseg, hdr->pgio_mirror_idx)) {
+                bool retry_pnfs;
+
+                retry_pnfs = ff_layout_has_available_ds(hdr->lseg);
+                dprintk("%s task %u reset io to %s\n", __func__,
+                        task->tk_pid, retry_pnfs ? "pNFS" : "MDS");
+                ff_layout_reset_write(hdr, retry_pnfs);
+                rpc_exit(task, 0);
+                return -EAGAIN;
+        }
+
+        return 0;
+}
+
+static void ff_layout_write_prepare_v3(struct rpc_task *task, void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        if (ff_layout_write_prepare_common(task, hdr))
+                return;
+
+        rpc_call_start(task);
+}
+
+static void ff_layout_write_prepare_v4(struct rpc_task *task, void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        if (ff_layout_write_prepare_common(task, hdr))
+                return;
+
+        if (ff_layout_setup_sequence(hdr->ds_clp,
+                                     &hdr->args.seq_args,
+                                     &hdr->res.seq_res,
+                                     task))
+                return;
+
+        if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context,
+                        hdr->args.lock_context, FMODE_WRITE) == -EIO)
+                rpc_exit(task, -EIO); /* lost lock, terminate I/O */
+}
+
+static void ff_layout_write_call_done(struct rpc_task *task, void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        if (test_bit(NFS_IOHDR_REDO, &hdr->flags) &&
+            task->tk_status == 0) {
+                nfs4_sequence_done(task, &hdr->res.seq_res);
+                return;
+        }
+
+        /* Note this may cause RPC to be resent */
+        hdr->mds_ops->rpc_call_done(task, hdr);
+}
+
+static void ff_layout_write_count_stats(struct rpc_task *task, void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        rpc_count_iostats_metrics(task,
+            &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_WRITE]);
+}
+
+static void ff_layout_commit_prepare_v3(struct rpc_task *task, void *data)
+{
+        rpc_call_start(task);
+}
+
+static void ff_layout_commit_prepare_v4(struct rpc_task *task, void *data)
+{
+        struct nfs_commit_data *wdata = data;
+
+        ff_layout_setup_sequence(wdata->ds_clp,
+                                 &wdata->args.seq_args,
+                                 &wdata->res.seq_res,
+                                 task);
+}
+
+static void ff_layout_commit_count_stats(struct rpc_task *task, void *data)
+{
+        struct nfs_commit_data *cdata = data;
+
+        rpc_count_iostats_metrics(task,
+            &NFS_CLIENT(cdata->inode)->cl_metrics[NFSPROC4_CLNT_COMMIT]);
+}
+
+static const struct rpc_call_ops ff_layout_read_call_ops_v3 = {
+        .rpc_call_prepare = ff_layout_read_prepare_v3,
+        .rpc_call_done = ff_layout_read_call_done,
+        .rpc_count_stats = ff_layout_read_count_stats,
+        .rpc_release = pnfs_generic_rw_release,
+};
+
+static const struct rpc_call_ops ff_layout_read_call_ops_v4 = {
+        .rpc_call_prepare = ff_layout_read_prepare_v4,
+        .rpc_call_done = ff_layout_read_call_done,
+        .rpc_count_stats = ff_layout_read_count_stats,
+        .rpc_release = pnfs_generic_rw_release,
+};
+
+static const struct rpc_call_ops ff_layout_write_call_ops_v3 = {
+        .rpc_call_prepare = ff_layout_write_prepare_v3,
+        .rpc_call_done = ff_layout_write_call_done,
+        .rpc_count_stats = ff_layout_write_count_stats,
+        .rpc_release = pnfs_generic_rw_release,
+};
+
+static const struct rpc_call_ops ff_layout_write_call_ops_v4 = {
+        .rpc_call_prepare = ff_layout_write_prepare_v4,
+        .rpc_call_done = ff_layout_write_call_done,
+        .rpc_count_stats = ff_layout_write_count_stats,
+        .rpc_release = pnfs_generic_rw_release,
+};
+
+static const struct rpc_call_ops ff_layout_commit_call_ops_v3 = {
+        .rpc_call_prepare = ff_layout_commit_prepare_v3,
+        .rpc_call_done = pnfs_generic_write_commit_done,
+        .rpc_count_stats = ff_layout_commit_count_stats,
+        .rpc_release = pnfs_generic_commit_release,
+};
+
+static const struct rpc_call_ops ff_layout_commit_call_ops_v4 = {
+        .rpc_call_prepare = ff_layout_commit_prepare_v4,
+        .rpc_call_done = pnfs_generic_write_commit_done,
+        .rpc_count_stats = ff_layout_commit_count_stats,
+        .rpc_release = pnfs_generic_commit_release,
+};
+
+static enum pnfs_try_status
+ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
+{
+        struct pnfs_layout_segment *lseg = hdr->lseg;
+        struct nfs4_pnfs_ds *ds;
+        struct rpc_clnt *ds_clnt;
+        struct rpc_cred *ds_cred;
+        loff_t offset = hdr->args.offset;
+        u32 idx = hdr->pgio_mirror_idx;
+        int vers;
+        struct nfs_fh *fh;
+
+        dprintk("--> %s ino %lu pgbase %u req %Zu@%llu\n",
+                __func__, hdr->inode->i_ino,
+                hdr->args.pgbase, (size_t)hdr->args.count, offset);
+
+        ds = nfs4_ff_layout_prepare_ds(lseg, idx, false);
+        if (!ds)
+                goto out_failed;
+
+        ds_clnt = nfs4_ff_find_or_create_ds_client(lseg, idx, ds->ds_clp,
+                                                   hdr->inode);
+        if (IS_ERR(ds_clnt))
+                goto out_failed;
+
+        ds_cred = ff_layout_get_ds_cred(lseg, idx, hdr->cred);
+        if (IS_ERR(ds_cred))
+                goto out_failed;
+
+        vers = nfs4_ff_layout_ds_version(lseg, idx);
+
+        dprintk("%s USE DS: %s cl_count %d vers %d\n", __func__,
+                ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count), vers);
+
+        atomic_inc(&ds->ds_clp->cl_count);
+        hdr->ds_clp = ds->ds_clp;
+        fh = nfs4_ff_layout_select_ds_fh(lseg, idx);
+        if (fh)
+                hdr->args.fh = fh;
+
+        /*
+         * Note that if we ever decide to split across DSes,
+         * then we may need to handle dense-like offsets.
+         */
+        hdr->args.offset = offset;
+        hdr->mds_offset = offset;
+
+        /* Perform an asynchronous read to ds */
+        nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,
+                          vers == 3 ? &ff_layout_read_call_ops_v3 :
+                                      &ff_layout_read_call_ops_v4,
+                          0, RPC_TASK_SOFTCONN);
+
+        return PNFS_ATTEMPTED;
+
+out_failed:
+        if (ff_layout_has_available_ds(lseg))
+                return PNFS_TRY_AGAIN;
+        return PNFS_NOT_ATTEMPTED;
+}
+
+/* Perform async writes. */
+static enum pnfs_try_status
+ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
+{
+        struct pnfs_layout_segment *lseg = hdr->lseg;
+        struct nfs4_pnfs_ds *ds;
+        struct rpc_clnt *ds_clnt;
+        struct rpc_cred *ds_cred;
+        loff_t offset = hdr->args.offset;
+        int vers;
+        struct nfs_fh *fh;
+        int idx = hdr->pgio_mirror_idx;
+
+        ds = nfs4_ff_layout_prepare_ds(lseg, idx, true);
+        if (!ds)
+                return PNFS_NOT_ATTEMPTED;
+
+        ds_clnt = nfs4_ff_find_or_create_ds_client(lseg, idx, ds->ds_clp,
+                                                   hdr->inode);
+        if (IS_ERR(ds_clnt))
+                return PNFS_NOT_ATTEMPTED;
+
+        ds_cred = ff_layout_get_ds_cred(lseg, idx, hdr->cred);
+        if (IS_ERR(ds_cred))
+                return PNFS_NOT_ATTEMPTED;
+
+        vers = nfs4_ff_layout_ds_version(lseg, idx);
+
+        dprintk("%s ino %lu sync %d req %Zu@%llu DS: %s cl_count %d vers %d\n",
+                __func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count,
+                offset, ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count),
+                vers);
+
+        hdr->pgio_done_cb = ff_layout_write_done_cb;
+        atomic_inc(&ds->ds_clp->cl_count);
+        hdr->ds_clp = ds->ds_clp;
+        hdr->ds_commit_idx = idx;
+        fh = nfs4_ff_layout_select_ds_fh(lseg, idx);
+        if (fh)
+                hdr->args.fh = fh;
+
+        /*
+         * Note that if we ever decide to split across DSes,
+         * then we may need to handle dense-like offsets.
+         */
+        hdr->args.offset = offset;
+
+        /* Perform an asynchronous write */
+        nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,
+                          vers == 3 ? &ff_layout_write_call_ops_v3 :
+                                      &ff_layout_write_call_ops_v4,
+                          sync, RPC_TASK_SOFTCONN);
+        return PNFS_ATTEMPTED;
+}
+
+static void
+ff_layout_mark_request_commit(struct nfs_page *req,
+                              struct pnfs_layout_segment *lseg,
+                              struct nfs_commit_info *cinfo,
+                              u32 ds_commit_idx)
+{
+        struct list_head *list;
+        struct pnfs_commit_bucket *buckets;
+
+        spin_lock(cinfo->lock);
+        buckets = cinfo->ds->buckets;
+        list = &buckets[ds_commit_idx].written;
+        if (list_empty(list)) {
+                /* Non-empty buckets hold a reference on the lseg.  That ref
+                 * is normally transferred to the COMMIT call and released
+                 * there.  It could also be released if the last req is pulled
+                 * off due to a rewrite, in which case it will be done in
+                 * pnfs_common_clear_request_commit
+                 */
+                WARN_ON_ONCE(buckets[ds_commit_idx].wlseg != NULL);
+                buckets[ds_commit_idx].wlseg = pnfs_get_lseg(lseg);
+        }
+        set_bit(PG_COMMIT_TO_DS, &req->wb_flags);
+        cinfo->ds->nwritten++;
+
+        /* nfs_request_add_commit_list(). We need to add req to list without
+         * dropping cinfo lock.
+         */
+        set_bit(PG_CLEAN, &(req)->wb_flags);
+        nfs_list_add_request(req, list);
+        cinfo->mds->ncommit++;
+        spin_unlock(cinfo->lock);
+        if (!cinfo->dreq) {
+                inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+                inc_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
+                             BDI_RECLAIMABLE);
+                __mark_inode_dirty(req->wb_context->dentry->d_inode,
+                                   I_DIRTY_DATASYNC);
+        }
+}
+
+static u32 calc_ds_index_from_commit(struct pnfs_layout_segment *lseg, u32 i)
+{
+        return i;
+}
+
+static struct nfs_fh *
+select_ds_fh_from_commit(struct pnfs_layout_segment *lseg, u32 i)
+{
+        struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg);
+
+        /* FIXME: Assume that there is only one NFS version available
+         * for the DS.
+         */
+        return &flseg->mirror_array[i]->fh_versions[0];
+}
+
+static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
+{
+        struct pnfs_layout_segment *lseg = data->lseg;
+        struct nfs4_pnfs_ds *ds;
+        struct rpc_clnt *ds_clnt;
+        struct rpc_cred *ds_cred;
+        u32 idx;
+        int vers;
+        struct nfs_fh *fh;
+
+        idx = calc_ds_index_from_commit(lseg, data->ds_commit_index);
+        ds = nfs4_ff_layout_prepare_ds(lseg, idx, true);
+        if (!ds)
+                goto out_err;
+
+        ds_clnt = nfs4_ff_find_or_create_ds_client(lseg, idx, ds->ds_clp,
+                                                   data->inode);
+        if (IS_ERR(ds_clnt))
+                goto out_err;
+
+        ds_cred = ff_layout_get_ds_cred(lseg, idx, data->cred);
+        if (IS_ERR(ds_cred))
+                goto out_err;
+
+        vers = nfs4_ff_layout_ds_version(lseg, idx);
+
+        dprintk("%s ino %lu, how %d cl_count %d vers %d\n", __func__,
+                data->inode->i_ino, how, atomic_read(&ds->ds_clp->cl_count),
+                vers);
+        data->commit_done_cb = ff_layout_commit_done_cb;
+        data->cred = ds_cred;
+        atomic_inc(&ds->ds_clp->cl_count);
+        data->ds_clp = ds->ds_clp;
+        fh = select_ds_fh_from_commit(lseg, data->ds_commit_index);
+        if (fh)
+                data->args.fh = fh;
+        return nfs_initiate_commit(ds_clnt, data, ds->ds_clp->rpc_ops,
+                                   vers == 3 ? &ff_layout_commit_call_ops_v3 :
+                                               &ff_layout_commit_call_ops_v4,
+                                   how, RPC_TASK_SOFTCONN);
+out_err:
+        pnfs_generic_prepare_to_resend_writes(data);
+        pnfs_generic_commit_release(data);
+        return -EAGAIN;
+}
+
+static int
+ff_layout_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
+                           int how, struct nfs_commit_info *cinfo)
+{
+        return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo,
+                                            ff_layout_initiate_commit);
+}
+
+static struct pnfs_ds_commit_info *
+ff_layout_get_ds_info(struct inode *inode)
+{
+        struct pnfs_layout_hdr *layout = NFS_I(inode)->layout;
+
+        if (layout == NULL)
+                return NULL;
+
+        return &FF_LAYOUT_FROM_HDR(layout)->commit_info;
+}
+
+static void
+ff_layout_free_deveiceid_node(struct nfs4_deviceid_node *d)
+{
+        nfs4_ff_layout_free_deviceid(container_of(d, struct nfs4_ff_layout_ds,
+                                                  id_node));
+}
+
+static int ff_layout_encode_ioerr(struct nfs4_flexfile_layout *flo,
+                                  struct xdr_stream *xdr,
+                                  const struct nfs4_layoutreturn_args *args)
+{
+        struct pnfs_layout_hdr *hdr = &flo->generic_hdr;
+        __be32 *start;
+        int count = 0, ret = 0;
+
+        start = xdr_reserve_space(xdr, 4);
+        if (unlikely(!start))
+                return -E2BIG;
+
+        /* This assume we always return _ALL_ layouts */
+        spin_lock(&hdr->plh_inode->i_lock);
+        ret = ff_layout_encode_ds_ioerr(flo, xdr, &count, &args->range);
+        spin_unlock(&hdr->plh_inode->i_lock);
+
+        *start = cpu_to_be32(count);
+
+        return ret;
+}
+
+/* report nothing for now */
+static void ff_layout_encode_iostats(struct nfs4_flexfile_layout *flo,
+                                     struct xdr_stream *xdr,
+                                     const struct nfs4_layoutreturn_args *args)
+{
+        __be32 *p;
+
+        p = xdr_reserve_space(xdr, 4);
+        if (likely(p))
+                *p = cpu_to_be32(0);
+}
+
+static struct nfs4_deviceid_node *
+ff_layout_alloc_deviceid_node(struct nfs_server *server,
+                              struct pnfs_device *pdev, gfp_t gfp_flags)
+{
+        struct nfs4_ff_layout_ds *dsaddr;
+
+        dsaddr = nfs4_ff_alloc_deviceid_node(server, pdev, gfp_flags);
+        if (!dsaddr)
+                return NULL;
+        return &dsaddr->id_node;
+}
+
+static void
+ff_layout_encode_layoutreturn(struct pnfs_layout_hdr *lo,
+                              struct xdr_stream *xdr,
+                              const struct nfs4_layoutreturn_args *args)
+{
+        struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
+        __be32 *start;
+
+        dprintk("%s: Begin\n", __func__);
+        start = xdr_reserve_space(xdr, 4);
+        BUG_ON(!start);
+
+        if (ff_layout_encode_ioerr(flo, xdr, args))
+                goto out;
+
+        ff_layout_encode_iostats(flo, xdr, args);
+out:
+        *start = cpu_to_be32((xdr->p - start - 1) * 4);
+        dprintk("%s: Return\n", __func__);
+}
+
+static struct pnfs_layoutdriver_type flexfilelayout_type = {
+        .id                     = LAYOUT_FLEX_FILES,
+        .name                   = "LAYOUT_FLEX_FILES",
+        .owner                  = THIS_MODULE,
+        .alloc_layout_hdr       = ff_layout_alloc_layout_hdr,
+        .free_layout_hdr        = ff_layout_free_layout_hdr,
+        .alloc_lseg             = ff_layout_alloc_lseg,
+        .free_lseg              = ff_layout_free_lseg,
+        .pg_read_ops            = &ff_layout_pg_read_ops,
+        .pg_write_ops           = &ff_layout_pg_write_ops,
+        .get_ds_info            = ff_layout_get_ds_info,
+        .free_deviceid_node     = ff_layout_free_deveiceid_node,
+        .mark_request_commit    = ff_layout_mark_request_commit,
+        .clear_request_commit   = pnfs_generic_clear_request_commit,
+        .scan_commit_lists      = pnfs_generic_scan_commit_lists,
+        .recover_commit_reqs    = pnfs_generic_recover_commit_reqs,
+        .commit_pagelist        = ff_layout_commit_pagelist,
+        .read_pagelist          = ff_layout_read_pagelist,
+        .write_pagelist         = ff_layout_write_pagelist,
+        .alloc_deviceid_node    = ff_layout_alloc_deviceid_node,
+        .encode_layoutreturn    = ff_layout_encode_layoutreturn,
+};
+
+static int __init nfs4flexfilelayout_init(void)
+{
+        printk(KERN_INFO "%s: NFSv4 Flexfile Layout Driver Registering...\n",
+               __func__);
+        return pnfs_register_layoutdriver(&flexfilelayout_type);
+}
+
+static void __exit nfs4flexfilelayout_exit(void)
+{
+        printk(KERN_INFO "%s: NFSv4 Flexfile Layout Driver Unregistering...\n",
+               __func__);
+        pnfs_unregister_layoutdriver(&flexfilelayout_type);
+}
+
+MODULE_ALIAS("nfs-layouttype4-4");
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("The NFSv4 flexfile layout driver");
+
+module_init(nfs4flexfilelayout_init);
+module_exit(nfs4flexfilelayout_exit);
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h
new file mode 100644
index 000000000000..070f20445b2d
--- /dev/null
+++ b/fs/nfs/flexfilelayout/flexfilelayout.h
@@ -0,0 +1,155 @@
+/*
+ * NFSv4 flexfile layout driver data structures.
+ *
+ * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
+ *
+ * Tao Peng <bergwolf@primarydata.com>
+ */
+
+#ifndef FS_NFS_NFS4FLEXFILELAYOUT_H
+#define FS_NFS_NFS4FLEXFILELAYOUT_H
+
+#include "../pnfs.h"
+
+/* XXX: Let's filter out insanely large mirror count for now to avoid oom
+ * due to network error etc. */
+#define NFS4_FLEXFILE_LAYOUT_MAX_MIRROR_CNT 4096
+
+struct nfs4_ff_ds_version {
+        u32                             version;
+        u32                             minor_version;
+        u32                             rsize;
+        u32                             wsize;
+        bool                            tightly_coupled;
+};
+
+/* chained in global deviceid hlist */
+struct nfs4_ff_layout_ds {
+        struct nfs4_deviceid_node       id_node;
+        u32                             ds_versions_cnt;
+        struct nfs4_ff_ds_version       *ds_versions;
+        struct nfs4_pnfs_ds             *ds;
+};
+
+struct nfs4_ff_layout_ds_err {
+        struct list_head                list; /* linked in mirror error_list */
+        u64                             offset;
+        u64                             length;
+        int                             status;
+        enum nfs_opnum4                 opnum;
+        nfs4_stateid                    stateid;
+        struct nfs4_deviceid            deviceid;
+};
+
+struct nfs4_ff_layout_mirror {
+        u32                             ds_count;
+        u32                             efficiency;
+        struct nfs4_ff_layout_ds        *mirror_ds;
+        u32                             fh_versions_cnt;
+        struct nfs_fh                   *fh_versions;
+        nfs4_stateid                    stateid;
+        struct nfs4_string              user_name;
+        struct nfs4_string              group_name;
+        u32                             uid;
+        u32                             gid;
+        struct rpc_cred                 *cred;
+        spinlock_t                      lock;
+};
+
+struct nfs4_ff_layout_segment {
+        struct pnfs_layout_segment      generic_hdr;
+        u64                             stripe_unit;
+        u32                             mirror_array_cnt;
+        struct nfs4_ff_layout_mirror    **mirror_array;
+};
+
+struct nfs4_flexfile_layout {
+        struct pnfs_layout_hdr generic_hdr;
+        struct pnfs_ds_commit_info commit_info;
+        struct list_head        error_list; /* nfs4_ff_layout_ds_err */
+};
+
+static inline struct nfs4_flexfile_layout *
+FF_LAYOUT_FROM_HDR(struct pnfs_layout_hdr *lo)
+{
+        return container_of(lo, struct nfs4_flexfile_layout, generic_hdr);
+}
+
+static inline struct nfs4_ff_layout_segment *
+FF_LAYOUT_LSEG(struct pnfs_layout_segment *lseg)
+{
+        return container_of(lseg,
+                            struct nfs4_ff_layout_segment,
+                            generic_hdr);
+}
+
+static inline struct nfs4_deviceid_node *
+FF_LAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg, u32 idx)
+{
+        if (idx >= FF_LAYOUT_LSEG(lseg)->mirror_array_cnt ||
+            FF_LAYOUT_LSEG(lseg)->mirror_array[idx] == NULL ||
+            FF_LAYOUT_LSEG(lseg)->mirror_array[idx]->mirror_ds == NULL)
+                return NULL;
+        return &FF_LAYOUT_LSEG(lseg)->mirror_array[idx]->mirror_ds->id_node;
+}
+
+static inline struct nfs4_ff_layout_ds *
+FF_LAYOUT_MIRROR_DS(struct nfs4_deviceid_node *node)
+{
+        return container_of(node, struct nfs4_ff_layout_ds, id_node);
+}
+
+static inline struct nfs4_ff_layout_mirror *
+FF_LAYOUT_COMP(struct pnfs_layout_segment *lseg, u32 idx)
+{
+        if (idx >= FF_LAYOUT_LSEG(lseg)->mirror_array_cnt)
+                return NULL;
+        return FF_LAYOUT_LSEG(lseg)->mirror_array[idx];
+}
+
+static inline u32
+FF_LAYOUT_MIRROR_COUNT(struct pnfs_layout_segment *lseg)
+{
+        return FF_LAYOUT_LSEG(lseg)->mirror_array_cnt;
+}
+
+static inline bool
+ff_layout_test_devid_unavailable(struct nfs4_deviceid_node *node)
+{
+        return nfs4_test_deviceid_unavailable(node);
+}
+
+static inline int
+nfs4_ff_layout_ds_version(struct pnfs_layout_segment *lseg, u32 ds_idx)
+{
+        return FF_LAYOUT_COMP(lseg, ds_idx)->mirror_ds->ds_versions[0].version;
+}
+
+struct nfs4_ff_layout_ds *
+nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
+                            gfp_t gfp_flags);
+void nfs4_ff_layout_put_deviceid(struct nfs4_ff_layout_ds *mirror_ds);
+void nfs4_ff_layout_free_deviceid(struct nfs4_ff_layout_ds *mirror_ds);
+int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
+                             struct nfs4_ff_layout_mirror *mirror, u64 offset,
+                             u64 length, int status, enum nfs_opnum4 opnum,
+                             gfp_t gfp_flags);
+int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
+                              struct xdr_stream *xdr, int *count,
+                              const struct pnfs_layout_range *range);
+struct nfs_fh *
+nfs4_ff_layout_select_ds_fh(struct pnfs_layout_segment *lseg, u32 mirror_idx);
+
+struct nfs4_pnfs_ds *
+nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx,
+                          bool fail_return);
+
+struct rpc_clnt *
+nfs4_ff_find_or_create_ds_client(struct pnfs_layout_segment *lseg,
+                                 u32 ds_idx,
+                                 struct nfs_client *ds_clp,
+                                 struct inode *inode);
+struct rpc_cred *ff_layout_get_ds_cred(struct pnfs_layout_segment *lseg,
+                                       u32 ds_idx, struct rpc_cred *mdscred);
+bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg);
+#endif /* FS_NFS_NFS4FLEXFILELAYOUT_H */
diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
new file mode 100644
index 000000000000..e2c01f204a95
--- /dev/null
+++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
@@ -0,0 +1,552 @@
+/*
+ * Device operations for the pnfs nfs4 file layout driver.
+ *
+ * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
+ *
+ * Tao Peng <bergwolf@primarydata.com>
+ */
+
+#include <linux/nfs_fs.h>
+#include <linux/vmalloc.h>
+#include <linux/module.h>
+#include <linux/sunrpc/addr.h>
+
+#include "../internal.h"
+#include "../nfs4session.h"
+#include "flexfilelayout.h"
+
+#define NFSDBG_FACILITY         NFSDBG_PNFS_LD
+
+static unsigned int dataserver_timeo = NFS4_DEF_DS_TIMEO;
+static unsigned int dataserver_retrans = NFS4_DEF_DS_RETRANS;
+
+void nfs4_ff_layout_put_deviceid(struct nfs4_ff_layout_ds *mirror_ds)
+{
+        if (mirror_ds)
+                nfs4_put_deviceid_node(&mirror_ds->id_node);
+}
+
+void nfs4_ff_layout_free_deviceid(struct nfs4_ff_layout_ds *mirror_ds)
+{
+        nfs4_print_deviceid(&mirror_ds->id_node.deviceid);
+        nfs4_pnfs_ds_put(mirror_ds->ds);
+        kfree(mirror_ds);
+}
+
+/* Decode opaque device data and construct new_ds using it */
+struct nfs4_ff_layout_ds *
+nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
+                            gfp_t gfp_flags)
+{
+        struct xdr_stream stream;
+        struct xdr_buf buf;
+        struct page *scratch;
+        struct list_head dsaddrs;
+        struct nfs4_pnfs_ds_addr *da;
+        struct nfs4_ff_layout_ds *new_ds = NULL;
+        struct nfs4_ff_ds_version *ds_versions = NULL;
+        u32 mp_count;
+        u32 version_count;
+        __be32 *p;
+        int i, ret = -ENOMEM;
+
+        /* set up xdr stream */
+        scratch = alloc_page(gfp_flags);
+        if (!scratch)
+                goto out_err;
+
+        new_ds = kzalloc(sizeof(struct nfs4_ff_layout_ds), gfp_flags);
+        if (!new_ds)
+                goto out_scratch;
+
+        nfs4_init_deviceid_node(&new_ds->id_node,
+                                server,
+                                &pdev->dev_id);
+        INIT_LIST_HEAD(&dsaddrs);
+
+        xdr_init_decode_pages(&stream, &buf, pdev->pages, pdev->pglen);
+        xdr_set_scratch_buffer(&stream, page_address(scratch), PAGE_SIZE);
+
+        /* multipath count */
+        p = xdr_inline_decode(&stream, 4);
+        if (unlikely(!p))
+                goto out_err_drain_dsaddrs;
+        mp_count = be32_to_cpup(p);
+        dprintk("%s: multipath ds count %d\n", __func__, mp_count);
+
+        for (i = 0; i < mp_count; i++) {
+                /* multipath ds */
+                da = nfs4_decode_mp_ds_addr(server->nfs_client->cl_net,
+                                            &stream, gfp_flags);
+                if (da)
+                        list_add_tail(&da->da_node, &dsaddrs);
+        }
+        if (list_empty(&dsaddrs)) {
+                dprintk("%s: no suitable DS addresses found\n",
+                        __func__);
+                ret = -ENOMEDIUM;
+                goto out_err_drain_dsaddrs;
+        }
+
+        /* version count */
+        p = xdr_inline_decode(&stream, 4);
+        if (unlikely(!p))
+                goto out_err_drain_dsaddrs;
+        version_count = be32_to_cpup(p);
+        dprintk("%s: version count %d\n", __func__, version_count);
+
+        ds_versions = kzalloc(version_count * sizeof(struct nfs4_ff_ds_version),
+                              gfp_flags);
+        if (!ds_versions)
+                goto out_scratch;
+
+        for (i = 0; i < version_count; i++) {
+                /* 20 = version(4) + minor_version(4) + rsize(4) + wsize(4) +
+                 * tightly_coupled(4) */
+                p = xdr_inline_decode(&stream, 20);
+                if (unlikely(!p))
+                        goto out_err_drain_dsaddrs;
+                ds_versions[i].version = be32_to_cpup(p++);
+                ds_versions[i].minor_version = be32_to_cpup(p++);
+                ds_versions[i].rsize = nfs_block_size(be32_to_cpup(p++), NULL);
+                ds_versions[i].wsize = nfs_block_size(be32_to_cpup(p++), NULL);
+                ds_versions[i].tightly_coupled = be32_to_cpup(p);
+
+                if (ds_versions[i].rsize > NFS_MAX_FILE_IO_SIZE)
+                        ds_versions[i].rsize = NFS_MAX_FILE_IO_SIZE;
+                if (ds_versions[i].wsize > NFS_MAX_FILE_IO_SIZE)
+                        ds_versions[i].wsize = NFS_MAX_FILE_IO_SIZE;
+
+                if (ds_versions[i].version != 3 || ds_versions[i].minor_version != 0) {
+                        dprintk("%s: [%d] unsupported ds version %d-%d\n", __func__,
+                                i, ds_versions[i].version,
+                                ds_versions[i].minor_version);
+                        ret = -EPROTONOSUPPORT;
+                        goto out_err_drain_dsaddrs;
+                }
+
+                dprintk("%s: [%d] vers %u minor_ver %u rsize %u wsize %u coupled %d\n",
+                        __func__, i, ds_versions[i].version,
+                        ds_versions[i].minor_version,
+                        ds_versions[i].rsize,
+                        ds_versions[i].wsize,
+                        ds_versions[i].tightly_coupled);
+        }
+
+        new_ds->ds_versions = ds_versions;
+        new_ds->ds_versions_cnt = version_count;
+
+        new_ds->ds = nfs4_pnfs_ds_add(&dsaddrs, gfp_flags);
+        if (!new_ds->ds)
+                goto out_err_drain_dsaddrs;
+
+        /* If DS was already in cache, free ds addrs */
+        while (!list_empty(&dsaddrs)) {
+                da = list_first_entry(&dsaddrs,
+                                      struct nfs4_pnfs_ds_addr,
+                                      da_node);
+                list_del_init(&da->da_node);
+                kfree(da->da_remotestr);
+                kfree(da);
+        }
+
+        __free_page(scratch);
+        return new_ds;
+
+out_err_drain_dsaddrs:
+        while (!list_empty(&dsaddrs)) {
+                da = list_first_entry(&dsaddrs, struct nfs4_pnfs_ds_addr,
+                                      da_node);
+                list_del_init(&da->da_node);
+                kfree(da->da_remotestr);
+                kfree(da);
+        }
+
+        kfree(ds_versions);
+out_scratch:
+        __free_page(scratch);
+out_err:
+        kfree(new_ds);
+
+        dprintk("%s ERROR: returning %d\n", __func__, ret);
+        return NULL;
+}
+
+static u64
+end_offset(u64 start, u64 len)
+{
+        u64 end;
+
+        end = start + len;
+        return end >= start ? end : NFS4_MAX_UINT64;
+}
+
+static void extend_ds_error(struct nfs4_ff_layout_ds_err *err,
+                            u64 offset, u64 length)
+{
+        u64 end;
+
+        end = max_t(u64, end_offset(err->offset, err->length),
+                    end_offset(offset, length));
+        err->offset = min_t(u64, err->offset, offset);
+        err->length = end - err->offset;
+}
+
+static bool ds_error_can_merge(struct nfs4_ff_layout_ds_err *err,  u64 offset,
+                               u64 length, int status, enum nfs_opnum4 opnum,
+                               nfs4_stateid *stateid,
+                               struct nfs4_deviceid *deviceid)
+{
+        return err->status == status && err->opnum == opnum &&
+               nfs4_stateid_match(&err->stateid, stateid) &&
+               !memcmp(&err->deviceid, deviceid, sizeof(*deviceid)) &&
+               end_offset(err->offset, err->length) >= offset &&
+               err->offset <= end_offset(offset, length);
+}
+
+static bool merge_ds_error(struct nfs4_ff_layout_ds_err *old,
+                           struct nfs4_ff_layout_ds_err *new)
+{
+        if (!ds_error_can_merge(old, new->offset, new->length, new->status,
+                                new->opnum, &new->stateid, &new->deviceid))
+                return false;
+
+        extend_ds_error(old, new->offset, new->length);
+        return true;
+}
+
+static bool
+ff_layout_add_ds_error_locked(struct nfs4_flexfile_layout *flo,
+                              struct nfs4_ff_layout_ds_err *dserr)
+{
+        struct nfs4_ff_layout_ds_err *err;
+
+        list_for_each_entry(err, &flo->error_list, list) {
+                if (merge_ds_error(err, dserr)) {
+                        return true;
+                }
+        }
+
+        list_add(&dserr->list, &flo->error_list);
+        return false;
+}
+
+static bool
+ff_layout_update_ds_error(struct nfs4_flexfile_layout *flo, u64 offset,
+                          u64 length, int status, enum nfs_opnum4 opnum,
+                          nfs4_stateid *stateid, struct nfs4_deviceid *deviceid)
+{
+        bool found = false;
+        struct nfs4_ff_layout_ds_err *err;
+
+        list_for_each_entry(err, &flo->error_list, list) {
+                if (ds_error_can_merge(err, offset, length, status, opnum,
+                                       stateid, deviceid)) {
+                        found = true;
+                        extend_ds_error(err, offset, length);
+                        break;
+                }
+        }
+
+        return found;
+}
+
+int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
+                             struct nfs4_ff_layout_mirror *mirror, u64 offset,
+                             u64 length, int status, enum nfs_opnum4 opnum,
+                             gfp_t gfp_flags)
+{
+        struct nfs4_ff_layout_ds_err *dserr;
+        bool needfree;
+
+        if (status == 0)
+                return 0;
+
+        if (mirror->mirror_ds == NULL)
+                return -EINVAL;
+
+        spin_lock(&flo->generic_hdr.plh_inode->i_lock);
+        if (ff_layout_update_ds_error(flo, offset, length, status, opnum,
+                                      &mirror->stateid,
+                                      &mirror->mirror_ds->id_node.deviceid)) {
+                spin_unlock(&flo->generic_hdr.plh_inode->i_lock);
+                return 0;
+        }
+        spin_unlock(&flo->generic_hdr.plh_inode->i_lock);
+        dserr = kmalloc(sizeof(*dserr), gfp_flags);
+        if (!dserr)
+                return -ENOMEM;
+
+        INIT_LIST_HEAD(&dserr->list);
+        dserr->offset = offset;
+        dserr->length = length;
+        dserr->status = status;
+        dserr->opnum = opnum;
+        nfs4_stateid_copy(&dserr->stateid, &mirror->stateid);
+        memcpy(&dserr->deviceid, &mirror->mirror_ds->id_node.deviceid,
+               NFS4_DEVICEID4_SIZE);
+
+        spin_lock(&flo->generic_hdr.plh_inode->i_lock);
+        needfree = ff_layout_add_ds_error_locked(flo, dserr);
+        spin_unlock(&flo->generic_hdr.plh_inode->i_lock);
+        if (needfree)
+                kfree(dserr);
+
+        return 0;
+}
+
+/* currently we only support AUTH_NONE and AUTH_SYS */
+static rpc_authflavor_t
+nfs4_ff_layout_choose_authflavor(struct nfs4_ff_layout_mirror *mirror)
+{
+        if (mirror->uid == (u32)-1)
+                return RPC_AUTH_NULL;
+        return RPC_AUTH_UNIX;
+}
+
+/* fetch cred for NFSv3 DS */
+static int ff_layout_update_mirror_cred(struct nfs4_ff_layout_mirror *mirror,
+                                      struct nfs4_pnfs_ds *ds)
+{
+        if (ds->ds_clp && !mirror->cred &&
+            mirror->mirror_ds->ds_versions[0].version == 3) {
+                struct rpc_auth *auth = ds->ds_clp->cl_rpcclient->cl_auth;
+                struct rpc_cred *cred;
+                struct auth_cred acred = {
+                        .uid = make_kuid(&init_user_ns, mirror->uid),
+                        .gid = make_kgid(&init_user_ns, mirror->gid),
+                };
+
+                /* AUTH_NULL ignores acred */
+                cred = auth->au_ops->lookup_cred(auth, &acred, 0);
+                if (IS_ERR(cred)) {
+                        dprintk("%s: lookup_cred failed with %ld\n",
+                                __func__, PTR_ERR(cred));
+                        return PTR_ERR(cred);
+                } else {
+                        mirror->cred = cred;
+                }
+        }
+        return 0;
+}
+
+struct nfs_fh *
+nfs4_ff_layout_select_ds_fh(struct pnfs_layout_segment *lseg, u32 mirror_idx)
+{
+        struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, mirror_idx);
+        struct nfs_fh *fh = NULL;
+        struct nfs4_deviceid_node *devid;
+
+        if (mirror == NULL || mirror->mirror_ds == NULL ||
+            mirror->mirror_ds->ds == NULL) {
+                printk(KERN_ERR "NFS: %s: No data server for mirror offset index %d\n",
+                        __func__, mirror_idx);
+                if (mirror && mirror->mirror_ds) {
+                        devid = &mirror->mirror_ds->id_node;
+                        pnfs_generic_mark_devid_invalid(devid);
+                }
+                goto out;
+        }
+
+        /* FIXME: For now assume there is only 1 version available for the DS */
+        fh = &mirror->fh_versions[0];
+out:
+        return fh;
+}
+
+/* Upon return, either ds is connected, or ds is NULL */
+struct nfs4_pnfs_ds *
+nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx,
+                          bool fail_return)
+{
+        struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, ds_idx);
+        struct nfs4_pnfs_ds *ds = NULL;
+        struct nfs4_deviceid_node *devid;
+        struct inode *ino = lseg->pls_layout->plh_inode;
+        struct nfs_server *s = NFS_SERVER(ino);
+        unsigned int max_payload;
+        rpc_authflavor_t flavor;
+
+        if (mirror == NULL || mirror->mirror_ds == NULL ||
+            mirror->mirror_ds->ds == NULL) {
+                printk(KERN_ERR "NFS: %s: No data server for offset index %d\n",
+                        __func__, ds_idx);
+                if (mirror && mirror->mirror_ds) {
+                        devid = &mirror->mirror_ds->id_node;
+                        pnfs_generic_mark_devid_invalid(devid);
+                }
+                goto out;
+        }
+
+        devid = &mirror->mirror_ds->id_node;
+        if (ff_layout_test_devid_unavailable(devid))
+                goto out;
+
+        ds = mirror->mirror_ds->ds;
+        /* matching smp_wmb() in _nfs4_pnfs_v3/4_ds_connect */
+        smp_rmb();
+        if (ds->ds_clp)
+                goto out;
+
+        flavor = nfs4_ff_layout_choose_authflavor(mirror);
+
+        /* FIXME: For now we assume the server sent only one version of NFS
+         * to use for the DS.
+         */
+        nfs4_pnfs_ds_connect(s, ds, devid, dataserver_timeo,
+                             dataserver_retrans,
+                             mirror->mirror_ds->ds_versions[0].version,
+                             mirror->mirror_ds->ds_versions[0].minor_version,
+                             flavor);
+
+        /* connect success, check rsize/wsize limit */
+        if (ds->ds_clp) {
+                max_payload =
+                        nfs_block_size(rpc_max_payload(ds->ds_clp->cl_rpcclient),
+                                       NULL);
+                if (mirror->mirror_ds->ds_versions[0].rsize > max_payload)
+                        mirror->mirror_ds->ds_versions[0].rsize = max_payload;
+                if (mirror->mirror_ds->ds_versions[0].wsize > max_payload)
+                        mirror->mirror_ds->ds_versions[0].wsize = max_payload;
+        } else {
+                ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout),
+                                         mirror, lseg->pls_range.offset,
+                                         lseg->pls_range.length, NFS4ERR_NXIO,
+                                         OP_ILLEGAL, GFP_NOIO);
+                if (fail_return) {
+                        pnfs_error_mark_layout_for_return(ino, lseg);
+                        if (ff_layout_has_available_ds(lseg))
+                                pnfs_set_retry_layoutget(lseg->pls_layout);
+                        else
+                                pnfs_clear_retry_layoutget(lseg->pls_layout);
+
+                } else {
+                        if (ff_layout_has_available_ds(lseg))
+                                set_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
+                                        &lseg->pls_layout->plh_flags);
+                        else {
+                                pnfs_error_mark_layout_for_return(ino, lseg);
+                                pnfs_clear_retry_layoutget(lseg->pls_layout);
+                        }
+                }
+        }
+
+        if (ff_layout_update_mirror_cred(mirror, ds))
+                ds = NULL;
+out:
+        return ds;
+}
+
+struct rpc_cred *
+ff_layout_get_ds_cred(struct pnfs_layout_segment *lseg, u32 ds_idx,
+                      struct rpc_cred *mdscred)
+{
+        struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, ds_idx);
+        struct rpc_cred *cred = ERR_PTR(-EINVAL);
+
+        if (!nfs4_ff_layout_prepare_ds(lseg, ds_idx, true))
+                goto out;
+
+        if (mirror && mirror->cred)
+                cred = mirror->cred;
+        else
+                cred = mdscred;
+out:
+        return cred;
+}
+
+/**
+* Find or create a DS rpc client with th MDS server rpc client auth flavor
+* in the nfs_client cl_ds_clients list.
+*/
+struct rpc_clnt *
+nfs4_ff_find_or_create_ds_client(struct pnfs_layout_segment *lseg, u32 ds_idx,
+                                 struct nfs_client *ds_clp, struct inode *inode)
+{
+        struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, ds_idx);
+
+        switch (mirror->mirror_ds->ds_versions[0].version) {
+        case 3:
+                /* For NFSv3 DS, flavor is set when creating DS connections */
+                return ds_clp->cl_rpcclient;
+        case 4:
+                return nfs4_find_or_create_ds_client(ds_clp, inode);
+        default:
+                BUG();
+        }
+}
+
+static bool is_range_intersecting(u64 offset1, u64 length1,
+                                  u64 offset2, u64 length2)
+{
+        u64 end1 = end_offset(offset1, length1);
+        u64 end2 = end_offset(offset2, length2);
+
+        return (end1 == NFS4_MAX_UINT64 || end1 > offset2) &&
+               (end2 == NFS4_MAX_UINT64 || end2 > offset1);
+}
+
+/* called with inode i_lock held */
+int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
+                              struct xdr_stream *xdr, int *count,
+                              const struct pnfs_layout_range *range)
+{
+        struct nfs4_ff_layout_ds_err *err, *n;
+        __be32 *p;
+
+        list_for_each_entry_safe(err, n, &flo->error_list, list) {
+                if (!is_range_intersecting(err->offset, err->length,
+                                           range->offset, range->length))
+                        continue;
+                /* offset(8) + length(8) + stateid(NFS4_STATEID_SIZE)
+                 * + deviceid(NFS4_DEVICEID4_SIZE) + status(4) + opnum(4)
+                 */
+                p = xdr_reserve_space(xdr,
+                                24 + NFS4_STATEID_SIZE + NFS4_DEVICEID4_SIZE);
+                if (unlikely(!p))
+                        return -ENOBUFS;
+                p = xdr_encode_hyper(p, err->offset);
+                p = xdr_encode_hyper(p, err->length);
+                p = xdr_encode_opaque_fixed(p, &err->stateid,
+                                            NFS4_STATEID_SIZE);
+                p = xdr_encode_opaque_fixed(p, &err->deviceid,
+                                            NFS4_DEVICEID4_SIZE);
+                *p++ = cpu_to_be32(err->status);
+                *p++ = cpu_to_be32(err->opnum);
+                *count += 1;
+                list_del(&err->list);
+                dprintk("%s: offset %llu length %llu status %d op %d count %d\n",
+                        __func__, err->offset, err->length, err->status,
+                        err->opnum, *count);
+                kfree(err);
+        }
+
+        return 0;
+}
+
+bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg)
+{
+        struct nfs4_ff_layout_mirror *mirror;
+        struct nfs4_deviceid_node *devid;
+        int idx;
+
+        for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) {
+                mirror = FF_LAYOUT_COMP(lseg, idx);
+                if (mirror && mirror->mirror_ds) {
+                        devid = &mirror->mirror_ds->id_node;
+                        if (!ff_layout_test_devid_unavailable(devid))
+                                return true;
+                }
+        }
+
+        return false;
+}
+
+module_param(dataserver_retrans, uint, 0644);
+MODULE_PARM_DESC(dataserver_retrans, "The  number of times the NFSv4.1 client "
+                        "retries a request before it attempts further "
+                        " recovery  action.");
+module_param(dataserver_timeo, uint, 0644);
+MODULE_PARM_DESC(dataserver_timeo, "The time (in tenths of a second) the "
+                        "NFSv4.1  client  waits for a response from a "
+                        " data server before it retries an NFS request.");
diff --git a/fs/nfs/idmap.c b/fs/nfs/idmap.c
index 2f5db844c172..857e2a99acc8 100644
--- a/fs/nfs/idmap.c
+++ b/fs/nfs/idmap.c
@@ -152,7 +152,7 @@ void nfs_fattr_map_and_free_names(struct nfs_server *server, struct nfs_fattr *f
                 nfs_fattr_free_group_name(fattr);
 }
 
-static int nfs_map_string_to_numeric(const char *name, size_t namelen, __u32 *res)
+int nfs_map_string_to_numeric(const char *name, size_t namelen, __u32 *res)
 {
         unsigned long val;
         char buf[16];
@@ -166,6 +166,7 @@ static int nfs_map_string_to_numeric(const char *name, size_t namelen, __u32 *re
         *res = val;
         return 1;
 }
+EXPORT_SYMBOL_GPL(nfs_map_string_to_numeric);
 
 static int nfs_map_numeric_to_string(__u32 id, char *buf, size_t buflen)
 {
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 2211f6ba8736..d2398c193bda 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -507,10 +507,15 @@ nfs_setattr(struct dentry *dentry, struct iattr *attr)
                 attr->ia_valid &= ~ATTR_MODE;
 
         if (attr->ia_valid & ATTR_SIZE) {
+                loff_t i_size;
+
                 BUG_ON(!S_ISREG(inode->i_mode));
 
-                if (attr->ia_size == i_size_read(inode))
+                i_size = i_size_read(inode);
+                if (attr->ia_size == i_size)
                         attr->ia_valid &= ~ATTR_SIZE;
+                else if (attr->ia_size < i_size && IS_SWAPFILE(inode))
+                        return -ETXTBSY;
         }
 
         /* Optimization: if the end result is no change, don't RPC */
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index b6f34bfa6fe8..21469e6e3834 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -6,6 +6,7 @@
 #include <linux/mount.h>
 #include <linux/security.h>
 #include <linux/crc32.h>
+#include <linux/nfs_page.h>
 
 #define NFS_MS_MASK (MS_RDONLY|MS_NOSUID|MS_NODEV|MS_NOEXEC|MS_SYNCHRONOUS)
 
@@ -187,9 +188,15 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_client* mds_clp,
                                              const struct sockaddr *ds_addr,
                                              int ds_addrlen, int ds_proto,
                                              unsigned int ds_timeo,
-                                             unsigned int ds_retrans);
+                                             unsigned int ds_retrans,
+                                             u32 minor_version,
+                                             rpc_authflavor_t au_flavor);
 extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
                                                 struct inode *);
+extern struct nfs_client *nfs3_set_ds_client(struct nfs_client *mds_clp,
+                        const struct sockaddr *ds_addr, int ds_addrlen,
+                        int ds_proto, unsigned int ds_timeo,
+                        unsigned int ds_retrans, rpc_authflavor_t au_flavor);
 #ifdef CONFIG_PROC_FS
 extern int __init nfs_fs_proc_init(void);
 extern void nfs_fs_proc_exit(void);
@@ -242,9 +249,12 @@ struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *);
 void nfs_pgio_header_free(struct nfs_pgio_header *);
 void nfs_pgio_data_destroy(struct nfs_pgio_header *);
 int nfs_generic_pgio(struct nfs_pageio_descriptor *, struct nfs_pgio_header *);
-int nfs_initiate_pgio(struct rpc_clnt *, struct nfs_pgio_header *,
-                      const struct rpc_call_ops *, int, int);
+int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
+                      struct rpc_cred *cred, const struct nfs_rpc_ops *rpc_ops,
+                      const struct rpc_call_ops *call_ops, int how, int flags);
 void nfs_free_request(struct nfs_page *req);
+struct nfs_pgio_mirror *
+nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc);
 
 static inline void nfs_iocounter_init(struct nfs_io_counter *c)
 {
@@ -252,6 +262,12 @@ static inline void nfs_iocounter_init(struct nfs_io_counter *c)
         atomic_set(&c->io_count, 0);
 }
 
+static inline bool nfs_pgio_has_mirroring(struct nfs_pageio_descriptor *desc)
+{
+        WARN_ON_ONCE(desc->pg_mirror_count < 1);
+        return desc->pg_mirror_count > 1;
+}
+
 /* nfs2xdr.c */
 extern struct rpc_procinfo nfs_procedures[];
 extern int nfs2_decode_dirent(struct xdr_stream *,
@@ -375,7 +391,7 @@ extern struct rpc_stat nfs_rpcstat;
 
 extern int __init register_nfs_fs(void);
 extern void __exit unregister_nfs_fs(void);
-extern void nfs_sb_active(struct super_block *sb);
+extern bool nfs_sb_active(struct super_block *sb);
 extern void nfs_sb_deactive(struct super_block *sb);
 
 /* namespace.c */
@@ -427,6 +443,7 @@ extern void nfs_write_prepare(struct rpc_task *task, void *calldata);
 extern void nfs_commit_prepare(struct rpc_task *task, void *calldata);
 extern int nfs_initiate_commit(struct rpc_clnt *clnt,
                                struct nfs_commit_data *data,
+                               const struct nfs_rpc_ops *nfs_ops,
                                const struct rpc_call_ops *call_ops,
                                int how, int flags);
 extern void nfs_init_commit(struct nfs_commit_data *data,
@@ -440,13 +457,15 @@ int nfs_scan_commit(struct inode *inode, struct list_head *dst,
                     struct nfs_commit_info *cinfo);
 void nfs_mark_request_commit(struct nfs_page *req,
                              struct pnfs_layout_segment *lseg,
-                             struct nfs_commit_info *cinfo);
+                             struct nfs_commit_info *cinfo,
+                             u32 ds_commit_idx);
 int nfs_write_need_commit(struct nfs_pgio_header *);
 int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
                             int how, struct nfs_commit_info *cinfo);
 void nfs_retry_commit(struct list_head *page_list,
                       struct pnfs_layout_segment *lseg,
-                      struct nfs_commit_info *cinfo);
+                      struct nfs_commit_info *cinfo,
+                      u32 ds_commit_idx);
 void nfs_commitdata_release(struct nfs_commit_data *data);
 void nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst,
                                  struct nfs_commit_info *cinfo);
@@ -457,6 +476,7 @@ void nfs_init_cinfo(struct nfs_commit_info *cinfo,
                     struct nfs_direct_req *dreq);
 int nfs_key_timeout_notify(struct file *filp, struct inode *inode);
 bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx);
+void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio);
 
 #ifdef CONFIG_MIGRATION
 extern int nfs_migrate_page(struct address_space *,
@@ -480,6 +500,7 @@ static inline void nfs_inode_dio_wait(struct inode *inode)
         inode_dio_wait(inode);
 }
 extern ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq);
+extern void nfs_direct_set_resched_writes(struct nfs_direct_req *dreq);
 
 /* nfs4proc.c */
 extern void __nfs4_read_done_cb(struct nfs_pgio_header *);
@@ -493,6 +514,26 @@ extern int nfs41_walk_client_list(struct nfs_client *clp,
                                 struct nfs_client **result,
                                 struct rpc_cred *cred);
 
+static inline struct inode *nfs_igrab_and_active(struct inode *inode)
+{
+        inode = igrab(inode);
+        if (inode != NULL && !nfs_sb_active(inode->i_sb)) {
+                iput(inode);
+                inode = NULL;
+        }
+        return inode;
+}
+
+static inline void nfs_iput_and_deactive(struct inode *inode)
+{
+        if (inode != NULL) {
+                struct super_block *sb = inode->i_sb;
+
+                iput(inode);
+                nfs_sb_deactive(sb);
+        }
+}
+
 /*
  * Determine the device name as a string
  */
diff --git a/fs/nfs/nfs2xdr.c b/fs/nfs/nfs2xdr.c
index 5f61b83f4a1c..b4e03ed8599d 100644
--- a/fs/nfs/nfs2xdr.c
+++ b/fs/nfs/nfs2xdr.c
@@ -481,7 +481,8 @@ out_overflow:
  *              void;
  *      };
  */
-static int decode_attrstat(struct xdr_stream *xdr, struct nfs_fattr *result)
+static int decode_attrstat(struct xdr_stream *xdr, struct nfs_fattr *result,
+                           __u32 *op_status)
 {
         enum nfs_stat status;
         int error;
@@ -489,6 +490,8 @@ static int decode_attrstat(struct xdr_stream *xdr, struct nfs_fattr *result)
         error = decode_stat(xdr, &status);
         if (unlikely(error))
                 goto out;
+        if (op_status)
+                *op_status = status;
         if (status != NFS_OK)
                 goto out_default;
         error = decode_fattr(xdr, result);
@@ -808,7 +811,7 @@ out_default:
 static int nfs2_xdr_dec_attrstat(struct rpc_rqst *req, struct xdr_stream *xdr,
                                  struct nfs_fattr *result)
 {
-        return decode_attrstat(xdr, result);
+        return decode_attrstat(xdr, result, NULL);
 }
 
 static int nfs2_xdr_dec_diropres(struct rpc_rqst *req, struct xdr_stream *xdr,
@@ -865,6 +868,7 @@ static int nfs2_xdr_dec_readres(struct rpc_rqst *req, struct xdr_stream *xdr,
         error = decode_stat(xdr, &status);
         if (unlikely(error))
                 goto out;
+        result->op_status = status;
         if (status != NFS_OK)
                 goto out_default;
         error = decode_fattr(xdr, result->fattr);
@@ -882,7 +886,7 @@ static int nfs2_xdr_dec_writeres(struct rpc_rqst *req, struct xdr_stream *xdr,
 {
         /* All NFSv2 writes are "file sync" writes */
         result->verf->committed = NFS_FILE_SYNC;
-        return decode_attrstat(xdr, result->fattr);
+        return decode_attrstat(xdr, result->fattr, &result->op_status);
 }
 
 /**
diff --git a/fs/nfs/nfs3_fs.h b/fs/nfs/nfs3_fs.h
index 333ae4068506..e134d6548ab7 100644
--- a/fs/nfs/nfs3_fs.h
+++ b/fs/nfs/nfs3_fs.h
@@ -30,5 +30,7 @@ struct nfs_server *nfs3_create_server(struct nfs_mount_info *, struct nfs_subver
 struct nfs_server *nfs3_clone_server(struct nfs_server *, struct nfs_fh *,
                                      struct nfs_fattr *, rpc_authflavor_t);
 
+/* nfs3super.c */
+extern struct nfs_subversion nfs_v3;
 
 #endif /* __LINUX_FS_NFS_NFS3_FS_H */
diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c
index 8c1b437c5403..9e9fa347a948 100644
--- a/fs/nfs/nfs3client.c
+++ b/fs/nfs/nfs3client.c
@@ -1,5 +1,6 @@
 #include <linux/nfs_fs.h>
 #include <linux/nfs_mount.h>
+#include <linux/sunrpc/addr.h>
 #include "internal.h"
 #include "nfs3_fs.h"
 
@@ -64,3 +65,43 @@ struct nfs_server *nfs3_clone_server(struct nfs_server *source,
                 nfs_init_server_aclclient(server);
         return server;
 }
+
+/*
+ * Set up a pNFS Data Server client over NFSv3.
+ *
+ * Return any existing nfs_client that matches server address,port,version
+ * and minorversion.
+ *
+ * For a new nfs_client, use a soft mount (default), a low retrans and a
+ * low timeout interval so that if a connection is lost, we retry through
+ * the MDS.
+ */
+struct nfs_client *nfs3_set_ds_client(struct nfs_client *mds_clp,
+                const struct sockaddr *ds_addr, int ds_addrlen,
+                int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans,
+                rpc_authflavor_t au_flavor)
+{
+        struct nfs_client_initdata cl_init = {
+                .addr = ds_addr,
+                .addrlen = ds_addrlen,
+                .nfs_mod = &nfs_v3,
+                .proto = ds_proto,
+                .net = mds_clp->cl_net,
+        };
+        struct rpc_timeout ds_timeout;
+        struct nfs_client *clp;
+        char buf[INET6_ADDRSTRLEN + 1];
+
+        /* fake a hostname because lockd wants it */
+        if (rpc_ntop(ds_addr, buf, sizeof(buf)) <= 0)
+                return ERR_PTR(-EINVAL);
+        cl_init.hostname = buf;
+
+        /* Use the MDS nfs_client cl_ipaddr. */
+        nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans);
+        clp = nfs_get_client(&cl_init, &ds_timeout, mds_clp->cl_ipaddr,
+                             au_flavor);
+
+        return clp;
+}
+EXPORT_SYMBOL_GPL(nfs3_set_ds_client);
diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c
index 524f9f837408..78e557c3ab87 100644
--- a/fs/nfs/nfs3proc.c
+++ b/fs/nfs/nfs3proc.c
@@ -800,6 +800,9 @@ static int nfs3_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr)
 {
         struct inode *inode = hdr->inode;
 
+        if (hdr->pgio_done_cb != NULL)
+                return hdr->pgio_done_cb(task, hdr);
+
         if (nfs3_async_handle_jukebox(task, inode))
                 return -EAGAIN;
 
@@ -825,6 +828,9 @@ static int nfs3_write_done(struct rpc_task *task, struct nfs_pgio_header *hdr)
 {
         struct inode *inode = hdr->inode;
 
+        if (hdr->pgio_done_cb != NULL)
+                return hdr->pgio_done_cb(task, hdr);
+
         if (nfs3_async_handle_jukebox(task, inode))
                 return -EAGAIN;
         if (task->tk_status >= 0)
@@ -845,6 +851,9 @@ static void nfs3_proc_commit_rpc_prepare(struct rpc_task *task, struct nfs_commi
 
 static int nfs3_commit_done(struct rpc_task *task, struct nfs_commit_data *data)
 {
+        if (data->commit_done_cb != NULL)
+                return data->commit_done_cb(task, data);
+
         if (nfs3_async_handle_jukebox(task, data->inode))
                 return -EAGAIN;
         nfs_refresh_inode(data->inode, data->res.fattr);
diff --git a/fs/nfs/nfs3super.c b/fs/nfs/nfs3super.c
index 6af29c2da352..5c4394e4656b 100644
--- a/fs/nfs/nfs3super.c
+++ b/fs/nfs/nfs3super.c
@@ -7,7 +7,7 @@
 #include "nfs3_fs.h"
 #include "nfs.h"
 
-static struct nfs_subversion nfs_v3 = {
+struct nfs_subversion nfs_v3 = {
         .owner = THIS_MODULE,
         .nfs_fs   = &nfs_fs_type,
         .rpc_vers = &nfs_version3,
diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c
index 8f4cbe7f4aa8..2a932fdc57cb 100644
--- a/fs/nfs/nfs3xdr.c
+++ b/fs/nfs/nfs3xdr.c
@@ -1636,6 +1636,7 @@ static int nfs3_xdr_dec_read3res(struct rpc_rqst *req, struct xdr_stream *xdr,
         error = decode_post_op_attr(xdr, result->fattr);
         if (unlikely(error))
                 goto out;
+        result->op_status = status;
         if (status != NFS3_OK)
                 goto out_status;
         error = decode_read3resok(xdr, result);
@@ -1708,6 +1709,7 @@ static int nfs3_xdr_dec_write3res(struct rpc_rqst *req, struct xdr_stream *xdr,
         error = decode_wcc_data(xdr, result->fattr);
         if (unlikely(error))
                 goto out;
+        result->op_status = status;
         if (status != NFS3_OK)
                 goto out_status;
         error = decode_write3resok(xdr, result);
@@ -2323,6 +2325,7 @@ static int nfs3_xdr_dec_commit3res(struct rpc_rqst *req,
         error = decode_wcc_data(xdr, result->fattr);
         if (unlikely(error))
                 goto out;
+        result->op_status = status;
         if (status != NFS3_OK)
                 goto out_status;
         error = decode_writeverf3(xdr, &result->verf->verifier);
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index a08178764cf9..fdef424b0cd3 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -44,6 +44,7 @@ enum nfs4_client_state {
 #define NFS4_RENEW_TIMEOUT              0x01
 #define NFS4_RENEW_DELEGATION_CB        0x02
 
+struct nfs_seqid_counter;
 struct nfs4_minor_version_ops {
         u32     minor_version;
         unsigned init_caps;
@@ -56,6 +57,8 @@ struct nfs4_minor_version_ops {
                         struct nfs_fsinfo *);
         void    (*free_lock_state)(struct nfs_server *,
                         struct nfs4_lock_state *);
+        struct nfs_seqid *
+                (*alloc_seqid)(struct nfs_seqid_counter *, gfp_t);
         const struct rpc_call_ops *call_sync_ops;
         const struct nfs4_state_recovery_ops *reboot_recovery_ops;
         const struct nfs4_state_recovery_ops *nograce_recovery_ops;
@@ -443,6 +446,12 @@ extern void nfs_increment_open_seqid(int status, struct nfs_seqid *seqid);
 extern void nfs_increment_lock_seqid(int status, struct nfs_seqid *seqid);
 extern void nfs_release_seqid(struct nfs_seqid *seqid);
 extern void nfs_free_seqid(struct nfs_seqid *seqid);
+extern int nfs40_setup_sequence(struct nfs4_slot_table *tbl,
+                                struct nfs4_sequence_args *args,
+                                struct nfs4_sequence_res *res,
+                                struct rpc_task *task);
+extern int nfs4_sequence_done(struct rpc_task *task,
+                              struct nfs4_sequence_res *res);
 
 extern void nfs4_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp);
 
diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
index 706ad10b8186..8646af9b11d2 100644
--- a/fs/nfs/nfs4client.c
+++ b/fs/nfs/nfs4client.c
@@ -849,14 +849,15 @@ error:
  */
 struct nfs_client *nfs4_set_ds_client(struct nfs_client* mds_clp,
                 const struct sockaddr *ds_addr, int ds_addrlen,
-                int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans)
+                int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans,
+                u32 minor_version, rpc_authflavor_t au_flavor)
 {
         struct nfs_client_initdata cl_init = {
                 .addr = ds_addr,
                 .addrlen = ds_addrlen,
                 .nfs_mod = &nfs_v4,
                 .proto = ds_proto,
-                .minorversion = mds_clp->cl_minorversion,
+                .minorversion = minor_version,
                 .net = mds_clp->cl_net,
         };
         struct rpc_timeout ds_timeout;
@@ -874,7 +875,7 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_client* mds_clp,
          */
         nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans);
         clp = nfs_get_client(&cl_init, &ds_timeout, mds_clp->cl_ipaddr,
-                             mds_clp->cl_rpcclient->cl_auth->au_flavor);
+                             au_flavor);
 
         dprintk("<-- %s %p\n", __func__, clp);
         return clp;
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index c347705b0161..2e7c9f7a6f7c 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -495,12 +495,11 @@ static void nfs4_set_sequence_privileged(struct nfs4_sequence_args *args)
         args->sa_privileged = 1;
 }
 
-static int nfs40_setup_sequence(const struct nfs_server *server,
-                                struct nfs4_sequence_args *args,
-                                struct nfs4_sequence_res *res,
-                                struct rpc_task *task)
+int nfs40_setup_sequence(struct nfs4_slot_table *tbl,
+                         struct nfs4_sequence_args *args,
+                         struct nfs4_sequence_res *res,
+                         struct rpc_task *task)
 {
-        struct nfs4_slot_table *tbl = server->nfs_client->cl_slot_tbl;
         struct nfs4_slot *slot;
 
         /* slot already allocated? */
@@ -535,6 +534,7 @@ out_sleep:
         spin_unlock(&tbl->slot_tbl_lock);
         return -EAGAIN;
 }
+EXPORT_SYMBOL_GPL(nfs40_setup_sequence);
 
 static int nfs40_sequence_done(struct rpc_task *task,
                                struct nfs4_sequence_res *res)
@@ -694,8 +694,7 @@ out_retry:
 }
 EXPORT_SYMBOL_GPL(nfs41_sequence_done);
 
-static int nfs4_sequence_done(struct rpc_task *task,
-                               struct nfs4_sequence_res *res)
+int nfs4_sequence_done(struct rpc_task *task, struct nfs4_sequence_res *res)
 {
         if (res->sr_slot == NULL)
                 return 1;
@@ -703,6 +702,7 @@ static int nfs4_sequence_done(struct rpc_task *task,
                 return nfs40_sequence_done(task, res);
         return nfs41_sequence_done(task, res);
 }
+EXPORT_SYMBOL_GPL(nfs4_sequence_done);
 
 int nfs41_setup_sequence(struct nfs4_session *session,
                                 struct nfs4_sequence_args *args,
@@ -777,7 +777,8 @@ static int nfs4_setup_sequence(const struct nfs_server *server,
         int ret = 0;
 
         if (!session)
-                return nfs40_setup_sequence(server, args, res, task);
+                return nfs40_setup_sequence(server->nfs_client->cl_slot_tbl,
+                                            args, res, task);
 
         dprintk("--> %s clp %p session %p sr_slot %u\n",
                 __func__, session->clp, session, res->sr_slot ?
@@ -818,14 +819,16 @@ static int nfs4_setup_sequence(const struct nfs_server *server,
                                struct nfs4_sequence_res *res,
                                struct rpc_task *task)
 {
-        return nfs40_setup_sequence(server, args, res, task);
+        return nfs40_setup_sequence(server->nfs_client->cl_slot_tbl,
+                                    args, res, task);
 }
 
-static int nfs4_sequence_done(struct rpc_task *task,
-                               struct nfs4_sequence_res *res)
+int nfs4_sequence_done(struct rpc_task *task,
+                       struct nfs4_sequence_res *res)
 {
         return nfs40_sequence_done(task, res);
 }
+EXPORT_SYMBOL_GPL(nfs4_sequence_done);
 
 #endif  /* !CONFIG_NFS_V4_1 */
 
@@ -937,6 +940,31 @@ static bool nfs4_clear_cap_atomic_open_v1(struct nfs_server *server,
         return true;
 }
 
+static u32
+nfs4_map_atomic_open_share(struct nfs_server *server,
+                fmode_t fmode, int openflags)
+{
+        u32 res = 0;
+
+        switch (fmode & (FMODE_READ | FMODE_WRITE)) {
+        case FMODE_READ:
+                res = NFS4_SHARE_ACCESS_READ;
+                break;
+        case FMODE_WRITE:
+                res = NFS4_SHARE_ACCESS_WRITE;
+                break;
+        case FMODE_READ|FMODE_WRITE:
+                res = NFS4_SHARE_ACCESS_BOTH;
+        }
+        if (!(server->caps & NFS_CAP_ATOMIC_OPEN_V1))
+                goto out;
+        /* Want no delegation if we're using O_DIRECT */
+        if (openflags & O_DIRECT)
+                res |= NFS4_SHARE_WANT_NO_DELEG;
+out:
+        return res;
+}
+
 static enum open_claim_type4
 nfs4_map_atomic_open_claim(struct nfs_server *server,
                 enum open_claim_type4 claim)
@@ -977,6 +1005,7 @@ static struct nfs4_opendata *nfs4_opendata_alloc(struct dentry *dentry,
         struct dentry *parent = dget_parent(dentry);
         struct inode *dir = parent->d_inode;
         struct nfs_server *server = NFS_SERVER(dir);
+        struct nfs_seqid *(*alloc_seqid)(struct nfs_seqid_counter *, gfp_t);
         struct nfs4_opendata *p;
 
         p = kzalloc(sizeof(*p), gfp_mask);
@@ -987,8 +1016,9 @@ static struct nfs4_opendata *nfs4_opendata_alloc(struct dentry *dentry,
         if (IS_ERR(p->f_label))
                 goto err_free_p;
 
-        p->o_arg.seqid = nfs_alloc_seqid(&sp->so_seqid, gfp_mask);
-        if (p->o_arg.seqid == NULL)
+        alloc_seqid = server->nfs_client->cl_mvops->alloc_seqid;
+        p->o_arg.seqid = alloc_seqid(&sp->so_seqid, gfp_mask);
+        if (IS_ERR(p->o_arg.seqid))
                 goto err_free_label;
         nfs_sb_active(dentry->d_sb);
         p->dentry = dget(dentry);
@@ -997,6 +1027,8 @@ static struct nfs4_opendata *nfs4_opendata_alloc(struct dentry *dentry,
         atomic_inc(&sp->so_count);
         p->o_arg.open_flags = flags;
         p->o_arg.fmode = fmode & (FMODE_READ|FMODE_WRITE);
+        p->o_arg.share_access = nfs4_map_atomic_open_share(server,
+                        fmode, flags);
         /* don't put an ACCESS op in OPEN compound if O_EXCL, because ACCESS
          * will return permission denied for all bits until close */
         if (!(flags & O_EXCL)) {
@@ -1167,6 +1199,16 @@ static bool nfs_need_update_open_stateid(struct nfs4_state *state,
         return false;
 }
 
+static void nfs_resync_open_stateid_locked(struct nfs4_state *state)
+{
+        if (state->n_wronly)
+                set_bit(NFS_O_WRONLY_STATE, &state->flags);
+        if (state->n_rdonly)
+                set_bit(NFS_O_RDONLY_STATE, &state->flags);
+        if (state->n_rdwr)
+                set_bit(NFS_O_RDWR_STATE, &state->flags);
+}
+
 static void nfs_clear_open_stateid_locked(struct nfs4_state *state,
                 nfs4_stateid *stateid, fmode_t fmode)
 {
@@ -1185,8 +1227,12 @@ static void nfs_clear_open_stateid_locked(struct nfs4_state *state,
         }
         if (stateid == NULL)
                 return;
-        if (!nfs_need_update_open_stateid(state, stateid))
+        /* Handle races with OPEN */
+        if (!nfs4_stateid_match_other(stateid, &state->open_stateid) ||
+            !nfs4_stateid_is_newer(stateid, &state->open_stateid)) {
+                nfs_resync_open_stateid_locked(state);
                 return;
+        }
         if (test_bit(NFS_DELEGATED_STATE, &state->flags) == 0)
                 nfs4_stateid_copy(&state->stateid, stateid);
         nfs4_stateid_copy(&state->open_stateid, stateid);
@@ -1281,6 +1327,23 @@ no_delegation:
         return ret;
 }
 
+static bool nfs4_update_lock_stateid(struct nfs4_lock_state *lsp,
+                const nfs4_stateid *stateid)
+{
+        struct nfs4_state *state = lsp->ls_state;
+        bool ret = false;
+
+        spin_lock(&state->state_lock);
+        if (!nfs4_stateid_match_other(stateid, &lsp->ls_stateid))
+                goto out_noupdate;
+        if (!nfs4_stateid_is_newer(stateid, &lsp->ls_stateid))
+                goto out_noupdate;
+        nfs4_stateid_copy(&lsp->ls_stateid, stateid);
+        ret = true;
+out_noupdate:
+        spin_unlock(&state->state_lock);
+        return ret;
+}
 
 static void nfs4_return_incompatible_delegation(struct inode *inode, fmode_t fmode)
 {
@@ -1679,8 +1742,8 @@ static void nfs4_open_confirm_prepare(struct rpc_task *task, void *calldata)
 {
         struct nfs4_opendata *data = calldata;
 
-        nfs40_setup_sequence(data->o_arg.server, &data->c_arg.seq_args,
-                                &data->c_res.seq_res, task);
+        nfs40_setup_sequence(data->o_arg.server->nfs_client->cl_slot_tbl,
+                             &data->c_arg.seq_args, &data->c_res.seq_res, task);
 }
 
 static void nfs4_open_confirm_done(struct rpc_task *task, void *calldata)
@@ -2587,6 +2650,11 @@ static void nfs4_close_done(struct rpc_task *task, void *data)
                 case -NFS4ERR_OLD_STATEID:
                 case -NFS4ERR_BAD_STATEID:
                 case -NFS4ERR_EXPIRED:
+                        if (!nfs4_stateid_match(&calldata->arg.stateid,
+                                                &state->stateid)) {
+                                rpc_restart_call_prepare(task);
+                                goto out_release;
+                        }
                         if (calldata->arg.fmode == 0)
                                 break;
                 default:
@@ -2619,6 +2687,7 @@ static void nfs4_close_prepare(struct rpc_task *task, void *data)
         is_rdwr = test_bit(NFS_O_RDWR_STATE, &state->flags);
         is_rdonly = test_bit(NFS_O_RDONLY_STATE, &state->flags);
         is_wronly = test_bit(NFS_O_WRONLY_STATE, &state->flags);
+        nfs4_stateid_copy(&calldata->arg.stateid, &state->stateid);
         /* Calculate the change in open mode */
         calldata->arg.fmode = 0;
         if (state->n_rdwr == 0) {
@@ -2653,6 +2722,9 @@ static void nfs4_close_prepare(struct rpc_task *task, void *data)
                         goto out_wait;
                     }
         }
+        calldata->arg.share_access =
+                nfs4_map_atomic_open_share(NFS_SERVER(inode),
+                                calldata->arg.fmode, 0);
 
         nfs_fattr_init(calldata->res.fattr);
         calldata->timestamp = jiffies;
@@ -2675,45 +2747,10 @@ static const struct rpc_call_ops nfs4_close_ops = {
         .rpc_release = nfs4_free_closedata,
 };
 
-static bool nfs4_state_has_opener(struct nfs4_state *state)
-{
-        /* first check existing openers */
-        if (test_bit(NFS_O_RDONLY_STATE, &state->flags) != 0 &&
-            state->n_rdonly != 0)
-                return true;
-
-        if (test_bit(NFS_O_WRONLY_STATE, &state->flags) != 0 &&
-            state->n_wronly != 0)
-                return true;
-
-        if (test_bit(NFS_O_RDWR_STATE, &state->flags) != 0 &&
-            state->n_rdwr != 0)
-                return true;
-
-        return false;
-}
-
 static bool nfs4_roc(struct inode *inode)
 {
-        struct nfs_inode *nfsi = NFS_I(inode);
-        struct nfs_open_context *ctx;
-        struct nfs4_state *state;
-
-        spin_lock(&inode->i_lock);
-        list_for_each_entry(ctx, &nfsi->open_files, list) {
-                state = ctx->state;
-                if (state == NULL)
-                        continue;
-                if (nfs4_state_has_opener(state)) {
-                        spin_unlock(&inode->i_lock);
-                        return false;
-                }
-        }
-        spin_unlock(&inode->i_lock);
-
-        if (nfs4_check_delegation(inode, FMODE_READ))
+        if (!nfs_have_layout(inode))
                 return false;
-
         return pnfs_roc(inode);
 }
 
@@ -2731,6 +2768,7 @@ static bool nfs4_roc(struct inode *inode)
 int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait)
 {
         struct nfs_server *server = NFS_SERVER(state->inode);
+        struct nfs_seqid *(*alloc_seqid)(struct nfs_seqid_counter *, gfp_t);
         struct nfs4_closedata *calldata;
         struct nfs4_state_owner *sp = state->owner;
         struct rpc_task *task;
@@ -2757,10 +2795,10 @@ int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait)
         calldata->inode = state->inode;
         calldata->state = state;
         calldata->arg.fh = NFS_FH(state->inode);
-        calldata->arg.stateid = &state->open_stateid;
         /* Serialization for the sequence id */
-        calldata->arg.seqid = nfs_alloc_seqid(&state->owner->so_seqid, gfp_mask);
-        if (calldata->arg.seqid == NULL)
+        alloc_seqid = server->nfs_client->cl_mvops->alloc_seqid;
+        calldata->arg.seqid = alloc_seqid(&state->owner->so_seqid, gfp_mask);
+        if (IS_ERR(calldata->arg.seqid))
                 goto out_free_calldata;
         calldata->arg.fmode = 0;
         calldata->arg.bitmask = server->cache_consistency_bitmask;
@@ -5137,9 +5175,13 @@ static void nfs4_delegreturn_done(struct rpc_task *task, void *calldata)
 static void nfs4_delegreturn_release(void *calldata)
 {
         struct nfs4_delegreturndata *data = calldata;
+        struct inode *inode = data->inode;
 
-        if (data->roc)
-                pnfs_roc_release(data->inode);
+        if (inode) {
+                if (data->roc)
+                        pnfs_roc_release(inode);
+                nfs_iput_and_deactive(inode);
+        }
         kfree(calldata);
 }
 
@@ -5196,9 +5238,9 @@ static int _nfs4_proc_delegreturn(struct inode *inode, struct rpc_cred *cred, co
         nfs_fattr_init(data->res.fattr);
         data->timestamp = jiffies;
         data->rpc_status = 0;
-        data->inode = inode;
-        data->roc = list_empty(&NFS_I(inode)->open_files) ?
-                    pnfs_roc(inode) : false;
+        data->inode = nfs_igrab_and_active(inode);
+        if (data->inode)
+                data->roc = nfs4_roc(inode);
 
         task_setup_data.callback_data = data;
         msg.rpc_argp = &data->args;
@@ -5353,7 +5395,6 @@ static struct nfs4_unlockdata *nfs4_alloc_unlockdata(struct file_lock *fl,
         p->arg.fl = &p->fl;
         p->arg.seqid = seqid;
         p->res.seqid = seqid;
-        p->arg.stateid = &lsp->ls_stateid;
         p->lsp = lsp;
         atomic_inc(&lsp->ls_count);
         /* Ensure we don't close file until we're done freeing locks! */
@@ -5380,14 +5421,18 @@ static void nfs4_locku_done(struct rpc_task *task, void *data)
                 return;
         switch (task->tk_status) {
                 case 0:
-                        nfs4_stateid_copy(&calldata->lsp->ls_stateid,
-                                        &calldata->res.stateid);
                         renew_lease(calldata->server, calldata->timestamp);
-                        break;
+                        do_vfs_lock(calldata->fl.fl_file, &calldata->fl);
+                        if (nfs4_update_lock_stateid(calldata->lsp,
+                                        &calldata->res.stateid))
+                                break;
                 case -NFS4ERR_BAD_STATEID:
                 case -NFS4ERR_OLD_STATEID:
                 case -NFS4ERR_STALE_STATEID:
                 case -NFS4ERR_EXPIRED:
+                        if (!nfs4_stateid_match(&calldata->arg.stateid,
+                                                &calldata->lsp->ls_stateid))
+                                rpc_restart_call_prepare(task);
                         break;
                 default:
                         if (nfs4_async_handle_error(task, calldata->server,
@@ -5403,6 +5448,7 @@ static void nfs4_locku_prepare(struct rpc_task *task, void *data)
 
         if (nfs_wait_on_sequence(calldata->arg.seqid, task) != 0)
                 goto out_wait;
+        nfs4_stateid_copy(&calldata->arg.stateid, &calldata->lsp->ls_stateid);
         if (test_bit(NFS_LOCK_INITIALIZED, &calldata->lsp->ls_flags) == 0) {
                 /* Note: exit _without_ running nfs4_locku_done */
                 goto out_no_action;
@@ -5473,6 +5519,7 @@ static int nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock *
         struct nfs_seqid *seqid;
         struct nfs4_lock_state *lsp;
         struct rpc_task *task;
+        struct nfs_seqid *(*alloc_seqid)(struct nfs_seqid_counter *, gfp_t);
         int status = 0;
         unsigned char fl_flags = request->fl_flags;
 
@@ -5496,9 +5543,10 @@ static int nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock *
         lsp = request->fl_u.nfs4_fl.owner;
         if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) == 0)
                 goto out;
-        seqid = nfs_alloc_seqid(&lsp->ls_seqid, GFP_KERNEL);
+        alloc_seqid = NFS_SERVER(inode)->nfs_client->cl_mvops->alloc_seqid;
+        seqid = alloc_seqid(&lsp->ls_seqid, GFP_KERNEL);
         status = -ENOMEM;
-        if (seqid == NULL)
+        if (IS_ERR(seqid))
                 goto out;
         task = nfs4_do_unlck(request, nfs_file_open_context(request->fl_file), lsp, seqid);
         status = PTR_ERR(task);
@@ -5531,6 +5579,7 @@ static struct nfs4_lockdata *nfs4_alloc_lockdata(struct file_lock *fl,
         struct nfs4_lockdata *p;
         struct inode *inode = lsp->ls_state->inode;
         struct nfs_server *server = NFS_SERVER(inode);
+        struct nfs_seqid *(*alloc_seqid)(struct nfs_seqid_counter *, gfp_t);
 
         p = kzalloc(sizeof(*p), gfp_mask);
         if (p == NULL)
@@ -5539,12 +5588,12 @@ static struct nfs4_lockdata *nfs4_alloc_lockdata(struct file_lock *fl,
         p->arg.fh = NFS_FH(inode);
         p->arg.fl = &p->fl;
         p->arg.open_seqid = nfs_alloc_seqid(&lsp->ls_state->owner->so_seqid, gfp_mask);
-        if (p->arg.open_seqid == NULL)
+        if (IS_ERR(p->arg.open_seqid))
                 goto out_free;
-        p->arg.lock_seqid = nfs_alloc_seqid(&lsp->ls_seqid, gfp_mask);
-        if (p->arg.lock_seqid == NULL)
+        alloc_seqid = server->nfs_client->cl_mvops->alloc_seqid;
+        p->arg.lock_seqid = alloc_seqid(&lsp->ls_seqid, gfp_mask);
+        if (IS_ERR(p->arg.lock_seqid))
                 goto out_free_seqid;
-        p->arg.lock_stateid = &lsp->ls_stateid;
         p->arg.lock_owner.clientid = server->nfs_client->cl_clientid;
         p->arg.lock_owner.id = lsp->ls_seqid.owner_id;
         p->arg.lock_owner.s_dev = server->s_dev;
@@ -5571,15 +5620,19 @@ static void nfs4_lock_prepare(struct rpc_task *task, void *calldata)
         if (nfs_wait_on_sequence(data->arg.lock_seqid, task) != 0)
                 goto out_wait;
         /* Do we need to do an open_to_lock_owner? */
-        if (!(data->arg.lock_seqid->sequence->flags & NFS_SEQID_CONFIRMED)) {
+        if (!test_bit(NFS_LOCK_INITIALIZED, &data->lsp->ls_flags)) {
                 if (nfs_wait_on_sequence(data->arg.open_seqid, task) != 0) {
                         goto out_release_lock_seqid;
                 }
-                data->arg.open_stateid = &state->open_stateid;
+                nfs4_stateid_copy(&data->arg.open_stateid,
+                                &state->open_stateid);
                 data->arg.new_lock_owner = 1;
                 data->res.open_seqid = data->arg.open_seqid;
-        } else
+        } else {
                 data->arg.new_lock_owner = 0;
+                nfs4_stateid_copy(&data->arg.lock_stateid,
+                                &data->lsp->ls_stateid);
+        }
         if (!nfs4_valid_open_stateid(state)) {
                 data->rpc_status = -EBADF;
                 task->tk_action = NULL;
@@ -5603,6 +5656,7 @@ out_wait:
 static void nfs4_lock_done(struct rpc_task *task, void *calldata)
 {
         struct nfs4_lockdata *data = calldata;
+        struct nfs4_lock_state *lsp = data->lsp;
 
         dprintk("%s: begin!\n", __func__);
 
@@ -5610,18 +5664,36 @@ static void nfs4_lock_done(struct rpc_task *task, void *calldata)
                 return;
 
         data->rpc_status = task->tk_status;
-        if (data->arg.new_lock_owner != 0) {
-                if (data->rpc_status == 0)
-                        nfs_confirm_seqid(&data->lsp->ls_seqid, 0);
-                else
-                        goto out;
-        }
-        if (data->rpc_status == 0) {
-                nfs4_stateid_copy(&data->lsp->ls_stateid, &data->res.stateid);
-                set_bit(NFS_LOCK_INITIALIZED, &data->lsp->ls_flags);
-                renew_lease(NFS_SERVER(data->ctx->dentry->d_inode), data->timestamp);
+        switch (task->tk_status) {
+        case 0:
+                renew_lease(NFS_SERVER(data->ctx->dentry->d_inode),
+                                data->timestamp);
+                if (data->arg.new_lock) {
+                        data->fl.fl_flags &= ~(FL_SLEEP | FL_ACCESS);
+                        if (do_vfs_lock(data->fl.fl_file, &data->fl) < 0) {
+                                rpc_restart_call_prepare(task);
+                                break;
+                        }
+                }
+                if (data->arg.new_lock_owner != 0) {
+                        nfs_confirm_seqid(&lsp->ls_seqid, 0);
+                        nfs4_stateid_copy(&lsp->ls_stateid, &data->res.stateid);
+                        set_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags);
+                } else if (!nfs4_update_lock_stateid(lsp, &data->res.stateid))
+                        rpc_restart_call_prepare(task);
+                break;
+        case -NFS4ERR_BAD_STATEID:
+        case -NFS4ERR_OLD_STATEID:
+        case -NFS4ERR_STALE_STATEID:
+        case -NFS4ERR_EXPIRED:
+                if (data->arg.new_lock_owner != 0) {
+                        if (!nfs4_stateid_match(&data->arg.open_stateid,
+                                                &lsp->ls_state->open_stateid))
+                                rpc_restart_call_prepare(task);
+                } else if (!nfs4_stateid_match(&data->arg.lock_stateid,
+                                                &lsp->ls_stateid))
+                                rpc_restart_call_prepare(task);
         }
-out:
         dprintk("%s: done, ret = %d!\n", __func__, data->rpc_status);
 }
 
@@ -5702,7 +5774,8 @@ static int _nfs4_do_setlk(struct nfs4_state *state, int cmd, struct file_lock *f
                 if (recovery_type == NFS_LOCK_RECLAIM)
                         data->arg.reclaim = NFS_LOCK_RECLAIM;
                 nfs4_set_sequence_privileged(&data->arg.seq_args);
-        }
+        } else
+                data->arg.new_lock = 1;
         task = rpc_run_task(&task_setup_data);
         if (IS_ERR(task))
                 return PTR_ERR(task);
@@ -5826,10 +5899,8 @@ static int nfs41_lock_expired(struct nfs4_state *state, struct file_lock *reques
 
 static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock *request)
 {
-        struct nfs4_state_owner *sp = state->owner;
         struct nfs_inode *nfsi = NFS_I(state->inode);
         unsigned char fl_flags = request->fl_flags;
-        unsigned int seq;
         int status = -ENOLCK;
 
         if ((fl_flags & FL_POSIX) &&
@@ -5849,25 +5920,11 @@ static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock
                 /* ...but avoid races with delegation recall... */
                 request->fl_flags = fl_flags & ~FL_SLEEP;
                 status = do_vfs_lock(request->fl_file, request);
-                goto out_unlock;
-        }
-        seq = raw_seqcount_begin(&sp->so_reclaim_seqcount);
-        up_read(&nfsi->rwsem);
-        status = _nfs4_do_setlk(state, cmd, request, NFS_LOCK_NEW);
-        if (status != 0)
+                up_read(&nfsi->rwsem);
                 goto out;
-        down_read(&nfsi->rwsem);
-        if (read_seqcount_retry(&sp->so_reclaim_seqcount, seq)) {
-                status = -NFS4ERR_DELAY;
-                goto out_unlock;
         }
-        /* Note: we always want to sleep here! */
-        request->fl_flags = fl_flags | FL_SLEEP;
-        if (do_vfs_lock(request->fl_file, request) < 0)
-                printk(KERN_WARNING "NFS: %s: VFS is out of sync with lock "
-                        "manager!\n", __func__);
-out_unlock:
         up_read(&nfsi->rwsem);
+        status = _nfs4_do_setlk(state, cmd, request, NFS_LOCK_NEW);
 out:
         request->fl_flags = fl_flags;
         return status;
@@ -5974,8 +6031,8 @@ static void nfs4_release_lockowner_prepare(struct rpc_task *task, void *calldata
 {
         struct nfs_release_lockowner_data *data = calldata;
         struct nfs_server *server = data->server;
-        nfs40_setup_sequence(server, &data->args.seq_args,
-                                &data->res.seq_res, task);
+        nfs40_setup_sequence(server->nfs_client->cl_slot_tbl,
+                             &data->args.seq_args, &data->res.seq_res, task);
         data->args.lock_owner.clientid = server->nfs_client->cl_clientid;
         data->timestamp = jiffies;
 }
@@ -7537,6 +7594,7 @@ nfs4_layoutget_prepare(struct rpc_task *task, void *calldata)
                 return;
         if (pnfs_choose_layoutget_stateid(&lgp->args.stateid,
                                           NFS_I(lgp->args.inode)->layout,
+                                          &lgp->args.range,
                                           lgp->args.ctx->state)) {
                 rpc_exit(task, NFS4_OK);
         }
@@ -7792,9 +7850,13 @@ static void nfs4_layoutreturn_release(void *calldata)
         spin_lock(&lo->plh_inode->i_lock);
         if (lrp->res.lrs_present)
                 pnfs_set_layout_stateid(lo, &lrp->res.stateid, true);
+        pnfs_clear_layoutreturn_waitbit(lo);
+        clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, &lo->plh_flags);
+        rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
         lo->plh_block_lgets--;
         spin_unlock(&lo->plh_inode->i_lock);
         pnfs_put_layout_hdr(lrp->args.layout);
+        nfs_iput_and_deactive(lrp->inode);
         kfree(calldata);
         dprintk("<-- %s\n", __func__);
 }
@@ -7805,7 +7867,7 @@ static const struct rpc_call_ops nfs4_layoutreturn_call_ops = {
         .rpc_release = nfs4_layoutreturn_release,
 };
 
-int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp)
+int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, bool sync)
 {
         struct rpc_task *task;
         struct rpc_message msg = {
@@ -7820,14 +7882,23 @@ int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp)
                 .callback_ops = &nfs4_layoutreturn_call_ops,
                 .callback_data = lrp,
         };
-        int status;
+        int status = 0;
 
         dprintk("--> %s\n", __func__);
+        if (!sync) {
+                lrp->inode = nfs_igrab_and_active(lrp->args.inode);
+                if (!lrp->inode) {
+                        nfs4_layoutreturn_release(lrp);
+                        return -EAGAIN;
+                }
+                task_setup_data.flags |= RPC_TASK_ASYNC;
+        }
         nfs4_init_sequence(&lrp->args.seq_args, &lrp->res.seq_res, 1);
         task = rpc_run_task(&task_setup_data);
         if (IS_ERR(task))
                 return PTR_ERR(task);
-        status = task->tk_status;
+        if (sync)
+                status = task->tk_status;
         trace_nfs4_layoutreturn(lrp->args.inode, status);
         dprintk("<-- %s status=%d\n", __func__, status);
         rpc_put_task(task);
@@ -7921,6 +7992,7 @@ static void nfs4_layoutcommit_release(void *calldata)
         nfs_post_op_update_inode_force_wcc(data->args.inode,
                                            data->res.fattr);
         put_rpccred(data->cred);
+        nfs_iput_and_deactive(data->inode);
         kfree(data);
 }
 
@@ -7945,7 +8017,6 @@ nfs4_proc_layoutcommit(struct nfs4_layoutcommit_data *data, bool sync)
                 .rpc_message = &msg,
                 .callback_ops = &nfs4_layoutcommit_ops,
                 .callback_data = data,
-                .flags = RPC_TASK_ASYNC,
         };
         struct rpc_task *task;
         int status = 0;
@@ -7956,18 +8027,21 @@ nfs4_proc_layoutcommit(struct nfs4_layoutcommit_data *data, bool sync)
                 data->args.lastbytewritten,
                 data->args.inode->i_ino);
 
+        if (!sync) {
+                data->inode = nfs_igrab_and_active(data->args.inode);
+                if (data->inode == NULL) {
+                        nfs4_layoutcommit_release(data);
+                        return -EAGAIN;
+                }
+                task_setup_data.flags = RPC_TASK_ASYNC;
+        }
         nfs4_init_sequence(&data->args.seq_args, &data->res.seq_res, 1);
         task = rpc_run_task(&task_setup_data);
         if (IS_ERR(task))
                 return PTR_ERR(task);
-        if (sync == false)
-                goto out;
-        status = nfs4_wait_for_completion_rpc_task(task);
-        if (status != 0)
-                goto out;
-        status = task->tk_status;
+        if (sync)
+                status = task->tk_status;
         trace_nfs4_layoutcommit(data->args.inode, status);
-out:
         dprintk("%s: status %d\n", __func__, status);
         rpc_put_task(task);
         return status;
@@ -8395,6 +8469,7 @@ static const struct nfs4_minor_version_ops nfs_v4_0_minor_ops = {
         .match_stateid = nfs4_match_stateid,
         .find_root_sec = nfs4_find_root_sec,
         .free_lock_state = nfs4_release_lockowner,
+        .alloc_seqid = nfs_alloc_seqid,
         .call_sync_ops = &nfs40_call_sync_ops,
         .reboot_recovery_ops = &nfs40_reboot_recovery_ops,
         .nograce_recovery_ops = &nfs40_nograce_recovery_ops,
@@ -8403,6 +8478,12 @@ static const struct nfs4_minor_version_ops nfs_v4_0_minor_ops = {
 };
 
 #if defined(CONFIG_NFS_V4_1)
+static struct nfs_seqid *
+nfs_alloc_no_seqid(struct nfs_seqid_counter *arg1, gfp_t arg2)
+{
+        return NULL;
+}
+
 static const struct nfs4_minor_version_ops nfs_v4_1_minor_ops = {
         .minor_version = 1,
         .init_caps = NFS_CAP_READDIRPLUS
@@ -8416,6 +8497,7 @@ static const struct nfs4_minor_version_ops nfs_v4_1_minor_ops = {
         .match_stateid = nfs41_match_stateid,
         .find_root_sec = nfs41_find_root_sec,
         .free_lock_state = nfs41_free_lock_state,
+        .alloc_seqid = nfs_alloc_no_seqid,
         .call_sync_ops = &nfs41_call_sync_ops,
         .reboot_recovery_ops = &nfs41_reboot_recovery_ops,
         .nograce_recovery_ops = &nfs41_nograce_recovery_ops,
@@ -8442,6 +8524,7 @@ static const struct nfs4_minor_version_ops nfs_v4_2_minor_ops = {
         .find_root_sec = nfs41_find_root_sec,
         .free_lock_state = nfs41_free_lock_state,
         .call_sync_ops = &nfs41_call_sync_ops,
+        .alloc_seqid = nfs_alloc_no_seqid,
         .reboot_recovery_ops = &nfs41_reboot_recovery_ops,
         .nograce_recovery_ops = &nfs41_nograce_recovery_ops,
         .state_renewal_ops = &nfs41_state_renewal_ops,
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index a3bb22ab68c5..5ad908e9ce9c 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -1003,11 +1003,11 @@ struct nfs_seqid *nfs_alloc_seqid(struct nfs_seqid_counter *counter, gfp_t gfp_m
         struct nfs_seqid *new;
 
         new = kmalloc(sizeof(*new), gfp_mask);
-        if (new != NULL) {
-                new->sequence = counter;
-                INIT_LIST_HEAD(&new->list);
-                new->task = NULL;
-        }
+        if (new == NULL)
+                return ERR_PTR(-ENOMEM);
+        new->sequence = counter;
+        INIT_LIST_HEAD(&new->list);
+        new->task = NULL;
         return new;
 }
 
@@ -1015,7 +1015,7 @@ void nfs_release_seqid(struct nfs_seqid *seqid)
 {
         struct nfs_seqid_counter *sequence;
 
-        if (list_empty(&seqid->list))
+        if (seqid == NULL || list_empty(&seqid->list))
                 return;
         sequence = seqid->sequence;
         spin_lock(&sequence->lock);
@@ -1071,13 +1071,15 @@ static void nfs_increment_seqid(int status, struct nfs_seqid *seqid)
 
 void nfs_increment_open_seqid(int status, struct nfs_seqid *seqid)
 {
-        struct nfs4_state_owner *sp = container_of(seqid->sequence,
-                                        struct nfs4_state_owner, so_seqid);
-        struct nfs_server *server = sp->so_server;
+        struct nfs4_state_owner *sp;
+
+        if (seqid == NULL)
+                return;
 
+        sp = container_of(seqid->sequence, struct nfs4_state_owner, so_seqid);
         if (status == -NFS4ERR_BAD_SEQID)
                 nfs4_drop_state_owner(sp);
-        if (!nfs4_has_session(server->nfs_client))
+        if (!nfs4_has_session(sp->so_server->nfs_client))
                 nfs_increment_seqid(status, seqid);
 }
 
@@ -1088,14 +1090,18 @@ void nfs_increment_open_seqid(int status, struct nfs_seqid *seqid)
  */
 void nfs_increment_lock_seqid(int status, struct nfs_seqid *seqid)
 {
-        nfs_increment_seqid(status, seqid);
+        if (seqid != NULL)
+                nfs_increment_seqid(status, seqid);
 }
 
 int nfs_wait_on_sequence(struct nfs_seqid *seqid, struct rpc_task *task)
 {
-        struct nfs_seqid_counter *sequence = seqid->sequence;
+        struct nfs_seqid_counter *sequence;
         int status = 0;
 
+        if (seqid == NULL)
+                goto out;
+        sequence = seqid->sequence;
         spin_lock(&sequence->lock);
         seqid->task = task;
         if (list_empty(&seqid->list))
@@ -1106,6 +1112,7 @@ int nfs_wait_on_sequence(struct nfs_seqid *seqid, struct rpc_task *task)
         status = -EAGAIN;
 unlock:
         spin_unlock(&sequence->lock);
+out:
         return status;
 }
 
diff --git a/fs/nfs/nfs4super.c b/fs/nfs/nfs4super.c
index 6f340f02f2ba..48cea3c30e5d 100644
--- a/fs/nfs/nfs4super.c
+++ b/fs/nfs/nfs4super.c
@@ -346,6 +346,9 @@ out:
 
 static void __exit exit_nfs_v4(void)
 {
+        /* Not called in the _init(), conditionally loaded */
+        nfs4_pnfs_v3_ds_connect_unload();
+
         unregister_nfs_version(&nfs_v4);
         nfs4_unregister_sysctl();
         nfs_idmap_quit();
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index cb4376b78ed9..e23a0a664e12 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -946,7 +946,10 @@ static void encode_uint64(struct xdr_stream *xdr, u64 n)
 static void encode_nfs4_seqid(struct xdr_stream *xdr,
                 const struct nfs_seqid *seqid)
 {
-        encode_uint32(xdr, seqid->sequence->counter);
+        if (seqid != NULL)
+                encode_uint32(xdr, seqid->sequence->counter);
+        else
+                encode_uint32(xdr, 0);
 }
 
 static void encode_compound_hdr(struct xdr_stream *xdr,
@@ -1125,7 +1128,7 @@ static void encode_close(struct xdr_stream *xdr, const struct nfs_closeargs *arg
 {
         encode_op_hdr(xdr, OP_CLOSE, decode_close_maxsz, hdr);
         encode_nfs4_seqid(xdr, arg->seqid);
-        encode_nfs4_stateid(xdr, arg->stateid);
+        encode_nfs4_stateid(xdr, &arg->stateid);
 }
 
 static void encode_commit(struct xdr_stream *xdr, const struct nfs_commitargs *args, struct compound_hdr *hdr)
@@ -1301,12 +1304,12 @@ static void encode_lock(struct xdr_stream *xdr, const struct nfs_lock_args *args
         *p = cpu_to_be32(args->new_lock_owner);
         if (args->new_lock_owner){
                 encode_nfs4_seqid(xdr, args->open_seqid);
-                encode_nfs4_stateid(xdr, args->open_stateid);
+                encode_nfs4_stateid(xdr, &args->open_stateid);
                 encode_nfs4_seqid(xdr, args->lock_seqid);
                 encode_lockowner(xdr, &args->lock_owner);
         }
         else {
-                encode_nfs4_stateid(xdr, args->lock_stateid);
+                encode_nfs4_stateid(xdr, &args->lock_stateid);
                 encode_nfs4_seqid(xdr, args->lock_seqid);
         }
 }
@@ -1330,7 +1333,7 @@ static void encode_locku(struct xdr_stream *xdr, const struct nfs_locku_args *ar
         encode_op_hdr(xdr, OP_LOCKU, decode_locku_maxsz, hdr);
         encode_uint32(xdr, nfs4_lock_type(args->fl, 0));
         encode_nfs4_seqid(xdr, args->seqid);
-        encode_nfs4_stateid(xdr, args->stateid);
+        encode_nfs4_stateid(xdr, &args->stateid);
         p = reserve_space(xdr, 16);
         p = xdr_encode_hyper(p, args->fl->fl_start);
         xdr_encode_hyper(p, nfs4_lock_length(args->fl));
@@ -1348,24 +1351,12 @@ static void encode_lookup(struct xdr_stream *xdr, const struct qstr *name, struc
         encode_string(xdr, name->len, name->name);
 }
 
-static void encode_share_access(struct xdr_stream *xdr, fmode_t fmode)
+static void encode_share_access(struct xdr_stream *xdr, u32 share_access)
 {
         __be32 *p;
 
         p = reserve_space(xdr, 8);
-        switch (fmode & (FMODE_READ|FMODE_WRITE)) {
-        case FMODE_READ:
-                *p++ = cpu_to_be32(NFS4_SHARE_ACCESS_READ);
-                break;
-        case FMODE_WRITE:
-                *p++ = cpu_to_be32(NFS4_SHARE_ACCESS_WRITE);
-                break;
-        case FMODE_READ|FMODE_WRITE:
-                *p++ = cpu_to_be32(NFS4_SHARE_ACCESS_BOTH);
-                break;
-        default:
-                *p++ = cpu_to_be32(0);
-        }
+        *p++ = cpu_to_be32(share_access);
         *p = cpu_to_be32(0);            /* for linux, share_deny = 0 always */
 }
 
@@ -1377,7 +1368,7 @@ static inline void encode_openhdr(struct xdr_stream *xdr, const struct nfs_opena
  * owner 4 = 32
  */
         encode_nfs4_seqid(xdr, arg->seqid);
-        encode_share_access(xdr, arg->fmode);
+        encode_share_access(xdr, arg->share_access);
         p = reserve_space(xdr, 36);
         p = xdr_encode_hyper(p, arg->clientid);
         *p++ = cpu_to_be32(24);
@@ -1530,9 +1521,9 @@ static void encode_open_confirm(struct xdr_stream *xdr, const struct nfs_open_co
 static void encode_open_downgrade(struct xdr_stream *xdr, const struct nfs_closeargs *arg, struct compound_hdr *hdr)
 {
         encode_op_hdr(xdr, OP_OPEN_DOWNGRADE, decode_open_downgrade_maxsz, hdr);
-        encode_nfs4_stateid(xdr, arg->stateid);
+        encode_nfs4_stateid(xdr, &arg->stateid);
         encode_nfs4_seqid(xdr, arg->seqid);
-        encode_share_access(xdr, arg->fmode);
+        encode_share_access(xdr, arg->share_access);
 }
 
 static void
@@ -1801,9 +1792,8 @@ static void encode_create_session(struct xdr_stream *xdr,
                                   struct compound_hdr *hdr)
 {
         __be32 *p;
-        char machine_name[NFS4_MAX_MACHINE_NAME_LEN];
-        uint32_t len;
         struct nfs_client *clp = args->client;
+        struct rpc_clnt *clnt = clp->cl_rpcclient;
         struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id);
         u32 max_resp_sz_cached;
 
@@ -1814,11 +1804,8 @@ static void encode_create_session(struct xdr_stream *xdr,
         max_resp_sz_cached = (NFS4_dec_open_sz + RPC_REPHDRSIZE +
                               RPC_MAX_AUTH_SIZE + 2) * XDR_UNIT;
 
-        len = scnprintf(machine_name, sizeof(machine_name), "%s",
-                        clp->cl_ipaddr);
-
         encode_op_hdr(xdr, OP_CREATE_SESSION, decode_create_session_maxsz, hdr);
-        p = reserve_space(xdr, 16 + 2*28 + 20 + len + 12);
+        p = reserve_space(xdr, 16 + 2*28 + 20 + clnt->cl_nodelen + 12);
         p = xdr_encode_hyper(p, clp->cl_clientid);
         *p++ = cpu_to_be32(clp->cl_seqid);                      /*Sequence id */
         *p++ = cpu_to_be32(args->flags);                        /*flags */
@@ -1847,7 +1834,7 @@ static void encode_create_session(struct xdr_stream *xdr,
 
         /* authsys_parms rfc1831 */
         *p++ = cpu_to_be32(nn->boot_time.tv_nsec);      /* stamp */
-        p = xdr_encode_opaque(p, machine_name, len);
+        p = xdr_encode_array(p, clnt->cl_nodename, clnt->cl_nodelen);
         *p++ = cpu_to_be32(0);                          /* UID */
         *p++ = cpu_to_be32(0);                          /* GID */
         *p = cpu_to_be32(0);                            /* No more gids */
@@ -2012,11 +1999,11 @@ encode_layoutreturn(struct xdr_stream *xdr,
         p = reserve_space(xdr, 16);
         *p++ = cpu_to_be32(0);          /* reclaim. always 0 for now */
         *p++ = cpu_to_be32(args->layout_type);
-        *p++ = cpu_to_be32(IOMODE_ANY);
+        *p++ = cpu_to_be32(args->range.iomode);
         *p = cpu_to_be32(RETURN_FILE);
         p = reserve_space(xdr, 16);
-        p = xdr_encode_hyper(p, 0);
-        p = xdr_encode_hyper(p, NFS4_MAX_UINT64);
+        p = xdr_encode_hyper(p, args->range.offset);
+        p = xdr_encode_hyper(p, args->range.length);
         spin_lock(&args->inode->i_lock);
         encode_nfs4_stateid(xdr, &args->stateid);
         spin_unlock(&args->inode->i_lock);
@@ -4936,20 +4923,13 @@ out_overflow:
         return -EIO;
 }
 
-static int decode_delegation(struct xdr_stream *xdr, struct nfs_openres *res)
+static int decode_rw_delegation(struct xdr_stream *xdr,
+                uint32_t delegation_type,
+                struct nfs_openres *res)
 {
         __be32 *p;
-        uint32_t delegation_type;
         int status;
 
-        p = xdr_inline_decode(xdr, 4);
-        if (unlikely(!p))
-                goto out_overflow;
-        delegation_type = be32_to_cpup(p);
-        if (delegation_type == NFS4_OPEN_DELEGATE_NONE) {
-                res->delegation_type = 0;
-                return 0;
-        }
         status = decode_stateid(xdr, &res->delegation);
         if (unlikely(status))
                 return status;
@@ -4973,6 +4953,52 @@ out_overflow:
         return -EIO;
 }
 
+static int decode_no_delegation(struct xdr_stream *xdr, struct nfs_openres *res)
+{
+        __be32 *p;
+        uint32_t why_no_delegation;
+
+        p = xdr_inline_decode(xdr, 4);
+        if (unlikely(!p))
+                goto out_overflow;
+        why_no_delegation = be32_to_cpup(p);
+        switch (why_no_delegation) {
+                case WND4_CONTENTION:
+                case WND4_RESOURCE:
+                        xdr_inline_decode(xdr, 4);
+                        /* Ignore for now */
+        }
+        return 0;
+out_overflow:
+        print_overflow_msg(__func__, xdr);
+        return -EIO;
+}
+
+static int decode_delegation(struct xdr_stream *xdr, struct nfs_openres *res)
+{
+        __be32 *p;
+        uint32_t delegation_type;
+
+        p = xdr_inline_decode(xdr, 4);
+        if (unlikely(!p))
+                goto out_overflow;
+        delegation_type = be32_to_cpup(p);
+        res->delegation_type = 0;
+        switch (delegation_type) {
+        case NFS4_OPEN_DELEGATE_NONE:
+                return 0;
+        case NFS4_OPEN_DELEGATE_READ:
+        case NFS4_OPEN_DELEGATE_WRITE:
+                return decode_rw_delegation(xdr, delegation_type, res);
+        case NFS4_OPEN_DELEGATE_NONE_EXT:
+                return decode_no_delegation(xdr, res);
+        }
+        return -EIO;
+out_overflow:
+        print_overflow_msg(__func__, xdr);
+        return -EIO;
+}
+
 static int decode_open(struct xdr_stream *xdr, struct nfs_openres *res)
 {
         __be32 *p;
@@ -6567,6 +6593,7 @@ static int nfs4_xdr_dec_read(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
         int status;
 
         status = decode_compound_hdr(xdr, &hdr);
+        res->op_status = hdr.status;
         if (status)
                 goto out;
         status = decode_sequence(xdr, &res->seq_res, rqstp);
@@ -6592,6 +6619,7 @@ static int nfs4_xdr_dec_write(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
         int status;
 
         status = decode_compound_hdr(xdr, &hdr);
+        res->op_status = hdr.status;
         if (status)
                 goto out;
         status = decode_sequence(xdr, &res->seq_res, rqstp);
@@ -6621,6 +6649,7 @@ static int nfs4_xdr_dec_commit(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
         int status;
 
         status = decode_compound_hdr(xdr, &hdr);
+        res->op_status = hdr.status;
         if (status)
                 goto out;
         status = decode_sequence(xdr, &res->seq_res, rqstp);
diff --git a/fs/nfs/nfsroot.c b/fs/nfs/nfsroot.c
index cd3c910d2d12..9bc9f04fb7f6 100644
--- a/fs/nfs/nfsroot.c
+++ b/fs/nfs/nfsroot.c
@@ -261,11 +261,11 @@ static int __init root_nfs_data(char *cmdline)
          */
         len = snprintf(nfs_export_path, sizeof(nfs_export_path),
                                 tmp, utsname()->nodename);
-        if (len > (int)sizeof(nfs_export_path))
+        if (len >= (int)sizeof(nfs_export_path))
                 goto out_devnametoolong;
         len = snprintf(nfs_root_device, sizeof(nfs_root_device),
                                 "%pI4:%s", &servaddr, nfs_export_path);
-        if (len > (int)sizeof(nfs_root_device))
+        if (len >= (int)sizeof(nfs_root_device))
                 goto out_devnametoolong;
 
         retval = 0;
diff --git a/fs/nfs/objlayout/objio_osd.c b/fs/nfs/objlayout/objio_osd.c
index 9e5bc42180e4..24e1d7403c0b 100644
--- a/fs/nfs/objlayout/objio_osd.c
+++ b/fs/nfs/objlayout/objio_osd.c
@@ -537,11 +537,12 @@ int objio_write_pagelist(struct nfs_pgio_header *hdr, int how)
 static size_t objio_pg_test(struct nfs_pageio_descriptor *pgio,
                           struct nfs_page *prev, struct nfs_page *req)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(pgio);
         unsigned int size;
 
         size = pnfs_generic_pg_test(pgio, prev, req);
 
-        if (!size || pgio->pg_count + req->wb_bytes >
+        if (!size || mirror->pg_count + req->wb_bytes >
             (unsigned long)pgio->pg_layout_private)
                 return 0;
 
@@ -607,12 +608,14 @@ static const struct nfs_pageio_ops objio_pg_read_ops = {
         .pg_init = objio_init_read,
         .pg_test = objio_pg_test,
         .pg_doio = pnfs_generic_pg_readpages,
+        .pg_cleanup = pnfs_generic_pg_cleanup,
 };
 
 static const struct nfs_pageio_ops objio_pg_write_ops = {
         .pg_init = objio_init_write,
         .pg_test = objio_pg_test,
         .pg_doio = pnfs_generic_pg_writepages,
+        .pg_cleanup = pnfs_generic_pg_cleanup,
 };
 
 static struct pnfs_layoutdriver_type objlayout_type = {
diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c
index 29c7f33c9cf1..d57190a0d533 100644
--- a/fs/nfs/pagelist.c
+++ b/fs/nfs/pagelist.c
@@ -42,21 +42,35 @@ static bool nfs_pgarray_set(struct nfs_page_array *p, unsigned int pagecount)
         return p->pagevec != NULL;
 }
 
+struct nfs_pgio_mirror *
+nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc)
+{
+        return nfs_pgio_has_mirroring(desc) ?
+                &desc->pg_mirrors[desc->pg_mirror_idx] :
+                &desc->pg_mirrors[0];
+}
+EXPORT_SYMBOL_GPL(nfs_pgio_current_mirror);
+
 void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
                        struct nfs_pgio_header *hdr,
                        void (*release)(struct nfs_pgio_header *hdr))
 {
-        hdr->req = nfs_list_entry(desc->pg_list.next);
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
+
+        hdr->req = nfs_list_entry(mirror->pg_list.next);
         hdr->inode = desc->pg_inode;
         hdr->cred = hdr->req->wb_context->cred;
         hdr->io_start = req_offset(hdr->req);
-        hdr->good_bytes = desc->pg_count;
+        hdr->good_bytes = mirror->pg_count;
         hdr->dreq = desc->pg_dreq;
         hdr->layout_private = desc->pg_layout_private;
         hdr->release = release;
         hdr->completion_ops = desc->pg_completion_ops;
         if (hdr->completion_ops->init_hdr)
                 hdr->completion_ops->init_hdr(hdr);
+
+        hdr->pgio_mirror_idx = desc->pg_mirror_idx;
 }
 EXPORT_SYMBOL_GPL(nfs_pgheader_init);
 
@@ -480,7 +494,10 @@ nfs_wait_on_request(struct nfs_page *req)
 size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
                            struct nfs_page *prev, struct nfs_page *req)
 {
-        if (desc->pg_count > desc->pg_bsize) {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
+
+        if (mirror->pg_count > mirror->pg_bsize) {
                 /* should never happen */
                 WARN_ON_ONCE(1);
                 return 0;
@@ -490,11 +507,11 @@ size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
          * Limit the request size so that we can still allocate a page array
          * for it without upsetting the slab allocator.
          */
-        if (((desc->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
+        if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
                         sizeof(struct page) > PAGE_SIZE)
                 return 0;
 
-        return min(desc->pg_bsize - desc->pg_count, (size_t)req->wb_bytes);
+        return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);
 }
 EXPORT_SYMBOL_GPL(nfs_generic_pg_test);
 
@@ -597,13 +614,14 @@ static void nfs_pgio_prepare(struct rpc_task *task, void *calldata)
 }
 
 int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
+                      struct rpc_cred *cred, const struct nfs_rpc_ops *rpc_ops,
                       const struct rpc_call_ops *call_ops, int how, int flags)
 {
         struct rpc_task *task;
         struct rpc_message msg = {
                 .rpc_argp = &hdr->args,
                 .rpc_resp = &hdr->res,
-                .rpc_cred = hdr->cred,
+                .rpc_cred = cred,
         };
         struct rpc_task_setup task_setup_data = {
                 .rpc_client = clnt,
@@ -616,7 +634,7 @@ int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
         };
         int ret = 0;
 
-        hdr->rw_ops->rw_initiate(hdr, &msg, &task_setup_data, how);
+        hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);
 
         dprintk("NFS: %5u initiated pgio call "
                 "(req %s/%llu, %u bytes @ offset %llu)\n",
@@ -650,10 +668,18 @@ EXPORT_SYMBOL_GPL(nfs_initiate_pgio);
 static int nfs_pgio_error(struct nfs_pageio_descriptor *desc,
                           struct nfs_pgio_header *hdr)
 {
+        struct nfs_pgio_mirror *mirror;
+        u32 midx;
+
         set_bit(NFS_IOHDR_REDO, &hdr->flags);
         nfs_pgio_data_destroy(hdr);
         hdr->completion_ops->completion(hdr);
-        desc->pg_completion_ops->error_cleanup(&desc->pg_list);
+        /* TODO: Make sure it's right to clean up all mirrors here
+         *       and not just hdr->pgio_mirror_idx */
+        for (midx = 0; midx < desc->pg_mirror_count; midx++) {
+                mirror = &desc->pg_mirrors[midx];
+                desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
+        }
         return -ENOMEM;
 }
 
@@ -670,6 +696,17 @@ static void nfs_pgio_release(void *calldata)
         hdr->completion_ops->completion(hdr);
 }
 
+static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,
+                                   unsigned int bsize)
+{
+        INIT_LIST_HEAD(&mirror->pg_list);
+        mirror->pg_bytes_written = 0;
+        mirror->pg_count = 0;
+        mirror->pg_bsize = bsize;
+        mirror->pg_base = 0;
+        mirror->pg_recoalesce = 0;
+}
+
 /**
  * nfs_pageio_init - initialise a page io descriptor
  * @desc: pointer to descriptor
@@ -686,13 +723,10 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
                      size_t bsize,
                      int io_flags)
 {
-        INIT_LIST_HEAD(&desc->pg_list);
-        desc->pg_bytes_written = 0;
-        desc->pg_count = 0;
-        desc->pg_bsize = bsize;
-        desc->pg_base = 0;
+        struct nfs_pgio_mirror *new;
+        int i;
+
         desc->pg_moreio = 0;
-        desc->pg_recoalesce = 0;
         desc->pg_inode = inode;
         desc->pg_ops = pg_ops;
         desc->pg_completion_ops = compl_ops;
@@ -702,6 +736,26 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
         desc->pg_lseg = NULL;
         desc->pg_dreq = NULL;
         desc->pg_layout_private = NULL;
+        desc->pg_bsize = bsize;
+
+        desc->pg_mirror_count = 1;
+        desc->pg_mirror_idx = 0;
+
+        if (pg_ops->pg_get_mirror_count) {
+                /* until we have a request, we don't have an lseg and no
+                 * idea how many mirrors there will be */
+                new = kcalloc(NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX,
+                              sizeof(struct nfs_pgio_mirror), GFP_KERNEL);
+                desc->pg_mirrors_dynamic = new;
+                desc->pg_mirrors = new;
+
+                for (i = 0; i < NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX; i++)
+                        nfs_pageio_mirror_init(&desc->pg_mirrors[i], bsize);
+        } else {
+                desc->pg_mirrors_dynamic = NULL;
+                desc->pg_mirrors = desc->pg_mirrors_static;
+                nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);
+        }
 }
 EXPORT_SYMBOL_GPL(nfs_pageio_init);
 
@@ -737,14 +791,16 @@ static void nfs_pgio_result(struct rpc_task *task, void *calldata)
 int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
                      struct nfs_pgio_header *hdr)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
         struct nfs_page         *req;
         struct page             **pages,
                                 *last_page;
-        struct list_head *head = &desc->pg_list;
+        struct list_head *head = &mirror->pg_list;
         struct nfs_commit_info cinfo;
         unsigned int pagecount, pageused;
 
-        pagecount = nfs_page_array_len(desc->pg_base, desc->pg_count);
+        pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);
         if (!nfs_pgarray_set(&hdr->page_array, pagecount))
                 return nfs_pgio_error(desc, hdr);
 
@@ -772,7 +828,7 @@ int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
                 desc->pg_ioflags &= ~FLUSH_COND_STABLE;
 
         /* Set up the argument struct */
-        nfs_pgio_rpcsetup(hdr, desc->pg_count, 0, desc->pg_ioflags, &cinfo);
+        nfs_pgio_rpcsetup(hdr, mirror->pg_count, 0, desc->pg_ioflags, &cinfo);
         desc->pg_rpc_callops = &nfs_pgio_common_ops;
         return 0;
 }
@@ -780,23 +836,74 @@ EXPORT_SYMBOL_GPL(nfs_generic_pgio);
 
 static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)
 {
+        struct nfs_pgio_mirror *mirror;
         struct nfs_pgio_header *hdr;
         int ret;
 
+        mirror = nfs_pgio_current_mirror(desc);
+
         hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
         if (!hdr) {
-                desc->pg_completion_ops->error_cleanup(&desc->pg_list);
+                /* TODO: make sure this is right with mirroring - or
+                 *       should it back out all mirrors? */
+                desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
                 return -ENOMEM;
         }
         nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);
         ret = nfs_generic_pgio(desc, hdr);
         if (ret == 0)
                 ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode),
-                                        hdr, desc->pg_rpc_callops,
+                                        hdr,
+                                        hdr->cred,
+                                        NFS_PROTO(hdr->inode),
+                                        desc->pg_rpc_callops,
                                         desc->pg_ioflags, 0);
         return ret;
 }
 
+/*
+ * nfs_pageio_setup_mirroring - determine if mirroring is to be used
+ *                              by calling the pg_get_mirror_count op
+ */
+static int nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio,
+                                       struct nfs_page *req)
+{
+        int mirror_count = 1;
+
+        if (!pgio->pg_ops->pg_get_mirror_count)
+                return 0;
+
+        mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
+
+        if (!mirror_count || mirror_count > NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX)
+                return -EINVAL;
+
+        if (WARN_ON_ONCE(!pgio->pg_mirrors_dynamic))
+                return -EINVAL;
+
+        pgio->pg_mirror_count = mirror_count;
+
+        return 0;
+}
+
+/*
+ * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
+ */
+void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio)
+{
+        pgio->pg_mirror_count = 1;
+        pgio->pg_mirror_idx = 0;
+}
+
+static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio)
+{
+        pgio->pg_mirror_count = 1;
+        pgio->pg_mirror_idx = 0;
+        pgio->pg_mirrors = pgio->pg_mirrors_static;
+        kfree(pgio->pg_mirrors_dynamic);
+        pgio->pg_mirrors_dynamic = NULL;
+}
+
 static bool nfs_match_open_context(const struct nfs_open_context *ctx1,
                 const struct nfs_open_context *ctx2)
 {
@@ -867,19 +974,22 @@ static bool nfs_can_coalesce_requests(struct nfs_page *prev,
 static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
                                      struct nfs_page *req)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
         struct nfs_page *prev = NULL;
-        if (desc->pg_count != 0) {
-                prev = nfs_list_entry(desc->pg_list.prev);
+
+        if (mirror->pg_count != 0) {
+                prev = nfs_list_entry(mirror->pg_list.prev);
         } else {
                 if (desc->pg_ops->pg_init)
                         desc->pg_ops->pg_init(desc, req);
-                desc->pg_base = req->wb_pgbase;
+                mirror->pg_base = req->wb_pgbase;
         }
         if (!nfs_can_coalesce_requests(prev, req, desc))
                 return 0;
         nfs_list_remove_request(req);
-        nfs_list_add_request(req, &desc->pg_list);
-        desc->pg_count += req->wb_bytes;
+        nfs_list_add_request(req, &mirror->pg_list);
+        mirror->pg_count += req->wb_bytes;
         return 1;
 }
 
@@ -888,16 +998,19 @@ static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
  */
 static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
 {
-        if (!list_empty(&desc->pg_list)) {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
+
+        if (!list_empty(&mirror->pg_list)) {
                 int error = desc->pg_ops->pg_doio(desc);
                 if (error < 0)
                         desc->pg_error = error;
                 else
-                        desc->pg_bytes_written += desc->pg_count;
+                        mirror->pg_bytes_written += mirror->pg_count;
         }
-        if (list_empty(&desc->pg_list)) {
-                desc->pg_count = 0;
-                desc->pg_base = 0;
+        if (list_empty(&mirror->pg_list)) {
+                mirror->pg_count = 0;
+                mirror->pg_base = 0;
         }
 }
 
@@ -915,6 +1028,8 @@ static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
 static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
                            struct nfs_page *req)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
         struct nfs_page *subreq;
         unsigned int bytes_left = 0;
         unsigned int offset, pgbase;
@@ -938,7 +1053,7 @@ static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
                         nfs_pageio_doio(desc);
                         if (desc->pg_error < 0)
                                 return 0;
-                        if (desc->pg_recoalesce)
+                        if (mirror->pg_recoalesce)
                                 return 0;
                         /* retry add_request for this subreq */
                         nfs_page_group_lock(req, false);
@@ -976,14 +1091,16 @@ err_ptr:
 
 static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
         LIST_HEAD(head);
 
         do {
-                list_splice_init(&desc->pg_list, &head);
-                desc->pg_bytes_written -= desc->pg_count;
-                desc->pg_count = 0;
-                desc->pg_base = 0;
-                desc->pg_recoalesce = 0;
+                list_splice_init(&mirror->pg_list, &head);
+                mirror->pg_bytes_written -= mirror->pg_count;
+                mirror->pg_count = 0;
+                mirror->pg_base = 0;
+                mirror->pg_recoalesce = 0;
+
                 desc->pg_moreio = 0;
 
                 while (!list_empty(&head)) {
@@ -997,11 +1114,11 @@ static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
                                 return 0;
                         break;
                 }
-        } while (desc->pg_recoalesce);
+        } while (mirror->pg_recoalesce);
         return 1;
 }
 
-int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
+static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc,
                 struct nfs_page *req)
 {
         int ret;
@@ -1014,9 +1131,80 @@ int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
                         break;
                 ret = nfs_do_recoalesce(desc);
         } while (ret);
+
         return ret;
 }
 
+int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
+                           struct nfs_page *req)
+{
+        u32 midx;
+        unsigned int pgbase, offset, bytes;
+        struct nfs_page *dupreq, *lastreq;
+
+        pgbase = req->wb_pgbase;
+        offset = req->wb_offset;
+        bytes = req->wb_bytes;
+
+        nfs_pageio_setup_mirroring(desc, req);
+
+        for (midx = 0; midx < desc->pg_mirror_count; midx++) {
+                if (midx) {
+                        nfs_page_group_lock(req, false);
+
+                        /* find the last request */
+                        for (lastreq = req->wb_head;
+                             lastreq->wb_this_page != req->wb_head;
+                             lastreq = lastreq->wb_this_page)
+                                ;
+
+                        dupreq = nfs_create_request(req->wb_context,
+                                        req->wb_page, lastreq, pgbase, bytes);
+
+                        if (IS_ERR(dupreq)) {
+                                nfs_page_group_unlock(req);
+                                return 0;
+                        }
+
+                        nfs_lock_request(dupreq);
+                        nfs_page_group_unlock(req);
+                        dupreq->wb_offset = offset;
+                        dupreq->wb_index = req->wb_index;
+                } else
+                        dupreq = req;
+
+                if (nfs_pgio_has_mirroring(desc))
+                        desc->pg_mirror_idx = midx;
+                if (!nfs_pageio_add_request_mirror(desc, dupreq))
+                        return 0;
+        }
+
+        return 1;
+}
+
+/*
+ * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
+ *                              nfs_pageio_descriptor
+ * @desc: pointer to io descriptor
+ */
+static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,
+                                       u32 mirror_idx)
+{
+        struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[mirror_idx];
+        u32 restore_idx = desc->pg_mirror_idx;
+
+        if (nfs_pgio_has_mirroring(desc))
+                desc->pg_mirror_idx = mirror_idx;
+        for (;;) {
+                nfs_pageio_doio(desc);
+                if (!mirror->pg_recoalesce)
+                        break;
+                if (!nfs_do_recoalesce(desc))
+                        break;
+        }
+        desc->pg_mirror_idx = restore_idx;
+}
+
 /*
  * nfs_pageio_resend - Transfer requests to new descriptor and resend
  * @hdr - the pgio header to move request from
@@ -1050,18 +1238,19 @@ int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,
 EXPORT_SYMBOL_GPL(nfs_pageio_resend);
 
 /**
- * nfs_pageio_complete - Complete I/O on an nfs_pageio_descriptor
+ * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
  * @desc: pointer to io descriptor
  */
 void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
 {
-        for (;;) {
-                nfs_pageio_doio(desc);
-                if (!desc->pg_recoalesce)
-                        break;
-                if (!nfs_do_recoalesce(desc))
-                        break;
-        }
+        u32 midx;
+
+        for (midx = 0; midx < desc->pg_mirror_count; midx++)
+                nfs_pageio_complete_mirror(desc, midx);
+
+        if (desc->pg_ops->pg_cleanup)
+                desc->pg_ops->pg_cleanup(desc);
+        nfs_pageio_cleanup_mirroring(desc);
 }
 
 /**
@@ -1077,10 +1266,17 @@ void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
  */
 void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
 {
-        if (!list_empty(&desc->pg_list)) {
-                struct nfs_page *prev = nfs_list_entry(desc->pg_list.prev);
-                if (index != prev->wb_index + 1)
-                        nfs_pageio_complete(desc);
+        struct nfs_pgio_mirror *mirror;
+        struct nfs_page *prev;
+        u32 midx;
+
+        for (midx = 0; midx < desc->pg_mirror_count; midx++) {
+                mirror = &desc->pg_mirrors[midx];
+                if (!list_empty(&mirror->pg_list)) {
+                        prev = nfs_list_entry(mirror->pg_list.prev);
+                        if (index != prev->wb_index + 1)
+                                nfs_pageio_complete_mirror(desc, midx);
+                }
         }
 }
 
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 0a5dda4d85c2..4f802b02fbb9 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -34,6 +34,7 @@
 #include "pnfs.h"
 #include "iostat.h"
 #include "nfs4trace.h"
+#include "delegation.h"
 
 #define NFSDBG_FACILITY         NFSDBG_PNFS
 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
@@ -50,6 +51,10 @@ static DEFINE_SPINLOCK(pnfs_spinlock);
  */
 static LIST_HEAD(pnfs_modules_tbl);
 
+static int
+pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid,
+                       enum pnfs_iomode iomode, bool sync);
+
 /* Return the registered pnfs layout driver module matching given id */
 static struct pnfs_layoutdriver_type *
 find_pnfs_driver_locked(u32 id)
@@ -238,6 +243,8 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
         struct inode *inode = lo->plh_inode;
 
         if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
+                if (!list_empty(&lo->plh_segs))
+                        WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
                 pnfs_detach_layout_hdr(lo);
                 spin_unlock(&inode->i_lock);
                 pnfs_free_layout_hdr(lo);
@@ -337,6 +344,48 @@ pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
         rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
 }
 
+/* Return true if layoutreturn is needed */
+static bool
+pnfs_layout_need_return(struct pnfs_layout_hdr *lo,
+                        struct pnfs_layout_segment *lseg)
+{
+        struct pnfs_layout_segment *s;
+
+        if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
+                return false;
+
+        list_for_each_entry(s, &lo->plh_segs, pls_list)
+                if (s != lseg && test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
+                        return false;
+
+        return true;
+}
+
+static void pnfs_layoutreturn_before_put_lseg(struct pnfs_layout_segment *lseg,
+                struct pnfs_layout_hdr *lo, struct inode *inode)
+{
+        lo = lseg->pls_layout;
+        inode = lo->plh_inode;
+
+        spin_lock(&inode->i_lock);
+        if (pnfs_layout_need_return(lo, lseg)) {
+                nfs4_stateid stateid;
+                enum pnfs_iomode iomode;
+
+                stateid = lo->plh_stateid;
+                iomode = lo->plh_return_iomode;
+                /* decreased in pnfs_send_layoutreturn() */
+                lo->plh_block_lgets++;
+                lo->plh_return_iomode = 0;
+                spin_unlock(&inode->i_lock);
+                pnfs_get_layout_hdr(lo);
+
+                /* Send an async layoutreturn so we dont deadlock */
+                pnfs_send_layoutreturn(lo, stateid, iomode, false);
+        } else
+                spin_unlock(&inode->i_lock);
+}
+
 void
 pnfs_put_lseg(struct pnfs_layout_segment *lseg)
 {
@@ -349,8 +398,17 @@ pnfs_put_lseg(struct pnfs_layout_segment *lseg)
         dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
                 atomic_read(&lseg->pls_refcount),
                 test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
+
+        /* Handle the case where refcount != 1 */
+        if (atomic_add_unless(&lseg->pls_refcount, -1, 1))
+                return;
+
         lo = lseg->pls_layout;
         inode = lo->plh_inode;
+        /* Do we need a layoutreturn? */
+        if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
+                pnfs_layoutreturn_before_put_lseg(lseg, lo, inode);
+
         if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
                 pnfs_get_layout_hdr(lo);
                 pnfs_layout_remove_lseg(lo, lseg);
@@ -543,6 +601,7 @@ pnfs_destroy_layout(struct nfs_inode *nfsi)
                 pnfs_get_layout_hdr(lo);
                 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
                 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
+                pnfs_clear_retry_layoutget(lo);
                 spin_unlock(&nfsi->vfs_inode.i_lock);
                 pnfs_free_lseg_list(&tmp_list);
                 pnfs_put_layout_hdr(lo);
@@ -740,25 +799,37 @@ pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
         return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
 }
 
+static bool
+pnfs_layout_returning(const struct pnfs_layout_hdr *lo,
+                      struct pnfs_layout_range *range)
+{
+        return test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) &&
+                (lo->plh_return_iomode == IOMODE_ANY ||
+                 lo->plh_return_iomode == range->iomode);
+}
+
 /* lget is set to 1 if called from inside send_layoutget call chain */
 static bool
-pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo, int lget)
+pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo,
+                        struct pnfs_layout_range *range, int lget)
 {
         return lo->plh_block_lgets ||
                 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags) ||
                 (list_empty(&lo->plh_segs) &&
-                 (atomic_read(&lo->plh_outstanding) > lget));
+                 (atomic_read(&lo->plh_outstanding) > lget)) ||
+                pnfs_layout_returning(lo, range);
 }
 
 int
 pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
+                              struct pnfs_layout_range *range,
                               struct nfs4_state *open_state)
 {
         int status = 0;
 
         dprintk("--> %s\n", __func__);
         spin_lock(&lo->plh_inode->i_lock);
-        if (pnfs_layoutgets_blocked(lo, 1)) {
+        if (pnfs_layoutgets_blocked(lo, range, 1)) {
                 status = -EAGAIN;
         } else if (!nfs4_valid_open_stateid(open_state)) {
                 status = -EBADF;
@@ -825,7 +896,9 @@ send_layoutget(struct pnfs_layout_hdr *lo,
                         pnfs_layout_io_set_failed(lo, range->iomode);
                 }
                 return NULL;
-        }
+        } else
+                pnfs_layout_clear_fail_bit(lo,
+                                pnfs_iomode_to_fail_bit(range->iomode));
 
         return lseg;
 }
@@ -845,6 +918,49 @@ static void pnfs_clear_layoutcommit(struct inode *inode,
         }
 }
 
+void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
+{
+        clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
+        smp_mb__after_atomic();
+        wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
+}
+
+static int
+pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid,
+                       enum pnfs_iomode iomode, bool sync)
+{
+        struct inode *ino = lo->plh_inode;
+        struct nfs4_layoutreturn *lrp;
+        int status = 0;
+
+        lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
+        if (unlikely(lrp == NULL)) {
+                status = -ENOMEM;
+                spin_lock(&ino->i_lock);
+                lo->plh_block_lgets--;
+                pnfs_clear_layoutreturn_waitbit(lo);
+                rpc_wake_up(&NFS_SERVER(ino)->roc_rpcwaitq);
+                spin_unlock(&ino->i_lock);
+                pnfs_put_layout_hdr(lo);
+                goto out;
+        }
+
+        lrp->args.stateid = stateid;
+        lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
+        lrp->args.inode = ino;
+        lrp->args.range.iomode = iomode;
+        lrp->args.range.offset = 0;
+        lrp->args.range.length = NFS4_MAX_UINT64;
+        lrp->args.layout = lo;
+        lrp->clp = NFS_SERVER(ino)->nfs_client;
+        lrp->cred = lo->plh_lc_cred;
+
+        status = nfs4_proc_layoutreturn(lrp, sync);
+out:
+        dprintk("<-- %s status: %d\n", __func__, status);
+        return status;
+}
+
 /*
  * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
  * when the layout segment list is empty.
@@ -859,7 +975,6 @@ _pnfs_return_layout(struct inode *ino)
         struct pnfs_layout_hdr *lo = NULL;
         struct nfs_inode *nfsi = NFS_I(ino);
         LIST_HEAD(tmp_list);
-        struct nfs4_layoutreturn *lrp;
         nfs4_stateid stateid;
         int status = 0, empty;
 
@@ -901,24 +1016,7 @@ _pnfs_return_layout(struct inode *ino)
         spin_unlock(&ino->i_lock);
         pnfs_free_lseg_list(&tmp_list);
 
-        lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
-        if (unlikely(lrp == NULL)) {
-                status = -ENOMEM;
-                spin_lock(&ino->i_lock);
-                lo->plh_block_lgets--;
-                spin_unlock(&ino->i_lock);
-                pnfs_put_layout_hdr(lo);
-                goto out;
-        }
-
-        lrp->args.stateid = stateid;
-        lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
-        lrp->args.inode = ino;
-        lrp->args.layout = lo;
-        lrp->clp = NFS_SERVER(ino)->nfs_client;
-        lrp->cred = lo->plh_lc_cred;
-
-        status = nfs4_proc_layoutreturn(lrp);
+        status = pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, true);
 out:
         dprintk("<-- %s status: %d\n", __func__, status);
         return status;
@@ -954,31 +1052,60 @@ pnfs_commit_and_return_layout(struct inode *inode)
 
 bool pnfs_roc(struct inode *ino)
 {
+        struct nfs_inode *nfsi = NFS_I(ino);
+        struct nfs_open_context *ctx;
+        struct nfs4_state *state;
         struct pnfs_layout_hdr *lo;
         struct pnfs_layout_segment *lseg, *tmp;
+        nfs4_stateid stateid;
         LIST_HEAD(tmp_list);
-        bool found = false;
+        bool found = false, layoutreturn = false;
 
         spin_lock(&ino->i_lock);
-        lo = NFS_I(ino)->layout;
+        lo = nfsi->layout;
         if (!lo || !test_and_clear_bit(NFS_LAYOUT_ROC, &lo->plh_flags) ||
             test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
-                goto out_nolayout;
+                goto out_noroc;
+
+        /* Don't return layout if we hold a delegation */
+        if (nfs4_check_delegation(ino, FMODE_READ))
+                goto out_noroc;
+
+        list_for_each_entry(ctx, &nfsi->open_files, list) {
+                state = ctx->state;
+                /* Don't return layout if there is open file state */
+                if (state != NULL && state->state != 0)
+                        goto out_noroc;
+        }
+
+        pnfs_clear_retry_layoutget(lo);
         list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
                 if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
                         mark_lseg_invalid(lseg, &tmp_list);
                         found = true;
                 }
         if (!found)
-                goto out_nolayout;
+                goto out_noroc;
         lo->plh_block_lgets++;
         pnfs_get_layout_hdr(lo); /* matched in pnfs_roc_release */
         spin_unlock(&ino->i_lock);
         pnfs_free_lseg_list(&tmp_list);
         return true;
 
-out_nolayout:
+out_noroc:
+        if (lo) {
+                stateid = lo->plh_stateid;
+                layoutreturn =
+                        test_and_clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
+                                           &lo->plh_flags);
+                if (layoutreturn) {
+                        lo->plh_block_lgets++;
+                        pnfs_get_layout_hdr(lo);
+                }
+        }
         spin_unlock(&ino->i_lock);
+        if (layoutreturn)
+                pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, true);
         return false;
 }
 
@@ -1013,8 +1140,9 @@ bool pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task)
         struct nfs_inode *nfsi = NFS_I(ino);
         struct pnfs_layout_hdr *lo;
         struct pnfs_layout_segment *lseg;
+        nfs4_stateid stateid;
         u32 current_seqid;
-        bool found = false;
+        bool found = false, layoutreturn = false;
 
         spin_lock(&ino->i_lock);
         list_for_each_entry(lseg, &nfsi->layout->plh_segs, pls_list)
@@ -1031,7 +1159,21 @@ bool pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task)
          */
         *barrier = current_seqid + atomic_read(&lo->plh_outstanding);
 out:
+        if (!found) {
+                stateid = lo->plh_stateid;
+                layoutreturn =
+                        test_and_clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
+                                           &lo->plh_flags);
+                if (layoutreturn) {
+                        lo->plh_block_lgets++;
+                        pnfs_get_layout_hdr(lo);
+                }
+        }
         spin_unlock(&ino->i_lock);
+        if (layoutreturn) {
+                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
+                pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, false);
+        }
         return found;
 }
 
@@ -1178,6 +1320,7 @@ pnfs_find_lseg(struct pnfs_layout_hdr *lo,
 
         list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
                 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
+                    !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
                     pnfs_lseg_range_match(&lseg->pls_range, range)) {
                         ret = pnfs_get_lseg(lseg);
                         break;
@@ -1266,6 +1409,35 @@ static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
         return ret;
 }
 
+/* stop waiting if someone clears NFS_LAYOUT_RETRY_LAYOUTGET bit. */
+static int pnfs_layoutget_retry_bit_wait(struct wait_bit_key *key)
+{
+        if (!test_bit(NFS_LAYOUT_RETRY_LAYOUTGET, key->flags))
+                return 1;
+        return nfs_wait_bit_killable(key);
+}
+
+static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
+{
+        /*
+         * send layoutcommit as it can hold up layoutreturn due to lseg
+         * reference
+         */
+        pnfs_layoutcommit_inode(lo->plh_inode, false);
+        return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
+                                   pnfs_layoutget_retry_bit_wait,
+                                   TASK_UNINTERRUPTIBLE);
+}
+
+static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
+{
+        unsigned long *bitlock = &lo->plh_flags;
+
+        clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
+        smp_mb__after_atomic();
+        wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
+}
+
 /*
  * Layout segment is retreived from the server if not cached.
  * The appropriate layout segment is referenced and returned to the caller.
@@ -1296,6 +1468,8 @@ pnfs_update_layout(struct inode *ino,
         if (pnfs_within_mdsthreshold(ctx, ino, iomode))
                 goto out;
 
+lookup_again:
+        first = false;
         spin_lock(&ino->i_lock);
         lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
         if (lo == NULL) {
@@ -1310,27 +1484,62 @@ pnfs_update_layout(struct inode *ino,
         }
 
         /* if LAYOUTGET already failed once we don't try again */
-        if (pnfs_layout_io_test_failed(lo, iomode))
+        if (pnfs_layout_io_test_failed(lo, iomode) &&
+            !pnfs_should_retry_layoutget(lo))
                 goto out_unlock;
 
-        /* Check to see if the layout for the given range already exists */
-        lseg = pnfs_find_lseg(lo, &arg);
-        if (lseg)
-                goto out_unlock;
+        first = list_empty(&lo->plh_segs);
+        if (first) {
+                /* The first layoutget for the file. Need to serialize per
+                 * RFC 5661 Errata 3208.
+                 */
+                if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
+                                     &lo->plh_flags)) {
+                        spin_unlock(&ino->i_lock);
+                        wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
+                                    TASK_UNINTERRUPTIBLE);
+                        pnfs_put_layout_hdr(lo);
+                        goto lookup_again;
+                }
+        } else {
+                /* Check to see if the layout for the given range
+                 * already exists
+                 */
+                lseg = pnfs_find_lseg(lo, &arg);
+                if (lseg)
+                        goto out_unlock;
+        }
+
+        /*
+         * Because we free lsegs before sending LAYOUTRETURN, we need to wait
+         * for LAYOUTRETURN even if first is true.
+         */
+        if (!lseg && pnfs_should_retry_layoutget(lo) &&
+            test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
+                spin_unlock(&ino->i_lock);
+                dprintk("%s wait for layoutreturn\n", __func__);
+                if (pnfs_prepare_to_retry_layoutget(lo)) {
+                        if (first)
+                                pnfs_clear_first_layoutget(lo);
+                        pnfs_put_layout_hdr(lo);
+                        dprintk("%s retrying\n", __func__);
+                        goto lookup_again;
+                }
+                goto out_put_layout_hdr;
+        }
 
-        if (pnfs_layoutgets_blocked(lo, 0))
+        if (pnfs_layoutgets_blocked(lo, &arg, 0))
                 goto out_unlock;
         atomic_inc(&lo->plh_outstanding);
-
-        first = list_empty(&lo->plh_layouts) ? true : false;
         spin_unlock(&ino->i_lock);
 
-        if (first) {
+        if (list_empty(&lo->plh_layouts)) {
                 /* The lo must be on the clp list if there is any
                  * chance of a CB_LAYOUTRECALL(FILE) coming in.
                  */
                 spin_lock(&clp->cl_lock);
-                list_add_tail(&lo->plh_layouts, &server->layouts);
+                if (list_empty(&lo->plh_layouts))
+                        list_add_tail(&lo->plh_layouts, &server->layouts);
                 spin_unlock(&clp->cl_lock);
         }
 
@@ -1343,8 +1552,11 @@ pnfs_update_layout(struct inode *ino,
                 arg.length = PAGE_CACHE_ALIGN(arg.length);
 
         lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
+        pnfs_clear_retry_layoutget(lo);
         atomic_dec(&lo->plh_outstanding);
 out_put_layout_hdr:
+        if (first)
+                pnfs_clear_first_layoutget(lo);
         pnfs_put_layout_hdr(lo);
 out:
         dprintk("%s: inode %s/%llu pNFS layout segment %s for "
@@ -1393,7 +1605,7 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
                 goto out_forget_reply;
         }
 
-        if (pnfs_layoutgets_blocked(lo, 1)) {
+        if (pnfs_layoutgets_blocked(lo, &lgp->args.range, 1)) {
                 dprintk("%s forget reply due to state\n", __func__);
                 goto out_forget_reply;
         }
@@ -1440,24 +1652,79 @@ out_forget_reply:
         goto out;
 }
 
+static void
+pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
+                                struct list_head *tmp_list,
+                                struct pnfs_layout_range *return_range)
+{
+        struct pnfs_layout_segment *lseg, *next;
+
+        dprintk("%s:Begin lo %p\n", __func__, lo);
+
+        if (list_empty(&lo->plh_segs))
+                return;
+
+        list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
+                if (should_free_lseg(&lseg->pls_range, return_range)) {
+                        dprintk("%s: marking lseg %p iomode %d "
+                                "offset %llu length %llu\n", __func__,
+                                lseg, lseg->pls_range.iomode,
+                                lseg->pls_range.offset,
+                                lseg->pls_range.length);
+                        set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
+                        mark_lseg_invalid(lseg, tmp_list);
+                }
+}
+
+void pnfs_error_mark_layout_for_return(struct inode *inode,
+                                       struct pnfs_layout_segment *lseg)
+{
+        struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
+        int iomode = pnfs_iomode_to_fail_bit(lseg->pls_range.iomode);
+        struct pnfs_layout_range range = {
+                .iomode = lseg->pls_range.iomode,
+                .offset = 0,
+                .length = NFS4_MAX_UINT64,
+        };
+        LIST_HEAD(free_me);
+
+        spin_lock(&inode->i_lock);
+        /* set failure bit so that pnfs path will be retried later */
+        pnfs_layout_set_fail_bit(lo, iomode);
+        set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
+        if (lo->plh_return_iomode == 0)
+                lo->plh_return_iomode = range.iomode;
+        else if (lo->plh_return_iomode != range.iomode)
+                lo->plh_return_iomode = IOMODE_ANY;
+        /*
+         * mark all matching lsegs so that we are sure to have no live
+         * segments at hand when sending layoutreturn. See pnfs_put_lseg()
+         * for how it works.
+         */
+        pnfs_mark_matching_lsegs_return(lo, &free_me, &range);
+        spin_unlock(&inode->i_lock);
+        pnfs_free_lseg_list(&free_me);
+}
+EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);
+
 void
 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
 {
         u64 rd_size = req->wb_bytes;
 
-        WARN_ON_ONCE(pgio->pg_lseg != NULL);
-
-        if (pgio->pg_dreq == NULL)
-                rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
-        else
-                rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);
-
-        pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
-                                           req->wb_context,
-                                           req_offset(req),
-                                           rd_size,
-                                           IOMODE_READ,
-                                           GFP_KERNEL);
+        if (pgio->pg_lseg == NULL) {
+                if (pgio->pg_dreq == NULL)
+                        rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
+                else
+                        rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);
+
+                pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
+                                                   req->wb_context,
+                                                   req_offset(req),
+                                                   rd_size,
+                                                   IOMODE_READ,
+                                                   GFP_KERNEL);
+        }
         /* If no lseg, fall back to read through mds */
         if (pgio->pg_lseg == NULL)
                 nfs_pageio_reset_read_mds(pgio);
@@ -1469,27 +1736,36 @@ void
 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
                            struct nfs_page *req, u64 wb_size)
 {
-        WARN_ON_ONCE(pgio->pg_lseg != NULL);
-
-        pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
-                                           req->wb_context,
-                                           req_offset(req),
-                                           wb_size,
-                                           IOMODE_RW,
-                                           GFP_NOFS);
+        if (pgio->pg_lseg == NULL)
+                pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
+                                                   req->wb_context,
+                                                   req_offset(req),
+                                                   wb_size,
+                                                   IOMODE_RW,
+                                                   GFP_NOFS);
         /* If no lseg, fall back to write through mds */
         if (pgio->pg_lseg == NULL)
                 nfs_pageio_reset_write_mds(pgio);
 }
 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);
 
+void
+pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
+{
+        if (desc->pg_lseg) {
+                pnfs_put_lseg(desc->pg_lseg);
+                desc->pg_lseg = NULL;
+        }
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);
+
 /*
  * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
  * of bytes (maximum @req->wb_bytes) that can be coalesced.
  */
 size_t
-pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
-                     struct nfs_page *req)
+pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
+                     struct nfs_page *prev, struct nfs_page *req)
 {
         unsigned int size;
         u64 seg_end, req_start, seg_left;
@@ -1513,10 +1789,16 @@ pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
                 seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
                                      pgio->pg_lseg->pls_range.length);
                 req_start = req_offset(req);
-                WARN_ON_ONCE(req_start > seg_end);
+                WARN_ON_ONCE(req_start >= seg_end);
                 /* start of request is past the last byte of this segment */
-                if (req_start >= seg_end)
+                if (req_start >= seg_end) {
+                        /* reference the new lseg */
+                        if (pgio->pg_ops->pg_cleanup)
+                                pgio->pg_ops->pg_cleanup(pgio);
+                        if (pgio->pg_ops->pg_init)
+                                pgio->pg_ops->pg_init(pgio, req);
                         return 0;
+                }
 
                 /* adjust 'size' iff there are fewer bytes left in the
                  * segment than what nfs_generic_pg_test returned */
@@ -1571,10 +1853,12 @@ static void
 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
                 struct nfs_pgio_header *hdr)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
         if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
-                list_splice_tail_init(&hdr->pages, &desc->pg_list);
+                list_splice_tail_init(&hdr->pages, &mirror->pg_list);
                 nfs_pageio_reset_write_mds(desc);
-                desc->pg_recoalesce = 1;
+                mirror->pg_recoalesce = 1;
         }
         nfs_pgio_data_destroy(hdr);
 }
@@ -1608,11 +1892,9 @@ pnfs_do_write(struct nfs_pageio_descriptor *desc,
         struct pnfs_layout_segment *lseg = desc->pg_lseg;
         enum pnfs_try_status trypnfs;
 
-        desc->pg_lseg = NULL;
         trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
         if (trypnfs == PNFS_NOT_ATTEMPTED)
                 pnfs_write_through_mds(desc, hdr);
-        pnfs_put_lseg(lseg);
 }
 
 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
@@ -1625,24 +1907,23 @@ EXPORT_SYMBOL_GPL(pnfs_writehdr_free);
 int
 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
         struct nfs_pgio_header *hdr;
         int ret;
 
         hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
         if (!hdr) {
-                desc->pg_completion_ops->error_cleanup(&desc->pg_list);
-                pnfs_put_lseg(desc->pg_lseg);
-                desc->pg_lseg = NULL;
+                desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
                 return -ENOMEM;
         }
         nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
+
         hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
         ret = nfs_generic_pgio(desc, hdr);
-        if (ret != 0) {
-                pnfs_put_lseg(desc->pg_lseg);
-                desc->pg_lseg = NULL;
-        } else
+        if (!ret)
                 pnfs_do_write(desc, hdr, desc->pg_ioflags);
+
         return ret;
 }
 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
@@ -1687,10 +1968,12 @@ static void
 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
                 struct nfs_pgio_header *hdr)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
         if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
-                list_splice_tail_init(&hdr->pages, &desc->pg_list);
+                list_splice_tail_init(&hdr->pages, &mirror->pg_list);
                 nfs_pageio_reset_read_mds(desc);
-                desc->pg_recoalesce = 1;
+                mirror->pg_recoalesce = 1;
         }
         nfs_pgio_data_destroy(hdr);
 }
@@ -1719,18 +2002,29 @@ pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
         return trypnfs;
 }
 
+/* Resend all requests through pnfs. */
+int pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
+{
+        struct nfs_pageio_descriptor pgio;
+
+        nfs_pageio_init_read(&pgio, hdr->inode, false, hdr->completion_ops);
+        return nfs_pageio_resend(&pgio, hdr);
+}
+EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);
+
 static void
 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
 {
         const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
         struct pnfs_layout_segment *lseg = desc->pg_lseg;
         enum pnfs_try_status trypnfs;
+        int err = 0;
 
-        desc->pg_lseg = NULL;
         trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
-        if (trypnfs == PNFS_NOT_ATTEMPTED)
+        if (trypnfs == PNFS_TRY_AGAIN)
+                err = pnfs_read_resend_pnfs(hdr);
+        if (trypnfs == PNFS_NOT_ATTEMPTED || err)
                 pnfs_read_through_mds(desc, hdr);
-        pnfs_put_lseg(lseg);
 }
 
 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
@@ -1743,24 +2037,20 @@ EXPORT_SYMBOL_GPL(pnfs_readhdr_free);
 int
 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
 {
+        struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
+
         struct nfs_pgio_header *hdr;
         int ret;
 
         hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
         if (!hdr) {
-                desc->pg_completion_ops->error_cleanup(&desc->pg_list);
-                ret = -ENOMEM;
-                pnfs_put_lseg(desc->pg_lseg);
-                desc->pg_lseg = NULL;
-                return ret;
+                desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
+                return -ENOMEM;
         }
         nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
         hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
         ret = nfs_generic_pgio(desc, hdr);
-        if (ret != 0) {
-                pnfs_put_lseg(desc->pg_lseg);
-                desc->pg_lseg = NULL;
-        } else
+        if (!ret)
                 pnfs_do_read(desc, hdr);
         return ret;
 }
@@ -1966,6 +2256,7 @@ clear_layoutcommitting:
         pnfs_clear_layoutcommitting(inode);
         goto out;
 }
+EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
 
 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
 {
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 9ae5b765b073..797cd6253adf 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -38,6 +38,25 @@ enum {
         NFS_LSEG_VALID = 0,     /* cleared when lseg is recalled/returned */
         NFS_LSEG_ROC,           /* roc bit received from server */
         NFS_LSEG_LAYOUTCOMMIT,  /* layoutcommit bit set for layoutcommit */
+        NFS_LSEG_LAYOUTRETURN,  /* layoutreturn bit set for layoutreturn */
+};
+
+/* Individual ip address */
+struct nfs4_pnfs_ds_addr {
+        struct sockaddr_storage da_addr;
+        size_t                  da_addrlen;
+        struct list_head        da_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
+        char                    *da_remotestr;  /* human readable addr+port */
+};
+
+struct nfs4_pnfs_ds {
+        struct list_head        ds_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
+        char                    *ds_remotestr;  /* comma sep list of addrs */
+        struct list_head        ds_addrs;
+        struct nfs_client       *ds_clp;
+        atomic_t                ds_count;
+        unsigned long           ds_state;
+#define NFS4DS_CONNECTING       0       /* ds is establishing connection */
 };
 
 struct pnfs_layout_segment {
@@ -53,19 +72,34 @@ struct pnfs_layout_segment {
 enum pnfs_try_status {
         PNFS_ATTEMPTED     = 0,
         PNFS_NOT_ATTEMPTED = 1,
+        PNFS_TRY_AGAIN     = 2,
 };
 
 #ifdef CONFIG_NFS_V4_1
 
 #define LAYOUT_NFSV4_1_MODULE_PREFIX "nfs-layouttype4"
 
+/*
+ * Default data server connection timeout and retrans vaules.
+ * Set by module parameters dataserver_timeo and dataserver_retrans.
+ */
+#define NFS4_DEF_DS_TIMEO   600 /* in tenths of a second */
+#define NFS4_DEF_DS_RETRANS 5
+
+/* error codes for internal use */
+#define NFS4ERR_RESET_TO_MDS   12001
+#define NFS4ERR_RESET_TO_PNFS  12002
+
 enum {
         NFS_LAYOUT_RO_FAILED = 0,       /* get ro layout failed stop trying */
         NFS_LAYOUT_RW_FAILED,           /* get rw layout failed stop trying */
         NFS_LAYOUT_BULK_RECALL,         /* bulk recall affecting layout */
         NFS_LAYOUT_ROC,                 /* some lseg had roc bit set */
         NFS_LAYOUT_RETURN,              /* Return this layout ASAP */
+        NFS_LAYOUT_RETURN_BEFORE_CLOSE, /* Return this layout before close */
         NFS_LAYOUT_INVALID_STID,        /* layout stateid id is invalid */
+        NFS_LAYOUT_FIRST_LAYOUTGET,     /* Serialize first layoutget */
+        NFS_LAYOUT_RETRY_LAYOUTGET,     /* Retry layoutget */
 };
 
 enum layoutdriver_policy_flags {
@@ -106,7 +140,8 @@ struct pnfs_layoutdriver_type {
         struct pnfs_ds_commit_info *(*get_ds_info) (struct inode *inode);
         void (*mark_request_commit) (struct nfs_page *req,
                                      struct pnfs_layout_segment *lseg,
-                                     struct nfs_commit_info *cinfo);
+                                     struct nfs_commit_info *cinfo,
+                                     u32 ds_commit_idx);
         void (*clear_request_commit) (struct nfs_page *req,
                                       struct nfs_commit_info *cinfo);
         int (*scan_commit_lists) (struct nfs_commit_info *cinfo,
@@ -154,6 +189,7 @@ struct pnfs_layout_hdr {
         u32                     plh_barrier; /* ignore lower seqids */
         unsigned long           plh_retry_timestamp;
         unsigned long           plh_flags;
+        enum pnfs_iomode        plh_return_iomode;
         loff_t                  plh_lwb; /* last write byte for layoutcommit */
         struct rpc_cred         *plh_lc_cred; /* layoutcommit cred */
         struct inode            *plh_inode;
@@ -185,7 +221,7 @@ extern int nfs4_proc_getdeviceinfo(struct nfs_server *server,
                                    struct pnfs_device *dev,
                                    struct rpc_cred *cred);
 extern struct pnfs_layout_segment* nfs4_proc_layoutget(struct nfs4_layoutget *lgp, gfp_t gfp_flags);
-extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp);
+extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, bool sync);
 
 /* pnfs.c */
 void pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo);
@@ -198,6 +234,7 @@ void pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *, struct nfs_page *
 int pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc);
 void pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
                                 struct nfs_page *req, u64 wb_size);
+void pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *);
 int pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc);
 size_t pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
                             struct nfs_page *prev, struct nfs_page *req);
@@ -217,6 +254,7 @@ void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo,
                              bool update_barrier);
 int pnfs_choose_layoutget_stateid(nfs4_stateid *dst,
                                   struct pnfs_layout_hdr *lo,
+                                  struct pnfs_layout_range *range,
                                   struct nfs4_state *open_state);
 int pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
                                 struct list_head *tmp_list,
@@ -233,17 +271,21 @@ int _pnfs_return_layout(struct inode *);
 int pnfs_commit_and_return_layout(struct inode *);
 void pnfs_ld_write_done(struct nfs_pgio_header *);
 void pnfs_ld_read_done(struct nfs_pgio_header *);
+int pnfs_read_resend_pnfs(struct nfs_pgio_header *);
 struct pnfs_layout_segment *pnfs_update_layout(struct inode *ino,
                                                struct nfs_open_context *ctx,
                                                loff_t pos,
                                                u64 count,
                                                enum pnfs_iomode iomode,
                                                gfp_t gfp_flags);
+void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo);
 
 void nfs4_deviceid_mark_client_invalid(struct nfs_client *clp);
 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *);
 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *);
 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void);
+void pnfs_error_mark_layout_for_return(struct inode *inode,
+                                       struct pnfs_layout_segment *lseg);
 
 /* nfs4_deviceid_flags */
 enum {
@@ -275,6 +317,39 @@ void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node);
 bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node);
 void nfs4_deviceid_purge_client(const struct nfs_client *);
 
+/* pnfs_nfs.c */
+void pnfs_generic_clear_request_commit(struct nfs_page *req,
+                                       struct nfs_commit_info *cinfo);
+void pnfs_generic_commit_release(void *calldata);
+void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data);
+void pnfs_generic_rw_release(void *data);
+void pnfs_generic_recover_commit_reqs(struct list_head *dst,
+                                      struct nfs_commit_info *cinfo);
+int pnfs_generic_commit_pagelist(struct inode *inode,
+                                 struct list_head *mds_pages,
+                                 int how,
+                                 struct nfs_commit_info *cinfo,
+                                 int (*initiate_commit)(struct nfs_commit_data *data,
+                                                        int how));
+int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max);
+void pnfs_generic_write_commit_done(struct rpc_task *task, void *data);
+void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds);
+struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(struct list_head *dsaddrs,
+                                      gfp_t gfp_flags);
+void nfs4_pnfs_v3_ds_connect_unload(void);
+void nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
+                          struct nfs4_deviceid_node *devid, unsigned int timeo,
+                          unsigned int retrans, u32 version, u32 minor_version,
+                          rpc_authflavor_t au_flavor);
+struct nfs4_pnfs_ds_addr *nfs4_decode_mp_ds_addr(struct net *net,
+                                                 struct xdr_stream *xdr,
+                                                 gfp_t gfp_flags);
+
+static inline bool nfs_have_layout(struct inode *inode)
+{
+        return NFS_I(inode)->layout != NULL;
+}
+
 static inline struct nfs4_deviceid_node *
 nfs4_get_deviceid(struct nfs4_deviceid_node *d)
 {
@@ -282,6 +357,26 @@ nfs4_get_deviceid(struct nfs4_deviceid_node *d)
         return d;
 }
 
+static inline void pnfs_set_retry_layoutget(struct pnfs_layout_hdr *lo)
+{
+        if (!test_and_set_bit(NFS_LAYOUT_RETRY_LAYOUTGET, &lo->plh_flags))
+                atomic_inc(&lo->plh_refcount);
+}
+
+static inline void pnfs_clear_retry_layoutget(struct pnfs_layout_hdr *lo)
+{
+        if (test_and_clear_bit(NFS_LAYOUT_RETRY_LAYOUTGET, &lo->plh_flags)) {
+                atomic_dec(&lo->plh_refcount);
+                /* wake up waiters for LAYOUTRETURN as that is not needed */
+                wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
+        }
+}
+
+static inline bool pnfs_should_retry_layoutget(struct pnfs_layout_hdr *lo)
+{
+        return test_bit(NFS_LAYOUT_RETRY_LAYOUTGET, &lo->plh_flags);
+}
+
 static inline struct pnfs_layout_segment *
 pnfs_get_lseg(struct pnfs_layout_segment *lseg)
 {
@@ -317,16 +412,22 @@ pnfs_get_ds_info(struct inode *inode)
         return ld->get_ds_info(inode);
 }
 
+static inline void
+pnfs_generic_mark_devid_invalid(struct nfs4_deviceid_node *node)
+{
+        set_bit(NFS_DEVICEID_INVALID, &node->flags);
+}
+
 static inline bool
 pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
-                         struct nfs_commit_info *cinfo)
+                         struct nfs_commit_info *cinfo, u32 ds_commit_idx)
 {
         struct inode *inode = req->wb_context->dentry->d_inode;
         struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
 
         if (lseg == NULL || ld->mark_request_commit == NULL)
                 return false;
-        ld->mark_request_commit(req, lseg, cinfo);
+        ld->mark_request_commit(req, lseg, cinfo, ds_commit_idx);
         return true;
 }
 
@@ -352,15 +453,6 @@ pnfs_scan_commit_lists(struct inode *inode, struct nfs_commit_info *cinfo,
                 return NFS_SERVER(inode)->pnfs_curr_ld->scan_commit_lists(cinfo, max);
 }
 
-static inline void
-pnfs_recover_commit_reqs(struct inode *inode, struct list_head *list,
-                         struct nfs_commit_info *cinfo)
-{
-        if (cinfo->ds == NULL || cinfo->ds->nwritten == 0)
-                return;
-        NFS_SERVER(inode)->pnfs_curr_ld->recover_commit_reqs(list, cinfo);
-}
-
 static inline struct nfs_page *
 pnfs_search_commit_reqs(struct inode *inode, struct nfs_commit_info *cinfo,
                         struct page *page)
@@ -427,6 +519,11 @@ static inline void nfs4_print_deviceid(const struct nfs4_deviceid *dev_id)
 #endif /* NFS_DEBUG */
 #else  /* CONFIG_NFS_V4_1 */
 
+static inline bool nfs_have_layout(struct inode *inode)
+{
+        return false;
+}
+
 static inline void pnfs_destroy_all_layouts(struct nfs_client *clp)
 {
 }
@@ -513,7 +610,7 @@ pnfs_get_ds_info(struct inode *inode)
 
 static inline bool
 pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
-                         struct nfs_commit_info *cinfo)
+                         struct nfs_commit_info *cinfo, u32 ds_commit_idx)
 {
         return false;
 }
@@ -531,12 +628,6 @@ pnfs_scan_commit_lists(struct inode *inode, struct nfs_commit_info *cinfo,
         return 0;
 }
 
-static inline void
-pnfs_recover_commit_reqs(struct inode *inode, struct list_head *list,
-                         struct nfs_commit_info *cinfo)
-{
-}
-
 static inline struct nfs_page *
 pnfs_search_commit_reqs(struct inode *inode, struct nfs_commit_info *cinfo,
                         struct page *page)
@@ -568,6 +659,10 @@ static inline struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
         return NULL;
 }
 
+static inline void nfs4_pnfs_v3_ds_connect_unload(void)
+{
+}
+
 #endif /* CONFIG_NFS_V4_1 */
 
 #endif /* FS_NFS_PNFS_H */
diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c
new file mode 100644
index 000000000000..fdc4f6562bb7
--- /dev/null
+++ b/fs/nfs/pnfs_nfs.c
@@ -0,0 +1,840 @@
+/*
+ * Common NFS I/O  operations for the pnfs file based
+ * layout drivers.
+ *
+ * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
+ *
+ * Tom Haynes <loghyr@primarydata.com>
+ */
+
+#include <linux/nfs_fs.h>
+#include <linux/nfs_page.h>
+#include <linux/sunrpc/addr.h>
+#include <linux/module.h>
+
+#include "nfs4session.h"
+#include "internal.h"
+#include "pnfs.h"
+
+#define NFSDBG_FACILITY         NFSDBG_PNFS
+
+void pnfs_generic_rw_release(void *data)
+{
+        struct nfs_pgio_header *hdr = data;
+
+        nfs_put_client(hdr->ds_clp);
+        hdr->mds_ops->rpc_release(data);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_rw_release);
+
+/* Fake up some data that will cause nfs_commit_release to retry the writes. */
+void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data)
+{
+        struct nfs_page *first = nfs_list_entry(data->pages.next);
+
+        data->task.tk_status = 0;
+        memcpy(&data->verf.verifier, &first->wb_verf,
+               sizeof(data->verf.verifier));
+        data->verf.verifier.data[0]++; /* ensure verifier mismatch */
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_prepare_to_resend_writes);
+
+void pnfs_generic_write_commit_done(struct rpc_task *task, void *data)
+{
+        struct nfs_commit_data *wdata = data;
+
+        /* Note this may cause RPC to be resent */
+        wdata->mds_ops->rpc_call_done(task, data);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_write_commit_done);
+
+void pnfs_generic_commit_release(void *calldata)
+{
+        struct nfs_commit_data *data = calldata;
+
+        data->completion_ops->completion(data);
+        pnfs_put_lseg(data->lseg);
+        nfs_put_client(data->ds_clp);
+        nfs_commitdata_release(data);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_commit_release);
+
+/* The generic layer is about to remove the req from the commit list.
+ * If this will make the bucket empty, it will need to put the lseg reference.
+ * Note this must be called holding the inode (/cinfo) lock
+ */
+void
+pnfs_generic_clear_request_commit(struct nfs_page *req,
+                                  struct nfs_commit_info *cinfo)
+{
+        struct pnfs_layout_segment *freeme = NULL;
+
+        if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
+                goto out;
+        cinfo->ds->nwritten--;
+        if (list_is_singular(&req->wb_list)) {
+                struct pnfs_commit_bucket *bucket;
+
+                bucket = list_first_entry(&req->wb_list,
+                                          struct pnfs_commit_bucket,
+                                          written);
+                freeme = bucket->wlseg;
+                bucket->wlseg = NULL;
+        }
+out:
+        nfs_request_remove_commit_list(req, cinfo);
+        pnfs_put_lseg_locked(freeme);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_clear_request_commit);
+
+static int
+pnfs_generic_transfer_commit_list(struct list_head *src, struct list_head *dst,
+                                  struct nfs_commit_info *cinfo, int max)
+{
+        struct nfs_page *req, *tmp;
+        int ret = 0;
+
+        list_for_each_entry_safe(req, tmp, src, wb_list) {
+                if (!nfs_lock_request(req))
+                        continue;
+                kref_get(&req->wb_kref);
+                if (cond_resched_lock(cinfo->lock))
+                        list_safe_reset_next(req, tmp, wb_list);
+                nfs_request_remove_commit_list(req, cinfo);
+                clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
+                nfs_list_add_request(req, dst);
+                ret++;
+                if ((ret == max) && !cinfo->dreq)
+                        break;
+        }
+        return ret;
+}
+
+static int
+pnfs_generic_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
+                                 struct nfs_commit_info *cinfo,
+                                 int max)
+{
+        struct list_head *src = &bucket->written;
+        struct list_head *dst = &bucket->committing;
+        int ret;
+
+        lockdep_assert_held(cinfo->lock);
+        ret = pnfs_generic_transfer_commit_list(src, dst, cinfo, max);
+        if (ret) {
+                cinfo->ds->nwritten -= ret;
+                cinfo->ds->ncommitting += ret;
+                bucket->clseg = bucket->wlseg;
+                if (list_empty(src))
+                        bucket->wlseg = NULL;
+                else
+                        pnfs_get_lseg(bucket->clseg);
+        }
+        return ret;
+}
+
+/* Move reqs from written to committing lists, returning count
+ * of number moved.
+ */
+int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo,
+                                   int max)
+{
+        int i, rv = 0, cnt;
+
+        lockdep_assert_held(cinfo->lock);
+        for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) {
+                cnt = pnfs_generic_scan_ds_commit_list(&cinfo->ds->buckets[i],
+                                                       cinfo, max);
+                max -= cnt;
+                rv += cnt;
+        }
+        return rv;
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists);
+
+/* Pull everything off the committing lists and dump into @dst.  */
+void pnfs_generic_recover_commit_reqs(struct list_head *dst,
+                                      struct nfs_commit_info *cinfo)
+{
+        struct pnfs_commit_bucket *b;
+        struct pnfs_layout_segment *freeme;
+        int i;
+
+        lockdep_assert_held(cinfo->lock);
+restart:
+        for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
+                if (pnfs_generic_transfer_commit_list(&b->written, dst,
+                                                      cinfo, 0)) {
+                        freeme = b->wlseg;
+                        b->wlseg = NULL;
+                        spin_unlock(cinfo->lock);
+                        pnfs_put_lseg(freeme);
+                        spin_lock(cinfo->lock);
+                        goto restart;
+                }
+        }
+        cinfo->ds->nwritten = 0;
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs);
+
+static void pnfs_generic_retry_commit(struct nfs_commit_info *cinfo, int idx)
+{
+        struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
+        struct pnfs_commit_bucket *bucket;
+        struct pnfs_layout_segment *freeme;
+        int i;
+
+        for (i = idx; i < fl_cinfo->nbuckets; i++) {
+                bucket = &fl_cinfo->buckets[i];
+                if (list_empty(&bucket->committing))
+                        continue;
+                nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo, i);
+                spin_lock(cinfo->lock);
+                freeme = bucket->clseg;
+                bucket->clseg = NULL;
+                spin_unlock(cinfo->lock);
+                pnfs_put_lseg(freeme);
+        }
+}
+
+static unsigned int
+pnfs_generic_alloc_ds_commits(struct nfs_commit_info *cinfo,
+                              struct list_head *list)
+{
+        struct pnfs_ds_commit_info *fl_cinfo;
+        struct pnfs_commit_bucket *bucket;
+        struct nfs_commit_data *data;
+        int i;
+        unsigned int nreq = 0;
+
+        fl_cinfo = cinfo->ds;
+        bucket = fl_cinfo->buckets;
+        for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) {
+                if (list_empty(&bucket->committing))
+                        continue;
+                data = nfs_commitdata_alloc();
+                if (!data)
+                        break;
+                data->ds_commit_index = i;
+                spin_lock(cinfo->lock);
+                data->lseg = bucket->clseg;
+                bucket->clseg = NULL;
+                spin_unlock(cinfo->lock);
+                list_add(&data->pages, list);
+                nreq++;
+        }
+
+        /* Clean up on error */
+        pnfs_generic_retry_commit(cinfo, i);
+        return nreq;
+}
+
+/* This follows nfs_commit_list pretty closely */
+int
+pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
+                             int how, struct nfs_commit_info *cinfo,
+                             int (*initiate_commit)(struct nfs_commit_data *data,
+                                                    int how))
+{
+        struct nfs_commit_data *data, *tmp;
+        LIST_HEAD(list);
+        unsigned int nreq = 0;
+
+        if (!list_empty(mds_pages)) {
+                data = nfs_commitdata_alloc();
+                if (data != NULL) {
+                        data->lseg = NULL;
+                        list_add(&data->pages, &list);
+                        nreq++;
+                } else {
+                        nfs_retry_commit(mds_pages, NULL, cinfo, 0);
+                        pnfs_generic_retry_commit(cinfo, 0);
+                        cinfo->completion_ops->error_cleanup(NFS_I(inode));
+                        return -ENOMEM;
+                }
+        }
+
+        nreq += pnfs_generic_alloc_ds_commits(cinfo, &list);
+
+        if (nreq == 0) {
+                cinfo->completion_ops->error_cleanup(NFS_I(inode));
+                goto out;
+        }
+
+        atomic_add(nreq, &cinfo->mds->rpcs_out);
+
+        list_for_each_entry_safe(data, tmp, &list, pages) {
+                list_del_init(&data->pages);
+                if (!data->lseg) {
+                        nfs_init_commit(data, mds_pages, NULL, cinfo);
+                        nfs_initiate_commit(NFS_CLIENT(inode), data,
+                                            NFS_PROTO(data->inode),
+                                            data->mds_ops, how, 0);
+                } else {
+                        struct pnfs_commit_bucket *buckets;
+
+                        buckets = cinfo->ds->buckets;
+                        nfs_init_commit(data,
+                                        &buckets[data->ds_commit_index].committing,
+                                        data->lseg,
+                                        cinfo);
+                        initiate_commit(data, how);
+                }
+        }
+out:
+        cinfo->ds->ncommitting = 0;
+        return PNFS_ATTEMPTED;
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_commit_pagelist);
+
+/*
+ * Data server cache
+ *
+ * Data servers can be mapped to different device ids.
+ * nfs4_pnfs_ds reference counting
+ *   - set to 1 on allocation
+ *   - incremented when a device id maps a data server already in the cache.
+ *   - decremented when deviceid is removed from the cache.
+ */
+static DEFINE_SPINLOCK(nfs4_ds_cache_lock);
+static LIST_HEAD(nfs4_data_server_cache);
+
+/* Debug routines */
+static void
+print_ds(struct nfs4_pnfs_ds *ds)
+{
+        if (ds == NULL) {
+                printk(KERN_WARNING "%s NULL device\n", __func__);
+                return;
+        }
+        printk(KERN_WARNING "        ds %s\n"
+                "        ref count %d\n"
+                "        client %p\n"
+                "        cl_exchange_flags %x\n",
+                ds->ds_remotestr,
+                atomic_read(&ds->ds_count), ds->ds_clp,
+                ds->ds_clp ? ds->ds_clp->cl_exchange_flags : 0);
+}
+
+static bool
+same_sockaddr(struct sockaddr *addr1, struct sockaddr *addr2)
+{
+        struct sockaddr_in *a, *b;
+        struct sockaddr_in6 *a6, *b6;
+
+        if (addr1->sa_family != addr2->sa_family)
+                return false;
+
+        switch (addr1->sa_family) {
+        case AF_INET:
+                a = (struct sockaddr_in *)addr1;
+                b = (struct sockaddr_in *)addr2;
+
+                if (a->sin_addr.s_addr == b->sin_addr.s_addr &&
+                    a->sin_port == b->sin_port)
+                        return true;
+                break;
+
+        case AF_INET6:
+                a6 = (struct sockaddr_in6 *)addr1;
+                b6 = (struct sockaddr_in6 *)addr2;
+
+                /* LINKLOCAL addresses must have matching scope_id */
+                if (ipv6_addr_src_scope(&a6->sin6_addr) ==
+                    IPV6_ADDR_SCOPE_LINKLOCAL &&
+                    a6->sin6_scope_id != b6->sin6_scope_id)
+                        return false;
+
+                if (ipv6_addr_equal(&a6->sin6_addr, &b6->sin6_addr) &&
+                    a6->sin6_port == b6->sin6_port)
+                        return true;
+                break;
+
+        default:
+                dprintk("%s: unhandled address family: %u\n",
+                        __func__, addr1->sa_family);
+                return false;
+        }
+
+        return false;
+}
+
+static bool
+_same_data_server_addrs_locked(const struct list_head *dsaddrs1,
+                               const struct list_head *dsaddrs2)
+{
+        struct nfs4_pnfs_ds_addr *da1, *da2;
+
+        /* step through both lists, comparing as we go */
+        for (da1 = list_first_entry(dsaddrs1, typeof(*da1), da_node),
+             da2 = list_first_entry(dsaddrs2, typeof(*da2), da_node);
+             da1 != NULL && da2 != NULL;
+             da1 = list_entry(da1->da_node.next, typeof(*da1), da_node),
+             da2 = list_entry(da2->da_node.next, typeof(*da2), da_node)) {
+                if (!same_sockaddr((struct sockaddr *)&da1->da_addr,
+                                   (struct sockaddr *)&da2->da_addr))
+                        return false;
+        }
+        if (da1 == NULL && da2 == NULL)
+                return true;
+
+        return false;
+}
+
+/*
+ * Lookup DS by addresses.  nfs4_ds_cache_lock is held
+ */
+static struct nfs4_pnfs_ds *
+_data_server_lookup_locked(const struct list_head *dsaddrs)
+{
+        struct nfs4_pnfs_ds *ds;
+
+        list_for_each_entry(ds, &nfs4_data_server_cache, ds_node)
+                if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs))
+                        return ds;
+        return NULL;
+}
+
+static void destroy_ds(struct nfs4_pnfs_ds *ds)
+{
+        struct nfs4_pnfs_ds_addr *da;
+
+        dprintk("--> %s\n", __func__);
+        ifdebug(FACILITY)
+                print_ds(ds);
+
+        nfs_put_client(ds->ds_clp);
+
+        while (!list_empty(&ds->ds_addrs)) {
+                da = list_first_entry(&ds->ds_addrs,
+                                      struct nfs4_pnfs_ds_addr,
+                                      da_node);
+                list_del_init(&da->da_node);
+                kfree(da->da_remotestr);
+                kfree(da);
+        }
+
+        kfree(ds->ds_remotestr);
+        kfree(ds);
+}
+
+void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds)
+{
+        if (atomic_dec_and_lock(&ds->ds_count,
+                                &nfs4_ds_cache_lock)) {
+                list_del_init(&ds->ds_node);
+                spin_unlock(&nfs4_ds_cache_lock);
+                destroy_ds(ds);
+        }
+}
+EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_put);
+
+/*
+ * Create a string with a human readable address and port to avoid
+ * complicated setup around many dprinks.
+ */
+static char *
+nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags)
+{
+        struct nfs4_pnfs_ds_addr *da;
+        char *remotestr;
+        size_t len;
+        char *p;
+
+        len = 3;        /* '{', '}' and eol */
+        list_for_each_entry(da, dsaddrs, da_node) {
+                len += strlen(da->da_remotestr) + 1;    /* string plus comma */
+        }
+
+        remotestr = kzalloc(len, gfp_flags);
+        if (!remotestr)
+                return NULL;
+
+        p = remotestr;
+        *(p++) = '{';
+        len--;
+        list_for_each_entry(da, dsaddrs, da_node) {
+                size_t ll = strlen(da->da_remotestr);
+
+                if (ll > len)
+                        goto out_err;
+
+                memcpy(p, da->da_remotestr, ll);
+                p += ll;
+                len -= ll;
+
+                if (len < 1)
+                        goto out_err;
+                (*p++) = ',';
+                len--;
+        }
+        if (len < 2)
+                goto out_err;
+        *(p++) = '}';
+        *p = '\0';
+        return remotestr;
+out_err:
+        kfree(remotestr);
+        return NULL;
+}
+
+/*
+ * Given a list of multipath struct nfs4_pnfs_ds_addr, add it to ds cache if
+ * uncached and return cached struct nfs4_pnfs_ds.
+ */
+struct nfs4_pnfs_ds *
+nfs4_pnfs_ds_add(struct list_head *dsaddrs, gfp_t gfp_flags)
+{
+        struct nfs4_pnfs_ds *tmp_ds, *ds = NULL;
+        char *remotestr;
+
+        if (list_empty(dsaddrs)) {
+                dprintk("%s: no addresses defined\n", __func__);
+                goto out;
+        }
+
+        ds = kzalloc(sizeof(*ds), gfp_flags);
+        if (!ds)
+                goto out;
+
+        /* this is only used for debugging, so it's ok if its NULL */
+        remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags);
+
+        spin_lock(&nfs4_ds_cache_lock);
+        tmp_ds = _data_server_lookup_locked(dsaddrs);
+        if (tmp_ds == NULL) {
+                INIT_LIST_HEAD(&ds->ds_addrs);
+                list_splice_init(dsaddrs, &ds->ds_addrs);
+                ds->ds_remotestr = remotestr;
+                atomic_set(&ds->ds_count, 1);
+                INIT_LIST_HEAD(&ds->ds_node);
+                ds->ds_clp = NULL;
+                list_add(&ds->ds_node, &nfs4_data_server_cache);
+                dprintk("%s add new data server %s\n", __func__,
+                        ds->ds_remotestr);
+        } else {
+                kfree(remotestr);
+                kfree(ds);
+                atomic_inc(&tmp_ds->ds_count);
+                dprintk("%s data server %s found, inc'ed ds_count to %d\n",
+                        __func__, tmp_ds->ds_remotestr,
+                        atomic_read(&tmp_ds->ds_count));
+                ds = tmp_ds;
+        }
+        spin_unlock(&nfs4_ds_cache_lock);
+out:
+        return ds;
+}
+EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_add);
+
+static void nfs4_wait_ds_connect(struct nfs4_pnfs_ds *ds)
+{
+        might_sleep();
+        wait_on_bit(&ds->ds_state, NFS4DS_CONNECTING,
+                        TASK_KILLABLE);
+}
+
+static void nfs4_clear_ds_conn_bit(struct nfs4_pnfs_ds *ds)
+{
+        smp_mb__before_atomic();
+        clear_bit(NFS4DS_CONNECTING, &ds->ds_state);
+        smp_mb__after_atomic();
+        wake_up_bit(&ds->ds_state, NFS4DS_CONNECTING);
+}
+
+static struct nfs_client *(*get_v3_ds_connect)(
+                        struct nfs_client *mds_clp,
+                        const struct sockaddr *ds_addr,
+                        int ds_addrlen,
+                        int ds_proto,
+                        unsigned int ds_timeo,
+                        unsigned int ds_retrans,
+                        rpc_authflavor_t au_flavor);
+
+static bool load_v3_ds_connect(void)
+{
+        if (!get_v3_ds_connect) {
+                get_v3_ds_connect = symbol_request(nfs3_set_ds_client);
+                WARN_ON_ONCE(!get_v3_ds_connect);
+        }
+
+        return(get_v3_ds_connect != NULL);
+}
+
+void __exit nfs4_pnfs_v3_ds_connect_unload(void)
+{
+        if (get_v3_ds_connect) {
+                symbol_put(nfs3_set_ds_client);
+                get_v3_ds_connect = NULL;
+        }
+}
+EXPORT_SYMBOL_GPL(nfs4_pnfs_v3_ds_connect_unload);
+
+static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
+                                 struct nfs4_pnfs_ds *ds,
+                                 unsigned int timeo,
+                                 unsigned int retrans,
+                                 rpc_authflavor_t au_flavor)
+{
+        struct nfs_client *clp = ERR_PTR(-EIO);
+        struct nfs4_pnfs_ds_addr *da;
+        int status = 0;
+
+        dprintk("--> %s DS %s au_flavor %d\n", __func__,
+                ds->ds_remotestr, au_flavor);
+
+        if (!load_v3_ds_connect())
+                goto out;
+
+        list_for_each_entry(da, &ds->ds_addrs, da_node) {
+                dprintk("%s: DS %s: trying address %s\n",
+                        __func__, ds->ds_remotestr, da->da_remotestr);
+
+                clp = get_v3_ds_connect(mds_srv->nfs_client,
+                                        (struct sockaddr *)&da->da_addr,
+                                        da->da_addrlen, IPPROTO_TCP,
+                                        timeo, retrans, au_flavor);
+                if (!IS_ERR(clp))
+                        break;
+        }
+
+        if (IS_ERR(clp)) {
+                status = PTR_ERR(clp);
+                goto out;
+        }
+
+        smp_wmb();
+        ds->ds_clp = clp;
+        dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr);
+out:
+        return status;
+}
+
+static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
+                                 struct nfs4_pnfs_ds *ds,
+                                 unsigned int timeo,
+                                 unsigned int retrans,
+                                 u32 minor_version,
+                                 rpc_authflavor_t au_flavor)
+{
+        struct nfs_client *clp = ERR_PTR(-EIO);
+        struct nfs4_pnfs_ds_addr *da;
+        int status = 0;
+
+        dprintk("--> %s DS %s au_flavor %d\n", __func__, ds->ds_remotestr,
+                au_flavor);
+
+        list_for_each_entry(da, &ds->ds_addrs, da_node) {
+                dprintk("%s: DS %s: trying address %s\n",
+                        __func__, ds->ds_remotestr, da->da_remotestr);
+
+                clp = nfs4_set_ds_client(mds_srv->nfs_client,
+                                        (struct sockaddr *)&da->da_addr,
+                                        da->da_addrlen, IPPROTO_TCP,
+                                        timeo, retrans, minor_version,
+                                        au_flavor);
+                if (!IS_ERR(clp))
+                        break;
+        }
+
+        if (IS_ERR(clp)) {
+                status = PTR_ERR(clp);
+                goto out;
+        }
+
+        status = nfs4_init_ds_session(clp, mds_srv->nfs_client->cl_lease_time);
+        if (status)
+                goto out_put;
+
+        smp_wmb();
+        ds->ds_clp = clp;
+        dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr);
+out:
+        return status;
+out_put:
+        nfs_put_client(clp);
+        goto out;
+}
+
+/*
+ * Create an rpc connection to the nfs4_pnfs_ds data server.
+ * Currently only supports IPv4 and IPv6 addresses.
+ * If connection fails, make devid unavailable.
+ */
+void nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
+                          struct nfs4_deviceid_node *devid, unsigned int timeo,
+                          unsigned int retrans, u32 version,
+                          u32 minor_version, rpc_authflavor_t au_flavor)
+{
+        if (test_and_set_bit(NFS4DS_CONNECTING, &ds->ds_state) == 0) {
+                int err = 0;
+
+                if (version == 3) {
+                        err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo,
+                                                       retrans, au_flavor);
+                } else if (version == 4) {
+                        err = _nfs4_pnfs_v4_ds_connect(mds_srv, ds, timeo,
+                                                       retrans, minor_version,
+                                                       au_flavor);
+                } else {
+                        dprintk("%s: unsupported DS version %d\n", __func__,
+                                version);
+                        err = -EPROTONOSUPPORT;
+                }
+
+                if (err)
+                        nfs4_mark_deviceid_unavailable(devid);
+                nfs4_clear_ds_conn_bit(ds);
+        } else {
+                nfs4_wait_ds_connect(ds);
+        }
+}
+EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_connect);
+
+/*
+ * Currently only supports ipv4, ipv6 and one multi-path address.
+ */
+struct nfs4_pnfs_ds_addr *
+nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
+{
+        struct nfs4_pnfs_ds_addr *da = NULL;
+        char *buf, *portstr;
+        __be16 port;
+        int nlen, rlen;
+        int tmp[2];
+        __be32 *p;
+        char *netid, *match_netid;
+        size_t len, match_netid_len;
+        char *startsep = "";
+        char *endsep = "";
+
+
+        /* r_netid */
+        p = xdr_inline_decode(xdr, 4);
+        if (unlikely(!p))
+                goto out_err;
+        nlen = be32_to_cpup(p++);
+
+        p = xdr_inline_decode(xdr, nlen);
+        if (unlikely(!p))
+                goto out_err;
+
+        netid = kmalloc(nlen+1, gfp_flags);
+        if (unlikely(!netid))
+                goto out_err;
+
+        netid[nlen] = '\0';
+        memcpy(netid, p, nlen);
+
+        /* r_addr: ip/ip6addr with port in dec octets - see RFC 5665 */
+        p = xdr_inline_decode(xdr, 4);
+        if (unlikely(!p))
+                goto out_free_netid;
+        rlen = be32_to_cpup(p);
+
+        p = xdr_inline_decode(xdr, rlen);
+        if (unlikely(!p))
+                goto out_free_netid;
+
+        /* port is ".ABC.DEF", 8 chars max */
+        if (rlen > INET6_ADDRSTRLEN + IPV6_SCOPE_ID_LEN + 8) {
+                dprintk("%s: Invalid address, length %d\n", __func__,
+                        rlen);
+                goto out_free_netid;
+        }
+        buf = kmalloc(rlen + 1, gfp_flags);
+        if (!buf) {
+                dprintk("%s: Not enough memory\n", __func__);
+                goto out_free_netid;
+        }
+        buf[rlen] = '\0';
+        memcpy(buf, p, rlen);
+
+        /* replace port '.' with '-' */
+        portstr = strrchr(buf, '.');
+        if (!portstr) {
+                dprintk("%s: Failed finding expected dot in port\n",
+                        __func__);
+                goto out_free_buf;
+        }
+        *portstr = '-';
+
+        /* find '.' between address and port */
+        portstr = strrchr(buf, '.');
+        if (!portstr) {
+                dprintk("%s: Failed finding expected dot between address and "
+                        "port\n", __func__);
+                goto out_free_buf;
+        }
+        *portstr = '\0';
+
+        da = kzalloc(sizeof(*da), gfp_flags);
+        if (unlikely(!da))
+                goto out_free_buf;
+
+        INIT_LIST_HEAD(&da->da_node);
+
+        if (!rpc_pton(net, buf, portstr-buf, (struct sockaddr *)&da->da_addr,
+                      sizeof(da->da_addr))) {
+                dprintk("%s: error parsing address %s\n", __func__, buf);
+                goto out_free_da;
+        }
+
+        portstr++;
+        sscanf(portstr, "%d-%d", &tmp[0], &tmp[1]);
+        port = htons((tmp[0] << 8) | (tmp[1]));
+
+        switch (da->da_addr.ss_family) {
+        case AF_INET:
+                ((struct sockaddr_in *)&da->da_addr)->sin_port = port;
+                da->da_addrlen = sizeof(struct sockaddr_in);
+                match_netid = "tcp";
+                match_netid_len = 3;
+                break;
+
+        case AF_INET6:
+                ((struct sockaddr_in6 *)&da->da_addr)->sin6_port = port;
+                da->da_addrlen = sizeof(struct sockaddr_in6);
+                match_netid = "tcp6";
+                match_netid_len = 4;
+                startsep = "[";
+                endsep = "]";
+                break;
+
+        default:
+                dprintk("%s: unsupported address family: %u\n",
+                        __func__, da->da_addr.ss_family);
+                goto out_free_da;
+        }
+
+        if (nlen != match_netid_len || strncmp(netid, match_netid, nlen)) {
+                dprintk("%s: ERROR: r_netid \"%s\" != \"%s\"\n",
+                        __func__, netid, match_netid);
+                goto out_free_da;
+        }
+
+        /* save human readable address */
+        len = strlen(startsep) + strlen(buf) + strlen(endsep) + 7;
+        da->da_remotestr = kzalloc(len, gfp_flags);
+
+        /* NULL is ok, only used for dprintk */
+        if (da->da_remotestr)
+                snprintf(da->da_remotestr, len, "%s%s%s:%u", startsep,
+                         buf, endsep, ntohs(port));
+
+        dprintk("%s: Parsed DS addr %s\n", __func__, da->da_remotestr);
+        kfree(buf);
+        kfree(netid);
+        return da;
+
+out_free_da:
+        kfree(da);
+out_free_buf:
+        dprintk("%s: Error parsing DS addr: %s\n", __func__, buf);
+        kfree(buf);
+out_free_netid:
+        kfree(netid);
+out_err:
+        return NULL;
+}
+EXPORT_SYMBOL_GPL(nfs4_decode_mp_ds_addr);
diff --git a/fs/nfs/read.c b/fs/nfs/read.c
index c91a4799c562..568ecf0a880f 100644
--- a/fs/nfs/read.c
+++ b/fs/nfs/read.c
@@ -70,8 +70,15 @@ EXPORT_SYMBOL_GPL(nfs_pageio_init_read);
 
 void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
 {
+        struct nfs_pgio_mirror *mirror;
+
         pgio->pg_ops = &nfs_pgio_rw_ops;
-        pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->rsize;
+
+        /* read path should never have more than one mirror */
+        WARN_ON_ONCE(pgio->pg_mirror_count != 1);
+
+        mirror = &pgio->pg_mirrors[0];
+        mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->rsize;
 }
 EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds);
 
@@ -81,6 +88,7 @@ int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode,
         struct nfs_page *new;
         unsigned int len;
         struct nfs_pageio_descriptor pgio;
+        struct nfs_pgio_mirror *pgm;
 
         len = nfs_page_length(page);
         if (len == 0)
@@ -97,7 +105,13 @@ int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode,
                              &nfs_async_read_completion_ops);
         nfs_pageio_add_request(&pgio, new);
         nfs_pageio_complete(&pgio);
-        NFS_I(inode)->read_io += pgio.pg_bytes_written;
+
+        /* It doesn't make sense to do mirrored reads! */
+        WARN_ON_ONCE(pgio.pg_mirror_count != 1);
+
+        pgm = &pgio.pg_mirrors[0];
+        NFS_I(inode)->read_io += pgm->pg_bytes_written;
+
         return 0;
 }
 
@@ -168,13 +182,14 @@ out:
 
 static void nfs_initiate_read(struct nfs_pgio_header *hdr,
                               struct rpc_message *msg,
+                              const struct nfs_rpc_ops *rpc_ops,
                               struct rpc_task_setup *task_setup_data, int how)
 {
         struct inode *inode = hdr->inode;
         int swap_flags = IS_SWAPFILE(inode) ? NFS_RPC_SWAPFLAGS : 0;
 
         task_setup_data->flags |= swap_flags;
-        NFS_PROTO(inode)->read_setup(hdr, msg);
+        rpc_ops->read_setup(hdr, msg);
 }
 
 static void
@@ -351,6 +366,7 @@ int nfs_readpages(struct file *filp, struct address_space *mapping,
                 struct list_head *pages, unsigned nr_pages)
 {
         struct nfs_pageio_descriptor pgio;
+        struct nfs_pgio_mirror *pgm;
         struct nfs_readdesc desc = {
                 .pgio = &pgio,
         };
@@ -386,10 +402,15 @@ int nfs_readpages(struct file *filp, struct address_space *mapping,
                              &nfs_async_read_completion_ops);
 
         ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc);
-
         nfs_pageio_complete(&pgio);
-        NFS_I(inode)->read_io += pgio.pg_bytes_written;
-        npages = (pgio.pg_bytes_written + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+
+        /* It doesn't make sense to do mirrored reads! */
+        WARN_ON_ONCE(pgio.pg_mirror_count != 1);
+
+        pgm = &pgio.pg_mirrors[0];
+        NFS_I(inode)->read_io += pgm->pg_bytes_written;
+        npages = (pgm->pg_bytes_written + PAGE_CACHE_SIZE - 1) >>
+                 PAGE_CACHE_SHIFT;
         nfs_add_stats(inode, NFSIOS_READPAGES, npages);
 read_complete:
         put_nfs_open_context(desc.ctx);
diff --git a/fs/nfs/super.c b/fs/nfs/super.c
index 31a11b0e885d..368d9395d2e7 100644
--- a/fs/nfs/super.c
+++ b/fs/nfs/super.c
@@ -405,12 +405,15 @@ void __exit unregister_nfs_fs(void)
         unregister_filesystem(&nfs_fs_type);
 }
 
-void nfs_sb_active(struct super_block *sb)
+bool nfs_sb_active(struct super_block *sb)
 {
         struct nfs_server *server = NFS_SB(sb);
 
-        if (atomic_inc_return(&server->active) == 1)
-                atomic_inc(&sb->s_active);
+        if (!atomic_inc_not_zero(&sb->s_active))
+                return false;
+        if (atomic_inc_return(&server->active) != 1)
+                atomic_dec(&sb->s_active);
+        return true;
 }
 EXPORT_SYMBOL_GPL(nfs_sb_active);
 
diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index 4ae66f416eb9..bcf83e535f29 100644
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -473,13 +473,18 @@ try_again:
         do {
                 /*
                  * Subrequests are always contiguous, non overlapping
-                 * and in order. If not, it's a programming error.
+                 * and in order - but may be repeated (mirrored writes).
                  */
-                WARN_ON_ONCE(subreq->wb_offset !=
-                     (head->wb_offset + total_bytes));
-
-                /* keep track of how many bytes this group covers */
-                total_bytes += subreq->wb_bytes;
+                if (subreq->wb_offset == (head->wb_offset + total_bytes)) {
+                        /* keep track of how many bytes this group covers */
+                        total_bytes += subreq->wb_bytes;
+                } else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset ||
+                            ((subreq->wb_offset + subreq->wb_bytes) >
+                             (head->wb_offset + total_bytes)))) {
+                        nfs_page_group_unlock(head);
+                        spin_unlock(&inode->i_lock);
+                        return ERR_PTR(-EIO);
+                }
 
                 if (!nfs_lock_request(subreq)) {
                         /* releases page group bit lock and
@@ -842,9 +847,9 @@ EXPORT_SYMBOL_GPL(nfs_init_cinfo);
  */
 void
 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
-                        struct nfs_commit_info *cinfo)
+                        struct nfs_commit_info *cinfo, u32 ds_commit_idx)
 {
-        if (pnfs_mark_request_commit(req, lseg, cinfo))
+        if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
                 return;
         nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo);
 }
@@ -900,7 +905,8 @@ static void nfs_write_completion(struct nfs_pgio_header *hdr)
                 }
                 if (nfs_write_need_commit(hdr)) {
                         memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
-                        nfs_mark_request_commit(req, hdr->lseg, &cinfo);
+                        nfs_mark_request_commit(req, hdr->lseg, &cinfo,
+                                hdr->pgio_mirror_idx);
                         goto next;
                 }
 remove_req:
@@ -1269,15 +1275,15 @@ static int flush_task_priority(int how)
 
 static void nfs_initiate_write(struct nfs_pgio_header *hdr,
                                struct rpc_message *msg,
+                               const struct nfs_rpc_ops *rpc_ops,
                                struct rpc_task_setup *task_setup_data, int how)
 {
-        struct inode *inode = hdr->inode;
         int priority = flush_task_priority(how);
 
         task_setup_data->priority = priority;
-        NFS_PROTO(inode)->write_setup(hdr, msg);
+        rpc_ops->write_setup(hdr, msg);
 
-        nfs4_state_protect_write(NFS_SERVER(inode)->nfs_client,
+        nfs4_state_protect_write(NFS_SERVER(hdr->inode)->nfs_client,
                                  &task_setup_data->rpc_client, msg, hdr);
 }
 
@@ -1327,8 +1333,14 @@ EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
 
 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
 {
+        struct nfs_pgio_mirror *mirror;
+
         pgio->pg_ops = &nfs_pgio_rw_ops;
-        pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
+
+        nfs_pageio_stop_mirroring(pgio);
+
+        mirror = &pgio->pg_mirrors[0];
+        mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
 }
 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
 
@@ -1494,6 +1506,7 @@ void nfs_commitdata_release(struct nfs_commit_data *data)
 EXPORT_SYMBOL_GPL(nfs_commitdata_release);
 
 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
+                        const struct nfs_rpc_ops *nfs_ops,
                         const struct rpc_call_ops *call_ops,
                         int how, int flags)
 {
@@ -1515,7 +1528,7 @@ int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
                 .priority = priority,
         };
         /* Set up the initial task struct.  */
-        NFS_PROTO(data->inode)->commit_setup(data, &msg);
+        nfs_ops->commit_setup(data, &msg);
 
         dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
 
@@ -1583,14 +1596,15 @@ EXPORT_SYMBOL_GPL(nfs_init_commit);
 
 void nfs_retry_commit(struct list_head *page_list,
                       struct pnfs_layout_segment *lseg,
-                      struct nfs_commit_info *cinfo)
+                      struct nfs_commit_info *cinfo,
+                      u32 ds_commit_idx)
 {
         struct nfs_page *req;
 
         while (!list_empty(page_list)) {
                 req = nfs_list_entry(page_list->next);
                 nfs_list_remove_request(req);
-                nfs_mark_request_commit(req, lseg, cinfo);
+                nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
                 if (!cinfo->dreq) {
                         dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
                         dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
@@ -1618,10 +1632,10 @@ nfs_commit_list(struct inode *inode, struct list_head *head, int how,
         /* Set up the argument struct */
         nfs_init_commit(data, head, NULL, cinfo);
         atomic_inc(&cinfo->mds->rpcs_out);
-        return nfs_initiate_commit(NFS_CLIENT(inode), data, data->mds_ops,
-                                   how, 0);
+        return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
+                                   data->mds_ops, how, 0);
  out_bad:
-        nfs_retry_commit(head, NULL, cinfo);
+        nfs_retry_commit(head, NULL, cinfo, 0);
         cinfo->completion_ops->error_cleanup(NFS_I(inode));
         return -ENOMEM;
 }
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index 022b761dbf0a..de7c91ca427e 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -516,6 +516,7 @@ enum pnfs_layouttype {
         LAYOUT_NFSV4_1_FILES  = 1,
         LAYOUT_OSD2_OBJECTS = 2,
         LAYOUT_BLOCK_VOLUME = 3,
+        LAYOUT_FLEX_FILES = 4,
 };
 
 /* used for both layout return and recall */
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index ddea982355f3..5e1273d4de14 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -77,10 +77,6 @@ struct nfs_client {
         /* Client owner identifier */
         const char *            cl_owner_id;
 
-        /* Our own IP address, as a null-terminated string.
-         * This is used to generate the mv0 callback address.
-         */
-        char                    cl_ipaddr[48];
         u32                     cl_cb_ident;    /* v4.0 callback identifier */
         const struct nfs4_minor_version_ops *cl_mvops;
         unsigned long           cl_mig_gen;
@@ -108,6 +104,11 @@ struct nfs_client {
 #define NFS_SP4_MACH_CRED_COMMIT   6    /* COMMIT */
 #endif /* CONFIG_NFS_V4 */
 
+        /* Our own IP address, as a null-terminated string.
+         * This is used to generate the mv0 callback address.
+         */
+        char                    cl_ipaddr[48];
+
 #ifdef CONFIG_NFS_FSCACHE
         struct fscache_cookie   *fscache;       /* client index cache cookie */
 #endif
diff --git a/include/linux/nfs_idmap.h b/include/linux/nfs_idmap.h
index 0f4b79da6584..333844e38f66 100644
--- a/include/linux/nfs_idmap.h
+++ b/include/linux/nfs_idmap.h
@@ -73,5 +73,7 @@ int nfs_map_group_to_gid(const struct nfs_server *, const char *, size_t, kgid_t
 int nfs_map_uid_to_name(const struct nfs_server *, kuid_t, char *, size_t);
 int nfs_map_gid_to_group(const struct nfs_server *, kgid_t, char *, size_t);
 
+int nfs_map_string_to_numeric(const char *name, size_t namelen, __u32 *res);
+
 extern unsigned int nfs_idmap_cache_timeout;
 #endif /* NFS_IDMAP_H */
diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h
index 6c3e06ee2fb7..3eb072dbce83 100644
--- a/include/linux/nfs_page.h
+++ b/include/linux/nfs_page.h
@@ -58,6 +58,9 @@ struct nfs_pageio_ops {
         size_t  (*pg_test)(struct nfs_pageio_descriptor *, struct nfs_page *,
                            struct nfs_page *);
         int     (*pg_doio)(struct nfs_pageio_descriptor *);
+        unsigned int    (*pg_get_mirror_count)(struct nfs_pageio_descriptor *,
+                                       struct nfs_page *);
+        void    (*pg_cleanup)(struct nfs_pageio_descriptor *);
 };
 
 struct nfs_rw_ops {
@@ -69,18 +72,21 @@ struct nfs_rw_ops {
                         struct inode *);
         void (*rw_result)(struct rpc_task *, struct nfs_pgio_header *);
         void (*rw_initiate)(struct nfs_pgio_header *, struct rpc_message *,
+                            const struct nfs_rpc_ops *,
                             struct rpc_task_setup *, int);
 };
 
-struct nfs_pageio_descriptor {
+struct nfs_pgio_mirror {
         struct list_head        pg_list;
         unsigned long           pg_bytes_written;
         size_t                  pg_count;
         size_t                  pg_bsize;
         unsigned int            pg_base;
-        unsigned char           pg_moreio : 1,
-                                pg_recoalesce : 1;
+        unsigned char           pg_recoalesce : 1;
+};
 
+struct nfs_pageio_descriptor {
+        unsigned char           pg_moreio : 1;
         struct inode            *pg_inode;
         const struct nfs_pageio_ops *pg_ops;
         const struct nfs_rw_ops *pg_rw_ops;
@@ -91,8 +97,18 @@ struct nfs_pageio_descriptor {
         struct pnfs_layout_segment *pg_lseg;
         struct nfs_direct_req   *pg_dreq;
         void                    *pg_layout_private;
+        unsigned int            pg_bsize;       /* default bsize for mirrors */
+
+        u32                     pg_mirror_count;
+        struct nfs_pgio_mirror  *pg_mirrors;
+        struct nfs_pgio_mirror  pg_mirrors_static[1];
+        struct nfs_pgio_mirror  *pg_mirrors_dynamic;
+        u32                     pg_mirror_idx;  /* current mirror */
 };
 
+/* arbitrarily selected limit to number of mirrors */
+#define NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX 16
+
 #define NFS_WBACK_BUSY(req)     (test_bit(PG_BUSY,&(req)->wb_flags))
 
 extern  struct nfs_page *nfs_create_request(struct nfs_open_context *ctx,
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index 467c84efb596..38d96ba935c2 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -285,6 +285,7 @@ struct nfs4_layoutcommit_data {
         struct nfs_fattr fattr;
         struct list_head lseg_list;
         struct rpc_cred *cred;
+        struct inode *inode;
         struct nfs4_layoutcommit_args args;
         struct nfs4_layoutcommit_res res;
 };
@@ -293,6 +294,7 @@ struct nfs4_layoutreturn_args {
         struct nfs4_sequence_args seq_args;
         struct pnfs_layout_hdr *layout;
         struct inode *inode;
+        struct pnfs_layout_range range;
         nfs4_stateid stateid;
         __u32   layout_type;
 };
@@ -308,6 +310,7 @@ struct nfs4_layoutreturn {
         struct nfs4_layoutreturn_res res;
         struct rpc_cred *cred;
         struct nfs_client *clp;
+        struct inode *inode;
         int rpc_status;
 };
 
@@ -325,6 +328,7 @@ struct nfs_openargs {
         struct nfs_seqid *      seqid;
         int                     open_flags;
         fmode_t                 fmode;
+        u32                     share_access;
         u32                     access;
         __u64                   clientid;
         struct stateowner_id    id;
@@ -389,9 +393,10 @@ struct nfs_open_confirmres {
 struct nfs_closeargs {
         struct nfs4_sequence_args       seq_args;
         struct nfs_fh *         fh;
-        nfs4_stateid *          stateid;
+        nfs4_stateid            stateid;
         struct nfs_seqid *      seqid;
         fmode_t                 fmode;
+        u32                     share_access;
         const u32 *             bitmask;
 };
 
@@ -416,12 +421,13 @@ struct nfs_lock_args {
         struct nfs_fh *         fh;
         struct file_lock *      fl;
         struct nfs_seqid *      lock_seqid;
-        nfs4_stateid *          lock_stateid;
+        nfs4_stateid            lock_stateid;
         struct nfs_seqid *      open_seqid;
-        nfs4_stateid *          open_stateid;
+        nfs4_stateid            open_stateid;
         struct nfs_lowner       lock_owner;
         unsigned char           block : 1;
         unsigned char           reclaim : 1;
+        unsigned char           new_lock : 1;
         unsigned char           new_lock_owner : 1;
 };
 
@@ -437,7 +443,7 @@ struct nfs_locku_args {
         struct nfs_fh *         fh;
         struct file_lock *      fl;
         struct nfs_seqid *      seqid;
-        nfs4_stateid *          stateid;
+        nfs4_stateid            stateid;
 };
 
 struct nfs_locku_res {
@@ -513,6 +519,7 @@ struct nfs_pgio_res {
         struct nfs4_sequence_res        seq_res;
         struct nfs_fattr *      fattr;
         __u32                   count;
+        __u32                   op_status;
         int                     eof;            /* used by read */
         struct nfs_writeverf *  verf;           /* used by write */
         const struct nfs_server *server;        /* used by write */
@@ -532,6 +539,7 @@ struct nfs_commitargs {
 
 struct nfs_commitres {
         struct nfs4_sequence_res        seq_res;
+        __u32                   op_status;
         struct nfs_fattr        *fattr;
         struct nfs_writeverf    *verf;
         const struct nfs_server *server;
@@ -1325,7 +1333,8 @@ struct nfs_pgio_header {
         __u64                   mds_offset;     /* Filelayout dense stripe */
         struct nfs_page_array   page_array;
         struct nfs_client       *ds_clp;        /* pNFS data server */
-        int                     ds_idx;         /* ds index if ds_clp is set */
+        int                     ds_commit_idx;  /* ds index if ds_clp is set */
+        int                     pgio_mirror_idx;/* mirror index in pgio layer */
 };
 
 struct nfs_mds_commit_info {
diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
index d86acc63b25f..598ba80ec30c 100644
--- a/include/linux/sunrpc/clnt.h
+++ b/include/linux/sunrpc/clnt.h
@@ -57,7 +57,7 @@ struct rpc_clnt {
         const struct rpc_timeout *cl_timeout;   /* Timeout strategy */
 
         int                     cl_nodelen;     /* nodename length */
-        char                    cl_nodename[UNX_MAXNODENAME];
+        char                    cl_nodename[UNX_MAXNODENAME+1];
         struct rpc_pipe_dir_head cl_pipedir_objects;
         struct rpc_clnt *       cl_parent;      /* Points to parent of clones */
         struct rpc_rtt          cl_rtt_default;
@@ -112,6 +112,7 @@ struct rpc_create_args {
         struct sockaddr         *saddress;
         const struct rpc_timeout *timeout;
         const char              *servername;
+        const char              *nodename;
         const struct rpc_program *program;
         u32                     prognumber;     /* overrides program->number */
         u32                     version;
diff --git a/include/linux/sunrpc/metrics.h b/include/linux/sunrpc/metrics.h
index eecb5a71e6c0..7e61a17030a4 100644
--- a/include/linux/sunrpc/metrics.h
+++ b/include/linux/sunrpc/metrics.h
@@ -79,6 +79,8 @@ struct rpc_clnt;
 struct rpc_iostats *    rpc_alloc_iostats(struct rpc_clnt *);
 void                    rpc_count_iostats(const struct rpc_task *,
                                           struct rpc_iostats *);
+void                    rpc_count_iostats_metrics(const struct rpc_task *,
+                                          struct rpc_iostats *);
 void                    rpc_print_iostats(struct seq_file *, struct rpc_clnt *);
 void                    rpc_free_iostats(struct rpc_iostats *);
 
@@ -87,6 +89,8 @@ void			rpc_free_iostats(struct rpc_iostats *);
 static inline struct rpc_iostats *rpc_alloc_iostats(struct rpc_clnt *clnt) { return NULL; }
 static inline void rpc_count_iostats(const struct rpc_task *task,
                                      struct rpc_iostats *stats) {}
+static inline void rpc_count_iostats_metrics(const struct rpc_task *,
+                                             struct rpc_iostats *) {}
 static inline void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt) {}
 static inline void rpc_free_iostats(struct rpc_iostats *stats) {}
 
diff --git a/include/linux/sunrpc/rpc_rdma.h b/include/linux/sunrpc/rpc_rdma.h
index b78f16b1dea3..f33c5a4d6fe4 100644
--- a/include/linux/sunrpc/rpc_rdma.h
+++ b/include/linux/sunrpc/rpc_rdma.h
@@ -42,6 +42,9 @@
 
 #include <linux/types.h>
 
+#define RPCRDMA_VERSION         1
+#define rpcrdma_version         cpu_to_be32(RPCRDMA_VERSION)
+
 struct rpcrdma_segment {
         __be32 rs_handle;       /* Registered memory handle */
         __be32 rs_length;       /* Length of the chunk in bytes */
@@ -95,7 +98,10 @@ struct rpcrdma_msg {
         } rm_body;
 };
 
-#define RPCRDMA_HDRLEN_MIN      28
+/*
+ * Smallest RPC/RDMA header: rm_xid through rm_type, then rm_nochunks
+ */
+#define RPCRDMA_HDRLEN_MIN      (sizeof(__be32) * 7)
 
 enum rpcrdma_errcode {
         ERR_VERS = 1,
@@ -115,4 +121,10 @@ enum rpcrdma_proc {
         RDMA_ERROR = 4          /* An RPC RDMA encoding error */
 };
 
+#define rdma_msg        cpu_to_be32(RDMA_MSG)
+#define rdma_nomsg      cpu_to_be32(RDMA_NOMSG)
+#define rdma_msgp       cpu_to_be32(RDMA_MSGP)
+#define rdma_done       cpu_to_be32(RDMA_DONE)
+#define rdma_error      cpu_to_be32(RDMA_ERROR)
+
 #endif                          /* _LINUX_SUNRPC_RPC_RDMA_H */
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 975da754c778..ddfe88f52219 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -63,8 +63,6 @@ extern atomic_t rdma_stat_rq_prod;
 extern atomic_t rdma_stat_sq_poll;
 extern atomic_t rdma_stat_sq_prod;
 
-#define RPCRDMA_VERSION 1
-
 /*
  * Contexts are built when an RDMA request is created and are a
  * record of the resources that can be recovered when the request
diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index 9d27ac45b909..8b93ef53df3c 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -347,6 +347,9 @@ void			xprt_force_disconnect(struct rpc_xprt *xprt);
 void                    xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie);
 int                     xs_swapper(struct rpc_xprt *xprt, int enable);
 
+bool                    xprt_lock_connect(struct rpc_xprt *, struct rpc_task *, void *);
+void                    xprt_unlock_connect(struct rpc_xprt *, void *);
+
 /*
  * Reserved bit positions in xprt->state
  */
@@ -357,10 +360,7 @@ int			xs_swapper(struct rpc_xprt *xprt, int enable);
 #define XPRT_BOUND              (4)
 #define XPRT_BINDING            (5)
 #define XPRT_CLOSING            (6)
-#define XPRT_CONNECTION_ABORT   (7)
-#define XPRT_CONNECTION_CLOSE   (8)
 #define XPRT_CONGESTED          (9)
-#define XPRT_CONNECTION_REUSE   (10)
 
 static inline void xprt_set_connected(struct rpc_xprt *xprt)
 {
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 05da12a33945..612aa73bbc60 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -286,10 +286,8 @@ static struct rpc_xprt *rpc_clnt_set_transport(struct rpc_clnt *clnt,
 
 static void rpc_clnt_set_nodename(struct rpc_clnt *clnt, const char *nodename)
 {
-        clnt->cl_nodelen = strlen(nodename);
-        if (clnt->cl_nodelen > UNX_MAXNODENAME)
-                clnt->cl_nodelen = UNX_MAXNODENAME;
-        memcpy(clnt->cl_nodename, nodename, clnt->cl_nodelen);
+        clnt->cl_nodelen = strlcpy(clnt->cl_nodename,
+                        nodename, sizeof(clnt->cl_nodename));
 }
 
 static int rpc_client_register(struct rpc_clnt *clnt,
@@ -365,6 +363,7 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args,
         const struct rpc_version *version;
         struct rpc_clnt *clnt = NULL;
         const struct rpc_timeout *timeout;
+        const char *nodename = args->nodename;
         int err;
 
         /* sanity check the name before trying to print it */
@@ -420,8 +419,10 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args,
 
         atomic_set(&clnt->cl_count, 1);
 
+        if (nodename == NULL)
+                nodename = utsname()->nodename;
         /* save the nodename */
-        rpc_clnt_set_nodename(clnt, utsname()->nodename);
+        rpc_clnt_set_nodename(clnt, nodename);
 
         err = rpc_client_register(clnt, args->authflavor, args->client_name);
         if (err)
@@ -576,6 +577,7 @@ static struct rpc_clnt *__rpc_clone_client(struct rpc_create_args *args,
         if (xprt == NULL)
                 goto out_err;
         args->servername = xprt->servername;
+        args->nodename = clnt->cl_nodename;
 
         new = rpc_new_client(args, xprt, clnt);
         if (IS_ERR(new)) {
@@ -1824,6 +1826,7 @@ call_connect_status(struct rpc_task *task)
         case -ECONNABORTED:
         case -ENETUNREACH:
         case -EHOSTUNREACH:
+        case -EADDRINUSE:
         case -ENOBUFS:
         case -EPIPE:
                 if (RPC_IS_SOFTCONN(task))
@@ -1932,6 +1935,7 @@ call_transmit_status(struct rpc_task *task)
                 }
         case -ECONNRESET:
         case -ECONNABORTED:
+        case -EADDRINUSE:
         case -ENOTCONN:
         case -ENOBUFS:
         case -EPIPE:
@@ -2051,6 +2055,7 @@ call_status(struct rpc_task *task)
         case -ECONNRESET:
         case -ECONNABORTED:
                 rpc_force_rebind(clnt);
+        case -EADDRINUSE:
         case -ENOBUFS:
                 rpc_delay(task, 3*HZ);
         case -EPIPE:
diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c
index 05202012bcfc..cf5770d8f49a 100644
--- a/net/sunrpc/rpcb_clnt.c
+++ b/net/sunrpc/rpcb_clnt.c
@@ -355,7 +355,8 @@ out:
         return result;
 }
 
-static struct rpc_clnt *rpcb_create(struct net *net, const char *hostname,
+static struct rpc_clnt *rpcb_create(struct net *net, const char *nodename,
+                                    const char *hostname,
                                     struct sockaddr *srvaddr, size_t salen,
                                     int proto, u32 version)
 {
@@ -365,6 +366,7 @@ static struct rpc_clnt *rpcb_create(struct net *net, const char *hostname,
                 .address        = srvaddr,
                 .addrsize       = salen,
                 .servername     = hostname,
+                .nodename       = nodename,
                 .program        = &rpcb_program,
                 .version        = version,
                 .authflavor     = RPC_AUTH_UNIX,
@@ -740,7 +742,9 @@ void rpcb_getport_async(struct rpc_task *task)
         dprintk("RPC: %5u %s: trying rpcbind version %u\n",
                 task->tk_pid, __func__, bind_version);
 
-        rpcb_clnt = rpcb_create(xprt->xprt_net, xprt->servername, sap, salen,
+        rpcb_clnt = rpcb_create(xprt->xprt_net,
+                                clnt->cl_nodename,
+                                xprt->servername, sap, salen,
                                 xprt->prot, bind_version);
         if (IS_ERR(rpcb_clnt)) {
                 status = PTR_ERR(rpcb_clnt);
diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
index d20f2329eea3..b91fd9c597b4 100644
--- a/net/sunrpc/sched.c
+++ b/net/sunrpc/sched.c
@@ -844,10 +844,10 @@ static void rpc_async_schedule(struct work_struct *work)
 void *rpc_malloc(struct rpc_task *task, size_t size)
 {
         struct rpc_buffer *buf;
-        gfp_t gfp = GFP_NOWAIT | __GFP_NOWARN;
+        gfp_t gfp = GFP_NOIO | __GFP_NOWARN;
 
         if (RPC_IS_SWAPPER(task))
-                gfp |= __GFP_MEMALLOC;
+                gfp = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN;
 
         size += sizeof(struct rpc_buffer);
         if (size <= RPC_BUFFER_MAXSIZE)
@@ -1069,7 +1069,8 @@ static int rpciod_start(void)
          * Create the rpciod thread and wait for it to start.
          */
         dprintk("RPC:       creating workqueue rpciod\n");
-        wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM, 1);
+        /* Note: highpri because network receive is latency sensitive */
+        wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
         rpciod_workqueue = wq;
         return rpciod_workqueue != NULL;
 }
diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
index 9711a155bc50..2ecb994314c1 100644
--- a/net/sunrpc/stats.c
+++ b/net/sunrpc/stats.c
@@ -140,22 +140,20 @@ void rpc_free_iostats(struct rpc_iostats *stats)
 EXPORT_SYMBOL_GPL(rpc_free_iostats);
 
 /**
- * rpc_count_iostats - tally up per-task stats
+ * rpc_count_iostats_metrics - tally up per-task stats
  * @task: completed rpc_task
- * @stats: array of stat structures
+ * @op_metrics: stat structure for OP that will accumulate stats from @task
  */
-void rpc_count_iostats(const struct rpc_task *task, struct rpc_iostats *stats)
+void rpc_count_iostats_metrics(const struct rpc_task *task,
+                               struct rpc_iostats *op_metrics)
 {
         struct rpc_rqst *req = task->tk_rqstp;
-        struct rpc_iostats *op_metrics;
         ktime_t delta, now;
 
-        if (!stats || !req)
+        if (!op_metrics || !req)
                 return;
 
         now = ktime_get();
-        op_metrics = &stats[task->tk_msg.rpc_proc->p_statidx];
-
         spin_lock(&op_metrics->om_lock);
 
         op_metrics->om_ops++;
@@ -175,6 +173,20 @@ void rpc_count_iostats(const struct rpc_task *task, struct rpc_iostats *stats)
 
         spin_unlock(&op_metrics->om_lock);
 }
+EXPORT_SYMBOL_GPL(rpc_count_iostats_metrics);
+
+/**
+ * rpc_count_iostats - tally up per-task stats
+ * @task: completed rpc_task
+ * @stats: array of stat structures
+ *
+ * Uses the statidx from @task
+ */
+void rpc_count_iostats(const struct rpc_task *task, struct rpc_iostats *stats)
+{
+        rpc_count_iostats_metrics(task,
+                                  &stats[task->tk_msg.rpc_proc->p_statidx]);
+}
 EXPORT_SYMBOL_GPL(rpc_count_iostats);
 
 static void _print_name(struct seq_file *seq, unsigned int op,
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index ebbefad21a37..e3015aede0d9 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -683,13 +683,43 @@ xprt_init_autodisconnect(unsigned long data)
         if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
                 goto out_abort;
         spin_unlock(&xprt->transport_lock);
-        set_bit(XPRT_CONNECTION_CLOSE, &xprt->state);
         queue_work(rpciod_workqueue, &xprt->task_cleanup);
         return;
 out_abort:
         spin_unlock(&xprt->transport_lock);
 }
 
+bool xprt_lock_connect(struct rpc_xprt *xprt,
+                struct rpc_task *task,
+                void *cookie)
+{
+        bool ret = false;
+
+        spin_lock_bh(&xprt->transport_lock);
+        if (!test_bit(XPRT_LOCKED, &xprt->state))
+                goto out;
+        if (xprt->snd_task != task)
+                goto out;
+        xprt->snd_task = cookie;
+        ret = true;
+out:
+        spin_unlock_bh(&xprt->transport_lock);
+        return ret;
+}
+
+void xprt_unlock_connect(struct rpc_xprt *xprt, void *cookie)
+{
+        spin_lock_bh(&xprt->transport_lock);
+        if (xprt->snd_task != cookie)
+                goto out;
+        if (!test_bit(XPRT_LOCKED, &xprt->state))
+                goto out;
+        xprt->snd_task =NULL;
+        xprt->ops->release_xprt(xprt, NULL);
+out:
+        spin_unlock_bh(&xprt->transport_lock);
+}
+
 /**
  * xprt_connect - schedule a transport connect operation
  * @task: RPC task that is requesting the connect
@@ -712,9 +742,7 @@ void xprt_connect(struct rpc_task *task)
         if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state))
                 xprt->ops->close(xprt);
 
-        if (xprt_connected(xprt))
-                xprt_release_write(xprt, task);
-        else {
+        if (!xprt_connected(xprt)) {
                 task->tk_rqstp->rq_bytes_sent = 0;
                 task->tk_timeout = task->tk_rqstp->rq_timeout;
                 rpc_sleep_on(&xprt->pending, task, xprt_connect_status);
@@ -726,6 +754,7 @@ void xprt_connect(struct rpc_task *task)
                 xprt->stat.connect_start = jiffies;
                 xprt->ops->connect(xprt, task);
         }
+        xprt_release_write(xprt, task);
 }
 
 static void xprt_connect_status(struct rpc_task *task)
@@ -758,7 +787,6 @@ static void xprt_connect_status(struct rpc_task *task)
                 dprintk("RPC: %5u xprt_connect_status: error %d connecting to "
                                 "server %s\n", task->tk_pid, -task->tk_status,
                                 xprt->servername);
-                xprt_release_write(xprt, task);
                 task->tk_status = -EIO;
         }
 }
diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index df01d124936c..7e9acd9361c5 100644
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -209,9 +209,11 @@ rpcrdma_create_chunks(struct rpc_rqst *rqst, struct xdr_buf *target,
                 if (cur_rchunk) {       /* read */
                         cur_rchunk->rc_discrim = xdr_one;
                         /* all read chunks have the same "position" */
-                        cur_rchunk->rc_position = htonl(pos);
-                        cur_rchunk->rc_target.rs_handle = htonl(seg->mr_rkey);
-                        cur_rchunk->rc_target.rs_length = htonl(seg->mr_len);
+                        cur_rchunk->rc_position = cpu_to_be32(pos);
+                        cur_rchunk->rc_target.rs_handle =
+                                                cpu_to_be32(seg->mr_rkey);
+                        cur_rchunk->rc_target.rs_length =
+                                                cpu_to_be32(seg->mr_len);
                         xdr_encode_hyper(
                                         (__be32 *)&cur_rchunk->rc_target.rs_offset,
                                         seg->mr_base);
@@ -222,8 +224,10 @@ rpcrdma_create_chunks(struct rpc_rqst *rqst, struct xdr_buf *target,
                         cur_rchunk++;
                         r_xprt->rx_stats.read_chunk_count++;
                 } else {                /* write/reply */
-                        cur_wchunk->wc_target.rs_handle = htonl(seg->mr_rkey);
-                        cur_wchunk->wc_target.rs_length = htonl(seg->mr_len);
+                        cur_wchunk->wc_target.rs_handle =
+                                                cpu_to_be32(seg->mr_rkey);
+                        cur_wchunk->wc_target.rs_length =
+                                                cpu_to_be32(seg->mr_len);
                         xdr_encode_hyper(
                                         (__be32 *)&cur_wchunk->wc_target.rs_offset,
                                         seg->mr_base);
@@ -257,7 +261,7 @@ rpcrdma_create_chunks(struct rpc_rqst *rqst, struct xdr_buf *target,
                 *iptr++ = xdr_zero;     /* encode a NULL reply chunk */
         } else {
                 warray->wc_discrim = xdr_one;
-                warray->wc_nchunks = htonl(nchunks);
+                warray->wc_nchunks = cpu_to_be32(nchunks);
                 iptr = (__be32 *) cur_wchunk;
                 if (type == rpcrdma_writech) {
                         *iptr++ = xdr_zero; /* finish the write chunk list */
@@ -290,7 +294,7 @@ ssize_t
 rpcrdma_marshal_chunks(struct rpc_rqst *rqst, ssize_t result)
 {
         struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
-        struct rpcrdma_msg *headerp = (struct rpcrdma_msg *)req->rl_base;
+        struct rpcrdma_msg *headerp = rdmab_to_msg(req->rl_rdmabuf);
 
         if (req->rl_rtype != rpcrdma_noch)
                 result = rpcrdma_create_chunks(rqst, &rqst->rq_snd_buf,
@@ -402,13 +406,12 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst)
         base = rqst->rq_svec[0].iov_base;
         rpclen = rqst->rq_svec[0].iov_len;
 
-        /* build RDMA header in private area at front */
-        headerp = (struct rpcrdma_msg *) req->rl_base;
-        /* don't htonl XID, it's already done in request */
+        headerp = rdmab_to_msg(req->rl_rdmabuf);
+        /* don't byte-swap XID, it's already done in request */
         headerp->rm_xid = rqst->rq_xid;
-        headerp->rm_vers = xdr_one;
-        headerp->rm_credit = htonl(r_xprt->rx_buf.rb_max_requests);
-        headerp->rm_type = htonl(RDMA_MSG);
+        headerp->rm_vers = rpcrdma_version;
+        headerp->rm_credit = cpu_to_be32(r_xprt->rx_buf.rb_max_requests);
+        headerp->rm_type = rdma_msg;
 
         /*
          * Chunks needed for results?
@@ -468,7 +471,7 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst)
                 return -EIO;
         }
 
-        hdrlen = 28; /*sizeof *headerp;*/
+        hdrlen = RPCRDMA_HDRLEN_MIN;
         padlen = 0;
 
         /*
@@ -482,11 +485,11 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst)
                                                 RPCRDMA_INLINE_PAD_VALUE(rqst));
 
                 if (padlen) {
-                        headerp->rm_type = htonl(RDMA_MSGP);
+                        headerp->rm_type = rdma_msgp;
                         headerp->rm_body.rm_padded.rm_align =
-                                htonl(RPCRDMA_INLINE_PAD_VALUE(rqst));
+                                cpu_to_be32(RPCRDMA_INLINE_PAD_VALUE(rqst));
                         headerp->rm_body.rm_padded.rm_thresh =
-                                htonl(RPCRDMA_INLINE_PAD_THRESH);
+                                cpu_to_be32(RPCRDMA_INLINE_PAD_THRESH);
                         headerp->rm_body.rm_padded.rm_pempty[0] = xdr_zero;
                         headerp->rm_body.rm_padded.rm_pempty[1] = xdr_zero;
                         headerp->rm_body.rm_padded.rm_pempty[2] = xdr_zero;
@@ -524,7 +527,7 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst)
         dprintk("RPC:       %s: %s: hdrlen %zd rpclen %zd padlen %zd"
                 " headerp 0x%p base 0x%p lkey 0x%x\n",
                 __func__, transfertypes[req->rl_wtype], hdrlen, rpclen, padlen,
-                headerp, base, req->rl_iov.lkey);
+                headerp, base, rdmab_lkey(req->rl_rdmabuf));
 
         /*
          * initialize send_iov's - normally only two: rdma chunk header and
@@ -533,26 +536,26 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst)
          * header and any write data. In all non-rdma cases, any following
          * data has been copied into the RPC header buffer.
          */
-        req->rl_send_iov[0].addr = req->rl_iov.addr;
+        req->rl_send_iov[0].addr = rdmab_addr(req->rl_rdmabuf);
         req->rl_send_iov[0].length = hdrlen;
-        req->rl_send_iov[0].lkey = req->rl_iov.lkey;
+        req->rl_send_iov[0].lkey = rdmab_lkey(req->rl_rdmabuf);
 
-        req->rl_send_iov[1].addr = req->rl_iov.addr + (base - req->rl_base);
+        req->rl_send_iov[1].addr = rdmab_addr(req->rl_sendbuf);
         req->rl_send_iov[1].length = rpclen;
-        req->rl_send_iov[1].lkey = req->rl_iov.lkey;
+        req->rl_send_iov[1].lkey = rdmab_lkey(req->rl_sendbuf);
 
         req->rl_niovs = 2;
 
         if (padlen) {
                 struct rpcrdma_ep *ep = &r_xprt->rx_ep;
 
-                req->rl_send_iov[2].addr = ep->rep_pad.addr;
+                req->rl_send_iov[2].addr = rdmab_addr(ep->rep_padbuf);
                 req->rl_send_iov[2].length = padlen;
-                req->rl_send_iov[2].lkey = ep->rep_pad.lkey;
+                req->rl_send_iov[2].lkey = rdmab_lkey(ep->rep_padbuf);
 
                 req->rl_send_iov[3].addr = req->rl_send_iov[1].addr + rpclen;
                 req->rl_send_iov[3].length = rqst->rq_slen - rpclen;
-                req->rl_send_iov[3].lkey = req->rl_iov.lkey;
+                req->rl_send_iov[3].lkey = rdmab_lkey(req->rl_sendbuf);
 
                 req->rl_niovs = 4;
         }
@@ -569,8 +572,9 @@ rpcrdma_count_chunks(struct rpcrdma_rep *rep, unsigned int max, int wrchunk, __b
 {
         unsigned int i, total_len;
         struct rpcrdma_write_chunk *cur_wchunk;
+        char *base = (char *)rdmab_to_msg(rep->rr_rdmabuf);
 
-        i = ntohl(**iptrp);     /* get array count */
+        i = be32_to_cpu(**iptrp);
         if (i > max)
                 return -1;
         cur_wchunk = (struct rpcrdma_write_chunk *) (*iptrp + 1);
@@ -582,11 +586,11 @@ rpcrdma_count_chunks(struct rpcrdma_rep *rep, unsigned int max, int wrchunk, __b
                         xdr_decode_hyper((__be32 *)&seg->rs_offset, &off);
                         dprintk("RPC:       %s: chunk %d@0x%llx:0x%x\n",
                                 __func__,
-                                ntohl(seg->rs_length),
+                                be32_to_cpu(seg->rs_length),
                                 (unsigned long long)off,
-                                ntohl(seg->rs_handle));
+                                be32_to_cpu(seg->rs_handle));
                 }
-                total_len += ntohl(seg->rs_length);
+                total_len += be32_to_cpu(seg->rs_length);
                 ++cur_wchunk;
         }
         /* check and adjust for properly terminated write chunk */
@@ -596,7 +600,7 @@ rpcrdma_count_chunks(struct rpcrdma_rep *rep, unsigned int max, int wrchunk, __b
                         return -1;
                 cur_wchunk = (struct rpcrdma_write_chunk *) w;
         }
-        if ((char *) cur_wchunk > rep->rr_base + rep->rr_len)
+        if ((char *)cur_wchunk > base + rep->rr_len)
                 return -1;
 
         *iptrp = (__be32 *) cur_wchunk;
@@ -691,7 +695,9 @@ rpcrdma_connect_worker(struct work_struct *work)
 {
         struct rpcrdma_ep *ep =
                 container_of(work, struct rpcrdma_ep, rep_connect_worker.work);
-        struct rpc_xprt *xprt = ep->rep_xprt;
+        struct rpcrdma_xprt *r_xprt =
+                container_of(ep, struct rpcrdma_xprt, rx_ep);
+        struct rpc_xprt *xprt = &r_xprt->rx_xprt;
 
         spin_lock_bh(&xprt->transport_lock);
         if (++xprt->connect_cookie == 0)        /* maintain a reserved value */
@@ -732,7 +738,7 @@ rpcrdma_reply_handler(struct rpcrdma_rep *rep)
         struct rpc_xprt *xprt = rep->rr_xprt;
         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
         __be32 *iptr;
-        int rdmalen, status;
+        int credits, rdmalen, status;
         unsigned long cwnd;
 
         /* Check status. If bad, signal disconnect and return rep to pool */
@@ -744,14 +750,14 @@ rpcrdma_reply_handler(struct rpcrdma_rep *rep)
                 }
                 return;
         }
-        if (rep->rr_len < 28) {
+        if (rep->rr_len < RPCRDMA_HDRLEN_MIN) {
                 dprintk("RPC:       %s: short/invalid reply\n", __func__);
                 goto repost;
         }
-        headerp = (struct rpcrdma_msg *) rep->rr_base;
-        if (headerp->rm_vers != xdr_one) {
+        headerp = rdmab_to_msg(rep->rr_rdmabuf);
+        if (headerp->rm_vers != rpcrdma_version) {
                 dprintk("RPC:       %s: invalid version %d\n",
-                        __func__, ntohl(headerp->rm_vers));
+                        __func__, be32_to_cpu(headerp->rm_vers));
                 goto repost;
         }
 
@@ -762,7 +768,8 @@ rpcrdma_reply_handler(struct rpcrdma_rep *rep)
                 spin_unlock(&xprt->transport_lock);
                 dprintk("RPC:       %s: reply 0x%p failed "
                         "to match any request xid 0x%08x len %d\n",
-                        __func__, rep, headerp->rm_xid, rep->rr_len);
+                        __func__, rep, be32_to_cpu(headerp->rm_xid),
+                        rep->rr_len);
 repost:
                 r_xprt->rx_stats.bad_reply_count++;
                 rep->rr_func = rpcrdma_reply_handler;
@@ -778,13 +785,14 @@ repost:
                 spin_unlock(&xprt->transport_lock);
                 dprintk("RPC:       %s: duplicate reply 0x%p to RPC "
                         "request 0x%p: xid 0x%08x\n", __func__, rep, req,
-                        headerp->rm_xid);
+                        be32_to_cpu(headerp->rm_xid));
                 goto repost;
         }
 
         dprintk("RPC:       %s: reply 0x%p completes request 0x%p\n"
                 "                   RPC request 0x%p xid 0x%08x\n",
-                        __func__, rep, req, rqst, headerp->rm_xid);
+                        __func__, rep, req, rqst,
+                        be32_to_cpu(headerp->rm_xid));
 
         /* from here on, the reply is no longer an orphan */
         req->rl_reply = rep;
@@ -793,7 +801,7 @@ repost:
         /* check for expected message types */
         /* The order of some of these tests is important. */
         switch (headerp->rm_type) {
-        case htonl(RDMA_MSG):
+        case rdma_msg:
                 /* never expect read chunks */
                 /* never expect reply chunks (two ways to check) */
                 /* never expect write chunks without having offered RDMA */
@@ -824,22 +832,24 @@ repost:
                 } else {
                         /* else ordinary inline */
                         rdmalen = 0;
-                        iptr = (__be32 *)((unsigned char *)headerp + 28);
-                        rep->rr_len -= 28; /*sizeof *headerp;*/
+                        iptr = (__be32 *)((unsigned char *)headerp +
+                                                        RPCRDMA_HDRLEN_MIN);
+                        rep->rr_len -= RPCRDMA_HDRLEN_MIN;
                         status = rep->rr_len;
                 }
                 /* Fix up the rpc results for upper layer */
                 rpcrdma_inline_fixup(rqst, (char *)iptr, rep->rr_len, rdmalen);
                 break;
 
-        case htonl(RDMA_NOMSG):
+        case rdma_nomsg:
                 /* never expect read or write chunks, always reply chunks */
                 if (headerp->rm_body.rm_chunks[0] != xdr_zero ||
                     headerp->rm_body.rm_chunks[1] != xdr_zero ||
                     headerp->rm_body.rm_chunks[2] != xdr_one ||
                     req->rl_nchunks == 0)
                         goto badheader;
-                iptr = (__be32 *)((unsigned char *)headerp + 28);
+                iptr = (__be32 *)((unsigned char *)headerp +
+                                                        RPCRDMA_HDRLEN_MIN);
                 rdmalen = rpcrdma_count_chunks(rep, req->rl_nchunks, 0, &iptr);
                 if (rdmalen < 0)
                         goto badheader;
@@ -853,7 +863,7 @@ badheader:
                 dprintk("%s: invalid rpcrdma reply header (type %d):"
                                 " chunks[012] == %d %d %d"
                                 " expected chunks <= %d\n",
-                                __func__, ntohl(headerp->rm_type),
+                                __func__, be32_to_cpu(headerp->rm_type),
                                 headerp->rm_body.rm_chunks[0],
                                 headerp->rm_body.rm_chunks[1],
                                 headerp->rm_body.rm_chunks[2],
@@ -863,8 +873,14 @@ badheader:
                 break;
         }
 
+        credits = be32_to_cpu(headerp->rm_credit);
+        if (credits == 0)
+                credits = 1;    /* don't deadlock */
+        else if (credits > r_xprt->rx_buf.rb_max_requests)
+                credits = r_xprt->rx_buf.rb_max_requests;
+
         cwnd = xprt->cwnd;
-        xprt->cwnd = atomic_read(&r_xprt->rx_buf.rb_credits) << RPC_CWNDSHIFT;
+        xprt->cwnd = credits << RPC_CWNDSHIFT;
         if (xprt->cwnd > cwnd)
                 xprt_release_rqst_cong(rqst->rq_task);
 
diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
index bbd6155d3e34..2e192baa59f3 100644
--- a/net/sunrpc/xprtrdma/transport.c
+++ b/net/sunrpc/xprtrdma/transport.c
@@ -200,9 +200,9 @@ xprt_rdma_free_addresses(struct rpc_xprt *xprt)
 static void
 xprt_rdma_connect_worker(struct work_struct *work)
 {
-        struct rpcrdma_xprt *r_xprt =
-                container_of(work, struct rpcrdma_xprt, rdma_connect.work);
-        struct rpc_xprt *xprt = &r_xprt->xprt;
+        struct rpcrdma_xprt *r_xprt = container_of(work, struct rpcrdma_xprt,
+                                                   rx_connect_worker.work);
+        struct rpc_xprt *xprt = &r_xprt->rx_xprt;
         int rc = 0;
 
         xprt_clear_connected(xprt);
@@ -235,7 +235,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
 
         dprintk("RPC:       %s: called\n", __func__);
 
-        cancel_delayed_work_sync(&r_xprt->rdma_connect);
+        cancel_delayed_work_sync(&r_xprt->rx_connect_worker);
 
         xprt_clear_connected(xprt);
 
@@ -364,8 +364,7 @@ xprt_setup_rdma(struct xprt_create *args)
          * any inline data. Also specify any padding which will be provided
          * from a preregistered zero buffer.
          */
-        rc = rpcrdma_buffer_create(&new_xprt->rx_buf, new_ep, &new_xprt->rx_ia,
-                                &new_xprt->rx_data);
+        rc = rpcrdma_buffer_create(new_xprt);
         if (rc)
                 goto out3;
 
@@ -374,9 +373,8 @@ xprt_setup_rdma(struct xprt_create *args)
          * connection loss notification is async. We also catch connection loss
          * when reaping receives.
          */
-        INIT_DELAYED_WORK(&new_xprt->rdma_connect, xprt_rdma_connect_worker);
-        new_ep->rep_func = rpcrdma_conn_func;
-        new_ep->rep_xprt = xprt;
+        INIT_DELAYED_WORK(&new_xprt->rx_connect_worker,
+                          xprt_rdma_connect_worker);
 
         xprt_rdma_format_addresses(xprt);
         xprt->max_payload = rpcrdma_max_payload(new_xprt);
@@ -434,94 +432,101 @@ xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task)
 
         if (r_xprt->rx_ep.rep_connected != 0) {
                 /* Reconnect */
-                schedule_delayed_work(&r_xprt->rdma_connect,
-                        xprt->reestablish_timeout);
+                schedule_delayed_work(&r_xprt->rx_connect_worker,
+                                      xprt->reestablish_timeout);
                 xprt->reestablish_timeout <<= 1;
                 if (xprt->reestablish_timeout > RPCRDMA_MAX_REEST_TO)
                         xprt->reestablish_timeout = RPCRDMA_MAX_REEST_TO;
                 else if (xprt->reestablish_timeout < RPCRDMA_INIT_REEST_TO)
                         xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;
         } else {
-                schedule_delayed_work(&r_xprt->rdma_connect, 0);
+                schedule_delayed_work(&r_xprt->rx_connect_worker, 0);
                 if (!RPC_IS_ASYNC(task))
-                        flush_delayed_work(&r_xprt->rdma_connect);
+                        flush_delayed_work(&r_xprt->rx_connect_worker);
         }
 }
 
 /*
  * The RDMA allocate/free functions need the task structure as a place
  * to hide the struct rpcrdma_req, which is necessary for the actual send/recv
- * sequence. For this reason, the recv buffers are attached to send
- * buffers for portions of the RPC. Note that the RPC layer allocates
- * both send and receive buffers in the same call. We may register
- * the receive buffer portion when using reply chunks.
+ * sequence.
+ *
+ * The RPC layer allocates both send and receive buffers in the same call
+ * (rq_send_buf and rq_rcv_buf are both part of a single contiguous buffer).
+ * We may register rq_rcv_buf when using reply chunks.
  */
 static void *
 xprt_rdma_allocate(struct rpc_task *task, size_t size)
 {
         struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;
-        struct rpcrdma_req *req, *nreq;
+        struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
+        struct rpcrdma_regbuf *rb;
+        struct rpcrdma_req *req;
+        size_t min_size;
+        gfp_t flags;
 
-        req = rpcrdma_buffer_get(&rpcx_to_rdmax(xprt)->rx_buf);
+        req = rpcrdma_buffer_get(&r_xprt->rx_buf);
         if (req == NULL)
                 return NULL;
 
-        if (size > req->rl_size) {
-                dprintk("RPC:       %s: size %zd too large for buffer[%zd]: "
-                        "prog %d vers %d proc %d\n",
-                        __func__, size, req->rl_size,
-                        task->tk_client->cl_prog, task->tk_client->cl_vers,
-                        task->tk_msg.rpc_proc->p_proc);
-                /*
-                 * Outgoing length shortage. Our inline write max must have
-                 * been configured to perform direct i/o.
-                 *
-                 * This is therefore a large metadata operation, and the
-                 * allocate call was made on the maximum possible message,
-                 * e.g. containing long filename(s) or symlink data. In
-                 * fact, while these metadata operations *might* carry
-                 * large outgoing payloads, they rarely *do*. However, we
-                 * have to commit to the request here, so reallocate and
-                 * register it now. The data path will never require this
-                 * reallocation.
-                 *
-                 * If the allocation or registration fails, the RPC framework
-                 * will (doggedly) retry.
-                 */
-                if (task->tk_flags & RPC_TASK_SWAPPER)
-                        nreq = kmalloc(sizeof *req + size, GFP_ATOMIC);
-                else
-                        nreq = kmalloc(sizeof *req + size, GFP_NOFS);
-                if (nreq == NULL)
-                        goto outfail;
-
-                if (rpcrdma_register_internal(&rpcx_to_rdmax(xprt)->rx_ia,
-                                nreq->rl_base, size + sizeof(struct rpcrdma_req)
-                                - offsetof(struct rpcrdma_req, rl_base),
-                                &nreq->rl_handle, &nreq->rl_iov)) {
-                        kfree(nreq);
-                        goto outfail;
-                }
-                rpcx_to_rdmax(xprt)->rx_stats.hardway_register_count += size;
-                nreq->rl_size = size;
-                nreq->rl_niovs = 0;
-                nreq->rl_nchunks = 0;
-                nreq->rl_buffer = (struct rpcrdma_buffer *)req;
-                nreq->rl_reply = req->rl_reply;
-                memcpy(nreq->rl_segments,
-                        req->rl_segments, sizeof nreq->rl_segments);
-                /* flag the swap with an unused field */
-                nreq->rl_iov.length = 0;
-                req->rl_reply = NULL;
-                req = nreq;
-        }
+        flags = GFP_NOIO | __GFP_NOWARN;
+        if (RPC_IS_SWAPPER(task))
+                flags = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN;
+
+        if (req->rl_rdmabuf == NULL)
+                goto out_rdmabuf;
+        if (req->rl_sendbuf == NULL)
+                goto out_sendbuf;
+        if (size > req->rl_sendbuf->rg_size)
+                goto out_sendbuf;
+
+out:
         dprintk("RPC:       %s: size %zd, request 0x%p\n", __func__, size, req);
         req->rl_connect_cookie = 0;     /* our reserved value */
-        return req->rl_xdr_buf;
-
-outfail:
+        return req->rl_sendbuf->rg_base;
+
+out_rdmabuf:
+        min_size = RPCRDMA_INLINE_WRITE_THRESHOLD(task->tk_rqstp);
+        rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, min_size, flags);
+        if (IS_ERR(rb))
+                goto out_fail;
+        req->rl_rdmabuf = rb;
+
+out_sendbuf:
+        /* XDR encoding and RPC/RDMA marshaling of this request has not
+         * yet occurred. Thus a lower bound is needed to prevent buffer
+         * overrun during marshaling.
+         *
+         * RPC/RDMA marshaling may choose to send payload bearing ops
+         * inline, if the result is smaller than the inline threshold.
+         * The value of the "size" argument accounts for header
+         * requirements but not for the payload in these cases.
+         *
+         * Likewise, allocate enough space to receive a reply up to the
+         * size of the inline threshold.
+         *
+         * It's unlikely that both the send header and the received
+         * reply will be large, but slush is provided here to allow
+         * flexibility when marshaling.
+         */
+        min_size = RPCRDMA_INLINE_READ_THRESHOLD(task->tk_rqstp);
+        min_size += RPCRDMA_INLINE_WRITE_THRESHOLD(task->tk_rqstp);
+        if (size < min_size)
+                size = min_size;
+
+        rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, flags);
+        if (IS_ERR(rb))
+                goto out_fail;
+        rb->rg_owner = req;
+
+        r_xprt->rx_stats.hardway_register_count += size;
+        rpcrdma_free_regbuf(&r_xprt->rx_ia, req->rl_sendbuf);
+        req->rl_sendbuf = rb;
+        goto out;
+
+out_fail:
         rpcrdma_buffer_put(req);
-        rpcx_to_rdmax(xprt)->rx_stats.failed_marshal_count++;
+        r_xprt->rx_stats.failed_marshal_count++;
         return NULL;
 }
 
@@ -533,47 +538,24 @@ xprt_rdma_free(void *buffer)
 {
         struct rpcrdma_req *req;
         struct rpcrdma_xprt *r_xprt;
-        struct rpcrdma_rep *rep;
+        struct rpcrdma_regbuf *rb;
         int i;
 
         if (buffer == NULL)
                 return;
 
-        req = container_of(buffer, struct rpcrdma_req, rl_xdr_buf[0]);
-        if (req->rl_iov.length == 0) {  /* see allocate above */
-                r_xprt = container_of(((struct rpcrdma_req *) req->rl_buffer)->rl_buffer,
-                                      struct rpcrdma_xprt, rx_buf);
-        } else
-                r_xprt = container_of(req->rl_buffer, struct rpcrdma_xprt, rx_buf);
-        rep = req->rl_reply;
+        rb = container_of(buffer, struct rpcrdma_regbuf, rg_base[0]);
+        req = rb->rg_owner;
+        r_xprt = container_of(req->rl_buffer, struct rpcrdma_xprt, rx_buf);
 
-        dprintk("RPC:       %s: called on 0x%p%s\n",
-                __func__, rep, (rep && rep->rr_func) ? " (with waiter)" : "");
+        dprintk("RPC:       %s: called on 0x%p\n", __func__, req->rl_reply);
 
-        /*
-         * Finish the deregistration.  The process is considered
-         * complete when the rr_func vector becomes NULL - this
-         * was put in place during rpcrdma_reply_handler() - the wait
-         * call below will not block if the dereg is "done". If
-         * interrupted, our framework will clean up.
-         */
         for (i = 0; req->rl_nchunks;) {
                 --req->rl_nchunks;
                 i += rpcrdma_deregister_external(
                         &req->rl_segments[i], r_xprt);
         }
 
-        if (req->rl_iov.length == 0) {  /* see allocate above */
-                struct rpcrdma_req *oreq = (struct rpcrdma_req *)req->rl_buffer;
-                oreq->rl_reply = req->rl_reply;
-                (void) rpcrdma_deregister_internal(&r_xprt->rx_ia,
-                                                   req->rl_handle,
-                                                   &req->rl_iov);
-                kfree(req);
-                req = oreq;
-        }
-
-        /* Put back request+reply buffers */
         rpcrdma_buffer_put(req);
 }
 
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index c98e40643910..124676c13780 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -49,6 +49,7 @@
 
 #include <linux/interrupt.h>
 #include <linux/slab.h>
+#include <linux/prefetch.h>
 #include <asm/bitops.h>
 
 #include "xprt_rdma.h"
@@ -153,7 +154,7 @@ rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context)
                 event->device->name, context);
         if (ep->rep_connected == 1) {
                 ep->rep_connected = -EIO;
-                ep->rep_func(ep);
+                rpcrdma_conn_func(ep);
                 wake_up_all(&ep->rep_connect_wait);
         }
 }
@@ -168,23 +169,59 @@ rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context)
                 event->device->name, context);
         if (ep->rep_connected == 1) {
                 ep->rep_connected = -EIO;
-                ep->rep_func(ep);
+                rpcrdma_conn_func(ep);
                 wake_up_all(&ep->rep_connect_wait);
         }
 }
 
+static const char * const wc_status[] = {
+        "success",
+        "local length error",
+        "local QP operation error",
+        "local EE context operation error",
+        "local protection error",
+        "WR flushed",
+        "memory management operation error",
+        "bad response error",
+        "local access error",
+        "remote invalid request error",
+        "remote access error",
+        "remote operation error",
+        "transport retry counter exceeded",
+        "RNR retrycounter exceeded",
+        "local RDD violation error",
+        "remove invalid RD request",
+        "operation aborted",
+        "invalid EE context number",
+        "invalid EE context state",
+        "fatal error",
+        "response timeout error",
+        "general error",
+};
+
+#define COMPLETION_MSG(status)                                  \
+        ((status) < ARRAY_SIZE(wc_status) ?                     \
+                wc_status[(status)] : "unexpected completion error")
+
 static void
 rpcrdma_sendcq_process_wc(struct ib_wc *wc)
 {
-        struct rpcrdma_mw *frmr = (struct rpcrdma_mw *)(unsigned long)wc->wr_id;
+        if (likely(wc->status == IB_WC_SUCCESS))
+                return;
 
-        dprintk("RPC:       %s: frmr %p status %X opcode %d\n",
-                __func__, frmr, wc->status, wc->opcode);
+        /* WARNING: Only wr_id and status are reliable at this point */
+        if (wc->wr_id == 0ULL) {
+                if (wc->status != IB_WC_WR_FLUSH_ERR)
+                        pr_err("RPC:       %s: SEND: %s\n",
+                               __func__, COMPLETION_MSG(wc->status));
+        } else {
+                struct rpcrdma_mw *r;
 
-        if (wc->wr_id == 0ULL)
-                return;
-        if (wc->status != IB_WC_SUCCESS)
-                frmr->r.frmr.fr_state = FRMR_IS_STALE;
+                r = (struct rpcrdma_mw *)(unsigned long)wc->wr_id;
+                r->r.frmr.fr_state = FRMR_IS_STALE;
+                pr_err("RPC:       %s: frmr %p (stale): %s\n",
+                       __func__, r, COMPLETION_MSG(wc->status));
+        }
 }
 
 static int
@@ -248,33 +285,32 @@ rpcrdma_recvcq_process_wc(struct ib_wc *wc, struct list_head *sched_list)
         struct rpcrdma_rep *rep =
                         (struct rpcrdma_rep *)(unsigned long)wc->wr_id;
 
-        dprintk("RPC:       %s: rep %p status %X opcode %X length %u\n",
-                __func__, rep, wc->status, wc->opcode, wc->byte_len);
+        /* WARNING: Only wr_id and status are reliable at this point */
+        if (wc->status != IB_WC_SUCCESS)
+                goto out_fail;
 
-        if (wc->status != IB_WC_SUCCESS) {
-                rep->rr_len = ~0U;
-                goto out_schedule;
-        }
+        /* status == SUCCESS means all fields in wc are trustworthy */
         if (wc->opcode != IB_WC_RECV)
                 return;
 
+        dprintk("RPC:       %s: rep %p opcode 'recv', length %u: success\n",
+                __func__, rep, wc->byte_len);
+
         rep->rr_len = wc->byte_len;
         ib_dma_sync_single_for_cpu(rdmab_to_ia(rep->rr_buffer)->ri_id->device,
-                        rep->rr_iov.addr, rep->rr_len, DMA_FROM_DEVICE);
-
-        if (rep->rr_len >= 16) {
-                struct rpcrdma_msg *p = (struct rpcrdma_msg *)rep->rr_base;
-                unsigned int credits = ntohl(p->rm_credit);
-
-                if (credits == 0)
-                        credits = 1;    /* don't deadlock */
-                else if (credits > rep->rr_buffer->rb_max_requests)
-                        credits = rep->rr_buffer->rb_max_requests;
-                atomic_set(&rep->rr_buffer->rb_credits, credits);
-        }
+                                   rdmab_addr(rep->rr_rdmabuf),
+                                   rep->rr_len, DMA_FROM_DEVICE);
+        prefetch(rdmab_to_msg(rep->rr_rdmabuf));
 
 out_schedule:
         list_add_tail(&rep->rr_list, sched_list);
+        return;
+out_fail:
+        if (wc->status != IB_WC_WR_FLUSH_ERR)
+                pr_err("RPC:       %s: rep %p: %s\n",
+                       __func__, rep, COMPLETION_MSG(wc->status));
+        rep->rr_len = ~0U;
+        goto out_schedule;
 }
 
 static int
@@ -390,8 +426,8 @@ rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
         struct sockaddr_in *addr = (struct sockaddr_in *) &ep->rep_remote_addr;
 #endif
-        struct ib_qp_attr attr;
-        struct ib_qp_init_attr iattr;
+        struct ib_qp_attr *attr = &ia->ri_qp_attr;
+        struct ib_qp_init_attr *iattr = &ia->ri_qp_init_attr;
         int connstate = 0;
 
         switch (event->event) {
@@ -414,12 +450,13 @@ rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
                 break;
         case RDMA_CM_EVENT_ESTABLISHED:
                 connstate = 1;
-                ib_query_qp(ia->ri_id->qp, &attr,
-                        IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC,
-                        &iattr);
+                ib_query_qp(ia->ri_id->qp, attr,
+                            IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC,
+                            iattr);
                 dprintk("RPC:       %s: %d responder resources"
                         " (%d initiator)\n",
-                        __func__, attr.max_dest_rd_atomic, attr.max_rd_atomic);
+                        __func__, attr->max_dest_rd_atomic,
+                        attr->max_rd_atomic);
                 goto connected;
         case RDMA_CM_EVENT_CONNECT_ERROR:
                 connstate = -ENOTCONN;
@@ -436,11 +473,10 @@ rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
         case RDMA_CM_EVENT_DEVICE_REMOVAL:
                 connstate = -ENODEV;
 connected:
-                atomic_set(&rpcx_to_rdmax(ep->rep_xprt)->rx_buf.rb_credits, 1);
                 dprintk("RPC:       %s: %sconnected\n",
                                         __func__, connstate > 0 ? "" : "dis");
                 ep->rep_connected = connstate;
-                ep->rep_func(ep);
+                rpcrdma_conn_func(ep);
                 wake_up_all(&ep->rep_connect_wait);
                 /*FALLTHROUGH*/
         default:
@@ -453,7 +489,7 @@ connected:
 
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
         if (connstate == 1) {
-                int ird = attr.max_dest_rd_atomic;
+                int ird = attr->max_dest_rd_atomic;
                 int tird = ep->rep_remote_cma.responder_resources;
                 printk(KERN_INFO "rpcrdma: connection to %pI4:%u "
                         "on %s, memreg %d slots %d ird %d%s\n",
@@ -554,8 +590,8 @@ int
 rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
 {
         int rc, mem_priv;
-        struct ib_device_attr devattr;
         struct rpcrdma_ia *ia = &xprt->rx_ia;
+        struct ib_device_attr *devattr = &ia->ri_devattr;
 
         ia->ri_id = rpcrdma_create_id(xprt, ia, addr);
         if (IS_ERR(ia->ri_id)) {
@@ -571,26 +607,21 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
                 goto out2;
         }
 
-        /*
-         * Query the device to determine if the requested memory
-         * registration strategy is supported. If it isn't, set the
-         * strategy to a globally supported model.
-         */
-        rc = ib_query_device(ia->ri_id->device, &devattr);
+        rc = ib_query_device(ia->ri_id->device, devattr);
         if (rc) {
                 dprintk("RPC:       %s: ib_query_device failed %d\n",
                         __func__, rc);
-                goto out2;
+                goto out3;
         }
 
-        if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {
+        if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {
                 ia->ri_have_dma_lkey = 1;
                 ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey;
         }
 
         if (memreg == RPCRDMA_FRMR) {
                 /* Requires both frmr reg and local dma lkey */
-                if ((devattr.device_cap_flags &
+                if ((devattr->device_cap_flags &
                      (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) !=
                     (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) {
                         dprintk("RPC:       %s: FRMR registration "
@@ -600,7 +631,7 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
                         /* Mind the ia limit on FRMR page list depth */
                         ia->ri_max_frmr_depth = min_t(unsigned int,
                                 RPCRDMA_MAX_DATA_SEGS,
-                                devattr.max_fast_reg_page_list_len);
+                                devattr->max_fast_reg_page_list_len);
                 }
         }
         if (memreg == RPCRDMA_MTHCAFMR) {
@@ -638,14 +669,14 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
                                 "phys register failed with %lX\n",
                                 __func__, PTR_ERR(ia->ri_bind_mem));
                         rc = -ENOMEM;
-                        goto out2;
+                        goto out3;
                 }
                 break;
         default:
                 printk(KERN_ERR "RPC: Unsupported memory "
                                 "registration mode: %d\n", memreg);
                 rc = -ENOMEM;
-                goto out2;
+                goto out3;
         }
         dprintk("RPC:       %s: memory registration strategy is %d\n",
                 __func__, memreg);
@@ -655,6 +686,10 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
 
         rwlock_init(&ia->ri_qplock);
         return 0;
+
+out3:
+        ib_dealloc_pd(ia->ri_pd);
+        ia->ri_pd = NULL;
 out2:
         rdma_destroy_id(ia->ri_id);
         ia->ri_id = NULL;
@@ -698,20 +733,13 @@ int
 rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
                                 struct rpcrdma_create_data_internal *cdata)
 {
-        struct ib_device_attr devattr;
+        struct ib_device_attr *devattr = &ia->ri_devattr;
         struct ib_cq *sendcq, *recvcq;
         int rc, err;
 
-        rc = ib_query_device(ia->ri_id->device, &devattr);
-        if (rc) {
-                dprintk("RPC:       %s: ib_query_device failed %d\n",
-                        __func__, rc);
-                return rc;
-        }
-
         /* check provider's send/recv wr limits */
-        if (cdata->max_requests > devattr.max_qp_wr)
-                cdata->max_requests = devattr.max_qp_wr;
+        if (cdata->max_requests > devattr->max_qp_wr)
+                cdata->max_requests = devattr->max_qp_wr;
 
         ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall;
         ep->rep_attr.qp_context = ep;
@@ -746,8 +774,8 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 
                 }
                 ep->rep_attr.cap.max_send_wr *= depth;
-                if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr) {
-                        cdata->max_requests = devattr.max_qp_wr / depth;
+                if (ep->rep_attr.cap.max_send_wr > devattr->max_qp_wr) {
+                        cdata->max_requests = devattr->max_qp_wr / depth;
                         if (!cdata->max_requests)
                                 return -EINVAL;
                         ep->rep_attr.cap.max_send_wr = cdata->max_requests *
@@ -766,6 +794,14 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
         ep->rep_attr.qp_type = IB_QPT_RC;
         ep->rep_attr.port_num = ~0;
 
+        if (cdata->padding) {
+                ep->rep_padbuf = rpcrdma_alloc_regbuf(ia, cdata->padding,
+                                                      GFP_KERNEL);
+                if (IS_ERR(ep->rep_padbuf))
+                        return PTR_ERR(ep->rep_padbuf);
+        } else
+                ep->rep_padbuf = NULL;
+
         dprintk("RPC:       %s: requested max: dtos: send %d recv %d; "
                 "iovs: send %d recv %d\n",
                 __func__,
@@ -781,7 +817,6 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
         else if (ep->rep_cqinit <= 2)
                 ep->rep_cqinit = 0;
         INIT_CQCOUNT(ep);
-        ep->rep_ia = ia;
         init_waitqueue_head(&ep->rep_connect_wait);
         INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker);
 
@@ -831,10 +866,11 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 
         /* Client offers RDMA Read but does not initiate */
         ep->rep_remote_cma.initiator_depth = 0;
-        if (devattr.max_qp_rd_atom > 32)        /* arbitrary but <= 255 */
+        if (devattr->max_qp_rd_atom > 32)       /* arbitrary but <= 255 */
                 ep->rep_remote_cma.responder_resources = 32;
         else
-                ep->rep_remote_cma.responder_resources = devattr.max_qp_rd_atom;
+                ep->rep_remote_cma.responder_resources =
+                                                devattr->max_qp_rd_atom;
 
         ep->rep_remote_cma.retry_count = 7;
         ep->rep_remote_cma.flow_control = 0;
@@ -848,6 +884,7 @@ out2:
                 dprintk("RPC:       %s: ib_destroy_cq returned %i\n",
                         __func__, err);
 out1:
+        rpcrdma_free_regbuf(ia, ep->rep_padbuf);
         return rc;
 }
 
@@ -874,11 +911,7 @@ rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
                 ia->ri_id->qp = NULL;
         }
 
-        /* padding - could be done in rpcrdma_buffer_destroy... */
-        if (ep->rep_pad_mr) {
-                rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad);
-                ep->rep_pad_mr = NULL;
-        }
+        rpcrdma_free_regbuf(ia, ep->rep_padbuf);
 
         rpcrdma_clean_cq(ep->rep_attr.recv_cq);
         rc = ib_destroy_cq(ep->rep_attr.recv_cq);
@@ -1048,6 +1081,48 @@ rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
         }
 }
 
+static struct rpcrdma_req *
+rpcrdma_create_req(struct rpcrdma_xprt *r_xprt)
+{
+        struct rpcrdma_req *req;
+
+        req = kzalloc(sizeof(*req), GFP_KERNEL);
+        if (req == NULL)
+                return ERR_PTR(-ENOMEM);
+
+        req->rl_buffer = &r_xprt->rx_buf;
+        return req;
+}
+
+static struct rpcrdma_rep *
+rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt)
+{
+        struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
+        struct rpcrdma_ia *ia = &r_xprt->rx_ia;
+        struct rpcrdma_rep *rep;
+        int rc;
+
+        rc = -ENOMEM;
+        rep = kzalloc(sizeof(*rep), GFP_KERNEL);
+        if (rep == NULL)
+                goto out;
+
+        rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize,
+                                               GFP_KERNEL);
+        if (IS_ERR(rep->rr_rdmabuf)) {
+                rc = PTR_ERR(rep->rr_rdmabuf);
+                goto out_free;
+        }
+
+        rep->rr_buffer = &r_xprt->rx_buf;
+        return rep;
+
+out_free:
+        kfree(rep);
+out:
+        return ERR_PTR(rc);
+}
+
 static int
 rpcrdma_init_fmrs(struct rpcrdma_ia *ia, struct rpcrdma_buffer *buf)
 {
@@ -1134,27 +1209,26 @@ out_free:
 }
 
 int
-rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
-        struct rpcrdma_ia *ia, struct rpcrdma_create_data_internal *cdata)
+rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)
 {
+        struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
+        struct rpcrdma_ia *ia = &r_xprt->rx_ia;
+        struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
         char *p;
-        size_t len, rlen, wlen;
+        size_t len;
         int i, rc;
 
         buf->rb_max_requests = cdata->max_requests;
         spin_lock_init(&buf->rb_lock);
-        atomic_set(&buf->rb_credits, 1);
 
         /* Need to allocate:
          *   1.  arrays for send and recv pointers
          *   2.  arrays of struct rpcrdma_req to fill in pointers
          *   3.  array of struct rpcrdma_rep for replies
-         *   4.  padding, if any
          * Send/recv buffers in req/rep need to be registered
          */
         len = buf->rb_max_requests *
                 (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *));
-        len += cdata->padding;
 
         p = kzalloc(len, GFP_KERNEL);
         if (p == NULL) {
@@ -1170,17 +1244,6 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
         buf->rb_recv_bufs = (struct rpcrdma_rep **) p;
         p = (char *) &buf->rb_recv_bufs[buf->rb_max_requests];
 
-        /*
-         * Register the zeroed pad buffer, if any.
-         */
-        if (cdata->padding) {
-                rc = rpcrdma_register_internal(ia, p, cdata->padding,
-                                            &ep->rep_pad_mr, &ep->rep_pad);
-                if (rc)
-                        goto out;
-        }
-        p += cdata->padding;
-
         INIT_LIST_HEAD(&buf->rb_mws);
         INIT_LIST_HEAD(&buf->rb_all);
         switch (ia->ri_memreg_strategy) {
@@ -1198,62 +1261,29 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
                 break;
         }
 
-        /*
-         * Allocate/init the request/reply buffers. Doing this
-         * using kmalloc for now -- one for each buf.
-         */
-        wlen = 1 << fls(cdata->inline_wsize + sizeof(struct rpcrdma_req));
-        rlen = 1 << fls(cdata->inline_rsize + sizeof(struct rpcrdma_rep));
-        dprintk("RPC:       %s: wlen = %zu, rlen = %zu\n",
-                __func__, wlen, rlen);
-
         for (i = 0; i < buf->rb_max_requests; i++) {
                 struct rpcrdma_req *req;
                 struct rpcrdma_rep *rep;
 
-                req = kmalloc(wlen, GFP_KERNEL);
-                if (req == NULL) {
+                req = rpcrdma_create_req(r_xprt);
+                if (IS_ERR(req)) {
                         dprintk("RPC:       %s: request buffer %d alloc"
                                 " failed\n", __func__, i);
-                        rc = -ENOMEM;
+                        rc = PTR_ERR(req);
                         goto out;
                 }
-                memset(req, 0, sizeof(struct rpcrdma_req));
                 buf->rb_send_bufs[i] = req;
-                buf->rb_send_bufs[i]->rl_buffer = buf;
 
-                rc = rpcrdma_register_internal(ia, req->rl_base,
-                                wlen - offsetof(struct rpcrdma_req, rl_base),
-                                &buf->rb_send_bufs[i]->rl_handle,
-                                &buf->rb_send_bufs[i]->rl_iov);
-                if (rc)
-                        goto out;
-
-                buf->rb_send_bufs[i]->rl_size = wlen -
-                                                sizeof(struct rpcrdma_req);
-
-                rep = kmalloc(rlen, GFP_KERNEL);
-                if (rep == NULL) {
+                rep = rpcrdma_create_rep(r_xprt);
+                if (IS_ERR(rep)) {
                         dprintk("RPC:       %s: reply buffer %d alloc failed\n",
                                 __func__, i);
-                        rc = -ENOMEM;
+                        rc = PTR_ERR(rep);
                         goto out;
                 }
-                memset(rep, 0, sizeof(struct rpcrdma_rep));
                 buf->rb_recv_bufs[i] = rep;
-                buf->rb_recv_bufs[i]->rr_buffer = buf;
-
-                rc = rpcrdma_register_internal(ia, rep->rr_base,
-                                rlen - offsetof(struct rpcrdma_rep, rr_base),
-                                &buf->rb_recv_bufs[i]->rr_handle,
-                                &buf->rb_recv_bufs[i]->rr_iov);
-                if (rc)
-                        goto out;
-
         }
-        dprintk("RPC:       %s: max_requests %d\n",
-                __func__, buf->rb_max_requests);
-        /* done */
+
         return 0;
 out:
         rpcrdma_buffer_destroy(buf);
@@ -1261,6 +1291,27 @@ out:
 }
 
 static void
+rpcrdma_destroy_rep(struct rpcrdma_ia *ia, struct rpcrdma_rep *rep)
+{
+        if (!rep)
+                return;
+
+        rpcrdma_free_regbuf(ia, rep->rr_rdmabuf);
+        kfree(rep);
+}
+
+static void
+rpcrdma_destroy_req(struct rpcrdma_ia *ia, struct rpcrdma_req *req)
+{
+        if (!req)
+                return;
+
+        rpcrdma_free_regbuf(ia, req->rl_sendbuf);
+        rpcrdma_free_regbuf(ia, req->rl_rdmabuf);
+        kfree(req);
+}
+
+static void
 rpcrdma_destroy_fmrs(struct rpcrdma_buffer *buf)
 {
         struct rpcrdma_mw *r;
@@ -1315,18 +1366,10 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
         dprintk("RPC:       %s: entering\n", __func__);
 
         for (i = 0; i < buf->rb_max_requests; i++) {
-                if (buf->rb_recv_bufs && buf->rb_recv_bufs[i]) {
-                        rpcrdma_deregister_internal(ia,
-                                        buf->rb_recv_bufs[i]->rr_handle,
-                                        &buf->rb_recv_bufs[i]->rr_iov);
-                        kfree(buf->rb_recv_bufs[i]);
-                }
-                if (buf->rb_send_bufs && buf->rb_send_bufs[i]) {
-                        rpcrdma_deregister_internal(ia,
-                                        buf->rb_send_bufs[i]->rl_handle,
-                                        &buf->rb_send_bufs[i]->rl_iov);
-                        kfree(buf->rb_send_bufs[i]);
-                }
+                if (buf->rb_recv_bufs)
+                        rpcrdma_destroy_rep(ia, buf->rb_recv_bufs[i]);
+                if (buf->rb_send_bufs)
+                        rpcrdma_destroy_req(ia, buf->rb_send_bufs[i]);
         }
 
         switch (ia->ri_memreg_strategy) {
@@ -1450,8 +1493,8 @@ rpcrdma_buffer_put_mrs(struct rpcrdma_req *req, struct rpcrdma_buffer *buf)
         int i;
 
         for (i = 1, seg++; i < RPCRDMA_MAX_SEGS; seg++, i++)
-                rpcrdma_buffer_put_mr(&seg->mr_chunk.rl_mw, buf);
-        rpcrdma_buffer_put_mr(&seg1->mr_chunk.rl_mw, buf);
+                rpcrdma_buffer_put_mr(&seg->rl_mw, buf);
+        rpcrdma_buffer_put_mr(&seg1->rl_mw, buf);
 }
 
 static void
@@ -1537,7 +1580,7 @@ rpcrdma_buffer_get_frmrs(struct rpcrdma_req *req, struct rpcrdma_buffer *buf,
                         list_add(&r->mw_list, stale);
                         continue;
                 }
-                req->rl_segments[i].mr_chunk.rl_mw = r;
+                req->rl_segments[i].rl_mw = r;
                 if (unlikely(i-- == 0))
                         return req;     /* Success */
         }
@@ -1559,7 +1602,7 @@ rpcrdma_buffer_get_fmrs(struct rpcrdma_req *req, struct rpcrdma_buffer *buf)
                 r = list_entry(buf->rb_mws.next,
                                struct rpcrdma_mw, mw_list);
                 list_del(&r->mw_list);
-                req->rl_segments[i].mr_chunk.rl_mw = r;
+                req->rl_segments[i].rl_mw = r;
                 if (unlikely(i-- == 0))
                         return req;     /* Success */
         }
@@ -1658,8 +1701,6 @@ rpcrdma_recv_buffer_get(struct rpcrdma_req *req)
         struct rpcrdma_buffer *buffers = req->rl_buffer;
         unsigned long flags;
 
-        if (req->rl_iov.length == 0)    /* special case xprt_rdma_allocate() */
-                buffers = ((struct rpcrdma_req *) buffers)->rl_buffer;
         spin_lock_irqsave(&buffers->rb_lock, flags);
         if (buffers->rb_recv_index < buffers->rb_max_requests) {
                 req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index];
@@ -1688,7 +1729,7 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
  * Wrappers for internal-use kmalloc memory registration, used by buffer code.
  */
 
-int
+static int
 rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len,
                                 struct ib_mr **mrp, struct ib_sge *iov)
 {
@@ -1739,7 +1780,7 @@ rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len,
         return rc;
 }
 
-int
+static int
 rpcrdma_deregister_internal(struct rpcrdma_ia *ia,
                                 struct ib_mr *mr, struct ib_sge *iov)
 {
@@ -1757,6 +1798,61 @@ rpcrdma_deregister_internal(struct rpcrdma_ia *ia,
         return rc;
 }
 
+/**
+ * rpcrdma_alloc_regbuf - kmalloc and register memory for SEND/RECV buffers
+ * @ia: controlling rpcrdma_ia
+ * @size: size of buffer to be allocated, in bytes
+ * @flags: GFP flags
+ *
+ * Returns pointer to private header of an area of internally
+ * registered memory, or an ERR_PTR. The registered buffer follows
+ * the end of the private header.
+ *
+ * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for
+ * receiving the payload of RDMA RECV operations. regbufs are not
+ * used for RDMA READ/WRITE operations, thus are registered only for
+ * LOCAL access.
+ */
+struct rpcrdma_regbuf *
+rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)
+{
+        struct rpcrdma_regbuf *rb;
+        int rc;
+
+        rc = -ENOMEM;
+        rb = kmalloc(sizeof(*rb) + size, flags);
+        if (rb == NULL)
+                goto out;
+
+        rb->rg_size = size;
+        rb->rg_owner = NULL;
+        rc = rpcrdma_register_internal(ia, rb->rg_base, size,
+                                       &rb->rg_mr, &rb->rg_iov);
+        if (rc)
+                goto out_free;
+
+        return rb;
+
+out_free:
+        kfree(rb);
+out:
+        return ERR_PTR(rc);
+}
+
+/**
+ * rpcrdma_free_regbuf - deregister and free registered buffer
+ * @ia: controlling rpcrdma_ia
+ * @rb: regbuf to be deregistered and freed
+ */
+void
+rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
+{
+        if (rb) {
+                rpcrdma_deregister_internal(ia, rb->rg_mr, &rb->rg_iov);
+                kfree(rb);
+        }
+}
+
 /*
  * Wrappers for chunk registration, shared by read/write chunk code.
  */
@@ -1799,7 +1895,7 @@ rpcrdma_register_frmr_external(struct rpcrdma_mr_seg *seg,
                         struct rpcrdma_xprt *r_xprt)
 {
         struct rpcrdma_mr_seg *seg1 = seg;
-        struct rpcrdma_mw *mw = seg1->mr_chunk.rl_mw;
+        struct rpcrdma_mw *mw = seg1->rl_mw;
         struct rpcrdma_frmr *frmr = &mw->r.frmr;
         struct ib_mr *mr = frmr->fr_mr;
         struct ib_send_wr fastreg_wr, *bad_wr;
@@ -1888,12 +1984,12 @@ rpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg,
         struct ib_send_wr invalidate_wr, *bad_wr;
         int rc;
 
-        seg1->mr_chunk.rl_mw->r.frmr.fr_state = FRMR_IS_INVALID;
+        seg1->rl_mw->r.frmr.fr_state = FRMR_IS_INVALID;
 
         memset(&invalidate_wr, 0, sizeof invalidate_wr);
-        invalidate_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw;
+        invalidate_wr.wr_id = (unsigned long)(void *)seg1->rl_mw;
         invalidate_wr.opcode = IB_WR_LOCAL_INV;
-        invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
+        invalidate_wr.ex.invalidate_rkey = seg1->rl_mw->r.frmr.fr_mr->rkey;
         DECR_CQCOUNT(&r_xprt->rx_ep);
 
         read_lock(&ia->ri_qplock);
@@ -1903,7 +1999,7 @@ rpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg,
         read_unlock(&ia->ri_qplock);
         if (rc) {
                 /* Force rpcrdma_buffer_get() to retry */
-                seg1->mr_chunk.rl_mw->r.frmr.fr_state = FRMR_IS_STALE;
+                seg1->rl_mw->r.frmr.fr_state = FRMR_IS_STALE;
                 dprintk("RPC:       %s: failed ib_post_send for invalidate,"
                         " status %i\n", __func__, rc);
         }
@@ -1935,8 +2031,7 @@ rpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg,
                     offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
                         break;
         }
-        rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr,
-                                physaddrs, i, seg1->mr_dma);
+        rc = ib_map_phys_fmr(seg1->rl_mw->r.fmr, physaddrs, i, seg1->mr_dma);
         if (rc) {
                 dprintk("RPC:       %s: failed ib_map_phys_fmr "
                         "%u@0x%llx+%i (%d)... status %i\n", __func__,
@@ -1945,7 +2040,7 @@ rpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg,
                 while (i--)
                         rpcrdma_unmap_one(ia, --seg);
         } else {
-                seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey;
+                seg1->mr_rkey = seg1->rl_mw->r.fmr->rkey;
                 seg1->mr_base = seg1->mr_dma + pageoff;
                 seg1->mr_nsegs = i;
                 seg1->mr_len = len;
@@ -1962,7 +2057,7 @@ rpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg,
         LIST_HEAD(l);
         int rc;
 
-        list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l);
+        list_add(&seg1->rl_mw->r.fmr->list, &l);
         rc = ib_unmap_fmr(&l);
         read_lock(&ia->ri_qplock);
         while (seg1->mr_nsegs--)
@@ -2104,11 +2199,13 @@ rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,
 
         recv_wr.next = NULL;
         recv_wr.wr_id = (u64) (unsigned long) rep;
-        recv_wr.sg_list = &rep->rr_iov;
+        recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;
         recv_wr.num_sge = 1;
 
         ib_dma_sync_single_for_cpu(ia->ri_id->device,
-                rep->rr_iov.addr, rep->rr_iov.length, DMA_BIDIRECTIONAL);
+                                   rdmab_addr(rep->rr_rdmabuf),
+                                   rdmab_length(rep->rr_rdmabuf),
+                                   DMA_BIDIRECTIONAL);
 
         rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);
 
diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index b799041b75bf..d1b70397c60f 100644
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -70,6 +70,9 @@ struct rpcrdma_ia {
         int                     ri_async_rc;
         enum rpcrdma_memreg     ri_memreg_strategy;
         unsigned int            ri_max_frmr_depth;
+        struct ib_device_attr   ri_devattr;
+        struct ib_qp_attr       ri_qp_attr;
+        struct ib_qp_init_attr  ri_qp_init_attr;
 };
 
 /*
@@ -83,13 +86,9 @@ struct rpcrdma_ep {
         atomic_t                rep_cqcount;
         int                     rep_cqinit;
         int                     rep_connected;
-        struct rpcrdma_ia       *rep_ia;
         struct ib_qp_init_attr  rep_attr;
         wait_queue_head_t       rep_connect_wait;
-        struct ib_sge           rep_pad;        /* holds zeroed pad */
-        struct ib_mr            *rep_pad_mr;    /* holds zeroed pad */
-        void                    (*rep_func)(struct rpcrdma_ep *);
-        struct rpc_xprt         *rep_xprt;      /* for rep_func */
+        struct rpcrdma_regbuf   *rep_padbuf;
         struct rdma_conn_param  rep_remote_cma;
         struct sockaddr_storage rep_remote_addr;
         struct delayed_work     rep_connect_worker;
@@ -106,6 +105,44 @@ struct rpcrdma_ep {
 #define INIT_CQCOUNT(ep) atomic_set(&(ep)->rep_cqcount, (ep)->rep_cqinit)
 #define DECR_CQCOUNT(ep) atomic_sub_return(1, &(ep)->rep_cqcount)
 
+/* Registered buffer -- registered kmalloc'd memory for RDMA SEND/RECV
+ *
+ * The below structure appears at the front of a large region of kmalloc'd
+ * memory, which always starts on a good alignment boundary.
+ */
+
+struct rpcrdma_regbuf {
+        size_t                  rg_size;
+        struct rpcrdma_req      *rg_owner;
+        struct ib_mr            *rg_mr;
+        struct ib_sge           rg_iov;
+        __be32                  rg_base[0] __attribute__ ((aligned(256)));
+};
+
+static inline u64
+rdmab_addr(struct rpcrdma_regbuf *rb)
+{
+        return rb->rg_iov.addr;
+}
+
+static inline u32
+rdmab_length(struct rpcrdma_regbuf *rb)
+{
+        return rb->rg_iov.length;
+}
+
+static inline u32
+rdmab_lkey(struct rpcrdma_regbuf *rb)
+{
+        return rb->rg_iov.lkey;
+}
+
+static inline struct rpcrdma_msg *
+rdmab_to_msg(struct rpcrdma_regbuf *rb)
+{
+        return (struct rpcrdma_msg *)rb->rg_base;
+}
+
 enum rpcrdma_chunktype {
         rpcrdma_noch = 0,
         rpcrdma_readch,
@@ -134,22 +171,16 @@ enum rpcrdma_chunktype {
 /* temporary static scatter/gather max */
 #define RPCRDMA_MAX_DATA_SEGS   (64)    /* max scatter/gather */
 #define RPCRDMA_MAX_SEGS        (RPCRDMA_MAX_DATA_SEGS + 2) /* head+tail = 2 */
-#define MAX_RPCRDMAHDR  (\
-        /* max supported RPC/RDMA header */ \
-        sizeof(struct rpcrdma_msg) + (2 * sizeof(u32)) + \
-        (sizeof(struct rpcrdma_read_chunk) * RPCRDMA_MAX_SEGS) + sizeof(u32))
 
 struct rpcrdma_buffer;
 
 struct rpcrdma_rep {
-        unsigned int    rr_len;         /* actual received reply length */
-        struct rpcrdma_buffer *rr_buffer; /* home base for this structure */
-        struct rpc_xprt *rr_xprt;       /* needed for request/reply matching */
-        void (*rr_func)(struct rpcrdma_rep *);/* called by tasklet in softint */
-        struct list_head rr_list;       /* tasklet list */
-        struct ib_sge   rr_iov;         /* for posting */
-        struct ib_mr    *rr_handle;     /* handle for mem in rr_iov */
-        char    rr_base[MAX_RPCRDMAHDR]; /* minimal inline receive buffer */
+        unsigned int            rr_len;
+        struct rpcrdma_buffer   *rr_buffer;
+        struct rpc_xprt         *rr_xprt;
+        void                    (*rr_func)(struct rpcrdma_rep *);
+        struct list_head        rr_list;
+        struct rpcrdma_regbuf   *rr_rdmabuf;
 };
 
 /*
@@ -211,10 +242,7 @@ struct rpcrdma_mw {
  */
 
 struct rpcrdma_mr_seg {         /* chunk descriptors */
-        union {                         /* chunk memory handles */
-                struct ib_mr    *rl_mr;         /* if registered directly */
-                struct rpcrdma_mw *rl_mw;       /* if registered from region */
-        } mr_chunk;
+        struct rpcrdma_mw *rl_mw;       /* registered MR */
         u64             mr_base;        /* registration result */
         u32             mr_rkey;        /* registration result */
         u32             mr_len;         /* length of chunk or segment */
@@ -227,22 +255,27 @@ struct rpcrdma_mr_seg {		/* chunk descriptors */
 };
 
 struct rpcrdma_req {
-        size_t          rl_size;        /* actual length of buffer */
         unsigned int    rl_niovs;       /* 0, 2 or 4 */
         unsigned int    rl_nchunks;     /* non-zero if chunks */
         unsigned int    rl_connect_cookie;      /* retry detection */
         enum rpcrdma_chunktype  rl_rtype, rl_wtype;
         struct rpcrdma_buffer *rl_buffer; /* home base for this structure */
         struct rpcrdma_rep      *rl_reply;/* holder for reply buffer */
-        struct rpcrdma_mr_seg rl_segments[RPCRDMA_MAX_SEGS];/* chunk segments */
         struct ib_sge   rl_send_iov[4]; /* for active requests */
-        struct ib_sge   rl_iov;         /* for posting */
-        struct ib_mr    *rl_handle;     /* handle for mem in rl_iov */
-        char            rl_base[MAX_RPCRDMAHDR]; /* start of actual buffer */
-        __u32           rl_xdr_buf[0];  /* start of returned rpc rq_buffer */
+        struct rpcrdma_regbuf *rl_rdmabuf;
+        struct rpcrdma_regbuf *rl_sendbuf;
+        struct rpcrdma_mr_seg rl_segments[RPCRDMA_MAX_SEGS];
 };
-#define rpcr_to_rdmar(r) \
-        container_of((r)->rq_buffer, struct rpcrdma_req, rl_xdr_buf[0])
+
+static inline struct rpcrdma_req *
+rpcr_to_rdmar(struct rpc_rqst *rqst)
+{
+        void *buffer = rqst->rq_buffer;
+        struct rpcrdma_regbuf *rb;
+
+        rb = container_of(buffer, struct rpcrdma_regbuf, rg_base);
+        return rb->rg_owner;
+}
 
 /*
  * struct rpcrdma_buffer -- holds list/queue of pre-registered memory for
@@ -252,7 +285,6 @@ struct rpcrdma_req {
  */
 struct rpcrdma_buffer {
         spinlock_t      rb_lock;        /* protects indexes */
-        atomic_t        rb_credits;     /* most recent server credits */
         int             rb_max_requests;/* client max requests */
         struct list_head rb_mws;        /* optional memory windows/fmrs/frmrs */
         struct list_head rb_all;
@@ -318,16 +350,16 @@ struct rpcrdma_stats {
  * during unmount.
  */
 struct rpcrdma_xprt {
-        struct rpc_xprt         xprt;
+        struct rpc_xprt         rx_xprt;
         struct rpcrdma_ia       rx_ia;
         struct rpcrdma_ep       rx_ep;
         struct rpcrdma_buffer   rx_buf;
         struct rpcrdma_create_data_internal rx_data;
-        struct delayed_work     rdma_connect;
+        struct delayed_work     rx_connect_worker;
         struct rpcrdma_stats    rx_stats;
 };
 
-#define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, xprt)
+#define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, rx_xprt)
 #define rpcx_to_rdmad(x) (rpcx_to_rdmax(x)->rx_data)
 
 /* Setting this to 0 ensures interoperability with early servers.
@@ -358,9 +390,7 @@ int rpcrdma_ep_post_recv(struct rpcrdma_ia *, struct rpcrdma_ep *,
 /*
  * Buffer calls - xprtrdma/verbs.c
  */
-int rpcrdma_buffer_create(struct rpcrdma_buffer *, struct rpcrdma_ep *,
-                                struct rpcrdma_ia *,
-                                struct rpcrdma_create_data_internal *);
+int rpcrdma_buffer_create(struct rpcrdma_xprt *);
 void rpcrdma_buffer_destroy(struct rpcrdma_buffer *);
 
 struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);
@@ -368,16 +398,16 @@ void rpcrdma_buffer_put(struct rpcrdma_req *);
 void rpcrdma_recv_buffer_get(struct rpcrdma_req *);
 void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
 
-int rpcrdma_register_internal(struct rpcrdma_ia *, void *, int,
-                                struct ib_mr **, struct ib_sge *);
-int rpcrdma_deregister_internal(struct rpcrdma_ia *,
-                                struct ib_mr *, struct ib_sge *);
-
 int rpcrdma_register_external(struct rpcrdma_mr_seg *,
                                 int, int, struct rpcrdma_xprt *);
 int rpcrdma_deregister_external(struct rpcrdma_mr_seg *,
                                 struct rpcrdma_xprt *);
 
+struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(struct rpcrdma_ia *,
+                                            size_t, gfp_t);
+void rpcrdma_free_regbuf(struct rpcrdma_ia *,
+                         struct rpcrdma_regbuf *);
+
 /*
  * RPC/RDMA connection management calls - xprtrdma/rpc_rdma.c
  */
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 87ce7e8bb8dc..66891e32c5e3 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -63,6 +63,8 @@ static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE;
 static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT;
 static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT;
 
+#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
+
 #define XS_TCP_LINGER_TO        (15U * HZ)
 static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO;
 
@@ -75,8 +77,6 @@ static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO;
  * someone else's file names!
  */
 
-#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
-
 static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE;
 static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE;
 static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT;
@@ -627,7 +627,7 @@ process_status:
  * @xprt: transport
  *
  * Initiates a graceful shutdown of the TCP socket by calling the
- * equivalent of shutdown(SHUT_WR);
+ * equivalent of shutdown(SHUT_RDWR);
  */
 static void xs_tcp_shutdown(struct rpc_xprt *xprt)
 {
@@ -635,7 +635,7 @@ static void xs_tcp_shutdown(struct rpc_xprt *xprt)
         struct socket *sock = transport->sock;
 
         if (sock != NULL) {
-                kernel_sock_shutdown(sock, SHUT_WR);
+                kernel_sock_shutdown(sock, SHUT_RDWR);
                 trace_rpc_socket_shutdown(xprt, sock);
         }
 }
@@ -718,9 +718,9 @@ static int xs_tcp_send_request(struct rpc_task *task)
                 dprintk("RPC:       sendmsg returned unrecognized error %d\n",
                         -status);
         case -ECONNRESET:
-                xs_tcp_shutdown(xprt);
         case -ECONNREFUSED:
         case -ENOTCONN:
+        case -EADDRINUSE:
         case -EPIPE:
                 clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
         }
@@ -773,6 +773,21 @@ static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *s
         sk->sk_error_report = transport->old_error_report;
 }
 
+static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt)
+{
+        smp_mb__before_atomic();
+        clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
+        clear_bit(XPRT_CLOSING, &xprt->state);
+        smp_mb__after_atomic();
+}
+
+static void xs_sock_mark_closed(struct rpc_xprt *xprt)
+{
+        xs_sock_reset_connection_flags(xprt);
+        /* Mark transport as closed and wake up all pending tasks */
+        xprt_disconnect_done(xprt);
+}
+
 /**
  * xs_error_report - callback to handle TCP socket state errors
  * @sk: socket
@@ -792,11 +807,12 @@ static void xs_error_report(struct sock *sk)
         err = -sk->sk_err;
         if (err == 0)
                 goto out;
+        /* Is this a reset event? */
+        if (sk->sk_state == TCP_CLOSE)
+                xs_sock_mark_closed(xprt);
         dprintk("RPC:       xs_error_report client %p, error=%d...\n",
                         xprt, -err);
         trace_rpc_socket_error(xprt, sk->sk_socket, err);
-        if (test_bit(XPRT_CONNECTION_REUSE, &xprt->state))
-                goto out;
         xprt_wake_pending_tasks(xprt, err);
  out:
         read_unlock_bh(&sk->sk_callback_lock);
@@ -806,12 +822,11 @@ static void xs_reset_transport(struct sock_xprt *transport)
 {
         struct socket *sock = transport->sock;
         struct sock *sk = transport->inet;
+        struct rpc_xprt *xprt = &transport->xprt;
 
         if (sk == NULL)
                 return;
 
-        transport->srcport = 0;
-
         write_lock_bh(&sk->sk_callback_lock);
         transport->inet = NULL;
         transport->sock = NULL;
@@ -820,8 +835,9 @@ static void xs_reset_transport(struct sock_xprt *transport)
 
         xs_restore_old_callbacks(transport, sk);
         write_unlock_bh(&sk->sk_callback_lock);
+        xs_sock_reset_connection_flags(xprt);
 
-        trace_rpc_socket_close(&transport->xprt, sock);
+        trace_rpc_socket_close(xprt, sock);
         sock_release(sock);
 }
 
@@ -841,27 +857,12 @@ static void xs_close(struct rpc_xprt *xprt)
 
         dprintk("RPC:       xs_close xprt %p\n", xprt);
 
-        cancel_delayed_work_sync(&transport->connect_worker);
-
         xs_reset_transport(transport);
         xprt->reestablish_timeout = 0;
 
-        smp_mb__before_atomic();
-        clear_bit(XPRT_CONNECTION_ABORT, &xprt->state);
-        clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
-        clear_bit(XPRT_CLOSING, &xprt->state);
-        smp_mb__after_atomic();
         xprt_disconnect_done(xprt);
 }
 
-static void xs_tcp_close(struct rpc_xprt *xprt)
-{
-        if (test_and_clear_bit(XPRT_CONNECTION_CLOSE, &xprt->state))
-                xs_close(xprt);
-        else
-                xs_tcp_shutdown(xprt);
-}
-
 static void xs_xprt_free(struct rpc_xprt *xprt)
 {
         xs_free_peer_addresses(xprt);
@@ -1032,7 +1033,6 @@ static void xs_udp_data_ready(struct sock *sk)
  */
 static void xs_tcp_force_close(struct rpc_xprt *xprt)
 {
-        set_bit(XPRT_CONNECTION_CLOSE, &xprt->state);
         xprt_force_disconnect(xprt);
 }
 
@@ -1425,54 +1425,6 @@ out:
         read_unlock_bh(&sk->sk_callback_lock);
 }
 
-/*
- * Do the equivalent of linger/linger2 handling for dealing with
- * broken servers that don't close the socket in a timely
- * fashion
- */
-static void xs_tcp_schedule_linger_timeout(struct rpc_xprt *xprt,
-                unsigned long timeout)
-{
-        struct sock_xprt *transport;
-
-        if (xprt_test_and_set_connecting(xprt))
-                return;
-        set_bit(XPRT_CONNECTION_ABORT, &xprt->state);
-        transport = container_of(xprt, struct sock_xprt, xprt);
-        queue_delayed_work(rpciod_workqueue, &transport->connect_worker,
-                           timeout);
-}
-
-static void xs_tcp_cancel_linger_timeout(struct rpc_xprt *xprt)
-{
-        struct sock_xprt *transport;
-
-        transport = container_of(xprt, struct sock_xprt, xprt);
-
-        if (!test_bit(XPRT_CONNECTION_ABORT, &xprt->state) ||
-            !cancel_delayed_work(&transport->connect_worker))
-                return;
-        clear_bit(XPRT_CONNECTION_ABORT, &xprt->state);
-        xprt_clear_connecting(xprt);
-}
-
-static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt)
-{
-        smp_mb__before_atomic();
-        clear_bit(XPRT_CONNECTION_ABORT, &xprt->state);
-        clear_bit(XPRT_CONNECTION_CLOSE, &xprt->state);
-        clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
-        clear_bit(XPRT_CLOSING, &xprt->state);
-        smp_mb__after_atomic();
-}
-
-static void xs_sock_mark_closed(struct rpc_xprt *xprt)
-{
-        xs_sock_reset_connection_flags(xprt);
-        /* Mark transport as closed and wake up all pending tasks */
-        xprt_disconnect_done(xprt);
-}
-
 /**
  * xs_tcp_state_change - callback to handle TCP socket state changes
  * @sk: socket whose state has changed
@@ -1521,7 +1473,6 @@ static void xs_tcp_state_change(struct sock *sk)
                 clear_bit(XPRT_CONNECTED, &xprt->state);
                 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
                 smp_mb__after_atomic();
-                xs_tcp_schedule_linger_timeout(xprt, xs_tcp_fin_timeout);
                 break;
         case TCP_CLOSE_WAIT:
                 /* The server initiated a shutdown of the socket */
@@ -1538,13 +1489,11 @@ static void xs_tcp_state_change(struct sock *sk)
                 break;
         case TCP_LAST_ACK:
                 set_bit(XPRT_CLOSING, &xprt->state);
-                xs_tcp_schedule_linger_timeout(xprt, xs_tcp_fin_timeout);
                 smp_mb__before_atomic();
                 clear_bit(XPRT_CONNECTED, &xprt->state);
                 smp_mb__after_atomic();
                 break;
         case TCP_CLOSE:
-                xs_tcp_cancel_linger_timeout(xprt);
                 xs_sock_mark_closed(xprt);
         }
  out:
@@ -1667,6 +1616,40 @@ static unsigned short xs_get_random_port(void)
 }
 
 /**
+ * xs_set_reuseaddr_port - set the socket's port and address reuse options
+ * @sock: socket
+ *
+ * Note that this function has to be called on all sockets that share the
+ * same port, and it must be called before binding.
+ */
+static void xs_sock_set_reuseport(struct socket *sock)
+{
+        int opt = 1;
+
+        kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEPORT,
+                        (char *)&opt, sizeof(opt));
+}
+
+static unsigned short xs_sock_getport(struct socket *sock)
+{
+        struct sockaddr_storage buf;
+        int buflen;
+        unsigned short port = 0;
+
+        if (kernel_getsockname(sock, (struct sockaddr *)&buf, &buflen) < 0)
+                goto out;
+        switch (buf.ss_family) {
+        case AF_INET6:
+                port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port);
+                break;
+        case AF_INET:
+                port = ntohs(((struct sockaddr_in *)&buf)->sin_port);
+        }
+out:
+        return port;
+}
+
+/**
  * xs_set_port - reset the port number in the remote endpoint address
  * @xprt: generic transport
  * @port: new port number
@@ -1680,6 +1663,12 @@ static void xs_set_port(struct rpc_xprt *xprt, unsigned short port)
         xs_update_peer_port(xprt);
 }
 
+static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock)
+{
+        if (transport->srcport == 0)
+                transport->srcport = xs_sock_getport(sock);
+}
+
 static unsigned short xs_get_srcport(struct sock_xprt *transport)
 {
         unsigned short port = transport->srcport;
@@ -1833,7 +1822,8 @@ static void xs_dummy_setup_socket(struct work_struct *work)
 }
 
 static struct socket *xs_create_sock(struct rpc_xprt *xprt,
-                struct sock_xprt *transport, int family, int type, int protocol)
+                struct sock_xprt *transport, int family, int type,
+                int protocol, bool reuseport)
 {
         struct socket *sock;
         int err;
@@ -1846,6 +1836,9 @@ static struct socket *xs_create_sock(struct rpc_xprt *xprt,
         }
         xs_reclassify_socket(family, sock);
 
+        if (reuseport)
+                xs_sock_set_reuseport(sock);
+
         err = xs_bind(transport, sock);
         if (err) {
                 sock_release(sock);
@@ -1903,7 +1896,6 @@ static int xs_local_setup_socket(struct sock_xprt *transport)
         struct socket *sock;
         int status = -EIO;
 
-        clear_bit(XPRT_CONNECTION_ABORT, &xprt->state);
         status = __sock_create(xprt->xprt_net, AF_LOCAL,
                                         SOCK_STREAM, 0, &sock, 1);
         if (status < 0) {
@@ -2044,10 +2036,9 @@ static void xs_udp_setup_socket(struct work_struct *work)
         struct socket *sock = transport->sock;
         int status = -EIO;
 
-        /* Start by resetting any existing state */
-        xs_reset_transport(transport);
         sock = xs_create_sock(xprt, transport,
-                        xs_addr(xprt)->sa_family, SOCK_DGRAM, IPPROTO_UDP);
+                        xs_addr(xprt)->sa_family, SOCK_DGRAM,
+                        IPPROTO_UDP, false);
         if (IS_ERR(sock))
                 goto out;
 
@@ -2061,61 +2052,11 @@ static void xs_udp_setup_socket(struct work_struct *work)
         trace_rpc_socket_connect(xprt, sock, 0);
         status = 0;
 out:
+        xprt_unlock_connect(xprt, transport);
         xprt_clear_connecting(xprt);
         xprt_wake_pending_tasks(xprt, status);
 }
 
-/*
- * We need to preserve the port number so the reply cache on the server can
- * find our cached RPC replies when we get around to reconnecting.
- */
-static void xs_abort_connection(struct sock_xprt *transport)
-{
-        int result;
-        struct sockaddr any;
-
-        dprintk("RPC:       disconnecting xprt %p to reuse port\n", transport);
-
-        /*
-         * Disconnect the transport socket by doing a connect operation
-         * with AF_UNSPEC.  This should return immediately...
-         */
-        memset(&any, 0, sizeof(any));
-        any.sa_family = AF_UNSPEC;
-        result = kernel_connect(transport->sock, &any, sizeof(any), 0);
-        trace_rpc_socket_reset_connection(&transport->xprt,
-                        transport->sock, result);
-        if (!result)
-                xs_sock_reset_connection_flags(&transport->xprt);
-        dprintk("RPC:       AF_UNSPEC connect return code %d\n", result);
-}
-
-static void xs_tcp_reuse_connection(struct sock_xprt *transport)
-{
-        unsigned int state = transport->inet->sk_state;
-
-        if (state == TCP_CLOSE && transport->sock->state == SS_UNCONNECTED) {
-                /* we don't need to abort the connection if the socket
-                 * hasn't undergone a shutdown
-                 */
-                if (transport->inet->sk_shutdown == 0)
-                        return;
-                dprintk("RPC:       %s: TCP_CLOSEd and sk_shutdown set to %d\n",
-                                __func__, transport->inet->sk_shutdown);
-        }
-        if ((1 << state) & (TCPF_ESTABLISHED|TCPF_SYN_SENT)) {
-                /* we don't need to abort the connection if the socket
-                 * hasn't undergone a shutdown
-                 */
-                if (transport->inet->sk_shutdown == 0)
-                        return;
-                dprintk("RPC:       %s: ESTABLISHED/SYN_SENT "
-                                "sk_shutdown set to %d\n",
-                                __func__, transport->inet->sk_shutdown);
-        }
-        xs_abort_connection(transport);
-}
-
 static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
 {
         struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
@@ -2149,9 +2090,7 @@ static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
                 sk->sk_allocation = GFP_ATOMIC;
 
                 /* socket options */
-                sk->sk_userlocks |= SOCK_BINDPORT_LOCK;
                 sock_reset_flag(sk, SOCK_LINGER);
-                tcp_sk(sk)->linger2 = 0;
                 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF;
 
                 xprt_clear_connected(xprt);
@@ -2174,6 +2113,7 @@ static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
         ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK);
         switch (ret) {
         case 0:
+                xs_set_srcport(transport, sock);
         case -EINPROGRESS:
                 /* SYN_SENT! */
                 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
@@ -2200,25 +2140,13 @@ static void xs_tcp_setup_socket(struct work_struct *work)
         int status = -EIO;
 
         if (!sock) {
-                clear_bit(XPRT_CONNECTION_ABORT, &xprt->state);
                 sock = xs_create_sock(xprt, transport,
-                                xs_addr(xprt)->sa_family, SOCK_STREAM, IPPROTO_TCP);
+                                xs_addr(xprt)->sa_family, SOCK_STREAM,
+                                IPPROTO_TCP, true);
                 if (IS_ERR(sock)) {
                         status = PTR_ERR(sock);
                         goto out;
                 }
-        } else {
-                int abort_and_exit;
-
-                abort_and_exit = test_and_clear_bit(XPRT_CONNECTION_ABORT,
-                                &xprt->state);
-                /* "close" the socket, preserving the local port */
-                set_bit(XPRT_CONNECTION_REUSE, &xprt->state);
-                xs_tcp_reuse_connection(transport);
-                clear_bit(XPRT_CONNECTION_REUSE, &xprt->state);
-
-                if (abort_and_exit)
-                        goto out_eagain;
         }
 
         dprintk("RPC:       worker connecting xprt %p via %s to "
@@ -2245,6 +2173,7 @@ static void xs_tcp_setup_socket(struct work_struct *work)
         case 0:
         case -EINPROGRESS:
         case -EALREADY:
+                xprt_unlock_connect(xprt, transport);
                 xprt_clear_connecting(xprt);
                 return;
         case -EINVAL:
@@ -2254,13 +2183,15 @@ static void xs_tcp_setup_socket(struct work_struct *work)
         case -ECONNREFUSED:
         case -ECONNRESET:
         case -ENETUNREACH:
+        case -EADDRINUSE:
         case -ENOBUFS:
                 /* retry with existing socket, after a delay */
+                xs_tcp_force_close(xprt);
                 goto out;
         }
-out_eagain:
         status = -EAGAIN;
 out:
+        xprt_unlock_connect(xprt, transport);
         xprt_clear_connecting(xprt);
         xprt_wake_pending_tasks(xprt, status);
 }
@@ -2283,6 +2214,11 @@ static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
 {
         struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
 
+        WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport));
+
+        /* Start by resetting any existing state */
+        xs_reset_transport(transport);
+
         if (transport->sock != NULL && !RPC_IS_SOFTCONN(task)) {
                 dprintk("RPC:       xs_connect delayed xprt %p for %lu "
                                 "seconds\n",
@@ -2559,7 +2495,7 @@ static struct rpc_xprt_ops xs_tcp_ops = {
         .buf_free               = rpc_free,
         .send_request           = xs_tcp_send_request,
         .set_retrans_timeout    = xprt_set_retrans_timeout_def,
-        .close                  = xs_tcp_close,
+        .close                  = xs_tcp_shutdown,
         .destroy                = xs_destroy,
         .print_stats            = xs_tcp_print_stats,
 };