pnfs: layoutreturn

NFSv4.1 LAYOUTRETURN implementation

Currently, does not support layout-type payload encoding.

Signed-off-by: Alexandros Batsakis <batsakis@netapp.com>
Signed-off-by: Andy Adamson <andros@citi.umich.edu>
Signed-off-by: Andy Adamson <andros@netapp.com>
Signed-off-by: Dean Hildebrand <dhildeb@us.ibm.com>
Signed-off-by: Fred Isaman <iisaman@citi.umich.edu>
Signed-off-by: Fred Isaman <iisaman@netapp.com>
Signed-off-by: Marc Eshel <eshel@almaden.ibm.com>
Signed-off-by: Zhang Jingwang <zhangjingwang@nrchpc.ac.cn>
[call pnfs_return_layout right before pnfs_destroy_layout]
[remove assert_spin_locked from pnfs_clear_lseg_list]
[remove wait parameter from the layoutreturn path.]
[remove return_type field from nfs4_layoutreturn_args]
[remove range from nfs4_layoutreturn_args]
[no need to send layoutcommit from _pnfs_return_layout]
[don't wait on sync layoutreturn]
[fix layout stateid in layoutreturn args]
[fixed NULL deref in _pnfs_return_layout]
[removed recaim member of nfs4_layoutreturn_args]
Signed-off-by: Benny Halevy <bhalevy@panasas.com>

5 files changed, 252 insertions, 7 deletions

diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 57bb31ad7a5e..e9c6d9f8f7e8 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -1424,9 +1424,10 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr)
  */
 void nfs4_evict_inode(struct inode *inode)
 {
-        pnfs_destroy_layout(NFS_I(inode));
         truncate_inode_pages(&inode->i_data, 0);
         end_writeback(inode);
+        pnfs_return_layout(inode);
+        pnfs_destroy_layout(NFS_I(inode));
         /* If we are holding a delegation, return it! */
         nfs_inode_return_delegation_noreclaim(inode);
         /* First call standard NFS clear_inode() code */
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 92c8bc4b5f97..5b4124e4c22f 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -5673,6 +5673,88 @@ int nfs4_proc_layoutget(struct nfs4_layoutget *lgp)
         return status;
 }
 
+static void
+nfs4_layoutreturn_prepare(struct rpc_task *task, void *calldata)
+{
+        struct nfs4_layoutreturn *lrp = calldata;
+
+        dprintk("--> %s\n", __func__);
+        if (nfs41_setup_sequence(lrp->clp->cl_session, &lrp->args.seq_args,
+                                &lrp->res.seq_res, 0, task))
+                return;
+        rpc_call_start(task);
+}
+
+static void nfs4_layoutreturn_done(struct rpc_task *task, void *calldata)
+{
+        struct nfs4_layoutreturn *lrp = calldata;
+        struct nfs_server *server;
+
+        dprintk("--> %s\n", __func__);
+
+        if (!nfs4_sequence_done(task, &lrp->res.seq_res))
+                return;
+
+        server = NFS_SERVER(lrp->args.inode);
+        if (nfs4_async_handle_error(task, server, NULL) == -EAGAIN) {
+                nfs_restart_rpc(task, lrp->clp);
+                return;
+        }
+        if (task->tk_status == 0) {
+                struct pnfs_layout_hdr *lo = NFS_I(lrp->args.inode)->layout;
+
+                if (lrp->res.lrs_present) {
+                        spin_lock(&lo->plh_inode->i_lock);
+                        pnfs_set_layout_stateid(lo, &lrp->res.stateid, true);
+                        spin_unlock(&lo->plh_inode->i_lock);
+                } else
+                        BUG_ON(!list_empty(&lo->plh_segs));
+        }
+        dprintk("<-- %s\n", __func__);
+}
+
+static void nfs4_layoutreturn_release(void *calldata)
+{
+        struct nfs4_layoutreturn *lrp = calldata;
+
+        dprintk("--> %s\n", __func__);
+        put_layout_hdr(NFS_I(lrp->args.inode)->layout);
+        kfree(calldata);
+        dprintk("<-- %s\n", __func__);
+}
+
+static const struct rpc_call_ops nfs4_layoutreturn_call_ops = {
+        .rpc_call_prepare = nfs4_layoutreturn_prepare,
+        .rpc_call_done = nfs4_layoutreturn_done,
+        .rpc_release = nfs4_layoutreturn_release,
+};
+
+int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp)
+{
+        struct rpc_task *task;
+        struct rpc_message msg = {
+                .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTRETURN],
+                .rpc_argp = &lrp->args,
+                .rpc_resp = &lrp->res,
+        };
+        struct rpc_task_setup task_setup_data = {
+                .rpc_client = lrp->clp->cl_rpcclient,
+                .rpc_message = &msg,
+                .callback_ops = &nfs4_layoutreturn_call_ops,
+                .callback_data = lrp,
+        };
+        int status;
+
+        dprintk("--> %s\n", __func__);
+        task = rpc_run_task(&task_setup_data);
+        if (IS_ERR(task))
+                return PTR_ERR(task);
+        status = task->tk_status;
+        dprintk("<-- %s status=%d\n", __func__, status);
+        rpc_put_task(task);
+        return status;
+}
+
 static int
 _nfs4_proc_getdeviceinfo(struct nfs_server *server, struct pnfs_device *pdev)
 {
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index c3ccd2c46834..f24212064356 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -338,7 +338,11 @@ static int nfs4_stat_to_errno(int);
                                 1 /* layoutupdate4 layout type */ + \
                                 1 /* NULL filelayout layoutupdate4 payload */)
 #define decode_layoutcommit_maxsz (op_decode_hdr_maxsz + 3)
-
+#define encode_layoutreturn_maxsz (8 + op_encode_hdr_maxsz + \
+                                encode_stateid_maxsz + \
+                                1 /* FIXME: opaque lrf_body always empty at the moment */)
+#define decode_layoutreturn_maxsz (op_decode_hdr_maxsz + \
+                                1 + decode_stateid_maxsz)
 #else /* CONFIG_NFS_V4_1 */
 #define encode_sequence_maxsz   0
 #define decode_sequence_maxsz   0
@@ -760,7 +764,14 @@ static int nfs4_stat_to_errno(int);
                                 decode_putfh_maxsz + \
                                 decode_layoutcommit_maxsz + \
                                 decode_getattr_maxsz)
-
+#define NFS4_enc_layoutreturn_sz (compound_encode_hdr_maxsz + \
+                                encode_sequence_maxsz + \
+                                encode_putfh_maxsz + \
+                                encode_layoutreturn_maxsz)
+#define NFS4_dec_layoutreturn_sz (compound_decode_hdr_maxsz + \
+                                decode_sequence_maxsz + \
+                                decode_putfh_maxsz + \
+                                decode_layoutreturn_maxsz)
 
 const u32 nfs41_maxwrite_overhead = ((RPC_MAX_HEADER_WITH_AUTH +
                                       compound_encode_hdr_maxsz +
@@ -1889,6 +1900,31 @@ encode_layoutcommit(struct xdr_stream *xdr,
         hdr->replen += decode_layoutcommit_maxsz;
         return 0;
 }
+
+static void
+encode_layoutreturn(struct xdr_stream *xdr,
+                    const struct nfs4_layoutreturn_args *args,
+                    struct compound_hdr *hdr)
+{
+        __be32 *p;
+
+        p = reserve_space(xdr, 20);
+        *p++ = cpu_to_be32(OP_LAYOUTRETURN);
+        *p++ = cpu_to_be32(0);          /* reclaim. always 0 for now */
+        *p++ = cpu_to_be32(args->layout_type);
+        *p++ = cpu_to_be32(IOMODE_ANY);
+        *p = cpu_to_be32(RETURN_FILE);
+        p = reserve_space(xdr, 16 + NFS4_STATEID_SIZE);
+        p = xdr_encode_hyper(p, 0);
+        p = xdr_encode_hyper(p, NFS4_MAX_UINT64);
+        spin_lock(&args->inode->i_lock);
+        xdr_encode_opaque_fixed(p, &args->stateid.data, NFS4_STATEID_SIZE);
+        spin_unlock(&args->inode->i_lock);
+        p = reserve_space(xdr, 4);
+        *p = cpu_to_be32(0);
+        hdr->nops++;
+        hdr->replen += decode_layoutreturn_maxsz;
+}
 #endif /* CONFIG_NFS_V4_1 */
 
 /*
@@ -2706,9 +2742,9 @@ static void nfs4_xdr_enc_layoutget(struct rpc_rqst *req,
 /*
  *  Encode LAYOUTCOMMIT request
  */
-static int nfs4_xdr_enc_layoutcommit(struct rpc_rqst *req,
-                                     struct xdr_stream *xdr,
-                                     struct nfs4_layoutcommit_args *args)
+static void nfs4_xdr_enc_layoutcommit(struct rpc_rqst *req,
+                                      struct xdr_stream *xdr,
+                                      struct nfs4_layoutcommit_args *args)
 {
         struct compound_hdr hdr = {
                 .minorversion = nfs4_xdr_minorversion(&args->seq_args),
@@ -2720,7 +2756,24 @@ static int nfs4_xdr_enc_layoutcommit(struct rpc_rqst *req,
         encode_layoutcommit(xdr, args, &hdr);
         encode_getfattr(xdr, args->bitmask, &hdr);
         encode_nops(&hdr);
-        return 0;
+}
+
+/*
+ * Encode LAYOUTRETURN request
+ */
+static void nfs4_xdr_enc_layoutreturn(struct rpc_rqst *req,
+                                      struct xdr_stream *xdr,
+                                      struct nfs4_layoutreturn_args *args)
+{
+        struct compound_hdr hdr = {
+                .minorversion = nfs4_xdr_minorversion(&args->seq_args),
+        };
+
+        encode_compound_hdr(xdr, req, &hdr);
+        encode_sequence(xdr, &args->seq_args, &hdr);
+        encode_putfh(xdr, NFS_FH(args->inode), &hdr);
+        encode_layoutreturn(xdr, args, &hdr);
+        encode_nops(&hdr);
 }
 #endif /* CONFIG_NFS_V4_1 */
 
@@ -5203,6 +5256,27 @@ out_overflow:
         return -EIO;
 }
 
+static int decode_layoutreturn(struct xdr_stream *xdr,
+                               struct nfs4_layoutreturn_res *res)
+{
+        __be32 *p;
+        int status;
+
+        status = decode_op_hdr(xdr, OP_LAYOUTRETURN);
+        if (status)
+                return status;
+        p = xdr_inline_decode(xdr, 4);
+        if (unlikely(!p))
+                goto out_overflow;
+        res->lrs_present = be32_to_cpup(p);
+        if (res->lrs_present)
+                status = decode_stateid(xdr, &res->stateid);
+        return status;
+out_overflow:
+        print_overflow_msg(__func__, xdr);
+        return -EIO;
+}
+
 static int decode_layoutcommit(struct xdr_stream *xdr,
                                struct rpc_rqst *req,
                                struct nfs4_layoutcommit_res *res)
@@ -6320,6 +6394,30 @@ out:
 }
 
 /*
+ * Decode LAYOUTRETURN response
+ */
+static int nfs4_xdr_dec_layoutreturn(struct rpc_rqst *rqstp,
+                                     struct xdr_stream *xdr,
+                                     struct nfs4_layoutreturn_res *res)
+{
+        struct compound_hdr hdr;
+        int status;
+
+        status = decode_compound_hdr(xdr, &hdr);
+        if (status)
+                goto out;
+        status = decode_sequence(xdr, &res->seq_res, rqstp);
+        if (status)
+                goto out;
+        status = decode_putfh(xdr);
+        if (status)
+                goto out;
+        status = decode_layoutreturn(xdr, res);
+out:
+        return status;
+}
+
+/*
  * Decode LAYOUTCOMMIT response
  */
 static int nfs4_xdr_dec_layoutcommit(struct rpc_rqst *rqstp,
@@ -6547,6 +6645,7 @@ struct rpc_procinfo	nfs4_procedures[] = {
         PROC(GETDEVICEINFO,     enc_getdeviceinfo,      dec_getdeviceinfo),
         PROC(LAYOUTGET,         enc_layoutget,          dec_layoutget),
         PROC(LAYOUTCOMMIT,      enc_layoutcommit,       dec_layoutcommit),
+        PROC(LAYOUTRETURN,      enc_layoutreturn,       dec_layoutreturn),
 #endif /* CONFIG_NFS_V4_1 */
 };
 
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 171662114fdd..00b128241746 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -619,6 +619,51 @@ out_err_free:
         return NULL;
 }
 
+/* Initiates a LAYOUTRETURN(FILE) */
+int
+_pnfs_return_layout(struct inode *ino)
+{
+        struct pnfs_layout_hdr *lo = NULL;
+        struct nfs_inode *nfsi = NFS_I(ino);
+        LIST_HEAD(tmp_list);
+        struct nfs4_layoutreturn *lrp;
+        nfs4_stateid stateid;
+        int status = 0;
+
+        dprintk("--> %s\n", __func__);
+
+        spin_lock(&ino->i_lock);
+        lo = nfsi->layout;
+        if (!lo || !mark_matching_lsegs_invalid(lo, &tmp_list, NULL)) {
+                spin_unlock(&ino->i_lock);
+                dprintk("%s: no layout segments to return\n", __func__);
+                goto out;
+        }
+        stateid = nfsi->layout->plh_stateid;
+        /* Reference matched in nfs4_layoutreturn_release */
+        get_layout_hdr(lo);
+        spin_unlock(&ino->i_lock);
+        pnfs_free_lseg_list(&tmp_list);
+
+        WARN_ON(test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags));
+
+        lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
+        if (unlikely(lrp == NULL)) {
+                status = -ENOMEM;
+                goto out;
+        }
+
+        lrp->args.stateid = stateid;
+        lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
+        lrp->args.inode = ino;
+        lrp->clp = NFS_SERVER(ino)->nfs_client;
+
+        status = nfs4_proc_layoutreturn(lrp);
+out:
+        dprintk("<-- %s status: %d\n", __func__, status);
+        return status;
+}
+
 bool pnfs_roc(struct inode *ino)
 {
         struct pnfs_layout_hdr *lo;
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 0383e66e71f0..c34f7a0e3bc2 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -129,6 +129,7 @@ extern void pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *);
 extern int nfs4_proc_getdeviceinfo(struct nfs_server *server,
                                    struct pnfs_device *dev);
 extern int nfs4_proc_layoutget(struct nfs4_layoutget *lgp);
+extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp);
 
 /* pnfs.c */
 void get_layout_hdr(struct pnfs_layout_hdr *lo);
@@ -165,6 +166,7 @@ void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
 bool pnfs_roc_drain(struct inode *ino, u32 *barrier);
 void pnfs_set_layoutcommit(struct nfs_write_data *wdata);
 int pnfs_layoutcommit_inode(struct inode *inode, bool sync);
+int _pnfs_return_layout(struct inode *);
 int pnfs_ld_write_done(struct nfs_write_data *);
 int pnfs_ld_read_done(struct nfs_read_data *);
 
@@ -256,6 +258,17 @@ static inline void pnfs_clear_request_commit(struct nfs_page *req)
                 put_lseg(req->wb_commit_lseg);
 }
 
+static inline int pnfs_return_layout(struct inode *ino)
+{
+        struct nfs_inode *nfsi = NFS_I(ino);
+        struct nfs_server *nfss = NFS_SERVER(ino);
+
+        if (pnfs_enabled_sb(nfss) && nfsi->layout)
+                return _pnfs_return_layout(ino);
+
+        return 0;
+}
+
 #else  /* CONFIG_NFS_V4_1 */
 
 static inline void pnfs_destroy_all_layouts(struct nfs_client *clp)
@@ -298,6 +311,11 @@ pnfs_try_to_write_data(struct nfs_write_data *data,
         return PNFS_NOT_ATTEMPTED;
 }
 
+static inline int pnfs_return_layout(struct inode *ino)
+{
+        return 0;
+}
+
 static inline bool
 pnfs_roc(struct inode *ino)
 {