pnfs/flexfiles: Add tracepoints for detecting pnfs fallback to MDS

Add tracepoints to allow debugging of the event chain leading to
a pnfs fallback to doing I/O through the MDS.

Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>

5 files changed, 112 insertions, 1 deletions

diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index bcff3bf5ae09..b04e20d28162 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -934,6 +934,10 @@ out_nolseg:
         if (pgio->pg_error < 0)
                 return;
 out_mds:
+        trace_pnfs_mds_fallback_pg_init_read(pgio->pg_inode,
+                        0, NFS4_MAX_UINT64, IOMODE_READ,
+                        NFS_I(pgio->pg_inode)->layout,
+                        pgio->pg_lseg);
         pnfs_put_lseg(pgio->pg_lseg);
         pgio->pg_lseg = NULL;
         nfs_pageio_reset_read_mds(pgio);
@@ -1000,6 +1004,10 @@ retry:
         return;
 
 out_mds:
+        trace_pnfs_mds_fallback_pg_init_write(pgio->pg_inode,
+                        0, NFS4_MAX_UINT64, IOMODE_RW,
+                        NFS_I(pgio->pg_inode)->layout,
+                        pgio->pg_lseg);
         pnfs_put_lseg(pgio->pg_lseg);
         pgio->pg_lseg = NULL;
         nfs_pageio_reset_write_mds(pgio);
@@ -1026,6 +1034,10 @@ ff_layout_pg_get_mirror_count_write(struct nfs_pageio_descriptor *pgio,
         if (pgio->pg_lseg)
                 return FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg);
 
+        trace_pnfs_mds_fallback_pg_get_mirror_count(pgio->pg_inode,
+                        0, NFS4_MAX_UINT64, IOMODE_RW,
+                        NFS_I(pgio->pg_inode)->layout,
+                        pgio->pg_lseg);
         /* no lseg means that pnfs is not in use, so no mirroring here */
         nfs_pageio_reset_write_mds(pgio);
 out:
@@ -1075,6 +1087,10 @@ static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs)
                         hdr->args.count,
                         (unsigned long long)hdr->args.offset);
 
+                trace_pnfs_mds_fallback_write_done(hdr->inode,
+                                hdr->args.offset, hdr->args.count,
+                                IOMODE_RW, NFS_I(hdr->inode)->layout,
+                                hdr->lseg);
                 task->tk_status = pnfs_write_done_resend_to_mds(hdr);
         }
 }
@@ -1094,6 +1110,10 @@ static void ff_layout_reset_read(struct nfs_pgio_header *hdr)
                         hdr->args.count,
                         (unsigned long long)hdr->args.offset);
 
+                trace_pnfs_mds_fallback_read_done(hdr->inode,
+                                hdr->args.offset, hdr->args.count,
+                                IOMODE_READ, NFS_I(hdr->inode)->layout,
+                                hdr->lseg);
                 task->tk_status = pnfs_read_done_resend_to_mds(hdr);
         }
 }
@@ -1827,6 +1847,9 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
 out_failed:
         if (ff_layout_avoid_mds_available_ds(lseg))
                 return PNFS_TRY_AGAIN;
+        trace_pnfs_mds_fallback_read_pagelist(hdr->inode,
+                        hdr->args.offset, hdr->args.count,
+                        IOMODE_READ, NFS_I(hdr->inode)->layout, lseg);
         return PNFS_NOT_ATTEMPTED;
 }
 
@@ -1892,6 +1915,9 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 out_failed:
         if (ff_layout_avoid_mds_available_ds(lseg))
                 return PNFS_TRY_AGAIN;
+        trace_pnfs_mds_fallback_write_pagelist(hdr->inode,
+                        hdr->args.offset, hdr->args.count,
+                        IOMODE_RW, NFS_I(hdr->inode)->layout, lseg);
         return PNFS_NOT_ATTEMPTED;
 }
 
diff --git a/fs/nfs/nfs4trace.c b/fs/nfs/nfs4trace.c
index e9fb3e50a999..1a8f376b3f73 100644
--- a/fs/nfs/nfs4trace.c
+++ b/fs/nfs/nfs4trace.c
@@ -16,4 +16,12 @@
 EXPORT_TRACEPOINT_SYMBOL_GPL(nfs4_pnfs_read);
 EXPORT_TRACEPOINT_SYMBOL_GPL(nfs4_pnfs_write);
 EXPORT_TRACEPOINT_SYMBOL_GPL(nfs4_pnfs_commit_ds);
+
+EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_pg_init_read);
+EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_pg_init_write);
+EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_pg_get_mirror_count);
+EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_read_done);
+EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_write_done);
+EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_read_pagelist);
+EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_write_pagelist);
 #endif
diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h
index d85f20945a2b..b2f395fa7350 100644
--- a/fs/nfs/nfs4trace.h
+++ b/fs/nfs/nfs4trace.h
@@ -1771,6 +1771,7 @@ TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_BLOCKED);
 TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_INVALID_OPEN);
 TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_RETRY);
 TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
+TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_EXIT);
 
 #define show_pnfs_update_layout_reason(reason)                          \
         __print_symbolic(reason,                                        \
@@ -1786,7 +1787,8 @@ TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
                 { PNFS_UPDATE_LAYOUT_BLOCKED, "layouts blocked" },      \
                 { PNFS_UPDATE_LAYOUT_INVALID_OPEN, "invalid open" },    \
                 { PNFS_UPDATE_LAYOUT_RETRY, "retrying" },       \
-                { PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET, "sent layoutget" })
+                { PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET, "sent layoutget" }, \
+                { PNFS_UPDATE_LAYOUT_EXIT, "exit" })
 
 TRACE_EVENT(pnfs_update_layout,
                 TP_PROTO(struct inode *inode,
@@ -1845,6 +1847,78 @@ TRACE_EVENT(pnfs_update_layout,
                 )
 );
 
+DECLARE_EVENT_CLASS(pnfs_layout_event,
+                TP_PROTO(struct inode *inode,
+                        loff_t pos,
+                        u64 count,
+                        enum pnfs_iomode iomode,
+                        struct pnfs_layout_hdr *lo,
+                        struct pnfs_layout_segment *lseg
+                ),
+                TP_ARGS(inode, pos, count, iomode, lo, lseg),
+                TP_STRUCT__entry(
+                        __field(dev_t, dev)
+                        __field(u64, fileid)
+                        __field(u32, fhandle)
+                        __field(loff_t, pos)
+                        __field(u64, count)
+                        __field(enum pnfs_iomode, iomode)
+                        __field(int, layoutstateid_seq)
+                        __field(u32, layoutstateid_hash)
+                        __field(long, lseg)
+                ),
+                TP_fast_assign(
+                        __entry->dev = inode->i_sb->s_dev;
+                        __entry->fileid = NFS_FILEID(inode);
+                        __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode));
+                        __entry->pos = pos;
+                        __entry->count = count;
+                        __entry->iomode = iomode;
+                        if (lo != NULL) {
+                                __entry->layoutstateid_seq =
+                                be32_to_cpu(lo->plh_stateid.seqid);
+                                __entry->layoutstateid_hash =
+                                nfs_stateid_hash(&lo->plh_stateid);
+                        } else {
+                                __entry->layoutstateid_seq = 0;
+                                __entry->layoutstateid_hash = 0;
+                        }
+                        __entry->lseg = (long)lseg;
+                ),
+                TP_printk(
+                        "fileid=%02x:%02x:%llu fhandle=0x%08x "
+                        "iomode=%s pos=%llu count=%llu "
+                        "layoutstateid=%d:0x%08x lseg=0x%lx",
+                        MAJOR(__entry->dev), MINOR(__entry->dev),
+                        (unsigned long long)__entry->fileid,
+                        __entry->fhandle,
+                        show_pnfs_iomode(__entry->iomode),
+                        (unsigned long long)__entry->pos,
+                        (unsigned long long)__entry->count,
+                        __entry->layoutstateid_seq, __entry->layoutstateid_hash,
+                        __entry->lseg
+                )
+);
+
+#define DEFINE_PNFS_LAYOUT_EVENT(name) \
+        DEFINE_EVENT(pnfs_layout_event, name, \
+                TP_PROTO(struct inode *inode, \
+                        loff_t pos, \
+                        u64 count, \
+                        enum pnfs_iomode iomode, \
+                        struct pnfs_layout_hdr *lo, \
+                        struct pnfs_layout_segment *lseg \
+                ), \
+                TP_ARGS(inode, pos, count, iomode, lo, lseg))
+
+DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_pg_init_read);
+DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_pg_init_write);
+DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_pg_get_mirror_count);
+DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_read_done);
+DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_write_done);
+DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_read_pagelist);
+DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_write_pagelist);
+
 #endif /* CONFIG_NFS_V4_1 */
 
 #endif /* _TRACE_NFS4_H */
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 758917463700..75bd5b552ba4 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -2037,6 +2037,8 @@ lookup_again:
 out_put_layout_hdr:
         if (first)
                 pnfs_clear_first_layoutget(lo);
+        trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
+                                 PNFS_UPDATE_LAYOUT_EXIT);
         pnfs_put_layout_hdr(lo);
 out:
         dprintk("%s: inode %s/%llu pNFS layout segment %s for "
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index 22494d170619..fd59904a282c 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -660,6 +660,7 @@ enum pnfs_update_layout_reason {
         PNFS_UPDATE_LAYOUT_BLOCKED,
         PNFS_UPDATE_LAYOUT_INVALID_OPEN,
         PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET,
+        PNFS_UPDATE_LAYOUT_EXIT,
 };
 
 #define NFS4_OP_MAP_NUM_LONGS                                   \