xfs: abort metadata writeback on permanent errors

If we are doing aysnc writeback of metadata, we can get write errors
but have nobody to report them to. At the moment, we simply attempt
to reissue the write from io completion in the hope that it's a
transient error.

When it's not a transient error, the buffer is stuck forever in
this loop, and we cannot break out of it. Eventually, unmount will
hang because the AIL cannot be emptied and everything goes downhill
from them.

To solve this problem, only retry the write IO once before aborting
it. We don't throw the buffer away because some transient errors can
last minutes (e.g.  FC path failover) or even hours (thin
provisioned devices that have run out of backing space) before they
go away. Hence we really want to keep trying until we can't try any
more.

Because the buffer was not cleaned, however, it does not get removed
from the AIL and hence the next pass across the AIL will start IO on
it again. As such, we still get the "retry forever" semantics that
we currently have, but we allow other access to the buffer in the
mean time. Meanwhile the filesystem can continue to modify the
buffer and relog it, so the IO errors won't hang the log or the
filesystem.

Now when we are pushing the AIL, we can see all these "permanent IO
error" buffers and we can issue a warning about failures before we
retry the IO. We can also catch these buffers when unmounting an
issue a corruption warning, too.

Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Ben Myers <bpm@sgi.com>

3 files changed, 32 insertions, 5 deletions

diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index 9fa9c4304613..afe7645e4b2b 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -1156,7 +1156,7 @@ xfs_bwrite(
         ASSERT(xfs_buf_islocked(bp));
 
         bp->b_flags |= XBF_WRITE;
-        bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q);
+        bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q | XBF_WRITE_FAIL);
 
         xfs_bdstrat_cb(bp);
 
@@ -1501,6 +1501,12 @@ xfs_wait_buftarg(
                         struct xfs_buf *bp;
                         bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
                         list_del_init(&bp->b_lru);
+                        if (bp->b_flags & XBF_WRITE_FAIL) {
+                                xfs_alert(btp->bt_mount,
+"Corruption Alert: Buffer at block 0x%llx had permanent write failures!\n"
+"Please run xfs_repair to determine the extent of the problem.",
+                                        (long long)bp->b_bn);
+                        }
                         xfs_buf_rele(bp);
                 }
                 if (loop++ != 0)
@@ -1784,7 +1790,7 @@ __xfs_buf_delwri_submit(
 
         blk_start_plug(&plug);
         list_for_each_entry_safe(bp, n, io_list, b_list) {
-                bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_ASYNC);
+                bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_ASYNC | XBF_WRITE_FAIL);
                 bp->b_flags |= XBF_WRITE;
 
                 if (!wait) {
diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h
index 7e41b08017f7..1cf21a4a9f22 100644
--- a/fs/xfs/xfs_buf.h
+++ b/fs/xfs/xfs_buf.h
@@ -45,6 +45,7 @@ typedef enum {
 #define XBF_ASYNC        (1 << 4) /* initiator will not wait for completion */
 #define XBF_DONE         (1 << 5) /* all pages in the buffer uptodate */
 #define XBF_STALE        (1 << 6) /* buffer has been staled, do not find it */
+#define XBF_WRITE_FAIL   (1 << 24)/* async writes have failed on this buffer */
 
 /* I/O hints for the BIO layer */
 #define XBF_SYNCIO       (1 << 10)/* treat this buffer as synchronous I/O */
@@ -70,6 +71,7 @@ typedef unsigned int xfs_buf_flags_t;
         { XBF_ASYNC,            "ASYNC" }, \
         { XBF_DONE,             "DONE" }, \
         { XBF_STALE,            "STALE" }, \
+        { XBF_WRITE_FAIL,       "WRITE_FAIL" }, \
         { XBF_SYNCIO,           "SYNCIO" }, \
         { XBF_FUA,              "FUA" }, \
         { XBF_FLUSH,            "FLUSH" }, \
@@ -80,6 +82,7 @@ typedef unsigned int xfs_buf_flags_t;
         { _XBF_DELWRI_Q,        "DELWRI_Q" }, \
         { _XBF_COMPOUND,        "COMPOUND" }
 
+
 /*
  * Internal state flags.
  */
@@ -300,7 +303,8 @@ extern void xfs_buf_terminate(void);
 
 #define XFS_BUF_ZEROFLAGS(bp) \
         ((bp)->b_flags &= ~(XBF_READ|XBF_WRITE|XBF_ASYNC| \
-                            XBF_SYNCIO|XBF_FUA|XBF_FLUSH))
+                            XBF_SYNCIO|XBF_FUA|XBF_FLUSH| \
+                            XBF_WRITE_FAIL))
 
 void xfs_buf_stale(struct xfs_buf *bp);
 #define XFS_BUF_UNSTALE(bp)     ((bp)->b_flags &= ~XBF_STALE)
diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c
index a64f67ba25d3..2227b9b050bb 100644
--- a/fs/xfs/xfs_buf_item.c
+++ b/fs/xfs/xfs_buf_item.c
@@ -496,6 +496,14 @@ xfs_buf_item_unpin(
         }
 }
 
+/*
+ * Buffer IO error rate limiting. Limit it to no more than 10 messages per 30
+ * seconds so as to not spam logs too much on repeated detection of the same
+ * buffer being bad..
+ */
+
+DEFINE_RATELIMIT_STATE(xfs_buf_write_fail_rl_state, 30 * HZ, 10);
+
 STATIC uint
 xfs_buf_item_push(
         struct xfs_log_item     *lip,
@@ -524,6 +532,14 @@ xfs_buf_item_push(
 
         trace_xfs_buf_item_push(bip);
 
+        /* has a previous flush failed due to IO errors? */
+        if ((bp->b_flags & XBF_WRITE_FAIL) &&
+            ___ratelimit(&xfs_buf_write_fail_rl_state, "XFS:")) {
+                xfs_warn(bp->b_target->bt_mount,
+"Detected failing async write on buffer block 0x%llx. Retrying async write.\n",
+                         (long long)bp->b_bn);
+        }
+
         if (!xfs_buf_delwri_queue(bp, buffer_list))
                 rval = XFS_ITEM_FLUSHING;
         xfs_buf_unlock(bp);
@@ -1096,8 +1112,9 @@ xfs_buf_iodone_callbacks(
 
                 xfs_buf_ioerror(bp, 0); /* errno of 0 unsets the flag */
 
-                if (!XFS_BUF_ISSTALE(bp)) {
-                        bp->b_flags |= XBF_WRITE | XBF_ASYNC | XBF_DONE;
+                if (!(bp->b_flags & (XBF_STALE|XBF_WRITE_FAIL))) {
+                        bp->b_flags |= XBF_WRITE | XBF_ASYNC |
+                                       XBF_DONE | XBF_WRITE_FAIL;
                         xfs_buf_iorequest(bp);
                 } else {
                         xfs_buf_relse(bp);