6 files changed, 45 insertions, 55 deletions

diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index 48b1e2989ea4..a046149e6099 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -998,26 +998,30 @@ xfs_buf_wait_unpin(
  *      Buffer Utility Routines
  */
 
-STATIC void
-xfs_buf_iodone_work(
-        struct work_struct      *work)
+void
+xfs_buf_ioend(
+        struct xfs_buf  *bp)
 {
-        struct xfs_buf          *bp =
-                container_of(work, xfs_buf_t, b_iodone_work);
-        bool                    read = !!(bp->b_flags & XBF_READ);
+        bool            read = bp->b_flags & XBF_READ;
+
+        trace_xfs_buf_iodone(bp, _RET_IP_);
 
         bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
 
-        /* only validate buffers that were read without errors */
-        if (read && bp->b_ops && !bp->b_error && (bp->b_flags & XBF_DONE))
+        /* Only validate buffers that were read without errors */
+        if (read && !bp->b_error && bp->b_ops) {
+                ASSERT(!bp->b_iodone);
                 bp->b_ops->verify_read(bp);
+        }
+
+        if (!bp->b_error)
+                bp->b_flags |= XBF_DONE;
 
         if (bp->b_iodone)
                 (*(bp->b_iodone))(bp);
         else if (bp->b_flags & XBF_ASYNC)
                 xfs_buf_relse(bp);
         else {
-                ASSERT(read && bp->b_ops);
                 complete(&bp->b_iowait);
 
                 /* release the !XBF_ASYNC ref now we are done. */
@@ -1025,30 +1029,22 @@ xfs_buf_iodone_work(
         }
 }
 
-void
-xfs_buf_ioend(
-        struct xfs_buf  *bp,
-        int             schedule)
+static void
+xfs_buf_ioend_work(
+        struct work_struct      *work)
 {
-        bool            read = !!(bp->b_flags & XBF_READ);
-
-        trace_xfs_buf_iodone(bp, _RET_IP_);
+        struct xfs_buf          *bp =
+                container_of(work, xfs_buf_t, b_iodone_work);
 
-        if (bp->b_error == 0)
-                bp->b_flags |= XBF_DONE;
+        xfs_buf_ioend(bp);
+}
 
-        if (bp->b_iodone || (read && bp->b_ops) || (bp->b_flags & XBF_ASYNC)) {
-                if (schedule) {
-                        INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
-                        queue_work(xfslogd_workqueue, &bp->b_iodone_work);
-                } else {
-                        xfs_buf_iodone_work(&bp->b_iodone_work);
-                }
-        } else {
-                bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
-                complete(&bp->b_iowait);
-                xfs_buf_rele(bp);
-        }
+void
+xfs_buf_ioend_async(
+        struct xfs_buf  *bp)
+{
+        INIT_WORK(&bp->b_iodone_work, xfs_buf_ioend_work);
+        queue_work(xfslogd_workqueue, &bp->b_iodone_work);
 }
 
 void
@@ -1099,7 +1095,7 @@ xfs_bioerror(
         XFS_BUF_UNDONE(bp);
         xfs_buf_stale(bp);
 
-        xfs_buf_ioend(bp, 0);
+        xfs_buf_ioend(bp);
 
         return -EIO;
 }
@@ -1186,15 +1182,6 @@ xfs_bwrite(
 }
 
 STATIC void
-_xfs_buf_ioend(
-        xfs_buf_t               *bp,
-        int                     schedule)
-{
-        if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
-                xfs_buf_ioend(bp, schedule);
-}
-
-STATIC void
 xfs_buf_bio_end_io(
         struct bio              *bio,
         int                     error)
@@ -1211,7 +1198,8 @@ xfs_buf_bio_end_io(
         if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
                 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
 
-        _xfs_buf_ioend(bp, 1);
+        if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
+                xfs_buf_ioend_async(bp);
         bio_put(bio);
 }
 
@@ -1423,15 +1411,17 @@ xfs_buf_iorequest(
         /*
          * If _xfs_buf_ioapply failed or we are doing synchronous IO that
          * completes extremely quickly, we can get back here with only the IO
-         * reference we took above. _xfs_buf_ioend will drop it to zero. Run
-         * completion processing synchronously so that we don't return to the
-         * caller with completion still pending. This avoids unnecessary context
-         * switches associated with the end_io workqueue.
+         * reference we took above. If we drop it to zero, run completion
+         * processing synchronously so that we don't return to the caller with
+         * completion still pending. This avoids unnecessary context switches
+         * associated with the end_io workqueue.
          */
-        if (bp->b_error || !(bp->b_flags & XBF_ASYNC))
-                _xfs_buf_ioend(bp, 0);
-        else
-                _xfs_buf_ioend(bp, 1);
+        if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
+                if (bp->b_error || !(bp->b_flags & XBF_ASYNC))
+                        xfs_buf_ioend(bp);
+                else
+                        xfs_buf_ioend_async(bp);
+        }
 
         xfs_buf_rele(bp);
 }
diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h
index c753183900b3..4585c1595a98 100644
--- a/fs/xfs/xfs_buf.h
+++ b/fs/xfs/xfs_buf.h
@@ -286,7 +286,7 @@ extern void xfs_buf_unlock(xfs_buf_t *);
 
 /* Buffer Read and Write Routines */
 extern int xfs_bwrite(struct xfs_buf *bp);
-extern void xfs_buf_ioend(xfs_buf_t *,  int);
+extern void xfs_buf_ioend(struct xfs_buf *bp);
 extern void xfs_buf_ioerror(xfs_buf_t *, int);
 extern void xfs_buf_ioerror_alert(struct xfs_buf *, const char *func);
 extern void xfs_buf_iorequest(xfs_buf_t *);
diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c
index 76007deed31f..4fd41b58e6d2 100644
--- a/fs/xfs/xfs_buf_item.c
+++ b/fs/xfs/xfs_buf_item.c
@@ -491,7 +491,7 @@ xfs_buf_item_unpin(
                 xfs_buf_ioerror(bp, -EIO);
                 XFS_BUF_UNDONE(bp);
                 xfs_buf_stale(bp);
-                xfs_buf_ioend(bp, 0);
+                xfs_buf_ioend(bp);
         }
 }
 
@@ -1115,7 +1115,7 @@ do_callbacks:
         xfs_buf_do_callbacks(bp);
         bp->b_fspriv = NULL;
         bp->b_iodone = NULL;
-        xfs_buf_ioend(bp, 0);
+        xfs_buf_ioend(bp);
 }
 
 /*
diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c
index fea3c92fb3f0..00d210bbf3c3 100644
--- a/fs/xfs/xfs_inode.c
+++ b/fs/xfs/xfs_inode.c
@@ -3056,7 +3056,7 @@ cluster_corrupt_out:
                         XFS_BUF_UNDONE(bp);
                         xfs_buf_stale(bp);
                         xfs_buf_ioerror(bp, -EIO);
-                        xfs_buf_ioend(bp, 0);
+                        xfs_buf_ioend(bp);
                 } else {
                         xfs_buf_stale(bp);
                         xfs_buf_relse(bp);
diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c
index 85f36f212641..3567396f4428 100644
--- a/fs/xfs/xfs_log.c
+++ b/fs/xfs/xfs_log.c
@@ -1678,7 +1678,7 @@ xlog_bdstrat(
         if (iclog->ic_state & XLOG_STATE_IOERROR) {
                 xfs_buf_ioerror(bp, -EIO);
                 xfs_buf_stale(bp);
-                xfs_buf_ioend(bp, 0);
+                xfs_buf_ioend(bp);
                 /*
                  * It would seem logical to return EIO here, but we rely on
                  * the log state machine to propagate I/O errors instead of
diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c
index 1fd5787add99..4ba19bf7da1f 100644
--- a/fs/xfs/xfs_log_recover.c
+++ b/fs/xfs/xfs_log_recover.c
@@ -383,7 +383,7 @@ xlog_recover_iodone(
                                         SHUTDOWN_META_IO_ERROR);
         }
         bp->b_iodone = NULL;
-        xfs_buf_ioend(bp, 0);
+        xfs_buf_ioend(bp);
 }
 
 /*