5 files changed, 35 insertions, 18 deletions

diff --git a/block/blk-flush.c b/block/blk-flush.c
index 20badd7b9d1b..9c423e53324a 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -73,6 +73,7 @@
 
 #include "blk.h"
 #include "blk-mq.h"
+#include "blk-mq-tag.h"
 
 /* FLUSH/FUA sequences */
 enum {
@@ -226,7 +227,12 @@ static void flush_end_io(struct request *flush_rq, int error)
         struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);
 
         if (q->mq_ops) {
+                struct blk_mq_hw_ctx *hctx;
+
+                /* release the tag's ownership to the req cloned from */
                 spin_lock_irqsave(&fq->mq_flush_lock, flags);
+                hctx = q->mq_ops->map_queue(q, flush_rq->mq_ctx->cpu);
+                blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq);
                 flush_rq->tag = -1;
         }
 
@@ -308,11 +314,18 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq)
 
         /*
          * Borrow tag from the first request since they can't
-         * be in flight at the same time.
+         * be in flight at the same time. And acquire the tag's
+         * ownership for flush req.
          */
         if (q->mq_ops) {
+                struct blk_mq_hw_ctx *hctx;
+
                 flush_rq->mq_ctx = first_rq->mq_ctx;
                 flush_rq->tag = first_rq->tag;
+                fq->orig_rq = first_rq;
+
+                hctx = q->mq_ops->map_queue(q, first_rq->mq_ctx->cpu);
+                blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq);
         }
 
         flush_rq->cmd_type = REQ_TYPE_FS;
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 9b6e28830b82..9115c6d59948 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -429,7 +429,7 @@ static void bt_for_each(struct blk_mq_hw_ctx *hctx,
                 for (bit = find_first_bit(&bm->word, bm->depth);
                      bit < bm->depth;
                      bit = find_next_bit(&bm->word, bm->depth, bit + 1)) {
-                        rq = blk_mq_tag_to_rq(hctx->tags, off + bit);
+                        rq = hctx->tags->rqs[off + bit];
                         if (rq->q == hctx->queue)
                                 fn(hctx, rq, data, reserved);
                 }
@@ -453,7 +453,7 @@ static void bt_tags_for_each(struct blk_mq_tags *tags,
                 for (bit = find_first_bit(&bm->word, bm->depth);
                      bit < bm->depth;
                      bit = find_next_bit(&bm->word, bm->depth, bit + 1)) {
-                        rq = blk_mq_tag_to_rq(tags, off + bit);
+                        rq = tags->rqs[off + bit];
                         fn(rq, data, reserved);
                 }
 
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 75893a34237d..9eb2cf4f01cb 100644
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -89,4 +89,16 @@ static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
         __blk_mq_tag_idle(hctx);
 }
 
+/*
+ * This helper should only be used for flush request to share tag
+ * with the request cloned from, and both the two requests can't be
+ * in flight at the same time. The caller has to make sure the tag
+ * can't be freed.
+ */
+static inline void blk_mq_tag_set_rq(struct blk_mq_hw_ctx *hctx,
+                unsigned int tag, struct request *rq)
+{
+        hctx->tags->rqs[tag] = rq;
+}
+
 #endif
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 81edbd95bda8..f2d67b4047a0 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -559,23 +559,9 @@ void blk_mq_abort_requeue_list(struct request_queue *q)
 }
 EXPORT_SYMBOL(blk_mq_abort_requeue_list);
 
-static inline bool is_flush_request(struct request *rq,
-                struct blk_flush_queue *fq, unsigned int tag)
-{
-        return ((rq->cmd_flags & REQ_FLUSH_SEQ) &&
-                        fq->flush_rq->tag == tag);
-}
-
 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
 {
-        struct request *rq = tags->rqs[tag];
-        /* mq_ctx of flush rq is always cloned from the corresponding req */
-        struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx);
-
-        if (!is_flush_request(rq, fq, tag))
-                return rq;
-
-        return fq->flush_rq;
+        return tags->rqs[tag];
 }
 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 
diff --git a/block/blk.h b/block/blk.h
index 026d9594142b..838188b35a83 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -22,6 +22,12 @@ struct blk_flush_queue {
         struct list_head        flush_queue[2];
         struct list_head        flush_data_in_flight;
         struct request          *flush_rq;
+
+        /*
+         * flush_rq shares tag with this rq, both can't be active
+         * at the same time
+         */
+        struct request          *orig_rq;
         spinlock_t              mq_flush_lock;
 };