blk-mq: call blk_mq_start_request from ->queue_rq

When we call blk_mq_start_request from the core blk-mq code before calling into
->queue_rq there is a racy window where the timeout handler can hit before we've
fully set up the driver specific part of the command.

Move the call to blk_mq_start_request into the driver so the driver can start
the request only once it is fully set up.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@fb.com>

1 files changed, 6 insertions, 7 deletions

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 32b4797f4186..141f2e06803a 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -384,7 +384,7 @@ void blk_mq_complete_request(struct request *rq)
 }
 EXPORT_SYMBOL(blk_mq_complete_request);
 
-static void blk_mq_start_request(struct request *rq)
+void blk_mq_start_request(struct request *rq)
 {
         struct request_queue *q = rq->q;
 
@@ -422,16 +422,18 @@ static void blk_mq_start_request(struct request *rq)
                 rq->nr_phys_segments++;
         }
 }
+EXPORT_SYMBOL(blk_mq_start_request);
 
 static void __blk_mq_requeue_request(struct request *rq)
 {
         struct request_queue *q = rq->q;
 
         trace_block_rq_requeue(q, rq);
-        clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 
-        if (q->dma_drain_size && blk_rq_bytes(rq))
-                rq->nr_phys_segments--;
+        if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
+                if (q->dma_drain_size && blk_rq_bytes(rq))
+                        rq->nr_phys_segments--;
+        }
 }
 
 void blk_mq_requeue_request(struct request *rq)
@@ -743,8 +745,6 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
                 rq = list_first_entry(&rq_list, struct request, queuelist);
                 list_del_init(&rq->queuelist);
 
-                blk_mq_start_request(rq);
-
                 ret = q->mq_ops->queue_rq(hctx, rq, list_empty(&rq_list));
                 switch (ret) {
                 case BLK_MQ_RQ_QUEUE_OK:
@@ -1186,7 +1186,6 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
                 int ret;
 
                 blk_mq_bio_to_request(rq, bio);
-                blk_mq_start_request(rq);
 
                 /*
                  * For OK queue, we are done. For error, kill it. Any other