blk-mq-sched: make completed_request() callback more useful

Currently, this callback is called right after put_request() and has no
distinguishable purpose. Instead, let's call it before put_request() as
soon as I/O has completed on the request, before we account it in
blk-stat. With this, Kyber can enable stats when it sees a latency
outlier and make sure the outlier gets accounted.

Signed-off-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@fb.com>

1 files changed, 4 insertions, 1 deletions

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 7138cd98146e..e2ef7b460924 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -350,7 +350,7 @@ void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
         if (rq->tag != -1)
                 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
         if (sched_tag != -1)
-                blk_mq_sched_completed_request(hctx, rq);
+                blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
         blk_mq_sched_restart(hctx);
         blk_queue_exit(q);
 }
@@ -444,6 +444,9 @@ static void __blk_mq_complete_request(struct request *rq)
 {
         struct request_queue *q = rq->q;
 
+        if (rq->internal_tag != -1)
+                blk_mq_sched_completed_request(rq);
+
         blk_mq_stat_add(rq);
 
         if (!q->softirq_done_fn)