blk-mq: add ->init_request and ->exit_request methods

The current blk_mq_init_commands/blk_mq_free_commands interface has a
two problems:

 1) Because only the constructor is passed to blk_mq_init_commands there
    is no easy way to clean up when a comman initialization failed.  The
    current code simply leaks the allocations done in the constructor.

 2) There is no good place to call blk_mq_free_commands: before
    blk_cleanup_queue there is no guarantee that all outstanding
    commands have completed, so we can't free them yet.  After
    blk_cleanup_queue the queue has usually been freed.  This can be
    worked around by grabbing an unconditional reference before calling
    blk_cleanup_queue and dropping it after blk_mq_free_commands is
    done, although that's not exatly pretty and driver writers are
    guaranteed to get it wrong sooner or later.

Both issues are easily fixed by making the request constructor and
destructor normal blk_mq_ops methods.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@fb.com>

1 files changed, 32 insertions, 73 deletions

diff --git a/block/blk-mq.c b/block/blk-mq.c
index e644feec068c..48d2d8495f5e 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1031,74 +1031,20 @@ static void blk_mq_hctx_notify(void *data, unsigned long action,
         blk_mq_put_ctx(ctx);
 }
 
-static int blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx,
-                                   int (*init)(void *, struct blk_mq_hw_ctx *,
-                                        struct request *, unsigned int),
-                                   void *data)
+static void blk_mq_free_rq_map(struct blk_mq_hw_ctx *hctx, void *driver_data)
 {
-        unsigned int i;
-        int ret = 0;
-
-        for (i = 0; i < hctx->queue_depth; i++) {
-                struct request *rq = hctx->rqs[i];
-
-                ret = init(data, hctx, rq, i);
-                if (ret)
-                        break;
-        }
-
-        return ret;
-}
-
-int blk_mq_init_commands(struct request_queue *q,
-                         int (*init)(void *, struct blk_mq_hw_ctx *,
-                                        struct request *, unsigned int),
-                         void *data)
-{
-        struct blk_mq_hw_ctx *hctx;
-        unsigned int i;
-        int ret = 0;
-
-        queue_for_each_hw_ctx(q, hctx, i) {
-                ret = blk_mq_init_hw_commands(hctx, init, data);
-                if (ret)
-                        break;
-        }
-
-        return ret;
-}
-EXPORT_SYMBOL(blk_mq_init_commands);
-
-static void blk_mq_free_hw_commands(struct blk_mq_hw_ctx *hctx,
-                                    void (*free)(void *, struct blk_mq_hw_ctx *,
-                                        struct request *, unsigned int),
-                                    void *data)
-{
-        unsigned int i;
+        struct page *page;
 
-        for (i = 0; i < hctx->queue_depth; i++) {
-                struct request *rq = hctx->rqs[i];
+        if (hctx->rqs && hctx->queue->mq_ops->exit_request) {
+                int i;
 
-                free(data, hctx, rq, i);
+                for (i = 0; i < hctx->queue_depth; i++) {
+                        if (!hctx->rqs[i])
+                                continue;
+                        hctx->queue->mq_ops->exit_request(driver_data, hctx,
+                                                          hctx->rqs[i], i);
+                }
         }
-}
-
-void blk_mq_free_commands(struct request_queue *q,
-                          void (*free)(void *, struct blk_mq_hw_ctx *,
-                                        struct request *, unsigned int),
-                          void *data)
-{
-        struct blk_mq_hw_ctx *hctx;
-        unsigned int i;
-
-        queue_for_each_hw_ctx(q, hctx, i)
-                blk_mq_free_hw_commands(hctx, free, data);
-}
-EXPORT_SYMBOL(blk_mq_free_commands);
-
-static void blk_mq_free_rq_map(struct blk_mq_hw_ctx *hctx)
-{
-        struct page *page;
 
         while (!list_empty(&hctx->page_list)) {
                 page = list_first_entry(&hctx->page_list, struct page, lru);
@@ -1123,10 +1069,12 @@ static size_t order_to_size(unsigned int order)
 }
 
 static int blk_mq_init_rq_map(struct blk_mq_hw_ctx *hctx,
-                              unsigned int reserved_tags, int node)
+                struct blk_mq_reg *reg, void *driver_data, int node)
 {
+        unsigned int reserved_tags = reg->reserved_tags;
         unsigned int i, j, entries_per_page, max_order = 4;
         size_t rq_size, left;
+        int error;
 
         INIT_LIST_HEAD(&hctx->page_list);
 
@@ -1175,14 +1123,23 @@ static int blk_mq_init_rq_map(struct blk_mq_hw_ctx *hctx,
                 for (j = 0; j < to_do; j++) {
                         hctx->rqs[i] = p;
                         blk_rq_init(hctx->queue, hctx->rqs[i]);
+                        if (reg->ops->init_request) {
+                                error = reg->ops->init_request(driver_data,
+                                                hctx, hctx->rqs[i], i);
+                                if (error)
+                                        goto err_rq_map;
+                        }
+
                         p += rq_size;
                         i++;
                 }
         }
 
-        if (i < (reserved_tags + BLK_MQ_TAG_MIN))
+        if (i < (reserved_tags + BLK_MQ_TAG_MIN)) {
+                error = -ENOMEM;
                 goto err_rq_map;
-        else if (i != hctx->queue_depth) {
+        }
+        if (i != hctx->queue_depth) {
                 hctx->queue_depth = i;
                 pr_warn("%s: queue depth set to %u because of low memory\n",
                                         __func__, i);
@@ -1190,12 +1147,14 @@ static int blk_mq_init_rq_map(struct blk_mq_hw_ctx *hctx,
 
         hctx->tags = blk_mq_init_tags(hctx->queue_depth, reserved_tags, node);
         if (!hctx->tags) {
-err_rq_map:
-                blk_mq_free_rq_map(hctx);
-                return -ENOMEM;
+                error = -ENOMEM;
+                goto err_rq_map;
         }
 
         return 0;
+err_rq_map:
+        blk_mq_free_rq_map(hctx, driver_data);
+        return error;
 }
 
 static int blk_mq_init_hw_queues(struct request_queue *q,
@@ -1228,7 +1187,7 @@ static int blk_mq_init_hw_queues(struct request_queue *q,
                                                 blk_mq_hctx_notify, hctx);
                 blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
 
-                if (blk_mq_init_rq_map(hctx, reg->reserved_tags, node))
+                if (blk_mq_init_rq_map(hctx, reg, driver_data, node))
                         break;
 
                 /*
@@ -1268,7 +1227,7 @@ static int blk_mq_init_hw_queues(struct request_queue *q,
                         reg->ops->exit_hctx(hctx, j);
 
                 blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
-                blk_mq_free_rq_map(hctx);
+                blk_mq_free_rq_map(hctx, driver_data);
                 kfree(hctx->ctxs);
         }
 
@@ -1455,7 +1414,7 @@ void blk_mq_free_queue(struct request_queue *q)
         queue_for_each_hw_ctx(q, hctx, i) {
                 kfree(hctx->ctx_map);
                 kfree(hctx->ctxs);
-                blk_mq_free_rq_map(hctx);
+                blk_mq_free_rq_map(hctx, q->queuedata);
                 blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
                 if (q->mq_ops->exit_hctx)
                         q->mq_ops->exit_hctx(hctx, i);