blk-mq: split tag ->rqs[] into two

This is in preparation for having two sets of tags available. For
that we need a static index, and a dynamically assignable one.

Signed-off-by: Jens Axboe <axboe@fb.com>
Reviewed-by: Omar Sandoval <osandov@fb.com>

3 files changed, 26 insertions, 9 deletions

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index ced752716878..9753747a34a2 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -290,11 +290,11 @@ int blk_mq_reinit_tagset(struct blk_mq_tag_set *set)
                 struct blk_mq_tags *tags = set->tags[i];
 
                 for (j = 0; j < tags->nr_tags; j++) {
-                        if (!tags->rqs[j])
+                        if (!tags->static_rqs[j])
                                 continue;
 
                         ret = set->ops->reinit_request(set->driver_data,
-                                                tags->rqs[j]);
+                                                tags->static_rqs[j]);
                         if (ret)
                                 goto out;
                 }
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 923602dd3bfb..41cd15fd1afd 100644
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -16,6 +16,7 @@ struct blk_mq_tags {
         struct sbitmap_queue breserved_tags;
 
         struct request **rqs;
+        struct request **static_rqs;
         struct list_head page_list;
 };
 
diff --git a/block/blk-mq.c b/block/blk-mq.c
index d40be641f3d5..89b81254201b 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -223,7 +223,7 @@ struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
 
         tag = blk_mq_get_tag(data);
         if (tag != BLK_MQ_TAG_FAIL) {
-                rq = data->hctx->tags->rqs[tag];
+                rq = data->hctx->tags->static_rqs[tag];
 
                 if (blk_mq_tag_busy(data->hctx)) {
                         rq->rq_flags = RQF_MQ_INFLIGHT;
@@ -231,6 +231,7 @@ struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
                 }
 
                 rq->tag = tag;
+                data->hctx->tags->rqs[tag] = rq;
                 blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
                 return rq;
         }
@@ -1567,11 +1568,13 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
                 int i;
 
                 for (i = 0; i < tags->nr_tags; i++) {
-                        if (!tags->rqs[i])
+                        struct request *rq = tags->static_rqs[i];
+
+                        if (!rq)
                                 continue;
-                        set->ops->exit_request(set->driver_data, tags->rqs[i],
+                        set->ops->exit_request(set->driver_data, rq,
                                                 hctx_idx, i);
-                        tags->rqs[i] = NULL;
+                        tags->static_rqs[i] = NULL;
                 }
         }
 
@@ -1591,6 +1594,8 @@ void blk_mq_free_rq_map(struct blk_mq_tags *tags)
 {
         kfree(tags->rqs);
         tags->rqs = NULL;
+        kfree(tags->static_rqs);
+        tags->static_rqs = NULL;
 
         blk_mq_free_tags(tags);
 }
@@ -1616,6 +1621,15 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
                 return NULL;
         }
 
+        tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
+                                 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
+                                 set->numa_node);
+        if (!tags->static_rqs) {
+                kfree(tags->rqs);
+                blk_mq_free_tags(tags);
+                return NULL;
+        }
+
         return tags;
 }
 
@@ -1677,12 +1691,14 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
                 to_do = min(entries_per_page, depth - i);
                 left -= to_do * rq_size;
                 for (j = 0; j < to_do; j++) {
-                        tags->rqs[i] = p;
+                        struct request *rq = p;
+
+                        tags->static_rqs[i] = rq;
                         if (set->ops->init_request) {
                                 if (set->ops->init_request(set->driver_data,
-                                                tags->rqs[i], hctx_idx, i,
+                                                rq, hctx_idx, i,
                                                 set->numa_node)) {
-                                        tags->rqs[i] = NULL;
+                                        tags->static_rqs[i] = NULL;
                                         goto fail;
                                 }
                         }