blk-mq: Reduce blk_mq_hw_ctx size

Since the srcu structure is rather large (184 bytes on an x86-64
system with kernel debugging disabled), only allocate it if needed.

Reported-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Bart Van Assche <bart.vanassche@sandisk.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Cc: Hannes Reinecke <hare@suse.com>
Cc: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

2 files changed, 25 insertions, 10 deletions

diff --git a/block/blk-mq.c b/block/blk-mq.c
index ca03cd4b263f..3e0cc11b1a90 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -172,7 +172,7 @@ void blk_mq_quiesce_queue(struct request_queue *q)
 
         queue_for_each_hw_ctx(q, hctx, i) {
                 if (hctx->flags & BLK_MQ_F_BLOCKING)
-                        synchronize_srcu(&hctx->queue_rq_srcu);
+                        synchronize_srcu(hctx->queue_rq_srcu);
                 else
                         rcu = true;
         }
@@ -1094,9 +1094,9 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
         } else {
                 might_sleep();
 
-                srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
+                srcu_idx = srcu_read_lock(hctx->queue_rq_srcu);
                 blk_mq_sched_dispatch_requests(hctx);
-                srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
+                srcu_read_unlock(hctx->queue_rq_srcu, srcu_idx);
         }
 }
 
@@ -1505,9 +1505,9 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
 
                 might_sleep();
 
-                srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
+                srcu_idx = srcu_read_lock(hctx->queue_rq_srcu);
                 __blk_mq_try_issue_directly(hctx, rq, cookie, true);
-                srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
+                srcu_read_unlock(hctx->queue_rq_srcu, srcu_idx);
         }
 }
 
@@ -1853,7 +1853,7 @@ static void blk_mq_exit_hctx(struct request_queue *q,
                 set->ops->exit_hctx(hctx, hctx_idx);
 
         if (hctx->flags & BLK_MQ_F_BLOCKING)
-                cleanup_srcu_struct(&hctx->queue_rq_srcu);
+                cleanup_srcu_struct(hctx->queue_rq_srcu);
 
         blk_mq_remove_cpuhp(hctx);
         blk_free_flush_queue(hctx->fq);
@@ -1926,7 +1926,7 @@ static int blk_mq_init_hctx(struct request_queue *q,
                 goto free_fq;
 
         if (hctx->flags & BLK_MQ_F_BLOCKING)
-                init_srcu_struct(&hctx->queue_rq_srcu);
+                init_srcu_struct(hctx->queue_rq_srcu);
 
         blk_mq_debugfs_register_hctx(q, hctx);
 
@@ -2201,6 +2201,20 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
 }
 EXPORT_SYMBOL(blk_mq_init_queue);
 
+static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set)
+{
+        int hw_ctx_size = sizeof(struct blk_mq_hw_ctx);
+
+        BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, queue_rq_srcu),
+                           __alignof__(struct blk_mq_hw_ctx)) !=
+                     sizeof(struct blk_mq_hw_ctx));
+
+        if (tag_set->flags & BLK_MQ_F_BLOCKING)
+                hw_ctx_size += sizeof(struct srcu_struct);
+
+        return hw_ctx_size;
+}
+
 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
                                                 struct request_queue *q)
 {
@@ -2215,7 +2229,7 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
                         continue;
 
                 node = blk_mq_hw_queue_to_node(q->mq_map, i);
-                hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
+                hctxs[i] = kzalloc_node(blk_mq_hw_ctx_size(set),
                                         GFP_KERNEL, node);
                 if (!hctxs[i])
                         break;
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index f1bd13ae8f57..3f2c22a42df6 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -39,8 +39,6 @@ struct blk_mq_hw_ctx {
         struct blk_mq_tags      *tags;
         struct blk_mq_tags      *sched_tags;
 
-        struct srcu_struct      queue_rq_srcu;
-
         unsigned long           queued;
         unsigned long           run;
 #define BLK_MQ_MAX_DISPATCH_ORDER       7
@@ -62,6 +60,9 @@ struct blk_mq_hw_ctx {
         struct dentry           *debugfs_dir;
         struct dentry           *sched_debugfs_dir;
 #endif
+
+        /* Must be the last member - see also blk_mq_hw_ctx_size(). */
+        struct srcu_struct      queue_rq_srcu[0];
 };
 
 struct blk_mq_tag_set {