blk-mq: move the cache friendly bitmap type of out blk-mq-tag

We will use it for the pending list in blk-mq core as well.

Signed-off-by: Jens Axboe <axboe@fb.com>

3 files changed, 17 insertions, 11 deletions

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 8d526a3e02f6..03ce6a11ba79 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -21,7 +21,7 @@ static bool bt_has_free_tags(struct blk_mq_bitmap_tags *bt)
         int i;
 
         for (i = 0; i < bt->map_nr; i++) {
-                struct blk_mq_bitmap *bm = &bt->map[i];
+                struct blk_align_bitmap *bm = &bt->map[i];
                 int ret;
 
                 ret = find_first_zero_bit(&bm->word, bm->depth);
@@ -40,7 +40,7 @@ bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
         return bt_has_free_tags(&tags->bitmap_tags);
 }
 
-static int __bt_get_word(struct blk_mq_bitmap *bm, unsigned int last_tag)
+static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag)
 {
         int tag, org_last_tag, end;
 
@@ -283,7 +283,7 @@ static void bt_for_each_free(struct blk_mq_bitmap_tags *bt,
         int i;
 
         for (i = 0; i < bt->map_nr; i++) {
-                struct blk_mq_bitmap *bm = &bt->map[i];
+                struct blk_align_bitmap *bm = &bt->map[i];
                 int bit = 0;
 
                 do {
@@ -323,7 +323,7 @@ static unsigned int bt_unused_tags(struct blk_mq_bitmap_tags *bt)
         unsigned int i, used;
 
         for (i = 0, used = 0; i < bt->map_nr; i++) {
-                struct blk_mq_bitmap *bm = &bt->map[i];
+                struct blk_align_bitmap *bm = &bt->map[i];
 
                 used += bitmap_weight(&bm->word, bm->depth);
         }
@@ -361,7 +361,7 @@ static int bt_alloc(struct blk_mq_bitmap_tags *bt, unsigned int depth,
                 }
 
                 nr = ALIGN(depth, tags_per_word) / tags_per_word;
-                bt->map = kzalloc_node(nr * sizeof(struct blk_mq_bitmap),
+                bt->map = kzalloc_node(nr * sizeof(struct blk_align_bitmap),
                                                 GFP_KERNEL, node);
                 if (!bt->map)
                         return -ENOMEM;
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 7aa9f0665489..9014269f3910 100644
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -1,6 +1,8 @@
 #ifndef INT_BLK_MQ_TAG_H
 #define INT_BLK_MQ_TAG_H
 
+#include "blk-mq.h"
+
 enum {
         BT_WAIT_QUEUES  = 8,
         BT_WAIT_BATCH   = 8,
@@ -14,18 +16,13 @@ struct bt_wait_state {
 #define TAG_TO_INDEX(bt, tag)   ((tag) >> (bt)->bits_per_word)
 #define TAG_TO_BIT(bt, tag)     ((tag) & ((1 << (bt)->bits_per_word) - 1))
 
-struct blk_mq_bitmap {
-        unsigned long word;
-        unsigned long depth;
-} ____cacheline_aligned_in_smp;
-
 struct blk_mq_bitmap_tags {
         unsigned int depth;
         unsigned int wake_cnt;
         unsigned int bits_per_word;
 
         unsigned int map_nr;
-        struct blk_mq_bitmap *map;
+        struct blk_align_bitmap *map;
 
         unsigned int wake_index;
         struct bt_wait_state *bs;
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 97cfab9c092f..5e5a378962b7 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -52,4 +52,13 @@ void blk_mq_disable_hotplug(void);
 extern unsigned int *blk_mq_make_queue_map(struct blk_mq_tag_set *set);
 extern int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues);
 
+/*
+ * Basic implementation of sparser bitmap, allowing the user to spread
+ * the bits over more cachelines.
+ */
+struct blk_align_bitmap {
+        unsigned long word;
+        unsigned long depth;
+} ____cacheline_aligned_in_smp;
+
 #endif