blk-mq: bitmap tag: fix races on shared ::wake_index fields

Fix racy updates of shared blk_mq_bitmap_tags::wake_index
and blk_mq_hw_ctx::wake_index fields.

Cc: Ming Lei <tom.leiming@gmail.com>
Signed-off-by: Alexander Gordeev <agordeev@redhat.com>
Signed-off-by: Jens Axboe <axboe@fb.com>

3 files changed, 23 insertions, 13 deletions

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 1aab39f71d95..6deb13055490 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -43,9 +43,16 @@ bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
         return bt_has_free_tags(&tags->bitmap_tags);
 }
 
-static inline void bt_index_inc(unsigned int *index)
+static inline int bt_index_inc(int index)
 {
-        *index = (*index + 1) & (BT_WAIT_QUEUES - 1);
+        return (index + 1) & (BT_WAIT_QUEUES - 1);
+}
+
+static inline void bt_index_atomic_inc(atomic_t *index)
+{
+        int old = atomic_read(index);
+        int new = bt_index_inc(old);
+        atomic_cmpxchg(index, old, new);
 }
 
 /*
@@ -69,14 +76,14 @@ static void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags)
         int i, wake_index;
 
         bt = &tags->bitmap_tags;
-        wake_index = bt->wake_index;
+        wake_index = atomic_read(&bt->wake_index);
         for (i = 0; i < BT_WAIT_QUEUES; i++) {
                 struct bt_wait_state *bs = &bt->bs[wake_index];
 
                 if (waitqueue_active(&bs->wait))
                         wake_up(&bs->wait);
 
-                bt_index_inc(&wake_index);
+                wake_index = bt_index_inc(wake_index);
         }
 }
 
@@ -212,12 +219,14 @@ static struct bt_wait_state *bt_wait_ptr(struct blk_mq_bitmap_tags *bt,
                                          struct blk_mq_hw_ctx *hctx)
 {
         struct bt_wait_state *bs;
+        int wait_index;
 
         if (!hctx)
                 return &bt->bs[0];
 
-        bs = &bt->bs[hctx->wait_index];
-        bt_index_inc(&hctx->wait_index);
+        wait_index = atomic_read(&hctx->wait_index);
+        bs = &bt->bs[wait_index];
+        bt_index_atomic_inc(&hctx->wait_index);
         return bs;
 }
 
@@ -313,18 +322,19 @@ static struct bt_wait_state *bt_wake_ptr(struct blk_mq_bitmap_tags *bt)
 {
         int i, wake_index;
 
-        wake_index = bt->wake_index;
+        wake_index = atomic_read(&bt->wake_index);
         for (i = 0; i < BT_WAIT_QUEUES; i++) {
                 struct bt_wait_state *bs = &bt->bs[wake_index];
 
                 if (waitqueue_active(&bs->wait)) {
-                        if (wake_index != bt->wake_index)
-                                bt->wake_index = wake_index;
+                        int o = atomic_read(&bt->wake_index);
+                        if (wake_index != o)
+                                atomic_cmpxchg(&bt->wake_index, o, wake_index);
 
                         return bs;
                 }
 
-                bt_index_inc(&wake_index);
+                wake_index = bt_index_inc(wake_index);
         }
 
         return NULL;
@@ -344,7 +354,7 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag)
         bs = bt_wake_ptr(bt);
         if (bs && atomic_dec_and_test(&bs->wait_cnt)) {
                 atomic_set(&bs->wait_cnt, bt->wake_cnt);
-                bt_index_inc(&bt->wake_index);
+                bt_index_atomic_inc(&bt->wake_index);
                 wake_up(&bs->wait);
         }
 }
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 98696a65d4d4..6206ed17ef76 100644
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -24,7 +24,7 @@ struct blk_mq_bitmap_tags {
         unsigned int map_nr;
         struct blk_align_bitmap *map;
 
-        unsigned int wake_index;
+        atomic_t wake_index;
         struct bt_wait_state *bs;
 };
 
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index a002cf191427..eb726b9c5762 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -42,7 +42,7 @@ struct blk_mq_hw_ctx {
         unsigned int            nr_ctx;
         struct blk_mq_ctx       **ctxs;
 
-        unsigned int            wait_index;
+        atomic_t                wait_index;
 
         struct blk_mq_tags      *tags;