1 files changed, 131 insertions, 8 deletions

diff --git a/lib/sbitmap.c b/lib/sbitmap.c
index 2cecf05c82fd..55e11c4b2f3b 100644
--- a/lib/sbitmap.c
+++ b/lib/sbitmap.c
@@ -17,6 +17,7 @@
 
 #include <linux/random.h>
 #include <linux/sbitmap.h>
+#include <linux/seq_file.h>
 
 int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
                       gfp_t flags, int node)
@@ -180,6 +181,62 @@ unsigned int sbitmap_weight(const struct sbitmap *sb)
 }
 EXPORT_SYMBOL_GPL(sbitmap_weight);
 
+void sbitmap_show(struct sbitmap *sb, struct seq_file *m)
+{
+        seq_printf(m, "depth=%u\n", sb->depth);
+        seq_printf(m, "busy=%u\n", sbitmap_weight(sb));
+        seq_printf(m, "bits_per_word=%u\n", 1U << sb->shift);
+        seq_printf(m, "map_nr=%u\n", sb->map_nr);
+}
+EXPORT_SYMBOL_GPL(sbitmap_show);
+
+static inline void emit_byte(struct seq_file *m, unsigned int offset, u8 byte)
+{
+        if ((offset & 0xf) == 0) {
+                if (offset != 0)
+                        seq_putc(m, '\n');
+                seq_printf(m, "%08x:", offset);
+        }
+        if ((offset & 0x1) == 0)
+                seq_putc(m, ' ');
+        seq_printf(m, "%02x", byte);
+}
+
+void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m)
+{
+        u8 byte = 0;
+        unsigned int byte_bits = 0;
+        unsigned int offset = 0;
+        int i;
+
+        for (i = 0; i < sb->map_nr; i++) {
+                unsigned long word = READ_ONCE(sb->map[i].word);
+                unsigned int word_bits = READ_ONCE(sb->map[i].depth);
+
+                while (word_bits > 0) {
+                        unsigned int bits = min(8 - byte_bits, word_bits);
+
+                        byte |= (word & (BIT(bits) - 1)) << byte_bits;
+                        byte_bits += bits;
+                        if (byte_bits == 8) {
+                                emit_byte(m, offset, byte);
+                                byte = 0;
+                                byte_bits = 0;
+                                offset++;
+                        }
+                        word >>= bits;
+                        word_bits -= bits;
+                }
+        }
+        if (byte_bits) {
+                emit_byte(m, offset, byte);
+                offset++;
+        }
+        if (offset)
+                seq_putc(m, '\n');
+}
+EXPORT_SYMBOL_GPL(sbitmap_bitmap_show);
+
 static unsigned int sbq_calc_wake_batch(unsigned int depth)
 {
         unsigned int wake_batch;
@@ -239,7 +296,19 @@ EXPORT_SYMBOL_GPL(sbitmap_queue_init_node);
 
 void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth)
 {
-        sbq->wake_batch = sbq_calc_wake_batch(depth);
+        unsigned int wake_batch = sbq_calc_wake_batch(depth);
+        int i;
+
+        if (sbq->wake_batch != wake_batch) {
+                WRITE_ONCE(sbq->wake_batch, wake_batch);
+                /*
+                 * Pairs with the memory barrier in sbq_wake_up() to ensure that
+                 * the batch size is updated before the wait counts.
+                 */
+                smp_mb__before_atomic();
+                for (i = 0; i < SBQ_WAIT_QUEUES; i++)
+                        atomic_set(&sbq->ws[i].wait_cnt, 1);
+        }
         sbitmap_resize(&sbq->sb, depth);
 }
 EXPORT_SYMBOL_GPL(sbitmap_queue_resize);
@@ -297,20 +366,39 @@ static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq)
 static void sbq_wake_up(struct sbitmap_queue *sbq)
 {
         struct sbq_wait_state *ws;
+        unsigned int wake_batch;
         int wait_cnt;
 
-        /* Ensure that the wait list checks occur after clear_bit(). */
-        smp_mb();
+        /*
+         * Pairs with the memory barrier in set_current_state() to ensure the
+         * proper ordering of clear_bit()/waitqueue_active() in the waker and
+         * test_and_set_bit()/prepare_to_wait()/finish_wait() in the waiter. See
+         * the comment on waitqueue_active(). This is __after_atomic because we
+         * just did clear_bit() in the caller.
+         */
+        smp_mb__after_atomic();
 
         ws = sbq_wake_ptr(sbq);
         if (!ws)
                 return;
 
         wait_cnt = atomic_dec_return(&ws->wait_cnt);
-        if (unlikely(wait_cnt < 0))
-                wait_cnt = atomic_inc_return(&ws->wait_cnt);
-        if (wait_cnt == 0) {
-                atomic_add(sbq->wake_batch, &ws->wait_cnt);
+        if (wait_cnt <= 0) {
+                wake_batch = READ_ONCE(sbq->wake_batch);
+                /*
+                 * Pairs with the memory barrier in sbitmap_queue_resize() to
+                 * ensure that we see the batch size update before the wait
+                 * count is reset.
+                 */
+                smp_mb__before_atomic();
+                /*
+                 * If there are concurrent callers to sbq_wake_up(), the last
+                 * one to decrement the wait count below zero will bump it back
+                 * up. If there is a concurrent resize, the count reset will
+                 * either cause the cmpxchg to fail or overwrite after the
+                 * cmpxchg.
+                 */
+                atomic_cmpxchg(&ws->wait_cnt, wait_cnt, wait_cnt + wake_batch);
                 sbq_index_atomic_inc(&sbq->wake_index);
                 wake_up(&ws->wait);
         }
@@ -331,7 +419,8 @@ void sbitmap_queue_wake_all(struct sbitmap_queue *sbq)
         int i, wake_index;
 
         /*
-         * Make sure all changes prior to this are visible from other CPUs.
+         * Pairs with the memory barrier in set_current_state() like in
+         * sbq_wake_up().
          */
         smp_mb();
         wake_index = atomic_read(&sbq->wake_index);
@@ -345,3 +434,37 @@ void sbitmap_queue_wake_all(struct sbitmap_queue *sbq)
         }
 }
 EXPORT_SYMBOL_GPL(sbitmap_queue_wake_all);
+
+void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m)
+{
+        bool first;
+        int i;
+
+        sbitmap_show(&sbq->sb, m);
+
+        seq_puts(m, "alloc_hint={");
+        first = true;
+        for_each_possible_cpu(i) {
+                if (!first)
+                        seq_puts(m, ", ");
+                first = false;
+                seq_printf(m, "%u", *per_cpu_ptr(sbq->alloc_hint, i));
+        }
+        seq_puts(m, "}\n");
+
+        seq_printf(m, "wake_batch=%u\n", sbq->wake_batch);
+        seq_printf(m, "wake_index=%d\n", atomic_read(&sbq->wake_index));
+
+        seq_puts(m, "ws={\n");
+        for (i = 0; i < SBQ_WAIT_QUEUES; i++) {
+                struct sbq_wait_state *ws = &sbq->ws[i];
+
+                seq_printf(m, "\t{.wait_cnt=%d, .wait=%s},\n",
+                           atomic_read(&ws->wait_cnt),
+                           waitqueue_active(&ws->wait) ? "active" : "inactive");
+        }
+        seq_puts(m, "}\n");
+
+        seq_printf(m, "round_robin=%d\n", sbq->round_robin);
+}
+EXPORT_SYMBOL_GPL(sbitmap_queue_show);