null_blk: multi queue aware block test driver

A driver that simply completes IO it receives, it does no
transfers. Written to fascilitate testing of the blk-mq code.
It supports various module options to use either bio queueing,
rq queueing, or mq mode.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

3 files changed, 639 insertions, 0 deletions

diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index e07a5fd58ad7..4682546c5da7 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -15,6 +15,9 @@ menuconfig BLK_DEV
 
 if BLK_DEV
 
+config BLK_DEV_NULL_BLK
+        tristate "Null test block driver"
+
 config BLK_DEV_FD
         tristate "Normal floppy disk support"
         depends on ARCH_MAY_HAVE_PC_FDC
diff --git a/drivers/block/Makefile b/drivers/block/Makefile
index ca07399a8d99..03b3b4a2bd8a 100644
--- a/drivers/block/Makefile
+++ b/drivers/block/Makefile
@@ -41,6 +41,7 @@ obj-$(CONFIG_BLK_DEV_RBD)     += rbd.o
 obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX)  += mtip32xx/
 
 obj-$(CONFIG_BLK_DEV_RSXX) += rsxx/
+obj-$(CONFIG_BLK_DEV_NULL_BLK)  += null_blk.o
 
 nvme-y          := nvme-core.o nvme-scsi.o
 swim_mod-y      := swim.o swim_asm.o
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
new file mode 100644
index 000000000000..b5d842370cc9
--- /dev/null
+++ b/drivers/block/null_blk.c
@@ -0,0 +1,635 @@
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/sched.h>
+#include <linux/fs.h>
+#include <linux/blkdev.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/blk-mq.h>
+#include <linux/hrtimer.h>
+
+struct nullb_cmd {
+        struct list_head list;
+        struct llist_node ll_list;
+        struct call_single_data csd;
+        struct request *rq;
+        struct bio *bio;
+        unsigned int tag;
+        struct nullb_queue *nq;
+};
+
+struct nullb_queue {
+        unsigned long *tag_map;
+        wait_queue_head_t wait;
+        unsigned int queue_depth;
+
+        struct nullb_cmd *cmds;
+};
+
+struct nullb {
+        struct list_head list;
+        unsigned int index;
+        struct request_queue *q;
+        struct gendisk *disk;
+        struct hrtimer timer;
+        unsigned int queue_depth;
+        spinlock_t lock;
+
+        struct nullb_queue *queues;
+        unsigned int nr_queues;
+};
+
+static LIST_HEAD(nullb_list);
+static struct mutex lock;
+static int null_major;
+static int nullb_indexes;
+
+struct completion_queue {
+        struct llist_head list;
+        struct hrtimer timer;
+};
+
+/*
+ * These are per-cpu for now, they will need to be configured by the
+ * complete_queues parameter and appropriately mapped.
+ */
+static DEFINE_PER_CPU(struct completion_queue, completion_queues);
+
+enum {
+        NULL_IRQ_NONE           = 0,
+        NULL_IRQ_SOFTIRQ        = 1,
+        NULL_IRQ_TIMER          = 2,
+
+        NULL_Q_BIO              = 0,
+        NULL_Q_RQ               = 1,
+        NULL_Q_MQ               = 2,
+};
+
+static int submit_queues = 1;
+module_param(submit_queues, int, S_IRUGO);
+MODULE_PARM_DESC(submit_queues, "Number of submission queues");
+
+static int home_node = NUMA_NO_NODE;
+module_param(home_node, int, S_IRUGO);
+MODULE_PARM_DESC(home_node, "Home node for the device");
+
+static int queue_mode = NULL_Q_MQ;
+module_param(queue_mode, int, S_IRUGO);
+MODULE_PARM_DESC(use_mq, "Use blk-mq interface (0=bio,1=rq,2=multiqueue)");
+
+static int gb = 250;
+module_param(gb, int, S_IRUGO);
+MODULE_PARM_DESC(gb, "Size in GB");
+
+static int bs = 512;
+module_param(bs, int, S_IRUGO);
+MODULE_PARM_DESC(bs, "Block size (in bytes)");
+
+static int nr_devices = 2;
+module_param(nr_devices, int, S_IRUGO);
+MODULE_PARM_DESC(nr_devices, "Number of devices to register");
+
+static int irqmode = NULL_IRQ_SOFTIRQ;
+module_param(irqmode, int, S_IRUGO);
+MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
+
+static int completion_nsec = 10000;
+module_param(completion_nsec, int, S_IRUGO);
+MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
+
+static int hw_queue_depth = 64;
+module_param(hw_queue_depth, int, S_IRUGO);
+MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
+
+static bool use_per_node_hctx = true;
+module_param(use_per_node_hctx, bool, S_IRUGO);
+MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: true");
+
+static void put_tag(struct nullb_queue *nq, unsigned int tag)
+{
+        clear_bit_unlock(tag, nq->tag_map);
+
+        if (waitqueue_active(&nq->wait))
+                wake_up(&nq->wait);
+}
+
+static unsigned int get_tag(struct nullb_queue *nq)
+{
+        unsigned int tag;
+
+        do {
+                tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
+                if (tag >= nq->queue_depth)
+                        return -1U;
+        } while (test_and_set_bit_lock(tag, nq->tag_map));
+
+        return tag;
+}
+
+static void free_cmd(struct nullb_cmd *cmd)
+{
+        put_tag(cmd->nq, cmd->tag);
+}
+
+static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
+{
+        struct nullb_cmd *cmd;
+        unsigned int tag;
+
+        tag = get_tag(nq);
+        if (tag != -1U) {
+                cmd = &nq->cmds[tag];
+                cmd->tag = tag;
+                cmd->nq = nq;
+                return cmd;
+        }
+
+        return NULL;
+}
+
+static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
+{
+        struct nullb_cmd *cmd;
+        DEFINE_WAIT(wait);
+
+        cmd = __alloc_cmd(nq);
+        if (cmd || !can_wait)
+                return cmd;
+
+        do {
+                prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
+                cmd = __alloc_cmd(nq);
+                if (cmd)
+                        break;
+
+                io_schedule();
+        } while (1);
+
+        finish_wait(&nq->wait, &wait);
+        return cmd;
+}
+
+static void end_cmd(struct nullb_cmd *cmd)
+{
+        if (cmd->rq) {
+                if (queue_mode == NULL_Q_MQ)