1 files changed, 51 insertions, 37 deletions

diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 6d8a87f252de..f63d358f3d93 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -30,6 +30,9 @@ struct virtio_blk
         /* The disk structure for the kernel. */
         struct gendisk *disk;
 
+        /* Block layer tags. */
+        struct blk_mq_tag_set tag_set;
+
         /* Process context for config space updates */
         struct work_struct config_work;
 
@@ -112,7 +115,7 @@ static int __virtblk_add_req(struct virtqueue *vq,
 
 static inline void virtblk_request_done(struct request *req)
 {
-        struct virtblk_req *vbr = req->special;
+        struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
         int error = virtblk_result(vbr);
 
         if (req->cmd_type == REQ_TYPE_BLOCK_PC) {
@@ -144,21 +147,22 @@ static void virtblk_done(struct virtqueue *vq)
                 if (unlikely(virtqueue_is_broken(vq)))
                         break;
         } while (!virtqueue_enable_cb(vq));
-        spin_unlock_irqrestore(&vblk->vq_lock, flags);
 
         /* In case queue is stopped waiting for more buffers. */
         if (req_done)
-                blk_mq_start_stopped_hw_queues(vblk->disk->queue);
+                blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
+        spin_unlock_irqrestore(&vblk->vq_lock, flags);
 }
 
 static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req)
 {
         struct virtio_blk *vblk = hctx->queue->queuedata;
-        struct virtblk_req *vbr = req->special;
+        struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
         unsigned long flags;
         unsigned int num;
         const bool last = (req->cmd_flags & REQ_END) != 0;
         int err;
+        bool notify = false;
 
         BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);
 
@@ -202,8 +206,8 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req)
         err = __virtblk_add_req(vblk->vq, vbr, vbr->sg, num);
         if (err) {
                 virtqueue_kick(vblk->vq);
-                spin_unlock_irqrestore(&vblk->vq_lock, flags);
                 blk_mq_stop_hw_queue(hctx);
+                spin_unlock_irqrestore(&vblk->vq_lock, flags);
                 /* Out of mem doesn't actually happen, since we fall back
                  * to direct descriptors */
                 if (err == -ENOMEM || err == -ENOSPC)
@@ -211,10 +215,12 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req)
                 return BLK_MQ_RQ_QUEUE_ERROR;
         }
 
-        if (last)
-                virtqueue_kick(vblk->vq);
-
+        if (last && virtqueue_kick_prepare(vblk->vq))
+                notify = true;
         spin_unlock_irqrestore(&vblk->vq_lock, flags);
+
+        if (notify)
+                virtqueue_notify(vblk->vq);
         return BLK_MQ_RQ_QUEUE_OK;
 }
 
@@ -480,33 +486,27 @@ static const struct device_attribute dev_attr_cache_type_rw =
         __ATTR(cache_type, S_IRUGO|S_IWUSR,
                virtblk_cache_type_show, virtblk_cache_type_store);
 
-static struct blk_mq_ops virtio_mq_ops = {
-        .queue_rq       = virtio_queue_rq,
-        .map_queue      = blk_mq_map_queue,
-        .alloc_hctx     = blk_mq_alloc_single_hw_queue,
-        .free_hctx      = blk_mq_free_single_hw_queue,
-        .complete       = virtblk_request_done,
-};
-
-static struct blk_mq_reg virtio_mq_reg = {
-        .ops            = &virtio_mq_ops,
-        .nr_hw_queues   = 1,
-        .queue_depth    = 0, /* Set in virtblk_probe */
-        .numa_node      = NUMA_NO_NODE,
-        .flags          = BLK_MQ_F_SHOULD_MERGE,
-};
-module_param_named(queue_depth, virtio_mq_reg.queue_depth, uint, 0444);
-
-static int virtblk_init_vbr(void *data, struct blk_mq_hw_ctx *hctx,
-                             struct request *rq, unsigned int nr)
+static int virtblk_init_request(void *data, struct request *rq,
+                unsigned int hctx_idx, unsigned int request_idx,
+                unsigned int numa_node)
 {
         struct virtio_blk *vblk = data;
-        struct virtblk_req *vbr = rq->special;
+        struct virtblk_req *vbr = blk_mq_rq_to_pdu(rq);
 
         sg_init_table(vbr->sg, vblk->sg_elems);
         return 0;
 }
 
+static struct blk_mq_ops virtio_mq_ops = {
+        .queue_rq       = virtio_queue_rq,
+        .map_queue      = blk_mq_map_queue,
+        .complete       = virtblk_request_done,
+        .init_request   = virtblk_init_request,
+};
+
+static unsigned int virtblk_queue_depth;
+module_param_named(queue_depth, virtblk_queue_depth, uint, 0444);
+
 static int virtblk_probe(struct virtio_device *vdev)
 {
         struct virtio_blk *vblk;
@@ -561,24 +561,34 @@ static int virtblk_probe(struct virtio_device *vdev)
         }
 
         /* Default queue sizing is to fill the ring. */
-        if (!virtio_mq_reg.queue_depth) {
-                virtio_mq_reg.queue_depth = vblk->vq->num_free;
+        if (!virtblk_queue_depth) {
+                virtblk_queue_depth = vblk->vq->num_free;
                 /* ... but without indirect descs, we use 2 descs per req */
                 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
-                        virtio_mq_reg.queue_depth /= 2;
+                        virtblk_queue_depth /= 2;
         }
-        virtio_mq_reg.cmd_size =
+
+        memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));
+        vblk->tag_set.ops = &virtio_mq_ops;
+        vblk->tag_set.nr_hw_queues = 1;
+        vblk->tag_set.queue_depth = virtblk_queue_depth;
+        vblk->tag_set.numa_node = NUMA_NO_NODE;
+        vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
+        vblk->tag_set.cmd_size =
                 sizeof(struct virtblk_req) +
                 sizeof(struct scatterlist) * sg_elems;
+        vblk->tag_set.driver_data = vblk;
+
+        err = blk_mq_alloc_tag_set(&vblk->tag_set);
+        if (err)
+                goto out_put_disk;
 
-        q = vblk->disk->queue = blk_mq_init_queue(&virtio_mq_reg, vblk);
+        q = vblk->disk->queue = blk_mq_init_queue(&vblk->tag_set);
         if (!q) {
                 err = -ENOMEM;
-                goto out_put_disk;
+                goto out_free_tags;
         }
 
-        blk_mq_init_commands(q, virtblk_init_vbr, vblk);
-
         q->queuedata = vblk;
 
         virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);
@@ -679,6 +689,8 @@ static int virtblk_probe(struct virtio_device *vdev)
 out_del_disk:
         del_gendisk(vblk->disk);
         blk_cleanup_queue(vblk->disk->queue);
+out_free_tags:
+        blk_mq_free_tag_set(&vblk->tag_set);
 out_put_disk:
         put_disk(vblk->disk);
 out_free_vq:
@@ -705,6 +717,8 @@ static void virtblk_remove(struct virtio_device *vdev)
         del_gendisk(vblk->disk);
         blk_cleanup_queue(vblk->disk->queue);
 
+        blk_mq_free_tag_set(&vblk->tag_set);
+
         /* Stop all the virtqueues. */
         vdev->config->reset(vdev);
 
@@ -749,7 +763,7 @@ static int virtblk_restore(struct virtio_device *vdev)
         vblk->config_enable = true;
         ret = init_vq(vdev->priv);
         if (!ret)
-                blk_mq_start_stopped_hw_queues(vblk->disk->queue);
+                blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
 
         return ret;
 }