aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2017-03-02 16:53:13 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2017-03-02 16:53:13 -0500
commit54d7989f476ca57fc3c5cc71524c480ccb74c481 (patch)
tree62a6b61edac708358d949af6c44b9ea3eb02c6fc
parent0f221a3102bba2d982d01bad38eb68507c343830 (diff)
parentc4baad50297d84bde1a7ad45e50c73adae4a2192 (diff)
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
Pull vhost updates from Michael Tsirkin: "virtio, vhost: optimizations, fixes Looks like a quiet cycle for vhost/virtio, just a couple of minor tweaks. Most notable is automatic interrupt affinity for blk and scsi. Hopefully other devices are not far behind" * tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: virtio-console: avoid DMA from stack vhost: introduce O(1) vq metadata cache virtio_scsi: use virtio IRQ affinity virtio_blk: use virtio IRQ affinity blk-mq: provide a default queue mapping for virtio device virtio: provide a method to get the IRQ affinity mask for a virtqueue virtio: allow drivers to request IRQ affinity when creating VQs virtio_pci: simplify MSI-X setup virtio_pci: don't duplicate the msix_enable flag in struct pci_dev virtio_pci: use shared interrupts for virtqueues virtio_pci: remove struct virtio_pci_vq_info vhost: try avoiding avail index access when getting descriptor virtio_mmio: expose header to userspace
Diffstat
-rw-r--r--block/Kconfig5
-rw-r--r--block/Makefile1
-rw-r--r--block/blk-mq-virtio.c54
-rw-r--r--drivers/block/virtio_blk.c14
-rw-r--r--drivers/char/virtio_console.c14
-rw-r--r--drivers/crypto/virtio/virtio_crypto_core.c2
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_kms.c2
-rw-r--r--drivers/misc/mic/vop/vop_main.c2
-rw-r--r--drivers/net/caif/caif_virtio.c3
-rw-r--r--drivers/net/virtio_net.c2
-rw-r--r--drivers/remoteproc/remoteproc_virtio.c3
-rw-r--r--drivers/rpmsg/virtio_rpmsg_bus.c2
-rw-r--r--drivers/s390/virtio/kvm_virtio.c3
-rw-r--r--drivers/s390/virtio/virtio_ccw.c3
-rw-r--r--drivers/scsi/virtio_scsi.c127
-rw-r--r--drivers/vhost/vhost.c173
-rw-r--r--drivers/vhost/vhost.h8
-rw-r--r--drivers/virtio/virtio_balloon.c3
-rw-r--r--drivers/virtio/virtio_input.c3
-rw-r--r--drivers/virtio/virtio_mmio.c5
-rw-r--r--drivers/virtio/virtio_pci_common.c376
-rw-r--r--drivers/virtio/virtio_pci_common.h50
-rw-r--r--drivers/virtio/virtio_pci_legacy.c9
-rw-r--r--drivers/virtio/virtio_pci_modern.c17
-rw-r--r--include/linux/blk-mq-virtio.h10
-rw-r--r--include/linux/cpuhotplug.h1
-rw-r--r--include/linux/virtio_config.h12
-rw-r--r--include/uapi/linux/Kbuild1
-rw-r--r--include/uapi/linux/virtio_mmio.h (renamed from include/linux/virtio_mmio.h)0
-rw-r--r--include/uapi/linux/virtio_pci.h2
-rw-r--r--net/vmw_vsock/virtio_transport.c3
31 files changed, 456 insertions, 454 deletions
diff --git a/block/Kconfig b/block/Kconfig
index a2a92e57a87d..e9f780f815f5 100644
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -189,4 +189,9 @@ config BLK_MQ_PCI
189 depends on BLOCK && PCI 189 depends on BLOCK && PCI
190 default y 190 default y
191 191
192config BLK_MQ_VIRTIO
193 bool
194 depends on BLOCK && VIRTIO
195 default y
196
192source block/Kconfig.iosched 197source block/Kconfig.iosched
diff --git a/block/Makefile b/block/Makefile
index 2ad7c304e3f5..081bb680789b 100644
--- a/block/Makefile
+++ b/block/Makefile
@@ -25,6 +25,7 @@ obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o
25obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o 25obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o
26obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o 26obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o
27obj-$(CONFIG_BLK_MQ_PCI) += blk-mq-pci.o 27obj-$(CONFIG_BLK_MQ_PCI) += blk-mq-pci.o
28obj-$(CONFIG_BLK_MQ_VIRTIO) += blk-mq-virtio.o
28obj-$(CONFIG_BLK_DEV_ZONED) += blk-zoned.o 29obj-$(CONFIG_BLK_DEV_ZONED) += blk-zoned.o
29obj-$(CONFIG_BLK_WBT) += blk-wbt.o 30obj-$(CONFIG_BLK_WBT) += blk-wbt.o
30obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o 31obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o
diff --git a/block/blk-mq-virtio.c b/block/blk-mq-virtio.c
new file mode 100644
index 000000000000..c3afbca11299
--- /dev/null
+++ b/block/blk-mq-virtio.c
@@ -0,0 +1,54 @@
1/*
2 * Copyright (c) 2016 Christoph Hellwig.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 */
13#include <linux/device.h>
14#include <linux/blk-mq.h>
15#include <linux/blk-mq-virtio.h>
16#include <linux/virtio_config.h>
17#include <linux/module.h>
18#include "blk-mq.h"
19
20/**
21 * blk_mq_virtio_map_queues - provide a default queue mapping for virtio device
22 * @set: tagset to provide the mapping for
23 * @vdev: virtio device associated with @set.
24 * @first_vec: first interrupt vectors to use for queues (usually 0)
25 *
26 * This function assumes the virtio device @vdev has at least as many available
27 * interrupt vetors as @set has queues. It will then queuery the vector
28 * corresponding to each queue for it's affinity mask and built queue mapping
29 * that maps a queue to the CPUs that have irq affinity for the corresponding
30 * vector.
31 */
32int blk_mq_virtio_map_queues(struct blk_mq_tag_set *set,
33 struct virtio_device *vdev, int first_vec)
34{
35 const struct cpumask *mask;
36 unsigned int queue, cpu;
37
38 if (!vdev->config->get_vq_affinity)
39 goto fallback;
40
41 for (queue = 0; queue < set->nr_hw_queues; queue++) {
42 mask = vdev->config->get_vq_affinity(vdev, first_vec + queue);
43 if (!mask)
44 goto fallback;
45
46 for_each_cpu(cpu, mask)
47 set->mq_map[cpu] = queue;
48 }
49
50 return 0;
51fallback:
52 return blk_mq_map_queues(set);
53}
54EXPORT_SYMBOL_GPL(blk_mq_virtio_map_queues);
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 024b473524c0..1d4c9f8bc1e1 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -5,6 +5,7 @@
5#include <linux/hdreg.h> 5#include <linux/hdreg.h>
6#include <linux/module.h> 6#include <linux/module.h>
7#include <linux/mutex.h> 7#include <linux/mutex.h>
8#include <linux/interrupt.h>
8#include <linux/virtio.h> 9#include <linux/virtio.h>
9#include <linux/virtio_blk.h> 10#include <linux/virtio_blk.h>
10#include <linux/scatterlist.h> 11#include <linux/scatterlist.h>
@@ -12,6 +13,7 @@
12#include <scsi/scsi_cmnd.h> 13#include <scsi/scsi_cmnd.h>
13#include <linux/idr.h> 14#include <linux/idr.h>
14#include <linux/blk-mq.h> 15#include <linux/blk-mq.h>
16#include <linux/blk-mq-virtio.h>
15#include <linux/numa.h> 17#include <linux/numa.h>
16 18
17#define PART_BITS 4 19#define PART_BITS 4
@@ -426,6 +428,7 @@ static int init_vq(struct virtio_blk *vblk)
426 struct virtqueue **vqs; 428 struct virtqueue **vqs;
427 unsigned short num_vqs; 429 unsigned short num_vqs;
428 struct virtio_device *vdev = vblk->vdev; 430 struct virtio_device *vdev = vblk->vdev;
431 struct irq_affinity desc = { 0, };
429 432
430 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ, 433 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ,
431 struct virtio_blk_config, num_queues, 434 struct virtio_blk_config, num_queues,
@@ -452,7 +455,8 @@ static int init_vq(struct virtio_blk *vblk)
452 } 455 }
453 456
454 /* Discover virtqueues and write information to configuration. */ 457 /* Discover virtqueues and write information to configuration. */
455 err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names); 458 err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names,
459 &desc);
456 if (err) 460 if (err)
457 goto out; 461 goto out;
458 462
@@ -586,10 +590,18 @@ static int virtblk_init_request(void *data, struct request *rq,
586 return 0; 590 return 0;
587} 591}
588 592
593static int virtblk_map_queues(struct blk_mq_tag_set *set)
594{
595 struct virtio_blk *vblk = set->driver_data;
596
597 return blk_mq_virtio_map_queues(set, vblk->vdev, 0);
598}
599
589static struct blk_mq_ops virtio_mq_ops = { 600static struct blk_mq_ops virtio_mq_ops = {
590 .queue_rq = virtio_queue_rq, 601 .queue_rq = virtio_queue_rq,
591 .complete = virtblk_request_done, 602 .complete = virtblk_request_done,
592 .init_request = virtblk_init_request, 603 .init_request = virtblk_init_request,
604 .map_queues = virtblk_map_queues,
593}; 605};
594 606
595static unsigned int virtblk_queue_depth; 607static unsigned int virtblk_queue_depth;
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index 17857beb4892..e9b7e0b3cabe 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -1136,6 +1136,8 @@ static int put_chars(u32 vtermno, const char *buf, int count)
1136{ 1136{
1137 struct port *port; 1137 struct port *port;
1138 struct scatterlist sg[1]; 1138 struct scatterlist sg[1];
1139 void *data;
1140 int ret;
1139 1141
1140 if (unlikely(early_put_chars)) 1142 if (unlikely(early_put_chars))
1141 return early_put_chars(vtermno, buf, count); 1143 return early_put_chars(vtermno, buf, count);
@@ -1144,8 +1146,14 @@ static int put_chars(u32 vtermno, const char *buf, int count)
1144 if (!port) 1146 if (!port)
1145 return -EPIPE; 1147 return -EPIPE;
1146 1148
1147 sg_init_one(sg, buf, count); 1149 data = kmemdup(buf, count, GFP_ATOMIC);
1148 return __send_to_port(port, sg, 1, count, (void *)buf, false); 1150 if (!data)
1151 return -ENOMEM;
1152
1153 sg_init_one(sg, data, count);
1154 ret = __send_to_port(port, sg, 1, count, data, false);
1155 kfree(data);
1156 return ret;
1149} 1157}
1150 1158
1151/* 1159/*
@@ -1939,7 +1947,7 @@ static int init_vqs(struct ports_device *portdev)
1939 /* Find the queues. */ 1947 /* Find the queues. */
1940 err = portdev->vdev->config->find_vqs(portdev->vdev, nr_queues, vqs, 1948 err = portdev->vdev->config->find_vqs(portdev->vdev, nr_queues, vqs,
1941 io_callbacks, 1949 io_callbacks,
1942 (const char **)io_names); 1950 (const char **)io_names, NULL);
1943 if (err) 1951 if (err)
1944 goto free; 1952 goto free;
1945 1953
diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c
index b5b153317376..21472e427f6f 100644
--- a/drivers/crypto/virtio/virtio_crypto_core.c
+++ b/drivers/crypto/virtio/virtio_crypto_core.c
@@ -120,7 +120,7 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi)
120 } 120 }
121 121
122 ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks, 122 ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
123 names); 123 names, NULL);
124 if (ret) 124 if (ret)
125 goto err_find; 125 goto err_find;
126 126
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 30f989a0cafc..491866865c33 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -176,7 +176,7 @@ int virtio_gpu_driver_load(struct drm_device *dev, unsigned long flags)
176#endif 176#endif
177 177
178 ret = vgdev->vdev->config->find_vqs(vgdev->vdev, 2, vqs, 178 ret = vgdev->vdev->config->find_vqs(vgdev->vdev, 2, vqs,
179 callbacks, names); 179 callbacks, names, NULL);
180 if (ret) { 180 if (ret) {
181 DRM_ERROR("failed to find virt queues\n"); 181 DRM_ERROR("failed to find virt queues\n");
182 goto err_vqs; 182 goto err_vqs;
diff --git a/drivers/misc/mic/vop/vop_main.c b/drivers/misc/mic/vop/vop_main.c
index 1a2b67f3183d..c2e29d7f0de8 100644
--- a/drivers/misc/mic/vop/vop_main.c
+++ b/drivers/misc/mic/vop/vop_main.c
@@ -374,7 +374,7 @@ unmap:
374static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs, 374static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
375 struct virtqueue *vqs[], 375 struct virtqueue *vqs[],
376 vq_callback_t *callbacks[], 376 vq_callback_t *callbacks[],
377 const char * const names[]) 377 const char * const names[], struct irq_affinity *desc)
378{ 378{
379 struct _vop_vdev *vdev = to_vopvdev(dev); 379 struct _vop_vdev *vdev = to_vopvdev(dev);
380 struct vop_device *vpdev = vdev->vpdev; 380 struct vop_device *vpdev = vdev->vpdev;
diff --git a/drivers/net/caif/caif_virtio.c b/drivers/net/caif/caif_virtio.c
index b306210b02b7..bc0eb47eccee 100644
--- a/drivers/net/caif/caif_virtio.c
+++ b/drivers/net/caif/caif_virtio.c
@@ -679,7 +679,8 @@ static int cfv_probe(struct virtio_device *vdev)
679 goto err; 679 goto err;
680 680
681 /* Get the TX virtio ring. This is a "guest side vring". */ 681 /* Get the TX virtio ring. This is a "guest side vring". */
682 err = vdev->config->find_vqs(vdev, 1, &cfv->vq_tx, &vq_cbs, &names); 682 err = vdev->config->find_vqs(vdev, 1, &cfv->vq_tx, &vq_cbs, &names,
683 NULL);
683 if (err) 684 if (err)
684 goto err; 685 goto err;
685 686
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index bf95016f442a..66fd3139be60 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -2080,7 +2080,7 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
2080 } 2080 }
2081 2081
2082 ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks, 2082 ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
2083 names); 2083 names, NULL);
2084 if (ret) 2084 if (ret)
2085 goto err_find; 2085 goto err_find;
2086 2086
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
index 364411fb7734..0142cc3f0c91 100644
--- a/drivers/remoteproc/remoteproc_virtio.c
+++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -137,7 +137,8 @@ static void rproc_virtio_del_vqs(struct virtio_device *vdev)
137static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, 137static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
138 struct virtqueue *vqs[], 138 struct virtqueue *vqs[],
139 vq_callback_t *callbacks[], 139 vq_callback_t *callbacks[],
140 const char * const names[]) 140 const char * const names[],
141 struct irq_affinity *desc)
141{ 142{
142 int i, ret; 143 int i, ret;
143 144
diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c
index 3090b0d3072f..5e66e081027e 100644
--- a/drivers/rpmsg/virtio_rpmsg_bus.c
+++ b/drivers/rpmsg/virtio_rpmsg_bus.c
@@ -869,7 +869,7 @@ static int rpmsg_probe(struct virtio_device *vdev)
869 init_waitqueue_head(&vrp->sendq); 869 init_waitqueue_head(&vrp->sendq);
870 870
871 /* We expect two virtqueues, rx and tx (and in this order) */ 871 /* We expect two virtqueues, rx and tx (and in this order) */
872 err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); 872 err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names, NULL);
873 if (err) 873 if (err)
874 goto free_vrp; 874 goto free_vrp;
875 875
diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c
index 5e5c11f37b24..2ce0b3eb2efe 100644
--- a/drivers/s390/virtio/kvm_virtio.c
+++ b/drivers/s390/virtio/kvm_virtio.c
@@ -255,7 +255,8 @@ static void kvm_del_vqs(struct virtio_device *vdev)
255static int kvm_find_vqs(struct virtio_device *vdev, unsigned nvqs, 255static int kvm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
256 struct virtqueue *vqs[], 256 struct virtqueue *vqs[],
257 vq_callback_t *callbacks[], 257 vq_callback_t *callbacks[],
258 const char * const names[]) 258 const char * const names[],
259 struct irq_affinity *desc)
259{ 260{
260 struct kvm_device *kdev = to_kvmdev(vdev); 261 struct kvm_device *kdev = to_kvmdev(vdev);
261 int i; 262 int i;
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 648373cde4a1..0ed209f3d8b0 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -628,7 +628,8 @@ out:
628static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, 628static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
629 struct virtqueue *vqs[], 629 struct virtqueue *vqs[],
630 vq_callback_t *callbacks[], 630 vq_callback_t *callbacks[],
631 const char * const names[]) 631 const char * const names[],
632 struct irq_affinity *desc)
632{ 633{
633 struct virtio_ccw_device *vcdev = to_vc_device(vdev); 634 struct virtio_ccw_device *vcdev = to_vc_device(vdev);
634 unsigned long *indicatorp = NULL; 635 unsigned long *indicatorp = NULL;
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index c680d7641311..939c47df73fa 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -18,6 +18,7 @@
18#include <linux/module.h> 18#include <linux/module.h>
19#include <linux/slab.h> 19#include <linux/slab.h>
20#include <linux/mempool.h> 20#include <linux/mempool.h>
21#include <linux/interrupt.h>
21#include <linux/virtio.h> 22#include <linux/virtio.h>
22#include <linux/virtio_ids.h> 23#include <linux/virtio_ids.h>
23#include <linux/virtio_config.h> 24#include <linux/virtio_config.h>
@@ -29,6 +30,7 @@
29#include <scsi/scsi_cmnd.h> 30#include <scsi/scsi_cmnd.h>
30#include <scsi/scsi_tcq.h> 31#include <scsi/scsi_tcq.h>
31#include <linux/seqlock.h> 32#include <linux/seqlock.h>
33#include <linux/blk-mq-virtio.h>
32 34
33#define VIRTIO_SCSI_MEMPOOL_SZ 64 35#define VIRTIO_SCSI_MEMPOOL_SZ 64
34#define VIRTIO_SCSI_EVENT_LEN 8 36#define VIRTIO_SCSI_EVENT_LEN 8
@@ -108,7 +110,6 @@ struct virtio_scsi {
108 bool affinity_hint_set; 110 bool affinity_hint_set;
109 111
110 struct hlist_node node; 112 struct hlist_node node;
111 struct hlist_node node_dead;
112 113
113 /* Protected by event_vq lock */ 114 /* Protected by event_vq lock */
114 bool stop_events; 115 bool stop_events;
@@ -118,7 +119,6 @@ struct virtio_scsi {
118 struct virtio_scsi_vq req_vqs[]; 119 struct virtio_scsi_vq req_vqs[];
119}; 120};
120 121
121static enum cpuhp_state virtioscsi_online;
122static struct kmem_cache *virtscsi_cmd_cache; 122static struct kmem_cache *virtscsi_cmd_cache;
123static mempool_t *virtscsi_cmd_pool; 123static mempool_t *virtscsi_cmd_pool;
124 124
@@ -766,6 +766,13 @@ static void virtscsi_target_destroy(struct scsi_target *starget)
766 kfree(tgt); 766 kfree(tgt);
767} 767}
768 768
769static int virtscsi_map_queues(struct Scsi_Host *shost)
770{
771 struct virtio_scsi *vscsi = shost_priv(shost);
772
773 return blk_mq_virtio_map_queues(&shost->tag_set, vscsi->vdev, 2);
774}
775
769static struct scsi_host_template virtscsi_host_template_single = { 776static struct scsi_host_template virtscsi_host_template_single = {
770 .module = THIS_MODULE, 777 .module = THIS_MODULE,
771 .name = "Virtio SCSI HBA", 778 .name = "Virtio SCSI HBA",
@@ -801,6 +808,7 @@ static struct scsi_host_template virtscsi_host_template_multi = {
801 .use_clustering = ENABLE_CLUSTERING, 808 .use_clustering = ENABLE_CLUSTERING,
802 .target_alloc = virtscsi_target_alloc, 809 .target_alloc = virtscsi_target_alloc,
803 .target_destroy = virtscsi_target_destroy, 810 .target_destroy = virtscsi_target_destroy,
811 .map_queues = virtscsi_map_queues,
804 .track_queue_depth = 1, 812 .track_queue_depth = 1,
805}; 813};
806 814
@@ -817,80 +825,6 @@ static struct scsi_host_template virtscsi_host_template_multi = {
817 virtio_cwrite(vdev, struct virtio_scsi_config, fld, &__val); \ 825 virtio_cwrite(vdev, struct virtio_scsi_config, fld, &__val); \
818 } while(0) 826 } while(0)
819 827
820static void __virtscsi_set_affinity(struct virtio_scsi *vscsi, bool affinity)
821{
822 int i;
823 int cpu;
824
825 /* In multiqueue mode, when the number of cpu is equal
826 * to the number of request queues, we let the qeueues
827 * to be private to one cpu by setting the affinity hint
828 * to eliminate the contention.
829 */
830 if ((vscsi->num_queues == 1 ||
831 vscsi->num_queues != num_online_cpus()) && affinity) {
832 if (vscsi->affinity_hint_set)
833 affinity = false;
834 else
835 return;
836 }
837
838 if (affinity) {
839 i = 0;
840 for_each_online_cpu(cpu) {
841 virtqueue_set_affinity(vscsi->req_vqs[i].vq, cpu);
842 i++;
843 }
844
845 vscsi->affinity_hint_set = true;
846 } else {
847 for (i = 0; i < vscsi->num_queues; i++) {
848 if (!vscsi->req_vqs[i].vq)
849 continue;
850
851 virtqueue_set_affinity(vscsi->req_vqs[i].vq, -1);
852 }
853
854 vscsi->affinity_hint_set = false;
855 }
856}
857
858static void virtscsi_set_affinity(struct virtio_scsi *vscsi, bool affinity)
859{
860 get_online_cpus();
861 __virtscsi_set_affinity(vscsi, affinity);
862 put_online_cpus();
863}
864
865static int virtscsi_cpu_online(unsigned int cpu, struct hlist_node *node)
866{
867 struct virtio_scsi *vscsi = hlist_entry_safe(node, struct virtio_scsi,
868 node);
869 __virtscsi_set_affinity(vscsi, true);
870 return 0;
871}
872
873static int virtscsi_cpu_notif_add(struct virtio_scsi *vi)
874{
875 int ret;
876
877 ret = cpuhp_state_add_instance(virtioscsi_online, &vi->node);
878 if (ret)
879 return ret;
880
881 ret = cpuhp_state_add_instance(CPUHP_VIRT_SCSI_DEAD, &vi->node_dead);
882 if (ret)
883 cpuhp_state_remove_instance(virtioscsi_online, &vi->node);
884 return ret;
885}
886
887static void virtscsi_cpu_notif_remove(struct virtio_scsi *vi)
888{
889 cpuhp_state_remove_instance_nocalls(virtioscsi_online, &vi->node);
890 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_SCSI_DEAD,
891 &vi->node_dead);
892}
893
894static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq, 828static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
895 struct virtqueue *vq) 829 struct virtqueue *vq)
896{ 830{
@@ -900,14 +834,8 @@ static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
900 834
901static void virtscsi_remove_vqs(struct virtio_device *vdev) 835static void virtscsi_remove_vqs(struct virtio_device *vdev)
902{ 836{
903 struct Scsi_Host *sh = virtio_scsi_host(vdev);
904 struct virtio_scsi *vscsi = shost_priv(sh);
905
906 virtscsi_set_affinity(vscsi, false);
907
908 /* Stop all the virtqueues. */ 837 /* Stop all the virtqueues. */
909 vdev->config->reset(vdev); 838 vdev->config->reset(vdev);
910
911 vdev->config->del_vqs(vdev); 839 vdev->config->del_vqs(vdev);
912} 840}
913 841
@@ -920,6 +848,7 @@ static int virtscsi_init(struct virtio_device *vdev,
920 vq_callback_t **callbacks; 848 vq_callback_t **callbacks;
921 const char **names; 849 const char **names;
922 struct virtqueue **vqs; 850 struct virtqueue **vqs;
851 struct irq_affinity desc = { .pre_vectors = 2 };
923 852
924 num_vqs = vscsi->num_queues + VIRTIO_SCSI_VQ_BASE; 853 num_vqs = vscsi->num_queues + VIRTIO_SCSI_VQ_BASE;
925 vqs = kmalloc(num_vqs * sizeof(struct virtqueue *), GFP_KERNEL); 854 vqs = kmalloc(num_vqs * sizeof(struct virtqueue *), GFP_KERNEL);
@@ -941,7 +870,8 @@ static int virtscsi_init(struct virtio_device *vdev,
941 } 870 }
942 871
943 /* Discover virtqueues and write information to configuration. */ 872 /* Discover virtqueues and write information to configuration. */
944 err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names); 873 err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names,
874 &desc);
945 if (err) 875 if (err)
946 goto out; 876 goto out;
947 877
@@ -1007,10 +937,6 @@ static int virtscsi_probe(struct virtio_device *vdev)
1007 if (err) 937 if (err)
1008 goto virtscsi_init_failed; 938 goto virtscsi_init_failed;
1009 939
1010 err = virtscsi_cpu_notif_add(vscsi);
1011 if (err)
1012 goto scsi_add_host_failed;
1013
1014 cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1; 940 cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1;
1015 shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue); 941 shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue);
1016 shost->max_sectors = virtscsi_config_get(vdev, max_sectors) ?: 0xFFFF; 942 shost->max_sectors = virtscsi_config_get(vdev, max_sectors) ?: 0xFFFF;
@@ -1065,9 +991,6 @@ static void virtscsi_remove(struct virtio_device *vdev)
1065 virtscsi_cancel_event_work(vscsi); 991 virtscsi_cancel_event_work(vscsi);
1066 992
1067 scsi_remove_host(shost); 993 scsi_remove_host(shost);
1068
1069 virtscsi_cpu_notif_remove(vscsi);
1070
1071 virtscsi_remove_vqs(vdev); 994 virtscsi_remove_vqs(vdev);
1072 scsi_host_put(shost); 995 scsi_host_put(shost);
1073} 996}
@@ -1075,10 +998,6 @@ static void virtscsi_remove(struct virtio_device *vdev)
1075#ifdef CONFIG_PM_SLEEP 998#ifdef CONFIG_PM_SLEEP
1076static int virtscsi_freeze(struct virtio_device *vdev) 999static int virtscsi_freeze(struct virtio_device *vdev)
1077{ 1000{
1078 struct Scsi_Host *sh = virtio_scsi_host(vdev);
1079 struct virtio_scsi *vscsi = shost_priv(sh);
1080
1081 virtscsi_cpu_notif_remove(vscsi);
1082 virtscsi_remove_vqs(vdev); 1001 virtscsi_remove_vqs(vdev);
1083 return 0; 1002 return 0;
1084} 1003}
@@ -1093,11 +1012,6 @@ static int virtscsi_restore(struct virtio_device *vdev)
1093 if (err) 1012 if (err)
1094 return err; 1013 return err;
1095 1014
1096 err = virtscsi_cpu_notif_add(vscsi);
1097 if (err) {
1098 vdev->config->del_vqs(vdev);
1099 return err;
1100 }
1101 virtio_device_ready(vdev); 1015 virtio_device_ready(vdev);
1102 1016
1103 if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) 1017 if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG))
@@ -1152,16 +1066,6 @@ static int __init init(void)
1152 pr_err("mempool_create() for virtscsi_cmd_pool failed\n"); 1066 pr_err("mempool_create() for virtscsi_cmd_pool failed\n");
1153 goto error; 1067 goto error;
1154 } 1068 }
1155 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1156 "scsi/virtio:online",
1157 virtscsi_cpu_online, NULL);
1158 if (ret < 0)
1159 goto error;
1160 virtioscsi_online = ret;
1161 ret = cpuhp_setup_state_multi(CPUHP_VIRT_SCSI_DEAD, "scsi/virtio:dead",
1162 NULL, virtscsi_cpu_online);
1163 if (ret)
1164 goto error;
1165 ret = register_virtio_driver(&virtio_scsi_driver); 1069 ret = register_virtio_driver(&virtio_scsi_driver);
1166 if (ret < 0) 1070 if (ret < 0)
1167 goto error; 1071 goto error;
@@ -1177,17 +1081,12 @@ error:
1177 kmem_cache_destroy(virtscsi_cmd_cache); 1081 kmem_cache_destroy(virtscsi_cmd_cache);
1178 virtscsi_cmd_cache = NULL; 1082 virtscsi_cmd_cache = NULL;
1179 } 1083 }
1180 if (virtioscsi_online)
1181 cpuhp_remove_multi_state(virtioscsi_online);
1182 cpuhp_remove_multi_state(CPUHP_VIRT_SCSI_DEAD);
1183 return ret; 1084 return ret;
1184} 1085}
1185 1086
1186static void __exit fini(void) 1087static void __exit fini(void)
1187{ 1088{
1188 unregister_virtio_driver(&virtio_scsi_driver); 1089 unregister_virtio_driver(&virtio_scsi_driver);
1189 cpuhp_remove_multi_state(virtioscsi_online);
1190 cpuhp_remove_multi_state(CPUHP_VIRT_SCSI_DEAD);
1191 mempool_destroy(virtscsi_cmd_pool); 1090 mempool_destroy(virtscsi_cmd_pool);
1192 kmem_cache_destroy(virtscsi_cmd_cache); 1091 kmem_cache_destroy(virtscsi_cmd_cache);
1193} 1092}
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 4269e621e254..9469364eefd7 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -282,6 +282,22 @@ void vhost_poll_queue(struct vhost_poll *poll)
282} 282}
283EXPORT_SYMBOL_GPL(vhost_poll_queue); 283EXPORT_SYMBOL_GPL(vhost_poll_queue);
284 284
285static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
286{
287 int j;
288
289 for (j = 0; j < VHOST_NUM_ADDRS; j++)
290 vq->meta_iotlb[j] = NULL;
291}
292
293static void vhost_vq_meta_reset(struct vhost_dev *d)
294{
295 int i;
296
297 for (i = 0; i < d->nvqs; ++i)
298 __vhost_vq_meta_reset(d->vqs[i]);
299}
300
285static void vhost_vq_reset(struct vhost_dev *dev, 301static void vhost_vq_reset(struct vhost_dev *dev,
286 struct vhost_virtqueue *vq) 302 struct vhost_virtqueue *vq)
287{ 303{
@@ -312,6 +328,7 @@ static void vhost_vq_reset(struct vhost_dev *dev,
312 vq->busyloop_timeout = 0; 328 vq->busyloop_timeout = 0;
313 vq->umem = NULL; 329 vq->umem = NULL;
314 vq->iotlb = NULL; 330 vq->iotlb = NULL;
331 __vhost_vq_meta_reset(vq);
315} 332}
316 333
317static int vhost_worker(void *data) 334static int vhost_worker(void *data)
@@ -691,6 +708,18 @@ static int vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem,
691 return 1; 708 return 1;
692} 709}
693 710
711static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq,
712 u64 addr, unsigned int size,
713 int type)
714{
715 const struct vhost_umem_node *node = vq->meta_iotlb[type];
716
717 if (!node)
718 return NULL;
719
720 return (void *)(uintptr_t)(node->userspace_addr + addr - node->start);
721}
722
694/* Can we switch to this memory table? */ 723/* Can we switch to this memory table? */
695/* Caller should have device mutex but not vq mutex */ 724/* Caller should have device mutex but not vq mutex */
696static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem, 725static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
@@ -733,8 +762,14 @@ static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to,
733 * could be access through iotlb. So -EAGAIN should 762 * could be access through iotlb. So -EAGAIN should
734 * not happen in this case. 763 * not happen in this case.
735 */ 764 */
736 /* TODO: more fast path */
737 struct iov_iter t; 765 struct iov_iter t;
766 void __user *uaddr = vhost_vq_meta_fetch(vq,
767 (u64)(uintptr_t)to, size,
768 VHOST_ADDR_DESC);
769
770 if (uaddr)
771 return __copy_to_user(uaddr, from, size);
772
738 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov, 773 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
739 ARRAY_SIZE(vq->iotlb_iov), 774 ARRAY_SIZE(vq->iotlb_iov),
740 VHOST_ACCESS_WO); 775 VHOST_ACCESS_WO);
@@ -762,8 +797,14 @@ static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,
762 * could be access through iotlb. So -EAGAIN should 797 * could be access through iotlb. So -EAGAIN should
763 * not happen in this case. 798 * not happen in this case.
764 */ 799 */
765 /* TODO: more fast path */ 800 void __user *uaddr = vhost_vq_meta_fetch(vq,
801 (u64)(uintptr_t)from, size,
802 VHOST_ADDR_DESC);
766 struct iov_iter f; 803 struct iov_iter f;
804
805 if (uaddr)
806 return __copy_from_user(to, uaddr, size);
807
767 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov, 808 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
768 ARRAY_SIZE(vq->iotlb_iov), 809 ARRAY_SIZE(vq->iotlb_iov),
769 VHOST_ACCESS_RO); 810 VHOST_ACCESS_RO);
@@ -783,17 +824,12 @@ out:
783 return ret; 824 return ret;
784} 825}
785 826
786static void __user *__vhost_get_user(struct vhost_virtqueue *vq, 827static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq,
787 void __user *addr, unsigned size) 828 void __user *addr, unsigned int size,
829 int type)
788{ 830{
789 int ret; 831 int ret;
790 832
791 /* This function should be called after iotlb
792 * prefetch, which means we're sure that vq
793 * could be access through iotlb. So -EAGAIN should
794 * not happen in this case.
795 */
796 /* TODO: more fast path */
797 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov, 833 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
798 ARRAY_SIZE(vq->iotlb_iov), 834 ARRAY_SIZE(vq->iotlb_iov),
799 VHOST_ACCESS_RO); 835 VHOST_ACCESS_RO);
@@ -814,14 +850,32 @@ static void __user *__vhost_get_user(struct vhost_virtqueue *vq,
814 return vq->iotlb_iov[0].iov_base; 850 return vq->iotlb_iov[0].iov_base;
815} 851}
816 852
817#define vhost_put_user(vq, x, ptr) \ 853/* This function should be called after iotlb
854 * prefetch, which means we're sure that vq
855 * could be access through iotlb. So -EAGAIN should
856 * not happen in this case.
857 */
858static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
859 void *addr, unsigned int size,
860 int type)
861{
862 void __user *uaddr = vhost_vq_meta_fetch(vq,
863 (u64)(uintptr_t)addr, size, type);
864 if (uaddr)
865 return uaddr;
866
867 return __vhost_get_user_slow(vq, addr, size, type);
868}
869
870#define vhost_put_user(vq, x, ptr) \
818({ \ 871({ \
819 int ret = -EFAULT; \ 872 int ret = -EFAULT; \
820 if (!vq->iotlb) { \ 873 if (!vq->iotlb) { \
821 ret = __put_user(x, ptr); \ 874 ret = __put_user(x, ptr); \
822 } else { \ 875 } else { \
823 __typeof__(ptr) to = \ 876 __typeof__(ptr) to = \
824 (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \ 877 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
878 sizeof(*ptr), VHOST_ADDR_USED); \
825 if (to != NULL) \ 879 if (to != NULL) \
826 ret = __put_user(x, to); \ 880 ret = __put_user(x, to); \
827 else \ 881 else \
@@ -830,14 +884,16 @@ static void __user *__vhost_get_user(struct vhost_virtqueue *vq,
830 ret; \ 884 ret; \
831}) 885})
832 886
833#define vhost_get_user(vq, x, ptr) \ 887#define vhost_get_user(vq, x, ptr, type) \
834({ \ 888({ \
835 int ret; \ 889 int ret; \
836 if (!vq->iotlb) { \ 890 if (!vq->iotlb) { \
837 ret = __get_user(x, ptr); \ 891 ret = __get_user(x, ptr); \
838 } else { \ 892 } else { \
839 __typeof__(ptr) from = \ 893 __typeof__(ptr) from = \
840 (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \ 894 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
895 sizeof(*ptr), \
896 type); \
841 if (from != NULL) \ 897 if (from != NULL) \
842 ret = __get_user(x, from); \ 898 ret = __get_user(x, from); \
843 else \ 899 else \
@@ -846,6 +902,12 @@ static void __user *__vhost_get_user(struct vhost_virtqueue *vq,
846 ret; \ 902 ret; \
847}) 903})
848 904
905#define vhost_get_avail(vq, x, ptr) \
906 vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
907
908#define vhost_get_used(vq, x, ptr) \
909 vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
910
849static void vhost_dev_lock_vqs(struct vhost_dev *d) 911static void vhost_dev_lock_vqs(struct vhost_dev *d)
850{ 912{
851 int i = 0; 913 int i = 0;
@@ -951,6 +1013,7 @@ static int vhost_process_iotlb_msg(struct vhost_dev *dev,
951 ret = -EFAULT; 1013 ret = -EFAULT;
952 break; 1014 break;
953 } 1015 }
1016 vhost_vq_meta_reset(dev);
954 if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size, 1017 if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
955 msg->iova + msg->size - 1, 1018 msg->iova + msg->size - 1,
956 msg->uaddr, msg->perm)) { 1019 msg->uaddr, msg->perm)) {
@@ -960,6 +1023,7 @@ static int vhost_process_iotlb_msg(struct vhost_dev *dev,
960 vhost_iotlb_notify_vq(dev, msg); 1023 vhost_iotlb_notify_vq(dev, msg);
961 break; 1024 break;
962 case VHOST_IOTLB_INVALIDATE: 1025 case VHOST_IOTLB_INVALIDATE:
1026 vhost_vq_meta_reset(dev);
963 vhost_del_umem_range(dev->iotlb, msg->iova, 1027 vhost_del_umem_range(dev->iotlb, msg->iova,
964 msg->iova + msg->size - 1); 1028 msg->iova + msg->size - 1);
965 break; 1029 break;
@@ -1103,12 +1167,26 @@ static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1103 sizeof *used + num * sizeof *used->ring + s); 1167 sizeof *used + num * sizeof *used->ring + s);
1104} 1168}
1105 1169
1170static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
1171 const struct vhost_umem_node *node,
1172 int type)
1173{
1174 int access = (type == VHOST_ADDR_USED) ?
1175 VHOST_ACCESS_WO : VHOST_ACCESS_RO;
1176
1177 if (likely(node->perm & access))
1178 vq->meta_iotlb[type] = node;
1179}
1180
1106static int iotlb_access_ok(struct vhost_virtqueue *vq, 1181static int iotlb_access_ok(struct vhost_virtqueue *vq,
1107 int access, u64 addr, u64 len) 1182 int access, u64 addr, u64 len, int type)
1108{ 1183{
1109 const struct vhost_umem_node *node; 1184 const struct vhost_umem_node *node;
1110 struct vhost_umem *umem = vq->iotlb; 1185 struct vhost_umem *umem = vq->iotlb;
1111 u64 s = 0, size; 1186 u64 s = 0, size, orig_addr = addr;
1187
1188 if (vhost_vq_meta_fetch(vq, addr, len, type))
1189 return true;
1112 1190
1113 while (len > s) { 1191 while (len > s) {
1114 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, 1192 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
@@ -1125,6 +1203,10 @@ static int iotlb_access_ok(struct vhost_virtqueue *vq,
1125 } 1203 }
1126 1204
1127 size = node->size - addr + node->start; 1205 size = node->size - addr + node->start;
1206
1207 if (orig_addr == addr && size >= len)
1208 vhost_vq_meta_update(vq, node, type);
1209
1128 s += size; 1210 s += size;
1129 addr += size; 1211 addr += size;
1130 } 1212 }
@@ -1141,13 +1223,15 @@ int vq_iotlb_prefetch(struct vhost_virtqueue *vq)
1141 return 1; 1223 return 1;
1142 1224
1143 return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, 1225 return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
1144 num * sizeof *vq->desc) && 1226 num * sizeof(*vq->desc), VHOST_ADDR_DESC) &&
1145 iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail, 1227 iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
1146 sizeof *vq->avail + 1228 sizeof *vq->avail +
1147 num * sizeof *vq->avail->ring + s) && 1229 num * sizeof(*vq->avail->ring) + s,
1230 VHOST_ADDR_AVAIL) &&
1148 iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used, 1231 iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
1149 sizeof *vq->used + 1232 sizeof *vq->used +
1150 num * sizeof *vq->used->ring + s); 1233 num * sizeof(*vq->used->ring) + s,
1234 VHOST_ADDR_USED);
1151} 1235}
1152EXPORT_SYMBOL_GPL(vq_iotlb_prefetch); 1236EXPORT_SYMBOL_GPL(vq_iotlb_prefetch);
1153 1237
@@ -1728,7 +1812,7 @@ int vhost_vq_init_access(struct vhost_virtqueue *vq)
1728 r = -EFAULT; 1812 r = -EFAULT;
1729 goto err; 1813 goto err;
1730 } 1814 }
1731 r = vhost_get_user(vq, last_used_idx, &vq->used->idx); 1815 r = vhost_get_used(vq, last_used_idx, &vq->used->idx);
1732 if (r) { 1816 if (r) {
1733 vq_err(vq, "Can't access used idx at %p\n", 1817 vq_err(vq, "Can't access used idx at %p\n",
1734 &vq->used->idx); 1818 &vq->used->idx);
@@ -1930,29 +2014,36 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1930 2014
1931 /* Check it isn't doing very strange things with descriptor numbers. */ 2015 /* Check it isn't doing very strange things with descriptor numbers. */
1932 last_avail_idx = vq->last_avail_idx; 2016 last_avail_idx = vq->last_avail_idx;
1933 if (unlikely(vhost_get_user(vq, avail_idx, &vq->avail->idx))) {
1934 vq_err(vq, "Failed to access avail idx at %p\n",
1935 &vq->avail->idx);
1936 return -EFAULT;
1937 }
1938 vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
1939 2017
1940 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { 2018 if (vq->avail_idx == vq->last_avail_idx) {
1941 vq_err(vq, "Guest moved used index from %u to %u", 2019 if (unlikely(vhost_get_avail(vq, avail_idx, &vq->avail->idx))) {
1942 last_avail_idx, vq->avail_idx); 2020 vq_err(vq, "Failed to access avail idx at %p\n",
1943 return -EFAULT; 2021 &vq->avail->idx);
1944 } 2022 return -EFAULT;
2023 }
2024 vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
1945 2025
1946 /* If there's nothing new since last we looked, return invalid. */ 2026 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
1947 if (vq->avail_idx == last_avail_idx) 2027 vq_err(vq, "Guest moved used index from %u to %u",
1948 return vq->num; 2028 last_avail_idx, vq->avail_idx);
2029 return -EFAULT;
2030 }
2031
2032 /* If there's nothing new since last we looked, return
2033 * invalid.
2034 */
2035 if (vq->avail_idx == last_avail_idx)
2036 return vq->num;
1949 2037
1950 /* Only get avail ring entries after they have been exposed by guest. */ 2038 /* Only get avail ring entries after they have been
1951 smp_rmb(); 2039 * exposed by guest.
2040 */
2041 smp_rmb();
2042 }
1952 2043
1953 /* Grab the next descriptor number they're advertising, and increment 2044 /* Grab the next descriptor number they're advertising, and increment
1954 * the index we've seen. */ 2045 * the index we've seen. */
1955 if (unlikely(vhost_get_user(vq, ring_head, 2046 if (unlikely(vhost_get_avail(vq, ring_head,
1956 &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) { 2047 &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) {
1957 vq_err(vq, "Failed to read head: idx %d address %p\n", 2048 vq_err(vq, "Failed to read head: idx %d address %p\n",
1958 last_avail_idx, 2049 last_avail_idx,
@@ -2168,7 +2259,7 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2168 * with the barrier that the Guest executes when enabling 2259 * with the barrier that the Guest executes when enabling
2169 * interrupts. */ 2260 * interrupts. */
2170 smp_mb(); 2261 smp_mb();
2171 if (vhost_get_user(vq, flags, &vq->avail->flags)) { 2262 if (vhost_get_avail(vq, flags, &vq->avail->flags)) {
2172 vq_err(vq, "Failed to get flags"); 2263 vq_err(vq, "Failed to get flags");
2173 return true; 2264 return true;
2174 } 2265 }
@@ -2195,7 +2286,7 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2195 * interrupts. */ 2286 * interrupts. */
2196 smp_mb(); 2287 smp_mb();
2197 2288
2198 if (vhost_get_user(vq, event, vhost_used_event(vq))) { 2289 if (vhost_get_avail(vq, event, vhost_used_event(vq))) {
2199 vq_err(vq, "Failed to get used event idx"); 2290 vq_err(vq, "Failed to get used event idx");
2200 return true; 2291 return true;
2201 } 2292 }
@@ -2242,7 +2333,7 @@ bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2242 if (vq->avail_idx != vq->last_avail_idx) 2333 if (vq->avail_idx != vq->last_avail_idx)
2243 return false; 2334 return false;
2244 2335
2245 r = vhost_get_user(vq, avail_idx, &vq->avail->idx); 2336 r = vhost_get_avail(vq, avail_idx, &vq->avail->idx);
2246 if (unlikely(r)) 2337 if (unlikely(r))
2247 return false; 2338 return false;
2248 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); 2339 vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
@@ -2278,7 +2369,7 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2278 /* They could have slipped one in as we were doing that: make 2369 /* They could have slipped one in as we were doing that: make
2279 * sure it's written, then check again. */ 2370 * sure it's written, then check again. */
2280 smp_mb(); 2371 smp_mb();
2281 r = vhost_get_user(vq, avail_idx, &vq->avail->idx); 2372 r = vhost_get_avail(vq, avail_idx, &vq->avail->idx);
2282 if (r) { 2373 if (r) {
2283 vq_err(vq, "Failed to check avail idx at %p: %d\n", 2374 vq_err(vq, "Failed to check avail idx at %p: %d\n",
2284 &vq->avail->idx, r); 2375 &vq->avail->idx, r);
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index a9cbbb148f46..f55671d53f28 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -76,6 +76,13 @@ struct vhost_umem {
76 int numem; 76 int numem;
77}; 77};
78 78
79enum vhost_uaddr_type {
80 VHOST_ADDR_DESC = 0,
81 VHOST_ADDR_AVAIL = 1,
82 VHOST_ADDR_USED = 2,
83 VHOST_NUM_ADDRS = 3,
84};
85
79/* The virtqueue structure describes a queue attached to a device. */ 86/* The virtqueue structure describes a queue attached to a device. */
80struct vhost_virtqueue { 87struct vhost_virtqueue {
81 struct vhost_dev *dev; 88 struct vhost_dev *dev;
@@ -86,6 +93,7 @@ struct vhost_virtqueue {
86 struct vring_desc __user *desc; 93 struct vring_desc __user *desc;
87 struct vring_avail __user *avail; 94 struct vring_avail __user *avail;
88 struct vring_used __user *used; 95 struct vring_used __user *used;
96 const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS];
89 struct file *kick; 97 struct file *kick;
90 struct file *call; 98 struct file *call;
91 struct file *error; 99 struct file *error;
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 9d2738e9217f..a2a4386d9836 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -413,7 +413,8 @@ static int init_vqs(struct virtio_balloon *vb)
413 * optionally stat. 413 * optionally stat.
414 */ 414 */
415 nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2; 415 nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
416 err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names); 416 err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names,
417 NULL);
417 if (err) 418 if (err)
418 return err; 419 return err;
419 420
diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c
index 350a2a5a49db..79f1293cda93 100644
--- a/drivers/virtio/virtio_input.c
+++ b/drivers/virtio/virtio_input.c
@@ -173,7 +173,8 @@ static int virtinput_init_vqs(struct virtio_input *vi)
173 static const char * const names[] = { "events", "status" }; 173 static const char * const names[] = { "events", "status" };
174 int err; 174 int err;
175 175
176 err = vi->vdev->config->find_vqs(vi->vdev, 2, vqs, cbs, names); 176 err = vi->vdev->config->find_vqs(vi->vdev, 2, vqs, cbs, names,
177 NULL);
177 if (err) 178 if (err)
178 return err; 179 return err;
179 vi->evt = vqs[0]; 180 vi->evt = vqs[0];
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index c71fde5fe835..78343b8f9034 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -70,7 +70,7 @@
70#include <linux/spinlock.h> 70#include <linux/spinlock.h>
71#include <linux/virtio.h> 71#include <linux/virtio.h>
72#include <linux/virtio_config.h> 72#include <linux/virtio_config.h>
73#include <linux/virtio_mmio.h> 73#include <uapi/linux/virtio_mmio.h>
74#include <linux/virtio_ring.h> 74#include <linux/virtio_ring.h>
75 75
76 76
@@ -446,7 +446,8 @@ error_available:
446static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs, 446static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
447 struct virtqueue *vqs[], 447 struct virtqueue *vqs[],
448 vq_callback_t *callbacks[], 448 vq_callback_t *callbacks[],
449 const char * const names[]) 449 const char * const names[],
450 struct irq_affinity *desc)
450{ 451{
451 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); 452 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
452 unsigned int irq = platform_get_irq(vm_dev->pdev, 0); 453 unsigned int irq = platform_get_irq(vm_dev->pdev, 0);
diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c
index 186cbab327b8..df548a6fb844 100644
--- a/drivers/virtio/virtio_pci_common.c
+++ b/drivers/virtio/virtio_pci_common.c
@@ -33,10 +33,8 @@ void vp_synchronize_vectors(struct virtio_device *vdev)
33 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 33 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
34 int i; 34 int i;
35 35
36 if (vp_dev->intx_enabled) 36 synchronize_irq(pci_irq_vector(vp_dev->pci_dev, 0));
37 synchronize_irq(vp_dev->pci_dev->irq); 37 for (i = 1; i < vp_dev->msix_vectors; i++)
38
39 for (i = 0; i < vp_dev->msix_vectors; ++i)
40 synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i)); 38 synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));
41} 39}
42 40
@@ -62,16 +60,13 @@ static irqreturn_t vp_config_changed(int irq, void *opaque)
62static irqreturn_t vp_vring_interrupt(int irq, void *opaque) 60static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
63{ 61{
64 struct virtio_pci_device *vp_dev = opaque; 62 struct virtio_pci_device *vp_dev = opaque;
65 struct virtio_pci_vq_info *info;
66 irqreturn_t ret = IRQ_NONE; 63 irqreturn_t ret = IRQ_NONE;
67 unsigned long flags; 64 struct virtqueue *vq;
68 65
69 spin_lock_irqsave(&vp_dev->lock, flags); 66 list_for_each_entry(vq, &vp_dev->vdev.vqs, list) {
70 list_for_each_entry(info, &vp_dev->virtqueues, node) { 67 if (vq->callback && vring_interrupt(irq, vq) == IRQ_HANDLED)
71 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
72 ret = IRQ_HANDLED; 68 ret = IRQ_HANDLED;
73 } 69 }
74 spin_unlock_irqrestore(&vp_dev->lock, flags);
75 70
76 return ret; 71 return ret;
77} 72}
@@ -102,237 +97,185 @@ static irqreturn_t vp_interrupt(int irq, void *opaque)
102 return vp_vring_interrupt(irq, opaque); 97 return vp_vring_interrupt(irq, opaque);
103} 98}
104 99
105static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, 100static void vp_remove_vqs(struct virtio_device *vdev)
106 bool per_vq_vectors)
107{ 101{
108 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 102 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
109 const char *name = dev_name(&vp_dev->vdev.dev); 103 struct virtqueue *vq, *n;
110 unsigned i, v;
111 int err = -ENOMEM;
112
113 vp_dev->msix_vectors = nvectors;
114
115 vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
116 GFP_KERNEL);
117 if (!vp_dev->msix_names)
118 goto error;
119 vp_dev->msix_affinity_masks
120 = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
121 GFP_KERNEL);
122 if (!vp_dev->msix_affinity_masks)
123 goto error;
124 for (i = 0; i < nvectors; ++i)
125 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
126 GFP_KERNEL))
127 goto error;
128
129 err = pci_alloc_irq_vectors(vp_dev->pci_dev, nvectors, nvectors,
130 PCI_IRQ_MSIX);
131 if (err < 0)
132 goto error;
133 vp_dev->msix_enabled = 1;
134
135 /* Set the vector used for configuration */
136 v = vp_dev->msix_used_vectors;
137 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
138 "%s-config", name);
139 err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
140 vp_config_changed, 0, vp_dev->msix_names[v],
141 vp_dev);
142 if (err)
143 goto error;
144 ++vp_dev->msix_used_vectors;
145
146 v = vp_dev->config_vector(vp_dev, v);
147 /* Verify we had enough resources to assign the vector */
148 if (v == VIRTIO_MSI_NO_VECTOR) {
149 err = -EBUSY;
150 goto error;
151 }
152
153 if (!per_vq_vectors) {
154 /* Shared vector for all VQs */
155 v = vp_dev->msix_used_vectors;
156 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
157 "%s-virtqueues", name);
158 err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
159 vp_vring_interrupt, 0, vp_dev->msix_names[v],
160 vp_dev);
161 if (err)
162 goto error;
163 ++vp_dev->msix_used_vectors;
164 }
165 return 0;
166error:
167 return err;
168}
169
170static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
171 void (*callback)(struct virtqueue *vq),
172 const char *name,
173 u16 msix_vec)
174{
175 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
176 struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
177 struct virtqueue *vq;
178 unsigned long flags;
179
180 /* fill out our structure that represents an active queue */
181 if (!info)
182 return ERR_PTR(-ENOMEM);
183 104
184 vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, msix_vec); 105 list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
185 if (IS_ERR(vq)) 106 if (vp_dev->msix_vector_map) {
186 goto out_info; 107 int v = vp_dev->msix_vector_map[vq->index];
187 108
188 info->vq = vq; 109 if (v != VIRTIO_MSI_NO_VECTOR)
189 if (callback) { 110 free_irq(pci_irq_vector(vp_dev->pci_dev, v),
190 spin_lock_irqsave(&vp_dev->lock, flags); 111 vq);
191 list_add(&info->node, &vp_dev->virtqueues); 112 }
192 spin_unlock_irqrestore(&vp_dev->lock, flags); 113 vp_dev->del_vq(vq);
193 } else {
194 INIT_LIST_HEAD(&info->node);
195 } 114 }
196
197 vp_dev->vqs[index] = info;
198 return vq;
199
200out_info:
201 kfree(info);
202 return vq;
203}
204
205static void vp_del_vq(struct virtqueue *vq)
206{
207 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
208 struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
209 unsigned long flags;
210
211 spin_lock_irqsave(&vp_dev->lock, flags);
212 list_del(&info->node);
213 spin_unlock_irqrestore(&vp_dev->lock, flags);
214
215 vp_dev->del_vq(info);
216 kfree(info);
217} 115}
218 116
219/* the config->del_vqs() implementation */ 117/* the config->del_vqs() implementation */
220void vp_del_vqs(struct virtio_device *vdev) 118void vp_del_vqs(struct virtio_device *vdev)
221{ 119{
222 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 120 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
223 struct virtqueue *vq, *n;
224 int i; 121 int i;
225 122
226 list_for_each_entry_safe(vq, n, &vdev->vqs, list) { 123 if (WARN_ON_ONCE(list_empty_careful(&vdev->vqs)))
227 if (vp_dev->per_vq_vectors) { 124 return;
228 int v = vp_dev->vqs[vq->index]->msix_vector;
229
230 if (v != VIRTIO_MSI_NO_VECTOR)
231 free_irq(pci_irq_vector(vp_dev->pci_dev, v),
232 vq);
233 }
234 vp_del_vq(vq);
235 }
236 vp_dev->per_vq_vectors = false;
237
238 if (vp_dev->intx_enabled) {
239 free_irq(vp_dev->pci_dev->irq, vp_dev);
240 vp_dev->intx_enabled = 0;
241 }
242 125
243 for (i = 0; i < vp_dev->msix_used_vectors; ++i) 126 vp_remove_vqs(vdev);
244 free_irq(pci_irq_vector(vp_dev->pci_dev, i), vp_dev);
245 127
246 for (i = 0; i < vp_dev->msix_vectors; i++) 128 if (vp_dev->pci_dev->msix_enabled) {
247 if (vp_dev->msix_affinity_masks[i]) 129 for (i = 0; i < vp_dev->msix_vectors; i++)
248 free_cpumask_var(vp_dev->msix_affinity_masks[i]); 130 free_cpumask_var(vp_dev->msix_affinity_masks[i]);
249 131
250 if (vp_dev->msix_enabled) {
251 /* Disable the vector used for configuration */ 132 /* Disable the vector used for configuration */
252 vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR); 133 vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
253 134
254 pci_free_irq_vectors(vp_dev->pci_dev); 135 kfree(vp_dev->msix_affinity_masks);
255 vp_dev->msix_enabled = 0; 136 kfree(vp_dev->msix_names);
137 kfree(vp_dev->msix_vector_map);
256 } 138 }
257 139
258 vp_dev->msix_vectors = 0; 140 free_irq(pci_irq_vector(vp_dev->pci_dev, 0), vp_dev);
259 vp_dev->msix_used_vectors = 0; 141 pci_free_irq_vectors(vp_dev->pci_dev);
260 kfree(vp_dev->msix_names);
261 vp_dev->msix_names = NULL;
262 kfree(vp_dev->msix_affinity_masks);
263 vp_dev->msix_affinity_masks = NULL;
264 kfree(vp_dev->vqs);
265 vp_dev->vqs = NULL;
266} 142}
267 143
268static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs, 144static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs,
269 struct virtqueue *vqs[], 145 struct virtqueue *vqs[], vq_callback_t *callbacks[],
270 vq_callback_t *callbacks[], 146 const char * const names[], struct irq_affinity *desc)
271 const char * const names[],
272 bool per_vq_vectors)
273{ 147{
274 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 148 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
149 const char *name = dev_name(&vp_dev->vdev.dev);
150 int i, err = -ENOMEM, allocated_vectors, nvectors;
151 unsigned flags = PCI_IRQ_MSIX;
152 bool shared = false;
275 u16 msix_vec; 153 u16 msix_vec;
276 int i, err, nvectors, allocated_vectors;
277 154
278 vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL); 155 if (desc) {
279 if (!vp_dev->vqs) 156 flags |= PCI_IRQ_AFFINITY;
280 return -ENOMEM; 157 desc->pre_vectors++; /* virtio config vector */
158 }
281 159
282 if (per_vq_vectors) { 160 nvectors = 1;
283 /* Best option: one for change interrupt, one per vq. */ 161 for (i = 0; i < nvqs; i++)
284 nvectors = 1; 162 if (callbacks[i])
285 for (i = 0; i < nvqs; ++i) 163 nvectors++;
286 if (callbacks[i]) 164
287 ++nvectors; 165 /* Try one vector per queue first. */
288 } else { 166 err = pci_alloc_irq_vectors_affinity(vp_dev->pci_dev, nvectors,
289 /* Second best: one for change, shared for all vqs. */ 167 nvectors, flags, desc);
290 nvectors = 2; 168 if (err < 0) {
169 /* Fallback to one vector for config, one shared for queues. */
170 shared = true;
171 err = pci_alloc_irq_vectors(vp_dev->pci_dev, 2, 2,
172 PCI_IRQ_MSIX);
173 if (err < 0)
174 return err;
175 }
176 if (err < 0)
177 return err;
178
179 vp_dev->msix_vectors = nvectors;
180 vp_dev->msix_names = kmalloc_array(nvectors,
181 sizeof(*vp_dev->msix_names), GFP_KERNEL);
182 if (!vp_dev->msix_names)
183 goto out_free_irq_vectors;
184
185 vp_dev->msix_affinity_masks = kcalloc(nvectors,
186 sizeof(*vp_dev->msix_affinity_masks), GFP_KERNEL);
187 if (!vp_dev->msix_affinity_masks)
188 goto out_free_msix_names;
189
190 for (i = 0; i < nvectors; ++i) {
191 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
192 GFP_KERNEL))
193 goto out_free_msix_affinity_masks;
291 } 194 }
292 195
293 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors); 196 /* Set the vector used for configuration */
197 snprintf(vp_dev->msix_names[0], sizeof(*vp_dev->msix_names),
198 "%s-config", name);
199 err = request_irq(pci_irq_vector(vp_dev->pci_dev, 0), vp_config_changed,
200 0, vp_dev->msix_names[0], vp_dev);
294 if (err) 201 if (err)
295 goto error_find; 202 goto out_free_msix_affinity_masks;
203
204 /* Verify we had enough resources to assign the vector */
205 if (vp_dev->config_vector(vp_dev, 0) == VIRTIO_MSI_NO_VECTOR) {
206 err = -EBUSY;
207 goto out_free_config_irq;
208 }
209
210 vp_dev->msix_vector_map = kmalloc_array(nvqs,
211 sizeof(*vp_dev->msix_vector_map), GFP_KERNEL);
212 if (!vp_dev->msix_vector_map)
213 goto out_disable_config_irq;
296 214
297 vp_dev->per_vq_vectors = per_vq_vectors; 215 allocated_vectors = 1; /* vector 0 is the config interrupt */
298 allocated_vectors = vp_dev->msix_used_vectors;
299 for (i = 0; i < nvqs; ++i) { 216 for (i = 0; i < nvqs; ++i) {
300 if (!names[i]) { 217 if (!names[i]) {
301 vqs[i] = NULL; 218 vqs[i] = NULL;
302 continue; 219 continue;
303 } 220 }
304 221
305 if (!callbacks[i]) 222 if (callbacks[i])
306 msix_vec = VIRTIO_MSI_NO_VECTOR; 223 msix_vec = allocated_vectors;
307 else if (vp_dev->per_vq_vectors)
308 msix_vec = allocated_vectors++;
309 else 224 else
310 msix_vec = VP_MSIX_VQ_VECTOR; 225 msix_vec = VIRTIO_MSI_NO_VECTOR;
311 vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], msix_vec); 226
227 vqs[i] = vp_dev->setup_vq(vp_dev, i, callbacks[i], names[i],
228 msix_vec);
312 if (IS_ERR(vqs[i])) { 229 if (IS_ERR(vqs[i])) {
313 err = PTR_ERR(vqs[i]); 230 err = PTR_ERR(vqs[i]);
314 goto error_find; 231 goto out_remove_vqs;
315 } 232 }
316 233
317 if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR) 234 if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
235 vp_dev->msix_vector_map[i] = VIRTIO_MSI_NO_VECTOR;
318 continue; 236 continue;
237 }
319 238
320 /* allocate per-vq irq if available and necessary */ 239 snprintf(vp_dev->msix_names[i + 1],
321 snprintf(vp_dev->msix_names[msix_vec], 240 sizeof(*vp_dev->msix_names), "%s-%s",
322 sizeof *vp_dev->msix_names,
323 "%s-%s",
324 dev_name(&vp_dev->vdev.dev), names[i]); 241 dev_name(&vp_dev->vdev.dev), names[i]);
325 err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec), 242 err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
326 vring_interrupt, 0, 243 vring_interrupt, IRQF_SHARED,
327 vp_dev->msix_names[msix_vec], 244 vp_dev->msix_names[i + 1], vqs[i]);
328 vqs[i]); 245 if (err) {
329 if (err) 246 /* don't free this irq on error */
330 goto error_find; 247 vp_dev->msix_vector_map[i] = VIRTIO_MSI_NO_VECTOR;
248 goto out_remove_vqs;
249 }
250 vp_dev->msix_vector_map[i] = msix_vec;
251
252 /*
253 * Use a different vector for each queue if they are available,
254 * else share the same vector for all VQs.
255 */
256 if (!shared)
257 allocated_vectors++;
331 } 258 }
259
332 return 0; 260 return 0;
333 261
334error_find: 262out_remove_vqs:
335 vp_del_vqs(vdev); 263 vp_remove_vqs(vdev);
264 kfree(vp_dev->msix_vector_map);
265out_disable_config_irq:
266 vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
267out_free_config_irq:
268 free_irq(pci_irq_vector(vp_dev->pci_dev, 0), vp_dev);
269out_free_msix_affinity_masks:
270 for (i = 0; i < nvectors; i++) {
271 if (vp_dev->msix_affinity_masks[i])
272 free_cpumask_var(vp_dev->msix_affinity_masks[i]);
273 }
274 kfree(vp_dev->msix_affinity_masks);
275out_free_msix_names:
276 kfree(vp_dev->msix_names);
277out_free_irq_vectors:
278 pci_free_irq_vectors(vp_dev->pci_dev);
336 return err; 279 return err;
337} 280}
338 281
@@ -343,53 +286,42 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned nvqs,
343 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 286 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
344 int i, err; 287 int i, err;
345 288
346 vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
347 if (!vp_dev->vqs)
348 return -ENOMEM;
349
350 err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED, 289 err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED,
351 dev_name(&vdev->dev), vp_dev); 290 dev_name(&vdev->dev), vp_dev);
352 if (err) 291 if (err)
353 goto out_del_vqs; 292 return err;
354 293
355 vp_dev->intx_enabled = 1;
356 vp_dev->per_vq_vectors = false;
357 for (i = 0; i < nvqs; ++i) { 294 for (i = 0; i < nvqs; ++i) {
358 if (!names[i]) { 295 if (!names[i]) {
359 vqs[i] = NULL; 296 vqs[i] = NULL;
360 continue; 297 continue;
361 } 298 }
362 vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], 299 vqs[i] = vp_dev->setup_vq(vp_dev, i, callbacks[i], names[i],
363 VIRTIO_MSI_NO_VECTOR); 300 VIRTIO_MSI_NO_VECTOR);
364 if (IS_ERR(vqs[i])) { 301 if (IS_ERR(vqs[i])) {
365 err = PTR_ERR(vqs[i]); 302 err = PTR_ERR(vqs[i]);
366 goto out_del_vqs; 303 goto out_remove_vqs;
367 } 304 }
368 } 305 }
369 306
370 return 0; 307 return 0;
371out_del_vqs: 308
372 vp_del_vqs(vdev); 309out_remove_vqs:
310 vp_remove_vqs(vdev);
311 free_irq(pci_irq_vector(vp_dev->pci_dev, 0), vp_dev);
373 return err; 312 return err;
374} 313}
375 314
376/* the config->find_vqs() implementation */ 315/* the config->find_vqs() implementation */
377int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs, 316int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
378 struct virtqueue *vqs[], 317 struct virtqueue *vqs[], vq_callback_t *callbacks[],
379 vq_callback_t *callbacks[], 318 const char * const names[], struct irq_affinity *desc)
380 const char * const names[])
381{ 319{
382 int err; 320 int err;
383 321
384 /* Try MSI-X with one vector per queue. */ 322 err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, desc);
385 err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true);
386 if (!err) 323 if (!err)
387 return 0; 324 return 0;
388 /* Fallback: MSI-X with one vector for config, one shared for queues. */
389 err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false);
390 if (!err)
391 return 0;
392 /* Finally fall back to regular interrupts. */
393 return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names); 325 return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names);
394} 326}
395 327
@@ -409,16 +341,15 @@ int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
409{ 341{
410 struct virtio_device *vdev = vq->vdev; 342 struct virtio_device *vdev = vq->vdev;
411 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 343 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
412 struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
413 struct cpumask *mask;
414 unsigned int irq;
415 344
416 if (!vq->callback) 345 if (!vq->callback)
417 return -EINVAL; 346 return -EINVAL;
418 347
419 if (vp_dev->msix_enabled) { 348 if (vp_dev->pci_dev->msix_enabled) {
420 mask = vp_dev->msix_affinity_masks[info->msix_vector]; 349 int vec = vp_dev->msix_vector_map[vq->index];
421 irq = pci_irq_vector(vp_dev->pci_dev, info->msix_vector); 350 struct cpumask *mask = vp_dev->msix_affinity_masks[vec];
351 unsigned int irq = pci_irq_vector(vp_dev->pci_dev, vec);
352
422 if (cpu == -1) 353 if (cpu == -1)
423 irq_set_affinity_hint(irq, NULL); 354 irq_set_affinity_hint(irq, NULL);
424 else { 355 else {
@@ -430,6 +361,17 @@ int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
430 return 0; 361 return 0;
431} 362}
432 363
364const struct cpumask *vp_get_vq_affinity(struct virtio_device *vdev, int index)
365{
366 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
367 unsigned int *map = vp_dev->msix_vector_map;
368
369 if (!map || map[index] == VIRTIO_MSI_NO_VECTOR)
370 return NULL;
371
372 return pci_irq_get_affinity(vp_dev->pci_dev, map[index]);
373}
374
433#ifdef CONFIG_PM_SLEEP 375#ifdef CONFIG_PM_SLEEP
434static int virtio_pci_freeze(struct device *dev) 376static int virtio_pci_freeze(struct device *dev)
435{ 377{
@@ -498,8 +440,6 @@ static int virtio_pci_probe(struct pci_dev *pci_dev,
498 vp_dev->vdev.dev.parent = &pci_dev->dev; 440 vp_dev->vdev.dev.parent = &pci_dev->dev;
499 vp_dev->vdev.dev.release = virtio_pci_release_dev; 441 vp_dev->vdev.dev.release = virtio_pci_release_dev;
500 vp_dev->pci_dev = pci_dev; 442 vp_dev->pci_dev = pci_dev;
501 INIT_LIST_HEAD(&vp_dev->virtqueues);
502 spin_lock_init(&vp_dev->lock);
503 443
504 /* enable the device */ 444 /* enable the device */
505 rc = pci_enable_device(pci_dev); 445 rc = pci_enable_device(pci_dev);
diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h
index b2f666250ae0..ac8c9d788964 100644
--- a/drivers/virtio/virtio_pci_common.h
+++ b/drivers/virtio/virtio_pci_common.h
@@ -31,17 +31,6 @@
31#include <linux/highmem.h> 31#include <linux/highmem.h>
32#include <linux/spinlock.h> 32#include <linux/spinlock.h>
33 33
34struct virtio_pci_vq_info {
35 /* the actual virtqueue */
36 struct virtqueue *vq;
37
38 /* the list node for the virtqueues list */
39 struct list_head node;
40
41 /* MSI-X vector (or none) */
42 unsigned msix_vector;
43};
44
45/* Our device structure */ 34/* Our device structure */
46struct virtio_pci_device { 35struct virtio_pci_device {
47 struct virtio_device vdev; 36 struct virtio_device vdev;
@@ -75,47 +64,25 @@ struct virtio_pci_device {
75 /* the IO mapping for the PCI config space */ 64 /* the IO mapping for the PCI config space */
76 void __iomem *ioaddr; 65 void __iomem *ioaddr;
77 66
78 /* a list of queues so we can dispatch IRQs */
79 spinlock_t lock;
80 struct list_head virtqueues;
81
82 /* array of all queues for house-keeping */
83 struct virtio_pci_vq_info **vqs;
84
85 /* MSI-X support */
86 int msix_enabled;
87 int intx_enabled;
88 cpumask_var_t *msix_affinity_masks; 67 cpumask_var_t *msix_affinity_masks;
89 /* Name strings for interrupts. This size should be enough, 68 /* Name strings for interrupts. This size should be enough,
90 * and I'm too lazy to allocate each name separately. */ 69 * and I'm too lazy to allocate each name separately. */
91 char (*msix_names)[256]; 70 char (*msix_names)[256];
92 /* Number of available vectors */ 71 /* Total Number of MSI-X vectors (including per-VQ ones). */
93 unsigned msix_vectors; 72 int msix_vectors;
94 /* Vectors allocated, excluding per-vq vectors if any */ 73 /* Map of per-VQ MSI-X vectors, may be NULL */
95 unsigned msix_used_vectors; 74 unsigned *msix_vector_map;
96
97 /* Whether we have vector per vq */
98 bool per_vq_vectors;
99 75
100 struct virtqueue *(*setup_vq)(struct virtio_pci_device *vp_dev, 76 struct virtqueue *(*setup_vq)(struct virtio_pci_device *vp_dev,
101 struct virtio_pci_vq_info *info,
102 unsigned idx, 77 unsigned idx,
103 void (*callback)(struct virtqueue *vq), 78 void (*callback)(struct virtqueue *vq),
104 const char *name, 79 const char *name,
105 u16 msix_vec); 80 u16 msix_vec);
106 void (*del_vq)(struct virtio_pci_vq_info *info); 81 void (*del_vq)(struct virtqueue *vq);
107 82
108 u16 (*config_vector)(struct virtio_pci_device *vp_dev, u16 vector); 83 u16 (*config_vector)(struct virtio_pci_device *vp_dev, u16 vector);
109}; 84};
110 85
111/* Constants for MSI-X */
112/* Use first vector for configuration changes, second and the rest for
113 * virtqueues Thus, we need at least 2 vectors for MSI. */
114enum {
115 VP_MSIX_CONFIG_VECTOR = 0,
116 VP_MSIX_VQ_VECTOR = 1,
117};
118
119/* Convert a generic virtio device to our structure */ 86/* Convert a generic virtio device to our structure */
120static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev) 87static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
121{ 88{
@@ -130,9 +97,8 @@ bool vp_notify(struct virtqueue *vq);
130void vp_del_vqs(struct virtio_device *vdev); 97void vp_del_vqs(struct virtio_device *vdev);
131/* the config->find_vqs() implementation */ 98/* the config->find_vqs() implementation */
132int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs, 99int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
133 struct virtqueue *vqs[], 100 struct virtqueue *vqs[], vq_callback_t *callbacks[],
134 vq_callback_t *callbacks[], 101 const char * const names[], struct irq_affinity *desc);
135 const char * const names[]);
136const char *vp_bus_name(struct virtio_device *vdev); 102const char *vp_bus_name(struct virtio_device *vdev);
137 103
138/* Setup the affinity for a virtqueue: 104/* Setup the affinity for a virtqueue:
@@ -142,6 +108,8 @@ const char *vp_bus_name(struct virtio_device *vdev);
142 */ 108 */
143int vp_set_vq_affinity(struct virtqueue *vq, int cpu); 109int vp_set_vq_affinity(struct virtqueue *vq, int cpu);
144 110
111const struct cpumask *vp_get_vq_affinity(struct virtio_device *vdev, int index);
112
145#if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY) 113#if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY)
146int virtio_pci_legacy_probe(struct virtio_pci_device *); 114int virtio_pci_legacy_probe(struct virtio_pci_device *);
147void virtio_pci_legacy_remove(struct virtio_pci_device *); 115void virtio_pci_legacy_remove(struct virtio_pci_device *);
diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
index 6d9e5173d5fa..f7362c5fe18a 100644
--- a/drivers/virtio/virtio_pci_legacy.c
+++ b/drivers/virtio/virtio_pci_legacy.c
@@ -112,7 +112,6 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
112} 112}
113 113
114static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, 114static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
115 struct virtio_pci_vq_info *info,
116 unsigned index, 115 unsigned index,
117 void (*callback)(struct virtqueue *vq), 116 void (*callback)(struct virtqueue *vq),
118 const char *name, 117 const char *name,
@@ -130,8 +129,6 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
130 if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN)) 129 if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
131 return ERR_PTR(-ENOENT); 130 return ERR_PTR(-ENOENT);
132 131
133 info->msix_vector = msix_vec;
134
135 /* create the vring */ 132 /* create the vring */
136 vq = vring_create_virtqueue(index, num, 133 vq = vring_create_virtqueue(index, num,
137 VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev, 134 VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev,
@@ -162,14 +159,13 @@ out_deactivate:
162 return ERR_PTR(err); 159 return ERR_PTR(err);
163} 160}
164 161
165static void del_vq(struct virtio_pci_vq_info *info) 162static void del_vq(struct virtqueue *vq)
166{ 163{
167 struct virtqueue *vq = info->vq;
168 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 164 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
169 165
170 iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL); 166 iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
171 167
172 if (vp_dev->msix_enabled) { 168 if (vp_dev->pci_dev->msix_enabled) {
173 iowrite16(VIRTIO_MSI_NO_VECTOR, 169 iowrite16(VIRTIO_MSI_NO_VECTOR,
174 vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR); 170 vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
175 /* Flush the write out to device */ 171 /* Flush the write out to device */
@@ -194,6 +190,7 @@ static const struct virtio_config_ops virtio_pci_config_ops = {
194 .finalize_features = vp_finalize_features, 190 .finalize_features = vp_finalize_features,
195 .bus_name = vp_bus_name, 191 .bus_name = vp_bus_name,
196 .set_vq_affinity = vp_set_vq_affinity, 192 .set_vq_affinity = vp_set_vq_affinity,
193 .get_vq_affinity = vp_get_vq_affinity,
197}; 194};
198 195
199/* the PCI probing function */ 196/* the PCI probing function */
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index 4bf7ab375894..7bc3004b840e 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -293,7 +293,6 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
293} 293}
294 294
295static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, 295static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
296 struct virtio_pci_vq_info *info,
297 unsigned index, 296 unsigned index,
298 void (*callback)(struct virtqueue *vq), 297 void (*callback)(struct virtqueue *vq),
299 const char *name, 298 const char *name,
@@ -323,8 +322,6 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
323 /* get offset of notification word for this vq */ 322 /* get offset of notification word for this vq */
324 off = vp_ioread16(&cfg->queue_notify_off); 323 off = vp_ioread16(&cfg->queue_notify_off);
325 324
326 info->msix_vector = msix_vec;
327
328 /* create the vring */ 325 /* create the vring */
329 vq = vring_create_virtqueue(index, num, 326 vq = vring_create_virtqueue(index, num,
330 SMP_CACHE_BYTES, &vp_dev->vdev, 327 SMP_CACHE_BYTES, &vp_dev->vdev,
@@ -387,13 +384,12 @@ err_map_notify:
387} 384}
388 385
389static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs, 386static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs,
390 struct virtqueue *vqs[], 387 struct virtqueue *vqs[], vq_callback_t *callbacks[],
391 vq_callback_t *callbacks[], 388 const char * const names[], struct irq_affinity *desc)
392 const char * const names[])
393{ 389{
394 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 390 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
395 struct virtqueue *vq; 391 struct virtqueue *vq;
396 int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names); 392 int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, desc);
397 393
398 if (rc) 394 if (rc)
399 return rc; 395 return rc;
@@ -409,14 +405,13 @@ static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs,
409 return 0; 405 return 0;
410} 406}
411 407
412static void del_vq(struct virtio_pci_vq_info *info) 408static void del_vq(struct virtqueue *vq)
413{ 409{
414 struct virtqueue *vq = info->vq;
415 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 410 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
416 411
417 vp_iowrite16(vq->index, &vp_dev->common->queue_select); 412 vp_iowrite16(vq->index, &vp_dev->common->queue_select);
418 413
419 if (vp_dev->msix_enabled) { 414 if (vp_dev->pci_dev->msix_enabled) {
420 vp_iowrite16(VIRTIO_MSI_NO_VECTOR, 415 vp_iowrite16(VIRTIO_MSI_NO_VECTOR,
421 &vp_dev->common->queue_msix_vector); 416 &vp_dev->common->queue_msix_vector);
422 /* Flush the write out to device */ 417 /* Flush the write out to device */
@@ -442,6 +437,7 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
442 .finalize_features = vp_finalize_features, 437 .finalize_features = vp_finalize_features,
443 .bus_name = vp_bus_name, 438 .bus_name = vp_bus_name,
444 .set_vq_affinity = vp_set_vq_affinity, 439 .set_vq_affinity = vp_set_vq_affinity,
440 .get_vq_affinity = vp_get_vq_affinity,
445}; 441};
446 442
447static const struct virtio_config_ops virtio_pci_config_ops = { 443static const struct virtio_config_ops virtio_pci_config_ops = {
@@ -457,6 +453,7 @@ static const struct virtio_config_ops virtio_pci_config_ops = {
457 .finalize_features = vp_finalize_features, 453 .finalize_features = vp_finalize_features,
458 .bus_name = vp_bus_name, 454 .bus_name = vp_bus_name,
459 .set_vq_affinity = vp_set_vq_affinity, 455 .set_vq_affinity = vp_set_vq_affinity,
456 .get_vq_affinity = vp_get_vq_affinity,
460}; 457};
461 458
462/** 459/**
diff --git a/include/linux/blk-mq-virtio.h b/include/linux/blk-mq-virtio.h
new file mode 100644
index 000000000000..b1ef6e14744f
--- /dev/null
+++ b/include/linux/blk-mq-virtio.h
@@ -0,0 +1,10 @@
1#ifndef _LINUX_BLK_MQ_VIRTIO_H
2#define _LINUX_BLK_MQ_VIRTIO_H
3
4struct blk_mq_tag_set;
5struct virtio_device;
6
7int blk_mq_virtio_map_queues(struct blk_mq_tag_set *set,
8 struct virtio_device *vdev, int first_vec);
9
10#endif /* _LINUX_BLK_MQ_VIRTIO_H */
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index bb790c4db0c5..62d240e962f0 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -26,7 +26,6 @@ enum cpuhp_state {
26 CPUHP_ARM_OMAP_WAKE_DEAD, 26 CPUHP_ARM_OMAP_WAKE_DEAD,
27 CPUHP_IRQ_POLL_DEAD, 27 CPUHP_IRQ_POLL_DEAD,
28 CPUHP_BLOCK_SOFTIRQ_DEAD, 28 CPUHP_BLOCK_SOFTIRQ_DEAD,
29 CPUHP_VIRT_SCSI_DEAD,
30 CPUHP_ACPI_CPUDRV_DEAD, 29 CPUHP_ACPI_CPUDRV_DEAD,
31 CPUHP_S390_PFAULT_DEAD, 30 CPUHP_S390_PFAULT_DEAD,
32 CPUHP_BLK_MQ_DEAD, 31 CPUHP_BLK_MQ_DEAD,
diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
index 26c155bb639b..8355bab175e1 100644
--- a/include/linux/virtio_config.h
+++ b/include/linux/virtio_config.h
@@ -7,6 +7,8 @@
7#include <linux/virtio_byteorder.h> 7#include <linux/virtio_byteorder.h>
8#include <uapi/linux/virtio_config.h> 8#include <uapi/linux/virtio_config.h>
9 9
10struct irq_affinity;
11
10/** 12/**
11 * virtio_config_ops - operations for configuring a virtio device 13 * virtio_config_ops - operations for configuring a virtio device
12 * @get: read the value of a configuration field 14 * @get: read the value of a configuration field
@@ -56,6 +58,7 @@
56 * This returns a pointer to the bus name a la pci_name from which 58 * This returns a pointer to the bus name a la pci_name from which
57 * the caller can then copy. 59 * the caller can then copy.
58 * @set_vq_affinity: set the affinity for a virtqueue. 60 * @set_vq_affinity: set the affinity for a virtqueue.
61 * @get_vq_affinity: get the affinity for a virtqueue (optional).
59 */ 62 */
60typedef void vq_callback_t(struct virtqueue *); 63typedef void vq_callback_t(struct virtqueue *);
61struct virtio_config_ops { 64struct virtio_config_ops {
@@ -68,14 +71,15 @@ struct virtio_config_ops {
68 void (*set_status)(struct virtio_device *vdev, u8 status); 71 void (*set_status)(struct virtio_device *vdev, u8 status);
69 void (*reset)(struct virtio_device *vdev); 72 void (*reset)(struct virtio_device *vdev);
70 int (*find_vqs)(struct virtio_device *, unsigned nvqs, 73 int (*find_vqs)(struct virtio_device *, unsigned nvqs,
71 struct virtqueue *vqs[], 74 struct virtqueue *vqs[], vq_callback_t *callbacks[],
72 vq_callback_t *callbacks[], 75 const char * const names[], struct irq_affinity *desc);
73 const char * const names[]);
74 void (*del_vqs)(struct virtio_device *); 76 void (*del_vqs)(struct virtio_device *);
75 u64 (*get_features)(struct virtio_device *vdev); 77 u64 (*get_features)(struct virtio_device *vdev);
76 int (*finalize_features)(struct virtio_device *vdev); 78 int (*finalize_features)(struct virtio_device *vdev);
77 const char *(*bus_name)(struct virtio_device *vdev); 79 const char *(*bus_name)(struct virtio_device *vdev);
78 int (*set_vq_affinity)(struct virtqueue *vq, int cpu); 80 int (*set_vq_affinity)(struct virtqueue *vq, int cpu);
81 const struct cpumask *(*get_vq_affinity)(struct virtio_device *vdev,
82 int index);
79}; 83};
80 84
81/* If driver didn't advertise the feature, it will never appear. */ 85/* If driver didn't advertise the feature, it will never appear. */
@@ -169,7 +173,7 @@ struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev,
169 vq_callback_t *callbacks[] = { c }; 173 vq_callback_t *callbacks[] = { c };
170 const char *names[] = { n }; 174 const char *names[] = { n };
171 struct virtqueue *vq; 175 struct virtqueue *vq;
172 int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names); 176 int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL);
173 if (err < 0) 177 if (err < 0)
174 return ERR_PTR(err); 178 return ERR_PTR(err);
175 return vq; 179 return vq;
diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild
index 1c80efb67d10..dd9820b1c779 100644
--- a/include/uapi/linux/Kbuild
+++ b/include/uapi/linux/Kbuild
@@ -466,6 +466,7 @@ header-y += virtio_console.h
466header-y += virtio_gpu.h 466header-y += virtio_gpu.h
467header-y += virtio_ids.h 467header-y += virtio_ids.h
468header-y += virtio_input.h 468header-y += virtio_input.h
469header-y += virtio_mmio.h
469header-y += virtio_net.h 470header-y += virtio_net.h
470header-y += virtio_pci.h 471header-y += virtio_pci.h
471header-y += virtio_ring.h 472header-y += virtio_ring.h
diff --git a/include/linux/virtio_mmio.h b/include/uapi/linux/virtio_mmio.h
index c4b09689ab64..c4b09689ab64 100644
--- a/include/linux/virtio_mmio.h
+++ b/include/uapi/linux/virtio_mmio.h
diff --git a/include/uapi/linux/virtio_pci.h b/include/uapi/linux/virtio_pci.h
index 90007a1abcab..15b4385a2be1 100644
--- a/include/uapi/linux/virtio_pci.h
+++ b/include/uapi/linux/virtio_pci.h
@@ -79,7 +79,7 @@
79 * configuration space */ 79 * configuration space */
80#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20) 80#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20)
81/* Deprecated: please use VIRTIO_PCI_CONFIG_OFF instead */ 81/* Deprecated: please use VIRTIO_PCI_CONFIG_OFF instead */
82#define VIRTIO_PCI_CONFIG(dev) VIRTIO_PCI_CONFIG_OFF((dev)->msix_enabled) 82#define VIRTIO_PCI_CONFIG(dev) VIRTIO_PCI_CONFIG_OFF((dev)->pci_dev->msix_enabled)
83 83
84/* Virtio ABI version, this must match exactly */ 84/* Virtio ABI version, this must match exactly */
85#define VIRTIO_PCI_ABI_VERSION 0 85#define VIRTIO_PCI_ABI_VERSION 0
diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c
index 6788264acc63..9d24c0e958b1 100644
--- a/net/vmw_vsock/virtio_transport.c
+++ b/net/vmw_vsock/virtio_transport.c
@@ -532,7 +532,8 @@ static int virtio_vsock_probe(struct virtio_device *vdev)
532 vsock->vdev = vdev; 532 vsock->vdev = vdev;
533 533
534 ret = vsock->vdev->config->find_vqs(vsock->vdev, VSOCK_VQ_MAX, 534 ret = vsock->vdev->config->find_vqs(vsock->vdev, VSOCK_VQ_MAX,
535 vsock->vqs, callbacks, names); 535 vsock->vqs, callbacks, names,
536 NULL);
536 if (ret < 0) 537 if (ret < 0)
537 goto out; 538 goto out;
538 539
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-25 20:31:03 -0400