aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/block/virtio_blk.c91
-rw-r--r--drivers/char/virtio_console.c5
-rw-r--r--drivers/net/virtio_net.c2
-rw-r--r--drivers/vhost/net.c12
-rw-r--r--drivers/vhost/test.c6
-rw-r--r--drivers/vhost/vhost.c138
-rw-r--r--drivers/vhost/vhost.h21
-rw-r--r--drivers/virtio/virtio_balloon.c21
-rw-r--r--drivers/virtio/virtio_ring.c53
9 files changed, 267 insertions, 82 deletions
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 6ecf89cdf006..079c08808d8a 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -6,10 +6,13 @@
6#include <linux/virtio.h> 6#include <linux/virtio.h>
7#include <linux/virtio_blk.h> 7#include <linux/virtio_blk.h>
8#include <linux/scatterlist.h> 8#include <linux/scatterlist.h>
9#include <linux/string_helpers.h>
10#include <scsi/scsi_cmnd.h>
9 11
10#define PART_BITS 4 12#define PART_BITS 4
11 13
12static int major, index; 14static int major, index;
15struct workqueue_struct *virtblk_wq;
13 16
14struct virtio_blk 17struct virtio_blk
15{ 18{
@@ -26,6 +29,9 @@ struct virtio_blk
26 29
27 mempool_t *pool; 30 mempool_t *pool;
28 31
32 /* Process context for config space updates */
33 struct work_struct config_work;
34
29 /* What host tells us, plus 2 for header & tailer. */ 35 /* What host tells us, plus 2 for header & tailer. */
30 unsigned int sg_elems; 36 unsigned int sg_elems;
31 37
@@ -141,7 +147,7 @@ static bool do_req(struct request_queue *q, struct virtio_blk *vblk,
141 num = blk_rq_map_sg(q, vbr->req, vblk->sg + out); 147 num = blk_rq_map_sg(q, vbr->req, vblk->sg + out);
142 148
143 if (vbr->req->cmd_type == REQ_TYPE_BLOCK_PC) { 149 if (vbr->req->cmd_type == REQ_TYPE_BLOCK_PC) {
144 sg_set_buf(&vblk->sg[num + out + in++], vbr->req->sense, 96); 150 sg_set_buf(&vblk->sg[num + out + in++], vbr->req->sense, SCSI_SENSE_BUFFERSIZE);
145 sg_set_buf(&vblk->sg[num + out + in++], &vbr->in_hdr, 151 sg_set_buf(&vblk->sg[num + out + in++], &vbr->in_hdr,
146 sizeof(vbr->in_hdr)); 152 sizeof(vbr->in_hdr));
147 } 153 }
@@ -291,6 +297,46 @@ static ssize_t virtblk_serial_show(struct device *dev,
291} 297}
292DEVICE_ATTR(serial, S_IRUGO, virtblk_serial_show, NULL); 298DEVICE_ATTR(serial, S_IRUGO, virtblk_serial_show, NULL);
293 299
300static void virtblk_config_changed_work(struct work_struct *work)
301{
302 struct virtio_blk *vblk =
303 container_of(work, struct virtio_blk, config_work);
304 struct virtio_device *vdev = vblk->vdev;
305 struct request_queue *q = vblk->disk->queue;
306 char cap_str_2[10], cap_str_10[10];
307 u64 capacity, size;
308
309 /* Host must always specify the capacity. */
310 vdev->config->get(vdev, offsetof(struct virtio_blk_config, capacity),
311 &capacity, sizeof(capacity));
312
313 /* If capacity is too big, truncate with warning. */
314 if ((sector_t)capacity != capacity) {
315 dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n",
316 (unsigned long long)capacity);
317 capacity = (sector_t)-1;
318 }
319
320 size = capacity * queue_logical_block_size(q);
321 string_get_size(size, STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
322 string_get_size(size, STRING_UNITS_10, cap_str_10, sizeof(cap_str_10));
323
324 dev_notice(&vdev->dev,
325 "new size: %llu %d-byte logical blocks (%s/%s)\n",
326 (unsigned long long)capacity,
327 queue_logical_block_size(q),
328 cap_str_10, cap_str_2);
329
330 set_capacity(vblk->disk, capacity);
331}
332
333static void virtblk_config_changed(struct virtio_device *vdev)
334{
335 struct virtio_blk *vblk = vdev->priv;
336
337 queue_work(virtblk_wq, &vblk->config_work);
338}
339
294static int __devinit virtblk_probe(struct virtio_device *vdev) 340static int __devinit virtblk_probe(struct virtio_device *vdev)
295{ 341{
296 struct virtio_blk *vblk; 342 struct virtio_blk *vblk;
@@ -327,6 +373,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev)
327 vblk->vdev = vdev; 373 vblk->vdev = vdev;
328 vblk->sg_elems = sg_elems; 374 vblk->sg_elems = sg_elems;
329 sg_init_table(vblk->sg, vblk->sg_elems); 375 sg_init_table(vblk->sg, vblk->sg_elems);
376 INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
330 377
331 /* We expect one virtqueue, for output. */ 378 /* We expect one virtqueue, for output. */
332 vblk->vq = virtio_find_single_vq(vdev, blk_done, "requests"); 379 vblk->vq = virtio_find_single_vq(vdev, blk_done, "requests");
@@ -477,6 +524,8 @@ static void __devexit virtblk_remove(struct virtio_device *vdev)
477{ 524{
478 struct virtio_blk *vblk = vdev->priv; 525 struct virtio_blk *vblk = vdev->priv;
479 526
527 flush_work(&vblk->config_work);
528
480 /* Nothing should be pending. */ 529 /* Nothing should be pending. */
481 BUG_ON(!list_empty(&vblk->reqs)); 530 BUG_ON(!list_empty(&vblk->reqs));
482 531
@@ -508,27 +557,47 @@ static unsigned int features[] = {
508 * Use __refdata to avoid this warning. 557 * Use __refdata to avoid this warning.
509 */ 558 */
510static struct virtio_driver __refdata virtio_blk = { 559static struct virtio_driver __refdata virtio_blk = {
511 .feature_table = features, 560 .feature_table = features,
512 .feature_table_size = ARRAY_SIZE(features), 561 .feature_table_size = ARRAY_SIZE(features),
513 .driver.name = KBUILD_MODNAME, 562 .driver.name = KBUILD_MODNAME,
514 .driver.owner = THIS_MODULE, 563 .driver.owner = THIS_MODULE,
515 .id_table = id_table, 564 .id_table = id_table,
516 .probe = virtblk_probe, 565 .probe = virtblk_probe,
517 .remove = __devexit_p(virtblk_remove), 566 .remove = __devexit_p(virtblk_remove),
567 .config_changed = virtblk_config_changed,
518}; 568};
519 569
520static int __init init(void) 570static int __init init(void)
521{ 571{
572 int error;
573
574 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0);
575 if (!virtblk_wq)
576 return -ENOMEM;
577
522 major = register_blkdev(0, "virtblk"); 578 major = register_blkdev(0, "virtblk");
523 if (major < 0) 579 if (major < 0) {
524 return major; 580 error = major;
525 return register_virtio_driver(&virtio_blk); 581 goto out_destroy_workqueue;
582 }
583
584 error = register_virtio_driver(&virtio_blk);
585 if (error)
586 goto out_unregister_blkdev;
587 return 0;
588
589out_unregister_blkdev:
590 unregister_blkdev(major, "virtblk");
591out_destroy_workqueue:
592 destroy_workqueue(virtblk_wq);
593 return error;
526} 594}
527 595
528static void __exit fini(void) 596static void __exit fini(void)
529{ 597{
530 unregister_blkdev(major, "virtblk"); 598 unregister_blkdev(major, "virtblk");
531 unregister_virtio_driver(&virtio_blk); 599 unregister_virtio_driver(&virtio_blk);
600 destroy_workqueue(virtblk_wq);
532} 601}
533module_init(init); 602module_init(init);
534module_exit(fini); 603module_exit(fini);
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index 838568a7dbf5..fb68b1295373 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -1677,17 +1677,12 @@ static int __devinit virtcons_probe(struct virtio_device *vdev)
1677 portdev->config.max_nr_ports = 1; 1677 portdev->config.max_nr_ports = 1;
1678 if (virtio_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT)) { 1678 if (virtio_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT)) {
1679 multiport = true; 1679 multiport = true;
1680 vdev->features[0] |= 1 << VIRTIO_CONSOLE_F_MULTIPORT;
1681
1682 vdev->config->get(vdev, offsetof(struct virtio_console_config, 1680 vdev->config->get(vdev, offsetof(struct virtio_console_config,
1683 max_nr_ports), 1681 max_nr_ports),
1684 &portdev->config.max_nr_ports, 1682 &portdev->config.max_nr_ports,
1685 sizeof(portdev->config.max_nr_ports)); 1683 sizeof(portdev->config.max_nr_ports));
1686 } 1684 }
1687 1685
1688 /* Let the Host know we support multiple ports.*/
1689 vdev->config->finalize_features(vdev);
1690
1691 err = init_vqs(portdev); 1686 err = init_vqs(portdev);
1692 if (err < 0) { 1687 if (err < 0) {
1693 dev_err(&vdev->dev, "Error %d initializing vqs\n", err); 1688 dev_err(&vdev->dev, "Error %d initializing vqs\n", err);
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 0cb0b0632672..f6853247a620 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -609,7 +609,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
609 * before it gets out of hand. Naturally, this wastes entries. */ 609 * before it gets out of hand. Naturally, this wastes entries. */
610 if (capacity < 2+MAX_SKB_FRAGS) { 610 if (capacity < 2+MAX_SKB_FRAGS) {
611 netif_stop_queue(dev); 611 netif_stop_queue(dev);
612 if (unlikely(!virtqueue_enable_cb(vi->svq))) { 612 if (unlikely(!virtqueue_enable_cb_delayed(vi->svq))) {
613 /* More just got used, free them then recheck. */ 613 /* More just got used, free them then recheck. */
614 capacity += free_old_xmit_skbs(vi); 614 capacity += free_old_xmit_skbs(vi);
615 if (capacity >= 2+MAX_SKB_FRAGS) { 615 if (capacity >= 2+MAX_SKB_FRAGS) {
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 2f7c76a85e53..e224a92baa16 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -144,7 +144,7 @@ static void handle_tx(struct vhost_net *net)
144 } 144 }
145 145
146 mutex_lock(&vq->mutex); 146 mutex_lock(&vq->mutex);
147 vhost_disable_notify(vq); 147 vhost_disable_notify(&net->dev, vq);
148 148
149 if (wmem < sock->sk->sk_sndbuf / 2) 149 if (wmem < sock->sk->sk_sndbuf / 2)
150 tx_poll_stop(net); 150 tx_poll_stop(net);
@@ -166,8 +166,8 @@ static void handle_tx(struct vhost_net *net)
166 set_bit(SOCK_ASYNC_NOSPACE, &sock->flags); 166 set_bit(SOCK_ASYNC_NOSPACE, &sock->flags);
167 break; 167 break;
168 } 168 }
169 if (unlikely(vhost_enable_notify(vq))) { 169 if (unlikely(vhost_enable_notify(&net->dev, vq))) {
170 vhost_disable_notify(vq); 170 vhost_disable_notify(&net->dev, vq);
171 continue; 171 continue;
172 } 172 }
173 break; 173 break;
@@ -315,7 +315,7 @@ static void handle_rx(struct vhost_net *net)
315 return; 315 return;
316 316
317 mutex_lock(&vq->mutex); 317 mutex_lock(&vq->mutex);
318 vhost_disable_notify(vq); 318 vhost_disable_notify(&net->dev, vq);
319 vhost_hlen = vq->vhost_hlen; 319 vhost_hlen = vq->vhost_hlen;
320 sock_hlen = vq->sock_hlen; 320 sock_hlen = vq->sock_hlen;
321 321
@@ -334,10 +334,10 @@ static void handle_rx(struct vhost_net *net)
334 break; 334 break;
335 /* OK, now we need to know about added descriptors. */ 335 /* OK, now we need to know about added descriptors. */
336 if (!headcount) { 336 if (!headcount) {
337 if (unlikely(vhost_enable_notify(vq))) { 337 if (unlikely(vhost_enable_notify(&net->dev, vq))) {
338 /* They have slipped one in as we were 338 /* They have slipped one in as we were
339 * doing that: check again. */ 339 * doing that: check again. */
340 vhost_disable_notify(vq); 340 vhost_disable_notify(&net->dev, vq);
341 continue; 341 continue;
342 } 342 }
343 /* Nothing new? Wait for eventfd to tell us 343 /* Nothing new? Wait for eventfd to tell us
diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c
index 099f30230d06..734e1d74ad80 100644
--- a/drivers/vhost/test.c
+++ b/drivers/vhost/test.c
@@ -49,7 +49,7 @@ static void handle_vq(struct vhost_test *n)
49 return; 49 return;
50 50
51 mutex_lock(&vq->mutex); 51 mutex_lock(&vq->mutex);
52 vhost_disable_notify(vq); 52 vhost_disable_notify(&n->dev, vq);
53 53
54 for (;;) { 54 for (;;) {
55 head = vhost_get_vq_desc(&n->dev, vq, vq->iov, 55 head = vhost_get_vq_desc(&n->dev, vq, vq->iov,
@@ -61,8 +61,8 @@ static void handle_vq(struct vhost_test *n)
61 break; 61 break;
62 /* Nothing new? Wait for eventfd to tell us they refilled. */ 62 /* Nothing new? Wait for eventfd to tell us they refilled. */
63 if (head == vq->num) { 63 if (head == vq->num) {
64 if (unlikely(vhost_enable_notify(vq))) { 64 if (unlikely(vhost_enable_notify(&n->dev, vq))) {
65 vhost_disable_notify(vq); 65 vhost_disable_notify(&n->dev, vq);
66 continue; 66 continue;
67 } 67 }
68 break; 68 break;
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 7aa4eea930f1..ea966b356352 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -37,6 +37,9 @@ enum {
37 VHOST_MEMORY_F_LOG = 0x1, 37 VHOST_MEMORY_F_LOG = 0x1,
38}; 38};
39 39
40#define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num])
41#define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num])
42
40static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, 43static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
41 poll_table *pt) 44 poll_table *pt)
42{ 45{
@@ -161,6 +164,8 @@ static void vhost_vq_reset(struct vhost_dev *dev,
161 vq->last_avail_idx = 0; 164 vq->last_avail_idx = 0;
162 vq->avail_idx = 0; 165 vq->avail_idx = 0;
163 vq->last_used_idx = 0; 166 vq->last_used_idx = 0;
167 vq->signalled_used = 0;
168 vq->signalled_used_valid = false;
164 vq->used_flags = 0; 169 vq->used_flags = 0;
165 vq->log_used = false; 170 vq->log_used = false;
166 vq->log_addr = -1ull; 171 vq->log_addr = -1ull;
@@ -489,16 +494,17 @@ static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem,
489 return 1; 494 return 1;
490} 495}
491 496
492static int vq_access_ok(unsigned int num, 497static int vq_access_ok(struct vhost_dev *d, unsigned int num,
493 struct vring_desc __user *desc, 498 struct vring_desc __user *desc,
494 struct vring_avail __user *avail, 499 struct vring_avail __user *avail,
495 struct vring_used __user *used) 500 struct vring_used __user *used)
496{ 501{
502 size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
497 return access_ok(VERIFY_READ, desc, num * sizeof *desc) && 503 return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
498 access_ok(VERIFY_READ, avail, 504 access_ok(VERIFY_READ, avail,
499 sizeof *avail + num * sizeof *avail->ring) && 505 sizeof *avail + num * sizeof *avail->ring + s) &&
500 access_ok(VERIFY_WRITE, used, 506 access_ok(VERIFY_WRITE, used,
501 sizeof *used + num * sizeof *used->ring); 507 sizeof *used + num * sizeof *used->ring + s);
502} 508}
503 509
504/* Can we log writes? */ 510/* Can we log writes? */
@@ -514,9 +520,11 @@ int vhost_log_access_ok(struct vhost_dev *dev)
514 520
515/* Verify access for write logging. */ 521/* Verify access for write logging. */
516/* Caller should have vq mutex and device mutex */ 522/* Caller should have vq mutex and device mutex */
517static int vq_log_access_ok(struct vhost_virtqueue *vq, void __user *log_base) 523static int vq_log_access_ok(struct vhost_dev *d, struct vhost_virtqueue *vq,
524 void __user *log_base)
518{ 525{
519 struct vhost_memory *mp; 526 struct vhost_memory *mp;
527 size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
520 528
521 mp = rcu_dereference_protected(vq->dev->memory, 529 mp = rcu_dereference_protected(vq->dev->memory,
522 lockdep_is_held(&vq->mutex)); 530 lockdep_is_held(&vq->mutex));
@@ -524,15 +532,15 @@ static int vq_log_access_ok(struct vhost_virtqueue *vq, void __user *log_base)
524 vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) && 532 vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) &&
525 (!vq->log_used || log_access_ok(log_base, vq->log_addr, 533 (!vq->log_used || log_access_ok(log_base, vq->log_addr,
526 sizeof *vq->used + 534 sizeof *vq->used +
527 vq->num * sizeof *vq->used->ring)); 535 vq->num * sizeof *vq->used->ring + s));
528} 536}
529 537
530/* Can we start vq? */ 538/* Can we start vq? */
531/* Caller should have vq mutex and device mutex */ 539/* Caller should have vq mutex and device mutex */
532int vhost_vq_access_ok(struct vhost_virtqueue *vq) 540int vhost_vq_access_ok(struct vhost_virtqueue *vq)
533{ 541{
534 return vq_access_ok(vq->num, vq->desc, vq->avail, vq->used) && 542 return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) &&
535 vq_log_access_ok(vq, vq->log_base); 543 vq_log_access_ok(vq->dev, vq, vq->log_base);
536} 544}
537 545
538static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) 546static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
@@ -577,6 +585,7 @@ static int init_used(struct vhost_virtqueue *vq,
577 585
578 if (r) 586 if (r)
579 return r; 587 return r;
588 vq->signalled_used_valid = false;
580 return get_user(vq->last_used_idx, &used->idx); 589 return get_user(vq->last_used_idx, &used->idx);
581} 590}
582 591
@@ -674,7 +683,7 @@ static long vhost_set_vring(struct vhost_dev *d, int ioctl, void __user *argp)
674 * If it is not, we don't as size might not have been setup. 683 * If it is not, we don't as size might not have been setup.
675 * We will verify when backend is configured. */ 684 * We will verify when backend is configured. */
676 if (vq->private_data) { 685 if (vq->private_data) {
677 if (!vq_access_ok(vq->num, 686 if (!vq_access_ok(d, vq->num,
678 (void __user *)(unsigned long)a.desc_user_addr, 687 (void __user *)(unsigned long)a.desc_user_addr,
679 (void __user *)(unsigned long)a.avail_user_addr, 688 (void __user *)(unsigned long)a.avail_user_addr,
680 (void __user *)(unsigned long)a.used_user_addr)) { 689 (void __user *)(unsigned long)a.used_user_addr)) {
@@ -818,7 +827,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, unsigned long arg)
818 vq = d->vqs + i; 827 vq = d->vqs + i;
819 mutex_lock(&vq->mutex); 828 mutex_lock(&vq->mutex);
820 /* If ring is inactive, will check when it's enabled. */ 829 /* If ring is inactive, will check when it's enabled. */
821 if (vq->private_data && !vq_log_access_ok(vq, base)) 830 if (vq->private_data && !vq_log_access_ok(d, vq, base))
822 r = -EFAULT; 831 r = -EFAULT;
823 else 832 else
824 vq->log_base = base; 833 vq->log_base = base;
@@ -1219,6 +1228,10 @@ int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
1219 1228
1220 /* On success, increment avail index. */ 1229 /* On success, increment avail index. */
1221 vq->last_avail_idx++; 1230 vq->last_avail_idx++;
1231
1232 /* Assume notifications from guest are disabled at this point,
1233 * if they aren't we would need to update avail_event index. */
1234 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
1222 return head; 1235 return head;
1223} 1236}
1224 1237
@@ -1267,6 +1280,12 @@ int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
1267 eventfd_signal(vq->log_ctx, 1); 1280 eventfd_signal(vq->log_ctx, 1);
1268 } 1281 }
1269 vq->last_used_idx++; 1282 vq->last_used_idx++;
1283 /* If the driver never bothers to signal in a very long while,
1284 * used index might wrap around. If that happens, invalidate
1285 * signalled_used index we stored. TODO: make sure driver
1286 * signals at least once in 2^16 and remove this. */
1287 if (unlikely(vq->last_used_idx == vq->signalled_used))
1288 vq->signalled_used_valid = false;
1270 return 0; 1289 return 0;
1271} 1290}
1272 1291
@@ -1275,6 +1294,7 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq,
1275 unsigned count) 1294 unsigned count)
1276{ 1295{
1277 struct vring_used_elem __user *used; 1296 struct vring_used_elem __user *used;
1297 u16 old, new;
1278 int start; 1298 int start;
1279 1299
1280 start = vq->last_used_idx % vq->num; 1300 start = vq->last_used_idx % vq->num;
@@ -1292,7 +1312,14 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq,
1292 ((void __user *)used - (void __user *)vq->used), 1312 ((void __user *)used - (void __user *)vq->used),
1293 count * sizeof *used); 1313 count * sizeof *used);
1294 } 1314 }
1295 vq->last_used_idx += count; 1315 old = vq->last_used_idx;
1316 new = (vq->last_used_idx += count);
1317 /* If the driver never bothers to signal in a very long while,
1318 * used index might wrap around. If that happens, invalidate
1319 * signalled_used index we stored. TODO: make sure driver
1320 * signals at least once in 2^16 and remove this. */
1321 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
1322 vq->signalled_used_valid = false;
1296 return 0; 1323 return 0;
1297} 1324}
1298 1325
@@ -1331,29 +1358,47 @@ int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
1331 return r; 1358 return r;
1332} 1359}
1333 1360
1334/* This actually signals the guest, using eventfd. */ 1361static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1335void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1336{ 1362{
1337 __u16 flags; 1363 __u16 old, new, event;
1338 1364 bool v;
1339 /* Flush out used index updates. This is paired 1365 /* Flush out used index updates. This is paired
1340 * with the barrier that the Guest executes when enabling 1366 * with the barrier that the Guest executes when enabling
1341 * interrupts. */ 1367 * interrupts. */
1342 smp_mb(); 1368 smp_mb();
1343 1369
1344 if (__get_user(flags, &vq->avail->flags)) { 1370 if (vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1345 vq_err(vq, "Failed to get flags"); 1371 unlikely(vq->avail_idx == vq->last_avail_idx))
1346 return; 1372 return true;
1373
1374 if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
1375 __u16 flags;
1376 if (__get_user(flags, &vq->avail->flags)) {
1377 vq_err(vq, "Failed to get flags");
1378 return true;
1379 }
1380 return !(flags & VRING_AVAIL_F_NO_INTERRUPT);
1347 } 1381 }
1382 old = vq->signalled_used;
1383 v = vq->signalled_used_valid;
1384 new = vq->signalled_used = vq->last_used_idx;
1385 vq->signalled_used_valid = true;
1348 1386
1349 /* If they don't want an interrupt, don't signal, unless empty. */ 1387 if (unlikely(!v))
1350 if ((flags & VRING_AVAIL_F_NO_INTERRUPT) && 1388 return true;
1351 (vq->avail_idx != vq->last_avail_idx ||
1352 !vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY)))
1353 return;
1354 1389
1390 if (get_user(event, vhost_used_event(vq))) {
1391 vq_err(vq, "Failed to get used event idx");
1392 return true;
1393 }
1394 return vring_need_event(event, new, old);
1395}
1396
1397/* This actually signals the guest, using eventfd. */
1398void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1399{
1355 /* Signal the Guest tell them we used something up. */ 1400 /* Signal the Guest tell them we used something up. */
1356 if (vq->call_ctx) 1401 if (vq->call_ctx && vhost_notify(dev, vq))
1357 eventfd_signal(vq->call_ctx, 1); 1402 eventfd_signal(vq->call_ctx, 1);
1358} 1403}
1359 1404
@@ -1376,7 +1421,7 @@ void vhost_add_used_and_signal_n(struct vhost_dev *dev,
1376} 1421}
1377 1422
1378/* OK, now we need to know about added descriptors. */ 1423/* OK, now we need to know about added descriptors. */
1379bool vhost_enable_notify(struct vhost_virtqueue *vq) 1424bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1380{ 1425{
1381 u16 avail_idx; 1426 u16 avail_idx;
1382 int r; 1427 int r;
@@ -1384,11 +1429,34 @@ bool vhost_enable_notify(struct vhost_virtqueue *vq)
1384 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) 1429 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
1385 return false; 1430 return false;
1386 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; 1431 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
1387 r = put_user(vq->used_flags, &vq->used->flags); 1432 if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
1388 if (r) { 1433 r = put_user(vq->used_flags, &vq->used->flags);
1389 vq_err(vq, "Failed to enable notification at %p: %d\n", 1434 if (r) {
1390 &vq->used->flags, r); 1435 vq_err(vq, "Failed to enable notification at %p: %d\n",
1391 return false; 1436 &vq->used->flags, r);
1437 return false;
1438 }
1439 } else {
1440 r = put_user(vq->avail_idx, vhost_avail_event(vq));
1441 if (r) {
1442 vq_err(vq, "Failed to update avail event index at %p: %d\n",
1443 vhost_avail_event(vq), r);
1444 return false;
1445 }
1446 }
1447 if (unlikely(vq->log_used)) {
1448 void __user *used;
1449 /* Make sure data is seen before log. */
1450 smp_wmb();
1451 used = vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX) ?
1452 &vq->used->flags : vhost_avail_event(vq);
1453 /* Log used flags or event index entry write. Both are 16 bit
1454 * fields. */
1455 log_write(vq->log_base, vq->log_addr +
1456 (used - (void __user *)vq->used),
1457 sizeof(u16));
1458 if (vq->log_ctx)
1459 eventfd_signal(vq->log_ctx, 1);
1392 } 1460 }
1393 /* They could have slipped one in as we were doing that: make 1461 /* They could have slipped one in as we were doing that: make
1394 * sure it's written, then check again. */ 1462 * sure it's written, then check again. */
@@ -1404,15 +1472,17 @@ bool vhost_enable_notify(struct vhost_virtqueue *vq)
1404} 1472}
1405 1473
1406/* We don't need to be notified again. */ 1474/* We don't need to be notified again. */
1407void vhost_disable_notify(struct vhost_virtqueue *vq) 1475void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1408{ 1476{
1409 int r; 1477 int r;
1410 1478
1411 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) 1479 if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
1412 return; 1480 return;
1413 vq->used_flags |= VRING_USED_F_NO_NOTIFY; 1481 vq->used_flags |= VRING_USED_F_NO_NOTIFY;
1414 r = put_user(vq->used_flags, &vq->used->flags); 1482 if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
1415 if (r) 1483 r = put_user(vq->used_flags, &vq->used->flags);
1416 vq_err(vq, "Failed to enable notification at %p: %d\n", 1484 if (r)
1417 &vq->used->flags, r); 1485 vq_err(vq, "Failed to enable notification at %p: %d\n",
1486 &vq->used->flags, r);
1487 }
1418} 1488}
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index b3363ae38518..8e03379dd30f 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -84,6 +84,12 @@ struct vhost_virtqueue {
84 /* Used flags */ 84 /* Used flags */
85 u16 used_flags; 85 u16 used_flags;
86 86
87 /* Last used index value we have signalled on */
88 u16 signalled_used;
89
90 /* Last used index value we have signalled on */
91 bool signalled_used_valid;
92
87 /* Log writes to used structure. */ 93 /* Log writes to used structure. */
88 bool log_used; 94 bool log_used;
89 u64 log_addr; 95 u64 log_addr;
@@ -149,8 +155,8 @@ void vhost_add_used_and_signal(struct vhost_dev *, struct vhost_virtqueue *,
149void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *, 155void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *,
150 struct vring_used_elem *heads, unsigned count); 156 struct vring_used_elem *heads, unsigned count);
151void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *); 157void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *);
152void vhost_disable_notify(struct vhost_virtqueue *); 158void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *);
153bool vhost_enable_notify(struct vhost_virtqueue *); 159bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *);
154 160
155int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 161int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
156 unsigned int log_num, u64 len); 162 unsigned int log_num, u64 len);
@@ -162,11 +168,12 @@ int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
162 } while (0) 168 } while (0)
163 169
164enum { 170enum {
165 VHOST_FEATURES = (1 << VIRTIO_F_NOTIFY_ON_EMPTY) | 171 VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) |
166 (1 << VIRTIO_RING_F_INDIRECT_DESC) | 172 (1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
167 (1 << VHOST_F_LOG_ALL) | 173 (1ULL << VIRTIO_RING_F_EVENT_IDX) |
168 (1 << VHOST_NET_F_VIRTIO_NET_HDR) | 174 (1ULL << VHOST_F_LOG_ALL) |
169 (1 << VIRTIO_NET_F_MRG_RXBUF), 175 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
176 (1ULL << VIRTIO_NET_F_MRG_RXBUF),
170}; 177};
171 178
172static inline int vhost_has_feature(struct vhost_dev *dev, int bit) 179static inline int vhost_has_feature(struct vhost_dev *dev, int bit)
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 0f1da45ba47d..e058ace2a4ad 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -40,9 +40,6 @@ struct virtio_balloon
40 /* Waiting for host to ack the pages we released. */ 40 /* Waiting for host to ack the pages we released. */
41 struct completion acked; 41 struct completion acked;
42 42
43 /* Do we have to tell Host *before* we reuse pages? */
44 bool tell_host_first;
45
46 /* The pages we've told the Host we're not using. */ 43 /* The pages we've told the Host we're not using. */
47 unsigned int num_pages; 44 unsigned int num_pages;
48 struct list_head pages; 45 struct list_head pages;
@@ -151,13 +148,14 @@ static void leak_balloon(struct virtio_balloon *vb, size_t num)
151 vb->num_pages--; 148 vb->num_pages--;
152 } 149 }
153 150
154 if (vb->tell_host_first) { 151
155 tell_host(vb, vb->deflate_vq); 152 /*
156 release_pages_by_pfn(vb->pfns, vb->num_pfns); 153 * Note that if
157 } else { 154 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
158 release_pages_by_pfn(vb->pfns, vb->num_pfns); 155 * is true, we *have* to do it in this order
159 tell_host(vb, vb->deflate_vq); 156 */
160 } 157 tell_host(vb, vb->deflate_vq);
158 release_pages_by_pfn(vb->pfns, vb->num_pfns);
161} 159}
162 160
163static inline void update_stat(struct virtio_balloon *vb, int idx, 161static inline void update_stat(struct virtio_balloon *vb, int idx,
@@ -325,9 +323,6 @@ static int virtballoon_probe(struct virtio_device *vdev)
325 goto out_del_vqs; 323 goto out_del_vqs;
326 } 324 }
327 325
328 vb->tell_host_first
329 = virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
330
331 return 0; 326 return 0;
332 327
333out_del_vqs: 328out_del_vqs:
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index b0043fb26a4d..68b9136847af 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -82,6 +82,9 @@ struct vring_virtqueue
82 /* Host supports indirect buffers */ 82 /* Host supports indirect buffers */
83 bool indirect; 83 bool indirect;
84 84
85 /* Host publishes avail event idx */
86 bool event;
87
85 /* Number of free buffers */ 88 /* Number of free buffers */
86 unsigned int num_free; 89 unsigned int num_free;
87 /* Head of free buffer list. */ 90 /* Head of free buffer list. */
@@ -237,18 +240,22 @@ EXPORT_SYMBOL_GPL(virtqueue_add_buf_gfp);
237void virtqueue_kick(struct virtqueue *_vq) 240void virtqueue_kick(struct virtqueue *_vq)
238{ 241{
239 struct vring_virtqueue *vq = to_vvq(_vq); 242 struct vring_virtqueue *vq = to_vvq(_vq);
243 u16 new, old;
240 START_USE(vq); 244 START_USE(vq);
241 /* Descriptors and available array need to be set before we expose the 245 /* Descriptors and available array need to be set before we expose the
242 * new available array entries. */ 246 * new available array entries. */
243 virtio_wmb(); 247 virtio_wmb();
244 248
245 vq->vring.avail->idx += vq->num_added; 249 old = vq->vring.avail->idx;
250 new = vq->vring.avail->idx = old + vq->num_added;
246 vq->num_added = 0; 251 vq->num_added = 0;
247 252
248 /* Need to update avail index before checking if we should notify */ 253 /* Need to update avail index before checking if we should notify */
249 virtio_mb(); 254 virtio_mb();
250 255
251 if (!(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY)) 256 if (vq->event ?
257 vring_need_event(vring_avail_event(&vq->vring), new, old) :
258 !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
252 /* Prod other side to tell it about changes. */ 259 /* Prod other side to tell it about changes. */
253 vq->notify(&vq->vq); 260 vq->notify(&vq->vq);
254 261
@@ -324,6 +331,14 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
324 ret = vq->data[i]; 331 ret = vq->data[i];
325 detach_buf(vq, i); 332 detach_buf(vq, i);
326 vq->last_used_idx++; 333 vq->last_used_idx++;
334 /* If we expect an interrupt for the next entry, tell host
335 * by writing event index and flush out the write before
336 * the read in the next get_buf call. */
337 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
338 vring_used_event(&vq->vring) = vq->last_used_idx;
339 virtio_mb();
340 }
341
327 END_USE(vq); 342 END_USE(vq);
328 return ret; 343 return ret;
329} 344}
@@ -345,7 +360,11 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
345 360
346 /* We optimistically turn back on interrupts, then check if there was 361 /* We optimistically turn back on interrupts, then check if there was
347 * more to do. */ 362 * more to do. */
363 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
364 * either clear the flags bit or point the event index at the next
365 * entry. Always do both to keep code simple. */
348 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 366 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
367 vring_used_event(&vq->vring) = vq->last_used_idx;
349 virtio_mb(); 368 virtio_mb();
350 if (unlikely(more_used(vq))) { 369 if (unlikely(more_used(vq))) {
351 END_USE(vq); 370 END_USE(vq);
@@ -357,6 +376,33 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
357} 376}
358EXPORT_SYMBOL_GPL(virtqueue_enable_cb); 377EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
359 378
379bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
380{
381 struct vring_virtqueue *vq = to_vvq(_vq);
382 u16 bufs;
383
384 START_USE(vq);
385
386 /* We optimistically turn back on interrupts, then check if there was
387 * more to do. */
388 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
389 * either clear the flags bit or point the event index at the next
390 * entry. Always do both to keep code simple. */
391 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
392 /* TODO: tune this threshold */
393 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
394 vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
395 virtio_mb();
396 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
397 END_USE(vq);
398 return false;
399 }
400
401 END_USE(vq);
402 return true;
403}
404EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
405
360void *virtqueue_detach_unused_buf(struct virtqueue *_vq) 406void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
361{ 407{
362 struct vring_virtqueue *vq = to_vvq(_vq); 408 struct vring_virtqueue *vq = to_vvq(_vq);
@@ -438,6 +484,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
438#endif 484#endif
439 485
440 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC); 486 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
487 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
441 488
442 /* No callback? Tell other side not to bother us. */ 489 /* No callback? Tell other side not to bother us. */
443 if (!callback) 490 if (!callback)
@@ -472,6 +519,8 @@ void vring_transport_features(struct virtio_device *vdev)
472 switch (i) { 519 switch (i) {
473 case VIRTIO_RING_F_INDIRECT_DESC: 520 case VIRTIO_RING_F_INDIRECT_DESC:
474 break; 521 break;
522 case VIRTIO_RING_F_EVENT_IDX:
523 break;
475 default: 524 default:
476 /* We don't understand this bit. */ 525 /* We don't understand this bit. */
477 clear_bit(i, vdev->features); 526 clear_bit(i, vdev->features);