14 files changed, 96 insertions, 122 deletions

diff --git a/MAINTAINERS b/MAINTAINERS
index 4012c2f98617..ccae35bb3904 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -12783,6 +12783,7 @@ F:	include/uapi/linux/virtio_console.h
 
 VIRTIO CORE, NET AND BLOCK DRIVERS
 M:      "Michael S. Tsirkin" <mst@redhat.com>
+M:      Jason Wang <jasowang@redhat.com>
 L:      virtualization@lists.linux-foundation.org
 S:      Maintained
 F:      Documentation/devicetree/bindings/virtio/
@@ -12813,6 +12814,7 @@ F:	include/uapi/linux/virtio_gpu.h
 
 VIRTIO HOST (VHOST)
 M:      "Michael S. Tsirkin" <mst@redhat.com>
+M:      Jason Wang <jasowang@redhat.com>
 L:      kvm@vger.kernel.org
 L:      virtualization@lists.linux-foundation.org
 L:      netdev@vger.kernel.org
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 2dc5c96c186a..5545a679abd8 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -376,7 +376,7 @@ static void virtblk_config_changed(struct virtio_device *vdev)
 
 static int init_vq(struct virtio_blk *vblk)
 {
-        int err = 0;
+        int err;
         int i;
         vq_callback_t **callbacks;
         const char **names;
@@ -390,13 +390,13 @@ static int init_vq(struct virtio_blk *vblk)
         if (err)
                 num_vqs = 1;
 
-        vblk->vqs = kmalloc(sizeof(*vblk->vqs) * num_vqs, GFP_KERNEL);
+        vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);
         if (!vblk->vqs)
                 return -ENOMEM;
 
-        names = kmalloc(sizeof(*names) * num_vqs, GFP_KERNEL);
-        callbacks = kmalloc(sizeof(*callbacks) * num_vqs, GFP_KERNEL);
-        vqs = kmalloc(sizeof(*vqs) * num_vqs, GFP_KERNEL);
+        names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL);
+        callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL);
+        vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL);
         if (!names || !callbacks || !vqs) {
                 err = -ENOMEM;
                 goto out;
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index d433b1db1fdd..5649234b7316 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -1539,19 +1539,29 @@ static void remove_port_data(struct port *port)
         spin_lock_irq(&port->inbuf_lock);
         /* Remove unused data this port might have received. */
         discard_port_data(port);
+        spin_unlock_irq(&port->inbuf_lock);
 
         /* Remove buffers we queued up for the Host to send us data in. */
-        while ((buf = virtqueue_detach_unused_buf(port->in_vq)))
-                free_buf(buf, true);
-        spin_unlock_irq(&port->inbuf_lock);
+        do {
+                spin_lock_irq(&port->inbuf_lock);
+                buf = virtqueue_detach_unused_buf(port->in_vq);
+                spin_unlock_irq(&port->inbuf_lock);
+                if (buf)
+                        free_buf(buf, true);
+        } while (buf);
 
         spin_lock_irq(&port->outvq_lock);
         reclaim_consumed_buffers(port);
+        spin_unlock_irq(&port->outvq_lock);
 
         /* Free pending buffers from the out-queue. */
-        while ((buf = virtqueue_detach_unused_buf(port->out_vq)))
-                free_buf(buf, true);
-        spin_unlock_irq(&port->outvq_lock);
+        do {
+                spin_lock_irq(&port->outvq_lock);
+                buf = virtqueue_detach_unused_buf(port->out_vq);
+                spin_unlock_irq(&port->outvq_lock);
+                if (buf)
+                        free_buf(buf, true);
+        } while (buf);
 }
 
 /*
diff --git a/drivers/virtio/config.c b/drivers/virtio/config.c
deleted file mode 100644
index f70bcd2ff98f..000000000000
--- a/drivers/virtio/config.c
+++ /dev/null
@@ -1,12 +0,0 @@
-/* Configuration space parsing helpers for virtio.
- *
- * The configuration is [type][len][... len bytes ...] fields.
- *
- * Copyright 2007 Rusty Russell, IBM Corporation.
- * GPL v2 or later.
- */
-#include <linux/err.h>
-#include <linux/virtio.h>
-#include <linux/virtio_config.h>
-#include <linux/bug.h>
-
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 4e7003db12c4..181793f07852 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -577,6 +577,8 @@ static int virtballoon_probe(struct virtio_device *vdev)
 
         virtio_device_ready(vdev);
 
+        if (towards_target(vb))
+                virtballoon_changed(vdev);
         return 0;
 
 out_del_vqs:
diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
index 8c4e61783441..6d9e5173d5fa 100644
--- a/drivers/virtio/virtio_pci_legacy.c
+++ b/drivers/virtio/virtio_pci_legacy.c
@@ -212,10 +212,18 @@ int virtio_pci_legacy_probe(struct virtio_pci_device *vp_dev)
                 return -ENODEV;
         }
 
-        rc = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64));
-        if (rc)
-                rc = dma_set_mask_and_coherent(&pci_dev->dev,
-                                                DMA_BIT_MASK(32));
+        rc = dma_set_mask(&pci_dev->dev, DMA_BIT_MASK(64));
+        if (rc) {
+                rc = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32));
+        } else {
+                /*
+                 * The virtio ring base address is expressed as a 32-bit PFN,
+                 * with a page size of 1 << VIRTIO_PCI_QUEUE_ADDR_SHIFT.
+                 */
+                dma_set_coherent_mask(&pci_dev->dev,
+                                DMA_BIT_MASK(32 + VIRTIO_PCI_QUEUE_ADDR_SHIFT));
+        }
+
         if (rc)
                 dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA.  Trying to continue, but this might not work.\n");
 
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index ed9c9eeedfe5..489bfc61cf30 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -167,7 +167,7 @@ static bool vring_use_dma_api(struct virtio_device *vdev)
  * making all of the arch DMA ops work on the vring device itself
  * is a mess.  For now, we use the parent device for DMA ops.
  */
-static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
+static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
 {
         return vq->vq.vdev->dev.parent;
 }
@@ -732,7 +732,8 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
 
         if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
                 vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
-                vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
+                if (!vq->event)
+                        vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
         }
 
 }
@@ -764,7 +765,8 @@ unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
          * entry. Always do both to keep code simple. */
         if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
                 vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
-                vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
+                if (!vq->event)
+                        vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
         }
         vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);
         END_USE(vq);
@@ -832,10 +834,11 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
          * more to do. */
         /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
          * either clear the flags bit or point the event index at the next
-         * entry. Always do both to keep code simple. */
+         * entry. Always update the event index to keep code simple. */
         if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
                 vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
-                vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
+                if (!vq->event)
+                        vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
         }
         /* TODO: tune this threshold */
         bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4;
@@ -953,7 +956,8 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index,
         /* No callback?  Tell other side not to bother us. */
         if (!callback) {
                 vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
-                vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);
+                if (!vq->event)
+                        vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);
         }
 
         /* Put everything in free lists. */
diff --git a/tools/virtio/ringtest/Makefile b/tools/virtio/ringtest/Makefile
index 877a8a4721b6..c012edbdb13b 100644
--- a/tools/virtio/ringtest/Makefile
+++ b/tools/virtio/ringtest/Makefile
@@ -3,8 +3,8 @@ all:
 all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder ptr_ring noring
 
 CFLAGS += -Wall
-CFLAGS += -pthread -O2 -ggdb
-LDFLAGS += -pthread -O2 -ggdb
+CFLAGS += -pthread -O2 -ggdb -flto -fwhole-program
+LDFLAGS += -pthread -O2 -ggdb -flto -fwhole-program
 
 main.o: main.c main.h
 ring.o: ring.c main.h
diff --git a/tools/virtio/ringtest/main.c b/tools/virtio/ringtest/main.c
index 147abb452a6c..f31353fac541 100644
--- a/tools/virtio/ringtest/main.c
+++ b/tools/virtio/ringtest/main.c
@@ -96,7 +96,13 @@ void set_affinity(const char *arg)
         assert(!ret);
 }
 
-static void run_guest(void)
+void poll_used(void)
+{
+        while (used_empty())
+                busy_wait();
+}
+
+static void __attribute__((__flatten__)) run_guest(void)
 {
         int completed_before;
         int completed = 0;
@@ -141,7 +147,7 @@ static void run_guest(void)
                 assert(completed <= bufs);
                 assert(started <= bufs);
                 if (do_sleep) {
-                        if (enable_call())
+                        if (used_empty() && enable_call())
                                 wait_for_call();
                 } else {
                         poll_used();
@@ -149,7 +155,13 @@ static void run_guest(void)
         }
 }
 
-static void run_host(void)
+void poll_avail(void)
+{
+        while (avail_empty())
+                busy_wait();
+}
+
+static void __attribute__((__flatten__)) run_host(void)
 {
         int completed_before;
         int completed = 0;
@@ -160,7 +172,7 @@ static void run_host(void)
 
         for (;;) {
                 if (do_sleep) {
-                        if (enable_kick())
+                        if (avail_empty() && enable_kick())
                                 wait_for_kick();
                 } else {
                         poll_avail();
diff --git a/tools/virtio/ringtest/main.h b/tools/virtio/ringtest/main.h
index 16917acb0ade..34e63cc4c572 100644
--- a/tools/virtio/ringtest/main.h
+++ b/tools/virtio/ringtest/main.h
@@ -56,15 +56,15 @@ void alloc_ring(void);
 int add_inbuf(unsigned, void *, void *);
 void *get_buf(unsigned *, void **);
 void disable_call();
+bool used_empty();
 bool enable_call();
 void kick_available();
-void poll_used();
 /* host side */
 void disable_kick();
+bool avail_empty();
 bool enable_kick();
 bool use_buf(unsigned *, void **);
 void call_used();
-void poll_avail();
 
 /* implemented by main */
 extern bool do_sleep;
diff --git a/tools/virtio/ringtest/noring.c b/tools/virtio/ringtest/noring.c
index eda2f4824130..b8d1c1daac7c 100644
--- a/tools/virtio/ringtest/noring.c
+++ b/tools/virtio/ringtest/noring.c
@@ -24,8 +24,9 @@ void *get_buf(unsigned *lenp, void **bufp)
         return "Buffer";
 }
 
-void poll_used(void)
+bool used_empty()
 {
+        return false;
 }
 
 void disable_call()
@@ -54,8 +55,9 @@ bool enable_kick()
         assert(0);
 }
 
-void poll_avail(void)
+bool avail_empty()
 {
+        return false;
 }
 
 bool use_buf(unsigned *lenp, void **bufp)
diff --git a/tools/virtio/ringtest/ptr_ring.c b/tools/virtio/ringtest/ptr_ring.c
index bd2ad1d3b7a9..635b07b4fdd3 100644
--- a/tools/virtio/ringtest/ptr_ring.c
+++ b/tools/virtio/ringtest/ptr_ring.c
@@ -133,18 +133,9 @@ void *get_buf(unsigned *lenp, void **bufp)
         return datap;
 }
 
-void poll_used(void)
+bool used_empty()
 {
-        void *b;
-
-        do {
-                if (tailcnt == headcnt || __ptr_ring_full(&array)) {
-                        b = NULL;
-                        barrier();
-                } else {
-                        b = "Buffer\n";
-                }
-        } while (!b);
+        return (tailcnt == headcnt || __ptr_ring_full(&array));
 }
 
 void disable_call()
@@ -173,14 +164,9 @@ bool enable_kick()
         assert(0);
 }
 
-void poll_avail(void)
+bool avail_empty()
 {
-        void *b;
-
-        do {
-                barrier();
-                b = __ptr_ring_peek(&array);
-        } while (!b);
+        return !__ptr_ring_peek(&array);
 }
 
 bool use_buf(unsigned *lenp, void **bufp)
diff --git a/tools/virtio/ringtest/ring.c b/tools/virtio/ringtest/ring.c
index c25c8d248b6b..747c5dd47be8 100644
--- a/tools/virtio/ringtest/ring.c
+++ b/tools/virtio/ringtest/ring.c
@@ -163,12 +163,11 @@ void *get_buf(unsigned *lenp, void **bufp)
         return datap;
 }
 
-void poll_used(void)
+bool used_empty()
 {
         unsigned head = (ring_size - 1) & guest.last_used_idx;
 
-        while (ring[head].flags & DESC_HW)
-                busy_wait();
+        return (ring[head].flags & DESC_HW);
 }
 
 void disable_call()
@@ -180,13 +179,11 @@ void disable_call()
 
 bool enable_call()
 {
-        unsigned head = (ring_size - 1) & guest.last_used_idx;
-
         event->call_index = guest.last_used_idx;
         /* Flush call index write */
         /* Barrier D (for pairing) */
         smp_mb();
-        return ring[head].flags & DESC_HW;
+        return used_empty();
 }
 
 void kick_available(void)
@@ -213,20 +210,17 @@ void disable_kick()
 
 bool enable_kick()
 {
-        unsigned head = (ring_size - 1) & host.used_idx;
-
         event->kick_index = host.used_idx;
         /* Barrier C (for pairing) */
         smp_mb();
-        return !(ring[head].flags & DESC_HW);
+        return avail_empty();
 }
 
-void poll_avail(void)
+bool avail_empty()
 {
         unsigned head = (ring_size - 1) & host.used_idx;
 
-        while (!(ring[head].flags & DESC_HW))
-                busy_wait();
+        return !(ring[head].flags & DESC_HW);
 }
 
 bool use_buf(unsigned *lenp, void **bufp)
diff --git a/tools/virtio/ringtest/virtio_ring_0_9.c b/tools/virtio/ringtest/virtio_ring_0_9.c
index 761866212aac..bbc3043b2fb1 100644
--- a/tools/virtio/ringtest/virtio_ring_0_9.c
+++ b/tools/virtio/ringtest/virtio_ring_0_9.c
@@ -194,24 +194,16 @@ void *get_buf(unsigned *lenp, void **bufp)
         return datap;
 }
 
-void poll_used(void)
+bool used_empty()
 {
+        unsigned short last_used_idx = guest.last_used_idx;
 #ifdef RING_POLL
-        unsigned head = (ring_size - 1) & guest.last_used_idx;
+        unsigned short head = last_used_idx & (ring_size - 1);
+        unsigned index = ring.used->ring[head].id;
 
-        for (;;) {
-                unsigned index = ring.used->ring[head].id;
-
-                if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))
-                        busy_wait();
-                else
-                        break;
-        }
+        return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1);
 #else
-        unsigned head = guest.last_used_idx;
-
-        while (ring.used->idx == head)
-                busy_wait();
+        return ring.used->idx == last_used_idx;
 #endif
 }
 
@@ -224,22 +216,11 @@ void disable_call()
 
 bool enable_call()
 {
-        unsigned short last_used_idx;
-
-        vring_used_event(&ring) = (last_used_idx = guest.last_used_idx);
+        vring_used_event(&ring) = guest.last_used_idx;
         /* Flush call index write */
         /* Barrier D (for pairing) */
         smp_mb();
-#ifdef RING_POLL
-        {
-                unsigned short head = last_used_idx & (ring_size - 1);
-                unsigned index = ring.used->ring[head].id;
-
-                return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1);
-        }
-#else
-        return ring.used->idx == last_used_idx;
-#endif
+        return used_empty();
 }
 
 void kick_available(void)
@@ -266,36 +247,21 @@ void disable_kick()
 
 bool enable_kick()
 {
-        unsigned head = host.used_idx;
-
-        vring_avail_event(&ring) = head;
+        vring_avail_event(&ring) = host.used_idx;
         /* Barrier C (for pairing) */
         smp_mb();
-#ifdef RING_POLL
-        {
-                unsigned index = ring.avail->ring[head & (ring_size - 1)];
-
-                return (index ^ head ^ 0x8000) & ~(ring_size - 1);
-        }
-#else
-        return head == ring.avail->idx;
-#endif
+        return avail_empty();
 }
 
-void poll_avail(void)
+bool avail_empty()
 {
         unsigned head = host.used_idx;
 #ifdef RING_POLL
-        for (;;) {
-                unsigned index = ring.avail->ring[head & (ring_size - 1)];
-                if ((index ^ head ^ 0x8000) & ~(ring_size - 1))
-                        busy_wait();
-                else
-                        break;
-        }
+        unsigned index = ring.avail->ring[head & (ring_size - 1)];
+
+        return ((index ^ head ^ 0x8000) & ~(ring_size - 1));
 #else
-        while (ring.avail->idx == head)
-                busy_wait();
+        return head == ring.avail->idx;
 #endif
 }