8 files changed, 35 insertions, 22 deletions

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
index 595d73197016..6a1d6447b864 100644
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -292,10 +292,12 @@ static struct virtqueue *lg_find_vq(struct virtio_device *vdev,
 
         /*
          * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-         * and we've got a pointer to its pages.
+         * and we've got a pointer to its pages.  Note that we set weak_barriers
+         * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+         * barriers.
          */
-        vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-                                 vdev, lvq->pages, lg_notify, callback, name);
+        vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+                                 true, lvq->pages, lg_notify, callback, name);
         if (!vq) {
                 err = -ENOMEM;
                 goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
index 8af868bab20b..7bc1955337ea 100644
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(struct virtio_device *vdev,
                 goto out;
 
         vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-                                 vdev, (void *) config->address,
+                                 vdev, true, (void *) config->address,
                                  kvm_notify, callback, name);
         if (!vq) {
                 err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index 0269717436af..01d6dc250d5c 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -310,8 +310,8 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index,
                         vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
         /* Create the vring */
-        vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-                                 vdev, info->queue, vm_notify, callback, name);
+        vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+                                 true, info->queue, vm_notify, callback, name);
         if (!vq) {
                 err = -ENOMEM;
                 goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
index baabb7937ec2..688b42d28dad 100644
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
                   vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
         /* create the vring */
-        vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-                                 vdev, info->queue, vp_notify, callback, name);
+        vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+                                 true, info->queue, vp_notify, callback, name);
         if (!vq) {
                 err = -ENOMEM;
                 goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index c7a2c208f6ea..50da92046092 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+        do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+        do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+        do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
         /* Actual memory layout for this queue */
         struct vring vring;
 
+        /* Can we use weak barriers? */
+        bool weak_barriers;
+
         /* Other side has made a mess, don't try any more. */
         bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_vq)
         START_USE(vq);
         /* Descriptors and available array need to be set before we expose the
          * new available array entries. */
-        virtio_wmb();
+        virtio_wmb(vq);
 
         old = vq->vring.avail->idx;
         new = vq->vring.avail->idx = old + vq->num_added;
         vq->num_added = 0;
 
         /* Need to update avail index before checking if we should notify */
-        virtio_mb();
+        virtio_mb(vq);
 
         if (vq->event ?
             vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
         }
 
         /* Only get used array entries after they have been exposed by host. */
-        virtio_rmb();
+        virtio_rmb(vq);
 
         i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
         *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
          * the read in the next get_buf call. */
         if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
                 vring_used_event(&vq->vring) = vq->last_used_idx;
-                virtio_mb();
+                virtio_mb(vq);
         }
 
         END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
          * entry. Always do both to keep code simple. */
         vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
         vring_used_event(&vq->vring) = vq->last_used_idx;
-        virtio_mb();
+        virtio_mb(vq);
         if (unlikely(more_used(vq))) {
                 END_USE(vq);
                 return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
         /* TODO: tune this threshold */
         bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
         vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-        virtio_mb();
+        virtio_mb(vq);
         if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
                 END_USE(vq);
                 return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
                                       unsigned int vring_align,
                                       struct virtio_device *vdev,
+                                      bool weak_barriers,
                                       void *pages,
                                       void (*notify)(struct virtqueue *),
                                       void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
         vq->vq.vdev = vdev;
         vq->vq.name = name;
         vq->notify = notify;
+        vq->weak_barriers = weak_barriers;
         vq->broken = false;
         vq->last_used_idx = 0;
         vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 36be0f6e18a9..e338730c2660 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
                                       unsigned int vring_align,
                                       struct virtio_device *vdev,
+                                      bool weak_barriers,
                                       void *pages,
                                       void (*notify)(struct virtqueue *vq),
                                       void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
index 669bcdd45805..953db2abf6b9 100644
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct virtqueue *vq);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
                                       unsigned int vring_align,
                                       struct virtio_device *vdev,
+                                      bool weak_barriers,
                                       void *pages,
                                       void (*notify)(struct virtqueue *vq),
                                       void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
index 74d3331bdaf9..0740284396c1 100644
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info *dev, int num)
         assert(r >= 0);
         memset(info->ring, 0, vring_size(num, 4096));
         vring_init(&info->vring, num, info->ring, 4096);
-        info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+        info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+                                       true, info->ring,
                                        vq_notify, vq_callback, "test");
         assert(info->vq);
         info->vq->priv = info;