1 files changed, 93 insertions, 9 deletions

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 5c52369ab9bb..a882f2606515 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -23,21 +23,30 @@
 
 #ifdef DEBUG
 /* For development, we want to crash whenever the ring is screwed. */
-#define BAD_RING(_vq, fmt...)                   \
-        do { dev_err(&(_vq)->vq.vdev->dev, fmt); BUG(); } while(0)
+#define BAD_RING(_vq, fmt, args...)                             \
+        do {                                                    \
+                dev_err(&(_vq)->vq.vdev->dev,                   \
+                        "%s:"fmt, (_vq)->vq.name, ##args);      \
+                BUG();                                          \
+        } while (0)
 /* Caller is supposed to guarantee no reentry. */
 #define START_USE(_vq)                                          \
         do {                                                    \
                 if ((_vq)->in_use)                              \
-                        panic("in_use = %i\n", (_vq)->in_use);  \
+                        panic("%s:in_use = %i\n",               \
+                              (_vq)->vq.name, (_vq)->in_use);   \
                 (_vq)->in_use = __LINE__;                       \
                 mb();                                           \
-        } while(0)
+        } while (0)
 #define END_USE(_vq) \
         do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; mb(); } while(0)
 #else
-#define BAD_RING(_vq, fmt...)                   \
-        do { dev_err(&_vq->vq.vdev->dev, fmt); (_vq)->broken = true; } while(0)
+#define BAD_RING(_vq, fmt, args...)                             \
+        do {                                                    \
+                dev_err(&_vq->vq.vdev->dev,                     \
+                        "%s:"fmt, (_vq)->vq.name, ##args);      \
+                (_vq)->broken = true;                           \
+        } while (0)
 #define START_USE(vq)
 #define END_USE(vq)
 #endif
@@ -52,6 +61,9 @@ struct vring_virtqueue
         /* Other side has made a mess, don't try any more. */
         bool broken;
 
+        /* Host supports indirect buffers */
+        bool indirect;
+
         /* Number of free buffers */
         unsigned int num_free;
         /* Head of free buffer list. */
@@ -76,6 +88,55 @@ struct vring_virtqueue
 
 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
 
+/* Set up an indirect table of descriptors and add it to the queue. */
+static int vring_add_indirect(struct vring_virtqueue *vq,
+                              struct scatterlist sg[],
+                              unsigned int out,
+                              unsigned int in)
+{
+        struct vring_desc *desc;
+        unsigned head;
+        int i;
+
+        desc = kmalloc((out + in) * sizeof(struct vring_desc), GFP_ATOMIC);
+        if (!desc)
+                return vq->vring.num;
+
+        /* Transfer entries from the sg list into the indirect page */
+        for (i = 0; i < out; i++) {
+                desc[i].flags = VRING_DESC_F_NEXT;
+                desc[i].addr = sg_phys(sg);
+                desc[i].len = sg->length;
+                desc[i].next = i+1;
+                sg++;
+        }
+        for (; i < (out + in); i++) {
+                desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
+                desc[i].addr = sg_phys(sg);
+                desc[i].len = sg->length;
+                desc[i].next = i+1;
+                sg++;
+        }
+
+        /* Last one doesn't continue. */
+        desc[i-1].flags &= ~VRING_DESC_F_NEXT;
+        desc[i-1].next = 0;
+
+        /* We're about to use a buffer */
+        vq->num_free--;
+
+        /* Use a single buffer which doesn't continue */
+        head = vq->free_head;
+        vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT;
+        vq->vring.desc[head].addr = virt_to_phys(desc);
+        vq->vring.desc[head].len = i * sizeof(struct vring_desc);
+
+        /* Update free pointer */
+        vq->free_head = vq->vring.desc[head].next;
+
+        return head;
+}
+
 static int vring_add_buf(struct virtqueue *_vq,
                          struct scatterlist sg[],
                          unsigned int out,
@@ -85,12 +146,21 @@ static int vring_add_buf(struct virtqueue *_vq,
         struct vring_virtqueue *vq = to_vvq(_vq);
         unsigned int i, avail, head, uninitialized_var(prev);
 
+        START_USE(vq);
+
         BUG_ON(data == NULL);
+
+        /* If the host supports indirect descriptor tables, and we have multiple
+         * buffers, then go indirect. FIXME: tune this threshold */
+        if (vq->indirect && (out + in) > 1 && vq->num_free) {
+                head = vring_add_indirect(vq, sg, out, in);
+                if (head != vq->vring.num)
+                        goto add_head;
+        }
+
         BUG_ON(out + in > vq->vring.num);
         BUG_ON(out + in == 0);
 
-        START_USE(vq);
-
         if (vq->num_free < out + in) {
                 pr_debug("Can't add buf len %i - avail = %i\n",
                          out + in, vq->num_free);
@@ -127,6 +197,7 @@ static int vring_add_buf(struct virtqueue *_vq,
         /* Update free pointer */
         vq->free_head = i;
 
+add_head:
         /* Set token. */
         vq->data[head] = data;
 
@@ -170,6 +241,11 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
 
         /* Put back on free list: find end */
         i = head;
+
+        /* Free the indirect table */
+        if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
+                kfree(phys_to_virt(vq->vring.desc[i].addr));
+
         while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
                 i = vq->vring.desc[i].next;
                 vq->num_free++;
@@ -284,7 +360,8 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
                                       struct virtio_device *vdev,
                                       void *pages,
                                       void (*notify)(struct virtqueue *),
-                                      void (*callback)(struct virtqueue *))
+                                      void (*callback)(struct virtqueue *),
+                                      const char *name)
 {
         struct vring_virtqueue *vq;
         unsigned int i;
@@ -303,14 +380,18 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
         vq->vq.callback = callback;
         vq->vq.vdev = vdev;
         vq->vq.vq_ops = &vring_vq_ops;
+        vq->vq.name = name;
         vq->notify = notify;
         vq->broken = false;
         vq->last_used_idx = 0;
         vq->num_added = 0;
+        list_add_tail(&vq->vq.list, &vdev->vqs);
 #ifdef DEBUG
         vq->in_use = false;
 #endif
 
+        vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
+
         /* No callback?  Tell other side not to bother us. */
         if (!callback)
                 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
@@ -327,6 +408,7 @@ EXPORT_SYMBOL_GPL(vring_new_virtqueue);
 
 void vring_del_virtqueue(struct virtqueue *vq)
 {
+        list_del(&vq->list);
         kfree(to_vvq(vq));
 }
 EXPORT_SYMBOL_GPL(vring_del_virtqueue);
@@ -338,6 +420,8 @@ void vring_transport_features(struct virtio_device *vdev)
 
         for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
                 switch (i) {
+                case VIRTIO_RING_F_INDIRECT_DESC:
+                        break;
                 default:
                         /* We don't understand this bit. */
                         clear_bit(i, vdev->features);