Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

5 files changed, 646 insertions, 259 deletions

diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 7c8008225ee3..e224a92baa16 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -10,7 +10,6 @@
 #include <linux/eventfd.h>
 #include <linux/vhost.h>
 #include <linux/virtio_net.h>
-#include <linux/mmu_context.h>
 #include <linux/miscdevice.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
@@ -61,6 +60,7 @@ static int move_iovec_hdr(struct iovec *from, struct iovec *to,
 {
         int seg = 0;
         size_t size;
+
         while (len && seg < iov_count) {
                 size = min(from->iov_len, len);
                 to->iov_base = from->iov_base;
@@ -80,6 +80,7 @@ static void copy_iovec_hdr(const struct iovec *from, struct iovec *to,
 {
         int seg = 0;
         size_t size;
+
         while (len && seg < iovcount) {
                 size = min(from->iov_len, len);
                 to->iov_base = from->iov_base;
@@ -127,7 +128,10 @@ static void handle_tx(struct vhost_net *net)
         size_t len, total_len = 0;
         int err, wmem;
         size_t hdr_size;
-        struct socket *sock = rcu_dereference(vq->private_data);
+        struct socket *sock;
+
+        /* TODO: check that we are running from vhost_worker? */
+        sock = rcu_dereference_check(vq->private_data, 1);
         if (!sock)
                 return;
 
@@ -139,9 +143,8 @@ static void handle_tx(struct vhost_net *net)
                 return;
         }
 
-        use_mm(net->dev.mm);
         mutex_lock(&vq->mutex);
-        vhost_disable_notify(vq);
+        vhost_disable_notify(&net->dev, vq);
 
         if (wmem < sock->sk->sk_sndbuf / 2)
                 tx_poll_stop(net);
@@ -163,8 +166,8 @@ static void handle_tx(struct vhost_net *net)
                                 set_bit(SOCK_ASYNC_NOSPACE, &sock->flags);
                                 break;
                         }
-                        if (unlikely(vhost_enable_notify(vq))) {
-                                vhost_disable_notify(vq);
+                        if (unlikely(vhost_enable_notify(&net->dev, vq))) {
+                                vhost_disable_notify(&net->dev, vq);
                                 continue;
                         }
                         break;
@@ -204,19 +207,19 @@ static void handle_tx(struct vhost_net *net)
         }
 
         mutex_unlock(&vq->mutex);
-        unuse_mm(net->dev.mm);
 }
 
 static int peek_head_len(struct sock *sk)
 {
         struct sk_buff *head;
         int len = 0;
+        unsigned long flags;
 
-        lock_sock(sk);
+        spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
         head = skb_peek(&sk->sk_receive_queue);
-        if (head)
+        if (likely(head))
                 len = head->len;
-        release_sock(sk);
+        spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags);
         return len;
 }
 
@@ -227,6 +230,7 @@ static int peek_head_len(struct sock *sk)
  * @iovcount    - returned count of io vectors we fill
  * @log         - vhost log
  * @log_num     - log offset
+ * @quota       - headcount quota, 1 for big buffer
  *      returns number of buffer heads allocated, negative on error
  */
 static int get_rx_bufs(struct vhost_virtqueue *vq,
@@ -234,7 +238,8 @@ static int get_rx_bufs(struct vhost_virtqueue *vq,
                        int datalen,
                        unsigned *iovcount,
                        struct vhost_log *log,
-                       unsigned *log_num)
+                       unsigned *log_num,
+                       unsigned int quota)
 {
         unsigned int out, in;
         int seg = 0;
@@ -242,8 +247,8 @@ static int get_rx_bufs(struct vhost_virtqueue *vq,
         unsigned d;
         int r, nlogs = 0;
 
-        while (datalen > 0) {
-                if (unlikely(seg >= VHOST_NET_MAX_SG)) {
+        while (datalen > 0 && headcount < quota) {
+                if (unlikely(seg >= UIO_MAXIOV)) {
                         r = -ENOBUFS;
                         goto err;
                 }
@@ -282,118 +287,7 @@ err:
 
 /* Expects to be always run from workqueue - which acts as
  * read-size critical section for our kind of RCU. */
-static void handle_rx_big(struct vhost_net *net)
-{
-        struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX];
-        unsigned out, in, log, s;
-        int head;
-        struct vhost_log *vq_log;
-        struct msghdr msg = {
-                .msg_name = NULL,
-                .msg_namelen = 0,
-                .msg_control = NULL, /* FIXME: get and handle RX aux data. */
-                .msg_controllen = 0,
-                .msg_iov = vq->iov,
-                .msg_flags = MSG_DONTWAIT,
-        };
-
-        struct virtio_net_hdr hdr = {
-                .flags = 0,
-                .gso_type = VIRTIO_NET_HDR_GSO_NONE
-        };
-
-        size_t len, total_len = 0;
-        int err;
-        size_t hdr_size;
-        struct socket *sock = rcu_dereference(vq->private_data);
-        if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue))
-                return;
-
-        use_mm(net->dev.mm);
-        mutex_lock(&vq->mutex);
-        vhost_disable_notify(vq);
-        hdr_size = vq->vhost_hlen;
-
-        vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ?
-                vq->log : NULL;
-
-        for (;;) {
-                head = vhost_get_vq_desc(&net->dev, vq, vq->iov,
-                                         ARRAY_SIZE(vq->iov),
-                                         &out, &in,
-                                         vq_log, &log);
-                /* On error, stop handling until the next kick. */
-                if (unlikely(head < 0))
-                        break;
-                /* OK, now we need to know about added descriptors. */
-                if (head == vq->num) {
-                        if (unlikely(vhost_enable_notify(vq))) {
-                                /* They have slipped one in as we were
-                                 * doing that: check again. */
-                                vhost_disable_notify(vq);
-                                continue;
-                        }
-                        /* Nothing new?  Wait for eventfd to tell us
-                         * they refilled. */
-                        break;
-                }
-                /* We don't need to be notified again. */
-                if (out) {
-                        vq_err(vq, "Unexpected descriptor format for RX: "
-                               "out %d, int %d\n",
-                               out, in);
-                        break;
-                }
-                /* Skip header. TODO: support TSO/mergeable rx buffers. */
-                s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, in);
-                msg.msg_iovlen = in;
-                len = iov_length(vq->iov, in);
-                /* Sanity check */
-                if (!len) {
-                        vq_err(vq, "Unexpected header len for RX: "
-                               "%zd expected %zd\n",
-                               iov_length(vq->hdr, s), hdr_size);
-                        break;
-                }
-                err = sock->ops->recvmsg(NULL, sock, &msg,
-                                         len, MSG_DONTWAIT | MSG_TRUNC);
-                /* TODO: Check specific error and bomb out unless EAGAIN? */
-                if (err < 0) {
-                        vhost_discard_vq_desc(vq, 1);
-                        break;
-                }
-                /* TODO: Should check and handle checksum. */
-                if (err > len) {
-                        pr_debug("Discarded truncated rx packet: "
-                                 " len %d > %zd\n", err, len);
-                        vhost_discard_vq_desc(vq, 1);
-                        continue;
-                }
-                len = err;
-                err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size);
-                if (err) {
-                        vq_err(vq, "Unable to write vnet_hdr at addr %p: %d\n",
-                               vq->iov->iov_base, err);
-                        break;
-                }
-                len += hdr_size;
-                vhost_add_used_and_signal(&net->dev, vq, head, len);
-                if (unlikely(vq_log))
-                        vhost_log_write(vq, vq_log, log, len);
-                total_len += len;
-                if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
-                        vhost_poll_queue(&vq->poll);
-                        break;
-                }
-        }
-
-        mutex_unlock(&vq->mutex);
-        unuse_mm(net->dev.mm);
-}
-
-/* Expects to be always run from workqueue - which acts as
- * read-size critical section for our kind of RCU. */
-static void handle_rx_mergeable(struct vhost_net *net)
+static void handle_rx(struct vhost_net *net)
 {
         struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX];
         unsigned uninitialized_var(in), log;
@@ -406,43 +300,44 @@ static void handle_rx_mergeable(struct vhost_net *net)
                 .msg_iov = vq->iov,
                 .msg_flags = MSG_DONTWAIT,
         };
-
         struct virtio_net_hdr_mrg_rxbuf hdr = {
                 .hdr.flags = 0,
                 .hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE
         };
-
         size_t total_len = 0;
-        int err, headcount;
+        int err, headcount, mergeable;
         size_t vhost_hlen, sock_hlen;
         size_t vhost_len, sock_len;
-        struct socket *sock = rcu_dereference(vq->private_data);
-        if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue))
+        /* TODO: check that we are running from vhost_worker? */
+        struct socket *sock = rcu_dereference_check(vq->private_data, 1);
+
+        if (!sock)
                 return;
 
-        use_mm(net->dev.mm);
         mutex_lock(&vq->mutex);
-        vhost_disable_notify(vq);
+        vhost_disable_notify(&net->dev, vq);
         vhost_hlen = vq->vhost_hlen;
         sock_hlen = vq->sock_hlen;
 
         vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ?
                 vq->log : NULL;
+        mergeable = vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF);
 
         while ((sock_len = peek_head_len(sock->sk))) {
                 sock_len += sock_hlen;
                 vhost_len = sock_len + vhost_hlen;
                 headcount = get_rx_bufs(vq, vq->heads, vhost_len,
-                                        &in, vq_log, &log);
+                                        &in, vq_log, &log,
+                                        likely(mergeable) ? UIO_MAXIOV : 1);
                 /* On error, stop handling until the next kick. */
                 if (unlikely(headcount < 0))
                         break;
                 /* OK, now we need to know about added descriptors. */
                 if (!headcount) {
-                        if (unlikely(vhost_enable_notify(vq))) {
+                        if (unlikely(vhost_enable_notify(&net->dev, vq))) {
                                 /* They have slipped one in as we were
                                  * doing that: check again. */
-                                vhost_disable_notify(vq);
+                                vhost_disable_notify(&net->dev, vq);
                                 continue;
                         }
                         /* Nothing new?  Wait for eventfd to tell us
@@ -455,7 +350,7 @@ static void handle_rx_mergeable(struct vhost_net *net)
                         move_iovec_hdr(vq->iov, vq->hdr, vhost_hlen, in);
                 else
                         /* Copy the header for use in VIRTIO_NET_F_MRG_RXBUF:
-                         * needed because sendmsg can modify msg_iov. */
+                         * needed because recvmsg can modify msg_iov. */
                         copy_iovec_hdr(vq->iov, vq->hdr, sock_hlen, in);
                 msg.msg_iovlen = in;
                 err = sock->ops->recvmsg(NULL, sock, &msg,
@@ -477,7 +372,7 @@ static void handle_rx_mergeable(struct vhost_net *net)
                         break;
                 }
                 /* TODO: Should check and handle checksum. */
-                if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF) &&
+                if (likely(mergeable) &&
                     memcpy_toiovecend(vq->hdr, (unsigned char *)&headcount,
                                       offsetof(typeof(hdr), num_buffers),
                                       sizeof hdr.num_buffers)) {
@@ -497,15 +392,6 @@ static void handle_rx_mergeable(struct vhost_net *net)
         }
 
         mutex_unlock(&vq->mutex);
-        unuse_mm(net->dev.mm);
-}
-
-static void handle_rx(struct vhost_net *net)
-{
-        if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF))
-                handle_rx_mergeable(net);
-        else
-                handle_rx_big(net);
 }
 
 static void handle_tx_kick(struct vhost_work *work)
@@ -582,7 +468,10 @@ static void vhost_net_disable_vq(struct vhost_net *n,
 static void vhost_net_enable_vq(struct vhost_net *n,
                                 struct vhost_virtqueue *vq)
 {
-        struct socket *sock = vq->private_data;
+        struct socket *sock;
+
+        sock = rcu_dereference_protected(vq->private_data,
+                                         lockdep_is_held(&vq->mutex));
         if (!sock)
                 return;
         if (vq == n->vqs + VHOST_NET_VQ_TX) {
@@ -598,7 +487,8 @@ static struct socket *vhost_net_stop_vq(struct vhost_net *n,
         struct socket *sock;
 
         mutex_lock(&vq->mutex);
-        sock = vq->private_data;
+        sock = rcu_dereference_protected(vq->private_data,
+                                         lockdep_is_held(&vq->mutex));
         vhost_net_disable_vq(n, vq);
         rcu_assign_pointer(vq->private_data, NULL);
         mutex_unlock(&vq->mutex);
@@ -652,6 +542,7 @@ static struct socket *get_raw_socket(int fd)
         } uaddr;
         int uaddr_len = sizeof uaddr, r;
         struct socket *sock = sockfd_lookup(fd, &r);
+
         if (!sock)
                 return ERR_PTR(-ENOTSOCK);
 
@@ -680,6 +571,7 @@ static struct socket *get_tap_socket(int fd)
 {
         struct file *file = fget(fd);
         struct socket *sock;
+
         if (!file)
                 return ERR_PTR(-EBADF);
         sock = tun_get_socket(file);
@@ -694,6 +586,7 @@ static struct socket *get_tap_socket(int fd)
 static struct socket *get_socket(int fd)
 {
         struct socket *sock;
+
         /* special case to disable backend */
         if (fd == -1)
                 return NULL;
@@ -736,11 +629,12 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
         }
 
         /* start polling new socket */
-        oldsock = vq->private_data;
+        oldsock = rcu_dereference_protected(vq->private_data,
+                                            lockdep_is_held(&vq->mutex));
         if (sock != oldsock) {
-                vhost_net_disable_vq(n, vq);
-                rcu_assign_pointer(vq->private_data, sock);
-                vhost_net_enable_vq(n, vq);
+                vhost_net_disable_vq(n, vq);
+                rcu_assign_pointer(vq->private_data, sock);
+                vhost_net_enable_vq(n, vq);
         }
 
         mutex_unlock(&vq->mutex);
@@ -765,6 +659,7 @@ static long vhost_net_reset_owner(struct vhost_net *n)
         struct socket *tx_sock = NULL;
         struct socket *rx_sock = NULL;
         long err;
+
         mutex_lock(&n->dev.mutex);
         err = vhost_dev_check_owner(&n->dev);
         if (err)
@@ -826,6 +721,7 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
         struct vhost_vring_file backend;
         u64 features;
         int r;
+
         switch (ioctl) {
         case VHOST_NET_SET_BACKEND:
                 if (copy_from_user(&backend, argp, sizeof backend))
@@ -869,6 +765,7 @@ static const struct file_operations vhost_net_fops = {
         .compat_ioctl   = vhost_net_compat_ioctl,
 #endif
         .open           = vhost_net_open,
+        .llseek         = noop_llseek,
 };
 
 static struct miscdevice vhost_net_misc = {
diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c
new file mode 100644
index 000000000000..734e1d74ad80
--- /dev/null
+++ b/drivers/vhost/test.c
@@ -0,0 +1,320 @@
+/* Copyright (C) 2009 Red Hat, Inc.
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ *
+ * test virtio server in host kernel.
+ */
+
+#include <linux/compat.h>
+#include <linux/eventfd.h>
+#include <linux/vhost.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+#include <linux/rcupdate.h>
+#include <linux/file.h>
+#include <linux/slab.h>
+
+#include "test.h"
+#include "vhost.c"
+
+/* Max number of bytes transferred before requeueing the job.
+ * Using this limit prevents one virtqueue from starving others. */
+#define VHOST_TEST_WEIGHT 0x80000
+
+enum {
+        VHOST_TEST_VQ = 0,
+        VHOST_TEST_VQ_MAX = 1,
+};
+
+struct vhost_test {
+        struct vhost_dev dev;
+        struct vhost_virtqueue vqs[VHOST_TEST_VQ_MAX];
+};
+
+/* Expects to be always run from workqueue - which acts as
+ * read-size critical section for our kind of RCU. */
+static void handle_vq(struct vhost_test *n)
+{
+        struct vhost_virtqueue *vq = &n->dev.vqs[VHOST_TEST_VQ];
+        unsigned out, in;
+        int head;
+        size_t len, total_len = 0;
+        void *private;
+
+        private = rcu_dereference_check(vq->private_data, 1);
+        if (!private)
+                return;
+
+        mutex_lock(&vq->mutex);
+        vhost_disable_notify(&n->dev, vq);
+
+        for (;;) {
+                head = vhost_get_vq_desc(&n->dev, vq, vq->iov,
+                                         ARRAY_SIZE(vq->iov),
+                                         &out, &in,
+                                         NULL, NULL);
+                /* On error, stop handling until the next kick. */
+                if (unlikely(head < 0))
+                        break;
+                /* Nothing new?  Wait for eventfd to tell us they refilled. */
+                if (head == vq->num) {
+                        if (unlikely(vhost_enable_notify(&n->dev, vq))) {
+                                vhost_disable_notify(&n->dev, vq);
+                                continue;
+                        }
+                        break;
+                }
+                if (in) {
+                        vq_err(vq, "Unexpected descriptor format for TX: "
+                               "out %d, int %d\n", out, in);
+                        break;
+                }
+                len = iov_length(vq->iov, out);
+                /* Sanity check */
+                if (!len) {
+                        vq_err(vq, "Unexpected 0 len for TX\n");
+                        break;
+                }
+                vhost_add_used_and_signal(&n->dev, vq, head, 0);
+                total_len += len;
+                if (unlikely(total_len >= VHOST_TEST_WEIGHT)) {
+                        vhost_poll_queue(&vq->poll);
+                        break;
+                }
+        }
+
+        mutex_unlock(&vq->mutex);
+}
+
+static void handle_vq_kick(struct vhost_work *work)
+{
+        struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
+                                                  poll.work);
+        struct vhost_test *n = container_of(vq->dev, struct vhost_test, dev);
+
+        handle_vq(n);
+}
+
+static int vhost_test_open(struct inode *inode, struct file *f)
+{
+        struct vhost_test *n = kmalloc(sizeof *n, GFP_KERNEL);
+        struct vhost_dev *dev;
+        int r;
+
+        if (!n)
+                return -ENOMEM;
+
+        dev = &n->dev;
+        n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick;
+        r = vhost_dev_init(dev, n->vqs, VHOST_TEST_VQ_MAX);
+        if (r < 0) {
+                kfree(n);
+                return r;
+        }
+
+        f->private_data = n;
+
+        return 0;
+}
+
+static void *vhost_test_stop_vq(struct vhost_test *n,
+                                struct vhost_virtqueue *vq)
+{
+        void *private;
+
+        mutex_lock(&vq->mutex);
+        private = rcu_dereference_protected(vq->private_data,
+                                         lockdep_is_held(&vq->mutex));
+        rcu_assign_pointer(vq->private_data, NULL);
+        mutex_unlock(&vq->mutex);
+        return private;
+}
+
+static void vhost_test_stop(struct vhost_test *n, void **privatep)
+{
+        *privatep = vhost_test_stop_vq(n, n->vqs + VHOST_TEST_VQ);
+}
+
+static void vhost_test_flush_vq(struct vhost_test *n, int index)
+{
+        vhost_poll_flush(&n->dev.vqs[index].poll);
+}
+
+static void vhost_test_flush(struct vhost_test *n)
+{
+        vhost_test_flush_vq(n, VHOST_TEST_VQ);
+}
+
+static int vhost_test_release(struct inode *inode, struct file *f)
+{
+        struct vhost_test *n = f->private_data;
+        void  *private;
+
+        vhost_test_stop(n, &private);
+        vhost_test_flush(n);
+        vhost_dev_cleanup(&n->dev);
+        /* We do an extra flush before freeing memory,
+         * since jobs can re-queue themselves. */
+        vhost_test_flush(n);
+        kfree(n);
+        return 0;
+}
+
+static long vhost_test_run(struct vhost_test *n, int test)
+{
+        void *priv, *oldpriv;
+        struct vhost_virtqueue *vq;
+        int r, index;
+
+        if (test < 0 || test > 1)
+                return -EINVAL;
+
+        mutex_lock(&n->dev.mutex);
+        r = vhost_dev_check_owner(&n->dev);
+        if (r)
+                goto err;
+
+        for (index = 0; index < n->dev.nvqs; ++index) {
+                /* Verify that ring has been setup correctly. */
+                if (!vhost_vq_access_ok(&n->vqs[index])) {
+                        r = -EFAULT;
+                        goto err;
+                }
+        }
+
+        for (index = 0; index < n->dev.nvqs; ++index) {
+                vq = n->vqs + index;
+                mutex_lock(&vq->mutex);
+                priv = test ? n : NULL;
+
+                /* start polling new socket */
+                oldpriv = rcu_dereference_protected(vq->private_data,
+                                                    lockdep_is_held(&vq->mutex));
+                rcu_assign_pointer(vq->private_data, priv);
+
+                mutex_unlock(&vq->mutex);
+
+                if (oldpriv) {
+                        vhost_test_flush_vq(n, index);
+                }
+        }
+
+        mutex_unlock(&n->dev.mutex);
+        return 0;
+
+err:
+        mutex_unlock(&n->dev.mutex);
+        return r;
+}
+
+static long vhost_test_reset_owner(struct vhost_test *n)
+{
+        void *priv = NULL;
+        long err;
+        mutex_lock(&n->dev.mutex);
+        err = vhost_dev_check_owner(&n->dev);
+        if (err)
+                goto done;
+        vhost_test_stop(n, &priv);
+        vhost_test_flush(n);
+        err = vhost_dev_reset_owner(&n->dev);
+done:
+        mutex_unlock(&n->dev.mutex);
+        return err;
+}
+
+static int vhost_test_set_features(struct vhost_test *n, u64 features)
+{
+        mutex_lock(&n->dev.mutex);
+        if ((features & (1 << VHOST_F_LOG_ALL)) &&
+            !vhost_log_access_ok(&n->dev)) {
+                mutex_unlock(&n->dev.mutex);
+                return -EFAULT;
+        }
+        n->dev.acked_features = features;
+        smp_wmb();
+        vhost_test_flush(n);
+        mutex_unlock(&n->dev.mutex);
+        return 0;
+}
+
+static long vhost_test_ioctl(struct file *f, unsigned int ioctl,
+                             unsigned long arg)
+{
+        struct vhost_test *n = f->private_data;
+        void __user *argp = (void __user *)arg;
+        u64 __user *featurep = argp;
+        int test;
+        u64 features;
+        int r;
+        switch (ioctl) {
+        case VHOST_TEST_RUN:
+                if (copy_from_user(&test, argp, sizeof test))
+                        return -EFAULT;
+                return vhost_test_run(n, test);
+        case VHOST_GET_FEATURES:
+                features = VHOST_FEATURES;
+                if (copy_to_user(featurep, &features, sizeof features))
+                        return -EFAULT;
+                return 0;
+        case VHOST_SET_FEATURES:
+                if (copy_from_user(&features, featurep, sizeof features))
+                        return -EFAULT;
+                if (features & ~VHOST_FEATURES)
+                        return -EOPNOTSUPP;
+                return vhost_test_set_features(n, features);
+        case VHOST_RESET_OWNER:
+                return vhost_test_reset_owner(n);
+        default:
+                mutex_lock(&n->dev.mutex);
+                r = vhost_dev_ioctl(&n->dev, ioctl, arg);
+                vhost_test_flush(n);
+                mutex_unlock(&n->dev.mutex);
+                return r;
+        }
+}
+
+#ifdef CONFIG_COMPAT
+static long vhost_test_compat_ioctl(struct file *f, unsigned int ioctl,
+                                   unsigned long arg)
+{
+        return vhost_test_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
+}
+#endif
+
+static const struct file_operations vhost_test_fops = {
+        .owner          = THIS_MODULE,
+        .release        = vhost_test_release,
+        .unlocked_ioctl = vhost_test_ioctl,
+#ifdef CONFIG_COMPAT
+        .compat_ioctl   = vhost_test_compat_ioctl,
+#endif
+        .open           = vhost_test_open,
+        .llseek         = noop_llseek,
+};
+
+static struct miscdevice vhost_test_misc = {
+        MISC_DYNAMIC_MINOR,
+        "vhost-test",
+        &vhost_test_fops,
+};
+
+static int vhost_test_init(void)
+{
+        return misc_register(&vhost_test_misc);
+}
+module_init(vhost_test_init);
+
+static void vhost_test_exit(void)
+{
+        misc_deregister(&vhost_test_misc);
+}
+module_exit(vhost_test_exit);
+
+MODULE_VERSION("0.0.1");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Michael S. Tsirkin");
+MODULE_DESCRIPTION("Host kernel side for virtio simulator");
diff --git a/drivers/vhost/test.h b/drivers/vhost/test.h
new file mode 100644
index 000000000000..1fef5df82153
--- /dev/null
+++ b/drivers/vhost/test.h
@@ -0,0 +1,7 @@
+#ifndef LINUX_VHOST_TEST_H
+#define LINUX_VHOST_TEST_H
+
+/* Start a given test on the virtio null device. 0 stops all tests. */
+#define VHOST_TEST_RUN _IOW(VHOST_VIRTIO, 0x31, int)
+
+#endif
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index dd3d6f7406f8..ea966b356352 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -4,7 +4,7 @@
  * Author: Michael S. Tsirkin <mst@redhat.com>
  *
  * Inspiration, some code, and most witty comments come from
- * Documentation/lguest/lguest.c, by Rusty Russell
+ * Documentation/virtual/lguest/lguest.c, by Rusty Russell
  *
  * This work is licensed under the terms of the GNU GPL, version 2.
  *
@@ -15,6 +15,7 @@
 #include <linux/vhost.h>
 #include <linux/virtio_net.h>
 #include <linux/mm.h>
+#include <linux/mmu_context.h>
 #include <linux/miscdevice.h>
 #include <linux/mutex.h>
 #include <linux/rcupdate.h>
@@ -29,8 +30,6 @@
 #include <linux/if_packet.h>
 #include <linux/if_arp.h>
 
-#include <net/sock.h>
-
 #include "vhost.h"
 
 enum {
@@ -38,12 +37,15 @@ enum {
         VHOST_MEMORY_F_LOG = 0x1,
 };
 
+#define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num])
+#define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num])
+
 static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
                             poll_table *pt)
 {
         struct vhost_poll *poll;
-        poll = container_of(pt, struct vhost_poll, table);
 
+        poll = container_of(pt, struct vhost_poll, table);
         poll->wqh = wqh;
         add_wait_queue(wqh, &poll->wait);
 }
@@ -86,6 +88,7 @@ void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
 void vhost_poll_start(struct vhost_poll *poll, struct file *file)
 {
         unsigned long mask;
+
         mask = file->f_op->poll(file, &poll->table);
         if (mask)
                 vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
@@ -98,22 +101,27 @@ void vhost_poll_stop(struct vhost_poll *poll)
         remove_wait_queue(poll->wqh, &poll->wait);
 }
 
+static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work,
+                                unsigned seq)
+{
+        int left;
+
+        spin_lock_irq(&dev->work_lock);
+        left = seq - work->done_seq;
+        spin_unlock_irq(&dev->work_lock);
+        return left <= 0;
+}
+
 static void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
 {
         unsigned seq;
-        int left;
         int flushing;
 
         spin_lock_irq(&dev->work_lock);
         seq = work->queue_seq;
         work->flushing++;
         spin_unlock_irq(&dev->work_lock);
-        wait_event(work->done, ({
-                   spin_lock_irq(&dev->work_lock);
-                   left = seq - work->done_seq <= 0;
-                   spin_unlock_irq(&dev->work_lock);
-                   left;
-        }));
+        wait_event(work->done, vhost_work_seq_done(dev, work, seq));
         spin_lock_irq(&dev->work_lock);
         flushing = --work->flushing;
         spin_unlock_irq(&dev->work_lock);
@@ -156,7 +164,8 @@ static void vhost_vq_reset(struct vhost_dev *dev,
         vq->last_avail_idx = 0;
         vq->avail_idx = 0;
         vq->last_used_idx = 0;
-        vq->used_flags = 0;
+        vq->signalled_used = 0;
+        vq->signalled_used_valid = false;
         vq->used_flags = 0;
         vq->log_used = false;
         vq->log_addr = -1ull;
@@ -178,6 +187,8 @@ static int vhost_worker(void *data)
         struct vhost_work *work = NULL;
         unsigned uninitialized_var(seq);
 
+        use_mm(dev->mm);
+
         for (;;) {
                 /* mb paired w/ kthread_stop */
                 set_current_state(TASK_INTERRUPTIBLE);
@@ -192,7 +203,7 @@ static int vhost_worker(void *data)
                 if (kthread_should_stop()) {
                         spin_unlock_irq(&dev->work_lock);
                         __set_current_state(TASK_RUNNING);
-                        return 0;
+                        break;
                 }
                 if (!list_empty(&dev->work_list)) {
                         work = list_first_entry(&dev->work_list,
@@ -210,6 +221,50 @@ static int vhost_worker(void *data)
                         schedule();
 
         }
+        unuse_mm(dev->mm);
+        return 0;
+}
+
+/* Helper to allocate iovec buffers for all vqs. */
+static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
+{
+        int i;
+
+        for (i = 0; i < dev->nvqs; ++i) {
+                dev->vqs[i].indirect = kmalloc(sizeof *dev->vqs[i].indirect *
+                                               UIO_MAXIOV, GFP_KERNEL);
+                dev->vqs[i].log = kmalloc(sizeof *dev->vqs[i].log * UIO_MAXIOV,
+                                          GFP_KERNEL);
+                dev->vqs[i].heads = kmalloc(sizeof *dev->vqs[i].heads *
+                                            UIO_MAXIOV, GFP_KERNEL);
+
+                if (!dev->vqs[i].indirect || !dev->vqs[i].log ||
+                        !dev->vqs[i].heads)
+                        goto err_nomem;
+        }
+        return 0;
+
+err_nomem:
+        for (; i >= 0; --i) {
+                kfree(dev->vqs[i].indirect);
+                kfree(dev->vqs[i].log);
+                kfree(dev->vqs[i].heads);
+        }
+        return -ENOMEM;
+}
+
+static void vhost_dev_free_iovecs(struct vhost_dev *dev)
+{
+        int i;
+
+        for (i = 0; i < dev->nvqs; ++i) {
+                kfree(dev->vqs[i].indirect);
+                dev->vqs[i].indirect = NULL;
+                kfree(dev->vqs[i].log);
+                dev->vqs[i].log = NULL;
+                kfree(dev->vqs[i].heads);
+                dev->vqs[i].heads = NULL;
+        }
 }
 
 long vhost_dev_init(struct vhost_dev *dev,
@@ -229,6 +284,9 @@ long vhost_dev_init(struct vhost_dev *dev,
         dev->worker = NULL;
 
         for (i = 0; i < dev->nvqs; ++i) {
+                dev->vqs[i].log = NULL;
+                dev->vqs[i].indirect = NULL;
+                dev->vqs[i].heads = NULL;
                 dev->vqs[i].dev = dev;
                 mutex_init(&dev->vqs[i].mutex);
                 vhost_vq_reset(dev, dev->vqs + i);
@@ -248,26 +306,28 @@ long vhost_dev_check_owner(struct vhost_dev *dev)
 }
 
 struct vhost_attach_cgroups_struct {
-        struct vhost_work work;
-        struct task_struct *owner;
-        int ret;
+        struct vhost_work work;
+        struct task_struct *owner;
+        int ret;
 };
 
 static void vhost_attach_cgroups_work(struct vhost_work *work)
 {
-        struct vhost_attach_cgroups_struct *s;
-        s = container_of(work, struct vhost_attach_cgroups_struct, work);
-        s->ret = cgroup_attach_task_all(s->owner, current);
+        struct vhost_attach_cgroups_struct *s;
+
+        s = container_of(work, struct vhost_attach_cgroups_struct, work);
+        s->ret = cgroup_attach_task_all(s->owner, current);
 }
 
 static int vhost_attach_cgroups(struct vhost_dev *dev)
 {
-        struct vhost_attach_cgroups_struct attach;
-        attach.owner = current;
-        vhost_work_init(&attach.work, vhost_attach_cgroups_work);
-        vhost_work_queue(dev, &attach.work);
-        vhost_work_flush(dev, &attach.work);
-        return attach.ret;
+        struct vhost_attach_cgroups_struct attach;
+
+        attach.owner = current;
+        vhost_work_init(&attach.work, vhost_attach_cgroups_work);
+        vhost_work_queue(dev, &attach.work);
+        vhost_work_flush(dev, &attach.work);
+        return attach.ret;
 }
 
 /* Caller should have device mutex */
@@ -275,11 +335,13 @@ static long vhost_dev_set_owner(struct vhost_dev *dev)
 {
         struct task_struct *worker;
         int err;
+
         /* Is there an owner already? */
         if (dev->mm) {
                 err = -EBUSY;
                 goto err_mm;
         }
+
         /* No owner, become one */
         dev->mm = get_task_mm(current);
         worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
@@ -295,6 +357,10 @@ static long vhost_dev_set_owner(struct vhost_dev *dev)
         if (err)
                 goto err_cgroup;
 
+        err = vhost_dev_alloc_iovecs(dev);
+        if (err)
+                goto err_cgroup;
+
         return 0;
 err_cgroup:
         kthread_stop(worker);
@@ -320,7 +386,7 @@ long vhost_dev_reset_owner(struct vhost_dev *dev)
         vhost_dev_cleanup(dev);
 
         memory->nregions = 0;
-        dev->memory = memory;
+        RCU_INIT_POINTER(dev->memory, memory);
         return 0;
 }
 
@@ -328,6 +394,7 @@ long vhost_dev_reset_owner(struct vhost_dev *dev)
 void vhost_dev_cleanup(struct vhost_dev *dev)
 {
         int i;
+
         for (i = 0; i < dev->nvqs; ++i) {
                 if (dev->vqs[i].kick && dev->vqs[i].handle_kick) {
                         vhost_poll_stop(&dev->vqs[i].poll);
@@ -345,6 +412,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev)
                         fput(dev->vqs[i].call);
                 vhost_vq_reset(dev, dev->vqs + i);
         }
+        vhost_dev_free_iovecs(dev);
         if (dev->log_ctx)
                 eventfd_ctx_put(dev->log_ctx);
         dev->log_ctx = NULL;
@@ -352,26 +420,27 @@ void vhost_dev_cleanup(struct vhost_dev *dev)
                 fput(dev->log_file);
         dev->log_file = NULL;
         /* No one will access memory at this point */
-        kfree(dev->memory);
-        dev->memory = NULL;
-        if (dev->mm)
-                mmput(dev->mm);
-        dev->mm = NULL;
-
+        kfree(rcu_dereference_protected(dev->memory,
+                                        lockdep_is_held(&dev->mutex)));
+        RCU_INIT_POINTER(dev->memory, NULL);
         WARN_ON(!list_empty(&dev->work_list));
         if (dev->worker) {
                 kthread_stop(dev->worker);
                 dev->worker = NULL;
         }
+        if (dev->mm)
+                mmput(dev->mm);
+        dev->mm = NULL;
 }
 
 static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
 {
         u64 a = addr / VHOST_PAGE_SIZE / 8;
+
         /* Make sure 64 bit math will not overflow. */
         if (a > ULONG_MAX - (unsigned long)log_base ||
             a + (unsigned long)log_base > ULONG_MAX)
-                return -EFAULT;
+                return 0;
 
         return access_ok(VERIFY_WRITE, log_base + a,
                          (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
@@ -408,6 +477,7 @@ static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem,
                             int log_all)
 {
         int i;
+
         for (i = 0; i < d->nvqs; ++i) {
                 int ok;
                 mutex_lock(&d->vqs[i].mutex);
@@ -424,48 +494,60 @@ static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem,
         return 1;
 }
 
-static int vq_access_ok(unsigned int num,
+static int vq_access_ok(struct vhost_dev *d, unsigned int num,
                         struct vring_desc __user *desc,
                         struct vring_avail __user *avail,
                         struct vring_used __user *used)
 {
+        size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
         return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
                access_ok(VERIFY_READ, avail,
-                         sizeof *avail + num * sizeof *avail->ring) &&
+                         sizeof *avail + num * sizeof *avail->ring + s) &&
                access_ok(VERIFY_WRITE, used,
-                        sizeof *used + num * sizeof *used->ring);
+                        sizeof *used + num * sizeof *used->ring + s);
 }
 
 /* Can we log writes? */
 /* Caller should have device mutex but not vq mutex */
 int vhost_log_access_ok(struct vhost_dev *dev)
 {
-        return memory_access_ok(dev, dev->memory, 1);
+        struct vhost_memory *mp;
+
+        mp = rcu_dereference_protected(dev->memory,
+                                       lockdep_is_held(&dev->mutex));
+        return memory_access_ok(dev, mp, 1);
 }
 
 /* Verify access for write logging. */
 /* Caller should have vq mutex and device mutex */
-static int vq_log_access_ok(struct vhost_virtqueue *vq, void __user *log_base)
+static int vq_log_access_ok(struct vhost_dev *d, struct vhost_virtqueue *vq,
+                            void __user *log_base)
 {
-        return vq_memory_access_ok(log_base, vq->dev->memory,
+        struct vhost_memory *mp;
+        size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
+
+        mp = rcu_dereference_protected(vq->dev->memory,
+                                       lockdep_is_held(&vq->mutex));
+        return vq_memory_access_ok(log_base, mp,
                             vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) &&
                 (!vq->log_used || log_access_ok(log_base, vq->log_addr,
                                         sizeof *vq->used +
-                                        vq->num * sizeof *vq->used->ring));
+                                        vq->num * sizeof *vq->used->ring + s));
 }
 
 /* Can we start vq? */
 /* Caller should have vq mutex and device mutex */
 int vhost_vq_access_ok(struct vhost_virtqueue *vq)
 {
-        return vq_access_ok(vq->num, vq->desc, vq->avail, vq->used) &&
-                vq_log_access_ok(vq, vq->log_base);
+        return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) &&
+                vq_log_access_ok(vq->dev, vq, vq->log_base);
 }
 
 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
 {
         struct vhost_memory mem, *newmem, *oldmem;
         unsigned long size = offsetof(struct vhost_memory, regions);
+
         if (copy_from_user(&mem, m, size))
                 return -EFAULT;
         if (mem.padding)
@@ -483,11 +565,13 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
                 return -EFAULT;
         }
 
-        if (!memory_access_ok(d, newmem, vhost_has_feature(d, VHOST_F_LOG_ALL))) {
+        if (!memory_access_ok(d, newmem,
+                              vhost_has_feature(d, VHOST_F_LOG_ALL))) {
                 kfree(newmem);
                 return -EFAULT;
         }
-        oldmem = d->memory;
+        oldmem = rcu_dereference_protected(d->memory,
+                                           lockdep_is_held(&d->mutex));
         rcu_assign_pointer(d->memory, newmem);
         synchronize_rcu();
         kfree(oldmem);
@@ -498,8 +582,10 @@ static int init_used(struct vhost_virtqueue *vq,
                      struct vring_used __user *used)
 {
         int r = put_user(vq->used_flags, &used->flags);
+
         if (r)
                 return r;
+        vq->signalled_used_valid = false;
         return get_user(vq->last_used_idx, &used->idx);
 }
 
@@ -597,7 +683,7 @@ static long vhost_set_vring(struct vhost_dev *d, int ioctl, void __user *argp)
                  * If it is not, we don't as size might not have been setup.
                  * We will verify when backend is configured. */
                 if (vq->private_data) {
-                        if (!vq_access_ok(vq->num,
+                        if (!vq_access_ok(d, vq->num,
                                 (void __user *)(unsigned long)a.desc_user_addr,
                                 (void __user *)(unsigned long)a.avail_user_addr,
                                 (void __user *)(unsigned long)a.used_user_addr)) {
@@ -741,7 +827,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, unsigned long arg)
                         vq = d->vqs + i;
                         mutex_lock(&vq->mutex);
                         /* If ring is inactive, will check when it's enabled. */
-                        if (vq->private_data && !vq_log_access_ok(vq, base))
+                        if (vq->private_data && !vq_log_access_ok(d, vq, base))
                                 r = -EFAULT;
                         else
                                 vq->log_base = base;
@@ -787,6 +873,7 @@ static const struct vhost_memory_region *find_region(struct vhost_memory *mem,
 {
         struct vhost_memory_region *reg;
         int i;
+
         /* linear search is not brilliant, but we really have on the order of 6
          * regions in practice */
         for (i = 0; i < mem->nregions; ++i) {
@@ -809,6 +896,7 @@ static int set_bit_to_user(int nr, void __user *addr)
         void *base;
         int bit = nr + (log % PAGE_SIZE) * 8;
         int r;
+
         r = get_user_pages_fast(log, 1, 1, &page);
         if (r < 0)
                 return r;
@@ -824,14 +912,16 @@ static int set_bit_to_user(int nr, void __user *addr)
 static int log_write(void __user *log_base,
                      u64 write_address, u64 write_length)
 {
+        u64 write_page = write_address / VHOST_PAGE_SIZE;
         int r;
+
         if (!write_length)
                 return 0;
-        write_address /= VHOST_PAGE_SIZE;
+        write_length += write_address % VHOST_PAGE_SIZE;
         for (;;) {
                 u64 base = (u64)(unsigned long)log_base;
-                u64 log = base + write_address / 8;
-                int bit = write_address % 8;
+                u64 log = base + write_page / 8;
+                int bit = write_page % 8;
                 if ((u64)(unsigned long)log != log)
                         return -EFAULT;
                 r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
@@ -840,7 +930,7 @@ static int log_write(void __user *log_base,
                 if (write_length <= VHOST_PAGE_SIZE)
                         break;
                 write_length -= VHOST_PAGE_SIZE;
-                write_address += VHOST_PAGE_SIZE;
+                write_page += 1;
         }
         return r;
 }
@@ -947,7 +1037,7 @@ static int get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq,
         }
 
         ret = translate_desc(dev, indirect->addr, indirect->len, vq->indirect,
-                             ARRAY_SIZE(vq->indirect));
+                             UIO_MAXIOV);
         if (unlikely(ret < 0)) {
                 vq_err(vq, "Translation failure %d in indirect.\n", ret);
                 return ret;
@@ -974,8 +1064,8 @@ static int get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq,
                                i, count);
                         return -EINVAL;
                 }
-                if (unlikely(memcpy_fromiovec((unsigned char *)&desc, vq->indirect,
-                                              sizeof desc))) {
+                if (unlikely(memcpy_fromiovec((unsigned char *)&desc,
+                                              vq->indirect, sizeof desc))) {
                         vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
                                i, (size_t)indirect->addr + i * sizeof desc);
                         return -EINVAL;
@@ -1035,7 +1125,7 @@ int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
 
         /* Check it isn't doing very strange things with descriptor numbers. */
         last_avail_idx = vq->last_avail_idx;
-        if (unlikely(get_user(vq->avail_idx, &vq->avail->idx))) {
+        if (unlikely(__get_user(vq->avail_idx, &vq->avail->idx))) {
                 vq_err(vq, "Failed to access avail idx at %p\n",
                        &vq->avail->idx);
                 return -EFAULT;
@@ -1056,8 +1146,8 @@ int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
 
         /* Grab the next descriptor number they're advertising, and increment
          * the index we've seen. */
-        if (unlikely(get_user(head,
-                              &vq->avail->ring[last_avail_idx % vq->num]))) {
+        if (unlikely(__get_user(head,
+                                &vq->avail->ring[last_avail_idx % vq->num]))) {
                 vq_err(vq, "Failed to read head: idx %d address %p\n",
                        last_avail_idx,
                        &vq->avail->ring[last_avail_idx % vq->num]);
@@ -1090,7 +1180,7 @@ int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
                                i, vq->num, head);
                         return -EINVAL;
                 }
-                ret = copy_from_user(&desc, vq->desc + i, sizeof desc);
+                ret = __copy_from_user(&desc, vq->desc + i, sizeof desc);
                 if (unlikely(ret)) {
                         vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
                                i, vq->desc + i);
@@ -1138,6 +1228,10 @@ int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
 
         /* On success, increment avail index. */
         vq->last_avail_idx++;
+
+        /* Assume notifications from guest are disabled at this point,
+         * if they aren't we would need to update avail_event index. */
+        BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
         return head;
 }
 
@@ -1156,17 +1250,17 @@ int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
         /* The virtqueue contains a ring of used buffers.  Get a pointer to the
          * next entry in that used ring. */
         used = &vq->used->ring[vq->last_used_idx % vq->num];
-        if (put_user(head, &used->id)) {
+        if (__put_user(head, &used->id)) {
                 vq_err(vq, "Failed to write used id");
                 return -EFAULT;
         }
-        if (put_user(len, &used->len)) {
+        if (__put_user(len, &used->len)) {
                 vq_err(vq, "Failed to write used len");
                 return -EFAULT;
         }
         /* Make sure buffer is written before we update index. */
         smp_wmb();
-        if (put_user(vq->last_used_idx + 1, &vq->used->idx)) {
+        if (__put_user(vq->last_used_idx + 1, &vq->used->idx)) {
                 vq_err(vq, "Failed to increment used idx");
                 return -EFAULT;
         }
@@ -1186,6 +1280,12 @@ int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
                         eventfd_signal(vq->log_ctx, 1);
         }
         vq->last_used_idx++;
+        /* If the driver never bothers to signal in a very long while,
+         * used index might wrap around. If that happens, invalidate
+         * signalled_used index we stored. TODO: make sure driver
+         * signals at least once in 2^16 and remove this. */
+        if (unlikely(vq->last_used_idx == vq->signalled_used))
+                vq->signalled_used_valid = false;
         return 0;
 }
 
@@ -1194,11 +1294,12 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq,
                             unsigned count)
 {
         struct vring_used_elem __user *used;
+        u16 old, new;
         int start;
 
         start = vq->last_used_idx % vq->num;
         used = vq->used->ring + start;
-        if (copy_to_user(used, heads, count * sizeof *used)) {
+        if (__copy_to_user(used, heads, count * sizeof *used)) {
                 vq_err(vq, "Failed to write used");
                 return -EFAULT;
         }
@@ -1211,7 +1312,14 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq,
                            ((void __user *)used - (void __user *)vq->used),
                           count * sizeof *used);
         }
-        vq->last_used_idx += count;
+        old = vq->last_used_idx;
+        new = (vq->last_used_idx += count);
+        /* If the driver never bothers to signal in a very long while,
+         * used index might wrap around. If that happens, invalidate
+         * signalled_used index we stored. TODO: make sure driver
+         * signals at least once in 2^16 and remove this. */
+        if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
+                vq->signalled_used_valid = false;
         return 0;
 }
 
@@ -1250,28 +1358,47 @@ int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
         return r;
 }
 
-/* This actually signals the guest, using eventfd. */
-void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
+static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
 {
-        __u16 flags;
+        __u16 old, new, event;
+        bool v;
         /* Flush out used index updates. This is paired
          * with the barrier that the Guest executes when enabling
          * interrupts. */
         smp_mb();
 
-        if (get_user(flags, &vq->avail->flags)) {
-                vq_err(vq, "Failed to get flags");
-                return;
+        if (vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
+            unlikely(vq->avail_idx == vq->last_avail_idx))
+                return true;
+
+        if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
+                __u16 flags;
+                if (__get_user(flags, &vq->avail->flags)) {
+                        vq_err(vq, "Failed to get flags");
+                        return true;
+                }
+                return !(flags & VRING_AVAIL_F_NO_INTERRUPT);
         }
+        old = vq->signalled_used;
+        v = vq->signalled_used_valid;
+        new = vq->signalled_used = vq->last_used_idx;
+        vq->signalled_used_valid = true;
 
-        /* If they don't want an interrupt, don't signal, unless empty. */
-        if ((flags & VRING_AVAIL_F_NO_INTERRUPT) &&
-            (vq->avail_idx != vq->last_avail_idx ||
-             !vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY)))
-                return;
+        if (unlikely(!v))
+                return true;
+
+        if (get_user(event, vhost_used_event(vq))) {
+                vq_err(vq, "Failed to get used event idx");
+                return true;
+        }
+        return vring_need_event(event, new, old);
+}
 
+/* This actually signals the guest, using eventfd. */
+void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
+{
         /* Signal the Guest tell them we used something up. */
-        if (vq->call_ctx)
+        if (vq->call_ctx && vhost_notify(dev, vq))
                 eventfd_signal(vq->call_ctx, 1);
 }
 
@@ -1294,23 +1421,47 @@ void vhost_add_used_and_signal_n(struct vhost_dev *dev,
 }
 
 /* OK, now we need to know about added descriptors. */
-bool vhost_enable_notify(struct vhost_virtqueue *vq)
+bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
 {
         u16 avail_idx;
         int r;
+
         if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
                 return false;
         vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
-        r = put_user(vq->used_flags, &vq->used->flags);
-        if (r) {
-                vq_err(vq, "Failed to enable notification at %p: %d\n",
-                       &vq->used->flags, r);
-                return false;
+        if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
+                r = put_user(vq->used_flags, &vq->used->flags);
+                if (r) {
+                        vq_err(vq, "Failed to enable notification at %p: %d\n",
+                               &vq->used->flags, r);
+                        return false;
+                }
+        } else {
+                r = put_user(vq->avail_idx, vhost_avail_event(vq));
+                if (r) {
+                        vq_err(vq, "Failed to update avail event index at %p: %d\n",
+                               vhost_avail_event(vq), r);
+                        return false;
+                }
+        }
+        if (unlikely(vq->log_used)) {
+                void __user *used;
+                /* Make sure data is seen before log. */
+                smp_wmb();
+                used = vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX) ?
+                        &vq->used->flags : vhost_avail_event(vq);
+                /* Log used flags or event index entry write. Both are 16 bit
+                 * fields. */
+                log_write(vq->log_base, vq->log_addr +
+                           (used - (void __user *)vq->used),
+                          sizeof(u16));
+                if (vq->log_ctx)
+                        eventfd_signal(vq->log_ctx, 1);
         }
         /* They could have slipped one in as we were doing that: make
          * sure it's written, then check again. */
         smp_mb();
-        r = get_user(avail_idx, &vq->avail->idx);
+        r = __get_user(avail_idx, &vq->avail->idx);
         if (r) {
                 vq_err(vq, "Failed to check avail idx at %p: %d\n",
                        &vq->avail->idx, r);
@@ -1321,14 +1472,17 @@ bool vhost_enable_notify(struct vhost_virtqueue *vq)
 }
 
 /* We don't need to be notified again. */
-void vhost_disable_notify(struct vhost_virtqueue *vq)
+void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
 {
         int r;
+
         if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
                 return;
         vq->used_flags |= VRING_USED_F_NO_NOTIFY;
-        r = put_user(vq->used_flags, &vq->used->flags);
-        if (r)
-                vq_err(vq, "Failed to enable notification at %p: %d\n",
-                       &vq->used->flags, r);
+        if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
+                r = put_user(vq->used_flags, &vq->used->flags);
+                if (r)
+                        vq_err(vq, "Failed to enable notification at %p: %d\n",
+                               &vq->used->flags, r);
+        }
 }
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index afd77295971c..8e03379dd30f 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -15,11 +15,6 @@
 
 struct vhost_device;
 
-enum {
-        /* Enough place for all fragments, head, and virtio net header. */
-        VHOST_NET_MAX_SG = MAX_SKB_FRAGS + 2,
-};
-
 struct vhost_work;
 typedef void (*vhost_work_fn_t)(struct vhost_work *work);
 
@@ -89,34 +84,43 @@ struct vhost_virtqueue {
         /* Used flags */
         u16 used_flags;
 
+        /* Last used index value we have signalled on */
+        u16 signalled_used;
+
+        /* Last used index value we have signalled on */
+        bool signalled_used_valid;
+
         /* Log writes to used structure. */
         bool log_used;
         u64 log_addr;
 
-        struct iovec indirect[VHOST_NET_MAX_SG];
-        struct iovec iov[VHOST_NET_MAX_SG];
-        struct iovec hdr[VHOST_NET_MAX_SG];
+        struct iovec iov[UIO_MAXIOV];
+        /* hdr is used to store the virtio header.
+         * Since each iovec has >= 1 byte length, we never need more than
+         * header length entries to store the header. */
+        struct iovec hdr[sizeof(struct virtio_net_hdr_mrg_rxbuf)];
+        struct iovec *indirect;
         size_t vhost_hlen;
         size_t sock_hlen;
-        struct vring_used_elem heads[VHOST_NET_MAX_SG];
+        struct vring_used_elem *heads;
         /* We use a kind of RCU to access private pointer.
          * All readers access it from worker, which makes it possible to
          * flush the vhost_work instead of synchronize_rcu. Therefore readers do
          * not need to call rcu_read_lock/rcu_read_unlock: the beginning of
          * vhost_work execution acts instead of rcu_read_lock() and the end of
-         * vhost_work execution acts instead of rcu_read_lock().
+         * vhost_work execution acts instead of rcu_read_unlock().
          * Writers use virtqueue mutex. */
-        void *private_data;
+        void __rcu *private_data;
         /* Log write descriptors */
         void __user *log_base;
-        struct vhost_log log[VHOST_NET_MAX_SG];
+        struct vhost_log *log;
 };
 
 struct vhost_dev {
         /* Readers use RCU to access memory table pointer
          * log base pointer and features.
          * Writers use mutex below.*/
-        struct vhost_memory *memory;
+        struct vhost_memory __rcu *memory;
         struct mm_struct *mm;
         struct mutex mutex;
         unsigned acked_features;
@@ -151,8 +155,8 @@ void vhost_add_used_and_signal(struct vhost_dev *, struct vhost_virtqueue *,
 void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *,
                                struct vring_used_elem *heads, unsigned count);
 void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *);
-void vhost_disable_notify(struct vhost_virtqueue *);
-bool vhost_enable_notify(struct vhost_virtqueue *);
+void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *);
+bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *);
 
 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
                     unsigned int log_num, u64 len);
@@ -164,16 +168,21 @@ int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
         } while (0)
 
 enum {
-        VHOST_FEATURES = (1 << VIRTIO_F_NOTIFY_ON_EMPTY) |
-                         (1 << VIRTIO_RING_F_INDIRECT_DESC) |
-                         (1 << VHOST_F_LOG_ALL) |
-                         (1 << VHOST_NET_F_VIRTIO_NET_HDR) |
-                         (1 << VIRTIO_NET_F_MRG_RXBUF),
+        VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) |
+                         (1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
+                         (1ULL << VIRTIO_RING_F_EVENT_IDX) |
+                         (1ULL << VHOST_F_LOG_ALL) |
+                         (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
+                         (1ULL << VIRTIO_NET_F_MRG_RXBUF),
 };
 
 static inline int vhost_has_feature(struct vhost_dev *dev, int bit)
 {
-        unsigned acked_features = rcu_dereference(dev->acked_features);
+        unsigned acked_features;
+
+        /* TODO: check that we are running from vhost_worker or dev mutex is
+         * held? */
+        acked_features = rcu_dereference_index_check(dev->acked_features, 1);
         return acked_features & (1 << bit);
 }