virtio-net: put virtio net header inline with data

For small packets we can simplify xmit processing
by linearizing buffers with the header:
most packets seem to have enough head room
we can use for this purpose.
Since existing hypervisors require that header
is the first s/g element, we need a feature bit
for this.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 34 insertions, 8 deletions

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 3d2a90a62649..f21600277583 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -106,6 +106,9 @@ struct virtnet_info {
         /* Has control virtqueue */
         bool has_cvq;
 
+        /* Host can handle any s/g split between our header and packet data */
+        bool any_header_sg;
+
         /* enable config space updates */
         bool config_enable;
 
@@ -669,12 +672,28 @@ static void free_old_xmit_skbs(struct send_queue *sq)
 
 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
 {
-        struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
+        struct skb_vnet_hdr *hdr;
         const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
         struct virtnet_info *vi = sq->vq->vdev->priv;
         unsigned num_sg;
+        unsigned hdr_len;
+        bool can_push;
 
         pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
+        if (vi->mergeable_rx_bufs)
+                hdr_len = sizeof hdr->mhdr;
+        else
+                hdr_len = sizeof hdr->hdr;
+
+        can_push = vi->any_header_sg &&
+                !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
+                !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
+        /* Even if we can, don't push here yet as this would skew
+         * csum_start offset below. */
+        if (can_push)
+                hdr = (struct skb_vnet_hdr *)(skb->data - hdr_len);
+        else
+                hdr = skb_vnet_hdr(skb);
 
         if (skb->ip_summed == CHECKSUM_PARTIAL) {
                 hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
@@ -703,15 +722,18 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
                 hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
         }
 
-        hdr->mhdr.num_buffers = 0;
-
-        /* Encode metadata header at front. */
         if (vi->mergeable_rx_bufs)
-                sg_set_buf(sq->sg, &hdr->mhdr, sizeof hdr->mhdr);
-        else
-                sg_set_buf(sq->sg, &hdr->hdr, sizeof hdr->hdr);
+                hdr->mhdr.num_buffers = 0;
 
-        num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
+        if (can_push) {
+                __skb_push(skb, hdr_len);
+                num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
+                /* Pull header back to avoid skew in tx bytes calculations. */
+                __skb_pull(skb, hdr_len);
+        } else {
+                sg_set_buf(sq->sg, hdr, hdr_len);
+                num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
+        }
         return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
 }
 
@@ -1552,6 +1574,9 @@ static int virtnet_probe(struct virtio_device *vdev)
         if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
                 vi->mergeable_rx_bufs = true;
 
+        if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT))
+                vi->any_header_sg = true;
+
         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
                 vi->has_cvq = true;
 
@@ -1727,6 +1752,7 @@ static unsigned int features[] = {
         VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
         VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ,
         VIRTIO_NET_F_CTRL_MAC_ADDR,
+        VIRTIO_F_ANY_LAYOUT,
 };
 
 static struct virtio_driver virtio_net_driver = {