veth: Add driver XDP

This is the basic implementation of veth driver XDP.

Incoming packets are sent from the peer veth device in the form of skb,
so this is generally doing the same thing as generic XDP.

This itself is not so useful, but a starting point to implement other
useful veth XDP features like TX and REDIRECT.

This introduces NAPI when XDP is enabled, because XDP is now heavily
relies on NAPI context. Use ptr_ring to emulate NIC ring. Tx function
enqueues packets to the ring and peer NAPI handler drains the ring.

Currently only one ring is allocated for each veth device, so it does
not scale on multiqueue env. This can be resolved by allocating rings
on the per-queue basis later.

Note that NAPI is not used but netif_rx is used when XDP is not loaded,
so this does not change the default behaviour.

v6:
- Check skb->len only when allocation is needed.
- Add __GFP_NOWARN to alloc_page() as it can be triggered by external
  events.

v3:
- Fix race on closing the device.
- Add extack messages in ndo_bpf.

v2:
- Squashed with the patch adding NAPI.
- Implement adjust_tail.
- Don't acquire consumer lock because it is guarded by NAPI.
- Make poll_controller noop since it is unnecessary.
- Register rxq_info on enabling XDP rather than on opening the device.

Signed-off-by: Toshiaki Makita <makita.toshiaki@lab.ntt.co.jp>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

1 files changed, 367 insertions, 7 deletions

diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index a69ad39ee57e..d3b9f10bea24 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -19,10 +19,18 @@
 #include <net/xfrm.h>
 #include <linux/veth.h>
 #include <linux/module.h>
+#include <linux/bpf.h>
+#include <linux/filter.h>
+#include <linux/ptr_ring.h>
+#include <linux/skb_array.h>
+#include <linux/bpf_trace.h>
 
 #define DRV_NAME        "veth"
 #define DRV_VERSION     "1.0"
 
+#define VETH_RING_SIZE          256
+#define VETH_XDP_HEADROOM       (XDP_PACKET_HEADROOM + NET_IP_ALIGN)
+
 struct pcpu_vstats {
         u64                     packets;
         u64                     bytes;
@@ -30,9 +38,16 @@ struct pcpu_vstats {
 };
 
 struct veth_priv {
+        struct napi_struct      xdp_napi;
+        struct net_device       *dev;
+        struct bpf_prog __rcu   *xdp_prog;
+        struct bpf_prog         *_xdp_prog;
         struct net_device __rcu *peer;
         atomic64_t              dropped;
         unsigned                requested_headroom;
+        bool                    rx_notify_masked;
+        struct ptr_ring         xdp_ring;
+        struct xdp_rxq_info     xdp_rxq;
 };
 
 /*
@@ -98,11 +113,43 @@ static const struct ethtool_ops veth_ethtool_ops = {
         .get_link_ksettings     = veth_get_link_ksettings,
 };
 
-static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
+/* general routines */
+
+static void __veth_xdp_flush(struct veth_priv *priv)
+{
+        /* Write ptr_ring before reading rx_notify_masked */
+        smp_mb();
+        if (!priv->rx_notify_masked) {
+                priv->rx_notify_masked = true;
+                napi_schedule(&priv->xdp_napi);
+        }
+}
+
+static int veth_xdp_rx(struct veth_priv *priv, struct sk_buff *skb)
+{
+        if (unlikely(ptr_ring_produce(&priv->xdp_ring, skb))) {
+                dev_kfree_skb_any(skb);
+                return NET_RX_DROP;
+        }
+
+        return NET_RX_SUCCESS;
+}
+
+static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, bool xdp)
 {
         struct veth_priv *priv = netdev_priv(dev);
+
+        return __dev_forward_skb(dev, skb) ?: xdp ?
+                veth_xdp_rx(priv, skb) :
+                netif_rx(skb);
+}
+
+static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+        struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
         struct net_device *rcv;
         int length = skb->len;
+        bool rcv_xdp = false;
 
         rcu_read_lock();
         rcv = rcu_dereference(priv->peer);
@@ -111,7 +158,10 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
                 goto drop;
         }
 
-        if (likely(dev_forward_skb(rcv, skb) == NET_RX_SUCCESS)) {
+        rcv_priv = netdev_priv(rcv);
+        rcv_xdp = rcu_access_pointer(rcv_priv->xdp_prog);
+
+        if (likely(veth_forward_skb(rcv, skb, rcv_xdp) == NET_RX_SUCCESS)) {
                 struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats);
 
                 u64_stats_update_begin(&stats->syncp);
@@ -122,14 +172,15 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
 drop:
                 atomic64_inc(&priv->dropped);
         }
+
+        if (rcv_xdp)
+                __veth_xdp_flush(rcv_priv);
+
         rcu_read_unlock();
+
         return NETDEV_TX_OK;
 }
 
-/*
- * general routines
- */
-
 static u64 veth_stats_one(struct pcpu_vstats *result, struct net_device *dev)
 {
         struct veth_priv *priv = netdev_priv(dev);
@@ -179,18 +230,254 @@ static void veth_set_multicast_list(struct net_device *dev)
 {
 }
 
+static struct sk_buff *veth_build_skb(void *head, int headroom, int len,
+                                      int buflen)
+{
+        struct sk_buff *skb;
+
+        if (!buflen) {
+                buflen = SKB_DATA_ALIGN(headroom + len) +
+                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+        }
+        skb = build_skb(head, buflen);
+        if (!skb)
+                return NULL;
+
+        skb_reserve(skb, headroom);
+        skb_put(skb, len);
+
+        return skb;
+}
+
+static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv,
+                                        struct sk_buff *skb)
+{
+        u32 pktlen, headroom, act, metalen;
+        void *orig_data, *orig_data_end;
+        struct bpf_prog *xdp_prog;
+        int mac_len, delta, off;
+        struct xdp_buff xdp;
+
+        rcu_read_lock();
+        xdp_prog = rcu_dereference(priv->xdp_prog);
+        if (unlikely(!xdp_prog)) {
+                rcu_read_unlock();
+                goto out;
+        }
+
+        mac_len = skb->data - skb_mac_header(skb);
+        pktlen = skb->len + mac_len;
+        headroom = skb_headroom(skb) - mac_len;
+
+        if (skb_shared(skb) || skb_head_is_locked(skb) ||
+            skb_is_nonlinear(skb) || headroom < XDP_PACKET_HEADROOM) {
+                struct sk_buff *nskb;
+                int size, head_off;
+                void *head, *start;
+                struct page *page;
+
+                size = SKB_DATA_ALIGN(VETH_XDP_HEADROOM + pktlen) +
+                       SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+                if (size > PAGE_SIZE)
+                        goto drop;
+
+                page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
+                if (!page)
+                        goto drop;
+
+                head = page_address(page);
+                start = head + VETH_XDP_HEADROOM;
+                if (skb_copy_bits(skb, -mac_len, start, pktlen)) {
+                        page_frag_free(head);
+                        goto drop;
+                }
+
+                nskb = veth_build_skb(head,
+                                      VETH_XDP_HEADROOM + mac_len, skb->len,
+                                      PAGE_SIZE);
+                if (!nskb) {
+                        page_frag_free(head);
+                        goto drop;
+                }
+
+                skb_copy_header(nskb, skb);
+                head_off = skb_headroom(nskb) - skb_headroom(skb);
+                skb_headers_offset_update(nskb, head_off);
+                if (skb->sk)
+                        skb_set_owner_w(nskb, skb->sk);
+                consume_skb(skb);
+                skb = nskb;
+        }
+
+        xdp.data_hard_start = skb->head;
+        xdp.data = skb_mac_header(skb);
+        xdp.data_end = xdp.data + pktlen;
+        xdp.data_meta = xdp.data;
+        xdp.rxq = &priv->xdp_rxq;
+        orig_data = xdp.data;
+        orig_data_end = xdp.data_end;
+
+        act = bpf_prog_run_xdp(xdp_prog, &xdp);
+
+        switch (act) {
+        case XDP_PASS:
+                break;
+        default:
+                bpf_warn_invalid_xdp_action(act);
+        case XDP_ABORTED:
+                trace_xdp_exception(priv->dev, xdp_prog, act);
+        case XDP_DROP:
+                goto drop;
+        }
+        rcu_read_unlock();
+
+        delta = orig_data - xdp.data;
+        off = mac_len + delta;
+        if (off > 0)
+                __skb_push(skb, off);
+        else if (off < 0)
+                __skb_pull(skb, -off);
+        skb->mac_header -= delta;
+        off = xdp.data_end - orig_data_end;
+        if (off != 0)
+                __skb_put(skb, off);
+        skb->protocol = eth_type_trans(skb, priv->dev);
+
+        metalen = xdp.data - xdp.data_meta;
+        if (metalen)
+                skb_metadata_set(skb, metalen);
+out:
+        return skb;
+drop:
+        rcu_read_unlock();
+        kfree_skb(skb);
+        return NULL;
+}
+
+static int veth_xdp_rcv(struct veth_priv *priv, int budget)
+{
+        int i, done = 0;
+
+        for (i = 0; i < budget; i++) {
+                struct sk_buff *skb = __ptr_ring_consume(&priv->xdp_ring);
+
+                if (!skb)
+                        break;
+
+                skb = veth_xdp_rcv_skb(priv, skb);
+
+                if (skb)
+                        napi_gro_receive(&priv->xdp_napi, skb);
+
+                done++;
+        }
+
+        return done;
+}
+
+static int veth_poll(struct napi_struct *napi, int budget)
+{
+        struct veth_priv *priv =
+                container_of(napi, struct veth_priv, xdp_napi);
+        int done;
+
+        done = veth_xdp_rcv(priv, budget);
+
+        if (done < budget && napi_complete_done(napi, done)) {
+                /* Write rx_notify_masked before reading ptr_ring */
+                smp_store_mb(priv->rx_notify_masked, false);
+                if (unlikely(!__ptr_ring_empty(&priv->xdp_ring))) {
+                        priv->rx_notify_masked = true;
+                        napi_schedule(&priv->xdp_napi);
+                }
+        }
+
+        return done;
+}
+
+static int veth_napi_add(struct net_device *dev)
+{
+        struct veth_priv *priv = netdev_priv(dev);
+        int err;
+
+        err = ptr_ring_init(&priv->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);
+        if (err)
+                return err;
+
+        netif_napi_add(dev, &priv->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
+        napi_enable(&priv->xdp_napi);
+
+        return 0;
+}
+
+static void veth_napi_del(struct net_device *dev)
+{
+        struct veth_priv *priv = netdev_priv(dev);
+
+        napi_disable(&priv->xdp_napi);
+        netif_napi_del(&priv->xdp_napi);
+        priv->rx_notify_masked = false;
+        ptr_ring_cleanup(&priv->xdp_ring, __skb_array_destroy_skb);
+}
+
+static int veth_enable_xdp(struct net_device *dev)
+{
+        struct veth_priv *priv = netdev_priv(dev);
+        int err;
+
+        if (!xdp_rxq_info_is_reg(&priv->xdp_rxq)) {
+                err = xdp_rxq_info_reg(&priv->xdp_rxq, dev, 0);
+                if (err < 0)
+                        return err;
+
+                err = xdp_rxq_info_reg_mem_model(&priv->xdp_rxq,
+                                                 MEM_TYPE_PAGE_SHARED, NULL);
+                if (err < 0)
+                        goto err;
+
+                err = veth_napi_add(dev);
+                if (err)
+                        goto err;
+        }
+
+        rcu_assign_pointer(priv->xdp_prog, priv->_xdp_prog);
+
+        return 0;
+err:
+        xdp_rxq_info_unreg(&priv->xdp_rxq);
+
+        return err;
+}
+
+static void veth_disable_xdp(struct net_device *dev)
+{
+        struct veth_priv *priv = netdev_priv(dev);
+
+        rcu_assign_pointer(priv->xdp_prog, NULL);
+        veth_napi_del(dev);
+        xdp_rxq_info_unreg(&priv->xdp_rxq);
+}
+
 static int veth_open(struct net_device *dev)
 {
         struct veth_priv *priv = netdev_priv(dev);
         struct net_device *peer = rtnl_dereference(priv->peer);
+        int err;
 
         if (!peer)
                 return -ENOTCONN;
 
+        if (priv->_xdp_prog) {
+                err = veth_enable_xdp(dev);
+                if (err)
+                        return err;
+        }
+
         if (peer->flags & IFF_UP) {
                 netif_carrier_on(dev);
                 netif_carrier_on(peer);
         }
+
         return 0;
 }
 
@@ -203,6 +490,9 @@ static int veth_close(struct net_device *dev)
         if (peer)
                 netif_carrier_off(peer);
 
+        if (priv->_xdp_prog)
+                veth_disable_xdp(dev);
+
         return 0;
 }
 
@@ -228,7 +518,7 @@ static void veth_dev_free(struct net_device *dev)
 static void veth_poll_controller(struct net_device *dev)
 {
         /* veth only receives frames when its peer sends one
-         * Since it's a synchronous operation, we are guaranteed
+         * Since it has nothing to do with disabling irqs, we are guaranteed
          * never to have pending data when we poll for it so
          * there is nothing to do here.
          *
@@ -276,6 +566,72 @@ out:
         rcu_read_unlock();
 }
 
+static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog,
+                        struct netlink_ext_ack *extack)
+{
+        struct veth_priv *priv = netdev_priv(dev);
+        struct bpf_prog *old_prog;
+        struct net_device *peer;
+        int err;
+
+        old_prog = priv->_xdp_prog;
+        priv->_xdp_prog = prog;
+        peer = rtnl_dereference(priv->peer);
+
+        if (prog) {
+                if (!peer) {
+                        NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached");
+                        err = -ENOTCONN;
+                        goto err;
+                }
+
+                if (dev->flags & IFF_UP) {
+                        err = veth_enable_xdp(dev);
+                        if (err) {
+                                NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed");
+                                goto err;
+                        }
+                }
+        }
+
+        if (old_prog) {
+                if (!prog && dev->flags & IFF_UP)
+                        veth_disable_xdp(dev);
+                bpf_prog_put(old_prog);
+        }
+
+        return 0;
+err:
+        priv->_xdp_prog = old_prog;
+
+        return err;
+}
+
+static u32 veth_xdp_query(struct net_device *dev)
+{
+        struct veth_priv *priv = netdev_priv(dev);
+        const struct bpf_prog *xdp_prog;
+
+        xdp_prog = priv->_xdp_prog;
+        if (xdp_prog)
+                return xdp_prog->aux->id;
+
+        return 0;
+}
+
+static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp)
+{
+        switch (xdp->command) {
+        case XDP_SETUP_PROG:
+                return veth_xdp_set(dev, xdp->prog, xdp->extack);
+        case XDP_QUERY_PROG:
+                xdp->prog_id = veth_xdp_query(dev);
+                return 0;
+        default:
+                return -EINVAL;
+        }
+}
+
 static const struct net_device_ops veth_netdev_ops = {
         .ndo_init            = veth_dev_init,
         .ndo_open            = veth_open,
@@ -290,6 +646,7 @@ static const struct net_device_ops veth_netdev_ops = {
         .ndo_get_iflink         = veth_get_iflink,
         .ndo_features_check     = passthru_features_check,
         .ndo_set_rx_headroom    = veth_set_rx_headroom,
+        .ndo_bpf                = veth_xdp,
 };
 
 #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \
@@ -451,10 +808,13 @@ static int veth_newlink(struct net *src_net, struct net_device *dev,
          */
 
         priv = netdev_priv(dev);
+        priv->dev = dev;
         rcu_assign_pointer(priv->peer, peer);
 
         priv = netdev_priv(peer);
+        priv->dev = peer;
         rcu_assign_pointer(priv->peer, dev);
+
         return 0;
 
 err_register_dev: