1 files changed, 150 insertions, 5 deletions

diff --git a/net/core/dev.c b/net/core/dev.c
index db6e31564d06..1b3317c026c6 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -95,6 +95,7 @@
 #include <linux/notifier.h>
 #include <linux/skbuff.h>
 #include <linux/bpf.h>
+#include <linux/bpf_trace.h>
 #include <net/net_namespace.h>
 #include <net/sock.h>
 #include <net/busy_poll.h>
@@ -4251,6 +4252,125 @@ static int __netif_receive_skb(struct sk_buff *skb)
         return ret;
 }
 
+static struct static_key generic_xdp_needed __read_mostly;
+
+static int generic_xdp_install(struct net_device *dev, struct netdev_xdp *xdp)
+{
+        struct bpf_prog *new = xdp->prog;
+        int ret = 0;
+
+        switch (xdp->command) {
+        case XDP_SETUP_PROG: {
+                struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
+
+                rcu_assign_pointer(dev->xdp_prog, new);
+                if (old)
+                        bpf_prog_put(old);
+
+                if (old && !new) {
+                        static_key_slow_dec(&generic_xdp_needed);
+                } else if (new && !old) {
+                        static_key_slow_inc(&generic_xdp_needed);
+                        dev_disable_lro(dev);
+                }
+                break;
+        }
+
+        case XDP_QUERY_PROG:
+                xdp->prog_attached = !!rcu_access_pointer(dev->xdp_prog);
+                break;
+
+        default:
+                ret = -EINVAL;
+                break;
+        }
+
+        return ret;
+}
+
+static u32 netif_receive_generic_xdp(struct sk_buff *skb,
+                                     struct bpf_prog *xdp_prog)
+{
+        struct xdp_buff xdp;
+        u32 act = XDP_DROP;
+        void *orig_data;
+        int hlen, off;
+        u32 mac_len;
+
+        /* Reinjected packets coming from act_mirred or similar should
+         * not get XDP generic processing.
+         */
+        if (skb_cloned(skb))
+                return XDP_PASS;
+
+        if (skb_linearize(skb))
+                goto do_drop;
+
+        /* The XDP program wants to see the packet starting at the MAC
+         * header.
+         */
+        mac_len = skb->data - skb_mac_header(skb);
+        hlen = skb_headlen(skb) + mac_len;
+        xdp.data = skb->data - mac_len;
+        xdp.data_end = xdp.data + hlen;
+        xdp.data_hard_start = skb->data - skb_headroom(skb);
+        orig_data = xdp.data;
+
+        act = bpf_prog_run_xdp(xdp_prog, &xdp);
+
+        off = xdp.data - orig_data;
+        if (off > 0)
+                __skb_pull(skb, off);
+        else if (off < 0)
+                __skb_push(skb, -off);
+
+        switch (act) {
+        case XDP_TX:
+                __skb_push(skb, mac_len);
+                /* fall through */
+        case XDP_PASS:
+                break;
+
+        default:
+                bpf_warn_invalid_xdp_action(act);
+                /* fall through */
+        case XDP_ABORTED:
+                trace_xdp_exception(skb->dev, xdp_prog, act);
+                /* fall through */
+        case XDP_DROP:
+        do_drop:
+                kfree_skb(skb);
+                break;
+        }
+
+        return act;
+}
+
+/* When doing generic XDP we have to bypass the qdisc layer and the
+ * network taps in order to match in-driver-XDP behavior.
+ */
+static void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
+{
+        struct net_device *dev = skb->dev;
+        struct netdev_queue *txq;
+        bool free_skb = true;
+        int cpu, rc;
+
+        txq = netdev_pick_tx(dev, skb, NULL);
+        cpu = smp_processor_id();
+        HARD_TX_LOCK(dev, txq, cpu);
+        if (!netif_xmit_stopped(txq)) {
+                rc = netdev_start_xmit(skb, dev, txq, 0);
+                if (dev_xmit_complete(rc))
+                        free_skb = false;
+        }
+        HARD_TX_UNLOCK(dev, txq);
+        if (free_skb) {
+                trace_xdp_exception(dev, xdp_prog, XDP_TX);
+                kfree_skb(skb);
+        }
+}
+
 static int netif_receive_skb_internal(struct sk_buff *skb)
 {
         int ret;
@@ -4262,6 +4382,21 @@ static int netif_receive_skb_internal(struct sk_buff *skb)
 
         rcu_read_lock();
 
+        if (static_key_false(&generic_xdp_needed)) {
+                struct bpf_prog *xdp_prog = rcu_dereference(skb->dev->xdp_prog);
+
+                if (xdp_prog) {
+                        u32 act = netif_receive_generic_xdp(skb, xdp_prog);
+
+                        if (act != XDP_PASS) {
+                                rcu_read_unlock();
+                                if (act == XDP_TX)
+                                        generic_xdp_tx(skb, xdp_prog);
+                                return NET_RX_DROP;
+                        }
+                }
+        }
+
 #ifdef CONFIG_RPS
         if (static_key_false(&rps_needed)) {
                 struct rps_dev_flow voidflow, *rflow = &voidflow;
@@ -4494,7 +4629,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
         enum gro_result ret;
         int grow;
 
-        if (!(skb->dev->features & NETIF_F_GRO))
+        if (netif_elide_gro(skb->dev))
                 goto normal;
 
         if (skb->csum_bad)
@@ -6723,6 +6858,7 @@ EXPORT_SYMBOL(dev_change_proto_down);
  */
 int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags)
 {
+        int (*xdp_op)(struct net_device *dev, struct netdev_xdp *xdp);
         const struct net_device_ops *ops = dev->netdev_ops;
         struct bpf_prog *prog = NULL;
         struct netdev_xdp xdp;
@@ -6730,14 +6866,16 @@ int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags)
 
         ASSERT_RTNL();
 
-        if (!ops->ndo_xdp)
-                return -EOPNOTSUPP;
+        xdp_op = ops->ndo_xdp;
+        if (!xdp_op || (flags & XDP_FLAGS_SKB_MODE))
+                xdp_op = generic_xdp_install;
+
         if (fd >= 0) {
                 if (flags & XDP_FLAGS_UPDATE_IF_NOEXIST) {
                         memset(&xdp, 0, sizeof(xdp));
                         xdp.command = XDP_QUERY_PROG;
 
-                        err = ops->ndo_xdp(dev, &xdp);
+                        err = xdp_op(dev, &xdp);
                         if (err < 0)
                                 return err;
                         if (xdp.prog_attached)
@@ -6753,7 +6891,7 @@ int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags)
         xdp.command = XDP_SETUP_PROG;
         xdp.prog = prog;
 
-        err = ops->ndo_xdp(dev, &xdp);
+        err = xdp_op(dev, &xdp);
         if (err < 0 && prog)
                 bpf_prog_put(prog);
 
@@ -7793,6 +7931,7 @@ EXPORT_SYMBOL(alloc_netdev_mqs);
 void free_netdev(struct net_device *dev)
 {
         struct napi_struct *p, *n;
+        struct bpf_prog *prog;
 
         might_sleep();
         netif_free_tx_queues(dev);
@@ -7811,6 +7950,12 @@ void free_netdev(struct net_device *dev)
         free_percpu(dev->pcpu_refcnt);
         dev->pcpu_refcnt = NULL;
 
+        prog = rcu_dereference_protected(dev->xdp_prog, 1);
+        if (prog) {
+                bpf_prog_put(prog);
+                static_key_slow_dec(&generic_xdp_needed);
+        }
+
         /*  Compatibility with error handling in drivers */
         if (dev->reg_state == NETREG_UNINITIALIZED) {
                 netdev_freemem(dev);