Merge branch 'xdp-offload-mode'

Jakub Kicinski says:

====================
xdp: offload mode

While we discuss the representors.. :)

This set adds XDP flag for forcing offload and a attachment mode
for reporting to user space that program has been offloaded.  The
nfp driver is modified to make use of the new flags, but also to
adhere to the DRV_MODE flag which should disable the HW offload.

The intended driver behaviour is:
            DRV mode   offload
no flags      yes     attempted
DRV_MODE      yes        no
 HW_MODE      no         yes

Where 'yes' means required, and error will be returned if setup fails.
'Attempted' means the offload will only happen automatically if HW is
capable and offloading the program will cause no change in system
behaviour (e.g. maps don't have to bound).

Thanks to loading the program both to the driver and HW by default we
can fallback to the driver mode without disruption in case user replaces
the program with one which cannot be offloaded later.

Note that the NFP driver currently claims XDP offload support but
lacks most basic features like direct packet access.

Only change compared to the RFC is fixing the double bpf_prog_put()
which Daniel has spotted (patch 5).
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

6 files changed, 73 insertions, 36 deletions

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index 02fd8d4e253c..b7446793106d 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -541,6 +541,8 @@ struct nfp_net_dp {
  * @rss_cfg:            RSS configuration
  * @rss_key:            RSS secret key
  * @rss_itbl:           RSS indirection table
+ * @xdp_flags:          Flags with which XDP prog was loaded
+ * @xdp_prog:           XDP prog (for ctrl path, both DRV and HW modes)
  * @max_r_vecs:         Number of allocated interrupt vectors for RX/TX
  * @max_tx_rings:       Maximum number of TX rings supported by the Firmware
  * @max_rx_rings:       Maximum number of RX rings supported by the Firmware
@@ -590,6 +592,9 @@ struct nfp_net {
         u8 rss_key[NFP_NET_CFG_RSS_KEY_SZ];
         u8 rss_itbl[NFP_NET_CFG_RSS_ITBL_SZ];
 
+        u32 xdp_flags;
+        struct bpf_prog *xdp_prog;
+
         unsigned int max_tx_rings;
         unsigned int max_rx_rings;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 2bdddd1ae666..2134493ec8a8 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -3274,19 +3274,14 @@ static void nfp_net_del_vxlan_port(struct net_device *netdev,
                 nfp_net_set_vxlan_port(nn, idx, 0);
 }
 
-static int nfp_net_xdp_setup(struct nfp_net *nn, struct netdev_xdp *xdp)
+static int
+nfp_net_xdp_setup_drv(struct nfp_net *nn, struct bpf_prog *prog,
+                      struct netlink_ext_ack *extack)
 {
-        struct bpf_prog *old_prog = nn->dp.xdp_prog;
-        struct bpf_prog *prog = xdp->prog;
         struct nfp_net_dp *dp;
-        int err;
 
-        if (!prog && !nn->dp.xdp_prog)
-                return 0;
-        if (prog && nn->dp.xdp_prog) {
-                prog = xchg(&nn->dp.xdp_prog, prog);
-                bpf_prog_put(prog);
-                nfp_app_xdp_offload(nn->app, nn, nn->dp.xdp_prog);
+        if (!prog == !nn->dp.xdp_prog) {
+                WRITE_ONCE(nn->dp.xdp_prog, prog);
                 return 0;
         }
 
@@ -3300,14 +3295,37 @@ static int nfp_net_xdp_setup(struct nfp_net *nn, struct netdev_xdp *xdp)
         dp->rx_dma_off = prog ? XDP_PACKET_HEADROOM - nn->dp.rx_offset : 0;
 
         /* We need RX reconfig to remap the buffers (BIDIR vs FROM_DEV) */
-        err = nfp_net_ring_reconfig(nn, dp, xdp->extack);
+        return nfp_net_ring_reconfig(nn, dp, extack);
+}
+
+static int
+nfp_net_xdp_setup(struct nfp_net *nn, struct bpf_prog *prog, u32 flags,
+                  struct netlink_ext_ack *extack)
+{
+        struct bpf_prog *drv_prog, *offload_prog;
+        int err;
+
+        if (nn->xdp_prog && (flags ^ nn->xdp_flags) & XDP_FLAGS_MODES)
+                return -EBUSY;
+
+        /* Load both when no flags set to allow easy activation of driver path
+         * when program is replaced by one which can't be offloaded.
+         */
+        drv_prog     = flags & XDP_FLAGS_HW_MODE  ? NULL : prog;
+        offload_prog = flags & XDP_FLAGS_DRV_MODE ? NULL : prog;
+
+        err = nfp_net_xdp_setup_drv(nn, drv_prog, extack);
         if (err)
                 return err;
 
-        if (old_prog)
-                bpf_prog_put(old_prog);
+        err = nfp_app_xdp_offload(nn->app, nn, offload_prog);
+        if (err && flags & XDP_FLAGS_HW_MODE)
+                return err;
 
-        nfp_app_xdp_offload(nn->app, nn, nn->dp.xdp_prog);
+        if (nn->xdp_prog)
+                bpf_prog_put(nn->xdp_prog);
+        nn->xdp_prog = prog;
+        nn->xdp_flags = flags;
 
         return 0;
 }
@@ -3318,10 +3336,14 @@ static int nfp_net_xdp(struct net_device *netdev, struct netdev_xdp *xdp)
 
         switch (xdp->command) {
         case XDP_SETUP_PROG:
-                return nfp_net_xdp_setup(nn, xdp);
+        case XDP_SETUP_PROG_HW:
+                return nfp_net_xdp_setup(nn, xdp->prog, xdp->flags,
+                                         xdp->extack);
         case XDP_QUERY_PROG:
-                xdp->prog_attached = !!nn->dp.xdp_prog;
-                xdp->prog_id = nn->dp.xdp_prog ? nn->dp.xdp_prog->aux->id : 0;
+                xdp->prog_attached = !!nn->xdp_prog;
+                if (nn->dp.bpf_offload_xdp)
+                        xdp->prog_attached = XDP_ATTACHED_HW;
+                xdp->prog_id = nn->xdp_prog ? nn->xdp_prog->aux->id : 0;
                 return 0;
         default:
                 return -EINVAL;
@@ -3479,6 +3501,9 @@ struct nfp_net *nfp_net_alloc(struct pci_dev *pdev, bool needs_netdev,
  */
 void nfp_net_free(struct nfp_net *nn)
 {
+        if (nn->xdp_prog)
+                bpf_prog_put(nn->xdp_prog);
+
         if (nn->dp.netdev)
                 free_netdev(nn->dp.netdev);
         else
@@ -3736,7 +3761,4 @@ void nfp_net_clean(struct nfp_net *nn)
                 return;
 
         unregister_netdev(nn->dp.netdev);
-
-        if (nn->dp.xdp_prog)
-                bpf_prog_put(nn->dp.xdp_prog);
 }
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 7c7118b3bd69..68f5d899d1e6 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -807,8 +807,10 @@ enum xdp_netdev_command {
          * when it is no longer used.
          */
         XDP_SETUP_PROG,
+        XDP_SETUP_PROG_HW,
         /* Check if a bpf program is set on the device.  The callee should
-         * return true if a program is currently attached and running.
+         * set @prog_attached to one of XDP_ATTACHED_* values, note that "true"
+         * is equivalent to XDP_ATTACHED_DRV.
          */
         XDP_QUERY_PROG,
 };
@@ -820,12 +822,13 @@ struct netdev_xdp {
         union {
                 /* XDP_SETUP_PROG */
                 struct {
+                        u32 flags;
                         struct bpf_prog *prog;
                         struct netlink_ext_ack *extack;
                 };
                 /* XDP_QUERY_PROG */
                 struct {
-                        bool prog_attached;
+                        u8 prog_attached;
                         u32 prog_id;
                 };
         };
@@ -3305,7 +3308,7 @@ struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
 typedef int (*xdp_op_t)(struct net_device *dev, struct netdev_xdp *xdp);
 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                       int fd, u32 flags);
-bool __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id);
+u8 __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id);
 
 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index dd88375a6580..8d062c58d5cb 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -891,15 +891,19 @@ enum {
 #define XDP_FLAGS_UPDATE_IF_NOEXIST     (1U << 0)
 #define XDP_FLAGS_SKB_MODE              (1U << 1)
 #define XDP_FLAGS_DRV_MODE              (1U << 2)
+#define XDP_FLAGS_HW_MODE               (1U << 3)
+#define XDP_FLAGS_MODES                 (XDP_FLAGS_SKB_MODE | \
+                                         XDP_FLAGS_DRV_MODE | \
+                                         XDP_FLAGS_HW_MODE)
 #define XDP_FLAGS_MASK                  (XDP_FLAGS_UPDATE_IF_NOEXIST | \
-                                         XDP_FLAGS_SKB_MODE | \
-                                         XDP_FLAGS_DRV_MODE)
+                                         XDP_FLAGS_MODES)
 
 /* These are stored into IFLA_XDP_ATTACHED on dump. */
 enum {
         XDP_ATTACHED_NONE = 0,
         XDP_ATTACHED_DRV,
         XDP_ATTACHED_SKB,
+        XDP_ATTACHED_HW,
 };
 
 enum {
diff --git a/net/core/dev.c b/net/core/dev.c
index df7637733e3c..a91572aa73d5 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6934,8 +6934,7 @@ int dev_change_proto_down(struct net_device *dev, bool proto_down)
 }
 EXPORT_SYMBOL(dev_change_proto_down);
 
-bool __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op,
-                        u32 *prog_id)
+u8 __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id)
 {
         struct netdev_xdp xdp;
 
@@ -6951,14 +6950,18 @@ bool __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op,
 }
 
 static int dev_xdp_install(struct net_device *dev, xdp_op_t xdp_op,
-                           struct netlink_ext_ack *extack,
+                           struct netlink_ext_ack *extack, u32 flags,
                            struct bpf_prog *prog)
 {
         struct netdev_xdp xdp;
 
         memset(&xdp, 0, sizeof(xdp));
-        xdp.command = XDP_SETUP_PROG;
+        if (flags & XDP_FLAGS_HW_MODE)
+                xdp.command = XDP_SETUP_PROG_HW;
+        else
+                xdp.command = XDP_SETUP_PROG;
         xdp.extack = extack;
+        xdp.flags = flags;
         xdp.prog = prog;
 
         return xdp_op(dev, &xdp);
@@ -6984,7 +6987,7 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
         ASSERT_RTNL();
 
         xdp_op = xdp_chk = ops->ndo_xdp;
-        if (!xdp_op && (flags & XDP_FLAGS_DRV_MODE))
+        if (!xdp_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))
                 return -EOPNOTSUPP;
         if (!xdp_op || (flags & XDP_FLAGS_SKB_MODE))
                 xdp_op = generic_xdp_install;
@@ -7003,7 +7006,7 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                         return PTR_ERR(prog);
         }
 
-        err = dev_xdp_install(dev, xdp_op, extack, prog);
+        err = dev_xdp_install(dev, xdp_op, extack, flags, prog);
         if (err < 0 && prog)
                 bpf_prog_put(prog);
 
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 8da89c1136e5..9a1bd510c812 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -16,6 +16,7 @@
  *      Vitaly E. Lavrov                RTA_OK arithmetics was wrong.
  */
 
+#include <linux/bitops.h>
 #include <linux/errno.h>
 #include <linux/module.h>
 #include <linux/types.h>
@@ -1264,10 +1265,10 @@ static u8 rtnl_xdp_attached_mode(struct net_device *dev, u32 *prog_id)
                 *prog_id = generic_xdp_prog->aux->id;
                 return XDP_ATTACHED_SKB;
         }
-        if (ops->ndo_xdp && __dev_xdp_attached(dev, ops->ndo_xdp, prog_id))
-                return XDP_ATTACHED_DRV;
+        if (!ops->ndo_xdp)
+                return XDP_ATTACHED_NONE;
 
-        return XDP_ATTACHED_NONE;
+        return __dev_xdp_attached(dev, ops->ndo_xdp, prog_id);
 }
 
 static int rtnl_xdp_fill(struct sk_buff *skb, struct net_device *dev)
@@ -2253,8 +2254,7 @@ static int do_setlink(const struct sk_buff *skb,
                                 err = -EINVAL;
                                 goto errout;
                         }
-                        if ((xdp_flags & XDP_FLAGS_SKB_MODE) &&
-                            (xdp_flags & XDP_FLAGS_DRV_MODE)) {
+                        if (hweight32(xdp_flags & XDP_FLAGS_MODES) > 1) {
                                 err = -EINVAL;
                                 goto errout;
                         }