6 files changed, 73 insertions, 36 deletions

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index 02fd8d4e253c..b7446793106d 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -541,6 +541,8 @@ struct nfp_net_dp {
  * @rss_cfg:            RSS configuration
  * @rss_key:            RSS secret key
  * @rss_itbl:           RSS indirection table
+ * @xdp_flags:          Flags with which XDP prog was loaded
+ * @xdp_prog:           XDP prog (for ctrl path, both DRV and HW modes)
  * @max_r_vecs:         Number of allocated interrupt vectors for RX/TX
  * @max_tx_rings:       Maximum number of TX rings supported by the Firmware
  * @max_rx_rings:       Maximum number of RX rings supported by the Firmware
@@ -590,6 +592,9 @@ struct nfp_net {
         u8 rss_key[NFP_NET_CFG_RSS_KEY_SZ];
         u8 rss_itbl[NFP_NET_CFG_RSS_ITBL_SZ];
 
+        u32 xdp_flags;
+        struct bpf_prog *xdp_prog;
+
         unsigned int max_tx_rings;
         unsigned int max_rx_rings;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 2bdddd1ae666..2134493ec8a8 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -3274,19 +3274,14 @@ static void nfp_net_del_vxlan_port(struct net_device *netdev,
                 nfp_net_set_vxlan_port(nn, idx, 0);
 }
 
-static int nfp_net_xdp_setup(struct nfp_net *nn, struct netdev_xdp *xdp)
+static int
+nfp_net_xdp_setup_drv(struct nfp_net *nn, struct bpf_prog *prog,
+                      struct netlink_ext_ack *extack)
 {
-        struct bpf_prog *old_prog = nn->dp.xdp_prog;
-        struct bpf_prog *prog = xdp->prog;
         struct nfp_net_dp *dp;
-        int err;
 
-        if (!prog && !nn->dp.xdp_prog)
-                return 0;
-        if (prog && nn->dp.xdp_prog) {
-                prog = xchg(&nn->dp.xdp_prog, prog);
-                bpf_prog_put(prog);
-                nfp_app_xdp_offload(nn->app, nn, nn->dp.xdp_prog);
+        if (!prog == !nn->dp.xdp_prog) {
+                WRITE_ONCE(nn->dp.xdp_prog, prog);
                 return 0;
         }
 
@@ -3300,14 +3295,37 @@ static int nfp_net_xdp_setup(struct nfp_net *nn, struct netdev_xdp *xdp)
         dp->rx_dma_off = prog ? XDP_PACKET_HEADROOM - nn->dp.rx_offset : 0;
 
         /* We need RX reconfig to remap the buffers (BIDIR vs FROM_DEV) */
-        err = nfp_net_ring_reconfig(nn, dp, xdp->extack);
+        return nfp_net_ring_reconfig(nn, dp, extack);
+}
+
+static int
+nfp_net_xdp_setup(struct nfp_net *nn, struct bpf_prog *prog, u32 flags,
+                  struct netlink_ext_ack *extack)
+{
+        struct bpf_prog *drv_prog, *offload_prog;
+        int err;
+
+        if (nn->xdp_prog && (flags ^ nn->xdp_flags) & XDP_FLAGS_MODES)
+                return -EBUSY;
+
+        /* Load both when no flags set to allow easy activation of driver path
+         * when program is replaced by one which can't be offloaded.
+         */
+        drv_prog     = flags & XDP_FLAGS_HW_MODE  ? NULL : prog;
+        offload_prog = flags & XDP_FLAGS_DRV_MODE ? NULL : prog;
+
+        err = nfp_net_xdp_setup_drv(nn, drv_prog, extack);
         if (err)
                 return err;
 
-        if (old_prog)
-                bpf_prog_put(old_prog);
+        err = nfp_app_xdp_offload(nn->app, nn, offload_prog);
+        if (err && flags & XDP_FLAGS_HW_MODE)
+                return err;
 
-        nfp_app_xdp_offload(nn->app, nn, nn->dp.xdp_prog);
+        if (nn->xdp_prog)
+                bpf_prog_put(nn->xdp_prog);
+        nn->xdp_prog = prog;
+        nn->xdp_flags = flags;
 
         return 0;
 }
@@ -3318,10 +3336,14 @@ static int nfp_net_xdp(struct net_device *netdev, struct netdev_xdp *xdp)
 
         switch (xdp->command) {
         case XDP_SETUP_PROG:
-                return nfp_net_xdp_setup(nn, xdp);
+        case XDP_SETUP_PROG_HW:
+                return nfp_net_xdp_setup(nn, xdp->prog, xdp->flags,
+                                         xdp->extack);
         case XDP_QUERY_PROG:
-                xdp->prog_attached = !!nn->dp.xdp_prog;
-                xdp->prog_id = nn->dp.xdp_prog ? nn->dp.xdp_prog->aux->id : 0;
+                xdp->prog_attached = !!nn->xdp_prog;
+                if (nn->dp.bpf_offload_xdp)
+                        xdp->prog_attached = XDP_ATTACHED_HW;
+                xdp->prog_id = nn->xdp_prog ? nn->xdp_prog->aux->id : 0;
                 return 0;
         default:
                 return -EINVAL;
@@ -3479,6 +3501,9 @@ struct nfp_net *nfp_net_alloc(struct pci_dev *pdev, bool needs_netdev,
  */
 void nfp_net_free(struct nfp_net *nn)
 {
+        if (nn->xdp_prog)
+                bpf_prog_put(nn->xdp_prog);
+
         if (nn->dp.netdev)
                 free_netdev(nn->dp.netdev);
         else
@@ -3736,7 +3761,4 @@ void nfp_net_clean(struct nfp_net *nn)
                 return;
 
         unregister_netdev(nn->dp.netdev);
-
-        if (nn->dp.xdp_prog)
-                bpf_prog_put(nn->dp.xdp_prog);
 }
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 7c7118b3bd69..68f5d899d1e6 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -807,8 +807,10 @@ enum xdp_netdev_command {
          * when it is no longer used.
          */
         XDP_SETUP_PROG,
+        XDP_SETUP_PROG_HW,
         /* Check if a bpf program is set on the device.  The callee should
-         * return true if a program is currently attached and running.
+         * set @prog_attached to one of XDP_ATTACHED_* values, note that "true"
+         * is equivalent to XDP_ATTACHED_DRV.
          */
         XDP_QUERY_PROG,
 };
@@ -820,12 +822,13 @@ struct netdev_xdp {
         union {
                 /* XDP_SETUP_PROG */
                 struct {
+                        u32 flags;
                         struct bpf_prog *prog;
                         struct netlink_ext_ack *extack;
                 };
                 /* XDP_QUERY_PROG */
                 struct {
-                        bool prog_attached;
+                        u8 prog_attached;
                         u32 prog_id;
                 };
         };
@@ -3305,7 +3308,7 @@ struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
 typedef int (*xdp_op_t)(struct net_device *dev, struct netdev_xdp *xdp);
 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                       int fd, u32 flags);
-bool __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id);
+u8 __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id);
 
 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index dd88375a6580..8d062c58d5cb 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -891,15 +891,19 @@ enum {
 #define XDP_FLAGS_UPDATE_IF_NOEXIST     (1U << 0)
 #define XDP_FLAGS_SKB_MODE              (1U << 1)
 #define XDP_FLAGS_DRV_MODE              (1U << 2)
+#define XDP_FLAGS_HW_MODE               (1U << 3)
+#define XDP_FLAGS_MODES                 (XDP_FLAGS_SKB_MODE | \
+                                         XDP_FLAGS_DRV_MODE | \
+                                         XDP_FLAGS_HW_MODE)
 #define XDP_FLAGS_MASK                  (XDP_FLAGS_UPDATE_IF_NOEXIST | \
-                                         XDP_FLAGS_SKB_MODE | \
-                                         XDP_FLAGS_DRV_MODE)
+                                         XDP_FLAGS_MODES)
 
 /* These are stored into IFLA_XDP_ATTACHED on dump. */
 enum {
         XDP_ATTACHED_NONE = 0,
         XDP_ATTACHED_DRV,
         XDP_ATTACHED_SKB,
+        XDP_ATTACHED_HW,
 };
 
 enum {
diff --git a/net/core/dev.c b/net/core/dev.c
index df7637733e3c..a91572aa73d5 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6934,8 +6934,7 @@ int dev_change_proto_down(struct net_device *dev, bool proto_down)
 }
 EXPORT_SYMBOL(dev_change_proto_down);
 
-bool __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op,
-                        u32 *prog_id)
+u8 __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id)
 {
         struct netdev_xdp xdp;
 
@@ -6951,14 +6950,18 @@ bool __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op,
 }
 
 static int dev_xdp_install(struct net_device *dev, xdp_op_t xdp_op,
-                           struct netlink_ext_ack *extack,
+                           struct netlink_ext_ack *extack, u32 flags,
                            struct bpf_prog *prog)
 {
         struct netdev_xdp xdp;
 
         memset(&xdp, 0, sizeof(xdp));
-        xdp.command = XDP_SETUP_PROG;
+        if (flags & XDP_FLAGS_HW_MODE)
+                xdp.command = XDP_SETUP_PROG_HW;
+        else
+                xdp.command = XDP_SETUP_PROG;
         xdp.extack = extack;
+        xdp.flags = flags;
         xdp.prog = prog;
 
         return xdp_op(dev, &xdp);
@@ -6984,7 +6987,7 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
         ASSERT_RTNL();
 
         xdp_op = xdp_chk = ops->ndo_xdp;
-        if (!xdp_op && (flags & XDP_FLAGS_DRV_MODE))
+        if (!xdp_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))
                 return -EOPNOTSUPP;
         if (!xdp_op || (flags & XDP_FLAGS_SKB_MODE))
                 xdp_op = generic_xdp_install;
@@ -7003,7 +7006,7 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                         return PTR_ERR(prog);
         }
 
-        err = dev_xdp_install(dev, xdp_op, extack, prog);
+        err = dev_xdp_install(dev, xdp_op, extack, flags, prog);
         if (err < 0 && prog)
                 bpf_prog_put(prog);
 
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 8da89c1136e5..9a1bd510c812 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -16,6 +16,7 @@
  *      Vitaly E. Lavrov                RTA_OK arithmetics was wrong.
  */
 
+#include <linux/bitops.h>
 #include <linux/errno.h>
 #include <linux/module.h>
 #include <linux/types.h>
@@ -1264,10 +1265,10 @@ static u8 rtnl_xdp_attached_mode(struct net_device *dev, u32 *prog_id)
                 *prog_id = generic_xdp_prog->aux->id;
                 return XDP_ATTACHED_SKB;
         }
-        if (ops->ndo_xdp && __dev_xdp_attached(dev, ops->ndo_xdp, prog_id))
-                return XDP_ATTACHED_DRV;
+        if (!ops->ndo_xdp)
+                return XDP_ATTACHED_NONE;
 
-        return XDP_ATTACHED_NONE;
+        return __dev_xdp_attached(dev, ops->ndo_xdp, prog_id);
 }
 
 static int rtnl_xdp_fill(struct sk_buff *skb, struct net_device *dev)
@@ -2253,8 +2254,7 @@ static int do_setlink(const struct sk_buff *skb,
                                 err = -EINVAL;
                                 goto errout;
                         }
-                        if ((xdp_flags & XDP_FLAGS_SKB_MODE) &&
-                            (xdp_flags & XDP_FLAGS_DRV_MODE)) {
+                        if (hweight32(xdp_flags & XDP_FLAGS_MODES) > 1) {
                                 err = -EINVAL;
                                 goto errout;
                         }