net: RPS: Enable hardware acceleration of RFS

Allow drivers for multiqueue hardware with flow filter tables to
accelerate RFS.  The driver must:

1. Set net_device::rx_cpu_rmap to a cpu_rmap of the RX completion
IRQs (in queue order).  This will provide a mapping from CPUs to the
queues for which completions are handled nearest to them.

2. Implement net_device_ops::ndo_rx_flow_steer.  This operation adds
or replaces a filter steering the given flow to the given RX queue, if
possible.

3. Periodically remove filters for which rps_may_expire_flow() returns
true.

Signed-off-by: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 30 insertions, 3 deletions

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 371fa8839d51..a335f2022690 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -554,14 +554,16 @@ struct rps_map {
 #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + (_num * sizeof(u16)))
 
 /*
- * The rps_dev_flow structure contains the mapping of a flow to a CPU and the
- * tail pointer for that CPU's input queue at the time of last enqueue.
+ * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
+ * tail pointer for that CPU's input queue at the time of last enqueue, and
+ * a hardware filter index.
  */
 struct rps_dev_flow {
         u16 cpu;
-        u16 fill;
+        u16 filter;
         unsigned int last_qtail;
 };
+#define RPS_NO_FILTER 0xffff
 
 /*
  * The rps_dev_flow_table structure contains a table of flow mappings.
@@ -611,6 +613,11 @@ static inline void rps_reset_sock_flow(struct rps_sock_flow_table *table,
 
 extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
 
+#ifdef CONFIG_RFS_ACCEL
+extern bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
+                                u32 flow_id, u16 filter_id);
+#endif
+
 /* This structure contains an instance of an RX queue. */
 struct netdev_rx_queue {
         struct rps_map __rcu            *rps_map;
@@ -769,6 +776,13 @@ struct netdev_tc_txq {
  *      is always called from the stack with the rtnl lock held and netif tx
  *      queues stopped. This allows the netdevice to perform queue management
  *      safely.
+ *
+ *      RFS acceleration.
+ * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
+ *                          u16 rxq_index, u32 flow_id);
+ *      Set hardware filter for RFS.  rxq_index is the target queue index;
+ *      flow_id is a flow ID to be passed to rps_may_expire_flow() later.
+ *      Return the filter ID on success, or a negative error code.
  */
 #define HAVE_NET_DEVICE_OPS
 struct net_device_ops {
@@ -842,6 +856,12 @@ struct net_device_ops {
         int                     (*ndo_fcoe_get_wwn)(struct net_device *dev,
                                                     u64 *wwn, int type);
 #endif
+#ifdef CONFIG_RFS_ACCEL
+        int                     (*ndo_rx_flow_steer)(struct net_device *dev,
+                                                     const struct sk_buff *skb,
+                                                     u16 rxq_index,
+                                                     u32 flow_id);
+#endif
 };
 
 /*
@@ -1056,6 +1076,13 @@ struct net_device {
 
         /* Number of RX queues currently active in device */
         unsigned int            real_num_rx_queues;
+
+#ifdef CONFIG_RFS_ACCEL
+        /* CPU reverse-mapping for RX completion interrupts, indexed
+         * by RX queue number.  Assigned by driver.  This must only be
+         * set if the ndo_rx_flow_steer operation is defined. */
+        struct cpu_rmap         *rx_cpu_rmap;
+#endif
 #endif
 
         rx_handler_func_t __rcu *rx_handler;