1 files changed, 68 insertions, 1 deletions

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 55c2086e1f06..649a0252686e 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -530,14 +530,73 @@ struct rps_map {
 };
 #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + (_num * sizeof(u16)))
 
+/*
+ * The rps_dev_flow structure contains the mapping of a flow to a CPU and the
+ * tail pointer for that CPU's input queue at the time of last enqueue.
+ */
+struct rps_dev_flow {
+        u16 cpu;
+        u16 fill;
+        unsigned int last_qtail;
+};
+
+/*
+ * The rps_dev_flow_table structure contains a table of flow mappings.
+ */
+struct rps_dev_flow_table {
+        unsigned int mask;
+        struct rcu_head rcu;
+        struct work_struct free_work;
+        struct rps_dev_flow flows[0];
+};
+#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
+    (_num * sizeof(struct rps_dev_flow)))
+
+/*
+ * The rps_sock_flow_table contains mappings of flows to the last CPU
+ * on which they were processed by the application (set in recvmsg).
+ */
+struct rps_sock_flow_table {
+        unsigned int mask;
+        u16 ents[0];
+};
+#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_sock_flow_table) + \
+    (_num * sizeof(u16)))
+
+#define RPS_NO_CPU 0xffff
+
+static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
+                                        u32 hash)
+{
+        if (table && hash) {
+                unsigned int cpu, index = hash & table->mask;
+
+                /* We only give a hint, preemption can change cpu under us */
+                cpu = raw_smp_processor_id();
+
+                if (table->ents[index] != cpu)
+                        table->ents[index] = cpu;
+        }
+}
+
+static inline void rps_reset_sock_flow(struct rps_sock_flow_table *table,
+                                       u32 hash)
+{
+        if (table && hash)
+                table->ents[hash & table->mask] = RPS_NO_CPU;
+}
+
+extern struct rps_sock_flow_table *rps_sock_flow_table;
+
 /* This structure contains an instance of an RX queue. */
 struct netdev_rx_queue {
         struct rps_map *rps_map;
+        struct rps_dev_flow_table *rps_flow_table;
         struct kobject kobj;
         struct netdev_rx_queue *first;
         atomic_t count;
 } ____cacheline_aligned_in_smp;
-#endif
+#endif /* CONFIG_RPS */
 
 /*
  * This structure defines the management hooks for network devices.
@@ -1333,11 +1392,19 @@ struct softnet_data {
         /* Elements below can be accessed between CPUs for RPS */
 #ifdef CONFIG_RPS
         struct call_single_data csd ____cacheline_aligned_in_smp;
+        unsigned int            input_queue_head;
 #endif
         struct sk_buff_head     input_pkt_queue;
         struct napi_struct      backlog;
 };
 
+static inline void incr_input_queue_head(struct softnet_data *queue)
+{
+#ifdef CONFIG_RPS
+        queue->input_queue_head++;
+#endif
+}
+
 DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 
 #define HAVE_NETIF_QUEUE