1 files changed, 27 insertions, 21 deletions

diff --git a/net/core/dev.c b/net/core/dev.c
index a3a96ffc67f4..8be38675e1a8 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3030,6 +3030,8 @@ static inline void ____napi_schedule(struct softnet_data *sd,
 /* One global table that all flow-based protocols share. */
 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
 EXPORT_SYMBOL(rps_sock_flow_table);
+u32 rps_cpu_mask __read_mostly;
+EXPORT_SYMBOL(rps_cpu_mask);
 
 struct static_key rps_needed __read_mostly;
 
@@ -3086,16 +3088,17 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
                        struct rps_dev_flow **rflowp)
 {
-        struct netdev_rx_queue *rxqueue;
-        struct rps_map *map;
+        const struct rps_sock_flow_table *sock_flow_table;
+        struct netdev_rx_queue *rxqueue = dev->_rx;
         struct rps_dev_flow_table *flow_table;
-        struct rps_sock_flow_table *sock_flow_table;
+        struct rps_map *map;
         int cpu = -1;
-        u16 tcpu;
+        u32 tcpu;
         u32 hash;
 
         if (skb_rx_queue_recorded(skb)) {
                 u16 index = skb_get_rx_queue(skb);
+
                 if (unlikely(index >= dev->real_num_rx_queues)) {
                         WARN_ONCE(dev->real_num_rx_queues > 1,
                                   "%s received packet on queue %u, but number "
@@ -3103,39 +3106,40 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
                                   dev->name, index, dev->real_num_rx_queues);
                         goto done;
                 }
-                rxqueue = dev->_rx + index;
-        } else
-                rxqueue = dev->_rx;
+                rxqueue += index;
+        }
 
+        /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
+
+        flow_table = rcu_dereference(rxqueue->rps_flow_table);
         map = rcu_dereference(rxqueue->rps_map);
-        if (map) {
-                if (map->len == 1 &&
-                    !rcu_access_pointer(rxqueue->rps_flow_table)) {
-                        tcpu = map->cpus[0];
-                        if (cpu_online(tcpu))
-                                cpu = tcpu;
-                        goto done;
-                }
-        } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
+        if (!flow_table && !map)
                 goto done;
-        }
 
         skb_reset_network_header(skb);
         hash = skb_get_hash(skb);
         if (!hash)
                 goto done;
 
-        flow_table = rcu_dereference(rxqueue->rps_flow_table);
         sock_flow_table = rcu_dereference(rps_sock_flow_table);
         if (flow_table && sock_flow_table) {
-                u16 next_cpu;
                 struct rps_dev_flow *rflow;
+                u32 next_cpu;
+                u32 ident;
+
+                /* First check into global flow table if there is a match */
+                ident = sock_flow_table->ents[hash & sock_flow_table->mask];
+                if ((ident ^ hash) & ~rps_cpu_mask)
+                        goto try_rps;
 
+                next_cpu = ident & rps_cpu_mask;
+
+                /* OK, now we know there is a match,
+                 * we can look at the local (per receive queue) flow table
+                 */
                 rflow = &flow_table->flows[hash & flow_table->mask];
                 tcpu = rflow->cpu;
 
-                next_cpu = sock_flow_table->ents[hash & sock_flow_table->mask];
-
                 /*
                  * If the desired CPU (where last recvmsg was done) is
                  * different from current CPU (one in the rx-queue flow
@@ -3162,6 +3166,8 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
                 }
         }
 
+try_rps:
+
         if (map) {
                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
                 if (cpu_online(tcpu)) {