net: Convert TCP & DCCP hash tables to use RCU / hlist_nulls

RCU was added to UDP lookups, using a fast infrastructure :
- sockets kmem_cache use SLAB_DESTROY_BY_RCU and dont pay the
  price of call_rcu() at freeing time.
- hlist_nulls permits to use few memory barriers.

This patch uses same infrastructure for TCP/DCCP established
and timewait sockets.

Thanks to SLAB_DESTROY_BY_RCU, no slowdown for applications
using short lived TCP connections. A followup patch, converting
rwlocks to spinlocks will even speedup this case.

__inet_lookup_established() is pretty fast now we dont have to
dirty a contended cache line (read_lock/read_unlock)

Only established and timewait hashtable are converted to RCU
(bind table and listen table are still using traditional locking)

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 5 insertions, 5 deletions

diff --git a/include/net/inet_timewait_sock.h b/include/net/inet_timewait_sock.h
index 80e4977631b8..4b8ece22b8e9 100644
--- a/include/net/inet_timewait_sock.h
+++ b/include/net/inet_timewait_sock.h
@@ -110,7 +110,7 @@ struct inet_timewait_sock {
 #define tw_state                __tw_common.skc_state
 #define tw_reuse                __tw_common.skc_reuse
 #define tw_bound_dev_if         __tw_common.skc_bound_dev_if
-#define tw_node                 __tw_common.skc_node
+#define tw_node                 __tw_common.skc_nulls_node
 #define tw_bind_node            __tw_common.skc_bind_node
 #define tw_refcnt               __tw_common.skc_refcnt
 #define tw_hash                 __tw_common.skc_hash
@@ -137,10 +137,10 @@ struct inet_timewait_sock {
         struct hlist_node       tw_death_node;
 };
 
-static inline void inet_twsk_add_node(struct inet_timewait_sock *tw,
-                                      struct hlist_head *list)
+static inline void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
+                                      struct hlist_nulls_head *list)
 {
-        hlist_add_head(&tw->tw_node, list);
+        hlist_nulls_add_head_rcu(&tw->tw_node, list);
 }
 
 static inline void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
@@ -175,7 +175,7 @@ static inline int inet_twsk_del_dead_node(struct inet_timewait_sock *tw)
 }
 
 #define inet_twsk_for_each(tw, node, head) \
-        hlist_for_each_entry(tw, node, head, tw_node)
+        hlist_nulls_for_each_entry(tw, node, head, tw_node)
 
 #define inet_twsk_for_each_inmate(tw, node, jail) \
         hlist_for_each_entry(tw, node, jail, tw_death_node)