net: listening_hash get a spinlock per bucket

This patch prepares RCU migration of listening_hash table for
TCP/DCCP protocols.

listening_hash table being small (32 slots per protocol), we add
a spinlock for each slot, instead of a single rwlock for whole table.

This should reduce hold time of readers, and writers concurrency.

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 12 insertions, 12 deletions

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 5559fea61e87..330b08a12274 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -97,11 +97,7 @@ struct tcp_md5sig_key *tcp_v4_md5_do_lookup(struct sock *sk, __be32 addr)
 }
 #endif
 
-struct inet_hashinfo __cacheline_aligned tcp_hashinfo = {
-        .lhash_lock  = __RW_LOCK_UNLOCKED(tcp_hashinfo.lhash_lock),
-        .lhash_users = ATOMIC_INIT(0),
-        .lhash_wait  = __WAIT_QUEUE_HEAD_INITIALIZER(tcp_hashinfo.lhash_wait),
-};
+struct inet_hashinfo tcp_hashinfo;
 
 static inline __u32 tcp_v4_init_sequence(struct sk_buff *skb)
 {
@@ -1874,15 +1870,18 @@ static void *listening_get_next(struct seq_file *seq, void *cur)
         struct inet_connection_sock *icsk;
         struct hlist_node *node;
         struct sock *sk = cur;
+        struct inet_listen_hashbucket *ilb;
         struct tcp_iter_state *st = seq->private;
         struct net *net = seq_file_net(seq);
 
         if (!sk) {
                 st->bucket = 0;
-                sk = sk_head(&tcp_hashinfo.listening_hash[0]);
+                ilb = &tcp_hashinfo.listening_hash[0];
+                spin_lock_bh(&ilb->lock);
+                sk = sk_head(&ilb->head);
                 goto get_sk;
         }
-
+        ilb = &tcp_hashinfo.listening_hash[st->bucket];
         ++st->num;
 
         if (st->state == TCP_SEQ_STATE_OPENREQ) {
@@ -1932,8 +1931,11 @@ start_req:
                 }
                 read_unlock_bh(&icsk->icsk_accept_queue.syn_wait_lock);
         }
+        spin_unlock_bh(&ilb->lock);
         if (++st->bucket < INET_LHTABLE_SIZE) {
-                sk = sk_head(&tcp_hashinfo.listening_hash[st->bucket]);
+                ilb = &tcp_hashinfo.listening_hash[st->bucket];
+                spin_lock_bh(&ilb->lock);
+                sk = sk_head(&ilb->head);
                 goto get_sk;
         }
         cur = NULL;
@@ -2066,12 +2068,10 @@ static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
         void *rc;
         struct tcp_iter_state *st = seq->private;
 
-        inet_listen_lock(&tcp_hashinfo);
         st->state = TCP_SEQ_STATE_LISTENING;
         rc        = listening_get_idx(seq, &pos);
 
         if (!rc) {
-                inet_listen_unlock(&tcp_hashinfo);
                 st->state = TCP_SEQ_STATE_ESTABLISHED;
                 rc        = established_get_idx(seq, pos);
         }
@@ -2103,7 +2103,6 @@ static void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
         case TCP_SEQ_STATE_LISTENING:
                 rc = listening_get_next(seq, v);
                 if (!rc) {
-                        inet_listen_unlock(&tcp_hashinfo);
                         st->state = TCP_SEQ_STATE_ESTABLISHED;
                         rc        = established_get_first(seq);
                 }
@@ -2130,7 +2129,7 @@ static void tcp_seq_stop(struct seq_file *seq, void *v)
                 }
         case TCP_SEQ_STATE_LISTENING:
                 if (v != SEQ_START_TOKEN)
-                        inet_listen_unlock(&tcp_hashinfo);
+                        spin_unlock_bh(&tcp_hashinfo.listening_hash[st->bucket].lock);
                 break;
         case TCP_SEQ_STATE_TIME_WAIT:
         case TCP_SEQ_STATE_ESTABLISHED:
@@ -2405,6 +2404,7 @@ static struct pernet_operations __net_initdata tcp_sk_ops = {
 
 void __init tcp_v4_init(void)
 {
+        inet_hashinfo_init(&tcp_hashinfo);
         if (register_pernet_device(&tcp_sk_ops))
                 panic("Failed to create the TCP control socket.\n");
 }