1 files changed, 57 insertions, 32 deletions

diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
index 1646a5658255..21544b9be259 100644
--- a/net/ipv6/inet6_hashtables.c
+++ b/net/ipv6/inet6_hashtables.c
@@ -25,26 +25,30 @@
 void __inet6_hash(struct sock *sk)
 {
         struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
-        struct hlist_head *list;
-        rwlock_t *lock;
 
         WARN_ON(!sk_unhashed(sk));
 
         if (sk->sk_state == TCP_LISTEN) {
-                list = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
-                lock = &hashinfo->lhash_lock;
-                inet_listen_wlock(hashinfo);
+                struct inet_listen_hashbucket *ilb;
+
+                ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
+                spin_lock(&ilb->lock);
+                __sk_add_node(sk, &ilb->head);
+                spin_unlock(&ilb->lock);
         } else {
                 unsigned int hash;
+                struct hlist_nulls_head *list;
+                rwlock_t *lock;
+
                 sk->sk_hash = hash = inet6_sk_ehashfn(sk);
                 list = &inet_ehash_bucket(hashinfo, hash)->chain;
                 lock = inet_ehash_lockp(hashinfo, hash);
                 write_lock(lock);
+                __sk_nulls_add_node_rcu(sk, list);
+                write_unlock(lock);
         }
 
-        __sk_add_node(sk, list);
         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
-        write_unlock(lock);
 }
 EXPORT_SYMBOL(__inet6_hash);
 
@@ -63,33 +67,53 @@ struct sock *__inet6_lookup_established(struct net *net,
                                            const int dif)
 {
         struct sock *sk;
-        const struct hlist_node *node;
+        const struct hlist_nulls_node *node;
         const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
         /* Optimize here for direct hit, only listening connections can
          * have wildcards anyways.
          */
         unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
-        struct inet_ehash_bucket *head = inet_ehash_bucket(hashinfo, hash);
-        rwlock_t *lock = inet_ehash_lockp(hashinfo, hash);
+        unsigned int slot = hash & (hashinfo->ehash_size - 1);
+        struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
 
-        prefetch(head->chain.first);
-        read_lock(lock);
-        sk_for_each(sk, node, &head->chain) {
+
+        rcu_read_lock();
+begin:
+        sk_nulls_for_each_rcu(sk, node, &head->chain) {
                 /* For IPV6 do the cheaper port and family tests first. */
-                if (INET6_MATCH(sk, net, hash, saddr, daddr, ports, dif))
-                        goto hit; /* You sunk my battleship! */
+                if (INET6_MATCH(sk, net, hash, saddr, daddr, ports, dif)) {
+                        if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
+                                goto begintw;
+                        if (!INET6_MATCH(sk, net, hash, saddr, daddr, ports, dif)) {
+                                sock_put(sk);
+                                goto begin;
+                        }
+                goto out;
+                }
         }
+        if (get_nulls_value(node) != slot)
+                goto begin;
+
+begintw:
         /* Must check for a TIME_WAIT'er before going to listener hash. */
-        sk_for_each(sk, node, &head->twchain) {
-                if (INET6_TW_MATCH(sk, net, hash, saddr, daddr, ports, dif))
-                        goto hit;
+        sk_nulls_for_each_rcu(sk, node, &head->twchain) {
+                if (INET6_TW_MATCH(sk, net, hash, saddr, daddr, ports, dif)) {
+                        if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) {
+                                sk = NULL;
+                                goto out;
+                        }
+                        if (!INET6_TW_MATCH(sk, net, hash, saddr, daddr, ports, dif)) {
+                                sock_put(sk);
+                                goto begintw;
+                        }
+                        goto out;
+                }
         }
-        read_unlock(lock);
-        return NULL;
-
-hit:
-        sock_hold(sk);
-        read_unlock(lock);
+        if (get_nulls_value(node) != slot)
+                goto begintw;
+        sk = NULL;
+out:
+        rcu_read_unlock();
         return sk;
 }
 EXPORT_SYMBOL(__inet6_lookup_established);
@@ -102,10 +126,11 @@ struct sock *inet6_lookup_listener(struct net *net,
         const struct hlist_node *node;
         struct sock *result = NULL;
         int score, hiscore = 0;
+        struct inet_listen_hashbucket *ilb;
 
-        read_lock(&hashinfo->lhash_lock);
-        sk_for_each(sk, node,
-                        &hashinfo->listening_hash[inet_lhashfn(net, hnum)]) {
+        ilb = &hashinfo->listening_hash[inet_lhashfn(net, hnum)];
+        spin_lock(&ilb->lock);
+        sk_for_each(sk, node, &ilb->head) {
                 if (net_eq(sock_net(sk), net) && inet_sk(sk)->num == hnum &&
                                 sk->sk_family == PF_INET6) {
                         const struct ipv6_pinfo *np = inet6_sk(sk);
@@ -133,7 +158,7 @@ struct sock *inet6_lookup_listener(struct net *net,
         }
         if (result)
                 sock_hold(result);
-        read_unlock(&hashinfo->lhash_lock);
+        spin_unlock(&ilb->lock);
         return result;
 }
 
@@ -172,14 +197,14 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row,
         struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
         rwlock_t *lock = inet_ehash_lockp(hinfo, hash);
         struct sock *sk2;
-        const struct hlist_node *node;
+        const struct hlist_nulls_node *node;
         struct inet_timewait_sock *tw;
 
         prefetch(head->chain.first);
         write_lock(lock);
 
         /* Check TIME-WAIT sockets first. */
-        sk_for_each(sk2, node, &head->twchain) {
+        sk_nulls_for_each(sk2, node, &head->twchain) {
                 tw = inet_twsk(sk2);
 
                 if (INET6_TW_MATCH(sk2, net, hash, saddr, daddr, ports, dif)) {
@@ -192,7 +217,7 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row,
         tw = NULL;
 
         /* And established part... */
-        sk_for_each(sk2, node, &head->chain) {
+        sk_nulls_for_each(sk2, node, &head->chain) {
                 if (INET6_MATCH(sk2, net, hash, saddr, daddr, ports, dif))
                         goto not_unique;
         }
@@ -203,7 +228,7 @@ unique:
         inet->num = lport;
         inet->sport = htons(lport);
         WARN_ON(!sk_unhashed(sk));
-        __sk_add_node(sk, &head->chain);
+        __sk_nulls_add_node_rcu(sk, &head->chain);
         sk->sk_hash = hash;
         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
         write_unlock(lock);