udp: Use hlist_nulls in UDP RCU code

This is a straightforward patch, using hlist_nulls infrastructure.

RCUification already done on UDP two weeks ago.

Using hlist_nulls permits us to avoid some memory barriers, both
at lookup time and delete time.

Patch is large because it adds new macros to include/net/sock.h.
These macros will be used by TCP & DCCP in next patch.

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 14 insertions, 12 deletions

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 8dafa36b1ba5..fd2d9ad4a8a3 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -98,7 +98,7 @@ static struct sock *__udp6_lib_lookup(struct net *net,
                                       int dif, struct udp_table *udptable)
 {
         struct sock *sk, *result;
-        struct hlist_node *node, *next;
+        struct hlist_nulls_node *node;
         unsigned short hnum = ntohs(dport);
         unsigned int hash = udp_hashfn(net, hnum);
         struct udp_hslot *hslot = &udptable->hash[hash];
@@ -108,19 +108,21 @@ static struct sock *__udp6_lib_lookup(struct net *net,
 begin:
         result = NULL;
         badness = -1;
-        sk_for_each_rcu_safenext(sk, node, &hslot->head, next) {
-                /*
-                 * lockless reader, and SLAB_DESTROY_BY_RCU items:
-                 * We must check this item was not moved to another chain
-                 */
-                if (udp_hashfn(net, sk->sk_hash) != hash)
-                        goto begin;
+        sk_nulls_for_each_rcu(sk, node, &hslot->head) {
                 score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);
                 if (score > badness) {
                         result = sk;
                         badness = score;
                 }
         }
+        /*
+         * if the nulls value we got at the end of this lookup is
+         * not the expected one, we must restart lookup.
+         * We probably met an item that was moved to another chain.
+         */
+        if (get_nulls_value(node) != hash)
+                goto begin;
+
         if (result) {
                 if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
                         result = NULL;
@@ -374,11 +376,11 @@ static struct sock *udp_v6_mcast_next(struct net *net, struct sock *sk,
                                       __be16 rmt_port, struct in6_addr *rmt_addr,
                                       int dif)
 {
-        struct hlist_node *node;
+        struct hlist_nulls_node *node;
         struct sock *s = sk;
         unsigned short num = ntohs(loc_port);
 
-        sk_for_each_from(s, node) {
+        sk_nulls_for_each_from(s, node) {
                 struct inet_sock *inet = inet_sk(s);
 
                 if (!net_eq(sock_net(s), net))
@@ -423,7 +425,7 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
         int dif;
 
         spin_lock(&hslot->lock);
-        sk = sk_head(&hslot->head);
+        sk = sk_nulls_head(&hslot->head);
         dif = inet6_iif(skb);
         sk = udp_v6_mcast_next(net, sk, uh->dest, daddr, uh->source, saddr, dif);
         if (!sk) {
@@ -432,7 +434,7 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
         }
 
         sk2 = sk;
-        while ((sk2 = udp_v6_mcast_next(net, sk_next(sk2), uh->dest, daddr,
+        while ((sk2 = udp_v6_mcast_next(net, sk_nulls_next(sk2), uh->dest, daddr,
                                         uh->source, saddr, dif))) {
                 struct sk_buff *buff = skb_clone(skb, GFP_ATOMIC);
                 if (buff) {