6 files changed, 90 insertions, 18 deletions

diff --git a/include/net/sock.h b/include/net/sock.h
index d200dfbe1ef6..0bea25db5471 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -363,6 +363,27 @@ static __inline__ int sk_del_node_init(struct sock *sk)
         return rc;
 }
 
+static __inline__ int __sk_del_node_init_rcu(struct sock *sk)
+{
+        if (sk_hashed(sk)) {
+                hlist_del_init_rcu(&sk->sk_node);
+                return 1;
+        }
+        return 0;
+}
+
+static __inline__ int sk_del_node_init_rcu(struct sock *sk)
+{
+        int rc = __sk_del_node_init_rcu(sk);
+
+        if (rc) {
+                /* paranoid for a while -acme */
+                WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
+                __sock_put(sk);
+        }
+        return rc;
+}
+
 static __inline__ void __sk_add_node(struct sock *sk, struct hlist_head *list)
 {
         hlist_add_head(&sk->sk_node, list);
@@ -374,6 +395,17 @@ static __inline__ void sk_add_node(struct sock *sk, struct hlist_head *list)
         __sk_add_node(sk, list);
 }
 
+static __inline__ void __sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
+{
+        hlist_add_head_rcu(&sk->sk_node, list);
+}
+
+static __inline__ void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
+{
+        sock_hold(sk);
+        __sk_add_node_rcu(sk, list);
+}
+
 static __inline__ void __sk_del_bind_node(struct sock *sk)
 {
         __hlist_del(&sk->sk_bind_node);
@@ -387,6 +419,8 @@ static __inline__ void sk_add_bind_node(struct sock *sk,
 
 #define sk_for_each(__sk, node, list) \
         hlist_for_each_entry(__sk, node, list, sk_node)
+#define sk_for_each_rcu(__sk, node, list) \
+        hlist_for_each_entry_rcu(__sk, node, list, sk_node)
 #define sk_for_each_from(__sk, node) \
         if (__sk && ({ node = &(__sk)->sk_node; 1; })) \
                 hlist_for_each_entry_from(__sk, node, sk_node)
@@ -589,8 +623,9 @@ struct proto {
         int                     *sysctl_rmem;
         int                     max_header;
 
-        struct kmem_cache               *slab;
+        struct kmem_cache       *slab;
         unsigned int            obj_size;
+        int                     slab_flags;
 
         atomic_t                *orphan_count;
 
diff --git a/net/core/sock.c b/net/core/sock.c
index 5e2a3132a8c9..ded1eb5d2fd4 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2042,7 +2042,8 @@ int proto_register(struct proto *prot, int alloc_slab)
 
         if (alloc_slab) {
                 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
-                                               SLAB_HWCACHE_ALIGN, NULL);
+                                        SLAB_HWCACHE_ALIGN | prot->slab_flags,
+                                        NULL);
 
                 if (prot->slab == NULL) {
                         printk(KERN_CRIT "%s: Can't create sock SLAB cache!\n",
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 2a6c491f97d7..0ea974bf7962 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -187,7 +187,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum,
         inet_sk(sk)->num = snum;
         sk->sk_hash = snum;
         if (sk_unhashed(sk)) {
-                sk_add_node(sk, &hslot->head);
+                sk_add_node_rcu(sk, &hslot->head);
                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
         }
         error = 0;
@@ -253,15 +253,24 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
                 __be16 sport, __be32 daddr, __be16 dport,
                 int dif, struct udp_table *udptable)
 {
-        struct sock *sk, *result = NULL;
+        struct sock *sk, *result;
         struct hlist_node *node;
         unsigned short hnum = ntohs(dport);
         unsigned int hash = udp_hashfn(net, hnum);
         struct udp_hslot *hslot = &udptable->hash[hash];
-        int score, badness = -1;
+        int score, badness;
 
-        spin_lock(&hslot->lock);
-        sk_for_each(sk, node, &hslot->head) {
+        rcu_read_lock();
+begin:
+        result = NULL;
+        badness = -1;
+        sk_for_each_rcu(sk, node, &hslot->head) {
+                /*
+                 * lockless reader, and SLAB_DESTROY_BY_RCU items:
+                 * We must check this item was not moved to another chain
+                 */
+                if (udp_hashfn(net, sk->sk_hash) != hash)
+                        goto begin;
                 score = compute_score(sk, net, saddr, hnum, sport,
                                       daddr, dport, dif);
                 if (score > badness) {
@@ -269,9 +278,16 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
                         badness = score;
                 }
         }
-        if (result)
-                sock_hold(result);
-        spin_unlock(&hslot->lock);
+        if (result) {
+                if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
+                        result = NULL;
+                else if (unlikely(compute_score(result, net, saddr, hnum, sport,
+                                  daddr, dport, dif) < badness)) {
+                        sock_put(result);
+                        goto begin;
+                }
+        }
+        rcu_read_unlock();
         return result;
 }
 
@@ -953,7 +969,7 @@ void udp_lib_unhash(struct sock *sk)
         struct udp_hslot *hslot = &udptable->hash[hash];
 
         spin_lock(&hslot->lock);
-        if (sk_del_node_init(sk)) {
+        if (sk_del_node_init_rcu(sk)) {
                 inet_sk(sk)->num = 0;
                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
         }
@@ -1517,6 +1533,7 @@ struct proto udp_prot = {
         .sysctl_wmem       = &sysctl_udp_wmem_min,
         .sysctl_rmem       = &sysctl_udp_rmem_min,
         .obj_size          = sizeof(struct udp_sock),
+        .slab_flags        = SLAB_DESTROY_BY_RCU,
         .h.udp_table       = &udp_table,
 #ifdef CONFIG_COMPAT
         .compat_setsockopt = compat_udp_setsockopt,
diff --git a/net/ipv4/udplite.c b/net/ipv4/udplite.c
index d8ea8e5f5ea3..c784891cb7e5 100644
--- a/net/ipv4/udplite.c
+++ b/net/ipv4/udplite.c
@@ -51,6 +51,7 @@ struct proto 	udplite_prot = {
         .unhash            = udp_lib_unhash,
         .get_port          = udp_v4_get_port,
         .obj_size          = sizeof(struct udp_sock),
+        .slab_flags        = SLAB_DESTROY_BY_RCU,
         .h.udp_table       = &udplite_table,
 #ifdef CONFIG_COMPAT
         .compat_setsockopt = compat_udp_setsockopt,
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index ccee7244ca0f..1d9790e43dfc 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -97,24 +97,40 @@ static struct sock *__udp6_lib_lookup(struct net *net,
                                       struct in6_addr *daddr, __be16 dport,
                                       int dif, struct udp_table *udptable)
 {
-        struct sock *sk, *result = NULL;
+        struct sock *sk, *result;
         struct hlist_node *node;
         unsigned short hnum = ntohs(dport);
         unsigned int hash = udp_hashfn(net, hnum);
         struct udp_hslot *hslot = &udptable->hash[hash];
-        int score, badness = -1;
+        int score, badness;
 
-        spin_lock(&hslot->lock);
-        sk_for_each(sk, node, &hslot->head) {
+        rcu_read_lock();
+begin:
+        result = NULL;
+        badness = -1;
+        sk_for_each_rcu(sk, node, &hslot->head) {
+                /*
+                 * lockless reader, and SLAB_DESTROY_BY_RCU items:
+                 * We must check this item was not moved to another chain
+                 */
+                if (udp_hashfn(net, sk->sk_hash) != hash)
+                        goto begin;
                 score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);
                 if (score > badness) {
                         result = sk;
                         badness = score;
                 }
         }
-        if (result)
-                sock_hold(result);
-        spin_unlock(&hslot->lock);
+        if (result) {
+                if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
+                        result = NULL;
+                else if (unlikely(compute_score(result, net, hnum, saddr, sport,
+                                        daddr, dport, dif) < badness)) {
+                        sock_put(result);
+                        goto begin;
+                }
+        }
+        rcu_read_unlock();
         return result;
 }
 
@@ -1062,6 +1078,7 @@ struct proto udpv6_prot = {
         .sysctl_wmem       = &sysctl_udp_wmem_min,
         .sysctl_rmem       = &sysctl_udp_rmem_min,
         .obj_size          = sizeof(struct udp6_sock),
+        .slab_flags        = SLAB_DESTROY_BY_RCU,
         .h.udp_table       = &udp_table,
 #ifdef CONFIG_COMPAT
         .compat_setsockopt = compat_udpv6_setsockopt,
diff --git a/net/ipv6/udplite.c b/net/ipv6/udplite.c
index f1e892a99e05..ba162a824585 100644
--- a/net/ipv6/udplite.c
+++ b/net/ipv6/udplite.c
@@ -49,6 +49,7 @@ struct proto udplitev6_prot = {
         .unhash            = udp_lib_unhash,
         .get_port          = udp_v6_get_port,
         .obj_size          = sizeof(struct udp6_sock),
+        .slab_flags        = SLAB_DESTROY_BY_RCU,
         .h.udp_table       = &udplite_table,
 #ifdef CONFIG_COMPAT
         .compat_setsockopt = compat_udpv6_setsockopt,