tcp: md5: rcu conversion

In order to be able to support proper RST messages for TCP MD5 flows, we
need to allow access to MD5 keys without locking listener socket.

This conversion is a nice cleanup, and shrinks size of timewait sockets
by 80 bytes.

IPv6 code reuses generic code found in IPv4 instead of duplicating it.

Control path uses GFP_KERNEL allocations instead of GFP_ATOMIC.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Shawn Lu <shawn.lu@ericsson.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

2 files changed, 99 insertions, 140 deletions

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 345e24928fa6..1d5fd82c5c08 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -90,16 +90,8 @@ EXPORT_SYMBOL(sysctl_tcp_low_latency);
 
 
 #ifdef CONFIG_TCP_MD5SIG
-static struct tcp_md5sig_key *tcp_v4_md5_do_lookup(struct sock *sk,
-                                                   __be32 addr);
-static int tcp_v4_md5_hash_hdr(char *md5_hash, struct tcp_md5sig_key *key,
+static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
                                __be32 daddr, __be32 saddr, const struct tcphdr *th);
-#else
-static inline
-struct tcp_md5sig_key *tcp_v4_md5_do_lookup(struct sock *sk, __be32 addr)
-{
-        return NULL;
-}
 #endif
 
 struct inet_hashinfo tcp_hashinfo;
@@ -631,7 +623,9 @@ static void tcp_v4_send_reset(struct sock *sk, struct sk_buff *skb)
         arg.iov[0].iov_len  = sizeof(rep.th);
 
 #ifdef CONFIG_TCP_MD5SIG
-        key = sk ? tcp_v4_md5_do_lookup(sk, ip_hdr(skb)->saddr) : NULL;
+        key = sk ? tcp_md5_do_lookup(sk,
+                                     (union tcp_md5_addr *)&ip_hdr(skb)->saddr,
+                                     AF_INET) : NULL;
         if (key) {
                 rep.opt[0] = htonl((TCPOPT_NOP << 24) |
                                    (TCPOPT_NOP << 16) |
@@ -759,7 +753,8 @@ static void tcp_v4_reqsk_send_ack(struct sock *sk, struct sk_buff *skb,
                         tcp_rsk(req)->rcv_isn + 1, req->rcv_wnd,
                         req->ts_recent,
                         0,
-                        tcp_v4_md5_do_lookup(sk, ip_hdr(skb)->daddr),
+                        tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&ip_hdr(skb)->daddr,
+                                          AF_INET),
                         inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
                         ip_hdr(skb)->tos);
 }
@@ -876,146 +871,124 @@ static struct ip_options_rcu *tcp_v4_save_options(struct sock *sk,
  */
 
 /* Find the Key structure for an address.  */
-static struct tcp_md5sig_key *
-                        tcp_v4_md5_do_lookup(struct sock *sk, __be32 addr)
+struct tcp_md5sig_key *tcp_md5_do_lookup(struct sock *sk,
+                                         const union tcp_md5_addr *addr,
+                                         int family)
 {
         struct tcp_sock *tp = tcp_sk(sk);
-        int i;
+        struct tcp_md5sig_key *key;
+        struct hlist_node *pos;
+        unsigned int size = sizeof(struct in_addr);
 
-        if (!tp->md5sig_info || !tp->md5sig_info->entries4)
+        if (!tp->md5sig_info)
                 return NULL;
-        for (i = 0; i < tp->md5sig_info->entries4; i++) {
-                if (tp->md5sig_info->keys4[i].addr == addr)
-                        return &tp->md5sig_info->keys4[i].base;
+#if IS_ENABLED(CONFIG_IPV6)
+        if (family == AF_INET6)
+                size = sizeof(struct in6_addr);
+#endif
+        hlist_for_each_entry_rcu(key, pos, &tp->md5sig_info->head, node) {
+                if (key->family != family)
+                        continue;
+                if (!memcmp(&key->addr, addr, size))
+                        return key;
         }
         return NULL;
 }
+EXPORT_SYMBOL(tcp_md5_do_lookup);
 
 struct tcp_md5sig_key *tcp_v4_md5_lookup(struct sock *sk,
                                          struct sock *addr_sk)
 {
-        return tcp_v4_md5_do_lookup(sk, inet_sk(addr_sk)->inet_daddr);
+        union tcp_md5_addr *addr;
+
+        addr = (union tcp_md5_addr *)&inet_sk(addr_sk)->inet_daddr;
+        return tcp_md5_do_lookup(sk, addr, AF_INET);
 }
 EXPORT_SYMBOL(tcp_v4_md5_lookup);
 
 static struct tcp_md5sig_key *tcp_v4_reqsk_md5_lookup(struct sock *sk,
                                                       struct request_sock *req)
 {
-        return tcp_v4_md5_do_lookup(sk, inet_rsk(req)->rmt_addr);
+        union tcp_md5_addr *addr;
+
+        addr = (union tcp_md5_addr *)&inet_rsk(req)->rmt_addr;
+        return tcp_md5_do_lookup(sk, addr, AF_INET);
 }
 
 /* This can be called on a newly created socket, from other files */
-int tcp_v4_md5_do_add(struct sock *sk, __be32 addr,
-                      u8 *newkey, u8 newkeylen)
+int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
+                   int family, const u8 *newkey, u8 newkeylen, gfp_t gfp)
 {
         /* Add Key to the list */
         struct tcp_md5sig_key *key;
         struct tcp_sock *tp = tcp_sk(sk);
-        struct tcp4_md5sig_key *keys;
+        struct tcp_md5sig_info *md5sig;
 
-        key = tcp_v4_md5_do_lookup(sk, addr);
+        key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&addr, AF_INET);
         if (key) {
                 /* Pre-existing entry - just update that one. */
-                kfree(key->key);
-                key->key = newkey;
+                memcpy(key->key, newkey, newkeylen);
                 key->keylen = newkeylen;
-        } else {
-                struct tcp_md5sig_info *md5sig;
-
-                if (!tp->md5sig_info) {
-                        tp->md5sig_info = kzalloc(sizeof(*tp->md5sig_info),
-                                                  GFP_ATOMIC);
-                        if (!tp->md5sig_info) {
-                                kfree(newkey);
-                                return -ENOMEM;
-                        }
-                        sk_nocaps_add(sk, NETIF_F_GSO_MASK);
-                }
+                return 0;
+        }
 
-                md5sig = tp->md5sig_info;
-                if (md5sig->entries4 == 0 &&
-                    tcp_alloc_md5sig_pool(sk) == NULL) {
-                        kfree(newkey);
+        md5sig = tp->md5sig_info;
+        if (!md5sig) {
+                md5sig = kmalloc(sizeof(*md5sig), gfp);
+                if (!md5sig)
                         return -ENOMEM;
-                }
-
-                if (md5sig->alloced4 == md5sig->entries4) {
-                        keys = kmalloc((sizeof(*keys) *
-                                        (md5sig->entries4 + 1)), GFP_ATOMIC);
-                        if (!keys) {
-                                kfree(newkey);
-                                if (md5sig->entries4 == 0)
-                                        tcp_free_md5sig_pool();
-                                return -ENOMEM;
-                        }
 
-                        if (md5sig->entries4)
-                                memcpy(keys, md5sig->keys4,
-                                       sizeof(*keys) * md5sig->entries4);
+                sk_nocaps_add(sk, NETIF_F_GSO_MASK);
+                INIT_HLIST_HEAD(&md5sig->head);
+                tp->md5sig_info = md5sig;
+        }
 
-                        /* Free old key list, and reference new one */
-                        kfree(md5sig->keys4);
-                        md5sig->keys4 = keys;
-                        md5sig->alloced4++;
-                }
-                md5sig->entries4++;
-                md5sig->keys4[md5sig->entries4 - 1].addr        = addr;
-                md5sig->keys4[md5sig->entries4 - 1].base.key    = newkey;
-                md5sig->keys4[md5sig->entries4 - 1].base.keylen = newkeylen;
+        key = kmalloc(sizeof(*key), gfp);
+        if (!key)
+                return -ENOMEM;
+        if (hlist_empty(&md5sig->head) && !tcp_alloc_md5sig_pool(sk)) {
+                kfree(key);
+                return -ENOMEM;
         }
+
+        memcpy(key->key, newkey, newkeylen);
+        key->keylen = newkeylen;
+        key->family = family;
+        memcpy(&key->addr, addr,
+               (family == AF_INET6) ? sizeof(struct in6_addr) :
+                                      sizeof(struct in_addr));
+        hlist_add_head_rcu(&key->node, &md5sig->head);
         return 0;
 }
-EXPORT_SYMBOL(tcp_v4_md5_do_add);
+EXPORT_SYMBOL(tcp_md5_do_add);
 
-int tcp_v4_md5_do_del(struct sock *sk, __be32 addr)
+int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family)
 {
         struct tcp_sock *tp = tcp_sk(sk);
-        int i;
-
-        for (i = 0; i < tp->md5sig_info->entries4; i++) {
-                if (tp->md5sig_info->keys4[i].addr == addr) {
-                        /* Free the key */
-                        kfree(tp->md5sig_info->keys4[i].base.key);
-                        tp->md5sig_info->entries4--;
-
-                        if (tp->md5sig_info->entries4 == 0) {
-                                kfree(tp->md5sig_info->keys4);
-                                tp->md5sig_info->keys4 = NULL;
-                                tp->md5sig_info->alloced4 = 0;
-                                tcp_free_md5sig_pool();
-                        } else if (tp->md5sig_info->entries4 != i) {
-                                /* Need to do some manipulation */
-                                memmove(&tp->md5sig_info->keys4[i],
-                                        &tp->md5sig_info->keys4[i+1],
-                                        (tp->md5sig_info->entries4 - i) *
-                                         sizeof(struct tcp4_md5sig_key));
-                        }
-                        return 0;
-                }
-        }
-        return -ENOENT;
+        struct tcp_md5sig_key *key;
+
+        key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&addr, AF_INET);
+        if (!key)
+                return -ENOENT;
+        hlist_del_rcu(&key->node);
+        kfree_rcu(key, rcu);
+        if (hlist_empty(&tp->md5sig_info->head))
+                tcp_free_md5sig_pool();
+        return 0;
 }
-EXPORT_SYMBOL(tcp_v4_md5_do_del);
+EXPORT_SYMBOL(tcp_md5_do_del);
 
-static void tcp_v4_clear_md5_list(struct sock *sk)
+void tcp_clear_md5_list(struct sock *sk)
 {
         struct tcp_sock *tp = tcp_sk(sk);
+        struct tcp_md5sig_key *key;
+        struct hlist_node *pos, *n;
 
-        /* Free each key, then the set of key keys,
-         * the crypto element, and then decrement our
-         * hold on the last resort crypto.
-         */
-        if (tp->md5sig_info->entries4) {
-                int i;
-                for (i = 0; i < tp->md5sig_info->entries4; i++)
-                        kfree(tp->md5sig_info->keys4[i].base.key);
-                tp->md5sig_info->entries4 = 0;
+        if (!hlist_empty(&tp->md5sig_info->head))
                 tcp_free_md5sig_pool();
-        }
-        if (tp->md5sig_info->keys4) {
-                kfree(tp->md5sig_info->keys4);
-                tp->md5sig_info->keys4 = NULL;
-                tp->md5sig_info->alloced4  = 0;
+        hlist_for_each_entry_safe(key, pos, n, &tp->md5sig_info->head, node) {
+                hlist_del_rcu(&key->node);
+                kfree_rcu(key, rcu);
         }
 }
 
@@ -1024,7 +997,6 @@ static int tcp_v4_parse_md5_keys(struct sock *sk, char __user *optval,
 {
         struct tcp_md5sig cmd;
         struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
-        u8 *newkey;
 
         if (optlen < sizeof(cmd))
                 return -EINVAL;
@@ -1038,29 +1010,16 @@ static int tcp_v4_parse_md5_keys(struct sock *sk, char __user *optval,
         if (!cmd.tcpm_key || !cmd.tcpm_keylen) {
                 if (!tcp_sk(sk)->md5sig_info)
                         return -ENOENT;
-                return tcp_v4_md5_do_del(sk, sin->sin_addr.s_addr);
+                return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
+                                      AF_INET);
         }
 
         if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
                 return -EINVAL;
 
-        if (!tcp_sk(sk)->md5sig_info) {
-                struct tcp_sock *tp = tcp_sk(sk);
-                struct tcp_md5sig_info *p;
-
-                p = kzalloc(sizeof(*p), sk->sk_allocation);
-                if (!p)
-                        return -EINVAL;
-
-                tp->md5sig_info = p;
-                sk_nocaps_add(sk, NETIF_F_GSO_MASK);
-        }
-
-        newkey = kmemdup(cmd.tcpm_key, cmd.tcpm_keylen, sk->sk_allocation);
-        if (!newkey)
-                return -ENOMEM;
-        return tcp_v4_md5_do_add(sk, sin->sin_addr.s_addr,
-                                 newkey, cmd.tcpm_keylen);
+        return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
+                              AF_INET, cmd.tcpm_key, cmd.tcpm_keylen,
+                              GFP_KERNEL);
 }
 
 static int tcp_v4_md5_hash_pseudoheader(struct tcp_md5sig_pool *hp,
@@ -1086,7 +1045,7 @@ static int tcp_v4_md5_hash_pseudoheader(struct tcp_md5sig_pool *hp,
         return crypto_hash_update(&hp->md5_desc, &sg, sizeof(*bp));
 }
 
-static int tcp_v4_md5_hash_hdr(char *md5_hash, struct tcp_md5sig_key *key,
+static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
                                __be32 daddr, __be32 saddr, const struct tcphdr *th)
 {
         struct tcp_md5sig_pool *hp;
@@ -1186,7 +1145,8 @@ static int tcp_v4_inbound_md5_hash(struct sock *sk, const struct sk_buff *skb)
         int genhash;
         unsigned char newhash[16];
 
-        hash_expected = tcp_v4_md5_do_lookup(sk, iph->saddr);
+        hash_expected = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&iph->saddr,
+                                          AF_INET);
         hash_location = tcp_parse_md5sig_option(th);
 
         /* We've parsed the options - do we have a hash? */
@@ -1474,7 +1434,8 @@ struct sock *tcp_v4_syn_recv_sock(struct sock *sk, struct sk_buff *skb,
 
 #ifdef CONFIG_TCP_MD5SIG
         /* Copy over the MD5 key from the original socket */
-        key = tcp_v4_md5_do_lookup(sk, newinet->inet_daddr);
+        key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&newinet->inet_daddr,
+                                AF_INET);
         if (key != NULL) {
                 /*
                  * We're using one, so create a matching key
@@ -1482,10 +1443,8 @@ struct sock *tcp_v4_syn_recv_sock(struct sock *sk, struct sk_buff *skb,
                  * memory, then we end up not copying the key
                  * across. Shucks.
                  */
-                char *newkey = kmemdup(key->key, key->keylen, GFP_ATOMIC);
-                if (newkey != NULL)
-                        tcp_v4_md5_do_add(newsk, newinet->inet_daddr,
-                                          newkey, key->keylen);
+                tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newinet->inet_daddr,
+                               AF_INET, key->key, key->keylen, GFP_ATOMIC);
                 sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
         }
 #endif
@@ -1934,7 +1893,7 @@ void tcp_v4_destroy_sock(struct sock *sk)
 #ifdef CONFIG_TCP_MD5SIG
         /* Clean up the MD5 key list, if any */
         if (tp->md5sig_info) {
-                tcp_v4_clear_md5_list(sk);
+                tcp_clear_md5_list(sk);
                 kfree(tp->md5sig_info);
                 tp->md5sig_info = NULL;
         }
diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c
index 550e755747e0..3cabafb5cdd1 100644
--- a/net/ipv4/tcp_minisocks.c
+++ b/net/ipv4/tcp_minisocks.c
@@ -359,13 +359,11 @@ void tcp_time_wait(struct sock *sk, int state, int timeo)
                  */
                 do {
                         struct tcp_md5sig_key *key;
-                        memset(tcptw->tw_md5_key, 0, sizeof(tcptw->tw_md5_key));
-                        tcptw->tw_md5_keylen = 0;
+                        tcptw->tw_md5_key = NULL;
                         key = tp->af_specific->md5_lookup(sk, sk);
                         if (key != NULL) {
-                                memcpy(&tcptw->tw_md5_key, key->key, key->keylen);
-                                tcptw->tw_md5_keylen = key->keylen;
-                                if (tcp_alloc_md5sig_pool(sk) == NULL)
+                                tcptw->tw_md5_key = kmemdup(key, sizeof(*key), GFP_ATOMIC);
+                                if (tcptw->tw_md5_key && tcp_alloc_md5sig_pool(sk) == NULL)
                                         BUG();
                         }
                 } while (0);
@@ -405,8 +403,10 @@ void tcp_twsk_destructor(struct sock *sk)
 {
 #ifdef CONFIG_TCP_MD5SIG
         struct tcp_timewait_sock *twsk = tcp_twsk(sk);
-        if (twsk->tw_md5_keylen)
+        if (twsk->tw_md5_key) {
                 tcp_free_md5sig_pool();
+                kfree_rcu(twsk->tw_md5_key, rcu);
+        }
 #endif
 }
 EXPORT_SYMBOL_GPL(tcp_twsk_destructor);