Merge branch 'net_get_random_once'

Hannes Frederic Sowa says:

====================
This series implements support for delaying the initialization of secret
keys, e.g. used for hashing, for as long as possible. This functionality
is implemented by a new macro, net_get_random_bytes.

I already used it to protect the socket hashes, the syncookie secret
(most important) and the tcp_fastopen secrets.

Changelog:
v2) Use static_keys in net_get_random_once to have as minimal impact to
    the fast-path as possible.
v3) added patch "static_key: WARN on usage before jump_label_init was called":
    Patch "x86/jump_label: expect default_nop if static_key gets enabled
    on boot-up" relaxes the checks for using static_key primitives before
    jump_label_init. So tighten them first.
v4) Update changelog on the patch "static_key: WARN on usage before
    jump_label_init was called"

Included patches:
 ipv4: split inet_ehashfn to hash functions per compilation unit
 ipv6: split inet6_ehashfn to hash functions per compilation unit
 static_key: WARN on usage before jump_label_init was called
 x86/jump_label: expect default_nop if static_key gets enabled on boot-up
 net: introduce new macro net_get_random_once
 inet: split syncookie keys for ipv4 and ipv6 and initialize with net_get_random_once
 inet: convert inet_ehash_secret and ipv6_hash_secret to net_get_random_once
 tcp: switch tcp_fastopen key generation to net_get_random_once
 net: switch net_secret key generation to net_get_random_once
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

13 files changed, 195 insertions, 79 deletions

diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c
index 3f1ec1586ae1..b02fd16b8942 100644
--- a/net/core/secure_seq.c
+++ b/net/core/secure_seq.c
@@ -7,6 +7,7 @@
 #include <linux/hrtimer.h>
 #include <linux/ktime.h>
 #include <linux/string.h>
+#include <linux/net.h>
 
 #include <net/secure_seq.h>
 
@@ -16,18 +17,7 @@ static u32 net_secret[NET_SECRET_SIZE] ____cacheline_aligned;
 
 static void net_secret_init(void)
 {
-        u32 tmp;
-        int i;
-
-        if (likely(net_secret[0]))
-                return;
-
-        for (i = NET_SECRET_SIZE; i > 0;) {
-                do {
-                        get_random_bytes(&tmp, sizeof(tmp));
-                } while (!tmp);
-                cmpxchg(&net_secret[--i], 0, tmp);
-        }
+        net_get_random_once(net_secret, sizeof(net_secret));
 }
 
 #ifdef CONFIG_INET
diff --git a/net/core/utils.c b/net/core/utils.c
index aa88e23fc87a..bf09371e19b1 100644
--- a/net/core/utils.c
+++ b/net/core/utils.c
@@ -338,3 +338,51 @@ void inet_proto_csum_replace16(__sum16 *sum, struct sk_buff *skb,
                                   csum_unfold(*sum)));
 }
 EXPORT_SYMBOL(inet_proto_csum_replace16);
+
+struct __net_random_once_work {
+        struct work_struct work;
+        struct static_key *key;
+};
+
+static void __net_random_once_deferred(struct work_struct *w)
+{
+        struct __net_random_once_work *work =
+                container_of(w, struct __net_random_once_work, work);
+        if (!static_key_enabled(work->key))
+                static_key_slow_inc(work->key);
+        kfree(work);
+}
+
+static void __net_random_once_disable_jump(struct static_key *key)
+{
+        struct __net_random_once_work *w;
+
+        w = kmalloc(sizeof(*w), GFP_ATOMIC);
+        if (!w)
+                return;
+
+        INIT_WORK(&w->work, __net_random_once_deferred);
+        w->key = key;
+        schedule_work(&w->work);
+}
+
+bool __net_get_random_once(void *buf, int nbytes, bool *done,
+                           struct static_key *done_key)
+{
+        static DEFINE_SPINLOCK(lock);
+
+        spin_lock_bh(&lock);
+        if (*done) {
+                spin_unlock_bh(&lock);
+                return false;
+        }
+
+        get_random_bytes(buf, nbytes);
+        *done = true;
+        spin_unlock_bh(&lock);
+
+        __net_random_once_disable_jump(done_key);
+
+        return true;
+}
+EXPORT_SYMBOL(__net_get_random_once);
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 4049906010f7..9433a6186f54 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -245,29 +245,6 @@ out:
 }
 EXPORT_SYMBOL(inet_listen);
 
-u32 inet_ehash_secret __read_mostly;
-EXPORT_SYMBOL(inet_ehash_secret);
-
-u32 ipv6_hash_secret __read_mostly;
-EXPORT_SYMBOL(ipv6_hash_secret);
-
-/*
- * inet_ehash_secret must be set exactly once, and to a non nul value
- * ipv6_hash_secret must be set exactly once.
- */
-void build_ehash_secret(void)
-{
-        u32 rnd;
-
-        do {
-                get_random_bytes(&rnd, sizeof(rnd));
-        } while (rnd == 0);
-
-        if (cmpxchg(&inet_ehash_secret, 0, rnd) == 0)
-                get_random_bytes(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
-}
-EXPORT_SYMBOL(build_ehash_secret);
-
 /*
  *      Create an inet socket.
  */
@@ -284,10 +261,6 @@ static int inet_create(struct net *net, struct socket *sock, int protocol,
         int try_loading_module = 0;
         int err;
 
-        if (unlikely(!inet_ehash_secret))
-                if (sock->type != SOCK_RAW && sock->type != SOCK_DGRAM)
-                        build_ehash_secret();
-
         sock->state = SS_UNCONNECTED;
 
         /* Look for the requested type/protocol pair. */
diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index a4b66bbe4f21..8b9cf279450d 100644
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -24,6 +24,31 @@
 #include <net/secure_seq.h>
 #include <net/ip.h>
 
+static unsigned int inet_ehashfn(struct net *net, const __be32 laddr,
+                                 const __u16 lport, const __be32 faddr,
+                                 const __be16 fport)
+{
+        static u32 inet_ehash_secret __read_mostly;
+
+        net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));
+
+        return __inet_ehashfn(laddr, lport, faddr, fport,
+                              inet_ehash_secret + net_hash_mix(net));
+}
+
+
+static unsigned int inet_sk_ehashfn(const struct sock *sk)
+{
+        const struct inet_sock *inet = inet_sk(sk);
+        const __be32 laddr = inet->inet_rcv_saddr;
+        const __u16 lport = inet->inet_num;
+        const __be32 faddr = inet->inet_daddr;
+        const __be16 fport = inet->inet_dport;
+        struct net *net = sock_net(sk);
+
+        return inet_ehashfn(net, laddr, lport, faddr, fport);
+}
+
 /*
  * Allocate and initialize a new local port bind bucket.
  * The bindhash mutex for snum's hash chain must be held here.
diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c
index 3b64c59b4109..b95331e6c077 100644
--- a/net/ipv4/syncookies.c
+++ b/net/ipv4/syncookies.c
@@ -25,15 +25,7 @@
 
 extern int sysctl_tcp_syncookies;
 
-__u32 syncookie_secret[2][16-4+SHA_DIGEST_WORDS];
-EXPORT_SYMBOL(syncookie_secret);
-
-static __init int init_syncookies(void)
-{
-        get_random_bytes(syncookie_secret, sizeof(syncookie_secret));
-        return 0;
-}
-__initcall(init_syncookies);
+static u32 syncookie_secret[2][16-4+SHA_DIGEST_WORDS];
 
 #define COOKIEBITS 24   /* Upper bits store count */
 #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1)
@@ -44,8 +36,11 @@ static DEFINE_PER_CPU(__u32 [16 + 5 + SHA_WORKSPACE_WORDS],
 static u32 cookie_hash(__be32 saddr, __be32 daddr, __be16 sport, __be16 dport,
                        u32 count, int c)
 {
-        __u32 *tmp = __get_cpu_var(ipv4_cookie_scratch);
+        __u32 *tmp;
+
+        net_get_random_once(syncookie_secret, sizeof(syncookie_secret));
 
+        tmp  = __get_cpu_var(ipv4_cookie_scratch);
         memcpy(tmp + 4, syncookie_secret[c], sizeof(syncookie_secret[c]));
         tmp[0] = (__force u32)saddr;
         tmp[1] = (__force u32)daddr;
diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c
index c08f096d46b5..4b161d5aba0b 100644
--- a/net/ipv4/sysctl_net_ipv4.c
+++ b/net/ipv4/sysctl_net_ipv4.c
@@ -274,6 +274,11 @@ static int proc_tcp_fastopen_key(struct ctl_table *ctl, int write,
                         ret = -EINVAL;
                         goto bad_key;
                 }
+                /* Generate a dummy secret but don't publish it. This
+                 * is needed so we don't regenerate a new key on the
+                 * first invocation of tcp_fastopen_cookie_gen
+                 */
+                tcp_fastopen_init_key_once(false);
                 tcp_fastopen_reset_cipher(user_key, TCP_FASTOPEN_KEY_LENGTH);
         }
 
diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c
index ab7bd35bb312..766032b4a6c3 100644
--- a/net/ipv4/tcp_fastopen.c
+++ b/net/ipv4/tcp_fastopen.c
@@ -14,6 +14,20 @@ struct tcp_fastopen_context __rcu *tcp_fastopen_ctx;
 
 static DEFINE_SPINLOCK(tcp_fastopen_ctx_lock);
 
+void tcp_fastopen_init_key_once(bool publish)
+{
+        static u8 key[TCP_FASTOPEN_KEY_LENGTH];
+
+        /* tcp_fastopen_reset_cipher publishes the new context
+         * atomically, so we allow this race happening here.
+         *
+         * All call sites of tcp_fastopen_cookie_gen also check
+         * for a valid cookie, so this is an acceptable risk.
+         */
+        if (net_get_random_once(key, sizeof(key)) && publish)
+                tcp_fastopen_reset_cipher(key, sizeof(key));
+}
+
 static void tcp_fastopen_ctx_free(struct rcu_head *head)
 {
         struct tcp_fastopen_context *ctx =
@@ -70,6 +84,8 @@ void tcp_fastopen_cookie_gen(__be32 src, __be32 dst,
         __be32 path[4] = { src, dst, 0, 0 };
         struct tcp_fastopen_context *ctx;
 
+        tcp_fastopen_init_key_once(true);
+
         rcu_read_lock();
         ctx = rcu_dereference(tcp_fastopen_ctx);
         if (ctx) {
@@ -78,14 +94,3 @@ void tcp_fastopen_cookie_gen(__be32 src, __be32 dst,
         }
         rcu_read_unlock();
 }
-
-static int __init tcp_fastopen_init(void)
-{
-        __u8 key[TCP_FASTOPEN_KEY_LENGTH];
-
-        get_random_bytes(key, sizeof(key));
-        tcp_fastopen_reset_cipher(key, sizeof(key));
-        return 0;
-}
-
-late_initcall(tcp_fastopen_init);
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 9f27bb800607..89909dd730dd 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -407,6 +407,18 @@ static inline int compute_score2(struct sock *sk, struct net *net,
         return score;
 }
 
+static unsigned int udp_ehashfn(struct net *net, const __be32 laddr,
+                                 const __u16 lport, const __be32 faddr,
+                                 const __be16 fport)
+{
+        static u32 udp_ehash_secret __read_mostly;
+
+        net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
+
+        return __inet_ehashfn(laddr, lport, faddr, fport,
+                              udp_ehash_secret + net_hash_mix(net));
+}
+
 
 /* called with read_rcu_lock() */
 static struct sock *udp4_lib_lookup2(struct net *net,
@@ -430,8 +442,8 @@ begin:
                         badness = score;
                         reuseport = sk->sk_reuseport;
                         if (reuseport) {
-                                hash = inet_ehashfn(net, daddr, hnum,
-                                                    saddr, sport);
+                                hash = udp_ehashfn(net, daddr, hnum,
+                                                   saddr, sport);
                                 matches = 1;
                         }
                 } else if (score == badness && reuseport) {
@@ -511,8 +523,8 @@ begin:
                         badness = score;
                         reuseport = sk->sk_reuseport;
                         if (reuseport) {
-                                hash = inet_ehashfn(net, daddr, hnum,
-                                                    saddr, sport);
+                                hash = udp_ehashfn(net, daddr, hnum,
+                                                   saddr, sport);
                                 matches = 1;
                         }
                 } else if (score == badness && reuseport) {
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index a2cb07cd3850..20af1fb81c83 100644
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -110,11 +110,6 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol,
         int try_loading_module = 0;
         int err;
 
-        if (sock->type != SOCK_RAW &&
-            sock->type != SOCK_DGRAM &&
-            !inet_ehash_secret)
-                build_ehash_secret();
-
         /* Look for the requested type/protocol pair. */
 lookup_protocol:
         err = -ESOCKTNOSUPPORT;
diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
index 842d833dfc18..262e13c02ec2 100644
--- a/net/ipv6/inet6_hashtables.c
+++ b/net/ipv6/inet6_hashtables.c
@@ -23,6 +23,39 @@
 #include <net/secure_seq.h>
 #include <net/ip.h>
 
+static unsigned int inet6_ehashfn(struct net *net,
+                                  const struct in6_addr *laddr,
+                                  const u16 lport,
+                                  const struct in6_addr *faddr,
+                                  const __be16 fport)
+{
+        static u32 inet6_ehash_secret __read_mostly;
+        static u32 ipv6_hash_secret __read_mostly;
+
+        u32 lhash, fhash;
+
+        net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
+        net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
+
+        lhash = (__force u32)laddr->s6_addr32[3];
+        fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);
+
+        return __inet6_ehashfn(lhash, lport, fhash, fport,
+                               inet6_ehash_secret + net_hash_mix(net));
+}
+
+static int inet6_sk_ehashfn(const struct sock *sk)
+{
+        const struct inet_sock *inet = inet_sk(sk);
+        const struct in6_addr *laddr = &sk->sk_v6_rcv_saddr;
+        const struct in6_addr *faddr = &sk->sk_v6_daddr;
+        const __u16 lport = inet->inet_num;
+        const __be16 fport = inet->inet_dport;
+        struct net *net = sock_net(sk);
+
+        return inet6_ehashfn(net, laddr, lport, faddr, fport);
+}
+
 int __inet6_hash(struct sock *sk, struct inet_timewait_sock *tw)
 {
         struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c
index d04d3f1dd9b7..535a3ad262f1 100644
--- a/net/ipv6/syncookies.c
+++ b/net/ipv6/syncookies.c
@@ -24,6 +24,8 @@
 #define COOKIEBITS 24   /* Upper bits store count */
 #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1)
 
+static u32 syncookie6_secret[2][16-4+SHA_DIGEST_WORDS];
+
 /* RFC 2460, Section 8.3:
  * [ipv6 tcp] MSS must be computed as the maximum packet size minus 60 [..]
  *
@@ -61,14 +63,18 @@ static DEFINE_PER_CPU(__u32 [16 + 5 + SHA_WORKSPACE_WORDS],
 static u32 cookie_hash(const struct in6_addr *saddr, const struct in6_addr *daddr,
                        __be16 sport, __be16 dport, u32 count, int c)
 {
-        __u32 *tmp = __get_cpu_var(ipv6_cookie_scratch);
+        __u32 *tmp;
+
+        net_get_random_once(syncookie6_secret, sizeof(syncookie6_secret));
+
+        tmp  = __get_cpu_var(ipv6_cookie_scratch);
 
         /*
          * we have 320 bits of information to hash, copy in the remaining
-         * 192 bits required for sha_transform, from the syncookie_secret
+         * 192 bits required for sha_transform, from the syncookie6_secret
          * and overwrite the digest with the secret
          */
-        memcpy(tmp + 10, syncookie_secret[c], 44);
+        memcpy(tmp + 10, syncookie6_secret[c], 44);
         memcpy(tmp, saddr, 16);
         memcpy(tmp + 4, daddr, 16);
         tmp[8] = ((__force u32)sport << 16) + (__force u32)dport;
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index b496de19a341..44fc4e3d661f 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -53,6 +53,29 @@
 #include <trace/events/skb.h>
 #include "udp_impl.h"
 
+static unsigned int udp6_ehashfn(struct net *net,
+                                  const struct in6_addr *laddr,
+                                  const u16 lport,
+                                  const struct in6_addr *faddr,
+                                  const __be16 fport)
+{
+        static u32 udp6_ehash_secret __read_mostly;
+        static u32 udp_ipv6_hash_secret __read_mostly;
+
+        u32 lhash, fhash;
+
+        net_get_random_once(&udp6_ehash_secret,
+                            sizeof(udp6_ehash_secret));
+        net_get_random_once(&udp_ipv6_hash_secret,
+                            sizeof(udp_ipv6_hash_secret));
+
+        lhash = (__force u32)laddr->s6_addr32[3];
+        fhash = __ipv6_addr_jhash(faddr, udp_ipv6_hash_secret);
+
+        return __inet6_ehashfn(lhash, lport, fhash, fport,
+                               udp_ipv6_hash_secret + net_hash_mix(net));
+}
+
 int ipv6_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2)
 {
         const struct in6_addr *sk2_rcv_saddr6 = inet6_rcv_saddr(sk2);
@@ -214,8 +237,8 @@ begin:
                         badness = score;
                         reuseport = sk->sk_reuseport;
                         if (reuseport) {
-                                hash = inet6_ehashfn(net, daddr, hnum,
-                                                     saddr, sport);
+                                hash = udp6_ehashfn(net, daddr, hnum,
+                                                    saddr, sport);
                                 matches = 1;
                         } else if (score == SCORE2_MAX)
                                 goto exact_match;
@@ -295,8 +318,8 @@ begin:
                         badness = score;
                         reuseport = sk->sk_reuseport;
                         if (reuseport) {
-                                hash = inet6_ehashfn(net, daddr, hnum,
-                                                     saddr, sport);
+                                hash = udp6_ehashfn(net, daddr, hnum,
+                                                    saddr, sport);
                                 matches = 1;
                         }
                 } else if (score == badness && reuseport) {
diff --git a/net/rds/connection.c b/net/rds/connection.c
index 642ad42c416b..378c3a6acf84 100644
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -51,10 +51,16 @@ static struct kmem_cache *rds_conn_slab;
 
 static struct hlist_head *rds_conn_bucket(__be32 laddr, __be32 faddr)
 {
+        static u32 rds_hash_secret __read_mostly;
+
+        unsigned long hash;
+
+        net_get_random_once(&rds_hash_secret, sizeof(rds_hash_secret));
+
         /* Pass NULL, don't need struct net for hash */
-        unsigned long hash = inet_ehashfn(NULL,
-                                          be32_to_cpu(laddr), 0,
-                                          be32_to_cpu(faddr), 0);
+        hash = __inet_ehashfn(be32_to_cpu(laddr), 0,
+                              be32_to_cpu(faddr), 0,
+                              rds_hash_secret);
         return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK];
 }