inetpeer: get rid of ip_id_count

Ideally, we would need to generate IP ID using a per destination IP
generator.

linux kernels used inet_peer cache for this purpose, but this had a huge
cost on servers disabling MTU discovery.

1) each inet_peer struct consumes 192 bytes

2) inetpeer cache uses a binary tree of inet_peer structs,
   with a nominal size of ~66000 elements under load.

3) lookups in this tree are hitting a lot of cache lines, as tree depth
   is about 20.

4) If server deals with many tcp flows, we have a high probability of
   not finding the inet_peer, allocating a fresh one, inserting it in
   the tree with same initial ip_id_count, (cf secure_ip_id())

5) We garbage collect inet_peer aggressively.

IP ID generation do not have to be 'perfect'

Goal is trying to avoid duplicates in a short period of time,
so that reassembly units have a chance to complete reassembly of
fragments belonging to one message before receiving other fragments
with a recycled ID.

We simply use an array of generators, and a Jenkin hash using the dst IP
as a key.

ipv6_select_ident() is put back into net/ipv6/ip6_output.c where it
belongs (it is only used from this file)

secure_ip_id() and secure_ipv6_id() no longer are needed.

Rename ip_select_ident_more() to ip_select_ident_segs() to avoid
unnecessary decrement/increment of the number of segments.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

8 files changed, 25 insertions, 57 deletions

diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c
index 17d34e3c2ac3..6748d420f714 100644
--- a/net/ipv4/igmp.c
+++ b/net/ipv4/igmp.c
@@ -369,7 +369,7 @@ static struct sk_buff *igmpv3_newpack(struct net_device *dev, int size)
         pip->saddr    = fl4.saddr;
         pip->protocol = IPPROTO_IGMP;
         pip->tot_len  = 0;      /* filled in later */
-        ip_select_ident(skb, &rt->dst, NULL);
+        ip_select_ident(skb, NULL);
         ((u8 *)&pip[1])[0] = IPOPT_RA;
         ((u8 *)&pip[1])[1] = 4;
         ((u8 *)&pip[1])[2] = 0;
@@ -714,7 +714,7 @@ static int igmp_send_report(struct in_device *in_dev, struct ip_mc_list *pmc,
         iph->daddr    = dst;
         iph->saddr    = fl4.saddr;
         iph->protocol = IPPROTO_IGMP;
-        ip_select_ident(skb, &rt->dst, NULL);
+        ip_select_ident(skb, NULL);
         ((u8 *)&iph[1])[0] = IPOPT_RA;
         ((u8 *)&iph[1])[1] = 4;
         ((u8 *)&iph[1])[2] = 0;
diff --git a/net/ipv4/inetpeer.c b/net/ipv4/inetpeer.c
index c98cf141f4ed..4ced1b9a97f0 100644
--- a/net/ipv4/inetpeer.c
+++ b/net/ipv4/inetpeer.c
@@ -26,20 +26,7 @@
  *  Theory of operations.
  *  We keep one entry for each peer IP address.  The nodes contains long-living
  *  information about the peer which doesn't depend on routes.
- *  At this moment this information consists only of ID field for the next
- *  outgoing IP packet.  This field is incremented with each packet as encoded
- *  in inet_getid() function (include/net/inetpeer.h).
- *  At the moment of writing this notes identifier of IP packets is generated
- *  to be unpredictable using this code only for packets subjected
- *  (actually or potentially) to defragmentation.  I.e. DF packets less than
- *  PMTU in size when local fragmentation is disabled use a constant ID and do
- *  not use this code (see ip_select_ident() in include/net/ip.h).
  *
- *  Route cache entries hold references to our nodes.
- *  New cache entries get references via lookup by destination IP address in
- *  the avl tree.  The reference is grabbed only when it's needed i.e. only
- *  when we try to output IP packet which needs an unpredictable ID (see
- *  __ip_select_ident() in net/ipv4/route.c).
  *  Nodes are removed only when reference counter goes to 0.
  *  When it's happened the node may be removed when a sufficient amount of
  *  time has been passed since its last use.  The less-recently-used entry can
@@ -62,7 +49,6 @@
  *              refcnt: atomically against modifications on other CPU;
  *                 usually under some other lock to prevent node disappearing
  *              daddr: unchangeable
- *              ip_id_count: atomic value (no lock needed)
  */
 
 static struct kmem_cache *peer_cachep __read_mostly;
@@ -497,10 +483,6 @@ relookup:
                 p->daddr = *daddr;
                 atomic_set(&p->refcnt, 1);
                 atomic_set(&p->rid, 0);
-                atomic_set(&p->ip_id_count,
-                                (daddr->family == AF_INET) ?
-                                        secure_ip_id(daddr->addr.a4) :
-                                        secure_ipv6_id(daddr->addr.a6));
                 p->metrics[RTAX_LOCK-1] = INETPEER_METRICS_NEW;
                 p->rate_tokens = 0;
                 /* 60*HZ is arbitrary, but chosen enough high so that the first
diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index 6e231ab58d65..8d3b6b0e9857 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -148,7 +148,7 @@ int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk,
         iph->daddr    = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
         iph->saddr    = saddr;
         iph->protocol = sk->sk_protocol;
-        ip_select_ident(skb, &rt->dst, sk);
+        ip_select_ident(skb, sk);
 
         if (opt && opt->opt.optlen) {
                 iph->ihl += opt->opt.optlen>>2;
@@ -430,8 +430,7 @@ packet_routed:
                 ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0);
         }
 
-        ip_select_ident_more(skb, &rt->dst, sk,
-                             (skb_shinfo(skb)->gso_segs ?: 1) - 1);
+        ip_select_ident_segs(skb, sk, skb_shinfo(skb)->gso_segs ?: 1);
 
         /* TODO : should we use skb->sk here instead of sk ? */
         skb->priority = sk->sk_priority;
@@ -1379,7 +1378,7 @@ struct sk_buff *__ip_make_skb(struct sock *sk,
         iph->ttl = ttl;
         iph->protocol = sk->sk_protocol;
         ip_copy_addrs(iph, fl4);
-        ip_select_ident(skb, &rt->dst, sk);
+        ip_select_ident(skb, sk);
 
         if (opt) {
                 iph->ihl += opt->optlen>>2;
diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c
index bcf206c79005..847e69cbff7e 100644
--- a/net/ipv4/ip_tunnel_core.c
+++ b/net/ipv4/ip_tunnel_core.c
@@ -74,7 +74,7 @@ int iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
         iph->daddr      =       dst;
         iph->saddr      =       src;
         iph->ttl        =       ttl;
-        __ip_select_ident(iph, &rt->dst, (skb_shinfo(skb)->gso_segs ?: 1) - 1);
+        __ip_select_ident(iph, skb_shinfo(skb)->gso_segs ?: 1);
 
         err = ip_local_out_sk(sk, skb);
         if (unlikely(net_xmit_eval(err)))
diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
index 2bc9cc47f246..65bcaa789043 100644
--- a/net/ipv4/ipmr.c
+++ b/net/ipv4/ipmr.c
@@ -1663,7 +1663,7 @@ static void ip_encap(struct sk_buff *skb, __be32 saddr, __be32 daddr)
         iph->protocol   =       IPPROTO_IPIP;
         iph->ihl        =       5;
         iph->tot_len    =       htons(skb->len);
-        ip_select_ident(skb, skb_dst(skb), NULL);
+        ip_select_ident(skb, NULL);
         ip_send_check(iph);
 
         memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c
index a9dbe58bdfe7..2c65160565e1 100644
--- a/net/ipv4/raw.c
+++ b/net/ipv4/raw.c
@@ -389,7 +389,7 @@ static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4,
                 iph->check   = 0;
                 iph->tot_len = htons(length);
                 if (!iph->id)
-                        ip_select_ident(skb, &rt->dst, NULL);
+                        ip_select_ident(skb, NULL);
 
                 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
         }
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 4154eb76b0ad..082239ffe34a 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -89,6 +89,7 @@
 #include <linux/rcupdate.h>
 #include <linux/times.h>
 #include <linux/slab.h>
+#include <linux/jhash.h>
 #include <net/dst.h>
 #include <net/net_namespace.h>
 #include <net/protocol.h>
@@ -456,39 +457,19 @@ static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
         return neigh_create(&arp_tbl, pkey, dev);
 }
 
-/*
- * Peer allocation may fail only in serious out-of-memory conditions.  However
- * we still can generate some output.
- * Random ID selection looks a bit dangerous because we have no chances to
- * select ID being unique in a reasonable period of time.
- * But broken packet identifier may be better than no packet at all.
- */
-static void ip_select_fb_ident(struct iphdr *iph)
-{
-        static DEFINE_SPINLOCK(ip_fb_id_lock);
-        static u32 ip_fallback_id;
-        u32 salt;
+atomic_t *ip_idents __read_mostly;
+EXPORT_SYMBOL(ip_idents);
 
-        spin_lock_bh(&ip_fb_id_lock);
-        salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
-        iph->id = htons(salt & 0xFFFF);
-        ip_fallback_id = salt;
-        spin_unlock_bh(&ip_fb_id_lock);
-}
-
-void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
+void __ip_select_ident(struct iphdr *iph, int segs)
 {
-        struct net *net = dev_net(dst->dev);
-        struct inet_peer *peer;
+        static u32 ip_idents_hashrnd __read_mostly;
+        u32 hash, id;
 
-        peer = inet_getpeer_v4(net->ipv4.peers, iph->daddr, 1);
-        if (peer) {
-                iph->id = htons(inet_getid(peer, more));
-                inet_putpeer(peer);
-                return;
-        }
+        net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 
-        ip_select_fb_ident(iph);
+        hash = jhash_1word((__force u32)iph->daddr, ip_idents_hashrnd);
+        id = ip_idents_reserve(hash, segs);
+        iph->id = htons(id);
 }
 EXPORT_SYMBOL(__ip_select_ident);
 
@@ -2711,6 +2692,12 @@ int __init ip_rt_init(void)
 {
         int rc = 0;
 
+        ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
+        if (!ip_idents)
+                panic("IP: failed to allocate ip_idents\n");
+
+        prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
+
 #ifdef CONFIG_IP_ROUTE_CLASSID
         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
         if (!ip_rt_acct)
diff --git a/net/ipv4/xfrm4_mode_tunnel.c b/net/ipv4/xfrm4_mode_tunnel.c
index 05f2b484954f..91771a7c802f 100644
--- a/net/ipv4/xfrm4_mode_tunnel.c
+++ b/net/ipv4/xfrm4_mode_tunnel.c
@@ -58,12 +58,12 @@ static int xfrm4_mode_tunnel_output(struct xfrm_state *x, struct sk_buff *skb)
 
         top_iph->frag_off = (flags & XFRM_STATE_NOPMTUDISC) ?
                 0 : (XFRM_MODE_SKB_CB(skb)->frag_off & htons(IP_DF));
-        ip_select_ident(skb, dst->child, NULL);
 
         top_iph->ttl = ip4_dst_hoplimit(dst->child);
 
         top_iph->saddr = x->props.saddr.a4;
         top_iph->daddr = x->id.daddr.a4;
+        ip_select_ident(skb, NULL);
 
         return 0;
 }