inetpeer: Optimize inet_getid()

While investigating for network latencies, I found inet_getid() was a
contention point for some workloads, as inet_peer_idlock is shared
by all inet_getid() users regardless of peers.

One way to fix this is to make ip_id_count an atomic_t instead
of __u16, and use atomic_add_return().

In order to keep sizeof(struct inet_peer) = 64 on 64bit arches
tcp_ts_stamp is also converted to __u32 instead of "unsigned long".

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

3 files changed, 10 insertions, 13 deletions

diff --git a/net/ipv4/inetpeer.c b/net/ipv4/inetpeer.c
index b1fbe18feb5a..6bcfe52a9c87 100644
--- a/net/ipv4/inetpeer.c
+++ b/net/ipv4/inetpeer.c
@@ -67,9 +67,6 @@
  *              ip_id_count: idlock
  */
 
-/* Exported for inet_getid inline function.  */
-DEFINE_SPINLOCK(inet_peer_idlock);
-
 static struct kmem_cache *peer_cachep __read_mostly;
 
 #define node_height(x) x->avl_height
@@ -390,7 +387,7 @@ struct inet_peer *inet_getpeer(__be32 daddr, int create)
         n->v4daddr = daddr;
         atomic_set(&n->refcnt, 1);
         atomic_set(&n->rid, 0);
-        n->ip_id_count = secure_ip_id(daddr);
+        atomic_set(&n->ip_id_count, secure_ip_id(daddr));
         n->tcp_ts_stamp = 0;
 
         write_lock_bh(&peer_pool_lock);
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index ff258b57680b..4284ceef7945 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -2852,7 +2852,7 @@ static int rt_fill_info(struct net *net,
         error = rt->u.dst.error;
         expires = rt->u.dst.expires ? rt->u.dst.expires - jiffies : 0;
         if (rt->peer) {
-                id = rt->peer->ip_id_count;
+                id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
                 if (rt->peer->tcp_ts_stamp) {
                         ts = rt->peer->tcp_ts;
                         tsage = get_seconds() - rt->peer->tcp_ts_stamp;
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index cf7f2086e6e0..df18ce04f41e 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -204,7 +204,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
                  * when trying new connection.
                  */
                 if (peer != NULL &&
-                    peer->tcp_ts_stamp + TCP_PAWS_MSL >= get_seconds()) {
+                    (u32)get_seconds() - peer->tcp_ts_stamp <= TCP_PAWS_MSL) {
                         tp->rx_opt.ts_recent_stamp = peer->tcp_ts_stamp;
                         tp->rx_opt.ts_recent = peer->tcp_ts;
                 }
@@ -1308,7 +1308,7 @@ int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
                     tcp_death_row.sysctl_tw_recycle &&
                     (peer = rt_get_peer((struct rtable *)dst)) != NULL &&
                     peer->v4daddr == saddr) {
-                        if (get_seconds() < peer->tcp_ts_stamp + TCP_PAWS_MSL &&
+                        if ((u32)get_seconds() - peer->tcp_ts_stamp < TCP_PAWS_MSL &&
                             (s32)(peer->tcp_ts - req->ts_recent) >
                                                         TCP_PAWS_WINDOW) {
                                 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSPASSIVEREJECTED);
@@ -1727,9 +1727,9 @@ int tcp_v4_remember_stamp(struct sock *sk)
 
         if (peer) {
                 if ((s32)(peer->tcp_ts - tp->rx_opt.ts_recent) <= 0 ||
-                    (peer->tcp_ts_stamp + TCP_PAWS_MSL < get_seconds() &&
-                     peer->tcp_ts_stamp <= tp->rx_opt.ts_recent_stamp)) {
-                        peer->tcp_ts_stamp = tp->rx_opt.ts_recent_stamp;
+                    ((u32)get_seconds() - peer->tcp_ts_stamp > TCP_PAWS_MSL &&
+                     peer->tcp_ts_stamp <= (u32)tp->rx_opt.ts_recent_stamp)) {
+                        peer->tcp_ts_stamp = (u32)tp->rx_opt.ts_recent_stamp;
                         peer->tcp_ts = tp->rx_opt.ts_recent;
                 }
                 if (release_it)
@@ -1748,9 +1748,9 @@ int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw)
                 const struct tcp_timewait_sock *tcptw = tcp_twsk((struct sock *)tw);
 
                 if ((s32)(peer->tcp_ts - tcptw->tw_ts_recent) <= 0 ||
-                    (peer->tcp_ts_stamp + TCP_PAWS_MSL < get_seconds() &&
-                     peer->tcp_ts_stamp <= tcptw->tw_ts_recent_stamp)) {
-                        peer->tcp_ts_stamp = tcptw->tw_ts_recent_stamp;
+                    ((u32)get_seconds() - peer->tcp_ts_stamp > TCP_PAWS_MSL &&
+                     peer->tcp_ts_stamp <= (u32)tcptw->tw_ts_recent_stamp)) {
+                        peer->tcp_ts_stamp = (u32)tcptw->tw_ts_recent_stamp;
                         peer->tcp_ts       = tcptw->tw_ts_recent;
                 }
                 inet_putpeer(peer);