inetpeer: Optimize inet_getid()

While investigating for network latencies, I found inet_getid() was a
contention point for some workloads, as inet_peer_idlock is shared
by all inet_getid() users regardless of peers.

One way to fix this is to make ip_id_count an atomic_t instead
of __u16, and use atomic_add_return().

In order to keep sizeof(struct inet_peer) = 64 on 64bit arches
tcp_ts_stamp is also converted to __u32 instead of "unsigned long".

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 5 insertions, 11 deletions

diff --git a/include/net/inetpeer.h b/include/net/inetpeer.h
index 35ad7b930467..87b1df0d4d8c 100644
--- a/include/net/inetpeer.h
+++ b/include/net/inetpeer.h
@@ -17,15 +17,15 @@ struct inet_peer {
         /* group together avl_left,avl_right,v4daddr to speedup lookups */
         struct inet_peer        *avl_left, *avl_right;
         __be32                  v4daddr;        /* peer's address */
-        __u16                   avl_height;
-        __u16                   ip_id_count;    /* IP ID for the next packet */
+        __u32                   avl_height;
         struct list_head        unused;
         __u32                   dtime;          /* the time of last use of not
                                                  * referenced entries */
         atomic_t                refcnt;
         atomic_t                rid;            /* Frag reception counter */
+        atomic_t                ip_id_count;    /* IP ID for the next packet */
         __u32                   tcp_ts;
-        unsigned long           tcp_ts_stamp;
+        __u32                   tcp_ts_stamp;
 };
 
 void                    inet_initpeers(void) __init;
@@ -36,17 +36,11 @@ struct inet_peer	*inet_getpeer(__be32 daddr, int create);
 /* can be called from BH context or outside */
 extern void inet_putpeer(struct inet_peer *p);
 
-extern spinlock_t inet_peer_idlock;
 /* can be called with or without local BH being disabled */
 static inline __u16     inet_getid(struct inet_peer *p, int more)
 {
-        __u16 id;
-
-        spin_lock_bh(&inet_peer_idlock);
-        id = p->ip_id_count;
-        p->ip_id_count += 1 + more;
-        spin_unlock_bh(&inet_peer_idlock);
-        return id;
+        more++;
+        return atomic_add_return(more, &p->ip_id_count) - more;
 }
 
 #endif /* _NET_INETPEER_H */