net: more accurate skb truesize

skb truesize currently accounts for sk_buff struct and part of skb head.
kmalloc() roundings are also ignored.

Considering that skb_shared_info is larger than sk_buff, its time to
take it into account for better memory accounting.

This patch introduces SKB_TRUESIZE(X) macro to centralize various
assumptions into a single place.

At skb alloc phase, we put skb_shared_info struct at the exact end of
skb head, to allow a better use of memory (lowering number of
reallocations), since kmalloc() gives us power-of-two memory blocks.

Unless SLUB/SLUB debug is active, both skb->head and skb_shared_info are
aligned to cache lines, as before.

Note: This patch might trigger performance regressions because of
misconfigured protocol stacks, hitting per socket or global memory
limits that were previously not reached. But its a necessary step for a
more accurate memory accounting.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
CC: Andi Kleen <ak@linux.intel.com>
CC: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

2 files changed, 9 insertions, 10 deletions

diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c
index 23ef31baa1af..ab188ae12fd9 100644
--- a/net/ipv4/icmp.c
+++ b/net/ipv4/icmp.c
@@ -1152,10 +1152,9 @@ static int __net_init icmp_sk_init(struct net *net)
                 net->ipv4.icmp_sk[i] = sk;
 
                 /* Enough space for 2 64K ICMP packets, including
-                 * sk_buff struct overhead.
+                 * sk_buff/skb_shared_info struct overhead.
                  */
-                sk->sk_sndbuf =
-                        (2 * ((64 * 1024) + sizeof(struct sk_buff)));
+                sk->sk_sndbuf = 2 * SKB_TRUESIZE(64 * 1024);
 
                 /*
                  * Speedup sock_wfree()
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 81cae641c9a9..c1653fe47255 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -265,8 +265,7 @@ static inline int TCP_ECN_rcv_ecn_echo(struct tcp_sock *tp, struct tcphdr *th)
 
 static void tcp_fixup_sndbuf(struct sock *sk)
 {
-        int sndmem = tcp_sk(sk)->rx_opt.mss_clamp + MAX_TCP_HEADER + 16 +
-                     sizeof(struct sk_buff);
+        int sndmem = SKB_TRUESIZE(tcp_sk(sk)->rx_opt.mss_clamp + MAX_TCP_HEADER);
 
         if (sk->sk_sndbuf < 3 * sndmem) {
                 sk->sk_sndbuf = 3 * sndmem;
@@ -349,7 +348,7 @@ static void tcp_grow_window(struct sock *sk, struct sk_buff *skb)
 static void tcp_fixup_rcvbuf(struct sock *sk)
 {
         struct tcp_sock *tp = tcp_sk(sk);
-        int rcvmem = tp->advmss + MAX_TCP_HEADER + 16 + sizeof(struct sk_buff);
+        int rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER);
 
         /* Try to select rcvbuf so that 4 mss-sized segments
          * will fit to window and corresponding skbs will fit to our rcvbuf.
@@ -540,8 +539,7 @@ void tcp_rcv_space_adjust(struct sock *sk)
                         space /= tp->advmss;
                         if (!space)
                                 space = 1;
-                        rcvmem = (tp->advmss + MAX_TCP_HEADER +
-                                  16 + sizeof(struct sk_buff));
+                        rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER);
                         while (tcp_win_from_space(rcvmem) < tp->advmss)
                                 rcvmem += 128;
                         space *= rcvmem;
@@ -4950,8 +4948,10 @@ static void tcp_new_space(struct sock *sk)
         struct tcp_sock *tp = tcp_sk(sk);
 
         if (tcp_should_expand_sndbuf(sk)) {
-                int sndmem = max_t(u32, tp->rx_opt.mss_clamp, tp->mss_cache) +
-                        MAX_TCP_HEADER + 16 + sizeof(struct sk_buff);
+                int sndmem = SKB_TRUESIZE(max_t(u32,
+                                                tp->rx_opt.mss_clamp,
+                                                tp->mss_cache) +
+                                          MAX_TCP_HEADER);
                 int demanded = max_t(unsigned int, tp->snd_cwnd,
                                      tp->reordering + 1);
                 sndmem *= 2 * demanded;