tcp: tsq: move tsq_flags close to sk_wmem_alloc

tsq_flags being in the same cache line than sk_wmem_alloc
makes a lot of sense. Both fields are changed from tcp_wfree()
and more generally by various TSQ related functions.

Prior patch made room in struct sock and added sk_tsq_flags,
this patch deletes tsq_flags from struct tcp_sock.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 11 insertions, 13 deletions

diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 5f04bee4c86a..b45101f3d2bd 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -767,14 +767,15 @@ static void tcp_tasklet_func(unsigned long data)
         list_for_each_safe(q, n, &list) {
                 tp = list_entry(q, struct tcp_sock, tsq_node);
                 list_del(&tp->tsq_node);
-                clear_bit(TSQ_QUEUED, &tp->tsq_flags);
 
                 sk = (struct sock *)tp;
+                clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
+
                 if (!sk->sk_lock.owned &&
-                    test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) {
+                    test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) {
                         bh_lock_sock(sk);
                         if (!sock_owned_by_user(sk)) {
-                                clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
+                                clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
                                 tcp_tsq_handler(sk);
                         }
                         bh_unlock_sock(sk);
@@ -797,16 +798,15 @@ static void tcp_tasklet_func(unsigned long data)
  */
 void tcp_release_cb(struct sock *sk)
 {
-        struct tcp_sock *tp = tcp_sk(sk);
         unsigned long flags, nflags;
 
         /* perform an atomic operation only if at least one flag is set */
         do {
-                flags = tp->tsq_flags;
+                flags = sk->sk_tsq_flags;
                 if (!(flags & TCP_DEFERRED_ALL))
                         return;
                 nflags = flags & ~TCP_DEFERRED_ALL;
-        } while (cmpxchg(&tp->tsq_flags, flags, nflags) != flags);
+        } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
 
         if (flags & TCPF_TSQ_DEFERRED)
                 tcp_tsq_handler(sk);
@@ -878,7 +878,7 @@ void tcp_wfree(struct sk_buff *skb)
         if (wmem >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
                 goto out;
 
-        for (oval = READ_ONCE(tp->tsq_flags);; oval = nval) {
+        for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
                 struct tsq_tasklet *tsq;
                 bool empty;
 
@@ -886,7 +886,7 @@ void tcp_wfree(struct sk_buff *skb)
                         goto out;
 
                 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
-                nval = cmpxchg(&tp->tsq_flags, oval, nval);
+                nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
                 if (nval != oval)
                         continue;
 
@@ -2100,7 +2100,7 @@ static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
                     skb->prev == sk->sk_write_queue.next)
                         return false;
 
-                set_bit(TSQ_THROTTLED, &tcp_sk(sk)->tsq_flags);
+                set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
                 /* It is possible TX completion already happened
                  * before we set TSQ_THROTTLED, so we must
                  * test again the condition.
@@ -2241,8 +2241,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
                     unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
                         break;
 
-                if (test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags))
-                        clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
+                if (test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
+                        clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
                 if (tcp_small_queue_check(sk, skb, 0))
                         break;
 
@@ -3545,8 +3545,6 @@ void tcp_send_ack(struct sock *sk)
         /* We do not want pure acks influencing TCP Small Queues or fq/pacing
          * too much.
          * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
-         * We also avoid tcp_wfree() overhead (cache line miss accessing
-         * tp->tsq_flags) by using regular sock_wfree()
          */
         skb_set_tcp_pure_ack(buff);