tcp: annotate sk->sk_wmem_queued lockless reads

For the sake of tcp_poll(), there are few places where we fetch
sk->sk_wmem_queued while this field can change from IRQ or other cpu.

We need to add READ_ONCE() annotations, and also make sure write
sides use corresponding WRITE_ONCE() to avoid store-tearing.

sk_wmem_queued_add() helper is added so that we can in
the future convert to ADD_ONCE() or equivalent if/when
available.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

6 files changed, 13 insertions, 13 deletions

diff --git a/net/core/datagram.c b/net/core/datagram.c
index 4cc8dc5db2b7..c210fc116103 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -640,7 +640,7 @@ int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,
                 skb->len += copied;
                 skb->truesize += truesize;
                 if (sk && sk->sk_type == SOCK_STREAM) {
-                        sk->sk_wmem_queued += truesize;
+                        sk_wmem_queued_add(sk, truesize);
                         sk_mem_charge(sk, truesize);
                 } else {
                         refcount_add(truesize, &skb->sk->sk_wmem_alloc);
diff --git a/net/core/sock.c b/net/core/sock.c
index cd075bc86407..a515392ba84b 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -3212,7 +3212,7 @@ void sk_get_meminfo(const struct sock *sk, u32 *mem)
         mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
         mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
         mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
-        mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued;
+        mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);
         mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
         mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);
         mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c
index bbb005eb5218..7dc79b973e6e 100644
--- a/net/ipv4/inet_diag.c
+++ b/net/ipv4/inet_diag.c
@@ -193,7 +193,7 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk,
         if (ext & (1 << (INET_DIAG_MEMINFO - 1))) {
                 struct inet_diag_meminfo minfo = {
                         .idiag_rmem = sk_rmem_alloc_get(sk),
-                        .idiag_wmem = sk->sk_wmem_queued,
+                        .idiag_wmem = READ_ONCE(sk->sk_wmem_queued),
                         .idiag_fmem = sk->sk_forward_alloc,
                         .idiag_tmem = sk_wmem_alloc_get(sk),
                 };
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 111853262972..b2ac4f074e2d 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -659,7 +659,7 @@ static void skb_entail(struct sock *sk, struct sk_buff *skb)
         tcb->sacked  = 0;
         __skb_header_release(skb);
         tcp_add_write_queue_tail(sk, skb);
-        sk->sk_wmem_queued += skb->truesize;
+        sk_wmem_queued_add(sk, skb->truesize);
         sk_mem_charge(sk, skb->truesize);
         if (tp->nonagle & TCP_NAGLE_PUSH)
                 tp->nonagle &= ~TCP_NAGLE_PUSH;
@@ -1034,7 +1034,7 @@ new_segment:
                 skb->len += copy;
                 skb->data_len += copy;
                 skb->truesize += copy;
-                sk->sk_wmem_queued += copy;
+                sk_wmem_queued_add(sk, copy);
                 sk_mem_charge(sk, copy);
                 skb->ip_summed = CHECKSUM_PARTIAL;
                 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index a115a991dfb5..0488607c5cd3 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -1199,7 +1199,7 @@ static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
         WRITE_ONCE(tp->write_seq, TCP_SKB_CB(skb)->end_seq);
         __skb_header_release(skb);
         tcp_add_write_queue_tail(sk, skb);
-        sk->sk_wmem_queued += skb->truesize;
+        sk_wmem_queued_add(sk, skb->truesize);
         sk_mem_charge(sk, skb->truesize);
 }
 
@@ -1333,7 +1333,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
                 return -ENOMEM; /* We'll just try again later. */
         skb_copy_decrypted(buff, skb);
 
-        sk->sk_wmem_queued += buff->truesize;
+        sk_wmem_queued_add(sk, buff->truesize);
         sk_mem_charge(sk, buff->truesize);
         nlen = skb->len - len - nsize;
         buff->truesize += nlen;
@@ -1443,7 +1443,7 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
 
         if (delta_truesize) {
                 skb->truesize      -= delta_truesize;
-                sk->sk_wmem_queued -= delta_truesize;
+                sk_wmem_queued_add(sk, -delta_truesize);
                 sk_mem_uncharge(sk, delta_truesize);
                 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
         }
@@ -1888,7 +1888,7 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
                 return -ENOMEM;
         skb_copy_decrypted(buff, skb);
 
-        sk->sk_wmem_queued += buff->truesize;
+        sk_wmem_queued_add(sk, buff->truesize);
         sk_mem_charge(sk, buff->truesize);
         buff->truesize += nlen;
         skb->truesize -= nlen;
@@ -2152,7 +2152,7 @@ static int tcp_mtu_probe(struct sock *sk)
         nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC, false);
         if (!nskb)
                 return -1;
-        sk->sk_wmem_queued += nskb->truesize;
+        sk_wmem_queued_add(sk, nskb->truesize);
         sk_mem_charge(sk, nskb->truesize);
 
         skb = tcp_send_head(sk);
@@ -3222,7 +3222,7 @@ int tcp_send_synack(struct sock *sk)
                         tcp_rtx_queue_unlink_and_free(skb, sk);
                         __skb_header_release(nskb);
                         tcp_rbtree_insert(&sk->tcp_rtx_queue, nskb);
-                        sk->sk_wmem_queued += nskb->truesize;
+                        sk_wmem_queued_add(sk, nskb->truesize);
                         sk_mem_charge(sk, nskb->truesize);
                         skb = nskb;
                 }
@@ -3447,7 +3447,7 @@ static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
 
         tcb->end_seq += skb->len;
         __skb_header_release(skb);
-        sk->sk_wmem_queued += skb->truesize;
+        sk_wmem_queued_add(sk, skb->truesize);
         sk_mem_charge(sk, skb->truesize);
         WRITE_ONCE(tp->write_seq, tcb->end_seq);
         tp->packets_out += tcp_skb_pcount(skb);
diff --git a/net/sched/em_meta.c b/net/sched/em_meta.c
index 4c9122fc35c9..3177dcb17316 100644
--- a/net/sched/em_meta.c
+++ b/net/sched/em_meta.c
@@ -446,7 +446,7 @@ META_COLLECTOR(int_sk_wmem_queued)
                 *err = -1;
                 return;
         }
-        dst->value = sk->sk_wmem_queued;
+        dst->value = READ_ONCE(sk->sk_wmem_queued);
 }
 
 META_COLLECTOR(int_sk_fwd_alloc)