net: extend sk_pacing_rate to unsigned long

sk_pacing_rate has beed introduced as a u32 field in 2013,
effectively limiting per flow pacing to 34Gbit.

We believe it is time to allow TCP to pace high speed flows
on 64bit hosts, as we now can reach 100Gbit on one TCP flow.

This patch adds no cost for 32bit kernels.

The tcpi_pacing_rate and tcpi_max_pacing_rate were already
exported as 64bit, so iproute2/ss command require no changes.

Unfortunately the SO_MAX_PACING_RATE socket option will stay
32bit and we will need to add a new option to let applications
control high pacing rates.

State      Recv-Q Send-Q Local Address:Port             Peer Address:Port
ESTAB      0      1787144  10.246.9.76:49992             10.246.9.77:36741
                 timer:(on,003ms,0) ino:91863 sk:2 <->
 skmem:(r0,rb540000,t66440,tb2363904,f605944,w1822984,o0,bl0,d0)
 ts sack bbr wscale:8,8 rto:201 rtt:0.057/0.006 mss:1448
 rcvmss:536 advmss:1448
 cwnd:138 ssthresh:178 bytes_acked:256699822585 segs_out:177279177
 segs_in:3916318 data_segs_out:177279175
 bbr:(bw:31276.8Mbps,mrtt:0,pacing_gain:1.25,cwnd_gain:2)
 send 28045.5Mbps lastrcv:73333
 pacing_rate 38705.0Mbps delivery_rate 22997.6Mbps
 busy:73333ms unacked:135 retrans:0/157 rcv_space:14480
 notsent:2085120 minrtt:0.013

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

2 files changed, 7 insertions, 6 deletions

diff --git a/net/core/filter.c b/net/core/filter.c
index 4bbc6567fcb8..80da21b097b8 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -3927,8 +3927,8 @@ BPF_CALL_5(bpf_setsockopt, struct bpf_sock_ops_kern *, bpf_sock,
                         sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
                         sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF);
                         break;
-                case SO_MAX_PACING_RATE:
-                        sk->sk_max_pacing_rate = val;
+                case SO_MAX_PACING_RATE: /* 32bit version */
+                        sk->sk_max_pacing_rate = (val == ~0U) ? ~0UL : val;
                         sk->sk_pacing_rate = min(sk->sk_pacing_rate,
                                                  sk->sk_max_pacing_rate);
                         break;
diff --git a/net/core/sock.c b/net/core/sock.c
index 7e8796a6a089..fdf9fc7d3f98 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -998,7 +998,7 @@ set_rcvbuf:
                         cmpxchg(&sk->sk_pacing_status,
                                 SK_PACING_NONE,
                                 SK_PACING_NEEDED);
-                sk->sk_max_pacing_rate = val;
+                sk->sk_max_pacing_rate = (val == ~0U) ? ~0UL : val;
                 sk->sk_pacing_rate = min(sk->sk_pacing_rate,
                                          sk->sk_max_pacing_rate);
                 break;
@@ -1336,7 +1336,8 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 #endif
 
         case SO_MAX_PACING_RATE:
-                v.val = sk->sk_max_pacing_rate;
+                /* 32bit version */
+                v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U);
                 break;
 
         case SO_INCOMING_CPU:
@@ -2810,8 +2811,8 @@ void sock_init_data(struct socket *sock, struct sock *sk)
         sk->sk_ll_usec          =       sysctl_net_busy_read;
 #endif
 
-        sk->sk_max_pacing_rate = ~0U;
-        sk->sk_pacing_rate = ~0U;
+        sk->sk_max_pacing_rate = ~0UL;
+        sk->sk_pacing_rate = ~0UL;
         sk->sk_pacing_shift = 10;
         sk->sk_incoming_cpu = -1;