1 files changed, 80 insertions, 46 deletions

diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 2d340bd2cd3d..d92bce0ea24e 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -627,7 +627,7 @@ static unsigned int tcp_synack_options(struct sock *sk,
                 if (unlikely(!ireq->tstamp_ok))
                         remaining -= TCPOLEN_SACKPERM_ALIGNED;
         }
-        if (foc != NULL) {
+        if (foc != NULL && foc->len >= 0) {
                 u32 need = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
                 need = (need + 3) & ~3U;  /* Align to 32 bits */
                 if (remaining >= need) {
@@ -878,15 +878,8 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
         BUG_ON(!skb || !tcp_skb_pcount(skb));
 
         if (clone_it) {
-                const struct sk_buff *fclone = skb + 1;
-
                 skb_mstamp_get(&skb->skb_mstamp);
 
-                if (unlikely(skb->fclone == SKB_FCLONE_ORIG &&
-                             fclone->fclone == SKB_FCLONE_CLONE))
-                        NET_INC_STATS(sock_net(sk),
-                                      LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
-
                 if (unlikely(skb_cloned(skb)))
                         skb = pskb_copy(skb, gfp_mask);
                 else
@@ -1081,7 +1074,7 @@ static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int de
  * Remember, these are still headerless SKBs at this point.
  */
 int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
-                 unsigned int mss_now)
+                 unsigned int mss_now, gfp_t gfp)
 {
         struct tcp_sock *tp = tcp_sk(sk);
         struct sk_buff *buff;
@@ -1096,11 +1089,11 @@ int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
         if (nsize < 0)
                 nsize = 0;
 
-        if (skb_unclone(skb, GFP_ATOMIC))
+        if (skb_unclone(skb, gfp))
                 return -ENOMEM;
 
         /* Get a new skb... force flag on. */
-        buff = sk_stream_alloc_skb(sk, nsize, GFP_ATOMIC);
+        buff = sk_stream_alloc_skb(sk, nsize, gfp);
         if (buff == NULL)
                 return -ENOMEM; /* We'll just try again later. */
 
@@ -1387,12 +1380,43 @@ unsigned int tcp_current_mss(struct sock *sk)
         return mss_now;
 }
 
-/* Congestion window validation. (RFC2861) */
-static void tcp_cwnd_validate(struct sock *sk)
+/* RFC2861, slow part. Adjust cwnd, after it was not full during one rto.
+ * As additional protections, we do not touch cwnd in retransmission phases,
+ * and if application hit its sndbuf limit recently.
+ */
+static void tcp_cwnd_application_limited(struct sock *sk)
+{
+        struct tcp_sock *tp = tcp_sk(sk);
+
+        if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
+            sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
+                /* Limited by application or receiver window. */
+                u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
+                u32 win_used = max(tp->snd_cwnd_used, init_win);
+                if (win_used < tp->snd_cwnd) {
+                        tp->snd_ssthresh = tcp_current_ssthresh(sk);
+                        tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1;
+                }
+                tp->snd_cwnd_used = 0;
+        }
+        tp->snd_cwnd_stamp = tcp_time_stamp;
+}
+
+static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
 {
         struct tcp_sock *tp = tcp_sk(sk);
 
-        if (tp->packets_out >= tp->snd_cwnd) {
+        /* Track the maximum number of outstanding packets in each
+         * window, and remember whether we were cwnd-limited then.
+         */
+        if (!before(tp->snd_una, tp->max_packets_seq) ||
+            tp->packets_out > tp->max_packets_out) {
+                tp->max_packets_out = tp->packets_out;
+                tp->max_packets_seq = tp->snd_nxt;
+                tp->is_cwnd_limited = is_cwnd_limited;
+        }
+
+        if (tcp_is_cwnd_limited(sk)) {
                 /* Network is feed fully. */
                 tp->snd_cwnd_used = 0;
                 tp->snd_cwnd_stamp = tcp_time_stamp;
@@ -1601,7 +1625,7 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
 
         /* All of a TSO frame must be composed of paged data.  */
         if (skb->len != skb->data_len)
-                return tcp_fragment(sk, skb, len, mss_now);
+                return tcp_fragment(sk, skb, len, mss_now, gfp);
 
         buff = sk_stream_alloc_skb(sk, 0, gfp);
         if (unlikely(buff == NULL))
@@ -1644,7 +1668,8 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
  *
  * This algorithm is from John Heffner.
  */
-static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
+static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
+                                 bool *is_cwnd_limited)
 {
         struct tcp_sock *tp = tcp_sk(sk);
         const struct inet_connection_sock *icsk = inet_csk(sk);
@@ -1708,6 +1733,9 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
         if (!tp->tso_deferred)
                 tp->tso_deferred = 1 | (jiffies << 1);
 
+        if (cong_win < send_win && cong_win < skb->len)
+                *is_cwnd_limited = true;
+
         return true;
 
 send_now:
@@ -1868,6 +1896,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
         unsigned int tso_segs, sent_pkts;
         int cwnd_quota;
         int result;
+        bool is_cwnd_limited = false;
 
         sent_pkts = 0;
 
@@ -1892,6 +1921,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
 
                 cwnd_quota = tcp_cwnd_test(tp, skb);
                 if (!cwnd_quota) {
+                        is_cwnd_limited = true;
                         if (push_one == 2)
                                 /* Force out a loss probe pkt. */
                                 cwnd_quota = 1;
@@ -1908,7 +1938,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
                                                       nonagle : TCP_NAGLE_PUSH))))
                                 break;
                 } else {
-                        if (!push_one && tcp_tso_should_defer(sk, skb))
+                        if (!push_one &&
+                            tcp_tso_should_defer(sk, skb, &is_cwnd_limited))
                                 break;
                 }
 
@@ -1973,7 +2004,7 @@ repair:
                 /* Send one loss probe per tail loss episode. */
                 if (push_one != 2)
                         tcp_schedule_loss_probe(sk);
-                tcp_cwnd_validate(sk);
+                tcp_cwnd_validate(sk, is_cwnd_limited);
                 return false;
         }
         return (push_one == 2) || (!tp->packets_out && tcp_send_head(sk));
@@ -2037,6 +2068,25 @@ bool tcp_schedule_loss_probe(struct sock *sk)
         return true;
 }
 
+/* Thanks to skb fast clones, we can detect if a prior transmit of
+ * a packet is still in a qdisc or driver queue.
+ * In this case, there is very little point doing a retransmit !
+ * Note: This is called from BH context only.
+ */
+static bool skb_still_in_host_queue(const struct sock *sk,
+                                    const struct sk_buff *skb)
+{
+        const struct sk_buff *fclone = skb + 1;
+
+        if (unlikely(skb->fclone == SKB_FCLONE_ORIG &&
+                     fclone->fclone == SKB_FCLONE_CLONE)) {
+                NET_INC_STATS_BH(sock_net(sk),
+                                 LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
+                return true;
+        }
+        return false;
+}
+
 /* When probe timeout (PTO) fires, send a new segment if one exists, else
  * retransmit the last segment.
  */
@@ -2062,12 +2112,16 @@ void tcp_send_loss_probe(struct sock *sk)
         if (WARN_ON(!skb))
                 goto rearm_timer;
 
+        if (skb_still_in_host_queue(sk, skb))
+                goto rearm_timer;
+
         pcount = tcp_skb_pcount(skb);
         if (WARN_ON(!pcount))
                 goto rearm_timer;
 
         if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
-                if (unlikely(tcp_fragment(sk, skb, (pcount - 1) * mss, mss)))
+                if (unlikely(tcp_fragment(sk, skb, (pcount - 1) * mss, mss,
+                                          GFP_ATOMIC)))
                         goto rearm_timer;
                 skb = tcp_write_queue_tail(sk);
         }
@@ -2075,9 +2129,7 @@ void tcp_send_loss_probe(struct sock *sk)
         if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
                 goto rearm_timer;
 
-        /* Probe with zero data doesn't trigger fast recovery. */
-        if (skb->len > 0)
-                err = __tcp_retransmit_skb(sk, skb);
+        err = __tcp_retransmit_skb(sk, skb);
 
         /* Record snd_nxt for loss detection. */
         if (likely(!err))
@@ -2383,6 +2435,9 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
             min(sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2), sk->sk_sndbuf))
                 return -EAGAIN;
 
+        if (skb_still_in_host_queue(sk, skb))
+                return -EBUSY;
+
         if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
                 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
                         BUG();
@@ -2405,7 +2460,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
                 return -EAGAIN;
 
         if (skb->len > cur_mss) {
-                if (tcp_fragment(sk, skb, cur_mss, cur_mss))
+                if (tcp_fragment(sk, skb, cur_mss, cur_mss, GFP_ATOMIC))
                         return -ENOMEM; /* We'll try again later. */
         } else {
                 int oldpcount = tcp_skb_pcount(skb);
@@ -2476,7 +2531,7 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
                  * see tcp_input.c tcp_sacktag_write_queue().
                  */
                 TCP_SKB_CB(skb)->ack_seq = tp->snd_nxt;
-        } else {
+        } else if (err != -EBUSY) {
                 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL);
         }
         return err;
@@ -2754,27 +2809,6 @@ struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst,
         if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < mss)
                 mss = tp->rx_opt.user_mss;
 
-        if (req->rcv_wnd == 0) { /* ignored for retransmitted syns */
-                __u8 rcv_wscale;
-                /* Set this up on the first call only */
-                req->window_clamp = tp->window_clamp ? : dst_metric(dst, RTAX_WINDOW);
-
-                /* limit the window selection if the user enforce a smaller rx buffer */
-                if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
-                    (req->window_clamp > tcp_full_space(sk) || req->window_clamp == 0))
-                        req->window_clamp = tcp_full_space(sk);
-
-                /* tcp_full_space because it is guaranteed to be the first packet */
-                tcp_select_initial_window(tcp_full_space(sk),
-                        mss - (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0),
-                        &req->rcv_wnd,
-                        &req->window_clamp,
-                        ireq->wscale_ok,
-                        &rcv_wscale,
-                        dst_metric(dst, RTAX_INITRWND));
-                ireq->rcv_wscale = rcv_wscale;
-        }
-
         memset(&opts, 0, sizeof(opts));
 #ifdef CONFIG_SYN_COOKIES
         if (unlikely(req->cookie_ts))
@@ -3207,7 +3241,7 @@ int tcp_write_wakeup(struct sock *sk)
                     skb->len > mss) {
                         seg_size = min(seg_size, mss);
                         TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
-                        if (tcp_fragment(sk, skb, seg_size, mss))
+                        if (tcp_fragment(sk, skb, seg_size, mss, GFP_ATOMIC))
                                 return -1;
                 } else if (!tcp_skb_pcount(skb))
                         tcp_set_skb_tso_segs(sk, skb, mss);