1 files changed, 65 insertions, 20 deletions

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index c4638e6f0238..97c8f5620c43 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -285,6 +285,8 @@ int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT;
 
 int sysctl_tcp_min_tso_segs __read_mostly = 2;
 
+int sysctl_tcp_autocorking __read_mostly = 1;
+
 struct percpu_counter tcp_orphan_count;
 EXPORT_SYMBOL_GPL(tcp_orphan_count);
 
@@ -379,7 +381,7 @@ void tcp_init_sock(struct sock *sk)
         struct inet_connection_sock *icsk = inet_csk(sk);
         struct tcp_sock *tp = tcp_sk(sk);
 
-        skb_queue_head_init(&tp->out_of_order_queue);
+        __skb_queue_head_init(&tp->out_of_order_queue);
         tcp_init_xmit_timers(sk);
         tcp_prequeue_init(tp);
         INIT_LIST_HEAD(&tp->tsq_node);
@@ -619,19 +621,58 @@ static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
                 tp->snd_up = tp->write_seq;
 }
 
-static inline void tcp_push(struct sock *sk, int flags, int mss_now,
-                            int nonagle)
+/* If a not yet filled skb is pushed, do not send it if
+ * we have data packets in Qdisc or NIC queues :
+ * Because TX completion will happen shortly, it gives a chance
+ * to coalesce future sendmsg() payload into this skb, without
+ * need for a timer, and with no latency trade off.
+ * As packets containing data payload have a bigger truesize
+ * than pure acks (dataless) packets, the last checks prevent
+ * autocorking if we only have an ACK in Qdisc/NIC queues,
+ * or if TX completion was delayed after we processed ACK packet.
+ */
+static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
+                                int size_goal)
 {
-        if (tcp_send_head(sk)) {
-                struct tcp_sock *tp = tcp_sk(sk);
+        return skb->len < size_goal &&
+               sysctl_tcp_autocorking &&
+               skb != tcp_write_queue_head(sk) &&
+               atomic_read(&sk->sk_wmem_alloc) > skb->truesize;
+}
+
+static void tcp_push(struct sock *sk, int flags, int mss_now,
+                     int nonagle, int size_goal)
+{
+        struct tcp_sock *tp = tcp_sk(sk);
+        struct sk_buff *skb;
 
-                if (!(flags & MSG_MORE) || forced_push(tp))
-                        tcp_mark_push(tp, tcp_write_queue_tail(sk));
+        if (!tcp_send_head(sk))
+                return;
+
+        skb = tcp_write_queue_tail(sk);
+        if (!(flags & MSG_MORE) || forced_push(tp))
+                tcp_mark_push(tp, skb);
+
+        tcp_mark_urg(tp, flags);
+
+        if (tcp_should_autocork(sk, skb, size_goal)) {
 
-                tcp_mark_urg(tp, flags);
-                __tcp_push_pending_frames(sk, mss_now,
-                                          (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle);
+                /* avoid atomic op if TSQ_THROTTLED bit is already set */
+                if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {
+                        NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
+                        set_bit(TSQ_THROTTLED, &tp->tsq_flags);
+                }
+                /* It is possible TX completion already happened
+                 * before we set TSQ_THROTTLED.
+                 */
+                if (atomic_read(&sk->sk_wmem_alloc) > skb->truesize)
+                        return;
         }
+
+        if (flags & MSG_MORE)
+                nonagle = TCP_NAGLE_CORK;
+
+        __tcp_push_pending_frames(sk, mss_now, nonagle);
 }
 
 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
@@ -934,7 +975,8 @@ new_segment:
 wait_for_sndbuf:
                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 wait_for_memory:
-                tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
+                tcp_push(sk, flags & ~MSG_MORE, mss_now,
+                         TCP_NAGLE_PUSH, size_goal);
 
                 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
                         goto do_error;
@@ -944,7 +986,7 @@ wait_for_memory:
 
 out:
         if (copied && !(flags & MSG_SENDPAGE_NOTLAST))
-                tcp_push(sk, flags, mss_now, tp->nonagle);
+                tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
         return copied;
 
 do_error:
@@ -1002,7 +1044,8 @@ void tcp_free_fastopen_req(struct tcp_sock *tp)
         }
 }
 
-static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *size)
+static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
+                                int *copied, size_t size)
 {
         struct tcp_sock *tp = tcp_sk(sk);
         int err, flags;
@@ -1017,11 +1060,12 @@ static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *size)
         if (unlikely(tp->fastopen_req == NULL))
                 return -ENOBUFS;
         tp->fastopen_req->data = msg;
+        tp->fastopen_req->size = size;
 
         flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
         err = __inet_stream_connect(sk->sk_socket, msg->msg_name,
                                     msg->msg_namelen, flags);
-        *size = tp->fastopen_req->copied;
+        *copied = tp->fastopen_req->copied;
         tcp_free_fastopen_req(tp);
         return err;
 }
@@ -1041,7 +1085,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 
         flags = msg->msg_flags;
         if (flags & MSG_FASTOPEN) {
-                err = tcp_sendmsg_fastopen(sk, msg, &copied_syn);
+                err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size);
                 if (err == -EINPROGRESS && copied_syn > 0)
                         goto out;
                 else if (err)
@@ -1225,7 +1269,8 @@ wait_for_sndbuf:
                         set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 wait_for_memory:
                         if (copied)
-                                tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
+                                tcp_push(sk, flags & ~MSG_MORE, mss_now,
+                                         TCP_NAGLE_PUSH, size_goal);
 
                         if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
                                 goto do_error;
@@ -1236,7 +1281,7 @@ wait_for_memory:
 
 out:
         if (copied)
-                tcp_push(sk, flags, mss_now, tp->nonagle);
+                tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
         release_sock(sk);
         return copied + copied_syn;
 
@@ -1623,11 +1668,11 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
                     (len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) &&
                     !sysctl_tcp_low_latency &&
                     net_dma_find_channel()) {
-                        preempt_enable_no_resched();
+                        preempt_enable();
                         tp->ucopy.pinned_list =
                                         dma_pin_iovec_pages(msg->msg_iov, len);
                 } else {
-                        preempt_enable_no_resched();
+                        preempt_enable();
                 }
         }
 #endif
@@ -2186,7 +2231,7 @@ adjudge_to_death:
         /*      This is a (useful) BSD violating of the RFC. There is a
          *      problem with TCP as specified in that the other end could
          *      keep a socket open forever with no application left this end.
-         *      We use a 3 minute timeout (about the same as BSD) then kill
+         *      We use a 1 minute timeout (about the same as BSD) then kill
          *      our end. If they send after that then tough - BUT: long enough
          *      that we won't make the old 4*rto = almost no time - whoops
          *      reset mistake.