1 files changed, 111 insertions, 57 deletions

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 0cd71b84e483..19a0612b8a20 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -339,7 +339,7 @@ unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
         struct sock *sk = sock->sk;
         struct tcp_sock *tp = tcp_sk(sk);
 
-        poll_wait(file, sk->sk_sleep, wait);
+        sock_poll_wait(file, sk->sk_sleep, wait);
         if (sk->sk_state == TCP_LISTEN)
                 return inet_csk_listen_poll(sk);
 
@@ -439,12 +439,14 @@ int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
                          !tp->urg_data ||
                          before(tp->urg_seq, tp->copied_seq) ||
                          !before(tp->urg_seq, tp->rcv_nxt)) {
+                        struct sk_buff *skb;
+
                         answ = tp->rcv_nxt - tp->copied_seq;
 
                         /* Subtract 1, if FIN is in queue. */
-                        if (answ && !skb_queue_empty(&sk->sk_receive_queue))
-                                answ -=
-                       tcp_hdr((struct sk_buff *)sk->sk_receive_queue.prev)->fin;
+                        skb = skb_peek_tail(&sk->sk_receive_queue);
+                        if (answ && skb)
+                                answ -= tcp_hdr(skb)->fin;
                 } else
                         answ = tp->urg_seq - tp->copied_seq;
                 release_sock(sk);
@@ -524,7 +526,8 @@ static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
         struct tcp_splice_state *tss = rd_desc->arg.data;
         int ret;
 
-        ret = skb_splice_bits(skb, offset, tss->pipe, rd_desc->count, tss->flags);
+        ret = skb_splice_bits(skb, offset, tss->pipe, min(rd_desc->count, len),
+                              tss->flags);
         if (ret > 0)
                 rd_desc->count -= ret;
         return ret;
@@ -660,6 +663,47 @@ struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp)
         return NULL;
 }
 
+static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
+                                       int large_allowed)
+{
+        struct tcp_sock *tp = tcp_sk(sk);
+        u32 xmit_size_goal, old_size_goal;
+
+        xmit_size_goal = mss_now;
+
+        if (large_allowed && sk_can_gso(sk)) {
+                xmit_size_goal = ((sk->sk_gso_max_size - 1) -
+                                  inet_csk(sk)->icsk_af_ops->net_header_len -
+                                  inet_csk(sk)->icsk_ext_hdr_len -
+                                  tp->tcp_header_len);
+
+                xmit_size_goal = tcp_bound_to_half_wnd(tp, xmit_size_goal);
+
+                /* We try hard to avoid divides here */
+                old_size_goal = tp->xmit_size_goal_segs * mss_now;
+
+                if (likely(old_size_goal <= xmit_size_goal &&
+                           old_size_goal + mss_now > xmit_size_goal)) {
+                        xmit_size_goal = old_size_goal;
+                } else {
+                        tp->xmit_size_goal_segs = xmit_size_goal / mss_now;
+                        xmit_size_goal = tp->xmit_size_goal_segs * mss_now;
+                }
+        }
+
+        return max(xmit_size_goal, mss_now);
+}
+
+static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
+{
+        int mss_now;
+
+        mss_now = tcp_current_mss(sk);
+        *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
+
+        return mss_now;
+}
+
 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
                          size_t psize, int flags)
 {
@@ -676,13 +720,12 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
 
         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 
-        mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
-        size_goal = tp->xmit_size_goal;
+        mss_now = tcp_send_mss(sk, &size_goal, flags);
         copied = 0;
 
         err = -EPIPE;
         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
-                goto do_error;
+                goto out_err;
 
         while (psize > 0) {
                 struct sk_buff *skb = tcp_write_queue_tail(sk);
@@ -760,8 +803,7 @@ wait_for_memory:
                 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
                         goto do_error;
 
-                mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
-                size_goal = tp->xmit_size_goal;
+                mss_now = tcp_send_mss(sk, &size_goal, flags);
         }
 
 out:
@@ -843,8 +885,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
         /* This should be in poll */
         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 
-        mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
-        size_goal = tp->xmit_size_goal;
+        mss_now = tcp_send_mss(sk, &size_goal, flags);
 
         /* Ok commence sending. */
         iovlen = msg->msg_iovlen;
@@ -853,7 +894,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
 
         err = -EPIPE;
         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
-                goto do_error;
+                goto out_err;
 
         while (--iovlen >= 0) {
                 int seglen = iov->iov_len;
@@ -862,13 +903,17 @@ int tcp_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
                 iov++;
 
                 while (seglen > 0) {
-                        int copy;
+                        int copy = 0;
+                        int max = size_goal;
 
                         skb = tcp_write_queue_tail(sk);
+                        if (tcp_send_head(sk)) {
+                                if (skb->ip_summed == CHECKSUM_NONE)
+                                        max = mss_now;
+                                copy = max - skb->len;
+                        }
 
-                        if (!tcp_send_head(sk) ||
-                            (copy = size_goal - skb->len) <= 0) {
-
+                        if (copy <= 0) {
 new_segment:
                                 /* Allocate new segment. If the interface is SG,
                                  * allocate skb fitting to single page.
@@ -889,6 +934,7 @@ new_segment:
 
                                 skb_entail(sk, skb);
                                 copy = size_goal;
+                                max = size_goal;
                         }
 
                         /* Try to append data to the end of skb. */
@@ -987,7 +1033,7 @@ new_segment:
                         if ((seglen -= copy) == 0 && iovlen == 0)
                                 goto out;
 
-                        if (skb->len < size_goal || (flags & MSG_OOB))
+                        if (skb->len < max || (flags & MSG_OOB))
                                 continue;
 
                         if (forced_push(tp)) {
@@ -1006,8 +1052,7 @@ wait_for_memory:
                         if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
                                 goto do_error;
 
-                        mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
-                        size_goal = tp->xmit_size_goal;
+                        mss_now = tcp_send_mss(sk, &size_goal, flags);
                 }
         }
 
@@ -1043,9 +1088,7 @@ out_err:
  *      this, no blocking and very strange errors 8)
  */
 
-static int tcp_recv_urg(struct sock *sk, long timeo,
-                        struct msghdr *msg, int len, int flags,
-                        int *addr_len)
+static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
 {
         struct tcp_sock *tp = tcp_sk(sk);
 
@@ -1285,6 +1328,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
         struct task_struct *user_recv = NULL;
         int copied_early = 0;
         struct sk_buff *skb;
+        u32 urg_hole = 0;
 
         lock_sock(sk);
 
@@ -1345,11 +1389,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 
                 /* Next get a buffer. */
 
-                skb = skb_peek(&sk->sk_receive_queue);
-                do {
-                        if (!skb)
-                                break;
-
+                skb_queue_walk(&sk->sk_receive_queue, skb) {
                         /* Now that we have two receive queues this
                          * shouldn't happen.
                          */
@@ -1366,8 +1406,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
                         if (tcp_hdr(skb)->fin)
                                 goto found_fin_ok;
                         WARN_ON(!(flags & MSG_PEEK));
-                        skb = skb->next;
-                } while (skb != (struct sk_buff *)&sk->sk_receive_queue);
+                }
 
                 /* Well, if we have backlog, try to process it now yet. */
 
@@ -1496,7 +1535,8 @@ do_prequeue:
                                 }
                         }
                 }
-                if ((flags & MSG_PEEK) && peek_seq != tp->copied_seq) {
+                if ((flags & MSG_PEEK) &&
+                    (peek_seq - copied - urg_hole != tp->copied_seq)) {
                         if (net_ratelimit())
                                 printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
                                        current->comm, task_pid_nr(current));
@@ -1517,6 +1557,7 @@ do_prequeue:
                                 if (!urg_offset) {
                                         if (!sock_flag(sk, SOCK_URGINLINE)) {
                                                 ++*seq;
+                                                urg_hole++;
                                                 offset++;
                                                 used--;
                                                 if (!used)
@@ -1660,7 +1701,7 @@ out:
         return err;
 
 recv_urg:
-        err = tcp_recv_urg(sk, timeo, msg, len, flags, addr_len);
+        err = tcp_recv_urg(sk, msg, len, flags);
         goto out;
 }
 
@@ -1798,7 +1839,7 @@ void tcp_close(struct sock *sk, long timeout)
                 /* Unread data was tossed, zap the connection. */
                 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
                 tcp_set_state(sk, TCP_CLOSE);
-                tcp_send_active_reset(sk, GFP_KERNEL);
+                tcp_send_active_reset(sk, sk->sk_allocation);
         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
                 /* Check zero linger _after_ checking for unread data. */
                 sk->sk_prot->disconnect(sk, 0);
@@ -1971,7 +2012,7 @@ int tcp_disconnect(struct sock *sk, int flags)
         tp->snd_cwnd = 2;
         icsk->icsk_probes_out = 0;
         tp->packets_out = 0;
-        tp->snd_ssthresh = 0x7fffffff;
+        tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
         tp->snd_cwnd_cnt = 0;
         tp->bytes_acked = 0;
         tcp_set_ca_state(sk, TCP_CA_Open);
@@ -2295,13 +2336,13 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
                 val = !!(tp->nonagle&TCP_NAGLE_CORK);
                 break;
         case TCP_KEEPIDLE:
-                val = (tp->keepalive_time ? : sysctl_tcp_keepalive_time) / HZ;
+                val = keepalive_time_when(tp) / HZ;
                 break;
         case TCP_KEEPINTVL:
-                val = (tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl) / HZ;
+                val = keepalive_intvl_when(tp) / HZ;
                 break;
         case TCP_KEEPCNT:
-                val = tp->keepalive_probes ? : sysctl_tcp_keepalive_probes;
+                val = keepalive_probes(tp);
                 break;
         case TCP_SYNCNT:
                 val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries;
@@ -2475,26 +2516,38 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
         struct sk_buff *p;
         struct tcphdr *th;
         struct tcphdr *th2;
+        unsigned int len;
         unsigned int thlen;
         unsigned int flags;
-        unsigned int total;
         unsigned int mss = 1;
+        unsigned int hlen;
+        unsigned int off;
         int flush = 1;
+        int i;
+
+        off = skb_gro_offset(skb);
+        hlen = off + sizeof(*th);
+        th = skb_gro_header_fast(skb, off);
+        if (skb_gro_header_hard(skb, hlen)) {
+                th = skb_gro_header_slow(skb, hlen, off);
+                if (unlikely(!th))
+                        goto out;
+        }
 
-        if (!pskb_may_pull(skb, sizeof(*th)))
-                goto out;
-
-        th = tcp_hdr(skb);
         thlen = th->doff * 4;
         if (thlen < sizeof(*th))
                 goto out;
 
-        if (!pskb_may_pull(skb, thlen))
-                goto out;
+        hlen = off + thlen;
+        if (skb_gro_header_hard(skb, hlen)) {
+                th = skb_gro_header_slow(skb, hlen, off);
+                if (unlikely(!th))
+                        goto out;
+        }
 
-        th = tcp_hdr(skb);
-        __skb_pull(skb, thlen);
+        skb_gro_pull(skb, thlen);
 
+        len = skb_gro_len(skb);
         flags = tcp_flag_word(th);
 
         for (; (p = *head); head = &p->next) {
@@ -2503,7 +2556,7 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
 
                 th2 = tcp_hdr(p);
 
-                if (th->source != th2->source || th->dest != th2->dest) {
+                if (*(u32 *)&th->source ^ *(u32 *)&th2->source) {
                         NAPI_GRO_CB(p)->same_flow = 0;
                         continue;
                 }
@@ -2518,14 +2571,15 @@ found:
         flush |= flags & TCP_FLAG_CWR;
         flush |= (flags ^ tcp_flag_word(th2)) &
                   ~(TCP_FLAG_CWR | TCP_FLAG_FIN | TCP_FLAG_PSH);
-        flush |= th->ack_seq != th2->ack_seq || th->window != th2->window;
-        flush |= memcmp(th + 1, th2 + 1, thlen - sizeof(*th));
+        flush |= th->ack_seq ^ th2->ack_seq;
+        for (i = sizeof(*th); i < thlen; i += 4)
+                flush |= *(u32 *)((u8 *)th + i) ^
+                         *(u32 *)((u8 *)th2 + i);
 
-        total = p->len;
         mss = skb_shinfo(p)->gso_size;
 
-        flush |= skb->len > mss || skb->len <= 0;
-        flush |= ntohl(th2->seq) + total != ntohl(th->seq);
+        flush |= (len - 1) >= mss;
+        flush |= (ntohl(th2->seq) + skb_gro_len(p)) ^ ntohl(th->seq);
 
         if (flush || skb_gro_receive(head, skb)) {
                 mss = 1;
@@ -2537,7 +2591,7 @@ found:
         tcp_flag_word(th2) |= flags & (TCP_FLAG_FIN | TCP_FLAG_PSH);
 
 out_check_final:
-        flush = skb->len < mss;
+        flush = len < mss;
         flush |= flags & (TCP_FLAG_URG | TCP_FLAG_PSH | TCP_FLAG_RST |
                           TCP_FLAG_SYN | TCP_FLAG_FIN);
 
@@ -2604,7 +2658,7 @@ void tcp_free_md5sig_pool(void)
 
 EXPORT_SYMBOL(tcp_free_md5sig_pool);
 
-static struct tcp_md5sig_pool **__tcp_alloc_md5sig_pool(void)
+static struct tcp_md5sig_pool **__tcp_alloc_md5sig_pool(struct sock *sk)
 {
         int cpu;
         struct tcp_md5sig_pool **pool;
@@ -2617,7 +2671,7 @@ static struct tcp_md5sig_pool **__tcp_alloc_md5sig_pool(void)
                 struct tcp_md5sig_pool *p;
                 struct crypto_hash *hash;
 
-                p = kzalloc(sizeof(*p), GFP_KERNEL);
+                p = kzalloc(sizeof(*p), sk->sk_allocation);
                 if (!p)
                         goto out_free;
                 *per_cpu_ptr(pool, cpu) = p;
@@ -2634,7 +2688,7 @@ out_free:
         return NULL;
 }
 
-struct tcp_md5sig_pool **tcp_alloc_md5sig_pool(void)
+struct tcp_md5sig_pool **tcp_alloc_md5sig_pool(struct sock *sk)
 {
         struct tcp_md5sig_pool **pool;
         int alloc = 0;
@@ -2655,7 +2709,7 @@ retry:
 
         if (alloc) {
                 /* we cannot hold spinlock here because this may sleep. */
-                struct tcp_md5sig_pool **p = __tcp_alloc_md5sig_pool();
+                struct tcp_md5sig_pool **p = __tcp_alloc_md5sig_pool(sk);
                 spin_lock_bh(&tcp_md5sig_pool_lock);
                 if (!p) {
                         tcp_md5sig_users--;