diff options
Diffstat (limited to 'net/ipv4/tcp_output.c')
-rw-r--r-- | net/ipv4/tcp_output.c | 154 |
1 files changed, 82 insertions, 72 deletions
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 882e0b0964d0..63170e297540 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c | |||
@@ -65,7 +65,7 @@ EXPORT_SYMBOL_GPL(sysctl_tcp_cookie_size); | |||
65 | 65 | ||
66 | 66 | ||
67 | /* Account for new data that has been sent to the network. */ | 67 | /* Account for new data that has been sent to the network. */ |
68 | static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb) | 68 | static void tcp_event_new_data_sent(struct sock *sk, const struct sk_buff *skb) |
69 | { | 69 | { |
70 | struct tcp_sock *tp = tcp_sk(sk); | 70 | struct tcp_sock *tp = tcp_sk(sk); |
71 | unsigned int prior_packets = tp->packets_out; | 71 | unsigned int prior_packets = tp->packets_out; |
@@ -89,9 +89,9 @@ static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb) | |||
89 | * Anything in between SND.UNA...SND.UNA+SND.WND also can be already | 89 | * Anything in between SND.UNA...SND.UNA+SND.WND also can be already |
90 | * invalid. OK, let's make this for now: | 90 | * invalid. OK, let's make this for now: |
91 | */ | 91 | */ |
92 | static inline __u32 tcp_acceptable_seq(struct sock *sk) | 92 | static inline __u32 tcp_acceptable_seq(const struct sock *sk) |
93 | { | 93 | { |
94 | struct tcp_sock *tp = tcp_sk(sk); | 94 | const struct tcp_sock *tp = tcp_sk(sk); |
95 | 95 | ||
96 | if (!before(tcp_wnd_end(tp), tp->snd_nxt)) | 96 | if (!before(tcp_wnd_end(tp), tp->snd_nxt)) |
97 | return tp->snd_nxt; | 97 | return tp->snd_nxt; |
@@ -116,7 +116,7 @@ static inline __u32 tcp_acceptable_seq(struct sock *sk) | |||
116 | static __u16 tcp_advertise_mss(struct sock *sk) | 116 | static __u16 tcp_advertise_mss(struct sock *sk) |
117 | { | 117 | { |
118 | struct tcp_sock *tp = tcp_sk(sk); | 118 | struct tcp_sock *tp = tcp_sk(sk); |
119 | struct dst_entry *dst = __sk_dst_get(sk); | 119 | const struct dst_entry *dst = __sk_dst_get(sk); |
120 | int mss = tp->advmss; | 120 | int mss = tp->advmss; |
121 | 121 | ||
122 | if (dst) { | 122 | if (dst) { |
@@ -133,7 +133,7 @@ static __u16 tcp_advertise_mss(struct sock *sk) | |||
133 | 133 | ||
134 | /* RFC2861. Reset CWND after idle period longer RTO to "restart window". | 134 | /* RFC2861. Reset CWND after idle period longer RTO to "restart window". |
135 | * This is the first part of cwnd validation mechanism. */ | 135 | * This is the first part of cwnd validation mechanism. */ |
136 | static void tcp_cwnd_restart(struct sock *sk, struct dst_entry *dst) | 136 | static void tcp_cwnd_restart(struct sock *sk, const struct dst_entry *dst) |
137 | { | 137 | { |
138 | struct tcp_sock *tp = tcp_sk(sk); | 138 | struct tcp_sock *tp = tcp_sk(sk); |
139 | s32 delta = tcp_time_stamp - tp->lsndtime; | 139 | s32 delta = tcp_time_stamp - tp->lsndtime; |
@@ -154,7 +154,7 @@ static void tcp_cwnd_restart(struct sock *sk, struct dst_entry *dst) | |||
154 | 154 | ||
155 | /* Congestion state accounting after a packet has been sent. */ | 155 | /* Congestion state accounting after a packet has been sent. */ |
156 | static void tcp_event_data_sent(struct tcp_sock *tp, | 156 | static void tcp_event_data_sent(struct tcp_sock *tp, |
157 | struct sk_buff *skb, struct sock *sk) | 157 | struct sock *sk) |
158 | { | 158 | { |
159 | struct inet_connection_sock *icsk = inet_csk(sk); | 159 | struct inet_connection_sock *icsk = inet_csk(sk); |
160 | const u32 now = tcp_time_stamp; | 160 | const u32 now = tcp_time_stamp; |
@@ -295,11 +295,11 @@ static u16 tcp_select_window(struct sock *sk) | |||
295 | } | 295 | } |
296 | 296 | ||
297 | /* Packet ECN state for a SYN-ACK */ | 297 | /* Packet ECN state for a SYN-ACK */ |
298 | static inline void TCP_ECN_send_synack(struct tcp_sock *tp, struct sk_buff *skb) | 298 | static inline void TCP_ECN_send_synack(const struct tcp_sock *tp, struct sk_buff *skb) |
299 | { | 299 | { |
300 | TCP_SKB_CB(skb)->flags &= ~TCPHDR_CWR; | 300 | TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR; |
301 | if (!(tp->ecn_flags & TCP_ECN_OK)) | 301 | if (!(tp->ecn_flags & TCP_ECN_OK)) |
302 | TCP_SKB_CB(skb)->flags &= ~TCPHDR_ECE; | 302 | TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; |
303 | } | 303 | } |
304 | 304 | ||
305 | /* Packet ECN state for a SYN. */ | 305 | /* Packet ECN state for a SYN. */ |
@@ -309,13 +309,13 @@ static inline void TCP_ECN_send_syn(struct sock *sk, struct sk_buff *skb) | |||
309 | 309 | ||
310 | tp->ecn_flags = 0; | 310 | tp->ecn_flags = 0; |
311 | if (sysctl_tcp_ecn == 1) { | 311 | if (sysctl_tcp_ecn == 1) { |
312 | TCP_SKB_CB(skb)->flags |= TCPHDR_ECE | TCPHDR_CWR; | 312 | TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; |
313 | tp->ecn_flags = TCP_ECN_OK; | 313 | tp->ecn_flags = TCP_ECN_OK; |
314 | } | 314 | } |
315 | } | 315 | } |
316 | 316 | ||
317 | static __inline__ void | 317 | static __inline__ void |
318 | TCP_ECN_make_synack(struct request_sock *req, struct tcphdr *th) | 318 | TCP_ECN_make_synack(const struct request_sock *req, struct tcphdr *th) |
319 | { | 319 | { |
320 | if (inet_rsk(req)->ecn_ok) | 320 | if (inet_rsk(req)->ecn_ok) |
321 | th->ece = 1; | 321 | th->ece = 1; |
@@ -356,7 +356,7 @@ static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags) | |||
356 | skb->ip_summed = CHECKSUM_PARTIAL; | 356 | skb->ip_summed = CHECKSUM_PARTIAL; |
357 | skb->csum = 0; | 357 | skb->csum = 0; |
358 | 358 | ||
359 | TCP_SKB_CB(skb)->flags = flags; | 359 | TCP_SKB_CB(skb)->tcp_flags = flags; |
360 | TCP_SKB_CB(skb)->sacked = 0; | 360 | TCP_SKB_CB(skb)->sacked = 0; |
361 | 361 | ||
362 | skb_shinfo(skb)->gso_segs = 1; | 362 | skb_shinfo(skb)->gso_segs = 1; |
@@ -565,7 +565,8 @@ static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp, | |||
565 | */ | 565 | */ |
566 | static unsigned tcp_syn_options(struct sock *sk, struct sk_buff *skb, | 566 | static unsigned tcp_syn_options(struct sock *sk, struct sk_buff *skb, |
567 | struct tcp_out_options *opts, | 567 | struct tcp_out_options *opts, |
568 | struct tcp_md5sig_key **md5) { | 568 | struct tcp_md5sig_key **md5) |
569 | { | ||
569 | struct tcp_sock *tp = tcp_sk(sk); | 570 | struct tcp_sock *tp = tcp_sk(sk); |
570 | struct tcp_cookie_values *cvp = tp->cookie_values; | 571 | struct tcp_cookie_values *cvp = tp->cookie_values; |
571 | unsigned remaining = MAX_TCP_OPTION_SPACE; | 572 | unsigned remaining = MAX_TCP_OPTION_SPACE; |
@@ -743,7 +744,8 @@ static unsigned tcp_synack_options(struct sock *sk, | |||
743 | */ | 744 | */ |
744 | static unsigned tcp_established_options(struct sock *sk, struct sk_buff *skb, | 745 | static unsigned tcp_established_options(struct sock *sk, struct sk_buff *skb, |
745 | struct tcp_out_options *opts, | 746 | struct tcp_out_options *opts, |
746 | struct tcp_md5sig_key **md5) { | 747 | struct tcp_md5sig_key **md5) |
748 | { | ||
747 | struct tcp_skb_cb *tcb = skb ? TCP_SKB_CB(skb) : NULL; | 749 | struct tcp_skb_cb *tcb = skb ? TCP_SKB_CB(skb) : NULL; |
748 | struct tcp_sock *tp = tcp_sk(sk); | 750 | struct tcp_sock *tp = tcp_sk(sk); |
749 | unsigned size = 0; | 751 | unsigned size = 0; |
@@ -826,7 +828,7 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, | |||
826 | tcb = TCP_SKB_CB(skb); | 828 | tcb = TCP_SKB_CB(skb); |
827 | memset(&opts, 0, sizeof(opts)); | 829 | memset(&opts, 0, sizeof(opts)); |
828 | 830 | ||
829 | if (unlikely(tcb->flags & TCPHDR_SYN)) | 831 | if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) |
830 | tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5); | 832 | tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5); |
831 | else | 833 | else |
832 | tcp_options_size = tcp_established_options(sk, skb, &opts, | 834 | tcp_options_size = tcp_established_options(sk, skb, &opts, |
@@ -850,9 +852,9 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, | |||
850 | th->seq = htonl(tcb->seq); | 852 | th->seq = htonl(tcb->seq); |
851 | th->ack_seq = htonl(tp->rcv_nxt); | 853 | th->ack_seq = htonl(tp->rcv_nxt); |
852 | *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) | | 854 | *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) | |
853 | tcb->flags); | 855 | tcb->tcp_flags); |
854 | 856 | ||
855 | if (unlikely(tcb->flags & TCPHDR_SYN)) { | 857 | if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) { |
856 | /* RFC1323: The window in SYN & SYN/ACK segments | 858 | /* RFC1323: The window in SYN & SYN/ACK segments |
857 | * is never scaled. | 859 | * is never scaled. |
858 | */ | 860 | */ |
@@ -875,7 +877,7 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, | |||
875 | } | 877 | } |
876 | 878 | ||
877 | tcp_options_write((__be32 *)(th + 1), tp, &opts); | 879 | tcp_options_write((__be32 *)(th + 1), tp, &opts); |
878 | if (likely((tcb->flags & TCPHDR_SYN) == 0)) | 880 | if (likely((tcb->tcp_flags & TCPHDR_SYN) == 0)) |
879 | TCP_ECN_send(sk, skb, tcp_header_size); | 881 | TCP_ECN_send(sk, skb, tcp_header_size); |
880 | 882 | ||
881 | #ifdef CONFIG_TCP_MD5SIG | 883 | #ifdef CONFIG_TCP_MD5SIG |
@@ -889,11 +891,11 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, | |||
889 | 891 | ||
890 | icsk->icsk_af_ops->send_check(sk, skb); | 892 | icsk->icsk_af_ops->send_check(sk, skb); |
891 | 893 | ||
892 | if (likely(tcb->flags & TCPHDR_ACK)) | 894 | if (likely(tcb->tcp_flags & TCPHDR_ACK)) |
893 | tcp_event_ack_sent(sk, tcp_skb_pcount(skb)); | 895 | tcp_event_ack_sent(sk, tcp_skb_pcount(skb)); |
894 | 896 | ||
895 | if (skb->len != tcp_header_size) | 897 | if (skb->len != tcp_header_size) |
896 | tcp_event_data_sent(tp, skb, sk); | 898 | tcp_event_data_sent(tp, sk); |
897 | 899 | ||
898 | if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) | 900 | if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) |
899 | TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS, | 901 | TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS, |
@@ -926,7 +928,7 @@ static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb) | |||
926 | } | 928 | } |
927 | 929 | ||
928 | /* Initialize TSO segments for a packet. */ | 930 | /* Initialize TSO segments for a packet. */ |
929 | static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, | 931 | static void tcp_set_skb_tso_segs(const struct sock *sk, struct sk_buff *skb, |
930 | unsigned int mss_now) | 932 | unsigned int mss_now) |
931 | { | 933 | { |
932 | if (skb->len <= mss_now || !sk_can_gso(sk) || | 934 | if (skb->len <= mss_now || !sk_can_gso(sk) || |
@@ -947,7 +949,7 @@ static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, | |||
947 | /* When a modification to fackets out becomes necessary, we need to check | 949 | /* When a modification to fackets out becomes necessary, we need to check |
948 | * skb is counted to fackets_out or not. | 950 | * skb is counted to fackets_out or not. |
949 | */ | 951 | */ |
950 | static void tcp_adjust_fackets_out(struct sock *sk, struct sk_buff *skb, | 952 | static void tcp_adjust_fackets_out(struct sock *sk, const struct sk_buff *skb, |
951 | int decr) | 953 | int decr) |
952 | { | 954 | { |
953 | struct tcp_sock *tp = tcp_sk(sk); | 955 | struct tcp_sock *tp = tcp_sk(sk); |
@@ -962,7 +964,7 @@ static void tcp_adjust_fackets_out(struct sock *sk, struct sk_buff *skb, | |||
962 | /* Pcount in the middle of the write queue got changed, we need to do various | 964 | /* Pcount in the middle of the write queue got changed, we need to do various |
963 | * tweaks to fix counters | 965 | * tweaks to fix counters |
964 | */ | 966 | */ |
965 | static void tcp_adjust_pcount(struct sock *sk, struct sk_buff *skb, int decr) | 967 | static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr) |
966 | { | 968 | { |
967 | struct tcp_sock *tp = tcp_sk(sk); | 969 | struct tcp_sock *tp = tcp_sk(sk); |
968 | 970 | ||
@@ -1032,9 +1034,9 @@ int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len, | |||
1032 | TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; | 1034 | TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; |
1033 | 1035 | ||
1034 | /* PSH and FIN should only be set in the second packet. */ | 1036 | /* PSH and FIN should only be set in the second packet. */ |
1035 | flags = TCP_SKB_CB(skb)->flags; | 1037 | flags = TCP_SKB_CB(skb)->tcp_flags; |
1036 | TCP_SKB_CB(skb)->flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); | 1038 | TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); |
1037 | TCP_SKB_CB(buff)->flags = flags; | 1039 | TCP_SKB_CB(buff)->tcp_flags = flags; |
1038 | TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked; | 1040 | TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked; |
1039 | 1041 | ||
1040 | if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) { | 1042 | if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) { |
@@ -1094,14 +1096,16 @@ static void __pskb_trim_head(struct sk_buff *skb, int len) | |||
1094 | eat = len; | 1096 | eat = len; |
1095 | k = 0; | 1097 | k = 0; |
1096 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { | 1098 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
1097 | if (skb_shinfo(skb)->frags[i].size <= eat) { | 1099 | int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); |
1098 | put_page(skb_shinfo(skb)->frags[i].page); | 1100 | |
1099 | eat -= skb_shinfo(skb)->frags[i].size; | 1101 | if (size <= eat) { |
1102 | skb_frag_unref(skb, i); | ||
1103 | eat -= size; | ||
1100 | } else { | 1104 | } else { |
1101 | skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; | 1105 | skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; |
1102 | if (eat) { | 1106 | if (eat) { |
1103 | skb_shinfo(skb)->frags[k].page_offset += eat; | 1107 | skb_shinfo(skb)->frags[k].page_offset += eat; |
1104 | skb_shinfo(skb)->frags[k].size -= eat; | 1108 | skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat); |
1105 | eat = 0; | 1109 | eat = 0; |
1106 | } | 1110 | } |
1107 | k++; | 1111 | k++; |
@@ -1144,10 +1148,10 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len) | |||
1144 | } | 1148 | } |
1145 | 1149 | ||
1146 | /* Calculate MSS. Not accounting for SACKs here. */ | 1150 | /* Calculate MSS. Not accounting for SACKs here. */ |
1147 | int tcp_mtu_to_mss(struct sock *sk, int pmtu) | 1151 | int tcp_mtu_to_mss(const struct sock *sk, int pmtu) |
1148 | { | 1152 | { |
1149 | struct tcp_sock *tp = tcp_sk(sk); | 1153 | const struct tcp_sock *tp = tcp_sk(sk); |
1150 | struct inet_connection_sock *icsk = inet_csk(sk); | 1154 | const struct inet_connection_sock *icsk = inet_csk(sk); |
1151 | int mss_now; | 1155 | int mss_now; |
1152 | 1156 | ||
1153 | /* Calculate base mss without TCP options: | 1157 | /* Calculate base mss without TCP options: |
@@ -1173,10 +1177,10 @@ int tcp_mtu_to_mss(struct sock *sk, int pmtu) | |||
1173 | } | 1177 | } |
1174 | 1178 | ||
1175 | /* Inverse of above */ | 1179 | /* Inverse of above */ |
1176 | int tcp_mss_to_mtu(struct sock *sk, int mss) | 1180 | int tcp_mss_to_mtu(const struct sock *sk, int mss) |
1177 | { | 1181 | { |
1178 | struct tcp_sock *tp = tcp_sk(sk); | 1182 | const struct tcp_sock *tp = tcp_sk(sk); |
1179 | struct inet_connection_sock *icsk = inet_csk(sk); | 1183 | const struct inet_connection_sock *icsk = inet_csk(sk); |
1180 | int mtu; | 1184 | int mtu; |
1181 | 1185 | ||
1182 | mtu = mss + | 1186 | mtu = mss + |
@@ -1250,8 +1254,8 @@ EXPORT_SYMBOL(tcp_sync_mss); | |||
1250 | */ | 1254 | */ |
1251 | unsigned int tcp_current_mss(struct sock *sk) | 1255 | unsigned int tcp_current_mss(struct sock *sk) |
1252 | { | 1256 | { |
1253 | struct tcp_sock *tp = tcp_sk(sk); | 1257 | const struct tcp_sock *tp = tcp_sk(sk); |
1254 | struct dst_entry *dst = __sk_dst_get(sk); | 1258 | const struct dst_entry *dst = __sk_dst_get(sk); |
1255 | u32 mss_now; | 1259 | u32 mss_now; |
1256 | unsigned header_len; | 1260 | unsigned header_len; |
1257 | struct tcp_out_options opts; | 1261 | struct tcp_out_options opts; |
@@ -1311,10 +1315,10 @@ static void tcp_cwnd_validate(struct sock *sk) | |||
1311 | * modulo only when the receiver window alone is the limiting factor or | 1315 | * modulo only when the receiver window alone is the limiting factor or |
1312 | * when we would be allowed to send the split-due-to-Nagle skb fully. | 1316 | * when we would be allowed to send the split-due-to-Nagle skb fully. |
1313 | */ | 1317 | */ |
1314 | static unsigned int tcp_mss_split_point(struct sock *sk, struct sk_buff *skb, | 1318 | static unsigned int tcp_mss_split_point(const struct sock *sk, const struct sk_buff *skb, |
1315 | unsigned int mss_now, unsigned int cwnd) | 1319 | unsigned int mss_now, unsigned int cwnd) |
1316 | { | 1320 | { |
1317 | struct tcp_sock *tp = tcp_sk(sk); | 1321 | const struct tcp_sock *tp = tcp_sk(sk); |
1318 | u32 needed, window, cwnd_len; | 1322 | u32 needed, window, cwnd_len; |
1319 | 1323 | ||
1320 | window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; | 1324 | window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; |
@@ -1334,13 +1338,14 @@ static unsigned int tcp_mss_split_point(struct sock *sk, struct sk_buff *skb, | |||
1334 | /* Can at least one segment of SKB be sent right now, according to the | 1338 | /* Can at least one segment of SKB be sent right now, according to the |
1335 | * congestion window rules? If so, return how many segments are allowed. | 1339 | * congestion window rules? If so, return how many segments are allowed. |
1336 | */ | 1340 | */ |
1337 | static inline unsigned int tcp_cwnd_test(struct tcp_sock *tp, | 1341 | static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp, |
1338 | struct sk_buff *skb) | 1342 | const struct sk_buff *skb) |
1339 | { | 1343 | { |
1340 | u32 in_flight, cwnd; | 1344 | u32 in_flight, cwnd; |
1341 | 1345 | ||
1342 | /* Don't be strict about the congestion window for the final FIN. */ | 1346 | /* Don't be strict about the congestion window for the final FIN. */ |
1343 | if ((TCP_SKB_CB(skb)->flags & TCPHDR_FIN) && tcp_skb_pcount(skb) == 1) | 1347 | if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) && |
1348 | tcp_skb_pcount(skb) == 1) | ||
1344 | return 1; | 1349 | return 1; |
1345 | 1350 | ||
1346 | in_flight = tcp_packets_in_flight(tp); | 1351 | in_flight = tcp_packets_in_flight(tp); |
@@ -1355,7 +1360,7 @@ static inline unsigned int tcp_cwnd_test(struct tcp_sock *tp, | |||
1355 | * This must be invoked the first time we consider transmitting | 1360 | * This must be invoked the first time we consider transmitting |
1356 | * SKB onto the wire. | 1361 | * SKB onto the wire. |
1357 | */ | 1362 | */ |
1358 | static int tcp_init_tso_segs(struct sock *sk, struct sk_buff *skb, | 1363 | static int tcp_init_tso_segs(const struct sock *sk, struct sk_buff *skb, |
1359 | unsigned int mss_now) | 1364 | unsigned int mss_now) |
1360 | { | 1365 | { |
1361 | int tso_segs = tcp_skb_pcount(skb); | 1366 | int tso_segs = tcp_skb_pcount(skb); |
@@ -1377,7 +1382,7 @@ static inline int tcp_minshall_check(const struct tcp_sock *tp) | |||
1377 | /* Return 0, if packet can be sent now without violation Nagle's rules: | 1382 | /* Return 0, if packet can be sent now without violation Nagle's rules: |
1378 | * 1. It is full sized. | 1383 | * 1. It is full sized. |
1379 | * 2. Or it contains FIN. (already checked by caller) | 1384 | * 2. Or it contains FIN. (already checked by caller) |
1380 | * 3. Or TCP_NODELAY was set. | 1385 | * 3. Or TCP_CORK is not set, and TCP_NODELAY is set. |
1381 | * 4. Or TCP_CORK is not set, and all sent packets are ACKed. | 1386 | * 4. Or TCP_CORK is not set, and all sent packets are ACKed. |
1382 | * With Minshall's modification: all sent small packets are ACKed. | 1387 | * With Minshall's modification: all sent small packets are ACKed. |
1383 | */ | 1388 | */ |
@@ -1393,7 +1398,7 @@ static inline int tcp_nagle_check(const struct tcp_sock *tp, | |||
1393 | /* Return non-zero if the Nagle test allows this packet to be | 1398 | /* Return non-zero if the Nagle test allows this packet to be |
1394 | * sent now. | 1399 | * sent now. |
1395 | */ | 1400 | */ |
1396 | static inline int tcp_nagle_test(struct tcp_sock *tp, struct sk_buff *skb, | 1401 | static inline int tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb, |
1397 | unsigned int cur_mss, int nonagle) | 1402 | unsigned int cur_mss, int nonagle) |
1398 | { | 1403 | { |
1399 | /* Nagle rule does not apply to frames, which sit in the middle of the | 1404 | /* Nagle rule does not apply to frames, which sit in the middle of the |
@@ -1409,7 +1414,7 @@ static inline int tcp_nagle_test(struct tcp_sock *tp, struct sk_buff *skb, | |||
1409 | * Nagle can be ignored during F-RTO too (see RFC4138). | 1414 | * Nagle can be ignored during F-RTO too (see RFC4138). |
1410 | */ | 1415 | */ |
1411 | if (tcp_urg_mode(tp) || (tp->frto_counter == 2) || | 1416 | if (tcp_urg_mode(tp) || (tp->frto_counter == 2) || |
1412 | (TCP_SKB_CB(skb)->flags & TCPHDR_FIN)) | 1417 | (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) |
1413 | return 1; | 1418 | return 1; |
1414 | 1419 | ||
1415 | if (!tcp_nagle_check(tp, skb, cur_mss, nonagle)) | 1420 | if (!tcp_nagle_check(tp, skb, cur_mss, nonagle)) |
@@ -1419,7 +1424,7 @@ static inline int tcp_nagle_test(struct tcp_sock *tp, struct sk_buff *skb, | |||
1419 | } | 1424 | } |
1420 | 1425 | ||
1421 | /* Does at least the first segment of SKB fit into the send window? */ | 1426 | /* Does at least the first segment of SKB fit into the send window? */ |
1422 | static inline int tcp_snd_wnd_test(struct tcp_sock *tp, struct sk_buff *skb, | 1427 | static inline int tcp_snd_wnd_test(const struct tcp_sock *tp, const struct sk_buff *skb, |
1423 | unsigned int cur_mss) | 1428 | unsigned int cur_mss) |
1424 | { | 1429 | { |
1425 | u32 end_seq = TCP_SKB_CB(skb)->end_seq; | 1430 | u32 end_seq = TCP_SKB_CB(skb)->end_seq; |
@@ -1434,10 +1439,10 @@ static inline int tcp_snd_wnd_test(struct tcp_sock *tp, struct sk_buff *skb, | |||
1434 | * should be put on the wire right now. If so, it returns the number of | 1439 | * should be put on the wire right now. If so, it returns the number of |
1435 | * packets allowed by the congestion window. | 1440 | * packets allowed by the congestion window. |
1436 | */ | 1441 | */ |
1437 | static unsigned int tcp_snd_test(struct sock *sk, struct sk_buff *skb, | 1442 | static unsigned int tcp_snd_test(const struct sock *sk, struct sk_buff *skb, |
1438 | unsigned int cur_mss, int nonagle) | 1443 | unsigned int cur_mss, int nonagle) |
1439 | { | 1444 | { |
1440 | struct tcp_sock *tp = tcp_sk(sk); | 1445 | const struct tcp_sock *tp = tcp_sk(sk); |
1441 | unsigned int cwnd_quota; | 1446 | unsigned int cwnd_quota; |
1442 | 1447 | ||
1443 | tcp_init_tso_segs(sk, skb, cur_mss); | 1448 | tcp_init_tso_segs(sk, skb, cur_mss); |
@@ -1455,7 +1460,7 @@ static unsigned int tcp_snd_test(struct sock *sk, struct sk_buff *skb, | |||
1455 | /* Test if sending is allowed right now. */ | 1460 | /* Test if sending is allowed right now. */ |
1456 | int tcp_may_send_now(struct sock *sk) | 1461 | int tcp_may_send_now(struct sock *sk) |
1457 | { | 1462 | { |
1458 | struct tcp_sock *tp = tcp_sk(sk); | 1463 | const struct tcp_sock *tp = tcp_sk(sk); |
1459 | struct sk_buff *skb = tcp_send_head(sk); | 1464 | struct sk_buff *skb = tcp_send_head(sk); |
1460 | 1465 | ||
1461 | return skb && | 1466 | return skb && |
@@ -1497,9 +1502,9 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len, | |||
1497 | TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; | 1502 | TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; |
1498 | 1503 | ||
1499 | /* PSH and FIN should only be set in the second packet. */ | 1504 | /* PSH and FIN should only be set in the second packet. */ |
1500 | flags = TCP_SKB_CB(skb)->flags; | 1505 | flags = TCP_SKB_CB(skb)->tcp_flags; |
1501 | TCP_SKB_CB(skb)->flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); | 1506 | TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); |
1502 | TCP_SKB_CB(buff)->flags = flags; | 1507 | TCP_SKB_CB(buff)->tcp_flags = flags; |
1503 | 1508 | ||
1504 | /* This packet was never sent out yet, so no SACK bits. */ | 1509 | /* This packet was never sent out yet, so no SACK bits. */ |
1505 | TCP_SKB_CB(buff)->sacked = 0; | 1510 | TCP_SKB_CB(buff)->sacked = 0; |
@@ -1530,7 +1535,7 @@ static int tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb) | |||
1530 | u32 send_win, cong_win, limit, in_flight; | 1535 | u32 send_win, cong_win, limit, in_flight; |
1531 | int win_divisor; | 1536 | int win_divisor; |
1532 | 1537 | ||
1533 | if (TCP_SKB_CB(skb)->flags & TCPHDR_FIN) | 1538 | if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) |
1534 | goto send_now; | 1539 | goto send_now; |
1535 | 1540 | ||
1536 | if (icsk->icsk_ca_state != TCP_CA_Open) | 1541 | if (icsk->icsk_ca_state != TCP_CA_Open) |
@@ -1657,7 +1662,7 @@ static int tcp_mtu_probe(struct sock *sk) | |||
1657 | 1662 | ||
1658 | TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; | 1663 | TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; |
1659 | TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; | 1664 | TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; |
1660 | TCP_SKB_CB(nskb)->flags = TCPHDR_ACK; | 1665 | TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK; |
1661 | TCP_SKB_CB(nskb)->sacked = 0; | 1666 | TCP_SKB_CB(nskb)->sacked = 0; |
1662 | nskb->csum = 0; | 1667 | nskb->csum = 0; |
1663 | nskb->ip_summed = skb->ip_summed; | 1668 | nskb->ip_summed = skb->ip_summed; |
@@ -1677,11 +1682,11 @@ static int tcp_mtu_probe(struct sock *sk) | |||
1677 | if (skb->len <= copy) { | 1682 | if (skb->len <= copy) { |
1678 | /* We've eaten all the data from this skb. | 1683 | /* We've eaten all the data from this skb. |
1679 | * Throw it away. */ | 1684 | * Throw it away. */ |
1680 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags; | 1685 | TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags; |
1681 | tcp_unlink_write_queue(skb, sk); | 1686 | tcp_unlink_write_queue(skb, sk); |
1682 | sk_wmem_free_skb(sk, skb); | 1687 | sk_wmem_free_skb(sk, skb); |
1683 | } else { | 1688 | } else { |
1684 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags & | 1689 | TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags & |
1685 | ~(TCPHDR_FIN|TCPHDR_PSH); | 1690 | ~(TCPHDR_FIN|TCPHDR_PSH); |
1686 | if (!skb_shinfo(skb)->nr_frags) { | 1691 | if (!skb_shinfo(skb)->nr_frags) { |
1687 | skb_pull(skb, copy); | 1692 | skb_pull(skb, copy); |
@@ -1796,11 +1801,13 @@ static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, | |||
1796 | tcp_event_new_data_sent(sk, skb); | 1801 | tcp_event_new_data_sent(sk, skb); |
1797 | 1802 | ||
1798 | tcp_minshall_update(tp, mss_now, skb); | 1803 | tcp_minshall_update(tp, mss_now, skb); |
1799 | sent_pkts++; | 1804 | sent_pkts += tcp_skb_pcount(skb); |
1800 | 1805 | ||
1801 | if (push_one) | 1806 | if (push_one) |
1802 | break; | 1807 | break; |
1803 | } | 1808 | } |
1809 | if (inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery) | ||
1810 | tp->prr_out += sent_pkts; | ||
1804 | 1811 | ||
1805 | if (likely(sent_pkts)) { | 1812 | if (likely(sent_pkts)) { |
1806 | tcp_cwnd_validate(sk); | 1813 | tcp_cwnd_validate(sk); |
@@ -1985,7 +1992,7 @@ static void tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb) | |||
1985 | TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq; | 1992 | TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq; |
1986 | 1993 | ||
1987 | /* Merge over control information. This moves PSH/FIN etc. over */ | 1994 | /* Merge over control information. This moves PSH/FIN etc. over */ |
1988 | TCP_SKB_CB(skb)->flags |= TCP_SKB_CB(next_skb)->flags; | 1995 | TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags; |
1989 | 1996 | ||
1990 | /* All done, get rid of second SKB and account for it so | 1997 | /* All done, get rid of second SKB and account for it so |
1991 | * packet counting does not break. | 1998 | * packet counting does not break. |
@@ -2003,7 +2010,7 @@ static void tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb) | |||
2003 | } | 2010 | } |
2004 | 2011 | ||
2005 | /* Check if coalescing SKBs is legal. */ | 2012 | /* Check if coalescing SKBs is legal. */ |
2006 | static int tcp_can_collapse(struct sock *sk, struct sk_buff *skb) | 2013 | static int tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb) |
2007 | { | 2014 | { |
2008 | if (tcp_skb_pcount(skb) > 1) | 2015 | if (tcp_skb_pcount(skb) > 1) |
2009 | return 0; | 2016 | return 0; |
@@ -2033,7 +2040,7 @@ static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to, | |||
2033 | 2040 | ||
2034 | if (!sysctl_tcp_retrans_collapse) | 2041 | if (!sysctl_tcp_retrans_collapse) |
2035 | return; | 2042 | return; |
2036 | if (TCP_SKB_CB(skb)->flags & TCPHDR_SYN) | 2043 | if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) |
2037 | return; | 2044 | return; |
2038 | 2045 | ||
2039 | tcp_for_write_queue_from_safe(skb, tmp, sk) { | 2046 | tcp_for_write_queue_from_safe(skb, tmp, sk) { |
@@ -2125,12 +2132,12 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb) | |||
2125 | * since it is cheap to do so and saves bytes on the network. | 2132 | * since it is cheap to do so and saves bytes on the network. |
2126 | */ | 2133 | */ |
2127 | if (skb->len > 0 && | 2134 | if (skb->len > 0 && |
2128 | (TCP_SKB_CB(skb)->flags & TCPHDR_FIN) && | 2135 | (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) && |
2129 | tp->snd_una == (TCP_SKB_CB(skb)->end_seq - 1)) { | 2136 | tp->snd_una == (TCP_SKB_CB(skb)->end_seq - 1)) { |
2130 | if (!pskb_trim(skb, 0)) { | 2137 | if (!pskb_trim(skb, 0)) { |
2131 | /* Reuse, even though it does some unnecessary work */ | 2138 | /* Reuse, even though it does some unnecessary work */ |
2132 | tcp_init_nondata_skb(skb, TCP_SKB_CB(skb)->end_seq - 1, | 2139 | tcp_init_nondata_skb(skb, TCP_SKB_CB(skb)->end_seq - 1, |
2133 | TCP_SKB_CB(skb)->flags); | 2140 | TCP_SKB_CB(skb)->tcp_flags); |
2134 | skb->ip_summed = CHECKSUM_NONE; | 2141 | skb->ip_summed = CHECKSUM_NONE; |
2135 | } | 2142 | } |
2136 | } | 2143 | } |
@@ -2179,7 +2186,7 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb) | |||
2179 | static int tcp_can_forward_retransmit(struct sock *sk) | 2186 | static int tcp_can_forward_retransmit(struct sock *sk) |
2180 | { | 2187 | { |
2181 | const struct inet_connection_sock *icsk = inet_csk(sk); | 2188 | const struct inet_connection_sock *icsk = inet_csk(sk); |
2182 | struct tcp_sock *tp = tcp_sk(sk); | 2189 | const struct tcp_sock *tp = tcp_sk(sk); |
2183 | 2190 | ||
2184 | /* Forward retransmissions are possible only during Recovery. */ | 2191 | /* Forward retransmissions are possible only during Recovery. */ |
2185 | if (icsk->icsk_ca_state != TCP_CA_Recovery) | 2192 | if (icsk->icsk_ca_state != TCP_CA_Recovery) |
@@ -2294,6 +2301,9 @@ begin_fwd: | |||
2294 | return; | 2301 | return; |
2295 | NET_INC_STATS_BH(sock_net(sk), mib_idx); | 2302 | NET_INC_STATS_BH(sock_net(sk), mib_idx); |
2296 | 2303 | ||
2304 | if (inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery) | ||
2305 | tp->prr_out += tcp_skb_pcount(skb); | ||
2306 | |||
2297 | if (skb == tcp_write_queue_head(sk)) | 2307 | if (skb == tcp_write_queue_head(sk)) |
2298 | inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, | 2308 | inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, |
2299 | inet_csk(sk)->icsk_rto, | 2309 | inet_csk(sk)->icsk_rto, |
@@ -2317,7 +2327,7 @@ void tcp_send_fin(struct sock *sk) | |||
2317 | mss_now = tcp_current_mss(sk); | 2327 | mss_now = tcp_current_mss(sk); |
2318 | 2328 | ||
2319 | if (tcp_send_head(sk) != NULL) { | 2329 | if (tcp_send_head(sk) != NULL) { |
2320 | TCP_SKB_CB(skb)->flags |= TCPHDR_FIN; | 2330 | TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_FIN; |
2321 | TCP_SKB_CB(skb)->end_seq++; | 2331 | TCP_SKB_CB(skb)->end_seq++; |
2322 | tp->write_seq++; | 2332 | tp->write_seq++; |
2323 | } else { | 2333 | } else { |
@@ -2379,11 +2389,11 @@ int tcp_send_synack(struct sock *sk) | |||
2379 | struct sk_buff *skb; | 2389 | struct sk_buff *skb; |
2380 | 2390 | ||
2381 | skb = tcp_write_queue_head(sk); | 2391 | skb = tcp_write_queue_head(sk); |
2382 | if (skb == NULL || !(TCP_SKB_CB(skb)->flags & TCPHDR_SYN)) { | 2392 | if (skb == NULL || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { |
2383 | printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n"); | 2393 | printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n"); |
2384 | return -EFAULT; | 2394 | return -EFAULT; |
2385 | } | 2395 | } |
2386 | if (!(TCP_SKB_CB(skb)->flags & TCPHDR_ACK)) { | 2396 | if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) { |
2387 | if (skb_cloned(skb)) { | 2397 | if (skb_cloned(skb)) { |
2388 | struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); | 2398 | struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); |
2389 | if (nskb == NULL) | 2399 | if (nskb == NULL) |
@@ -2397,7 +2407,7 @@ int tcp_send_synack(struct sock *sk) | |||
2397 | skb = nskb; | 2407 | skb = nskb; |
2398 | } | 2408 | } |
2399 | 2409 | ||
2400 | TCP_SKB_CB(skb)->flags |= TCPHDR_ACK; | 2410 | TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK; |
2401 | TCP_ECN_send_synack(tcp_sk(sk), skb); | 2411 | TCP_ECN_send_synack(tcp_sk(sk), skb); |
2402 | } | 2412 | } |
2403 | TCP_SKB_CB(skb)->when = tcp_time_stamp; | 2413 | TCP_SKB_CB(skb)->when = tcp_time_stamp; |
@@ -2542,7 +2552,7 @@ EXPORT_SYMBOL(tcp_make_synack); | |||
2542 | /* Do all connect socket setups that can be done AF independent. */ | 2552 | /* Do all connect socket setups that can be done AF independent. */ |
2543 | static void tcp_connect_init(struct sock *sk) | 2553 | static void tcp_connect_init(struct sock *sk) |
2544 | { | 2554 | { |
2545 | struct dst_entry *dst = __sk_dst_get(sk); | 2555 | const struct dst_entry *dst = __sk_dst_get(sk); |
2546 | struct tcp_sock *tp = tcp_sk(sk); | 2556 | struct tcp_sock *tp = tcp_sk(sk); |
2547 | __u8 rcv_wscale; | 2557 | __u8 rcv_wscale; |
2548 | 2558 | ||
@@ -2794,13 +2804,13 @@ int tcp_write_wakeup(struct sock *sk) | |||
2794 | if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq || | 2804 | if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq || |
2795 | skb->len > mss) { | 2805 | skb->len > mss) { |
2796 | seg_size = min(seg_size, mss); | 2806 | seg_size = min(seg_size, mss); |
2797 | TCP_SKB_CB(skb)->flags |= TCPHDR_PSH; | 2807 | TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; |
2798 | if (tcp_fragment(sk, skb, seg_size, mss)) | 2808 | if (tcp_fragment(sk, skb, seg_size, mss)) |
2799 | return -1; | 2809 | return -1; |
2800 | } else if (!tcp_skb_pcount(skb)) | 2810 | } else if (!tcp_skb_pcount(skb)) |
2801 | tcp_set_skb_tso_segs(sk, skb, mss); | 2811 | tcp_set_skb_tso_segs(sk, skb, mss); |
2802 | 2812 | ||
2803 | TCP_SKB_CB(skb)->flags |= TCPHDR_PSH; | 2813 | TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; |
2804 | TCP_SKB_CB(skb)->when = tcp_time_stamp; | 2814 | TCP_SKB_CB(skb)->when = tcp_time_stamp; |
2805 | err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); | 2815 | err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); |
2806 | if (!err) | 2816 | if (!err) |