diff options
author | David S. Miller <davem@sunset.davemloft.net> | 2007-03-07 15:12:44 -0500 |
---|---|---|
committer | David S. Miller <davem@sunset.davemloft.net> | 2007-04-26 01:24:02 -0400 |
commit | fe067e8ab5e0dc5ca3c54634924c628da92090b4 (patch) | |
tree | 98f5a6ebbb770f16682cfc52caea2da1e7eeb73b | |
parent | 02ea4923b4997d7e1310c027081f46d584b9d714 (diff) |
[TCP]: Abstract out all write queue operations.
This allows the write queue implementation to be changed,
for example, to one which allows fast interval searching.
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | include/net/sock.h | 21 | ||||
-rw-r--r-- | include/net/tcp.h | 114 | ||||
-rw-r--r-- | net/ipv4/tcp.c | 32 | ||||
-rw-r--r-- | net/ipv4/tcp_input.c | 64 | ||||
-rw-r--r-- | net/ipv4/tcp_ipv4.c | 2 | ||||
-rw-r--r-- | net/ipv4/tcp_output.c | 95 | ||||
-rw-r--r-- | net/ipv4/tcp_timer.c | 10 |
7 files changed, 221 insertions, 117 deletions
diff --git a/include/net/sock.h b/include/net/sock.h index 9583639090d2..2974bacc8850 100644 --- a/include/net/sock.h +++ b/include/net/sock.h | |||
@@ -710,15 +710,6 @@ static inline void sk_stream_mem_reclaim(struct sock *sk) | |||
710 | __sk_stream_mem_reclaim(sk); | 710 | __sk_stream_mem_reclaim(sk); |
711 | } | 711 | } |
712 | 712 | ||
713 | static inline void sk_stream_writequeue_purge(struct sock *sk) | ||
714 | { | ||
715 | struct sk_buff *skb; | ||
716 | |||
717 | while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) | ||
718 | sk_stream_free_skb(sk, skb); | ||
719 | sk_stream_mem_reclaim(sk); | ||
720 | } | ||
721 | |||
722 | static inline int sk_stream_rmem_schedule(struct sock *sk, struct sk_buff *skb) | 713 | static inline int sk_stream_rmem_schedule(struct sock *sk, struct sk_buff *skb) |
723 | { | 714 | { |
724 | return (int)skb->truesize <= sk->sk_forward_alloc || | 715 | return (int)skb->truesize <= sk->sk_forward_alloc || |
@@ -1256,18 +1247,6 @@ static inline struct page *sk_stream_alloc_page(struct sock *sk) | |||
1256 | return page; | 1247 | return page; |
1257 | } | 1248 | } |
1258 | 1249 | ||
1259 | #define sk_stream_for_retrans_queue(skb, sk) \ | ||
1260 | for (skb = (sk)->sk_write_queue.next; \ | ||
1261 | (skb != (sk)->sk_send_head) && \ | ||
1262 | (skb != (struct sk_buff *)&(sk)->sk_write_queue); \ | ||
1263 | skb = skb->next) | ||
1264 | |||
1265 | /*from STCP for fast SACK Process*/ | ||
1266 | #define sk_stream_for_retrans_queue_from(skb, sk) \ | ||
1267 | for (; (skb != (sk)->sk_send_head) && \ | ||
1268 | (skb != (struct sk_buff *)&(sk)->sk_write_queue); \ | ||
1269 | skb = skb->next) | ||
1270 | |||
1271 | /* | 1250 | /* |
1272 | * Default write policy as shown to user space via poll/select/SIGIO | 1251 | * Default write policy as shown to user space via poll/select/SIGIO |
1273 | */ | 1252 | */ |
diff --git a/include/net/tcp.h b/include/net/tcp.h index 181c0600af1c..6dacc352dcf1 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h | |||
@@ -1162,6 +1162,120 @@ static inline void tcp_put_md5sig_pool(void) | |||
1162 | put_cpu(); | 1162 | put_cpu(); |
1163 | } | 1163 | } |
1164 | 1164 | ||
1165 | /* write queue abstraction */ | ||
1166 | static inline void tcp_write_queue_purge(struct sock *sk) | ||
1167 | { | ||
1168 | struct sk_buff *skb; | ||
1169 | |||
1170 | while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) | ||
1171 | sk_stream_free_skb(sk, skb); | ||
1172 | sk_stream_mem_reclaim(sk); | ||
1173 | } | ||
1174 | |||
1175 | static inline struct sk_buff *tcp_write_queue_head(struct sock *sk) | ||
1176 | { | ||
1177 | struct sk_buff *skb = sk->sk_write_queue.next; | ||
1178 | if (skb == (struct sk_buff *) &sk->sk_write_queue) | ||
1179 | return NULL; | ||
1180 | return skb; | ||
1181 | } | ||
1182 | |||
1183 | static inline struct sk_buff *tcp_write_queue_tail(struct sock *sk) | ||
1184 | { | ||
1185 | struct sk_buff *skb = sk->sk_write_queue.prev; | ||
1186 | if (skb == (struct sk_buff *) &sk->sk_write_queue) | ||
1187 | return NULL; | ||
1188 | return skb; | ||
1189 | } | ||
1190 | |||
1191 | static inline struct sk_buff *tcp_write_queue_next(struct sock *sk, struct sk_buff *skb) | ||
1192 | { | ||
1193 | return skb->next; | ||
1194 | } | ||
1195 | |||
1196 | #define tcp_for_write_queue(skb, sk) \ | ||
1197 | for (skb = (sk)->sk_write_queue.next; \ | ||
1198 | (skb != (struct sk_buff *)&(sk)->sk_write_queue); \ | ||
1199 | skb = skb->next) | ||
1200 | |||
1201 | #define tcp_for_write_queue_from(skb, sk) \ | ||
1202 | for (; (skb != (struct sk_buff *)&(sk)->sk_write_queue);\ | ||
1203 | skb = skb->next) | ||
1204 | |||
1205 | static inline struct sk_buff *tcp_send_head(struct sock *sk) | ||
1206 | { | ||
1207 | return sk->sk_send_head; | ||
1208 | } | ||
1209 | |||
1210 | static inline void tcp_advance_send_head(struct sock *sk, struct sk_buff *skb) | ||
1211 | { | ||
1212 | sk->sk_send_head = skb->next; | ||
1213 | if (sk->sk_send_head == (struct sk_buff *)&sk->sk_write_queue) | ||
1214 | sk->sk_send_head = NULL; | ||
1215 | } | ||
1216 | |||
1217 | static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unlinked) | ||
1218 | { | ||
1219 | if (sk->sk_send_head == skb_unlinked) | ||
1220 | sk->sk_send_head = NULL; | ||
1221 | } | ||
1222 | |||
1223 | static inline void tcp_init_send_head(struct sock *sk) | ||
1224 | { | ||
1225 | sk->sk_send_head = NULL; | ||
1226 | } | ||
1227 | |||
1228 | static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) | ||
1229 | { | ||
1230 | __skb_queue_tail(&sk->sk_write_queue, skb); | ||
1231 | } | ||
1232 | |||
1233 | static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) | ||
1234 | { | ||
1235 | __tcp_add_write_queue_tail(sk, skb); | ||
1236 | |||
1237 | /* Queue it, remembering where we must start sending. */ | ||
1238 | if (sk->sk_send_head == NULL) | ||
1239 | sk->sk_send_head = skb; | ||
1240 | } | ||
1241 | |||
1242 | static inline void __tcp_add_write_queue_head(struct sock *sk, struct sk_buff *skb) | ||
1243 | { | ||
1244 | __skb_queue_head(&sk->sk_write_queue, skb); | ||
1245 | } | ||
1246 | |||
1247 | /* Insert buff after skb on the write queue of sk. */ | ||
1248 | static inline void tcp_insert_write_queue_after(struct sk_buff *skb, | ||
1249 | struct sk_buff *buff, | ||
1250 | struct sock *sk) | ||
1251 | { | ||
1252 | __skb_append(skb, buff, &sk->sk_write_queue); | ||
1253 | } | ||
1254 | |||
1255 | /* Insert skb between prev and next on the write queue of sk. */ | ||
1256 | static inline void tcp_insert_write_queue_before(struct sk_buff *new, | ||
1257 | struct sk_buff *skb, | ||
1258 | struct sock *sk) | ||
1259 | { | ||
1260 | __skb_insert(new, skb->prev, skb, &sk->sk_write_queue); | ||
1261 | } | ||
1262 | |||
1263 | static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) | ||
1264 | { | ||
1265 | __skb_unlink(skb, &sk->sk_write_queue); | ||
1266 | } | ||
1267 | |||
1268 | static inline int tcp_skb_is_last(const struct sock *sk, | ||
1269 | const struct sk_buff *skb) | ||
1270 | { | ||
1271 | return skb->next == (struct sk_buff *)&sk->sk_write_queue; | ||
1272 | } | ||
1273 | |||
1274 | static inline int tcp_write_queue_empty(struct sock *sk) | ||
1275 | { | ||
1276 | return skb_queue_empty(&sk->sk_write_queue); | ||
1277 | } | ||
1278 | |||
1165 | /* /proc */ | 1279 | /* /proc */ |
1166 | enum tcp_seq_states { | 1280 | enum tcp_seq_states { |
1167 | TCP_SEQ_STATE_LISTENING, | 1281 | TCP_SEQ_STATE_LISTENING, |
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 3834b10b5115..689f9330f1b9 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c | |||
@@ -470,10 +470,8 @@ static inline void skb_entail(struct sock *sk, struct tcp_sock *tp, | |||
470 | tcb->flags = TCPCB_FLAG_ACK; | 470 | tcb->flags = TCPCB_FLAG_ACK; |
471 | tcb->sacked = 0; | 471 | tcb->sacked = 0; |
472 | skb_header_release(skb); | 472 | skb_header_release(skb); |
473 | __skb_queue_tail(&sk->sk_write_queue, skb); | 473 | tcp_add_write_queue_tail(sk, skb); |
474 | sk_charge_skb(sk, skb); | 474 | sk_charge_skb(sk, skb); |
475 | if (!sk->sk_send_head) | ||
476 | sk->sk_send_head = skb; | ||
477 | if (tp->nonagle & TCP_NAGLE_PUSH) | 475 | if (tp->nonagle & TCP_NAGLE_PUSH) |
478 | tp->nonagle &= ~TCP_NAGLE_PUSH; | 476 | tp->nonagle &= ~TCP_NAGLE_PUSH; |
479 | } | 477 | } |
@@ -491,8 +489,8 @@ static inline void tcp_mark_urg(struct tcp_sock *tp, int flags, | |||
491 | static inline void tcp_push(struct sock *sk, struct tcp_sock *tp, int flags, | 489 | static inline void tcp_push(struct sock *sk, struct tcp_sock *tp, int flags, |
492 | int mss_now, int nonagle) | 490 | int mss_now, int nonagle) |
493 | { | 491 | { |
494 | if (sk->sk_send_head) { | 492 | if (tcp_send_head(sk)) { |
495 | struct sk_buff *skb = sk->sk_write_queue.prev; | 493 | struct sk_buff *skb = tcp_write_queue_tail(sk); |
496 | if (!(flags & MSG_MORE) || forced_push(tp)) | 494 | if (!(flags & MSG_MORE) || forced_push(tp)) |
497 | tcp_mark_push(tp, skb); | 495 | tcp_mark_push(tp, skb); |
498 | tcp_mark_urg(tp, flags, skb); | 496 | tcp_mark_urg(tp, flags, skb); |
@@ -526,13 +524,13 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse | |||
526 | goto do_error; | 524 | goto do_error; |
527 | 525 | ||
528 | while (psize > 0) { | 526 | while (psize > 0) { |
529 | struct sk_buff *skb = sk->sk_write_queue.prev; | 527 | struct sk_buff *skb = tcp_write_queue_tail(sk); |
530 | struct page *page = pages[poffset / PAGE_SIZE]; | 528 | struct page *page = pages[poffset / PAGE_SIZE]; |
531 | int copy, i, can_coalesce; | 529 | int copy, i, can_coalesce; |
532 | int offset = poffset % PAGE_SIZE; | 530 | int offset = poffset % PAGE_SIZE; |
533 | int size = min_t(size_t, psize, PAGE_SIZE - offset); | 531 | int size = min_t(size_t, psize, PAGE_SIZE - offset); |
534 | 532 | ||
535 | if (!sk->sk_send_head || (copy = size_goal - skb->len) <= 0) { | 533 | if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) { |
536 | new_segment: | 534 | new_segment: |
537 | if (!sk_stream_memory_free(sk)) | 535 | if (!sk_stream_memory_free(sk)) |
538 | goto wait_for_sndbuf; | 536 | goto wait_for_sndbuf; |
@@ -589,7 +587,7 @@ new_segment: | |||
589 | if (forced_push(tp)) { | 587 | if (forced_push(tp)) { |
590 | tcp_mark_push(tp, skb); | 588 | tcp_mark_push(tp, skb); |
591 | __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); | 589 | __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); |
592 | } else if (skb == sk->sk_send_head) | 590 | } else if (skb == tcp_send_head(sk)) |
593 | tcp_push_one(sk, mss_now); | 591 | tcp_push_one(sk, mss_now); |
594 | continue; | 592 | continue; |
595 | 593 | ||
@@ -704,9 +702,9 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, | |||
704 | while (seglen > 0) { | 702 | while (seglen > 0) { |
705 | int copy; | 703 | int copy; |
706 | 704 | ||
707 | skb = sk->sk_write_queue.prev; | 705 | skb = tcp_write_queue_tail(sk); |
708 | 706 | ||
709 | if (!sk->sk_send_head || | 707 | if (!tcp_send_head(sk) || |
710 | (copy = size_goal - skb->len) <= 0) { | 708 | (copy = size_goal - skb->len) <= 0) { |
711 | 709 | ||
712 | new_segment: | 710 | new_segment: |
@@ -833,7 +831,7 @@ new_segment: | |||
833 | if (forced_push(tp)) { | 831 | if (forced_push(tp)) { |
834 | tcp_mark_push(tp, skb); | 832 | tcp_mark_push(tp, skb); |
835 | __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); | 833 | __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); |
836 | } else if (skb == sk->sk_send_head) | 834 | } else if (skb == tcp_send_head(sk)) |
837 | tcp_push_one(sk, mss_now); | 835 | tcp_push_one(sk, mss_now); |
838 | continue; | 836 | continue; |
839 | 837 | ||
@@ -860,9 +858,11 @@ out: | |||
860 | 858 | ||
861 | do_fault: | 859 | do_fault: |
862 | if (!skb->len) { | 860 | if (!skb->len) { |
863 | if (sk->sk_send_head == skb) | 861 | tcp_unlink_write_queue(skb, sk); |
864 | sk->sk_send_head = NULL; | 862 | /* It is the one place in all of TCP, except connection |
865 | __skb_unlink(skb, &sk->sk_write_queue); | 863 | * reset, where we can be unlinking the send_head. |
864 | */ | ||
865 | tcp_check_send_head(sk, skb); | ||
866 | sk_stream_free_skb(sk, skb); | 866 | sk_stream_free_skb(sk, skb); |
867 | } | 867 | } |
868 | 868 | ||
@@ -1732,7 +1732,7 @@ int tcp_disconnect(struct sock *sk, int flags) | |||
1732 | 1732 | ||
1733 | tcp_clear_xmit_timers(sk); | 1733 | tcp_clear_xmit_timers(sk); |
1734 | __skb_queue_purge(&sk->sk_receive_queue); | 1734 | __skb_queue_purge(&sk->sk_receive_queue); |
1735 | sk_stream_writequeue_purge(sk); | 1735 | tcp_write_queue_purge(sk); |
1736 | __skb_queue_purge(&tp->out_of_order_queue); | 1736 | __skb_queue_purge(&tp->out_of_order_queue); |
1737 | #ifdef CONFIG_NET_DMA | 1737 | #ifdef CONFIG_NET_DMA |
1738 | __skb_queue_purge(&sk->sk_async_wait_queue); | 1738 | __skb_queue_purge(&sk->sk_async_wait_queue); |
@@ -1758,7 +1758,7 @@ int tcp_disconnect(struct sock *sk, int flags) | |||
1758 | tcp_set_ca_state(sk, TCP_CA_Open); | 1758 | tcp_set_ca_state(sk, TCP_CA_Open); |
1759 | tcp_clear_retrans(tp); | 1759 | tcp_clear_retrans(tp); |
1760 | inet_csk_delack_init(sk); | 1760 | inet_csk_delack_init(sk); |
1761 | sk->sk_send_head = NULL; | 1761 | tcp_init_send_head(sk); |
1762 | tp->rx_opt.saw_tstamp = 0; | 1762 | tp->rx_opt.saw_tstamp = 0; |
1763 | tcp_sack_reset(&tp->rx_opt); | 1763 | tcp_sack_reset(&tp->rx_opt); |
1764 | __sk_dst_reset(sk); | 1764 | __sk_dst_reset(sk); |
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index d0a3630f41a7..22d0bb03c5da 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c | |||
@@ -1044,7 +1044,7 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_ | |||
1044 | cached_skb = tp->fastpath_skb_hint; | 1044 | cached_skb = tp->fastpath_skb_hint; |
1045 | cached_fack_count = tp->fastpath_cnt_hint; | 1045 | cached_fack_count = tp->fastpath_cnt_hint; |
1046 | if (!cached_skb) { | 1046 | if (!cached_skb) { |
1047 | cached_skb = sk->sk_write_queue.next; | 1047 | cached_skb = tcp_write_queue_head(sk); |
1048 | cached_fack_count = 0; | 1048 | cached_fack_count = 0; |
1049 | } | 1049 | } |
1050 | 1050 | ||
@@ -1061,10 +1061,13 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_ | |||
1061 | if (after(end_seq, tp->high_seq)) | 1061 | if (after(end_seq, tp->high_seq)) |
1062 | flag |= FLAG_DATA_LOST; | 1062 | flag |= FLAG_DATA_LOST; |
1063 | 1063 | ||
1064 | sk_stream_for_retrans_queue_from(skb, sk) { | 1064 | tcp_for_write_queue_from(skb, sk) { |
1065 | int in_sack, pcount; | 1065 | int in_sack, pcount; |
1066 | u8 sacked; | 1066 | u8 sacked; |
1067 | 1067 | ||
1068 | if (skb == tcp_send_head(sk)) | ||
1069 | break; | ||
1070 | |||
1068 | cached_skb = skb; | 1071 | cached_skb = skb; |
1069 | cached_fack_count = fack_count; | 1072 | cached_fack_count = fack_count; |
1070 | if (i == first_sack_index) { | 1073 | if (i == first_sack_index) { |
@@ -1213,7 +1216,9 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_ | |||
1213 | if (lost_retrans && icsk->icsk_ca_state == TCP_CA_Recovery) { | 1216 | if (lost_retrans && icsk->icsk_ca_state == TCP_CA_Recovery) { |
1214 | struct sk_buff *skb; | 1217 | struct sk_buff *skb; |
1215 | 1218 | ||
1216 | sk_stream_for_retrans_queue(skb, sk) { | 1219 | tcp_for_write_queue(skb, sk) { |
1220 | if (skb == tcp_send_head(sk)) | ||
1221 | break; | ||
1217 | if (after(TCP_SKB_CB(skb)->seq, lost_retrans)) | 1222 | if (after(TCP_SKB_CB(skb)->seq, lost_retrans)) |
1218 | break; | 1223 | break; |
1219 | if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) | 1224 | if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) |
@@ -1266,8 +1271,8 @@ int tcp_use_frto(struct sock *sk) | |||
1266 | const struct tcp_sock *tp = tcp_sk(sk); | 1271 | const struct tcp_sock *tp = tcp_sk(sk); |
1267 | struct sk_buff *skb; | 1272 | struct sk_buff *skb; |
1268 | 1273 | ||
1269 | if (!sysctl_tcp_frto || !sk->sk_send_head || | 1274 | if (!sysctl_tcp_frto || !tcp_send_head(sk) || |
1270 | after(TCP_SKB_CB(sk->sk_send_head)->end_seq, | 1275 | after(TCP_SKB_CB(tcp_send_head(sk))->end_seq, |
1271 | tp->snd_una + tp->snd_wnd)) | 1276 | tp->snd_una + tp->snd_wnd)) |
1272 | return 0; | 1277 | return 0; |
1273 | 1278 | ||
@@ -1278,8 +1283,11 @@ int tcp_use_frto(struct sock *sk) | |||
1278 | if (tp->retrans_out > 1) | 1283 | if (tp->retrans_out > 1) |
1279 | return 0; | 1284 | return 0; |
1280 | 1285 | ||
1281 | skb = skb_peek(&sk->sk_write_queue)->next; /* Skips head */ | 1286 | skb = tcp_write_queue_head(sk); |
1282 | sk_stream_for_retrans_queue_from(skb, sk) { | 1287 | skb = tcp_write_queue_next(sk, skb); /* Skips head */ |
1288 | tcp_for_write_queue_from(skb, sk) { | ||
1289 | if (skb == tcp_send_head(sk)) | ||
1290 | break; | ||
1283 | if (TCP_SKB_CB(skb)->sacked&TCPCB_RETRANS) | 1291 | if (TCP_SKB_CB(skb)->sacked&TCPCB_RETRANS) |
1284 | return 0; | 1292 | return 0; |
1285 | /* Short-circuit when first non-SACKed skb has been checked */ | 1293 | /* Short-circuit when first non-SACKed skb has been checked */ |
@@ -1343,7 +1351,7 @@ void tcp_enter_frto(struct sock *sk) | |||
1343 | tp->undo_marker = tp->snd_una; | 1351 | tp->undo_marker = tp->snd_una; |
1344 | tp->undo_retrans = 0; | 1352 | tp->undo_retrans = 0; |
1345 | 1353 | ||
1346 | skb = skb_peek(&sk->sk_write_queue); | 1354 | skb = tcp_write_queue_head(sk); |
1347 | if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { | 1355 | if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { |
1348 | TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS; | 1356 | TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS; |
1349 | tp->retrans_out -= tcp_skb_pcount(skb); | 1357 | tp->retrans_out -= tcp_skb_pcount(skb); |
@@ -1380,7 +1388,9 @@ static void tcp_enter_frto_loss(struct sock *sk, int allowed_segments, int flag) | |||
1380 | tp->fackets_out = 0; | 1388 | tp->fackets_out = 0; |
1381 | tp->retrans_out = 0; | 1389 | tp->retrans_out = 0; |
1382 | 1390 | ||
1383 | sk_stream_for_retrans_queue(skb, sk) { | 1391 | tcp_for_write_queue(skb, sk) { |
1392 | if (skb == tcp_send_head(sk)) | ||
1393 | break; | ||
1384 | cnt += tcp_skb_pcount(skb); | 1394 | cnt += tcp_skb_pcount(skb); |
1385 | /* | 1395 | /* |
1386 | * Count the retransmission made on RTO correctly (only when | 1396 | * Count the retransmission made on RTO correctly (only when |
@@ -1468,7 +1478,9 @@ void tcp_enter_loss(struct sock *sk, int how) | |||
1468 | if (!how) | 1478 | if (!how) |
1469 | tp->undo_marker = tp->snd_una; | 1479 | tp->undo_marker = tp->snd_una; |
1470 | 1480 | ||
1471 | sk_stream_for_retrans_queue(skb, sk) { | 1481 | tcp_for_write_queue(skb, sk) { |
1482 | if (skb == tcp_send_head(sk)) | ||
1483 | break; | ||
1472 | cnt += tcp_skb_pcount(skb); | 1484 | cnt += tcp_skb_pcount(skb); |
1473 | if (TCP_SKB_CB(skb)->sacked&TCPCB_RETRANS) | 1485 | if (TCP_SKB_CB(skb)->sacked&TCPCB_RETRANS) |
1474 | tp->undo_marker = 0; | 1486 | tp->undo_marker = 0; |
@@ -1503,14 +1515,14 @@ static int tcp_check_sack_reneging(struct sock *sk) | |||
1503 | * receiver _host_ is heavily congested (or buggy). | 1515 | * receiver _host_ is heavily congested (or buggy). |
1504 | * Do processing similar to RTO timeout. | 1516 | * Do processing similar to RTO timeout. |
1505 | */ | 1517 | */ |
1506 | if ((skb = skb_peek(&sk->sk_write_queue)) != NULL && | 1518 | if ((skb = tcp_write_queue_head(sk)) != NULL && |
1507 | (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) { | 1519 | (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) { |
1508 | struct inet_connection_sock *icsk = inet_csk(sk); | 1520 | struct inet_connection_sock *icsk = inet_csk(sk); |
1509 | NET_INC_STATS_BH(LINUX_MIB_TCPSACKRENEGING); | 1521 | NET_INC_STATS_BH(LINUX_MIB_TCPSACKRENEGING); |
1510 | 1522 | ||
1511 | tcp_enter_loss(sk, 1); | 1523 | tcp_enter_loss(sk, 1); |
1512 | icsk->icsk_retransmits++; | 1524 | icsk->icsk_retransmits++; |
1513 | tcp_retransmit_skb(sk, skb_peek(&sk->sk_write_queue)); | 1525 | tcp_retransmit_skb(sk, tcp_write_queue_head(sk)); |
1514 | inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, | 1526 | inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, |
1515 | icsk->icsk_rto, TCP_RTO_MAX); | 1527 | icsk->icsk_rto, TCP_RTO_MAX); |
1516 | return 1; | 1528 | return 1; |
@@ -1531,7 +1543,7 @@ static inline int tcp_skb_timedout(struct sock *sk, struct sk_buff *skb) | |||
1531 | static inline int tcp_head_timedout(struct sock *sk, struct tcp_sock *tp) | 1543 | static inline int tcp_head_timedout(struct sock *sk, struct tcp_sock *tp) |
1532 | { | 1544 | { |
1533 | return tp->packets_out && | 1545 | return tp->packets_out && |
1534 | tcp_skb_timedout(sk, skb_peek(&sk->sk_write_queue)); | 1546 | tcp_skb_timedout(sk, tcp_write_queue_head(sk)); |
1535 | } | 1547 | } |
1536 | 1548 | ||
1537 | /* Linux NewReno/SACK/FACK/ECN state machine. | 1549 | /* Linux NewReno/SACK/FACK/ECN state machine. |
@@ -1726,11 +1738,13 @@ static void tcp_mark_head_lost(struct sock *sk, struct tcp_sock *tp, | |||
1726 | skb = tp->lost_skb_hint; | 1738 | skb = tp->lost_skb_hint; |
1727 | cnt = tp->lost_cnt_hint; | 1739 | cnt = tp->lost_cnt_hint; |
1728 | } else { | 1740 | } else { |
1729 | skb = sk->sk_write_queue.next; | 1741 | skb = tcp_write_queue_head(sk); |
1730 | cnt = 0; | 1742 | cnt = 0; |
1731 | } | 1743 | } |
1732 | 1744 | ||
1733 | sk_stream_for_retrans_queue_from(skb, sk) { | 1745 | tcp_for_write_queue_from(skb, sk) { |
1746 | if (skb == tcp_send_head(sk)) | ||
1747 | break; | ||
1734 | /* TODO: do this better */ | 1748 | /* TODO: do this better */ |
1735 | /* this is not the most efficient way to do this... */ | 1749 | /* this is not the most efficient way to do this... */ |
1736 | tp->lost_skb_hint = skb; | 1750 | tp->lost_skb_hint = skb; |
@@ -1777,9 +1791,11 @@ static void tcp_update_scoreboard(struct sock *sk, struct tcp_sock *tp) | |||
1777 | struct sk_buff *skb; | 1791 | struct sk_buff *skb; |
1778 | 1792 | ||
1779 | skb = tp->scoreboard_skb_hint ? tp->scoreboard_skb_hint | 1793 | skb = tp->scoreboard_skb_hint ? tp->scoreboard_skb_hint |
1780 | : sk->sk_write_queue.next; | 1794 | : tcp_write_queue_head(sk); |
1781 | 1795 | ||
1782 | sk_stream_for_retrans_queue_from(skb, sk) { | 1796 | tcp_for_write_queue_from(skb, sk) { |
1797 | if (skb == tcp_send_head(sk)) | ||
1798 | break; | ||
1783 | if (!tcp_skb_timedout(sk, skb)) | 1799 | if (!tcp_skb_timedout(sk, skb)) |
1784 | break; | 1800 | break; |
1785 | 1801 | ||
@@ -1970,7 +1986,9 @@ static int tcp_try_undo_loss(struct sock *sk, struct tcp_sock *tp) | |||
1970 | { | 1986 | { |
1971 | if (tcp_may_undo(tp)) { | 1987 | if (tcp_may_undo(tp)) { |
1972 | struct sk_buff *skb; | 1988 | struct sk_buff *skb; |
1973 | sk_stream_for_retrans_queue(skb, sk) { | 1989 | tcp_for_write_queue(skb, sk) { |
1990 | if (skb == tcp_send_head(sk)) | ||
1991 | break; | ||
1974 | TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST; | 1992 | TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST; |
1975 | } | 1993 | } |
1976 | 1994 | ||
@@ -2382,8 +2400,8 @@ static int tcp_clean_rtx_queue(struct sock *sk, __s32 *seq_rtt_p) | |||
2382 | = icsk->icsk_ca_ops->rtt_sample; | 2400 | = icsk->icsk_ca_ops->rtt_sample; |
2383 | struct timeval tv = { .tv_sec = 0, .tv_usec = 0 }; | 2401 | struct timeval tv = { .tv_sec = 0, .tv_usec = 0 }; |
2384 | 2402 | ||
2385 | while ((skb = skb_peek(&sk->sk_write_queue)) && | 2403 | while ((skb = tcp_write_queue_head(sk)) && |
2386 | skb != sk->sk_send_head) { | 2404 | skb != tcp_send_head(sk)) { |
2387 | struct tcp_skb_cb *scb = TCP_SKB_CB(skb); | 2405 | struct tcp_skb_cb *scb = TCP_SKB_CB(skb); |
2388 | __u8 sacked = scb->sacked; | 2406 | __u8 sacked = scb->sacked; |
2389 | 2407 | ||
@@ -2446,7 +2464,7 @@ static int tcp_clean_rtx_queue(struct sock *sk, __s32 *seq_rtt_p) | |||
2446 | } | 2464 | } |
2447 | tcp_dec_pcount_approx(&tp->fackets_out, skb); | 2465 | tcp_dec_pcount_approx(&tp->fackets_out, skb); |
2448 | tcp_packets_out_dec(tp, skb); | 2466 | tcp_packets_out_dec(tp, skb); |
2449 | __skb_unlink(skb, &sk->sk_write_queue); | 2467 | tcp_unlink_write_queue(skb, sk); |
2450 | sk_stream_free_skb(sk, skb); | 2468 | sk_stream_free_skb(sk, skb); |
2451 | clear_all_retrans_hints(tp); | 2469 | clear_all_retrans_hints(tp); |
2452 | } | 2470 | } |
@@ -2495,7 +2513,7 @@ static void tcp_ack_probe(struct sock *sk) | |||
2495 | 2513 | ||
2496 | /* Was it a usable window open? */ | 2514 | /* Was it a usable window open? */ |
2497 | 2515 | ||
2498 | if (!after(TCP_SKB_CB(sk->sk_send_head)->end_seq, | 2516 | if (!after(TCP_SKB_CB(tcp_send_head(sk))->end_seq, |
2499 | tp->snd_una + tp->snd_wnd)) { | 2517 | tp->snd_una + tp->snd_wnd)) { |
2500 | icsk->icsk_backoff = 0; | 2518 | icsk->icsk_backoff = 0; |
2501 | inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0); | 2519 | inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0); |
@@ -2795,7 +2813,7 @@ no_queue: | |||
2795 | * being used to time the probes, and is probably far higher than | 2813 | * being used to time the probes, and is probably far higher than |
2796 | * it needs to be for normal retransmission. | 2814 | * it needs to be for normal retransmission. |
2797 | */ | 2815 | */ |
2798 | if (sk->sk_send_head) | 2816 | if (tcp_send_head(sk)) |
2799 | tcp_ack_probe(sk); | 2817 | tcp_ack_probe(sk); |
2800 | return 1; | 2818 | return 1; |
2801 | 2819 | ||
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index addac1110f94..3326681b8429 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c | |||
@@ -1890,7 +1890,7 @@ int tcp_v4_destroy_sock(struct sock *sk) | |||
1890 | tcp_cleanup_congestion_control(sk); | 1890 | tcp_cleanup_congestion_control(sk); |
1891 | 1891 | ||
1892 | /* Cleanup up the write buffer. */ | 1892 | /* Cleanup up the write buffer. */ |
1893 | sk_stream_writequeue_purge(sk); | 1893 | tcp_write_queue_purge(sk); |
1894 | 1894 | ||
1895 | /* Cleans up our, hopefully empty, out_of_order_queue. */ | 1895 | /* Cleans up our, hopefully empty, out_of_order_queue. */ |
1896 | __skb_queue_purge(&tp->out_of_order_queue); | 1896 | __skb_queue_purge(&tp->out_of_order_queue); |
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index d19b2f3b70fd..2a62b55b15f1 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c | |||
@@ -65,9 +65,7 @@ int sysctl_tcp_slow_start_after_idle __read_mostly = 1; | |||
65 | static void update_send_head(struct sock *sk, struct tcp_sock *tp, | 65 | static void update_send_head(struct sock *sk, struct tcp_sock *tp, |
66 | struct sk_buff *skb) | 66 | struct sk_buff *skb) |
67 | { | 67 | { |
68 | sk->sk_send_head = skb->next; | 68 | tcp_advance_send_head(sk, skb); |
69 | if (sk->sk_send_head == (struct sk_buff *)&sk->sk_write_queue) | ||
70 | sk->sk_send_head = NULL; | ||
71 | tp->snd_nxt = TCP_SKB_CB(skb)->end_seq; | 69 | tp->snd_nxt = TCP_SKB_CB(skb)->end_seq; |
72 | tcp_packets_out_inc(sk, tp, skb); | 70 | tcp_packets_out_inc(sk, tp, skb); |
73 | } | 71 | } |
@@ -567,12 +565,8 @@ static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb) | |||
567 | /* Advance write_seq and place onto the write_queue. */ | 565 | /* Advance write_seq and place onto the write_queue. */ |
568 | tp->write_seq = TCP_SKB_CB(skb)->end_seq; | 566 | tp->write_seq = TCP_SKB_CB(skb)->end_seq; |
569 | skb_header_release(skb); | 567 | skb_header_release(skb); |
570 | __skb_queue_tail(&sk->sk_write_queue, skb); | 568 | tcp_add_write_queue_tail(sk, skb); |
571 | sk_charge_skb(sk, skb); | 569 | sk_charge_skb(sk, skb); |
572 | |||
573 | /* Queue it, remembering where we must start sending. */ | ||
574 | if (sk->sk_send_head == NULL) | ||
575 | sk->sk_send_head = skb; | ||
576 | } | 570 | } |
577 | 571 | ||
578 | static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, unsigned int mss_now) | 572 | static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, unsigned int mss_now) |
@@ -705,7 +699,7 @@ int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len, unsigned int mss | |||
705 | 699 | ||
706 | /* Link BUFF into the send queue. */ | 700 | /* Link BUFF into the send queue. */ |
707 | skb_header_release(buff); | 701 | skb_header_release(buff); |
708 | __skb_append(skb, buff, &sk->sk_write_queue); | 702 | tcp_insert_write_queue_after(skb, buff, sk); |
709 | 703 | ||
710 | return 0; | 704 | return 0; |
711 | } | 705 | } |
@@ -1056,7 +1050,7 @@ static inline int tcp_snd_wnd_test(struct tcp_sock *tp, struct sk_buff *skb, uns | |||
1056 | return !after(end_seq, tp->snd_una + tp->snd_wnd); | 1050 | return !after(end_seq, tp->snd_una + tp->snd_wnd); |
1057 | } | 1051 | } |
1058 | 1052 | ||
1059 | /* This checks if the data bearing packet SKB (usually sk->sk_send_head) | 1053 | /* This checks if the data bearing packet SKB (usually tcp_send_head(sk)) |
1060 | * should be put on the wire right now. If so, it returns the number of | 1054 | * should be put on the wire right now. If so, it returns the number of |
1061 | * packets allowed by the congestion window. | 1055 | * packets allowed by the congestion window. |
1062 | */ | 1056 | */ |
@@ -1079,15 +1073,9 @@ static unsigned int tcp_snd_test(struct sock *sk, struct sk_buff *skb, | |||
1079 | return cwnd_quota; | 1073 | return cwnd_quota; |
1080 | } | 1074 | } |
1081 | 1075 | ||
1082 | static inline int tcp_skb_is_last(const struct sock *sk, | ||
1083 | const struct sk_buff *skb) | ||
1084 | { | ||
1085 | return skb->next == (struct sk_buff *)&sk->sk_write_queue; | ||
1086 | } | ||
1087 | |||
1088 | int tcp_may_send_now(struct sock *sk, struct tcp_sock *tp) | 1076 | int tcp_may_send_now(struct sock *sk, struct tcp_sock *tp) |
1089 | { | 1077 | { |
1090 | struct sk_buff *skb = sk->sk_send_head; | 1078 | struct sk_buff *skb = tcp_send_head(sk); |
1091 | 1079 | ||
1092 | return (skb && | 1080 | return (skb && |
1093 | tcp_snd_test(sk, skb, tcp_current_mss(sk, 1), | 1081 | tcp_snd_test(sk, skb, tcp_current_mss(sk, 1), |
@@ -1143,7 +1131,7 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len, | |||
1143 | 1131 | ||
1144 | /* Link BUFF into the send queue. */ | 1132 | /* Link BUFF into the send queue. */ |
1145 | skb_header_release(buff); | 1133 | skb_header_release(buff); |
1146 | __skb_append(skb, buff, &sk->sk_write_queue); | 1134 | tcp_insert_write_queue_after(skb, buff, sk); |
1147 | 1135 | ||
1148 | return 0; | 1136 | return 0; |
1149 | } | 1137 | } |
@@ -1249,10 +1237,10 @@ static int tcp_mtu_probe(struct sock *sk) | |||
1249 | 1237 | ||
1250 | /* Have enough data in the send queue to probe? */ | 1238 | /* Have enough data in the send queue to probe? */ |
1251 | len = 0; | 1239 | len = 0; |
1252 | if ((skb = sk->sk_send_head) == NULL) | 1240 | if ((skb = tcp_send_head(sk)) == NULL) |
1253 | return -1; | 1241 | return -1; |
1254 | while ((len += skb->len) < probe_size && !tcp_skb_is_last(sk, skb)) | 1242 | while ((len += skb->len) < probe_size && !tcp_skb_is_last(sk, skb)) |
1255 | skb = skb->next; | 1243 | skb = tcp_write_queue_next(sk, skb); |
1256 | if (len < probe_size) | 1244 | if (len < probe_size) |
1257 | return -1; | 1245 | return -1; |
1258 | 1246 | ||
@@ -1279,9 +1267,9 @@ static int tcp_mtu_probe(struct sock *sk) | |||
1279 | return -1; | 1267 | return -1; |
1280 | sk_charge_skb(sk, nskb); | 1268 | sk_charge_skb(sk, nskb); |
1281 | 1269 | ||
1282 | skb = sk->sk_send_head; | 1270 | skb = tcp_send_head(sk); |
1283 | __skb_insert(nskb, skb->prev, skb, &sk->sk_write_queue); | 1271 | tcp_insert_write_queue_before(nskb, skb, sk); |
1284 | sk->sk_send_head = nskb; | 1272 | tcp_advance_send_head(sk, skb); |
1285 | 1273 | ||
1286 | TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; | 1274 | TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; |
1287 | TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; | 1275 | TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; |
@@ -1292,7 +1280,7 @@ static int tcp_mtu_probe(struct sock *sk) | |||
1292 | 1280 | ||
1293 | len = 0; | 1281 | len = 0; |
1294 | while (len < probe_size) { | 1282 | while (len < probe_size) { |
1295 | next = skb->next; | 1283 | next = tcp_write_queue_next(sk, skb); |
1296 | 1284 | ||
1297 | copy = min_t(int, skb->len, probe_size - len); | 1285 | copy = min_t(int, skb->len, probe_size - len); |
1298 | if (nskb->ip_summed) | 1286 | if (nskb->ip_summed) |
@@ -1305,7 +1293,7 @@ static int tcp_mtu_probe(struct sock *sk) | |||
1305 | /* We've eaten all the data from this skb. | 1293 | /* We've eaten all the data from this skb. |
1306 | * Throw it away. */ | 1294 | * Throw it away. */ |
1307 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags; | 1295 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags; |
1308 | __skb_unlink(skb, &sk->sk_write_queue); | 1296 | tcp_unlink_write_queue(skb, sk); |
1309 | sk_stream_free_skb(sk, skb); | 1297 | sk_stream_free_skb(sk, skb); |
1310 | } else { | 1298 | } else { |
1311 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags & | 1299 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags & |
@@ -1377,7 +1365,7 @@ static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle) | |||
1377 | sent_pkts = 1; | 1365 | sent_pkts = 1; |
1378 | } | 1366 | } |
1379 | 1367 | ||
1380 | while ((skb = sk->sk_send_head)) { | 1368 | while ((skb = tcp_send_head(sk))) { |
1381 | unsigned int limit; | 1369 | unsigned int limit; |
1382 | 1370 | ||
1383 | tso_segs = tcp_init_tso_segs(sk, skb, mss_now); | 1371 | tso_segs = tcp_init_tso_segs(sk, skb, mss_now); |
@@ -1435,7 +1423,7 @@ static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle) | |||
1435 | tcp_cwnd_validate(sk, tp); | 1423 | tcp_cwnd_validate(sk, tp); |
1436 | return 0; | 1424 | return 0; |
1437 | } | 1425 | } |
1438 | return !tp->packets_out && sk->sk_send_head; | 1426 | return !tp->packets_out && tcp_send_head(sk); |
1439 | } | 1427 | } |
1440 | 1428 | ||
1441 | /* Push out any pending frames which were held back due to | 1429 | /* Push out any pending frames which were held back due to |
@@ -1445,7 +1433,7 @@ static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle) | |||
1445 | void __tcp_push_pending_frames(struct sock *sk, struct tcp_sock *tp, | 1433 | void __tcp_push_pending_frames(struct sock *sk, struct tcp_sock *tp, |
1446 | unsigned int cur_mss, int nonagle) | 1434 | unsigned int cur_mss, int nonagle) |
1447 | { | 1435 | { |
1448 | struct sk_buff *skb = sk->sk_send_head; | 1436 | struct sk_buff *skb = tcp_send_head(sk); |
1449 | 1437 | ||
1450 | if (skb) { | 1438 | if (skb) { |
1451 | if (tcp_write_xmit(sk, cur_mss, nonagle)) | 1439 | if (tcp_write_xmit(sk, cur_mss, nonagle)) |
@@ -1459,7 +1447,7 @@ void __tcp_push_pending_frames(struct sock *sk, struct tcp_sock *tp, | |||
1459 | void tcp_push_one(struct sock *sk, unsigned int mss_now) | 1447 | void tcp_push_one(struct sock *sk, unsigned int mss_now) |
1460 | { | 1448 | { |
1461 | struct tcp_sock *tp = tcp_sk(sk); | 1449 | struct tcp_sock *tp = tcp_sk(sk); |
1462 | struct sk_buff *skb = sk->sk_send_head; | 1450 | struct sk_buff *skb = tcp_send_head(sk); |
1463 | unsigned int tso_segs, cwnd_quota; | 1451 | unsigned int tso_segs, cwnd_quota; |
1464 | 1452 | ||
1465 | BUG_ON(!skb || skb->len < mss_now); | 1453 | BUG_ON(!skb || skb->len < mss_now); |
@@ -1620,7 +1608,7 @@ u32 __tcp_select_window(struct sock *sk) | |||
1620 | static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, int mss_now) | 1608 | static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, int mss_now) |
1621 | { | 1609 | { |
1622 | struct tcp_sock *tp = tcp_sk(sk); | 1610 | struct tcp_sock *tp = tcp_sk(sk); |
1623 | struct sk_buff *next_skb = skb->next; | 1611 | struct sk_buff *next_skb = tcp_write_queue_next(sk, skb); |
1624 | 1612 | ||
1625 | /* The first test we must make is that neither of these two | 1613 | /* The first test we must make is that neither of these two |
1626 | * SKB's are still referenced by someone else. | 1614 | * SKB's are still referenced by someone else. |
@@ -1652,7 +1640,7 @@ static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, int m | |||
1652 | clear_all_retrans_hints(tp); | 1640 | clear_all_retrans_hints(tp); |
1653 | 1641 | ||
1654 | /* Ok. We will be able to collapse the packet. */ | 1642 | /* Ok. We will be able to collapse the packet. */ |
1655 | __skb_unlink(next_skb, &sk->sk_write_queue); | 1643 | tcp_unlink_write_queue(next_skb, sk); |
1656 | 1644 | ||
1657 | memcpy(skb_put(skb, next_skb_size), next_skb->data, next_skb_size); | 1645 | memcpy(skb_put(skb, next_skb_size), next_skb->data, next_skb_size); |
1658 | 1646 | ||
@@ -1706,7 +1694,9 @@ void tcp_simple_retransmit(struct sock *sk) | |||
1706 | unsigned int mss = tcp_current_mss(sk, 0); | 1694 | unsigned int mss = tcp_current_mss(sk, 0); |
1707 | int lost = 0; | 1695 | int lost = 0; |
1708 | 1696 | ||
1709 | sk_stream_for_retrans_queue(skb, sk) { | 1697 | tcp_for_write_queue(skb, sk) { |
1698 | if (skb == tcp_send_head(sk)) | ||
1699 | break; | ||
1710 | if (skb->len > mss && | 1700 | if (skb->len > mss && |
1711 | !(TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_ACKED)) { | 1701 | !(TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_ACKED)) { |
1712 | if (TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_RETRANS) { | 1702 | if (TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_RETRANS) { |
@@ -1790,10 +1780,10 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb) | |||
1790 | /* Collapse two adjacent packets if worthwhile and we can. */ | 1780 | /* Collapse two adjacent packets if worthwhile and we can. */ |
1791 | if(!(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN) && | 1781 | if(!(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN) && |
1792 | (skb->len < (cur_mss >> 1)) && | 1782 | (skb->len < (cur_mss >> 1)) && |
1793 | (skb->next != sk->sk_send_head) && | 1783 | (tcp_write_queue_next(sk, skb) != tcp_send_head(sk)) && |
1794 | (skb->next != (struct sk_buff *)&sk->sk_write_queue) && | 1784 | (!tcp_skb_is_last(sk, skb)) && |
1795 | (skb_shinfo(skb)->nr_frags == 0 && skb_shinfo(skb->next)->nr_frags == 0) && | 1785 | (skb_shinfo(skb)->nr_frags == 0 && skb_shinfo(tcp_write_queue_next(sk, skb))->nr_frags == 0) && |
1796 | (tcp_skb_pcount(skb) == 1 && tcp_skb_pcount(skb->next) == 1) && | 1786 | (tcp_skb_pcount(skb) == 1 && tcp_skb_pcount(tcp_write_queue_next(sk, skb)) == 1) && |
1797 | (sysctl_tcp_retrans_collapse != 0)) | 1787 | (sysctl_tcp_retrans_collapse != 0)) |
1798 | tcp_retrans_try_collapse(sk, skb, cur_mss); | 1788 | tcp_retrans_try_collapse(sk, skb, cur_mss); |
1799 | 1789 | ||
@@ -1872,15 +1862,17 @@ void tcp_xmit_retransmit_queue(struct sock *sk) | |||
1872 | skb = tp->retransmit_skb_hint; | 1862 | skb = tp->retransmit_skb_hint; |
1873 | packet_cnt = tp->retransmit_cnt_hint; | 1863 | packet_cnt = tp->retransmit_cnt_hint; |
1874 | }else{ | 1864 | }else{ |
1875 | skb = sk->sk_write_queue.next; | 1865 | skb = tcp_write_queue_head(sk); |
1876 | packet_cnt = 0; | 1866 | packet_cnt = 0; |
1877 | } | 1867 | } |
1878 | 1868 | ||
1879 | /* First pass: retransmit lost packets. */ | 1869 | /* First pass: retransmit lost packets. */ |
1880 | if (tp->lost_out) { | 1870 | if (tp->lost_out) { |
1881 | sk_stream_for_retrans_queue_from(skb, sk) { | 1871 | tcp_for_write_queue_from(skb, sk) { |
1882 | __u8 sacked = TCP_SKB_CB(skb)->sacked; | 1872 | __u8 sacked = TCP_SKB_CB(skb)->sacked; |
1883 | 1873 | ||
1874 | if (skb == tcp_send_head(sk)) | ||
1875 | break; | ||
1884 | /* we could do better than to assign each time */ | 1876 | /* we could do better than to assign each time */ |
1885 | tp->retransmit_skb_hint = skb; | 1877 | tp->retransmit_skb_hint = skb; |
1886 | tp->retransmit_cnt_hint = packet_cnt; | 1878 | tp->retransmit_cnt_hint = packet_cnt; |
@@ -1906,8 +1898,7 @@ void tcp_xmit_retransmit_queue(struct sock *sk) | |||
1906 | else | 1898 | else |
1907 | NET_INC_STATS_BH(LINUX_MIB_TCPSLOWSTARTRETRANS); | 1899 | NET_INC_STATS_BH(LINUX_MIB_TCPSLOWSTARTRETRANS); |
1908 | 1900 | ||
1909 | if (skb == | 1901 | if (skb == tcp_write_queue_head(sk)) |
1910 | skb_peek(&sk->sk_write_queue)) | ||
1911 | inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, | 1902 | inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, |
1912 | inet_csk(sk)->icsk_rto, | 1903 | inet_csk(sk)->icsk_rto, |
1913 | TCP_RTO_MAX); | 1904 | TCP_RTO_MAX); |
@@ -1944,11 +1935,13 @@ void tcp_xmit_retransmit_queue(struct sock *sk) | |||
1944 | skb = tp->forward_skb_hint; | 1935 | skb = tp->forward_skb_hint; |
1945 | packet_cnt = tp->forward_cnt_hint; | 1936 | packet_cnt = tp->forward_cnt_hint; |
1946 | } else{ | 1937 | } else{ |
1947 | skb = sk->sk_write_queue.next; | 1938 | skb = tcp_write_queue_head(sk); |
1948 | packet_cnt = 0; | 1939 | packet_cnt = 0; |
1949 | } | 1940 | } |
1950 | 1941 | ||
1951 | sk_stream_for_retrans_queue_from(skb, sk) { | 1942 | tcp_for_write_queue_from(skb, sk) { |
1943 | if (skb == tcp_send_head(sk)) | ||
1944 | break; | ||
1952 | tp->forward_cnt_hint = packet_cnt; | 1945 | tp->forward_cnt_hint = packet_cnt; |
1953 | tp->forward_skb_hint = skb; | 1946 | tp->forward_skb_hint = skb; |
1954 | 1947 | ||
@@ -1973,7 +1966,7 @@ void tcp_xmit_retransmit_queue(struct sock *sk) | |||
1973 | break; | 1966 | break; |
1974 | } | 1967 | } |
1975 | 1968 | ||
1976 | if (skb == skb_peek(&sk->sk_write_queue)) | 1969 | if (skb == tcp_write_queue_head(sk)) |
1977 | inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, | 1970 | inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, |
1978 | inet_csk(sk)->icsk_rto, | 1971 | inet_csk(sk)->icsk_rto, |
1979 | TCP_RTO_MAX); | 1972 | TCP_RTO_MAX); |
@@ -1989,7 +1982,7 @@ void tcp_xmit_retransmit_queue(struct sock *sk) | |||
1989 | void tcp_send_fin(struct sock *sk) | 1982 | void tcp_send_fin(struct sock *sk) |
1990 | { | 1983 | { |
1991 | struct tcp_sock *tp = tcp_sk(sk); | 1984 | struct tcp_sock *tp = tcp_sk(sk); |
1992 | struct sk_buff *skb = skb_peek_tail(&sk->sk_write_queue); | 1985 | struct sk_buff *skb = tcp_write_queue_tail(sk); |
1993 | int mss_now; | 1986 | int mss_now; |
1994 | 1987 | ||
1995 | /* Optimization, tack on the FIN if we have a queue of | 1988 | /* Optimization, tack on the FIN if we have a queue of |
@@ -1998,7 +1991,7 @@ void tcp_send_fin(struct sock *sk) | |||
1998 | */ | 1991 | */ |
1999 | mss_now = tcp_current_mss(sk, 1); | 1992 | mss_now = tcp_current_mss(sk, 1); |
2000 | 1993 | ||
2001 | if (sk->sk_send_head != NULL) { | 1994 | if (tcp_send_head(sk) != NULL) { |
2002 | TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN; | 1995 | TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN; |
2003 | TCP_SKB_CB(skb)->end_seq++; | 1996 | TCP_SKB_CB(skb)->end_seq++; |
2004 | tp->write_seq++; | 1997 | tp->write_seq++; |
@@ -2071,7 +2064,7 @@ int tcp_send_synack(struct sock *sk) | |||
2071 | { | 2064 | { |
2072 | struct sk_buff* skb; | 2065 | struct sk_buff* skb; |
2073 | 2066 | ||
2074 | skb = skb_peek(&sk->sk_write_queue); | 2067 | skb = tcp_write_queue_head(sk); |
2075 | if (skb == NULL || !(TCP_SKB_CB(skb)->flags&TCPCB_FLAG_SYN)) { | 2068 | if (skb == NULL || !(TCP_SKB_CB(skb)->flags&TCPCB_FLAG_SYN)) { |
2076 | printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n"); | 2069 | printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n"); |
2077 | return -EFAULT; | 2070 | return -EFAULT; |
@@ -2081,9 +2074,9 @@ int tcp_send_synack(struct sock *sk) | |||
2081 | struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); | 2074 | struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); |
2082 | if (nskb == NULL) | 2075 | if (nskb == NULL) |
2083 | return -ENOMEM; | 2076 | return -ENOMEM; |
2084 | __skb_unlink(skb, &sk->sk_write_queue); | 2077 | tcp_unlink_write_queue(skb, sk); |
2085 | skb_header_release(nskb); | 2078 | skb_header_release(nskb); |
2086 | __skb_queue_head(&sk->sk_write_queue, nskb); | 2079 | __tcp_add_write_queue_head(sk, nskb); |
2087 | sk_stream_free_skb(sk, skb); | 2080 | sk_stream_free_skb(sk, skb); |
2088 | sk_charge_skb(sk, nskb); | 2081 | sk_charge_skb(sk, nskb); |
2089 | skb = nskb; | 2082 | skb = nskb; |
@@ -2285,7 +2278,7 @@ int tcp_connect(struct sock *sk) | |||
2285 | TCP_SKB_CB(buff)->when = tcp_time_stamp; | 2278 | TCP_SKB_CB(buff)->when = tcp_time_stamp; |
2286 | tp->retrans_stamp = TCP_SKB_CB(buff)->when; | 2279 | tp->retrans_stamp = TCP_SKB_CB(buff)->when; |
2287 | skb_header_release(buff); | 2280 | skb_header_release(buff); |
2288 | __skb_queue_tail(&sk->sk_write_queue, buff); | 2281 | __tcp_add_write_queue_tail(sk, buff); |
2289 | sk_charge_skb(sk, buff); | 2282 | sk_charge_skb(sk, buff); |
2290 | tp->packets_out += tcp_skb_pcount(buff); | 2283 | tp->packets_out += tcp_skb_pcount(buff); |
2291 | tcp_transmit_skb(sk, buff, 1, GFP_KERNEL); | 2284 | tcp_transmit_skb(sk, buff, 1, GFP_KERNEL); |
@@ -2441,7 +2434,7 @@ int tcp_write_wakeup(struct sock *sk) | |||
2441 | struct tcp_sock *tp = tcp_sk(sk); | 2434 | struct tcp_sock *tp = tcp_sk(sk); |
2442 | struct sk_buff *skb; | 2435 | struct sk_buff *skb; |
2443 | 2436 | ||
2444 | if ((skb = sk->sk_send_head) != NULL && | 2437 | if ((skb = tcp_send_head(sk)) != NULL && |
2445 | before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) { | 2438 | before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) { |
2446 | int err; | 2439 | int err; |
2447 | unsigned int mss = tcp_current_mss(sk, 0); | 2440 | unsigned int mss = tcp_current_mss(sk, 0); |
@@ -2491,7 +2484,7 @@ void tcp_send_probe0(struct sock *sk) | |||
2491 | 2484 | ||
2492 | err = tcp_write_wakeup(sk); | 2485 | err = tcp_write_wakeup(sk); |
2493 | 2486 | ||
2494 | if (tp->packets_out || !sk->sk_send_head) { | 2487 | if (tp->packets_out || !tcp_send_head(sk)) { |
2495 | /* Cancel probe timer, if it is not required. */ | 2488 | /* Cancel probe timer, if it is not required. */ |
2496 | icsk->icsk_probes_out = 0; | 2489 | icsk->icsk_probes_out = 0; |
2497 | icsk->icsk_backoff = 0; | 2490 | icsk->icsk_backoff = 0; |
diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index a9243cfc1bea..2ca97b20929d 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c | |||
@@ -233,7 +233,7 @@ static void tcp_probe_timer(struct sock *sk) | |||
233 | struct tcp_sock *tp = tcp_sk(sk); | 233 | struct tcp_sock *tp = tcp_sk(sk); |
234 | int max_probes; | 234 | int max_probes; |
235 | 235 | ||
236 | if (tp->packets_out || !sk->sk_send_head) { | 236 | if (tp->packets_out || !tcp_send_head(sk)) { |
237 | icsk->icsk_probes_out = 0; | 237 | icsk->icsk_probes_out = 0; |
238 | return; | 238 | return; |
239 | } | 239 | } |
@@ -284,7 +284,7 @@ static void tcp_retransmit_timer(struct sock *sk) | |||
284 | if (!tp->packets_out) | 284 | if (!tp->packets_out) |
285 | goto out; | 285 | goto out; |
286 | 286 | ||
287 | BUG_TRAP(!skb_queue_empty(&sk->sk_write_queue)); | 287 | BUG_TRAP(!tcp_write_queue_empty(sk)); |
288 | 288 | ||
289 | if (!tp->snd_wnd && !sock_flag(sk, SOCK_DEAD) && | 289 | if (!tp->snd_wnd && !sock_flag(sk, SOCK_DEAD) && |
290 | !((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))) { | 290 | !((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))) { |
@@ -306,7 +306,7 @@ static void tcp_retransmit_timer(struct sock *sk) | |||
306 | goto out; | 306 | goto out; |
307 | } | 307 | } |
308 | tcp_enter_loss(sk, 0); | 308 | tcp_enter_loss(sk, 0); |
309 | tcp_retransmit_skb(sk, skb_peek(&sk->sk_write_queue)); | 309 | tcp_retransmit_skb(sk, tcp_write_queue_head(sk)); |
310 | __sk_dst_reset(sk); | 310 | __sk_dst_reset(sk); |
311 | goto out_reset_timer; | 311 | goto out_reset_timer; |
312 | } | 312 | } |
@@ -341,7 +341,7 @@ static void tcp_retransmit_timer(struct sock *sk) | |||
341 | tcp_enter_loss(sk, 0); | 341 | tcp_enter_loss(sk, 0); |
342 | } | 342 | } |
343 | 343 | ||
344 | if (tcp_retransmit_skb(sk, skb_peek(&sk->sk_write_queue)) > 0) { | 344 | if (tcp_retransmit_skb(sk, tcp_write_queue_head(sk)) > 0) { |
345 | /* Retransmission failed because of local congestion, | 345 | /* Retransmission failed because of local congestion, |
346 | * do not backoff. | 346 | * do not backoff. |
347 | */ | 347 | */ |
@@ -482,7 +482,7 @@ static void tcp_keepalive_timer (unsigned long data) | |||
482 | elapsed = keepalive_time_when(tp); | 482 | elapsed = keepalive_time_when(tp); |
483 | 483 | ||
484 | /* It is alive without keepalive 8) */ | 484 | /* It is alive without keepalive 8) */ |
485 | if (tp->packets_out || sk->sk_send_head) | 485 | if (tp->packets_out || tcp_send_head(sk)) |
486 | goto resched; | 486 | goto resched; |
487 | 487 | ||
488 | elapsed = tcp_time_stamp - tp->rcv_tstamp; | 488 | elapsed = tcp_time_stamp - tp->rcv_tstamp; |