aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@sunset.davemloft.net>2007-03-07 15:12:44 -0500
committerDavid S. Miller <davem@sunset.davemloft.net>2007-04-26 01:24:02 -0400
commitfe067e8ab5e0dc5ca3c54634924c628da92090b4 (patch)
tree98f5a6ebbb770f16682cfc52caea2da1e7eeb73b
parent02ea4923b4997d7e1310c027081f46d584b9d714 (diff)
[TCP]: Abstract out all write queue operations.
This allows the write queue implementation to be changed, for example, to one which allows fast interval searching. Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--include/net/sock.h21
-rw-r--r--include/net/tcp.h114
-rw-r--r--net/ipv4/tcp.c32
-rw-r--r--net/ipv4/tcp_input.c64
-rw-r--r--net/ipv4/tcp_ipv4.c2
-rw-r--r--net/ipv4/tcp_output.c95
-rw-r--r--net/ipv4/tcp_timer.c10
7 files changed, 221 insertions, 117 deletions
diff --git a/include/net/sock.h b/include/net/sock.h
index 9583639090d2..2974bacc8850 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -710,15 +710,6 @@ static inline void sk_stream_mem_reclaim(struct sock *sk)
710 __sk_stream_mem_reclaim(sk); 710 __sk_stream_mem_reclaim(sk);
711} 711}
712 712
713static inline void sk_stream_writequeue_purge(struct sock *sk)
714{
715 struct sk_buff *skb;
716
717 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL)
718 sk_stream_free_skb(sk, skb);
719 sk_stream_mem_reclaim(sk);
720}
721
722static inline int sk_stream_rmem_schedule(struct sock *sk, struct sk_buff *skb) 713static inline int sk_stream_rmem_schedule(struct sock *sk, struct sk_buff *skb)
723{ 714{
724 return (int)skb->truesize <= sk->sk_forward_alloc || 715 return (int)skb->truesize <= sk->sk_forward_alloc ||
@@ -1256,18 +1247,6 @@ static inline struct page *sk_stream_alloc_page(struct sock *sk)
1256 return page; 1247 return page;
1257} 1248}
1258 1249
1259#define sk_stream_for_retrans_queue(skb, sk) \
1260 for (skb = (sk)->sk_write_queue.next; \
1261 (skb != (sk)->sk_send_head) && \
1262 (skb != (struct sk_buff *)&(sk)->sk_write_queue); \
1263 skb = skb->next)
1264
1265/*from STCP for fast SACK Process*/
1266#define sk_stream_for_retrans_queue_from(skb, sk) \
1267 for (; (skb != (sk)->sk_send_head) && \
1268 (skb != (struct sk_buff *)&(sk)->sk_write_queue); \
1269 skb = skb->next)
1270
1271/* 1250/*
1272 * Default write policy as shown to user space via poll/select/SIGIO 1251 * Default write policy as shown to user space via poll/select/SIGIO
1273 */ 1252 */
diff --git a/include/net/tcp.h b/include/net/tcp.h
index 181c0600af1c..6dacc352dcf1 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -1162,6 +1162,120 @@ static inline void tcp_put_md5sig_pool(void)
1162 put_cpu(); 1162 put_cpu();
1163} 1163}
1164 1164
1165/* write queue abstraction */
1166static inline void tcp_write_queue_purge(struct sock *sk)
1167{
1168 struct sk_buff *skb;
1169
1170 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL)
1171 sk_stream_free_skb(sk, skb);
1172 sk_stream_mem_reclaim(sk);
1173}
1174
1175static inline struct sk_buff *tcp_write_queue_head(struct sock *sk)
1176{
1177 struct sk_buff *skb = sk->sk_write_queue.next;
1178 if (skb == (struct sk_buff *) &sk->sk_write_queue)
1179 return NULL;
1180 return skb;
1181}
1182
1183static inline struct sk_buff *tcp_write_queue_tail(struct sock *sk)
1184{
1185 struct sk_buff *skb = sk->sk_write_queue.prev;
1186 if (skb == (struct sk_buff *) &sk->sk_write_queue)
1187 return NULL;
1188 return skb;
1189}
1190
1191static inline struct sk_buff *tcp_write_queue_next(struct sock *sk, struct sk_buff *skb)
1192{
1193 return skb->next;
1194}
1195
1196#define tcp_for_write_queue(skb, sk) \
1197 for (skb = (sk)->sk_write_queue.next; \
1198 (skb != (struct sk_buff *)&(sk)->sk_write_queue); \
1199 skb = skb->next)
1200
1201#define tcp_for_write_queue_from(skb, sk) \
1202 for (; (skb != (struct sk_buff *)&(sk)->sk_write_queue);\
1203 skb = skb->next)
1204
1205static inline struct sk_buff *tcp_send_head(struct sock *sk)
1206{
1207 return sk->sk_send_head;
1208}
1209
1210static inline void tcp_advance_send_head(struct sock *sk, struct sk_buff *skb)
1211{
1212 sk->sk_send_head = skb->next;
1213 if (sk->sk_send_head == (struct sk_buff *)&sk->sk_write_queue)
1214 sk->sk_send_head = NULL;
1215}
1216
1217static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unlinked)
1218{
1219 if (sk->sk_send_head == skb_unlinked)
1220 sk->sk_send_head = NULL;
1221}
1222
1223static inline void tcp_init_send_head(struct sock *sk)
1224{
1225 sk->sk_send_head = NULL;
1226}
1227
1228static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1229{
1230 __skb_queue_tail(&sk->sk_write_queue, skb);
1231}
1232
1233static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1234{
1235 __tcp_add_write_queue_tail(sk, skb);
1236
1237 /* Queue it, remembering where we must start sending. */
1238 if (sk->sk_send_head == NULL)
1239 sk->sk_send_head = skb;
1240}
1241
1242static inline void __tcp_add_write_queue_head(struct sock *sk, struct sk_buff *skb)
1243{
1244 __skb_queue_head(&sk->sk_write_queue, skb);
1245}
1246
1247/* Insert buff after skb on the write queue of sk. */
1248static inline void tcp_insert_write_queue_after(struct sk_buff *skb,
1249 struct sk_buff *buff,
1250 struct sock *sk)
1251{
1252 __skb_append(skb, buff, &sk->sk_write_queue);
1253}
1254
1255/* Insert skb between prev and next on the write queue of sk. */
1256static inline void tcp_insert_write_queue_before(struct sk_buff *new,
1257 struct sk_buff *skb,
1258 struct sock *sk)
1259{
1260 __skb_insert(new, skb->prev, skb, &sk->sk_write_queue);
1261}
1262
1263static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
1264{
1265 __skb_unlink(skb, &sk->sk_write_queue);
1266}
1267
1268static inline int tcp_skb_is_last(const struct sock *sk,
1269 const struct sk_buff *skb)
1270{
1271 return skb->next == (struct sk_buff *)&sk->sk_write_queue;
1272}
1273
1274static inline int tcp_write_queue_empty(struct sock *sk)
1275{
1276 return skb_queue_empty(&sk->sk_write_queue);
1277}
1278
1165/* /proc */ 1279/* /proc */
1166enum tcp_seq_states { 1280enum tcp_seq_states {
1167 TCP_SEQ_STATE_LISTENING, 1281 TCP_SEQ_STATE_LISTENING,
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 3834b10b5115..689f9330f1b9 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -470,10 +470,8 @@ static inline void skb_entail(struct sock *sk, struct tcp_sock *tp,
470 tcb->flags = TCPCB_FLAG_ACK; 470 tcb->flags = TCPCB_FLAG_ACK;
471 tcb->sacked = 0; 471 tcb->sacked = 0;
472 skb_header_release(skb); 472 skb_header_release(skb);
473 __skb_queue_tail(&sk->sk_write_queue, skb); 473 tcp_add_write_queue_tail(sk, skb);
474 sk_charge_skb(sk, skb); 474 sk_charge_skb(sk, skb);
475 if (!sk->sk_send_head)
476 sk->sk_send_head = skb;
477 if (tp->nonagle & TCP_NAGLE_PUSH) 475 if (tp->nonagle & TCP_NAGLE_PUSH)
478 tp->nonagle &= ~TCP_NAGLE_PUSH; 476 tp->nonagle &= ~TCP_NAGLE_PUSH;
479} 477}
@@ -491,8 +489,8 @@ static inline void tcp_mark_urg(struct tcp_sock *tp, int flags,
491static inline void tcp_push(struct sock *sk, struct tcp_sock *tp, int flags, 489static inline void tcp_push(struct sock *sk, struct tcp_sock *tp, int flags,
492 int mss_now, int nonagle) 490 int mss_now, int nonagle)
493{ 491{
494 if (sk->sk_send_head) { 492 if (tcp_send_head(sk)) {
495 struct sk_buff *skb = sk->sk_write_queue.prev; 493 struct sk_buff *skb = tcp_write_queue_tail(sk);
496 if (!(flags & MSG_MORE) || forced_push(tp)) 494 if (!(flags & MSG_MORE) || forced_push(tp))
497 tcp_mark_push(tp, skb); 495 tcp_mark_push(tp, skb);
498 tcp_mark_urg(tp, flags, skb); 496 tcp_mark_urg(tp, flags, skb);
@@ -526,13 +524,13 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
526 goto do_error; 524 goto do_error;
527 525
528 while (psize > 0) { 526 while (psize > 0) {
529 struct sk_buff *skb = sk->sk_write_queue.prev; 527 struct sk_buff *skb = tcp_write_queue_tail(sk);
530 struct page *page = pages[poffset / PAGE_SIZE]; 528 struct page *page = pages[poffset / PAGE_SIZE];
531 int copy, i, can_coalesce; 529 int copy, i, can_coalesce;
532 int offset = poffset % PAGE_SIZE; 530 int offset = poffset % PAGE_SIZE;
533 int size = min_t(size_t, psize, PAGE_SIZE - offset); 531 int size = min_t(size_t, psize, PAGE_SIZE - offset);
534 532
535 if (!sk->sk_send_head || (copy = size_goal - skb->len) <= 0) { 533 if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) {
536new_segment: 534new_segment:
537 if (!sk_stream_memory_free(sk)) 535 if (!sk_stream_memory_free(sk))
538 goto wait_for_sndbuf; 536 goto wait_for_sndbuf;
@@ -589,7 +587,7 @@ new_segment:
589 if (forced_push(tp)) { 587 if (forced_push(tp)) {
590 tcp_mark_push(tp, skb); 588 tcp_mark_push(tp, skb);
591 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); 589 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH);
592 } else if (skb == sk->sk_send_head) 590 } else if (skb == tcp_send_head(sk))
593 tcp_push_one(sk, mss_now); 591 tcp_push_one(sk, mss_now);
594 continue; 592 continue;
595 593
@@ -704,9 +702,9 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
704 while (seglen > 0) { 702 while (seglen > 0) {
705 int copy; 703 int copy;
706 704
707 skb = sk->sk_write_queue.prev; 705 skb = tcp_write_queue_tail(sk);
708 706
709 if (!sk->sk_send_head || 707 if (!tcp_send_head(sk) ||
710 (copy = size_goal - skb->len) <= 0) { 708 (copy = size_goal - skb->len) <= 0) {
711 709
712new_segment: 710new_segment:
@@ -833,7 +831,7 @@ new_segment:
833 if (forced_push(tp)) { 831 if (forced_push(tp)) {
834 tcp_mark_push(tp, skb); 832 tcp_mark_push(tp, skb);
835 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); 833 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH);
836 } else if (skb == sk->sk_send_head) 834 } else if (skb == tcp_send_head(sk))
837 tcp_push_one(sk, mss_now); 835 tcp_push_one(sk, mss_now);
838 continue; 836 continue;
839 837
@@ -860,9 +858,11 @@ out:
860 858
861do_fault: 859do_fault:
862 if (!skb->len) { 860 if (!skb->len) {
863 if (sk->sk_send_head == skb) 861 tcp_unlink_write_queue(skb, sk);
864 sk->sk_send_head = NULL; 862 /* It is the one place in all of TCP, except connection
865 __skb_unlink(skb, &sk->sk_write_queue); 863 * reset, where we can be unlinking the send_head.
864 */
865 tcp_check_send_head(sk, skb);
866 sk_stream_free_skb(sk, skb); 866 sk_stream_free_skb(sk, skb);
867 } 867 }
868 868
@@ -1732,7 +1732,7 @@ int tcp_disconnect(struct sock *sk, int flags)
1732 1732
1733 tcp_clear_xmit_timers(sk); 1733 tcp_clear_xmit_timers(sk);
1734 __skb_queue_purge(&sk->sk_receive_queue); 1734 __skb_queue_purge(&sk->sk_receive_queue);
1735 sk_stream_writequeue_purge(sk); 1735 tcp_write_queue_purge(sk);
1736 __skb_queue_purge(&tp->out_of_order_queue); 1736 __skb_queue_purge(&tp->out_of_order_queue);
1737#ifdef CONFIG_NET_DMA 1737#ifdef CONFIG_NET_DMA
1738 __skb_queue_purge(&sk->sk_async_wait_queue); 1738 __skb_queue_purge(&sk->sk_async_wait_queue);
@@ -1758,7 +1758,7 @@ int tcp_disconnect(struct sock *sk, int flags)
1758 tcp_set_ca_state(sk, TCP_CA_Open); 1758 tcp_set_ca_state(sk, TCP_CA_Open);
1759 tcp_clear_retrans(tp); 1759 tcp_clear_retrans(tp);
1760 inet_csk_delack_init(sk); 1760 inet_csk_delack_init(sk);
1761 sk->sk_send_head = NULL; 1761 tcp_init_send_head(sk);
1762 tp->rx_opt.saw_tstamp = 0; 1762 tp->rx_opt.saw_tstamp = 0;
1763 tcp_sack_reset(&tp->rx_opt); 1763 tcp_sack_reset(&tp->rx_opt);
1764 __sk_dst_reset(sk); 1764 __sk_dst_reset(sk);
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index d0a3630f41a7..22d0bb03c5da 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -1044,7 +1044,7 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_
1044 cached_skb = tp->fastpath_skb_hint; 1044 cached_skb = tp->fastpath_skb_hint;
1045 cached_fack_count = tp->fastpath_cnt_hint; 1045 cached_fack_count = tp->fastpath_cnt_hint;
1046 if (!cached_skb) { 1046 if (!cached_skb) {
1047 cached_skb = sk->sk_write_queue.next; 1047 cached_skb = tcp_write_queue_head(sk);
1048 cached_fack_count = 0; 1048 cached_fack_count = 0;
1049 } 1049 }
1050 1050
@@ -1061,10 +1061,13 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_
1061 if (after(end_seq, tp->high_seq)) 1061 if (after(end_seq, tp->high_seq))
1062 flag |= FLAG_DATA_LOST; 1062 flag |= FLAG_DATA_LOST;
1063 1063
1064 sk_stream_for_retrans_queue_from(skb, sk) { 1064 tcp_for_write_queue_from(skb, sk) {
1065 int in_sack, pcount; 1065 int in_sack, pcount;
1066 u8 sacked; 1066 u8 sacked;
1067 1067
1068 if (skb == tcp_send_head(sk))
1069 break;
1070
1068 cached_skb = skb; 1071 cached_skb = skb;
1069 cached_fack_count = fack_count; 1072 cached_fack_count = fack_count;
1070 if (i == first_sack_index) { 1073 if (i == first_sack_index) {
@@ -1213,7 +1216,9 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_
1213 if (lost_retrans && icsk->icsk_ca_state == TCP_CA_Recovery) { 1216 if (lost_retrans && icsk->icsk_ca_state == TCP_CA_Recovery) {
1214 struct sk_buff *skb; 1217 struct sk_buff *skb;
1215 1218
1216 sk_stream_for_retrans_queue(skb, sk) { 1219 tcp_for_write_queue(skb, sk) {
1220 if (skb == tcp_send_head(sk))
1221 break;
1217 if (after(TCP_SKB_CB(skb)->seq, lost_retrans)) 1222 if (after(TCP_SKB_CB(skb)->seq, lost_retrans))
1218 break; 1223 break;
1219 if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) 1224 if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
@@ -1266,8 +1271,8 @@ int tcp_use_frto(struct sock *sk)
1266 const struct tcp_sock *tp = tcp_sk(sk); 1271 const struct tcp_sock *tp = tcp_sk(sk);
1267 struct sk_buff *skb; 1272 struct sk_buff *skb;
1268 1273
1269 if (!sysctl_tcp_frto || !sk->sk_send_head || 1274 if (!sysctl_tcp_frto || !tcp_send_head(sk) ||
1270 after(TCP_SKB_CB(sk->sk_send_head)->end_seq, 1275 after(TCP_SKB_CB(tcp_send_head(sk))->end_seq,
1271 tp->snd_una + tp->snd_wnd)) 1276 tp->snd_una + tp->snd_wnd))
1272 return 0; 1277 return 0;
1273 1278
@@ -1278,8 +1283,11 @@ int tcp_use_frto(struct sock *sk)
1278 if (tp->retrans_out > 1) 1283 if (tp->retrans_out > 1)
1279 return 0; 1284 return 0;
1280 1285
1281 skb = skb_peek(&sk->sk_write_queue)->next; /* Skips head */ 1286 skb = tcp_write_queue_head(sk);
1282 sk_stream_for_retrans_queue_from(skb, sk) { 1287 skb = tcp_write_queue_next(sk, skb); /* Skips head */
1288 tcp_for_write_queue_from(skb, sk) {
1289 if (skb == tcp_send_head(sk))
1290 break;
1283 if (TCP_SKB_CB(skb)->sacked&TCPCB_RETRANS) 1291 if (TCP_SKB_CB(skb)->sacked&TCPCB_RETRANS)
1284 return 0; 1292 return 0;
1285 /* Short-circuit when first non-SACKed skb has been checked */ 1293 /* Short-circuit when first non-SACKed skb has been checked */
@@ -1343,7 +1351,7 @@ void tcp_enter_frto(struct sock *sk)
1343 tp->undo_marker = tp->snd_una; 1351 tp->undo_marker = tp->snd_una;
1344 tp->undo_retrans = 0; 1352 tp->undo_retrans = 0;
1345 1353
1346 skb = skb_peek(&sk->sk_write_queue); 1354 skb = tcp_write_queue_head(sk);
1347 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { 1355 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
1348 TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS; 1356 TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
1349 tp->retrans_out -= tcp_skb_pcount(skb); 1357 tp->retrans_out -= tcp_skb_pcount(skb);
@@ -1380,7 +1388,9 @@ static void tcp_enter_frto_loss(struct sock *sk, int allowed_segments, int flag)
1380 tp->fackets_out = 0; 1388 tp->fackets_out = 0;
1381 tp->retrans_out = 0; 1389 tp->retrans_out = 0;
1382 1390
1383 sk_stream_for_retrans_queue(skb, sk) { 1391 tcp_for_write_queue(skb, sk) {
1392 if (skb == tcp_send_head(sk))
1393 break;
1384 cnt += tcp_skb_pcount(skb); 1394 cnt += tcp_skb_pcount(skb);
1385 /* 1395 /*
1386 * Count the retransmission made on RTO correctly (only when 1396 * Count the retransmission made on RTO correctly (only when
@@ -1468,7 +1478,9 @@ void tcp_enter_loss(struct sock *sk, int how)
1468 if (!how) 1478 if (!how)
1469 tp->undo_marker = tp->snd_una; 1479 tp->undo_marker = tp->snd_una;
1470 1480
1471 sk_stream_for_retrans_queue(skb, sk) { 1481 tcp_for_write_queue(skb, sk) {
1482 if (skb == tcp_send_head(sk))
1483 break;
1472 cnt += tcp_skb_pcount(skb); 1484 cnt += tcp_skb_pcount(skb);
1473 if (TCP_SKB_CB(skb)->sacked&TCPCB_RETRANS) 1485 if (TCP_SKB_CB(skb)->sacked&TCPCB_RETRANS)
1474 tp->undo_marker = 0; 1486 tp->undo_marker = 0;
@@ -1503,14 +1515,14 @@ static int tcp_check_sack_reneging(struct sock *sk)
1503 * receiver _host_ is heavily congested (or buggy). 1515 * receiver _host_ is heavily congested (or buggy).
1504 * Do processing similar to RTO timeout. 1516 * Do processing similar to RTO timeout.
1505 */ 1517 */
1506 if ((skb = skb_peek(&sk->sk_write_queue)) != NULL && 1518 if ((skb = tcp_write_queue_head(sk)) != NULL &&
1507 (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) { 1519 (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) {
1508 struct inet_connection_sock *icsk = inet_csk(sk); 1520 struct inet_connection_sock *icsk = inet_csk(sk);
1509 NET_INC_STATS_BH(LINUX_MIB_TCPSACKRENEGING); 1521 NET_INC_STATS_BH(LINUX_MIB_TCPSACKRENEGING);
1510 1522
1511 tcp_enter_loss(sk, 1); 1523 tcp_enter_loss(sk, 1);
1512 icsk->icsk_retransmits++; 1524 icsk->icsk_retransmits++;
1513 tcp_retransmit_skb(sk, skb_peek(&sk->sk_write_queue)); 1525 tcp_retransmit_skb(sk, tcp_write_queue_head(sk));
1514 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 1526 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
1515 icsk->icsk_rto, TCP_RTO_MAX); 1527 icsk->icsk_rto, TCP_RTO_MAX);
1516 return 1; 1528 return 1;
@@ -1531,7 +1543,7 @@ static inline int tcp_skb_timedout(struct sock *sk, struct sk_buff *skb)
1531static inline int tcp_head_timedout(struct sock *sk, struct tcp_sock *tp) 1543static inline int tcp_head_timedout(struct sock *sk, struct tcp_sock *tp)
1532{ 1544{
1533 return tp->packets_out && 1545 return tp->packets_out &&
1534 tcp_skb_timedout(sk, skb_peek(&sk->sk_write_queue)); 1546 tcp_skb_timedout(sk, tcp_write_queue_head(sk));
1535} 1547}
1536 1548
1537/* Linux NewReno/SACK/FACK/ECN state machine. 1549/* Linux NewReno/SACK/FACK/ECN state machine.
@@ -1726,11 +1738,13 @@ static void tcp_mark_head_lost(struct sock *sk, struct tcp_sock *tp,
1726 skb = tp->lost_skb_hint; 1738 skb = tp->lost_skb_hint;
1727 cnt = tp->lost_cnt_hint; 1739 cnt = tp->lost_cnt_hint;
1728 } else { 1740 } else {
1729 skb = sk->sk_write_queue.next; 1741 skb = tcp_write_queue_head(sk);
1730 cnt = 0; 1742 cnt = 0;
1731 } 1743 }
1732 1744
1733 sk_stream_for_retrans_queue_from(skb, sk) { 1745 tcp_for_write_queue_from(skb, sk) {
1746 if (skb == tcp_send_head(sk))
1747 break;
1734 /* TODO: do this better */ 1748 /* TODO: do this better */
1735 /* this is not the most efficient way to do this... */ 1749 /* this is not the most efficient way to do this... */
1736 tp->lost_skb_hint = skb; 1750 tp->lost_skb_hint = skb;
@@ -1777,9 +1791,11 @@ static void tcp_update_scoreboard(struct sock *sk, struct tcp_sock *tp)
1777 struct sk_buff *skb; 1791 struct sk_buff *skb;
1778 1792
1779 skb = tp->scoreboard_skb_hint ? tp->scoreboard_skb_hint 1793 skb = tp->scoreboard_skb_hint ? tp->scoreboard_skb_hint
1780 : sk->sk_write_queue.next; 1794 : tcp_write_queue_head(sk);
1781 1795
1782 sk_stream_for_retrans_queue_from(skb, sk) { 1796 tcp_for_write_queue_from(skb, sk) {
1797 if (skb == tcp_send_head(sk))
1798 break;
1783 if (!tcp_skb_timedout(sk, skb)) 1799 if (!tcp_skb_timedout(sk, skb))
1784 break; 1800 break;
1785 1801
@@ -1970,7 +1986,9 @@ static int tcp_try_undo_loss(struct sock *sk, struct tcp_sock *tp)
1970{ 1986{
1971 if (tcp_may_undo(tp)) { 1987 if (tcp_may_undo(tp)) {
1972 struct sk_buff *skb; 1988 struct sk_buff *skb;
1973 sk_stream_for_retrans_queue(skb, sk) { 1989 tcp_for_write_queue(skb, sk) {
1990 if (skb == tcp_send_head(sk))
1991 break;
1974 TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST; 1992 TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST;
1975 } 1993 }
1976 1994
@@ -2382,8 +2400,8 @@ static int tcp_clean_rtx_queue(struct sock *sk, __s32 *seq_rtt_p)
2382 = icsk->icsk_ca_ops->rtt_sample; 2400 = icsk->icsk_ca_ops->rtt_sample;
2383 struct timeval tv = { .tv_sec = 0, .tv_usec = 0 }; 2401 struct timeval tv = { .tv_sec = 0, .tv_usec = 0 };
2384 2402
2385 while ((skb = skb_peek(&sk->sk_write_queue)) && 2403 while ((skb = tcp_write_queue_head(sk)) &&
2386 skb != sk->sk_send_head) { 2404 skb != tcp_send_head(sk)) {
2387 struct tcp_skb_cb *scb = TCP_SKB_CB(skb); 2405 struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
2388 __u8 sacked = scb->sacked; 2406 __u8 sacked = scb->sacked;
2389 2407
@@ -2446,7 +2464,7 @@ static int tcp_clean_rtx_queue(struct sock *sk, __s32 *seq_rtt_p)
2446 } 2464 }
2447 tcp_dec_pcount_approx(&tp->fackets_out, skb); 2465 tcp_dec_pcount_approx(&tp->fackets_out, skb);
2448 tcp_packets_out_dec(tp, skb); 2466 tcp_packets_out_dec(tp, skb);
2449 __skb_unlink(skb, &sk->sk_write_queue); 2467 tcp_unlink_write_queue(skb, sk);
2450 sk_stream_free_skb(sk, skb); 2468 sk_stream_free_skb(sk, skb);
2451 clear_all_retrans_hints(tp); 2469 clear_all_retrans_hints(tp);
2452 } 2470 }
@@ -2495,7 +2513,7 @@ static void tcp_ack_probe(struct sock *sk)
2495 2513
2496 /* Was it a usable window open? */ 2514 /* Was it a usable window open? */
2497 2515
2498 if (!after(TCP_SKB_CB(sk->sk_send_head)->end_seq, 2516 if (!after(TCP_SKB_CB(tcp_send_head(sk))->end_seq,
2499 tp->snd_una + tp->snd_wnd)) { 2517 tp->snd_una + tp->snd_wnd)) {
2500 icsk->icsk_backoff = 0; 2518 icsk->icsk_backoff = 0;
2501 inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0); 2519 inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0);
@@ -2795,7 +2813,7 @@ no_queue:
2795 * being used to time the probes, and is probably far higher than 2813 * being used to time the probes, and is probably far higher than
2796 * it needs to be for normal retransmission. 2814 * it needs to be for normal retransmission.
2797 */ 2815 */
2798 if (sk->sk_send_head) 2816 if (tcp_send_head(sk))
2799 tcp_ack_probe(sk); 2817 tcp_ack_probe(sk);
2800 return 1; 2818 return 1;
2801 2819
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index addac1110f94..3326681b8429 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -1890,7 +1890,7 @@ int tcp_v4_destroy_sock(struct sock *sk)
1890 tcp_cleanup_congestion_control(sk); 1890 tcp_cleanup_congestion_control(sk);
1891 1891
1892 /* Cleanup up the write buffer. */ 1892 /* Cleanup up the write buffer. */
1893 sk_stream_writequeue_purge(sk); 1893 tcp_write_queue_purge(sk);
1894 1894
1895 /* Cleans up our, hopefully empty, out_of_order_queue. */ 1895 /* Cleans up our, hopefully empty, out_of_order_queue. */
1896 __skb_queue_purge(&tp->out_of_order_queue); 1896 __skb_queue_purge(&tp->out_of_order_queue);
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index d19b2f3b70fd..2a62b55b15f1 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -65,9 +65,7 @@ int sysctl_tcp_slow_start_after_idle __read_mostly = 1;
65static void update_send_head(struct sock *sk, struct tcp_sock *tp, 65static void update_send_head(struct sock *sk, struct tcp_sock *tp,
66 struct sk_buff *skb) 66 struct sk_buff *skb)
67{ 67{
68 sk->sk_send_head = skb->next; 68 tcp_advance_send_head(sk, skb);
69 if (sk->sk_send_head == (struct sk_buff *)&sk->sk_write_queue)
70 sk->sk_send_head = NULL;
71 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq; 69 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;
72 tcp_packets_out_inc(sk, tp, skb); 70 tcp_packets_out_inc(sk, tp, skb);
73} 71}
@@ -567,12 +565,8 @@ static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
567 /* Advance write_seq and place onto the write_queue. */ 565 /* Advance write_seq and place onto the write_queue. */
568 tp->write_seq = TCP_SKB_CB(skb)->end_seq; 566 tp->write_seq = TCP_SKB_CB(skb)->end_seq;
569 skb_header_release(skb); 567 skb_header_release(skb);
570 __skb_queue_tail(&sk->sk_write_queue, skb); 568 tcp_add_write_queue_tail(sk, skb);
571 sk_charge_skb(sk, skb); 569 sk_charge_skb(sk, skb);
572
573 /* Queue it, remembering where we must start sending. */
574 if (sk->sk_send_head == NULL)
575 sk->sk_send_head = skb;
576} 570}
577 571
578static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, unsigned int mss_now) 572static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, unsigned int mss_now)
@@ -705,7 +699,7 @@ int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len, unsigned int mss
705 699
706 /* Link BUFF into the send queue. */ 700 /* Link BUFF into the send queue. */
707 skb_header_release(buff); 701 skb_header_release(buff);
708 __skb_append(skb, buff, &sk->sk_write_queue); 702 tcp_insert_write_queue_after(skb, buff, sk);
709 703
710 return 0; 704 return 0;
711} 705}
@@ -1056,7 +1050,7 @@ static inline int tcp_snd_wnd_test(struct tcp_sock *tp, struct sk_buff *skb, uns
1056 return !after(end_seq, tp->snd_una + tp->snd_wnd); 1050 return !after(end_seq, tp->snd_una + tp->snd_wnd);
1057} 1051}
1058 1052
1059/* This checks if the data bearing packet SKB (usually sk->sk_send_head) 1053/* This checks if the data bearing packet SKB (usually tcp_send_head(sk))
1060 * should be put on the wire right now. If so, it returns the number of 1054 * should be put on the wire right now. If so, it returns the number of
1061 * packets allowed by the congestion window. 1055 * packets allowed by the congestion window.
1062 */ 1056 */
@@ -1079,15 +1073,9 @@ static unsigned int tcp_snd_test(struct sock *sk, struct sk_buff *skb,
1079 return cwnd_quota; 1073 return cwnd_quota;
1080} 1074}
1081 1075
1082static inline int tcp_skb_is_last(const struct sock *sk,
1083 const struct sk_buff *skb)
1084{
1085 return skb->next == (struct sk_buff *)&sk->sk_write_queue;
1086}
1087
1088int tcp_may_send_now(struct sock *sk, struct tcp_sock *tp) 1076int tcp_may_send_now(struct sock *sk, struct tcp_sock *tp)
1089{ 1077{
1090 struct sk_buff *skb = sk->sk_send_head; 1078 struct sk_buff *skb = tcp_send_head(sk);
1091 1079
1092 return (skb && 1080 return (skb &&
1093 tcp_snd_test(sk, skb, tcp_current_mss(sk, 1), 1081 tcp_snd_test(sk, skb, tcp_current_mss(sk, 1),
@@ -1143,7 +1131,7 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
1143 1131
1144 /* Link BUFF into the send queue. */ 1132 /* Link BUFF into the send queue. */
1145 skb_header_release(buff); 1133 skb_header_release(buff);
1146 __skb_append(skb, buff, &sk->sk_write_queue); 1134 tcp_insert_write_queue_after(skb, buff, sk);
1147 1135
1148 return 0; 1136 return 0;
1149} 1137}
@@ -1249,10 +1237,10 @@ static int tcp_mtu_probe(struct sock *sk)
1249 1237
1250 /* Have enough data in the send queue to probe? */ 1238 /* Have enough data in the send queue to probe? */
1251 len = 0; 1239 len = 0;
1252 if ((skb = sk->sk_send_head) == NULL) 1240 if ((skb = tcp_send_head(sk)) == NULL)
1253 return -1; 1241 return -1;
1254 while ((len += skb->len) < probe_size && !tcp_skb_is_last(sk, skb)) 1242 while ((len += skb->len) < probe_size && !tcp_skb_is_last(sk, skb))
1255 skb = skb->next; 1243 skb = tcp_write_queue_next(sk, skb);
1256 if (len < probe_size) 1244 if (len < probe_size)
1257 return -1; 1245 return -1;
1258 1246
@@ -1279,9 +1267,9 @@ static int tcp_mtu_probe(struct sock *sk)
1279 return -1; 1267 return -1;
1280 sk_charge_skb(sk, nskb); 1268 sk_charge_skb(sk, nskb);
1281 1269
1282 skb = sk->sk_send_head; 1270 skb = tcp_send_head(sk);
1283 __skb_insert(nskb, skb->prev, skb, &sk->sk_write_queue); 1271 tcp_insert_write_queue_before(nskb, skb, sk);
1284 sk->sk_send_head = nskb; 1272 tcp_advance_send_head(sk, skb);
1285 1273
1286 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; 1274 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
1287 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; 1275 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
@@ -1292,7 +1280,7 @@ static int tcp_mtu_probe(struct sock *sk)
1292 1280
1293 len = 0; 1281 len = 0;
1294 while (len < probe_size) { 1282 while (len < probe_size) {
1295 next = skb->next; 1283 next = tcp_write_queue_next(sk, skb);
1296 1284
1297 copy = min_t(int, skb->len, probe_size - len); 1285 copy = min_t(int, skb->len, probe_size - len);
1298 if (nskb->ip_summed) 1286 if (nskb->ip_summed)
@@ -1305,7 +1293,7 @@ static int tcp_mtu_probe(struct sock *sk)
1305 /* We've eaten all the data from this skb. 1293 /* We've eaten all the data from this skb.
1306 * Throw it away. */ 1294 * Throw it away. */
1307 TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags; 1295 TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags;
1308 __skb_unlink(skb, &sk->sk_write_queue); 1296 tcp_unlink_write_queue(skb, sk);
1309 sk_stream_free_skb(sk, skb); 1297 sk_stream_free_skb(sk, skb);
1310 } else { 1298 } else {
1311 TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags & 1299 TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags &
@@ -1377,7 +1365,7 @@ static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle)
1377 sent_pkts = 1; 1365 sent_pkts = 1;
1378 } 1366 }
1379 1367
1380 while ((skb = sk->sk_send_head)) { 1368 while ((skb = tcp_send_head(sk))) {
1381 unsigned int limit; 1369 unsigned int limit;
1382 1370
1383 tso_segs = tcp_init_tso_segs(sk, skb, mss_now); 1371 tso_segs = tcp_init_tso_segs(sk, skb, mss_now);
@@ -1435,7 +1423,7 @@ static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle)
1435 tcp_cwnd_validate(sk, tp); 1423 tcp_cwnd_validate(sk, tp);
1436 return 0; 1424 return 0;
1437 } 1425 }
1438 return !tp->packets_out && sk->sk_send_head; 1426 return !tp->packets_out && tcp_send_head(sk);
1439} 1427}
1440 1428
1441/* Push out any pending frames which were held back due to 1429/* Push out any pending frames which were held back due to
@@ -1445,7 +1433,7 @@ static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle)
1445void __tcp_push_pending_frames(struct sock *sk, struct tcp_sock *tp, 1433void __tcp_push_pending_frames(struct sock *sk, struct tcp_sock *tp,
1446 unsigned int cur_mss, int nonagle) 1434 unsigned int cur_mss, int nonagle)
1447{ 1435{
1448 struct sk_buff *skb = sk->sk_send_head; 1436 struct sk_buff *skb = tcp_send_head(sk);
1449 1437
1450 if (skb) { 1438 if (skb) {
1451 if (tcp_write_xmit(sk, cur_mss, nonagle)) 1439 if (tcp_write_xmit(sk, cur_mss, nonagle))
@@ -1459,7 +1447,7 @@ void __tcp_push_pending_frames(struct sock *sk, struct tcp_sock *tp,
1459void tcp_push_one(struct sock *sk, unsigned int mss_now) 1447void tcp_push_one(struct sock *sk, unsigned int mss_now)
1460{ 1448{
1461 struct tcp_sock *tp = tcp_sk(sk); 1449 struct tcp_sock *tp = tcp_sk(sk);
1462 struct sk_buff *skb = sk->sk_send_head; 1450 struct sk_buff *skb = tcp_send_head(sk);
1463 unsigned int tso_segs, cwnd_quota; 1451 unsigned int tso_segs, cwnd_quota;
1464 1452
1465 BUG_ON(!skb || skb->len < mss_now); 1453 BUG_ON(!skb || skb->len < mss_now);
@@ -1620,7 +1608,7 @@ u32 __tcp_select_window(struct sock *sk)
1620static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, int mss_now) 1608static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, int mss_now)
1621{ 1609{
1622 struct tcp_sock *tp = tcp_sk(sk); 1610 struct tcp_sock *tp = tcp_sk(sk);
1623 struct sk_buff *next_skb = skb->next; 1611 struct sk_buff *next_skb = tcp_write_queue_next(sk, skb);
1624 1612
1625 /* The first test we must make is that neither of these two 1613 /* The first test we must make is that neither of these two
1626 * SKB's are still referenced by someone else. 1614 * SKB's are still referenced by someone else.
@@ -1652,7 +1640,7 @@ static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, int m
1652 clear_all_retrans_hints(tp); 1640 clear_all_retrans_hints(tp);
1653 1641
1654 /* Ok. We will be able to collapse the packet. */ 1642 /* Ok. We will be able to collapse the packet. */
1655 __skb_unlink(next_skb, &sk->sk_write_queue); 1643 tcp_unlink_write_queue(next_skb, sk);
1656 1644
1657 memcpy(skb_put(skb, next_skb_size), next_skb->data, next_skb_size); 1645 memcpy(skb_put(skb, next_skb_size), next_skb->data, next_skb_size);
1658 1646
@@ -1706,7 +1694,9 @@ void tcp_simple_retransmit(struct sock *sk)
1706 unsigned int mss = tcp_current_mss(sk, 0); 1694 unsigned int mss = tcp_current_mss(sk, 0);
1707 int lost = 0; 1695 int lost = 0;
1708 1696
1709 sk_stream_for_retrans_queue(skb, sk) { 1697 tcp_for_write_queue(skb, sk) {
1698 if (skb == tcp_send_head(sk))
1699 break;
1710 if (skb->len > mss && 1700 if (skb->len > mss &&
1711 !(TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_ACKED)) { 1701 !(TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_ACKED)) {
1712 if (TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_RETRANS) { 1702 if (TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_RETRANS) {
@@ -1790,10 +1780,10 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
1790 /* Collapse two adjacent packets if worthwhile and we can. */ 1780 /* Collapse two adjacent packets if worthwhile and we can. */
1791 if(!(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN) && 1781 if(!(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN) &&
1792 (skb->len < (cur_mss >> 1)) && 1782 (skb->len < (cur_mss >> 1)) &&
1793 (skb->next != sk->sk_send_head) && 1783 (tcp_write_queue_next(sk, skb) != tcp_send_head(sk)) &&
1794 (skb->next != (struct sk_buff *)&sk->sk_write_queue) && 1784 (!tcp_skb_is_last(sk, skb)) &&
1795 (skb_shinfo(skb)->nr_frags == 0 && skb_shinfo(skb->next)->nr_frags == 0) && 1785 (skb_shinfo(skb)->nr_frags == 0 && skb_shinfo(tcp_write_queue_next(sk, skb))->nr_frags == 0) &&
1796 (tcp_skb_pcount(skb) == 1 && tcp_skb_pcount(skb->next) == 1) && 1786 (tcp_skb_pcount(skb) == 1 && tcp_skb_pcount(tcp_write_queue_next(sk, skb)) == 1) &&
1797 (sysctl_tcp_retrans_collapse != 0)) 1787 (sysctl_tcp_retrans_collapse != 0))
1798 tcp_retrans_try_collapse(sk, skb, cur_mss); 1788 tcp_retrans_try_collapse(sk, skb, cur_mss);
1799 1789
@@ -1872,15 +1862,17 @@ void tcp_xmit_retransmit_queue(struct sock *sk)
1872 skb = tp->retransmit_skb_hint; 1862 skb = tp->retransmit_skb_hint;
1873 packet_cnt = tp->retransmit_cnt_hint; 1863 packet_cnt = tp->retransmit_cnt_hint;
1874 }else{ 1864 }else{
1875 skb = sk->sk_write_queue.next; 1865 skb = tcp_write_queue_head(sk);
1876 packet_cnt = 0; 1866 packet_cnt = 0;
1877 } 1867 }
1878 1868
1879 /* First pass: retransmit lost packets. */ 1869 /* First pass: retransmit lost packets. */
1880 if (tp->lost_out) { 1870 if (tp->lost_out) {
1881 sk_stream_for_retrans_queue_from(skb, sk) { 1871 tcp_for_write_queue_from(skb, sk) {
1882 __u8 sacked = TCP_SKB_CB(skb)->sacked; 1872 __u8 sacked = TCP_SKB_CB(skb)->sacked;
1883 1873
1874 if (skb == tcp_send_head(sk))
1875 break;
1884 /* we could do better than to assign each time */ 1876 /* we could do better than to assign each time */
1885 tp->retransmit_skb_hint = skb; 1877 tp->retransmit_skb_hint = skb;
1886 tp->retransmit_cnt_hint = packet_cnt; 1878 tp->retransmit_cnt_hint = packet_cnt;
@@ -1906,8 +1898,7 @@ void tcp_xmit_retransmit_queue(struct sock *sk)
1906 else 1898 else
1907 NET_INC_STATS_BH(LINUX_MIB_TCPSLOWSTARTRETRANS); 1899 NET_INC_STATS_BH(LINUX_MIB_TCPSLOWSTARTRETRANS);
1908 1900
1909 if (skb == 1901 if (skb == tcp_write_queue_head(sk))
1910 skb_peek(&sk->sk_write_queue))
1911 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 1902 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
1912 inet_csk(sk)->icsk_rto, 1903 inet_csk(sk)->icsk_rto,
1913 TCP_RTO_MAX); 1904 TCP_RTO_MAX);
@@ -1944,11 +1935,13 @@ void tcp_xmit_retransmit_queue(struct sock *sk)
1944 skb = tp->forward_skb_hint; 1935 skb = tp->forward_skb_hint;
1945 packet_cnt = tp->forward_cnt_hint; 1936 packet_cnt = tp->forward_cnt_hint;
1946 } else{ 1937 } else{
1947 skb = sk->sk_write_queue.next; 1938 skb = tcp_write_queue_head(sk);
1948 packet_cnt = 0; 1939 packet_cnt = 0;
1949 } 1940 }
1950 1941
1951 sk_stream_for_retrans_queue_from(skb, sk) { 1942 tcp_for_write_queue_from(skb, sk) {
1943 if (skb == tcp_send_head(sk))
1944 break;
1952 tp->forward_cnt_hint = packet_cnt; 1945 tp->forward_cnt_hint = packet_cnt;
1953 tp->forward_skb_hint = skb; 1946 tp->forward_skb_hint = skb;
1954 1947
@@ -1973,7 +1966,7 @@ void tcp_xmit_retransmit_queue(struct sock *sk)
1973 break; 1966 break;
1974 } 1967 }
1975 1968
1976 if (skb == skb_peek(&sk->sk_write_queue)) 1969 if (skb == tcp_write_queue_head(sk))
1977 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 1970 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
1978 inet_csk(sk)->icsk_rto, 1971 inet_csk(sk)->icsk_rto,
1979 TCP_RTO_MAX); 1972 TCP_RTO_MAX);
@@ -1989,7 +1982,7 @@ void tcp_xmit_retransmit_queue(struct sock *sk)
1989void tcp_send_fin(struct sock *sk) 1982void tcp_send_fin(struct sock *sk)
1990{ 1983{
1991 struct tcp_sock *tp = tcp_sk(sk); 1984 struct tcp_sock *tp = tcp_sk(sk);
1992 struct sk_buff *skb = skb_peek_tail(&sk->sk_write_queue); 1985 struct sk_buff *skb = tcp_write_queue_tail(sk);
1993 int mss_now; 1986 int mss_now;
1994 1987
1995 /* Optimization, tack on the FIN if we have a queue of 1988 /* Optimization, tack on the FIN if we have a queue of
@@ -1998,7 +1991,7 @@ void tcp_send_fin(struct sock *sk)
1998 */ 1991 */
1999 mss_now = tcp_current_mss(sk, 1); 1992 mss_now = tcp_current_mss(sk, 1);
2000 1993
2001 if (sk->sk_send_head != NULL) { 1994 if (tcp_send_head(sk) != NULL) {
2002 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN; 1995 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN;
2003 TCP_SKB_CB(skb)->end_seq++; 1996 TCP_SKB_CB(skb)->end_seq++;
2004 tp->write_seq++; 1997 tp->write_seq++;
@@ -2071,7 +2064,7 @@ int tcp_send_synack(struct sock *sk)
2071{ 2064{
2072 struct sk_buff* skb; 2065 struct sk_buff* skb;
2073 2066
2074 skb = skb_peek(&sk->sk_write_queue); 2067 skb = tcp_write_queue_head(sk);
2075 if (skb == NULL || !(TCP_SKB_CB(skb)->flags&TCPCB_FLAG_SYN)) { 2068 if (skb == NULL || !(TCP_SKB_CB(skb)->flags&TCPCB_FLAG_SYN)) {
2076 printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n"); 2069 printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n");
2077 return -EFAULT; 2070 return -EFAULT;
@@ -2081,9 +2074,9 @@ int tcp_send_synack(struct sock *sk)
2081 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); 2074 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
2082 if (nskb == NULL) 2075 if (nskb == NULL)
2083 return -ENOMEM; 2076 return -ENOMEM;
2084 __skb_unlink(skb, &sk->sk_write_queue); 2077 tcp_unlink_write_queue(skb, sk);
2085 skb_header_release(nskb); 2078 skb_header_release(nskb);
2086 __skb_queue_head(&sk->sk_write_queue, nskb); 2079 __tcp_add_write_queue_head(sk, nskb);
2087 sk_stream_free_skb(sk, skb); 2080 sk_stream_free_skb(sk, skb);
2088 sk_charge_skb(sk, nskb); 2081 sk_charge_skb(sk, nskb);
2089 skb = nskb; 2082 skb = nskb;
@@ -2285,7 +2278,7 @@ int tcp_connect(struct sock *sk)
2285 TCP_SKB_CB(buff)->when = tcp_time_stamp; 2278 TCP_SKB_CB(buff)->when = tcp_time_stamp;
2286 tp->retrans_stamp = TCP_SKB_CB(buff)->when; 2279 tp->retrans_stamp = TCP_SKB_CB(buff)->when;
2287 skb_header_release(buff); 2280 skb_header_release(buff);
2288 __skb_queue_tail(&sk->sk_write_queue, buff); 2281 __tcp_add_write_queue_tail(sk, buff);
2289 sk_charge_skb(sk, buff); 2282 sk_charge_skb(sk, buff);
2290 tp->packets_out += tcp_skb_pcount(buff); 2283 tp->packets_out += tcp_skb_pcount(buff);
2291 tcp_transmit_skb(sk, buff, 1, GFP_KERNEL); 2284 tcp_transmit_skb(sk, buff, 1, GFP_KERNEL);
@@ -2441,7 +2434,7 @@ int tcp_write_wakeup(struct sock *sk)
2441 struct tcp_sock *tp = tcp_sk(sk); 2434 struct tcp_sock *tp = tcp_sk(sk);
2442 struct sk_buff *skb; 2435 struct sk_buff *skb;
2443 2436
2444 if ((skb = sk->sk_send_head) != NULL && 2437 if ((skb = tcp_send_head(sk)) != NULL &&
2445 before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) { 2438 before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) {
2446 int err; 2439 int err;
2447 unsigned int mss = tcp_current_mss(sk, 0); 2440 unsigned int mss = tcp_current_mss(sk, 0);
@@ -2491,7 +2484,7 @@ void tcp_send_probe0(struct sock *sk)
2491 2484
2492 err = tcp_write_wakeup(sk); 2485 err = tcp_write_wakeup(sk);
2493 2486
2494 if (tp->packets_out || !sk->sk_send_head) { 2487 if (tp->packets_out || !tcp_send_head(sk)) {
2495 /* Cancel probe timer, if it is not required. */ 2488 /* Cancel probe timer, if it is not required. */
2496 icsk->icsk_probes_out = 0; 2489 icsk->icsk_probes_out = 0;
2497 icsk->icsk_backoff = 0; 2490 icsk->icsk_backoff = 0;
diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c
index a9243cfc1bea..2ca97b20929d 100644
--- a/net/ipv4/tcp_timer.c
+++ b/net/ipv4/tcp_timer.c
@@ -233,7 +233,7 @@ static void tcp_probe_timer(struct sock *sk)
233 struct tcp_sock *tp = tcp_sk(sk); 233 struct tcp_sock *tp = tcp_sk(sk);
234 int max_probes; 234 int max_probes;
235 235
236 if (tp->packets_out || !sk->sk_send_head) { 236 if (tp->packets_out || !tcp_send_head(sk)) {
237 icsk->icsk_probes_out = 0; 237 icsk->icsk_probes_out = 0;
238 return; 238 return;
239 } 239 }
@@ -284,7 +284,7 @@ static void tcp_retransmit_timer(struct sock *sk)
284 if (!tp->packets_out) 284 if (!tp->packets_out)
285 goto out; 285 goto out;
286 286
287 BUG_TRAP(!skb_queue_empty(&sk->sk_write_queue)); 287 BUG_TRAP(!tcp_write_queue_empty(sk));
288 288
289 if (!tp->snd_wnd && !sock_flag(sk, SOCK_DEAD) && 289 if (!tp->snd_wnd && !sock_flag(sk, SOCK_DEAD) &&
290 !((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))) { 290 !((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))) {
@@ -306,7 +306,7 @@ static void tcp_retransmit_timer(struct sock *sk)
306 goto out; 306 goto out;
307 } 307 }
308 tcp_enter_loss(sk, 0); 308 tcp_enter_loss(sk, 0);
309 tcp_retransmit_skb(sk, skb_peek(&sk->sk_write_queue)); 309 tcp_retransmit_skb(sk, tcp_write_queue_head(sk));
310 __sk_dst_reset(sk); 310 __sk_dst_reset(sk);
311 goto out_reset_timer; 311 goto out_reset_timer;
312 } 312 }
@@ -341,7 +341,7 @@ static void tcp_retransmit_timer(struct sock *sk)
341 tcp_enter_loss(sk, 0); 341 tcp_enter_loss(sk, 0);
342 } 342 }
343 343
344 if (tcp_retransmit_skb(sk, skb_peek(&sk->sk_write_queue)) > 0) { 344 if (tcp_retransmit_skb(sk, tcp_write_queue_head(sk)) > 0) {
345 /* Retransmission failed because of local congestion, 345 /* Retransmission failed because of local congestion,
346 * do not backoff. 346 * do not backoff.
347 */ 347 */
@@ -482,7 +482,7 @@ static void tcp_keepalive_timer (unsigned long data)
482 elapsed = keepalive_time_when(tp); 482 elapsed = keepalive_time_when(tp);
483 483
484 /* It is alive without keepalive 8) */ 484 /* It is alive without keepalive 8) */
485 if (tp->packets_out || sk->sk_send_head) 485 if (tp->packets_out || tcp_send_head(sk))
486 goto resched; 486 goto resched;
487 487
488 elapsed = tcp_time_stamp - tp->rcv_tstamp; 488 elapsed = tcp_time_stamp - tp->rcv_tstamp;