diff options
Diffstat (limited to 'net')
-rw-r--r-- | net/core/datagram.c | 2 | ||||
-rw-r--r-- | net/core/sock.c | 104 | ||||
-rw-r--r-- | net/core/stream.c | 84 | ||||
-rw-r--r-- | net/ipv4/tcp.c | 23 | ||||
-rw-r--r-- | net/ipv4/tcp_input.c | 26 | ||||
-rw-r--r-- | net/ipv4/tcp_output.c | 26 | ||||
-rw-r--r-- | net/ipv4/tcp_timer.c | 8 | ||||
-rw-r--r-- | net/sctp/protocol.c | 2 | ||||
-rw-r--r-- | net/sctp/sm_statefuns.c | 2 | ||||
-rw-r--r-- | net/sctp/socket.c | 11 | ||||
-rw-r--r-- | net/sctp/ulpevent.c | 2 | ||||
-rw-r--r-- | net/sctp/ulpqueue.c | 2 |
12 files changed, 162 insertions, 130 deletions
diff --git a/net/core/datagram.c b/net/core/datagram.c index 2d733131d7ce..8a28fc93b724 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c | |||
@@ -209,6 +209,7 @@ struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, | |||
209 | void skb_free_datagram(struct sock *sk, struct sk_buff *skb) | 209 | void skb_free_datagram(struct sock *sk, struct sk_buff *skb) |
210 | { | 210 | { |
211 | kfree_skb(skb); | 211 | kfree_skb(skb); |
212 | sk_mem_reclaim(sk); | ||
212 | } | 213 | } |
213 | 214 | ||
214 | /** | 215 | /** |
@@ -248,6 +249,7 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags) | |||
248 | } | 249 | } |
249 | 250 | ||
250 | kfree_skb(skb); | 251 | kfree_skb(skb); |
252 | sk_mem_reclaim(sk); | ||
251 | return err; | 253 | return err; |
252 | } | 254 | } |
253 | 255 | ||
diff --git a/net/core/sock.c b/net/core/sock.c index 118214047ed2..8c184c4a3811 100644 --- a/net/core/sock.c +++ b/net/core/sock.c | |||
@@ -282,6 +282,11 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) | |||
282 | if (err) | 282 | if (err) |
283 | goto out; | 283 | goto out; |
284 | 284 | ||
285 | if (!sk_rmem_schedule(sk, skb->truesize)) { | ||
286 | err = -ENOBUFS; | ||
287 | goto out; | ||
288 | } | ||
289 | |||
285 | skb->dev = NULL; | 290 | skb->dev = NULL; |
286 | skb_set_owner_r(skb, sk); | 291 | skb_set_owner_r(skb, sk); |
287 | 292 | ||
@@ -1107,7 +1112,9 @@ void sock_rfree(struct sk_buff *skb) | |||
1107 | { | 1112 | { |
1108 | struct sock *sk = skb->sk; | 1113 | struct sock *sk = skb->sk; |
1109 | 1114 | ||
1115 | skb_truesize_check(skb); | ||
1110 | atomic_sub(skb->truesize, &sk->sk_rmem_alloc); | 1116 | atomic_sub(skb->truesize, &sk->sk_rmem_alloc); |
1117 | sk_mem_uncharge(skb->sk, skb->truesize); | ||
1111 | } | 1118 | } |
1112 | 1119 | ||
1113 | 1120 | ||
@@ -1384,6 +1391,103 @@ int sk_wait_data(struct sock *sk, long *timeo) | |||
1384 | 1391 | ||
1385 | EXPORT_SYMBOL(sk_wait_data); | 1392 | EXPORT_SYMBOL(sk_wait_data); |
1386 | 1393 | ||
1394 | /** | ||
1395 | * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated | ||
1396 | * @sk: socket | ||
1397 | * @size: memory size to allocate | ||
1398 | * @kind: allocation type | ||
1399 | * | ||
1400 | * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means | ||
1401 | * rmem allocation. This function assumes that protocols which have | ||
1402 | * memory_pressure use sk_wmem_queued as write buffer accounting. | ||
1403 | */ | ||
1404 | int __sk_mem_schedule(struct sock *sk, int size, int kind) | ||
1405 | { | ||
1406 | struct proto *prot = sk->sk_prot; | ||
1407 | int amt = sk_mem_pages(size); | ||
1408 | int allocated; | ||
1409 | |||
1410 | sk->sk_forward_alloc += amt * SK_MEM_QUANTUM; | ||
1411 | allocated = atomic_add_return(amt, prot->memory_allocated); | ||
1412 | |||
1413 | /* Under limit. */ | ||
1414 | if (allocated <= prot->sysctl_mem[0]) { | ||
1415 | if (prot->memory_pressure && *prot->memory_pressure) | ||
1416 | *prot->memory_pressure = 0; | ||
1417 | return 1; | ||
1418 | } | ||
1419 | |||
1420 | /* Under pressure. */ | ||
1421 | if (allocated > prot->sysctl_mem[1]) | ||
1422 | if (prot->enter_memory_pressure) | ||
1423 | prot->enter_memory_pressure(); | ||
1424 | |||
1425 | /* Over hard limit. */ | ||
1426 | if (allocated > prot->sysctl_mem[2]) | ||
1427 | goto suppress_allocation; | ||
1428 | |||
1429 | /* guarantee minimum buffer size under pressure */ | ||
1430 | if (kind == SK_MEM_RECV) { | ||
1431 | if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0]) | ||
1432 | return 1; | ||
1433 | } else { /* SK_MEM_SEND */ | ||
1434 | if (sk->sk_type == SOCK_STREAM) { | ||
1435 | if (sk->sk_wmem_queued < prot->sysctl_wmem[0]) | ||
1436 | return 1; | ||
1437 | } else if (atomic_read(&sk->sk_wmem_alloc) < | ||
1438 | prot->sysctl_wmem[0]) | ||
1439 | return 1; | ||
1440 | } | ||
1441 | |||
1442 | if (prot->memory_pressure) { | ||
1443 | if (!*prot->memory_pressure || | ||
1444 | prot->sysctl_mem[2] > atomic_read(prot->sockets_allocated) * | ||
1445 | sk_mem_pages(sk->sk_wmem_queued + | ||
1446 | atomic_read(&sk->sk_rmem_alloc) + | ||
1447 | sk->sk_forward_alloc)) | ||
1448 | return 1; | ||
1449 | } | ||
1450 | |||
1451 | suppress_allocation: | ||
1452 | |||
1453 | if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { | ||
1454 | sk_stream_moderate_sndbuf(sk); | ||
1455 | |||
1456 | /* Fail only if socket is _under_ its sndbuf. | ||
1457 | * In this case we cannot block, so that we have to fail. | ||
1458 | */ | ||
1459 | if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) | ||
1460 | return 1; | ||
1461 | } | ||
1462 | |||
1463 | /* Alas. Undo changes. */ | ||
1464 | sk->sk_forward_alloc -= amt * SK_MEM_QUANTUM; | ||
1465 | atomic_sub(amt, prot->memory_allocated); | ||
1466 | return 0; | ||
1467 | } | ||
1468 | |||
1469 | EXPORT_SYMBOL(__sk_mem_schedule); | ||
1470 | |||
1471 | /** | ||
1472 | * __sk_reclaim - reclaim memory_allocated | ||
1473 | * @sk: socket | ||
1474 | */ | ||
1475 | void __sk_mem_reclaim(struct sock *sk) | ||
1476 | { | ||
1477 | struct proto *prot = sk->sk_prot; | ||
1478 | |||
1479 | atomic_sub(sk->sk_forward_alloc / SK_MEM_QUANTUM, | ||
1480 | prot->memory_allocated); | ||
1481 | sk->sk_forward_alloc &= SK_MEM_QUANTUM - 1; | ||
1482 | |||
1483 | if (prot->memory_pressure && *prot->memory_pressure && | ||
1484 | (atomic_read(prot->memory_allocated) < prot->sysctl_mem[0])) | ||
1485 | *prot->memory_pressure = 0; | ||
1486 | } | ||
1487 | |||
1488 | EXPORT_SYMBOL(__sk_mem_reclaim); | ||
1489 | |||
1490 | |||
1387 | /* | 1491 | /* |
1388 | * Set of default routines for initialising struct proto_ops when | 1492 | * Set of default routines for initialising struct proto_ops when |
1389 | * the protocol does not support a particular function. In certain | 1493 | * the protocol does not support a particular function. In certain |
diff --git a/net/core/stream.c b/net/core/stream.c index bf188ffdbdbe..4a0ad152c9c4 100644 --- a/net/core/stream.c +++ b/net/core/stream.c | |||
@@ -172,17 +172,6 @@ do_interrupted: | |||
172 | 172 | ||
173 | EXPORT_SYMBOL(sk_stream_wait_memory); | 173 | EXPORT_SYMBOL(sk_stream_wait_memory); |
174 | 174 | ||
175 | void sk_stream_rfree(struct sk_buff *skb) | ||
176 | { | ||
177 | struct sock *sk = skb->sk; | ||
178 | |||
179 | skb_truesize_check(skb); | ||
180 | atomic_sub(skb->truesize, &sk->sk_rmem_alloc); | ||
181 | sk->sk_forward_alloc += skb->truesize; | ||
182 | } | ||
183 | |||
184 | EXPORT_SYMBOL(sk_stream_rfree); | ||
185 | |||
186 | int sk_stream_error(struct sock *sk, int flags, int err) | 175 | int sk_stream_error(struct sock *sk, int flags, int err) |
187 | { | 176 | { |
188 | if (err == -EPIPE) | 177 | if (err == -EPIPE) |
@@ -194,77 +183,6 @@ int sk_stream_error(struct sock *sk, int flags, int err) | |||
194 | 183 | ||
195 | EXPORT_SYMBOL(sk_stream_error); | 184 | EXPORT_SYMBOL(sk_stream_error); |
196 | 185 | ||
197 | void __sk_stream_mem_reclaim(struct sock *sk) | ||
198 | { | ||
199 | atomic_sub(sk->sk_forward_alloc >> SK_STREAM_MEM_QUANTUM_SHIFT, | ||
200 | sk->sk_prot->memory_allocated); | ||
201 | sk->sk_forward_alloc &= SK_STREAM_MEM_QUANTUM - 1; | ||
202 | if (*sk->sk_prot->memory_pressure && | ||
203 | (atomic_read(sk->sk_prot->memory_allocated) < | ||
204 | sk->sk_prot->sysctl_mem[0])) | ||
205 | *sk->sk_prot->memory_pressure = 0; | ||
206 | } | ||
207 | |||
208 | EXPORT_SYMBOL(__sk_stream_mem_reclaim); | ||
209 | |||
210 | int sk_stream_mem_schedule(struct sock *sk, int size, int kind) | ||
211 | { | ||
212 | int amt = sk_stream_pages(size); | ||
213 | struct proto *prot = sk->sk_prot; | ||
214 | |||
215 | sk->sk_forward_alloc += amt * SK_STREAM_MEM_QUANTUM; | ||
216 | atomic_add(amt, prot->memory_allocated); | ||
217 | |||
218 | /* Under limit. */ | ||
219 | if (atomic_read(prot->memory_allocated) < prot->sysctl_mem[0]) { | ||
220 | if (*prot->memory_pressure) | ||
221 | *prot->memory_pressure = 0; | ||
222 | return 1; | ||
223 | } | ||
224 | |||
225 | /* Over hard limit. */ | ||
226 | if (atomic_read(prot->memory_allocated) > prot->sysctl_mem[2]) { | ||
227 | prot->enter_memory_pressure(); | ||
228 | goto suppress_allocation; | ||
229 | } | ||
230 | |||
231 | /* Under pressure. */ | ||
232 | if (atomic_read(prot->memory_allocated) > prot->sysctl_mem[1]) | ||
233 | prot->enter_memory_pressure(); | ||
234 | |||
235 | if (kind) { | ||
236 | if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0]) | ||
237 | return 1; | ||
238 | } else if (sk->sk_wmem_queued < prot->sysctl_wmem[0]) | ||
239 | return 1; | ||
240 | |||
241 | if (!*prot->memory_pressure || | ||
242 | prot->sysctl_mem[2] > atomic_read(prot->sockets_allocated) * | ||
243 | sk_stream_pages(sk->sk_wmem_queued + | ||
244 | atomic_read(&sk->sk_rmem_alloc) + | ||
245 | sk->sk_forward_alloc)) | ||
246 | return 1; | ||
247 | |||
248 | suppress_allocation: | ||
249 | |||
250 | if (!kind) { | ||
251 | sk_stream_moderate_sndbuf(sk); | ||
252 | |||
253 | /* Fail only if socket is _under_ its sndbuf. | ||
254 | * In this case we cannot block, so that we have to fail. | ||
255 | */ | ||
256 | if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) | ||
257 | return 1; | ||
258 | } | ||
259 | |||
260 | /* Alas. Undo changes. */ | ||
261 | sk->sk_forward_alloc -= amt * SK_STREAM_MEM_QUANTUM; | ||
262 | atomic_sub(amt, prot->memory_allocated); | ||
263 | return 0; | ||
264 | } | ||
265 | |||
266 | EXPORT_SYMBOL(sk_stream_mem_schedule); | ||
267 | |||
268 | void sk_stream_kill_queues(struct sock *sk) | 186 | void sk_stream_kill_queues(struct sock *sk) |
269 | { | 187 | { |
270 | /* First the read buffer. */ | 188 | /* First the read buffer. */ |
@@ -277,7 +195,7 @@ void sk_stream_kill_queues(struct sock *sk) | |||
277 | BUG_TRAP(skb_queue_empty(&sk->sk_write_queue)); | 195 | BUG_TRAP(skb_queue_empty(&sk->sk_write_queue)); |
278 | 196 | ||
279 | /* Account for returned memory. */ | 197 | /* Account for returned memory. */ |
280 | sk_stream_mem_reclaim(sk); | 198 | sk_mem_reclaim(sk); |
281 | 199 | ||
282 | BUG_TRAP(!sk->sk_wmem_queued); | 200 | BUG_TRAP(!sk->sk_wmem_queued); |
283 | BUG_TRAP(!sk->sk_forward_alloc); | 201 | BUG_TRAP(!sk->sk_forward_alloc); |
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index fdaf965a6794..2cbfa6df7976 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c | |||
@@ -308,7 +308,7 @@ struct tcp_splice_state { | |||
308 | /* | 308 | /* |
309 | * Pressure flag: try to collapse. | 309 | * Pressure flag: try to collapse. |
310 | * Technical note: it is used by multiple contexts non atomically. | 310 | * Technical note: it is used by multiple contexts non atomically. |
311 | * All the sk_stream_mem_schedule() is of this nature: accounting | 311 | * All the __sk_mem_schedule() is of this nature: accounting |
312 | * is strict, actions are advisory and have some latency. | 312 | * is strict, actions are advisory and have some latency. |
313 | */ | 313 | */ |
314 | int tcp_memory_pressure __read_mostly; | 314 | int tcp_memory_pressure __read_mostly; |
@@ -485,7 +485,8 @@ static inline void skb_entail(struct sock *sk, struct sk_buff *skb) | |||
485 | tcb->sacked = 0; | 485 | tcb->sacked = 0; |
486 | skb_header_release(skb); | 486 | skb_header_release(skb); |
487 | tcp_add_write_queue_tail(sk, skb); | 487 | tcp_add_write_queue_tail(sk, skb); |
488 | sk_charge_skb(sk, skb); | 488 | sk->sk_wmem_queued += skb->truesize; |
489 | sk_mem_charge(sk, skb->truesize); | ||
489 | if (tp->nonagle & TCP_NAGLE_PUSH) | 490 | if (tp->nonagle & TCP_NAGLE_PUSH) |
490 | tp->nonagle &= ~TCP_NAGLE_PUSH; | 491 | tp->nonagle &= ~TCP_NAGLE_PUSH; |
491 | } | 492 | } |
@@ -638,7 +639,7 @@ struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) | |||
638 | 639 | ||
639 | skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp); | 640 | skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp); |
640 | if (skb) { | 641 | if (skb) { |
641 | if (sk_stream_wmem_schedule(sk, skb->truesize)) { | 642 | if (sk_wmem_schedule(sk, skb->truesize)) { |
642 | /* | 643 | /* |
643 | * Make sure that we have exactly size bytes | 644 | * Make sure that we have exactly size bytes |
644 | * available to the caller, no more, no less. | 645 | * available to the caller, no more, no less. |
@@ -707,7 +708,7 @@ new_segment: | |||
707 | tcp_mark_push(tp, skb); | 708 | tcp_mark_push(tp, skb); |
708 | goto new_segment; | 709 | goto new_segment; |
709 | } | 710 | } |
710 | if (!sk_stream_wmem_schedule(sk, copy)) | 711 | if (!sk_wmem_schedule(sk, copy)) |
711 | goto wait_for_memory; | 712 | goto wait_for_memory; |
712 | 713 | ||
713 | if (can_coalesce) { | 714 | if (can_coalesce) { |
@@ -721,7 +722,7 @@ new_segment: | |||
721 | skb->data_len += copy; | 722 | skb->data_len += copy; |
722 | skb->truesize += copy; | 723 | skb->truesize += copy; |
723 | sk->sk_wmem_queued += copy; | 724 | sk->sk_wmem_queued += copy; |
724 | sk->sk_forward_alloc -= copy; | 725 | sk_mem_charge(sk, copy); |
725 | skb->ip_summed = CHECKSUM_PARTIAL; | 726 | skb->ip_summed = CHECKSUM_PARTIAL; |
726 | tp->write_seq += copy; | 727 | tp->write_seq += copy; |
727 | TCP_SKB_CB(skb)->end_seq += copy; | 728 | TCP_SKB_CB(skb)->end_seq += copy; |
@@ -928,7 +929,7 @@ new_segment: | |||
928 | if (copy > PAGE_SIZE - off) | 929 | if (copy > PAGE_SIZE - off) |
929 | copy = PAGE_SIZE - off; | 930 | copy = PAGE_SIZE - off; |
930 | 931 | ||
931 | if (!sk_stream_wmem_schedule(sk, copy)) | 932 | if (!sk_wmem_schedule(sk, copy)) |
932 | goto wait_for_memory; | 933 | goto wait_for_memory; |
933 | 934 | ||
934 | if (!page) { | 935 | if (!page) { |
@@ -1019,7 +1020,7 @@ do_fault: | |||
1019 | * reset, where we can be unlinking the send_head. | 1020 | * reset, where we can be unlinking the send_head. |
1020 | */ | 1021 | */ |
1021 | tcp_check_send_head(sk, skb); | 1022 | tcp_check_send_head(sk, skb); |
1022 | sk_stream_free_skb(sk, skb); | 1023 | sk_wmem_free_skb(sk, skb); |
1023 | } | 1024 | } |
1024 | 1025 | ||
1025 | do_error: | 1026 | do_error: |
@@ -1738,7 +1739,7 @@ void tcp_close(struct sock *sk, long timeout) | |||
1738 | __kfree_skb(skb); | 1739 | __kfree_skb(skb); |
1739 | } | 1740 | } |
1740 | 1741 | ||
1741 | sk_stream_mem_reclaim(sk); | 1742 | sk_mem_reclaim(sk); |
1742 | 1743 | ||
1743 | /* As outlined in RFC 2525, section 2.17, we send a RST here because | 1744 | /* As outlined in RFC 2525, section 2.17, we send a RST here because |
1744 | * data was lost. To witness the awful effects of the old behavior of | 1745 | * data was lost. To witness the awful effects of the old behavior of |
@@ -1841,7 +1842,7 @@ adjudge_to_death: | |||
1841 | } | 1842 | } |
1842 | } | 1843 | } |
1843 | if (sk->sk_state != TCP_CLOSE) { | 1844 | if (sk->sk_state != TCP_CLOSE) { |
1844 | sk_stream_mem_reclaim(sk); | 1845 | sk_mem_reclaim(sk); |
1845 | if (tcp_too_many_orphans(sk, | 1846 | if (tcp_too_many_orphans(sk, |
1846 | atomic_read(sk->sk_prot->orphan_count))) { | 1847 | atomic_read(sk->sk_prot->orphan_count))) { |
1847 | if (net_ratelimit()) | 1848 | if (net_ratelimit()) |
@@ -2658,11 +2659,11 @@ void __init tcp_init(void) | |||
2658 | limit = ((unsigned long)sysctl_tcp_mem[1]) << (PAGE_SHIFT - 7); | 2659 | limit = ((unsigned long)sysctl_tcp_mem[1]) << (PAGE_SHIFT - 7); |
2659 | max_share = min(4UL*1024*1024, limit); | 2660 | max_share = min(4UL*1024*1024, limit); |
2660 | 2661 | ||
2661 | sysctl_tcp_wmem[0] = SK_STREAM_MEM_QUANTUM; | 2662 | sysctl_tcp_wmem[0] = SK_MEM_QUANTUM; |
2662 | sysctl_tcp_wmem[1] = 16*1024; | 2663 | sysctl_tcp_wmem[1] = 16*1024; |
2663 | sysctl_tcp_wmem[2] = max(64*1024, max_share); | 2664 | sysctl_tcp_wmem[2] = max(64*1024, max_share); |
2664 | 2665 | ||
2665 | sysctl_tcp_rmem[0] = SK_STREAM_MEM_QUANTUM; | 2666 | sysctl_tcp_rmem[0] = SK_MEM_QUANTUM; |
2666 | sysctl_tcp_rmem[1] = 87380; | 2667 | sysctl_tcp_rmem[1] = 87380; |
2667 | sysctl_tcp_rmem[2] = max(87380, max_share); | 2668 | sysctl_tcp_rmem[2] = max(87380, max_share); |
2668 | 2669 | ||
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index efea9873208e..722c9cbb91e3 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c | |||
@@ -591,7 +591,7 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) | |||
591 | * restart window, so that we send ACKs quickly. | 591 | * restart window, so that we send ACKs quickly. |
592 | */ | 592 | */ |
593 | tcp_incr_quickack(sk); | 593 | tcp_incr_quickack(sk); |
594 | sk_stream_mem_reclaim(sk); | 594 | sk_mem_reclaim(sk); |
595 | } | 595 | } |
596 | } | 596 | } |
597 | icsk->icsk_ack.lrcvtime = now; | 597 | icsk->icsk_ack.lrcvtime = now; |
@@ -2851,7 +2851,7 @@ static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets) | |||
2851 | break; | 2851 | break; |
2852 | 2852 | ||
2853 | tcp_unlink_write_queue(skb, sk); | 2853 | tcp_unlink_write_queue(skb, sk); |
2854 | sk_stream_free_skb(sk, skb); | 2854 | sk_wmem_free_skb(sk, skb); |
2855 | tcp_clear_all_retrans_hints(tp); | 2855 | tcp_clear_all_retrans_hints(tp); |
2856 | } | 2856 | } |
2857 | 2857 | ||
@@ -3567,7 +3567,7 @@ static void tcp_fin(struct sk_buff *skb, struct sock *sk, struct tcphdr *th) | |||
3567 | __skb_queue_purge(&tp->out_of_order_queue); | 3567 | __skb_queue_purge(&tp->out_of_order_queue); |
3568 | if (tcp_is_sack(tp)) | 3568 | if (tcp_is_sack(tp)) |
3569 | tcp_sack_reset(&tp->rx_opt); | 3569 | tcp_sack_reset(&tp->rx_opt); |
3570 | sk_stream_mem_reclaim(sk); | 3570 | sk_mem_reclaim(sk); |
3571 | 3571 | ||
3572 | if (!sock_flag(sk, SOCK_DEAD)) { | 3572 | if (!sock_flag(sk, SOCK_DEAD)) { |
3573 | sk->sk_state_change(sk); | 3573 | sk->sk_state_change(sk); |
@@ -3850,12 +3850,12 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) | |||
3850 | queue_and_out: | 3850 | queue_and_out: |
3851 | if (eaten < 0 && | 3851 | if (eaten < 0 && |
3852 | (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || | 3852 | (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || |
3853 | !sk_stream_rmem_schedule(sk, skb))) { | 3853 | !sk_rmem_schedule(sk, skb->truesize))) { |
3854 | if (tcp_prune_queue(sk) < 0 || | 3854 | if (tcp_prune_queue(sk) < 0 || |
3855 | !sk_stream_rmem_schedule(sk, skb)) | 3855 | !sk_rmem_schedule(sk, skb->truesize)) |
3856 | goto drop; | 3856 | goto drop; |
3857 | } | 3857 | } |
3858 | sk_stream_set_owner_r(skb, sk); | 3858 | skb_set_owner_r(skb, sk); |
3859 | __skb_queue_tail(&sk->sk_receive_queue, skb); | 3859 | __skb_queue_tail(&sk->sk_receive_queue, skb); |
3860 | } | 3860 | } |
3861 | tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; | 3861 | tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; |
@@ -3924,9 +3924,9 @@ drop: | |||
3924 | TCP_ECN_check_ce(tp, skb); | 3924 | TCP_ECN_check_ce(tp, skb); |
3925 | 3925 | ||
3926 | if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || | 3926 | if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || |
3927 | !sk_stream_rmem_schedule(sk, skb)) { | 3927 | !sk_rmem_schedule(sk, skb->truesize)) { |
3928 | if (tcp_prune_queue(sk) < 0 || | 3928 | if (tcp_prune_queue(sk) < 0 || |
3929 | !sk_stream_rmem_schedule(sk, skb)) | 3929 | !sk_rmem_schedule(sk, skb->truesize)) |
3930 | goto drop; | 3930 | goto drop; |
3931 | } | 3931 | } |
3932 | 3932 | ||
@@ -3937,7 +3937,7 @@ drop: | |||
3937 | SOCK_DEBUG(sk, "out of order segment: rcv_next %X seq %X - %X\n", | 3937 | SOCK_DEBUG(sk, "out of order segment: rcv_next %X seq %X - %X\n", |
3938 | tp->rcv_nxt, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); | 3938 | tp->rcv_nxt, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); |
3939 | 3939 | ||
3940 | sk_stream_set_owner_r(skb, sk); | 3940 | skb_set_owner_r(skb, sk); |
3941 | 3941 | ||
3942 | if (!skb_peek(&tp->out_of_order_queue)) { | 3942 | if (!skb_peek(&tp->out_of_order_queue)) { |
3943 | /* Initial out of order segment, build 1 SACK. */ | 3943 | /* Initial out of order segment, build 1 SACK. */ |
@@ -4079,7 +4079,7 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, | |||
4079 | memcpy(nskb->cb, skb->cb, sizeof(skb->cb)); | 4079 | memcpy(nskb->cb, skb->cb, sizeof(skb->cb)); |
4080 | TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start; | 4080 | TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start; |
4081 | __skb_insert(nskb, skb->prev, skb, list); | 4081 | __skb_insert(nskb, skb->prev, skb, list); |
4082 | sk_stream_set_owner_r(nskb, sk); | 4082 | skb_set_owner_r(nskb, sk); |
4083 | 4083 | ||
4084 | /* Copy data, releasing collapsed skbs. */ | 4084 | /* Copy data, releasing collapsed skbs. */ |
4085 | while (copy > 0) { | 4085 | while (copy > 0) { |
@@ -4177,7 +4177,7 @@ static int tcp_prune_queue(struct sock *sk) | |||
4177 | sk->sk_receive_queue.next, | 4177 | sk->sk_receive_queue.next, |
4178 | (struct sk_buff*)&sk->sk_receive_queue, | 4178 | (struct sk_buff*)&sk->sk_receive_queue, |
4179 | tp->copied_seq, tp->rcv_nxt); | 4179 | tp->copied_seq, tp->rcv_nxt); |
4180 | sk_stream_mem_reclaim(sk); | 4180 | sk_mem_reclaim(sk); |
4181 | 4181 | ||
4182 | if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) | 4182 | if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) |
4183 | return 0; | 4183 | return 0; |
@@ -4197,7 +4197,7 @@ static int tcp_prune_queue(struct sock *sk) | |||
4197 | */ | 4197 | */ |
4198 | if (tcp_is_sack(tp)) | 4198 | if (tcp_is_sack(tp)) |
4199 | tcp_sack_reset(&tp->rx_opt); | 4199 | tcp_sack_reset(&tp->rx_opt); |
4200 | sk_stream_mem_reclaim(sk); | 4200 | sk_mem_reclaim(sk); |
4201 | } | 4201 | } |
4202 | 4202 | ||
4203 | if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) | 4203 | if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) |
@@ -4699,7 +4699,7 @@ int tcp_rcv_established(struct sock *sk, struct sk_buff *skb, | |||
4699 | /* Bulk data transfer: receiver */ | 4699 | /* Bulk data transfer: receiver */ |
4700 | __skb_pull(skb,tcp_header_len); | 4700 | __skb_pull(skb,tcp_header_len); |
4701 | __skb_queue_tail(&sk->sk_receive_queue, skb); | 4701 | __skb_queue_tail(&sk->sk_receive_queue, skb); |
4702 | sk_stream_set_owner_r(skb, sk); | 4702 | skb_set_owner_r(skb, sk); |
4703 | tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; | 4703 | tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; |
4704 | } | 4704 | } |
4705 | 4705 | ||
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 9058e0a25107..7a4834a2ae84 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c | |||
@@ -637,7 +637,8 @@ static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb) | |||
637 | tp->write_seq = TCP_SKB_CB(skb)->end_seq; | 637 | tp->write_seq = TCP_SKB_CB(skb)->end_seq; |
638 | skb_header_release(skb); | 638 | skb_header_release(skb); |
639 | tcp_add_write_queue_tail(sk, skb); | 639 | tcp_add_write_queue_tail(sk, skb); |
640 | sk_charge_skb(sk, skb); | 640 | sk->sk_wmem_queued += skb->truesize; |
641 | sk_mem_charge(sk, skb->truesize); | ||
641 | } | 642 | } |
642 | 643 | ||
643 | static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, unsigned int mss_now) | 644 | static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, unsigned int mss_now) |
@@ -701,7 +702,8 @@ int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len, unsigned int mss | |||
701 | if (buff == NULL) | 702 | if (buff == NULL) |
702 | return -ENOMEM; /* We'll just try again later. */ | 703 | return -ENOMEM; /* We'll just try again later. */ |
703 | 704 | ||
704 | sk_charge_skb(sk, buff); | 705 | sk->sk_wmem_queued += buff->truesize; |
706 | sk_mem_charge(sk, buff->truesize); | ||
705 | nlen = skb->len - len - nsize; | 707 | nlen = skb->len - len - nsize; |
706 | buff->truesize += nlen; | 708 | buff->truesize += nlen; |
707 | skb->truesize -= nlen; | 709 | skb->truesize -= nlen; |
@@ -825,7 +827,7 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len) | |||
825 | 827 | ||
826 | skb->truesize -= len; | 828 | skb->truesize -= len; |
827 | sk->sk_wmem_queued -= len; | 829 | sk->sk_wmem_queued -= len; |
828 | sk->sk_forward_alloc += len; | 830 | sk_mem_uncharge(sk, len); |
829 | sock_set_flag(sk, SOCK_QUEUE_SHRUNK); | 831 | sock_set_flag(sk, SOCK_QUEUE_SHRUNK); |
830 | 832 | ||
831 | /* Any change of skb->len requires recalculation of tso | 833 | /* Any change of skb->len requires recalculation of tso |
@@ -1197,7 +1199,8 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len, | |||
1197 | if (unlikely(buff == NULL)) | 1199 | if (unlikely(buff == NULL)) |
1198 | return -ENOMEM; | 1200 | return -ENOMEM; |
1199 | 1201 | ||
1200 | sk_charge_skb(sk, buff); | 1202 | sk->sk_wmem_queued += buff->truesize; |
1203 | sk_mem_charge(sk, buff->truesize); | ||
1201 | buff->truesize += nlen; | 1204 | buff->truesize += nlen; |
1202 | skb->truesize -= nlen; | 1205 | skb->truesize -= nlen; |
1203 | 1206 | ||
@@ -1350,7 +1353,8 @@ static int tcp_mtu_probe(struct sock *sk) | |||
1350 | /* We're allowed to probe. Build it now. */ | 1353 | /* We're allowed to probe. Build it now. */ |
1351 | if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL) | 1354 | if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL) |
1352 | return -1; | 1355 | return -1; |
1353 | sk_charge_skb(sk, nskb); | 1356 | sk->sk_wmem_queued += nskb->truesize; |
1357 | sk_mem_charge(sk, nskb->truesize); | ||
1354 | 1358 | ||
1355 | skb = tcp_send_head(sk); | 1359 | skb = tcp_send_head(sk); |
1356 | 1360 | ||
@@ -1377,7 +1381,7 @@ static int tcp_mtu_probe(struct sock *sk) | |||
1377 | * Throw it away. */ | 1381 | * Throw it away. */ |
1378 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags; | 1382 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags; |
1379 | tcp_unlink_write_queue(skb, sk); | 1383 | tcp_unlink_write_queue(skb, sk); |
1380 | sk_stream_free_skb(sk, skb); | 1384 | sk_wmem_free_skb(sk, skb); |
1381 | } else { | 1385 | } else { |
1382 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags & | 1386 | TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags & |
1383 | ~(TCPCB_FLAG_FIN|TCPCB_FLAG_PSH); | 1387 | ~(TCPCB_FLAG_FIN|TCPCB_FLAG_PSH); |
@@ -1744,7 +1748,7 @@ static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, int m | |||
1744 | /* changed transmit queue under us so clear hints */ | 1748 | /* changed transmit queue under us so clear hints */ |
1745 | tcp_clear_retrans_hints_partial(tp); | 1749 | tcp_clear_retrans_hints_partial(tp); |
1746 | 1750 | ||
1747 | sk_stream_free_skb(sk, next_skb); | 1751 | sk_wmem_free_skb(sk, next_skb); |
1748 | } | 1752 | } |
1749 | } | 1753 | } |
1750 | 1754 | ||
@@ -2139,8 +2143,9 @@ int tcp_send_synack(struct sock *sk) | |||
2139 | tcp_unlink_write_queue(skb, sk); | 2143 | tcp_unlink_write_queue(skb, sk); |
2140 | skb_header_release(nskb); | 2144 | skb_header_release(nskb); |
2141 | __tcp_add_write_queue_head(sk, nskb); | 2145 | __tcp_add_write_queue_head(sk, nskb); |
2142 | sk_stream_free_skb(sk, skb); | 2146 | sk_wmem_free_skb(sk, skb); |
2143 | sk_charge_skb(sk, nskb); | 2147 | sk->sk_wmem_queued += nskb->truesize; |
2148 | sk_mem_charge(sk, nskb->truesize); | ||
2144 | skb = nskb; | 2149 | skb = nskb; |
2145 | } | 2150 | } |
2146 | 2151 | ||
@@ -2343,7 +2348,8 @@ int tcp_connect(struct sock *sk) | |||
2343 | tp->retrans_stamp = TCP_SKB_CB(buff)->when; | 2348 | tp->retrans_stamp = TCP_SKB_CB(buff)->when; |
2344 | skb_header_release(buff); | 2349 | skb_header_release(buff); |
2345 | __tcp_add_write_queue_tail(sk, buff); | 2350 | __tcp_add_write_queue_tail(sk, buff); |
2346 | sk_charge_skb(sk, buff); | 2351 | sk->sk_wmem_queued += buff->truesize; |
2352 | sk_mem_charge(sk, buff->truesize); | ||
2347 | tp->packets_out += tcp_skb_pcount(buff); | 2353 | tp->packets_out += tcp_skb_pcount(buff); |
2348 | tcp_transmit_skb(sk, buff, 1, GFP_KERNEL); | 2354 | tcp_transmit_skb(sk, buff, 1, GFP_KERNEL); |
2349 | 2355 | ||
diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index ea85bc00c61f..17931be6d584 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c | |||
@@ -186,7 +186,7 @@ static void tcp_delack_timer(unsigned long data) | |||
186 | goto out_unlock; | 186 | goto out_unlock; |
187 | } | 187 | } |
188 | 188 | ||
189 | sk_stream_mem_reclaim(sk); | 189 | sk_mem_reclaim(sk); |
190 | 190 | ||
191 | if (sk->sk_state == TCP_CLOSE || !(icsk->icsk_ack.pending & ICSK_ACK_TIMER)) | 191 | if (sk->sk_state == TCP_CLOSE || !(icsk->icsk_ack.pending & ICSK_ACK_TIMER)) |
192 | goto out; | 192 | goto out; |
@@ -226,7 +226,7 @@ static void tcp_delack_timer(unsigned long data) | |||
226 | 226 | ||
227 | out: | 227 | out: |
228 | if (tcp_memory_pressure) | 228 | if (tcp_memory_pressure) |
229 | sk_stream_mem_reclaim(sk); | 229 | sk_mem_reclaim(sk); |
230 | out_unlock: | 230 | out_unlock: |
231 | bh_unlock_sock(sk); | 231 | bh_unlock_sock(sk); |
232 | sock_put(sk); | 232 | sock_put(sk); |
@@ -420,7 +420,7 @@ static void tcp_write_timer(unsigned long data) | |||
420 | TCP_CHECK_TIMER(sk); | 420 | TCP_CHECK_TIMER(sk); |
421 | 421 | ||
422 | out: | 422 | out: |
423 | sk_stream_mem_reclaim(sk); | 423 | sk_mem_reclaim(sk); |
424 | out_unlock: | 424 | out_unlock: |
425 | bh_unlock_sock(sk); | 425 | bh_unlock_sock(sk); |
426 | sock_put(sk); | 426 | sock_put(sk); |
@@ -514,7 +514,7 @@ static void tcp_keepalive_timer (unsigned long data) | |||
514 | } | 514 | } |
515 | 515 | ||
516 | TCP_CHECK_TIMER(sk); | 516 | TCP_CHECK_TIMER(sk); |
517 | sk_stream_mem_reclaim(sk); | 517 | sk_mem_reclaim(sk); |
518 | 518 | ||
519 | resched: | 519 | resched: |
520 | inet_csk_reset_keepalive_timer (sk, elapsed); | 520 | inet_csk_reset_keepalive_timer (sk, elapsed); |
diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index e466e00b9a9f..b92196495027 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c | |||
@@ -1109,7 +1109,7 @@ SCTP_STATIC __init int sctp_init(void) | |||
1109 | sysctl_sctp_rmem[1] = (1500 *(sizeof(struct sk_buff) + 1)); | 1109 | sysctl_sctp_rmem[1] = (1500 *(sizeof(struct sk_buff) + 1)); |
1110 | sysctl_sctp_rmem[2] = max(sysctl_sctp_rmem[1], max_share); | 1110 | sysctl_sctp_rmem[2] = max(sysctl_sctp_rmem[1], max_share); |
1111 | 1111 | ||
1112 | sysctl_sctp_wmem[0] = SK_STREAM_MEM_QUANTUM; | 1112 | sysctl_sctp_wmem[0] = SK_MEM_QUANTUM; |
1113 | sysctl_sctp_wmem[1] = 16*1024; | 1113 | sysctl_sctp_wmem[1] = 16*1024; |
1114 | sysctl_sctp_wmem[2] = max(64*1024, max_share); | 1114 | sysctl_sctp_wmem[2] = max(64*1024, max_share); |
1115 | 1115 | ||
diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 511d8c9a171a..b1267519183b 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c | |||
@@ -5844,7 +5844,7 @@ static int sctp_eat_data(const struct sctp_association *asoc, | |||
5844 | /* | 5844 | /* |
5845 | * Also try to renege to limit our memory usage in the event that | 5845 | * Also try to renege to limit our memory usage in the event that |
5846 | * we are under memory pressure | 5846 | * we are under memory pressure |
5847 | * If we can't renege, don't worry about it, the sk_stream_rmem_schedule | 5847 | * If we can't renege, don't worry about it, the sk_rmem_schedule |
5848 | * in sctp_ulpevent_make_rcvmsg will drop the frame if we grow our | 5848 | * in sctp_ulpevent_make_rcvmsg will drop the frame if we grow our |
5849 | * memory usage too much | 5849 | * memory usage too much |
5850 | */ | 5850 | */ |
diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 7a8650f01d08..710df67a6785 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c | |||
@@ -174,7 +174,8 @@ static inline void sctp_set_owner_w(struct sctp_chunk *chunk) | |||
174 | sizeof(struct sctp_chunk); | 174 | sizeof(struct sctp_chunk); |
175 | 175 | ||
176 | atomic_add(sizeof(struct sctp_chunk), &sk->sk_wmem_alloc); | 176 | atomic_add(sizeof(struct sctp_chunk), &sk->sk_wmem_alloc); |
177 | sk_charge_skb(sk, chunk->skb); | 177 | sk->sk_wmem_queued += chunk->skb->truesize; |
178 | sk_mem_charge(sk, chunk->skb->truesize); | ||
178 | } | 179 | } |
179 | 180 | ||
180 | /* Verify that this is a valid address. */ | 181 | /* Verify that this is a valid address. */ |
@@ -6035,10 +6036,10 @@ static void sctp_wfree(struct sk_buff *skb) | |||
6035 | atomic_sub(sizeof(struct sctp_chunk), &sk->sk_wmem_alloc); | 6036 | atomic_sub(sizeof(struct sctp_chunk), &sk->sk_wmem_alloc); |
6036 | 6037 | ||
6037 | /* | 6038 | /* |
6038 | * This undoes what is done via sk_charge_skb | 6039 | * This undoes what is done via sctp_set_owner_w and sk_mem_charge |
6039 | */ | 6040 | */ |
6040 | sk->sk_wmem_queued -= skb->truesize; | 6041 | sk->sk_wmem_queued -= skb->truesize; |
6041 | sk->sk_forward_alloc += skb->truesize; | 6042 | sk_mem_uncharge(sk, skb->truesize); |
6042 | 6043 | ||
6043 | sock_wfree(skb); | 6044 | sock_wfree(skb); |
6044 | __sctp_write_space(asoc); | 6045 | __sctp_write_space(asoc); |
@@ -6059,9 +6060,9 @@ void sctp_sock_rfree(struct sk_buff *skb) | |||
6059 | atomic_sub(event->rmem_len, &sk->sk_rmem_alloc); | 6060 | atomic_sub(event->rmem_len, &sk->sk_rmem_alloc); |
6060 | 6061 | ||
6061 | /* | 6062 | /* |
6062 | * Mimic the behavior of sk_stream_rfree | 6063 | * Mimic the behavior of sock_rfree |
6063 | */ | 6064 | */ |
6064 | sk->sk_forward_alloc += event->rmem_len; | 6065 | sk_mem_uncharge(sk, event->rmem_len); |
6065 | } | 6066 | } |
6066 | 6067 | ||
6067 | 6068 | ||
diff --git a/net/sctp/ulpevent.c b/net/sctp/ulpevent.c index 307314356e16..047c27df98f4 100644 --- a/net/sctp/ulpevent.c +++ b/net/sctp/ulpevent.c | |||
@@ -700,7 +700,7 @@ struct sctp_ulpevent *sctp_ulpevent_make_rcvmsg(struct sctp_association *asoc, | |||
700 | if (rx_count >= asoc->base.sk->sk_rcvbuf) { | 700 | if (rx_count >= asoc->base.sk->sk_rcvbuf) { |
701 | 701 | ||
702 | if ((asoc->base.sk->sk_userlocks & SOCK_RCVBUF_LOCK) || | 702 | if ((asoc->base.sk->sk_userlocks & SOCK_RCVBUF_LOCK) || |
703 | (!sk_stream_rmem_schedule(asoc->base.sk, chunk->skb))) | 703 | (!sk_rmem_schedule(asoc->base.sk, chunk->skb->truesize))) |
704 | goto fail; | 704 | goto fail; |
705 | } | 705 | } |
706 | 706 | ||
diff --git a/net/sctp/ulpqueue.c b/net/sctp/ulpqueue.c index 1733fa29a501..c25caefa3bcb 100644 --- a/net/sctp/ulpqueue.c +++ b/net/sctp/ulpqueue.c | |||
@@ -1046,7 +1046,7 @@ void sctp_ulpq_renege(struct sctp_ulpq *ulpq, struct sctp_chunk *chunk, | |||
1046 | sctp_ulpq_partial_delivery(ulpq, chunk, gfp); | 1046 | sctp_ulpq_partial_delivery(ulpq, chunk, gfp); |
1047 | } | 1047 | } |
1048 | 1048 | ||
1049 | sk_stream_mem_reclaim(asoc->base.sk); | 1049 | sk_mem_reclaim(asoc->base.sk); |
1050 | return; | 1050 | return; |
1051 | } | 1051 | } |
1052 | 1052 | ||