aboutsummaryrefslogtreecommitdiffstats
path: root/net/core
diff options
context:
space:
mode:
authorRussell King <rmk@dyn-67.arm.linux.org.uk>2009-03-28 16:29:51 -0400
committerRussell King <rmk+kernel@arm.linux.org.uk>2009-03-28 16:29:51 -0400
commited40d0c472b136682b2fcba05f89762859c7374f (patch)
tree076b83a26bcd63d6158463735dd34c10bbc591dc /net/core
parent9e495834e59ca9b29f1a1f63b9f5533bb022ac49 (diff)
parent5d80f8e5a9dc9c9a94d4aeaa567e219a808b8a4a (diff)
Merge branch 'origin' into devel
Conflicts: sound/soc/pxa/pxa2xx-i2s.c
Diffstat (limited to 'net/core')
-rw-r--r--net/core/Makefile3
-rw-r--r--net/core/datagram.c2
-rw-r--r--net/core/dev.c267
-rw-r--r--net/core/drop_monitor.c263
-rw-r--r--net/core/ethtool.c58
-rw-r--r--net/core/fib_rules.c3
-rw-r--r--net/core/neighbour.c15
-rw-r--r--net/core/net-sysfs.c2
-rw-r--r--net/core/net-traces.c29
-rw-r--r--net/core/pktgen.c18
-rw-r--r--net/core/rtnetlink.c9
-rw-r--r--net/core/skbuff.c235
-rw-r--r--net/core/sock.c95
-rw-r--r--net/core/sysctl_net_core.c1
-rw-r--r--net/core/utils.c1
15 files changed, 772 insertions, 229 deletions
diff --git a/net/core/Makefile b/net/core/Makefile
index 26a37cb31923..796f46eece5f 100644
--- a/net/core/Makefile
+++ b/net/core/Makefile
@@ -17,3 +17,6 @@ obj-$(CONFIG_NET_PKTGEN) += pktgen.o
17obj-$(CONFIG_NETPOLL) += netpoll.o 17obj-$(CONFIG_NETPOLL) += netpoll.o
18obj-$(CONFIG_NET_DMA) += user_dma.o 18obj-$(CONFIG_NET_DMA) += user_dma.o
19obj-$(CONFIG_FIB_RULES) += fib_rules.o 19obj-$(CONFIG_FIB_RULES) += fib_rules.o
20obj-$(CONFIG_TRACEPOINTS) += net-traces.o
21obj-$(CONFIG_NET_DROP_MONITOR) += drop_monitor.o
22
diff --git a/net/core/datagram.c b/net/core/datagram.c
index 5e2ac0c4b07c..d0de644b378d 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -208,7 +208,7 @@ struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags,
208 208
209void skb_free_datagram(struct sock *sk, struct sk_buff *skb) 209void skb_free_datagram(struct sock *sk, struct sk_buff *skb)
210{ 210{
211 kfree_skb(skb); 211 consume_skb(skb);
212 sk_mem_reclaim_partial(sk); 212 sk_mem_reclaim_partial(sk);
213} 213}
214 214
diff --git a/net/core/dev.c b/net/core/dev.c
index e3fe5c705606..63ec4bf89b29 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1668,8 +1668,8 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1668 struct netdev_queue *txq) 1668 struct netdev_queue *txq)
1669{ 1669{
1670 const struct net_device_ops *ops = dev->netdev_ops; 1670 const struct net_device_ops *ops = dev->netdev_ops;
1671 int rc;
1671 1672
1672 prefetch(&dev->netdev_ops->ndo_start_xmit);
1673 if (likely(!skb->next)) { 1673 if (likely(!skb->next)) {
1674 if (!list_empty(&ptype_all)) 1674 if (!list_empty(&ptype_all))
1675 dev_queue_xmit_nit(skb, dev); 1675 dev_queue_xmit_nit(skb, dev);
@@ -1681,13 +1681,27 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1681 goto gso; 1681 goto gso;
1682 } 1682 }
1683 1683
1684 return ops->ndo_start_xmit(skb, dev); 1684 rc = ops->ndo_start_xmit(skb, dev);
1685 /*
1686 * TODO: if skb_orphan() was called by
1687 * dev->hard_start_xmit() (for example, the unmodified
1688 * igb driver does that; bnx2 doesn't), then
1689 * skb_tx_software_timestamp() will be unable to send
1690 * back the time stamp.
1691 *
1692 * How can this be prevented? Always create another
1693 * reference to the socket before calling
1694 * dev->hard_start_xmit()? Prevent that skb_orphan()
1695 * does anything in dev->hard_start_xmit() by clearing
1696 * the skb destructor before the call and restoring it
1697 * afterwards, then doing the skb_orphan() ourselves?
1698 */
1699 return rc;
1685 } 1700 }
1686 1701
1687gso: 1702gso:
1688 do { 1703 do {
1689 struct sk_buff *nskb = skb->next; 1704 struct sk_buff *nskb = skb->next;
1690 int rc;
1691 1705
1692 skb->next = nskb->next; 1706 skb->next = nskb->next;
1693 nskb->next = NULL; 1707 nskb->next = NULL;
@@ -1708,59 +1722,24 @@ out_kfree_skb:
1708 return 0; 1722 return 0;
1709} 1723}
1710 1724
1711static u32 simple_tx_hashrnd; 1725static u32 skb_tx_hashrnd;
1712static int simple_tx_hashrnd_initialized = 0;
1713 1726
1714static u16 simple_tx_hash(struct net_device *dev, struct sk_buff *skb) 1727u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
1715{ 1728{
1716 u32 addr1, addr2, ports; 1729 u32 hash;
1717 u32 hash, ihl;
1718 u8 ip_proto = 0;
1719
1720 if (unlikely(!simple_tx_hashrnd_initialized)) {
1721 get_random_bytes(&simple_tx_hashrnd, 4);
1722 simple_tx_hashrnd_initialized = 1;
1723 }
1724 1730
1725 switch (skb->protocol) { 1731 if (skb_rx_queue_recorded(skb)) {
1726 case htons(ETH_P_IP): 1732 hash = skb_get_rx_queue(skb);
1727 if (!(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET))) 1733 } else if (skb->sk && skb->sk->sk_hash) {
1728 ip_proto = ip_hdr(skb)->protocol; 1734 hash = skb->sk->sk_hash;
1729 addr1 = ip_hdr(skb)->saddr; 1735 } else
1730 addr2 = ip_hdr(skb)->daddr; 1736 hash = skb->protocol;
1731 ihl = ip_hdr(skb)->ihl;
1732 break;
1733 case htons(ETH_P_IPV6):
1734 ip_proto = ipv6_hdr(skb)->nexthdr;
1735 addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
1736 addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
1737 ihl = (40 >> 2);
1738 break;
1739 default:
1740 return 0;
1741 }
1742
1743
1744 switch (ip_proto) {
1745 case IPPROTO_TCP:
1746 case IPPROTO_UDP:
1747 case IPPROTO_DCCP:
1748 case IPPROTO_ESP:
1749 case IPPROTO_AH:
1750 case IPPROTO_SCTP:
1751 case IPPROTO_UDPLITE:
1752 ports = *((u32 *) (skb_network_header(skb) + (ihl * 4)));
1753 break;
1754
1755 default:
1756 ports = 0;
1757 break;
1758 }
1759 1737
1760 hash = jhash_3words(addr1, addr2, ports, simple_tx_hashrnd); 1738 hash = jhash_1word(hash, skb_tx_hashrnd);
1761 1739
1762 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32); 1740 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
1763} 1741}
1742EXPORT_SYMBOL(skb_tx_hash);
1764 1743
1765static struct netdev_queue *dev_pick_tx(struct net_device *dev, 1744static struct netdev_queue *dev_pick_tx(struct net_device *dev,
1766 struct sk_buff *skb) 1745 struct sk_buff *skb)
@@ -1771,7 +1750,7 @@ static struct netdev_queue *dev_pick_tx(struct net_device *dev,
1771 if (ops->ndo_select_queue) 1750 if (ops->ndo_select_queue)
1772 queue_index = ops->ndo_select_queue(dev, skb); 1751 queue_index = ops->ndo_select_queue(dev, skb);
1773 else if (dev->real_num_tx_queues > 1) 1752 else if (dev->real_num_tx_queues > 1)
1774 queue_index = simple_tx_hash(dev, skb); 1753 queue_index = skb_tx_hash(dev, skb);
1775 1754
1776 skb_set_queue_mapping(skb, queue_index); 1755 skb_set_queue_mapping(skb, queue_index);
1777 return netdev_get_tx_queue(dev, queue_index); 1756 return netdev_get_tx_queue(dev, queue_index);
@@ -2297,6 +2276,8 @@ ncls:
2297 if (!skb) 2276 if (!skb)
2298 goto out; 2277 goto out;
2299 2278
2279 skb_orphan(skb);
2280
2300 type = skb->protocol; 2281 type = skb->protocol;
2301 list_for_each_entry_rcu(ptype, 2282 list_for_each_entry_rcu(ptype,
2302 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) { 2283 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
@@ -2366,7 +2347,6 @@ static int napi_gro_complete(struct sk_buff *skb)
2366 2347
2367out: 2348out:
2368 skb_shinfo(skb)->gso_size = 0; 2349 skb_shinfo(skb)->gso_size = 0;
2369 __skb_push(skb, -skb_network_offset(skb));
2370 return netif_receive_skb(skb); 2350 return netif_receive_skb(skb);
2371} 2351}
2372 2352
@@ -2380,20 +2360,40 @@ void napi_gro_flush(struct napi_struct *napi)
2380 napi_gro_complete(skb); 2360 napi_gro_complete(skb);
2381 } 2361 }
2382 2362
2363 napi->gro_count = 0;
2383 napi->gro_list = NULL; 2364 napi->gro_list = NULL;
2384} 2365}
2385EXPORT_SYMBOL(napi_gro_flush); 2366EXPORT_SYMBOL(napi_gro_flush);
2386 2367
2368void *skb_gro_header(struct sk_buff *skb, unsigned int hlen)
2369{
2370 unsigned int offset = skb_gro_offset(skb);
2371
2372 hlen += offset;
2373 if (hlen <= skb_headlen(skb))
2374 return skb->data + offset;
2375
2376 if (unlikely(!skb_shinfo(skb)->nr_frags ||
2377 skb_shinfo(skb)->frags[0].size <=
2378 hlen - skb_headlen(skb) ||
2379 PageHighMem(skb_shinfo(skb)->frags[0].page)))
2380 return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL;
2381
2382 return page_address(skb_shinfo(skb)->frags[0].page) +
2383 skb_shinfo(skb)->frags[0].page_offset +
2384 offset - skb_headlen(skb);
2385}
2386EXPORT_SYMBOL(skb_gro_header);
2387
2387int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 2388int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2388{ 2389{
2389 struct sk_buff **pp = NULL; 2390 struct sk_buff **pp = NULL;
2390 struct packet_type *ptype; 2391 struct packet_type *ptype;
2391 __be16 type = skb->protocol; 2392 __be16 type = skb->protocol;
2392 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK]; 2393 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
2393 int count = 0;
2394 int same_flow; 2394 int same_flow;
2395 int mac_len; 2395 int mac_len;
2396 int free; 2396 int ret;
2397 2397
2398 if (!(skb->dev->features & NETIF_F_GRO)) 2398 if (!(skb->dev->features & NETIF_F_GRO))
2399 goto normal; 2399 goto normal;
@@ -2403,30 +2403,16 @@ int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2403 2403
2404 rcu_read_lock(); 2404 rcu_read_lock();
2405 list_for_each_entry_rcu(ptype, head, list) { 2405 list_for_each_entry_rcu(ptype, head, list) {
2406 struct sk_buff *p;
2407
2408 if (ptype->type != type || ptype->dev || !ptype->gro_receive) 2406 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
2409 continue; 2407 continue;
2410 2408
2411 skb_reset_network_header(skb); 2409 skb_set_network_header(skb, skb_gro_offset(skb));
2412 mac_len = skb->network_header - skb->mac_header; 2410 mac_len = skb->network_header - skb->mac_header;
2413 skb->mac_len = mac_len; 2411 skb->mac_len = mac_len;
2414 NAPI_GRO_CB(skb)->same_flow = 0; 2412 NAPI_GRO_CB(skb)->same_flow = 0;
2415 NAPI_GRO_CB(skb)->flush = 0; 2413 NAPI_GRO_CB(skb)->flush = 0;
2416 NAPI_GRO_CB(skb)->free = 0; 2414 NAPI_GRO_CB(skb)->free = 0;
2417 2415
2418 for (p = napi->gro_list; p; p = p->next) {
2419 count++;
2420
2421 if (!NAPI_GRO_CB(p)->same_flow)
2422 continue;
2423
2424 if (p->mac_len != mac_len ||
2425 memcmp(skb_mac_header(p), skb_mac_header(skb),
2426 mac_len))
2427 NAPI_GRO_CB(p)->same_flow = 0;
2428 }
2429
2430 pp = ptype->gro_receive(&napi->gro_list, skb); 2416 pp = ptype->gro_receive(&napi->gro_list, skb);
2431 break; 2417 break;
2432 } 2418 }
@@ -2436,7 +2422,7 @@ int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2436 goto normal; 2422 goto normal;
2437 2423
2438 same_flow = NAPI_GRO_CB(skb)->same_flow; 2424 same_flow = NAPI_GRO_CB(skb)->same_flow;
2439 free = NAPI_GRO_CB(skb)->free; 2425 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
2440 2426
2441 if (pp) { 2427 if (pp) {
2442 struct sk_buff *nskb = *pp; 2428 struct sk_buff *nskb = *pp;
@@ -2444,27 +2430,35 @@ int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2444 *pp = nskb->next; 2430 *pp = nskb->next;
2445 nskb->next = NULL; 2431 nskb->next = NULL;
2446 napi_gro_complete(nskb); 2432 napi_gro_complete(nskb);
2447 count--; 2433 napi->gro_count--;
2448 } 2434 }
2449 2435
2450 if (same_flow) 2436 if (same_flow)
2451 goto ok; 2437 goto ok;
2452 2438
2453 if (NAPI_GRO_CB(skb)->flush || count >= MAX_GRO_SKBS) { 2439 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
2454 __skb_push(skb, -skb_network_offset(skb));
2455 goto normal; 2440 goto normal;
2456 }
2457 2441
2442 napi->gro_count++;
2458 NAPI_GRO_CB(skb)->count = 1; 2443 NAPI_GRO_CB(skb)->count = 1;
2459 skb_shinfo(skb)->gso_size = skb->len; 2444 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
2460 skb->next = napi->gro_list; 2445 skb->next = napi->gro_list;
2461 napi->gro_list = skb; 2446 napi->gro_list = skb;
2447 ret = GRO_HELD;
2448
2449pull:
2450 if (unlikely(!pskb_may_pull(skb, skb_gro_offset(skb)))) {
2451 if (napi->gro_list == skb)
2452 napi->gro_list = skb->next;
2453 ret = GRO_DROP;
2454 }
2462 2455
2463ok: 2456ok:
2464 return free; 2457 return ret;
2465 2458
2466normal: 2459normal:
2467 return -1; 2460 ret = GRO_NORMAL;
2461 goto pull;
2468} 2462}
2469EXPORT_SYMBOL(dev_gro_receive); 2463EXPORT_SYMBOL(dev_gro_receive);
2470 2464
@@ -2472,29 +2466,44 @@ static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2472{ 2466{
2473 struct sk_buff *p; 2467 struct sk_buff *p;
2474 2468
2469 if (netpoll_rx_on(skb))
2470 return GRO_NORMAL;
2471
2475 for (p = napi->gro_list; p; p = p->next) { 2472 for (p = napi->gro_list; p; p = p->next) {
2476 NAPI_GRO_CB(p)->same_flow = 1; 2473 NAPI_GRO_CB(p)->same_flow = !compare_ether_header(
2474 skb_mac_header(p), skb_gro_mac_header(skb));
2477 NAPI_GRO_CB(p)->flush = 0; 2475 NAPI_GRO_CB(p)->flush = 0;
2478 } 2476 }
2479 2477
2480 return dev_gro_receive(napi, skb); 2478 return dev_gro_receive(napi, skb);
2481} 2479}
2482 2480
2483int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 2481int napi_skb_finish(int ret, struct sk_buff *skb)
2484{ 2482{
2485 if (netpoll_receive_skb(skb)) 2483 int err = NET_RX_SUCCESS;
2486 return NET_RX_DROP;
2487 2484
2488 switch (__napi_gro_receive(napi, skb)) { 2485 switch (ret) {
2489 case -1: 2486 case GRO_NORMAL:
2490 return netif_receive_skb(skb); 2487 return netif_receive_skb(skb);
2491 2488
2492 case 1: 2489 case GRO_DROP:
2490 err = NET_RX_DROP;
2491 /* fall through */
2492
2493 case GRO_MERGED_FREE:
2493 kfree_skb(skb); 2494 kfree_skb(skb);
2494 break; 2495 break;
2495 } 2496 }
2496 2497
2497 return NET_RX_SUCCESS; 2498 return err;
2499}
2500EXPORT_SYMBOL(napi_skb_finish);
2501
2502int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2503{
2504 skb_gro_reset_offset(skb);
2505
2506 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
2498} 2507}
2499EXPORT_SYMBOL(napi_gro_receive); 2508EXPORT_SYMBOL(napi_gro_receive);
2500 2509
@@ -2512,6 +2521,9 @@ struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi,
2512{ 2521{
2513 struct net_device *dev = napi->dev; 2522 struct net_device *dev = napi->dev;
2514 struct sk_buff *skb = napi->skb; 2523 struct sk_buff *skb = napi->skb;
2524 struct ethhdr *eth;
2525 skb_frag_t *frag;
2526 int i;
2515 2527
2516 napi->skb = NULL; 2528 napi->skb = NULL;
2517 2529
@@ -2524,20 +2536,36 @@ struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi,
2524 } 2536 }
2525 2537
2526 BUG_ON(info->nr_frags > MAX_SKB_FRAGS); 2538 BUG_ON(info->nr_frags > MAX_SKB_FRAGS);
2539 frag = &info->frags[info->nr_frags - 1];
2540
2541 for (i = skb_shinfo(skb)->nr_frags; i < info->nr_frags; i++) {
2542 skb_fill_page_desc(skb, i, frag->page, frag->page_offset,
2543 frag->size);
2544 frag++;
2545 }
2527 skb_shinfo(skb)->nr_frags = info->nr_frags; 2546 skb_shinfo(skb)->nr_frags = info->nr_frags;
2528 memcpy(skb_shinfo(skb)->frags, info->frags, sizeof(info->frags));
2529 2547
2530 skb->data_len = info->len; 2548 skb->data_len = info->len;
2531 skb->len += info->len; 2549 skb->len += info->len;
2532 skb->truesize += info->len; 2550 skb->truesize += info->len;
2533 2551
2534 if (!pskb_may_pull(skb, ETH_HLEN)) { 2552 skb_reset_mac_header(skb);
2553 skb_gro_reset_offset(skb);
2554
2555 eth = skb_gro_header(skb, sizeof(*eth));
2556 if (!eth) {
2535 napi_reuse_skb(napi, skb); 2557 napi_reuse_skb(napi, skb);
2536 skb = NULL; 2558 skb = NULL;
2537 goto out; 2559 goto out;
2538 } 2560 }
2539 2561
2540 skb->protocol = eth_type_trans(skb, dev); 2562 skb_gro_pull(skb, sizeof(*eth));
2563
2564 /*
2565 * This works because the only protocols we care about don't require
2566 * special handling. We'll fix it up properly at the end.
2567 */
2568 skb->protocol = eth->h_proto;
2541 2569
2542 skb->ip_summed = info->ip_summed; 2570 skb->ip_summed = info->ip_summed;
2543 skb->csum = info->csum; 2571 skb->csum = info->csum;
@@ -2547,32 +2575,43 @@ out:
2547} 2575}
2548EXPORT_SYMBOL(napi_fraginfo_skb); 2576EXPORT_SYMBOL(napi_fraginfo_skb);
2549 2577
2550int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info) 2578int napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb, int ret)
2551{ 2579{
2552 struct sk_buff *skb = napi_fraginfo_skb(napi, info); 2580 int err = NET_RX_SUCCESS;
2553 int err = NET_RX_DROP;
2554 2581
2555 if (!skb) 2582 switch (ret) {
2556 goto out; 2583 case GRO_NORMAL:
2584 case GRO_HELD:
2585 skb->protocol = eth_type_trans(skb, napi->dev);
2557 2586
2558 if (netpoll_receive_skb(skb)) 2587 if (ret == GRO_NORMAL)
2559 goto out; 2588 return netif_receive_skb(skb);
2560 2589
2561 err = NET_RX_SUCCESS; 2590 skb_gro_pull(skb, -ETH_HLEN);
2591 break;
2562 2592
2563 switch (__napi_gro_receive(napi, skb)) { 2593 case GRO_DROP:
2564 case -1: 2594 err = NET_RX_DROP;
2565 return netif_receive_skb(skb); 2595 /* fall through */
2566 2596
2567 case 0: 2597 case GRO_MERGED_FREE:
2568 goto out; 2598 napi_reuse_skb(napi, skb);
2599 break;
2569 } 2600 }
2570 2601
2571 napi_reuse_skb(napi, skb);
2572
2573out:
2574 return err; 2602 return err;
2575} 2603}
2604EXPORT_SYMBOL(napi_frags_finish);
2605
2606int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info)
2607{
2608 struct sk_buff *skb = napi_fraginfo_skb(napi, info);
2609
2610 if (!skb)
2611 return NET_RX_DROP;
2612
2613 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
2614}
2576EXPORT_SYMBOL(napi_gro_frags); 2615EXPORT_SYMBOL(napi_gro_frags);
2577 2616
2578static int process_backlog(struct napi_struct *napi, int quota) 2617static int process_backlog(struct napi_struct *napi, int quota)
@@ -2588,18 +2627,15 @@ static int process_backlog(struct napi_struct *napi, int quota)
2588 local_irq_disable(); 2627 local_irq_disable();
2589 skb = __skb_dequeue(&queue->input_pkt_queue); 2628 skb = __skb_dequeue(&queue->input_pkt_queue);
2590 if (!skb) { 2629 if (!skb) {
2630 __napi_complete(napi);
2591 local_irq_enable(); 2631 local_irq_enable();
2592 napi_complete(napi); 2632 break;
2593 goto out;
2594 } 2633 }
2595 local_irq_enable(); 2634 local_irq_enable();
2596 2635
2597 napi_gro_receive(napi, skb); 2636 netif_receive_skb(skb);
2598 } while (++work < quota && jiffies == start_time); 2637 } while (++work < quota && jiffies == start_time);
2599 2638
2600 napi_gro_flush(napi);
2601
2602out:
2603 return work; 2639 return work;
2604} 2640}
2605 2641
@@ -2653,6 +2689,7 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2653 int (*poll)(struct napi_struct *, int), int weight) 2689 int (*poll)(struct napi_struct *, int), int weight)
2654{ 2690{
2655 INIT_LIST_HEAD(&napi->poll_list); 2691 INIT_LIST_HEAD(&napi->poll_list);
2692 napi->gro_count = 0;
2656 napi->gro_list = NULL; 2693 napi->gro_list = NULL;
2657 napi->skb = NULL; 2694 napi->skb = NULL;
2658 napi->poll = poll; 2695 napi->poll = poll;
@@ -2681,6 +2718,7 @@ void netif_napi_del(struct napi_struct *napi)
2681 } 2718 }
2682 2719
2683 napi->gro_list = NULL; 2720 napi->gro_list = NULL;
2721 napi->gro_count = 0;
2684} 2722}
2685EXPORT_SYMBOL(netif_napi_del); 2723EXPORT_SYMBOL(netif_napi_del);
2686 2724
@@ -3949,6 +3987,7 @@ static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
3949 cmd == SIOCSMIIREG || 3987 cmd == SIOCSMIIREG ||
3950 cmd == SIOCBRADDIF || 3988 cmd == SIOCBRADDIF ||
3951 cmd == SIOCBRDELIF || 3989 cmd == SIOCBRDELIF ||
3990 cmd == SIOCSHWTSTAMP ||
3952 cmd == SIOCWANDEV) { 3991 cmd == SIOCWANDEV) {
3953 err = -EOPNOTSUPP; 3992 err = -EOPNOTSUPP;
3954 if (ops->ndo_do_ioctl) { 3993 if (ops->ndo_do_ioctl) {
@@ -4103,6 +4142,7 @@ int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
4103 case SIOCBONDCHANGEACTIVE: 4142 case SIOCBONDCHANGEACTIVE:
4104 case SIOCBRADDIF: 4143 case SIOCBRADDIF:
4105 case SIOCBRDELIF: 4144 case SIOCBRDELIF:
4145 case SIOCSHWTSTAMP:
4106 if (!capable(CAP_NET_ADMIN)) 4146 if (!capable(CAP_NET_ADMIN))
4107 return -EPERM; 4147 return -EPERM;
4108 /* fall through */ 4148 /* fall through */
@@ -5199,6 +5239,7 @@ static int __init net_dev_init(void)
5199 queue->backlog.poll = process_backlog; 5239 queue->backlog.poll = process_backlog;
5200 queue->backlog.weight = weight_p; 5240 queue->backlog.weight = weight_p;
5201 queue->backlog.gro_list = NULL; 5241 queue->backlog.gro_list = NULL;
5242 queue->backlog.gro_count = 0;
5202 } 5243 }
5203 5244
5204 dev_boot_phase = 0; 5245 dev_boot_phase = 0;
@@ -5231,6 +5272,14 @@ out:
5231 5272
5232subsys_initcall(net_dev_init); 5273subsys_initcall(net_dev_init);
5233 5274
5275static int __init initialize_hashrnd(void)
5276{
5277 get_random_bytes(&skb_tx_hashrnd, sizeof(skb_tx_hashrnd));
5278 return 0;
5279}
5280
5281late_initcall_sync(initialize_hashrnd);
5282
5234EXPORT_SYMBOL(__dev_get_by_index); 5283EXPORT_SYMBOL(__dev_get_by_index);
5235EXPORT_SYMBOL(__dev_get_by_name); 5284EXPORT_SYMBOL(__dev_get_by_name);
5236EXPORT_SYMBOL(__dev_remove_pack); 5285EXPORT_SYMBOL(__dev_remove_pack);
diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c
new file mode 100644
index 000000000000..9fd0dc3cca99
--- /dev/null
+++ b/net/core/drop_monitor.c
@@ -0,0 +1,263 @@
1/*
2 * Monitoring code for network dropped packet alerts
3 *
4 * Copyright (C) 2009 Neil Horman <nhorman@tuxdriver.com>
5 */
6
7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>
9#include <linux/string.h>
10#include <linux/if_arp.h>
11#include <linux/inetdevice.h>
12#include <linux/inet.h>
13#include <linux/interrupt.h>
14#include <linux/netpoll.h>
15#include <linux/sched.h>
16#include <linux/delay.h>
17#include <linux/types.h>
18#include <linux/workqueue.h>
19#include <linux/netlink.h>
20#include <linux/net_dropmon.h>
21#include <linux/percpu.h>
22#include <linux/timer.h>
23#include <linux/bitops.h>
24#include <net/genetlink.h>
25
26#include <trace/skb.h>
27
28#include <asm/unaligned.h>
29
30#define TRACE_ON 1
31#define TRACE_OFF 0
32
33static void send_dm_alert(struct work_struct *unused);
34
35
36/*
37 * Globals, our netlink socket pointer
38 * and the work handle that will send up
39 * netlink alerts
40 */
41struct sock *dm_sock;
42
43struct per_cpu_dm_data {
44 struct work_struct dm_alert_work;
45 struct sk_buff *skb;
46 atomic_t dm_hit_count;
47 struct timer_list send_timer;
48};
49
50static struct genl_family net_drop_monitor_family = {
51 .id = GENL_ID_GENERATE,
52 .hdrsize = 0,
53 .name = "NET_DM",
54 .version = 1,
55 .maxattr = NET_DM_CMD_MAX,
56};
57
58static DEFINE_PER_CPU(struct per_cpu_dm_data, dm_cpu_data);
59
60static int dm_hit_limit = 64;
61static int dm_delay = 1;
62
63
64static void reset_per_cpu_data(struct per_cpu_dm_data *data)
65{
66 size_t al;
67 struct net_dm_alert_msg *msg;
68
69 al = sizeof(struct net_dm_alert_msg);
70 al += dm_hit_limit * sizeof(struct net_dm_drop_point);
71 data->skb = genlmsg_new(al, GFP_KERNEL);
72 genlmsg_put(data->skb, 0, 0, &net_drop_monitor_family,
73 0, NET_DM_CMD_ALERT);
74 msg = __nla_reserve_nohdr(data->skb, sizeof(struct net_dm_alert_msg));
75 memset(msg, 0, al);
76 atomic_set(&data->dm_hit_count, dm_hit_limit);
77}
78
79static void send_dm_alert(struct work_struct *unused)
80{
81 struct sk_buff *skb;
82 struct per_cpu_dm_data *data = &__get_cpu_var(dm_cpu_data);
83
84 /*
85 * Grab the skb we're about to send
86 */
87 skb = data->skb;
88
89 /*
90 * Replace it with a new one
91 */
92 reset_per_cpu_data(data);
93
94 /*
95 * Ship it!
96 */
97 genlmsg_multicast(skb, 0, NET_DM_GRP_ALERT, GFP_KERNEL);
98
99}
100
101/*
102 * This is the timer function to delay the sending of an alert
103 * in the event that more drops will arrive during the
104 * hysteresis period. Note that it operates under the timer interrupt
105 * so we don't need to disable preemption here
106 */
107static void sched_send_work(unsigned long unused)
108{
109 struct per_cpu_dm_data *data = &__get_cpu_var(dm_cpu_data);
110
111 schedule_work(&data->dm_alert_work);
112}
113
114static void trace_kfree_skb_hit(struct sk_buff *skb, void *location)
115{
116 struct net_dm_alert_msg *msg;
117 struct nlmsghdr *nlh;
118 int i;
119 struct per_cpu_dm_data *data = &__get_cpu_var(dm_cpu_data);
120
121
122 if (!atomic_add_unless(&data->dm_hit_count, -1, 0)) {
123 /*
124 * we're already at zero, discard this hit
125 */
126 goto out;
127 }
128
129 nlh = (struct nlmsghdr *)data->skb->data;
130 msg = genlmsg_data(nlmsg_data(nlh));
131 for (i = 0; i < msg->entries; i++) {
132 if (!memcmp(&location, msg->points[i].pc, sizeof(void *))) {
133 msg->points[i].count++;
134 goto out;
135 }
136 }
137
138 /*
139 * We need to create a new entry
140 */
141 __nla_reserve_nohdr(data->skb, sizeof(struct net_dm_drop_point));
142 memcpy(msg->points[msg->entries].pc, &location, sizeof(void *));
143 msg->points[msg->entries].count = 1;
144 msg->entries++;
145
146 if (!timer_pending(&data->send_timer)) {
147 data->send_timer.expires = jiffies + dm_delay * HZ;
148 add_timer_on(&data->send_timer, smp_processor_id());
149 }
150
151out:
152 return;
153}
154
155static int set_all_monitor_traces(int state)
156{
157 int rc = 0;
158
159 switch (state) {
160 case TRACE_ON:
161 rc |= register_trace_kfree_skb(trace_kfree_skb_hit);
162 break;
163 case TRACE_OFF:
164 rc |= unregister_trace_kfree_skb(trace_kfree_skb_hit);
165
166 tracepoint_synchronize_unregister();
167 break;
168 default:
169 rc = 1;
170 break;
171 }
172
173 if (rc)
174 return -EINPROGRESS;
175 return rc;
176}
177
178
179static int net_dm_cmd_config(struct sk_buff *skb,
180 struct genl_info *info)
181{
182 return -ENOTSUPP;
183}
184
185static int net_dm_cmd_trace(struct sk_buff *skb,
186 struct genl_info *info)
187{
188 switch (info->genlhdr->cmd) {
189 case NET_DM_CMD_START:
190 return set_all_monitor_traces(TRACE_ON);
191 break;
192 case NET_DM_CMD_STOP:
193 return set_all_monitor_traces(TRACE_OFF);
194 break;
195 }
196
197 return -ENOTSUPP;
198}
199
200
201static struct genl_ops dropmon_ops[] = {
202 {
203 .cmd = NET_DM_CMD_CONFIG,
204 .doit = net_dm_cmd_config,
205 },
206 {
207 .cmd = NET_DM_CMD_START,
208 .doit = net_dm_cmd_trace,
209 },
210 {
211 .cmd = NET_DM_CMD_STOP,
212 .doit = net_dm_cmd_trace,
213 },
214};
215
216static int __init init_net_drop_monitor(void)
217{
218 int cpu;
219 int rc, i, ret;
220 struct per_cpu_dm_data *data;
221 printk(KERN_INFO "Initalizing network drop monitor service\n");
222
223 if (sizeof(void *) > 8) {
224 printk(KERN_ERR "Unable to store program counters on this arch, Drop monitor failed\n");
225 return -ENOSPC;
226 }
227
228 if (genl_register_family(&net_drop_monitor_family) < 0) {
229 printk(KERN_ERR "Could not create drop monitor netlink family\n");
230 return -EFAULT;
231 }
232
233 rc = -EFAULT;
234
235 for (i = 0; i < ARRAY_SIZE(dropmon_ops); i++) {
236 ret = genl_register_ops(&net_drop_monitor_family,
237 &dropmon_ops[i]);
238 if (ret) {
239 printk(KERN_CRIT "failed to register operation %d\n",
240 dropmon_ops[i].cmd);
241 goto out_unreg;
242 }
243 }
244
245 rc = 0;
246
247 for_each_present_cpu(cpu) {
248 data = &per_cpu(dm_cpu_data, cpu);
249 reset_per_cpu_data(data);
250 INIT_WORK(&data->dm_alert_work, send_dm_alert);
251 init_timer(&data->send_timer);
252 data->send_timer.data = cpu;
253 data->send_timer.function = sched_send_work;
254 }
255 goto out;
256
257out_unreg:
258 genl_unregister_family(&net_drop_monitor_family);
259out:
260 return rc;
261}
262
263late_initcall(init_net_drop_monitor);
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index 947710a36ced..244ca56dffac 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -209,34 +209,62 @@ static int ethtool_get_drvinfo(struct net_device *dev, void __user *useraddr)
209 return 0; 209 return 0;
210} 210}
211 211
212static int ethtool_set_rxhash(struct net_device *dev, void __user *useraddr) 212static int ethtool_set_rxnfc(struct net_device *dev, void __user *useraddr)
213{ 213{
214 struct ethtool_rxnfc cmd; 214 struct ethtool_rxnfc cmd;
215 215
216 if (!dev->ethtool_ops->set_rxhash) 216 if (!dev->ethtool_ops->set_rxnfc)
217 return -EOPNOTSUPP; 217 return -EOPNOTSUPP;
218 218
219 if (copy_from_user(&cmd, useraddr, sizeof(cmd))) 219 if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
220 return -EFAULT; 220 return -EFAULT;
221 221
222 return dev->ethtool_ops->set_rxhash(dev, &cmd); 222 return dev->ethtool_ops->set_rxnfc(dev, &cmd);
223} 223}
224 224
225static int ethtool_get_rxhash(struct net_device *dev, void __user *useraddr) 225static int ethtool_get_rxnfc(struct net_device *dev, void __user *useraddr)
226{ 226{
227 struct ethtool_rxnfc info; 227 struct ethtool_rxnfc info;
228 const struct ethtool_ops *ops = dev->ethtool_ops;
229 int ret;
230 void *rule_buf = NULL;
228 231
229 if (!dev->ethtool_ops->get_rxhash) 232 if (!ops->get_rxnfc)
230 return -EOPNOTSUPP; 233 return -EOPNOTSUPP;
231 234
232 if (copy_from_user(&info, useraddr, sizeof(info))) 235 if (copy_from_user(&info, useraddr, sizeof(info)))
233 return -EFAULT; 236 return -EFAULT;
234 237
235 dev->ethtool_ops->get_rxhash(dev, &info); 238 if (info.cmd == ETHTOOL_GRXCLSRLALL) {
239 if (info.rule_cnt > 0) {
240 rule_buf = kmalloc(info.rule_cnt * sizeof(u32),
241 GFP_USER);
242 if (!rule_buf)
243 return -ENOMEM;
244 }
245 }
236 246
247 ret = ops->get_rxnfc(dev, &info, rule_buf);
248 if (ret < 0)
249 goto err_out;
250
251 ret = -EFAULT;
237 if (copy_to_user(useraddr, &info, sizeof(info))) 252 if (copy_to_user(useraddr, &info, sizeof(info)))
238 return -EFAULT; 253 goto err_out;
239 return 0; 254
255 if (rule_buf) {
256 useraddr += offsetof(struct ethtool_rxnfc, rule_locs);
257 if (copy_to_user(useraddr, rule_buf,
258 info.rule_cnt * sizeof(u32)))
259 goto err_out;
260 }
261 ret = 0;
262
263err_out:
264 if (rule_buf)
265 kfree(rule_buf);
266
267 return ret;
240} 268}
241 269
242static int ethtool_get_regs(struct net_device *dev, char __user *useraddr) 270static int ethtool_get_regs(struct net_device *dev, char __user *useraddr)
@@ -901,6 +929,10 @@ int dev_ethtool(struct net *net, struct ifreq *ifr)
901 case ETHTOOL_GFLAGS: 929 case ETHTOOL_GFLAGS:
902 case ETHTOOL_GPFLAGS: 930 case ETHTOOL_GPFLAGS:
903 case ETHTOOL_GRXFH: 931 case ETHTOOL_GRXFH:
932 case ETHTOOL_GRXRINGS:
933 case ETHTOOL_GRXCLSRLCNT:
934 case ETHTOOL_GRXCLSRULE:
935 case ETHTOOL_GRXCLSRLALL:
904 break; 936 break;
905 default: 937 default:
906 if (!capable(CAP_NET_ADMIN)) 938 if (!capable(CAP_NET_ADMIN))
@@ -1052,10 +1084,16 @@ int dev_ethtool(struct net *net, struct ifreq *ifr)
1052 dev->ethtool_ops->set_priv_flags); 1084 dev->ethtool_ops->set_priv_flags);
1053 break; 1085 break;
1054 case ETHTOOL_GRXFH: 1086 case ETHTOOL_GRXFH:
1055 rc = ethtool_get_rxhash(dev, useraddr); 1087 case ETHTOOL_GRXRINGS:
1088 case ETHTOOL_GRXCLSRLCNT:
1089 case ETHTOOL_GRXCLSRULE:
1090 case ETHTOOL_GRXCLSRLALL:
1091 rc = ethtool_get_rxnfc(dev, useraddr);
1056 break; 1092 break;
1057 case ETHTOOL_SRXFH: 1093 case ETHTOOL_SRXFH:
1058 rc = ethtool_set_rxhash(dev, useraddr); 1094 case ETHTOOL_SRXCLSRLDEL:
1095 case ETHTOOL_SRXCLSRLINS:
1096 rc = ethtool_set_rxnfc(dev, useraddr);
1059 break; 1097 break;
1060 case ETHTOOL_GGRO: 1098 case ETHTOOL_GGRO:
1061 rc = ethtool_get_gro(dev, useraddr); 1099 rc = ethtool_get_gro(dev, useraddr);
diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c
index 32b3a0152d7a..98691e1466b8 100644
--- a/net/core/fib_rules.c
+++ b/net/core/fib_rules.c
@@ -588,7 +588,8 @@ static void notify_rule_change(int event, struct fib_rule *rule,
588 goto errout; 588 goto errout;
589 } 589 }
590 590
591 err = rtnl_notify(skb, net, pid, ops->nlgroup, nlh, GFP_KERNEL); 591 rtnl_notify(skb, net, pid, ops->nlgroup, nlh, GFP_KERNEL);
592 return;
592errout: 593errout:
593 if (err < 0) 594 if (err < 0)
594 rtnl_set_sk_err(net, ops->nlgroup, err); 595 rtnl_set_sk_err(net, ops->nlgroup, err);
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 278a142d1047..a1cbce7fdae5 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -871,8 +871,7 @@ static void neigh_timer_handler(unsigned long arg)
871 write_unlock(&neigh->lock); 871 write_unlock(&neigh->lock);
872 neigh->ops->solicit(neigh, skb); 872 neigh->ops->solicit(neigh, skb);
873 atomic_inc(&neigh->probes); 873 atomic_inc(&neigh->probes);
874 if (skb) 874 kfree_skb(skb);
875 kfree_skb(skb);
876 } else { 875 } else {
877out: 876out:
878 write_unlock(&neigh->lock); 877 write_unlock(&neigh->lock);
@@ -908,8 +907,7 @@ int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb)
908 neigh->updated = jiffies; 907 neigh->updated = jiffies;
909 write_unlock_bh(&neigh->lock); 908 write_unlock_bh(&neigh->lock);
910 909
911 if (skb) 910 kfree_skb(skb);
912 kfree_skb(skb);
913 return 1; 911 return 1;
914 } 912 }
915 } else if (neigh->nud_state & NUD_STALE) { 913 } else if (neigh->nud_state & NUD_STALE) {
@@ -1656,7 +1654,11 @@ static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh, void *arg)
1656 flags &= ~NEIGH_UPDATE_F_OVERRIDE; 1654 flags &= ~NEIGH_UPDATE_F_OVERRIDE;
1657 } 1655 }
1658 1656
1659 err = neigh_update(neigh, lladdr, ndm->ndm_state, flags); 1657 if (ndm->ndm_flags & NTF_USE) {
1658 neigh_event_send(neigh, NULL);
1659 err = 0;
1660 } else
1661 err = neigh_update(neigh, lladdr, ndm->ndm_state, flags);
1660 neigh_release(neigh); 1662 neigh_release(neigh);
1661 goto out_dev_put; 1663 goto out_dev_put;
1662 } 1664 }
@@ -2534,7 +2536,8 @@ static void __neigh_notify(struct neighbour *n, int type, int flags)
2534 kfree_skb(skb); 2536 kfree_skb(skb);
2535 goto errout; 2537 goto errout;
2536 } 2538 }
2537 err = rtnl_notify(skb, net, 0, RTNLGRP_NEIGH, NULL, GFP_ATOMIC); 2539 rtnl_notify(skb, net, 0, RTNLGRP_NEIGH, NULL, GFP_ATOMIC);
2540 return;
2538errout: 2541errout:
2539 if (err < 0) 2542 if (err < 0)
2540 rtnl_set_sk_err(net, RTNLGRP_NEIGH, err); 2543 rtnl_set_sk_err(net, RTNLGRP_NEIGH, err);
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index 484f58750eba..2da59a0ac4ac 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -498,7 +498,7 @@ int netdev_register_kobject(struct net_device *net)
498 dev->groups = groups; 498 dev->groups = groups;
499 499
500 BUILD_BUG_ON(BUS_ID_SIZE < IFNAMSIZ); 500 BUILD_BUG_ON(BUS_ID_SIZE < IFNAMSIZ);
501 dev_set_name(dev, net->name); 501 dev_set_name(dev, "%s", net->name);
502 502
503#ifdef CONFIG_SYSFS 503#ifdef CONFIG_SYSFS
504 *groups++ = &netstat_group; 504 *groups++ = &netstat_group;
diff --git a/net/core/net-traces.c b/net/core/net-traces.c
new file mode 100644
index 000000000000..c8fb45665e4f
--- /dev/null
+++ b/net/core/net-traces.c
@@ -0,0 +1,29 @@
1/*
2 * consolidates trace point definitions
3 *
4 * Copyright (C) 2009 Neil Horman <nhorman@tuxdriver.com>
5 */
6
7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>
9#include <linux/string.h>
10#include <linux/if_arp.h>
11#include <linux/inetdevice.h>
12#include <linux/inet.h>
13#include <linux/interrupt.h>
14#include <linux/netpoll.h>
15#include <linux/sched.h>
16#include <linux/delay.h>
17#include <linux/rcupdate.h>
18#include <linux/types.h>
19#include <linux/workqueue.h>
20#include <linux/netlink.h>
21#include <linux/net_dropmon.h>
22#include <trace/skb.h>
23
24#include <asm/unaligned.h>
25#include <asm/bitops.h>
26
27
28DEFINE_TRACE(kfree_skb);
29EXPORT_TRACEPOINT_SYMBOL_GPL(kfree_skb);
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index 65498483325a..32d419f5ac98 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -3275,8 +3275,7 @@ static void pktgen_stop(struct pktgen_thread *t)
3275 3275
3276 list_for_each_entry(pkt_dev, &t->if_list, list) { 3276 list_for_each_entry(pkt_dev, &t->if_list, list) {
3277 pktgen_stop_device(pkt_dev); 3277 pktgen_stop_device(pkt_dev);
3278 if (pkt_dev->skb) 3278 kfree_skb(pkt_dev->skb);
3279 kfree_skb(pkt_dev->skb);
3280 3279
3281 pkt_dev->skb = NULL; 3280 pkt_dev->skb = NULL;
3282 } 3281 }
@@ -3303,8 +3302,7 @@ static void pktgen_rem_one_if(struct pktgen_thread *t)
3303 if (!cur->removal_mark) 3302 if (!cur->removal_mark)
3304 continue; 3303 continue;
3305 3304
3306 if (cur->skb) 3305 kfree_skb(cur->skb);
3307 kfree_skb(cur->skb);
3308 cur->skb = NULL; 3306 cur->skb = NULL;
3309 3307
3310 pktgen_remove_device(t, cur); 3308 pktgen_remove_device(t, cur);
@@ -3328,8 +3326,7 @@ static void pktgen_rem_all_ifs(struct pktgen_thread *t)
3328 list_for_each_safe(q, n, &t->if_list) { 3326 list_for_each_safe(q, n, &t->if_list) {
3329 cur = list_entry(q, struct pktgen_dev, list); 3327 cur = list_entry(q, struct pktgen_dev, list);
3330 3328
3331 if (cur->skb) 3329 kfree_skb(cur->skb);
3332 kfree_skb(cur->skb);
3333 cur->skb = NULL; 3330 cur->skb = NULL;
3334 3331
3335 pktgen_remove_device(t, cur); 3332 pktgen_remove_device(t, cur);
@@ -3393,8 +3390,7 @@ static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev)
3393 3390
3394 if (!netif_running(odev)) { 3391 if (!netif_running(odev)) {
3395 pktgen_stop_device(pkt_dev); 3392 pktgen_stop_device(pkt_dev);
3396 if (pkt_dev->skb) 3393 kfree_skb(pkt_dev->skb);
3397 kfree_skb(pkt_dev->skb);
3398 pkt_dev->skb = NULL; 3394 pkt_dev->skb = NULL;
3399 goto out; 3395 goto out;
3400 } 3396 }
@@ -3415,8 +3411,7 @@ static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev)
3415 if ((++pkt_dev->clone_count >= pkt_dev->clone_skb) 3411 if ((++pkt_dev->clone_count >= pkt_dev->clone_skb)
3416 || (!pkt_dev->skb)) { 3412 || (!pkt_dev->skb)) {
3417 /* build a new pkt */ 3413 /* build a new pkt */
3418 if (pkt_dev->skb) 3414 kfree_skb(pkt_dev->skb);
3419 kfree_skb(pkt_dev->skb);
3420 3415
3421 pkt_dev->skb = fill_packet(odev, pkt_dev); 3416 pkt_dev->skb = fill_packet(odev, pkt_dev);
3422 if (pkt_dev->skb == NULL) { 3417 if (pkt_dev->skb == NULL) {
@@ -3498,8 +3493,7 @@ static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev)
3498 3493
3499 /* Done with this */ 3494 /* Done with this */
3500 pktgen_stop_device(pkt_dev); 3495 pktgen_stop_device(pkt_dev);
3501 if (pkt_dev->skb) 3496 kfree_skb(pkt_dev->skb);
3502 kfree_skb(pkt_dev->skb);
3503 pkt_dev->skb = NULL; 3497 pkt_dev->skb = NULL;
3504 } 3498 }
3505out:; 3499out:;
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 790dd205bb5d..d78030f88bd0 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -455,8 +455,8 @@ int rtnl_unicast(struct sk_buff *skb, struct net *net, u32 pid)
455 return nlmsg_unicast(rtnl, skb, pid); 455 return nlmsg_unicast(rtnl, skb, pid);
456} 456}
457 457
458int rtnl_notify(struct sk_buff *skb, struct net *net, u32 pid, u32 group, 458void rtnl_notify(struct sk_buff *skb, struct net *net, u32 pid, u32 group,
459 struct nlmsghdr *nlh, gfp_t flags) 459 struct nlmsghdr *nlh, gfp_t flags)
460{ 460{
461 struct sock *rtnl = net->rtnl; 461 struct sock *rtnl = net->rtnl;
462 int report = 0; 462 int report = 0;
@@ -464,7 +464,7 @@ int rtnl_notify(struct sk_buff *skb, struct net *net, u32 pid, u32 group,
464 if (nlh) 464 if (nlh)
465 report = nlmsg_report(nlh); 465 report = nlmsg_report(nlh);
466 466
467 return nlmsg_notify(rtnl, skb, pid, group, report, flags); 467 nlmsg_notify(rtnl, skb, pid, group, report, flags);
468} 468}
469 469
470void rtnl_set_sk_err(struct net *net, u32 group, int error) 470void rtnl_set_sk_err(struct net *net, u32 group, int error)
@@ -1246,7 +1246,8 @@ void rtmsg_ifinfo(int type, struct net_device *dev, unsigned change)
1246 kfree_skb(skb); 1246 kfree_skb(skb);
1247 goto errout; 1247 goto errout;
1248 } 1248 }
1249 err = rtnl_notify(skb, net, 0, RTNLGRP_LINK, NULL, GFP_KERNEL); 1249 rtnl_notify(skb, net, 0, RTNLGRP_LINK, NULL, GFP_KERNEL);
1250 return;
1250errout: 1251errout:
1251 if (err < 0) 1252 if (err < 0)
1252 rtnl_set_sk_err(net, RTNLGRP_LINK, err); 1253 rtnl_set_sk_err(net, RTNLGRP_LINK, err);
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index c6a6b166f8d6..6acbf9e79eb1 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -55,6 +55,7 @@
55#include <linux/rtnetlink.h> 55#include <linux/rtnetlink.h>
56#include <linux/init.h> 56#include <linux/init.h>
57#include <linux/scatterlist.h> 57#include <linux/scatterlist.h>
58#include <linux/errqueue.h>
58 59
59#include <net/protocol.h> 60#include <net/protocol.h>
60#include <net/dst.h> 61#include <net/dst.h>
@@ -64,6 +65,7 @@
64 65
65#include <asm/uaccess.h> 66#include <asm/uaccess.h>
66#include <asm/system.h> 67#include <asm/system.h>
68#include <trace/skb.h>
67 69
68#include "kmap_skb.h" 70#include "kmap_skb.h"
69 71
@@ -123,6 +125,7 @@ void skb_over_panic(struct sk_buff *skb, int sz, void *here)
123 skb->dev ? skb->dev->name : "<NULL>"); 125 skb->dev ? skb->dev->name : "<NULL>");
124 BUG(); 126 BUG();
125} 127}
128EXPORT_SYMBOL(skb_over_panic);
126 129
127/** 130/**
128 * skb_under_panic - private function 131 * skb_under_panic - private function
@@ -142,6 +145,7 @@ void skb_under_panic(struct sk_buff *skb, int sz, void *here)
142 skb->dev ? skb->dev->name : "<NULL>"); 145 skb->dev ? skb->dev->name : "<NULL>");
143 BUG(); 146 BUG();
144} 147}
148EXPORT_SYMBOL(skb_under_panic);
145 149
146/* Allocate a new skbuff. We do this ourselves so we can fill in a few 150/* Allocate a new skbuff. We do this ourselves so we can fill in a few
147 * 'private' fields and also do memory statistics to find all the 151 * 'private' fields and also do memory statistics to find all the
@@ -205,7 +209,9 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
205 shinfo->gso_segs = 0; 209 shinfo->gso_segs = 0;
206 shinfo->gso_type = 0; 210 shinfo->gso_type = 0;
207 shinfo->ip6_frag_id = 0; 211 shinfo->ip6_frag_id = 0;
212 shinfo->tx_flags.flags = 0;
208 shinfo->frag_list = NULL; 213 shinfo->frag_list = NULL;
214 memset(&shinfo->hwtstamps, 0, sizeof(shinfo->hwtstamps));
209 215
210 if (fclone) { 216 if (fclone) {
211 struct sk_buff *child = skb + 1; 217 struct sk_buff *child = skb + 1;
@@ -223,6 +229,7 @@ nodata:
223 skb = NULL; 229 skb = NULL;
224 goto out; 230 goto out;
225} 231}
232EXPORT_SYMBOL(__alloc_skb);
226 233
227/** 234/**
228 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 235 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
@@ -250,6 +257,7 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
250 } 257 }
251 return skb; 258 return skb;
252} 259}
260EXPORT_SYMBOL(__netdev_alloc_skb);
253 261
254struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask) 262struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask)
255{ 263{
@@ -418,6 +426,7 @@ void __kfree_skb(struct sk_buff *skb)
418 skb_release_all(skb); 426 skb_release_all(skb);
419 kfree_skbmem(skb); 427 kfree_skbmem(skb);
420} 428}
429EXPORT_SYMBOL(__kfree_skb);
421 430
422/** 431/**
423 * kfree_skb - free an sk_buff 432 * kfree_skb - free an sk_buff
@@ -434,8 +443,30 @@ void kfree_skb(struct sk_buff *skb)
434 smp_rmb(); 443 smp_rmb();
435 else if (likely(!atomic_dec_and_test(&skb->users))) 444 else if (likely(!atomic_dec_and_test(&skb->users)))
436 return; 445 return;
446 trace_kfree_skb(skb, __builtin_return_address(0));
447 __kfree_skb(skb);
448}
449EXPORT_SYMBOL(kfree_skb);
450
451/**
452 * consume_skb - free an skbuff
453 * @skb: buffer to free
454 *
455 * Drop a ref to the buffer and free it if the usage count has hit zero
456 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
457 * is being dropped after a failure and notes that
458 */
459void consume_skb(struct sk_buff *skb)
460{
461 if (unlikely(!skb))
462 return;
463 if (likely(atomic_read(&skb->users) == 1))
464 smp_rmb();
465 else if (likely(!atomic_dec_and_test(&skb->users)))
466 return;
437 __kfree_skb(skb); 467 __kfree_skb(skb);
438} 468}
469EXPORT_SYMBOL(consume_skb);
439 470
440/** 471/**
441 * skb_recycle_check - check if skb can be reused for receive 472 * skb_recycle_check - check if skb can be reused for receive
@@ -605,6 +636,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
605 636
606 return __skb_clone(n, skb); 637 return __skb_clone(n, skb);
607} 638}
639EXPORT_SYMBOL(skb_clone);
608 640
609static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 641static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
610{ 642{
@@ -671,7 +703,7 @@ struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
671 copy_skb_header(n, skb); 703 copy_skb_header(n, skb);
672 return n; 704 return n;
673} 705}
674 706EXPORT_SYMBOL(skb_copy);
675 707
676/** 708/**
677 * pskb_copy - create copy of an sk_buff with private head. 709 * pskb_copy - create copy of an sk_buff with private head.
@@ -730,6 +762,7 @@ struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
730out: 762out:
731 return n; 763 return n;
732} 764}
765EXPORT_SYMBOL(pskb_copy);
733 766
734/** 767/**
735 * pskb_expand_head - reallocate header of &sk_buff 768 * pskb_expand_head - reallocate header of &sk_buff
@@ -813,6 +846,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
813nodata: 846nodata:
814 return -ENOMEM; 847 return -ENOMEM;
815} 848}
849EXPORT_SYMBOL(pskb_expand_head);
816 850
817/* Make private copy of skb with writable head and some headroom */ 851/* Make private copy of skb with writable head and some headroom */
818 852
@@ -833,7 +867,7 @@ struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
833 } 867 }
834 return skb2; 868 return skb2;
835} 869}
836 870EXPORT_SYMBOL(skb_realloc_headroom);
837 871
838/** 872/**
839 * skb_copy_expand - copy and expand sk_buff 873 * skb_copy_expand - copy and expand sk_buff
@@ -898,6 +932,7 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
898 932
899 return n; 933 return n;
900} 934}
935EXPORT_SYMBOL(skb_copy_expand);
901 936
902/** 937/**
903 * skb_pad - zero pad the tail of an skb 938 * skb_pad - zero pad the tail of an skb
@@ -943,6 +978,7 @@ free_skb:
943 kfree_skb(skb); 978 kfree_skb(skb);
944 return err; 979 return err;
945} 980}
981EXPORT_SYMBOL(skb_pad);
946 982
947/** 983/**
948 * skb_put - add data to a buffer 984 * skb_put - add data to a buffer
@@ -1100,6 +1136,7 @@ done:
1100 1136
1101 return 0; 1137 return 0;
1102} 1138}
1139EXPORT_SYMBOL(___pskb_trim);
1103 1140
1104/** 1141/**
1105 * __pskb_pull_tail - advance tail of skb header 1142 * __pskb_pull_tail - advance tail of skb header
@@ -1193,8 +1230,7 @@ unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
1193 insp = list; 1230 insp = list;
1194 } 1231 }
1195 if (!pskb_pull(list, eat)) { 1232 if (!pskb_pull(list, eat)) {
1196 if (clone) 1233 kfree_skb(clone);
1197 kfree_skb(clone);
1198 return NULL; 1234 return NULL;
1199 } 1235 }
1200 break; 1236 break;
@@ -1238,6 +1274,7 @@ pull_pages:
1238 1274
1239 return skb_tail_pointer(skb); 1275 return skb_tail_pointer(skb);
1240} 1276}
1277EXPORT_SYMBOL(__pskb_pull_tail);
1241 1278
1242/* Copy some data bits from skb to kernel buffer. */ 1279/* Copy some data bits from skb to kernel buffer. */
1243 1280
@@ -1315,6 +1352,7 @@ int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1315fault: 1352fault:
1316 return -EFAULT; 1353 return -EFAULT;
1317} 1354}
1355EXPORT_SYMBOL(skb_copy_bits);
1318 1356
1319/* 1357/*
1320 * Callback from splice_to_pipe(), if we need to release some pages 1358 * Callback from splice_to_pipe(), if we need to release some pages
@@ -1325,14 +1363,39 @@ static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
1325 put_page(spd->pages[i]); 1363 put_page(spd->pages[i]);
1326} 1364}
1327 1365
1328static inline struct page *linear_to_page(struct page *page, unsigned int len, 1366static inline struct page *linear_to_page(struct page *page, unsigned int *len,
1329 unsigned int offset) 1367 unsigned int *offset,
1368 struct sk_buff *skb)
1330{ 1369{
1331 struct page *p = alloc_pages(GFP_KERNEL, 0); 1370 struct sock *sk = skb->sk;
1371 struct page *p = sk->sk_sndmsg_page;
1372 unsigned int off;
1332 1373
1333 if (!p) 1374 if (!p) {
1334 return NULL; 1375new_page:
1335 memcpy(page_address(p) + offset, page_address(page) + offset, len); 1376 p = sk->sk_sndmsg_page = alloc_pages(sk->sk_allocation, 0);
1377 if (!p)
1378 return NULL;
1379
1380 off = sk->sk_sndmsg_off = 0;
1381 /* hold one ref to this page until it's full */
1382 } else {
1383 unsigned int mlen;
1384
1385 off = sk->sk_sndmsg_off;
1386 mlen = PAGE_SIZE - off;
1387 if (mlen < 64 && mlen < *len) {
1388 put_page(p);
1389 goto new_page;
1390 }
1391
1392 *len = min_t(unsigned int, *len, mlen);
1393 }
1394
1395 memcpy(page_address(p) + off, page_address(page) + *offset, *len);
1396 sk->sk_sndmsg_off += *len;
1397 *offset = off;
1398 get_page(p);
1336 1399
1337 return p; 1400 return p;
1338} 1401}
@@ -1341,21 +1404,21 @@ static inline struct page *linear_to_page(struct page *page, unsigned int len,
1341 * Fill page/offset/length into spd, if it can hold more pages. 1404 * Fill page/offset/length into spd, if it can hold more pages.
1342 */ 1405 */
1343static inline int spd_fill_page(struct splice_pipe_desc *spd, struct page *page, 1406static inline int spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
1344 unsigned int len, unsigned int offset, 1407 unsigned int *len, unsigned int offset,
1345 struct sk_buff *skb, int linear) 1408 struct sk_buff *skb, int linear)
1346{ 1409{
1347 if (unlikely(spd->nr_pages == PIPE_BUFFERS)) 1410 if (unlikely(spd->nr_pages == PIPE_BUFFERS))
1348 return 1; 1411 return 1;
1349 1412
1350 if (linear) { 1413 if (linear) {
1351 page = linear_to_page(page, len, offset); 1414 page = linear_to_page(page, len, &offset, skb);
1352 if (!page) 1415 if (!page)
1353 return 1; 1416 return 1;
1354 } else 1417 } else
1355 get_page(page); 1418 get_page(page);
1356 1419
1357 spd->pages[spd->nr_pages] = page; 1420 spd->pages[spd->nr_pages] = page;
1358 spd->partial[spd->nr_pages].len = len; 1421 spd->partial[spd->nr_pages].len = *len;
1359 spd->partial[spd->nr_pages].offset = offset; 1422 spd->partial[spd->nr_pages].offset = offset;
1360 spd->nr_pages++; 1423 spd->nr_pages++;
1361 1424
@@ -1365,8 +1428,13 @@ static inline int spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
1365static inline void __segment_seek(struct page **page, unsigned int *poff, 1428static inline void __segment_seek(struct page **page, unsigned int *poff,
1366 unsigned int *plen, unsigned int off) 1429 unsigned int *plen, unsigned int off)
1367{ 1430{
1431 unsigned long n;
1432
1368 *poff += off; 1433 *poff += off;
1369 *page += *poff / PAGE_SIZE; 1434 n = *poff / PAGE_SIZE;
1435 if (n)
1436 *page = nth_page(*page, n);
1437
1370 *poff = *poff % PAGE_SIZE; 1438 *poff = *poff % PAGE_SIZE;
1371 *plen -= off; 1439 *plen -= off;
1372} 1440}
@@ -1397,7 +1465,7 @@ static inline int __splice_segment(struct page *page, unsigned int poff,
1397 /* the linear region may spread across several pages */ 1465 /* the linear region may spread across several pages */
1398 flen = min_t(unsigned int, flen, PAGE_SIZE - poff); 1466 flen = min_t(unsigned int, flen, PAGE_SIZE - poff);
1399 1467
1400 if (spd_fill_page(spd, page, flen, poff, skb, linear)) 1468 if (spd_fill_page(spd, page, &flen, poff, skb, linear))
1401 return 1; 1469 return 1;
1402 1470
1403 __segment_seek(&page, &poff, &plen, flen); 1471 __segment_seek(&page, &poff, &plen, flen);
@@ -1590,7 +1658,6 @@ int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
1590fault: 1658fault:
1591 return -EFAULT; 1659 return -EFAULT;
1592} 1660}
1593
1594EXPORT_SYMBOL(skb_store_bits); 1661EXPORT_SYMBOL(skb_store_bits);
1595 1662
1596/* Checksum skb data. */ 1663/* Checksum skb data. */
@@ -1667,6 +1734,7 @@ __wsum skb_checksum(const struct sk_buff *skb, int offset,
1667 1734
1668 return csum; 1735 return csum;
1669} 1736}
1737EXPORT_SYMBOL(skb_checksum);
1670 1738
1671/* Both of above in one bottle. */ 1739/* Both of above in one bottle. */
1672 1740
@@ -1748,6 +1816,7 @@ __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
1748 BUG_ON(len); 1816 BUG_ON(len);
1749 return csum; 1817 return csum;
1750} 1818}
1819EXPORT_SYMBOL(skb_copy_and_csum_bits);
1751 1820
1752void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 1821void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
1753{ 1822{
@@ -1774,6 +1843,7 @@ void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
1774 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 1843 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
1775 } 1844 }
1776} 1845}
1846EXPORT_SYMBOL(skb_copy_and_csum_dev);
1777 1847
1778/** 1848/**
1779 * skb_dequeue - remove from the head of the queue 1849 * skb_dequeue - remove from the head of the queue
@@ -1794,6 +1864,7 @@ struct sk_buff *skb_dequeue(struct sk_buff_head *list)
1794 spin_unlock_irqrestore(&list->lock, flags); 1864 spin_unlock_irqrestore(&list->lock, flags);
1795 return result; 1865 return result;
1796} 1866}
1867EXPORT_SYMBOL(skb_dequeue);
1797 1868
1798/** 1869/**
1799 * skb_dequeue_tail - remove from the tail of the queue 1870 * skb_dequeue_tail - remove from the tail of the queue
@@ -1813,6 +1884,7 @@ struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
1813 spin_unlock_irqrestore(&list->lock, flags); 1884 spin_unlock_irqrestore(&list->lock, flags);
1814 return result; 1885 return result;
1815} 1886}
1887EXPORT_SYMBOL(skb_dequeue_tail);
1816 1888
1817/** 1889/**
1818 * skb_queue_purge - empty a list 1890 * skb_queue_purge - empty a list
@@ -1828,6 +1900,7 @@ void skb_queue_purge(struct sk_buff_head *list)
1828 while ((skb = skb_dequeue(list)) != NULL) 1900 while ((skb = skb_dequeue(list)) != NULL)
1829 kfree_skb(skb); 1901 kfree_skb(skb);
1830} 1902}
1903EXPORT_SYMBOL(skb_queue_purge);
1831 1904
1832/** 1905/**
1833 * skb_queue_head - queue a buffer at the list head 1906 * skb_queue_head - queue a buffer at the list head
@@ -1848,6 +1921,7 @@ void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
1848 __skb_queue_head(list, newsk); 1921 __skb_queue_head(list, newsk);
1849 spin_unlock_irqrestore(&list->lock, flags); 1922 spin_unlock_irqrestore(&list->lock, flags);
1850} 1923}
1924EXPORT_SYMBOL(skb_queue_head);
1851 1925
1852/** 1926/**
1853 * skb_queue_tail - queue a buffer at the list tail 1927 * skb_queue_tail - queue a buffer at the list tail
@@ -1868,6 +1942,7 @@ void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
1868 __skb_queue_tail(list, newsk); 1942 __skb_queue_tail(list, newsk);
1869 spin_unlock_irqrestore(&list->lock, flags); 1943 spin_unlock_irqrestore(&list->lock, flags);
1870} 1944}
1945EXPORT_SYMBOL(skb_queue_tail);
1871 1946
1872/** 1947/**
1873 * skb_unlink - remove a buffer from a list 1948 * skb_unlink - remove a buffer from a list
@@ -1887,6 +1962,7 @@ void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1887 __skb_unlink(skb, list); 1962 __skb_unlink(skb, list);
1888 spin_unlock_irqrestore(&list->lock, flags); 1963 spin_unlock_irqrestore(&list->lock, flags);
1889} 1964}
1965EXPORT_SYMBOL(skb_unlink);
1890 1966
1891/** 1967/**
1892 * skb_append - append a buffer 1968 * skb_append - append a buffer
@@ -1906,7 +1982,7 @@ void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head
1906 __skb_queue_after(list, old, newsk); 1982 __skb_queue_after(list, old, newsk);
1907 spin_unlock_irqrestore(&list->lock, flags); 1983 spin_unlock_irqrestore(&list->lock, flags);
1908} 1984}
1909 1985EXPORT_SYMBOL(skb_append);
1910 1986
1911/** 1987/**
1912 * skb_insert - insert a buffer 1988 * skb_insert - insert a buffer
@@ -1928,6 +2004,7 @@ void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head
1928 __skb_insert(newsk, old->prev, old, list); 2004 __skb_insert(newsk, old->prev, old, list);
1929 spin_unlock_irqrestore(&list->lock, flags); 2005 spin_unlock_irqrestore(&list->lock, flags);
1930} 2006}
2007EXPORT_SYMBOL(skb_insert);
1931 2008
1932static inline void skb_split_inside_header(struct sk_buff *skb, 2009static inline void skb_split_inside_header(struct sk_buff *skb,
1933 struct sk_buff* skb1, 2010 struct sk_buff* skb1,
@@ -2006,6 +2083,7 @@ void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
2006 else /* Second chunk has no header, nothing to copy. */ 2083 else /* Second chunk has no header, nothing to copy. */
2007 skb_split_no_header(skb, skb1, len, pos); 2084 skb_split_no_header(skb, skb1, len, pos);
2008} 2085}
2086EXPORT_SYMBOL(skb_split);
2009 2087
2010/* Shifting from/to a cloned skb is a no-go. 2088/* Shifting from/to a cloned skb is a no-go.
2011 * 2089 *
@@ -2168,6 +2246,7 @@ void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
2168 st->frag_idx = st->stepped_offset = 0; 2246 st->frag_idx = st->stepped_offset = 0;
2169 st->frag_data = NULL; 2247 st->frag_data = NULL;
2170} 2248}
2249EXPORT_SYMBOL(skb_prepare_seq_read);
2171 2250
2172/** 2251/**
2173 * skb_seq_read - Sequentially read skb data 2252 * skb_seq_read - Sequentially read skb data
@@ -2255,6 +2334,7 @@ next_skb:
2255 2334
2256 return 0; 2335 return 0;
2257} 2336}
2337EXPORT_SYMBOL(skb_seq_read);
2258 2338
2259/** 2339/**
2260 * skb_abort_seq_read - Abort a sequential read of skb data 2340 * skb_abort_seq_read - Abort a sequential read of skb data
@@ -2268,6 +2348,7 @@ void skb_abort_seq_read(struct skb_seq_state *st)
2268 if (st->frag_data) 2348 if (st->frag_data)
2269 kunmap_skb_frag(st->frag_data); 2349 kunmap_skb_frag(st->frag_data);
2270} 2350}
2351EXPORT_SYMBOL(skb_abort_seq_read);
2271 2352
2272#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 2353#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
2273 2354
@@ -2310,6 +2391,7 @@ unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
2310 ret = textsearch_find(config, state); 2391 ret = textsearch_find(config, state);
2311 return (ret <= to - from ? ret : UINT_MAX); 2392 return (ret <= to - from ? ret : UINT_MAX);
2312} 2393}
2394EXPORT_SYMBOL(skb_find_text);
2313 2395
2314/** 2396/**
2315 * skb_append_datato_frags: - append the user data to a skb 2397 * skb_append_datato_frags: - append the user data to a skb
@@ -2382,6 +2464,7 @@ int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
2382 2464
2383 return 0; 2465 return 0;
2384} 2466}
2467EXPORT_SYMBOL(skb_append_datato_frags);
2385 2468
2386/** 2469/**
2387 * skb_pull_rcsum - pull skb and update receive checksum 2470 * skb_pull_rcsum - pull skb and update receive checksum
@@ -2569,7 +2652,6 @@ err:
2569 } 2652 }
2570 return ERR_PTR(err); 2653 return ERR_PTR(err);
2571} 2654}
2572
2573EXPORT_SYMBOL_GPL(skb_segment); 2655EXPORT_SYMBOL_GPL(skb_segment);
2574 2656
2575int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) 2657int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
@@ -2577,17 +2659,23 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2577 struct sk_buff *p = *head; 2659 struct sk_buff *p = *head;
2578 struct sk_buff *nskb; 2660 struct sk_buff *nskb;
2579 unsigned int headroom; 2661 unsigned int headroom;
2580 unsigned int hlen = p->data - skb_mac_header(p); 2662 unsigned int len = skb_gro_len(skb);
2581 unsigned int len = skb->len;
2582 2663
2583 if (hlen + p->len + len >= 65536) 2664 if (p->len + len >= 65536)
2584 return -E2BIG; 2665 return -E2BIG;
2585 2666
2586 if (skb_shinfo(p)->frag_list) 2667 if (skb_shinfo(p)->frag_list)
2587 goto merge; 2668 goto merge;
2588 else if (!skb_headlen(p) && !skb_headlen(skb) && 2669 else if (skb_headlen(skb) <= skb_gro_offset(skb)) {
2589 skb_shinfo(p)->nr_frags + skb_shinfo(skb)->nr_frags < 2670 if (skb_shinfo(p)->nr_frags + skb_shinfo(skb)->nr_frags >
2590 MAX_SKB_FRAGS) { 2671 MAX_SKB_FRAGS)
2672 return -E2BIG;
2673
2674 skb_shinfo(skb)->frags[0].page_offset +=
2675 skb_gro_offset(skb) - skb_headlen(skb);
2676 skb_shinfo(skb)->frags[0].size -=
2677 skb_gro_offset(skb) - skb_headlen(skb);
2678
2591 memcpy(skb_shinfo(p)->frags + skb_shinfo(p)->nr_frags, 2679 memcpy(skb_shinfo(p)->frags + skb_shinfo(p)->nr_frags,
2592 skb_shinfo(skb)->frags, 2680 skb_shinfo(skb)->frags,
2593 skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t)); 2681 skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
@@ -2604,7 +2692,7 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2604 } 2692 }
2605 2693
2606 headroom = skb_headroom(p); 2694 headroom = skb_headroom(p);
2607 nskb = netdev_alloc_skb(p->dev, headroom); 2695 nskb = netdev_alloc_skb(p->dev, headroom + skb_gro_offset(p));
2608 if (unlikely(!nskb)) 2696 if (unlikely(!nskb))
2609 return -ENOMEM; 2697 return -ENOMEM;
2610 2698
@@ -2612,12 +2700,15 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2612 nskb->mac_len = p->mac_len; 2700 nskb->mac_len = p->mac_len;
2613 2701
2614 skb_reserve(nskb, headroom); 2702 skb_reserve(nskb, headroom);
2703 __skb_put(nskb, skb_gro_offset(p));
2615 2704
2616 skb_set_mac_header(nskb, -hlen); 2705 skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
2617 skb_set_network_header(nskb, skb_network_offset(p)); 2706 skb_set_network_header(nskb, skb_network_offset(p));
2618 skb_set_transport_header(nskb, skb_transport_offset(p)); 2707 skb_set_transport_header(nskb, skb_transport_offset(p));
2619 2708
2620 memcpy(skb_mac_header(nskb), skb_mac_header(p), hlen); 2709 __skb_pull(p, skb_gro_offset(p));
2710 memcpy(skb_mac_header(nskb), skb_mac_header(p),
2711 p->data - skb_mac_header(p));
2621 2712
2622 *NAPI_GRO_CB(nskb) = *NAPI_GRO_CB(p); 2713 *NAPI_GRO_CB(nskb) = *NAPI_GRO_CB(p);
2623 skb_shinfo(nskb)->frag_list = p; 2714 skb_shinfo(nskb)->frag_list = p;
@@ -2636,6 +2727,17 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2636 p = nskb; 2727 p = nskb;
2637 2728
2638merge: 2729merge:
2730 if (skb_gro_offset(skb) > skb_headlen(skb)) {
2731 skb_shinfo(skb)->frags[0].page_offset +=
2732 skb_gro_offset(skb) - skb_headlen(skb);
2733 skb_shinfo(skb)->frags[0].size -=
2734 skb_gro_offset(skb) - skb_headlen(skb);
2735 skb_gro_reset_offset(skb);
2736 skb_gro_pull(skb, skb_headlen(skb));
2737 }
2738
2739 __skb_pull(skb, skb_gro_offset(skb));
2740
2639 p->prev->next = skb; 2741 p->prev->next = skb;
2640 p->prev = skb; 2742 p->prev = skb;
2641 skb_header_release(skb); 2743 skb_header_release(skb);
@@ -2747,6 +2849,7 @@ int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int le
2747 2849
2748 return nsg; 2850 return nsg;
2749} 2851}
2852EXPORT_SYMBOL_GPL(skb_to_sgvec);
2750 2853
2751/** 2854/**
2752 * skb_cow_data - Check that a socket buffer's data buffers are writable 2855 * skb_cow_data - Check that a socket buffer's data buffers are writable
@@ -2856,6 +2959,45 @@ int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
2856 2959
2857 return elt; 2960 return elt;
2858} 2961}
2962EXPORT_SYMBOL_GPL(skb_cow_data);
2963
2964void skb_tstamp_tx(struct sk_buff *orig_skb,
2965 struct skb_shared_hwtstamps *hwtstamps)
2966{
2967 struct sock *sk = orig_skb->sk;
2968 struct sock_exterr_skb *serr;
2969 struct sk_buff *skb;
2970 int err;
2971
2972 if (!sk)
2973 return;
2974
2975 skb = skb_clone(orig_skb, GFP_ATOMIC);
2976 if (!skb)
2977 return;
2978
2979 if (hwtstamps) {
2980 *skb_hwtstamps(skb) =
2981 *hwtstamps;
2982 } else {
2983 /*
2984 * no hardware time stamps available,
2985 * so keep the skb_shared_tx and only
2986 * store software time stamp
2987 */
2988 skb->tstamp = ktime_get_real();
2989 }
2990
2991 serr = SKB_EXT_ERR(skb);
2992 memset(serr, 0, sizeof(*serr));
2993 serr->ee.ee_errno = ENOMSG;
2994 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
2995 err = sock_queue_err_skb(sk, skb);
2996 if (err)
2997 kfree_skb(skb);
2998}
2999EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3000
2859 3001
2860/** 3002/**
2861 * skb_partial_csum_set - set up and verify partial csum values for packet 3003 * skb_partial_csum_set - set up and verify partial csum values for packet
@@ -2884,6 +3026,7 @@ bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
2884 skb->csum_offset = off; 3026 skb->csum_offset = off;
2885 return true; 3027 return true;
2886} 3028}
3029EXPORT_SYMBOL_GPL(skb_partial_csum_set);
2887 3030
2888void __skb_warn_lro_forwarding(const struct sk_buff *skb) 3031void __skb_warn_lro_forwarding(const struct sk_buff *skb)
2889{ 3032{
@@ -2891,42 +3034,4 @@ void __skb_warn_lro_forwarding(const struct sk_buff *skb)
2891 pr_warning("%s: received packets cannot be forwarded" 3034 pr_warning("%s: received packets cannot be forwarded"
2892 " while LRO is enabled\n", skb->dev->name); 3035 " while LRO is enabled\n", skb->dev->name);
2893} 3036}
2894
2895EXPORT_SYMBOL(___pskb_trim);
2896EXPORT_SYMBOL(__kfree_skb);
2897EXPORT_SYMBOL(kfree_skb);
2898EXPORT_SYMBOL(__pskb_pull_tail);
2899EXPORT_SYMBOL(__alloc_skb);
2900EXPORT_SYMBOL(__netdev_alloc_skb);
2901EXPORT_SYMBOL(pskb_copy);
2902EXPORT_SYMBOL(pskb_expand_head);
2903EXPORT_SYMBOL(skb_checksum);
2904EXPORT_SYMBOL(skb_clone);
2905EXPORT_SYMBOL(skb_copy);
2906EXPORT_SYMBOL(skb_copy_and_csum_bits);
2907EXPORT_SYMBOL(skb_copy_and_csum_dev);
2908EXPORT_SYMBOL(skb_copy_bits);
2909EXPORT_SYMBOL(skb_copy_expand);
2910EXPORT_SYMBOL(skb_over_panic);
2911EXPORT_SYMBOL(skb_pad);
2912EXPORT_SYMBOL(skb_realloc_headroom);
2913EXPORT_SYMBOL(skb_under_panic);
2914EXPORT_SYMBOL(skb_dequeue);
2915EXPORT_SYMBOL(skb_dequeue_tail);
2916EXPORT_SYMBOL(skb_insert);
2917EXPORT_SYMBOL(skb_queue_purge);
2918EXPORT_SYMBOL(skb_queue_head);
2919EXPORT_SYMBOL(skb_queue_tail);
2920EXPORT_SYMBOL(skb_unlink);
2921EXPORT_SYMBOL(skb_append);
2922EXPORT_SYMBOL(skb_split);
2923EXPORT_SYMBOL(skb_prepare_seq_read);
2924EXPORT_SYMBOL(skb_seq_read);
2925EXPORT_SYMBOL(skb_abort_seq_read);
2926EXPORT_SYMBOL(skb_find_text);
2927EXPORT_SYMBOL(skb_append_datato_frags);
2928EXPORT_SYMBOL(__skb_warn_lro_forwarding); 3037EXPORT_SYMBOL(__skb_warn_lro_forwarding);
2929
2930EXPORT_SYMBOL_GPL(skb_to_sgvec);
2931EXPORT_SYMBOL_GPL(skb_cow_data);
2932EXPORT_SYMBOL_GPL(skb_partial_csum_set);
diff --git a/net/core/sock.c b/net/core/sock.c
index 5f97caa158e8..0620046e4eba 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -120,6 +120,7 @@
120#include <net/net_namespace.h> 120#include <net/net_namespace.h>
121#include <net/request_sock.h> 121#include <net/request_sock.h>
122#include <net/sock.h> 122#include <net/sock.h>
123#include <linux/net_tstamp.h>
123#include <net/xfrm.h> 124#include <net/xfrm.h>
124#include <linux/ipsec.h> 125#include <linux/ipsec.h>
125 126
@@ -149,7 +150,7 @@ static const char *af_family_key_strings[AF_MAX+1] = {
149 "sk_lock-AF_DECnet", "sk_lock-AF_NETBEUI" , "sk_lock-AF_SECURITY" , 150 "sk_lock-AF_DECnet", "sk_lock-AF_NETBEUI" , "sk_lock-AF_SECURITY" ,
150 "sk_lock-AF_KEY" , "sk_lock-AF_NETLINK" , "sk_lock-AF_PACKET" , 151 "sk_lock-AF_KEY" , "sk_lock-AF_NETLINK" , "sk_lock-AF_PACKET" ,
151 "sk_lock-AF_ASH" , "sk_lock-AF_ECONET" , "sk_lock-AF_ATMSVC" , 152 "sk_lock-AF_ASH" , "sk_lock-AF_ECONET" , "sk_lock-AF_ATMSVC" ,
152 "sk_lock-21" , "sk_lock-AF_SNA" , "sk_lock-AF_IRDA" , 153 "sk_lock-AF_RDS" , "sk_lock-AF_SNA" , "sk_lock-AF_IRDA" ,
153 "sk_lock-AF_PPPOX" , "sk_lock-AF_WANPIPE" , "sk_lock-AF_LLC" , 154 "sk_lock-AF_PPPOX" , "sk_lock-AF_WANPIPE" , "sk_lock-AF_LLC" ,
154 "sk_lock-27" , "sk_lock-28" , "sk_lock-AF_CAN" , 155 "sk_lock-27" , "sk_lock-28" , "sk_lock-AF_CAN" ,
155 "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-IUCV" , 156 "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-IUCV" ,
@@ -164,7 +165,7 @@ static const char *af_family_slock_key_strings[AF_MAX+1] = {
164 "slock-AF_DECnet", "slock-AF_NETBEUI" , "slock-AF_SECURITY" , 165 "slock-AF_DECnet", "slock-AF_NETBEUI" , "slock-AF_SECURITY" ,
165 "slock-AF_KEY" , "slock-AF_NETLINK" , "slock-AF_PACKET" , 166 "slock-AF_KEY" , "slock-AF_NETLINK" , "slock-AF_PACKET" ,
166 "slock-AF_ASH" , "slock-AF_ECONET" , "slock-AF_ATMSVC" , 167 "slock-AF_ASH" , "slock-AF_ECONET" , "slock-AF_ATMSVC" ,
167 "slock-21" , "slock-AF_SNA" , "slock-AF_IRDA" , 168 "slock-AF_RDS" , "slock-AF_SNA" , "slock-AF_IRDA" ,
168 "slock-AF_PPPOX" , "slock-AF_WANPIPE" , "slock-AF_LLC" , 169 "slock-AF_PPPOX" , "slock-AF_WANPIPE" , "slock-AF_LLC" ,
169 "slock-27" , "slock-28" , "slock-AF_CAN" , 170 "slock-27" , "slock-28" , "slock-AF_CAN" ,
170 "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_IUCV" , 171 "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_IUCV" ,
@@ -179,7 +180,7 @@ static const char *af_family_clock_key_strings[AF_MAX+1] = {
179 "clock-AF_DECnet", "clock-AF_NETBEUI" , "clock-AF_SECURITY" , 180 "clock-AF_DECnet", "clock-AF_NETBEUI" , "clock-AF_SECURITY" ,
180 "clock-AF_KEY" , "clock-AF_NETLINK" , "clock-AF_PACKET" , 181 "clock-AF_KEY" , "clock-AF_NETLINK" , "clock-AF_PACKET" ,
181 "clock-AF_ASH" , "clock-AF_ECONET" , "clock-AF_ATMSVC" , 182 "clock-AF_ASH" , "clock-AF_ECONET" , "clock-AF_ATMSVC" ,
182 "clock-21" , "clock-AF_SNA" , "clock-AF_IRDA" , 183 "clock-AF_RDS" , "clock-AF_SNA" , "clock-AF_IRDA" ,
183 "clock-AF_PPPOX" , "clock-AF_WANPIPE" , "clock-AF_LLC" , 184 "clock-AF_PPPOX" , "clock-AF_WANPIPE" , "clock-AF_LLC" ,
184 "clock-27" , "clock-28" , "clock-AF_CAN" , 185 "clock-27" , "clock-28" , "clock-AF_CAN" ,
185 "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" , 186 "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" ,
@@ -255,11 +256,14 @@ static void sock_warn_obsolete_bsdism(const char *name)
255 } 256 }
256} 257}
257 258
258static void sock_disable_timestamp(struct sock *sk) 259static void sock_disable_timestamp(struct sock *sk, int flag)
259{ 260{
260 if (sock_flag(sk, SOCK_TIMESTAMP)) { 261 if (sock_flag(sk, flag)) {
261 sock_reset_flag(sk, SOCK_TIMESTAMP); 262 sock_reset_flag(sk, flag);
262 net_disable_timestamp(); 263 if (!sock_flag(sk, SOCK_TIMESTAMP) &&
264 !sock_flag(sk, SOCK_TIMESTAMPING_RX_SOFTWARE)) {
265 net_disable_timestamp();
266 }
263 } 267 }
264} 268}
265 269
@@ -614,13 +618,38 @@ set_rcvbuf:
614 else 618 else
615 sock_set_flag(sk, SOCK_RCVTSTAMPNS); 619 sock_set_flag(sk, SOCK_RCVTSTAMPNS);
616 sock_set_flag(sk, SOCK_RCVTSTAMP); 620 sock_set_flag(sk, SOCK_RCVTSTAMP);
617 sock_enable_timestamp(sk); 621 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
618 } else { 622 } else {
619 sock_reset_flag(sk, SOCK_RCVTSTAMP); 623 sock_reset_flag(sk, SOCK_RCVTSTAMP);
620 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 624 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
621 } 625 }
622 break; 626 break;
623 627
628 case SO_TIMESTAMPING:
629 if (val & ~SOF_TIMESTAMPING_MASK) {
630 ret = EINVAL;
631 break;
632 }
633 sock_valbool_flag(sk, SOCK_TIMESTAMPING_TX_HARDWARE,
634 val & SOF_TIMESTAMPING_TX_HARDWARE);
635 sock_valbool_flag(sk, SOCK_TIMESTAMPING_TX_SOFTWARE,
636 val & SOF_TIMESTAMPING_TX_SOFTWARE);
637 sock_valbool_flag(sk, SOCK_TIMESTAMPING_RX_HARDWARE,
638 val & SOF_TIMESTAMPING_RX_HARDWARE);
639 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
640 sock_enable_timestamp(sk,
641 SOCK_TIMESTAMPING_RX_SOFTWARE);
642 else
643 sock_disable_timestamp(sk,
644 SOCK_TIMESTAMPING_RX_SOFTWARE);
645 sock_valbool_flag(sk, SOCK_TIMESTAMPING_SOFTWARE,
646 val & SOF_TIMESTAMPING_SOFTWARE);
647 sock_valbool_flag(sk, SOCK_TIMESTAMPING_SYS_HARDWARE,
648 val & SOF_TIMESTAMPING_SYS_HARDWARE);
649 sock_valbool_flag(sk, SOCK_TIMESTAMPING_RAW_HARDWARE,
650 val & SOF_TIMESTAMPING_RAW_HARDWARE);
651 break;
652
624 case SO_RCVLOWAT: 653 case SO_RCVLOWAT:
625 if (val < 0) 654 if (val < 0)
626 val = INT_MAX; 655 val = INT_MAX;
@@ -768,6 +797,24 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
768 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS); 797 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS);
769 break; 798 break;
770 799
800 case SO_TIMESTAMPING:
801 v.val = 0;
802 if (sock_flag(sk, SOCK_TIMESTAMPING_TX_HARDWARE))
803 v.val |= SOF_TIMESTAMPING_TX_HARDWARE;
804 if (sock_flag(sk, SOCK_TIMESTAMPING_TX_SOFTWARE))
805 v.val |= SOF_TIMESTAMPING_TX_SOFTWARE;
806 if (sock_flag(sk, SOCK_TIMESTAMPING_RX_HARDWARE))
807 v.val |= SOF_TIMESTAMPING_RX_HARDWARE;
808 if (sock_flag(sk, SOCK_TIMESTAMPING_RX_SOFTWARE))
809 v.val |= SOF_TIMESTAMPING_RX_SOFTWARE;
810 if (sock_flag(sk, SOCK_TIMESTAMPING_SOFTWARE))
811 v.val |= SOF_TIMESTAMPING_SOFTWARE;
812 if (sock_flag(sk, SOCK_TIMESTAMPING_SYS_HARDWARE))
813 v.val |= SOF_TIMESTAMPING_SYS_HARDWARE;
814 if (sock_flag(sk, SOCK_TIMESTAMPING_RAW_HARDWARE))
815 v.val |= SOF_TIMESTAMPING_RAW_HARDWARE;
816 break;
817
771 case SO_RCVTIMEO: 818 case SO_RCVTIMEO:
772 lv=sizeof(struct timeval); 819 lv=sizeof(struct timeval);
773 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) { 820 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) {
@@ -969,7 +1016,8 @@ void sk_free(struct sock *sk)
969 rcu_assign_pointer(sk->sk_filter, NULL); 1016 rcu_assign_pointer(sk->sk_filter, NULL);
970 } 1017 }
971 1018
972 sock_disable_timestamp(sk); 1019 sock_disable_timestamp(sk, SOCK_TIMESTAMP);
1020 sock_disable_timestamp(sk, SOCK_TIMESTAMPING_RX_SOFTWARE);
973 1021
974 if (atomic_read(&sk->sk_omem_alloc)) 1022 if (atomic_read(&sk->sk_omem_alloc))
975 printk(KERN_DEBUG "%s: optmem leakage (%d bytes) detected.\n", 1023 printk(KERN_DEBUG "%s: optmem leakage (%d bytes) detected.\n",
@@ -1255,10 +1303,9 @@ static long sock_wait_for_wmem(struct sock * sk, long timeo)
1255 * Generic send/receive buffer handlers 1303 * Generic send/receive buffer handlers
1256 */ 1304 */
1257 1305
1258static struct sk_buff *sock_alloc_send_pskb(struct sock *sk, 1306struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1259 unsigned long header_len, 1307 unsigned long data_len, int noblock,
1260 unsigned long data_len, 1308 int *errcode)
1261 int noblock, int *errcode)
1262{ 1309{
1263 struct sk_buff *skb; 1310 struct sk_buff *skb;
1264 gfp_t gfp_mask; 1311 gfp_t gfp_mask;
@@ -1338,6 +1385,7 @@ failure:
1338 *errcode = err; 1385 *errcode = err;
1339 return NULL; 1386 return NULL;
1340} 1387}
1388EXPORT_SYMBOL(sock_alloc_send_pskb);
1341 1389
1342struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size, 1390struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1343 int noblock, int *errcode) 1391 int noblock, int *errcode)
@@ -1786,7 +1834,7 @@ int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp)
1786{ 1834{
1787 struct timeval tv; 1835 struct timeval tv;
1788 if (!sock_flag(sk, SOCK_TIMESTAMP)) 1836 if (!sock_flag(sk, SOCK_TIMESTAMP))
1789 sock_enable_timestamp(sk); 1837 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
1790 tv = ktime_to_timeval(sk->sk_stamp); 1838 tv = ktime_to_timeval(sk->sk_stamp);
1791 if (tv.tv_sec == -1) 1839 if (tv.tv_sec == -1)
1792 return -ENOENT; 1840 return -ENOENT;
@@ -1802,7 +1850,7 @@ int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
1802{ 1850{
1803 struct timespec ts; 1851 struct timespec ts;
1804 if (!sock_flag(sk, SOCK_TIMESTAMP)) 1852 if (!sock_flag(sk, SOCK_TIMESTAMP))
1805 sock_enable_timestamp(sk); 1853 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
1806 ts = ktime_to_timespec(sk->sk_stamp); 1854 ts = ktime_to_timespec(sk->sk_stamp);
1807 if (ts.tv_sec == -1) 1855 if (ts.tv_sec == -1)
1808 return -ENOENT; 1856 return -ENOENT;
@@ -1814,11 +1862,20 @@ int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
1814} 1862}
1815EXPORT_SYMBOL(sock_get_timestampns); 1863EXPORT_SYMBOL(sock_get_timestampns);
1816 1864
1817void sock_enable_timestamp(struct sock *sk) 1865void sock_enable_timestamp(struct sock *sk, int flag)
1818{ 1866{
1819 if (!sock_flag(sk, SOCK_TIMESTAMP)) { 1867 if (!sock_flag(sk, flag)) {
1820 sock_set_flag(sk, SOCK_TIMESTAMP); 1868 sock_set_flag(sk, flag);
1821 net_enable_timestamp(); 1869 /*
1870 * we just set one of the two flags which require net
1871 * time stamping, but time stamping might have been on
1872 * already because of the other one
1873 */
1874 if (!sock_flag(sk,
1875 flag == SOCK_TIMESTAMP ?
1876 SOCK_TIMESTAMPING_RX_SOFTWARE :
1877 SOCK_TIMESTAMP))
1878 net_enable_timestamp();
1822 } 1879 }
1823} 1880}
1824 1881
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index 83d3398559ea..7db1de0497c6 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -11,6 +11,7 @@
11#include <linux/socket.h> 11#include <linux/socket.h>
12#include <linux/netdevice.h> 12#include <linux/netdevice.h>
13#include <linux/init.h> 13#include <linux/init.h>
14#include <net/ip.h>
14#include <net/sock.h> 15#include <net/sock.h>
15 16
16static struct ctl_table net_core_table[] = { 17static struct ctl_table net_core_table[] = {
diff --git a/net/core/utils.c b/net/core/utils.c
index 72e0ebe964a0..83221aee7084 100644
--- a/net/core/utils.c
+++ b/net/core/utils.c
@@ -22,7 +22,6 @@
22#include <linux/net.h> 22#include <linux/net.h>
23#include <linux/string.h> 23#include <linux/string.h>
24#include <linux/types.h> 24#include <linux/types.h>
25#include <linux/random.h>
26#include <linux/percpu.h> 25#include <linux/percpu.h>
27#include <linux/init.h> 26#include <linux/init.h>
28#include <net/sock.h> 27#include <net/sock.h>
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-09 00:23:15 -0400