diff options
Diffstat (limited to 'net/core')
-rw-r--r-- | net/core/datagram.c | 8 | ||||
-rw-r--r-- | net/core/dev.c | 144 | ||||
-rw-r--r-- | net/core/dev_addr_lists.c | 4 | ||||
-rw-r--r-- | net/core/dst.c | 15 | ||||
-rw-r--r-- | net/core/ethtool.c | 20 | ||||
-rw-r--r-- | net/core/fib_rules.c | 4 | ||||
-rw-r--r-- | net/core/filter.c | 2 | ||||
-rw-r--r-- | net/core/kmap_skb.h | 2 | ||||
-rw-r--r-- | net/core/link_watch.c | 9 | ||||
-rw-r--r-- | net/core/neighbour.c | 40 | ||||
-rw-r--r-- | net/core/net-sysfs.c | 12 | ||||
-rw-r--r-- | net/core/netpoll.c | 4 | ||||
-rw-r--r-- | net/core/pktgen.c | 3 | ||||
-rw-r--r-- | net/core/rtnetlink.c | 1 | ||||
-rw-r--r-- | net/core/skbuff.c | 31 | ||||
-rw-r--r-- | net/core/sock.c | 16 | ||||
-rw-r--r-- | net/core/user_dma.c | 2 |
17 files changed, 187 insertions, 130 deletions
diff --git a/net/core/datagram.c b/net/core/datagram.c index 18ac112ea7ae..6449bed457d4 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c | |||
@@ -332,7 +332,7 @@ int skb_copy_datagram_iovec(const struct sk_buff *skb, int offset, | |||
332 | int err; | 332 | int err; |
333 | u8 *vaddr; | 333 | u8 *vaddr; |
334 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; | 334 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
335 | struct page *page = frag->page; | 335 | struct page *page = skb_frag_page(frag); |
336 | 336 | ||
337 | if (copy > len) | 337 | if (copy > len) |
338 | copy = len; | 338 | copy = len; |
@@ -418,7 +418,7 @@ int skb_copy_datagram_const_iovec(const struct sk_buff *skb, int offset, | |||
418 | int err; | 418 | int err; |
419 | u8 *vaddr; | 419 | u8 *vaddr; |
420 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; | 420 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
421 | struct page *page = frag->page; | 421 | struct page *page = skb_frag_page(frag); |
422 | 422 | ||
423 | if (copy > len) | 423 | if (copy > len) |
424 | copy = len; | 424 | copy = len; |
@@ -508,7 +508,7 @@ int skb_copy_datagram_from_iovec(struct sk_buff *skb, int offset, | |||
508 | int err; | 508 | int err; |
509 | u8 *vaddr; | 509 | u8 *vaddr; |
510 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; | 510 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
511 | struct page *page = frag->page; | 511 | struct page *page = skb_frag_page(frag); |
512 | 512 | ||
513 | if (copy > len) | 513 | if (copy > len) |
514 | copy = len; | 514 | copy = len; |
@@ -594,7 +594,7 @@ static int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset, | |||
594 | int err = 0; | 594 | int err = 0; |
595 | u8 *vaddr; | 595 | u8 *vaddr; |
596 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; | 596 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
597 | struct page *page = frag->page; | 597 | struct page *page = skb_frag_page(frag); |
598 | 598 | ||
599 | if (copy > len) | 599 | if (copy > len) |
600 | copy = len; | 600 | copy = len; |
diff --git a/net/core/dev.c b/net/core/dev.c index b10ff0a71855..bf49a47ddfdb 100644 --- a/net/core/dev.c +++ b/net/core/dev.c | |||
@@ -133,6 +133,8 @@ | |||
133 | #include <linux/pci.h> | 133 | #include <linux/pci.h> |
134 | #include <linux/inetdevice.h> | 134 | #include <linux/inetdevice.h> |
135 | #include <linux/cpu_rmap.h> | 135 | #include <linux/cpu_rmap.h> |
136 | #include <linux/if_tunnel.h> | ||
137 | #include <linux/if_pppox.h> | ||
136 | 138 | ||
137 | #include "net-sysfs.h" | 139 | #include "net-sysfs.h" |
138 | 140 | ||
@@ -1955,9 +1957,11 @@ static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) | |||
1955 | #ifdef CONFIG_HIGHMEM | 1957 | #ifdef CONFIG_HIGHMEM |
1956 | int i; | 1958 | int i; |
1957 | if (!(dev->features & NETIF_F_HIGHDMA)) { | 1959 | if (!(dev->features & NETIF_F_HIGHDMA)) { |
1958 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) | 1960 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
1959 | if (PageHighMem(skb_shinfo(skb)->frags[i].page)) | 1961 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
1962 | if (PageHighMem(skb_frag_page(frag))) | ||
1960 | return 1; | 1963 | return 1; |
1964 | } | ||
1961 | } | 1965 | } |
1962 | 1966 | ||
1963 | if (PCI_DMA_BUS_IS_PHYS) { | 1967 | if (PCI_DMA_BUS_IS_PHYS) { |
@@ -1966,7 +1970,8 @@ static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) | |||
1966 | if (!pdev) | 1970 | if (!pdev) |
1967 | return 0; | 1971 | return 0; |
1968 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { | 1972 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
1969 | dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page); | 1973 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
1974 | dma_addr_t addr = page_to_phys(skb_frag_page(frag)); | ||
1970 | if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) | 1975 | if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) |
1971 | return 1; | 1976 | return 1; |
1972 | } | 1977 | } |
@@ -2527,24 +2532,29 @@ static inline void ____napi_schedule(struct softnet_data *sd, | |||
2527 | 2532 | ||
2528 | /* | 2533 | /* |
2529 | * __skb_get_rxhash: calculate a flow hash based on src/dst addresses | 2534 | * __skb_get_rxhash: calculate a flow hash based on src/dst addresses |
2530 | * and src/dst port numbers. Returns a non-zero hash number on success | 2535 | * and src/dst port numbers. Sets rxhash in skb to non-zero hash value |
2531 | * and 0 on failure. | 2536 | * on success, zero indicates no valid hash. Also, sets l4_rxhash in skb |
2537 | * if hash is a canonical 4-tuple hash over transport ports. | ||
2532 | */ | 2538 | */ |
2533 | __u32 __skb_get_rxhash(struct sk_buff *skb) | 2539 | void __skb_get_rxhash(struct sk_buff *skb) |
2534 | { | 2540 | { |
2535 | int nhoff, hash = 0, poff; | 2541 | int nhoff, hash = 0, poff; |
2536 | const struct ipv6hdr *ip6; | 2542 | const struct ipv6hdr *ip6; |
2537 | const struct iphdr *ip; | 2543 | const struct iphdr *ip; |
2544 | const struct vlan_hdr *vlan; | ||
2538 | u8 ip_proto; | 2545 | u8 ip_proto; |
2539 | u32 addr1, addr2, ihl; | 2546 | u32 addr1, addr2; |
2547 | u16 proto; | ||
2540 | union { | 2548 | union { |
2541 | u32 v32; | 2549 | u32 v32; |
2542 | u16 v16[2]; | 2550 | u16 v16[2]; |
2543 | } ports; | 2551 | } ports; |
2544 | 2552 | ||
2545 | nhoff = skb_network_offset(skb); | 2553 | nhoff = skb_network_offset(skb); |
2554 | proto = skb->protocol; | ||
2546 | 2555 | ||
2547 | switch (skb->protocol) { | 2556 | again: |
2557 | switch (proto) { | ||
2548 | case __constant_htons(ETH_P_IP): | 2558 | case __constant_htons(ETH_P_IP): |
2549 | if (!pskb_may_pull(skb, sizeof(*ip) + nhoff)) | 2559 | if (!pskb_may_pull(skb, sizeof(*ip) + nhoff)) |
2550 | goto done; | 2560 | goto done; |
@@ -2556,7 +2566,7 @@ __u32 __skb_get_rxhash(struct sk_buff *skb) | |||
2556 | ip_proto = ip->protocol; | 2566 | ip_proto = ip->protocol; |
2557 | addr1 = (__force u32) ip->saddr; | 2567 | addr1 = (__force u32) ip->saddr; |
2558 | addr2 = (__force u32) ip->daddr; | 2568 | addr2 = (__force u32) ip->daddr; |
2559 | ihl = ip->ihl; | 2569 | nhoff += ip->ihl * 4; |
2560 | break; | 2570 | break; |
2561 | case __constant_htons(ETH_P_IPV6): | 2571 | case __constant_htons(ETH_P_IPV6): |
2562 | if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff)) | 2572 | if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff)) |
@@ -2566,20 +2576,64 @@ __u32 __skb_get_rxhash(struct sk_buff *skb) | |||
2566 | ip_proto = ip6->nexthdr; | 2576 | ip_proto = ip6->nexthdr; |
2567 | addr1 = (__force u32) ip6->saddr.s6_addr32[3]; | 2577 | addr1 = (__force u32) ip6->saddr.s6_addr32[3]; |
2568 | addr2 = (__force u32) ip6->daddr.s6_addr32[3]; | 2578 | addr2 = (__force u32) ip6->daddr.s6_addr32[3]; |
2569 | ihl = (40 >> 2); | 2579 | nhoff += 40; |
2570 | break; | 2580 | break; |
2581 | case __constant_htons(ETH_P_8021Q): | ||
2582 | if (!pskb_may_pull(skb, sizeof(*vlan) + nhoff)) | ||
2583 | goto done; | ||
2584 | vlan = (const struct vlan_hdr *) (skb->data + nhoff); | ||
2585 | proto = vlan->h_vlan_encapsulated_proto; | ||
2586 | nhoff += sizeof(*vlan); | ||
2587 | goto again; | ||
2588 | case __constant_htons(ETH_P_PPP_SES): | ||
2589 | if (!pskb_may_pull(skb, PPPOE_SES_HLEN + nhoff)) | ||
2590 | goto done; | ||
2591 | proto = *((__be16 *) (skb->data + nhoff + | ||
2592 | sizeof(struct pppoe_hdr))); | ||
2593 | nhoff += PPPOE_SES_HLEN; | ||
2594 | goto again; | ||
2571 | default: | 2595 | default: |
2572 | goto done; | 2596 | goto done; |
2573 | } | 2597 | } |
2574 | 2598 | ||
2599 | switch (ip_proto) { | ||
2600 | case IPPROTO_GRE: | ||
2601 | if (pskb_may_pull(skb, nhoff + 16)) { | ||
2602 | u8 *h = skb->data + nhoff; | ||
2603 | __be16 flags = *(__be16 *)h; | ||
2604 | |||
2605 | /* | ||
2606 | * Only look inside GRE if version zero and no | ||
2607 | * routing | ||
2608 | */ | ||
2609 | if (!(flags & (GRE_VERSION|GRE_ROUTING))) { | ||
2610 | proto = *(__be16 *)(h + 2); | ||
2611 | nhoff += 4; | ||
2612 | if (flags & GRE_CSUM) | ||
2613 | nhoff += 4; | ||
2614 | if (flags & GRE_KEY) | ||
2615 | nhoff += 4; | ||
2616 | if (flags & GRE_SEQ) | ||
2617 | nhoff += 4; | ||
2618 | goto again; | ||
2619 | } | ||
2620 | } | ||
2621 | break; | ||
2622 | case IPPROTO_IPIP: | ||
2623 | goto again; | ||
2624 | default: | ||
2625 | break; | ||
2626 | } | ||
2627 | |||
2575 | ports.v32 = 0; | 2628 | ports.v32 = 0; |
2576 | poff = proto_ports_offset(ip_proto); | 2629 | poff = proto_ports_offset(ip_proto); |
2577 | if (poff >= 0) { | 2630 | if (poff >= 0) { |
2578 | nhoff += ihl * 4 + poff; | 2631 | nhoff += poff; |
2579 | if (pskb_may_pull(skb, nhoff + 4)) { | 2632 | if (pskb_may_pull(skb, nhoff + 4)) { |
2580 | ports.v32 = * (__force u32 *) (skb->data + nhoff); | 2633 | ports.v32 = * (__force u32 *) (skb->data + nhoff); |
2581 | if (ports.v16[1] < ports.v16[0]) | 2634 | if (ports.v16[1] < ports.v16[0]) |
2582 | swap(ports.v16[0], ports.v16[1]); | 2635 | swap(ports.v16[0], ports.v16[1]); |
2636 | skb->l4_rxhash = 1; | ||
2583 | } | 2637 | } |
2584 | } | 2638 | } |
2585 | 2639 | ||
@@ -2592,7 +2646,7 @@ __u32 __skb_get_rxhash(struct sk_buff *skb) | |||
2592 | hash = 1; | 2646 | hash = 1; |
2593 | 2647 | ||
2594 | done: | 2648 | done: |
2595 | return hash; | 2649 | skb->rxhash = hash; |
2596 | } | 2650 | } |
2597 | EXPORT_SYMBOL(__skb_get_rxhash); | 2651 | EXPORT_SYMBOL(__skb_get_rxhash); |
2598 | 2652 | ||
@@ -2681,13 +2735,13 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, | |||
2681 | map = rcu_dereference(rxqueue->rps_map); | 2735 | map = rcu_dereference(rxqueue->rps_map); |
2682 | if (map) { | 2736 | if (map) { |
2683 | if (map->len == 1 && | 2737 | if (map->len == 1 && |
2684 | !rcu_dereference_raw(rxqueue->rps_flow_table)) { | 2738 | !rcu_access_pointer(rxqueue->rps_flow_table)) { |
2685 | tcpu = map->cpus[0]; | 2739 | tcpu = map->cpus[0]; |
2686 | if (cpu_online(tcpu)) | 2740 | if (cpu_online(tcpu)) |
2687 | cpu = tcpu; | 2741 | cpu = tcpu; |
2688 | goto done; | 2742 | goto done; |
2689 | } | 2743 | } |
2690 | } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) { | 2744 | } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) { |
2691 | goto done; | 2745 | goto done; |
2692 | } | 2746 | } |
2693 | 2747 | ||
@@ -3102,8 +3156,8 @@ void netdev_rx_handler_unregister(struct net_device *dev) | |||
3102 | { | 3156 | { |
3103 | 3157 | ||
3104 | ASSERT_RTNL(); | 3158 | ASSERT_RTNL(); |
3105 | rcu_assign_pointer(dev->rx_handler, NULL); | 3159 | RCU_INIT_POINTER(dev->rx_handler, NULL); |
3106 | rcu_assign_pointer(dev->rx_handler_data, NULL); | 3160 | RCU_INIT_POINTER(dev->rx_handler_data, NULL); |
3107 | } | 3161 | } |
3108 | EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); | 3162 | EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); |
3109 | 3163 | ||
@@ -3195,10 +3249,9 @@ ncls: | |||
3195 | ret = deliver_skb(skb, pt_prev, orig_dev); | 3249 | ret = deliver_skb(skb, pt_prev, orig_dev); |
3196 | pt_prev = NULL; | 3250 | pt_prev = NULL; |
3197 | } | 3251 | } |
3198 | if (vlan_do_receive(&skb)) { | 3252 | if (vlan_do_receive(&skb)) |
3199 | ret = __netif_receive_skb(skb); | 3253 | goto another_round; |
3200 | goto out; | 3254 | else if (unlikely(!skb)) |
3201 | } else if (unlikely(!skb)) | ||
3202 | goto out; | 3255 | goto out; |
3203 | } | 3256 | } |
3204 | 3257 | ||
@@ -3432,7 +3485,7 @@ pull: | |||
3432 | skb_shinfo(skb)->frags[0].size -= grow; | 3485 | skb_shinfo(skb)->frags[0].size -= grow; |
3433 | 3486 | ||
3434 | if (unlikely(!skb_shinfo(skb)->frags[0].size)) { | 3487 | if (unlikely(!skb_shinfo(skb)->frags[0].size)) { |
3435 | put_page(skb_shinfo(skb)->frags[0].page); | 3488 | skb_frag_unref(skb, 0); |
3436 | memmove(skb_shinfo(skb)->frags, | 3489 | memmove(skb_shinfo(skb)->frags, |
3437 | skb_shinfo(skb)->frags + 1, | 3490 | skb_shinfo(skb)->frags + 1, |
3438 | --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t)); | 3491 | --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t)); |
@@ -3496,10 +3549,9 @@ void skb_gro_reset_offset(struct sk_buff *skb) | |||
3496 | NAPI_GRO_CB(skb)->frag0_len = 0; | 3549 | NAPI_GRO_CB(skb)->frag0_len = 0; |
3497 | 3550 | ||
3498 | if (skb->mac_header == skb->tail && | 3551 | if (skb->mac_header == skb->tail && |
3499 | !PageHighMem(skb_shinfo(skb)->frags[0].page)) { | 3552 | !PageHighMem(skb_frag_page(&skb_shinfo(skb)->frags[0]))) { |
3500 | NAPI_GRO_CB(skb)->frag0 = | 3553 | NAPI_GRO_CB(skb)->frag0 = |
3501 | page_address(skb_shinfo(skb)->frags[0].page) + | 3554 | skb_frag_address(&skb_shinfo(skb)->frags[0]); |
3502 | skb_shinfo(skb)->frags[0].page_offset; | ||
3503 | NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size; | 3555 | NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size; |
3504 | } | 3556 | } |
3505 | } | 3557 | } |
@@ -4497,9 +4549,7 @@ void __dev_set_rx_mode(struct net_device *dev) | |||
4497 | if (!netif_device_present(dev)) | 4549 | if (!netif_device_present(dev)) |
4498 | return; | 4550 | return; |
4499 | 4551 | ||
4500 | if (ops->ndo_set_rx_mode) | 4552 | if (!(dev->priv_flags & IFF_UNICAST_FLT)) { |
4501 | ops->ndo_set_rx_mode(dev); | ||
4502 | else { | ||
4503 | /* Unicast addresses changes may only happen under the rtnl, | 4553 | /* Unicast addresses changes may only happen under the rtnl, |
4504 | * therefore calling __dev_set_promiscuity here is safe. | 4554 | * therefore calling __dev_set_promiscuity here is safe. |
4505 | */ | 4555 | */ |
@@ -4510,10 +4560,10 @@ void __dev_set_rx_mode(struct net_device *dev) | |||
4510 | __dev_set_promiscuity(dev, -1); | 4560 | __dev_set_promiscuity(dev, -1); |
4511 | dev->uc_promisc = false; | 4561 | dev->uc_promisc = false; |
4512 | } | 4562 | } |
4513 | |||
4514 | if (ops->ndo_set_multicast_list) | ||
4515 | ops->ndo_set_multicast_list(dev); | ||
4516 | } | 4563 | } |
4564 | |||
4565 | if (ops->ndo_set_rx_mode) | ||
4566 | ops->ndo_set_rx_mode(dev); | ||
4517 | } | 4567 | } |
4518 | 4568 | ||
4519 | void dev_set_rx_mode(struct net_device *dev) | 4569 | void dev_set_rx_mode(struct net_device *dev) |
@@ -4524,30 +4574,6 @@ void dev_set_rx_mode(struct net_device *dev) | |||
4524 | } | 4574 | } |
4525 | 4575 | ||
4526 | /** | 4576 | /** |
4527 | * dev_ethtool_get_settings - call device's ethtool_ops::get_settings() | ||
4528 | * @dev: device | ||
4529 | * @cmd: memory area for ethtool_ops::get_settings() result | ||
4530 | * | ||
4531 | * The cmd arg is initialized properly (cleared and | ||
4532 | * ethtool_cmd::cmd field set to ETHTOOL_GSET). | ||
4533 | * | ||
4534 | * Return device's ethtool_ops::get_settings() result value or | ||
4535 | * -EOPNOTSUPP when device doesn't expose | ||
4536 | * ethtool_ops::get_settings() operation. | ||
4537 | */ | ||
4538 | int dev_ethtool_get_settings(struct net_device *dev, | ||
4539 | struct ethtool_cmd *cmd) | ||
4540 | { | ||
4541 | if (!dev->ethtool_ops || !dev->ethtool_ops->get_settings) | ||
4542 | return -EOPNOTSUPP; | ||
4543 | |||
4544 | memset(cmd, 0, sizeof(struct ethtool_cmd)); | ||
4545 | cmd->cmd = ETHTOOL_GSET; | ||
4546 | return dev->ethtool_ops->get_settings(dev, cmd); | ||
4547 | } | ||
4548 | EXPORT_SYMBOL(dev_ethtool_get_settings); | ||
4549 | |||
4550 | /** | ||
4551 | * dev_get_flags - get flags reported to userspace | 4577 | * dev_get_flags - get flags reported to userspace |
4552 | * @dev: device | 4578 | * @dev: device |
4553 | * | 4579 | * |
@@ -4863,7 +4889,7 @@ static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd) | |||
4863 | return -EOPNOTSUPP; | 4889 | return -EOPNOTSUPP; |
4864 | 4890 | ||
4865 | case SIOCADDMULTI: | 4891 | case SIOCADDMULTI: |
4866 | if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) || | 4892 | if (!ops->ndo_set_rx_mode || |
4867 | ifr->ifr_hwaddr.sa_family != AF_UNSPEC) | 4893 | ifr->ifr_hwaddr.sa_family != AF_UNSPEC) |
4868 | return -EINVAL; | 4894 | return -EINVAL; |
4869 | if (!netif_device_present(dev)) | 4895 | if (!netif_device_present(dev)) |
@@ -4871,7 +4897,7 @@ static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd) | |||
4871 | return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data); | 4897 | return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data); |
4872 | 4898 | ||
4873 | case SIOCDELMULTI: | 4899 | case SIOCDELMULTI: |
4874 | if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) || | 4900 | if (!ops->ndo_set_rx_mode || |
4875 | ifr->ifr_hwaddr.sa_family != AF_UNSPEC) | 4901 | ifr->ifr_hwaddr.sa_family != AF_UNSPEC) |
4876 | return -EINVAL; | 4902 | return -EINVAL; |
4877 | if (!netif_device_present(dev)) | 4903 | if (!netif_device_present(dev)) |
@@ -5735,8 +5761,8 @@ void netdev_run_todo(void) | |||
5735 | 5761 | ||
5736 | /* paranoia */ | 5762 | /* paranoia */ |
5737 | BUG_ON(netdev_refcnt_read(dev)); | 5763 | BUG_ON(netdev_refcnt_read(dev)); |
5738 | WARN_ON(rcu_dereference_raw(dev->ip_ptr)); | 5764 | WARN_ON(rcu_access_pointer(dev->ip_ptr)); |
5739 | WARN_ON(rcu_dereference_raw(dev->ip6_ptr)); | 5765 | WARN_ON(rcu_access_pointer(dev->ip6_ptr)); |
5740 | WARN_ON(dev->dn_ptr); | 5766 | WARN_ON(dev->dn_ptr); |
5741 | 5767 | ||
5742 | if (dev->destructor) | 5768 | if (dev->destructor) |
@@ -5940,7 +5966,7 @@ void free_netdev(struct net_device *dev) | |||
5940 | kfree(dev->_rx); | 5966 | kfree(dev->_rx); |
5941 | #endif | 5967 | #endif |
5942 | 5968 | ||
5943 | kfree(rcu_dereference_raw(dev->ingress_queue)); | 5969 | kfree(rcu_dereference_protected(dev->ingress_queue, 1)); |
5944 | 5970 | ||
5945 | /* Flush device addresses */ | 5971 | /* Flush device addresses */ |
5946 | dev_addr_flush(dev); | 5972 | dev_addr_flush(dev); |
diff --git a/net/core/dev_addr_lists.c b/net/core/dev_addr_lists.c index e2e66939ed00..283d1b863876 100644 --- a/net/core/dev_addr_lists.c +++ b/net/core/dev_addr_lists.c | |||
@@ -591,8 +591,8 @@ EXPORT_SYMBOL(dev_mc_del_global); | |||
591 | * addresses that have no users left. The source device must be | 591 | * addresses that have no users left. The source device must be |
592 | * locked by netif_tx_lock_bh. | 592 | * locked by netif_tx_lock_bh. |
593 | * | 593 | * |
594 | * This function is intended to be called from the dev->set_multicast_list | 594 | * This function is intended to be called from the ndo_set_rx_mode |
595 | * or dev->set_rx_mode function of layered software devices. | 595 | * function of layered software devices. |
596 | */ | 596 | */ |
597 | int dev_mc_sync(struct net_device *to, struct net_device *from) | 597 | int dev_mc_sync(struct net_device *to, struct net_device *from) |
598 | { | 598 | { |
diff --git a/net/core/dst.c b/net/core/dst.c index 14b33baf0733..d5e2c4c09107 100644 --- a/net/core/dst.c +++ b/net/core/dst.c | |||
@@ -171,7 +171,7 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev, | |||
171 | dst_init_metrics(dst, dst_default_metrics, true); | 171 | dst_init_metrics(dst, dst_default_metrics, true); |
172 | dst->expires = 0UL; | 172 | dst->expires = 0UL; |
173 | dst->path = dst; | 173 | dst->path = dst; |
174 | dst->_neighbour = NULL; | 174 | RCU_INIT_POINTER(dst->_neighbour, NULL); |
175 | #ifdef CONFIG_XFRM | 175 | #ifdef CONFIG_XFRM |
176 | dst->xfrm = NULL; | 176 | dst->xfrm = NULL; |
177 | #endif | 177 | #endif |
@@ -229,11 +229,11 @@ struct dst_entry *dst_destroy(struct dst_entry * dst) | |||
229 | smp_rmb(); | 229 | smp_rmb(); |
230 | 230 | ||
231 | again: | 231 | again: |
232 | neigh = dst->_neighbour; | 232 | neigh = rcu_dereference_protected(dst->_neighbour, 1); |
233 | child = dst->child; | 233 | child = dst->child; |
234 | 234 | ||
235 | if (neigh) { | 235 | if (neigh) { |
236 | dst->_neighbour = NULL; | 236 | RCU_INIT_POINTER(dst->_neighbour, NULL); |
237 | neigh_release(neigh); | 237 | neigh_release(neigh); |
238 | } | 238 | } |
239 | 239 | ||
@@ -360,14 +360,19 @@ static void dst_ifdown(struct dst_entry *dst, struct net_device *dev, | |||
360 | if (!unregister) { | 360 | if (!unregister) { |
361 | dst->input = dst->output = dst_discard; | 361 | dst->input = dst->output = dst_discard; |
362 | } else { | 362 | } else { |
363 | struct neighbour *neigh; | ||
364 | |||
363 | dst->dev = dev_net(dst->dev)->loopback_dev; | 365 | dst->dev = dev_net(dst->dev)->loopback_dev; |
364 | dev_hold(dst->dev); | 366 | dev_hold(dst->dev); |
365 | dev_put(dev); | 367 | dev_put(dev); |
366 | if (dst->_neighbour && dst->_neighbour->dev == dev) { | 368 | rcu_read_lock(); |
367 | dst->_neighbour->dev = dst->dev; | 369 | neigh = dst_get_neighbour(dst); |
370 | if (neigh && neigh->dev == dev) { | ||
371 | neigh->dev = dst->dev; | ||
368 | dev_hold(dst->dev); | 372 | dev_hold(dst->dev); |
369 | dev_put(dev); | 373 | dev_put(dev); |
370 | } | 374 | } |
375 | rcu_read_unlock(); | ||
371 | } | 376 | } |
372 | } | 377 | } |
373 | 378 | ||
diff --git a/net/core/ethtool.c b/net/core/ethtool.c index 6cdba5fc2bed..f44481707124 100644 --- a/net/core/ethtool.c +++ b/net/core/ethtool.c | |||
@@ -569,15 +569,25 @@ int __ethtool_set_flags(struct net_device *dev, u32 data) | |||
569 | return 0; | 569 | return 0; |
570 | } | 570 | } |
571 | 571 | ||
572 | static int ethtool_get_settings(struct net_device *dev, void __user *useraddr) | 572 | int __ethtool_get_settings(struct net_device *dev, struct ethtool_cmd *cmd) |
573 | { | 573 | { |
574 | struct ethtool_cmd cmd = { .cmd = ETHTOOL_GSET }; | 574 | ASSERT_RTNL(); |
575 | int err; | ||
576 | 575 | ||
577 | if (!dev->ethtool_ops->get_settings) | 576 | if (!dev->ethtool_ops || !dev->ethtool_ops->get_settings) |
578 | return -EOPNOTSUPP; | 577 | return -EOPNOTSUPP; |
579 | 578 | ||
580 | err = dev->ethtool_ops->get_settings(dev, &cmd); | 579 | memset(cmd, 0, sizeof(struct ethtool_cmd)); |
580 | cmd->cmd = ETHTOOL_GSET; | ||
581 | return dev->ethtool_ops->get_settings(dev, cmd); | ||
582 | } | ||
583 | EXPORT_SYMBOL(__ethtool_get_settings); | ||
584 | |||
585 | static int ethtool_get_settings(struct net_device *dev, void __user *useraddr) | ||
586 | { | ||
587 | int err; | ||
588 | struct ethtool_cmd cmd; | ||
589 | |||
590 | err = __ethtool_get_settings(dev, &cmd); | ||
581 | if (err < 0) | 591 | if (err < 0) |
582 | return err; | 592 | return err; |
583 | 593 | ||
diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c index 3231b468bb72..38be4744133f 100644 --- a/net/core/fib_rules.c +++ b/net/core/fib_rules.c | |||
@@ -487,7 +487,7 @@ static int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr* nlh, void *arg) | |||
487 | if (ops->nr_goto_rules > 0) { | 487 | if (ops->nr_goto_rules > 0) { |
488 | list_for_each_entry(tmp, &ops->rules_list, list) { | 488 | list_for_each_entry(tmp, &ops->rules_list, list) { |
489 | if (rtnl_dereference(tmp->ctarget) == rule) { | 489 | if (rtnl_dereference(tmp->ctarget) == rule) { |
490 | rcu_assign_pointer(tmp->ctarget, NULL); | 490 | RCU_INIT_POINTER(tmp->ctarget, NULL); |
491 | ops->unresolved_rules++; | 491 | ops->unresolved_rules++; |
492 | } | 492 | } |
493 | } | 493 | } |
@@ -545,7 +545,7 @@ static int fib_nl_fill_rule(struct sk_buff *skb, struct fib_rule *rule, | |||
545 | frh->flags = rule->flags; | 545 | frh->flags = rule->flags; |
546 | 546 | ||
547 | if (rule->action == FR_ACT_GOTO && | 547 | if (rule->action == FR_ACT_GOTO && |
548 | rcu_dereference_raw(rule->ctarget) == NULL) | 548 | rcu_access_pointer(rule->ctarget) == NULL) |
549 | frh->flags |= FIB_RULE_UNRESOLVED; | 549 | frh->flags |= FIB_RULE_UNRESOLVED; |
550 | 550 | ||
551 | if (rule->iifname[0]) { | 551 | if (rule->iifname[0]) { |
diff --git a/net/core/filter.c b/net/core/filter.c index 36f975fa87cb..8fcc2d776e09 100644 --- a/net/core/filter.c +++ b/net/core/filter.c | |||
@@ -645,7 +645,7 @@ int sk_detach_filter(struct sock *sk) | |||
645 | filter = rcu_dereference_protected(sk->sk_filter, | 645 | filter = rcu_dereference_protected(sk->sk_filter, |
646 | sock_owned_by_user(sk)); | 646 | sock_owned_by_user(sk)); |
647 | if (filter) { | 647 | if (filter) { |
648 | rcu_assign_pointer(sk->sk_filter, NULL); | 648 | RCU_INIT_POINTER(sk->sk_filter, NULL); |
649 | sk_filter_uncharge(sk, filter); | 649 | sk_filter_uncharge(sk, filter); |
650 | ret = 0; | 650 | ret = 0; |
651 | } | 651 | } |
diff --git a/net/core/kmap_skb.h b/net/core/kmap_skb.h index 283c2b993fb8..81e1ed7c8383 100644 --- a/net/core/kmap_skb.h +++ b/net/core/kmap_skb.h | |||
@@ -7,7 +7,7 @@ static inline void *kmap_skb_frag(const skb_frag_t *frag) | |||
7 | 7 | ||
8 | local_bh_disable(); | 8 | local_bh_disable(); |
9 | #endif | 9 | #endif |
10 | return kmap_atomic(frag->page, KM_SKB_DATA_SOFTIRQ); | 10 | return kmap_atomic(skb_frag_page(frag), KM_SKB_DATA_SOFTIRQ); |
11 | } | 11 | } |
12 | 12 | ||
13 | static inline void kunmap_skb_frag(void *vaddr) | 13 | static inline void kunmap_skb_frag(void *vaddr) |
diff --git a/net/core/link_watch.c b/net/core/link_watch.c index 357bd4ee4baa..c3519c6d1b16 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c | |||
@@ -78,8 +78,13 @@ static void rfc2863_policy(struct net_device *dev) | |||
78 | 78 | ||
79 | static bool linkwatch_urgent_event(struct net_device *dev) | 79 | static bool linkwatch_urgent_event(struct net_device *dev) |
80 | { | 80 | { |
81 | return netif_running(dev) && netif_carrier_ok(dev) && | 81 | if (!netif_running(dev)) |
82 | qdisc_tx_changing(dev); | 82 | return false; |
83 | |||
84 | if (dev->ifindex != dev->iflink) | ||
85 | return true; | ||
86 | |||
87 | return netif_carrier_ok(dev) && qdisc_tx_changing(dev); | ||
83 | } | 88 | } |
84 | 89 | ||
85 | 90 | ||
diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 1334d7e56f02..43449649cf73 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c | |||
@@ -844,6 +844,19 @@ static void neigh_invalidate(struct neighbour *neigh) | |||
844 | skb_queue_purge(&neigh->arp_queue); | 844 | skb_queue_purge(&neigh->arp_queue); |
845 | } | 845 | } |
846 | 846 | ||
847 | static void neigh_probe(struct neighbour *neigh) | ||
848 | __releases(neigh->lock) | ||
849 | { | ||
850 | struct sk_buff *skb = skb_peek(&neigh->arp_queue); | ||
851 | /* keep skb alive even if arp_queue overflows */ | ||
852 | if (skb) | ||
853 | skb = skb_copy(skb, GFP_ATOMIC); | ||
854 | write_unlock(&neigh->lock); | ||
855 | neigh->ops->solicit(neigh, skb); | ||
856 | atomic_inc(&neigh->probes); | ||
857 | kfree_skb(skb); | ||
858 | } | ||
859 | |||
847 | /* Called when a timer expires for a neighbour entry. */ | 860 | /* Called when a timer expires for a neighbour entry. */ |
848 | 861 | ||
849 | static void neigh_timer_handler(unsigned long arg) | 862 | static void neigh_timer_handler(unsigned long arg) |
@@ -920,14 +933,7 @@ static void neigh_timer_handler(unsigned long arg) | |||
920 | neigh_hold(neigh); | 933 | neigh_hold(neigh); |
921 | } | 934 | } |
922 | if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) { | 935 | if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) { |
923 | struct sk_buff *skb = skb_peek(&neigh->arp_queue); | 936 | neigh_probe(neigh); |
924 | /* keep skb alive even if arp_queue overflows */ | ||
925 | if (skb) | ||
926 | skb = skb_copy(skb, GFP_ATOMIC); | ||
927 | write_unlock(&neigh->lock); | ||
928 | neigh->ops->solicit(neigh, skb); | ||
929 | atomic_inc(&neigh->probes); | ||
930 | kfree_skb(skb); | ||
931 | } else { | 937 | } else { |
932 | out: | 938 | out: |
933 | write_unlock(&neigh->lock); | 939 | write_unlock(&neigh->lock); |
@@ -942,7 +948,7 @@ out: | |||
942 | int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb) | 948 | int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb) |
943 | { | 949 | { |
944 | int rc; | 950 | int rc; |
945 | unsigned long now; | 951 | bool immediate_probe = false; |
946 | 952 | ||
947 | write_lock_bh(&neigh->lock); | 953 | write_lock_bh(&neigh->lock); |
948 | 954 | ||
@@ -950,14 +956,16 @@ int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb) | |||
950 | if (neigh->nud_state & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE)) | 956 | if (neigh->nud_state & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE)) |
951 | goto out_unlock_bh; | 957 | goto out_unlock_bh; |
952 | 958 | ||
953 | now = jiffies; | ||
954 | |||
955 | if (!(neigh->nud_state & (NUD_STALE | NUD_INCOMPLETE))) { | 959 | if (!(neigh->nud_state & (NUD_STALE | NUD_INCOMPLETE))) { |
956 | if (neigh->parms->mcast_probes + neigh->parms->app_probes) { | 960 | if (neigh->parms->mcast_probes + neigh->parms->app_probes) { |
961 | unsigned long next, now = jiffies; | ||
962 | |||
957 | atomic_set(&neigh->probes, neigh->parms->ucast_probes); | 963 | atomic_set(&neigh->probes, neigh->parms->ucast_probes); |
958 | neigh->nud_state = NUD_INCOMPLETE; | 964 | neigh->nud_state = NUD_INCOMPLETE; |
959 | neigh->updated = jiffies; | 965 | neigh->updated = now; |
960 | neigh_add_timer(neigh, now + 1); | 966 | next = now + max(neigh->parms->retrans_time, HZ/2); |
967 | neigh_add_timer(neigh, next); | ||
968 | immediate_probe = true; | ||
961 | } else { | 969 | } else { |
962 | neigh->nud_state = NUD_FAILED; | 970 | neigh->nud_state = NUD_FAILED; |
963 | neigh->updated = jiffies; | 971 | neigh->updated = jiffies; |
@@ -989,7 +997,11 @@ int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb) | |||
989 | rc = 1; | 997 | rc = 1; |
990 | } | 998 | } |
991 | out_unlock_bh: | 999 | out_unlock_bh: |
992 | write_unlock_bh(&neigh->lock); | 1000 | if (immediate_probe) |
1001 | neigh_probe(neigh); | ||
1002 | else | ||
1003 | write_unlock(&neigh->lock); | ||
1004 | local_bh_enable(); | ||
993 | return rc; | 1005 | return rc; |
994 | } | 1006 | } |
995 | EXPORT_SYMBOL(__neigh_event_send); | 1007 | EXPORT_SYMBOL(__neigh_event_send); |
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 1683e5db2f27..7604a635376b 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c | |||
@@ -147,7 +147,7 @@ static ssize_t show_speed(struct device *dev, | |||
147 | 147 | ||
148 | if (netif_running(netdev)) { | 148 | if (netif_running(netdev)) { |
149 | struct ethtool_cmd cmd; | 149 | struct ethtool_cmd cmd; |
150 | if (!dev_ethtool_get_settings(netdev, &cmd)) | 150 | if (!__ethtool_get_settings(netdev, &cmd)) |
151 | ret = sprintf(buf, fmt_udec, ethtool_cmd_speed(&cmd)); | 151 | ret = sprintf(buf, fmt_udec, ethtool_cmd_speed(&cmd)); |
152 | } | 152 | } |
153 | rtnl_unlock(); | 153 | rtnl_unlock(); |
@@ -165,7 +165,7 @@ static ssize_t show_duplex(struct device *dev, | |||
165 | 165 | ||
166 | if (netif_running(netdev)) { | 166 | if (netif_running(netdev)) { |
167 | struct ethtool_cmd cmd; | 167 | struct ethtool_cmd cmd; |
168 | if (!dev_ethtool_get_settings(netdev, &cmd)) | 168 | if (!__ethtool_get_settings(netdev, &cmd)) |
169 | ret = sprintf(buf, "%s\n", | 169 | ret = sprintf(buf, "%s\n", |
170 | cmd.duplex ? "full" : "half"); | 170 | cmd.duplex ? "full" : "half"); |
171 | } | 171 | } |
@@ -712,13 +712,13 @@ static void rx_queue_release(struct kobject *kobj) | |||
712 | struct rps_dev_flow_table *flow_table; | 712 | struct rps_dev_flow_table *flow_table; |
713 | 713 | ||
714 | 714 | ||
715 | map = rcu_dereference_raw(queue->rps_map); | 715 | map = rcu_dereference_protected(queue->rps_map, 1); |
716 | if (map) { | 716 | if (map) { |
717 | RCU_INIT_POINTER(queue->rps_map, NULL); | 717 | RCU_INIT_POINTER(queue->rps_map, NULL); |
718 | kfree_rcu(map, rcu); | 718 | kfree_rcu(map, rcu); |
719 | } | 719 | } |
720 | 720 | ||
721 | flow_table = rcu_dereference_raw(queue->rps_flow_table); | 721 | flow_table = rcu_dereference_protected(queue->rps_flow_table, 1); |
722 | if (flow_table) { | 722 | if (flow_table) { |
723 | RCU_INIT_POINTER(queue->rps_flow_table, NULL); | 723 | RCU_INIT_POINTER(queue->rps_flow_table, NULL); |
724 | call_rcu(&flow_table->rcu, rps_dev_flow_table_release); | 724 | call_rcu(&flow_table->rcu, rps_dev_flow_table_release); |
@@ -987,10 +987,10 @@ static ssize_t store_xps_map(struct netdev_queue *queue, | |||
987 | } | 987 | } |
988 | 988 | ||
989 | if (nonempty) | 989 | if (nonempty) |
990 | rcu_assign_pointer(dev->xps_maps, new_dev_maps); | 990 | RCU_INIT_POINTER(dev->xps_maps, new_dev_maps); |
991 | else { | 991 | else { |
992 | kfree(new_dev_maps); | 992 | kfree(new_dev_maps); |
993 | rcu_assign_pointer(dev->xps_maps, NULL); | 993 | RCU_INIT_POINTER(dev->xps_maps, NULL); |
994 | } | 994 | } |
995 | 995 | ||
996 | if (dev_maps) | 996 | if (dev_maps) |
diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 52622517e0d8..f57d94627a2a 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c | |||
@@ -762,7 +762,7 @@ int __netpoll_setup(struct netpoll *np) | |||
762 | } | 762 | } |
763 | 763 | ||
764 | /* last thing to do is link it to the net device structure */ | 764 | /* last thing to do is link it to the net device structure */ |
765 | rcu_assign_pointer(ndev->npinfo, npinfo); | 765 | RCU_INIT_POINTER(ndev->npinfo, npinfo); |
766 | 766 | ||
767 | return 0; | 767 | return 0; |
768 | 768 | ||
@@ -903,7 +903,7 @@ void __netpoll_cleanup(struct netpoll *np) | |||
903 | if (ops->ndo_netpoll_cleanup) | 903 | if (ops->ndo_netpoll_cleanup) |
904 | ops->ndo_netpoll_cleanup(np->dev); | 904 | ops->ndo_netpoll_cleanup(np->dev); |
905 | 905 | ||
906 | rcu_assign_pointer(np->dev->npinfo, NULL); | 906 | RCU_INIT_POINTER(np->dev->npinfo, NULL); |
907 | 907 | ||
908 | /* avoid racing with NAPI reading npinfo */ | 908 | /* avoid racing with NAPI reading npinfo */ |
909 | synchronize_rcu_bh(); | 909 | synchronize_rcu_bh(); |
diff --git a/net/core/pktgen.c b/net/core/pktgen.c index e35a6fbb8110..796044ac0bf3 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c | |||
@@ -2602,8 +2602,7 @@ static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb, | |||
2602 | if (!pkt_dev->page) | 2602 | if (!pkt_dev->page) |
2603 | break; | 2603 | break; |
2604 | } | 2604 | } |
2605 | skb_shinfo(skb)->frags[i].page = pkt_dev->page; | 2605 | skb_frag_set_page(skb, i, pkt_dev->page); |
2606 | get_page(pkt_dev->page); | ||
2607 | skb_shinfo(skb)->frags[i].page_offset = 0; | 2606 | skb_shinfo(skb)->frags[i].page_offset = 0; |
2608 | /*last fragment, fill rest of data*/ | 2607 | /*last fragment, fill rest of data*/ |
2609 | if (i == (frags - 1)) | 2608 | if (i == (frags - 1)) |
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 99d9e953fe39..39f8dd6a2821 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c | |||
@@ -1604,7 +1604,6 @@ struct net_device *rtnl_create_link(struct net *src_net, struct net *net, | |||
1604 | dev_net_set(dev, net); | 1604 | dev_net_set(dev, net); |
1605 | dev->rtnl_link_ops = ops; | 1605 | dev->rtnl_link_ops = ops; |
1606 | dev->rtnl_link_state = RTNL_LINK_INITIALIZING; | 1606 | dev->rtnl_link_state = RTNL_LINK_INITIALIZING; |
1607 | dev->real_num_tx_queues = real_num_queues; | ||
1608 | 1607 | ||
1609 | if (tb[IFLA_MTU]) | 1608 | if (tb[IFLA_MTU]) |
1610 | dev->mtu = nla_get_u32(tb[IFLA_MTU]); | 1609 | dev->mtu = nla_get_u32(tb[IFLA_MTU]); |
diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 387703f56fce..5b2c5f1d4dba 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c | |||
@@ -326,7 +326,7 @@ static void skb_release_data(struct sk_buff *skb) | |||
326 | if (skb_shinfo(skb)->nr_frags) { | 326 | if (skb_shinfo(skb)->nr_frags) { |
327 | int i; | 327 | int i; |
328 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) | 328 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) |
329 | put_page(skb_shinfo(skb)->frags[i].page); | 329 | skb_frag_unref(skb, i); |
330 | } | 330 | } |
331 | 331 | ||
332 | /* | 332 | /* |
@@ -529,6 +529,8 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) | |||
529 | new->mac_header = old->mac_header; | 529 | new->mac_header = old->mac_header; |
530 | skb_dst_copy(new, old); | 530 | skb_dst_copy(new, old); |
531 | new->rxhash = old->rxhash; | 531 | new->rxhash = old->rxhash; |
532 | new->ooo_okay = old->ooo_okay; | ||
533 | new->l4_rxhash = old->l4_rxhash; | ||
532 | #ifdef CONFIG_XFRM | 534 | #ifdef CONFIG_XFRM |
533 | new->sp = secpath_get(old->sp); | 535 | new->sp = secpath_get(old->sp); |
534 | #endif | 536 | #endif |
@@ -820,7 +822,7 @@ struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask) | |||
820 | } | 822 | } |
821 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { | 823 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
822 | skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; | 824 | skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; |
823 | get_page(skb_shinfo(n)->frags[i].page); | 825 | skb_frag_ref(skb, i); |
824 | } | 826 | } |
825 | skb_shinfo(n)->nr_frags = i; | 827 | skb_shinfo(n)->nr_frags = i; |
826 | } | 828 | } |
@@ -911,7 +913,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, | |||
911 | goto nofrags; | 913 | goto nofrags; |
912 | } | 914 | } |
913 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) | 915 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) |
914 | get_page(skb_shinfo(skb)->frags[i].page); | 916 | skb_frag_ref(skb, i); |
915 | 917 | ||
916 | if (skb_has_frag_list(skb)) | 918 | if (skb_has_frag_list(skb)) |
917 | skb_clone_fraglist(skb); | 919 | skb_clone_fraglist(skb); |
@@ -1191,7 +1193,7 @@ drop_pages: | |||
1191 | skb_shinfo(skb)->nr_frags = i; | 1193 | skb_shinfo(skb)->nr_frags = i; |
1192 | 1194 | ||
1193 | for (; i < nfrags; i++) | 1195 | for (; i < nfrags; i++) |
1194 | put_page(skb_shinfo(skb)->frags[i].page); | 1196 | skb_frag_unref(skb, i); |
1195 | 1197 | ||
1196 | if (skb_has_frag_list(skb)) | 1198 | if (skb_has_frag_list(skb)) |
1197 | skb_drop_fraglist(skb); | 1199 | skb_drop_fraglist(skb); |
@@ -1360,7 +1362,7 @@ pull_pages: | |||
1360 | k = 0; | 1362 | k = 0; |
1361 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { | 1363 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
1362 | if (skb_shinfo(skb)->frags[i].size <= eat) { | 1364 | if (skb_shinfo(skb)->frags[i].size <= eat) { |
1363 | put_page(skb_shinfo(skb)->frags[i].page); | 1365 | skb_frag_unref(skb, i); |
1364 | eat -= skb_shinfo(skb)->frags[i].size; | 1366 | eat -= skb_shinfo(skb)->frags[i].size; |
1365 | } else { | 1367 | } else { |
1366 | skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; | 1368 | skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; |
@@ -1619,7 +1621,8 @@ static int __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, | |||
1619 | for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { | 1621 | for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { |
1620 | const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; | 1622 | const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; |
1621 | 1623 | ||
1622 | if (__splice_segment(f->page, f->page_offset, f->size, | 1624 | if (__splice_segment(skb_frag_page(f), |
1625 | f->page_offset, f->size, | ||
1623 | offset, len, skb, spd, 0, sk, pipe)) | 1626 | offset, len, skb, spd, 0, sk, pipe)) |
1624 | return 1; | 1627 | return 1; |
1625 | } | 1628 | } |
@@ -2164,7 +2167,7 @@ static inline void skb_split_no_header(struct sk_buff *skb, | |||
2164 | * where splitting is expensive. | 2167 | * where splitting is expensive. |
2165 | * 2. Split is accurately. We make this. | 2168 | * 2. Split is accurately. We make this. |
2166 | */ | 2169 | */ |
2167 | get_page(skb_shinfo(skb)->frags[i].page); | 2170 | skb_frag_ref(skb, i); |
2168 | skb_shinfo(skb1)->frags[0].page_offset += len - pos; | 2171 | skb_shinfo(skb1)->frags[0].page_offset += len - pos; |
2169 | skb_shinfo(skb1)->frags[0].size -= len - pos; | 2172 | skb_shinfo(skb1)->frags[0].size -= len - pos; |
2170 | skb_shinfo(skb)->frags[i].size = len - pos; | 2173 | skb_shinfo(skb)->frags[i].size = len - pos; |
@@ -2239,7 +2242,8 @@ int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) | |||
2239 | * commit all, so that we don't have to undo partial changes | 2242 | * commit all, so that we don't have to undo partial changes |
2240 | */ | 2243 | */ |
2241 | if (!to || | 2244 | if (!to || |
2242 | !skb_can_coalesce(tgt, to, fragfrom->page, fragfrom->page_offset)) { | 2245 | !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), |
2246 | fragfrom->page_offset)) { | ||
2243 | merge = -1; | 2247 | merge = -1; |
2244 | } else { | 2248 | } else { |
2245 | merge = to - 1; | 2249 | merge = to - 1; |
@@ -2286,7 +2290,7 @@ int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) | |||
2286 | to++; | 2290 | to++; |
2287 | 2291 | ||
2288 | } else { | 2292 | } else { |
2289 | get_page(fragfrom->page); | 2293 | __skb_frag_ref(fragfrom); |
2290 | fragto->page = fragfrom->page; | 2294 | fragto->page = fragfrom->page; |
2291 | fragto->page_offset = fragfrom->page_offset; | 2295 | fragto->page_offset = fragfrom->page_offset; |
2292 | fragto->size = todo; | 2296 | fragto->size = todo; |
@@ -2308,7 +2312,7 @@ int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) | |||
2308 | fragto = &skb_shinfo(tgt)->frags[merge]; | 2312 | fragto = &skb_shinfo(tgt)->frags[merge]; |
2309 | 2313 | ||
2310 | fragto->size += fragfrom->size; | 2314 | fragto->size += fragfrom->size; |
2311 | put_page(fragfrom->page); | 2315 | __skb_frag_unref(fragfrom); |
2312 | } | 2316 | } |
2313 | 2317 | ||
2314 | /* Reposition in the original skb */ | 2318 | /* Reposition in the original skb */ |
@@ -2553,8 +2557,7 @@ int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, | |||
2553 | left = PAGE_SIZE - frag->page_offset; | 2557 | left = PAGE_SIZE - frag->page_offset; |
2554 | copy = (length > left)? left : length; | 2558 | copy = (length > left)? left : length; |
2555 | 2559 | ||
2556 | ret = getfrag(from, (page_address(frag->page) + | 2560 | ret = getfrag(from, skb_frag_address(frag) + frag->size, |
2557 | frag->page_offset + frag->size), | ||
2558 | offset, copy, 0, skb); | 2561 | offset, copy, 0, skb); |
2559 | if (ret < 0) | 2562 | if (ret < 0) |
2560 | return -EFAULT; | 2563 | return -EFAULT; |
@@ -2706,7 +2709,7 @@ struct sk_buff *skb_segment(struct sk_buff *skb, u32 features) | |||
2706 | 2709 | ||
2707 | while (pos < offset + len && i < nfrags) { | 2710 | while (pos < offset + len && i < nfrags) { |
2708 | *frag = skb_shinfo(skb)->frags[i]; | 2711 | *frag = skb_shinfo(skb)->frags[i]; |
2709 | get_page(frag->page); | 2712 | __skb_frag_ref(frag); |
2710 | size = frag->size; | 2713 | size = frag->size; |
2711 | 2714 | ||
2712 | if (pos < offset) { | 2715 | if (pos < offset) { |
@@ -2929,7 +2932,7 @@ __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) | |||
2929 | 2932 | ||
2930 | if (copy > len) | 2933 | if (copy > len) |
2931 | copy = len; | 2934 | copy = len; |
2932 | sg_set_page(&sg[elt], frag->page, copy, | 2935 | sg_set_page(&sg[elt], skb_frag_page(frag), copy, |
2933 | frag->page_offset+offset-start); | 2936 | frag->page_offset+offset-start); |
2934 | elt++; | 2937 | elt++; |
2935 | if (!(len -= copy)) | 2938 | if (!(len -= copy)) |
diff --git a/net/core/sock.c b/net/core/sock.c index bc745d00ea4d..b29ab61b029c 100644 --- a/net/core/sock.c +++ b/net/core/sock.c | |||
@@ -387,7 +387,7 @@ struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) | |||
387 | 387 | ||
388 | if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { | 388 | if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { |
389 | sk_tx_queue_clear(sk); | 389 | sk_tx_queue_clear(sk); |
390 | rcu_assign_pointer(sk->sk_dst_cache, NULL); | 390 | RCU_INIT_POINTER(sk->sk_dst_cache, NULL); |
391 | dst_release(dst); | 391 | dst_release(dst); |
392 | return NULL; | 392 | return NULL; |
393 | } | 393 | } |
@@ -1158,7 +1158,7 @@ static void __sk_free(struct sock *sk) | |||
1158 | atomic_read(&sk->sk_wmem_alloc) == 0); | 1158 | atomic_read(&sk->sk_wmem_alloc) == 0); |
1159 | if (filter) { | 1159 | if (filter) { |
1160 | sk_filter_uncharge(sk, filter); | 1160 | sk_filter_uncharge(sk, filter); |
1161 | rcu_assign_pointer(sk->sk_filter, NULL); | 1161 | RCU_INIT_POINTER(sk->sk_filter, NULL); |
1162 | } | 1162 | } |
1163 | 1163 | ||
1164 | sock_disable_timestamp(sk, SOCK_TIMESTAMP); | 1164 | sock_disable_timestamp(sk, SOCK_TIMESTAMP); |
@@ -1533,7 +1533,6 @@ struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, | |||
1533 | skb_shinfo(skb)->nr_frags = npages; | 1533 | skb_shinfo(skb)->nr_frags = npages; |
1534 | for (i = 0; i < npages; i++) { | 1534 | for (i = 0; i < npages; i++) { |
1535 | struct page *page; | 1535 | struct page *page; |
1536 | skb_frag_t *frag; | ||
1537 | 1536 | ||
1538 | page = alloc_pages(sk->sk_allocation, 0); | 1537 | page = alloc_pages(sk->sk_allocation, 0); |
1539 | if (!page) { | 1538 | if (!page) { |
@@ -1543,12 +1542,11 @@ struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, | |||
1543 | goto failure; | 1542 | goto failure; |
1544 | } | 1543 | } |
1545 | 1544 | ||
1546 | frag = &skb_shinfo(skb)->frags[i]; | 1545 | __skb_fill_page_desc(skb, i, |
1547 | frag->page = page; | 1546 | page, 0, |
1548 | frag->page_offset = 0; | 1547 | (data_len >= PAGE_SIZE ? |
1549 | frag->size = (data_len >= PAGE_SIZE ? | 1548 | PAGE_SIZE : |
1550 | PAGE_SIZE : | 1549 | data_len)); |
1551 | data_len); | ||
1552 | data_len -= PAGE_SIZE; | 1550 | data_len -= PAGE_SIZE; |
1553 | } | 1551 | } |
1554 | 1552 | ||
diff --git a/net/core/user_dma.c b/net/core/user_dma.c index 25d717ebc92e..34e9664cae3b 100644 --- a/net/core/user_dma.c +++ b/net/core/user_dma.c | |||
@@ -78,7 +78,7 @@ int dma_skb_copy_datagram_iovec(struct dma_chan *chan, | |||
78 | copy = end - offset; | 78 | copy = end - offset; |
79 | if (copy > 0) { | 79 | if (copy > 0) { |
80 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; | 80 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
81 | struct page *page = frag->page; | 81 | struct page *page = skb_frag_page(frag); |
82 | 82 | ||
83 | if (copy > len) | 83 | if (copy > len) |
84 | copy = len; | 84 | copy = len; |