diff options
Diffstat (limited to 'net/core/dev.c')
-rw-r--r-- | net/core/dev.c | 408 |
1 files changed, 259 insertions, 149 deletions
diff --git a/net/core/dev.c b/net/core/dev.c index 723a34710ad4..e1c1cdcc2bb0 100644 --- a/net/core/dev.c +++ b/net/core/dev.c | |||
@@ -101,8 +101,6 @@ | |||
101 | #include <linux/proc_fs.h> | 101 | #include <linux/proc_fs.h> |
102 | #include <linux/seq_file.h> | 102 | #include <linux/seq_file.h> |
103 | #include <linux/stat.h> | 103 | #include <linux/stat.h> |
104 | #include <linux/if_bridge.h> | ||
105 | #include <linux/if_macvlan.h> | ||
106 | #include <net/dst.h> | 104 | #include <net/dst.h> |
107 | #include <net/pkt_sched.h> | 105 | #include <net/pkt_sched.h> |
108 | #include <net/checksum.h> | 106 | #include <net/checksum.h> |
@@ -803,35 +801,31 @@ struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) | |||
803 | EXPORT_SYMBOL(dev_getfirstbyhwtype); | 801 | EXPORT_SYMBOL(dev_getfirstbyhwtype); |
804 | 802 | ||
805 | /** | 803 | /** |
806 | * dev_get_by_flags - find any device with given flags | 804 | * dev_get_by_flags_rcu - find any device with given flags |
807 | * @net: the applicable net namespace | 805 | * @net: the applicable net namespace |
808 | * @if_flags: IFF_* values | 806 | * @if_flags: IFF_* values |
809 | * @mask: bitmask of bits in if_flags to check | 807 | * @mask: bitmask of bits in if_flags to check |
810 | * | 808 | * |
811 | * Search for any interface with the given flags. Returns NULL if a device | 809 | * Search for any interface with the given flags. Returns NULL if a device |
812 | * is not found or a pointer to the device. The device returned has | 810 | * is not found or a pointer to the device. Must be called inside |
813 | * had a reference added and the pointer is safe until the user calls | 811 | * rcu_read_lock(), and result refcount is unchanged. |
814 | * dev_put to indicate they have finished with it. | ||
815 | */ | 812 | */ |
816 | 813 | ||
817 | struct net_device *dev_get_by_flags(struct net *net, unsigned short if_flags, | 814 | struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags, |
818 | unsigned short mask) | 815 | unsigned short mask) |
819 | { | 816 | { |
820 | struct net_device *dev, *ret; | 817 | struct net_device *dev, *ret; |
821 | 818 | ||
822 | ret = NULL; | 819 | ret = NULL; |
823 | rcu_read_lock(); | ||
824 | for_each_netdev_rcu(net, dev) { | 820 | for_each_netdev_rcu(net, dev) { |
825 | if (((dev->flags ^ if_flags) & mask) == 0) { | 821 | if (((dev->flags ^ if_flags) & mask) == 0) { |
826 | dev_hold(dev); | ||
827 | ret = dev; | 822 | ret = dev; |
828 | break; | 823 | break; |
829 | } | 824 | } |
830 | } | 825 | } |
831 | rcu_read_unlock(); | ||
832 | return ret; | 826 | return ret; |
833 | } | 827 | } |
834 | EXPORT_SYMBOL(dev_get_by_flags); | 828 | EXPORT_SYMBOL(dev_get_by_flags_rcu); |
835 | 829 | ||
836 | /** | 830 | /** |
837 | * dev_valid_name - check if name is okay for network device | 831 | * dev_valid_name - check if name is okay for network device |
@@ -1488,6 +1482,7 @@ static inline void net_timestamp_check(struct sk_buff *skb) | |||
1488 | int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) | 1482 | int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) |
1489 | { | 1483 | { |
1490 | skb_orphan(skb); | 1484 | skb_orphan(skb); |
1485 | nf_reset(skb); | ||
1491 | 1486 | ||
1492 | if (!(dev->flags & IFF_UP) || | 1487 | if (!(dev->flags & IFF_UP) || |
1493 | (skb->len > (dev->mtu + dev->hard_header_len))) { | 1488 | (skb->len > (dev->mtu + dev->hard_header_len))) { |
@@ -1541,7 +1536,8 @@ static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) | |||
1541 | if (net_ratelimit()) | 1536 | if (net_ratelimit()) |
1542 | printk(KERN_CRIT "protocol %04x is " | 1537 | printk(KERN_CRIT "protocol %04x is " |
1543 | "buggy, dev %s\n", | 1538 | "buggy, dev %s\n", |
1544 | skb2->protocol, dev->name); | 1539 | ntohs(skb2->protocol), |
1540 | dev->name); | ||
1545 | skb_reset_network_header(skb2); | 1541 | skb_reset_network_header(skb2); |
1546 | } | 1542 | } |
1547 | 1543 | ||
@@ -1911,8 +1907,32 @@ static int dev_gso_segment(struct sk_buff *skb) | |||
1911 | */ | 1907 | */ |
1912 | static inline void skb_orphan_try(struct sk_buff *skb) | 1908 | static inline void skb_orphan_try(struct sk_buff *skb) |
1913 | { | 1909 | { |
1914 | if (!skb_tx(skb)->flags) | 1910 | struct sock *sk = skb->sk; |
1911 | |||
1912 | if (sk && !skb_tx(skb)->flags) { | ||
1913 | /* skb_tx_hash() wont be able to get sk. | ||
1914 | * We copy sk_hash into skb->rxhash | ||
1915 | */ | ||
1916 | if (!skb->rxhash) | ||
1917 | skb->rxhash = sk->sk_hash; | ||
1915 | skb_orphan(skb); | 1918 | skb_orphan(skb); |
1919 | } | ||
1920 | } | ||
1921 | |||
1922 | /* | ||
1923 | * Returns true if either: | ||
1924 | * 1. skb has frag_list and the device doesn't support FRAGLIST, or | ||
1925 | * 2. skb is fragmented and the device does not support SG, or if | ||
1926 | * at least one of fragments is in highmem and device does not | ||
1927 | * support DMA from it. | ||
1928 | */ | ||
1929 | static inline int skb_needs_linearize(struct sk_buff *skb, | ||
1930 | struct net_device *dev) | ||
1931 | { | ||
1932 | return skb_is_nonlinear(skb) && | ||
1933 | ((skb_has_frags(skb) && !(dev->features & NETIF_F_FRAGLIST)) || | ||
1934 | (skb_shinfo(skb)->nr_frags && (!(dev->features & NETIF_F_SG) || | ||
1935 | illegal_highdma(dev, skb)))); | ||
1916 | } | 1936 | } |
1917 | 1937 | ||
1918 | int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, | 1938 | int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, |
@@ -1939,6 +1959,22 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, | |||
1939 | goto out_kfree_skb; | 1959 | goto out_kfree_skb; |
1940 | if (skb->next) | 1960 | if (skb->next) |
1941 | goto gso; | 1961 | goto gso; |
1962 | } else { | ||
1963 | if (skb_needs_linearize(skb, dev) && | ||
1964 | __skb_linearize(skb)) | ||
1965 | goto out_kfree_skb; | ||
1966 | |||
1967 | /* If packet is not checksummed and device does not | ||
1968 | * support checksumming for this protocol, complete | ||
1969 | * checksumming here. | ||
1970 | */ | ||
1971 | if (skb->ip_summed == CHECKSUM_PARTIAL) { | ||
1972 | skb_set_transport_header(skb, skb->csum_start - | ||
1973 | skb_headroom(skb)); | ||
1974 | if (!dev_can_checksum(dev, skb) && | ||
1975 | skb_checksum_help(skb)) | ||
1976 | goto out_kfree_skb; | ||
1977 | } | ||
1942 | } | 1978 | } |
1943 | 1979 | ||
1944 | rc = ops->ndo_start_xmit(skb, dev); | 1980 | rc = ops->ndo_start_xmit(skb, dev); |
@@ -1998,8 +2034,7 @@ u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb) | |||
1998 | if (skb->sk && skb->sk->sk_hash) | 2034 | if (skb->sk && skb->sk->sk_hash) |
1999 | hash = skb->sk->sk_hash; | 2035 | hash = skb->sk->sk_hash; |
2000 | else | 2036 | else |
2001 | hash = (__force u16) skb->protocol; | 2037 | hash = (__force u16) skb->protocol ^ skb->rxhash; |
2002 | |||
2003 | hash = jhash_1word(hash, hashrnd); | 2038 | hash = jhash_1word(hash, hashrnd); |
2004 | 2039 | ||
2005 | return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32); | 2040 | return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32); |
@@ -2022,12 +2057,11 @@ static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index) | |||
2022 | static struct netdev_queue *dev_pick_tx(struct net_device *dev, | 2057 | static struct netdev_queue *dev_pick_tx(struct net_device *dev, |
2023 | struct sk_buff *skb) | 2058 | struct sk_buff *skb) |
2024 | { | 2059 | { |
2025 | u16 queue_index; | 2060 | int queue_index; |
2026 | struct sock *sk = skb->sk; | 2061 | struct sock *sk = skb->sk; |
2027 | 2062 | ||
2028 | if (sk_tx_queue_recorded(sk)) { | 2063 | queue_index = sk_tx_queue_get(sk); |
2029 | queue_index = sk_tx_queue_get(sk); | 2064 | if (queue_index < 0) { |
2030 | } else { | ||
2031 | const struct net_device_ops *ops = dev->netdev_ops; | 2065 | const struct net_device_ops *ops = dev->netdev_ops; |
2032 | 2066 | ||
2033 | if (ops->ndo_select_queue) { | 2067 | if (ops->ndo_select_queue) { |
@@ -2056,14 +2090,24 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, | |||
2056 | struct netdev_queue *txq) | 2090 | struct netdev_queue *txq) |
2057 | { | 2091 | { |
2058 | spinlock_t *root_lock = qdisc_lock(q); | 2092 | spinlock_t *root_lock = qdisc_lock(q); |
2093 | bool contended = qdisc_is_running(q); | ||
2059 | int rc; | 2094 | int rc; |
2060 | 2095 | ||
2096 | /* | ||
2097 | * Heuristic to force contended enqueues to serialize on a | ||
2098 | * separate lock before trying to get qdisc main lock. | ||
2099 | * This permits __QDISC_STATE_RUNNING owner to get the lock more often | ||
2100 | * and dequeue packets faster. | ||
2101 | */ | ||
2102 | if (unlikely(contended)) | ||
2103 | spin_lock(&q->busylock); | ||
2104 | |||
2061 | spin_lock(root_lock); | 2105 | spin_lock(root_lock); |
2062 | if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { | 2106 | if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { |
2063 | kfree_skb(skb); | 2107 | kfree_skb(skb); |
2064 | rc = NET_XMIT_DROP; | 2108 | rc = NET_XMIT_DROP; |
2065 | } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && | 2109 | } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && |
2066 | !test_and_set_bit(__QDISC_STATE_RUNNING, &q->state)) { | 2110 | qdisc_run_begin(q)) { |
2067 | /* | 2111 | /* |
2068 | * This is a work-conserving queue; there are no old skbs | 2112 | * This is a work-conserving queue; there are no old skbs |
2069 | * waiting to be sent out; and the qdisc is not running - | 2113 | * waiting to be sent out; and the qdisc is not running - |
@@ -2072,37 +2116,33 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, | |||
2072 | if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE)) | 2116 | if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE)) |
2073 | skb_dst_force(skb); | 2117 | skb_dst_force(skb); |
2074 | __qdisc_update_bstats(q, skb->len); | 2118 | __qdisc_update_bstats(q, skb->len); |
2075 | if (sch_direct_xmit(skb, q, dev, txq, root_lock)) | 2119 | if (sch_direct_xmit(skb, q, dev, txq, root_lock)) { |
2120 | if (unlikely(contended)) { | ||
2121 | spin_unlock(&q->busylock); | ||
2122 | contended = false; | ||
2123 | } | ||
2076 | __qdisc_run(q); | 2124 | __qdisc_run(q); |
2077 | else | 2125 | } else |
2078 | clear_bit(__QDISC_STATE_RUNNING, &q->state); | 2126 | qdisc_run_end(q); |
2079 | 2127 | ||
2080 | rc = NET_XMIT_SUCCESS; | 2128 | rc = NET_XMIT_SUCCESS; |
2081 | } else { | 2129 | } else { |
2082 | skb_dst_force(skb); | 2130 | skb_dst_force(skb); |
2083 | rc = qdisc_enqueue_root(skb, q); | 2131 | rc = qdisc_enqueue_root(skb, q); |
2084 | qdisc_run(q); | 2132 | if (qdisc_run_begin(q)) { |
2133 | if (unlikely(contended)) { | ||
2134 | spin_unlock(&q->busylock); | ||
2135 | contended = false; | ||
2136 | } | ||
2137 | __qdisc_run(q); | ||
2138 | } | ||
2085 | } | 2139 | } |
2086 | spin_unlock(root_lock); | 2140 | spin_unlock(root_lock); |
2087 | 2141 | if (unlikely(contended)) | |
2142 | spin_unlock(&q->busylock); | ||
2088 | return rc; | 2143 | return rc; |
2089 | } | 2144 | } |
2090 | 2145 | ||
2091 | /* | ||
2092 | * Returns true if either: | ||
2093 | * 1. skb has frag_list and the device doesn't support FRAGLIST, or | ||
2094 | * 2. skb is fragmented and the device does not support SG, or if | ||
2095 | * at least one of fragments is in highmem and device does not | ||
2096 | * support DMA from it. | ||
2097 | */ | ||
2098 | static inline int skb_needs_linearize(struct sk_buff *skb, | ||
2099 | struct net_device *dev) | ||
2100 | { | ||
2101 | return (skb_has_frags(skb) && !(dev->features & NETIF_F_FRAGLIST)) || | ||
2102 | (skb_shinfo(skb)->nr_frags && (!(dev->features & NETIF_F_SG) || | ||
2103 | illegal_highdma(dev, skb))); | ||
2104 | } | ||
2105 | |||
2106 | /** | 2146 | /** |
2107 | * dev_queue_xmit - transmit a buffer | 2147 | * dev_queue_xmit - transmit a buffer |
2108 | * @skb: buffer to transmit | 2148 | * @skb: buffer to transmit |
@@ -2135,25 +2175,6 @@ int dev_queue_xmit(struct sk_buff *skb) | |||
2135 | struct Qdisc *q; | 2175 | struct Qdisc *q; |
2136 | int rc = -ENOMEM; | 2176 | int rc = -ENOMEM; |
2137 | 2177 | ||
2138 | /* GSO will handle the following emulations directly. */ | ||
2139 | if (netif_needs_gso(dev, skb)) | ||
2140 | goto gso; | ||
2141 | |||
2142 | /* Convert a paged skb to linear, if required */ | ||
2143 | if (skb_needs_linearize(skb, dev) && __skb_linearize(skb)) | ||
2144 | goto out_kfree_skb; | ||
2145 | |||
2146 | /* If packet is not checksummed and device does not support | ||
2147 | * checksumming for this protocol, complete checksumming here. | ||
2148 | */ | ||
2149 | if (skb->ip_summed == CHECKSUM_PARTIAL) { | ||
2150 | skb_set_transport_header(skb, skb->csum_start - | ||
2151 | skb_headroom(skb)); | ||
2152 | if (!dev_can_checksum(dev, skb) && skb_checksum_help(skb)) | ||
2153 | goto out_kfree_skb; | ||
2154 | } | ||
2155 | |||
2156 | gso: | ||
2157 | /* Disable soft irqs for various locks below. Also | 2178 | /* Disable soft irqs for various locks below. Also |
2158 | * stops preemption for RCU. | 2179 | * stops preemption for RCU. |
2159 | */ | 2180 | */ |
@@ -2212,7 +2233,6 @@ gso: | |||
2212 | rc = -ENETDOWN; | 2233 | rc = -ENETDOWN; |
2213 | rcu_read_unlock_bh(); | 2234 | rcu_read_unlock_bh(); |
2214 | 2235 | ||
2215 | out_kfree_skb: | ||
2216 | kfree_skb(skb); | 2236 | kfree_skb(skb); |
2217 | return rc; | 2237 | return rc; |
2218 | out: | 2238 | out: |
@@ -2597,70 +2617,14 @@ static inline int deliver_skb(struct sk_buff *skb, | |||
2597 | return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); | 2617 | return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); |
2598 | } | 2618 | } |
2599 | 2619 | ||
2600 | #if defined(CONFIG_BRIDGE) || defined (CONFIG_BRIDGE_MODULE) | 2620 | #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \ |
2601 | 2621 | (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)) | |
2602 | #if defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE) | ||
2603 | /* This hook is defined here for ATM LANE */ | 2622 | /* This hook is defined here for ATM LANE */ |
2604 | int (*br_fdb_test_addr_hook)(struct net_device *dev, | 2623 | int (*br_fdb_test_addr_hook)(struct net_device *dev, |
2605 | unsigned char *addr) __read_mostly; | 2624 | unsigned char *addr) __read_mostly; |
2606 | EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); | 2625 | EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); |
2607 | #endif | 2626 | #endif |
2608 | 2627 | ||
2609 | /* | ||
2610 | * If bridge module is loaded call bridging hook. | ||
2611 | * returns NULL if packet was consumed. | ||
2612 | */ | ||
2613 | struct sk_buff *(*br_handle_frame_hook)(struct net_bridge_port *p, | ||
2614 | struct sk_buff *skb) __read_mostly; | ||
2615 | EXPORT_SYMBOL_GPL(br_handle_frame_hook); | ||
2616 | |||
2617 | static inline struct sk_buff *handle_bridge(struct sk_buff *skb, | ||
2618 | struct packet_type **pt_prev, int *ret, | ||
2619 | struct net_device *orig_dev) | ||
2620 | { | ||
2621 | struct net_bridge_port *port; | ||
2622 | |||
2623 | if (skb->pkt_type == PACKET_LOOPBACK || | ||
2624 | (port = rcu_dereference(skb->dev->br_port)) == NULL) | ||
2625 | return skb; | ||
2626 | |||
2627 | if (*pt_prev) { | ||
2628 | *ret = deliver_skb(skb, *pt_prev, orig_dev); | ||
2629 | *pt_prev = NULL; | ||
2630 | } | ||
2631 | |||
2632 | return br_handle_frame_hook(port, skb); | ||
2633 | } | ||
2634 | #else | ||
2635 | #define handle_bridge(skb, pt_prev, ret, orig_dev) (skb) | ||
2636 | #endif | ||
2637 | |||
2638 | #if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) | ||
2639 | struct sk_buff *(*macvlan_handle_frame_hook)(struct macvlan_port *p, | ||
2640 | struct sk_buff *skb) __read_mostly; | ||
2641 | EXPORT_SYMBOL_GPL(macvlan_handle_frame_hook); | ||
2642 | |||
2643 | static inline struct sk_buff *handle_macvlan(struct sk_buff *skb, | ||
2644 | struct packet_type **pt_prev, | ||
2645 | int *ret, | ||
2646 | struct net_device *orig_dev) | ||
2647 | { | ||
2648 | struct macvlan_port *port; | ||
2649 | |||
2650 | port = rcu_dereference(skb->dev->macvlan_port); | ||
2651 | if (!port) | ||
2652 | return skb; | ||
2653 | |||
2654 | if (*pt_prev) { | ||
2655 | *ret = deliver_skb(skb, *pt_prev, orig_dev); | ||
2656 | *pt_prev = NULL; | ||
2657 | } | ||
2658 | return macvlan_handle_frame_hook(port, skb); | ||
2659 | } | ||
2660 | #else | ||
2661 | #define handle_macvlan(skb, pt_prev, ret, orig_dev) (skb) | ||
2662 | #endif | ||
2663 | |||
2664 | #ifdef CONFIG_NET_CLS_ACT | 2628 | #ifdef CONFIG_NET_CLS_ACT |
2665 | /* TODO: Maybe we should just force sch_ingress to be compiled in | 2629 | /* TODO: Maybe we should just force sch_ingress to be compiled in |
2666 | * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions | 2630 | * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions |
@@ -2678,10 +2642,10 @@ static int ing_filter(struct sk_buff *skb) | |||
2678 | int result = TC_ACT_OK; | 2642 | int result = TC_ACT_OK; |
2679 | struct Qdisc *q; | 2643 | struct Qdisc *q; |
2680 | 2644 | ||
2681 | if (MAX_RED_LOOP < ttl++) { | 2645 | if (unlikely(MAX_RED_LOOP < ttl++)) { |
2682 | printk(KERN_WARNING | 2646 | if (net_ratelimit()) |
2683 | "Redir loop detected Dropping packet (%d->%d)\n", | 2647 | pr_warning( "Redir loop detected Dropping packet (%d->%d)\n", |
2684 | skb->skb_iif, dev->ifindex); | 2648 | skb->skb_iif, dev->ifindex); |
2685 | return TC_ACT_SHOT; | 2649 | return TC_ACT_SHOT; |
2686 | } | 2650 | } |
2687 | 2651 | ||
@@ -2711,9 +2675,6 @@ static inline struct sk_buff *handle_ing(struct sk_buff *skb, | |||
2711 | if (*pt_prev) { | 2675 | if (*pt_prev) { |
2712 | *ret = deliver_skb(skb, *pt_prev, orig_dev); | 2676 | *ret = deliver_skb(skb, *pt_prev, orig_dev); |
2713 | *pt_prev = NULL; | 2677 | *pt_prev = NULL; |
2714 | } else { | ||
2715 | /* Huh? Why does turning on AF_PACKET affect this? */ | ||
2716 | skb->tc_verd = SET_TC_OK2MUNGE(skb->tc_verd); | ||
2717 | } | 2678 | } |
2718 | 2679 | ||
2719 | switch (ing_filter(skb)) { | 2680 | switch (ing_filter(skb)) { |
@@ -2756,6 +2717,51 @@ void netif_nit_deliver(struct sk_buff *skb) | |||
2756 | rcu_read_unlock(); | 2717 | rcu_read_unlock(); |
2757 | } | 2718 | } |
2758 | 2719 | ||
2720 | /** | ||
2721 | * netdev_rx_handler_register - register receive handler | ||
2722 | * @dev: device to register a handler for | ||
2723 | * @rx_handler: receive handler to register | ||
2724 | * @rx_handler_data: data pointer that is used by rx handler | ||
2725 | * | ||
2726 | * Register a receive hander for a device. This handler will then be | ||
2727 | * called from __netif_receive_skb. A negative errno code is returned | ||
2728 | * on a failure. | ||
2729 | * | ||
2730 | * The caller must hold the rtnl_mutex. | ||
2731 | */ | ||
2732 | int netdev_rx_handler_register(struct net_device *dev, | ||
2733 | rx_handler_func_t *rx_handler, | ||
2734 | void *rx_handler_data) | ||
2735 | { | ||
2736 | ASSERT_RTNL(); | ||
2737 | |||
2738 | if (dev->rx_handler) | ||
2739 | return -EBUSY; | ||
2740 | |||
2741 | rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); | ||
2742 | rcu_assign_pointer(dev->rx_handler, rx_handler); | ||
2743 | |||
2744 | return 0; | ||
2745 | } | ||
2746 | EXPORT_SYMBOL_GPL(netdev_rx_handler_register); | ||
2747 | |||
2748 | /** | ||
2749 | * netdev_rx_handler_unregister - unregister receive handler | ||
2750 | * @dev: device to unregister a handler from | ||
2751 | * | ||
2752 | * Unregister a receive hander from a device. | ||
2753 | * | ||
2754 | * The caller must hold the rtnl_mutex. | ||
2755 | */ | ||
2756 | void netdev_rx_handler_unregister(struct net_device *dev) | ||
2757 | { | ||
2758 | |||
2759 | ASSERT_RTNL(); | ||
2760 | rcu_assign_pointer(dev->rx_handler, NULL); | ||
2761 | rcu_assign_pointer(dev->rx_handler_data, NULL); | ||
2762 | } | ||
2763 | EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); | ||
2764 | |||
2759 | static inline void skb_bond_set_mac_by_master(struct sk_buff *skb, | 2765 | static inline void skb_bond_set_mac_by_master(struct sk_buff *skb, |
2760 | struct net_device *master) | 2766 | struct net_device *master) |
2761 | { | 2767 | { |
@@ -2777,7 +2783,8 @@ int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master) | |||
2777 | if (master->priv_flags & IFF_MASTER_ARPMON) | 2783 | if (master->priv_flags & IFF_MASTER_ARPMON) |
2778 | dev->last_rx = jiffies; | 2784 | dev->last_rx = jiffies; |
2779 | 2785 | ||
2780 | if ((master->priv_flags & IFF_MASTER_ALB) && master->br_port) { | 2786 | if ((master->priv_flags & IFF_MASTER_ALB) && |
2787 | (master->priv_flags & IFF_BRIDGE_PORT)) { | ||
2781 | /* Do address unmangle. The local destination address | 2788 | /* Do address unmangle. The local destination address |
2782 | * will be always the one master has. Provides the right | 2789 | * will be always the one master has. Provides the right |
2783 | * functionality in a bridge. | 2790 | * functionality in a bridge. |
@@ -2808,6 +2815,7 @@ EXPORT_SYMBOL(__skb_bond_should_drop); | |||
2808 | static int __netif_receive_skb(struct sk_buff *skb) | 2815 | static int __netif_receive_skb(struct sk_buff *skb) |
2809 | { | 2816 | { |
2810 | struct packet_type *ptype, *pt_prev; | 2817 | struct packet_type *ptype, *pt_prev; |
2818 | rx_handler_func_t *rx_handler; | ||
2811 | struct net_device *orig_dev; | 2819 | struct net_device *orig_dev; |
2812 | struct net_device *master; | 2820 | struct net_device *master; |
2813 | struct net_device *null_or_orig; | 2821 | struct net_device *null_or_orig; |
@@ -2849,8 +2857,7 @@ static int __netif_receive_skb(struct sk_buff *skb) | |||
2849 | skb->dev = master; | 2857 | skb->dev = master; |
2850 | } | 2858 | } |
2851 | 2859 | ||
2852 | __get_cpu_var(softnet_data).processed++; | 2860 | __this_cpu_inc(softnet_data.processed); |
2853 | |||
2854 | skb_reset_network_header(skb); | 2861 | skb_reset_network_header(skb); |
2855 | skb_reset_transport_header(skb); | 2862 | skb_reset_transport_header(skb); |
2856 | skb->mac_len = skb->network_header - skb->mac_header; | 2863 | skb->mac_len = skb->network_header - skb->mac_header; |
@@ -2882,12 +2889,17 @@ static int __netif_receive_skb(struct sk_buff *skb) | |||
2882 | ncls: | 2889 | ncls: |
2883 | #endif | 2890 | #endif |
2884 | 2891 | ||
2885 | skb = handle_bridge(skb, &pt_prev, &ret, orig_dev); | 2892 | /* Handle special case of bridge or macvlan */ |
2886 | if (!skb) | 2893 | rx_handler = rcu_dereference(skb->dev->rx_handler); |
2887 | goto out; | 2894 | if (rx_handler) { |
2888 | skb = handle_macvlan(skb, &pt_prev, &ret, orig_dev); | 2895 | if (pt_prev) { |
2889 | if (!skb) | 2896 | ret = deliver_skb(skb, pt_prev, orig_dev); |
2890 | goto out; | 2897 | pt_prev = NULL; |
2898 | } | ||
2899 | skb = rx_handler(skb); | ||
2900 | if (!skb) | ||
2901 | goto out; | ||
2902 | } | ||
2891 | 2903 | ||
2892 | /* | 2904 | /* |
2893 | * Make sure frames received on VLAN interfaces stacked on | 2905 | * Make sure frames received on VLAN interfaces stacked on |
@@ -2948,6 +2960,9 @@ int netif_receive_skb(struct sk_buff *skb) | |||
2948 | if (netdev_tstamp_prequeue) | 2960 | if (netdev_tstamp_prequeue) |
2949 | net_timestamp_check(skb); | 2961 | net_timestamp_check(skb); |
2950 | 2962 | ||
2963 | if (skb_defer_rx_timestamp(skb)) | ||
2964 | return NET_RX_SUCCESS; | ||
2965 | |||
2951 | #ifdef CONFIG_RPS | 2966 | #ifdef CONFIG_RPS |
2952 | { | 2967 | { |
2953 | struct rps_dev_flow voidflow, *rflow = &voidflow; | 2968 | struct rps_dev_flow voidflow, *rflow = &voidflow; |
@@ -3712,10 +3727,11 @@ void dev_seq_stop(struct seq_file *seq, void *v) | |||
3712 | 3727 | ||
3713 | static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev) | 3728 | static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev) |
3714 | { | 3729 | { |
3715 | const struct net_device_stats *stats = dev_get_stats(dev); | 3730 | struct rtnl_link_stats64 temp; |
3731 | const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp); | ||
3716 | 3732 | ||
3717 | seq_printf(seq, "%6s: %7lu %7lu %4lu %4lu %4lu %5lu %10lu %9lu " | 3733 | seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu " |
3718 | "%8lu %7lu %4lu %4lu %4lu %5lu %7lu %10lu\n", | 3734 | "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n", |
3719 | dev->name, stats->rx_bytes, stats->rx_packets, | 3735 | dev->name, stats->rx_bytes, stats->rx_packets, |
3720 | stats->rx_errors, | 3736 | stats->rx_errors, |
3721 | stats->rx_dropped + stats->rx_missed_errors, | 3737 | stats->rx_dropped + stats->rx_missed_errors, |
@@ -5264,20 +5280,22 @@ void netdev_run_todo(void) | |||
5264 | /** | 5280 | /** |
5265 | * dev_txq_stats_fold - fold tx_queues stats | 5281 | * dev_txq_stats_fold - fold tx_queues stats |
5266 | * @dev: device to get statistics from | 5282 | * @dev: device to get statistics from |
5267 | * @stats: struct net_device_stats to hold results | 5283 | * @stats: struct rtnl_link_stats64 to hold results |
5268 | */ | 5284 | */ |
5269 | void dev_txq_stats_fold(const struct net_device *dev, | 5285 | void dev_txq_stats_fold(const struct net_device *dev, |
5270 | struct net_device_stats *stats) | 5286 | struct rtnl_link_stats64 *stats) |
5271 | { | 5287 | { |
5272 | unsigned long tx_bytes = 0, tx_packets = 0, tx_dropped = 0; | 5288 | u64 tx_bytes = 0, tx_packets = 0, tx_dropped = 0; |
5273 | unsigned int i; | 5289 | unsigned int i; |
5274 | struct netdev_queue *txq; | 5290 | struct netdev_queue *txq; |
5275 | 5291 | ||
5276 | for (i = 0; i < dev->num_tx_queues; i++) { | 5292 | for (i = 0; i < dev->num_tx_queues; i++) { |
5277 | txq = netdev_get_tx_queue(dev, i); | 5293 | txq = netdev_get_tx_queue(dev, i); |
5294 | spin_lock_bh(&txq->_xmit_lock); | ||
5278 | tx_bytes += txq->tx_bytes; | 5295 | tx_bytes += txq->tx_bytes; |
5279 | tx_packets += txq->tx_packets; | 5296 | tx_packets += txq->tx_packets; |
5280 | tx_dropped += txq->tx_dropped; | 5297 | tx_dropped += txq->tx_dropped; |
5298 | spin_unlock_bh(&txq->_xmit_lock); | ||
5281 | } | 5299 | } |
5282 | if (tx_bytes || tx_packets || tx_dropped) { | 5300 | if (tx_bytes || tx_packets || tx_dropped) { |
5283 | stats->tx_bytes = tx_bytes; | 5301 | stats->tx_bytes = tx_bytes; |
@@ -5287,23 +5305,53 @@ void dev_txq_stats_fold(const struct net_device *dev, | |||
5287 | } | 5305 | } |
5288 | EXPORT_SYMBOL(dev_txq_stats_fold); | 5306 | EXPORT_SYMBOL(dev_txq_stats_fold); |
5289 | 5307 | ||
5308 | /* Convert net_device_stats to rtnl_link_stats64. They have the same | ||
5309 | * fields in the same order, with only the type differing. | ||
5310 | */ | ||
5311 | static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, | ||
5312 | const struct net_device_stats *netdev_stats) | ||
5313 | { | ||
5314 | #if BITS_PER_LONG == 64 | ||
5315 | BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats)); | ||
5316 | memcpy(stats64, netdev_stats, sizeof(*stats64)); | ||
5317 | #else | ||
5318 | size_t i, n = sizeof(*stats64) / sizeof(u64); | ||
5319 | const unsigned long *src = (const unsigned long *)netdev_stats; | ||
5320 | u64 *dst = (u64 *)stats64; | ||
5321 | |||
5322 | BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) != | ||
5323 | sizeof(*stats64) / sizeof(u64)); | ||
5324 | for (i = 0; i < n; i++) | ||
5325 | dst[i] = src[i]; | ||
5326 | #endif | ||
5327 | } | ||
5328 | |||
5290 | /** | 5329 | /** |
5291 | * dev_get_stats - get network device statistics | 5330 | * dev_get_stats - get network device statistics |
5292 | * @dev: device to get statistics from | 5331 | * @dev: device to get statistics from |
5332 | * @storage: place to store stats | ||
5293 | * | 5333 | * |
5294 | * Get network statistics from device. The device driver may provide | 5334 | * Get network statistics from device. Return @storage. |
5295 | * its own method by setting dev->netdev_ops->get_stats; otherwise | 5335 | * The device driver may provide its own method by setting |
5296 | * the internal statistics structure is used. | 5336 | * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; |
5337 | * otherwise the internal statistics structure is used. | ||
5297 | */ | 5338 | */ |
5298 | const struct net_device_stats *dev_get_stats(struct net_device *dev) | 5339 | struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, |
5340 | struct rtnl_link_stats64 *storage) | ||
5299 | { | 5341 | { |
5300 | const struct net_device_ops *ops = dev->netdev_ops; | 5342 | const struct net_device_ops *ops = dev->netdev_ops; |
5301 | 5343 | ||
5302 | if (ops->ndo_get_stats) | 5344 | if (ops->ndo_get_stats64) { |
5303 | return ops->ndo_get_stats(dev); | 5345 | memset(storage, 0, sizeof(*storage)); |
5304 | 5346 | return ops->ndo_get_stats64(dev, storage); | |
5305 | dev_txq_stats_fold(dev, &dev->stats); | 5347 | } |
5306 | return &dev->stats; | 5348 | if (ops->ndo_get_stats) { |
5349 | netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); | ||
5350 | return storage; | ||
5351 | } | ||
5352 | netdev_stats_to_stats64(storage, &dev->stats); | ||
5353 | dev_txq_stats_fold(dev, storage); | ||
5354 | return storage; | ||
5307 | } | 5355 | } |
5308 | EXPORT_SYMBOL(dev_get_stats); | 5356 | EXPORT_SYMBOL(dev_get_stats); |
5309 | 5357 | ||
@@ -5808,6 +5856,68 @@ char *netdev_drivername(const struct net_device *dev, char *buffer, int len) | |||
5808 | return buffer; | 5856 | return buffer; |
5809 | } | 5857 | } |
5810 | 5858 | ||
5859 | static int __netdev_printk(const char *level, const struct net_device *dev, | ||
5860 | struct va_format *vaf) | ||
5861 | { | ||
5862 | int r; | ||
5863 | |||
5864 | if (dev && dev->dev.parent) | ||
5865 | r = dev_printk(level, dev->dev.parent, "%s: %pV", | ||
5866 | netdev_name(dev), vaf); | ||
5867 | else if (dev) | ||
5868 | r = printk("%s%s: %pV", level, netdev_name(dev), vaf); | ||
5869 | else | ||
5870 | r = printk("%s(NULL net_device): %pV", level, vaf); | ||
5871 | |||
5872 | return r; | ||
5873 | } | ||
5874 | |||
5875 | int netdev_printk(const char *level, const struct net_device *dev, | ||
5876 | const char *format, ...) | ||
5877 | { | ||
5878 | struct va_format vaf; | ||
5879 | va_list args; | ||
5880 | int r; | ||
5881 | |||
5882 | va_start(args, format); | ||
5883 | |||
5884 | vaf.fmt = format; | ||
5885 | vaf.va = &args; | ||
5886 | |||
5887 | r = __netdev_printk(level, dev, &vaf); | ||
5888 | va_end(args); | ||
5889 | |||
5890 | return r; | ||
5891 | } | ||
5892 | EXPORT_SYMBOL(netdev_printk); | ||
5893 | |||
5894 | #define define_netdev_printk_level(func, level) \ | ||
5895 | int func(const struct net_device *dev, const char *fmt, ...) \ | ||
5896 | { \ | ||
5897 | int r; \ | ||
5898 | struct va_format vaf; \ | ||
5899 | va_list args; \ | ||
5900 | \ | ||
5901 | va_start(args, fmt); \ | ||
5902 | \ | ||
5903 | vaf.fmt = fmt; \ | ||
5904 | vaf.va = &args; \ | ||
5905 | \ | ||
5906 | r = __netdev_printk(level, dev, &vaf); \ | ||
5907 | va_end(args); \ | ||
5908 | \ | ||
5909 | return r; \ | ||
5910 | } \ | ||
5911 | EXPORT_SYMBOL(func); | ||
5912 | |||
5913 | define_netdev_printk_level(netdev_emerg, KERN_EMERG); | ||
5914 | define_netdev_printk_level(netdev_alert, KERN_ALERT); | ||
5915 | define_netdev_printk_level(netdev_crit, KERN_CRIT); | ||
5916 | define_netdev_printk_level(netdev_err, KERN_ERR); | ||
5917 | define_netdev_printk_level(netdev_warn, KERN_WARNING); | ||
5918 | define_netdev_printk_level(netdev_notice, KERN_NOTICE); | ||
5919 | define_netdev_printk_level(netdev_info, KERN_INFO); | ||
5920 | |||
5811 | static void __net_exit netdev_exit(struct net *net) | 5921 | static void __net_exit netdev_exit(struct net *net) |
5812 | { | 5922 | { |
5813 | kfree(net->dev_name_head); | 5923 | kfree(net->dev_name_head); |