aboutsummaryrefslogtreecommitdiffstats
path: root/net/core
diff options
context:
space:
mode:
Diffstat (limited to 'net/core')
-rw-r--r--net/core/Makefile1
-rw-r--r--net/core/dev.c138
-rw-r--r--net/core/ethtool.c556
-rw-r--r--net/core/neighbour.c162
-rw-r--r--net/core/net-sysfs.c56
-rw-r--r--net/core/netpoll.c6
-rw-r--r--net/core/netprio_cgroup.c344
-rw-r--r--net/core/pktgen.c15
-rw-r--r--net/core/skbuff.c71
-rw-r--r--net/core/sock.c42
-rw-r--r--net/core/sysctl_net_core.c9
11 files changed, 791 insertions, 609 deletions
diff --git a/net/core/Makefile b/net/core/Makefile
index 0d357b1c4e57..3606d40aae62 100644
--- a/net/core/Makefile
+++ b/net/core/Makefile
@@ -19,3 +19,4 @@ obj-$(CONFIG_FIB_RULES) += fib_rules.o
19obj-$(CONFIG_TRACEPOINTS) += net-traces.o 19obj-$(CONFIG_TRACEPOINTS) += net-traces.o
20obj-$(CONFIG_NET_DROP_MONITOR) += drop_monitor.o 20obj-$(CONFIG_NET_DROP_MONITOR) += drop_monitor.o
21obj-$(CONFIG_NETWORK_PHY_TIMESTAMPING) += timestamping.o 21obj-$(CONFIG_NETWORK_PHY_TIMESTAMPING) += timestamping.o
22obj-$(CONFIG_NETPRIO_CGROUP) += netprio_cgroup.o
diff --git a/net/core/dev.c b/net/core/dev.c
index 6ba50a1e404c..8afb244b205f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -137,6 +137,7 @@
137#include <linux/if_pppox.h> 137#include <linux/if_pppox.h>
138#include <linux/ppp_defs.h> 138#include <linux/ppp_defs.h>
139#include <linux/net_tstamp.h> 139#include <linux/net_tstamp.h>
140#include <linux/jump_label.h>
140 141
141#include "net-sysfs.h" 142#include "net-sysfs.h"
142 143
@@ -1320,8 +1321,6 @@ EXPORT_SYMBOL(dev_close);
1320 */ 1321 */
1321void dev_disable_lro(struct net_device *dev) 1322void dev_disable_lro(struct net_device *dev)
1322{ 1323{
1323 u32 flags;
1324
1325 /* 1324 /*
1326 * If we're trying to disable lro on a vlan device 1325 * If we're trying to disable lro on a vlan device
1327 * use the underlying physical device instead 1326 * use the underlying physical device instead
@@ -1329,15 +1328,9 @@ void dev_disable_lro(struct net_device *dev)
1329 if (is_vlan_dev(dev)) 1328 if (is_vlan_dev(dev))
1330 dev = vlan_dev_real_dev(dev); 1329 dev = vlan_dev_real_dev(dev);
1331 1330
1332 if (dev->ethtool_ops && dev->ethtool_ops->get_flags) 1331 dev->wanted_features &= ~NETIF_F_LRO;
1333 flags = dev->ethtool_ops->get_flags(dev); 1332 netdev_update_features(dev);
1334 else
1335 flags = ethtool_op_get_flags(dev);
1336 1333
1337 if (!(flags & ETH_FLAG_LRO))
1338 return;
1339
1340 __ethtool_set_flags(dev, flags & ~ETH_FLAG_LRO);
1341 if (unlikely(dev->features & NETIF_F_LRO)) 1334 if (unlikely(dev->features & NETIF_F_LRO))
1342 netdev_WARN(dev, "failed to disable LRO!\n"); 1335 netdev_WARN(dev, "failed to disable LRO!\n");
1343} 1336}
@@ -1449,34 +1442,32 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1449} 1442}
1450EXPORT_SYMBOL(call_netdevice_notifiers); 1443EXPORT_SYMBOL(call_netdevice_notifiers);
1451 1444
1452/* When > 0 there are consumers of rx skb time stamps */ 1445static struct jump_label_key netstamp_needed __read_mostly;
1453static atomic_t netstamp_needed = ATOMIC_INIT(0);
1454 1446
1455void net_enable_timestamp(void) 1447void net_enable_timestamp(void)
1456{ 1448{
1457 atomic_inc(&netstamp_needed); 1449 jump_label_inc(&netstamp_needed);
1458} 1450}
1459EXPORT_SYMBOL(net_enable_timestamp); 1451EXPORT_SYMBOL(net_enable_timestamp);
1460 1452
1461void net_disable_timestamp(void) 1453void net_disable_timestamp(void)
1462{ 1454{
1463 atomic_dec(&netstamp_needed); 1455 jump_label_dec(&netstamp_needed);
1464} 1456}
1465EXPORT_SYMBOL(net_disable_timestamp); 1457EXPORT_SYMBOL(net_disable_timestamp);
1466 1458
1467static inline void net_timestamp_set(struct sk_buff *skb) 1459static inline void net_timestamp_set(struct sk_buff *skb)
1468{ 1460{
1469 if (atomic_read(&netstamp_needed)) 1461 skb->tstamp.tv64 = 0;
1462 if (static_branch(&netstamp_needed))
1470 __net_timestamp(skb); 1463 __net_timestamp(skb);
1471 else
1472 skb->tstamp.tv64 = 0;
1473} 1464}
1474 1465
1475static inline void net_timestamp_check(struct sk_buff *skb) 1466#define net_timestamp_check(COND, SKB) \
1476{ 1467 if (static_branch(&netstamp_needed)) { \
1477 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed)) 1468 if ((COND) && !(SKB)->tstamp.tv64) \
1478 __net_timestamp(skb); 1469 __net_timestamp(SKB); \
1479} 1470 } \
1480 1471
1481static int net_hwtstamp_validate(struct ifreq *ifr) 1472static int net_hwtstamp_validate(struct ifreq *ifr)
1482{ 1473{
@@ -1923,7 +1914,8 @@ EXPORT_SYMBOL(skb_checksum_help);
1923 * It may return NULL if the skb requires no segmentation. This is 1914 * It may return NULL if the skb requires no segmentation. This is
1924 * only possible when GSO is used for verifying header integrity. 1915 * only possible when GSO is used for verifying header integrity.
1925 */ 1916 */
1926struct sk_buff *skb_gso_segment(struct sk_buff *skb, u32 features) 1917struct sk_buff *skb_gso_segment(struct sk_buff *skb,
1918 netdev_features_t features)
1927{ 1919{
1928 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 1920 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1929 struct packet_type *ptype; 1921 struct packet_type *ptype;
@@ -1953,9 +1945,9 @@ struct sk_buff *skb_gso_segment(struct sk_buff *skb, u32 features)
1953 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) 1945 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1954 dev->ethtool_ops->get_drvinfo(dev, &info); 1946 dev->ethtool_ops->get_drvinfo(dev, &info);
1955 1947
1956 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n", 1948 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d ip_summed=%d\n",
1957 info.driver, dev ? dev->features : 0L, 1949 info.driver, dev ? &dev->features : NULL,
1958 skb->sk ? skb->sk->sk_route_caps : 0L, 1950 skb->sk ? &skb->sk->sk_route_caps : NULL,
1959 skb->len, skb->data_len, skb->ip_summed); 1951 skb->len, skb->data_len, skb->ip_summed);
1960 1952
1961 if (skb_header_cloned(skb) && 1953 if (skb_header_cloned(skb) &&
@@ -2064,7 +2056,7 @@ static void dev_gso_skb_destructor(struct sk_buff *skb)
2064 * This function segments the given skb and stores the list of segments 2056 * This function segments the given skb and stores the list of segments
2065 * in skb->next. 2057 * in skb->next.
2066 */ 2058 */
2067static int dev_gso_segment(struct sk_buff *skb, int features) 2059static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
2068{ 2060{
2069 struct sk_buff *segs; 2061 struct sk_buff *segs;
2070 2062
@@ -2103,7 +2095,7 @@ static inline void skb_orphan_try(struct sk_buff *skb)
2103 } 2095 }
2104} 2096}
2105 2097
2106static bool can_checksum_protocol(unsigned long features, __be16 protocol) 2098static bool can_checksum_protocol(netdev_features_t features, __be16 protocol)
2107{ 2099{
2108 return ((features & NETIF_F_GEN_CSUM) || 2100 return ((features & NETIF_F_GEN_CSUM) ||
2109 ((features & NETIF_F_V4_CSUM) && 2101 ((features & NETIF_F_V4_CSUM) &&
@@ -2114,7 +2106,8 @@ static bool can_checksum_protocol(unsigned long features, __be16 protocol)
2114 protocol == htons(ETH_P_FCOE))); 2106 protocol == htons(ETH_P_FCOE)));
2115} 2107}
2116 2108
2117static u32 harmonize_features(struct sk_buff *skb, __be16 protocol, u32 features) 2109static netdev_features_t harmonize_features(struct sk_buff *skb,
2110 __be16 protocol, netdev_features_t features)
2118{ 2111{
2119 if (!can_checksum_protocol(features, protocol)) { 2112 if (!can_checksum_protocol(features, protocol)) {
2120 features &= ~NETIF_F_ALL_CSUM; 2113 features &= ~NETIF_F_ALL_CSUM;
@@ -2126,10 +2119,10 @@ static u32 harmonize_features(struct sk_buff *skb, __be16 protocol, u32 features
2126 return features; 2119 return features;
2127} 2120}
2128 2121
2129u32 netif_skb_features(struct sk_buff *skb) 2122netdev_features_t netif_skb_features(struct sk_buff *skb)
2130{ 2123{
2131 __be16 protocol = skb->protocol; 2124 __be16 protocol = skb->protocol;
2132 u32 features = skb->dev->features; 2125 netdev_features_t features = skb->dev->features;
2133 2126
2134 if (protocol == htons(ETH_P_8021Q)) { 2127 if (protocol == htons(ETH_P_8021Q)) {
2135 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 2128 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
@@ -2175,7 +2168,7 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2175 unsigned int skb_len; 2168 unsigned int skb_len;
2176 2169
2177 if (likely(!skb->next)) { 2170 if (likely(!skb->next)) {
2178 u32 features; 2171 netdev_features_t features;
2179 2172
2180 /* 2173 /*
2181 * If device doesn't need skb->dst, release it right now while 2174 * If device doesn't need skb->dst, release it right now while
@@ -2456,6 +2449,18 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2456 return rc; 2449 return rc;
2457} 2450}
2458 2451
2452#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
2453static void skb_update_prio(struct sk_buff *skb)
2454{
2455 struct netprio_map *map = rcu_dereference(skb->dev->priomap);
2456
2457 if ((!skb->priority) && (skb->sk) && map)
2458 skb->priority = map->priomap[skb->sk->sk_cgrp_prioidx];
2459}
2460#else
2461#define skb_update_prio(skb)
2462#endif
2463
2459static DEFINE_PER_CPU(int, xmit_recursion); 2464static DEFINE_PER_CPU(int, xmit_recursion);
2460#define RECURSION_LIMIT 10 2465#define RECURSION_LIMIT 10
2461 2466
@@ -2496,6 +2501,8 @@ int dev_queue_xmit(struct sk_buff *skb)
2496 */ 2501 */
2497 rcu_read_lock_bh(); 2502 rcu_read_lock_bh();
2498 2503
2504 skb_update_prio(skb);
2505
2499 txq = dev_pick_tx(dev, skb); 2506 txq = dev_pick_tx(dev, skb);
2500 q = rcu_dereference_bh(txq->qdisc); 2507 q = rcu_dereference_bh(txq->qdisc);
2501 2508
@@ -2718,6 +2725,8 @@ EXPORT_SYMBOL(__skb_get_rxhash);
2718struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 2725struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2719EXPORT_SYMBOL(rps_sock_flow_table); 2726EXPORT_SYMBOL(rps_sock_flow_table);
2720 2727
2728struct jump_label_key rps_needed __read_mostly;
2729
2721static struct rps_dev_flow * 2730static struct rps_dev_flow *
2722set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 2731set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2723 struct rps_dev_flow *rflow, u16 next_cpu) 2732 struct rps_dev_flow *rflow, u16 next_cpu)
@@ -2997,12 +3006,11 @@ int netif_rx(struct sk_buff *skb)
2997 if (netpoll_rx(skb)) 3006 if (netpoll_rx(skb))
2998 return NET_RX_DROP; 3007 return NET_RX_DROP;
2999 3008
3000 if (netdev_tstamp_prequeue) 3009 net_timestamp_check(netdev_tstamp_prequeue, skb);
3001 net_timestamp_check(skb);
3002 3010
3003 trace_netif_rx(skb); 3011 trace_netif_rx(skb);
3004#ifdef CONFIG_RPS 3012#ifdef CONFIG_RPS
3005 { 3013 if (static_branch(&rps_needed)) {
3006 struct rps_dev_flow voidflow, *rflow = &voidflow; 3014 struct rps_dev_flow voidflow, *rflow = &voidflow;
3007 int cpu; 3015 int cpu;
3008 3016
@@ -3017,14 +3025,13 @@ int netif_rx(struct sk_buff *skb)
3017 3025
3018 rcu_read_unlock(); 3026 rcu_read_unlock();
3019 preempt_enable(); 3027 preempt_enable();
3020 } 3028 } else
3021#else 3029#endif
3022 { 3030 {
3023 unsigned int qtail; 3031 unsigned int qtail;
3024 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 3032 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3025 put_cpu(); 3033 put_cpu();
3026 } 3034 }
3027#endif
3028 return ret; 3035 return ret;
3029} 3036}
3030EXPORT_SYMBOL(netif_rx); 3037EXPORT_SYMBOL(netif_rx);
@@ -3230,8 +3237,7 @@ static int __netif_receive_skb(struct sk_buff *skb)
3230 int ret = NET_RX_DROP; 3237 int ret = NET_RX_DROP;
3231 __be16 type; 3238 __be16 type;
3232 3239
3233 if (!netdev_tstamp_prequeue) 3240 net_timestamp_check(!netdev_tstamp_prequeue, skb);
3234 net_timestamp_check(skb);
3235 3241
3236 trace_netif_receive_skb(skb); 3242 trace_netif_receive_skb(skb);
3237 3243
@@ -3362,14 +3368,13 @@ out:
3362 */ 3368 */
3363int netif_receive_skb(struct sk_buff *skb) 3369int netif_receive_skb(struct sk_buff *skb)
3364{ 3370{
3365 if (netdev_tstamp_prequeue) 3371 net_timestamp_check(netdev_tstamp_prequeue, skb);
3366 net_timestamp_check(skb);
3367 3372
3368 if (skb_defer_rx_timestamp(skb)) 3373 if (skb_defer_rx_timestamp(skb))
3369 return NET_RX_SUCCESS; 3374 return NET_RX_SUCCESS;
3370 3375
3371#ifdef CONFIG_RPS 3376#ifdef CONFIG_RPS
3372 { 3377 if (static_branch(&rps_needed)) {
3373 struct rps_dev_flow voidflow, *rflow = &voidflow; 3378 struct rps_dev_flow voidflow, *rflow = &voidflow;
3374 int cpu, ret; 3379 int cpu, ret;
3375 3380
@@ -3380,16 +3385,12 @@ int netif_receive_skb(struct sk_buff *skb)
3380 if (cpu >= 0) { 3385 if (cpu >= 0) {
3381 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 3386 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3382 rcu_read_unlock(); 3387 rcu_read_unlock();
3383 } else { 3388 return ret;
3384 rcu_read_unlock();
3385 ret = __netif_receive_skb(skb);
3386 } 3389 }
3387 3390 rcu_read_unlock();
3388 return ret;
3389 } 3391 }
3390#else
3391 return __netif_receive_skb(skb);
3392#endif 3392#endif
3393 return __netif_receive_skb(skb);
3393} 3394}
3394EXPORT_SYMBOL(netif_receive_skb); 3395EXPORT_SYMBOL(netif_receive_skb);
3395 3396
@@ -5362,7 +5363,8 @@ static void rollback_registered(struct net_device *dev)
5362 list_del(&single); 5363 list_del(&single);
5363} 5364}
5364 5365
5365static u32 netdev_fix_features(struct net_device *dev, u32 features) 5366static netdev_features_t netdev_fix_features(struct net_device *dev,
5367 netdev_features_t features)
5366{ 5368{
5367 /* Fix illegal checksum combinations */ 5369 /* Fix illegal checksum combinations */
5368 if ((features & NETIF_F_HW_CSUM) && 5370 if ((features & NETIF_F_HW_CSUM) &&
@@ -5371,12 +5373,6 @@ static u32 netdev_fix_features(struct net_device *dev, u32 features)
5371 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 5373 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5372 } 5374 }
5373 5375
5374 if ((features & NETIF_F_NO_CSUM) &&
5375 (features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5376 netdev_warn(dev, "mixed no checksumming and other settings.\n");
5377 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5378 }
5379
5380 /* Fix illegal SG+CSUM combinations. */ 5376 /* Fix illegal SG+CSUM combinations. */
5381 if ((features & NETIF_F_SG) && 5377 if ((features & NETIF_F_SG) &&
5382 !(features & NETIF_F_ALL_CSUM)) { 5378 !(features & NETIF_F_ALL_CSUM)) {
@@ -5424,7 +5420,7 @@ static u32 netdev_fix_features(struct net_device *dev, u32 features)
5424 5420
5425int __netdev_update_features(struct net_device *dev) 5421int __netdev_update_features(struct net_device *dev)
5426{ 5422{
5427 u32 features; 5423 netdev_features_t features;
5428 int err = 0; 5424 int err = 0;
5429 5425
5430 ASSERT_RTNL(); 5426 ASSERT_RTNL();
@@ -5440,16 +5436,16 @@ int __netdev_update_features(struct net_device *dev)
5440 if (dev->features == features) 5436 if (dev->features == features)
5441 return 0; 5437 return 0;
5442 5438
5443 netdev_dbg(dev, "Features changed: 0x%08x -> 0x%08x\n", 5439 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5444 dev->features, features); 5440 &dev->features, &features);
5445 5441
5446 if (dev->netdev_ops->ndo_set_features) 5442 if (dev->netdev_ops->ndo_set_features)
5447 err = dev->netdev_ops->ndo_set_features(dev, features); 5443 err = dev->netdev_ops->ndo_set_features(dev, features);
5448 5444
5449 if (unlikely(err < 0)) { 5445 if (unlikely(err < 0)) {
5450 netdev_err(dev, 5446 netdev_err(dev,
5451 "set_features() failed (%d); wanted 0x%08x, left 0x%08x\n", 5447 "set_features() failed (%d); wanted %pNF, left %pNF\n",
5452 err, features, dev->features); 5448 err, &features, &dev->features);
5453 return -1; 5449 return -1;
5454 } 5450 }
5455 5451
@@ -5633,11 +5629,12 @@ int register_netdevice(struct net_device *dev)
5633 dev->wanted_features = dev->features & dev->hw_features; 5629 dev->wanted_features = dev->features & dev->hw_features;
5634 5630
5635 /* Turn on no cache copy if HW is doing checksum */ 5631 /* Turn on no cache copy if HW is doing checksum */
5636 dev->hw_features |= NETIF_F_NOCACHE_COPY; 5632 if (!(dev->flags & IFF_LOOPBACK)) {
5637 if ((dev->features & NETIF_F_ALL_CSUM) && 5633 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5638 !(dev->features & NETIF_F_NO_CSUM)) { 5634 if (dev->features & NETIF_F_ALL_CSUM) {
5639 dev->wanted_features |= NETIF_F_NOCACHE_COPY; 5635 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5640 dev->features |= NETIF_F_NOCACHE_COPY; 5636 dev->features |= NETIF_F_NOCACHE_COPY;
5637 }
5641 } 5638 }
5642 5639
5643 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 5640 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
@@ -6373,7 +6370,8 @@ static int dev_cpu_callback(struct notifier_block *nfb,
6373 * @one to the master device with current feature set @all. Will not 6370 * @one to the master device with current feature set @all. Will not
6374 * enable anything that is off in @mask. Returns the new feature set. 6371 * enable anything that is off in @mask. Returns the new feature set.
6375 */ 6372 */
6376u32 netdev_increment_features(u32 all, u32 one, u32 mask) 6373netdev_features_t netdev_increment_features(netdev_features_t all,
6374 netdev_features_t one, netdev_features_t mask)
6377{ 6375{
6378 if (mask & NETIF_F_GEN_CSUM) 6376 if (mask & NETIF_F_GEN_CSUM)
6379 mask |= NETIF_F_ALL_CSUM; 6377 mask |= NETIF_F_ALL_CSUM;
@@ -6382,10 +6380,6 @@ u32 netdev_increment_features(u32 all, u32 one, u32 mask)
6382 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask; 6380 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6383 all &= one | ~NETIF_F_ALL_FOR_ALL; 6381 all &= one | ~NETIF_F_ALL_FOR_ALL;
6384 6382
6385 /* If device needs checksumming, downgrade to it. */
6386 if (all & (NETIF_F_ALL_CSUM & ~NETIF_F_NO_CSUM))
6387 all &= ~NETIF_F_NO_CSUM;
6388
6389 /* If one device supports hw checksumming, set for all. */ 6383 /* If one device supports hw checksumming, set for all. */
6390 if (all & NETIF_F_GEN_CSUM) 6384 if (all & NETIF_F_GEN_CSUM)
6391 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM); 6385 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index f44481707124..31b0b7f5383e 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -36,235 +36,44 @@ u32 ethtool_op_get_link(struct net_device *dev)
36} 36}
37EXPORT_SYMBOL(ethtool_op_get_link); 37EXPORT_SYMBOL(ethtool_op_get_link);
38 38
39u32 ethtool_op_get_tx_csum(struct net_device *dev)
40{
41 return (dev->features & NETIF_F_ALL_CSUM) != 0;
42}
43EXPORT_SYMBOL(ethtool_op_get_tx_csum);
44
45int ethtool_op_set_tx_csum(struct net_device *dev, u32 data)
46{
47 if (data)
48 dev->features |= NETIF_F_IP_CSUM;
49 else
50 dev->features &= ~NETIF_F_IP_CSUM;
51
52 return 0;
53}
54EXPORT_SYMBOL(ethtool_op_set_tx_csum);
55
56int ethtool_op_set_tx_hw_csum(struct net_device *dev, u32 data)
57{
58 if (data)
59 dev->features |= NETIF_F_HW_CSUM;
60 else
61 dev->features &= ~NETIF_F_HW_CSUM;
62
63 return 0;
64}
65EXPORT_SYMBOL(ethtool_op_set_tx_hw_csum);
66
67int ethtool_op_set_tx_ipv6_csum(struct net_device *dev, u32 data)
68{
69 if (data)
70 dev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
71 else
72 dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
73
74 return 0;
75}
76EXPORT_SYMBOL(ethtool_op_set_tx_ipv6_csum);
77
78u32 ethtool_op_get_sg(struct net_device *dev)
79{
80 return (dev->features & NETIF_F_SG) != 0;
81}
82EXPORT_SYMBOL(ethtool_op_get_sg);
83
84int ethtool_op_set_sg(struct net_device *dev, u32 data)
85{
86 if (data)
87 dev->features |= NETIF_F_SG;
88 else
89 dev->features &= ~NETIF_F_SG;
90
91 return 0;
92}
93EXPORT_SYMBOL(ethtool_op_set_sg);
94
95u32 ethtool_op_get_tso(struct net_device *dev)
96{
97 return (dev->features & NETIF_F_TSO) != 0;
98}
99EXPORT_SYMBOL(ethtool_op_get_tso);
100
101int ethtool_op_set_tso(struct net_device *dev, u32 data)
102{
103 if (data)
104 dev->features |= NETIF_F_TSO;
105 else
106 dev->features &= ~NETIF_F_TSO;
107
108 return 0;
109}
110EXPORT_SYMBOL(ethtool_op_set_tso);
111
112u32 ethtool_op_get_ufo(struct net_device *dev)
113{
114 return (dev->features & NETIF_F_UFO) != 0;
115}
116EXPORT_SYMBOL(ethtool_op_get_ufo);
117
118int ethtool_op_set_ufo(struct net_device *dev, u32 data)
119{
120 if (data)
121 dev->features |= NETIF_F_UFO;
122 else
123 dev->features &= ~NETIF_F_UFO;
124 return 0;
125}
126EXPORT_SYMBOL(ethtool_op_set_ufo);
127
128/* the following list of flags are the same as their associated
129 * NETIF_F_xxx values in include/linux/netdevice.h
130 */
131static const u32 flags_dup_features =
132 (ETH_FLAG_LRO | ETH_FLAG_RXVLAN | ETH_FLAG_TXVLAN | ETH_FLAG_NTUPLE |
133 ETH_FLAG_RXHASH);
134
135u32 ethtool_op_get_flags(struct net_device *dev)
136{
137 /* in the future, this function will probably contain additional
138 * handling for flags which are not so easily handled
139 * by a simple masking operation
140 */
141
142 return dev->features & flags_dup_features;
143}
144EXPORT_SYMBOL(ethtool_op_get_flags);
145
146/* Check if device can enable (or disable) particular feature coded in "data"
147 * argument. Flags "supported" describe features that can be toggled by device.
148 * If feature can not be toggled, it state (enabled or disabled) must match
149 * hardcoded device features state, otherwise flags are marked as invalid.
150 */
151bool ethtool_invalid_flags(struct net_device *dev, u32 data, u32 supported)
152{
153 u32 features = dev->features & flags_dup_features;
154 /* "data" can contain only flags_dup_features bits,
155 * see __ethtool_set_flags */
156
157 return (features & ~supported) != (data & ~supported);
158}
159EXPORT_SYMBOL(ethtool_invalid_flags);
160
161int ethtool_op_set_flags(struct net_device *dev, u32 data, u32 supported)
162{
163 if (ethtool_invalid_flags(dev, data, supported))
164 return -EINVAL;
165
166 dev->features = ((dev->features & ~flags_dup_features) |
167 (data & flags_dup_features));
168 return 0;
169}
170EXPORT_SYMBOL(ethtool_op_set_flags);
171
172/* Handlers for each ethtool command */ 39/* Handlers for each ethtool command */
173 40
174#define ETHTOOL_DEV_FEATURE_WORDS 1 41#define ETHTOOL_DEV_FEATURE_WORDS ((NETDEV_FEATURE_COUNT + 31) / 32)
175 42
176static void ethtool_get_features_compat(struct net_device *dev, 43static const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = {
177 struct ethtool_get_features_block *features) 44 [NETIF_F_SG_BIT] = "tx-scatter-gather",
178{ 45 [NETIF_F_IP_CSUM_BIT] = "tx-checksum-ipv4",
179 if (!dev->ethtool_ops) 46 [NETIF_F_HW_CSUM_BIT] = "tx-checksum-ip-generic",
180 return; 47 [NETIF_F_IPV6_CSUM_BIT] = "tx-checksum-ipv6",
181 48 [NETIF_F_HIGHDMA_BIT] = "highdma",
182 /* getting RX checksum */ 49 [NETIF_F_FRAGLIST_BIT] = "tx-scatter-gather-fraglist",
183 if (dev->ethtool_ops->get_rx_csum) 50 [NETIF_F_HW_VLAN_TX_BIT] = "tx-vlan-hw-insert",
184 if (dev->ethtool_ops->get_rx_csum(dev)) 51
185 features[0].active |= NETIF_F_RXCSUM; 52 [NETIF_F_HW_VLAN_RX_BIT] = "rx-vlan-hw-parse",
186 53 [NETIF_F_HW_VLAN_FILTER_BIT] = "rx-vlan-filter",
187 /* mark legacy-changeable features */ 54 [NETIF_F_VLAN_CHALLENGED_BIT] = "vlan-challenged",
188 if (dev->ethtool_ops->set_sg) 55 [NETIF_F_GSO_BIT] = "tx-generic-segmentation",
189 features[0].available |= NETIF_F_SG; 56 [NETIF_F_LLTX_BIT] = "tx-lockless",
190 if (dev->ethtool_ops->set_tx_csum) 57 [NETIF_F_NETNS_LOCAL_BIT] = "netns-local",
191 features[0].available |= NETIF_F_ALL_CSUM; 58 [NETIF_F_GRO_BIT] = "rx-gro",
192 if (dev->ethtool_ops->set_tso) 59 [NETIF_F_LRO_BIT] = "rx-lro",
193 features[0].available |= NETIF_F_ALL_TSO; 60
194 if (dev->ethtool_ops->set_rx_csum) 61 [NETIF_F_TSO_BIT] = "tx-tcp-segmentation",
195 features[0].available |= NETIF_F_RXCSUM; 62 [NETIF_F_UFO_BIT] = "tx-udp-fragmentation",
196 if (dev->ethtool_ops->set_flags) 63 [NETIF_F_GSO_ROBUST_BIT] = "tx-gso-robust",
197 features[0].available |= flags_dup_features; 64 [NETIF_F_TSO_ECN_BIT] = "tx-tcp-ecn-segmentation",
198} 65 [NETIF_F_TSO6_BIT] = "tx-tcp6-segmentation",
199 66 [NETIF_F_FSO_BIT] = "tx-fcoe-segmentation",
200static int ethtool_set_feature_compat(struct net_device *dev, 67
201 int (*legacy_set)(struct net_device *, u32), 68 [NETIF_F_FCOE_CRC_BIT] = "tx-checksum-fcoe-crc",
202 struct ethtool_set_features_block *features, u32 mask) 69 [NETIF_F_SCTP_CSUM_BIT] = "tx-checksum-sctp",
203{ 70 [NETIF_F_FCOE_MTU_BIT] = "fcoe-mtu",
204 u32 do_set; 71 [NETIF_F_NTUPLE_BIT] = "rx-ntuple-filter",
205 72 [NETIF_F_RXHASH_BIT] = "rx-hashing",
206 if (!legacy_set) 73 [NETIF_F_RXCSUM_BIT] = "rx-checksum",
207 return 0; 74 [NETIF_F_NOCACHE_COPY_BIT] = "tx-nocache-copy",
208 75 [NETIF_F_LOOPBACK_BIT] = "loopback",
209 if (!(features[0].valid & mask)) 76};
210 return 0;
211
212 features[0].valid &= ~mask;
213
214 do_set = !!(features[0].requested & mask);
215
216 if (legacy_set(dev, do_set) < 0)
217 netdev_info(dev,
218 "Legacy feature change (%s) failed for 0x%08x\n",
219 do_set ? "set" : "clear", mask);
220
221 return 1;
222}
223
224static int ethtool_set_flags_compat(struct net_device *dev,
225 int (*legacy_set)(struct net_device *, u32),
226 struct ethtool_set_features_block *features, u32 mask)
227{
228 u32 value;
229
230 if (!legacy_set)
231 return 0;
232
233 if (!(features[0].valid & mask))
234 return 0;
235
236 value = dev->features & ~features[0].valid;
237 value |= features[0].requested;
238
239 features[0].valid &= ~mask;
240
241 if (legacy_set(dev, value & mask) < 0)
242 netdev_info(dev, "Legacy flags change failed\n");
243
244 return 1;
245}
246
247static int ethtool_set_features_compat(struct net_device *dev,
248 struct ethtool_set_features_block *features)
249{
250 int compat;
251
252 if (!dev->ethtool_ops)
253 return 0;
254
255 compat = ethtool_set_feature_compat(dev, dev->ethtool_ops->set_sg,
256 features, NETIF_F_SG);
257 compat |= ethtool_set_feature_compat(dev, dev->ethtool_ops->set_tx_csum,
258 features, NETIF_F_ALL_CSUM);
259 compat |= ethtool_set_feature_compat(dev, dev->ethtool_ops->set_tso,
260 features, NETIF_F_ALL_TSO);
261 compat |= ethtool_set_feature_compat(dev, dev->ethtool_ops->set_rx_csum,
262 features, NETIF_F_RXCSUM);
263 compat |= ethtool_set_flags_compat(dev, dev->ethtool_ops->set_flags,
264 features, flags_dup_features);
265
266 return compat;
267}
268 77
269static int ethtool_get_features(struct net_device *dev, void __user *useraddr) 78static int ethtool_get_features(struct net_device *dev, void __user *useraddr)
270{ 79{
@@ -272,18 +81,21 @@ static int ethtool_get_features(struct net_device *dev, void __user *useraddr)
272 .cmd = ETHTOOL_GFEATURES, 81 .cmd = ETHTOOL_GFEATURES,
273 .size = ETHTOOL_DEV_FEATURE_WORDS, 82 .size = ETHTOOL_DEV_FEATURE_WORDS,
274 }; 83 };
275 struct ethtool_get_features_block features[ETHTOOL_DEV_FEATURE_WORDS] = { 84 struct ethtool_get_features_block features[ETHTOOL_DEV_FEATURE_WORDS];
276 {
277 .available = dev->hw_features,
278 .requested = dev->wanted_features,
279 .active = dev->features,
280 .never_changed = NETIF_F_NEVER_CHANGE,
281 },
282 };
283 u32 __user *sizeaddr; 85 u32 __user *sizeaddr;
284 u32 copy_size; 86 u32 copy_size;
87 int i;
285 88
286 ethtool_get_features_compat(dev, features); 89 /* in case feature bits run out again */
90 BUILD_BUG_ON(ETHTOOL_DEV_FEATURE_WORDS * sizeof(u32) > sizeof(netdev_features_t));
91
92 for (i = 0; i < ETHTOOL_DEV_FEATURE_WORDS; ++i) {
93 features[i].available = (u32)(dev->hw_features >> (32 * i));
94 features[i].requested = (u32)(dev->wanted_features >> (32 * i));
95 features[i].active = (u32)(dev->features >> (32 * i));
96 features[i].never_changed =
97 (u32)(NETIF_F_NEVER_CHANGE >> (32 * i));
98 }
287 99
288 sizeaddr = useraddr + offsetof(struct ethtool_gfeatures, size); 100 sizeaddr = useraddr + offsetof(struct ethtool_gfeatures, size);
289 if (get_user(copy_size, sizeaddr)) 101 if (get_user(copy_size, sizeaddr))
@@ -305,7 +117,8 @@ static int ethtool_set_features(struct net_device *dev, void __user *useraddr)
305{ 117{
306 struct ethtool_sfeatures cmd; 118 struct ethtool_sfeatures cmd;
307 struct ethtool_set_features_block features[ETHTOOL_DEV_FEATURE_WORDS]; 119 struct ethtool_set_features_block features[ETHTOOL_DEV_FEATURE_WORDS];
308 int ret = 0; 120 netdev_features_t wanted = 0, valid = 0;
121 int i, ret = 0;
309 122
310 if (copy_from_user(&cmd, useraddr, sizeof(cmd))) 123 if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
311 return -EFAULT; 124 return -EFAULT;
@@ -317,65 +130,29 @@ static int ethtool_set_features(struct net_device *dev, void __user *useraddr)
317 if (copy_from_user(features, useraddr, sizeof(features))) 130 if (copy_from_user(features, useraddr, sizeof(features)))
318 return -EFAULT; 131 return -EFAULT;
319 132
320 if (features[0].valid & ~NETIF_F_ETHTOOL_BITS) 133 for (i = 0; i < ETHTOOL_DEV_FEATURE_WORDS; ++i) {
321 return -EINVAL; 134 valid |= (netdev_features_t)features[i].valid << (32 * i);
135 wanted |= (netdev_features_t)features[i].requested << (32 * i);
136 }
322 137
323 if (ethtool_set_features_compat(dev, features)) 138 if (valid & ~NETIF_F_ETHTOOL_BITS)
324 ret |= ETHTOOL_F_COMPAT; 139 return -EINVAL;
325 140
326 if (features[0].valid & ~dev->hw_features) { 141 if (valid & ~dev->hw_features) {
327 features[0].valid &= dev->hw_features; 142 valid &= dev->hw_features;
328 ret |= ETHTOOL_F_UNSUPPORTED; 143 ret |= ETHTOOL_F_UNSUPPORTED;
329 } 144 }
330 145
331 dev->wanted_features &= ~features[0].valid; 146 dev->wanted_features &= ~valid;
332 dev->wanted_features |= features[0].valid & features[0].requested; 147 dev->wanted_features |= wanted & valid;
333 __netdev_update_features(dev); 148 __netdev_update_features(dev);
334 149
335 if ((dev->wanted_features ^ dev->features) & features[0].valid) 150 if ((dev->wanted_features ^ dev->features) & valid)
336 ret |= ETHTOOL_F_WISH; 151 ret |= ETHTOOL_F_WISH;
337 152
338 return ret; 153 return ret;
339} 154}
340 155
341static const char netdev_features_strings[ETHTOOL_DEV_FEATURE_WORDS * 32][ETH_GSTRING_LEN] = {
342 /* NETIF_F_SG */ "tx-scatter-gather",
343 /* NETIF_F_IP_CSUM */ "tx-checksum-ipv4",
344 /* NETIF_F_NO_CSUM */ "tx-checksum-unneeded",
345 /* NETIF_F_HW_CSUM */ "tx-checksum-ip-generic",
346 /* NETIF_F_IPV6_CSUM */ "tx-checksum-ipv6",
347 /* NETIF_F_HIGHDMA */ "highdma",
348 /* NETIF_F_FRAGLIST */ "tx-scatter-gather-fraglist",
349 /* NETIF_F_HW_VLAN_TX */ "tx-vlan-hw-insert",
350
351 /* NETIF_F_HW_VLAN_RX */ "rx-vlan-hw-parse",
352 /* NETIF_F_HW_VLAN_FILTER */ "rx-vlan-filter",
353 /* NETIF_F_VLAN_CHALLENGED */ "vlan-challenged",
354 /* NETIF_F_GSO */ "tx-generic-segmentation",
355 /* NETIF_F_LLTX */ "tx-lockless",
356 /* NETIF_F_NETNS_LOCAL */ "netns-local",
357 /* NETIF_F_GRO */ "rx-gro",
358 /* NETIF_F_LRO */ "rx-lro",
359
360 /* NETIF_F_TSO */ "tx-tcp-segmentation",
361 /* NETIF_F_UFO */ "tx-udp-fragmentation",
362 /* NETIF_F_GSO_ROBUST */ "tx-gso-robust",
363 /* NETIF_F_TSO_ECN */ "tx-tcp-ecn-segmentation",
364 /* NETIF_F_TSO6 */ "tx-tcp6-segmentation",
365 /* NETIF_F_FSO */ "tx-fcoe-segmentation",
366 "",
367 "",
368
369 /* NETIF_F_FCOE_CRC */ "tx-checksum-fcoe-crc",
370 /* NETIF_F_SCTP_CSUM */ "tx-checksum-sctp",
371 /* NETIF_F_FCOE_MTU */ "fcoe-mtu",
372 /* NETIF_F_NTUPLE */ "rx-ntuple-filter",
373 /* NETIF_F_RXHASH */ "rx-hashing",
374 /* NETIF_F_RXCSUM */ "rx-checksum",
375 /* NETIF_F_NOCACHE_COPY */ "tx-nocache-copy",
376 /* NETIF_F_LOOPBACK */ "loopback",
377};
378
379static int __ethtool_get_sset_count(struct net_device *dev, int sset) 156static int __ethtool_get_sset_count(struct net_device *dev, int sset)
380{ 157{
381 const struct ethtool_ops *ops = dev->ethtool_ops; 158 const struct ethtool_ops *ops = dev->ethtool_ops;
@@ -402,7 +179,7 @@ static void __ethtool_get_strings(struct net_device *dev,
402 ops->get_strings(dev, stringset, data); 179 ops->get_strings(dev, stringset, data);
403} 180}
404 181
405static u32 ethtool_get_feature_mask(u32 eth_cmd) 182static netdev_features_t ethtool_get_feature_mask(u32 eth_cmd)
406{ 183{
407 /* feature masks of legacy discrete ethtool ops */ 184 /* feature masks of legacy discrete ethtool ops */
408 185
@@ -433,136 +210,82 @@ static u32 ethtool_get_feature_mask(u32 eth_cmd)
433 } 210 }
434} 211}
435 212
436static void *__ethtool_get_one_feature_actor(struct net_device *dev, u32 ethcmd)
437{
438 const struct ethtool_ops *ops = dev->ethtool_ops;
439
440 if (!ops)
441 return NULL;
442
443 switch (ethcmd) {
444 case ETHTOOL_GTXCSUM:
445 return ops->get_tx_csum;
446 case ETHTOOL_GRXCSUM:
447 return ops->get_rx_csum;
448 case ETHTOOL_SSG:
449 return ops->get_sg;
450 case ETHTOOL_STSO:
451 return ops->get_tso;
452 case ETHTOOL_SUFO:
453 return ops->get_ufo;
454 default:
455 return NULL;
456 }
457}
458
459static u32 __ethtool_get_rx_csum_oldbug(struct net_device *dev)
460{
461 return !!(dev->features & NETIF_F_ALL_CSUM);
462}
463
464static int ethtool_get_one_feature(struct net_device *dev, 213static int ethtool_get_one_feature(struct net_device *dev,
465 char __user *useraddr, u32 ethcmd) 214 char __user *useraddr, u32 ethcmd)
466{ 215{
467 u32 mask = ethtool_get_feature_mask(ethcmd); 216 netdev_features_t mask = ethtool_get_feature_mask(ethcmd);
468 struct ethtool_value edata = { 217 struct ethtool_value edata = {
469 .cmd = ethcmd, 218 .cmd = ethcmd,
470 .data = !!(dev->features & mask), 219 .data = !!(dev->features & mask),
471 }; 220 };
472 221
473 /* compatibility with discrete get_ ops */
474 if (!(dev->hw_features & mask)) {
475 u32 (*actor)(struct net_device *);
476
477 actor = __ethtool_get_one_feature_actor(dev, ethcmd);
478
479 /* bug compatibility with old get_rx_csum */
480 if (ethcmd == ETHTOOL_GRXCSUM && !actor)
481 actor = __ethtool_get_rx_csum_oldbug;
482
483 if (actor)
484 edata.data = actor(dev);
485 }
486
487 if (copy_to_user(useraddr, &edata, sizeof(edata))) 222 if (copy_to_user(useraddr, &edata, sizeof(edata)))
488 return -EFAULT; 223 return -EFAULT;
489 return 0; 224 return 0;
490} 225}
491 226
492static int __ethtool_set_tx_csum(struct net_device *dev, u32 data);
493static int __ethtool_set_rx_csum(struct net_device *dev, u32 data);
494static int __ethtool_set_sg(struct net_device *dev, u32 data);
495static int __ethtool_set_tso(struct net_device *dev, u32 data);
496static int __ethtool_set_ufo(struct net_device *dev, u32 data);
497
498static int ethtool_set_one_feature(struct net_device *dev, 227static int ethtool_set_one_feature(struct net_device *dev,
499 void __user *useraddr, u32 ethcmd) 228 void __user *useraddr, u32 ethcmd)
500{ 229{
501 struct ethtool_value edata; 230 struct ethtool_value edata;
502 u32 mask; 231 netdev_features_t mask;
503 232
504 if (copy_from_user(&edata, useraddr, sizeof(edata))) 233 if (copy_from_user(&edata, useraddr, sizeof(edata)))
505 return -EFAULT; 234 return -EFAULT;
506 235
507 mask = ethtool_get_feature_mask(ethcmd); 236 mask = ethtool_get_feature_mask(ethcmd);
508 mask &= dev->hw_features; 237 mask &= dev->hw_features;
509 if (mask) { 238 if (!mask)
510 if (edata.data) 239 return -EOPNOTSUPP;
511 dev->wanted_features |= mask;
512 else
513 dev->wanted_features &= ~mask;
514 240
515 __netdev_update_features(dev); 241 if (edata.data)
516 return 0; 242 dev->wanted_features |= mask;
517 } 243 else
244 dev->wanted_features &= ~mask;
518 245
519 /* Driver is not converted to ndo_fix_features or does not 246 __netdev_update_features(dev);
520 * support changing this offload. In the latter case it won't
521 * have corresponding ethtool_ops field set.
522 *
523 * Following part is to be removed after all drivers advertise
524 * their changeable features in netdev->hw_features and stop
525 * using discrete offload setting ops.
526 */
527 247
528 switch (ethcmd) { 248 return 0;
529 case ETHTOOL_STXCSUM: 249}
530 return __ethtool_set_tx_csum(dev, edata.data); 250
531 case ETHTOOL_SRXCSUM: 251#define ETH_ALL_FLAGS (ETH_FLAG_LRO | ETH_FLAG_RXVLAN | ETH_FLAG_TXVLAN | \
532 return __ethtool_set_rx_csum(dev, edata.data); 252 ETH_FLAG_NTUPLE | ETH_FLAG_RXHASH)
533 case ETHTOOL_SSG: 253#define ETH_ALL_FEATURES (NETIF_F_LRO | NETIF_F_HW_VLAN_RX | \
534 return __ethtool_set_sg(dev, edata.data); 254 NETIF_F_HW_VLAN_TX | NETIF_F_NTUPLE | NETIF_F_RXHASH)
535 case ETHTOOL_STSO: 255
536 return __ethtool_set_tso(dev, edata.data); 256static u32 __ethtool_get_flags(struct net_device *dev)
537 case ETHTOOL_SUFO: 257{
538 return __ethtool_set_ufo(dev, edata.data); 258 u32 flags = 0;
539 default: 259
540 return -EOPNOTSUPP; 260 if (dev->features & NETIF_F_LRO) flags |= ETH_FLAG_LRO;
541 } 261 if (dev->features & NETIF_F_HW_VLAN_RX) flags |= ETH_FLAG_RXVLAN;
262 if (dev->features & NETIF_F_HW_VLAN_TX) flags |= ETH_FLAG_TXVLAN;
263 if (dev->features & NETIF_F_NTUPLE) flags |= ETH_FLAG_NTUPLE;
264 if (dev->features & NETIF_F_RXHASH) flags |= ETH_FLAG_RXHASH;
265
266 return flags;
542} 267}
543 268
544int __ethtool_set_flags(struct net_device *dev, u32 data) 269static int __ethtool_set_flags(struct net_device *dev, u32 data)
545{ 270{
546 u32 changed; 271 netdev_features_t features = 0, changed;
547 272
548 if (data & ~flags_dup_features) 273 if (data & ~ETH_ALL_FLAGS)
549 return -EINVAL; 274 return -EINVAL;
550 275
551 /* legacy set_flags() op */ 276 if (data & ETH_FLAG_LRO) features |= NETIF_F_LRO;
552 if (dev->ethtool_ops->set_flags) { 277 if (data & ETH_FLAG_RXVLAN) features |= NETIF_F_HW_VLAN_RX;
553 if (unlikely(dev->hw_features & flags_dup_features)) 278 if (data & ETH_FLAG_TXVLAN) features |= NETIF_F_HW_VLAN_TX;
554 netdev_warn(dev, 279 if (data & ETH_FLAG_NTUPLE) features |= NETIF_F_NTUPLE;
555 "driver BUG: mixed hw_features and set_flags()\n"); 280 if (data & ETH_FLAG_RXHASH) features |= NETIF_F_RXHASH;
556 return dev->ethtool_ops->set_flags(dev, data);
557 }
558 281
559 /* allow changing only bits set in hw_features */ 282 /* allow changing only bits set in hw_features */
560 changed = (data ^ dev->features) & flags_dup_features; 283 changed = (features ^ dev->features) & ETH_ALL_FEATURES;
561 if (changed & ~dev->hw_features) 284 if (changed & ~dev->hw_features)
562 return (changed & dev->hw_features) ? -EINVAL : -EOPNOTSUPP; 285 return (changed & dev->hw_features) ? -EINVAL : -EOPNOTSUPP;
563 286
564 dev->wanted_features = 287 dev->wanted_features =
565 (dev->wanted_features & ~changed) | (data & dev->hw_features); 288 (dev->wanted_features & ~changed) | (features & changed);
566 289
567 __netdev_update_features(dev); 290 __netdev_update_features(dev);
568 291
@@ -1231,81 +954,6 @@ static int ethtool_set_pauseparam(struct net_device *dev, void __user *useraddr)
1231 return dev->ethtool_ops->set_pauseparam(dev, &pauseparam); 954 return dev->ethtool_ops->set_pauseparam(dev, &pauseparam);
1232} 955}
1233 956
1234static int __ethtool_set_sg(struct net_device *dev, u32 data)
1235{
1236 int err;
1237
1238 if (!dev->ethtool_ops->set_sg)
1239 return -EOPNOTSUPP;
1240
1241 if (data && !(dev->features & NETIF_F_ALL_CSUM))
1242 return -EINVAL;
1243
1244 if (!data && dev->ethtool_ops->set_tso) {
1245 err = dev->ethtool_ops->set_tso(dev, 0);
1246 if (err)
1247 return err;
1248 }
1249
1250 if (!data && dev->ethtool_ops->set_ufo) {
1251 err = dev->ethtool_ops->set_ufo(dev, 0);
1252 if (err)
1253 return err;
1254 }
1255 return dev->ethtool_ops->set_sg(dev, data);
1256}
1257
1258static int __ethtool_set_tx_csum(struct net_device *dev, u32 data)
1259{
1260 int err;
1261
1262 if (!dev->ethtool_ops->set_tx_csum)
1263 return -EOPNOTSUPP;
1264
1265 if (!data && dev->ethtool_ops->set_sg) {
1266 err = __ethtool_set_sg(dev, 0);
1267 if (err)
1268 return err;
1269 }
1270
1271 return dev->ethtool_ops->set_tx_csum(dev, data);
1272}
1273
1274static int __ethtool_set_rx_csum(struct net_device *dev, u32 data)
1275{
1276 if (!dev->ethtool_ops->set_rx_csum)
1277 return -EOPNOTSUPP;
1278
1279 if (!data)
1280 dev->features &= ~NETIF_F_GRO;
1281
1282 return dev->ethtool_ops->set_rx_csum(dev, data);
1283}
1284
1285static int __ethtool_set_tso(struct net_device *dev, u32 data)
1286{
1287 if (!dev->ethtool_ops->set_tso)
1288 return -EOPNOTSUPP;
1289
1290 if (data && !(dev->features & NETIF_F_SG))
1291 return -EINVAL;
1292
1293 return dev->ethtool_ops->set_tso(dev, data);
1294}
1295
1296static int __ethtool_set_ufo(struct net_device *dev, u32 data)
1297{
1298 if (!dev->ethtool_ops->set_ufo)
1299 return -EOPNOTSUPP;
1300 if (data && !(dev->features & NETIF_F_SG))
1301 return -EINVAL;
1302 if (data && !((dev->features & NETIF_F_GEN_CSUM) ||
1303 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
1304 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM)))
1305 return -EINVAL;
1306 return dev->ethtool_ops->set_ufo(dev, data);
1307}
1308
1309static int ethtool_self_test(struct net_device *dev, char __user *useraddr) 957static int ethtool_self_test(struct net_device *dev, char __user *useraddr)
1310{ 958{
1311 struct ethtool_test test; 959 struct ethtool_test test;
@@ -1771,9 +1419,7 @@ int dev_ethtool(struct net *net, struct ifreq *ifr)
1771 break; 1419 break;
1772 case ETHTOOL_GFLAGS: 1420 case ETHTOOL_GFLAGS:
1773 rc = ethtool_get_value(dev, useraddr, ethcmd, 1421 rc = ethtool_get_value(dev, useraddr, ethcmd,
1774 (dev->ethtool_ops->get_flags ? 1422 __ethtool_get_flags);
1775 dev->ethtool_ops->get_flags :
1776 ethtool_op_get_flags));
1777 break; 1423 break;
1778 case ETHTOOL_SFLAGS: 1424 case ETHTOOL_SFLAGS:
1779 rc = ethtool_set_value(dev, useraddr, __ethtool_set_flags); 1425 rc = ethtool_set_value(dev, useraddr, __ethtool_set_flags);
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 5ac07d31fbc9..27d3fefeaa13 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -238,6 +238,7 @@ static void neigh_flush_dev(struct neigh_table *tbl, struct net_device *dev)
238 it to safe state. 238 it to safe state.
239 */ 239 */
240 skb_queue_purge(&n->arp_queue); 240 skb_queue_purge(&n->arp_queue);
241 n->arp_queue_len_bytes = 0;
241 n->output = neigh_blackhole; 242 n->output = neigh_blackhole;
242 if (n->nud_state & NUD_VALID) 243 if (n->nud_state & NUD_VALID)
243 n->nud_state = NUD_NOARP; 244 n->nud_state = NUD_NOARP;
@@ -702,6 +703,7 @@ void neigh_destroy(struct neighbour *neigh)
702 printk(KERN_WARNING "Impossible event.\n"); 703 printk(KERN_WARNING "Impossible event.\n");
703 704
704 skb_queue_purge(&neigh->arp_queue); 705 skb_queue_purge(&neigh->arp_queue);
706 neigh->arp_queue_len_bytes = 0;
705 707
706 dev_put(neigh->dev); 708 dev_put(neigh->dev);
707 neigh_parms_put(neigh->parms); 709 neigh_parms_put(neigh->parms);
@@ -842,6 +844,7 @@ static void neigh_invalidate(struct neighbour *neigh)
842 write_lock(&neigh->lock); 844 write_lock(&neigh->lock);
843 } 845 }
844 skb_queue_purge(&neigh->arp_queue); 846 skb_queue_purge(&neigh->arp_queue);
847 neigh->arp_queue_len_bytes = 0;
845} 848}
846 849
847static void neigh_probe(struct neighbour *neigh) 850static void neigh_probe(struct neighbour *neigh)
@@ -980,15 +983,20 @@ int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb)
980 983
981 if (neigh->nud_state == NUD_INCOMPLETE) { 984 if (neigh->nud_state == NUD_INCOMPLETE) {
982 if (skb) { 985 if (skb) {
983 if (skb_queue_len(&neigh->arp_queue) >= 986 while (neigh->arp_queue_len_bytes + skb->truesize >
984 neigh->parms->queue_len) { 987 neigh->parms->queue_len_bytes) {
985 struct sk_buff *buff; 988 struct sk_buff *buff;
989
986 buff = __skb_dequeue(&neigh->arp_queue); 990 buff = __skb_dequeue(&neigh->arp_queue);
991 if (!buff)
992 break;
993 neigh->arp_queue_len_bytes -= buff->truesize;
987 kfree_skb(buff); 994 kfree_skb(buff);
988 NEIGH_CACHE_STAT_INC(neigh->tbl, unres_discards); 995 NEIGH_CACHE_STAT_INC(neigh->tbl, unres_discards);
989 } 996 }
990 skb_dst_force(skb); 997 skb_dst_force(skb);
991 __skb_queue_tail(&neigh->arp_queue, skb); 998 __skb_queue_tail(&neigh->arp_queue, skb);
999 neigh->arp_queue_len_bytes += skb->truesize;
992 } 1000 }
993 rc = 1; 1001 rc = 1;
994 } 1002 }
@@ -1175,6 +1183,7 @@ int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new,
1175 write_lock_bh(&neigh->lock); 1183 write_lock_bh(&neigh->lock);
1176 } 1184 }
1177 skb_queue_purge(&neigh->arp_queue); 1185 skb_queue_purge(&neigh->arp_queue);
1186 neigh->arp_queue_len_bytes = 0;
1178 } 1187 }
1179out: 1188out:
1180 if (update_isrouter) { 1189 if (update_isrouter) {
@@ -1747,7 +1756,11 @@ static int neightbl_fill_parms(struct sk_buff *skb, struct neigh_parms *parms)
1747 NLA_PUT_U32(skb, NDTPA_IFINDEX, parms->dev->ifindex); 1756 NLA_PUT_U32(skb, NDTPA_IFINDEX, parms->dev->ifindex);
1748 1757
1749 NLA_PUT_U32(skb, NDTPA_REFCNT, atomic_read(&parms->refcnt)); 1758 NLA_PUT_U32(skb, NDTPA_REFCNT, atomic_read(&parms->refcnt));
1750 NLA_PUT_U32(skb, NDTPA_QUEUE_LEN, parms->queue_len); 1759 NLA_PUT_U32(skb, NDTPA_QUEUE_LENBYTES, parms->queue_len_bytes);
1760 /* approximative value for deprecated QUEUE_LEN (in packets) */
1761 NLA_PUT_U32(skb, NDTPA_QUEUE_LEN,
1762 DIV_ROUND_UP(parms->queue_len_bytes,
1763 SKB_TRUESIZE(ETH_FRAME_LEN)));
1751 NLA_PUT_U32(skb, NDTPA_PROXY_QLEN, parms->proxy_qlen); 1764 NLA_PUT_U32(skb, NDTPA_PROXY_QLEN, parms->proxy_qlen);
1752 NLA_PUT_U32(skb, NDTPA_APP_PROBES, parms->app_probes); 1765 NLA_PUT_U32(skb, NDTPA_APP_PROBES, parms->app_probes);
1753 NLA_PUT_U32(skb, NDTPA_UCAST_PROBES, parms->ucast_probes); 1766 NLA_PUT_U32(skb, NDTPA_UCAST_PROBES, parms->ucast_probes);
@@ -1974,7 +1987,11 @@ static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh, void *arg)
1974 1987
1975 switch (i) { 1988 switch (i) {
1976 case NDTPA_QUEUE_LEN: 1989 case NDTPA_QUEUE_LEN:
1977 p->queue_len = nla_get_u32(tbp[i]); 1990 p->queue_len_bytes = nla_get_u32(tbp[i]) *
1991 SKB_TRUESIZE(ETH_FRAME_LEN);
1992 break;
1993 case NDTPA_QUEUE_LENBYTES:
1994 p->queue_len_bytes = nla_get_u32(tbp[i]);
1978 break; 1995 break;
1979 case NDTPA_PROXY_QLEN: 1996 case NDTPA_PROXY_QLEN:
1980 p->proxy_qlen = nla_get_u32(tbp[i]); 1997 p->proxy_qlen = nla_get_u32(tbp[i]);
@@ -2638,117 +2655,158 @@ EXPORT_SYMBOL(neigh_app_ns);
2638 2655
2639#ifdef CONFIG_SYSCTL 2656#ifdef CONFIG_SYSCTL
2640 2657
2641#define NEIGH_VARS_MAX 19 2658static int proc_unres_qlen(ctl_table *ctl, int write, void __user *buffer,
2659 size_t *lenp, loff_t *ppos)
2660{
2661 int size, ret;
2662 ctl_table tmp = *ctl;
2663
2664 tmp.data = &size;
2665 size = DIV_ROUND_UP(*(int *)ctl->data, SKB_TRUESIZE(ETH_FRAME_LEN));
2666 ret = proc_dointvec(&tmp, write, buffer, lenp, ppos);
2667 if (write && !ret)
2668 *(int *)ctl->data = size * SKB_TRUESIZE(ETH_FRAME_LEN);
2669 return ret;
2670}
2671
2672enum {
2673 NEIGH_VAR_MCAST_PROBE,
2674 NEIGH_VAR_UCAST_PROBE,
2675 NEIGH_VAR_APP_PROBE,
2676 NEIGH_VAR_RETRANS_TIME,
2677 NEIGH_VAR_BASE_REACHABLE_TIME,
2678 NEIGH_VAR_DELAY_PROBE_TIME,
2679 NEIGH_VAR_GC_STALETIME,
2680 NEIGH_VAR_QUEUE_LEN,
2681 NEIGH_VAR_QUEUE_LEN_BYTES,
2682 NEIGH_VAR_PROXY_QLEN,
2683 NEIGH_VAR_ANYCAST_DELAY,
2684 NEIGH_VAR_PROXY_DELAY,
2685 NEIGH_VAR_LOCKTIME,
2686 NEIGH_VAR_RETRANS_TIME_MS,
2687 NEIGH_VAR_BASE_REACHABLE_TIME_MS,
2688 NEIGH_VAR_GC_INTERVAL,
2689 NEIGH_VAR_GC_THRESH1,
2690 NEIGH_VAR_GC_THRESH2,
2691 NEIGH_VAR_GC_THRESH3,
2692 NEIGH_VAR_MAX
2693};
2642 2694
2643static struct neigh_sysctl_table { 2695static struct neigh_sysctl_table {
2644 struct ctl_table_header *sysctl_header; 2696 struct ctl_table_header *sysctl_header;
2645 struct ctl_table neigh_vars[NEIGH_VARS_MAX]; 2697 struct ctl_table neigh_vars[NEIGH_VAR_MAX + 1];
2646 char *dev_name; 2698 char *dev_name;
2647} neigh_sysctl_template __read_mostly = { 2699} neigh_sysctl_template __read_mostly = {
2648 .neigh_vars = { 2700 .neigh_vars = {
2649 { 2701 [NEIGH_VAR_MCAST_PROBE] = {
2650 .procname = "mcast_solicit", 2702 .procname = "mcast_solicit",
2651 .maxlen = sizeof(int), 2703 .maxlen = sizeof(int),
2652 .mode = 0644, 2704 .mode = 0644,
2653 .proc_handler = proc_dointvec, 2705 .proc_handler = proc_dointvec,
2654 }, 2706 },
2655 { 2707 [NEIGH_VAR_UCAST_PROBE] = {
2656 .procname = "ucast_solicit", 2708 .procname = "ucast_solicit",
2657 .maxlen = sizeof(int), 2709 .maxlen = sizeof(int),
2658 .mode = 0644, 2710 .mode = 0644,
2659 .proc_handler = proc_dointvec, 2711 .proc_handler = proc_dointvec,
2660 }, 2712 },
2661 { 2713 [NEIGH_VAR_APP_PROBE] = {
2662 .procname = "app_solicit", 2714 .procname = "app_solicit",
2663 .maxlen = sizeof(int), 2715 .maxlen = sizeof(int),
2664 .mode = 0644, 2716 .mode = 0644,
2665 .proc_handler = proc_dointvec, 2717 .proc_handler = proc_dointvec,
2666 }, 2718 },
2667 { 2719 [NEIGH_VAR_RETRANS_TIME] = {
2668 .procname = "retrans_time", 2720 .procname = "retrans_time",
2669 .maxlen = sizeof(int), 2721 .maxlen = sizeof(int),
2670 .mode = 0644, 2722 .mode = 0644,
2671 .proc_handler = proc_dointvec_userhz_jiffies, 2723 .proc_handler = proc_dointvec_userhz_jiffies,
2672 }, 2724 },
2673 { 2725 [NEIGH_VAR_BASE_REACHABLE_TIME] = {
2674 .procname = "base_reachable_time", 2726 .procname = "base_reachable_time",
2675 .maxlen = sizeof(int), 2727 .maxlen = sizeof(int),
2676 .mode = 0644, 2728 .mode = 0644,
2677 .proc_handler = proc_dointvec_jiffies, 2729 .proc_handler = proc_dointvec_jiffies,
2678 }, 2730 },
2679 { 2731 [NEIGH_VAR_DELAY_PROBE_TIME] = {
2680 .procname = "delay_first_probe_time", 2732 .procname = "delay_first_probe_time",
2681 .maxlen = sizeof(int), 2733 .maxlen = sizeof(int),
2682 .mode = 0644, 2734 .mode = 0644,
2683 .proc_handler = proc_dointvec_jiffies, 2735 .proc_handler = proc_dointvec_jiffies,
2684 }, 2736 },
2685 { 2737 [NEIGH_VAR_GC_STALETIME] = {
2686 .procname = "gc_stale_time", 2738 .procname = "gc_stale_time",
2687 .maxlen = sizeof(int), 2739 .maxlen = sizeof(int),
2688 .mode = 0644, 2740 .mode = 0644,
2689 .proc_handler = proc_dointvec_jiffies, 2741 .proc_handler = proc_dointvec_jiffies,
2690 }, 2742 },
2691 { 2743 [NEIGH_VAR_QUEUE_LEN] = {
2692 .procname = "unres_qlen", 2744 .procname = "unres_qlen",
2693 .maxlen = sizeof(int), 2745 .maxlen = sizeof(int),
2694 .mode = 0644, 2746 .mode = 0644,
2747 .proc_handler = proc_unres_qlen,
2748 },
2749 [NEIGH_VAR_QUEUE_LEN_BYTES] = {
2750 .procname = "unres_qlen_bytes",
2751 .maxlen = sizeof(int),
2752 .mode = 0644,
2695 .proc_handler = proc_dointvec, 2753 .proc_handler = proc_dointvec,
2696 }, 2754 },
2697 { 2755 [NEIGH_VAR_PROXY_QLEN] = {
2698 .procname = "proxy_qlen", 2756 .procname = "proxy_qlen",
2699 .maxlen = sizeof(int), 2757 .maxlen = sizeof(int),
2700 .mode = 0644, 2758 .mode = 0644,
2701 .proc_handler = proc_dointvec, 2759 .proc_handler = proc_dointvec,
2702 }, 2760 },
2703 { 2761 [NEIGH_VAR_ANYCAST_DELAY] = {
2704 .procname = "anycast_delay", 2762 .procname = "anycast_delay",
2705 .maxlen = sizeof(int), 2763 .maxlen = sizeof(int),
2706 .mode = 0644, 2764 .mode = 0644,
2707 .proc_handler = proc_dointvec_userhz_jiffies, 2765 .proc_handler = proc_dointvec_userhz_jiffies,
2708 }, 2766 },
2709 { 2767 [NEIGH_VAR_PROXY_DELAY] = {
2710 .procname = "proxy_delay", 2768 .procname = "proxy_delay",
2711 .maxlen = sizeof(int), 2769 .maxlen = sizeof(int),
2712 .mode = 0644, 2770 .mode = 0644,
2713 .proc_handler = proc_dointvec_userhz_jiffies, 2771 .proc_handler = proc_dointvec_userhz_jiffies,
2714 }, 2772 },
2715 { 2773 [NEIGH_VAR_LOCKTIME] = {
2716 .procname = "locktime", 2774 .procname = "locktime",
2717 .maxlen = sizeof(int), 2775 .maxlen = sizeof(int),
2718 .mode = 0644, 2776 .mode = 0644,
2719 .proc_handler = proc_dointvec_userhz_jiffies, 2777 .proc_handler = proc_dointvec_userhz_jiffies,
2720 }, 2778 },
2721 { 2779 [NEIGH_VAR_RETRANS_TIME_MS] = {
2722 .procname = "retrans_time_ms", 2780 .procname = "retrans_time_ms",
2723 .maxlen = sizeof(int), 2781 .maxlen = sizeof(int),
2724 .mode = 0644, 2782 .mode = 0644,
2725 .proc_handler = proc_dointvec_ms_jiffies, 2783 .proc_handler = proc_dointvec_ms_jiffies,
2726 }, 2784 },
2727 { 2785 [NEIGH_VAR_BASE_REACHABLE_TIME_MS] = {
2728 .procname = "base_reachable_time_ms", 2786 .procname = "base_reachable_time_ms",
2729 .maxlen = sizeof(int), 2787 .maxlen = sizeof(int),
2730 .mode = 0644, 2788 .mode = 0644,
2731 .proc_handler = proc_dointvec_ms_jiffies, 2789 .proc_handler = proc_dointvec_ms_jiffies,
2732 }, 2790 },
2733 { 2791 [NEIGH_VAR_GC_INTERVAL] = {
2734 .procname = "gc_interval", 2792 .procname = "gc_interval",
2735 .maxlen = sizeof(int), 2793 .maxlen = sizeof(int),
2736 .mode = 0644, 2794 .mode = 0644,
2737 .proc_handler = proc_dointvec_jiffies, 2795 .proc_handler = proc_dointvec_jiffies,
2738 }, 2796 },
2739 { 2797 [NEIGH_VAR_GC_THRESH1] = {
2740 .procname = "gc_thresh1", 2798 .procname = "gc_thresh1",
2741 .maxlen = sizeof(int), 2799 .maxlen = sizeof(int),
2742 .mode = 0644, 2800 .mode = 0644,
2743 .proc_handler = proc_dointvec, 2801 .proc_handler = proc_dointvec,
2744 }, 2802 },
2745 { 2803 [NEIGH_VAR_GC_THRESH2] = {
2746 .procname = "gc_thresh2", 2804 .procname = "gc_thresh2",
2747 .maxlen = sizeof(int), 2805 .maxlen = sizeof(int),
2748 .mode = 0644, 2806 .mode = 0644,
2749 .proc_handler = proc_dointvec, 2807 .proc_handler = proc_dointvec,
2750 }, 2808 },
2751 { 2809 [NEIGH_VAR_GC_THRESH3] = {
2752 .procname = "gc_thresh3", 2810 .procname = "gc_thresh3",
2753 .maxlen = sizeof(int), 2811 .maxlen = sizeof(int),
2754 .mode = 0644, 2812 .mode = 0644,
@@ -2781,47 +2839,49 @@ int neigh_sysctl_register(struct net_device *dev, struct neigh_parms *p,
2781 if (!t) 2839 if (!t)
2782 goto err; 2840 goto err;
2783 2841
2784 t->neigh_vars[0].data = &p->mcast_probes; 2842 t->neigh_vars[NEIGH_VAR_MCAST_PROBE].data = &p->mcast_probes;
2785 t->neigh_vars[1].data = &p->ucast_probes; 2843 t->neigh_vars[NEIGH_VAR_UCAST_PROBE].data = &p->ucast_probes;
2786 t->neigh_vars[2].data = &p->app_probes; 2844 t->neigh_vars[NEIGH_VAR_APP_PROBE].data = &p->app_probes;
2787 t->neigh_vars[3].data = &p->retrans_time; 2845 t->neigh_vars[NEIGH_VAR_RETRANS_TIME].data = &p->retrans_time;
2788 t->neigh_vars[4].data = &p->base_reachable_time; 2846 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].data = &p->base_reachable_time;
2789 t->neigh_vars[5].data = &p->delay_probe_time; 2847 t->neigh_vars[NEIGH_VAR_DELAY_PROBE_TIME].data = &p->delay_probe_time;
2790 t->neigh_vars[6].data = &p->gc_staletime; 2848 t->neigh_vars[NEIGH_VAR_GC_STALETIME].data = &p->gc_staletime;
2791 t->neigh_vars[7].data = &p->queue_len; 2849 t->neigh_vars[NEIGH_VAR_QUEUE_LEN].data = &p->queue_len_bytes;
2792 t->neigh_vars[8].data = &p->proxy_qlen; 2850 t->neigh_vars[NEIGH_VAR_QUEUE_LEN_BYTES].data = &p->queue_len_bytes;
2793 t->neigh_vars[9].data = &p->anycast_delay; 2851 t->neigh_vars[NEIGH_VAR_PROXY_QLEN].data = &p->proxy_qlen;
2794 t->neigh_vars[10].data = &p->proxy_delay; 2852 t->neigh_vars[NEIGH_VAR_ANYCAST_DELAY].data = &p->anycast_delay;
2795 t->neigh_vars[11].data = &p->locktime; 2853 t->neigh_vars[NEIGH_VAR_PROXY_DELAY].data = &p->proxy_delay;
2796 t->neigh_vars[12].data = &p->retrans_time; 2854 t->neigh_vars[NEIGH_VAR_LOCKTIME].data = &p->locktime;
2797 t->neigh_vars[13].data = &p->base_reachable_time; 2855 t->neigh_vars[NEIGH_VAR_RETRANS_TIME_MS].data = &p->retrans_time;
2856 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].data = &p->base_reachable_time;
2798 2857
2799 if (dev) { 2858 if (dev) {
2800 dev_name_source = dev->name; 2859 dev_name_source = dev->name;
2801 /* Terminate the table early */ 2860 /* Terminate the table early */
2802 memset(&t->neigh_vars[14], 0, sizeof(t->neigh_vars[14])); 2861 memset(&t->neigh_vars[NEIGH_VAR_GC_INTERVAL], 0,
2862 sizeof(t->neigh_vars[NEIGH_VAR_GC_INTERVAL]));
2803 } else { 2863 } else {
2804 dev_name_source = neigh_path[NEIGH_CTL_PATH_DEV].procname; 2864 dev_name_source = neigh_path[NEIGH_CTL_PATH_DEV].procname;
2805 t->neigh_vars[14].data = (int *)(p + 1); 2865 t->neigh_vars[NEIGH_VAR_GC_INTERVAL].data = (int *)(p + 1);
2806 t->neigh_vars[15].data = (int *)(p + 1) + 1; 2866 t->neigh_vars[NEIGH_VAR_GC_THRESH1].data = (int *)(p + 1) + 1;
2807 t->neigh_vars[16].data = (int *)(p + 1) + 2; 2867 t->neigh_vars[NEIGH_VAR_GC_THRESH2].data = (int *)(p + 1) + 2;
2808 t->neigh_vars[17].data = (int *)(p + 1) + 3; 2868 t->neigh_vars[NEIGH_VAR_GC_THRESH3].data = (int *)(p + 1) + 3;
2809 } 2869 }
2810 2870
2811 2871
2812 if (handler) { 2872 if (handler) {
2813 /* RetransTime */ 2873 /* RetransTime */
2814 t->neigh_vars[3].proc_handler = handler; 2874 t->neigh_vars[NEIGH_VAR_RETRANS_TIME].proc_handler = handler;
2815 t->neigh_vars[3].extra1 = dev; 2875 t->neigh_vars[NEIGH_VAR_RETRANS_TIME].extra1 = dev;
2816 /* ReachableTime */ 2876 /* ReachableTime */
2817 t->neigh_vars[4].proc_handler = handler; 2877 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].proc_handler = handler;
2818 t->neigh_vars[4].extra1 = dev; 2878 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].extra1 = dev;
2819 /* RetransTime (in milliseconds)*/ 2879 /* RetransTime (in milliseconds)*/
2820 t->neigh_vars[12].proc_handler = handler; 2880 t->neigh_vars[NEIGH_VAR_RETRANS_TIME_MS].proc_handler = handler;
2821 t->neigh_vars[12].extra1 = dev; 2881 t->neigh_vars[NEIGH_VAR_RETRANS_TIME_MS].extra1 = dev;
2822 /* ReachableTime (in milliseconds) */ 2882 /* ReachableTime (in milliseconds) */
2823 t->neigh_vars[13].proc_handler = handler; 2883 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].proc_handler = handler;
2824 t->neigh_vars[13].extra1 = dev; 2884 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].extra1 = dev;
2825 } 2885 }
2826 2886
2827 t->dev_name = kstrdup(dev_name_source, GFP_KERNEL); 2887 t->dev_name = kstrdup(dev_name_source, GFP_KERNEL);
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index c71c434a4c05..db6c2f83633f 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -606,9 +606,12 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue,
606 rcu_assign_pointer(queue->rps_map, map); 606 rcu_assign_pointer(queue->rps_map, map);
607 spin_unlock(&rps_map_lock); 607 spin_unlock(&rps_map_lock);
608 608
609 if (old_map) 609 if (map)
610 jump_label_inc(&rps_needed);
611 if (old_map) {
610 kfree_rcu(old_map, rcu); 612 kfree_rcu(old_map, rcu);
611 613 jump_label_dec(&rps_needed);
614 }
612 free_cpumask_var(mask); 615 free_cpumask_var(mask);
613 return len; 616 return len;
614} 617}
@@ -780,7 +783,7 @@ net_rx_queue_update_kobjects(struct net_device *net, int old_num, int new_num)
780#endif 783#endif
781} 784}
782 785
783#ifdef CONFIG_XPS 786#ifdef CONFIG_SYSFS
784/* 787/*
785 * netdev_queue sysfs structures and functions. 788 * netdev_queue sysfs structures and functions.
786 */ 789 */
@@ -826,6 +829,23 @@ static const struct sysfs_ops netdev_queue_sysfs_ops = {
826 .store = netdev_queue_attr_store, 829 .store = netdev_queue_attr_store,
827}; 830};
828 831
832static ssize_t show_trans_timeout(struct netdev_queue *queue,
833 struct netdev_queue_attribute *attribute,
834 char *buf)
835{
836 unsigned long trans_timeout;
837
838 spin_lock_irq(&queue->_xmit_lock);
839 trans_timeout = queue->trans_timeout;
840 spin_unlock_irq(&queue->_xmit_lock);
841
842 return sprintf(buf, "%lu", trans_timeout);
843}
844
845static struct netdev_queue_attribute queue_trans_timeout =
846 __ATTR(tx_timeout, S_IRUGO, show_trans_timeout, NULL);
847
848#ifdef CONFIG_XPS
829static inline unsigned int get_netdev_queue_index(struct netdev_queue *queue) 849static inline unsigned int get_netdev_queue_index(struct netdev_queue *queue)
830{ 850{
831 struct net_device *dev = queue->dev; 851 struct net_device *dev = queue->dev;
@@ -901,7 +921,7 @@ static ssize_t store_xps_map(struct netdev_queue *queue,
901 struct xps_map *map, *new_map; 921 struct xps_map *map, *new_map;
902 struct xps_dev_maps *dev_maps, *new_dev_maps; 922 struct xps_dev_maps *dev_maps, *new_dev_maps;
903 int nonempty = 0; 923 int nonempty = 0;
904 int numa_node = -2; 924 int numa_node_id = -2;
905 925
906 if (!capable(CAP_NET_ADMIN)) 926 if (!capable(CAP_NET_ADMIN))
907 return -EPERM; 927 return -EPERM;
@@ -944,10 +964,10 @@ static ssize_t store_xps_map(struct netdev_queue *queue,
944 need_set = cpumask_test_cpu(cpu, mask) && cpu_online(cpu); 964 need_set = cpumask_test_cpu(cpu, mask) && cpu_online(cpu);
945#ifdef CONFIG_NUMA 965#ifdef CONFIG_NUMA
946 if (need_set) { 966 if (need_set) {
947 if (numa_node == -2) 967 if (numa_node_id == -2)
948 numa_node = cpu_to_node(cpu); 968 numa_node_id = cpu_to_node(cpu);
949 else if (numa_node != cpu_to_node(cpu)) 969 else if (numa_node_id != cpu_to_node(cpu))
950 numa_node = -1; 970 numa_node_id = -1;
951 } 971 }
952#endif 972#endif
953 if (need_set && pos >= map_len) { 973 if (need_set && pos >= map_len) {
@@ -997,7 +1017,7 @@ static ssize_t store_xps_map(struct netdev_queue *queue,
997 if (dev_maps) 1017 if (dev_maps)
998 kfree_rcu(dev_maps, rcu); 1018 kfree_rcu(dev_maps, rcu);
999 1019
1000 netdev_queue_numa_node_write(queue, (numa_node >= 0) ? numa_node : 1020 netdev_queue_numa_node_write(queue, (numa_node_id >= 0) ? numa_node_id :
1001 NUMA_NO_NODE); 1021 NUMA_NO_NODE);
1002 1022
1003 mutex_unlock(&xps_map_mutex); 1023 mutex_unlock(&xps_map_mutex);
@@ -1020,12 +1040,17 @@ error:
1020 1040
1021static struct netdev_queue_attribute xps_cpus_attribute = 1041static struct netdev_queue_attribute xps_cpus_attribute =
1022 __ATTR(xps_cpus, S_IRUGO | S_IWUSR, show_xps_map, store_xps_map); 1042 __ATTR(xps_cpus, S_IRUGO | S_IWUSR, show_xps_map, store_xps_map);
1043#endif /* CONFIG_XPS */
1023 1044
1024static struct attribute *netdev_queue_default_attrs[] = { 1045static struct attribute *netdev_queue_default_attrs[] = {
1046 &queue_trans_timeout.attr,
1047#ifdef CONFIG_XPS
1025 &xps_cpus_attribute.attr, 1048 &xps_cpus_attribute.attr,
1049#endif
1026 NULL 1050 NULL
1027}; 1051};
1028 1052
1053#ifdef CONFIG_XPS
1029static void netdev_queue_release(struct kobject *kobj) 1054static void netdev_queue_release(struct kobject *kobj)
1030{ 1055{
1031 struct netdev_queue *queue = to_netdev_queue(kobj); 1056 struct netdev_queue *queue = to_netdev_queue(kobj);
@@ -1076,10 +1101,13 @@ static void netdev_queue_release(struct kobject *kobj)
1076 memset(kobj, 0, sizeof(*kobj)); 1101 memset(kobj, 0, sizeof(*kobj));
1077 dev_put(queue->dev); 1102 dev_put(queue->dev);
1078} 1103}
1104#endif /* CONFIG_XPS */
1079 1105
1080static struct kobj_type netdev_queue_ktype = { 1106static struct kobj_type netdev_queue_ktype = {
1081 .sysfs_ops = &netdev_queue_sysfs_ops, 1107 .sysfs_ops = &netdev_queue_sysfs_ops,
1108#ifdef CONFIG_XPS
1082 .release = netdev_queue_release, 1109 .release = netdev_queue_release,
1110#endif
1083 .default_attrs = netdev_queue_default_attrs, 1111 .default_attrs = netdev_queue_default_attrs,
1084}; 1112};
1085 1113
@@ -1102,12 +1130,12 @@ static int netdev_queue_add_kobject(struct net_device *net, int index)
1102 1130
1103 return error; 1131 return error;
1104} 1132}
1105#endif /* CONFIG_XPS */ 1133#endif /* CONFIG_SYSFS */
1106 1134
1107int 1135int
1108netdev_queue_update_kobjects(struct net_device *net, int old_num, int new_num) 1136netdev_queue_update_kobjects(struct net_device *net, int old_num, int new_num)
1109{ 1137{
1110#ifdef CONFIG_XPS 1138#ifdef CONFIG_SYSFS
1111 int i; 1139 int i;
1112 int error = 0; 1140 int error = 0;
1113 1141
@@ -1125,14 +1153,14 @@ netdev_queue_update_kobjects(struct net_device *net, int old_num, int new_num)
1125 return error; 1153 return error;
1126#else 1154#else
1127 return 0; 1155 return 0;
1128#endif 1156#endif /* CONFIG_SYSFS */
1129} 1157}
1130 1158
1131static int register_queue_kobjects(struct net_device *net) 1159static int register_queue_kobjects(struct net_device *net)
1132{ 1160{
1133 int error = 0, txq = 0, rxq = 0, real_rx = 0, real_tx = 0; 1161 int error = 0, txq = 0, rxq = 0, real_rx = 0, real_tx = 0;
1134 1162
1135#if defined(CONFIG_RPS) || defined(CONFIG_XPS) 1163#ifdef CONFIG_SYSFS
1136 net->queues_kset = kset_create_and_add("queues", 1164 net->queues_kset = kset_create_and_add("queues",
1137 NULL, &net->dev.kobj); 1165 NULL, &net->dev.kobj);
1138 if (!net->queues_kset) 1166 if (!net->queues_kset)
@@ -1173,7 +1201,7 @@ static void remove_queue_kobjects(struct net_device *net)
1173 1201
1174 net_rx_queue_update_kobjects(net, real_rx, 0); 1202 net_rx_queue_update_kobjects(net, real_rx, 0);
1175 netdev_queue_update_kobjects(net, real_tx, 0); 1203 netdev_queue_update_kobjects(net, real_tx, 0);
1176#if defined(CONFIG_RPS) || defined(CONFIG_XPS) 1204#ifdef CONFIG_SYSFS
1177 kset_unregister(net->queues_kset); 1205 kset_unregister(net->queues_kset);
1178#endif 1206#endif
1179} 1207}
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index cf64c1ffa4cd..1a7d8e2c9768 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -422,6 +422,7 @@ static void arp_reply(struct sk_buff *skb)
422 struct sk_buff *send_skb; 422 struct sk_buff *send_skb;
423 struct netpoll *np, *tmp; 423 struct netpoll *np, *tmp;
424 unsigned long flags; 424 unsigned long flags;
425 int hlen, tlen;
425 int hits = 0; 426 int hits = 0;
426 427
427 if (list_empty(&npinfo->rx_np)) 428 if (list_empty(&npinfo->rx_np))
@@ -479,8 +480,9 @@ static void arp_reply(struct sk_buff *skb)
479 if (tip != np->local_ip) 480 if (tip != np->local_ip)
480 continue; 481 continue;
481 482
482 send_skb = find_skb(np, size + LL_ALLOCATED_SPACE(np->dev), 483 hlen = LL_RESERVED_SPACE(np->dev);
483 LL_RESERVED_SPACE(np->dev)); 484 tlen = np->dev->needed_tailroom;
485 send_skb = find_skb(np, size + hlen + tlen, hlen);
484 if (!send_skb) 486 if (!send_skb)
485 continue; 487 continue;
486 488
diff --git a/net/core/netprio_cgroup.c b/net/core/netprio_cgroup.c
new file mode 100644
index 000000000000..3a9fd4826b75
--- /dev/null
+++ b/net/core/netprio_cgroup.c
@@ -0,0 +1,344 @@
1/*
2 * net/core/netprio_cgroup.c Priority Control Group
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Authors: Neil Horman <nhorman@tuxdriver.com>
10 */
11
12#include <linux/module.h>
13#include <linux/slab.h>
14#include <linux/types.h>
15#include <linux/string.h>
16#include <linux/errno.h>
17#include <linux/skbuff.h>
18#include <linux/cgroup.h>
19#include <linux/rcupdate.h>
20#include <linux/atomic.h>
21#include <net/rtnetlink.h>
22#include <net/pkt_cls.h>
23#include <net/sock.h>
24#include <net/netprio_cgroup.h>
25
26static struct cgroup_subsys_state *cgrp_create(struct cgroup_subsys *ss,
27 struct cgroup *cgrp);
28static void cgrp_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp);
29static int cgrp_populate(struct cgroup_subsys *ss, struct cgroup *cgrp);
30
31struct cgroup_subsys net_prio_subsys = {
32 .name = "net_prio",
33 .create = cgrp_create,
34 .destroy = cgrp_destroy,
35 .populate = cgrp_populate,
36#ifdef CONFIG_NETPRIO_CGROUP
37 .subsys_id = net_prio_subsys_id,
38#endif
39 .module = THIS_MODULE
40};
41
42#define PRIOIDX_SZ 128
43
44static unsigned long prioidx_map[PRIOIDX_SZ];
45static DEFINE_SPINLOCK(prioidx_map_lock);
46static atomic_t max_prioidx = ATOMIC_INIT(0);
47
48static inline struct cgroup_netprio_state *cgrp_netprio_state(struct cgroup *cgrp)
49{
50 return container_of(cgroup_subsys_state(cgrp, net_prio_subsys_id),
51 struct cgroup_netprio_state, css);
52}
53
54static int get_prioidx(u32 *prio)
55{
56 unsigned long flags;
57 u32 prioidx;
58
59 spin_lock_irqsave(&prioidx_map_lock, flags);
60 prioidx = find_first_zero_bit(prioidx_map, sizeof(unsigned long) * PRIOIDX_SZ);
61 set_bit(prioidx, prioidx_map);
62 spin_unlock_irqrestore(&prioidx_map_lock, flags);
63 if (prioidx == sizeof(unsigned long) * PRIOIDX_SZ)
64 return -ENOSPC;
65
66 atomic_set(&max_prioidx, prioidx);
67 *prio = prioidx;
68 return 0;
69}
70
71static void put_prioidx(u32 idx)
72{
73 unsigned long flags;
74
75 spin_lock_irqsave(&prioidx_map_lock, flags);
76 clear_bit(idx, prioidx_map);
77 spin_unlock_irqrestore(&prioidx_map_lock, flags);
78}
79
80static void extend_netdev_table(struct net_device *dev, u32 new_len)
81{
82 size_t new_size = sizeof(struct netprio_map) +
83 ((sizeof(u32) * new_len));
84 struct netprio_map *new_priomap = kzalloc(new_size, GFP_KERNEL);
85 struct netprio_map *old_priomap;
86 int i;
87
88 old_priomap = rtnl_dereference(dev->priomap);
89
90 if (!new_priomap) {
91 printk(KERN_WARNING "Unable to alloc new priomap!\n");
92 return;
93 }
94
95 for (i = 0;
96 old_priomap && (i < old_priomap->priomap_len);
97 i++)
98 new_priomap->priomap[i] = old_priomap->priomap[i];
99
100 new_priomap->priomap_len = new_len;
101
102 rcu_assign_pointer(dev->priomap, new_priomap);
103 if (old_priomap)
104 kfree_rcu(old_priomap, rcu);
105}
106
107static void update_netdev_tables(void)
108{
109 struct net_device *dev;
110 u32 max_len = atomic_read(&max_prioidx);
111 struct netprio_map *map;
112
113 rtnl_lock();
114 for_each_netdev(&init_net, dev) {
115 map = rtnl_dereference(dev->priomap);
116 if ((!map) ||
117 (map->priomap_len < max_len))
118 extend_netdev_table(dev, max_len);
119 }
120 rtnl_unlock();
121}
122
123static struct cgroup_subsys_state *cgrp_create(struct cgroup_subsys *ss,
124 struct cgroup *cgrp)
125{
126 struct cgroup_netprio_state *cs;
127 int ret;
128
129 cs = kzalloc(sizeof(*cs), GFP_KERNEL);
130 if (!cs)
131 return ERR_PTR(-ENOMEM);
132
133 if (cgrp->parent && cgrp_netprio_state(cgrp->parent)->prioidx) {
134 kfree(cs);
135 return ERR_PTR(-EINVAL);
136 }
137
138 ret = get_prioidx(&cs->prioidx);
139 if (ret != 0) {
140 printk(KERN_WARNING "No space in priority index array\n");
141 kfree(cs);
142 return ERR_PTR(ret);
143 }
144
145 return &cs->css;
146}
147
148static void cgrp_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
149{
150 struct cgroup_netprio_state *cs;
151 struct net_device *dev;
152 struct netprio_map *map;
153
154 cs = cgrp_netprio_state(cgrp);
155 rtnl_lock();
156 for_each_netdev(&init_net, dev) {
157 map = rtnl_dereference(dev->priomap);
158 if (map)
159 map->priomap[cs->prioidx] = 0;
160 }
161 rtnl_unlock();
162 put_prioidx(cs->prioidx);
163 kfree(cs);
164}
165
166static u64 read_prioidx(struct cgroup *cgrp, struct cftype *cft)
167{
168 return (u64)cgrp_netprio_state(cgrp)->prioidx;
169}
170
171static int read_priomap(struct cgroup *cont, struct cftype *cft,
172 struct cgroup_map_cb *cb)
173{
174 struct net_device *dev;
175 u32 prioidx = cgrp_netprio_state(cont)->prioidx;
176 u32 priority;
177 struct netprio_map *map;
178
179 rcu_read_lock();
180 for_each_netdev_rcu(&init_net, dev) {
181 map = rcu_dereference(dev->priomap);
182 priority = map ? map->priomap[prioidx] : 0;
183 cb->fill(cb, dev->name, priority);
184 }
185 rcu_read_unlock();
186 return 0;
187}
188
189static int write_priomap(struct cgroup *cgrp, struct cftype *cft,
190 const char *buffer)
191{
192 char *devname = kstrdup(buffer, GFP_KERNEL);
193 int ret = -EINVAL;
194 u32 prioidx = cgrp_netprio_state(cgrp)->prioidx;
195 unsigned long priority;
196 char *priostr;
197 struct net_device *dev;
198 struct netprio_map *map;
199
200 if (!devname)
201 return -ENOMEM;
202
203 /*
204 * Minimally sized valid priomap string
205 */
206 if (strlen(devname) < 3)
207 goto out_free_devname;
208
209 priostr = strstr(devname, " ");
210 if (!priostr)
211 goto out_free_devname;
212
213 /*
214 *Separate the devname from the associated priority
215 *and advance the priostr poitner to the priority value
216 */
217 *priostr = '\0';
218 priostr++;
219
220 /*
221 * If the priostr points to NULL, we're at the end of the passed
222 * in string, and its not a valid write
223 */
224 if (*priostr == '\0')
225 goto out_free_devname;
226
227 ret = kstrtoul(priostr, 10, &priority);
228 if (ret < 0)
229 goto out_free_devname;
230
231 ret = -ENODEV;
232
233 dev = dev_get_by_name(&init_net, devname);
234 if (!dev)
235 goto out_free_devname;
236
237 update_netdev_tables();
238 ret = 0;
239 rcu_read_lock();
240 map = rcu_dereference(dev->priomap);
241 if (map)
242 map->priomap[prioidx] = priority;
243 rcu_read_unlock();
244 dev_put(dev);
245
246out_free_devname:
247 kfree(devname);
248 return ret;
249}
250
251static struct cftype ss_files[] = {
252 {
253 .name = "prioidx",
254 .read_u64 = read_prioidx,
255 },
256 {
257 .name = "ifpriomap",
258 .read_map = read_priomap,
259 .write_string = write_priomap,
260 },
261};
262
263static int cgrp_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
264{
265 return cgroup_add_files(cgrp, ss, ss_files, ARRAY_SIZE(ss_files));
266}
267
268static int netprio_device_event(struct notifier_block *unused,
269 unsigned long event, void *ptr)
270{
271 struct net_device *dev = ptr;
272 struct netprio_map *old;
273 u32 max_len = atomic_read(&max_prioidx);
274
275 /*
276 * Note this is called with rtnl_lock held so we have update side
277 * protection on our rcu assignments
278 */
279
280 switch (event) {
281
282 case NETDEV_REGISTER:
283 if (max_len)
284 extend_netdev_table(dev, max_len);
285 break;
286 case NETDEV_UNREGISTER:
287 old = rtnl_dereference(dev->priomap);
288 RCU_INIT_POINTER(dev->priomap, NULL);
289 if (old)
290 kfree_rcu(old, rcu);
291 break;
292 }
293 return NOTIFY_DONE;
294}
295
296static struct notifier_block netprio_device_notifier = {
297 .notifier_call = netprio_device_event
298};
299
300static int __init init_cgroup_netprio(void)
301{
302 int ret;
303
304 ret = cgroup_load_subsys(&net_prio_subsys);
305 if (ret)
306 goto out;
307#ifndef CONFIG_NETPRIO_CGROUP
308 smp_wmb();
309 net_prio_subsys_id = net_prio_subsys.subsys_id;
310#endif
311
312 register_netdevice_notifier(&netprio_device_notifier);
313
314out:
315 return ret;
316}
317
318static void __exit exit_cgroup_netprio(void)
319{
320 struct netprio_map *old;
321 struct net_device *dev;
322
323 unregister_netdevice_notifier(&netprio_device_notifier);
324
325 cgroup_unload_subsys(&net_prio_subsys);
326
327#ifndef CONFIG_NETPRIO_CGROUP
328 net_prio_subsys_id = -1;
329 synchronize_rcu();
330#endif
331
332 rtnl_lock();
333 for_each_netdev(&init_net, dev) {
334 old = rtnl_dereference(dev->priomap);
335 RCU_INIT_POINTER(dev->priomap, NULL);
336 if (old)
337 kfree_rcu(old, rcu);
338 }
339 rtnl_unlock();
340}
341
342module_init(init_cgroup_netprio);
343module_exit(exit_cgroup_netprio);
344MODULE_LICENSE("GPL v2");
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index 0001c243b35c..aa53a35a631b 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -1304,7 +1304,7 @@ static ssize_t pktgen_if_write(struct file *file,
1304 scan_ip6(buf, pkt_dev->in6_daddr.s6_addr); 1304 scan_ip6(buf, pkt_dev->in6_daddr.s6_addr);
1305 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr); 1305 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr);
1306 1306
1307 ipv6_addr_copy(&pkt_dev->cur_in6_daddr, &pkt_dev->in6_daddr); 1307 pkt_dev->cur_in6_daddr = pkt_dev->in6_daddr;
1308 1308
1309 if (debug) 1309 if (debug)
1310 printk(KERN_DEBUG "pktgen: dst6 set to: %s\n", buf); 1310 printk(KERN_DEBUG "pktgen: dst6 set to: %s\n", buf);
@@ -1327,8 +1327,7 @@ static ssize_t pktgen_if_write(struct file *file,
1327 scan_ip6(buf, pkt_dev->min_in6_daddr.s6_addr); 1327 scan_ip6(buf, pkt_dev->min_in6_daddr.s6_addr);
1328 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr); 1328 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr);
1329 1329
1330 ipv6_addr_copy(&pkt_dev->cur_in6_daddr, 1330 pkt_dev->cur_in6_daddr = pkt_dev->min_in6_daddr;
1331 &pkt_dev->min_in6_daddr);
1332 if (debug) 1331 if (debug)
1333 printk(KERN_DEBUG "pktgen: dst6_min set to: %s\n", buf); 1332 printk(KERN_DEBUG "pktgen: dst6_min set to: %s\n", buf);
1334 1333
@@ -1371,7 +1370,7 @@ static ssize_t pktgen_if_write(struct file *file,
1371 scan_ip6(buf, pkt_dev->in6_saddr.s6_addr); 1370 scan_ip6(buf, pkt_dev->in6_saddr.s6_addr);
1372 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr); 1371 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr);
1373 1372
1374 ipv6_addr_copy(&pkt_dev->cur_in6_saddr, &pkt_dev->in6_saddr); 1373 pkt_dev->cur_in6_saddr = pkt_dev->in6_saddr;
1375 1374
1376 if (debug) 1375 if (debug)
1377 printk(KERN_DEBUG "pktgen: src6 set to: %s\n", buf); 1376 printk(KERN_DEBUG "pktgen: src6 set to: %s\n", buf);
@@ -2079,9 +2078,7 @@ static void pktgen_setup_inject(struct pktgen_dev *pkt_dev)
2079 ifp = ifp->if_next) { 2078 ifp = ifp->if_next) {
2080 if (ifp->scope == IFA_LINK && 2079 if (ifp->scope == IFA_LINK &&
2081 !(ifp->flags & IFA_F_TENTATIVE)) { 2080 !(ifp->flags & IFA_F_TENTATIVE)) {
2082 ipv6_addr_copy(&pkt_dev-> 2081 pkt_dev->cur_in6_saddr = ifp->addr;
2083 cur_in6_saddr,
2084 &ifp->addr);
2085 err = 0; 2082 err = 0;
2086 break; 2083 break;
2087 } 2084 }
@@ -2958,8 +2955,8 @@ static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
2958 iph->payload_len = htons(sizeof(struct udphdr) + datalen); 2955 iph->payload_len = htons(sizeof(struct udphdr) + datalen);
2959 iph->nexthdr = IPPROTO_UDP; 2956 iph->nexthdr = IPPROTO_UDP;
2960 2957
2961 ipv6_addr_copy(&iph->daddr, &pkt_dev->cur_in6_daddr); 2958 iph->daddr = pkt_dev->cur_in6_daddr;
2962 ipv6_addr_copy(&iph->saddr, &pkt_dev->cur_in6_saddr); 2959 iph->saddr = pkt_dev->cur_in6_saddr;
2963 2960
2964 skb->mac_header = (skb->network_header - ETH_HLEN - 2961 skb->mac_header = (skb->network_header - ETH_HLEN -
2965 pkt_dev->pkt_overhead); 2962 pkt_dev->pkt_overhead);
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 3c30ee4a5710..678ae4e783aa 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -245,6 +245,55 @@ nodata:
245EXPORT_SYMBOL(__alloc_skb); 245EXPORT_SYMBOL(__alloc_skb);
246 246
247/** 247/**
248 * build_skb - build a network buffer
249 * @data: data buffer provided by caller
250 *
251 * Allocate a new &sk_buff. Caller provides space holding head and
252 * skb_shared_info. @data must have been allocated by kmalloc()
253 * The return is the new skb buffer.
254 * On a failure the return is %NULL, and @data is not freed.
255 * Notes :
256 * Before IO, driver allocates only data buffer where NIC put incoming frame
257 * Driver should add room at head (NET_SKB_PAD) and
258 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
259 * After IO, driver calls build_skb(), to allocate sk_buff and populate it
260 * before giving packet to stack.
261 * RX rings only contains data buffers, not full skbs.
262 */
263struct sk_buff *build_skb(void *data)
264{
265 struct skb_shared_info *shinfo;
266 struct sk_buff *skb;
267 unsigned int size;
268
269 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
270 if (!skb)
271 return NULL;
272
273 size = ksize(data) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
274
275 memset(skb, 0, offsetof(struct sk_buff, tail));
276 skb->truesize = SKB_TRUESIZE(size);
277 atomic_set(&skb->users, 1);
278 skb->head = data;
279 skb->data = data;
280 skb_reset_tail_pointer(skb);
281 skb->end = skb->tail + size;
282#ifdef NET_SKBUFF_DATA_USES_OFFSET
283 skb->mac_header = ~0U;
284#endif
285
286 /* make sure we initialize shinfo sequentially */
287 shinfo = skb_shinfo(skb);
288 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
289 atomic_set(&shinfo->dataref, 1);
290 kmemcheck_annotate_variable(shinfo->destructor_arg);
291
292 return skb;
293}
294EXPORT_SYMBOL(build_skb);
295
296/**
248 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 297 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
249 * @dev: network device to receive on 298 * @dev: network device to receive on
250 * @length: length to allocate 299 * @length: length to allocate
@@ -2621,7 +2670,7 @@ EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2621 * a pointer to the first in a list of new skbs for the segments. 2670 * a pointer to the first in a list of new skbs for the segments.
2622 * In case of error it returns ERR_PTR(err). 2671 * In case of error it returns ERR_PTR(err).
2623 */ 2672 */
2624struct sk_buff *skb_segment(struct sk_buff *skb, u32 features) 2673struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
2625{ 2674{
2626 struct sk_buff *segs = NULL; 2675 struct sk_buff *segs = NULL;
2627 struct sk_buff *tail = NULL; 2676 struct sk_buff *tail = NULL;
@@ -3169,6 +3218,26 @@ void skb_tstamp_tx(struct sk_buff *orig_skb,
3169} 3218}
3170EXPORT_SYMBOL_GPL(skb_tstamp_tx); 3219EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3171 3220
3221void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
3222{
3223 struct sock *sk = skb->sk;
3224 struct sock_exterr_skb *serr;
3225 int err;
3226
3227 skb->wifi_acked_valid = 1;
3228 skb->wifi_acked = acked;
3229
3230 serr = SKB_EXT_ERR(skb);
3231 memset(serr, 0, sizeof(*serr));
3232 serr->ee.ee_errno = ENOMSG;
3233 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
3234
3235 err = sock_queue_err_skb(sk, skb);
3236 if (err)
3237 kfree_skb(skb);
3238}
3239EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
3240
3172 3241
3173/** 3242/**
3174 * skb_partial_csum_set - set up and verify partial csum values for packet 3243 * skb_partial_csum_set - set up and verify partial csum values for packet
diff --git a/net/core/sock.c b/net/core/sock.c
index 4ed7b1d12f5e..16069139797c 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -125,6 +125,7 @@
125#include <net/xfrm.h> 125#include <net/xfrm.h>
126#include <linux/ipsec.h> 126#include <linux/ipsec.h>
127#include <net/cls_cgroup.h> 127#include <net/cls_cgroup.h>
128#include <net/netprio_cgroup.h>
128 129
129#include <linux/filter.h> 130#include <linux/filter.h>
130 131
@@ -221,10 +222,16 @@ __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
221int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 222int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
222EXPORT_SYMBOL(sysctl_optmem_max); 223EXPORT_SYMBOL(sysctl_optmem_max);
223 224
224#if defined(CONFIG_CGROUPS) && !defined(CONFIG_NET_CLS_CGROUP) 225#if defined(CONFIG_CGROUPS)
226#if !defined(CONFIG_NET_CLS_CGROUP)
225int net_cls_subsys_id = -1; 227int net_cls_subsys_id = -1;
226EXPORT_SYMBOL_GPL(net_cls_subsys_id); 228EXPORT_SYMBOL_GPL(net_cls_subsys_id);
227#endif 229#endif
230#if !defined(CONFIG_NETPRIO_CGROUP)
231int net_prio_subsys_id = -1;
232EXPORT_SYMBOL_GPL(net_prio_subsys_id);
233#endif
234#endif
228 235
229static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen) 236static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
230{ 237{
@@ -740,6 +747,11 @@ set_rcvbuf:
740 case SO_RXQ_OVFL: 747 case SO_RXQ_OVFL:
741 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 748 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
742 break; 749 break;
750
751 case SO_WIFI_STATUS:
752 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
753 break;
754
743 default: 755 default:
744 ret = -ENOPROTOOPT; 756 ret = -ENOPROTOOPT;
745 break; 757 break;
@@ -961,6 +973,10 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
961 v.val = !!sock_flag(sk, SOCK_RXQ_OVFL); 973 v.val = !!sock_flag(sk, SOCK_RXQ_OVFL);
962 break; 974 break;
963 975
976 case SO_WIFI_STATUS:
977 v.val = !!sock_flag(sk, SOCK_WIFI_STATUS);
978 break;
979
964 default: 980 default:
965 return -ENOPROTOOPT; 981 return -ENOPROTOOPT;
966 } 982 }
@@ -1111,6 +1127,18 @@ void sock_update_classid(struct sock *sk)
1111 sk->sk_classid = classid; 1127 sk->sk_classid = classid;
1112} 1128}
1113EXPORT_SYMBOL(sock_update_classid); 1129EXPORT_SYMBOL(sock_update_classid);
1130
1131void sock_update_netprioidx(struct sock *sk)
1132{
1133 struct cgroup_netprio_state *state;
1134 if (in_interrupt())
1135 return;
1136 rcu_read_lock();
1137 state = task_netprio_state(current);
1138 sk->sk_cgrp_prioidx = state ? state->prioidx : 0;
1139 rcu_read_unlock();
1140}
1141EXPORT_SYMBOL_GPL(sock_update_netprioidx);
1114#endif 1142#endif
1115 1143
1116/** 1144/**
@@ -1138,6 +1166,7 @@ struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1138 atomic_set(&sk->sk_wmem_alloc, 1); 1166 atomic_set(&sk->sk_wmem_alloc, 1);
1139 1167
1140 sock_update_classid(sk); 1168 sock_update_classid(sk);
1169 sock_update_netprioidx(sk);
1141 } 1170 }
1142 1171
1143 return sk; 1172 return sk;
@@ -1204,7 +1233,14 @@ void sk_release_kernel(struct sock *sk)
1204} 1233}
1205EXPORT_SYMBOL(sk_release_kernel); 1234EXPORT_SYMBOL(sk_release_kernel);
1206 1235
1207struct sock *sk_clone(const struct sock *sk, const gfp_t priority) 1236/**
1237 * sk_clone_lock - clone a socket, and lock its clone
1238 * @sk: the socket to clone
1239 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1240 *
1241 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1242 */
1243struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
1208{ 1244{
1209 struct sock *newsk; 1245 struct sock *newsk;
1210 1246
@@ -1297,7 +1333,7 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
1297out: 1333out:
1298 return newsk; 1334 return newsk;
1299} 1335}
1300EXPORT_SYMBOL_GPL(sk_clone); 1336EXPORT_SYMBOL_GPL(sk_clone_lock);
1301 1337
1302void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 1338void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
1303{ 1339{
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index 77a65f031488..d05559d4d9cd 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -68,8 +68,13 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write,
68 68
69 if (sock_table != orig_sock_table) { 69 if (sock_table != orig_sock_table) {
70 rcu_assign_pointer(rps_sock_flow_table, sock_table); 70 rcu_assign_pointer(rps_sock_flow_table, sock_table);
71 synchronize_rcu(); 71 if (sock_table)
72 vfree(orig_sock_table); 72 jump_label_inc(&rps_needed);
73 if (orig_sock_table) {
74 jump_label_dec(&rps_needed);
75 synchronize_rcu();
76 vfree(orig_sock_table);
77 }
73 } 78 }
74 } 79 }
75 80
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-26 01:53:30 -0400