aboutsummaryrefslogtreecommitdiffstats
path: root/net/core
diff options
context:
space:
mode:
Diffstat (limited to 'net/core')
-rw-r--r--net/core/datagram.c2
-rw-r--r--net/core/dev.c383
-rw-r--r--net/core/ethtool.c21
-rw-r--r--net/core/filter.c514
-rw-r--r--net/core/neighbour.c1
-rw-r--r--net/core/net-sysfs.c430
-rw-r--r--net/core/net-sysfs.h4
-rw-r--r--net/core/netpoll.c4
-rw-r--r--net/core/pktgen.c44
-rw-r--r--net/core/request_sock.c1
-rw-r--r--net/core/rtnetlink.c166
-rw-r--r--net/core/scm.c10
-rw-r--r--net/core/skbuff.c36
-rw-r--r--net/core/sock.c11
-rw-r--r--net/core/timestamping.c4
15 files changed, 1199 insertions, 432 deletions
diff --git a/net/core/datagram.c b/net/core/datagram.c
index cd1e039c875..18ac112ea7a 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -177,7 +177,7 @@ struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
177 * interrupt level will suddenly eat the receive_queue. 177 * interrupt level will suddenly eat the receive_queue.
178 * 178 *
179 * Look at current nfs client by the way... 179 * Look at current nfs client by the way...
180 * However, this function was corrent in any case. 8) 180 * However, this function was correct in any case. 8)
181 */ 181 */
182 unsigned long cpu_flags; 182 unsigned long cpu_flags;
183 183
diff --git a/net/core/dev.c b/net/core/dev.c
index 0dd54a69dac..a215269d2e3 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -743,34 +743,31 @@ struct net_device *dev_get_by_index(struct net *net, int ifindex)
743EXPORT_SYMBOL(dev_get_by_index); 743EXPORT_SYMBOL(dev_get_by_index);
744 744
745/** 745/**
746 * dev_getbyhwaddr - find a device by its hardware address 746 * dev_getbyhwaddr_rcu - find a device by its hardware address
747 * @net: the applicable net namespace 747 * @net: the applicable net namespace
748 * @type: media type of device 748 * @type: media type of device
749 * @ha: hardware address 749 * @ha: hardware address
750 * 750 *
751 * Search for an interface by MAC address. Returns NULL if the device 751 * Search for an interface by MAC address. Returns NULL if the device
752 * is not found or a pointer to the device. The caller must hold the 752 * is not found or a pointer to the device. The caller must hold RCU
753 * rtnl semaphore. The returned device has not had its ref count increased 753 * The returned device has not had its ref count increased
754 * and the caller must therefore be careful about locking 754 * and the caller must therefore be careful about locking
755 * 755 *
756 * BUGS:
757 * If the API was consistent this would be __dev_get_by_hwaddr
758 */ 756 */
759 757
760struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha) 758struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
759 const char *ha)
761{ 760{
762 struct net_device *dev; 761 struct net_device *dev;
763 762
764 ASSERT_RTNL(); 763 for_each_netdev_rcu(net, dev)
765
766 for_each_netdev(net, dev)
767 if (dev->type == type && 764 if (dev->type == type &&
768 !memcmp(dev->dev_addr, ha, dev->addr_len)) 765 !memcmp(dev->dev_addr, ha, dev->addr_len))
769 return dev; 766 return dev;
770 767
771 return NULL; 768 return NULL;
772} 769}
773EXPORT_SYMBOL(dev_getbyhwaddr); 770EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
774 771
775struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 772struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
776{ 773{
@@ -1225,52 +1222,90 @@ int dev_open(struct net_device *dev)
1225} 1222}
1226EXPORT_SYMBOL(dev_open); 1223EXPORT_SYMBOL(dev_open);
1227 1224
1228static int __dev_close(struct net_device *dev) 1225static int __dev_close_many(struct list_head *head)
1229{ 1226{
1230 const struct net_device_ops *ops = dev->netdev_ops; 1227 struct net_device *dev;
1231 1228
1232 ASSERT_RTNL(); 1229 ASSERT_RTNL();
1233 might_sleep(); 1230 might_sleep();
1234 1231
1235 /* 1232 list_for_each_entry(dev, head, unreg_list) {
1236 * Tell people we are going down, so that they can 1233 /*
1237 * prepare to death, when device is still operating. 1234 * Tell people we are going down, so that they can
1238 */ 1235 * prepare to death, when device is still operating.
1239 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1236 */
1237 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1240 1238
1241 clear_bit(__LINK_STATE_START, &dev->state); 1239 clear_bit(__LINK_STATE_START, &dev->state);
1242 1240
1243 /* Synchronize to scheduled poll. We cannot touch poll list, 1241 /* Synchronize to scheduled poll. We cannot touch poll list, it
1244 * it can be even on different cpu. So just clear netif_running(). 1242 * can be even on different cpu. So just clear netif_running().
1245 * 1243 *
1246 * dev->stop() will invoke napi_disable() on all of it's 1244 * dev->stop() will invoke napi_disable() on all of it's
1247 * napi_struct instances on this device. 1245 * napi_struct instances on this device.
1248 */ 1246 */
1249 smp_mb__after_clear_bit(); /* Commit netif_running(). */ 1247 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1248 }
1250 1249
1251 dev_deactivate(dev); 1250 dev_deactivate_many(head);
1252 1251
1253 /* 1252 list_for_each_entry(dev, head, unreg_list) {
1254 * Call the device specific close. This cannot fail. 1253 const struct net_device_ops *ops = dev->netdev_ops;
1255 * Only if device is UP
1256 *
1257 * We allow it to be called even after a DETACH hot-plug
1258 * event.
1259 */
1260 if (ops->ndo_stop)
1261 ops->ndo_stop(dev);
1262 1254
1263 /* 1255 /*
1264 * Device is now down. 1256 * Call the device specific close. This cannot fail.
1265 */ 1257 * Only if device is UP
1258 *
1259 * We allow it to be called even after a DETACH hot-plug
1260 * event.
1261 */
1262 if (ops->ndo_stop)
1263 ops->ndo_stop(dev);
1264
1265 /*
1266 * Device is now down.
1267 */
1268
1269 dev->flags &= ~IFF_UP;
1270
1271 /*
1272 * Shutdown NET_DMA
1273 */
1274 net_dmaengine_put();
1275 }
1276
1277 return 0;
1278}
1279
1280static int __dev_close(struct net_device *dev)
1281{
1282 LIST_HEAD(single);
1283
1284 list_add(&dev->unreg_list, &single);
1285 return __dev_close_many(&single);
1286}
1287
1288int dev_close_many(struct list_head *head)
1289{
1290 struct net_device *dev, *tmp;
1291 LIST_HEAD(tmp_list);
1292
1293 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1294 if (!(dev->flags & IFF_UP))
1295 list_move(&dev->unreg_list, &tmp_list);
1266 1296
1267 dev->flags &= ~IFF_UP; 1297 __dev_close_many(head);
1268 1298
1269 /* 1299 /*
1270 * Shutdown NET_DMA 1300 * Tell people we are down
1271 */ 1301 */
1272 net_dmaengine_put(); 1302 list_for_each_entry(dev, head, unreg_list) {
1303 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1304 call_netdevice_notifiers(NETDEV_DOWN, dev);
1305 }
1273 1306
1307 /* rollback_registered_many needs the complete original list */
1308 list_splice(&tmp_list, head);
1274 return 0; 1309 return 0;
1275} 1310}
1276 1311
@@ -1285,16 +1320,10 @@ static int __dev_close(struct net_device *dev)
1285 */ 1320 */
1286int dev_close(struct net_device *dev) 1321int dev_close(struct net_device *dev)
1287{ 1322{
1288 if (!(dev->flags & IFF_UP)) 1323 LIST_HEAD(single);
1289 return 0;
1290
1291 __dev_close(dev);
1292 1324
1293 /* 1325 list_add(&dev->unreg_list, &single);
1294 * Tell people we are down 1326 dev_close_many(&single);
1295 */
1296 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1297 call_netdevice_notifiers(NETDEV_DOWN, dev);
1298 1327
1299 return 0; 1328 return 0;
1300} 1329}
@@ -1499,6 +1528,14 @@ int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1499} 1528}
1500EXPORT_SYMBOL_GPL(dev_forward_skb); 1529EXPORT_SYMBOL_GPL(dev_forward_skb);
1501 1530
1531static inline int deliver_skb(struct sk_buff *skb,
1532 struct packet_type *pt_prev,
1533 struct net_device *orig_dev)
1534{
1535 atomic_inc(&skb->users);
1536 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1537}
1538
1502/* 1539/*
1503 * Support routine. Sends outgoing frames to any network 1540 * Support routine. Sends outgoing frames to any network
1504 * taps currently in use. 1541 * taps currently in use.
@@ -1507,13 +1544,8 @@ EXPORT_SYMBOL_GPL(dev_forward_skb);
1507static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 1544static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1508{ 1545{
1509 struct packet_type *ptype; 1546 struct packet_type *ptype;
1510 1547 struct sk_buff *skb2 = NULL;
1511#ifdef CONFIG_NET_CLS_ACT 1548 struct packet_type *pt_prev = NULL;
1512 if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))
1513 net_timestamp_set(skb);
1514#else
1515 net_timestamp_set(skb);
1516#endif
1517 1549
1518 rcu_read_lock(); 1550 rcu_read_lock();
1519 list_for_each_entry_rcu(ptype, &ptype_all, list) { 1551 list_for_each_entry_rcu(ptype, &ptype_all, list) {
@@ -1523,10 +1555,18 @@ static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1523 if ((ptype->dev == dev || !ptype->dev) && 1555 if ((ptype->dev == dev || !ptype->dev) &&
1524 (ptype->af_packet_priv == NULL || 1556 (ptype->af_packet_priv == NULL ||
1525 (struct sock *)ptype->af_packet_priv != skb->sk)) { 1557 (struct sock *)ptype->af_packet_priv != skb->sk)) {
1526 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC); 1558 if (pt_prev) {
1559 deliver_skb(skb2, pt_prev, skb->dev);
1560 pt_prev = ptype;
1561 continue;
1562 }
1563
1564 skb2 = skb_clone(skb, GFP_ATOMIC);
1527 if (!skb2) 1565 if (!skb2)
1528 break; 1566 break;
1529 1567
1568 net_timestamp_set(skb2);
1569
1530 /* skb->nh should be correctly 1570 /* skb->nh should be correctly
1531 set by sender, so that the second statement is 1571 set by sender, so that the second statement is
1532 just protection against buggy protocols. 1572 just protection against buggy protocols.
@@ -1545,9 +1585,11 @@ static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1545 1585
1546 skb2->transport_header = skb2->network_header; 1586 skb2->transport_header = skb2->network_header;
1547 skb2->pkt_type = PACKET_OUTGOING; 1587 skb2->pkt_type = PACKET_OUTGOING;
1548 ptype->func(skb2, skb->dev, ptype, skb->dev); 1588 pt_prev = ptype;
1549 } 1589 }
1550 } 1590 }
1591 if (pt_prev)
1592 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1551 rcu_read_unlock(); 1593 rcu_read_unlock();
1552} 1594}
1553 1595
@@ -1557,12 +1599,19 @@ static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1557 */ 1599 */
1558int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 1600int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
1559{ 1601{
1602 int rc;
1603
1560 if (txq < 1 || txq > dev->num_tx_queues) 1604 if (txq < 1 || txq > dev->num_tx_queues)
1561 return -EINVAL; 1605 return -EINVAL;
1562 1606
1563 if (dev->reg_state == NETREG_REGISTERED) { 1607 if (dev->reg_state == NETREG_REGISTERED) {
1564 ASSERT_RTNL(); 1608 ASSERT_RTNL();
1565 1609
1610 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1611 txq);
1612 if (rc)
1613 return rc;
1614
1566 if (txq < dev->real_num_tx_queues) 1615 if (txq < dev->real_num_tx_queues)
1567 qdisc_reset_all_tx_gt(dev, txq); 1616 qdisc_reset_all_tx_gt(dev, txq);
1568 } 1617 }
@@ -1757,7 +1806,7 @@ int skb_checksum_help(struct sk_buff *skb)
1757 goto out_set_summed; 1806 goto out_set_summed;
1758 } 1807 }
1759 1808
1760 offset = skb->csum_start - skb_headroom(skb); 1809 offset = skb_checksum_start_offset(skb);
1761 BUG_ON(offset >= skb_headlen(skb)); 1810 BUG_ON(offset >= skb_headlen(skb));
1762 csum = skb_checksum(skb, offset, skb->len - offset, 0); 1811 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1763 1812
@@ -1794,16 +1843,18 @@ struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
1794 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 1843 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1795 struct packet_type *ptype; 1844 struct packet_type *ptype;
1796 __be16 type = skb->protocol; 1845 __be16 type = skb->protocol;
1846 int vlan_depth = ETH_HLEN;
1797 int err; 1847 int err;
1798 1848
1799 if (type == htons(ETH_P_8021Q)) { 1849 while (type == htons(ETH_P_8021Q)) {
1800 struct vlan_ethhdr *veh; 1850 struct vlan_hdr *vh;
1801 1851
1802 if (unlikely(!pskb_may_pull(skb, VLAN_ETH_HLEN))) 1852 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
1803 return ERR_PTR(-EINVAL); 1853 return ERR_PTR(-EINVAL);
1804 1854
1805 veh = (struct vlan_ethhdr *)skb->data; 1855 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1806 type = veh->h_vlan_encapsulated_proto; 1856 type = vh->h_vlan_encapsulated_proto;
1857 vlan_depth += VLAN_HLEN;
1807 } 1858 }
1808 1859
1809 skb_reset_mac_header(skb); 1860 skb_reset_mac_header(skb);
@@ -1817,8 +1868,7 @@ struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
1817 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) 1868 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1818 dev->ethtool_ops->get_drvinfo(dev, &info); 1869 dev->ethtool_ops->get_drvinfo(dev, &info);
1819 1870
1820 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d " 1871 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n",
1821 "ip_summed=%d",
1822 info.driver, dev ? dev->features : 0L, 1872 info.driver, dev ? dev->features : 0L,
1823 skb->sk ? skb->sk->sk_route_caps : 0L, 1873 skb->sk ? skb->sk->sk_route_caps : 0L,
1824 skb->len, skb->data_len, skb->ip_summed); 1874 skb->len, skb->data_len, skb->ip_summed);
@@ -1967,6 +2017,23 @@ static inline void skb_orphan_try(struct sk_buff *skb)
1967 } 2017 }
1968} 2018}
1969 2019
2020int netif_get_vlan_features(struct sk_buff *skb, struct net_device *dev)
2021{
2022 __be16 protocol = skb->protocol;
2023
2024 if (protocol == htons(ETH_P_8021Q)) {
2025 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2026 protocol = veh->h_vlan_encapsulated_proto;
2027 } else if (!skb->vlan_tci)
2028 return dev->features;
2029
2030 if (protocol != htons(ETH_P_8021Q))
2031 return dev->features & dev->vlan_features;
2032 else
2033 return 0;
2034}
2035EXPORT_SYMBOL(netif_get_vlan_features);
2036
1970/* 2037/*
1971 * Returns true if either: 2038 * Returns true if either:
1972 * 1. skb has frag_list and the device doesn't support FRAGLIST, or 2039 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
@@ -1977,15 +2044,20 @@ static inline void skb_orphan_try(struct sk_buff *skb)
1977static inline int skb_needs_linearize(struct sk_buff *skb, 2044static inline int skb_needs_linearize(struct sk_buff *skb,
1978 struct net_device *dev) 2045 struct net_device *dev)
1979{ 2046{
1980 int features = dev->features; 2047 if (skb_is_nonlinear(skb)) {
2048 int features = dev->features;
1981 2049
1982 if (skb->protocol == htons(ETH_P_8021Q) || vlan_tx_tag_present(skb)) 2050 if (vlan_tx_tag_present(skb))
1983 features &= dev->vlan_features; 2051 features &= dev->vlan_features;
1984 2052
1985 return skb_is_nonlinear(skb) && 2053 return (skb_has_frag_list(skb) &&
1986 ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) || 2054 !(features & NETIF_F_FRAGLIST)) ||
1987 (skb_shinfo(skb)->nr_frags && (!(features & NETIF_F_SG) || 2055 (skb_shinfo(skb)->nr_frags &&
1988 illegal_highdma(dev, skb)))); 2056 (!(features & NETIF_F_SG) ||
2057 illegal_highdma(dev, skb)));
2058 }
2059
2060 return 0;
1989} 2061}
1990 2062
1991int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, 2063int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
@@ -1995,9 +2067,6 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1995 int rc = NETDEV_TX_OK; 2067 int rc = NETDEV_TX_OK;
1996 2068
1997 if (likely(!skb->next)) { 2069 if (likely(!skb->next)) {
1998 if (!list_empty(&ptype_all))
1999 dev_queue_xmit_nit(skb, dev);
2000
2001 /* 2070 /*
2002 * If device doesnt need skb->dst, release it right now while 2071 * If device doesnt need skb->dst, release it right now while
2003 * its hot in this cpu cache 2072 * its hot in this cpu cache
@@ -2005,6 +2074,9 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2005 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 2074 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2006 skb_dst_drop(skb); 2075 skb_dst_drop(skb);
2007 2076
2077 if (!list_empty(&ptype_all))
2078 dev_queue_xmit_nit(skb, dev);
2079
2008 skb_orphan_try(skb); 2080 skb_orphan_try(skb);
2009 2081
2010 if (vlan_tx_tag_present(skb) && 2082 if (vlan_tx_tag_present(skb) &&
@@ -2031,8 +2103,8 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2031 * checksumming here. 2103 * checksumming here.
2032 */ 2104 */
2033 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2105 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2034 skb_set_transport_header(skb, skb->csum_start - 2106 skb_set_transport_header(skb,
2035 skb_headroom(skb)); 2107 skb_checksum_start_offset(skb));
2036 if (!dev_can_checksum(dev, skb) && 2108 if (!dev_can_checksum(dev, skb) &&
2037 skb_checksum_help(skb)) 2109 skb_checksum_help(skb))
2038 goto out_kfree_skb; 2110 goto out_kfree_skb;
@@ -2085,14 +2157,19 @@ out:
2085 2157
2086static u32 hashrnd __read_mostly; 2158static u32 hashrnd __read_mostly;
2087 2159
2088u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb) 2160/*
2161 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2162 * to be used as a distribution range.
2163 */
2164u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2165 unsigned int num_tx_queues)
2089{ 2166{
2090 u32 hash; 2167 u32 hash;
2091 2168
2092 if (skb_rx_queue_recorded(skb)) { 2169 if (skb_rx_queue_recorded(skb)) {
2093 hash = skb_get_rx_queue(skb); 2170 hash = skb_get_rx_queue(skb);
2094 while (unlikely(hash >= dev->real_num_tx_queues)) 2171 while (unlikely(hash >= num_tx_queues))
2095 hash -= dev->real_num_tx_queues; 2172 hash -= num_tx_queues;
2096 return hash; 2173 return hash;
2097 } 2174 }
2098 2175
@@ -2102,9 +2179,9 @@ u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
2102 hash = (__force u16) skb->protocol ^ skb->rxhash; 2179 hash = (__force u16) skb->protocol ^ skb->rxhash;
2103 hash = jhash_1word(hash, hashrnd); 2180 hash = jhash_1word(hash, hashrnd);
2104 2181
2105 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32); 2182 return (u16) (((u64) hash * num_tx_queues) >> 32);
2106} 2183}
2107EXPORT_SYMBOL(skb_tx_hash); 2184EXPORT_SYMBOL(__skb_tx_hash);
2108 2185
2109static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index) 2186static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2110{ 2187{
@@ -2119,26 +2196,70 @@ static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2119 return queue_index; 2196 return queue_index;
2120} 2197}
2121 2198
2199static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2200{
2201#ifdef CONFIG_XPS
2202 struct xps_dev_maps *dev_maps;
2203 struct xps_map *map;
2204 int queue_index = -1;
2205
2206 rcu_read_lock();
2207 dev_maps = rcu_dereference(dev->xps_maps);
2208 if (dev_maps) {
2209 map = rcu_dereference(
2210 dev_maps->cpu_map[raw_smp_processor_id()]);
2211 if (map) {
2212 if (map->len == 1)
2213 queue_index = map->queues[0];
2214 else {
2215 u32 hash;
2216 if (skb->sk && skb->sk->sk_hash)
2217 hash = skb->sk->sk_hash;
2218 else
2219 hash = (__force u16) skb->protocol ^
2220 skb->rxhash;
2221 hash = jhash_1word(hash, hashrnd);
2222 queue_index = map->queues[
2223 ((u64)hash * map->len) >> 32];
2224 }
2225 if (unlikely(queue_index >= dev->real_num_tx_queues))
2226 queue_index = -1;
2227 }
2228 }
2229 rcu_read_unlock();
2230
2231 return queue_index;
2232#else
2233 return -1;
2234#endif
2235}
2236
2122static struct netdev_queue *dev_pick_tx(struct net_device *dev, 2237static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2123 struct sk_buff *skb) 2238 struct sk_buff *skb)
2124{ 2239{
2125 int queue_index; 2240 int queue_index;
2126 const struct net_device_ops *ops = dev->netdev_ops; 2241 const struct net_device_ops *ops = dev->netdev_ops;
2127 2242
2128 if (ops->ndo_select_queue) { 2243 if (dev->real_num_tx_queues == 1)
2244 queue_index = 0;
2245 else if (ops->ndo_select_queue) {
2129 queue_index = ops->ndo_select_queue(dev, skb); 2246 queue_index = ops->ndo_select_queue(dev, skb);
2130 queue_index = dev_cap_txqueue(dev, queue_index); 2247 queue_index = dev_cap_txqueue(dev, queue_index);
2131 } else { 2248 } else {
2132 struct sock *sk = skb->sk; 2249 struct sock *sk = skb->sk;
2133 queue_index = sk_tx_queue_get(sk); 2250 queue_index = sk_tx_queue_get(sk);
2134 if (queue_index < 0 || queue_index >= dev->real_num_tx_queues) {
2135 2251
2136 queue_index = 0; 2252 if (queue_index < 0 || skb->ooo_okay ||
2137 if (dev->real_num_tx_queues > 1) 2253 queue_index >= dev->real_num_tx_queues) {
2254 int old_index = queue_index;
2255
2256 queue_index = get_xps_queue(dev, skb);
2257 if (queue_index < 0)
2138 queue_index = skb_tx_hash(dev, skb); 2258 queue_index = skb_tx_hash(dev, skb);
2139 2259
2140 if (sk) { 2260 if (queue_index != old_index && sk) {
2141 struct dst_entry *dst = rcu_dereference_check(sk->sk_dst_cache, 1); 2261 struct dst_entry *dst =
2262 rcu_dereference_check(sk->sk_dst_cache, 1);
2142 2263
2143 if (dst && skb_dst(skb) == dst) 2264 if (dst && skb_dst(skb) == dst)
2144 sk_tx_queue_set(sk, queue_index); 2265 sk_tx_queue_set(sk, queue_index);
@@ -2712,14 +2833,6 @@ static void net_tx_action(struct softirq_action *h)
2712 } 2833 }
2713} 2834}
2714 2835
2715static inline int deliver_skb(struct sk_buff *skb,
2716 struct packet_type *pt_prev,
2717 struct net_device *orig_dev)
2718{
2719 atomic_inc(&skb->users);
2720 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2721}
2722
2723#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \ 2836#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
2724 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)) 2837 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
2725/* This hook is defined here for ATM LANE */ 2838/* This hook is defined here for ATM LANE */
@@ -4887,10 +5000,12 @@ static void rollback_registered_many(struct list_head *head)
4887 } 5000 }
4888 5001
4889 BUG_ON(dev->reg_state != NETREG_REGISTERED); 5002 BUG_ON(dev->reg_state != NETREG_REGISTERED);
5003 }
4890 5004
4891 /* If device is running, close it first. */ 5005 /* If device is running, close it first. */
4892 dev_close(dev); 5006 dev_close_many(head);
4893 5007
5008 list_for_each_entry(dev, head, unreg_list) {
4894 /* And unlink it from device chain. */ 5009 /* And unlink it from device chain. */
4895 unlist_netdevice(dev); 5010 unlist_netdevice(dev);
4896 5011
@@ -4967,10 +5082,13 @@ unsigned long netdev_fix_features(unsigned long features, const char *name)
4967 } 5082 }
4968 5083
4969 if (features & NETIF_F_UFO) { 5084 if (features & NETIF_F_UFO) {
4970 if (!(features & NETIF_F_GEN_CSUM)) { 5085 /* maybe split UFO into V4 and V6? */
5086 if (!((features & NETIF_F_GEN_CSUM) ||
5087 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5088 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
4971 if (name) 5089 if (name)
4972 printk(KERN_ERR "%s: Dropping NETIF_F_UFO " 5090 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4973 "since no NETIF_F_HW_CSUM feature.\n", 5091 "since no checksum offload features.\n",
4974 name); 5092 name);
4975 features &= ~NETIF_F_UFO; 5093 features &= ~NETIF_F_UFO;
4976 } 5094 }
@@ -5014,9 +5132,9 @@ void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5014} 5132}
5015EXPORT_SYMBOL(netif_stacked_transfer_operstate); 5133EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5016 5134
5135#ifdef CONFIG_RPS
5017static int netif_alloc_rx_queues(struct net_device *dev) 5136static int netif_alloc_rx_queues(struct net_device *dev)
5018{ 5137{
5019#ifdef CONFIG_RPS
5020 unsigned int i, count = dev->num_rx_queues; 5138 unsigned int i, count = dev->num_rx_queues;
5021 struct netdev_rx_queue *rx; 5139 struct netdev_rx_queue *rx;
5022 5140
@@ -5029,15 +5147,22 @@ static int netif_alloc_rx_queues(struct net_device *dev)
5029 } 5147 }
5030 dev->_rx = rx; 5148 dev->_rx = rx;
5031 5149
5032 /*
5033 * Set a pointer to first element in the array which holds the
5034 * reference count.
5035 */
5036 for (i = 0; i < count; i++) 5150 for (i = 0; i < count; i++)
5037 rx[i].first = rx; 5151 rx[i].dev = dev;
5038#endif
5039 return 0; 5152 return 0;
5040} 5153}
5154#endif
5155
5156static void netdev_init_one_queue(struct net_device *dev,
5157 struct netdev_queue *queue, void *_unused)
5158{
5159 /* Initialize queue lock */
5160 spin_lock_init(&queue->_xmit_lock);
5161 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5162 queue->xmit_lock_owner = -1;
5163 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
5164 queue->dev = dev;
5165}
5041 5166
5042static int netif_alloc_netdev_queues(struct net_device *dev) 5167static int netif_alloc_netdev_queues(struct net_device *dev)
5043{ 5168{
@@ -5053,25 +5178,11 @@ static int netif_alloc_netdev_queues(struct net_device *dev)
5053 return -ENOMEM; 5178 return -ENOMEM;
5054 } 5179 }
5055 dev->_tx = tx; 5180 dev->_tx = tx;
5056 return 0;
5057}
5058 5181
5059static void netdev_init_one_queue(struct net_device *dev,
5060 struct netdev_queue *queue,
5061 void *_unused)
5062{
5063 queue->dev = dev;
5064
5065 /* Initialize queue lock */
5066 spin_lock_init(&queue->_xmit_lock);
5067 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5068 queue->xmit_lock_owner = -1;
5069}
5070
5071static void netdev_init_queues(struct net_device *dev)
5072{
5073 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 5182 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5074 spin_lock_init(&dev->tx_global_lock); 5183 spin_lock_init(&dev->tx_global_lock);
5184
5185 return 0;
5075} 5186}
5076 5187
5077/** 5188/**
@@ -5110,16 +5221,6 @@ int register_netdevice(struct net_device *dev)
5110 5221
5111 dev->iflink = -1; 5222 dev->iflink = -1;
5112 5223
5113 ret = netif_alloc_rx_queues(dev);
5114 if (ret)
5115 goto out;
5116
5117 ret = netif_alloc_netdev_queues(dev);
5118 if (ret)
5119 goto out;
5120
5121 netdev_init_queues(dev);
5122
5123 /* Init, if this function is available */ 5224 /* Init, if this function is available */
5124 if (dev->netdev_ops->ndo_init) { 5225 if (dev->netdev_ops->ndo_init) {
5125 ret = dev->netdev_ops->ndo_init(dev); 5226 ret = dev->netdev_ops->ndo_init(dev);
@@ -5577,10 +5678,14 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
5577 5678
5578 dev->num_tx_queues = queue_count; 5679 dev->num_tx_queues = queue_count;
5579 dev->real_num_tx_queues = queue_count; 5680 dev->real_num_tx_queues = queue_count;
5681 if (netif_alloc_netdev_queues(dev))
5682 goto free_pcpu;
5580 5683
5581#ifdef CONFIG_RPS 5684#ifdef CONFIG_RPS
5582 dev->num_rx_queues = queue_count; 5685 dev->num_rx_queues = queue_count;
5583 dev->real_num_rx_queues = queue_count; 5686 dev->real_num_rx_queues = queue_count;
5687 if (netif_alloc_rx_queues(dev))
5688 goto free_pcpu;
5584#endif 5689#endif
5585 5690
5586 dev->gso_max_size = GSO_MAX_SIZE; 5691 dev->gso_max_size = GSO_MAX_SIZE;
@@ -5597,6 +5702,11 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
5597 5702
5598free_pcpu: 5703free_pcpu:
5599 free_percpu(dev->pcpu_refcnt); 5704 free_percpu(dev->pcpu_refcnt);
5705 kfree(dev->_tx);
5706#ifdef CONFIG_RPS
5707 kfree(dev->_rx);
5708#endif
5709
5600free_p: 5710free_p:
5601 kfree(p); 5711 kfree(p);
5602 return NULL; 5712 return NULL;
@@ -5618,6 +5728,9 @@ void free_netdev(struct net_device *dev)
5618 release_net(dev_net(dev)); 5728 release_net(dev_net(dev));
5619 5729
5620 kfree(dev->_tx); 5730 kfree(dev->_tx);
5731#ifdef CONFIG_RPS
5732 kfree(dev->_rx);
5733#endif
5621 5734
5622 kfree(rcu_dereference_raw(dev->ingress_queue)); 5735 kfree(rcu_dereference_raw(dev->ingress_queue));
5623 5736
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index 956a9f4971c..17741782a34 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -891,6 +891,20 @@ static int ethtool_nway_reset(struct net_device *dev)
891 return dev->ethtool_ops->nway_reset(dev); 891 return dev->ethtool_ops->nway_reset(dev);
892} 892}
893 893
894static int ethtool_get_link(struct net_device *dev, char __user *useraddr)
895{
896 struct ethtool_value edata = { .cmd = ETHTOOL_GLINK };
897
898 if (!dev->ethtool_ops->get_link)
899 return -EOPNOTSUPP;
900
901 edata.data = netif_running(dev) && dev->ethtool_ops->get_link(dev);
902
903 if (copy_to_user(useraddr, &edata, sizeof(edata)))
904 return -EFAULT;
905 return 0;
906}
907
894static int ethtool_get_eeprom(struct net_device *dev, void __user *useraddr) 908static int ethtool_get_eeprom(struct net_device *dev, void __user *useraddr)
895{ 909{
896 struct ethtool_eeprom eeprom; 910 struct ethtool_eeprom eeprom;
@@ -1171,7 +1185,9 @@ static int ethtool_set_ufo(struct net_device *dev, char __user *useraddr)
1171 return -EFAULT; 1185 return -EFAULT;
1172 if (edata.data && !(dev->features & NETIF_F_SG)) 1186 if (edata.data && !(dev->features & NETIF_F_SG))
1173 return -EINVAL; 1187 return -EINVAL;
1174 if (edata.data && !(dev->features & NETIF_F_HW_CSUM)) 1188 if (edata.data && !((dev->features & NETIF_F_GEN_CSUM) ||
1189 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
1190 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM)))
1175 return -EINVAL; 1191 return -EINVAL;
1176 return dev->ethtool_ops->set_ufo(dev, edata.data); 1192 return dev->ethtool_ops->set_ufo(dev, edata.data);
1177} 1193}
@@ -1528,8 +1544,7 @@ int dev_ethtool(struct net *net, struct ifreq *ifr)
1528 rc = ethtool_nway_reset(dev); 1544 rc = ethtool_nway_reset(dev);
1529 break; 1545 break;
1530 case ETHTOOL_GLINK: 1546 case ETHTOOL_GLINK:
1531 rc = ethtool_get_value(dev, useraddr, ethcmd, 1547 rc = ethtool_get_link(dev, useraddr);
1532 dev->ethtool_ops->get_link);
1533 break; 1548 break;
1534 case ETHTOOL_GEEPROM: 1549 case ETHTOOL_GEEPROM:
1535 rc = ethtool_get_eeprom(dev, useraddr); 1550 rc = ethtool_get_eeprom(dev, useraddr);
diff --git a/net/core/filter.c b/net/core/filter.c
index ae21a0d3c4a..2b27d4efdd4 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -37,9 +37,69 @@
37#include <asm/uaccess.h> 37#include <asm/uaccess.h>
38#include <asm/unaligned.h> 38#include <asm/unaligned.h>
39#include <linux/filter.h> 39#include <linux/filter.h>
40#include <linux/reciprocal_div.h>
41
42enum {
43 BPF_S_RET_K = 1,
44 BPF_S_RET_A,
45 BPF_S_ALU_ADD_K,
46 BPF_S_ALU_ADD_X,
47 BPF_S_ALU_SUB_K,
48 BPF_S_ALU_SUB_X,
49 BPF_S_ALU_MUL_K,
50 BPF_S_ALU_MUL_X,
51 BPF_S_ALU_DIV_X,
52 BPF_S_ALU_AND_K,
53 BPF_S_ALU_AND_X,
54 BPF_S_ALU_OR_K,
55 BPF_S_ALU_OR_X,
56 BPF_S_ALU_LSH_K,
57 BPF_S_ALU_LSH_X,
58 BPF_S_ALU_RSH_K,
59 BPF_S_ALU_RSH_X,
60 BPF_S_ALU_NEG,
61 BPF_S_LD_W_ABS,
62 BPF_S_LD_H_ABS,
63 BPF_S_LD_B_ABS,
64 BPF_S_LD_W_LEN,
65 BPF_S_LD_W_IND,
66 BPF_S_LD_H_IND,
67 BPF_S_LD_B_IND,
68 BPF_S_LD_IMM,
69 BPF_S_LDX_W_LEN,
70 BPF_S_LDX_B_MSH,
71 BPF_S_LDX_IMM,
72 BPF_S_MISC_TAX,
73 BPF_S_MISC_TXA,
74 BPF_S_ALU_DIV_K,
75 BPF_S_LD_MEM,
76 BPF_S_LDX_MEM,
77 BPF_S_ST,
78 BPF_S_STX,
79 BPF_S_JMP_JA,
80 BPF_S_JMP_JEQ_K,
81 BPF_S_JMP_JEQ_X,
82 BPF_S_JMP_JGE_K,
83 BPF_S_JMP_JGE_X,
84 BPF_S_JMP_JGT_K,
85 BPF_S_JMP_JGT_X,
86 BPF_S_JMP_JSET_K,
87 BPF_S_JMP_JSET_X,
88 /* Ancillary data */
89 BPF_S_ANC_PROTOCOL,
90 BPF_S_ANC_PKTTYPE,
91 BPF_S_ANC_IFINDEX,
92 BPF_S_ANC_NLATTR,
93 BPF_S_ANC_NLATTR_NEST,
94 BPF_S_ANC_MARK,
95 BPF_S_ANC_QUEUE,
96 BPF_S_ANC_HATYPE,
97 BPF_S_ANC_RXHASH,
98 BPF_S_ANC_CPU,
99};
40 100
41/* No hurry in this branch */ 101/* No hurry in this branch */
42static void *__load_pointer(struct sk_buff *skb, int k) 102static void *__load_pointer(const struct sk_buff *skb, int k, unsigned int size)
43{ 103{
44 u8 *ptr = NULL; 104 u8 *ptr = NULL;
45 105
@@ -48,21 +108,17 @@ static void *__load_pointer(struct sk_buff *skb, int k)
48 else if (k >= SKF_LL_OFF) 108 else if (k >= SKF_LL_OFF)
49 ptr = skb_mac_header(skb) + k - SKF_LL_OFF; 109 ptr = skb_mac_header(skb) + k - SKF_LL_OFF;
50 110
51 if (ptr >= skb->head && ptr < skb_tail_pointer(skb)) 111 if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb))
52 return ptr; 112 return ptr;
53 return NULL; 113 return NULL;
54} 114}
55 115
56static inline void *load_pointer(struct sk_buff *skb, int k, 116static inline void *load_pointer(const struct sk_buff *skb, int k,
57 unsigned int size, void *buffer) 117 unsigned int size, void *buffer)
58{ 118{
59 if (k >= 0) 119 if (k >= 0)
60 return skb_header_pointer(skb, k, size, buffer); 120 return skb_header_pointer(skb, k, size, buffer);
61 else { 121 return __load_pointer(skb, k, size);
62 if (k >= SKF_AD_OFF)
63 return NULL;
64 return __load_pointer(skb, k);
65 }
66} 122}
67 123
68/** 124/**
@@ -89,7 +145,7 @@ int sk_filter(struct sock *sk, struct sk_buff *skb)
89 rcu_read_lock_bh(); 145 rcu_read_lock_bh();
90 filter = rcu_dereference_bh(sk->sk_filter); 146 filter = rcu_dereference_bh(sk->sk_filter);
91 if (filter) { 147 if (filter) {
92 unsigned int pkt_len = sk_run_filter(skb, filter->insns, filter->len); 148 unsigned int pkt_len = sk_run_filter(skb, filter->insns);
93 149
94 err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM; 150 err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM;
95 } 151 }
@@ -103,50 +159,52 @@ EXPORT_SYMBOL(sk_filter);
103 * sk_run_filter - run a filter on a socket 159 * sk_run_filter - run a filter on a socket
104 * @skb: buffer to run the filter on 160 * @skb: buffer to run the filter on
105 * @filter: filter to apply 161 * @filter: filter to apply
106 * @flen: length of filter
107 * 162 *
108 * Decode and apply filter instructions to the skb->data. 163 * Decode and apply filter instructions to the skb->data.
109 * Return length to keep, 0 for none. skb is the data we are 164 * Return length to keep, 0 for none. @skb is the data we are
110 * filtering, filter is the array of filter instructions, and 165 * filtering, @filter is the array of filter instructions.
111 * len is the number of filter blocks in the array. 166 * Because all jumps are guaranteed to be before last instruction,
167 * and last instruction guaranteed to be a RET, we dont need to check
168 * flen. (We used to pass to this function the length of filter)
112 */ 169 */
113unsigned int sk_run_filter(struct sk_buff *skb, struct sock_filter *filter, int flen) 170unsigned int sk_run_filter(const struct sk_buff *skb,
171 const struct sock_filter *fentry)
114{ 172{
115 void *ptr; 173 void *ptr;
116 u32 A = 0; /* Accumulator */ 174 u32 A = 0; /* Accumulator */
117 u32 X = 0; /* Index Register */ 175 u32 X = 0; /* Index Register */
118 u32 mem[BPF_MEMWORDS]; /* Scratch Memory Store */ 176 u32 mem[BPF_MEMWORDS]; /* Scratch Memory Store */
119 unsigned long memvalid = 0;
120 u32 tmp; 177 u32 tmp;
121 int k; 178 int k;
122 int pc;
123 179
124 BUILD_BUG_ON(BPF_MEMWORDS > BITS_PER_LONG);
125 /* 180 /*
126 * Process array of filter instructions. 181 * Process array of filter instructions.
127 */ 182 */
128 for (pc = 0; pc < flen; pc++) { 183 for (;; fentry++) {
129 const struct sock_filter *fentry = &filter[pc]; 184#if defined(CONFIG_X86_32)
130 u32 f_k = fentry->k; 185#define K (fentry->k)
186#else
187 const u32 K = fentry->k;
188#endif
131 189
132 switch (fentry->code) { 190 switch (fentry->code) {
133 case BPF_S_ALU_ADD_X: 191 case BPF_S_ALU_ADD_X:
134 A += X; 192 A += X;
135 continue; 193 continue;
136 case BPF_S_ALU_ADD_K: 194 case BPF_S_ALU_ADD_K:
137 A += f_k; 195 A += K;
138 continue; 196 continue;
139 case BPF_S_ALU_SUB_X: 197 case BPF_S_ALU_SUB_X:
140 A -= X; 198 A -= X;
141 continue; 199 continue;
142 case BPF_S_ALU_SUB_K: 200 case BPF_S_ALU_SUB_K:
143 A -= f_k; 201 A -= K;
144 continue; 202 continue;
145 case BPF_S_ALU_MUL_X: 203 case BPF_S_ALU_MUL_X:
146 A *= X; 204 A *= X;
147 continue; 205 continue;
148 case BPF_S_ALU_MUL_K: 206 case BPF_S_ALU_MUL_K:
149 A *= f_k; 207 A *= K;
150 continue; 208 continue;
151 case BPF_S_ALU_DIV_X: 209 case BPF_S_ALU_DIV_X:
152 if (X == 0) 210 if (X == 0)
@@ -154,89 +212,89 @@ unsigned int sk_run_filter(struct sk_buff *skb, struct sock_filter *filter, int
154 A /= X; 212 A /= X;
155 continue; 213 continue;
156 case BPF_S_ALU_DIV_K: 214 case BPF_S_ALU_DIV_K:
157 A /= f_k; 215 A = reciprocal_divide(A, K);
158 continue; 216 continue;
159 case BPF_S_ALU_AND_X: 217 case BPF_S_ALU_AND_X:
160 A &= X; 218 A &= X;
161 continue; 219 continue;
162 case BPF_S_ALU_AND_K: 220 case BPF_S_ALU_AND_K:
163 A &= f_k; 221 A &= K;
164 continue; 222 continue;
165 case BPF_S_ALU_OR_X: 223 case BPF_S_ALU_OR_X:
166 A |= X; 224 A |= X;
167 continue; 225 continue;
168 case BPF_S_ALU_OR_K: 226 case BPF_S_ALU_OR_K:
169 A |= f_k; 227 A |= K;
170 continue; 228 continue;
171 case BPF_S_ALU_LSH_X: 229 case BPF_S_ALU_LSH_X:
172 A <<= X; 230 A <<= X;
173 continue; 231 continue;
174 case BPF_S_ALU_LSH_K: 232 case BPF_S_ALU_LSH_K:
175 A <<= f_k; 233 A <<= K;
176 continue; 234 continue;
177 case BPF_S_ALU_RSH_X: 235 case BPF_S_ALU_RSH_X:
178 A >>= X; 236 A >>= X;
179 continue; 237 continue;
180 case BPF_S_ALU_RSH_K: 238 case BPF_S_ALU_RSH_K:
181 A >>= f_k; 239 A >>= K;
182 continue; 240 continue;
183 case BPF_S_ALU_NEG: 241 case BPF_S_ALU_NEG:
184 A = -A; 242 A = -A;
185 continue; 243 continue;
186 case BPF_S_JMP_JA: 244 case BPF_S_JMP_JA:
187 pc += f_k; 245 fentry += K;
188 continue; 246 continue;
189 case BPF_S_JMP_JGT_K: 247 case BPF_S_JMP_JGT_K:
190 pc += (A > f_k) ? fentry->jt : fentry->jf; 248 fentry += (A > K) ? fentry->jt : fentry->jf;
191 continue; 249 continue;
192 case BPF_S_JMP_JGE_K: 250 case BPF_S_JMP_JGE_K:
193 pc += (A >= f_k) ? fentry->jt : fentry->jf; 251 fentry += (A >= K) ? fentry->jt : fentry->jf;
194 continue; 252 continue;
195 case BPF_S_JMP_JEQ_K: 253 case BPF_S_JMP_JEQ_K:
196 pc += (A == f_k) ? fentry->jt : fentry->jf; 254 fentry += (A == K) ? fentry->jt : fentry->jf;
197 continue; 255 continue;
198 case BPF_S_JMP_JSET_K: 256 case BPF_S_JMP_JSET_K:
199 pc += (A & f_k) ? fentry->jt : fentry->jf; 257 fentry += (A & K) ? fentry->jt : fentry->jf;
200 continue; 258 continue;
201 case BPF_S_JMP_JGT_X: 259 case BPF_S_JMP_JGT_X:
202 pc += (A > X) ? fentry->jt : fentry->jf; 260 fentry += (A > X) ? fentry->jt : fentry->jf;
203 continue; 261 continue;
204 case BPF_S_JMP_JGE_X: 262 case BPF_S_JMP_JGE_X:
205 pc += (A >= X) ? fentry->jt : fentry->jf; 263 fentry += (A >= X) ? fentry->jt : fentry->jf;
206 continue; 264 continue;
207 case BPF_S_JMP_JEQ_X: 265 case BPF_S_JMP_JEQ_X:
208 pc += (A == X) ? fentry->jt : fentry->jf; 266 fentry += (A == X) ? fentry->jt : fentry->jf;
209 continue; 267 continue;
210 case BPF_S_JMP_JSET_X: 268 case BPF_S_JMP_JSET_X:
211 pc += (A & X) ? fentry->jt : fentry->jf; 269 fentry += (A & X) ? fentry->jt : fentry->jf;
212 continue; 270 continue;
213 case BPF_S_LD_W_ABS: 271 case BPF_S_LD_W_ABS:
214 k = f_k; 272 k = K;
215load_w: 273load_w:
216 ptr = load_pointer(skb, k, 4, &tmp); 274 ptr = load_pointer(skb, k, 4, &tmp);
217 if (ptr != NULL) { 275 if (ptr != NULL) {
218 A = get_unaligned_be32(ptr); 276 A = get_unaligned_be32(ptr);
219 continue; 277 continue;
220 } 278 }
221 break; 279 return 0;
222 case BPF_S_LD_H_ABS: 280 case BPF_S_LD_H_ABS:
223 k = f_k; 281 k = K;
224load_h: 282load_h:
225 ptr = load_pointer(skb, k, 2, &tmp); 283 ptr = load_pointer(skb, k, 2, &tmp);
226 if (ptr != NULL) { 284 if (ptr != NULL) {
227 A = get_unaligned_be16(ptr); 285 A = get_unaligned_be16(ptr);
228 continue; 286 continue;
229 } 287 }
230 break; 288 return 0;
231 case BPF_S_LD_B_ABS: 289 case BPF_S_LD_B_ABS:
232 k = f_k; 290 k = K;
233load_b: 291load_b:
234 ptr = load_pointer(skb, k, 1, &tmp); 292 ptr = load_pointer(skb, k, 1, &tmp);
235 if (ptr != NULL) { 293 if (ptr != NULL) {
236 A = *(u8 *)ptr; 294 A = *(u8 *)ptr;
237 continue; 295 continue;
238 } 296 }
239 break; 297 return 0;
240 case BPF_S_LD_W_LEN: 298 case BPF_S_LD_W_LEN:
241 A = skb->len; 299 A = skb->len;
242 continue; 300 continue;
@@ -244,34 +302,32 @@ load_b:
244 X = skb->len; 302 X = skb->len;
245 continue; 303 continue;
246 case BPF_S_LD_W_IND: 304 case BPF_S_LD_W_IND:
247 k = X + f_k; 305 k = X + K;
248 goto load_w; 306 goto load_w;
249 case BPF_S_LD_H_IND: 307 case BPF_S_LD_H_IND:
250 k = X + f_k; 308 k = X + K;
251 goto load_h; 309 goto load_h;
252 case BPF_S_LD_B_IND: 310 case BPF_S_LD_B_IND:
253 k = X + f_k; 311 k = X + K;
254 goto load_b; 312 goto load_b;
255 case BPF_S_LDX_B_MSH: 313 case BPF_S_LDX_B_MSH:
256 ptr = load_pointer(skb, f_k, 1, &tmp); 314 ptr = load_pointer(skb, K, 1, &tmp);
257 if (ptr != NULL) { 315 if (ptr != NULL) {
258 X = (*(u8 *)ptr & 0xf) << 2; 316 X = (*(u8 *)ptr & 0xf) << 2;
259 continue; 317 continue;
260 } 318 }
261 return 0; 319 return 0;
262 case BPF_S_LD_IMM: 320 case BPF_S_LD_IMM:
263 A = f_k; 321 A = K;
264 continue; 322 continue;
265 case BPF_S_LDX_IMM: 323 case BPF_S_LDX_IMM:
266 X = f_k; 324 X = K;
267 continue; 325 continue;
268 case BPF_S_LD_MEM: 326 case BPF_S_LD_MEM:
269 A = (memvalid & (1UL << f_k)) ? 327 A = mem[K];
270 mem[f_k] : 0;
271 continue; 328 continue;
272 case BPF_S_LDX_MEM: 329 case BPF_S_LDX_MEM:
273 X = (memvalid & (1UL << f_k)) ? 330 X = mem[K];
274 mem[f_k] : 0;
275 continue; 331 continue;
276 case BPF_S_MISC_TAX: 332 case BPF_S_MISC_TAX:
277 X = A; 333 X = A;
@@ -280,50 +336,44 @@ load_b:
280 A = X; 336 A = X;
281 continue; 337 continue;
282 case BPF_S_RET_K: 338 case BPF_S_RET_K:
283 return f_k; 339 return K;
284 case BPF_S_RET_A: 340 case BPF_S_RET_A:
285 return A; 341 return A;
286 case BPF_S_ST: 342 case BPF_S_ST:
287 memvalid |= 1UL << f_k; 343 mem[K] = A;
288 mem[f_k] = A;
289 continue; 344 continue;
290 case BPF_S_STX: 345 case BPF_S_STX:
291 memvalid |= 1UL << f_k; 346 mem[K] = X;
292 mem[f_k] = X;
293 continue; 347 continue;
294 default: 348 case BPF_S_ANC_PROTOCOL:
295 WARN_ON(1);
296 return 0;
297 }
298
299 /*
300 * Handle ancillary data, which are impossible
301 * (or very difficult) to get parsing packet contents.
302 */
303 switch (k-SKF_AD_OFF) {
304 case SKF_AD_PROTOCOL:
305 A = ntohs(skb->protocol); 349 A = ntohs(skb->protocol);
306 continue; 350 continue;
307 case SKF_AD_PKTTYPE: 351 case BPF_S_ANC_PKTTYPE:
308 A = skb->pkt_type; 352 A = skb->pkt_type;
309 continue; 353 continue;
310 case SKF_AD_IFINDEX: 354 case BPF_S_ANC_IFINDEX:
311 if (!skb->dev) 355 if (!skb->dev)
312 return 0; 356 return 0;
313 A = skb->dev->ifindex; 357 A = skb->dev->ifindex;
314 continue; 358 continue;
315 case SKF_AD_MARK: 359 case BPF_S_ANC_MARK:
316 A = skb->mark; 360 A = skb->mark;
317 continue; 361 continue;
318 case SKF_AD_QUEUE: 362 case BPF_S_ANC_QUEUE:
319 A = skb->queue_mapping; 363 A = skb->queue_mapping;
320 continue; 364 continue;
321 case SKF_AD_HATYPE: 365 case BPF_S_ANC_HATYPE:
322 if (!skb->dev) 366 if (!skb->dev)
323 return 0; 367 return 0;
324 A = skb->dev->type; 368 A = skb->dev->type;
325 continue; 369 continue;
326 case SKF_AD_NLATTR: { 370 case BPF_S_ANC_RXHASH:
371 A = skb->rxhash;
372 continue;
373 case BPF_S_ANC_CPU:
374 A = raw_smp_processor_id();
375 continue;
376 case BPF_S_ANC_NLATTR: {
327 struct nlattr *nla; 377 struct nlattr *nla;
328 378
329 if (skb_is_nonlinear(skb)) 379 if (skb_is_nonlinear(skb))
@@ -339,7 +389,7 @@ load_b:
339 A = 0; 389 A = 0;
340 continue; 390 continue;
341 } 391 }
342 case SKF_AD_NLATTR_NEST: { 392 case BPF_S_ANC_NLATTR_NEST: {
343 struct nlattr *nla; 393 struct nlattr *nla;
344 394
345 if (skb_is_nonlinear(skb)) 395 if (skb_is_nonlinear(skb))
@@ -359,6 +409,7 @@ load_b:
359 continue; 409 continue;
360 } 410 }
361 default: 411 default:
412 WARN_ON(1);
362 return 0; 413 return 0;
363 } 414 }
364 } 415 }
@@ -367,6 +418,66 @@ load_b:
367} 418}
368EXPORT_SYMBOL(sk_run_filter); 419EXPORT_SYMBOL(sk_run_filter);
369 420
421/*
422 * Security :
423 * A BPF program is able to use 16 cells of memory to store intermediate
424 * values (check u32 mem[BPF_MEMWORDS] in sk_run_filter())
425 * As we dont want to clear mem[] array for each packet going through
426 * sk_run_filter(), we check that filter loaded by user never try to read
427 * a cell if not previously written, and we check all branches to be sure
428 * a malicious user doesnt try to abuse us.
429 */
430static int check_load_and_stores(struct sock_filter *filter, int flen)
431{
432 u16 *masks, memvalid = 0; /* one bit per cell, 16 cells */
433 int pc, ret = 0;
434
435 BUILD_BUG_ON(BPF_MEMWORDS > 16);
436 masks = kmalloc(flen * sizeof(*masks), GFP_KERNEL);
437 if (!masks)
438 return -ENOMEM;
439 memset(masks, 0xff, flen * sizeof(*masks));
440
441 for (pc = 0; pc < flen; pc++) {
442 memvalid &= masks[pc];
443
444 switch (filter[pc].code) {
445 case BPF_S_ST:
446 case BPF_S_STX:
447 memvalid |= (1 << filter[pc].k);
448 break;
449 case BPF_S_LD_MEM:
450 case BPF_S_LDX_MEM:
451 if (!(memvalid & (1 << filter[pc].k))) {
452 ret = -EINVAL;
453 goto error;
454 }
455 break;
456 case BPF_S_JMP_JA:
457 /* a jump must set masks on target */
458 masks[pc + 1 + filter[pc].k] &= memvalid;
459 memvalid = ~0;
460 break;
461 case BPF_S_JMP_JEQ_K:
462 case BPF_S_JMP_JEQ_X:
463 case BPF_S_JMP_JGE_K:
464 case BPF_S_JMP_JGE_X:
465 case BPF_S_JMP_JGT_K:
466 case BPF_S_JMP_JGT_X:
467 case BPF_S_JMP_JSET_X:
468 case BPF_S_JMP_JSET_K:
469 /* a jump must set masks on targets */
470 masks[pc + 1 + filter[pc].jt] &= memvalid;
471 masks[pc + 1 + filter[pc].jf] &= memvalid;
472 memvalid = ~0;
473 break;
474 }
475 }
476error:
477 kfree(masks);
478 return ret;
479}
480
370/** 481/**
371 * sk_chk_filter - verify socket filter code 482 * sk_chk_filter - verify socket filter code
372 * @filter: filter to verify 483 * @filter: filter to verify
@@ -383,7 +494,57 @@ EXPORT_SYMBOL(sk_run_filter);
383 */ 494 */
384int sk_chk_filter(struct sock_filter *filter, int flen) 495int sk_chk_filter(struct sock_filter *filter, int flen)
385{ 496{
386 struct sock_filter *ftest; 497 /*
498 * Valid instructions are initialized to non-0.
499 * Invalid instructions are initialized to 0.
500 */
501 static const u8 codes[] = {
502 [BPF_ALU|BPF_ADD|BPF_K] = BPF_S_ALU_ADD_K,
503 [BPF_ALU|BPF_ADD|BPF_X] = BPF_S_ALU_ADD_X,
504 [BPF_ALU|BPF_SUB|BPF_K] = BPF_S_ALU_SUB_K,
505 [BPF_ALU|BPF_SUB|BPF_X] = BPF_S_ALU_SUB_X,
506 [BPF_ALU|BPF_MUL|BPF_K] = BPF_S_ALU_MUL_K,
507 [BPF_ALU|BPF_MUL|BPF_X] = BPF_S_ALU_MUL_X,
508 [BPF_ALU|BPF_DIV|BPF_X] = BPF_S_ALU_DIV_X,
509 [BPF_ALU|BPF_AND|BPF_K] = BPF_S_ALU_AND_K,
510 [BPF_ALU|BPF_AND|BPF_X] = BPF_S_ALU_AND_X,
511 [BPF_ALU|BPF_OR|BPF_K] = BPF_S_ALU_OR_K,
512 [BPF_ALU|BPF_OR|BPF_X] = BPF_S_ALU_OR_X,
513 [BPF_ALU|BPF_LSH|BPF_K] = BPF_S_ALU_LSH_K,
514 [BPF_ALU|BPF_LSH|BPF_X] = BPF_S_ALU_LSH_X,
515 [BPF_ALU|BPF_RSH|BPF_K] = BPF_S_ALU_RSH_K,
516 [BPF_ALU|BPF_RSH|BPF_X] = BPF_S_ALU_RSH_X,
517 [BPF_ALU|BPF_NEG] = BPF_S_ALU_NEG,
518 [BPF_LD|BPF_W|BPF_ABS] = BPF_S_LD_W_ABS,
519 [BPF_LD|BPF_H|BPF_ABS] = BPF_S_LD_H_ABS,
520 [BPF_LD|BPF_B|BPF_ABS] = BPF_S_LD_B_ABS,
521 [BPF_LD|BPF_W|BPF_LEN] = BPF_S_LD_W_LEN,
522 [BPF_LD|BPF_W|BPF_IND] = BPF_S_LD_W_IND,
523 [BPF_LD|BPF_H|BPF_IND] = BPF_S_LD_H_IND,
524 [BPF_LD|BPF_B|BPF_IND] = BPF_S_LD_B_IND,
525 [BPF_LD|BPF_IMM] = BPF_S_LD_IMM,
526 [BPF_LDX|BPF_W|BPF_LEN] = BPF_S_LDX_W_LEN,
527 [BPF_LDX|BPF_B|BPF_MSH] = BPF_S_LDX_B_MSH,
528 [BPF_LDX|BPF_IMM] = BPF_S_LDX_IMM,
529 [BPF_MISC|BPF_TAX] = BPF_S_MISC_TAX,
530 [BPF_MISC|BPF_TXA] = BPF_S_MISC_TXA,
531 [BPF_RET|BPF_K] = BPF_S_RET_K,
532 [BPF_RET|BPF_A] = BPF_S_RET_A,
533 [BPF_ALU|BPF_DIV|BPF_K] = BPF_S_ALU_DIV_K,
534 [BPF_LD|BPF_MEM] = BPF_S_LD_MEM,
535 [BPF_LDX|BPF_MEM] = BPF_S_LDX_MEM,
536 [BPF_ST] = BPF_S_ST,
537 [BPF_STX] = BPF_S_STX,
538 [BPF_JMP|BPF_JA] = BPF_S_JMP_JA,
539 [BPF_JMP|BPF_JEQ|BPF_K] = BPF_S_JMP_JEQ_K,
540 [BPF_JMP|BPF_JEQ|BPF_X] = BPF_S_JMP_JEQ_X,
541 [BPF_JMP|BPF_JGE|BPF_K] = BPF_S_JMP_JGE_K,
542 [BPF_JMP|BPF_JGE|BPF_X] = BPF_S_JMP_JGE_X,
543 [BPF_JMP|BPF_JGT|BPF_K] = BPF_S_JMP_JGT_K,
544 [BPF_JMP|BPF_JGT|BPF_X] = BPF_S_JMP_JGT_X,
545 [BPF_JMP|BPF_JSET|BPF_K] = BPF_S_JMP_JSET_K,
546 [BPF_JMP|BPF_JSET|BPF_X] = BPF_S_JMP_JSET_X,
547 };
387 int pc; 548 int pc;
388 549
389 if (flen == 0 || flen > BPF_MAXINSNS) 550 if (flen == 0 || flen > BPF_MAXINSNS)
@@ -391,136 +552,31 @@ int sk_chk_filter(struct sock_filter *filter, int flen)
391 552
392 /* check the filter code now */ 553 /* check the filter code now */
393 for (pc = 0; pc < flen; pc++) { 554 for (pc = 0; pc < flen; pc++) {
394 ftest = &filter[pc]; 555 struct sock_filter *ftest = &filter[pc];
395 556 u16 code = ftest->code;
396 /* Only allow valid instructions */
397 switch (ftest->code) {
398 case BPF_ALU|BPF_ADD|BPF_K:
399 ftest->code = BPF_S_ALU_ADD_K;
400 break;
401 case BPF_ALU|BPF_ADD|BPF_X:
402 ftest->code = BPF_S_ALU_ADD_X;
403 break;
404 case BPF_ALU|BPF_SUB|BPF_K:
405 ftest->code = BPF_S_ALU_SUB_K;
406 break;
407 case BPF_ALU|BPF_SUB|BPF_X:
408 ftest->code = BPF_S_ALU_SUB_X;
409 break;
410 case BPF_ALU|BPF_MUL|BPF_K:
411 ftest->code = BPF_S_ALU_MUL_K;
412 break;
413 case BPF_ALU|BPF_MUL|BPF_X:
414 ftest->code = BPF_S_ALU_MUL_X;
415 break;
416 case BPF_ALU|BPF_DIV|BPF_X:
417 ftest->code = BPF_S_ALU_DIV_X;
418 break;
419 case BPF_ALU|BPF_AND|BPF_K:
420 ftest->code = BPF_S_ALU_AND_K;
421 break;
422 case BPF_ALU|BPF_AND|BPF_X:
423 ftest->code = BPF_S_ALU_AND_X;
424 break;
425 case BPF_ALU|BPF_OR|BPF_K:
426 ftest->code = BPF_S_ALU_OR_K;
427 break;
428 case BPF_ALU|BPF_OR|BPF_X:
429 ftest->code = BPF_S_ALU_OR_X;
430 break;
431 case BPF_ALU|BPF_LSH|BPF_K:
432 ftest->code = BPF_S_ALU_LSH_K;
433 break;
434 case BPF_ALU|BPF_LSH|BPF_X:
435 ftest->code = BPF_S_ALU_LSH_X;
436 break;
437 case BPF_ALU|BPF_RSH|BPF_K:
438 ftest->code = BPF_S_ALU_RSH_K;
439 break;
440 case BPF_ALU|BPF_RSH|BPF_X:
441 ftest->code = BPF_S_ALU_RSH_X;
442 break;
443 case BPF_ALU|BPF_NEG:
444 ftest->code = BPF_S_ALU_NEG;
445 break;
446 case BPF_LD|BPF_W|BPF_ABS:
447 ftest->code = BPF_S_LD_W_ABS;
448 break;
449 case BPF_LD|BPF_H|BPF_ABS:
450 ftest->code = BPF_S_LD_H_ABS;
451 break;
452 case BPF_LD|BPF_B|BPF_ABS:
453 ftest->code = BPF_S_LD_B_ABS;
454 break;
455 case BPF_LD|BPF_W|BPF_LEN:
456 ftest->code = BPF_S_LD_W_LEN;
457 break;
458 case BPF_LD|BPF_W|BPF_IND:
459 ftest->code = BPF_S_LD_W_IND;
460 break;
461 case BPF_LD|BPF_H|BPF_IND:
462 ftest->code = BPF_S_LD_H_IND;
463 break;
464 case BPF_LD|BPF_B|BPF_IND:
465 ftest->code = BPF_S_LD_B_IND;
466 break;
467 case BPF_LD|BPF_IMM:
468 ftest->code = BPF_S_LD_IMM;
469 break;
470 case BPF_LDX|BPF_W|BPF_LEN:
471 ftest->code = BPF_S_LDX_W_LEN;
472 break;
473 case BPF_LDX|BPF_B|BPF_MSH:
474 ftest->code = BPF_S_LDX_B_MSH;
475 break;
476 case BPF_LDX|BPF_IMM:
477 ftest->code = BPF_S_LDX_IMM;
478 break;
479 case BPF_MISC|BPF_TAX:
480 ftest->code = BPF_S_MISC_TAX;
481 break;
482 case BPF_MISC|BPF_TXA:
483 ftest->code = BPF_S_MISC_TXA;
484 break;
485 case BPF_RET|BPF_K:
486 ftest->code = BPF_S_RET_K;
487 break;
488 case BPF_RET|BPF_A:
489 ftest->code = BPF_S_RET_A;
490 break;
491 557
558 if (code >= ARRAY_SIZE(codes))
559 return -EINVAL;
560 code = codes[code];
561 if (!code)
562 return -EINVAL;
492 /* Some instructions need special checks */ 563 /* Some instructions need special checks */
493 564 switch (code) {
565 case BPF_S_ALU_DIV_K:
494 /* check for division by zero */ 566 /* check for division by zero */
495 case BPF_ALU|BPF_DIV|BPF_K:
496 if (ftest->k == 0) 567 if (ftest->k == 0)
497 return -EINVAL; 568 return -EINVAL;
498 ftest->code = BPF_S_ALU_DIV_K; 569 ftest->k = reciprocal_value(ftest->k);
499 break;
500
501 /* check for invalid memory addresses */
502 case BPF_LD|BPF_MEM:
503 if (ftest->k >= BPF_MEMWORDS)
504 return -EINVAL;
505 ftest->code = BPF_S_LD_MEM;
506 break;
507 case BPF_LDX|BPF_MEM:
508 if (ftest->k >= BPF_MEMWORDS)
509 return -EINVAL;
510 ftest->code = BPF_S_LDX_MEM;
511 break;
512 case BPF_ST:
513 if (ftest->k >= BPF_MEMWORDS)
514 return -EINVAL;
515 ftest->code = BPF_S_ST;
516 break; 570 break;
517 case BPF_STX: 571 case BPF_S_LD_MEM:
572 case BPF_S_LDX_MEM:
573 case BPF_S_ST:
574 case BPF_S_STX:
575 /* check for invalid memory addresses */
518 if (ftest->k >= BPF_MEMWORDS) 576 if (ftest->k >= BPF_MEMWORDS)
519 return -EINVAL; 577 return -EINVAL;
520 ftest->code = BPF_S_STX;
521 break; 578 break;
522 579 case BPF_S_JMP_JA:
523 case BPF_JMP|BPF_JA:
524 /* 580 /*
525 * Note, the large ftest->k might cause loops. 581 * Note, the large ftest->k might cause loops.
526 * Compare this with conditional jumps below, 582 * Compare this with conditional jumps below,
@@ -528,40 +584,7 @@ int sk_chk_filter(struct sock_filter *filter, int flen)
528 */ 584 */
529 if (ftest->k >= (unsigned)(flen-pc-1)) 585 if (ftest->k >= (unsigned)(flen-pc-1))
530 return -EINVAL; 586 return -EINVAL;
531 ftest->code = BPF_S_JMP_JA;
532 break;
533
534 case BPF_JMP|BPF_JEQ|BPF_K:
535 ftest->code = BPF_S_JMP_JEQ_K;
536 break;
537 case BPF_JMP|BPF_JEQ|BPF_X:
538 ftest->code = BPF_S_JMP_JEQ_X;
539 break;
540 case BPF_JMP|BPF_JGE|BPF_K:
541 ftest->code = BPF_S_JMP_JGE_K;
542 break;
543 case BPF_JMP|BPF_JGE|BPF_X:
544 ftest->code = BPF_S_JMP_JGE_X;
545 break;
546 case BPF_JMP|BPF_JGT|BPF_K:
547 ftest->code = BPF_S_JMP_JGT_K;
548 break;
549 case BPF_JMP|BPF_JGT|BPF_X:
550 ftest->code = BPF_S_JMP_JGT_X;
551 break;
552 case BPF_JMP|BPF_JSET|BPF_K:
553 ftest->code = BPF_S_JMP_JSET_K;
554 break;
555 case BPF_JMP|BPF_JSET|BPF_X:
556 ftest->code = BPF_S_JMP_JSET_X;
557 break; 587 break;
558
559 default:
560 return -EINVAL;
561 }
562
563 /* for conditionals both must be safe */
564 switch (ftest->code) {
565 case BPF_S_JMP_JEQ_K: 588 case BPF_S_JMP_JEQ_K:
566 case BPF_S_JMP_JEQ_X: 589 case BPF_S_JMP_JEQ_X:
567 case BPF_S_JMP_JGE_K: 590 case BPF_S_JMP_JGE_K:
@@ -570,21 +593,40 @@ int sk_chk_filter(struct sock_filter *filter, int flen)
570 case BPF_S_JMP_JGT_X: 593 case BPF_S_JMP_JGT_X:
571 case BPF_S_JMP_JSET_X: 594 case BPF_S_JMP_JSET_X:
572 case BPF_S_JMP_JSET_K: 595 case BPF_S_JMP_JSET_K:
596 /* for conditionals both must be safe */
573 if (pc + ftest->jt + 1 >= flen || 597 if (pc + ftest->jt + 1 >= flen ||
574 pc + ftest->jf + 1 >= flen) 598 pc + ftest->jf + 1 >= flen)
575 return -EINVAL; 599 return -EINVAL;
600 break;
601 case BPF_S_LD_W_ABS:
602 case BPF_S_LD_H_ABS:
603 case BPF_S_LD_B_ABS:
604#define ANCILLARY(CODE) case SKF_AD_OFF + SKF_AD_##CODE: \
605 code = BPF_S_ANC_##CODE; \
606 break
607 switch (ftest->k) {
608 ANCILLARY(PROTOCOL);
609 ANCILLARY(PKTTYPE);
610 ANCILLARY(IFINDEX);
611 ANCILLARY(NLATTR);
612 ANCILLARY(NLATTR_NEST);
613 ANCILLARY(MARK);
614 ANCILLARY(QUEUE);
615 ANCILLARY(HATYPE);
616 ANCILLARY(RXHASH);
617 ANCILLARY(CPU);
618 }
576 } 619 }
620 ftest->code = code;
577 } 621 }
578 622
579 /* last instruction must be a RET code */ 623 /* last instruction must be a RET code */
580 switch (filter[flen - 1].code) { 624 switch (filter[flen - 1].code) {
581 case BPF_S_RET_K: 625 case BPF_S_RET_K:
582 case BPF_S_RET_A: 626 case BPF_S_RET_A:
583 return 0; 627 return check_load_and_stores(filter, flen);
584 break; 628 }
585 default: 629 return -EINVAL;
586 return -EINVAL;
587 }
588} 630}
589EXPORT_SYMBOL(sk_chk_filter); 631EXPORT_SYMBOL(sk_chk_filter);
590 632
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 8cc8f9a79db..60a90291342 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -41,7 +41,6 @@
41 41
42#define NEIGH_PRINTK(x...) printk(x) 42#define NEIGH_PRINTK(x...) printk(x)
43#define NEIGH_NOPRINTK(x...) do { ; } while(0) 43#define NEIGH_NOPRINTK(x...) do { ; } while(0)
44#define NEIGH_PRINTK0 NEIGH_PRINTK
45#define NEIGH_PRINTK1 NEIGH_NOPRINTK 44#define NEIGH_PRINTK1 NEIGH_NOPRINTK
46#define NEIGH_PRINTK2 NEIGH_NOPRINTK 45#define NEIGH_PRINTK2 NEIGH_NOPRINTK
47 46
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index 7f902cad10f..e23c01be5a5 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -706,7 +706,6 @@ static struct attribute *rx_queue_default_attrs[] = {
706static void rx_queue_release(struct kobject *kobj) 706static void rx_queue_release(struct kobject *kobj)
707{ 707{
708 struct netdev_rx_queue *queue = to_rx_queue(kobj); 708 struct netdev_rx_queue *queue = to_rx_queue(kobj);
709 struct netdev_rx_queue *first = queue->first;
710 struct rps_map *map; 709 struct rps_map *map;
711 struct rps_dev_flow_table *flow_table; 710 struct rps_dev_flow_table *flow_table;
712 711
@@ -723,10 +722,8 @@ static void rx_queue_release(struct kobject *kobj)
723 call_rcu(&flow_table->rcu, rps_dev_flow_table_release); 722 call_rcu(&flow_table->rcu, rps_dev_flow_table_release);
724 } 723 }
725 724
726 if (atomic_dec_and_test(&first->count)) 725 memset(kobj, 0, sizeof(*kobj));
727 kfree(first); 726 dev_put(queue->dev);
728 else
729 memset(kobj, 0, sizeof(*kobj));
730} 727}
731 728
732static struct kobj_type rx_queue_ktype = { 729static struct kobj_type rx_queue_ktype = {
@@ -738,7 +735,6 @@ static struct kobj_type rx_queue_ktype = {
738static int rx_queue_add_kobject(struct net_device *net, int index) 735static int rx_queue_add_kobject(struct net_device *net, int index)
739{ 736{
740 struct netdev_rx_queue *queue = net->_rx + index; 737 struct netdev_rx_queue *queue = net->_rx + index;
741 struct netdev_rx_queue *first = queue->first;
742 struct kobject *kobj = &queue->kobj; 738 struct kobject *kobj = &queue->kobj;
743 int error = 0; 739 int error = 0;
744 740
@@ -751,14 +747,16 @@ static int rx_queue_add_kobject(struct net_device *net, int index)
751 } 747 }
752 748
753 kobject_uevent(kobj, KOBJ_ADD); 749 kobject_uevent(kobj, KOBJ_ADD);
754 atomic_inc(&first->count); 750 dev_hold(queue->dev);
755 751
756 return error; 752 return error;
757} 753}
754#endif /* CONFIG_RPS */
758 755
759int 756int
760net_rx_queue_update_kobjects(struct net_device *net, int old_num, int new_num) 757net_rx_queue_update_kobjects(struct net_device *net, int old_num, int new_num)
761{ 758{
759#ifdef CONFIG_RPS
762 int i; 760 int i;
763 int error = 0; 761 int error = 0;
764 762
@@ -774,23 +772,423 @@ net_rx_queue_update_kobjects(struct net_device *net, int old_num, int new_num)
774 kobject_put(&net->_rx[i].kobj); 772 kobject_put(&net->_rx[i].kobj);
775 773
776 return error; 774 return error;
775#else
776 return 0;
777#endif
778}
779
780#ifdef CONFIG_XPS
781/*
782 * netdev_queue sysfs structures and functions.
783 */
784struct netdev_queue_attribute {
785 struct attribute attr;
786 ssize_t (*show)(struct netdev_queue *queue,
787 struct netdev_queue_attribute *attr, char *buf);
788 ssize_t (*store)(struct netdev_queue *queue,
789 struct netdev_queue_attribute *attr, const char *buf, size_t len);
790};
791#define to_netdev_queue_attr(_attr) container_of(_attr, \
792 struct netdev_queue_attribute, attr)
793
794#define to_netdev_queue(obj) container_of(obj, struct netdev_queue, kobj)
795
796static ssize_t netdev_queue_attr_show(struct kobject *kobj,
797 struct attribute *attr, char *buf)
798{
799 struct netdev_queue_attribute *attribute = to_netdev_queue_attr(attr);
800 struct netdev_queue *queue = to_netdev_queue(kobj);
801
802 if (!attribute->show)
803 return -EIO;
804
805 return attribute->show(queue, attribute, buf);
777} 806}
778 807
779static int rx_queue_register_kobjects(struct net_device *net) 808static ssize_t netdev_queue_attr_store(struct kobject *kobj,
809 struct attribute *attr,
810 const char *buf, size_t count)
780{ 811{
812 struct netdev_queue_attribute *attribute = to_netdev_queue_attr(attr);
813 struct netdev_queue *queue = to_netdev_queue(kobj);
814
815 if (!attribute->store)
816 return -EIO;
817
818 return attribute->store(queue, attribute, buf, count);
819}
820
821static const struct sysfs_ops netdev_queue_sysfs_ops = {
822 .show = netdev_queue_attr_show,
823 .store = netdev_queue_attr_store,
824};
825
826static inline unsigned int get_netdev_queue_index(struct netdev_queue *queue)
827{
828 struct net_device *dev = queue->dev;
829 int i;
830
831 for (i = 0; i < dev->num_tx_queues; i++)
832 if (queue == &dev->_tx[i])
833 break;
834
835 BUG_ON(i >= dev->num_tx_queues);
836
837 return i;
838}
839
840
841static ssize_t show_xps_map(struct netdev_queue *queue,
842 struct netdev_queue_attribute *attribute, char *buf)
843{
844 struct net_device *dev = queue->dev;
845 struct xps_dev_maps *dev_maps;
846 cpumask_var_t mask;
847 unsigned long index;
848 size_t len = 0;
849 int i;
850
851 if (!zalloc_cpumask_var(&mask, GFP_KERNEL))
852 return -ENOMEM;
853
854 index = get_netdev_queue_index(queue);
855
856 rcu_read_lock();
857 dev_maps = rcu_dereference(dev->xps_maps);
858 if (dev_maps) {
859 for_each_possible_cpu(i) {
860 struct xps_map *map =
861 rcu_dereference(dev_maps->cpu_map[i]);
862 if (map) {
863 int j;
864 for (j = 0; j < map->len; j++) {
865 if (map->queues[j] == index) {
866 cpumask_set_cpu(i, mask);
867 break;
868 }
869 }
870 }
871 }
872 }
873 rcu_read_unlock();
874
875 len += cpumask_scnprintf(buf + len, PAGE_SIZE, mask);
876 if (PAGE_SIZE - len < 3) {
877 free_cpumask_var(mask);
878 return -EINVAL;
879 }
880
881 free_cpumask_var(mask);
882 len += sprintf(buf + len, "\n");
883 return len;
884}
885
886static void xps_map_release(struct rcu_head *rcu)
887{
888 struct xps_map *map = container_of(rcu, struct xps_map, rcu);
889
890 kfree(map);
891}
892
893static void xps_dev_maps_release(struct rcu_head *rcu)
894{
895 struct xps_dev_maps *dev_maps =
896 container_of(rcu, struct xps_dev_maps, rcu);
897
898 kfree(dev_maps);
899}
900
901static DEFINE_MUTEX(xps_map_mutex);
902#define xmap_dereference(P) \
903 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
904
905static ssize_t store_xps_map(struct netdev_queue *queue,
906 struct netdev_queue_attribute *attribute,
907 const char *buf, size_t len)
908{
909 struct net_device *dev = queue->dev;
910 cpumask_var_t mask;
911 int err, i, cpu, pos, map_len, alloc_len, need_set;
912 unsigned long index;
913 struct xps_map *map, *new_map;
914 struct xps_dev_maps *dev_maps, *new_dev_maps;
915 int nonempty = 0;
916 int numa_node = -2;
917
918 if (!capable(CAP_NET_ADMIN))
919 return -EPERM;
920
921 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
922 return -ENOMEM;
923
924 index = get_netdev_queue_index(queue);
925
926 err = bitmap_parse(buf, len, cpumask_bits(mask), nr_cpumask_bits);
927 if (err) {
928 free_cpumask_var(mask);
929 return err;
930 }
931
932 new_dev_maps = kzalloc(max_t(unsigned,
933 XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES), GFP_KERNEL);
934 if (!new_dev_maps) {
935 free_cpumask_var(mask);
936 return -ENOMEM;
937 }
938
939 mutex_lock(&xps_map_mutex);
940
941 dev_maps = xmap_dereference(dev->xps_maps);
942
943 for_each_possible_cpu(cpu) {
944 map = dev_maps ?
945 xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;
946 new_map = map;
947 if (map) {
948 for (pos = 0; pos < map->len; pos++)
949 if (map->queues[pos] == index)
950 break;
951 map_len = map->len;
952 alloc_len = map->alloc_len;
953 } else
954 pos = map_len = alloc_len = 0;
955
956 need_set = cpu_isset(cpu, *mask) && cpu_online(cpu);
957#ifdef CONFIG_NUMA
958 if (need_set) {
959 if (numa_node == -2)
960 numa_node = cpu_to_node(cpu);
961 else if (numa_node != cpu_to_node(cpu))
962 numa_node = -1;
963 }
964#endif
965 if (need_set && pos >= map_len) {
966 /* Need to add queue to this CPU's map */
967 if (map_len >= alloc_len) {
968 alloc_len = alloc_len ?
969 2 * alloc_len : XPS_MIN_MAP_ALLOC;
970 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len),
971 GFP_KERNEL,
972 cpu_to_node(cpu));
973 if (!new_map)
974 goto error;
975 new_map->alloc_len = alloc_len;
976 for (i = 0; i < map_len; i++)
977 new_map->queues[i] = map->queues[i];
978 new_map->len = map_len;
979 }
980 new_map->queues[new_map->len++] = index;
981 } else if (!need_set && pos < map_len) {
982 /* Need to remove queue from this CPU's map */
983 if (map_len > 1)
984 new_map->queues[pos] =
985 new_map->queues[--new_map->len];
986 else
987 new_map = NULL;
988 }
989 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], new_map);
990 }
991
992 /* Cleanup old maps */
993 for_each_possible_cpu(cpu) {
994 map = dev_maps ?
995 xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;
996 if (map && xmap_dereference(new_dev_maps->cpu_map[cpu]) != map)
997 call_rcu(&map->rcu, xps_map_release);
998 if (new_dev_maps->cpu_map[cpu])
999 nonempty = 1;
1000 }
1001
1002 if (nonempty)
1003 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
1004 else {
1005 kfree(new_dev_maps);
1006 rcu_assign_pointer(dev->xps_maps, NULL);
1007 }
1008
1009 if (dev_maps)
1010 call_rcu(&dev_maps->rcu, xps_dev_maps_release);
1011
1012 netdev_queue_numa_node_write(queue, (numa_node >= 0) ? numa_node :
1013 NUMA_NO_NODE);
1014
1015 mutex_unlock(&xps_map_mutex);
1016
1017 free_cpumask_var(mask);
1018 return len;
1019
1020error:
1021 mutex_unlock(&xps_map_mutex);
1022
1023 if (new_dev_maps)
1024 for_each_possible_cpu(i)
1025 kfree(rcu_dereference_protected(
1026 new_dev_maps->cpu_map[i],
1027 1));
1028 kfree(new_dev_maps);
1029 free_cpumask_var(mask);
1030 return -ENOMEM;
1031}
1032
1033static struct netdev_queue_attribute xps_cpus_attribute =
1034 __ATTR(xps_cpus, S_IRUGO | S_IWUSR, show_xps_map, store_xps_map);
1035
1036static struct attribute *netdev_queue_default_attrs[] = {
1037 &xps_cpus_attribute.attr,
1038 NULL
1039};
1040
1041static void netdev_queue_release(struct kobject *kobj)
1042{
1043 struct netdev_queue *queue = to_netdev_queue(kobj);
1044 struct net_device *dev = queue->dev;
1045 struct xps_dev_maps *dev_maps;
1046 struct xps_map *map;
1047 unsigned long index;
1048 int i, pos, nonempty = 0;
1049
1050 index = get_netdev_queue_index(queue);
1051
1052 mutex_lock(&xps_map_mutex);
1053 dev_maps = xmap_dereference(dev->xps_maps);
1054
1055 if (dev_maps) {
1056 for_each_possible_cpu(i) {
1057 map = xmap_dereference(dev_maps->cpu_map[i]);
1058 if (!map)
1059 continue;
1060
1061 for (pos = 0; pos < map->len; pos++)
1062 if (map->queues[pos] == index)
1063 break;
1064
1065 if (pos < map->len) {
1066 if (map->len > 1)
1067 map->queues[pos] =
1068 map->queues[--map->len];
1069 else {
1070 RCU_INIT_POINTER(dev_maps->cpu_map[i],
1071 NULL);
1072 call_rcu(&map->rcu, xps_map_release);
1073 map = NULL;
1074 }
1075 }
1076 if (map)
1077 nonempty = 1;
1078 }
1079
1080 if (!nonempty) {
1081 RCU_INIT_POINTER(dev->xps_maps, NULL);
1082 call_rcu(&dev_maps->rcu, xps_dev_maps_release);
1083 }
1084 }
1085
1086 mutex_unlock(&xps_map_mutex);
1087
1088 memset(kobj, 0, sizeof(*kobj));
1089 dev_put(queue->dev);
1090}
1091
1092static struct kobj_type netdev_queue_ktype = {
1093 .sysfs_ops = &netdev_queue_sysfs_ops,
1094 .release = netdev_queue_release,
1095 .default_attrs = netdev_queue_default_attrs,
1096};
1097
1098static int netdev_queue_add_kobject(struct net_device *net, int index)
1099{
1100 struct netdev_queue *queue = net->_tx + index;
1101 struct kobject *kobj = &queue->kobj;
1102 int error = 0;
1103
1104 kobj->kset = net->queues_kset;
1105 error = kobject_init_and_add(kobj, &netdev_queue_ktype, NULL,
1106 "tx-%u", index);
1107 if (error) {
1108 kobject_put(kobj);
1109 return error;
1110 }
1111
1112 kobject_uevent(kobj, KOBJ_ADD);
1113 dev_hold(queue->dev);
1114
1115 return error;
1116}
1117#endif /* CONFIG_XPS */
1118
1119int
1120netdev_queue_update_kobjects(struct net_device *net, int old_num, int new_num)
1121{
1122#ifdef CONFIG_XPS
1123 int i;
1124 int error = 0;
1125
1126 for (i = old_num; i < new_num; i++) {
1127 error = netdev_queue_add_kobject(net, i);
1128 if (error) {
1129 new_num = old_num;
1130 break;
1131 }
1132 }
1133
1134 while (--i >= new_num)
1135 kobject_put(&net->_tx[i].kobj);
1136
1137 return error;
1138#else
1139 return 0;
1140#endif
1141}
1142
1143static int register_queue_kobjects(struct net_device *net)
1144{
1145 int error = 0, txq = 0, rxq = 0, real_rx = 0, real_tx = 0;
1146
1147#if defined(CONFIG_RPS) || defined(CONFIG_XPS)
781 net->queues_kset = kset_create_and_add("queues", 1148 net->queues_kset = kset_create_and_add("queues",
782 NULL, &net->dev.kobj); 1149 NULL, &net->dev.kobj);
783 if (!net->queues_kset) 1150 if (!net->queues_kset)
784 return -ENOMEM; 1151 return -ENOMEM;
785 return net_rx_queue_update_kobjects(net, 0, net->real_num_rx_queues); 1152#endif
1153
1154#ifdef CONFIG_RPS
1155 real_rx = net->real_num_rx_queues;
1156#endif
1157 real_tx = net->real_num_tx_queues;
1158
1159 error = net_rx_queue_update_kobjects(net, 0, real_rx);
1160 if (error)
1161 goto error;
1162 rxq = real_rx;
1163
1164 error = netdev_queue_update_kobjects(net, 0, real_tx);
1165 if (error)
1166 goto error;
1167 txq = real_tx;
1168
1169 return 0;
1170
1171error:
1172 netdev_queue_update_kobjects(net, txq, 0);
1173 net_rx_queue_update_kobjects(net, rxq, 0);
1174 return error;
786} 1175}
787 1176
788static void rx_queue_remove_kobjects(struct net_device *net) 1177static void remove_queue_kobjects(struct net_device *net)
789{ 1178{
790 net_rx_queue_update_kobjects(net, net->real_num_rx_queues, 0); 1179 int real_rx = 0, real_tx = 0;
1180
1181#ifdef CONFIG_RPS
1182 real_rx = net->real_num_rx_queues;
1183#endif
1184 real_tx = net->real_num_tx_queues;
1185
1186 net_rx_queue_update_kobjects(net, real_rx, 0);
1187 netdev_queue_update_kobjects(net, real_tx, 0);
1188#if defined(CONFIG_RPS) || defined(CONFIG_XPS)
791 kset_unregister(net->queues_kset); 1189 kset_unregister(net->queues_kset);
1190#endif
792} 1191}
793#endif /* CONFIG_RPS */
794 1192
795static const void *net_current_ns(void) 1193static const void *net_current_ns(void)
796{ 1194{
@@ -889,9 +1287,7 @@ void netdev_unregister_kobject(struct net_device * net)
889 1287
890 kobject_get(&dev->kobj); 1288 kobject_get(&dev->kobj);
891 1289
892#ifdef CONFIG_RPS 1290 remove_queue_kobjects(net);
893 rx_queue_remove_kobjects(net);
894#endif
895 1291
896 device_del(dev); 1292 device_del(dev);
897} 1293}
@@ -930,13 +1326,11 @@ int netdev_register_kobject(struct net_device *net)
930 if (error) 1326 if (error)
931 return error; 1327 return error;
932 1328
933#ifdef CONFIG_RPS 1329 error = register_queue_kobjects(net);
934 error = rx_queue_register_kobjects(net);
935 if (error) { 1330 if (error) {
936 device_del(dev); 1331 device_del(dev);
937 return error; 1332 return error;
938 } 1333 }
939#endif
940 1334
941 return error; 1335 return error;
942} 1336}
diff --git a/net/core/net-sysfs.h b/net/core/net-sysfs.h
index 778e1571548..bd7751ec1c4 100644
--- a/net/core/net-sysfs.h
+++ b/net/core/net-sysfs.h
@@ -4,8 +4,8 @@
4int netdev_kobject_init(void); 4int netdev_kobject_init(void);
5int netdev_register_kobject(struct net_device *); 5int netdev_register_kobject(struct net_device *);
6void netdev_unregister_kobject(struct net_device *); 6void netdev_unregister_kobject(struct net_device *);
7#ifdef CONFIG_RPS
8int net_rx_queue_update_kobjects(struct net_device *, int old_num, int new_num); 7int net_rx_queue_update_kobjects(struct net_device *, int old_num, int new_num);
9#endif 8int netdev_queue_update_kobjects(struct net_device *net,
9 int old_num, int new_num);
10 10
11#endif 11#endif
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index 4e98ffac3af..72d9b50109f 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -35,7 +35,6 @@
35 35
36#define MAX_UDP_CHUNK 1460 36#define MAX_UDP_CHUNK 1460
37#define MAX_SKBS 32 37#define MAX_SKBS 32
38#define MAX_QUEUE_DEPTH (MAX_SKBS / 2)
39 38
40static struct sk_buff_head skb_pool; 39static struct sk_buff_head skb_pool;
41 40
@@ -76,8 +75,7 @@ static void queue_process(struct work_struct *work)
76 75
77 local_irq_save(flags); 76 local_irq_save(flags);
78 __netif_tx_lock(txq, smp_processor_id()); 77 __netif_tx_lock(txq, smp_processor_id());
79 if (netif_tx_queue_stopped(txq) || 78 if (netif_tx_queue_frozen_or_stopped(txq) ||
80 netif_tx_queue_frozen(txq) ||
81 ops->ndo_start_xmit(skb, dev) != NETDEV_TX_OK) { 79 ops->ndo_start_xmit(skb, dev) != NETDEV_TX_OK) {
82 skb_queue_head(&npinfo->txq, skb); 80 skb_queue_head(&npinfo->txq, skb);
83 __netif_tx_unlock(txq); 81 __netif_tx_unlock(txq);
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index 33bc3823ac6..a9e7fc4c461 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -378,6 +378,7 @@ struct pktgen_dev {
378 378
379 u16 queue_map_min; 379 u16 queue_map_min;
380 u16 queue_map_max; 380 u16 queue_map_max;
381 __u32 skb_priority; /* skb priority field */
381 int node; /* Memory node */ 382 int node; /* Memory node */
382 383
383#ifdef CONFIG_XFRM 384#ifdef CONFIG_XFRM
@@ -394,6 +395,8 @@ struct pktgen_hdr {
394 __be32 tv_usec; 395 __be32 tv_usec;
395}; 396};
396 397
398static bool pktgen_exiting __read_mostly;
399
397struct pktgen_thread { 400struct pktgen_thread {
398 spinlock_t if_lock; /* for list of devices */ 401 spinlock_t if_lock; /* for list of devices */
399 struct list_head if_list; /* All device here */ 402 struct list_head if_list; /* All device here */
@@ -547,6 +550,10 @@ static int pktgen_if_show(struct seq_file *seq, void *v)
547 pkt_dev->queue_map_min, 550 pkt_dev->queue_map_min,
548 pkt_dev->queue_map_max); 551 pkt_dev->queue_map_max);
549 552
553 if (pkt_dev->skb_priority)
554 seq_printf(seq, " skb_priority: %u\n",
555 pkt_dev->skb_priority);
556
550 if (pkt_dev->flags & F_IPV6) { 557 if (pkt_dev->flags & F_IPV6) {
551 char b1[128], b2[128], b3[128]; 558 char b1[128], b2[128], b3[128];
552 fmt_ip6(b1, pkt_dev->in6_saddr.s6_addr); 559 fmt_ip6(b1, pkt_dev->in6_saddr.s6_addr);
@@ -1711,6 +1718,18 @@ static ssize_t pktgen_if_write(struct file *file,
1711 return count; 1718 return count;
1712 } 1719 }
1713 1720
1721 if (!strcmp(name, "skb_priority")) {
1722 len = num_arg(&user_buffer[i], 9, &value);
1723 if (len < 0)
1724 return len;
1725
1726 i += len;
1727 pkt_dev->skb_priority = value;
1728 sprintf(pg_result, "OK: skb_priority=%i",
1729 pkt_dev->skb_priority);
1730 return count;
1731 }
1732
1714 sprintf(pkt_dev->result, "No such parameter \"%s\"", name); 1733 sprintf(pkt_dev->result, "No such parameter \"%s\"", name);
1715 return -EINVAL; 1734 return -EINVAL;
1716} 1735}
@@ -2641,6 +2660,7 @@ static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
2641 sprintf(pkt_dev->result, "No memory"); 2660 sprintf(pkt_dev->result, "No memory");
2642 return NULL; 2661 return NULL;
2643 } 2662 }
2663 prefetchw(skb->data);
2644 2664
2645 skb_reserve(skb, datalen); 2665 skb_reserve(skb, datalen);
2646 2666
@@ -2671,6 +2691,8 @@ static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
2671 skb->transport_header = skb->network_header + sizeof(struct iphdr); 2691 skb->transport_header = skb->network_header + sizeof(struct iphdr);
2672 skb_put(skb, sizeof(struct iphdr) + sizeof(struct udphdr)); 2692 skb_put(skb, sizeof(struct iphdr) + sizeof(struct udphdr));
2673 skb_set_queue_mapping(skb, queue_map); 2693 skb_set_queue_mapping(skb, queue_map);
2694 skb->priority = pkt_dev->skb_priority;
2695
2674 iph = ip_hdr(skb); 2696 iph = ip_hdr(skb);
2675 udph = udp_hdr(skb); 2697 udph = udp_hdr(skb);
2676 2698
@@ -2986,6 +3008,7 @@ static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
2986 sprintf(pkt_dev->result, "No memory"); 3008 sprintf(pkt_dev->result, "No memory");
2987 return NULL; 3009 return NULL;
2988 } 3010 }
3011 prefetchw(skb->data);
2989 3012
2990 skb_reserve(skb, 16); 3013 skb_reserve(skb, 16);
2991 3014
@@ -3016,6 +3039,7 @@ static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
3016 skb->transport_header = skb->network_header + sizeof(struct ipv6hdr); 3039 skb->transport_header = skb->network_header + sizeof(struct ipv6hdr);
3017 skb_put(skb, sizeof(struct ipv6hdr) + sizeof(struct udphdr)); 3040 skb_put(skb, sizeof(struct ipv6hdr) + sizeof(struct udphdr));
3018 skb_set_queue_mapping(skb, queue_map); 3041 skb_set_queue_mapping(skb, queue_map);
3042 skb->priority = pkt_dev->skb_priority;
3019 iph = ipv6_hdr(skb); 3043 iph = ipv6_hdr(skb);
3020 udph = udp_hdr(skb); 3044 udph = udp_hdr(skb);
3021 3045
@@ -3431,11 +3455,6 @@ static void pktgen_rem_thread(struct pktgen_thread *t)
3431 3455
3432 remove_proc_entry(t->tsk->comm, pg_proc_dir); 3456 remove_proc_entry(t->tsk->comm, pg_proc_dir);
3433 3457
3434 mutex_lock(&pktgen_thread_lock);
3435
3436 list_del(&t->th_list);
3437
3438 mutex_unlock(&pktgen_thread_lock);
3439} 3458}
3440 3459
3441static void pktgen_resched(struct pktgen_dev *pkt_dev) 3460static void pktgen_resched(struct pktgen_dev *pkt_dev)
@@ -3510,7 +3529,7 @@ static void pktgen_xmit(struct pktgen_dev *pkt_dev)
3510 3529
3511 __netif_tx_lock_bh(txq); 3530 __netif_tx_lock_bh(txq);
3512 3531
3513 if (unlikely(netif_tx_queue_stopped(txq) || netif_tx_queue_frozen(txq))) { 3532 if (unlikely(netif_tx_queue_frozen_or_stopped(txq))) {
3514 ret = NETDEV_TX_BUSY; 3533 ret = NETDEV_TX_BUSY;
3515 pkt_dev->last_ok = 0; 3534 pkt_dev->last_ok = 0;
3516 goto unlock; 3535 goto unlock;
@@ -3534,8 +3553,7 @@ static void pktgen_xmit(struct pktgen_dev *pkt_dev)
3534 break; 3553 break;
3535 default: /* Drivers are not supposed to return other values! */ 3554 default: /* Drivers are not supposed to return other values! */
3536 if (net_ratelimit()) 3555 if (net_ratelimit())
3537 pr_info("pktgen: %s xmit error: %d\n", 3556 pr_info("%s xmit error: %d\n", pkt_dev->odevname, ret);
3538 pkt_dev->odevname, ret);
3539 pkt_dev->errors++; 3557 pkt_dev->errors++;
3540 /* fallthru */ 3558 /* fallthru */
3541 case NETDEV_TX_LOCKED: 3559 case NETDEV_TX_LOCKED:
@@ -3582,6 +3600,8 @@ static int pktgen_thread_worker(void *arg)
3582 pkt_dev = next_to_run(t); 3600 pkt_dev = next_to_run(t);
3583 3601
3584 if (unlikely(!pkt_dev && t->control == 0)) { 3602 if (unlikely(!pkt_dev && t->control == 0)) {
3603 if (pktgen_exiting)
3604 break;
3585 wait_event_interruptible_timeout(t->queue, 3605 wait_event_interruptible_timeout(t->queue,
3586 t->control != 0, 3606 t->control != 0,
3587 HZ/10); 3607 HZ/10);
@@ -3634,6 +3654,13 @@ static int pktgen_thread_worker(void *arg)
3634 pr_debug("%s removing thread\n", t->tsk->comm); 3654 pr_debug("%s removing thread\n", t->tsk->comm);
3635 pktgen_rem_thread(t); 3655 pktgen_rem_thread(t);
3636 3656
3657 /* Wait for kthread_stop */
3658 while (!kthread_should_stop()) {
3659 set_current_state(TASK_INTERRUPTIBLE);
3660 schedule();
3661 }
3662 __set_current_state(TASK_RUNNING);
3663
3637 return 0; 3664 return 0;
3638} 3665}
3639 3666
@@ -3908,6 +3935,7 @@ static void __exit pg_cleanup(void)
3908 struct list_head *q, *n; 3935 struct list_head *q, *n;
3909 3936
3910 /* Stop all interfaces & threads */ 3937 /* Stop all interfaces & threads */
3938 pktgen_exiting = true;
3911 3939
3912 list_for_each_safe(q, n, &pktgen_threads) { 3940 list_for_each_safe(q, n, &pktgen_threads) {
3913 t = list_entry(q, struct pktgen_thread, th_list); 3941 t = list_entry(q, struct pktgen_thread, th_list);
diff --git a/net/core/request_sock.c b/net/core/request_sock.c
index fceeb37d716..182236b2510 100644
--- a/net/core/request_sock.c
+++ b/net/core/request_sock.c
@@ -33,6 +33,7 @@
33 * Note : Dont forget somaxconn that may limit backlog too. 33 * Note : Dont forget somaxconn that may limit backlog too.
34 */ 34 */
35int sysctl_max_syn_backlog = 256; 35int sysctl_max_syn_backlog = 256;
36EXPORT_SYMBOL(sysctl_max_syn_backlog);
36 37
37int reqsk_queue_alloc(struct request_sock_queue *queue, 38int reqsk_queue_alloc(struct request_sock_queue *queue,
38 unsigned int nr_table_entries) 39 unsigned int nr_table_entries)
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 841c287ef40..750db57f3bb 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -362,6 +362,95 @@ static size_t rtnl_link_get_size(const struct net_device *dev)
362 return size; 362 return size;
363} 363}
364 364
365static LIST_HEAD(rtnl_af_ops);
366
367static const struct rtnl_af_ops *rtnl_af_lookup(const int family)
368{
369 const struct rtnl_af_ops *ops;
370
371 list_for_each_entry(ops, &rtnl_af_ops, list) {
372 if (ops->family == family)
373 return ops;
374 }
375
376 return NULL;
377}
378
379/**
380 * __rtnl_af_register - Register rtnl_af_ops with rtnetlink.
381 * @ops: struct rtnl_af_ops * to register
382 *
383 * The caller must hold the rtnl_mutex.
384 *
385 * Returns 0 on success or a negative error code.
386 */
387int __rtnl_af_register(struct rtnl_af_ops *ops)
388{
389 list_add_tail(&ops->list, &rtnl_af_ops);
390 return 0;
391}
392EXPORT_SYMBOL_GPL(__rtnl_af_register);
393
394/**
395 * rtnl_af_register - Register rtnl_af_ops with rtnetlink.
396 * @ops: struct rtnl_af_ops * to register
397 *
398 * Returns 0 on success or a negative error code.
399 */
400int rtnl_af_register(struct rtnl_af_ops *ops)
401{
402 int err;
403
404 rtnl_lock();
405 err = __rtnl_af_register(ops);
406 rtnl_unlock();
407 return err;
408}
409EXPORT_SYMBOL_GPL(rtnl_af_register);
410
411/**
412 * __rtnl_af_unregister - Unregister rtnl_af_ops from rtnetlink.
413 * @ops: struct rtnl_af_ops * to unregister
414 *
415 * The caller must hold the rtnl_mutex.
416 */
417void __rtnl_af_unregister(struct rtnl_af_ops *ops)
418{
419 list_del(&ops->list);
420}
421EXPORT_SYMBOL_GPL(__rtnl_af_unregister);
422
423/**
424 * rtnl_af_unregister - Unregister rtnl_af_ops from rtnetlink.
425 * @ops: struct rtnl_af_ops * to unregister
426 */
427void rtnl_af_unregister(struct rtnl_af_ops *ops)
428{
429 rtnl_lock();
430 __rtnl_af_unregister(ops);
431 rtnl_unlock();
432}
433EXPORT_SYMBOL_GPL(rtnl_af_unregister);
434
435static size_t rtnl_link_get_af_size(const struct net_device *dev)
436{
437 struct rtnl_af_ops *af_ops;
438 size_t size;
439
440 /* IFLA_AF_SPEC */
441 size = nla_total_size(sizeof(struct nlattr));
442
443 list_for_each_entry(af_ops, &rtnl_af_ops, list) {
444 if (af_ops->get_link_af_size) {
445 /* AF_* + nested data */
446 size += nla_total_size(sizeof(struct nlattr)) +
447 af_ops->get_link_af_size(dev);
448 }
449 }
450
451 return size;
452}
453
365static int rtnl_link_fill(struct sk_buff *skb, const struct net_device *dev) 454static int rtnl_link_fill(struct sk_buff *skb, const struct net_device *dev)
366{ 455{
367 const struct rtnl_link_ops *ops = dev->rtnl_link_ops; 456 const struct rtnl_link_ops *ops = dev->rtnl_link_ops;
@@ -671,7 +760,8 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev)
671 + nla_total_size(4) /* IFLA_NUM_VF */ 760 + nla_total_size(4) /* IFLA_NUM_VF */
672 + rtnl_vfinfo_size(dev) /* IFLA_VFINFO_LIST */ 761 + rtnl_vfinfo_size(dev) /* IFLA_VFINFO_LIST */
673 + rtnl_port_size(dev) /* IFLA_VF_PORTS + IFLA_PORT_SELF */ 762 + rtnl_port_size(dev) /* IFLA_VF_PORTS + IFLA_PORT_SELF */
674 + rtnl_link_get_size(dev); /* IFLA_LINKINFO */ 763 + rtnl_link_get_size(dev) /* IFLA_LINKINFO */
764 + rtnl_link_get_af_size(dev); /* IFLA_AF_SPEC */
675} 765}
676 766
677static int rtnl_vf_ports_fill(struct sk_buff *skb, struct net_device *dev) 767static int rtnl_vf_ports_fill(struct sk_buff *skb, struct net_device *dev)
@@ -757,7 +847,8 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
757 struct nlmsghdr *nlh; 847 struct nlmsghdr *nlh;
758 struct rtnl_link_stats64 temp; 848 struct rtnl_link_stats64 temp;
759 const struct rtnl_link_stats64 *stats; 849 const struct rtnl_link_stats64 *stats;
760 struct nlattr *attr; 850 struct nlattr *attr, *af_spec;
851 struct rtnl_af_ops *af_ops;
761 852
762 nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ifm), flags); 853 nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ifm), flags);
763 if (nlh == NULL) 854 if (nlh == NULL)
@@ -866,6 +957,36 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
866 goto nla_put_failure; 957 goto nla_put_failure;
867 } 958 }
868 959
960 if (!(af_spec = nla_nest_start(skb, IFLA_AF_SPEC)))
961 goto nla_put_failure;
962
963 list_for_each_entry(af_ops, &rtnl_af_ops, list) {
964 if (af_ops->fill_link_af) {
965 struct nlattr *af;
966 int err;
967
968 if (!(af = nla_nest_start(skb, af_ops->family)))
969 goto nla_put_failure;
970
971 err = af_ops->fill_link_af(skb, dev);
972
973 /*
974 * Caller may return ENODATA to indicate that there
975 * was no data to be dumped. This is not an error, it
976 * means we should trim the attribute header and
977 * continue.
978 */
979 if (err == -ENODATA)
980 nla_nest_cancel(skb, af);
981 else if (err < 0)
982 goto nla_put_failure;
983
984 nla_nest_end(skb, af);
985 }
986 }
987
988 nla_nest_end(skb, af_spec);
989
869 return nlmsg_end(skb, nlh); 990 return nlmsg_end(skb, nlh);
870 991
871nla_put_failure: 992nla_put_failure:
@@ -924,6 +1045,7 @@ const struct nla_policy ifla_policy[IFLA_MAX+1] = {
924 [IFLA_VFINFO_LIST] = {. type = NLA_NESTED }, 1045 [IFLA_VFINFO_LIST] = {. type = NLA_NESTED },
925 [IFLA_VF_PORTS] = { .type = NLA_NESTED }, 1046 [IFLA_VF_PORTS] = { .type = NLA_NESTED },
926 [IFLA_PORT_SELF] = { .type = NLA_NESTED }, 1047 [IFLA_PORT_SELF] = { .type = NLA_NESTED },
1048 [IFLA_AF_SPEC] = { .type = NLA_NESTED },
927}; 1049};
928EXPORT_SYMBOL(ifla_policy); 1050EXPORT_SYMBOL(ifla_policy);
929 1051
@@ -985,6 +1107,28 @@ static int validate_linkmsg(struct net_device *dev, struct nlattr *tb[])
985 return -EINVAL; 1107 return -EINVAL;
986 } 1108 }
987 1109
1110 if (tb[IFLA_AF_SPEC]) {
1111 struct nlattr *af;
1112 int rem, err;
1113
1114 nla_for_each_nested(af, tb[IFLA_AF_SPEC], rem) {
1115 const struct rtnl_af_ops *af_ops;
1116
1117 if (!(af_ops = rtnl_af_lookup(nla_type(af))))
1118 return -EAFNOSUPPORT;
1119
1120 if (!af_ops->set_link_af)
1121 return -EOPNOTSUPP;
1122
1123 if (af_ops->validate_link_af) {
1124 err = af_ops->validate_link_af(dev,
1125 tb[IFLA_AF_SPEC]);
1126 if (err < 0)
1127 return err;
1128 }
1129 }
1130 }
1131
988 return 0; 1132 return 0;
989} 1133}
990 1134
@@ -1225,6 +1369,24 @@ static int do_setlink(struct net_device *dev, struct ifinfomsg *ifm,
1225 goto errout; 1369 goto errout;
1226 modified = 1; 1370 modified = 1;
1227 } 1371 }
1372
1373 if (tb[IFLA_AF_SPEC]) {
1374 struct nlattr *af;
1375 int rem;
1376
1377 nla_for_each_nested(af, tb[IFLA_AF_SPEC], rem) {
1378 const struct rtnl_af_ops *af_ops;
1379
1380 if (!(af_ops = rtnl_af_lookup(nla_type(af))))
1381 BUG();
1382
1383 err = af_ops->set_link_af(dev, af);
1384 if (err < 0)
1385 goto errout;
1386
1387 modified = 1;
1388 }
1389 }
1228 err = 0; 1390 err = 0;
1229 1391
1230errout: 1392errout:
diff --git a/net/core/scm.c b/net/core/scm.c
index 413cab89017..bbe45445080 100644
--- a/net/core/scm.c
+++ b/net/core/scm.c
@@ -79,10 +79,11 @@ static int scm_fp_copy(struct cmsghdr *cmsg, struct scm_fp_list **fplp)
79 return -ENOMEM; 79 return -ENOMEM;
80 *fplp = fpl; 80 *fplp = fpl;
81 fpl->count = 0; 81 fpl->count = 0;
82 fpl->max = SCM_MAX_FD;
82 } 83 }
83 fpp = &fpl->fp[fpl->count]; 84 fpp = &fpl->fp[fpl->count];
84 85
85 if (fpl->count + num > SCM_MAX_FD) 86 if (fpl->count + num > fpl->max)
86 return -EINVAL; 87 return -EINVAL;
87 88
88 /* 89 /*
@@ -331,11 +332,12 @@ struct scm_fp_list *scm_fp_dup(struct scm_fp_list *fpl)
331 if (!fpl) 332 if (!fpl)
332 return NULL; 333 return NULL;
333 334
334 new_fpl = kmalloc(sizeof(*fpl), GFP_KERNEL); 335 new_fpl = kmemdup(fpl, offsetof(struct scm_fp_list, fp[fpl->count]),
336 GFP_KERNEL);
335 if (new_fpl) { 337 if (new_fpl) {
336 for (i=fpl->count-1; i>=0; i--) 338 for (i = 0; i < fpl->count; i++)
337 get_file(fpl->fp[i]); 339 get_file(fpl->fp[i]);
338 memcpy(new_fpl, fpl, sizeof(*fpl)); 340 new_fpl->max = new_fpl->count;
339 } 341 }
340 return new_fpl; 342 return new_fpl;
341} 343}
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 104f8444754..19d6c21220f 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -778,6 +778,28 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
778 778
779 size = SKB_DATA_ALIGN(size); 779 size = SKB_DATA_ALIGN(size);
780 780
781 /* Check if we can avoid taking references on fragments if we own
782 * the last reference on skb->head. (see skb_release_data())
783 */
784 if (!skb->cloned)
785 fastpath = true;
786 else {
787 int delta = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;
788
789 fastpath = atomic_read(&skb_shinfo(skb)->dataref) == delta;
790 }
791
792 if (fastpath &&
793 size + sizeof(struct skb_shared_info) <= ksize(skb->head)) {
794 memmove(skb->head + size, skb_shinfo(skb),
795 offsetof(struct skb_shared_info,
796 frags[skb_shinfo(skb)->nr_frags]));
797 memmove(skb->head + nhead, skb->head,
798 skb_tail_pointer(skb) - skb->head);
799 off = nhead;
800 goto adjust_others;
801 }
802
781 data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask); 803 data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
782 if (!data) 804 if (!data)
783 goto nodata; 805 goto nodata;
@@ -791,17 +813,6 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
791 skb_shinfo(skb), 813 skb_shinfo(skb),
792 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 814 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
793 815
794 /* Check if we can avoid taking references on fragments if we own
795 * the last reference on skb->head. (see skb_release_data())
796 */
797 if (!skb->cloned)
798 fastpath = true;
799 else {
800 int delta = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;
801
802 fastpath = atomic_read(&skb_shinfo(skb)->dataref) == delta;
803 }
804
805 if (fastpath) { 816 if (fastpath) {
806 kfree(skb->head); 817 kfree(skb->head);
807 } else { 818 } else {
@@ -816,6 +827,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
816 off = (data + nhead) - skb->head; 827 off = (data + nhead) - skb->head;
817 828
818 skb->head = data; 829 skb->head = data;
830adjust_others:
819 skb->data += off; 831 skb->data += off;
820#ifdef NET_SKBUFF_DATA_USES_OFFSET 832#ifdef NET_SKBUFF_DATA_USES_OFFSET
821 skb->end = size; 833 skb->end = size;
@@ -1812,7 +1824,7 @@ void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
1812 long csstart; 1824 long csstart;
1813 1825
1814 if (skb->ip_summed == CHECKSUM_PARTIAL) 1826 if (skb->ip_summed == CHECKSUM_PARTIAL)
1815 csstart = skb->csum_start - skb_headroom(skb); 1827 csstart = skb_checksum_start_offset(skb);
1816 else 1828 else
1817 csstart = skb_headlen(skb); 1829 csstart = skb_headlen(skb);
1818 1830
diff --git a/net/core/sock.c b/net/core/sock.c
index e5af8d5d5b5..a6b9e8061f3 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -992,17 +992,18 @@ static inline void sock_lock_init(struct sock *sk)
992/* 992/*
993 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 993 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
994 * even temporarly, because of RCU lookups. sk_node should also be left as is. 994 * even temporarly, because of RCU lookups. sk_node should also be left as is.
995 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
995 */ 996 */
996static void sock_copy(struct sock *nsk, const struct sock *osk) 997static void sock_copy(struct sock *nsk, const struct sock *osk)
997{ 998{
998#ifdef CONFIG_SECURITY_NETWORK 999#ifdef CONFIG_SECURITY_NETWORK
999 void *sptr = nsk->sk_security; 1000 void *sptr = nsk->sk_security;
1000#endif 1001#endif
1001 BUILD_BUG_ON(offsetof(struct sock, sk_copy_start) != 1002 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1002 sizeof(osk->sk_node) + sizeof(osk->sk_refcnt) + 1003
1003 sizeof(osk->sk_tx_queue_mapping)); 1004 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1004 memcpy(&nsk->sk_copy_start, &osk->sk_copy_start, 1005 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1005 osk->sk_prot->obj_size - offsetof(struct sock, sk_copy_start)); 1006
1006#ifdef CONFIG_SECURITY_NETWORK 1007#ifdef CONFIG_SECURITY_NETWORK
1007 nsk->sk_security = sptr; 1008 nsk->sk_security = sptr;
1008 security_sk_clone(osk, nsk); 1009 security_sk_clone(osk, nsk);
diff --git a/net/core/timestamping.c b/net/core/timestamping.c
index c19bb4ee405..7e7ca375d43 100644
--- a/net/core/timestamping.c
+++ b/net/core/timestamping.c
@@ -26,12 +26,12 @@ static struct sock_filter ptp_filter[] = {
26 PTP_FILTER 26 PTP_FILTER
27}; 27};
28 28
29static unsigned int classify(struct sk_buff *skb) 29static unsigned int classify(const struct sk_buff *skb)
30{ 30{
31 if (likely(skb->dev && 31 if (likely(skb->dev &&
32 skb->dev->phydev && 32 skb->dev->phydev &&
33 skb->dev->phydev->drv)) 33 skb->dev->phydev->drv))
34 return sk_run_filter(skb, ptp_filter, ARRAY_SIZE(ptp_filter)); 34 return sk_run_filter(skb, ptp_filter);
35 else 35 else
36 return PTP_CLASS_NONE; 36 return PTP_CLASS_NONE;
37} 37}
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-21 02:37:08 -0400