Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

1 files changed, 470 insertions, 91 deletions

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 46c36ffe20ee..9e19477991ad 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -138,6 +138,9 @@ static inline bool dev_xmit_complete(int rc)
 
 #define MAX_ADDR_LEN    32              /* Largest hardware address length */
 
+/* Initial net device group. All devices belong to group 0 by default. */
+#define INIT_NETDEV_GROUP       0
+
 #ifdef  __KERNEL__
 /*
  *      Compute the worst case header length according to the protocols
@@ -228,9 +231,9 @@ struct netdev_hw_addr {
 #define NETDEV_HW_ADDR_T_SLAVE          3
 #define NETDEV_HW_ADDR_T_UNICAST        4
 #define NETDEV_HW_ADDR_T_MULTICAST      5
-        int                     refcount;
         bool                    synced;
         bool                    global_use;
+        int                     refcount;
         struct rcu_head         rcu_head;
 };
 
@@ -281,6 +284,12 @@ struct hh_cache {
         unsigned long   hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
 };
 
+static inline void hh_cache_put(struct hh_cache *hh)
+{
+        if (atomic_dec_and_test(&hh->hh_refcnt))
+                kfree(hh);
+}
+
 /* Reserve HH_DATA_MOD byte aligned hard_header_len, but at least that much.
  * Alternative is:
  *   dev->hard_header_len ? (dev->hard_header_len +
@@ -381,7 +390,55 @@ enum gro_result {
 };
 typedef enum gro_result gro_result_t;
 
-typedef struct sk_buff *rx_handler_func_t(struct sk_buff *skb);
+/*
+ * enum rx_handler_result - Possible return values for rx_handlers.
+ * @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
+ * further.
+ * @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
+ * case skb->dev was changed by rx_handler.
+ * @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
+ * @RX_HANDLER_PASS: Do nothing, passe the skb as if no rx_handler was called.
+ *
+ * rx_handlers are functions called from inside __netif_receive_skb(), to do
+ * special processing of the skb, prior to delivery to protocol handlers.
+ *
+ * Currently, a net_device can only have a single rx_handler registered. Trying
+ * to register a second rx_handler will return -EBUSY.
+ *
+ * To register a rx_handler on a net_device, use netdev_rx_handler_register().
+ * To unregister a rx_handler on a net_device, use
+ * netdev_rx_handler_unregister().
+ *
+ * Upon return, rx_handler is expected to tell __netif_receive_skb() what to
+ * do with the skb.
+ *
+ * If the rx_handler consumed to skb in some way, it should return
+ * RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
+ * the skb to be delivered in some other ways.
+ *
+ * If the rx_handler changed skb->dev, to divert the skb to another
+ * net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
+ * new device will be called if it exists.
+ *
+ * If the rx_handler consider the skb should be ignored, it should return
+ * RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
+ * are registred on exact device (ptype->dev == skb->dev).
+ *
+ * If the rx_handler didn't changed skb->dev, but want the skb to be normally
+ * delivered, it should return RX_HANDLER_PASS.
+ *
+ * A device without a registered rx_handler will behave as if rx_handler
+ * returned RX_HANDLER_PASS.
+ */
+
+enum rx_handler_result {
+        RX_HANDLER_CONSUMED,
+        RX_HANDLER_ANOTHER,
+        RX_HANDLER_EXACT,
+        RX_HANDLER_PASS,
+};
+typedef enum rx_handler_result rx_handler_result_t;
+typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
 
 extern void __napi_schedule(struct napi_struct *n);
 
@@ -487,6 +544,8 @@ static inline void napi_synchronize(const struct napi_struct *n)
 enum netdev_queue_state_t {
         __QUEUE_STATE_XOFF,
         __QUEUE_STATE_FROZEN,
+#define QUEUE_STATE_XOFF_OR_FROZEN ((1 << __QUEUE_STATE_XOFF)           | \
+                                    (1 << __QUEUE_STATE_FROZEN))
 };
 
 struct netdev_queue {
@@ -497,6 +556,12 @@ struct netdev_queue {
         struct Qdisc            *qdisc;
         unsigned long           state;
         struct Qdisc            *qdisc_sleeping;
+#ifdef CONFIG_RPS
+        struct kobject          kobj;
+#endif
+#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
+        int                     numa_node;
+#endif
 /*
  * write mostly part
  */
@@ -506,11 +571,24 @@ struct netdev_queue {
          * please use this field instead of dev->trans_start
          */
         unsigned long           trans_start;
-        u64                     tx_bytes;
-        u64                     tx_packets;
-        u64                     tx_dropped;
 } ____cacheline_aligned_in_smp;
 
+static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
+{
+#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
+        return q->numa_node;
+#else
+        return NUMA_NO_NODE;
+#endif
+}
+
+static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
+{
+#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
+        q->numa_node = node;
+#endif
+}
+
 #ifdef CONFIG_RPS
 /*
  * This structure holds an RPS map which can be of variable length.  The
@@ -524,14 +602,16 @@ struct rps_map {
 #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + (_num * sizeof(u16)))
 
 /*
- * The rps_dev_flow structure contains the mapping of a flow to a CPU and the
- * tail pointer for that CPU's input queue at the time of last enqueue.
+ * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
+ * tail pointer for that CPU's input queue at the time of last enqueue, and
+ * a hardware filter index.
  */
 struct rps_dev_flow {
         u16 cpu;
-        u16 fill;
+        u16 filter;
         unsigned int last_qtail;
 };
+#define RPS_NO_FILTER 0xffff
 
 /*
  * The rps_dev_flow_table structure contains a table of flow mappings.
@@ -579,18 +659,56 @@ static inline void rps_reset_sock_flow(struct rps_sock_flow_table *table,
                 table->ents[hash & table->mask] = RPS_NO_CPU;
 }
 
-extern struct rps_sock_flow_table *rps_sock_flow_table;
+extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
+
+#ifdef CONFIG_RFS_ACCEL
+extern bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
+                                u32 flow_id, u16 filter_id);
+#endif
 
 /* This structure contains an instance of an RX queue. */
 struct netdev_rx_queue {
-        struct rps_map *rps_map;
-        struct rps_dev_flow_table *rps_flow_table;
-        struct kobject kobj;
-        struct netdev_rx_queue *first;
-        atomic_t count;
+        struct rps_map __rcu            *rps_map;
+        struct rps_dev_flow_table __rcu *rps_flow_table;
+        struct kobject                  kobj;
+        struct net_device               *dev;
 } ____cacheline_aligned_in_smp;
 #endif /* CONFIG_RPS */
 
+#ifdef CONFIG_XPS
+/*
+ * This structure holds an XPS map which can be of variable length.  The
+ * map is an array of queues.
+ */
+struct xps_map {
+        unsigned int len;
+        unsigned int alloc_len;
+        struct rcu_head rcu;
+        u16 queues[0];
+};
+#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + (_num * sizeof(u16)))
+#define XPS_MIN_MAP_ALLOC ((L1_CACHE_BYTES - sizeof(struct xps_map))    \
+    / sizeof(u16))
+
+/*
+ * This structure holds all XPS maps for device.  Maps are indexed by CPU.
+ */
+struct xps_dev_maps {
+        struct rcu_head rcu;
+        struct xps_map __rcu *cpu_map[0];
+};
+#define XPS_DEV_MAPS_SIZE (sizeof(struct xps_dev_maps) +                \
+    (nr_cpu_ids * sizeof(struct xps_map *)))
+#endif /* CONFIG_XPS */
+
+#define TC_MAX_QUEUE    16
+#define TC_BITMASK      15
+/* HW offloaded queuing disciplines txq count and offset maps */
+struct netdev_tc_txq {
+        u16 count;
+        u16 offset;
+};
+
 /*
  * This structure defines the management hooks for network devices.
  * The following hooks can be defined; unless noted otherwise, they are
@@ -677,7 +795,7 @@ struct netdev_rx_queue {
  *         neither operation.
  *
  * void (*ndo_vlan_rx_register)(struct net_device *dev, struct vlan_group *grp);
- *      If device support VLAN receive accleration
+ *      If device support VLAN receive acceleration
  *      (ie. dev->features & NETIF_F_HW_VLAN_RX), then this function is called
  *      when vlan groups for the device changes.  Note: grp is NULL
  *      if no vlan's groups are being used.
@@ -701,6 +819,74 @@ struct netdev_rx_queue {
  * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
  *                        struct nlattr *port[]);
  * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
+ * int (*ndo_setup_tc)(struct net_device *dev, u8 tc)
+ *      Called to setup 'tc' number of traffic classes in the net device. This
+ *      is always called from the stack with the rtnl lock held and netif tx
+ *      queues stopped. This allows the netdevice to perform queue management
+ *      safely.
+ *
+ *      Fiber Channel over Ethernet (FCoE) offload functions.
+ * int (*ndo_fcoe_enable)(struct net_device *dev);
+ *      Called when the FCoE protocol stack wants to start using LLD for FCoE
+ *      so the underlying device can perform whatever needed configuration or
+ *      initialization to support acceleration of FCoE traffic.
+ *
+ * int (*ndo_fcoe_disable)(struct net_device *dev);
+ *      Called when the FCoE protocol stack wants to stop using LLD for FCoE
+ *      so the underlying device can perform whatever needed clean-ups to
+ *      stop supporting acceleration of FCoE traffic.
+ *
+ * int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
+ *                           struct scatterlist *sgl, unsigned int sgc);
+ *      Called when the FCoE Initiator wants to initialize an I/O that
+ *      is a possible candidate for Direct Data Placement (DDP). The LLD can
+ *      perform necessary setup and returns 1 to indicate the device is set up
+ *      successfully to perform DDP on this I/O, otherwise this returns 0.
+ *
+ * int (*ndo_fcoe_ddp_done)(struct net_device *dev,  u16 xid);
+ *      Called when the FCoE Initiator/Target is done with the DDPed I/O as
+ *      indicated by the FC exchange id 'xid', so the underlying device can
+ *      clean up and reuse resources for later DDP requests.
+ *
+ * int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
+ *                            struct scatterlist *sgl, unsigned int sgc);
+ *      Called when the FCoE Target wants to initialize an I/O that
+ *      is a possible candidate for Direct Data Placement (DDP). The LLD can
+ *      perform necessary setup and returns 1 to indicate the device is set up
+ *      successfully to perform DDP on this I/O, otherwise this returns 0.
+ *
+ * int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
+ *      Called when the underlying device wants to override default World Wide
+ *      Name (WWN) generation mechanism in FCoE protocol stack to pass its own
+ *      World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
+ *      protocol stack to use.
+ *
+ *      RFS acceleration.
+ * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
+ *                          u16 rxq_index, u32 flow_id);
+ *      Set hardware filter for RFS.  rxq_index is the target queue index;
+ *      flow_id is a flow ID to be passed to rps_may_expire_flow() later.
+ *      Return the filter ID on success, or a negative error code.
+ *
+ *      Slave management functions (for bridge, bonding, etc). User should
+ *      call netdev_set_master() to set dev->master properly.
+ * int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
+ *      Called to make another netdev an underling.
+ *
+ * int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
+ *      Called to release previously enslaved netdev.
+ *
+ *      Feature/offload setting functions.
+ * u32 (*ndo_fix_features)(struct net_device *dev, u32 features);
+ *      Adjusts the requested feature flags according to device-specific
+ *      constraints, and returns the resulting flags. Must not modify
+ *      the device state.
+ *
+ * int (*ndo_set_features)(struct net_device *dev, u32 features);
+ *      Called to update device configuration to new features. Passed
+ *      feature set might be less than what was returned by ndo_fix_features()).
+ *      Must return >0 or -errno if it changed dev->features itself.
+ *
  */
 #define HAVE_NET_DEVICE_OPS
 struct net_device_ops {
@@ -759,6 +945,7 @@ struct net_device_ops {
                                                    struct nlattr *port[]);
         int                     (*ndo_get_vf_port)(struct net_device *dev,
                                                    int vf, struct sk_buff *skb);
+        int                     (*ndo_setup_tc)(struct net_device *dev, u8 tc);
 #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
         int                     (*ndo_fcoe_enable)(struct net_device *dev);
         int                     (*ndo_fcoe_disable)(struct net_device *dev);
@@ -768,11 +955,29 @@ struct net_device_ops {
                                                       unsigned int sgc);
         int                     (*ndo_fcoe_ddp_done)(struct net_device *dev,
                                                      u16 xid);
+        int                     (*ndo_fcoe_ddp_target)(struct net_device *dev,
+                                                       u16 xid,
+                                                       struct scatterlist *sgl,
+                                                       unsigned int sgc);
 #define NETDEV_FCOE_WWNN 0
 #define NETDEV_FCOE_WWPN 1
         int                     (*ndo_fcoe_get_wwn)(struct net_device *dev,
                                                     u64 *wwn, int type);
 #endif
+#ifdef CONFIG_RFS_ACCEL
+        int                     (*ndo_rx_flow_steer)(struct net_device *dev,
+                                                     const struct sk_buff *skb,
+                                                     u16 rxq_index,
+                                                     u32 flow_id);
+#endif
+        int                     (*ndo_add_slave)(struct net_device *dev,
+                                                 struct net_device *slave_dev);
+        int                     (*ndo_del_slave)(struct net_device *dev,
+                                                 struct net_device *slave_dev);
+        u32                     (*ndo_fix_features)(struct net_device *dev,
+                                                    u32 features);
+        int                     (*ndo_set_features)(struct net_device *dev,
+                                                    u32 features);
 };
 
 /*
@@ -815,17 +1020,24 @@ struct net_device {
          *      part of the usual set specified in Space.c.
          */
 
-        unsigned char           if_port;        /* Selectable AUI, TP,..*/
-        unsigned char           dma;            /* DMA channel          */
-
         unsigned long           state;
 
         struct list_head        dev_list;
         struct list_head        napi_list;
         struct list_head        unreg_list;
 
-        /* Net device features */
-        unsigned long           features;
+        /* currently active device features */
+        u32                     features;
+        /* user-changeable features */
+        u32                     hw_features;
+        /* user-requested features */
+        u32                     wanted_features;
+        /* mask of features inheritable by VLAN devices */
+        u32                     vlan_features;
+
+        /* Net device feature bits; if you change something,
+         * also update netdev_features_strings[] in ethtool.c */
+
 #define NETIF_F_SG              1       /* Scatter/gather IO. */
 #define NETIF_F_IP_CSUM         2       /* Can checksum TCP/UDP over IPv4. */
 #define NETIF_F_NO_CSUM         4       /* Does not require checksum. F.e. loopack. */
@@ -850,6 +1062,9 @@ struct net_device {
 #define NETIF_F_FCOE_MTU        (1 << 26) /* Supports max FCoE MTU, 2158 bytes*/
 #define NETIF_F_NTUPLE          (1 << 27) /* N-tuple filters supported */
 #define NETIF_F_RXHASH          (1 << 28) /* Receive hashing offload */
+#define NETIF_F_RXCSUM          (1 << 29) /* Receive checksumming offload */
+#define NETIF_F_NOCACHE_COPY    (1 << 30) /* Use no-cache copyfromuser */
+#define NETIF_F_LOOPBACK        (1 << 31) /* Enable loopback */
 
         /* Segmentation offload features */
 #define NETIF_F_GSO_SHIFT       16
@@ -861,6 +1076,12 @@ struct net_device {
 #define NETIF_F_TSO6            (SKB_GSO_TCPV6 << NETIF_F_GSO_SHIFT)
 #define NETIF_F_FSO             (SKB_GSO_FCOE << NETIF_F_GSO_SHIFT)
 
+        /* Features valid for ethtool to change */
+        /* = all defined minus driver/device-class-related */
+#define NETIF_F_NEVER_CHANGE    (NETIF_F_VLAN_CHALLENGED | \
+                                  NETIF_F_LLTX | NETIF_F_NETNS_LOCAL)
+#define NETIF_F_ETHTOOL_BITS    (0xff3fffff & ~NETIF_F_NEVER_CHANGE)
+
         /* List of features with software fallbacks. */
 #define NETIF_F_GSO_SOFTWARE    (NETIF_F_TSO | NETIF_F_TSO_ECN | \
                                  NETIF_F_TSO6 | NETIF_F_UFO)
@@ -871,19 +1092,35 @@ struct net_device {
 #define NETIF_F_V6_CSUM         (NETIF_F_GEN_CSUM | NETIF_F_IPV6_CSUM)
 #define NETIF_F_ALL_CSUM        (NETIF_F_V4_CSUM | NETIF_F_V6_CSUM)
 
+#define NETIF_F_ALL_TSO         (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
+
+#define NETIF_F_ALL_FCOE        (NETIF_F_FCOE_CRC | NETIF_F_FCOE_MTU | \
+                                 NETIF_F_FSO)
+
         /*
          * If one device supports one of these features, then enable them
          * for all in netdev_increment_features.
          */
 #define NETIF_F_ONE_FOR_ALL     (NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ROBUST | \
                                  NETIF_F_SG | NETIF_F_HIGHDMA |         \
-                                 NETIF_F_FRAGLIST)
+                                 NETIF_F_FRAGLIST | NETIF_F_VLAN_CHALLENGED)
+        /*
+         * If one device doesn't support one of these features, then disable it
+         * for all in netdev_increment_features.
+         */
+#define NETIF_F_ALL_FOR_ALL     (NETIF_F_NOCACHE_COPY | NETIF_F_FSO)
+
+        /* changeable features with no special hardware requirements */
+#define NETIF_F_SOFT_FEATURES   (NETIF_F_GSO | NETIF_F_GRO)
 
         /* Interface index. Unique device identifier    */
         int                     ifindex;
         int                     iflink;
 
         struct net_device_stats stats;
+        atomic_long_t           rx_dropped; /* dropped packets by core network
+                                             * Do not use this in drivers.
+                                             */
 
 #ifdef CONFIG_WIRELESS_EXT
         /* List of functions to handle Wireless Extensions (instead of ioctl).
@@ -900,13 +1137,16 @@ struct net_device {
         const struct header_ops *header_ops;
 
         unsigned int            flags;  /* interface flags (a la BSD)   */
+        unsigned int            priv_flags; /* Like 'flags' but invisible to userspace. */
         unsigned short          gflags;
-        unsigned short          priv_flags; /* Like 'flags' but invisible to userspace. */
         unsigned short          padded; /* How much padding added by alloc_netdev() */
 
         unsigned char           operstate; /* RFC2863 operstate */
         unsigned char           link_mode; /* mapping policy to operstate */
 
+        unsigned char           if_port;        /* Selectable AUI, TP,..*/
+        unsigned char           dma;            /* DMA channel          */
+
         unsigned int            mtu;    /* interface MTU value          */
         unsigned short          type;   /* interface hardware type      */
         unsigned short          hard_header_len;        /* hardware hdr length  */
@@ -918,10 +1158,6 @@ struct net_device {
         unsigned short          needed_headroom;
         unsigned short          needed_tailroom;
 
-        struct net_device       *master; /* Pointer to master device of a group,
-                                          * which this device is member of.
-                                          */
-
         /* Interface address info. */
         unsigned char           perm_addr[MAX_ADDR_LEN]; /* permanent hw address */
         unsigned char           addr_assign_type; /* hw address assignment type */
@@ -937,23 +1173,37 @@ struct net_device {
 
 
         /* Protocol specific pointers */
-        
+
+#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+        struct vlan_group __rcu *vlgrp;         /* VLAN group */
+#endif
 #ifdef CONFIG_NET_DSA
         void                    *dsa_ptr;       /* dsa specific data */
 #endif
         void                    *atalk_ptr;     /* AppleTalk link       */
-        void                    *ip_ptr;        /* IPv4 specific data   */
-        void                    *dn_ptr;        /* DECnet specific data */
-        void                    *ip6_ptr;       /* IPv6 specific data */
+        struct in_device __rcu  *ip_ptr;        /* IPv4 specific data   */
+        struct dn_dev __rcu     *dn_ptr;        /* DECnet specific data */
+        struct inet6_dev __rcu  *ip6_ptr;       /* IPv6 specific data */
         void                    *ec_ptr;        /* Econet specific data */
         void                    *ax25_ptr;      /* AX.25 specific data */
         struct wireless_dev     *ieee80211_ptr; /* IEEE 802.11 specific data,
                                                    assign before registering */
 
 /*
- * Cache line mostly used on receive path (including eth_type_trans())
+ * Cache lines mostly used on receive path (including eth_type_trans())
  */
-        unsigned long           last_rx;        /* Time of last Rx      */
+        unsigned long           last_rx;        /* Time of last Rx
+                                                 * This should not be set in
+                                                 * drivers, unless really needed,
+                                                 * because network stack (bonding)
+                                                 * use it if/when necessary, to
+                                                 * avoid dirtying this cache line.
+                                                 */
+
+        struct net_device       *master; /* Pointer to master device of a group,
+                                          * which this device is member of.
+                                          */
+
         /* Interface address info used in eth_type_trans() */
         unsigned char           *dev_addr;      /* hw address, (before bcast
                                                    because most packets are
@@ -969,14 +1219,28 @@ struct net_device {
 
         struct netdev_rx_queue  *_rx;
 
-        /* Number of RX queues allocated at alloc_netdev_mq() time  */
+        /* Number of RX queues allocated at register_netdev() time */
         unsigned int            num_rx_queues;
+
+        /* Number of RX queues currently active in device */
+        unsigned int            real_num_rx_queues;
+
+#ifdef CONFIG_RFS_ACCEL
+        /* CPU reverse-mapping for RX completion interrupts, indexed
+         * by RX queue number.  Assigned by driver.  This must only be
+         * set if the ndo_rx_flow_steer operation is defined. */
+        struct cpu_rmap         *rx_cpu_rmap;
+#endif
 #endif
 
-        struct netdev_queue     rx_queue;
-        rx_handler_func_t       *rx_handler;
-        void                    *rx_handler_data;
+        rx_handler_func_t __rcu *rx_handler;
+        void __rcu              *rx_handler_data;
+
+        struct netdev_queue __rcu *ingress_queue;
 
+/*
+ * Cache lines mostly used on transmit path
+ */
         struct netdev_queue     *_tx ____cacheline_aligned_in_smp;
 
         /* Number of TX queues allocated at alloc_netdev_mq() time  */
@@ -990,9 +1254,11 @@ struct net_device {
 
         unsigned long           tx_queue_len;   /* Max frames per queue allowed */
         spinlock_t              tx_global_lock;
-/*
- * One part is mostly used on xmit path (device)
- */
+
+#ifdef CONFIG_XPS
+        struct xps_dev_maps __rcu *xps_maps;
+#endif
+
         /* These may be needed for future network-power-down code. */
 
         /*
@@ -1005,7 +1271,7 @@ struct net_device {
         struct timer_list       watchdog_timer;
 
         /* Number of references to this device */
-        atomic_t                refcnt ____cacheline_aligned_in_smp;
+        int __percpu            *pcpu_refcnt;
 
         /* delayed register/unregister */
         struct list_head        todo_list;
@@ -1021,7 +1287,9 @@ struct net_device {
                NETREG_UNREGISTERED,     /* completed unregister todo */
                NETREG_RELEASED,         /* called free_netdev */
                NETREG_DUMMY,            /* dummy device for NAPI poll */
-        } reg_state:16;
+        } reg_state:8;
+
+        bool dismantle; /* device is going do be freed */
 
         enum {
                 RTNL_LINK_INITIALIZED,
@@ -1041,10 +1309,14 @@ struct net_device {
 #endif
 
         /* mid-layer private */
-        void                    *ml_priv;
-
+        union {
+                void                            *ml_priv;
+                struct pcpu_lstats __percpu     *lstats; /* loopback stats */
+                struct pcpu_tstats __percpu     *tstats; /* tunnel stats */
+                struct pcpu_dstats __percpu     *dstats; /* dummy stats */
+        };
         /* GARP */
-        struct garp_port        *garp_port;
+        struct garp_port __rcu  *garp_port;
 
         /* class/net/name entry */
         struct device           dev;
@@ -1054,9 +1326,6 @@ struct net_device {
         /* rtnetlink link ops */
         const struct rtnl_link_ops *rtnl_link_ops;
 
-        /* VLAN feature mask */
-        unsigned long vlan_features;
-
         /* for setting kernel sock attribute on TCP connection setup */
 #define GSO_MAX_SIZE            65536
         unsigned int            gso_max_size;
@@ -1065,6 +1334,9 @@ struct net_device {
         /* Data Center Bridging netlink ops */
         const struct dcbnl_rtnl_ops *dcbnl_ops;
 #endif
+        u8 num_tc;
+        struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
+        u8 prio_tc_map[TC_BITMASK + 1];
 
 #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
         /* max exchange id for FCoE LRO by ddp */
@@ -1075,12 +1347,66 @@ struct net_device {
 
         /* phy device may attach itself for hardware timestamping */
         struct phy_device *phydev;
+
+        /* group the device belongs to */
+        int group;
 };
 #define to_net_dev(d) container_of(d, struct net_device, dev)
 
 #define NETDEV_ALIGN            32
 
 static inline
+int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
+{
+        return dev->prio_tc_map[prio & TC_BITMASK];
+}
+
+static inline
+int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
+{
+        if (tc >= dev->num_tc)
+                return -EINVAL;
+
+        dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
+        return 0;
+}
+
+static inline
+void netdev_reset_tc(struct net_device *dev)
+{
+        dev->num_tc = 0;
+        memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
+        memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
+}
+
+static inline
+int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
+{
+        if (tc >= dev->num_tc)
+                return -EINVAL;
+
+        dev->tc_to_txq[tc].count = count;
+        dev->tc_to_txq[tc].offset = offset;
+        return 0;
+}
+
+static inline
+int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
+{
+        if (num_tc > TC_MAX_QUEUE)
+                return -EINVAL;
+
+        dev->num_tc = num_tc;
+        return 0;
+}
+
+static inline
+int netdev_get_num_tc(struct net_device *dev)
+{
+        return dev->num_tc;
+}
+
+static inline
 struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
                                          unsigned int index)
 {
@@ -1222,7 +1548,7 @@ struct packet_type {
                                          struct packet_type *,
                                          struct net_device *);
         struct sk_buff          *(*gso_segment)(struct sk_buff *skb,
-                                                int features);
+                                                u32 features);
         int                     (*gso_send_check)(struct sk_buff *skb);
         struct sk_buff          **(*gro_receive)(struct sk_buff **head,
                                                struct sk_buff *skb);
@@ -1267,7 +1593,7 @@ static inline struct net_device *next_net_device_rcu(struct net_device *dev)
         struct net *net;
 
         net = dev_net(dev);
-        lh = rcu_dereference(dev->dev_list.next);
+        lh = rcu_dereference(list_next_rcu(&dev->dev_list));
         return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
 }
 
@@ -1277,9 +1603,17 @@ static inline struct net_device *first_net_device(struct net *net)
                 net_device_entry(net->dev_base_head.next);
 }
 
+static inline struct net_device *first_net_device_rcu(struct net *net)
+{
+        struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
+
+        return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
+}
+
 extern int                      netdev_boot_setup_check(struct net_device *dev);
 extern unsigned long            netdev_boot_base(const char *prefix, int unit);
-extern struct net_device    *dev_getbyhwaddr(struct net *net, unsigned short type, char *hwaddr);
+extern struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
+                                              const char *hwaddr);
 extern struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
 extern struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type);
 extern void             dev_add_pack(struct packet_type *pt);
@@ -1305,6 +1639,7 @@ static inline void unregister_netdevice(struct net_device *dev)
         unregister_netdevice_queue(dev, NULL);
 }
 
+extern int              netdev_refcnt_read(const struct net_device *dev);
 extern void             free_netdev(struct net_device *dev);
 extern void             synchronize_net(void);
 extern int              register_netdevice_notifier(struct notifier_block *nb);
@@ -1525,6 +1860,10 @@ static inline void netif_tx_wake_all_queues(struct net_device *dev)
 
 static inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
 {
+        if (WARN_ON(!dev_queue)) {
+                pr_info("netif_stop_queue() cannot be called before register_netdev()\n");
+                return;
+        }
         set_bit(__QUEUE_STATE_XOFF, &dev_queue->state);
 }
 
@@ -1566,9 +1905,9 @@ static inline int netif_queue_stopped(const struct net_device *dev)
         return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
 }
 
-static inline int netif_tx_queue_frozen(const struct netdev_queue *dev_queue)
+static inline int netif_tx_queue_frozen_or_stopped(const struct netdev_queue *dev_queue)
 {
-        return test_bit(__QUEUE_STATE_FROZEN, &dev_queue->state);
+        return dev_queue->state & QUEUE_STATE_XOFF_OR_FROZEN;
 }
 
 /**
@@ -1659,6 +1998,16 @@ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
                 __netif_schedule(txq->qdisc);
 }
 
+/*
+ * Returns a Tx hash for the given packet when dev->real_num_tx_queues is used
+ * as a distribution range limit for the returned value.
+ */
+static inline u16 skb_tx_hash(const struct net_device *dev,
+                              const struct sk_buff *skb)
+{
+        return __skb_tx_hash(dev, skb, dev->real_num_tx_queues);
+}
+
 /**
  *      netif_is_multiqueue - test if device has multiple transmit queues
  *      @dev: network device
@@ -1667,11 +2016,34 @@ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
  */
 static inline int netif_is_multiqueue(const struct net_device *dev)
 {
-        return (dev->num_tx_queues > 1);
+        return dev->num_tx_queues > 1;
+}
+
+extern int netif_set_real_num_tx_queues(struct net_device *dev,
+                                        unsigned int txq);
+
+#ifdef CONFIG_RPS
+extern int netif_set_real_num_rx_queues(struct net_device *dev,
+                                        unsigned int rxq);
+#else
+static inline int netif_set_real_num_rx_queues(struct net_device *dev,
+                                                unsigned int rxq)
+{
+        return 0;
 }
+#endif
 
-extern void netif_set_real_num_tx_queues(struct net_device *dev,
-                                         unsigned int txq);
+static inline int netif_copy_real_num_queues(struct net_device *to_dev,
+                                             const struct net_device *from_dev)
+{
+        netif_set_real_num_tx_queues(to_dev, from_dev->real_num_tx_queues);
+#ifdef CONFIG_RPS
+        return netif_set_real_num_rx_queues(to_dev,
+                                            from_dev->real_num_rx_queues);
+#else
+        return 0;
+#endif
+}
 
 /* Use this variant when it is known for sure that it
  * is executing from hardware interrupt context or with hardware interrupts
@@ -1695,8 +2067,7 @@ extern gro_result_t	dev_gro_receive(struct napi_struct *napi,
 extern gro_result_t     napi_skb_finish(gro_result_t ret, struct sk_buff *skb);
 extern gro_result_t     napi_gro_receive(struct napi_struct *napi,
                                          struct sk_buff *skb);
-extern void             napi_reuse_skb(struct napi_struct *napi,
-                                       struct sk_buff *skb);
+extern void             napi_gro_flush(struct napi_struct *napi);
 extern struct sk_buff * napi_get_frags(struct napi_struct *napi);
 extern gro_result_t     napi_frags_finish(struct napi_struct *napi,
                                           struct sk_buff *skb,
@@ -1715,7 +2086,6 @@ extern int netdev_rx_handler_register(struct net_device *dev,
                                       void *rx_handler_data);
 extern void netdev_rx_handler_unregister(struct net_device *dev);
 
-extern void             netif_nit_deliver(struct sk_buff *skb);
 extern int              dev_valid_name(const char *name);
 extern int              dev_ioctl(struct net *net, unsigned int cmd, void __user *);
 extern int              dev_ethtool(struct net *net, struct ifreq *);
@@ -1728,6 +2098,7 @@ extern int		dev_set_alias(struct net_device *, const char *, size_t);
 extern int              dev_change_net_namespace(struct net_device *,
                                                  struct net *, const char *);
 extern int              dev_set_mtu(struct net_device *, int);
+extern void             dev_set_group(struct net_device *, int);
 extern int              dev_set_mac_address(struct net_device *,
                                             struct sockaddr *);
 extern int              dev_hard_start_xmit(struct sk_buff *skb,
@@ -1749,7 +2120,7 @@ extern void netdev_run_todo(void);
  */
 static inline void dev_put(struct net_device *dev)
 {
-        atomic_dec(&dev->refcnt);
+        irqsafe_cpu_dec(*dev->pcpu_refcnt);
 }
 
 /**
@@ -1760,7 +2131,7 @@ static inline void dev_put(struct net_device *dev)
  */
 static inline void dev_hold(struct net_device *dev)
 {
-        atomic_inc(&dev->refcnt);
+        irqsafe_cpu_inc(*dev->pcpu_refcnt);
 }
 
 /* Carrier loss detection, dial on demand. The functions netif_carrier_on
@@ -2072,11 +2443,15 @@ static inline void netif_addr_unlock_bh(struct net_device *dev)
 extern void             ether_setup(struct net_device *dev);
 
 /* Support for loadable net-drivers */
-extern struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
+extern struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
                                        void (*setup)(struct net_device *),
-                                       unsigned int queue_count);
+                                       unsigned int txqs, unsigned int rxqs);
 #define alloc_netdev(sizeof_priv, name, setup) \
-        alloc_netdev_mq(sizeof_priv, name, setup, 1)
+        alloc_netdev_mqs(sizeof_priv, name, setup, 1, 1)
+
+#define alloc_netdev_mq(sizeof_priv, name, setup, count) \
+        alloc_netdev_mqs(sizeof_priv, name, setup, count, count)
+
 extern int              register_netdev(struct net_device *dev);
 extern void             unregister_netdev(struct net_device *dev);
 
@@ -2142,15 +2517,16 @@ extern void		dev_load(struct net *net, const char *name);
 extern void             dev_mcast_init(void);
 extern struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
                                                struct rtnl_link_stats64 *storage);
-extern void             dev_txq_stats_fold(const struct net_device *dev,
-                                           struct rtnl_link_stats64 *stats);
 
 extern int              netdev_max_backlog;
 extern int              netdev_tstamp_prequeue;
 extern int              weight_p;
+extern int              bpf_jit_enable;
 extern int              netdev_set_master(struct net_device *dev, struct net_device *master);
+extern int netdev_set_bond_master(struct net_device *dev,
+                                  struct net_device *master);
 extern int skb_checksum_help(struct sk_buff *skb);
-extern struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features);
+extern struct sk_buff *skb_gso_segment(struct sk_buff *skb, u32 features);
 #ifdef CONFIG_BUG
 extern void netdev_rx_csum_fault(struct net_device *dev);
 #else
@@ -2171,33 +2547,42 @@ extern void dev_seq_stop(struct seq_file *seq, void *v);
 extern int netdev_class_create_file(struct class_attribute *class_attr);
 extern void netdev_class_remove_file(struct class_attribute *class_attr);
 
-extern char *netdev_drivername(const struct net_device *dev, char *buffer, int len);
+extern struct kobj_ns_type_operations net_ns_type_operations;
+
+extern const char *netdev_drivername(const struct net_device *dev);
 
 extern void linkwatch_run_queue(void);
 
-unsigned long netdev_increment_features(unsigned long all, unsigned long one,
-                                        unsigned long mask);
-unsigned long netdev_fix_features(unsigned long features, const char *name);
+static inline u32 netdev_get_wanted_features(struct net_device *dev)
+{
+        return (dev->features & ~dev->hw_features) | dev->wanted_features;
+}
+u32 netdev_increment_features(u32 all, u32 one, u32 mask);
+u32 netdev_fix_features(struct net_device *dev, u32 features);
+int __netdev_update_features(struct net_device *dev);
+void netdev_update_features(struct net_device *dev);
+void netdev_change_features(struct net_device *dev);
 
 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
                                         struct net_device *dev);
 
-static inline int net_gso_ok(int features, int gso_type)
+u32 netif_skb_features(struct sk_buff *skb);
+
+static inline int net_gso_ok(u32 features, int gso_type)
 {
         int feature = gso_type << NETIF_F_GSO_SHIFT;
         return (features & feature) == feature;
 }
 
-static inline int skb_gso_ok(struct sk_buff *skb, int features)
+static inline int skb_gso_ok(struct sk_buff *skb, u32 features)
 {
         return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
-               (!skb_has_frags(skb) || (features & NETIF_F_FRAGLIST));
+               (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
 }
 
-static inline int netif_needs_gso(struct net_device *dev, struct sk_buff *skb)
+static inline int netif_needs_gso(struct sk_buff *skb, int features)
 {
-        return skb_is_gso(skb) &&
-               (!skb_gso_ok(skb, dev->features) ||
+        return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
                 unlikely(skb->ip_summed != CHECKSUM_PARTIAL));
 }
 
@@ -2207,29 +2592,20 @@ static inline void netif_set_gso_max_size(struct net_device *dev,
         dev->gso_max_size = size;
 }
 
-extern int __skb_bond_should_drop(struct sk_buff *skb,
-                                  struct net_device *master);
-
-static inline int skb_bond_should_drop(struct sk_buff *skb,
-                                       struct net_device *master)
+static inline int netif_is_bond_slave(struct net_device *dev)
 {
-        if (master)
-                return __skb_bond_should_drop(skb, master);
-        return 0;
+        return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
 }
 
 extern struct pernet_operations __net_initdata loopback_net_ops;
 
-static inline int dev_ethtool_get_settings(struct net_device *dev,
-                                           struct ethtool_cmd *cmd)
-{
-        if (!dev->ethtool_ops || !dev->ethtool_ops->get_settings)
-                return -EOPNOTSUPP;
-        return dev->ethtool_ops->get_settings(dev, cmd);
-}
+int dev_ethtool_get_settings(struct net_device *dev,
+                             struct ethtool_cmd *cmd);
 
 static inline u32 dev_ethtool_get_rx_csum(struct net_device *dev)
 {
+        if (dev->features & NETIF_F_RXCSUM)
+                return 1;
         if (!dev->ethtool_ops || !dev->ethtool_ops->get_rx_csum)
                 return 0;
         return dev->ethtool_ops->get_rx_csum(dev);
@@ -2271,6 +2647,9 @@ extern int netdev_notice(const struct net_device *dev, const char *format, ...)
 extern int netdev_info(const struct net_device *dev, const char *format, ...)
         __attribute__ ((format (printf, 2, 3)));
 
+#define MODULE_ALIAS_NETDEV(device) \
+        MODULE_ALIAS("netdev-" device)
+
 #if defined(DEBUG)
 #define netdev_dbg(__dev, format, args...)                      \
         netdev_printk(KERN_DEBUG, __dev, format, ##args)