Merge branch 'bonding_neighbours'

bonding: use neighbours instead of own lists

Veaceslav Falico says:

====================
This patchset introduces all the needed infrastructure, on top of current
adjacent lists, to be able to remove bond's slave_list/slave->list. The
overhead in memory/CPU is minimal, and after the patchset bonding can rely
on its slave-related functions, given the proper locking. I've done some
netperf benchmarks on a vm, and the delta was about 0.1gbps for 35gbps as a
whole, so no speed fluctuations.

It also automatically creates lower/upper and master symlinks in dev's
sysfs directory.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

11 files changed, 638 insertions, 444 deletions

diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c
index 0d8f427ade93..1337eafe4311 100644
--- a/drivers/net/bonding/bond_3ad.c
+++ b/drivers/net/bonding/bond_3ad.c
@@ -2117,7 +2117,7 @@ void bond_3ad_state_machine_handler(struct work_struct *work)
         read_lock(&bond->lock);
 
         //check if there are any slaves
-        if (list_empty(&bond->slave_list))
+        if (!bond_has_slaves(bond))
                 goto re_arm;
 
         // check if agg_select_timer timer after initialize is timed out
@@ -2417,14 +2417,15 @@ int bond_3ad_get_active_agg_info(struct bonding *bond, struct ad_info *ad_info)
 
 int bond_3ad_xmit_xor(struct sk_buff *skb, struct net_device *dev)
 {
-        struct slave *slave, *start_at;
         struct bonding *bond = netdev_priv(dev);
-        int slave_agg_no;
-        int slaves_in_agg;
-        int agg_id;
-        int i;
+        struct slave *slave, *first_ok_slave;
+        struct aggregator *agg;
         struct ad_info ad_info;
+        struct list_head *iter;
+        int slaves_in_agg;
+        int slave_agg_no;
         int res = 1;
+        int agg_id;
 
         read_lock(&bond->lock);
         if (__bond_3ad_get_active_agg_info(bond, &ad_info)) {
@@ -2437,20 +2438,28 @@ int bond_3ad_xmit_xor(struct sk_buff *skb, struct net_device *dev)
         agg_id = ad_info.aggregator_id;
 
         if (slaves_in_agg == 0) {
-                /*the aggregator is empty*/
                 pr_debug("%s: Error: active aggregator is empty\n", dev->name);
                 goto out;
         }
 
         slave_agg_no = bond->xmit_hash_policy(skb, slaves_in_agg);
+        first_ok_slave = NULL;
 
-        bond_for_each_slave(bond, slave) {
-                struct aggregator *agg = SLAVE_AD_INFO(slave).port.aggregator;
+        bond_for_each_slave(bond, slave, iter) {
+                agg = SLAVE_AD_INFO(slave).port.aggregator;
+                if (!agg || agg->aggregator_identifier != agg_id)
+                        continue;
 
-                if (agg && (agg->aggregator_identifier == agg_id)) {
+                if (slave_agg_no >= 0) {
+                        if (!first_ok_slave && SLAVE_IS_OK(slave))
+                                first_ok_slave = slave;
                         slave_agg_no--;
-                        if (slave_agg_no < 0)
-                                break;
+                        continue;
+                }
+
+                if (SLAVE_IS_OK(slave)) {
+                        res = bond_dev_queue_xmit(bond, skb, slave->dev);
+                        goto out;
                 }
         }
 
@@ -2460,20 +2469,10 @@ int bond_3ad_xmit_xor(struct sk_buff *skb, struct net_device *dev)
                 goto out;
         }
 
-        start_at = slave;
-
-        bond_for_each_slave_from(bond, slave, i, start_at) {
-                int slave_agg_id = 0;
-                struct aggregator *agg = SLAVE_AD_INFO(slave).port.aggregator;
-
-                if (agg)
-                        slave_agg_id = agg->aggregator_identifier;
-
-                if (SLAVE_IS_OK(slave) && agg && (slave_agg_id == agg_id)) {
-                        res = bond_dev_queue_xmit(bond, skb, slave->dev);
-                        break;
-                }
-        }
+        /* we couldn't find any suitable slave after the agg_no, so use the
+         * first suitable found, if found. */
+        if (first_ok_slave)
+                res = bond_dev_queue_xmit(bond, skb, first_ok_slave->dev);
 
 out:
         read_unlock(&bond->lock);
@@ -2515,11 +2514,12 @@ int bond_3ad_lacpdu_recv(const struct sk_buff *skb, struct bonding *bond,
 void bond_3ad_update_lacp_rate(struct bonding *bond)
 {
         struct port *port = NULL;
+        struct list_head *iter;
         struct slave *slave;
         int lacp_fast;
 
         lacp_fast = bond->params.lacp_fast;
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 port = &(SLAVE_AD_INFO(slave).port);
                 __get_state_machine_lock(port);
                 if (lacp_fast)
diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c
index f428ef574372..e96041816b5b 100644
--- a/drivers/net/bonding/bond_alb.c
+++ b/drivers/net/bonding/bond_alb.c
@@ -223,13 +223,14 @@ static long long compute_gap(struct slave *slave)
 static struct slave *tlb_get_least_loaded_slave(struct bonding *bond)
 {
         struct slave *slave, *least_loaded;
+        struct list_head *iter;
         long long max_gap;
 
         least_loaded = NULL;
         max_gap = LLONG_MIN;
 
         /* Find the slave with the largest gap */
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (SLAVE_IS_OK(slave)) {
                         long long gap = compute_gap(slave);
 
@@ -382,30 +383,31 @@ out:
 static struct slave *rlb_next_rx_slave(struct bonding *bond)
 {
         struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond));
-        struct slave *rx_slave, *slave, *start_at;
-        int i = 0;
-
-        if (bond_info->next_rx_slave)
-                start_at = bond_info->next_rx_slave;
-        else
-                start_at = bond_first_slave(bond);
-
-        rx_slave = NULL;
+        struct slave *before = NULL, *rx_slave = NULL, *slave;
+        struct list_head *iter;
+        bool found = false;
 
-        bond_for_each_slave_from(bond, slave, i, start_at) {
-                if (SLAVE_IS_OK(slave)) {
-                        if (!rx_slave) {
-                                rx_slave = slave;
-                        } else if (slave->speed > rx_slave->speed) {
+        bond_for_each_slave(bond, slave, iter) {
+                if (!SLAVE_IS_OK(slave))
+                        continue;
+                if (!found) {
+                        if (!before || before->speed < slave->speed)
+                                before = slave;
+                } else {
+                        if (!rx_slave || rx_slave->speed < slave->speed)
                                 rx_slave = slave;
-                        }
                 }
+                if (slave == bond_info->rx_slave)
+                        found = true;
         }
+        /* we didn't find anything after the current or we have something
+         * better before and up to the current slave
+         */
+        if (!rx_slave || (before && rx_slave->speed < before->speed))
+                rx_slave = before;
 
-        if (rx_slave) {
-                slave = bond_next_slave(bond, rx_slave);
-                bond_info->next_rx_slave = slave;
-        }
+        if (rx_slave)
+                bond_info->rx_slave = rx_slave;
 
         return rx_slave;
 }
@@ -1019,7 +1021,7 @@ static void alb_send_learning_packets(struct slave *slave, u8 mac_addr[])
 
         /* loop through vlans and send one packet for each */
         rcu_read_lock();
-        netdev_for_each_upper_dev_rcu(bond->dev, upper, iter) {
+        netdev_for_each_all_upper_dev_rcu(bond->dev, upper, iter) {
                 if (upper->priv_flags & IFF_802_1Q_VLAN)
                         alb_send_lp_vid(slave, mac_addr,
                                         vlan_dev_vlan_id(upper));
@@ -1172,10 +1174,11 @@ static void alb_change_hw_addr_on_detach(struct bonding *bond, struct slave *sla
  */
 static int alb_handle_addr_collision_on_attach(struct bonding *bond, struct slave *slave)
 {
-        struct slave *tmp_slave1, *free_mac_slave = NULL;
         struct slave *has_bond_addr = bond->curr_active_slave;
+        struct slave *tmp_slave1, *free_mac_slave = NULL;
+        struct list_head *iter;
 
-        if (list_empty(&bond->slave_list)) {
+        if (!bond_has_slaves(bond)) {
                 /* this is the first slave */
                 return 0;
         }
@@ -1196,7 +1199,7 @@ static int alb_handle_addr_collision_on_attach(struct bonding *bond, struct slav
         /* The slave's address is equal to the address of the bond.
          * Search for a spare address in the bond for this slave.
          */
-        bond_for_each_slave(bond, tmp_slave1) {
+        bond_for_each_slave(bond, tmp_slave1, iter) {
                 if (!bond_slave_has_mac(bond, tmp_slave1->perm_hwaddr)) {
                         /* no slave has tmp_slave1's perm addr
                          * as its curr addr
@@ -1246,15 +1249,16 @@ static int alb_handle_addr_collision_on_attach(struct bonding *bond, struct slav
  */
 static int alb_set_mac_address(struct bonding *bond, void *addr)
 {
-        char tmp_addr[ETH_ALEN];
-        struct slave *slave;
+        struct slave *slave, *rollback_slave;
+        struct list_head *iter;
         struct sockaddr sa;
+        char tmp_addr[ETH_ALEN];
         int res;
 
         if (bond->alb_info.rlb_enabled)
                 return 0;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 /* save net_device's current hw address */
                 memcpy(tmp_addr, slave->dev->dev_addr, ETH_ALEN);
 
@@ -1274,10 +1278,12 @@ unwind:
         sa.sa_family = bond->dev->type;
 
         /* unwind from head to the slave that failed */
-        bond_for_each_slave_continue_reverse(bond, slave) {
-                memcpy(tmp_addr, slave->dev->dev_addr, ETH_ALEN);
-                dev_set_mac_address(slave->dev, &sa);
-                memcpy(slave->dev->dev_addr, tmp_addr, ETH_ALEN);
+        bond_for_each_slave(bond, rollback_slave, iter) {
+                if (rollback_slave == slave)
+                        break;
+                memcpy(tmp_addr, rollback_slave->dev->dev_addr, ETH_ALEN);
+                dev_set_mac_address(rollback_slave->dev, &sa);
+                memcpy(rollback_slave->dev->dev_addr, tmp_addr, ETH_ALEN);
         }
 
         return res;
@@ -1458,11 +1464,12 @@ void bond_alb_monitor(struct work_struct *work)
         struct bonding *bond = container_of(work, struct bonding,
                                             alb_work.work);
         struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond));
+        struct list_head *iter;
         struct slave *slave;
 
         read_lock(&bond->lock);
 
-        if (list_empty(&bond->slave_list)) {
+        if (!bond_has_slaves(bond)) {
                 bond_info->tx_rebalance_counter = 0;
                 bond_info->lp_counter = 0;
                 goto re_arm;
@@ -1480,7 +1487,7 @@ void bond_alb_monitor(struct work_struct *work)
                  */
                 read_lock(&bond->curr_slave_lock);
 
-                bond_for_each_slave(bond, slave)
+                bond_for_each_slave(bond, slave, iter)
                         alb_send_learning_packets(slave, slave->dev->dev_addr);
 
                 read_unlock(&bond->curr_slave_lock);
@@ -1493,7 +1500,7 @@ void bond_alb_monitor(struct work_struct *work)
 
                 read_lock(&bond->curr_slave_lock);
 
-                bond_for_each_slave(bond, slave) {
+                bond_for_each_slave(bond, slave, iter) {
                         tlb_clear_slave(bond, slave, 1);
                         if (slave == bond->curr_active_slave) {
                                 SLAVE_TLB_INFO(slave).load =
@@ -1599,13 +1606,13 @@ int bond_alb_init_slave(struct bonding *bond, struct slave *slave)
  */
 void bond_alb_deinit_slave(struct bonding *bond, struct slave *slave)
 {
-        if (!list_empty(&bond->slave_list))
+        if (bond_has_slaves(bond))
                 alb_change_hw_addr_on_detach(bond, slave);
 
         tlb_clear_slave(bond, slave, 0);
 
         if (bond->alb_info.rlb_enabled) {
-                bond->alb_info.next_rx_slave = NULL;
+                bond->alb_info.rx_slave = NULL;
                 rlb_clear_slave(bond, slave);
         }
 }
@@ -1669,7 +1676,7 @@ void bond_alb_handle_active_change(struct bonding *bond, struct slave *new_slave
         swap_slave = bond->curr_active_slave;
         rcu_assign_pointer(bond->curr_active_slave, new_slave);
 
-        if (!new_slave || list_empty(&bond->slave_list))
+        if (!new_slave || !bond_has_slaves(bond))
                 return;
 
         /* set the new curr_active_slave to the bonds mac address
diff --git a/drivers/net/bonding/bond_alb.h b/drivers/net/bonding/bond_alb.h
index c5eff5dafdfe..4226044efd08 100644
--- a/drivers/net/bonding/bond_alb.h
+++ b/drivers/net/bonding/bond_alb.h
@@ -154,9 +154,7 @@ struct alb_bond_info {
         u8                      rx_ntt; /* flag - need to transmit
                                          * to all rx clients
                                          */
-        struct slave            *next_rx_slave;/* next slave to be assigned
-                                                * to a new rx client for
-                                                */
+        struct slave            *rx_slave;/* last slave to xmit from */
         u8                      primary_is_promisc;        /* boolean */
         u32                     rlb_promisc_timeout_counter;/* counts primary
                                                              * promiscuity time
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 55bbb8b8200c..d5c3153226b7 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -332,10 +332,11 @@ static int bond_vlan_rx_add_vid(struct net_device *bond_dev,
                                 __be16 proto, u16 vid)
 {
         struct bonding *bond = netdev_priv(bond_dev);
-        struct slave *slave;
+        struct slave *slave, *rollback_slave;
+        struct list_head *iter;
         int res;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 res = vlan_vid_add(slave->dev, proto, vid);
                 if (res)
                         goto unwind;
@@ -344,9 +345,13 @@ static int bond_vlan_rx_add_vid(struct net_device *bond_dev,
         return 0;
 
 unwind:
-        /* unwind from the slave that failed */
-        bond_for_each_slave_continue_reverse(bond, slave)
-                vlan_vid_del(slave->dev, proto, vid);
+        /* unwind to the slave that failed */
+        bond_for_each_slave(bond, rollback_slave, iter) {
+                if (rollback_slave == slave)
+                        break;
+
+                vlan_vid_del(rollback_slave->dev, proto, vid);
+        }
 
         return res;
 }
@@ -360,9 +365,10 @@ static int bond_vlan_rx_kill_vid(struct net_device *bond_dev,
                                  __be16 proto, u16 vid)
 {
         struct bonding *bond = netdev_priv(bond_dev);
+        struct list_head *iter;
         struct slave *slave;
 
-        bond_for_each_slave(bond, slave)
+        bond_for_each_slave(bond, slave, iter)
                 vlan_vid_del(slave->dev, proto, vid);
 
         if (bond_is_lb(bond))
@@ -382,15 +388,16 @@ static int bond_vlan_rx_kill_vid(struct net_device *bond_dev,
  */
 static int bond_set_carrier(struct bonding *bond)
 {
+        struct list_head *iter;
         struct slave *slave;
 
-        if (list_empty(&bond->slave_list))
+        if (!bond_has_slaves(bond))
                 goto down;
 
         if (bond->params.mode == BOND_MODE_8023AD)
                 return bond_3ad_set_carrier(bond);
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (slave->link == BOND_LINK_UP) {
                         if (!netif_carrier_ok(bond->dev)) {
                                 netif_carrier_on(bond->dev);
@@ -522,7 +529,9 @@ static int bond_check_dev_link(struct bonding *bond,
  */
 static int bond_set_promiscuity(struct bonding *bond, int inc)
 {
+        struct list_head *iter;
         int err = 0;
+
         if (USES_PRIMARY(bond->params.mode)) {
                 /* write lock already acquired */
                 if (bond->curr_active_slave) {
@@ -532,7 +541,7 @@ static int bond_set_promiscuity(struct bonding *bond, int inc)
         } else {
                 struct slave *slave;
 
-                bond_for_each_slave(bond, slave) {
+                bond_for_each_slave(bond, slave, iter) {
                         err = dev_set_promiscuity(slave->dev, inc);
                         if (err)
                                 return err;
@@ -546,7 +555,9 @@ static int bond_set_promiscuity(struct bonding *bond, int inc)
  */
 static int bond_set_allmulti(struct bonding *bond, int inc)
 {
+        struct list_head *iter;
         int err = 0;
+
         if (USES_PRIMARY(bond->params.mode)) {
                 /* write lock already acquired */
                 if (bond->curr_active_slave) {
@@ -556,7 +567,7 @@ static int bond_set_allmulti(struct bonding *bond, int inc)
         } else {
                 struct slave *slave;
 
-                bond_for_each_slave(bond, slave) {
+                bond_for_each_slave(bond, slave, iter) {
                         err = dev_set_allmulti(slave->dev, inc);
                         if (err)
                                 return err;
@@ -774,43 +785,24 @@ static bool bond_should_change_active(struct bonding *bond)
 /**
  * find_best_interface - select the best available slave to be the active one
  * @bond: our bonding struct
- *
- * Warning: Caller must hold curr_slave_lock for writing.
  */
 static struct slave *bond_find_best_slave(struct bonding *bond)
 {
-        struct slave *new_active, *old_active;
-        struct slave *bestslave = NULL;
+        struct slave *slave, *bestslave = NULL;
+        struct list_head *iter;
         int mintime = bond->params.updelay;
-        int i;
 
-        new_active = bond->curr_active_slave;
-
-        if (!new_active) { /* there were no active slaves left */
-                new_active = bond_first_slave(bond);
-                if (!new_active)
-                        return NULL; /* still no slave, return NULL */
-        }
-
-        if ((bond->primary_slave) &&
-            bond->primary_slave->link == BOND_LINK_UP &&
-            bond_should_change_active(bond)) {
-                new_active = bond->primary_slave;
-        }
-
-        /* remember where to stop iterating over the slaves */
-        old_active = new_active;
-
-        bond_for_each_slave_from(bond, new_active, i, old_active) {
-                if (new_active->link == BOND_LINK_UP) {
-                        return new_active;
-                } else if (new_active->link == BOND_LINK_BACK &&
-                           IS_UP(new_active->dev)) {
-                        /* link up, but waiting for stabilization */
-                        if (new_active->delay < mintime) {
-                                mintime = new_active->delay;
-                                bestslave = new_active;
-                        }
+        if (bond->primary_slave && bond->primary_slave->link == BOND_LINK_UP &&
+            bond_should_change_active(bond))
+                return bond->primary_slave;
+
+        bond_for_each_slave(bond, slave, iter) {
+                if (slave->link == BOND_LINK_UP)
+                        return slave;
+                if (slave->link == BOND_LINK_BACK && IS_UP(slave->dev) &&
+                    slave->delay < mintime) {
+                        mintime = slave->delay;
+                        bestslave = slave;
                 }
         }
 
@@ -980,7 +972,6 @@ void bond_select_active_slave(struct bonding *bond)
  */
 static void bond_attach_slave(struct bonding *bond, struct slave *new_slave)
 {
-        list_add_tail_rcu(&new_slave->list, &bond->slave_list);
         bond->slave_cnt++;
 }
 
@@ -996,7 +987,6 @@ static void bond_attach_slave(struct bonding *bond, struct slave *new_slave)
  */
 static void bond_detach_slave(struct bonding *bond, struct slave *slave)
 {
-        list_del_rcu(&slave->list);
         bond->slave_cnt--;
 }
 
@@ -1046,9 +1036,10 @@ static void bond_poll_controller(struct net_device *bond_dev)
 static void bond_netpoll_cleanup(struct net_device *bond_dev)
 {
         struct bonding *bond = netdev_priv(bond_dev);
+        struct list_head *iter;
         struct slave *slave;
 
-        bond_for_each_slave(bond, slave)
+        bond_for_each_slave(bond, slave, iter)
                 if (IS_UP(slave->dev))
                         slave_disable_netpoll(slave);
 }
@@ -1056,10 +1047,11 @@ static void bond_netpoll_cleanup(struct net_device *bond_dev)
 static int bond_netpoll_setup(struct net_device *dev, struct netpoll_info *ni, gfp_t gfp)
 {
         struct bonding *bond = netdev_priv(dev);
+        struct list_head *iter;
         struct slave *slave;
         int err = 0;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 err = slave_enable_netpoll(slave);
                 if (err) {
                         bond_netpoll_cleanup(dev);
@@ -1087,10 +1079,11 @@ static netdev_features_t bond_fix_features(struct net_device *dev,
                                            netdev_features_t features)
 {
         struct bonding *bond = netdev_priv(dev);
+        struct list_head *iter;
         netdev_features_t mask;
         struct slave *slave;
 
-        if (list_empty(&bond->slave_list)) {
+        if (!bond_has_slaves(bond)) {
                 /* Disable adding VLANs to empty bond. But why? --mq */
                 features |= NETIF_F_VLAN_CHALLENGED;
                 return features;
@@ -1100,7 +1093,7 @@ static netdev_features_t bond_fix_features(struct net_device *dev,
         features &= ~NETIF_F_ONE_FOR_ALL;
         features |= NETIF_F_ALL_FOR_ALL;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 features = netdev_increment_features(features,
                                                      slave->dev->features,
                                                      mask);
@@ -1118,16 +1111,17 @@ static void bond_compute_features(struct bonding *bond)
 {
         unsigned int flags, dst_release_flag = IFF_XMIT_DST_RELEASE;
         netdev_features_t vlan_features = BOND_VLAN_FEATURES;
+        struct net_device *bond_dev = bond->dev;
+        struct list_head *iter;
+        struct slave *slave;
         unsigned short max_hard_header_len = ETH_HLEN;
         unsigned int gso_max_size = GSO_MAX_SIZE;
-        struct net_device *bond_dev = bond->dev;
         u16 gso_max_segs = GSO_MAX_SEGS;
-        struct slave *slave;
 
-        if (list_empty(&bond->slave_list))
+        if (!bond_has_slaves(bond))
                 goto done;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 vlan_features = netdev_increment_features(vlan_features,
                         slave->dev->vlan_features, BOND_VLAN_FEATURES);
 
@@ -1233,11 +1227,12 @@ static rx_handler_result_t bond_handle_frame(struct sk_buff **pskb)
 }
 
 static int bond_master_upper_dev_link(struct net_device *bond_dev,
-                                      struct net_device *slave_dev)
+                                      struct net_device *slave_dev,
+                                      struct slave *slave)
 {
         int err;
 
-        err = netdev_master_upper_dev_link(slave_dev, bond_dev);
+        err = netdev_master_upper_dev_link_private(slave_dev, bond_dev, slave);
         if (err)
                 return err;
         slave_dev->flags |= IFF_SLAVE;
@@ -1258,7 +1253,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
 {
         struct bonding *bond = netdev_priv(bond_dev);
         const struct net_device_ops *slave_ops = slave_dev->netdev_ops;
-        struct slave *new_slave = NULL;
+        struct slave *new_slave = NULL, *prev_slave;
         struct sockaddr addr;
         int link_reporting;
         int res = 0, i;
@@ -1313,7 +1308,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
          * bond ether type mutual exclusion - don't allow slaves of dissimilar
          * ether type (eg ARPHRD_ETHER and ARPHRD_INFINIBAND) share the same bond
          */
-        if (list_empty(&bond->slave_list)) {
+        if (!bond_has_slaves(bond)) {
                 if (bond_dev->type != slave_dev->type) {
                         pr_debug("%s: change device type from %d to %d\n",
                                  bond_dev->name,
@@ -1352,7 +1347,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
         }
 
         if (slave_ops->ndo_set_mac_address == NULL) {
-                if (list_empty(&bond->slave_list)) {
+                if (!bond_has_slaves(bond)) {
                         pr_warning("%s: Warning: The first slave device specified does not support setting the MAC address. Setting fail_over_mac to active.",
                                    bond_dev->name);
                         bond->params.fail_over_mac = BOND_FOM_ACTIVE;
@@ -1368,7 +1363,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
 
         /* If this is the first slave, then we need to set the master's hardware
          * address to be the same as the slave's. */
-        if (list_empty(&bond->slave_list) &&
+        if (!bond_has_slaves(bond) &&
             bond->dev->addr_assign_type == NET_ADDR_RANDOM)
                 bond_set_dev_addr(bond->dev, slave_dev);
 
@@ -1377,7 +1372,6 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
                 res = -ENOMEM;
                 goto err_undo_flags;
         }
-        INIT_LIST_HEAD(&new_slave->list);
         /*
          * Set the new_slave's queue_id to be zero.  Queue ID mapping
          * is set via sysfs or module option if desired.
@@ -1413,17 +1407,11 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
                 }
         }
 
-        res = bond_master_upper_dev_link(bond_dev, slave_dev);
-        if (res) {
-                pr_debug("Error %d calling bond_master_upper_dev_link\n", res);
-                goto err_restore_mac;
-        }
-
         /* open the slave since the application closed it */
         res = dev_open(slave_dev);
         if (res) {
                 pr_debug("Opening slave %s failed\n", slave_dev->name);
-                goto err_unset_master;
+                goto err_restore_mac;
         }
 
         new_slave->bond = bond;
@@ -1481,6 +1469,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
 
         write_lock_bh(&bond->lock);
 
+        prev_slave = bond_last_slave(bond);
         bond_attach_slave(bond, new_slave);
 
         new_slave->delay = 0;
@@ -1575,9 +1564,6 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
                          */
                         bond_3ad_initialize(bond, 1000/AD_TIMER_INTERVAL);
                 } else {
-                        struct slave *prev_slave;
-
-                        prev_slave = bond_prev_slave(bond, new_slave);
                         SLAVE_AD_INFO(new_slave).id =
                                 SLAVE_AD_INFO(prev_slave).id + 1;
                 }
@@ -1626,17 +1612,20 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
 
         read_unlock(&bond->lock);
 
-        res = bond_create_slave_symlinks(bond_dev, slave_dev);
-        if (res)
-                goto err_detach;
-
         res = netdev_rx_handler_register(slave_dev, bond_handle_frame,
                                          new_slave);
         if (res) {
                 pr_debug("Error %d calling netdev_rx_handler_register\n", res);
-                goto err_dest_symlinks;
+                goto err_detach;
+        }
+
+        res = bond_master_upper_dev_link(bond_dev, slave_dev, new_slave);
+        if (res) {
+                pr_debug("Error %d calling bond_master_upper_dev_link\n", res);
+                goto err_unregister;
         }
 
+
         pr_info("%s: enslaving %s as a%s interface with a%s link.\n",
                 bond_dev->name, slave_dev->name,
                 bond_is_active_slave(new_slave) ? "n active" : " backup",
@@ -1646,8 +1635,8 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
         return 0;
 
 /* Undo stages on error */
-err_dest_symlinks:
-        bond_destroy_slave_symlinks(bond_dev, slave_dev);
+err_unregister:
+        netdev_rx_handler_unregister(slave_dev);
 
 err_detach:
         if (!USES_PRIMARY(bond->params.mode))
@@ -1675,9 +1664,6 @@ err_close:
         slave_dev->priv_flags &= ~IFF_BONDING;
         dev_close(slave_dev);
 
-err_unset_master:
-        bond_upper_dev_unlink(bond_dev, slave_dev);
-
 err_restore_mac:
         if (!bond->params.fail_over_mac) {
                 /* XXX TODO - fom follow mode needs to change master's
@@ -1698,7 +1684,7 @@ err_free:
 err_undo_flags:
         bond_compute_features(bond);
         /* Enslave of first slave has failed and we need to fix master's mac */
-        if (list_empty(&bond->slave_list) &&
+        if (!bond_has_slaves(bond) &&
             ether_addr_equal(bond_dev->dev_addr, slave_dev->dev_addr))
                 eth_hw_addr_random(bond_dev);
 
@@ -1748,6 +1734,8 @@ static int __bond_release_one(struct net_device *bond_dev,
         }
 
         write_unlock_bh(&bond->lock);
+
+        bond_upper_dev_unlink(bond_dev, slave_dev);
         /* unregister rx_handler early so bond_handle_frame wouldn't be called
          * for this slave anymore.
          */
@@ -1776,7 +1764,7 @@ static int __bond_release_one(struct net_device *bond_dev,
 
         if (!all && !bond->params.fail_over_mac) {
                 if (ether_addr_equal(bond_dev->dev_addr, slave->perm_hwaddr) &&
-                    !list_empty(&bond->slave_list))
+                    bond_has_slaves(bond))
                         pr_warn("%s: Warning: the permanent HWaddr of %s - %pM - is still in use by %s. Set the HWaddr of %s to a different address to avoid conflicts.\n",
                                    bond_dev->name, slave_dev->name,
                                    slave->perm_hwaddr,
@@ -1819,7 +1807,7 @@ static int __bond_release_one(struct net_device *bond_dev,
                 write_lock_bh(&bond->lock);
         }
 
-        if (list_empty(&bond->slave_list)) {
+        if (!bond_has_slaves(bond)) {
                 bond_set_carrier(bond);
                 eth_hw_addr_random(bond_dev);
 
@@ -1835,7 +1823,7 @@ static int __bond_release_one(struct net_device *bond_dev,
         unblock_netpoll_tx();
         synchronize_rcu();
 
-        if (list_empty(&bond->slave_list)) {
+        if (!bond_has_slaves(bond)) {
                 call_netdevice_notifiers(NETDEV_CHANGEADDR, bond->dev);
                 call_netdevice_notifiers(NETDEV_RELEASE, bond->dev);
         }
@@ -1847,8 +1835,6 @@ static int __bond_release_one(struct net_device *bond_dev,
                         bond_dev->name, slave_dev->name, bond_dev->name);
 
         /* must do this from outside any spinlocks */
-        bond_destroy_slave_symlinks(bond_dev, slave_dev);
-
         vlan_vids_del_by_dev(slave_dev, bond_dev);
 
         /* If the mode USES_PRIMARY, then this cases was handled above by
@@ -1866,8 +1852,6 @@ static int __bond_release_one(struct net_device *bond_dev,
                 bond_hw_addr_flush(bond_dev, slave_dev);
         }
 
-        bond_upper_dev_unlink(bond_dev, slave_dev);
-
         slave_disable_netpoll(slave);
 
         /* close slave before restoring its mac address */
@@ -1906,7 +1890,7 @@ static int  bond_release_and_destroy(struct net_device *bond_dev,
         int ret;
 
         ret = bond_release(bond_dev, slave_dev);
-        if (ret == 0 && list_empty(&bond->slave_list)) {
+        if (ret == 0 && !bond_has_slaves(bond)) {
                 bond_dev->priv_flags |= IFF_DISABLE_NETPOLL;
                 pr_info("%s: destroying bond %s.\n",
                         bond_dev->name, bond_dev->name);
@@ -1987,11 +1971,12 @@ static int bond_info_query(struct net_device *bond_dev, struct ifbond *info)
 static int bond_slave_info_query(struct net_device *bond_dev, struct ifslave *info)
 {
         struct bonding *bond = netdev_priv(bond_dev);
+        struct list_head *iter;
         int i = 0, res = -ENODEV;
         struct slave *slave;
 
         read_lock(&bond->lock);
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (i++ == (int)info->slave_id) {
                         res = 0;
                         strcpy(info->slave_name, slave->dev->name);
@@ -2012,12 +1997,13 @@ static int bond_slave_info_query(struct net_device *bond_dev, struct ifslave *in
 static int bond_miimon_inspect(struct bonding *bond)
 {
         int link_state, commit = 0;
+        struct list_head *iter;
         struct slave *slave;
         bool ignore_updelay;
 
         ignore_updelay = !bond->curr_active_slave ? true : false;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 slave->new_link = BOND_LINK_NOCHANGE;
 
                 link_state = bond_check_dev_link(bond, slave->dev, 0);
@@ -2111,9 +2097,10 @@ static int bond_miimon_inspect(struct bonding *bond)
 
 static void bond_miimon_commit(struct bonding *bond)
 {
+        struct list_head *iter;
         struct slave *slave;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 switch (slave->new_link) {
                 case BOND_LINK_NOCHANGE:
                         continue;
@@ -2218,7 +2205,7 @@ void bond_mii_monitor(struct work_struct *work)
 
         delay = msecs_to_jiffies(bond->params.miimon);
 
-        if (list_empty(&bond->slave_list))
+        if (!bond_has_slaves(bond))
                 goto re_arm;
 
         should_notify_peers = bond_should_notify_peers(bond);
@@ -2267,7 +2254,7 @@ static bool bond_has_this_ip(struct bonding *bond, __be32 ip)
                 return true;
 
         rcu_read_lock();
-        netdev_for_each_upper_dev_rcu(bond->dev, upper, iter) {
+        netdev_for_each_all_upper_dev_rcu(bond->dev, upper, iter) {
                 if (ip == bond_confirm_addr(upper, 0, ip)) {
                         ret = true;
                         break;
@@ -2342,10 +2329,12 @@ static void bond_arp_send_all(struct bonding *bond, struct slave *slave)
                  *
                  * TODO: QinQ?
                  */
-                netdev_for_each_upper_dev_rcu(bond->dev, vlan_upper, vlan_iter) {
+                netdev_for_each_all_upper_dev_rcu(bond->dev, vlan_upper,
+                                                  vlan_iter) {
                         if (!is_vlan_dev(vlan_upper))
                                 continue;
-                        netdev_for_each_upper_dev_rcu(vlan_upper, upper, iter) {
+                        netdev_for_each_all_upper_dev_rcu(vlan_upper, upper,
+                                                          iter) {
                                 if (upper == rt->dst.dev) {
                                         vlan_id = vlan_dev_vlan_id(vlan_upper);
                                         rcu_read_unlock();
@@ -2358,7 +2347,7 @@ static void bond_arp_send_all(struct bonding *bond, struct slave *slave)
                  * our upper vlans, then just search for any dev that
                  * matches, and in case it's a vlan - save the id
                  */
-                netdev_for_each_upper_dev_rcu(bond->dev, upper, iter) {
+                netdev_for_each_all_upper_dev_rcu(bond->dev, upper, iter) {
                         if (upper == rt->dst.dev) {
                                 /* if it's a vlan - get its VID */
                                 if (is_vlan_dev(upper))
@@ -2505,11 +2494,12 @@ void bond_loadbalance_arp_mon(struct work_struct *work)
         struct bonding *bond = container_of(work, struct bonding,
                                             arp_work.work);
         struct slave *slave, *oldcurrent;
+        struct list_head *iter;
         int do_failover = 0;
 
         read_lock(&bond->lock);
 
-        if (list_empty(&bond->slave_list))
+        if (!bond_has_slaves(bond))
                 goto re_arm;
 
         oldcurrent = bond->curr_active_slave;
@@ -2521,7 +2511,7 @@ void bond_loadbalance_arp_mon(struct work_struct *work)
          * TODO: what about up/down delay in arp mode? it wasn't here before
          *       so it can wait
          */
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 unsigned long trans_start = dev_trans_start(slave->dev);
 
                 if (slave->link != BOND_LINK_UP) {
@@ -2612,10 +2602,11 @@ re_arm:
 static int bond_ab_arp_inspect(struct bonding *bond)
 {
         unsigned long trans_start, last_rx;
+        struct list_head *iter;
         struct slave *slave;
         int commit = 0;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 slave->new_link = BOND_LINK_NOCHANGE;
                 last_rx = slave_last_rx(bond, slave);
 
@@ -2682,9 +2673,10 @@ static int bond_ab_arp_inspect(struct bonding *bond)
 static void bond_ab_arp_commit(struct bonding *bond)
 {
         unsigned long trans_start;
+        struct list_head *iter;
         struct slave *slave;
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 switch (slave->new_link) {
                 case BOND_LINK_NOCHANGE:
                         continue;
@@ -2755,8 +2747,9 @@ do_failover:
  */
 static void bond_ab_arp_probe(struct bonding *bond)
 {
-        struct slave *slave, *next_slave;
-        int i;
+        struct slave *slave, *before = NULL, *new_slave = NULL;
+        struct list_head *iter;
+        bool found = false;
 
         read_lock(&bond->curr_slave_lock);
 
@@ -2786,18 +2779,12 @@ static void bond_ab_arp_probe(struct bonding *bond)
 
         bond_set_slave_inactive_flags(bond->current_arp_slave);
 
-        /* search for next candidate */
-        next_slave = bond_next_slave(bond, bond->current_arp_slave);
-        bond_for_each_slave_from(bond, slave, i, next_slave) {
-                if (IS_UP(slave->dev)) {
-                        slave->link = BOND_LINK_BACK;
-                        bond_set_slave_active_flags(slave);
-                        bond_arp_send_all(bond, slave);
-                        slave->jiffies = jiffies;
-                        bond->current_arp_slave = slave;
-                        break;
-                }
+        bond_for_each_slave(bond, slave, iter) {
+                if (!found && !before && IS_UP(slave->dev))
+                        before = slave;
 
+                if (found && !new_slave && IS_UP(slave->dev))
+                        new_slave = slave;
                 /* if the link state is up at this point, we
                  * mark it down - this can happen if we have
                  * simultaneous link failures and
@@ -2805,7 +2792,7 @@ static void bond_ab_arp_probe(struct bonding *bond)
                  * one the current slave so it is still marked
                  * up when it is actually down
                  */
-                if (slave->link == BOND_LINK_UP) {
+                if (!IS_UP(slave->dev) && slave->link == BOND_LINK_UP) {
                         slave->link = BOND_LINK_DOWN;
                         if (slave->link_failure_count < UINT_MAX)
                                 slave->link_failure_count++;
@@ -2815,7 +2802,22 @@ static void bond_ab_arp_probe(struct bonding *bond)
                         pr_info("%s: backup interface %s is now down.\n",
                                 bond->dev->name, slave->dev->name);
                 }
+                if (slave == bond->current_arp_slave)
+                        found = true;
         }
+
+        if (!new_slave && before)
+                new_slave = before;
+
+        if (!new_slave)
+                return;
+
+        new_slave->link = BOND_LINK_BACK;
+        bond_set_slave_active_flags(new_slave);
+        bond_arp_send_all(bond, new_slave);
+        new_slave->jiffies = jiffies;
+        bond->current_arp_slave = new_slave;
+
 }
 
 void bond_activebackup_arp_mon(struct work_struct *work)
@@ -2829,7 +2831,7 @@ void bond_activebackup_arp_mon(struct work_struct *work)
 
         delta_in_ticks = msecs_to_jiffies(bond->params.arp_interval);
 
-        if (list_empty(&bond->slave_list))
+        if (!bond_has_slaves(bond))
                 goto re_arm;
 
         should_notify_peers = bond_should_notify_peers(bond);
@@ -3148,13 +3150,14 @@ static void bond_work_cancel_all(struct bonding *bond)
 static int bond_open(struct net_device *bond_dev)
 {
         struct bonding *bond = netdev_priv(bond_dev);
+        struct list_head *iter;
         struct slave *slave;
 
         /* reset slave->backup and slave->inactive */
         read_lock(&bond->lock);
-        if (!list_empty(&bond->slave_list)) {
+        if (bond_has_slaves(bond)) {
                 read_lock(&bond->curr_slave_lock);
-                bond_for_each_slave(bond, slave) {
+                bond_for_each_slave(bond, slave, iter) {
                         if ((bond->params.mode == BOND_MODE_ACTIVEBACKUP)
                                 && (slave != bond->curr_active_slave)) {
                                 bond_set_slave_inactive_flags(slave);
@@ -3214,12 +3217,13 @@ static struct rtnl_link_stats64 *bond_get_stats(struct net_device *bond_dev,
 {
         struct bonding *bond = netdev_priv(bond_dev);
         struct rtnl_link_stats64 temp;
+        struct list_head *iter;
         struct slave *slave;
 
         memset(stats, 0, sizeof(*stats));
 
         read_lock_bh(&bond->lock);
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 const struct rtnl_link_stats64 *sstats =
                         dev_get_stats(slave->dev, &temp);
 
@@ -3386,6 +3390,7 @@ static void bond_change_rx_flags(struct net_device *bond_dev, int change)
 static void bond_set_rx_mode(struct net_device *bond_dev)
 {
         struct bonding *bond = netdev_priv(bond_dev);
+        struct list_head *iter;
         struct slave *slave;
 
         ASSERT_RTNL();
@@ -3397,7 +3402,7 @@ static void bond_set_rx_mode(struct net_device *bond_dev)
                         dev_mc_sync(slave->dev, bond_dev);
                 }
         } else {
-                bond_for_each_slave(bond, slave) {
+                bond_for_each_slave(bond, slave, iter) {
                         dev_uc_sync_multiple(slave->dev, bond_dev);
                         dev_mc_sync_multiple(slave->dev, bond_dev);
                 }
@@ -3464,7 +3469,8 @@ static int bond_neigh_setup(struct net_device *dev,
 static int bond_change_mtu(struct net_device *bond_dev, int new_mtu)
 {
         struct bonding *bond = netdev_priv(bond_dev);
-        struct slave *slave;
+        struct slave *slave, *rollback_slave;
+        struct list_head *iter;
         int res = 0;
 
         pr_debug("bond=%p, name=%s, new_mtu=%d\n", bond,
@@ -3485,10 +3491,9 @@ static int bond_change_mtu(struct net_device *bond_dev, int new_mtu)
          * call to the base driver.
          */
 
-        bond_for_each_slave(bond, slave) {
-                pr_debug("s %p s->p %p c_m %p\n",
+        bond_for_each_slave(bond, slave, iter) {
+                pr_debug("s %p c_m %p\n",
                          slave,
-                         bond_prev_slave(bond, slave),
                          slave->dev->netdev_ops->ndo_change_mtu);
 
                 res = dev_set_mtu(slave->dev, new_mtu);
@@ -3513,13 +3518,16 @@ static int bond_change_mtu(struct net_device *bond_dev, int new_mtu)
 
 unwind:
         /* unwind from head to the slave that failed */
-        bond_for_each_slave_continue_reverse(bond, slave) {
+        bond_for_each_slave(bond, rollback_slave, iter) {
                 int tmp_res;
 
-                tmp_res = dev_set_mtu(slave->dev, bond_dev->mtu);
+                if (rollback_slave == slave)
+                        break;
+
+                tmp_res = dev_set_mtu(rollback_slave->dev, bond_dev->mtu);
                 if (tmp_res) {
                         pr_debug("unwind err %d dev %s\n",
-                                 tmp_res, slave->dev->name);
+                                 tmp_res, rollback_slave->dev->name);
                 }
         }
 
@@ -3536,8 +3544,9 @@ unwind:
 static int bond_set_mac_address(struct net_device *bond_dev, void *addr)
 {
         struct bonding *bond = netdev_priv(bond_dev);
+        struct slave *slave, *rollback_slave;
         struct sockaddr *sa = addr, tmp_sa;
-        struct slave *slave;
+        struct list_head *iter;
         int res = 0;
 
         if (bond->params.mode == BOND_MODE_ALB)
@@ -3571,7 +3580,7 @@ static int bond_set_mac_address(struct net_device *bond_dev, void *addr)
          * call to the base driver.
          */
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 const struct net_device_ops *slave_ops = slave->dev->netdev_ops;
                 pr_debug("slave %p %s\n", slave, slave->dev->name);
 
@@ -3603,13 +3612,16 @@ unwind:
         tmp_sa.sa_family = bond_dev->type;
 
         /* unwind from head to the slave that failed */
-        bond_for_each_slave_continue_reverse(bond, slave) {
+        bond_for_each_slave(bond, rollback_slave, iter) {
                 int tmp_res;
 
-                tmp_res = dev_set_mac_address(slave->dev, &tmp_sa);
+                if (rollback_slave == slave)
+                        break;
+
+                tmp_res = dev_set_mac_address(rollback_slave->dev, &tmp_sa);
                 if (tmp_res) {
                         pr_debug("unwind err %d dev %s\n",
-                                 tmp_res, slave->dev->name);
+                                 tmp_res, rollback_slave->dev->name);
                 }
         }
 
@@ -3628,11 +3640,12 @@ unwind:
  */
 void bond_xmit_slave_id(struct bonding *bond, struct sk_buff *skb, int slave_id)
 {
+        struct list_head *iter;
         struct slave *slave;
         int i = slave_id;
 
         /* Here we start from the slave with slave_id */
-        bond_for_each_slave_rcu(bond, slave) {
+        bond_for_each_slave_rcu(bond, slave, iter) {
                 if (--i < 0) {
                         if (slave_can_tx(slave)) {
                                 bond_dev_queue_xmit(bond, skb, slave->dev);
@@ -3643,7 +3656,7 @@ void bond_xmit_slave_id(struct bonding *bond, struct sk_buff *skb, int slave_id)
 
         /* Here we start from the first slave up to slave_id */
         i = slave_id;
-        bond_for_each_slave_rcu(bond, slave) {
+        bond_for_each_slave_rcu(bond, slave, iter) {
                 if (--i < 0)
                         break;
                 if (slave_can_tx(slave)) {
@@ -3720,8 +3733,9 @@ static int bond_xmit_broadcast(struct sk_buff *skb, struct net_device *bond_dev)
 {
         struct bonding *bond = netdev_priv(bond_dev);
         struct slave *slave = NULL;
+        struct list_head *iter;
 
-        bond_for_each_slave_rcu(bond, slave) {
+        bond_for_each_slave_rcu(bond, slave, iter) {
                 if (bond_is_last_slave(bond, slave))
                         break;
                 if (IS_UP(slave->dev) && slave->link == BOND_LINK_UP) {
@@ -3770,13 +3784,14 @@ static inline int bond_slave_override(struct bonding *bond,
 {
         struct slave *slave = NULL;
         struct slave *check_slave;
+        struct list_head *iter;
         int res = 1;
 
         if (!skb->queue_mapping)
                 return 1;
 
         /* Find out if any slaves have the same mapping as this skb. */
-        bond_for_each_slave_rcu(bond, check_slave) {
+        bond_for_each_slave_rcu(bond, check_slave, iter) {
                 if (check_slave->queue_id == skb->queue_mapping) {
                         slave = check_slave;
                         break;
@@ -3862,7 +3877,7 @@ static netdev_tx_t bond_start_xmit(struct sk_buff *skb, struct net_device *dev)
                 return NETDEV_TX_BUSY;
 
         rcu_read_lock();
-        if (!list_empty(&bond->slave_list))
+        if (bond_has_slaves(bond))
                 ret = __bond_start_xmit(skb, dev);
         else
                 kfree_skb(skb);
@@ -3908,6 +3923,7 @@ static int bond_ethtool_get_settings(struct net_device *bond_dev,
 {
         struct bonding *bond = netdev_priv(bond_dev);
         unsigned long speed = 0;
+        struct list_head *iter;
         struct slave *slave;
 
         ecmd->duplex = DUPLEX_UNKNOWN;
@@ -3919,7 +3935,7 @@ static int bond_ethtool_get_settings(struct net_device *bond_dev,
          * this is an accurate maximum.
          */
         read_lock(&bond->lock);
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (SLAVE_IS_OK(slave)) {
                         if (slave->speed != SPEED_UNKNOWN)
                                 speed += slave->speed;
@@ -3994,7 +4010,6 @@ static void bond_setup(struct net_device *bond_dev)
         /* initialize rwlocks */
         rwlock_init(&bond->lock);
         rwlock_init(&bond->curr_slave_lock);
-        INIT_LIST_HEAD(&bond->slave_list);
         bond->params = bonding_defaults;
 
         /* Initialize pointers */
@@ -4050,12 +4065,13 @@ static void bond_setup(struct net_device *bond_dev)
 static void bond_uninit(struct net_device *bond_dev)
 {
         struct bonding *bond = netdev_priv(bond_dev);
-        struct slave *slave, *tmp_slave;
+        struct list_head *iter;
+        struct slave *slave;
 
         bond_netpoll_cleanup(bond_dev);
 
         /* Release the bonded slaves */
-        list_for_each_entry_safe(slave, tmp_slave, &bond->slave_list, list)
+        bond_for_each_slave(bond, slave, iter)
                 __bond_release_one(bond_dev, slave->dev, true);
         pr_info("%s: released all slaves\n", bond_dev->name);
 
diff --git a/drivers/net/bonding/bond_procfs.c b/drivers/net/bonding/bond_procfs.c
index 20a6ee25bb63..7af5646e4410 100644
--- a/drivers/net/bonding/bond_procfs.c
+++ b/drivers/net/bonding/bond_procfs.c
@@ -10,8 +10,9 @@ static void *bond_info_seq_start(struct seq_file *seq, loff_t *pos)
         __acquires(&bond->lock)
 {
         struct bonding *bond = seq->private;
-        loff_t off = 0;
+        struct list_head *iter;
         struct slave *slave;
+        loff_t off = 0;
 
         /* make sure the bond won't be taken away */
         rcu_read_lock();
@@ -20,7 +21,7 @@ static void *bond_info_seq_start(struct seq_file *seq, loff_t *pos)
         if (*pos == 0)
                 return SEQ_START_TOKEN;
 
-        bond_for_each_slave(bond, slave)
+        bond_for_each_slave(bond, slave, iter)
                 if (++off == *pos)
                         return slave;
 
diff --git a/drivers/net/bonding/bond_sysfs.c b/drivers/net/bonding/bond_sysfs.c
index c29b836749b6..e06c644470b1 100644
--- a/drivers/net/bonding/bond_sysfs.c
+++ b/drivers/net/bonding/bond_sysfs.c
@@ -168,41 +168,6 @@ static const struct class_attribute class_attr_bonding_masters = {
         .namespace = bonding_namespace,
 };
 
-int bond_create_slave_symlinks(struct net_device *master,
-                               struct net_device *slave)
-{
-        char linkname[IFNAMSIZ+7];
-        int ret = 0;
-
-        /* first, create a link from the slave back to the master */
-        ret = sysfs_create_link(&(slave->dev.kobj), &(master->dev.kobj),
-                                "master");
-        if (ret)
-                return ret;
-        /* next, create a link from the master to the slave */
-        sprintf(linkname, "slave_%s", slave->name);
-        ret = sysfs_create_link(&(master->dev.kobj), &(slave->dev.kobj),
-                                linkname);
-
-        /* free the master link created earlier in case of error */
-        if (ret)
-                sysfs_remove_link(&(slave->dev.kobj), "master");
-
-        return ret;
-
-}
-
-void bond_destroy_slave_symlinks(struct net_device *master,
-                                 struct net_device *slave)
-{
-        char linkname[IFNAMSIZ+7];
-
-        sysfs_remove_link(&(slave->dev.kobj), "master");
-        sprintf(linkname, "slave_%s", slave->name);
-        sysfs_remove_link(&(master->dev.kobj), linkname);
-}
-
-
 /*
  * Show the slaves in the current bond.
  */
@@ -210,11 +175,12 @@ static ssize_t bonding_show_slaves(struct device *d,
                                    struct device_attribute *attr, char *buf)
 {
         struct bonding *bond = to_bond(d);
+        struct list_head *iter;
         struct slave *slave;
         int res = 0;
 
         read_lock(&bond->lock);
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (res > (PAGE_SIZE - IFNAMSIZ)) {
                         /* not enough space for another interface name */
                         if ((PAGE_SIZE - res) > 10)
@@ -326,7 +292,7 @@ static ssize_t bonding_store_mode(struct device *d,
                 goto out;
         }
 
-        if (!list_empty(&bond->slave_list)) {
+        if (bond_has_slaves(bond)) {
                 pr_err("unable to update mode of %s because it has slaves.\n",
                         bond->dev->name);
                 ret = -EPERM;
@@ -522,7 +488,7 @@ static ssize_t bonding_store_fail_over_mac(struct device *d,
         if (!rtnl_trylock())
                 return restart_syscall();
 
-        if (!list_empty(&bond->slave_list)) {
+        if (bond_has_slaves(bond)) {
                 pr_err("%s: Can't alter fail_over_mac with slaves in bond.\n",
                        bond->dev->name);
                 ret = -EPERM;
@@ -656,6 +622,7 @@ static ssize_t bonding_store_arp_targets(struct device *d,
                                          const char *buf, size_t count)
 {
         struct bonding *bond = to_bond(d);
+        struct list_head *iter;
         struct slave *slave;
         __be32 newtarget, *targets;
         unsigned long *targets_rx;
@@ -688,7 +655,7 @@ static ssize_t bonding_store_arp_targets(struct device *d,
                          &newtarget);
                 /* not to race with bond_arp_rcv */
                 write_lock_bh(&bond->lock);
-                bond_for_each_slave(bond, slave)
+                bond_for_each_slave(bond, slave, iter)
                         slave->target_last_arp_rx[ind] = jiffies;
                 targets[ind] = newtarget;
                 write_unlock_bh(&bond->lock);
@@ -714,7 +681,7 @@ static ssize_t bonding_store_arp_targets(struct device *d,
                         &newtarget);
 
                 write_lock_bh(&bond->lock);
-                bond_for_each_slave(bond, slave) {
+                bond_for_each_slave(bond, slave, iter) {
                         targets_rx = slave->target_last_arp_rx;
                         j = ind;
                         for (; (j < BOND_MAX_ARP_TARGETS-1) && targets[j+1]; j++)
@@ -1111,6 +1078,7 @@ static ssize_t bonding_store_primary(struct device *d,
                                      const char *buf, size_t count)
 {
         struct bonding *bond = to_bond(d);
+        struct list_head *iter;
         char ifname[IFNAMSIZ];
         struct slave *slave;
 
@@ -1138,7 +1106,7 @@ static ssize_t bonding_store_primary(struct device *d,
                 goto out;
         }
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (strncmp(slave->dev->name, ifname, IFNAMSIZ) == 0) {
                         pr_info("%s: Setting %s as primary slave.\n",
                                 bond->dev->name, slave->dev->name);
@@ -1286,6 +1254,7 @@ static ssize_t bonding_store_active_slave(struct device *d,
 {
         struct slave *slave, *old_active, *new_active;
         struct bonding *bond = to_bond(d);
+        struct list_head *iter;
         char ifname[IFNAMSIZ];
 
         if (!rtnl_trylock())
@@ -1313,7 +1282,7 @@ static ssize_t bonding_store_active_slave(struct device *d,
                 goto out;
         }
 
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (strncmp(slave->dev->name, ifname, IFNAMSIZ) == 0) {
                         old_active = bond->curr_active_slave;
                         new_active = slave;
@@ -1493,6 +1462,7 @@ static ssize_t bonding_show_queue_id(struct device *d,
                                      char *buf)
 {
         struct bonding *bond = to_bond(d);
+        struct list_head *iter;
         struct slave *slave;
         int res = 0;
 
@@ -1500,7 +1470,7 @@ static ssize_t bonding_show_queue_id(struct device *d,
                 return restart_syscall();
 
         read_lock(&bond->lock);
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (res > (PAGE_SIZE - IFNAMSIZ - 6)) {
                         /* not enough space for another interface_name:queue_id pair */
                         if ((PAGE_SIZE - res) > 10)
@@ -1529,6 +1499,7 @@ static ssize_t bonding_store_queue_id(struct device *d,
 {
         struct slave *slave, *update_slave;
         struct bonding *bond = to_bond(d);
+        struct list_head *iter;
         u16 qid;
         int ret = count;
         char *delim;
@@ -1565,7 +1536,7 @@ static ssize_t bonding_store_queue_id(struct device *d,
 
         /* Search for thes slave and check for duplicate qids */
         update_slave = NULL;
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (sdev == slave->dev)
                         /*
                          * We don't need to check the matching
@@ -1619,6 +1590,7 @@ static ssize_t bonding_store_slaves_active(struct device *d,
 {
         struct bonding *bond = to_bond(d);
         int new_value, ret = count;
+        struct list_head *iter;
         struct slave *slave;
 
         if (sscanf(buf, "%d", &new_value) != 1) {
@@ -1641,7 +1613,7 @@ static ssize_t bonding_store_slaves_active(struct device *d,
         }
 
         read_lock(&bond->lock);
-        bond_for_each_slave(bond, slave) {
+        bond_for_each_slave(bond, slave, iter) {
                 if (!bond_is_active_slave(slave)) {
                         if (new_value)
                                 slave->inactive = 0;
diff --git a/drivers/net/bonding/bonding.h b/drivers/net/bonding/bonding.h
index 03cf3fd14490..5b71601666cd 100644
--- a/drivers/net/bonding/bonding.h
+++ b/drivers/net/bonding/bonding.h
@@ -72,63 +72,40 @@
         res; })
 
 /* slave list primitives */
-#define bond_to_slave(ptr) list_entry(ptr, struct slave, list)
+#define bond_slave_list(bond) (&(bond)->dev->adj_list.lower)
+
+#define bond_has_slaves(bond) !list_empty(bond_slave_list(bond))
 
 /* IMPORTANT: bond_first/last_slave can return NULL in case of an empty list */
 #define bond_first_slave(bond) \
-        list_first_entry_or_null(&(bond)->slave_list, struct slave, list)
+        (bond_has_slaves(bond) ? \
+                netdev_adjacent_get_private(bond_slave_list(bond)->next) : \
+                NULL)
 #define bond_last_slave(bond) \
-        (list_empty(&(bond)->slave_list) ? NULL : \
-                                           bond_to_slave((bond)->slave_list.prev))
+        (bond_has_slaves(bond) ? \
+                netdev_adjacent_get_private(bond_slave_list(bond)->prev) : \
+                NULL)
 
-#define bond_is_first_slave(bond, pos) ((pos)->list.prev == &(bond)->slave_list)
-#define bond_is_last_slave(bond, pos) ((pos)->list.next == &(bond)->slave_list)
+#define bond_is_first_slave(bond, pos) (pos == bond_first_slave(bond))
+#define bond_is_last_slave(bond, pos) (pos == bond_last_slave(bond))
 
 /* Since bond_first/last_slave can return NULL, these can return NULL too */
-#define bond_next_slave(bond, pos) \
-        (bond_is_last_slave(bond, pos) ? bond_first_slave(bond) : \
-                                         bond_to_slave((pos)->list.next))
-
-#define bond_prev_slave(bond, pos) \
-        (bond_is_first_slave(bond, pos) ? bond_last_slave(bond) : \
-                                          bond_to_slave((pos)->list.prev))
-
-/**
- * bond_for_each_slave_from - iterate the slaves list from a starting point
- * @bond:       the bond holding this list.
- * @pos:        current slave.
- * @cnt:        counter for max number of moves
- * @start:      starting point.
- *
- * Caller must hold bond->lock
- */
-#define bond_for_each_slave_from(bond, pos, cnt, start) \
-        for (cnt = 0, pos = start; pos && cnt < (bond)->slave_cnt; \
-             cnt++, pos = bond_next_slave(bond, pos))
+#define bond_next_slave(bond, pos) __bond_next_slave(bond, pos)
 
 /**
  * bond_for_each_slave - iterate over all slaves
  * @bond:       the bond holding this list
  * @pos:        current slave
+ * @iter:       list_head * iterator
  *
  * Caller must hold bond->lock
  */
-#define bond_for_each_slave(bond, pos) \
-        list_for_each_entry(pos, &(bond)->slave_list, list)
+#define bond_for_each_slave(bond, pos, iter) \
+        netdev_for_each_lower_private((bond)->dev, pos, iter)
 
 /* Caller must have rcu_read_lock */
-#define bond_for_each_slave_rcu(bond, pos) \
-        list_for_each_entry_rcu(pos, &(bond)->slave_list, list)
-
-/**
- * bond_for_each_slave_reverse - iterate in reverse from a given position
- * @bond:       the bond holding this list
- * @pos:        slave to continue from
- *
- * Caller must hold bond->lock
- */
-#define bond_for_each_slave_continue_reverse(bond, pos) \
-        list_for_each_entry_continue_reverse(pos, &(bond)->slave_list, list)
+#define bond_for_each_slave_rcu(bond, pos, iter) \
+        netdev_for_each_lower_private_rcu((bond)->dev, pos, iter)
 
 #ifdef CONFIG_NET_POLL_CONTROLLER
 extern atomic_t netpoll_block_tx;
@@ -188,7 +165,6 @@ struct bond_parm_tbl {
 
 struct slave {
         struct net_device *dev; /* first - useful for panic debug */
-        struct list_head list;
         struct bonding *bond; /* our master */
         int    delay;
         unsigned long jiffies;
@@ -228,7 +204,6 @@ struct slave {
  */
 struct bonding {
         struct   net_device *dev; /* first - useful for panic debug */
-        struct   list_head slave_list;
         struct   slave *curr_active_slave;
         struct   slave *current_arp_slave;
         struct   slave *primary_slave;
@@ -269,6 +244,34 @@ struct bonding {
         ((struct slave *) rtnl_dereference(dev->rx_handler_data))
 
 /**
+ * __bond_next_slave - get the next slave after the one provided
+ * @bond - bonding struct
+ * @slave - the slave provided
+ *
+ * Returns the next slave after the slave provided, first slave if the
+ * slave provided is the last slave and NULL if slave is not found
+ */
+static inline struct slave *__bond_next_slave(struct bonding *bond,
+                                              struct slave *slave)
+{
+        struct slave *slave_iter;
+        struct list_head *iter;
+        bool found = false;
+
+        netdev_for_each_lower_private(bond->dev, slave_iter, iter) {
+                if (found)
+                        return slave_iter;
+                if (slave_iter == slave)
+                        found = true;
+        }
+
+        if (found)
+                return bond_first_slave(bond);
+
+        return NULL;
+}
+
+/**
  * Returns NULL if the net_device does not belong to any of the bond's slaves
  *
  * Caller must hold bond lock for read
@@ -276,13 +279,7 @@ struct bonding {
 static inline struct slave *bond_get_slave_by_dev(struct bonding *bond,
                                                   struct net_device *slave_dev)
 {
-        struct slave *slave = NULL;
-
-        bond_for_each_slave(bond, slave)
-                if (slave->dev == slave_dev)
-                        return slave;
-
-        return NULL;
+        return netdev_lower_dev_get_private(bond->dev, slave_dev);
 }
 
 static inline struct bonding *bond_get_bond_by_slave(struct slave *slave)
@@ -439,8 +436,6 @@ int bond_create(struct net *net, const char *name);
 int bond_create_sysfs(struct bond_net *net);
 void bond_destroy_sysfs(struct bond_net *net);
 void bond_prepare_sysfs_group(struct bonding *bond);
-int bond_create_slave_symlinks(struct net_device *master, struct net_device *slave);
-void bond_destroy_slave_symlinks(struct net_device *master, struct net_device *slave);
 int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev);
 int bond_release(struct net_device *bond_dev, struct net_device *slave_dev);
 void bond_mii_monitor(struct work_struct *);
@@ -492,9 +487,10 @@ static inline void bond_destroy_proc_dir(struct bond_net *bn)
 static inline struct slave *bond_slave_has_mac(struct bonding *bond,
                                                const u8 *mac)
 {
+        struct list_head *iter;
         struct slave *tmp;
 
-        bond_for_each_slave(bond, tmp)
+        bond_for_each_slave(bond, tmp, iter)
                 if (ether_addr_equal_64bits(mac, tmp->dev->dev_addr))
                         return tmp;
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index c73cabdbd4c0..85d0cda5fbfa 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -3983,6 +3983,7 @@ static int cxgb4_inet6addr_handler(struct notifier_block *this,
         struct net_device *event_dev;
         int ret = NOTIFY_DONE;
         struct bonding *bond = netdev_priv(ifa->idev->dev);
+        struct list_head *iter;
         struct slave *slave;
         struct pci_dev *first_pdev = NULL;
 
@@ -3995,7 +3996,7 @@ static int cxgb4_inet6addr_handler(struct notifier_block *this,
                  * in all of them only once.
                  */
                 read_lock(&bond->lock);
-                bond_for_each_slave(bond, slave) {
+                bond_for_each_slave(bond, slave, iter) {
                         if (!first_pdev) {
                                 ret = clip_add(slave->dev, ifa, event);
                                 /* If clip_add is success then only initialize
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 3de49aca4519..b4cfb63f264e 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1143,8 +1143,18 @@ struct net_device {
         struct list_head        dev_list;
         struct list_head        napi_list;
         struct list_head        unreg_list;
-        struct list_head        upper_dev_list; /* List of upper devices */
-        struct list_head        lower_dev_list;
+
+        /* directly linked devices, like slaves for bonding */
+        struct {
+                struct list_head upper;
+                struct list_head lower;
+        } adj_list;
+
+        /* all linked devices, *including* neighbours */
+        struct {
+                struct list_head upper;
+                struct list_head lower;
+        } all_adj_list;
 
 
         /* currently active device features */
@@ -2813,24 +2823,49 @@ extern int		bpf_jit_enable;
 extern bool netdev_has_upper_dev(struct net_device *dev,
                                  struct net_device *upper_dev);
 extern bool netdev_has_any_upper_dev(struct net_device *dev);
-extern struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
-                                                        struct list_head **iter);
+extern struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
+                                                            struct list_head **iter);
 
 /* iterate through upper list, must be called under RCU read lock */
-#define netdev_for_each_upper_dev_rcu(dev, upper, iter) \
-        for (iter = &(dev)->upper_dev_list, \
-             upper = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
-             upper; \
-             upper = netdev_upper_get_next_dev_rcu(dev, &(iter)))
-
+#define netdev_for_each_all_upper_dev_rcu(dev, updev, iter) \
+        for (iter = &(dev)->all_adj_list.upper, \
+             updev = netdev_all_upper_get_next_dev_rcu(dev, &(iter)); \
+             updev; \
+             updev = netdev_all_upper_get_next_dev_rcu(dev, &(iter)))
+
+extern void *netdev_lower_get_next_private(struct net_device *dev,
+                                           struct list_head **iter);
+extern void *netdev_lower_get_next_private_rcu(struct net_device *dev,
+                                               struct list_head **iter);
+
+#define netdev_for_each_lower_private(dev, priv, iter) \
+        for (iter = (dev)->adj_list.lower.next, \
+             priv = netdev_lower_get_next_private(dev, &(iter)); \
+             priv; \
+             priv = netdev_lower_get_next_private(dev, &(iter)))
+
+#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
+        for (iter = &(dev)->adj_list.lower, \
+             priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
+             priv; \
+             priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
+
+extern void *netdev_adjacent_get_private(struct list_head *adj_list);
 extern struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
 extern struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
 extern int netdev_upper_dev_link(struct net_device *dev,
                                  struct net_device *upper_dev);
 extern int netdev_master_upper_dev_link(struct net_device *dev,
                                         struct net_device *upper_dev);
+extern int netdev_master_upper_dev_link_private(struct net_device *dev,
+                                                struct net_device *upper_dev,
+                                                void *private);
 extern void netdev_upper_dev_unlink(struct net_device *dev,
                                     struct net_device *upper_dev);
+extern void *netdev_lower_dev_get_private_rcu(struct net_device *dev,
+                                              struct net_device *lower_dev);
+extern void *netdev_lower_dev_get_private(struct net_device *dev,
+                                          struct net_device *lower_dev);
 extern int skb_checksum_help(struct sk_buff *skb);
 extern struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
         netdev_features_t features, bool tx_path);
diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c
index 61fc573f1142..b3d17d1c49c3 100644
--- a/net/8021q/vlan.c
+++ b/net/8021q/vlan.c
@@ -98,14 +98,14 @@ void unregister_vlan_dev(struct net_device *dev, struct list_head *head)
                 vlan_gvrp_request_leave(dev);
 
         vlan_group_set_device(grp, vlan->vlan_proto, vlan_id, NULL);
+
+        netdev_upper_dev_unlink(real_dev, dev);
         /* Because unregister_netdevice_queue() makes sure at least one rcu
          * grace period is respected before device freeing,
          * we dont need to call synchronize_net() here.
          */
         unregister_netdevice_queue(dev, head);
 
-        netdev_upper_dev_unlink(real_dev, dev);
-
         if (grp->nr_vlan_devs == 0) {
                 vlan_mvrp_uninit_applicant(real_dev);
                 vlan_gvrp_uninit_applicant(real_dev);
@@ -169,13 +169,13 @@ int register_vlan_dev(struct net_device *dev)
         if (err < 0)
                 goto out_uninit_mvrp;
 
-        err = netdev_upper_dev_link(real_dev, dev);
-        if (err)
-                goto out_uninit_mvrp;
-
         err = register_netdevice(dev);
         if (err < 0)
-                goto out_upper_dev_unlink;
+                goto out_uninit_mvrp;
+
+        err = netdev_upper_dev_link(real_dev, dev);
+        if (err)
+                goto out_unregister_netdev;
 
         /* Account for reference in struct vlan_dev_priv */
         dev_hold(real_dev);
@@ -191,8 +191,8 @@ int register_vlan_dev(struct net_device *dev)
 
         return 0;
 
-out_upper_dev_unlink:
-        netdev_upper_dev_unlink(real_dev, dev);
+out_unregister_netdev:
+        unregister_netdevice(dev);
 out_uninit_mvrp:
         if (grp->nr_vlan_devs == 0)
                 vlan_mvrp_uninit_applicant(real_dev);
diff --git a/net/core/dev.c b/net/core/dev.c
index 5c713f2239cc..25ab6fe80da2 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4373,42 +4373,40 @@ struct netdev_adjacent {
         /* upper master flag, there can only be one master device per list */
         bool master;
 
-        /* indicates that this dev is our first-level lower/upper device */
-        bool neighbour;
-
         /* counter for the number of times this device was added to us */
         u16 ref_nr;
 
+        /* private field for the users */
+        void *private;
+
         struct list_head list;
         struct rcu_head rcu;
 };
 
-static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev,
-                                                 struct net_device *adj_dev,
-                                                 bool upper)
+static struct netdev_adjacent *__netdev_find_adj_rcu(struct net_device *dev,
+                                                     struct net_device *adj_dev,
+                                                     struct list_head *adj_list)
 {
         struct netdev_adjacent *adj;
-        struct list_head *dev_list;
 
-        dev_list = upper ? &dev->upper_dev_list : &dev->lower_dev_list;
-
-        list_for_each_entry(adj, dev_list, list) {
+        list_for_each_entry_rcu(adj, adj_list, list) {
                 if (adj->dev == adj_dev)
                         return adj;
         }
         return NULL;
 }
 
-static inline struct netdev_adjacent *__netdev_find_upper(struct net_device *dev,
-                                                          struct net_device *udev)
+static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev,
+                                                 struct net_device *adj_dev,
+                                                 struct list_head *adj_list)
 {
-        return __netdev_find_adj(dev, udev, true);
-}
+        struct netdev_adjacent *adj;
 
-static inline struct netdev_adjacent *__netdev_find_lower(struct net_device *dev,
-                                                          struct net_device *ldev)
-{
-        return __netdev_find_adj(dev, ldev, false);
+        list_for_each_entry(adj, adj_list, list) {
+                if (adj->dev == adj_dev)
+                        return adj;
+        }
+        return NULL;
 }
 
 /**
@@ -4425,7 +4423,7 @@ bool netdev_has_upper_dev(struct net_device *dev,
 {
         ASSERT_RTNL();
 
-        return __netdev_find_upper(dev, upper_dev);
+        return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper);
 }
 EXPORT_SYMBOL(netdev_has_upper_dev);
 
@@ -4440,7 +4438,7 @@ bool netdev_has_any_upper_dev(struct net_device *dev)
 {
         ASSERT_RTNL();
 
-        return !list_empty(&dev->upper_dev_list);
+        return !list_empty(&dev->all_adj_list.upper);
 }
 EXPORT_SYMBOL(netdev_has_any_upper_dev);
 
@@ -4457,10 +4455,10 @@ struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
 
         ASSERT_RTNL();
 
-        if (list_empty(&dev->upper_dev_list))
+        if (list_empty(&dev->adj_list.upper))
                 return NULL;
 
-        upper = list_first_entry(&dev->upper_dev_list,
+        upper = list_first_entry(&dev->adj_list.upper,
                                  struct netdev_adjacent, list);
         if (likely(upper->master))
                 return upper->dev;
@@ -4468,15 +4466,26 @@ struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
 }
 EXPORT_SYMBOL(netdev_master_upper_dev_get);
 
-/* netdev_upper_get_next_dev_rcu - Get the next dev from upper list
+void *netdev_adjacent_get_private(struct list_head *adj_list)
+{
+        struct netdev_adjacent *adj;
+
+        adj = list_entry(adj_list, struct netdev_adjacent, list);
+
+        return adj->private;
+}
+EXPORT_SYMBOL(netdev_adjacent_get_private);
+
+/**
+ * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
  * @dev: device
  * @iter: list_head ** of the current position
  *
  * Gets the next device from the dev's upper list, starting from iter
  * position. The caller must hold RCU read lock.
  */
-struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
-                                                 struct list_head **iter)
+struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
+                                                     struct list_head **iter)
 {
         struct netdev_adjacent *upper;
 
@@ -4484,14 +4493,71 @@ struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
 
         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
 
-        if (&upper->list == &dev->upper_dev_list)
+        if (&upper->list == &dev->all_adj_list.upper)
                 return NULL;
 
         *iter = &upper->list;
 
         return upper->dev;
 }
-EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
+EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
+
+/**
+ * netdev_lower_get_next_private - Get the next ->private from the
+ *                                 lower neighbour list
+ * @dev: device
+ * @iter: list_head ** of the current position
+ *
+ * Gets the next netdev_adjacent->private from the dev's lower neighbour
+ * list, starting from iter position. The caller must hold either hold the
+ * RTNL lock or its own locking that guarantees that the neighbour lower
+ * list will remain unchainged.
+ */
+void *netdev_lower_get_next_private(struct net_device *dev,
+                                    struct list_head **iter)
+{
+        struct netdev_adjacent *lower;
+
+        lower = list_entry(*iter, struct netdev_adjacent, list);
+
+        if (&lower->list == &dev->adj_list.lower)
+                return NULL;
+
+        if (iter)
+                *iter = lower->list.next;
+
+        return lower->private;
+}
+EXPORT_SYMBOL(netdev_lower_get_next_private);
+
+/**
+ * netdev_lower_get_next_private_rcu - Get the next ->private from the
+ *                                     lower neighbour list, RCU
+ *                                     variant
+ * @dev: device
+ * @iter: list_head ** of the current position
+ *
+ * Gets the next netdev_adjacent->private from the dev's lower neighbour
+ * list, starting from iter position. The caller must hold RCU read lock.
+ */
+void *netdev_lower_get_next_private_rcu(struct net_device *dev,
+                                        struct list_head **iter)
+{
+        struct netdev_adjacent *lower;
+
+        WARN_ON_ONCE(!rcu_read_lock_held());
+
+        lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
+
+        if (&lower->list == &dev->adj_list.lower)
+                return NULL;
+
+        if (iter)
+                *iter = &lower->list;
+
+        return lower->private;
+}
+EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
 
 /**
  * netdev_master_upper_dev_get_rcu - Get master upper device
@@ -4504,7 +4570,7 @@ struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
 {
         struct netdev_adjacent *upper;
 
-        upper = list_first_or_null_rcu(&dev->upper_dev_list,
+        upper = list_first_or_null_rcu(&dev->adj_list.upper,
                                        struct netdev_adjacent, list);
         if (upper && likely(upper->master))
                 return upper->dev;
@@ -4514,15 +4580,16 @@ EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
 
 static int __netdev_adjacent_dev_insert(struct net_device *dev,
                                         struct net_device *adj_dev,
-                                        bool neighbour, bool master,
-                                        bool upper)
+                                        struct list_head *dev_list,
+                                        void *private, bool master)
 {
         struct netdev_adjacent *adj;
+        char linkname[IFNAMSIZ+7];
+        int ret;
 
-        adj = __netdev_find_adj(dev, adj_dev, upper);
+        adj = __netdev_find_adj(dev, adj_dev, dev_list);
 
         if (adj) {
-                BUG_ON(neighbour);
                 adj->ref_nr++;
                 return 0;
         }
@@ -4533,124 +4600,178 @@ static int __netdev_adjacent_dev_insert(struct net_device *dev,
 
         adj->dev = adj_dev;
         adj->master = master;
-        adj->neighbour = neighbour;
         adj->ref_nr = 1;
-
+        adj->private = private;
         dev_hold(adj_dev);
-        pr_debug("dev_hold for %s, because of %s link added from %s to %s\n",
-                 adj_dev->name, upper ? "upper" : "lower", dev->name,
-                 adj_dev->name);
 
-        if (!upper) {
-                list_add_tail_rcu(&adj->list, &dev->lower_dev_list);
-                return 0;
+        pr_debug("dev_hold for %s, because of link added from %s to %s\n",
+                 adj_dev->name, dev->name, adj_dev->name);
+
+        if (dev_list == &dev->adj_list.lower) {
+                sprintf(linkname, "lower_%s", adj_dev->name);
+                ret = sysfs_create_link(&(dev->dev.kobj),
+                                        &(adj_dev->dev.kobj), linkname);
+                if (ret)
+                        goto free_adj;
+        } else if (dev_list == &dev->adj_list.upper) {
+                sprintf(linkname, "upper_%s", adj_dev->name);
+                ret = sysfs_create_link(&(dev->dev.kobj),
+                                        &(adj_dev->dev.kobj), linkname);
+                if (ret)
+                        goto free_adj;
         }
 
-        /* Ensure that master upper link is always the first item in list. */
-        if (master)
-                list_add_rcu(&adj->list, &dev->upper_dev_list);
-        else
-                list_add_tail_rcu(&adj->list, &dev->upper_dev_list);
+        /* Ensure that master link is always the first item in list. */
+        if (master) {
+                ret = sysfs_create_link(&(dev->dev.kobj),
+                                        &(adj_dev->dev.kobj), "master");
+                if (ret)
+                        goto remove_symlinks;
+
+                list_add_rcu(&adj->list, dev_list);
+        } else {
+                list_add_tail_rcu(&adj->list, dev_list);
+        }
 
         return 0;
-}
 
-static inline int __netdev_upper_dev_insert(struct net_device *dev,
-                                            struct net_device *udev,
-                                            bool master, bool neighbour)
-{
-        return __netdev_adjacent_dev_insert(dev, udev, neighbour, master,
-                                            true);
-}
+remove_symlinks:
+        if (dev_list == &dev->adj_list.lower) {
+                sprintf(linkname, "lower_%s", adj_dev->name);
+                sysfs_remove_link(&(dev->dev.kobj), linkname);
+        } else if (dev_list == &dev->adj_list.upper) {
+                sprintf(linkname, "upper_%s", adj_dev->name);
+                sysfs_remove_link(&(dev->dev.kobj), linkname);
+        }
 
-static inline int __netdev_lower_dev_insert(struct net_device *dev,
-                                            struct net_device *ldev,
-                                            bool neighbour)
-{
-        return __netdev_adjacent_dev_insert(dev, ldev, neighbour, false,
-                                            false);
+free_adj:
+        kfree(adj);
+
+        return ret;
 }
 
 void __netdev_adjacent_dev_remove(struct net_device *dev,
-                                  struct net_device *adj_dev, bool upper)
+                                  struct net_device *adj_dev,
+                                  struct list_head *dev_list)
 {
         struct netdev_adjacent *adj;
+        char linkname[IFNAMSIZ+7];
 
-        if (upper)
-                adj = __netdev_find_upper(dev, adj_dev);
-        else
-                adj = __netdev_find_lower(dev, adj_dev);
+        adj = __netdev_find_adj(dev, adj_dev, dev_list);
 
-        if (!adj)
+        if (!adj) {
+                pr_err("tried to remove device %s from %s\n",
+                       dev->name, adj_dev->name);
                 BUG();
+        }
 
         if (adj->ref_nr > 1) {
+                pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
+                         adj->ref_nr-1);
                 adj->ref_nr--;
                 return;
         }
 
+        if (adj->master)
+                sysfs_remove_link(&(dev->dev.kobj), "master");
+
+        if (dev_list == &dev->adj_list.lower) {
+                sprintf(linkname, "lower_%s", adj_dev->name);
+                sysfs_remove_link(&(dev->dev.kobj), linkname);
+        } else if (dev_list == &dev->adj_list.upper) {
+                sprintf(linkname, "upper_%s", adj_dev->name);
+                sysfs_remove_link(&(dev->dev.kobj), linkname);
+        }
+
         list_del_rcu(&adj->list);
-        pr_debug("dev_put for %s, because of %s link removed from %s to %s\n",
-                 adj_dev->name, upper ? "upper" : "lower", dev->name,
-                 adj_dev->name);
+        pr_debug("dev_put for %s, because link removed from %s to %s\n",
+                 adj_dev->name, dev->name, adj_dev->name);
         dev_put(adj_dev);
         kfree_rcu(adj, rcu);
 }
 
-static inline void __netdev_upper_dev_remove(struct net_device *dev,
-                                             struct net_device *udev)
-{
-        return __netdev_adjacent_dev_remove(dev, udev, true);
-}
-
-static inline void __netdev_lower_dev_remove(struct net_device *dev,
-                                             struct net_device *ldev)
-{
-        return __netdev_adjacent_dev_remove(dev, ldev, false);
-}
-
-int __netdev_adjacent_dev_insert_link(struct net_device *dev,
-                                      struct net_device *upper_dev,
-                                      bool master, bool neighbour)
+int __netdev_adjacent_dev_link_lists(struct net_device *dev,
+                                     struct net_device *upper_dev,
+                                     struct list_head *up_list,
+                                     struct list_head *down_list,
+                                     void *private, bool master)
 {
         int ret;
 
-        ret = __netdev_upper_dev_insert(dev, upper_dev, master, neighbour);
+        ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
+                                           master);
         if (ret)
                 return ret;
 
-        ret = __netdev_lower_dev_insert(upper_dev, dev, neighbour);
+        ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
+                                           false);
         if (ret) {
-                __netdev_upper_dev_remove(dev, upper_dev);
+                __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
                 return ret;
         }
 
         return 0;
 }
 
-static inline int __netdev_adjacent_dev_link(struct net_device *dev,
-                                             struct net_device *udev)
+int __netdev_adjacent_dev_link(struct net_device *dev,
+                               struct net_device *upper_dev)
 {
-        return __netdev_adjacent_dev_insert_link(dev, udev, false, false);
+        return __netdev_adjacent_dev_link_lists(dev, upper_dev,
+                                                &dev->all_adj_list.upper,
+                                                &upper_dev->all_adj_list.lower,
+                                                NULL, false);
 }
 
-static inline int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
-                                                       struct net_device *udev,
-                                                       bool master)
+void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
+                                        struct net_device *upper_dev,
+                                        struct list_head *up_list,
+                                        struct list_head *down_list)
 {
-        return __netdev_adjacent_dev_insert_link(dev, udev, master, true);
+        __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
+        __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
 }
 
 void __netdev_adjacent_dev_unlink(struct net_device *dev,
                                   struct net_device *upper_dev)
 {
-        __netdev_upper_dev_remove(dev, upper_dev);
-        __netdev_lower_dev_remove(upper_dev, dev);
+        __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
+                                           &dev->all_adj_list.upper,
+                                           &upper_dev->all_adj_list.lower);
+}
+
+int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
+                                         struct net_device *upper_dev,
+                                         void *private, bool master)
+{
+        int ret = __netdev_adjacent_dev_link(dev, upper_dev);
+
+        if (ret)
+                return ret;
+
+        ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
+                                               &dev->adj_list.upper,
+                                               &upper_dev->adj_list.lower,
+                                               private, master);
+        if (ret) {
+                __netdev_adjacent_dev_unlink(dev, upper_dev);
+                return ret;
+        }
+
+        return 0;
 }
 
+void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
+                                            struct net_device *upper_dev)
+{
+        __netdev_adjacent_dev_unlink(dev, upper_dev);
+        __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
+                                           &dev->adj_list.upper,
+                                           &upper_dev->adj_list.lower);
+}
 
 static int __netdev_upper_dev_link(struct net_device *dev,
-                                   struct net_device *upper_dev, bool master)
+                                   struct net_device *upper_dev, bool master,
+                                   void *private)
 {
         struct netdev_adjacent *i, *j, *to_i, *to_j;
         int ret = 0;
@@ -4661,26 +4782,29 @@ static int __netdev_upper_dev_link(struct net_device *dev,
                 return -EBUSY;
 
         /* To prevent loops, check if dev is not upper device to upper_dev. */
-        if (__netdev_find_upper(upper_dev, dev))
+        if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper))
                 return -EBUSY;
 
-        if (__netdev_find_upper(dev, upper_dev))
+        if (__netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper))
                 return -EEXIST;
 
         if (master && netdev_master_upper_dev_get(dev))
                 return -EBUSY;
 
-        ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, master);
+        ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private,
+                                                   master);
         if (ret)
                 return ret;
 
         /* Now that we linked these devs, make all the upper_dev's
-         * upper_dev_list visible to every dev's lower_dev_list and vice
+         * all_adj_list.upper visible to every dev's all_adj_list.lower an
          * versa, and don't forget the devices itself. All of these
          * links are non-neighbours.
          */
-        list_for_each_entry(i, &dev->lower_dev_list, list) {
-                list_for_each_entry(j, &upper_dev->upper_dev_list, list) {
+        list_for_each_entry(i, &dev->all_adj_list.lower, list) {
+                list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
+                        pr_debug("Interlinking %s with %s, non-neighbour\n",
+                                 i->dev->name, j->dev->name);
                         ret = __netdev_adjacent_dev_link(i->dev, j->dev);
                         if (ret)
                                 goto rollback_mesh;
@@ -4688,14 +4812,18 @@ static int __netdev_upper_dev_link(struct net_device *dev,
         }
 
         /* add dev to every upper_dev's upper device */
-        list_for_each_entry(i, &upper_dev->upper_dev_list, list) {
+        list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
+                pr_debug("linking %s's upper device %s with %s\n",
+                         upper_dev->name, i->dev->name, dev->name);
                 ret = __netdev_adjacent_dev_link(dev, i->dev);
                 if (ret)
                         goto rollback_upper_mesh;
         }
 
         /* add upper_dev to every dev's lower device */
-        list_for_each_entry(i, &dev->lower_dev_list, list) {
+        list_for_each_entry(i, &dev->all_adj_list.lower, list) {
+                pr_debug("linking %s's lower device %s with %s\n", dev->name,
+                         i->dev->name, upper_dev->name);
                 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
                 if (ret)
                         goto rollback_lower_mesh;
@@ -4706,7 +4834,7 @@ static int __netdev_upper_dev_link(struct net_device *dev,
 
 rollback_lower_mesh:
         to_i = i;
-        list_for_each_entry(i, &dev->lower_dev_list, list) {
+        list_for_each_entry(i, &dev->all_adj_list.lower, list) {
                 if (i == to_i)
                         break;
                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
@@ -4716,7 +4844,7 @@ rollback_lower_mesh:
 
 rollback_upper_mesh:
         to_i = i;
-        list_for_each_entry(i, &upper_dev->upper_dev_list, list) {
+        list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
                 if (i == to_i)
                         break;
                 __netdev_adjacent_dev_unlink(dev, i->dev);
@@ -4727,8 +4855,8 @@ rollback_upper_mesh:
 rollback_mesh:
         to_i = i;
         to_j = j;
-        list_for_each_entry(i, &dev->lower_dev_list, list) {
-                list_for_each_entry(j, &upper_dev->upper_dev_list, list) {
+        list_for_each_entry(i, &dev->all_adj_list.lower, list) {
+                list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
                         if (i == to_i && j == to_j)
                                 break;
                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
@@ -4737,7 +4865,7 @@ rollback_mesh:
                         break;
         }
 
-        __netdev_adjacent_dev_unlink(dev, upper_dev);
+        __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
 
         return ret;
 }
@@ -4755,7 +4883,7 @@ rollback_mesh:
 int netdev_upper_dev_link(struct net_device *dev,
                           struct net_device *upper_dev)
 {
-        return __netdev_upper_dev_link(dev, upper_dev, false);
+        return __netdev_upper_dev_link(dev, upper_dev, false, NULL);
 }
 EXPORT_SYMBOL(netdev_upper_dev_link);
 
@@ -4773,10 +4901,18 @@ EXPORT_SYMBOL(netdev_upper_dev_link);
 int netdev_master_upper_dev_link(struct net_device *dev,
                                  struct net_device *upper_dev)
 {
-        return __netdev_upper_dev_link(dev, upper_dev, true);
+        return __netdev_upper_dev_link(dev, upper_dev, true, NULL);
 }
 EXPORT_SYMBOL(netdev_master_upper_dev_link);
 
+int netdev_master_upper_dev_link_private(struct net_device *dev,
+                                         struct net_device *upper_dev,
+                                         void *private)
+{
+        return __netdev_upper_dev_link(dev, upper_dev, true, private);
+}
+EXPORT_SYMBOL(netdev_master_upper_dev_link_private);
+
 /**
  * netdev_upper_dev_unlink - Removes a link to upper device
  * @dev: device
@@ -4791,29 +4927,59 @@ void netdev_upper_dev_unlink(struct net_device *dev,
         struct netdev_adjacent *i, *j;
         ASSERT_RTNL();
 
-        __netdev_adjacent_dev_unlink(dev, upper_dev);
+        __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
 
         /* Here is the tricky part. We must remove all dev's lower
          * devices from all upper_dev's upper devices and vice
          * versa, to maintain the graph relationship.
          */
-        list_for_each_entry(i, &dev->lower_dev_list, list)
-                list_for_each_entry(j, &upper_dev->upper_dev_list, list)
+        list_for_each_entry(i, &dev->all_adj_list.lower, list)
+                list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
 
         /* remove also the devices itself from lower/upper device
          * list
          */
-        list_for_each_entry(i, &dev->lower_dev_list, list)
+        list_for_each_entry(i, &dev->all_adj_list.lower, list)
                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
 
-        list_for_each_entry(i, &upper_dev->upper_dev_list, list)
+        list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
                 __netdev_adjacent_dev_unlink(dev, i->dev);
 
         call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
 }
 EXPORT_SYMBOL(netdev_upper_dev_unlink);
 
+void *netdev_lower_dev_get_private_rcu(struct net_device *dev,
+                                       struct net_device *lower_dev)
+{
+        struct netdev_adjacent *lower;
+
+        if (!lower_dev)
+                return NULL;
+        lower = __netdev_find_adj_rcu(dev, lower_dev, &dev->adj_list.lower);
+        if (!lower)
+                return NULL;
+
+        return lower->private;
+}
+EXPORT_SYMBOL(netdev_lower_dev_get_private_rcu);
+
+void *netdev_lower_dev_get_private(struct net_device *dev,
+                                   struct net_device *lower_dev)
+{
+        struct netdev_adjacent *lower;
+
+        if (!lower_dev)
+                return NULL;
+        lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower);
+        if (!lower)
+                return NULL;
+
+        return lower->private;
+}
+EXPORT_SYMBOL(netdev_lower_dev_get_private);
+
 static void dev_change_rx_flags(struct net_device *dev, int flags)
 {
         const struct net_device_ops *ops = dev->netdev_ops;
@@ -6069,8 +6235,10 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
         INIT_LIST_HEAD(&dev->napi_list);
         INIT_LIST_HEAD(&dev->unreg_list);
         INIT_LIST_HEAD(&dev->link_watch_list);
-        INIT_LIST_HEAD(&dev->upper_dev_list);
-        INIT_LIST_HEAD(&dev->lower_dev_list);
+        INIT_LIST_HEAD(&dev->adj_list.upper);
+        INIT_LIST_HEAD(&dev->adj_list.lower);
+        INIT_LIST_HEAD(&dev->all_adj_list.upper);
+        INIT_LIST_HEAD(&dev->all_adj_list.lower);
         dev->priv_flags = IFF_XMIT_DST_RELEASE;
         setup(dev);