5 files changed, 143 insertions, 109 deletions

diff --git a/net/core/dev.c b/net/core/dev.c
index a76021c71207..29cf00c5d865 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -220,7 +220,8 @@ static RAW_NOTIFIER_HEAD(netdev_chain);
  *      Device drivers call our routines to queue packets here. We empty the
  *      queue in the local softnet handler.
  */
-DEFINE_PER_CPU(struct softnet_data, softnet_data) = { NULL };
+
+DEFINE_PER_CPU(struct softnet_data, softnet_data);
 
 #ifdef CONFIG_SYSFS
 extern int netdev_sysfs_init(void);
@@ -1018,16 +1019,12 @@ int dev_close(struct net_device *dev)
         clear_bit(__LINK_STATE_START, &dev->state);
 
         /* Synchronize to scheduled poll. We cannot touch poll list,
-         * it can be even on different cpu. So just clear netif_running(),
-         * and wait when poll really will happen. Actually, the best place
-         * for this is inside dev->stop() after device stopped its irq
-         * engine, but this requires more changes in devices. */
-
+         * it can be even on different cpu. So just clear netif_running().
+         *
+         * dev->stop() will invoke napi_disable() on all of it's
+         * napi_struct instances on this device.
+         */
         smp_mb__after_clear_bit(); /* Commit netif_running(). */
-        while (test_bit(__LINK_STATE_RX_SCHED, &dev->state)) {
-                /* No hurry. */
-                msleep(1);
-        }
 
         /*
          *      Call the device specific close. This cannot fail.
@@ -1233,21 +1230,21 @@ void __netif_schedule(struct net_device *dev)
 }
 EXPORT_SYMBOL(__netif_schedule);
 
-void __netif_rx_schedule(struct net_device *dev)
+void dev_kfree_skb_irq(struct sk_buff *skb)
 {
-        unsigned long flags;
+        if (atomic_dec_and_test(&skb->users)) {
+                struct softnet_data *sd;
+                unsigned long flags;
 
-        local_irq_save(flags);
-        dev_hold(dev);
-        list_add_tail(&dev->poll_list, &__get_cpu_var(softnet_data).poll_list);
-        if (dev->quota < 0)
-                dev->quota += dev->weight;
-        else
-                dev->quota = dev->weight;
-        __raise_softirq_irqoff(NET_RX_SOFTIRQ);
-        local_irq_restore(flags);
+                local_irq_save(flags);
+                sd = &__get_cpu_var(softnet_data);
+                skb->next = sd->completion_queue;
+                sd->completion_queue = skb;
+                raise_softirq_irqoff(NET_TX_SOFTIRQ);
+                local_irq_restore(flags);
+        }
 }
-EXPORT_SYMBOL(__netif_rx_schedule);
+EXPORT_SYMBOL(dev_kfree_skb_irq);
 
 void dev_kfree_skb_any(struct sk_buff *skb)
 {
@@ -1259,7 +1256,12 @@ void dev_kfree_skb_any(struct sk_buff *skb)
 EXPORT_SYMBOL(dev_kfree_skb_any);
 
 
-/* Hot-plugging. */
+/**
+ * netif_device_detach - mark device as removed
+ * @dev: network device
+ *
+ * Mark device as removed from system and therefore no longer available.
+ */
 void netif_device_detach(struct net_device *dev)
 {
         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
@@ -1269,6 +1271,12 @@ void netif_device_detach(struct net_device *dev)
 }
 EXPORT_SYMBOL(netif_device_detach);
 
+/**
+ * netif_device_attach - mark device as attached
+ * @dev: network device
+ *
+ * Mark device as attached from system and restart if needed.
+ */
 void netif_device_attach(struct net_device *dev)
 {
         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
@@ -1730,7 +1738,7 @@ enqueue:
                         return NET_RX_SUCCESS;
                 }
 
-                netif_rx_schedule(&queue->backlog_dev);
+                napi_schedule(&queue->backlog);
                 goto enqueue;
         }
 
@@ -1771,6 +1779,7 @@ static inline struct net_device *skb_bond(struct sk_buff *skb)
         return dev;
 }
 
+
 static void net_tx_action(struct softirq_action *h)
 {
         struct softnet_data *sd = &__get_cpu_var(softnet_data);
@@ -1927,7 +1936,7 @@ int netif_receive_skb(struct sk_buff *skb)
         __be16 type;
 
         /* if we've gotten here through NAPI, check netpoll */
-        if (skb->dev->poll && netpoll_rx(skb))
+        if (netpoll_receive_skb(skb))
                 return NET_RX_DROP;
 
         if (!skb->tstamp.tv64)
@@ -2017,22 +2026,25 @@ out:
         return ret;
 }
 
-static int process_backlog(struct net_device *backlog_dev, int *budget)
+static int process_backlog(struct napi_struct *napi, int quota)
 {
         int work = 0;
-        int quota = min(backlog_dev->quota, *budget);
         struct softnet_data *queue = &__get_cpu_var(softnet_data);
         unsigned long start_time = jiffies;
 
-        backlog_dev->weight = weight_p;
-        for (;;) {
+        napi->weight = weight_p;
+        do {
                 struct sk_buff *skb;
                 struct net_device *dev;
 
                 local_irq_disable();
                 skb = __skb_dequeue(&queue->input_pkt_queue);
-                if (!skb)
-                        goto job_done;
+                if (!skb) {
+                        __napi_complete(napi);
+                        local_irq_enable();
+                        break;
+                }
+
                 local_irq_enable();
 
                 dev = skb->dev;
@@ -2040,67 +2052,86 @@ static int process_backlog(struct net_device *backlog_dev, int *budget)
                 netif_receive_skb(skb);
 
                 dev_put(dev);
+        } while (++work < quota && jiffies == start_time);
 
-                work++;
-
-                if (work >= quota || jiffies - start_time > 1)
-                        break;
-
-        }
-
-        backlog_dev->quota -= work;
-        *budget -= work;
-        return -1;
-
-job_done:
-        backlog_dev->quota -= work;
-        *budget -= work;
+        return work;
+}
 
-        list_del(&backlog_dev->poll_list);
-        smp_mb__before_clear_bit();
-        netif_poll_enable(backlog_dev);
+/**
+ * __napi_schedule - schedule for receive
+ * @napi: entry to schedule
+ *
+ * The entry's receive function will be scheduled to run
+ */
+void fastcall __napi_schedule(struct napi_struct *n)
+{
+        unsigned long flags;
 
-        local_irq_enable();
-        return 0;
+        local_irq_save(flags);
+        list_add_tail(&n->poll_list, &__get_cpu_var(softnet_data).poll_list);
+        __raise_softirq_irqoff(NET_RX_SOFTIRQ);
+        local_irq_restore(flags);
 }
+EXPORT_SYMBOL(__napi_schedule);
+
 
 static void net_rx_action(struct softirq_action *h)
 {
-        struct softnet_data *queue = &__get_cpu_var(softnet_data);
+        struct list_head *list = &__get_cpu_var(softnet_data).poll_list;
         unsigned long start_time = jiffies;
         int budget = netdev_budget;
         void *have;
 
         local_irq_disable();
 
-        while (!list_empty(&queue->poll_list)) {
-                struct net_device *dev;
+        while (!list_empty(list)) {
+                struct napi_struct *n;
+                int work, weight;
 
-                if (budget <= 0 || jiffies - start_time > 1)
+                /* If softirq window is exhuasted then punt.
+                 *
+                 * Note that this is a slight policy change from the
+                 * previous NAPI code, which would allow up to 2
+                 * jiffies to pass before breaking out.  The test
+                 * used to be "jiffies - start_time > 1".
+                 */
+                if (unlikely(budget <= 0 || jiffies != start_time))
                         goto softnet_break;
 
                 local_irq_enable();
 
-                dev = list_entry(queue->poll_list.next,
-                                 struct net_device, poll_list);
-                have = netpoll_poll_lock(dev);
+                /* Even though interrupts have been re-enabled, this
+                 * access is safe because interrupts can only add new
+                 * entries to the tail of this list, and only ->poll()
+                 * calls can remove this head entry from the list.
+                 */
+                n = list_entry(list->next, struct napi_struct, poll_list);
 
-                if (dev->quota <= 0 || dev->poll(dev, &budget)) {
-                        netpoll_poll_unlock(have);
-                        local_irq_disable();
-                        list_move_tail(&dev->poll_list, &queue->poll_list);
-                        if (dev->quota < 0)
-                                dev->quota += dev->weight;
-                        else
-                                dev->quota = dev->weight;
-                } else {
-                        netpoll_poll_unlock(have);
-                        dev_put(dev);
-                        local_irq_disable();
-                }
+                have = netpoll_poll_lock(n);
+
+                weight = n->weight;
+
+                work = n->poll(n, weight);
+
+                WARN_ON_ONCE(work > weight);
+
+                budget -= work;
+
+                local_irq_disable();
+
+                /* Drivers must not modify the NAPI state if they
+                 * consume the entire weight.  In such cases this code
+                 * still "owns" the NAPI instance and therefore can
+                 * move the instance around on the list at-will.
+                 */
+                if (unlikely(work == weight))
+                        list_move_tail(&n->poll_list, list);
+
+                netpoll_poll_unlock(have);
         }
 out:
         local_irq_enable();
+
 #ifdef CONFIG_NET_DMA
         /*
          * There may not be any more sk_buffs coming right now, so push
@@ -2115,6 +2146,7 @@ out:
                 }
         }
 #endif
+
         return;
 
 softnet_break:
@@ -3704,6 +3736,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
         dev->egress_subqueue_count = queue_count;
 
         dev->get_stats = internal_stats;
+        netpoll_netdev_init(dev);
         setup(dev);
         strcpy(dev->name, name);
         return dev;
@@ -4076,10 +4109,9 @@ static int __init net_dev_init(void)
                 skb_queue_head_init(&queue->input_pkt_queue);
                 queue->completion_queue = NULL;
                 INIT_LIST_HEAD(&queue->poll_list);
-                set_bit(__LINK_STATE_START, &queue->backlog_dev.state);
-                queue->backlog_dev.weight = weight_p;
-                queue->backlog_dev.poll = process_backlog;
-                atomic_set(&queue->backlog_dev.refcnt, 1);
+
+                queue->backlog.poll = process_backlog;
+                queue->backlog.weight = weight_p;
         }
 
         netdev_dma_register();
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index 5c19b0646d7a..79159db6acb9 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -216,20 +216,6 @@ static ssize_t store_tx_queue_len(struct device *dev,
         return netdev_store(dev, attr, buf, len, change_tx_queue_len);
 }
 
-NETDEVICE_SHOW(weight, fmt_dec);
-
-static int change_weight(struct net_device *net, unsigned long new_weight)
-{
-        net->weight = new_weight;
-        return 0;
-}
-
-static ssize_t store_weight(struct device *dev, struct device_attribute *attr,
-                            const char *buf, size_t len)
-{
-        return netdev_store(dev, attr, buf, len, change_weight);
-}
-
 static struct device_attribute net_class_attributes[] = {
         __ATTR(addr_len, S_IRUGO, show_addr_len, NULL),
         __ATTR(iflink, S_IRUGO, show_iflink, NULL),
@@ -246,7 +232,6 @@ static struct device_attribute net_class_attributes[] = {
         __ATTR(flags, S_IRUGO | S_IWUSR, show_flags, store_flags),
         __ATTR(tx_queue_len, S_IRUGO | S_IWUSR, show_tx_queue_len,
                store_tx_queue_len),
-        __ATTR(weight, S_IRUGO | S_IWUSR, show_weight, store_weight),
         {}
 };
 
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index de1b26aa5720..abe6e3a4cc44 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -119,19 +119,22 @@ static __sum16 checksum_udp(struct sk_buff *skb, struct udphdr *uh,
 static void poll_napi(struct netpoll *np)
 {
         struct netpoll_info *npinfo = np->dev->npinfo;
+        struct napi_struct *napi;
         int budget = 16;
 
-        if (test_bit(__LINK_STATE_RX_SCHED, &np->dev->state) &&
-            npinfo->poll_owner != smp_processor_id() &&
-            spin_trylock(&npinfo->poll_lock)) {
-                npinfo->rx_flags |= NETPOLL_RX_DROP;
-                atomic_inc(&trapped);
+        list_for_each_entry(napi, &np->dev->napi_list, dev_list) {
+                if (test_bit(NAPI_STATE_SCHED, &napi->state) &&
+                    napi->poll_owner != smp_processor_id() &&
+                    spin_trylock(&napi->poll_lock)) {
+                        npinfo->rx_flags |= NETPOLL_RX_DROP;
+                        atomic_inc(&trapped);
 
-                np->dev->poll(np->dev, &budget);
+                        napi->poll(napi, budget);
 
-                atomic_dec(&trapped);
-                npinfo->rx_flags &= ~NETPOLL_RX_DROP;
-                spin_unlock(&npinfo->poll_lock);
+                        atomic_dec(&trapped);
+                        npinfo->rx_flags &= ~NETPOLL_RX_DROP;
+                        spin_unlock(&napi->poll_lock);
+                }
         }
 }
 
@@ -157,7 +160,7 @@ void netpoll_poll(struct netpoll *np)
 
         /* Process pending work on NIC */
         np->dev->poll_controller(np->dev);
-        if (np->dev->poll)
+        if (!list_empty(&np->dev->napi_list))
                 poll_napi(np);
 
         service_arp_queue(np->dev->npinfo);
@@ -233,6 +236,17 @@ repeat:
         return skb;
 }
 
+static int netpoll_owner_active(struct net_device *dev)
+{
+        struct napi_struct *napi;
+
+        list_for_each_entry(napi, &dev->napi_list, dev_list) {
+                if (napi->poll_owner == smp_processor_id())
+                        return 1;
+        }
+        return 0;
+}
+
 static void netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
 {
         int status = NETDEV_TX_BUSY;
@@ -246,8 +260,7 @@ static void netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
         }
 
         /* don't get messages out of order, and no recursion */
-        if (skb_queue_len(&npinfo->txq) == 0 &&
-                    npinfo->poll_owner != smp_processor_id()) {
+        if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) {
                 unsigned long flags;
 
                 local_irq_save(flags);
@@ -652,8 +665,6 @@ int netpoll_setup(struct netpoll *np)
 
                 npinfo->rx_flags = 0;
                 npinfo->rx_np = NULL;
-                spin_lock_init(&npinfo->poll_lock);
-                npinfo->poll_owner = -1;
 
                 spin_lock_init(&npinfo->rx_lock);
                 skb_queue_head_init(&npinfo->arp_tx);
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 4756d5857abf..2b0b6fac6cef 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -634,7 +634,6 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
 
         NLA_PUT_STRING(skb, IFLA_IFNAME, dev->name);
         NLA_PUT_U32(skb, IFLA_TXQLEN, dev->tx_queue_len);
-        NLA_PUT_U32(skb, IFLA_WEIGHT, dev->weight);
         NLA_PUT_U8(skb, IFLA_OPERSTATE,
                    netif_running(dev) ? dev->operstate : IF_OPER_DOWN);
         NLA_PUT_U8(skb, IFLA_LINKMODE, dev->link_mode);
@@ -834,9 +833,6 @@ static int do_setlink(struct net_device *dev, struct ifinfomsg *ifm,
         if (tb[IFLA_TXQLEN])
                 dev->tx_queue_len = nla_get_u32(tb[IFLA_TXQLEN]);
 
-        if (tb[IFLA_WEIGHT])
-                dev->weight = nla_get_u32(tb[IFLA_WEIGHT]);
-
         if (tb[IFLA_OPERSTATE])
                 set_operstate(dev, nla_get_u8(tb[IFLA_OPERSTATE]));
 
@@ -1074,8 +1070,6 @@ replay:
                                nla_len(tb[IFLA_BROADCAST]));
                 if (tb[IFLA_TXQLEN])
                         dev->tx_queue_len = nla_get_u32(tb[IFLA_TXQLEN]);
-                if (tb[IFLA_WEIGHT])
-                        dev->weight = nla_get_u32(tb[IFLA_WEIGHT]);
                 if (tb[IFLA_OPERSTATE])
                         set_operstate(dev, nla_get_u8(tb[IFLA_OPERSTATE]));
                 if (tb[IFLA_LINKMODE])
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index c81649cf0b9e..e970e8e75720 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -256,6 +256,12 @@ static void dev_watchdog_down(struct net_device *dev)
         netif_tx_unlock_bh(dev);
 }
 
+/**
+ *      netif_carrier_on - set carrier
+ *      @dev: network device
+ *
+ * Device has detected that carrier.
+ */
 void netif_carrier_on(struct net_device *dev)
 {
         if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state))
@@ -264,6 +270,12 @@ void netif_carrier_on(struct net_device *dev)
                 __netdev_watchdog_up(dev);
 }
 
+/**
+ *      netif_carrier_off - clear carrier
+ *      @dev: network device
+ *
+ * Device has detected loss of carrier.
+ */
 void netif_carrier_off(struct net_device *dev)
 {
         if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state))