Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

1 files changed, 381 insertions, 158 deletions

diff --git a/drivers/net/sfc/efx.c b/drivers/net/sfc/efx.c
index ba674c5ca29e..c914729f9554 100644
--- a/drivers/net/sfc/efx.c
+++ b/drivers/net/sfc/efx.c
@@ -1,7 +1,7 @@
 /****************************************************************************
  * Driver for Solarflare Solarstorm network controllers and boards
  * Copyright 2005-2006 Fen Systems Ltd.
- * Copyright 2005-2009 Solarflare Communications Inc.
+ * Copyright 2005-2011 Solarflare Communications Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
@@ -21,9 +21,9 @@
 #include <linux/ethtool.h>
 #include <linux/topology.h>
 #include <linux/gfp.h>
+#include <linux/cpu_rmap.h>
 #include "net_driver.h"
 #include "efx.h"
-#include "mdio_10g.h"
 #include "nic.h"
 
 #include "mcdi.h"
@@ -68,14 +68,6 @@ const char *efx_loopback_mode_names[] = {
         [LOOPBACK_PHYXS_WS]     = "PHYXS_WS",
 };
 
-/* Interrupt mode names (see INT_MODE())) */
-const unsigned int efx_interrupt_mode_max = EFX_INT_MODE_MAX;
-const char *efx_interrupt_mode_names[] = {
-        [EFX_INT_MODE_MSIX]   = "MSI-X",
-        [EFX_INT_MODE_MSI]    = "MSI",
-        [EFX_INT_MODE_LEGACY] = "legacy",
-};
-
 const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
 const char *efx_reset_type_names[] = {
         [RESET_TYPE_INVISIBLE]     = "INVISIBLE",
@@ -114,7 +106,7 @@ static struct workqueue_struct *reset_workqueue;
  * This is only used in MSI-X interrupt mode
  */
 static unsigned int separate_tx_channels;
-module_param(separate_tx_channels, uint, 0644);
+module_param(separate_tx_channels, uint, 0444);
 MODULE_PARM_DESC(separate_tx_channels,
                  "Use separate channels for TX and RX");
 
@@ -124,10 +116,11 @@ MODULE_PARM_DESC(separate_tx_channels,
 static int napi_weight = 64;
 
 /* This is the time (in jiffies) between invocations of the hardware
- * monitor, which checks for known hardware bugs and resets the
- * hardware and driver as necessary.
+ * monitor.  On Falcon-based NICs, this will:
+ * - Check the on-board hardware monitor;
+ * - Poll the link state and reconfigure the hardware as necessary.
  */
-unsigned int efx_monitor_interval = 1 * HZ;
+static unsigned int efx_monitor_interval = 1 * HZ;
 
 /* This controls whether or not the driver will initialise devices
  * with invalid MAC addresses stored in the EEPROM or flash.  If true,
@@ -201,10 +194,15 @@ MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
  * Utility functions and prototypes
  *
  *************************************************************************/
-static void efx_remove_channel(struct efx_channel *channel);
+
+static void efx_remove_channels(struct efx_nic *efx);
 static void efx_remove_port(struct efx_nic *efx);
+static void efx_init_napi(struct efx_nic *efx);
 static void efx_fini_napi(struct efx_nic *efx);
-static void efx_fini_channels(struct efx_nic *efx);
+static void efx_fini_napi_channel(struct efx_channel *channel);
+static void efx_fini_struct(struct efx_nic *efx);
+static void efx_start_all(struct efx_nic *efx);
+static void efx_stop_all(struct efx_nic *efx);
 
 #define EFX_ASSERT_RESET_SERIALISED(efx)                \
         do {                                            \
@@ -248,7 +246,7 @@ static int efx_process_channel(struct efx_channel *channel, int budget)
 
         efx_rx_strategy(channel);
 
-        efx_fast_push_rx_descriptors(&efx->rx_queue[channel->channel]);
+        efx_fast_push_rx_descriptors(efx_channel_get_rx_queue(channel));
 
         return spent;
 }
@@ -310,6 +308,8 @@ static int efx_poll(struct napi_struct *napi, int budget)
                         channel->irq_mod_score = 0;
                 }
 
+                efx_filter_rfs_expire(channel);
+
                 /* There is no race here; although napi_disable() will
                  * only wait for napi_complete(), this isn't a problem
                  * since efx_channel_processed() will have no effect if
@@ -328,18 +328,23 @@ static int efx_poll(struct napi_struct *napi, int budget)
  * processing to finish, then directly poll (and ack ) the eventq.
  * Finally reenable NAPI and interrupts.
  *
- * Since we are touching interrupts the caller should hold the suspend lock
+ * This is for use only during a loopback self-test.  It must not
+ * deliver any packets up the stack as this can result in deadlock.
  */
 void efx_process_channel_now(struct efx_channel *channel)
 {
         struct efx_nic *efx = channel->efx;
 
+        BUG_ON(channel->channel >= efx->n_channels);
         BUG_ON(!channel->enabled);
+        BUG_ON(!efx->loopback_selftest);
 
         /* Disable interrupts and wait for ISRs to complete */
         efx_nic_disable_interrupts(efx);
-        if (efx->legacy_irq)
+        if (efx->legacy_irq) {
                 synchronize_irq(efx->legacy_irq);
+                efx->legacy_irq_enabled = false;
+        }
         if (channel->irq)
                 synchronize_irq(channel->irq);
 
@@ -347,13 +352,15 @@ void efx_process_channel_now(struct efx_channel *channel)
         napi_disable(&channel->napi_str);
 
         /* Poll the channel */
-        efx_process_channel(channel, EFX_EVQ_SIZE);
+        efx_process_channel(channel, channel->eventq_mask + 1);
 
         /* Ack the eventq. This may cause an interrupt to be generated
          * when they are reenabled */
         efx_channel_processed(channel);
 
         napi_enable(&channel->napi_str);
+        if (efx->legacy_irq)
+                efx->legacy_irq_enabled = true;
         efx_nic_enable_interrupts(efx);
 }
 
@@ -364,9 +371,18 @@ void efx_process_channel_now(struct efx_channel *channel)
  */
 static int efx_probe_eventq(struct efx_channel *channel)
 {
+        struct efx_nic *efx = channel->efx;
+        unsigned long entries;
+
         netif_dbg(channel->efx, probe, channel->efx->net_dev,
                   "chan %d create event queue\n", channel->channel);
 
+        /* Build an event queue with room for one event per tx and rx buffer,
+         * plus some extra for link state events and MCDI completions. */
+        entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
+        EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
+        channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
+
         return efx_nic_probe_eventq(channel);
 }
 
@@ -403,6 +419,61 @@ static void efx_remove_eventq(struct efx_channel *channel)
  *
  *************************************************************************/
 
+/* Allocate and initialise a channel structure, optionally copying
+ * parameters (but not resources) from an old channel structure. */
+static struct efx_channel *
+efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
+{
+        struct efx_channel *channel;
+        struct efx_rx_queue *rx_queue;
+        struct efx_tx_queue *tx_queue;
+        int j;
+
+        if (old_channel) {
+                channel = kmalloc(sizeof(*channel), GFP_KERNEL);
+                if (!channel)
+                        return NULL;
+
+                *channel = *old_channel;
+
+                channel->napi_dev = NULL;
+                memset(&channel->eventq, 0, sizeof(channel->eventq));
+
+                rx_queue = &channel->rx_queue;
+                rx_queue->buffer = NULL;
+                memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
+
+                for (j = 0; j < EFX_TXQ_TYPES; j++) {
+                        tx_queue = &channel->tx_queue[j];
+                        if (tx_queue->channel)
+                                tx_queue->channel = channel;
+                        tx_queue->buffer = NULL;
+                        memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
+                }
+        } else {
+                channel = kzalloc(sizeof(*channel), GFP_KERNEL);
+                if (!channel)
+                        return NULL;
+
+                channel->efx = efx;
+                channel->channel = i;
+
+                for (j = 0; j < EFX_TXQ_TYPES; j++) {
+                        tx_queue = &channel->tx_queue[j];
+                        tx_queue->efx = efx;
+                        tx_queue->queue = i * EFX_TXQ_TYPES + j;
+                        tx_queue->channel = channel;
+                }
+        }
+
+        rx_queue = &channel->rx_queue;
+        rx_queue->efx = efx;
+        setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,
+                    (unsigned long)rx_queue);
+
+        return channel;
+}
+
 static int efx_probe_channel(struct efx_channel *channel)
 {
         struct efx_tx_queue *tx_queue;
@@ -459,11 +530,38 @@ static void efx_set_channel_names(struct efx_nic *efx)
                                 number -= efx->n_rx_channels;
                         }
                 }
-                snprintf(channel->name, sizeof(channel->name),
+                snprintf(efx->channel_name[channel->channel],
+                         sizeof(efx->channel_name[0]),
                          "%s%s-%d", efx->name, type, number);
         }
 }
 
+static int efx_probe_channels(struct efx_nic *efx)
+{
+        struct efx_channel *channel;
+        int rc;
+
+        /* Restart special buffer allocation */
+        efx->next_buffer_table = 0;
+
+        efx_for_each_channel(channel, efx) {
+                rc = efx_probe_channel(channel);
+                if (rc) {
+                        netif_err(efx, probe, efx->net_dev,
+                                  "failed to create channel %d\n",
+                                  channel->channel);
+                        goto fail;
+                }
+        }
+        efx_set_channel_names(efx);
+
+        return 0;
+
+fail:
+        efx_remove_channels(efx);
+        return rc;
+}
+
 /* Channels are shutdown and reinitialised whilst the NIC is running
  * to propagate configuration changes (mtu, checksum offload), or
  * to clear hardware error conditions
@@ -580,7 +678,7 @@ static void efx_fini_channels(struct efx_nic *efx)
 
                 efx_for_each_channel_rx_queue(rx_queue, channel)
                         efx_fini_rx_queue(rx_queue);
-                efx_for_each_channel_tx_queue(tx_queue, channel)
+                efx_for_each_possible_channel_tx_queue(tx_queue, channel)
                         efx_fini_tx_queue(tx_queue);
                 efx_fini_eventq(channel);
         }
@@ -596,11 +694,84 @@ static void efx_remove_channel(struct efx_channel *channel)
 
         efx_for_each_channel_rx_queue(rx_queue, channel)
                 efx_remove_rx_queue(rx_queue);
-        efx_for_each_channel_tx_queue(tx_queue, channel)
+        efx_for_each_possible_channel_tx_queue(tx_queue, channel)
                 efx_remove_tx_queue(tx_queue);
         efx_remove_eventq(channel);
 }
 
+static void efx_remove_channels(struct efx_nic *efx)
+{
+        struct efx_channel *channel;
+
+        efx_for_each_channel(channel, efx)
+                efx_remove_channel(channel);
+}
+
+int
+efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
+{
+        struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
+        u32 old_rxq_entries, old_txq_entries;
+        unsigned i;
+        int rc;
+
+        efx_stop_all(efx);
+        efx_fini_channels(efx);
+
+        /* Clone channels */
+        memset(other_channel, 0, sizeof(other_channel));
+        for (i = 0; i < efx->n_channels; i++) {
+                channel = efx_alloc_channel(efx, i, efx->channel[i]);
+                if (!channel) {
+                        rc = -ENOMEM;
+                        goto out;
+                }
+                other_channel[i] = channel;
+        }
+
+        /* Swap entry counts and channel pointers */
+        old_rxq_entries = efx->rxq_entries;
+        old_txq_entries = efx->txq_entries;
+        efx->rxq_entries = rxq_entries;
+        efx->txq_entries = txq_entries;
+        for (i = 0; i < efx->n_channels; i++) {
+                channel = efx->channel[i];
+                efx->channel[i] = other_channel[i];
+                other_channel[i] = channel;
+        }
+
+        rc = efx_probe_channels(efx);
+        if (rc)
+                goto rollback;
+
+        efx_init_napi(efx);
+
+        /* Destroy old channels */
+        for (i = 0; i < efx->n_channels; i++) {
+                efx_fini_napi_channel(other_channel[i]);
+                efx_remove_channel(other_channel[i]);
+        }
+out:
+        /* Free unused channel structures */
+        for (i = 0; i < efx->n_channels; i++)
+                kfree(other_channel[i]);
+
+        efx_init_channels(efx);
+        efx_start_all(efx);
+        return rc;
+
+rollback:
+        /* Swap back */
+        efx->rxq_entries = old_rxq_entries;
+        efx->txq_entries = old_txq_entries;
+        for (i = 0; i < efx->n_channels; i++) {
+                channel = efx->channel[i];
+                efx->channel[i] = other_channel[i];
+                other_channel[i] = channel;
+        }
+        goto out;
+}
+
 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
 {
         mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));
@@ -627,11 +798,6 @@ void efx_link_status_changed(struct efx_nic *efx)
         if (!netif_running(efx->net_dev))
                 return;
 
-        if (efx->port_inhibited) {
-                netif_carrier_off(efx->net_dev);
-                return;
-        }
-
         if (link_state->up != netif_carrier_ok(efx->net_dev)) {
                 efx->n_link_state_changes++;
 
@@ -667,7 +833,7 @@ void efx_link_set_advertising(struct efx_nic *efx, u32 advertising)
         }
 }
 
-void efx_link_set_wanted_fc(struct efx_nic *efx, enum efx_fc_type wanted_fc)
+void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
 {
         efx->wanted_fc = wanted_fc;
         if (efx->link_advertising) {
@@ -751,6 +917,7 @@ static void efx_mac_work(struct work_struct *data)
 
 static int efx_probe_port(struct efx_nic *efx)
 {
+        unsigned char *perm_addr;
         int rc;
 
         netif_dbg(efx, probe, efx->net_dev, "create port\n");
@@ -761,14 +928,15 @@ static int efx_probe_port(struct efx_nic *efx)
         /* Connect up MAC/PHY operations table */
         rc = efx->type->probe_port(efx);
         if (rc)
-                goto err;
+                return rc;
 
         /* Sanity check MAC address */
-        if (is_valid_ether_addr(efx->mac_address)) {
-                memcpy(efx->net_dev->dev_addr, efx->mac_address, ETH_ALEN);
+        perm_addr = efx->net_dev->perm_addr;
+        if (is_valid_ether_addr(perm_addr)) {
+                memcpy(efx->net_dev->dev_addr, perm_addr, ETH_ALEN);
         } else {
                 netif_err(efx, probe, efx->net_dev, "invalid MAC address %pM\n",
-                          efx->mac_address);
+                          perm_addr);
                 if (!allow_bad_hwaddr) {
                         rc = -EINVAL;
                         goto err;
@@ -782,7 +950,7 @@ static int efx_probe_port(struct efx_nic *efx)
         return 0;
 
  err:
-        efx_remove_port(efx);
+        efx->type->remove_port(efx);
         return rc;
 }
 
@@ -883,6 +1051,7 @@ static int efx_init_io(struct efx_nic *efx)
 {
         struct pci_dev *pci_dev = efx->pci_dev;
         dma_addr_t dma_mask = efx->type->max_dma_mask;
+        bool use_wc;
         int rc;
 
         netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
@@ -933,8 +1102,21 @@ static int efx_init_io(struct efx_nic *efx)
                 rc = -EIO;
                 goto fail3;
         }
-        efx->membase = ioremap_nocache(efx->membase_phys,
-                                       efx->type->mem_map_size);
+
+        /* bug22643: If SR-IOV is enabled then tx push over a write combined
+         * mapping is unsafe. We need to disable write combining in this case.
+         * MSI is unsupported when SR-IOV is enabled, and the firmware will
+         * have removed the MSI capability. So write combining is safe if
+         * there is an MSI capability.
+         */
+        use_wc = (!EFX_WORKAROUND_22643(efx) ||
+                  pci_find_capability(pci_dev, PCI_CAP_ID_MSI));
+        if (use_wc)
+                efx->membase = ioremap_wc(efx->membase_phys,
+                                          efx->type->mem_map_size);
+        else
+                efx->membase = ioremap_nocache(efx->membase_phys,
+                                               efx->type->mem_map_size);
         if (!efx->membase) {
                 netif_err(efx, probe, efx->net_dev,
                           "could not map memory BAR at %llx+%x\n",
@@ -985,6 +1167,9 @@ static int efx_wanted_channels(void)
         int count;
         int cpu;
 
+        if (rss_cpus)
+                return rss_cpus;
+
         if (unlikely(!zalloc_cpumask_var(&core_mask, GFP_KERNEL))) {
                 printk(KERN_WARNING
                        "sfc: RSS disabled due to allocation failure\n");
@@ -1004,10 +1189,32 @@ static int efx_wanted_channels(void)
         return count;
 }
 
+static int
+efx_init_rx_cpu_rmap(struct efx_nic *efx, struct msix_entry *xentries)
+{
+#ifdef CONFIG_RFS_ACCEL
+        int i, rc;
+
+        efx->net_dev->rx_cpu_rmap = alloc_irq_cpu_rmap(efx->n_rx_channels);
+        if (!efx->net_dev->rx_cpu_rmap)
+                return -ENOMEM;
+        for (i = 0; i < efx->n_rx_channels; i++) {
+                rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,
+                                      xentries[i].vector);
+                if (rc) {
+                        free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
+                        efx->net_dev->rx_cpu_rmap = NULL;
+                        return rc;
+                }
+        }
+#endif
+        return 0;
+}
+
 /* Probe the number and type of interrupts we are able to obtain, and
  * the resulting numbers of channels and RX queues.
  */
-static void efx_probe_interrupts(struct efx_nic *efx)
+static int efx_probe_interrupts(struct efx_nic *efx)
 {
         int max_channels =
                 min_t(int, efx->type->phys_addr_channels, EFX_MAX_CHANNELS);
@@ -1049,8 +1256,14 @@ static void efx_probe_interrupts(struct efx_nic *efx)
                                 efx->n_tx_channels = efx->n_channels;
                                 efx->n_rx_channels = efx->n_channels;
                         }
+                        rc = efx_init_rx_cpu_rmap(efx, xentries);
+                        if (rc) {
+                                pci_disable_msix(efx->pci_dev);
+                                return rc;
+                        }
                         for (i = 0; i < n_channels; i++)
-                                efx->channel[i].irq = xentries[i].vector;
+                                efx_get_channel(efx, i)->irq =
+                                        xentries[i].vector;
                 } else {
                         /* Fall back to single channel MSI */
                         efx->interrupt_mode = EFX_INT_MODE_MSI;
@@ -1066,7 +1279,7 @@ static void efx_probe_interrupts(struct efx_nic *efx)
                 efx->n_tx_channels = 1;
                 rc = pci_enable_msi(efx->pci_dev);
                 if (rc == 0) {
-                        efx->channel[0].irq = efx->pci_dev->irq;
+                        efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
                 } else {
                         netif_err(efx, drv, efx->net_dev,
                                   "could not enable MSI\n");
@@ -1081,6 +1294,8 @@ static void efx_probe_interrupts(struct efx_nic *efx)
                 efx->n_tx_channels = 1;
                 efx->legacy_irq = efx->pci_dev->irq;
         }
+
+        return 0;
 }
 
 static void efx_remove_interrupts(struct efx_nic *efx)
@@ -1101,22 +1316,18 @@ static void efx_set_channels(struct efx_nic *efx)
 {
         struct efx_channel *channel;
         struct efx_tx_queue *tx_queue;
-        struct efx_rx_queue *rx_queue;
-        unsigned tx_channel_offset =
+
+        efx->tx_channel_offset =
                 separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0;
 
+        /* We need to adjust the TX queue numbers if we have separate
+         * RX-only and TX-only channels.
+         */
         efx_for_each_channel(channel, efx) {
-                if (channel->channel - tx_channel_offset < efx->n_tx_channels) {
-                        channel->tx_queue = &efx->tx_queue[
-                                (channel->channel - tx_channel_offset) *
-                                EFX_TXQ_TYPES];
-                        efx_for_each_channel_tx_queue(tx_queue, channel)
-                                tx_queue->channel = channel;
-                }
+                efx_for_each_channel_tx_queue(tx_queue, channel)
+                        tx_queue->queue -= (efx->tx_channel_offset *
+                                            EFX_TXQ_TYPES);
         }
-
-        efx_for_each_rx_queue(rx_queue, efx)
-                rx_queue->channel = &efx->channel[rx_queue->queue];
 }
 
 static int efx_probe_nic(struct efx_nic *efx)
@@ -1133,7 +1344,9 @@ static int efx_probe_nic(struct efx_nic *efx)
 
         /* Determine the number of channels and queues by trying to hook
          * in MSI-X interrupts. */
-        efx_probe_interrupts(efx);
+        rc = efx_probe_interrupts(efx);
+        if (rc)
+                goto fail;
 
         if (efx->n_channels > 1)
                 get_random_bytes(&efx->rx_hash_key, sizeof(efx->rx_hash_key));
@@ -1141,12 +1354,17 @@ static int efx_probe_nic(struct efx_nic *efx)
                 efx->rx_indir_table[i] = i % efx->n_rx_channels;
 
         efx_set_channels(efx);
-        efx->net_dev->real_num_tx_queues = efx->n_tx_channels;
+        netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
+        netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
 
         /* Initialise the interrupt moderation settings */
         efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true);
 
         return 0;
+
+fail:
+        efx->type->remove(efx);
+        return rc;
 }
 
 static void efx_remove_nic(struct efx_nic *efx)
@@ -1165,40 +1383,37 @@ static void efx_remove_nic(struct efx_nic *efx)
 
 static int efx_probe_all(struct efx_nic *efx)
 {
-        struct efx_channel *channel;
         int rc;
 
-        /* Create NIC */
         rc = efx_probe_nic(efx);
         if (rc) {
                 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
                 goto fail1;
         }
 
-        /* Create port */
         rc = efx_probe_port(efx);
         if (rc) {
                 netif_err(efx, probe, efx->net_dev, "failed to create port\n");
                 goto fail2;
         }
 
-        /* Create channels */
-        efx_for_each_channel(channel, efx) {
-                rc = efx_probe_channel(channel);
-                if (rc) {
-                        netif_err(efx, probe, efx->net_dev,
-                                  "failed to create channel %d\n",
-                                  channel->channel);
-                        goto fail3;
-                }
+        efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
+        rc = efx_probe_channels(efx);
+        if (rc)
+                goto fail3;
+
+        rc = efx_probe_filters(efx);
+        if (rc) {
+                netif_err(efx, probe, efx->net_dev,
+                          "failed to create filter tables\n");
+                goto fail4;
         }
-        efx_set_channel_names(efx);
 
         return 0;
 
+ fail4:
+        efx_remove_channels(efx);
  fail3:
-        efx_for_each_channel(channel, efx)
-                efx_remove_channel(channel);
         efx_remove_port(efx);
  fail2:
         efx_remove_nic(efx);
@@ -1230,12 +1445,14 @@ static void efx_start_all(struct efx_nic *efx)
          * restart the transmit interface early so the watchdog timer stops */
         efx_start_port(efx);
 
-        efx_for_each_channel(channel, efx) {
-                if (efx_dev_registered(efx))
-                        efx_wake_queue(channel);
+        if (efx_dev_registered(efx) && netif_device_present(efx->net_dev))
+                netif_tx_wake_all_queues(efx->net_dev);
+
+        efx_for_each_channel(channel, efx)
                 efx_start_channel(channel);
-        }
 
+        if (efx->legacy_irq)
+                efx->legacy_irq_enabled = true;
         efx_nic_enable_interrupts(efx);
 
         /* Switch to event based MCDI completions after enabling interrupts.
@@ -1296,8 +1513,10 @@ static void efx_stop_all(struct efx_nic *efx)
 
         /* Disable interrupts and wait for ISR to complete */
         efx_nic_disable_interrupts(efx);
-        if (efx->legacy_irq)
+        if (efx->legacy_irq) {
                 synchronize_irq(efx->legacy_irq);
+                efx->legacy_irq_enabled = false;
+        }
         efx_for_each_channel(channel, efx) {
                 if (channel->irq)
                         synchronize_irq(channel->irq);
@@ -1318,9 +1537,7 @@ static void efx_stop_all(struct efx_nic *efx)
         /* Stop the kernel transmit interface late, so the watchdog
          * timer isn't ticking over the flush */
         if (efx_dev_registered(efx)) {
-                struct efx_channel *channel;
-                efx_for_each_channel(channel, efx)
-                        efx_stop_queue(channel);
+                netif_tx_stop_all_queues(efx->net_dev);
                 netif_tx_lock_bh(efx->net_dev);
                 netif_tx_unlock_bh(efx->net_dev);
         }
@@ -1328,10 +1545,8 @@ static void efx_stop_all(struct efx_nic *efx)
 
 static void efx_remove_all(struct efx_nic *efx)
 {
-        struct efx_channel *channel;
-
-        efx_for_each_channel(channel, efx)
-                efx_remove_channel(channel);
+        efx_remove_filters(efx);
+        efx_remove_channels(efx);
         efx_remove_port(efx);
         efx_remove_nic(efx);
 }
@@ -1355,20 +1570,20 @@ static unsigned irq_mod_ticks(int usecs, int resolution)
 void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs,
                              bool rx_adaptive)
 {
-        struct efx_tx_queue *tx_queue;
-        struct efx_rx_queue *rx_queue;
+        struct efx_channel *channel;
         unsigned tx_ticks = irq_mod_ticks(tx_usecs, EFX_IRQ_MOD_RESOLUTION);
         unsigned rx_ticks = irq_mod_ticks(rx_usecs, EFX_IRQ_MOD_RESOLUTION);
 
         EFX_ASSERT_RESET_SERIALISED(efx);
 
-        efx_for_each_tx_queue(tx_queue, efx)
-                tx_queue->channel->irq_moderation = tx_ticks;
-
         efx->irq_rx_adaptive = rx_adaptive;
         efx->irq_rx_moderation = rx_ticks;
-        efx_for_each_rx_queue(rx_queue, efx)
-                rx_queue->channel->irq_moderation = rx_ticks;
+        efx_for_each_channel(channel, efx) {
+                if (efx_channel_has_rx_queue(channel))
+                        channel->irq_moderation = rx_ticks;
+                else if (efx_channel_has_tx_queues(channel))
+                        channel->irq_moderation = tx_ticks;
+        }
 }
 
 /**************************************************************************
@@ -1377,8 +1592,7 @@ void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs,
  *
  **************************************************************************/
 
-/* Run periodically off the general workqueue. Serialised against
- * efx_reconfigure_port via the mac_lock */
+/* Run periodically off the general workqueue */
 static void efx_monitor(struct work_struct *data)
 {
         struct efx_nic *efx = container_of(data, struct efx_nic,
@@ -1391,16 +1605,13 @@ static void efx_monitor(struct work_struct *data)
 
         /* If the mac_lock is already held then it is likely a port
          * reconfiguration is already in place, which will likely do
-         * most of the work of check_hw() anyway. */
-        if (!mutex_trylock(&efx->mac_lock))
-                goto out_requeue;
-        if (!efx->port_enabled)
-                goto out_unlock;
-        efx->type->monitor(efx);
+         * most of the work of monitor() anyway. */
+        if (mutex_trylock(&efx->mac_lock)) {
+                if (efx->port_enabled)
+                        efx->type->monitor(efx);
+                mutex_unlock(&efx->mac_lock);
+        }
 
-out_unlock:
-        mutex_unlock(&efx->mac_lock);
-out_requeue:
         queue_delayed_work(efx->workqueue, &efx->monitor_work,
                            efx_monitor_interval);
 }
@@ -1435,7 +1646,7 @@ static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
  *
  **************************************************************************/
 
-static int efx_init_napi(struct efx_nic *efx)
+static void efx_init_napi(struct efx_nic *efx)
 {
         struct efx_channel *channel;
 
@@ -1444,18 +1655,21 @@ static int efx_init_napi(struct efx_nic *efx)
                 netif_napi_add(channel->napi_dev, &channel->napi_str,
                                efx_poll, napi_weight);
         }
-        return 0;
+}
+
+static void efx_fini_napi_channel(struct efx_channel *channel)
+{
+        if (channel->napi_dev)
+                netif_napi_del(&channel->napi_str);
+        channel->napi_dev = NULL;
 }
 
 static void efx_fini_napi(struct efx_nic *efx)
 {
         struct efx_channel *channel;
 
-        efx_for_each_channel(channel, efx) {
-                if (channel->napi_dev)
-                        netif_napi_del(&channel->napi_str);
-                channel->napi_dev = NULL;
-        }
+        efx_for_each_channel(channel, efx)
+                efx_fini_napi_channel(channel);
 }
 
 /**************************************************************************
@@ -1546,11 +1760,11 @@ static struct rtnl_link_stats64 *efx_net_stats(struct net_device *net_dev, struc
         stats->tx_packets = mac_stats->tx_packets;
         stats->rx_bytes = mac_stats->rx_bytes;
         stats->tx_bytes = mac_stats->tx_bytes;
+        stats->rx_dropped = efx->n_rx_nodesc_drop_cnt;
         stats->multicast = mac_stats->rx_multicast;
         stats->collisions = mac_stats->tx_collision;
         stats->rx_length_errors = (mac_stats->rx_gtjumbo +
                                    mac_stats->rx_length_error);
-        stats->rx_over_errors = efx->n_rx_nodesc_drop_cnt;
         stats->rx_crc_errors = mac_stats->rx_bad;
         stats->rx_frame_errors = mac_stats->rx_align_error;
         stats->rx_fifo_errors = mac_stats->rx_overflow;
@@ -1669,6 +1883,17 @@ static void efx_set_multicast_list(struct net_device *net_dev)
         /* Otherwise efx_start_port() will do this */
 }
 
+static int efx_set_features(struct net_device *net_dev, u32 data)
+{
+        struct efx_nic *efx = netdev_priv(net_dev);
+
+        /* If disabling RX n-tuple filtering, clear existing filters */
+        if (net_dev->features & ~data & NETIF_F_NTUPLE)
+                efx_filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
+
+        return 0;
+}
+
 static const struct net_device_ops efx_netdev_ops = {
         .ndo_open               = efx_net_open,
         .ndo_stop               = efx_net_stop,
@@ -1680,9 +1905,14 @@ static const struct net_device_ops efx_netdev_ops = {
         .ndo_change_mtu         = efx_change_mtu,
         .ndo_set_mac_address    = efx_set_mac_address,
         .ndo_set_multicast_list = efx_set_multicast_list,
+        .ndo_set_features       = efx_set_features,
 #ifdef CONFIG_NET_POLL_CONTROLLER
         .ndo_poll_controller = efx_netpoll,
 #endif
+        .ndo_setup_tc           = efx_setup_tc,
+#ifdef CONFIG_RFS_ACCEL
+        .ndo_rx_flow_steer      = efx_filter_rfs,
+#endif
 };
 
 static void efx_update_name(struct efx_nic *efx)
@@ -1719,6 +1949,7 @@ static DEVICE_ATTR(phy_type, 0644, show_phy_type, NULL);
 static int efx_register_netdev(struct efx_nic *efx)
 {
         struct net_device *net_dev = efx->net_dev;
+        struct efx_channel *channel;
         int rc;
 
         net_dev->watchdog_timeo = 5 * HZ;
@@ -1741,6 +1972,12 @@ static int efx_register_netdev(struct efx_nic *efx)
         if (rc)
                 goto fail_locked;
 
+        efx_for_each_channel(channel, efx) {
+                struct efx_tx_queue *tx_queue;
+                efx_for_each_channel_tx_queue(tx_queue, channel)
+                        efx_init_tx_queue_core_txq(tx_queue);
+        }
+
         /* Always start with carrier off; PHY events will detect the link */
         netif_carrier_off(efx->net_dev);
 
@@ -1767,6 +2004,7 @@ fail_registered:
 
 static void efx_unregister_netdev(struct efx_nic *efx)
 {
+        struct efx_channel *channel;
         struct efx_tx_queue *tx_queue;
 
         if (!efx->net_dev)
@@ -1777,8 +2015,10 @@ static void efx_unregister_netdev(struct efx_nic *efx)
         /* Free up any skbs still remaining. This has to happen before
          * we try to unregister the netdev as running their destructors
          * may be needed to get the device ref. count to 0. */
-        efx_for_each_tx_queue(tx_queue, efx)
-                efx_release_tx_buffers(tx_queue);
+        efx_for_each_channel(channel, efx) {
+                efx_for_each_channel_tx_queue(tx_queue, channel)
+                        efx_release_tx_buffers(tx_queue);
+        }
 
         if (efx_dev_registered(efx)) {
                 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
@@ -1801,7 +2041,6 @@ void efx_reset_down(struct efx_nic *efx, enum reset_type method)
 
         efx_stop_all(efx);
         mutex_lock(&efx->mac_lock);
-        mutex_lock(&efx->spi_lock);
 
         efx_fini_channels(efx);
         if (efx->port_initialized && method != RESET_TYPE_INVISIBLE)
@@ -1841,8 +2080,8 @@ int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
         efx->mac_op->reconfigure(efx);
 
         efx_init_channels(efx);
+        efx_restore_filters(efx);
 
-        mutex_unlock(&efx->spi_lock);
         mutex_unlock(&efx->mac_lock);
 
         efx_start_all(efx);
@@ -1852,7 +2091,6 @@ int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
 fail:
         efx->port_initialized = false;
 
-        mutex_unlock(&efx->spi_lock);
         mutex_unlock(&efx->mac_lock);
 
         return rc;
@@ -1871,6 +2109,7 @@ int efx_reset(struct efx_nic *efx, enum reset_type method)
         netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
                    RESET_TYPE(method));
 
+        netif_device_detach(efx->net_dev);
         efx_reset_down(efx, method);
 
         rc = efx->type->reset(efx, method);
@@ -1904,6 +2143,7 @@ out:
                 efx->state = STATE_DISABLED;
         } else {
                 netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
+                netif_device_attach(efx->net_dev);
         }
         return rc;
 }
@@ -2010,15 +2250,13 @@ int efx_port_dummy_op_int(struct efx_nic *efx)
         return 0;
 }
 void efx_port_dummy_op_void(struct efx_nic *efx) {}
-void efx_port_dummy_op_set_id_led(struct efx_nic *efx, enum efx_led_mode mode)
-{
-}
-bool efx_port_dummy_op_poll(struct efx_nic *efx)
+
+static bool efx_port_dummy_op_poll(struct efx_nic *efx)
 {
         return false;
 }
 
-static struct efx_phy_operations efx_dummy_phy_operations = {
+static const struct efx_phy_operations efx_dummy_phy_operations = {
         .init            = efx_port_dummy_op_int,
         .reconfigure     = efx_port_dummy_op_int,
         .poll            = efx_port_dummy_op_poll,
@@ -2034,19 +2272,14 @@ static struct efx_phy_operations efx_dummy_phy_operations = {
 /* This zeroes out and then fills in the invariants in a struct
  * efx_nic (including all sub-structures).
  */
-static int efx_init_struct(struct efx_nic *efx, struct efx_nic_type *type,
+static int efx_init_struct(struct efx_nic *efx, const struct efx_nic_type *type,
                            struct pci_dev *pci_dev, struct net_device *net_dev)
 {
-        struct efx_channel *channel;
-        struct efx_tx_queue *tx_queue;
-        struct efx_rx_queue *rx_queue;
         int i;
 
         /* Initialise common structures */
         memset(efx, 0, sizeof(*efx));
         spin_lock_init(&efx->biu_lock);
-        mutex_init(&efx->mdio_lock);
-        mutex_init(&efx->spi_lock);
 #ifdef CONFIG_SFC_MTD
         INIT_LIST_HEAD(&efx->mtd_list);
 #endif
@@ -2059,7 +2292,6 @@ static int efx_init_struct(struct efx_nic *efx, struct efx_nic_type *type,
         strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
 
         efx->net_dev = net_dev;
-        efx->rx_checksum_enabled = true;
         spin_lock_init(&efx->stats_lock);
         mutex_init(&efx->mac_lock);
         efx->mac_op = type->default_mac_ops;
@@ -2068,36 +2300,13 @@ static int efx_init_struct(struct efx_nic *efx, struct efx_nic_type *type,
         INIT_WORK(&efx->mac_work, efx_mac_work);
 
         for (i = 0; i < EFX_MAX_CHANNELS; i++) {
-                channel = &efx->channel[i];
-                channel->efx = efx;
-                channel->channel = i;
-                channel->work_pending = false;
-                spin_lock_init(&channel->tx_stop_lock);
-                atomic_set(&channel->tx_stop_count, 1);
-        }
-        for (i = 0; i < EFX_MAX_TX_QUEUES; i++) {
-                tx_queue = &efx->tx_queue[i];
-                tx_queue->efx = efx;
-                tx_queue->queue = i;
-                tx_queue->buffer = NULL;
-                tx_queue->channel = &efx->channel[0]; /* for safety */
-                tx_queue->tso_headers_free = NULL;
-        }
-        for (i = 0; i < EFX_MAX_RX_QUEUES; i++) {
-                rx_queue = &efx->rx_queue[i];
-                rx_queue->efx = efx;
-                rx_queue->queue = i;
-                rx_queue->channel = &efx->channel[0]; /* for safety */
-                rx_queue->buffer = NULL;
-                setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,
-                            (unsigned long)rx_queue);
+                efx->channel[i] = efx_alloc_channel(efx, i, NULL);
+                if (!efx->channel[i])
+                        goto fail;
         }
 
         efx->type = type;
 
-        /* As close as we can get to guaranteeing that we don't overflow */
-        BUILD_BUG_ON(EFX_EVQ_SIZE < EFX_TXQ_SIZE + EFX_RXQ_SIZE);
-
         EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS);
 
         /* Higher numbered interrupt modes are less capable! */
@@ -2109,13 +2318,22 @@ static int efx_init_struct(struct efx_nic *efx, struct efx_nic_type *type,
                  pci_name(pci_dev));
         efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
         if (!efx->workqueue)
-                return -ENOMEM;
+                goto fail;
 
         return 0;
+
+fail:
+        efx_fini_struct(efx);
+        return -ENOMEM;
 }
 
 static void efx_fini_struct(struct efx_nic *efx)
 {
+        int i;
+
+        for (i = 0; i < EFX_MAX_CHANNELS; i++)
+                kfree(efx->channel[i]);
+
         if (efx->workqueue) {
                 destroy_workqueue(efx->workqueue);
                 efx->workqueue = NULL;
@@ -2133,6 +2351,10 @@ static void efx_fini_struct(struct efx_nic *efx)
  */
 static void efx_pci_remove_main(struct efx_nic *efx)
 {
+#ifdef CONFIG_RFS_ACCEL
+        free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
+        efx->net_dev->rx_cpu_rmap = NULL;
+#endif
         efx_nic_fini_interrupt(efx);
         efx_fini_channels(efx);
         efx_fini_port(efx);
@@ -2192,9 +2414,7 @@ static int efx_pci_probe_main(struct efx_nic *efx)
         if (rc)
                 goto fail1;
 
-        rc = efx_init_napi(efx);
-        if (rc)
-                goto fail2;
+        efx_init_napi(efx);
 
         rc = efx->type->init(efx);
         if (rc) {
@@ -2225,7 +2445,6 @@ static int efx_pci_probe_main(struct efx_nic *efx)
         efx->type->fini(efx);
  fail3:
         efx_fini_napi(efx);
- fail2:
         efx_remove_all(efx);
  fail1:
         return rc;
@@ -2243,23 +2462,27 @@ static int efx_pci_probe_main(struct efx_nic *efx)
 static int __devinit efx_pci_probe(struct pci_dev *pci_dev,
                                    const struct pci_device_id *entry)
 {
-        struct efx_nic_type *type = (struct efx_nic_type *) entry->driver_data;
+        const struct efx_nic_type *type = (const struct efx_nic_type *) entry->driver_data;
         struct net_device *net_dev;
         struct efx_nic *efx;
         int i, rc;
 
         /* Allocate and initialise a struct net_device and struct efx_nic */
-        net_dev = alloc_etherdev_mq(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES);
+        net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
+                                     EFX_MAX_RX_QUEUES);
         if (!net_dev)
                 return -ENOMEM;
         net_dev->features |= (type->offload_features | NETIF_F_SG |
                               NETIF_F_HIGHDMA | NETIF_F_TSO |
-                              NETIF_F_GRO);
+                              NETIF_F_RXCSUM);
         if (type->offload_features & NETIF_F_V6_CSUM)
                 net_dev->features |= NETIF_F_TSO6;
         /* Mask for features that also apply to VLAN devices */
         net_dev->vlan_features |= (NETIF_F_ALL_CSUM | NETIF_F_SG |
-                                   NETIF_F_HIGHDMA | NETIF_F_TSO);
+                                   NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
+                                   NETIF_F_RXCSUM);
+        /* All offloads can be toggled */
+        net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA;
         efx = netdev_priv(net_dev);
         pci_set_drvdata(pci_dev, efx);
         SET_NETDEV_DEV(net_dev, &pci_dev->dev);