1 files changed, 138 insertions, 122 deletions

diff --git a/drivers/net/sfc/falcon.c b/drivers/net/sfc/falcon.c
index e0c0b23f94ef..31ed1f49de00 100644
--- a/drivers/net/sfc/falcon.c
+++ b/drivers/net/sfc/falcon.c
@@ -108,10 +108,10 @@ MODULE_PARM_DESC(rx_xon_thresh_bytes, "RX fifo XON threshold");
 /* Max number of internal errors. After this resets will not be performed */
 #define FALCON_MAX_INT_ERRORS 4
 
-/* Maximum period that we wait for flush events. If the flush event
- * doesn't arrive in this period of time then we check if the queue
- * was disabled anyway. */
-#define FALCON_FLUSH_TIMEOUT 10 /* 10ms */
+/* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times
+ */
+#define FALCON_FLUSH_INTERVAL 10
+#define FALCON_FLUSH_POLL_COUNT 100
 
 /**************************************************************************
  *
@@ -452,6 +452,8 @@ void falcon_init_tx(struct efx_tx_queue *tx_queue)
         efx_oword_t tx_desc_ptr;
         struct efx_nic *efx = tx_queue->efx;
 
+        tx_queue->flushed = false;
+
         /* Pin TX descriptor ring */
         falcon_init_special_buffer(efx, &tx_queue->txd);
 
@@ -492,60 +494,16 @@ void falcon_init_tx(struct efx_tx_queue *tx_queue)
         }
 }
 
-static int falcon_flush_tx_queue(struct efx_tx_queue *tx_queue)
+static void falcon_flush_tx_queue(struct efx_tx_queue *tx_queue)
 {
         struct efx_nic *efx = tx_queue->efx;
-        struct efx_channel *channel = &efx->channel[0];
         efx_oword_t tx_flush_descq;
-        unsigned int read_ptr, i;
 
         /* Post a flush command */
         EFX_POPULATE_OWORD_2(tx_flush_descq,
                              TX_FLUSH_DESCQ_CMD, 1,
                              TX_FLUSH_DESCQ, tx_queue->queue);
         falcon_write(efx, &tx_flush_descq, TX_FLUSH_DESCQ_REG_KER);
-        msleep(FALCON_FLUSH_TIMEOUT);
-
-        if (EFX_WORKAROUND_7803(efx))
-                return 0;
-
-        /* Look for a flush completed event */
-        read_ptr = channel->eventq_read_ptr;
-        for (i = 0; i < FALCON_EVQ_SIZE; ++i) {
-                efx_qword_t *event = falcon_event(channel, read_ptr);
-                int ev_code, ev_sub_code, ev_queue;
-                if (!falcon_event_present(event))
-                        break;
-
-                ev_code = EFX_QWORD_FIELD(*event, EV_CODE);
-                ev_sub_code = EFX_QWORD_FIELD(*event, DRIVER_EV_SUB_CODE);
-                ev_queue = EFX_QWORD_FIELD(*event, DRIVER_EV_TX_DESCQ_ID);
-                if ((ev_sub_code == TX_DESCQ_FLS_DONE_EV_DECODE) &&
-                    (ev_queue == tx_queue->queue)) {
-                        EFX_LOG(efx, "tx queue %d flush command succesful\n",
-                                tx_queue->queue);
-                        return 0;
-                }
-
-                read_ptr = (read_ptr + 1) & FALCON_EVQ_MASK;
-        }
-
-        if (EFX_WORKAROUND_11557(efx)) {
-                efx_oword_t reg;
-                bool enabled;
-
-                falcon_read_table(efx, &reg, efx->type->txd_ptr_tbl_base,
-                                  tx_queue->queue);
-                enabled = EFX_OWORD_FIELD(reg, TX_DESCQ_EN);
-                if (!enabled) {
-                        EFX_LOG(efx, "tx queue %d disabled without a "
-                                "flush event seen\n", tx_queue->queue);
-                        return 0;
-                }
-        }
-
-        EFX_ERR(efx, "tx queue %d flush command timed out\n", tx_queue->queue);
-        return -ETIMEDOUT;
 }
 
 void falcon_fini_tx(struct efx_tx_queue *tx_queue)
@@ -553,9 +511,8 @@ void falcon_fini_tx(struct efx_tx_queue *tx_queue)
         struct efx_nic *efx = tx_queue->efx;
         efx_oword_t tx_desc_ptr;
 
-        /* Stop the hardware using the queue */
-        if (falcon_flush_tx_queue(tx_queue))
-                EFX_ERR(efx, "failed to flush tx queue %d\n", tx_queue->queue);
+        /* The queue should have been flushed */
+        WARN_ON(!tx_queue->flushed);
 
         /* Remove TX descriptor ring from card */
         EFX_ZERO_OWORD(tx_desc_ptr);
@@ -643,6 +600,8 @@ void falcon_init_rx(struct efx_rx_queue *rx_queue)
                 rx_queue->queue, rx_queue->rxd.index,
                 rx_queue->rxd.index + rx_queue->rxd.entries - 1);
 
+        rx_queue->flushed = false;
+
         /* Pin RX descriptor ring */
         falcon_init_special_buffer(efx, &rx_queue->rxd);
 
@@ -663,11 +622,9 @@ void falcon_init_rx(struct efx_rx_queue *rx_queue)
                            rx_queue->queue);
 }
 
-static int falcon_flush_rx_queue(struct efx_rx_queue *rx_queue)
+static void falcon_flush_rx_queue(struct efx_rx_queue *rx_queue)
 {
         struct efx_nic *efx = rx_queue->efx;
-        struct efx_channel *channel = &efx->channel[0];
-        unsigned int read_ptr, i;
         efx_oword_t rx_flush_descq;
 
         /* Post a flush command */
@@ -675,76 +632,15 @@ static int falcon_flush_rx_queue(struct efx_rx_queue *rx_queue)
                              RX_FLUSH_DESCQ_CMD, 1,
                              RX_FLUSH_DESCQ, rx_queue->queue);
         falcon_write(efx, &rx_flush_descq, RX_FLUSH_DESCQ_REG_KER);
-        msleep(FALCON_FLUSH_TIMEOUT);
-
-        if (EFX_WORKAROUND_7803(efx))
-                return 0;
-
-        /* Look for a flush completed event */
-        read_ptr = channel->eventq_read_ptr;
-        for (i = 0; i < FALCON_EVQ_SIZE; ++i) {
-                efx_qword_t *event = falcon_event(channel, read_ptr);
-                int ev_code, ev_sub_code, ev_queue;
-                bool ev_failed;
-                if (!falcon_event_present(event))
-                        break;
-
-                ev_code = EFX_QWORD_FIELD(*event, EV_CODE);
-                ev_sub_code = EFX_QWORD_FIELD(*event, DRIVER_EV_SUB_CODE);
-                ev_queue = EFX_QWORD_FIELD(*event, DRIVER_EV_RX_DESCQ_ID);
-                ev_failed = EFX_QWORD_FIELD(*event, DRIVER_EV_RX_FLUSH_FAIL);
-
-                if ((ev_sub_code == RX_DESCQ_FLS_DONE_EV_DECODE) &&
-                    (ev_queue == rx_queue->queue)) {
-                        if (ev_failed) {
-                                EFX_INFO(efx, "rx queue %d flush command "
-                                         "failed\n", rx_queue->queue);
-                                return -EAGAIN;
-                        } else {
-                                EFX_LOG(efx, "rx queue %d flush command "
-                                        "succesful\n", rx_queue->queue);
-                                return 0;
-                        }
-                }
-
-                read_ptr = (read_ptr + 1) & FALCON_EVQ_MASK;
-        }
-
-        if (EFX_WORKAROUND_11557(efx)) {
-                efx_oword_t reg;
-                bool enabled;
-
-                falcon_read_table(efx, &reg, efx->type->rxd_ptr_tbl_base,
-                                  rx_queue->queue);
-                enabled = EFX_OWORD_FIELD(reg, RX_DESCQ_EN);
-                if (!enabled) {
-                        EFX_LOG(efx, "rx queue %d disabled without a "
-                                "flush event seen\n", rx_queue->queue);
-                        return 0;
-                }
-        }
-
-        EFX_ERR(efx, "rx queue %d flush command timed out\n", rx_queue->queue);
-        return -ETIMEDOUT;
 }
 
 void falcon_fini_rx(struct efx_rx_queue *rx_queue)
 {
         efx_oword_t rx_desc_ptr;
         struct efx_nic *efx = rx_queue->efx;
-        int i, rc;
 
-        /* Try and flush the rx queue. This may need to be repeated */
-        for (i = 0; i < 5; i++) {
-                rc = falcon_flush_rx_queue(rx_queue);
-                if (rc == -EAGAIN)
-                        continue;
-                break;
-        }
-        if (rc) {
-                EFX_ERR(efx, "failed to flush rx queue %d\n", rx_queue->queue);
-                efx_schedule_reset(efx, RESET_TYPE_INVISIBLE);
-        }
+        /* The queue should already have been flushed */
+        WARN_ON(!rx_queue->flushed);
 
         /* Remove RX descriptor ring from card */
         EFX_ZERO_OWORD(rx_desc_ptr);
@@ -1007,7 +903,7 @@ static void falcon_handle_global_event(struct efx_channel *channel,
                 is_phy_event = true;
 
         if ((falcon_rev(efx) >= FALCON_REV_B0) &&
-            EFX_OWORD_FIELD(*event, XG_MNT_INTR_B0))
+            EFX_QWORD_FIELD(*event, XG_MNT_INTR_B0))
                 is_phy_event = true;
 
         if (is_phy_event) {
@@ -1255,6 +1151,121 @@ void falcon_generate_test_event(struct efx_channel *channel, unsigned int magic)
         falcon_generate_event(channel, &test_event);
 }
 
+/**************************************************************************
+ *
+ * Flush handling
+ *
+ **************************************************************************/
+
+
+static void falcon_poll_flush_events(struct efx_nic *efx)
+{
+        struct efx_channel *channel = &efx->channel[0];
+        struct efx_tx_queue *tx_queue;
+        struct efx_rx_queue *rx_queue;
+        unsigned int read_ptr, i;
+
+        read_ptr = channel->eventq_read_ptr;
+        for (i = 0; i < FALCON_EVQ_SIZE; ++i) {
+                efx_qword_t *event = falcon_event(channel, read_ptr);
+                int ev_code, ev_sub_code, ev_queue;
+                bool ev_failed;
+                if (!falcon_event_present(event))
+                        break;
+
+                ev_code = EFX_QWORD_FIELD(*event, EV_CODE);
+                if (ev_code != DRIVER_EV_DECODE)
+                        continue;
+
+                ev_sub_code = EFX_QWORD_FIELD(*event, DRIVER_EV_SUB_CODE);
+                switch (ev_sub_code) {
+                case TX_DESCQ_FLS_DONE_EV_DECODE:
+                        ev_queue = EFX_QWORD_FIELD(*event,
+                                                   DRIVER_EV_TX_DESCQ_ID);
+                        if (ev_queue < EFX_TX_QUEUE_COUNT) {
+                                tx_queue = efx->tx_queue + ev_queue;
+                                tx_queue->flushed = true;
+                        }
+                        break;
+                case RX_DESCQ_FLS_DONE_EV_DECODE:
+                        ev_queue = EFX_QWORD_FIELD(*event,
+                                                   DRIVER_EV_RX_DESCQ_ID);
+                        ev_failed = EFX_QWORD_FIELD(*event,
+                                                    DRIVER_EV_RX_FLUSH_FAIL);
+                        if (ev_queue < efx->n_rx_queues) {
+                                rx_queue = efx->rx_queue + ev_queue;
+
+                                /* retry the rx flush */
+                                if (ev_failed)
+                                        falcon_flush_rx_queue(rx_queue);
+                                else
+                                        rx_queue->flushed = true;
+                        }
+                        break;
+                }
+
+                read_ptr = (read_ptr + 1) & FALCON_EVQ_MASK;
+        }
+}
+
+/* Handle tx and rx flushes at the same time, since they run in
+ * parallel in the hardware and there's no reason for us to
+ * serialise them */
+int falcon_flush_queues(struct efx_nic *efx)
+{
+        struct efx_rx_queue *rx_queue;
+        struct efx_tx_queue *tx_queue;
+        int i;
+        bool outstanding;
+
+        /* Issue flush requests */
+        efx_for_each_tx_queue(tx_queue, efx) {
+                tx_queue->flushed = false;
+                falcon_flush_tx_queue(tx_queue);
+        }
+        efx_for_each_rx_queue(rx_queue, efx) {
+                rx_queue->flushed = false;
+                falcon_flush_rx_queue(rx_queue);
+        }
+
+        /* Poll the evq looking for flush completions. Since we're not pushing
+         * any more rx or tx descriptors at this point, we're in no danger of
+         * overflowing the evq whilst we wait */
+        for (i = 0; i < FALCON_FLUSH_POLL_COUNT; ++i) {
+                msleep(FALCON_FLUSH_INTERVAL);
+                falcon_poll_flush_events(efx);
+
+                /* Check if every queue has been succesfully flushed */
+                outstanding = false;
+                efx_for_each_tx_queue(tx_queue, efx)
+                        outstanding |= !tx_queue->flushed;
+                efx_for_each_rx_queue(rx_queue, efx)
+                        outstanding |= !rx_queue->flushed;
+                if (!outstanding)
+                        return 0;
+        }
+
+        /* Mark the queues as all flushed. We're going to return failure
+         * leading to a reset, or fake up success anyway. "flushed" now
+         * indicates that we tried to flush. */
+        efx_for_each_tx_queue(tx_queue, efx) {
+                if (!tx_queue->flushed)
+                        EFX_ERR(efx, "tx queue %d flush command timed out\n",
+                                tx_queue->queue);
+                tx_queue->flushed = true;
+        }
+        efx_for_each_rx_queue(rx_queue, efx) {
+                if (!rx_queue->flushed)
+                        EFX_ERR(efx, "rx queue %d flush command timed out\n",
+                                rx_queue->queue);
+                rx_queue->flushed = true;
+        }
+
+        if (EFX_WORKAROUND_7803(efx))
+                return 0;
+
+        return -ETIMEDOUT;
+}
 
 /**************************************************************************
  *
@@ -1363,10 +1374,11 @@ static irqreturn_t falcon_fatal_interrupt(struct efx_nic *efx)
                         EFX_OWORD_FMT "\n", EFX_OWORD_VAL(reg));
         }
 
-        /* Disable DMA bus mastering on both devices */
+        /* Disable both devices */
         pci_disable_device(efx->pci_dev);
         if (FALCON_IS_DUAL_FUNC(efx))
                 pci_disable_device(nic_data->pci_dev2);
+        falcon_disable_interrupts(efx);
 
         if (++n_int_errors < FALCON_MAX_INT_ERRORS) {
                 EFX_ERR(efx, "SYSTEM ERROR - reset scheduled\n");
@@ -1593,7 +1605,7 @@ void falcon_fini_interrupt(struct efx_nic *efx)
  **************************************************************************
  */
 
-#define FALCON_SPI_MAX_LEN sizeof(efx_oword_t)
+#define FALCON_SPI_MAX_LEN ((unsigned) sizeof(efx_oword_t))
 
 /* Wait for SPI command completion */
 static int falcon_spi_wait(struct efx_nic *efx)
@@ -1942,8 +1954,10 @@ int falcon_dma_stats(struct efx_nic *efx, unsigned int done_offset)
 
         /* Wait for transfer to complete */
         for (i = 0; i < 400; i++) {
-                if (*(volatile u32 *)dma_done == FALCON_STATS_DONE)
+                if (*(volatile u32 *)dma_done == FALCON_STATS_DONE) {
+                        rmb(); /* Ensure the stats are valid. */
                         return 0;
+                }
                 udelay(10);
         }
 
@@ -2758,6 +2772,8 @@ int falcon_probe_nic(struct efx_nic *efx)
 
         /* Allocate storage for hardware specific data */
         nic_data = kzalloc(sizeof(*nic_data), GFP_KERNEL);
+        if (!nic_data)
+                return -ENOMEM;
         efx->nic_data = nic_data;
 
         /* Determine number of ports etc. */