sfc: Enable RX DMA scattering where possible

Enable RX DMA scattering iff an RX buffer large enough for the current
MTU will not fit into a single page and the NIC supports DMA
scattering for kernel-mode RX queues.

On Falcon and Siena, the RX_USR_BUF_SIZE field is used as the DMA
limit for both all RX queues with scatter enabled.  Set it to 1824,
matching what Onload uses now.

Maintain a statistic for frames truncated due to lack of descriptors
(rx_nodesc_trunc).  This is distinct from rx_frm_trunc which may be
incremented when scattering is disabled and implies an over-length
frame.

Whenever an MTU change causes scattering to be turned on or off,
update filters that point to the PF queues, but leave others
unchanged, as VF drivers assume scattering is off.

Add n_frags parameters to various functions, and make them iterate:
- efx_rx_packet()
- efx_recycle_rx_buffers()
- efx_rx_mk_skb()
- efx_rx_deliver()

Make efx_handle_rx_event() responsible for updating
efx_rx_queue::removed_count.

Change the RX pipeline state to a starting ring index and number of
fragments, and make __efx_rx_packet() responsible for clearing it.

Based on earlier versions by David Riddoch and Jon Cooper.

Signed-off-by: Ben Hutchings <bhutchings@solarflare.com>

1 files changed, 75 insertions, 15 deletions

diff --git a/drivers/net/ethernet/sfc/nic.c b/drivers/net/ethernet/sfc/nic.c
index 0ad790cc473c..f9f5df8b51fe 100644
--- a/drivers/net/ethernet/sfc/nic.c
+++ b/drivers/net/ethernet/sfc/nic.c
@@ -591,12 +591,22 @@ void efx_nic_init_rx(struct efx_rx_queue *rx_queue)
         struct efx_nic *efx = rx_queue->efx;
         bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0;
         bool iscsi_digest_en = is_b0;
+        bool jumbo_en;
+
+        /* For kernel-mode queues in Falcon A1, the JUMBO flag enables
+         * DMA to continue after a PCIe page boundary (and scattering
+         * is not possible).  In Falcon B0 and Siena, it enables
+         * scatter.
+         */
+        jumbo_en = !is_b0 || efx->rx_scatter;
 
         netif_dbg(efx, hw, efx->net_dev,
                   "RX queue %d ring in special buffers %d-%d\n",
                   efx_rx_queue_index(rx_queue), rx_queue->rxd.index,
                   rx_queue->rxd.index + rx_queue->rxd.entries - 1);
 
+        rx_queue->scatter_n = 0;
+
         /* Pin RX descriptor ring */
         efx_init_special_buffer(efx, &rx_queue->rxd);
 
@@ -613,8 +623,7 @@ void efx_nic_init_rx(struct efx_rx_queue *rx_queue)
                               FRF_AZ_RX_DESCQ_SIZE,
                               __ffs(rx_queue->rxd.entries),
                               FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
-                              /* For >=B0 this is scatter so disable */
-                              FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
+                              FRF_AZ_RX_DESCQ_JUMBO, jumbo_en,
                               FRF_AZ_RX_DESCQ_EN, 1);
         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
                          efx_rx_queue_index(rx_queue));
@@ -968,13 +977,24 @@ static u16 efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
                 EFX_RX_PKT_DISCARD : 0;
 }
 
-/* Handle receive events that are not in-order. */
-static void
+/* Handle receive events that are not in-order. Return true if this
+ * can be handled as a partial packet discard, false if it's more
+ * serious.
+ */
+static bool
 efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
 {
+        struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
         struct efx_nic *efx = rx_queue->efx;
         unsigned expected, dropped;
 
+        if (rx_queue->scatter_n &&
+            index == ((rx_queue->removed_count + rx_queue->scatter_n - 1) &
+                      rx_queue->ptr_mask)) {
+                ++channel->n_rx_nodesc_trunc;
+                return true;
+        }
+
         expected = rx_queue->removed_count & rx_queue->ptr_mask;
         dropped = (index - expected) & rx_queue->ptr_mask;
         netif_info(efx, rx_err, efx->net_dev,
@@ -983,6 +1003,7 @@ efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
 
         efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
                            RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
+        return false;
 }
 
 /* Handle a packet received event
@@ -998,7 +1019,7 @@ efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
         unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
         unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
         unsigned expected_ptr;
-        bool rx_ev_pkt_ok;
+        bool rx_ev_pkt_ok, rx_ev_sop, rx_ev_cont;
         u16 flags;
         struct efx_rx_queue *rx_queue;
         struct efx_nic *efx = channel->efx;
@@ -1006,21 +1027,56 @@ efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
         if (unlikely(ACCESS_ONCE(efx->reset_pending)))
                 return;
 
-        /* Basic packet information */
-        rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
-        rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
-        rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
-        WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
-        WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
+        rx_ev_cont = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT);
+        rx_ev_sop = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP);
         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
                 channel->channel);
 
         rx_queue = efx_channel_get_rx_queue(channel);
 
         rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
-        expected_ptr = rx_queue->removed_count & rx_queue->ptr_mask;
-        if (unlikely(rx_ev_desc_ptr != expected_ptr))
-                efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);
+        expected_ptr = ((rx_queue->removed_count + rx_queue->scatter_n) &
+                        rx_queue->ptr_mask);
+
+        /* Check for partial drops and other errors */
+        if (unlikely(rx_ev_desc_ptr != expected_ptr) ||
+            unlikely(rx_ev_sop != (rx_queue->scatter_n == 0))) {
+                if (rx_ev_desc_ptr != expected_ptr &&
+                    !efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr))
+                        return;
+
+                /* Discard all pending fragments */
+                if (rx_queue->scatter_n) {
+                        efx_rx_packet(
+                                rx_queue,
+                                rx_queue->removed_count & rx_queue->ptr_mask,
+                                rx_queue->scatter_n, 0, EFX_RX_PKT_DISCARD);
+                        rx_queue->removed_count += rx_queue->scatter_n;
+                        rx_queue->scatter_n = 0;
+                }
+
+                /* Return if there is no new fragment */
+                if (rx_ev_desc_ptr != expected_ptr)
+                        return;
+
+                /* Discard new fragment if not SOP */
+                if (!rx_ev_sop) {
+                        efx_rx_packet(
+                                rx_queue,
+                                rx_queue->removed_count & rx_queue->ptr_mask,
+                                1, 0, EFX_RX_PKT_DISCARD);
+                        ++rx_queue->removed_count;
+                        return;
+                }
+        }
+
+        ++rx_queue->scatter_n;
+        if (rx_ev_cont)
+                return;
+
+        rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
+        rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
+        rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 
         if (likely(rx_ev_pkt_ok)) {
                 /* If packet is marked as OK and packet type is TCP/IP or
@@ -1048,7 +1104,11 @@ efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
         channel->irq_mod_score += 2;
 
         /* Handle received packet */
-        efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt, flags);
+        efx_rx_packet(rx_queue,
+                      rx_queue->removed_count & rx_queue->ptr_mask,
+                      rx_queue->scatter_n, rx_ev_byte_cnt, flags);
+        rx_queue->removed_count += rx_queue->scatter_n;
+        rx_queue->scatter_n = 0;
 }
 
 /* If this flush done event corresponds to a &struct efx_tx_queue, then