aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e.h3
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_debugfs.c12
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_ethtool.c81
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_main.c54
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_txrx.c421
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_txrx.h68
-rw-r--r--drivers/net/ethernet/intel/i40evf/i40e_txrx.c425
-rw-r--r--drivers/net/ethernet/intel/i40evf/i40e_txrx.h67
-rw-r--r--drivers/net/ethernet/intel/i40evf/i40evf.h3
-rw-r--r--drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c46
-rw-r--r--drivers/net/ethernet/intel/i40evf/i40evf_main.c22
-rw-r--r--drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c24
12 files changed, 769 insertions, 457 deletions
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index 46e9f4e0a02c..ebe795a7f5f9 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -824,6 +824,7 @@ struct i40e_q_vector {
824 struct i40e_ring_container rx; 824 struct i40e_ring_container rx;
825 struct i40e_ring_container tx; 825 struct i40e_ring_container tx;
826 826
827 u8 itr_countdown; /* when 0 should adjust adaptive ITR */
827 u8 num_ringpairs; /* total number of ring pairs in vector */ 828 u8 num_ringpairs; /* total number of ring pairs in vector */
828 829
829 cpumask_t affinity_mask; 830 cpumask_t affinity_mask;
@@ -832,8 +833,6 @@ struct i40e_q_vector {
832 struct rcu_head rcu; /* to avoid race with update stats on free */ 833 struct rcu_head rcu; /* to avoid race with update stats on free */
833 char name[I40E_INT_NAME_STR_LEN]; 834 char name[I40E_INT_NAME_STR_LEN];
834 bool arm_wb_state; 835 bool arm_wb_state;
835#define ITR_COUNTDOWN_START 100
836 u8 itr_countdown; /* when 0 should adjust ITR */
837} ____cacheline_internodealigned_in_smp; 836} ____cacheline_internodealigned_in_smp;
838 837
839/* lan device */ 838/* lan device */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
index 4c3b4243cf65..e9fc51bd6c95 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
@@ -315,9 +315,9 @@ static void i40e_dbg_dump_vsi_seid(struct i40e_pf *pf, int seid)
315 i, rx_ring->vsi, 315 i, rx_ring->vsi,
316 rx_ring->q_vector); 316 rx_ring->q_vector);
317 dev_info(&pf->pdev->dev, 317 dev_info(&pf->pdev->dev,
318 " rx_rings[%i]: rx_itr_setting = %d (%s)\n", 318 " rx_rings[%i]: itr_setting = %d (%s)\n",
319 i, rx_ring->rx_itr_setting, 319 i, rx_ring->itr_setting,
320 ITR_IS_DYNAMIC(rx_ring->rx_itr_setting) ? "dynamic" : "fixed"); 320 ITR_IS_DYNAMIC(rx_ring->itr_setting) ? "dynamic" : "fixed");
321 } 321 }
322 for (i = 0; i < vsi->num_queue_pairs; i++) { 322 for (i = 0; i < vsi->num_queue_pairs; i++) {
323 struct i40e_ring *tx_ring = READ_ONCE(vsi->tx_rings[i]); 323 struct i40e_ring *tx_ring = READ_ONCE(vsi->tx_rings[i]);
@@ -366,9 +366,9 @@ static void i40e_dbg_dump_vsi_seid(struct i40e_pf *pf, int seid)
366 " tx_rings[%i]: DCB tc = %d\n", 366 " tx_rings[%i]: DCB tc = %d\n",
367 i, tx_ring->dcb_tc); 367 i, tx_ring->dcb_tc);
368 dev_info(&pf->pdev->dev, 368 dev_info(&pf->pdev->dev,
369 " tx_rings[%i]: tx_itr_setting = %d (%s)\n", 369 " tx_rings[%i]: itr_setting = %d (%s)\n",
370 i, tx_ring->tx_itr_setting, 370 i, tx_ring->itr_setting,
371 ITR_IS_DYNAMIC(tx_ring->tx_itr_setting) ? "dynamic" : "fixed"); 371 ITR_IS_DYNAMIC(tx_ring->itr_setting) ? "dynamic" : "fixed");
372 } 372 }
373 rcu_read_unlock(); 373 rcu_read_unlock();
374 dev_info(&pf->pdev->dev, 374 dev_info(&pf->pdev->dev,
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index 2f5bee713fef..29a7412b2fa6 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -2244,14 +2244,14 @@ static int __i40e_get_coalesce(struct net_device *netdev,
2244 rx_ring = vsi->rx_rings[queue]; 2244 rx_ring = vsi->rx_rings[queue];
2245 tx_ring = vsi->tx_rings[queue]; 2245 tx_ring = vsi->tx_rings[queue];
2246 2246
2247 if (ITR_IS_DYNAMIC(rx_ring->rx_itr_setting)) 2247 if (ITR_IS_DYNAMIC(rx_ring->itr_setting))
2248 ec->use_adaptive_rx_coalesce = 1; 2248 ec->use_adaptive_rx_coalesce = 1;
2249 2249
2250 if (ITR_IS_DYNAMIC(tx_ring->tx_itr_setting)) 2250 if (ITR_IS_DYNAMIC(tx_ring->itr_setting))
2251 ec->use_adaptive_tx_coalesce = 1; 2251 ec->use_adaptive_tx_coalesce = 1;
2252 2252
2253 ec->rx_coalesce_usecs = rx_ring->rx_itr_setting & ~I40E_ITR_DYNAMIC; 2253 ec->rx_coalesce_usecs = rx_ring->itr_setting & ~I40E_ITR_DYNAMIC;
2254 ec->tx_coalesce_usecs = tx_ring->tx_itr_setting & ~I40E_ITR_DYNAMIC; 2254 ec->tx_coalesce_usecs = tx_ring->itr_setting & ~I40E_ITR_DYNAMIC;
2255 2255
2256 /* we use the _usecs_high to store/set the interrupt rate limit 2256 /* we use the _usecs_high to store/set the interrupt rate limit
2257 * that the hardware supports, that almost but not quite 2257 * that the hardware supports, that almost but not quite
@@ -2311,34 +2311,35 @@ static void i40e_set_itr_per_queue(struct i40e_vsi *vsi,
2311 struct i40e_pf *pf = vsi->back; 2311 struct i40e_pf *pf = vsi->back;
2312 struct i40e_hw *hw = &pf->hw; 2312 struct i40e_hw *hw = &pf->hw;
2313 struct i40e_q_vector *q_vector; 2313 struct i40e_q_vector *q_vector;
2314 u16 vector, intrl; 2314 u16 intrl;
2315 2315
2316 intrl = i40e_intrl_usec_to_reg(vsi->int_rate_limit); 2316 intrl = i40e_intrl_usec_to_reg(vsi->int_rate_limit);
2317 2317
2318 rx_ring->rx_itr_setting = ec->rx_coalesce_usecs; 2318 rx_ring->itr_setting = ITR_REG_ALIGN(ec->rx_coalesce_usecs);
2319 tx_ring->tx_itr_setting = ec->tx_coalesce_usecs; 2319 tx_ring->itr_setting = ITR_REG_ALIGN(ec->tx_coalesce_usecs);
2320 2320
2321 if (ec->use_adaptive_rx_coalesce) 2321 if (ec->use_adaptive_rx_coalesce)
2322 rx_ring->rx_itr_setting |= I40E_ITR_DYNAMIC; 2322 rx_ring->itr_setting |= I40E_ITR_DYNAMIC;
2323 else 2323 else
2324 rx_ring->rx_itr_setting &= ~I40E_ITR_DYNAMIC; 2324 rx_ring->itr_setting &= ~I40E_ITR_DYNAMIC;
2325 2325
2326 if (ec->use_adaptive_tx_coalesce) 2326 if (ec->use_adaptive_tx_coalesce)
2327 tx_ring->tx_itr_setting |= I40E_ITR_DYNAMIC; 2327 tx_ring->itr_setting |= I40E_ITR_DYNAMIC;
2328 else 2328 else
2329 tx_ring->tx_itr_setting &= ~I40E_ITR_DYNAMIC; 2329 tx_ring->itr_setting &= ~I40E_ITR_DYNAMIC;
2330 2330
2331 q_vector = rx_ring->q_vector; 2331 q_vector = rx_ring->q_vector;
2332 q_vector->rx.itr = ITR_TO_REG(rx_ring->rx_itr_setting); 2332 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
2333 vector = vsi->base_vector + q_vector->v_idx;
2334 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), q_vector->rx.itr);
2335 2333
2336 q_vector = tx_ring->q_vector; 2334 q_vector = tx_ring->q_vector;
2337 q_vector->tx.itr = ITR_TO_REG(tx_ring->tx_itr_setting); 2335 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
2338 vector = vsi->base_vector + q_vector->v_idx;
2339 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), q_vector->tx.itr);
2340 2336
2341 wr32(hw, I40E_PFINT_RATEN(vector - 1), intrl); 2337 /* The interrupt handler itself will take care of programming
2338 * the Tx and Rx ITR values based on the values we have entered
2339 * into the q_vector, no need to write the values now.
2340 */
2341
2342 wr32(hw, I40E_PFINT_RATEN(q_vector->reg_idx), intrl);
2342 i40e_flush(hw); 2343 i40e_flush(hw);
2343} 2344}
2344 2345
@@ -2364,11 +2365,11 @@ static int __i40e_set_coalesce(struct net_device *netdev,
2364 vsi->work_limit = ec->tx_max_coalesced_frames_irq; 2365 vsi->work_limit = ec->tx_max_coalesced_frames_irq;
2365 2366
2366 if (queue < 0) { 2367 if (queue < 0) {
2367 cur_rx_itr = vsi->rx_rings[0]->rx_itr_setting; 2368 cur_rx_itr = vsi->rx_rings[0]->itr_setting;
2368 cur_tx_itr = vsi->tx_rings[0]->tx_itr_setting; 2369 cur_tx_itr = vsi->tx_rings[0]->itr_setting;
2369 } else if (queue < vsi->num_queue_pairs) { 2370 } else if (queue < vsi->num_queue_pairs) {
2370 cur_rx_itr = vsi->rx_rings[queue]->rx_itr_setting; 2371 cur_rx_itr = vsi->rx_rings[queue]->itr_setting;
2371 cur_tx_itr = vsi->tx_rings[queue]->tx_itr_setting; 2372 cur_tx_itr = vsi->tx_rings[queue]->itr_setting;
2372 } else { 2373 } else {
2373 netif_info(pf, drv, netdev, "Invalid queue value, queue range is 0 - %d\n", 2374 netif_info(pf, drv, netdev, "Invalid queue value, queue range is 0 - %d\n",
2374 vsi->num_queue_pairs - 1); 2375 vsi->num_queue_pairs - 1);
@@ -2396,7 +2397,7 @@ static int __i40e_set_coalesce(struct net_device *netdev,
2396 return -EINVAL; 2397 return -EINVAL;
2397 } 2398 }
2398 2399
2399 if (ec->rx_coalesce_usecs > (I40E_MAX_ITR << 1)) { 2400 if (ec->rx_coalesce_usecs > I40E_MAX_ITR) {
2400 netif_info(pf, drv, netdev, "Invalid value, rx-usecs range is 0-8160\n"); 2401 netif_info(pf, drv, netdev, "Invalid value, rx-usecs range is 0-8160\n");
2401 return -EINVAL; 2402 return -EINVAL;
2402 } 2403 }
@@ -2407,16 +2408,16 @@ static int __i40e_set_coalesce(struct net_device *netdev,
2407 return -EINVAL; 2408 return -EINVAL;
2408 } 2409 }
2409 2410
2410 if (ec->tx_coalesce_usecs > (I40E_MAX_ITR << 1)) { 2411 if (ec->tx_coalesce_usecs > I40E_MAX_ITR) {
2411 netif_info(pf, drv, netdev, "Invalid value, tx-usecs range is 0-8160\n"); 2412 netif_info(pf, drv, netdev, "Invalid value, tx-usecs range is 0-8160\n");
2412 return -EINVAL; 2413 return -EINVAL;
2413 } 2414 }
2414 2415
2415 if (ec->use_adaptive_rx_coalesce && !cur_rx_itr) 2416 if (ec->use_adaptive_rx_coalesce && !cur_rx_itr)
2416 ec->rx_coalesce_usecs = I40E_MIN_ITR << 1; 2417 ec->rx_coalesce_usecs = I40E_MIN_ITR;
2417 2418
2418 if (ec->use_adaptive_tx_coalesce && !cur_tx_itr) 2419 if (ec->use_adaptive_tx_coalesce && !cur_tx_itr)
2419 ec->tx_coalesce_usecs = I40E_MIN_ITR << 1; 2420 ec->tx_coalesce_usecs = I40E_MIN_ITR;
2420 2421
2421 intrl_reg = i40e_intrl_usec_to_reg(ec->rx_coalesce_usecs_high); 2422 intrl_reg = i40e_intrl_usec_to_reg(ec->rx_coalesce_usecs_high);
2422 vsi->int_rate_limit = INTRL_REG_TO_USEC(intrl_reg); 2423 vsi->int_rate_limit = INTRL_REG_TO_USEC(intrl_reg);
@@ -4406,6 +4407,8 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
4406 } 4407 }
4407 4408
4408flags_complete: 4409flags_complete:
4410 changed_flags = orig_flags ^ new_flags;
4411
4409 /* Before we finalize any flag changes, we need to perform some 4412 /* Before we finalize any flag changes, we need to perform some
4410 * checks to ensure that the changes are supported and safe. 4413 * checks to ensure that the changes are supported and safe.
4411 */ 4414 */
@@ -4415,13 +4418,17 @@ flags_complete:
4415 !(pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)) 4418 !(pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE))
4416 return -EOPNOTSUPP; 4419 return -EOPNOTSUPP;
4417 4420
4418 /* Disable FW LLDP not supported if NPAR active or if FW 4421 /* If the driver detected FW LLDP was disabled on init, this flag could
4419 * API version < 1.7 4422 * be set, however we do not support _changing_ the flag if NPAR is
4423 * enabled or FW API version < 1.7. There are situations where older
4424 * FW versions/NPAR enabled PFs could disable LLDP, however we _must_
4425 * not allow the user to enable/disable LLDP with this flag on
4426 * unsupported FW versions.
4420 */ 4427 */
4421 if (new_flags & I40E_FLAG_DISABLE_FW_LLDP) { 4428 if (changed_flags & I40E_FLAG_DISABLE_FW_LLDP) {
4422 if (pf->hw.func_caps.npar_enable) { 4429 if (pf->hw.func_caps.npar_enable) {
4423 dev_warn(&pf->pdev->dev, 4430 dev_warn(&pf->pdev->dev,
4424 "Unable to stop FW LLDP if NPAR active\n"); 4431 "Unable to change FW LLDP if NPAR active\n");
4425 return -EOPNOTSUPP; 4432 return -EOPNOTSUPP;
4426 } 4433 }
4427 4434
@@ -4429,7 +4436,7 @@ flags_complete:
4429 (pf->hw.aq.api_maj_ver == 1 && 4436 (pf->hw.aq.api_maj_ver == 1 &&
4430 pf->hw.aq.api_min_ver < 7)) { 4437 pf->hw.aq.api_min_ver < 7)) {
4431 dev_warn(&pf->pdev->dev, 4438 dev_warn(&pf->pdev->dev,
4432 "FW ver does not support stopping FW LLDP\n"); 4439 "FW ver does not support changing FW LLDP\n");
4433 return -EOPNOTSUPP; 4440 return -EOPNOTSUPP;
4434 } 4441 }
4435 } 4442 }
@@ -4439,6 +4446,10 @@ flags_complete:
4439 * something else has modified the flags variable since we copied it 4446 * something else has modified the flags variable since we copied it
4440 * originally. We'll just punt with an error and log something in the 4447 * originally. We'll just punt with an error and log something in the
4441 * message buffer. 4448 * message buffer.
4449 *
4450 * This is the point of no return for this function. We need to have
4451 * checked any discrepancies or misconfigurations and returned
4452 * EOPNOTSUPP before updating pf->flags here.
4442 */ 4453 */
4443 if (cmpxchg64(&pf->flags, orig_flags, new_flags) != orig_flags) { 4454 if (cmpxchg64(&pf->flags, orig_flags, new_flags) != orig_flags) {
4444 dev_warn(&pf->pdev->dev, 4455 dev_warn(&pf->pdev->dev,
@@ -4446,8 +4457,6 @@ flags_complete:
4446 return -EAGAIN; 4457 return -EAGAIN;
4447 } 4458 }
4448 4459
4449 changed_flags = orig_flags ^ new_flags;
4450
4451 /* Process any additional changes needed as a result of flag changes. 4460 /* Process any additional changes needed as a result of flag changes.
4452 * The changed_flags value reflects the list of bits that were 4461 * The changed_flags value reflects the list of bits that were
4453 * changed in the code above. 4462 * changed in the code above.
@@ -4479,6 +4488,12 @@ flags_complete:
4479 } 4488 }
4480 } 4489 }
4481 4490
4491 if ((changed_flags & pf->flags &
4492 I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) &&
4493 (pf->flags & I40E_FLAG_MFP_ENABLED))
4494 dev_warn(&pf->pdev->dev,
4495 "Turning on link-down-on-close flag may affect other partitions\n");
4496
4482 if (changed_flags & I40E_FLAG_DISABLE_FW_LLDP) { 4497 if (changed_flags & I40E_FLAG_DISABLE_FW_LLDP) {
4483 if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) { 4498 if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) {
4484 struct i40e_dcbx_config *dcbcfg; 4499 struct i40e_dcbx_config *dcbcfg;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index e31adbc75f9c..70ecd9c3a163 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -3449,15 +3449,20 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3449 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3449 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3450 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3450 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3451 3451
3452 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3452 q_vector->rx.next_update = jiffies + 1;
3453 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting); 3453 q_vector->rx.target_itr =
3454 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3454 ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3455 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3455 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3456 q_vector->rx.itr); 3456 q_vector->rx.target_itr);
3457 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting); 3457 q_vector->rx.current_itr = q_vector->rx.target_itr;
3458 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3458
3459 q_vector->tx.next_update = jiffies + 1;
3460 q_vector->tx.target_itr =
3461 ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3459 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3462 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3460 q_vector->tx.itr); 3463 q_vector->tx.target_itr);
3464 q_vector->tx.current_itr = q_vector->tx.target_itr;
3465
3461 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3466 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3462 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3467 i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3463 3468
@@ -3558,13 +3563,14 @@ static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3558 u32 val; 3563 u32 val;
3559 3564
3560 /* set the ITR configuration */ 3565 /* set the ITR configuration */
3561 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3566 q_vector->rx.next_update = jiffies + 1;
3562 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting); 3567 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3563 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3568 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr);
3564 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 3569 q_vector->rx.current_itr = q_vector->rx.target_itr;
3565 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting); 3570 q_vector->tx.next_update = jiffies + 1;
3566 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3571 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3567 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 3572 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr);
3573 q_vector->tx.current_itr = q_vector->tx.target_itr;
3568 3574
3569 i40e_enable_misc_int_causes(pf); 3575 i40e_enable_misc_int_causes(pf);
3570 3576
@@ -9215,6 +9221,17 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
9215 } 9221 }
9216 i40e_get_oem_version(&pf->hw); 9222 i40e_get_oem_version(&pf->hw);
9217 9223
9224 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
9225 ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) ||
9226 hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) {
9227 /* The following delay is necessary for 4.33 firmware and older
9228 * to recover after EMP reset. 200 ms should suffice but we
9229 * put here 300 ms to be sure that FW is ready to operate
9230 * after reset.
9231 */
9232 mdelay(300);
9233 }
9234
9218 /* re-verify the eeprom if we just had an EMP reset */ 9235 /* re-verify the eeprom if we just had an EMP reset */
9219 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 9236 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
9220 i40e_verify_eeprom(pf); 9237 i40e_verify_eeprom(pf);
@@ -10018,7 +10035,7 @@ static int i40e_alloc_rings(struct i40e_vsi *vsi)
10018 ring->dcb_tc = 0; 10035 ring->dcb_tc = 0;
10019 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10036 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10020 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10037 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10021 ring->tx_itr_setting = pf->tx_itr_default; 10038 ring->itr_setting = pf->tx_itr_default;
10022 vsi->tx_rings[i] = ring++; 10039 vsi->tx_rings[i] = ring++;
10023 10040
10024 if (!i40e_enabled_xdp_vsi(vsi)) 10041 if (!i40e_enabled_xdp_vsi(vsi))
@@ -10036,7 +10053,7 @@ static int i40e_alloc_rings(struct i40e_vsi *vsi)
10036 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10053 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10037 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10054 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10038 set_ring_xdp(ring); 10055 set_ring_xdp(ring);
10039 ring->tx_itr_setting = pf->tx_itr_default; 10056 ring->itr_setting = pf->tx_itr_default;
10040 vsi->xdp_rings[i] = ring++; 10057 vsi->xdp_rings[i] = ring++;
10041 10058
10042setup_rx: 10059setup_rx:
@@ -10049,7 +10066,7 @@ setup_rx:
10049 ring->count = vsi->num_desc; 10066 ring->count = vsi->num_desc;
10050 ring->size = 0; 10067 ring->size = 0;
10051 ring->dcb_tc = 0; 10068 ring->dcb_tc = 0;
10052 ring->rx_itr_setting = pf->rx_itr_default; 10069 ring->itr_setting = pf->rx_itr_default;
10053 vsi->rx_rings[i] = ring; 10070 vsi->rx_rings[i] = ring;
10054 } 10071 }
10055 10072
@@ -10328,9 +10345,6 @@ static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
10328 netif_napi_add(vsi->netdev, &q_vector->napi, 10345 netif_napi_add(vsi->netdev, &q_vector->napi,
10329 i40e_napi_poll, NAPI_POLL_WEIGHT); 10346 i40e_napi_poll, NAPI_POLL_WEIGHT);
10330 10347
10331 q_vector->rx.latency_range = I40E_LOW_LATENCY;
10332 q_vector->tx.latency_range = I40E_LOW_LATENCY;
10333
10334 /* tie q_vector and vsi together */ 10348 /* tie q_vector and vsi together */
10335 vsi->q_vectors[v_idx] = q_vector; 10349 vsi->q_vectors[v_idx] = q_vector;
10336 10350
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index e554aa6cf070..1ec9b1d8023d 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -995,99 +995,241 @@ void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
995 } 995 }
996} 996}
997 997
998static inline bool i40e_container_is_rx(struct i40e_q_vector *q_vector,
999 struct i40e_ring_container *rc)
1000{
1001 return &q_vector->rx == rc;
1002}
1003
1004static inline unsigned int i40e_itr_divisor(struct i40e_q_vector *q_vector)
1005{
1006 unsigned int divisor;
1007
1008 switch (q_vector->vsi->back->hw.phy.link_info.link_speed) {
1009 case I40E_LINK_SPEED_40GB:
1010 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 1024;
1011 break;
1012 case I40E_LINK_SPEED_25GB:
1013 case I40E_LINK_SPEED_20GB:
1014 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 512;
1015 break;
1016 default:
1017 case I40E_LINK_SPEED_10GB:
1018 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 256;
1019 break;
1020 case I40E_LINK_SPEED_1GB:
1021 case I40E_LINK_SPEED_100MB:
1022 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 32;
1023 break;
1024 }
1025
1026 return divisor;
1027}
1028
998/** 1029/**
999 * i40e_set_new_dynamic_itr - Find new ITR level 1030 * i40e_update_itr - update the dynamic ITR value based on statistics
1031 * @q_vector: structure containing interrupt and ring information
1000 * @rc: structure containing ring performance data 1032 * @rc: structure containing ring performance data
1001 * 1033 *
1002 * Returns true if ITR changed, false if not 1034 * Stores a new ITR value based on packets and byte
1003 * 1035 * counts during the last interrupt. The advantage of per interrupt
1004 * Stores a new ITR value based on packets and byte counts during 1036 * computation is faster updates and more accurate ITR for the current
1005 * the last interrupt. The advantage of per interrupt computation 1037 * traffic pattern. Constants in this function were computed
1006 * is faster updates and more accurate ITR for the current traffic 1038 * based on theoretical maximum wire speed and thresholds were set based
1007 * pattern. Constants in this function were computed based on 1039 * on testing data as well as attempting to minimize response time
1008 * theoretical maximum wire speed and thresholds were set based on
1009 * testing data as well as attempting to minimize response time
1010 * while increasing bulk throughput. 1040 * while increasing bulk throughput.
1011 **/ 1041 **/
1012static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc) 1042static void i40e_update_itr(struct i40e_q_vector *q_vector,
1043 struct i40e_ring_container *rc)
1013{ 1044{
1014 enum i40e_latency_range new_latency_range = rc->latency_range; 1045 unsigned int avg_wire_size, packets, bytes, itr;
1015 u32 new_itr = rc->itr; 1046 unsigned long next_update = jiffies;
1016 int bytes_per_usec;
1017 unsigned int usecs, estimated_usecs;
1018 1047
1019 if (rc->total_packets == 0 || !rc->itr) 1048 /* If we don't have any rings just leave ourselves set for maximum
1020 return false; 1049 * possible latency so we take ourselves out of the equation.
1050 */
1051 if (!rc->ring || !ITR_IS_DYNAMIC(rc->ring->itr_setting))
1052 return;
1021 1053
1022 usecs = (rc->itr << 1) * ITR_COUNTDOWN_START; 1054 /* For Rx we want to push the delay up and default to low latency.
1023 bytes_per_usec = rc->total_bytes / usecs; 1055 * for Tx we want to pull the delay down and default to high latency.
1056 */
1057 itr = i40e_container_is_rx(q_vector, rc) ?
1058 I40E_ITR_ADAPTIVE_MIN_USECS | I40E_ITR_ADAPTIVE_LATENCY :
1059 I40E_ITR_ADAPTIVE_MAX_USECS | I40E_ITR_ADAPTIVE_LATENCY;
1060
1061 /* If we didn't update within up to 1 - 2 jiffies we can assume
1062 * that either packets are coming in so slow there hasn't been
1063 * any work, or that there is so much work that NAPI is dealing
1064 * with interrupt moderation and we don't need to do anything.
1065 */
1066 if (time_after(next_update, rc->next_update))
1067 goto clear_counts;
1068
1069 /* If itr_countdown is set it means we programmed an ITR within
1070 * the last 4 interrupt cycles. This has a side effect of us
1071 * potentially firing an early interrupt. In order to work around
1072 * this we need to throw out any data received for a few
1073 * interrupts following the update.
1074 */
1075 if (q_vector->itr_countdown) {
1076 itr = rc->target_itr;
1077 goto clear_counts;
1078 }
1079
1080 packets = rc->total_packets;
1081 bytes = rc->total_bytes;
1024 1082
1025 /* The calculations in this algorithm depend on interrupts actually 1083 if (i40e_container_is_rx(q_vector, rc)) {
1026 * firing at the ITR rate. This may not happen if the packet rate is 1084 /* If Rx there are 1 to 4 packets and bytes are less than
1027 * really low, or if we've been napi polling. Check to make sure 1085 * 9000 assume insufficient data to use bulk rate limiting
1028 * that's not the case before we continue. 1086 * approach unless Tx is already in bulk rate limiting. We
1087 * are likely latency driven.
1088 */
1089 if (packets && packets < 4 && bytes < 9000 &&
1090 (q_vector->tx.target_itr & I40E_ITR_ADAPTIVE_LATENCY)) {
1091 itr = I40E_ITR_ADAPTIVE_LATENCY;
1092 goto adjust_by_size;
1093 }
1094 } else if (packets < 4) {
1095 /* If we have Tx and Rx ITR maxed and Tx ITR is running in
1096 * bulk mode and we are receiving 4 or fewer packets just
1097 * reset the ITR_ADAPTIVE_LATENCY bit for latency mode so
1098 * that the Rx can relax.
1099 */
1100 if (rc->target_itr == I40E_ITR_ADAPTIVE_MAX_USECS &&
1101 (q_vector->rx.target_itr & I40E_ITR_MASK) ==
1102 I40E_ITR_ADAPTIVE_MAX_USECS)
1103 goto clear_counts;
1104 } else if (packets > 32) {
1105 /* If we have processed over 32 packets in a single interrupt
1106 * for Tx assume we need to switch over to "bulk" mode.
1107 */
1108 rc->target_itr &= ~I40E_ITR_ADAPTIVE_LATENCY;
1109 }
1110
1111 /* We have no packets to actually measure against. This means
1112 * either one of the other queues on this vector is active or
1113 * we are a Tx queue doing TSO with too high of an interrupt rate.
1114 *
1115 * Between 4 and 56 we can assume that our current interrupt delay
1116 * is only slightly too low. As such we should increase it by a small
1117 * fixed amount.
1029 */ 1118 */
1030 estimated_usecs = jiffies_to_usecs(jiffies - rc->last_itr_update); 1119 if (packets < 56) {
1031 if (estimated_usecs > usecs) { 1120 itr = rc->target_itr + I40E_ITR_ADAPTIVE_MIN_INC;
1032 new_latency_range = I40E_LOW_LATENCY; 1121 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
1033 goto reset_latency; 1122 itr &= I40E_ITR_ADAPTIVE_LATENCY;
1123 itr += I40E_ITR_ADAPTIVE_MAX_USECS;
1124 }
1125 goto clear_counts;
1034 } 1126 }
1035 1127
1036 /* simple throttlerate management 1128 if (packets <= 256) {
1037 * 0-10MB/s lowest (50000 ints/s) 1129 itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr);
1038 * 10-20MB/s low (20000 ints/s) 1130 itr &= I40E_ITR_MASK;
1039 * 20-1249MB/s bulk (18000 ints/s) 1131
1132 /* Between 56 and 112 is our "goldilocks" zone where we are
1133 * working out "just right". Just report that our current
1134 * ITR is good for us.
1135 */
1136 if (packets <= 112)
1137 goto clear_counts;
1138
1139 /* If packet count is 128 or greater we are likely looking
1140 * at a slight overrun of the delay we want. Try halving
1141 * our delay to see if that will cut the number of packets
1142 * in half per interrupt.
1143 */
1144 itr /= 2;
1145 itr &= I40E_ITR_MASK;
1146 if (itr < I40E_ITR_ADAPTIVE_MIN_USECS)
1147 itr = I40E_ITR_ADAPTIVE_MIN_USECS;
1148
1149 goto clear_counts;
1150 }
1151
1152 /* The paths below assume we are dealing with a bulk ITR since
1153 * number of packets is greater than 256. We are just going to have
1154 * to compute a value and try to bring the count under control,
1155 * though for smaller packet sizes there isn't much we can do as
1156 * NAPI polling will likely be kicking in sooner rather than later.
1157 */
1158 itr = I40E_ITR_ADAPTIVE_BULK;
1159
1160adjust_by_size:
1161 /* If packet counts are 256 or greater we can assume we have a gross
1162 * overestimation of what the rate should be. Instead of trying to fine
1163 * tune it just use the formula below to try and dial in an exact value
1164 * give the current packet size of the frame.
1165 */
1166 avg_wire_size = bytes / packets;
1167
1168 /* The following is a crude approximation of:
1169 * wmem_default / (size + overhead) = desired_pkts_per_int
1170 * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
1171 * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
1040 * 1172 *
1041 * The math works out because the divisor is in 10^(-6) which 1173 * Assuming wmem_default is 212992 and overhead is 640 bytes per
1042 * turns the bytes/us input value into MB/s values, but 1174 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
1043 * make sure to use usecs, as the register values written 1175 * formula down to
1044 * are in 2 usec increments in the ITR registers, and make sure 1176 *
1045 * to use the smoothed values that the countdown timer gives us. 1177 * (170 * (size + 24)) / (size + 640) = ITR
1178 *
1179 * We first do some math on the packet size and then finally bitshift
1180 * by 8 after rounding up. We also have to account for PCIe link speed
1181 * difference as ITR scales based on this.
1046 */ 1182 */
1047 switch (new_latency_range) { 1183 if (avg_wire_size <= 60) {
1048 case I40E_LOWEST_LATENCY: 1184 /* Start at 250k ints/sec */
1049 if (bytes_per_usec > 10) 1185 avg_wire_size = 4096;
1050 new_latency_range = I40E_LOW_LATENCY; 1186 } else if (avg_wire_size <= 380) {
1051 break; 1187 /* 250K ints/sec to 60K ints/sec */
1052 case I40E_LOW_LATENCY: 1188 avg_wire_size *= 40;
1053 if (bytes_per_usec > 20) 1189 avg_wire_size += 1696;
1054 new_latency_range = I40E_BULK_LATENCY; 1190 } else if (avg_wire_size <= 1084) {
1055 else if (bytes_per_usec <= 10) 1191 /* 60K ints/sec to 36K ints/sec */
1056 new_latency_range = I40E_LOWEST_LATENCY; 1192 avg_wire_size *= 15;
1057 break; 1193 avg_wire_size += 11452;
1058 case I40E_BULK_LATENCY: 1194 } else if (avg_wire_size <= 1980) {
1059 default: 1195 /* 36K ints/sec to 30K ints/sec */
1060 if (bytes_per_usec <= 20) 1196 avg_wire_size *= 5;
1061 new_latency_range = I40E_LOW_LATENCY; 1197 avg_wire_size += 22420;
1062 break; 1198 } else {
1199 /* plateau at a limit of 30K ints/sec */
1200 avg_wire_size = 32256;
1063 } 1201 }
1064 1202
1065reset_latency: 1203 /* If we are in low latency mode halve our delay which doubles the
1066 rc->latency_range = new_latency_range; 1204 * rate to somewhere between 100K to 16K ints/sec
1205 */
1206 if (itr & I40E_ITR_ADAPTIVE_LATENCY)
1207 avg_wire_size /= 2;
1067 1208
1068 switch (new_latency_range) { 1209 /* Resultant value is 256 times larger than it needs to be. This
1069 case I40E_LOWEST_LATENCY: 1210 * gives us room to adjust the value as needed to either increase
1070 new_itr = I40E_ITR_50K; 1211 * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
1071 break; 1212 *
1072 case I40E_LOW_LATENCY: 1213 * Use addition as we have already recorded the new latency flag
1073 new_itr = I40E_ITR_20K; 1214 * for the ITR value.
1074 break; 1215 */
1075 case I40E_BULK_LATENCY: 1216 itr += DIV_ROUND_UP(avg_wire_size, i40e_itr_divisor(q_vector)) *
1076 new_itr = I40E_ITR_18K; 1217 I40E_ITR_ADAPTIVE_MIN_INC;
1077 break; 1218
1078 default: 1219 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
1079 break; 1220 itr &= I40E_ITR_ADAPTIVE_LATENCY;
1221 itr += I40E_ITR_ADAPTIVE_MAX_USECS;
1080 } 1222 }
1081 1223
1224clear_counts:
1225 /* write back value */
1226 rc->target_itr = itr;
1227
1228 /* next update should occur within next jiffy */
1229 rc->next_update = next_update + 1;
1230
1082 rc->total_bytes = 0; 1231 rc->total_bytes = 0;
1083 rc->total_packets = 0; 1232 rc->total_packets = 0;
1084 rc->last_itr_update = jiffies;
1085
1086 if (new_itr != rc->itr) {
1087 rc->itr = new_itr;
1088 return true;
1089 }
1090 return false;
1091} 1233}
1092 1234
1093/** 1235/**
@@ -1991,7 +2133,7 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
1991 * @rx_buffer: rx buffer to pull data from 2133 * @rx_buffer: rx buffer to pull data from
1992 * 2134 *
1993 * This function will clean up the contents of the rx_buffer. It will 2135 * This function will clean up the contents of the rx_buffer. It will
1994 * either recycle the bufer or unmap it and free the associated resources. 2136 * either recycle the buffer or unmap it and free the associated resources.
1995 */ 2137 */
1996static void i40e_put_rx_buffer(struct i40e_ring *rx_ring, 2138static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
1997 struct i40e_rx_buffer *rx_buffer) 2139 struct i40e_rx_buffer *rx_buffer)
@@ -2274,29 +2416,45 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
2274 return failure ? budget : (int)total_rx_packets; 2416 return failure ? budget : (int)total_rx_packets;
2275} 2417}
2276 2418
2277static u32 i40e_buildreg_itr(const int type, const u16 itr) 2419static inline u32 i40e_buildreg_itr(const int type, u16 itr)
2278{ 2420{
2279 u32 val; 2421 u32 val;
2280 2422
2423 /* We don't bother with setting the CLEARPBA bit as the data sheet
2424 * points out doing so is "meaningless since it was already
2425 * auto-cleared". The auto-clearing happens when the interrupt is
2426 * asserted.
2427 *
2428 * Hardware errata 28 for also indicates that writing to a
2429 * xxINT_DYN_CTLx CSR with INTENA_MSK (bit 31) set to 0 will clear
2430 * an event in the PBA anyway so we need to rely on the automask
2431 * to hold pending events for us until the interrupt is re-enabled
2432 *
2433 * The itr value is reported in microseconds, and the register
2434 * value is recorded in 2 microsecond units. For this reason we
2435 * only need to shift by the interval shift - 1 instead of the
2436 * full value.
2437 */
2438 itr &= I40E_ITR_MASK;
2439
2281 val = I40E_PFINT_DYN_CTLN_INTENA_MASK | 2440 val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
2282 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
2283 (type << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | 2441 (type << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
2284 (itr << I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT); 2442 (itr << (I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT - 1));
2285 2443
2286 return val; 2444 return val;
2287} 2445}
2288 2446
2289/* a small macro to shorten up some long lines */ 2447/* a small macro to shorten up some long lines */
2290#define INTREG I40E_PFINT_DYN_CTLN 2448#define INTREG I40E_PFINT_DYN_CTLN
2291static inline int get_rx_itr(struct i40e_vsi *vsi, int idx)
2292{
2293 return vsi->rx_rings[idx]->rx_itr_setting;
2294}
2295 2449
2296static inline int get_tx_itr(struct i40e_vsi *vsi, int idx) 2450/* The act of updating the ITR will cause it to immediately trigger. In order
2297{ 2451 * to prevent this from throwing off adaptive update statistics we defer the
2298 return vsi->tx_rings[idx]->tx_itr_setting; 2452 * update so that it can only happen so often. So after either Tx or Rx are
2299} 2453 * updated we make the adaptive scheme wait until either the ITR completely
2454 * expires via the next_update expiration or we have been through at least
2455 * 3 interrupts.
2456 */
2457#define ITR_COUNTDOWN_START 3
2300 2458
2301/** 2459/**
2302 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt 2460 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
@@ -2308,10 +2466,7 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
2308 struct i40e_q_vector *q_vector) 2466 struct i40e_q_vector *q_vector)
2309{ 2467{
2310 struct i40e_hw *hw = &vsi->back->hw; 2468 struct i40e_hw *hw = &vsi->back->hw;
2311 bool rx = false, tx = false; 2469 u32 intval;
2312 u32 rxval, txval;
2313 int idx = q_vector->v_idx;
2314 int rx_itr_setting, tx_itr_setting;
2315 2470
2316 /* If we don't have MSIX, then we only need to re-enable icr0 */ 2471 /* If we don't have MSIX, then we only need to re-enable icr0 */
2317 if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED)) { 2472 if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED)) {
@@ -2319,65 +2474,49 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
2319 return; 2474 return;
2320 } 2475 }
2321 2476
2322 /* avoid dynamic calculation if in countdown mode OR if 2477 /* These will do nothing if dynamic updates are not enabled */
2323 * all dynamic is disabled 2478 i40e_update_itr(q_vector, &q_vector->tx);
2324 */ 2479 i40e_update_itr(q_vector, &q_vector->rx);
2325 rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
2326
2327 rx_itr_setting = get_rx_itr(vsi, idx);
2328 tx_itr_setting = get_tx_itr(vsi, idx);
2329
2330 if (q_vector->itr_countdown > 0 ||
2331 (!ITR_IS_DYNAMIC(rx_itr_setting) &&
2332 !ITR_IS_DYNAMIC(tx_itr_setting))) {
2333 goto enable_int;
2334 }
2335
2336 if (ITR_IS_DYNAMIC(rx_itr_setting)) {
2337 rx = i40e_set_new_dynamic_itr(&q_vector->rx);
2338 rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
2339 }
2340
2341 if (ITR_IS_DYNAMIC(tx_itr_setting)) {
2342 tx = i40e_set_new_dynamic_itr(&q_vector->tx);
2343 txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr);
2344 }
2345 2480
2346 if (rx || tx) { 2481 /* This block of logic allows us to get away with only updating
2347 /* get the higher of the two ITR adjustments and 2482 * one ITR value with each interrupt. The idea is to perform a
2348 * use the same value for both ITR registers 2483 * pseudo-lazy update with the following criteria.
2349 * when in adaptive mode (Rx and/or Tx) 2484 *
2350 */ 2485 * 1. Rx is given higher priority than Tx if both are in same state
2351 u16 itr = max(q_vector->tx.itr, q_vector->rx.itr); 2486 * 2. If we must reduce an ITR that is given highest priority.
2352 2487 * 3. We then give priority to increasing ITR based on amount.
2353 q_vector->tx.itr = q_vector->rx.itr = itr;
2354 txval = i40e_buildreg_itr(I40E_TX_ITR, itr);
2355 tx = true;
2356 rxval = i40e_buildreg_itr(I40E_RX_ITR, itr);
2357 rx = true;
2358 }
2359
2360 /* only need to enable the interrupt once, but need
2361 * to possibly update both ITR values
2362 */ 2488 */
2363 if (rx) { 2489 if (q_vector->rx.target_itr < q_vector->rx.current_itr) {
2364 /* set the INTENA_MSK_MASK so that this first write 2490 /* Rx ITR needs to be reduced, this is highest priority */
2365 * won't actually enable the interrupt, instead just 2491 intval = i40e_buildreg_itr(I40E_RX_ITR,
2366 * updating the ITR (it's bit 31 PF and VF) 2492 q_vector->rx.target_itr);
2493 q_vector->rx.current_itr = q_vector->rx.target_itr;
2494 q_vector->itr_countdown = ITR_COUNTDOWN_START;
2495 } else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) ||
2496 ((q_vector->rx.target_itr - q_vector->rx.current_itr) <
2497 (q_vector->tx.target_itr - q_vector->tx.current_itr))) {
2498 /* Tx ITR needs to be reduced, this is second priority
2499 * Tx ITR needs to be increased more than Rx, fourth priority
2367 */ 2500 */
2368 rxval |= BIT(31); 2501 intval = i40e_buildreg_itr(I40E_TX_ITR,
2369 /* don't check _DOWN because interrupt isn't being enabled */ 2502 q_vector->tx.target_itr);
2370 wr32(hw, INTREG(q_vector->reg_idx), rxval); 2503 q_vector->tx.current_itr = q_vector->tx.target_itr;
2504 q_vector->itr_countdown = ITR_COUNTDOWN_START;
2505 } else if (q_vector->rx.current_itr != q_vector->rx.target_itr) {
2506 /* Rx ITR needs to be increased, third priority */
2507 intval = i40e_buildreg_itr(I40E_RX_ITR,
2508 q_vector->rx.target_itr);
2509 q_vector->rx.current_itr = q_vector->rx.target_itr;
2510 q_vector->itr_countdown = ITR_COUNTDOWN_START;
2511 } else {
2512 /* No ITR update, lowest priority */
2513 intval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
2514 if (q_vector->itr_countdown)
2515 q_vector->itr_countdown--;
2371 } 2516 }
2372 2517
2373enable_int:
2374 if (!test_bit(__I40E_VSI_DOWN, vsi->state)) 2518 if (!test_bit(__I40E_VSI_DOWN, vsi->state))
2375 wr32(hw, INTREG(q_vector->reg_idx), txval); 2519 wr32(hw, INTREG(q_vector->reg_idx), intval);
2376
2377 if (q_vector->itr_countdown)
2378 q_vector->itr_countdown--;
2379 else
2380 q_vector->itr_countdown = ITR_COUNTDOWN_START;
2381} 2520}
2382 2521
2383/** 2522/**
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index 701b708628b0..f75a8fe68fcf 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -30,32 +30,37 @@
30#include <net/xdp.h> 30#include <net/xdp.h>
31 31
32/* Interrupt Throttling and Rate Limiting Goodies */ 32/* Interrupt Throttling and Rate Limiting Goodies */
33
34#define I40E_MAX_ITR 0x0FF0 /* reg uses 2 usec resolution */
35#define I40E_MIN_ITR 0x0001 /* reg uses 2 usec resolution */
36#define I40E_ITR_100K 0x0005
37#define I40E_ITR_50K 0x000A
38#define I40E_ITR_20K 0x0019
39#define I40E_ITR_18K 0x001B
40#define I40E_ITR_8K 0x003E
41#define I40E_ITR_4K 0x007A
42#define I40E_MAX_INTRL 0x3B /* reg uses 4 usec resolution */
43#define I40E_ITR_RX_DEF (ITR_REG_TO_USEC(I40E_ITR_20K) | \
44 I40E_ITR_DYNAMIC)
45#define I40E_ITR_TX_DEF (ITR_REG_TO_USEC(I40E_ITR_20K) | \
46 I40E_ITR_DYNAMIC)
47#define I40E_ITR_DYNAMIC 0x8000 /* use top bit as a flag */
48#define I40E_MIN_INT_RATE 250 /* ~= 1000000 / (I40E_MAX_ITR * 2) */
49#define I40E_MAX_INT_RATE 500000 /* == 1000000 / (I40E_MIN_ITR * 2) */
50#define I40E_DEFAULT_IRQ_WORK 256 33#define I40E_DEFAULT_IRQ_WORK 256
51#define ITR_TO_REG(setting) ((setting & ~I40E_ITR_DYNAMIC) >> 1) 34
52#define ITR_IS_DYNAMIC(setting) (!!(setting & I40E_ITR_DYNAMIC)) 35/* The datasheet for the X710 and XL710 indicate that the maximum value for
53#define ITR_REG_TO_USEC(itr_reg) (itr_reg << 1) 36 * the ITR is 8160usec which is then called out as 0xFF0 with a 2usec
37 * resolution. 8160 is 0x1FE0 when written out in hex. So instead of storing
38 * the register value which is divided by 2 lets use the actual values and
39 * avoid an excessive amount of translation.
40 */
41#define I40E_ITR_DYNAMIC 0x8000 /* use top bit as a flag */
42#define I40E_ITR_MASK 0x1FFE /* mask for ITR register value */
43#define I40E_MIN_ITR 2 /* reg uses 2 usec resolution */
44#define I40E_ITR_100K 10 /* all values below must be even */
45#define I40E_ITR_50K 20
46#define I40E_ITR_20K 50
47#define I40E_ITR_18K 60
48#define I40E_ITR_8K 122
49#define I40E_MAX_ITR 8160 /* maximum value as per datasheet */
50#define ITR_TO_REG(setting) ((setting) & ~I40E_ITR_DYNAMIC)
51#define ITR_REG_ALIGN(setting) __ALIGN_MASK(setting, ~I40E_ITR_MASK)
52#define ITR_IS_DYNAMIC(setting) (!!((setting) & I40E_ITR_DYNAMIC))
53
54#define I40E_ITR_RX_DEF (I40E_ITR_20K | I40E_ITR_DYNAMIC)
55#define I40E_ITR_TX_DEF (I40E_ITR_20K | I40E_ITR_DYNAMIC)
56
54/* 0x40 is the enable bit for interrupt rate limiting, and must be set if 57/* 0x40 is the enable bit for interrupt rate limiting, and must be set if
55 * the value of the rate limit is non-zero 58 * the value of the rate limit is non-zero
56 */ 59 */
57#define INTRL_ENA BIT(6) 60#define INTRL_ENA BIT(6)
61#define I40E_MAX_INTRL 0x3B /* reg uses 4 usec resolution */
58#define INTRL_REG_TO_USEC(intrl) ((intrl & ~INTRL_ENA) << 2) 62#define INTRL_REG_TO_USEC(intrl) ((intrl & ~INTRL_ENA) << 2)
63
59/** 64/**
60 * i40e_intrl_usec_to_reg - convert interrupt rate limit to register 65 * i40e_intrl_usec_to_reg - convert interrupt rate limit to register
61 * @intrl: interrupt rate limit to convert 66 * @intrl: interrupt rate limit to convert
@@ -382,8 +387,7 @@ struct i40e_ring {
382 * these values always store the USER setting, and must be converted 387 * these values always store the USER setting, and must be converted
383 * before programming to a register. 388 * before programming to a register.
384 */ 389 */
385 u16 rx_itr_setting; 390 u16 itr_setting;
386 u16 tx_itr_setting;
387 391
388 u16 count; /* Number of descriptors */ 392 u16 count; /* Number of descriptors */
389 u16 reg_idx; /* HW register index of the ring */ 393 u16 reg_idx; /* HW register index of the ring */
@@ -459,21 +463,21 @@ static inline void set_ring_xdp(struct i40e_ring *ring)
459 ring->flags |= I40E_TXR_FLAGS_XDP; 463 ring->flags |= I40E_TXR_FLAGS_XDP;
460} 464}
461 465
462enum i40e_latency_range { 466#define I40E_ITR_ADAPTIVE_MIN_INC 0x0002
463 I40E_LOWEST_LATENCY = 0, 467#define I40E_ITR_ADAPTIVE_MIN_USECS 0x0002
464 I40E_LOW_LATENCY = 1, 468#define I40E_ITR_ADAPTIVE_MAX_USECS 0x007e
465 I40E_BULK_LATENCY = 2, 469#define I40E_ITR_ADAPTIVE_LATENCY 0x8000
466}; 470#define I40E_ITR_ADAPTIVE_BULK 0x0000
471#define ITR_IS_BULK(x) (!((x) & I40E_ITR_ADAPTIVE_LATENCY))
467 472
468struct i40e_ring_container { 473struct i40e_ring_container {
469 /* array of pointers to rings */ 474 struct i40e_ring *ring; /* pointer to linked list of ring(s) */
470 struct i40e_ring *ring; 475 unsigned long next_update; /* jiffies value of next update */
471 unsigned int total_bytes; /* total bytes processed this int */ 476 unsigned int total_bytes; /* total bytes processed this int */
472 unsigned int total_packets; /* total packets processed this int */ 477 unsigned int total_packets; /* total packets processed this int */
473 unsigned long last_itr_update; /* jiffies of last ITR update */
474 u16 count; 478 u16 count;
475 enum i40e_latency_range latency_range; 479 u16 target_itr; /* target ITR setting for ring(s) */
476 u16 itr; 480 u16 current_itr; /* current ITR setting for ring(s) */
477}; 481};
478 482
479/* iterator for handling rings in ring container */ 483/* iterator for handling rings in ring container */
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
index 357d6051281f..eb8f3e327f6b 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
@@ -392,99 +392,241 @@ void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
392 val); 392 val);
393} 393}
394 394
395static inline bool i40e_container_is_rx(struct i40e_q_vector *q_vector,
396 struct i40e_ring_container *rc)
397{
398 return &q_vector->rx == rc;
399}
400
401static inline unsigned int i40e_itr_divisor(struct i40e_q_vector *q_vector)
402{
403 unsigned int divisor;
404
405 switch (q_vector->adapter->link_speed) {
406 case I40E_LINK_SPEED_40GB:
407 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 1024;
408 break;
409 case I40E_LINK_SPEED_25GB:
410 case I40E_LINK_SPEED_20GB:
411 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 512;
412 break;
413 default:
414 case I40E_LINK_SPEED_10GB:
415 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 256;
416 break;
417 case I40E_LINK_SPEED_1GB:
418 case I40E_LINK_SPEED_100MB:
419 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 32;
420 break;
421 }
422
423 return divisor;
424}
425
395/** 426/**
396 * i40e_set_new_dynamic_itr - Find new ITR level 427 * i40e_update_itr - update the dynamic ITR value based on statistics
428 * @q_vector: structure containing interrupt and ring information
397 * @rc: structure containing ring performance data 429 * @rc: structure containing ring performance data
398 * 430 *
399 * Returns true if ITR changed, false if not 431 * Stores a new ITR value based on packets and byte
400 * 432 * counts during the last interrupt. The advantage of per interrupt
401 * Stores a new ITR value based on packets and byte counts during 433 * computation is faster updates and more accurate ITR for the current
402 * the last interrupt. The advantage of per interrupt computation 434 * traffic pattern. Constants in this function were computed
403 * is faster updates and more accurate ITR for the current traffic 435 * based on theoretical maximum wire speed and thresholds were set based
404 * pattern. Constants in this function were computed based on 436 * on testing data as well as attempting to minimize response time
405 * theoretical maximum wire speed and thresholds were set based on
406 * testing data as well as attempting to minimize response time
407 * while increasing bulk throughput. 437 * while increasing bulk throughput.
408 **/ 438 **/
409static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc) 439static void i40e_update_itr(struct i40e_q_vector *q_vector,
440 struct i40e_ring_container *rc)
410{ 441{
411 enum i40e_latency_range new_latency_range = rc->latency_range; 442 unsigned int avg_wire_size, packets, bytes, itr;
412 u32 new_itr = rc->itr; 443 unsigned long next_update = jiffies;
413 int bytes_per_usec;
414 unsigned int usecs, estimated_usecs;
415 444
416 if (rc->total_packets == 0 || !rc->itr) 445 /* If we don't have any rings just leave ourselves set for maximum
417 return false; 446 * possible latency so we take ourselves out of the equation.
447 */
448 if (!rc->ring || !ITR_IS_DYNAMIC(rc->ring->itr_setting))
449 return;
450
451 /* For Rx we want to push the delay up and default to low latency.
452 * for Tx we want to pull the delay down and default to high latency.
453 */
454 itr = i40e_container_is_rx(q_vector, rc) ?
455 I40E_ITR_ADAPTIVE_MIN_USECS | I40E_ITR_ADAPTIVE_LATENCY :
456 I40E_ITR_ADAPTIVE_MAX_USECS | I40E_ITR_ADAPTIVE_LATENCY;
457
458 /* If we didn't update within up to 1 - 2 jiffies we can assume
459 * that either packets are coming in so slow there hasn't been
460 * any work, or that there is so much work that NAPI is dealing
461 * with interrupt moderation and we don't need to do anything.
462 */
463 if (time_after(next_update, rc->next_update))
464 goto clear_counts;
465
466 /* If itr_countdown is set it means we programmed an ITR within
467 * the last 4 interrupt cycles. This has a side effect of us
468 * potentially firing an early interrupt. In order to work around
469 * this we need to throw out any data received for a few
470 * interrupts following the update.
471 */
472 if (q_vector->itr_countdown) {
473 itr = rc->target_itr;
474 goto clear_counts;
475 }
476
477 packets = rc->total_packets;
478 bytes = rc->total_bytes;
418 479
419 usecs = (rc->itr << 1) * ITR_COUNTDOWN_START; 480 if (i40e_container_is_rx(q_vector, rc)) {
420 bytes_per_usec = rc->total_bytes / usecs; 481 /* If Rx there are 1 to 4 packets and bytes are less than
482 * 9000 assume insufficient data to use bulk rate limiting
483 * approach unless Tx is already in bulk rate limiting. We
484 * are likely latency driven.
485 */
486 if (packets && packets < 4 && bytes < 9000 &&
487 (q_vector->tx.target_itr & I40E_ITR_ADAPTIVE_LATENCY)) {
488 itr = I40E_ITR_ADAPTIVE_LATENCY;
489 goto adjust_by_size;
490 }
491 } else if (packets < 4) {
492 /* If we have Tx and Rx ITR maxed and Tx ITR is running in
493 * bulk mode and we are receiving 4 or fewer packets just
494 * reset the ITR_ADAPTIVE_LATENCY bit for latency mode so
495 * that the Rx can relax.
496 */
497 if (rc->target_itr == I40E_ITR_ADAPTIVE_MAX_USECS &&
498 (q_vector->rx.target_itr & I40E_ITR_MASK) ==
499 I40E_ITR_ADAPTIVE_MAX_USECS)
500 goto clear_counts;
501 } else if (packets > 32) {
502 /* If we have processed over 32 packets in a single interrupt
503 * for Tx assume we need to switch over to "bulk" mode.
504 */
505 rc->target_itr &= ~I40E_ITR_ADAPTIVE_LATENCY;
506 }
421 507
422 /* The calculations in this algorithm depend on interrupts actually 508 /* We have no packets to actually measure against. This means
423 * firing at the ITR rate. This may not happen if the packet rate is 509 * either one of the other queues on this vector is active or
424 * really low, or if we've been napi polling. Check to make sure 510 * we are a Tx queue doing TSO with too high of an interrupt rate.
425 * that's not the case before we continue. 511 *
512 * Between 4 and 56 we can assume that our current interrupt delay
513 * is only slightly too low. As such we should increase it by a small
514 * fixed amount.
426 */ 515 */
427 estimated_usecs = jiffies_to_usecs(jiffies - rc->last_itr_update); 516 if (packets < 56) {
428 if (estimated_usecs > usecs) { 517 itr = rc->target_itr + I40E_ITR_ADAPTIVE_MIN_INC;
429 new_latency_range = I40E_LOW_LATENCY; 518 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
430 goto reset_latency; 519 itr &= I40E_ITR_ADAPTIVE_LATENCY;
520 itr += I40E_ITR_ADAPTIVE_MAX_USECS;
521 }
522 goto clear_counts;
523 }
524
525 if (packets <= 256) {
526 itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr);
527 itr &= I40E_ITR_MASK;
528
529 /* Between 56 and 112 is our "goldilocks" zone where we are
530 * working out "just right". Just report that our current
531 * ITR is good for us.
532 */
533 if (packets <= 112)
534 goto clear_counts;
535
536 /* If packet count is 128 or greater we are likely looking
537 * at a slight overrun of the delay we want. Try halving
538 * our delay to see if that will cut the number of packets
539 * in half per interrupt.
540 */
541 itr /= 2;
542 itr &= I40E_ITR_MASK;
543 if (itr < I40E_ITR_ADAPTIVE_MIN_USECS)
544 itr = I40E_ITR_ADAPTIVE_MIN_USECS;
545
546 goto clear_counts;
431 } 547 }
432 548
433 /* simple throttlerate management 549 /* The paths below assume we are dealing with a bulk ITR since
434 * 0-10MB/s lowest (50000 ints/s) 550 * number of packets is greater than 256. We are just going to have
435 * 10-20MB/s low (20000 ints/s) 551 * to compute a value and try to bring the count under control,
436 * 20-1249MB/s bulk (18000 ints/s) 552 * though for smaller packet sizes there isn't much we can do as
553 * NAPI polling will likely be kicking in sooner rather than later.
554 */
555 itr = I40E_ITR_ADAPTIVE_BULK;
556
557adjust_by_size:
558 /* If packet counts are 256 or greater we can assume we have a gross
559 * overestimation of what the rate should be. Instead of trying to fine
560 * tune it just use the formula below to try and dial in an exact value
561 * give the current packet size of the frame.
562 */
563 avg_wire_size = bytes / packets;
564
565 /* The following is a crude approximation of:
566 * wmem_default / (size + overhead) = desired_pkts_per_int
567 * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
568 * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
437 * 569 *
438 * The math works out because the divisor is in 10^(-6) which 570 * Assuming wmem_default is 212992 and overhead is 640 bytes per
439 * turns the bytes/us input value into MB/s values, but 571 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
440 * make sure to use usecs, as the register values written 572 * formula down to
441 * are in 2 usec increments in the ITR registers, and make sure 573 *
442 * to use the smoothed values that the countdown timer gives us. 574 * (170 * (size + 24)) / (size + 640) = ITR
575 *
576 * We first do some math on the packet size and then finally bitshift
577 * by 8 after rounding up. We also have to account for PCIe link speed
578 * difference as ITR scales based on this.
443 */ 579 */
444 switch (new_latency_range) { 580 if (avg_wire_size <= 60) {
445 case I40E_LOWEST_LATENCY: 581 /* Start at 250k ints/sec */
446 if (bytes_per_usec > 10) 582 avg_wire_size = 4096;
447 new_latency_range = I40E_LOW_LATENCY; 583 } else if (avg_wire_size <= 380) {
448 break; 584 /* 250K ints/sec to 60K ints/sec */
449 case I40E_LOW_LATENCY: 585 avg_wire_size *= 40;
450 if (bytes_per_usec > 20) 586 avg_wire_size += 1696;
451 new_latency_range = I40E_BULK_LATENCY; 587 } else if (avg_wire_size <= 1084) {
452 else if (bytes_per_usec <= 10) 588 /* 60K ints/sec to 36K ints/sec */
453 new_latency_range = I40E_LOWEST_LATENCY; 589 avg_wire_size *= 15;
454 break; 590 avg_wire_size += 11452;
455 case I40E_BULK_LATENCY: 591 } else if (avg_wire_size <= 1980) {
456 default: 592 /* 36K ints/sec to 30K ints/sec */
457 if (bytes_per_usec <= 20) 593 avg_wire_size *= 5;
458 new_latency_range = I40E_LOW_LATENCY; 594 avg_wire_size += 22420;
459 break; 595 } else {
596 /* plateau at a limit of 30K ints/sec */
597 avg_wire_size = 32256;
460 } 598 }
461 599
462reset_latency: 600 /* If we are in low latency mode halve our delay which doubles the
463 rc->latency_range = new_latency_range; 601 * rate to somewhere between 100K to 16K ints/sec
602 */
603 if (itr & I40E_ITR_ADAPTIVE_LATENCY)
604 avg_wire_size /= 2;
464 605
465 switch (new_latency_range) { 606 /* Resultant value is 256 times larger than it needs to be. This
466 case I40E_LOWEST_LATENCY: 607 * gives us room to adjust the value as needed to either increase
467 new_itr = I40E_ITR_50K; 608 * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
468 break; 609 *
469 case I40E_LOW_LATENCY: 610 * Use addition as we have already recorded the new latency flag
470 new_itr = I40E_ITR_20K; 611 * for the ITR value.
471 break; 612 */
472 case I40E_BULK_LATENCY: 613 itr += DIV_ROUND_UP(avg_wire_size, i40e_itr_divisor(q_vector)) *
473 new_itr = I40E_ITR_18K; 614 I40E_ITR_ADAPTIVE_MIN_INC;
474 break; 615
475 default: 616 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
476 break; 617 itr &= I40E_ITR_ADAPTIVE_LATENCY;
618 itr += I40E_ITR_ADAPTIVE_MAX_USECS;
477 } 619 }
478 620
621clear_counts:
622 /* write back value */
623 rc->target_itr = itr;
624
625 /* next update should occur within next jiffy */
626 rc->next_update = next_update + 1;
627
479 rc->total_bytes = 0; 628 rc->total_bytes = 0;
480 rc->total_packets = 0; 629 rc->total_packets = 0;
481 rc->last_itr_update = jiffies;
482
483 if (new_itr != rc->itr) {
484 rc->itr = new_itr;
485 return true;
486 }
487 return false;
488} 630}
489 631
490/** 632/**
@@ -1273,7 +1415,7 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
1273 * @rx_buffer: rx buffer to pull data from 1415 * @rx_buffer: rx buffer to pull data from
1274 * 1416 *
1275 * This function will clean up the contents of the rx_buffer. It will 1417 * This function will clean up the contents of the rx_buffer. It will
1276 * either recycle the bufer or unmap it and free the associated resources. 1418 * either recycle the buffer or unmap it and free the associated resources.
1277 */ 1419 */
1278static void i40e_put_rx_buffer(struct i40e_ring *rx_ring, 1420static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
1279 struct i40e_rx_buffer *rx_buffer) 1421 struct i40e_rx_buffer *rx_buffer)
@@ -1457,33 +1599,45 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1457 return failure ? budget : (int)total_rx_packets; 1599 return failure ? budget : (int)total_rx_packets;
1458} 1600}
1459 1601
1460static u32 i40e_buildreg_itr(const int type, const u16 itr) 1602static inline u32 i40e_buildreg_itr(const int type, u16 itr)
1461{ 1603{
1462 u32 val; 1604 u32 val;
1463 1605
1606 /* We don't bother with setting the CLEARPBA bit as the data sheet
1607 * points out doing so is "meaningless since it was already
1608 * auto-cleared". The auto-clearing happens when the interrupt is
1609 * asserted.
1610 *
1611 * Hardware errata 28 for also indicates that writing to a
1612 * xxINT_DYN_CTLx CSR with INTENA_MSK (bit 31) set to 0 will clear
1613 * an event in the PBA anyway so we need to rely on the automask
1614 * to hold pending events for us until the interrupt is re-enabled
1615 *
1616 * The itr value is reported in microseconds, and the register
1617 * value is recorded in 2 microsecond units. For this reason we
1618 * only need to shift by the interval shift - 1 instead of the
1619 * full value.
1620 */
1621 itr &= I40E_ITR_MASK;
1622
1464 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK | 1623 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1465 I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK |
1466 (type << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) | 1624 (type << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1467 (itr << I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT); 1625 (itr << (I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT - 1));
1468 1626
1469 return val; 1627 return val;
1470} 1628}
1471 1629
1472/* a small macro to shorten up some long lines */ 1630/* a small macro to shorten up some long lines */
1473#define INTREG I40E_VFINT_DYN_CTLN1 1631#define INTREG I40E_VFINT_DYN_CTLN1
1474static inline int get_rx_itr(struct i40e_vsi *vsi, int idx)
1475{
1476 struct i40evf_adapter *adapter = vsi->back;
1477 1632
1478 return adapter->rx_rings[idx].rx_itr_setting; 1633/* The act of updating the ITR will cause it to immediately trigger. In order
1479} 1634 * to prevent this from throwing off adaptive update statistics we defer the
1480 1635 * update so that it can only happen so often. So after either Tx or Rx are
1481static inline int get_tx_itr(struct i40e_vsi *vsi, int idx) 1636 * updated we make the adaptive scheme wait until either the ITR completely
1482{ 1637 * expires via the next_update expiration or we have been through at least
1483 struct i40evf_adapter *adapter = vsi->back; 1638 * 3 interrupts.
1484 1639 */
1485 return adapter->tx_rings[idx].tx_itr_setting; 1640#define ITR_COUNTDOWN_START 3
1486}
1487 1641
1488/** 1642/**
1489 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt 1643 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
@@ -1495,70 +1649,51 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
1495 struct i40e_q_vector *q_vector) 1649 struct i40e_q_vector *q_vector)
1496{ 1650{
1497 struct i40e_hw *hw = &vsi->back->hw; 1651 struct i40e_hw *hw = &vsi->back->hw;
1498 bool rx = false, tx = false; 1652 u32 intval;
1499 u32 rxval, txval;
1500 int idx = q_vector->v_idx;
1501 int rx_itr_setting, tx_itr_setting;
1502
1503 /* avoid dynamic calculation if in countdown mode OR if
1504 * all dynamic is disabled
1505 */
1506 rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
1507
1508 rx_itr_setting = get_rx_itr(vsi, idx);
1509 tx_itr_setting = get_tx_itr(vsi, idx);
1510 1653
1511 if (q_vector->itr_countdown > 0 || 1654 /* These will do nothing if dynamic updates are not enabled */
1512 (!ITR_IS_DYNAMIC(rx_itr_setting) && 1655 i40e_update_itr(q_vector, &q_vector->tx);
1513 !ITR_IS_DYNAMIC(tx_itr_setting))) { 1656 i40e_update_itr(q_vector, &q_vector->rx);
1514 goto enable_int;
1515 }
1516
1517 if (ITR_IS_DYNAMIC(rx_itr_setting)) {
1518 rx = i40e_set_new_dynamic_itr(&q_vector->rx);
1519 rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
1520 }
1521 1657
1522 if (ITR_IS_DYNAMIC(tx_itr_setting)) { 1658 /* This block of logic allows us to get away with only updating
1523 tx = i40e_set_new_dynamic_itr(&q_vector->tx); 1659 * one ITR value with each interrupt. The idea is to perform a
1524 txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr); 1660 * pseudo-lazy update with the following criteria.
1525 } 1661 *
1526 1662 * 1. Rx is given higher priority than Tx if both are in same state
1527 if (rx || tx) { 1663 * 2. If we must reduce an ITR that is given highest priority.
1528 /* get the higher of the two ITR adjustments and 1664 * 3. We then give priority to increasing ITR based on amount.
1529 * use the same value for both ITR registers
1530 * when in adaptive mode (Rx and/or Tx)
1531 */
1532 u16 itr = max(q_vector->tx.itr, q_vector->rx.itr);
1533
1534 q_vector->tx.itr = q_vector->rx.itr = itr;
1535 txval = i40e_buildreg_itr(I40E_TX_ITR, itr);
1536 tx = true;
1537 rxval = i40e_buildreg_itr(I40E_RX_ITR, itr);
1538 rx = true;
1539 }
1540
1541 /* only need to enable the interrupt once, but need
1542 * to possibly update both ITR values
1543 */ 1665 */
1544 if (rx) { 1666 if (q_vector->rx.target_itr < q_vector->rx.current_itr) {
1545 /* set the INTENA_MSK_MASK so that this first write 1667 /* Rx ITR needs to be reduced, this is highest priority */
1546 * won't actually enable the interrupt, instead just 1668 intval = i40e_buildreg_itr(I40E_RX_ITR,
1547 * updating the ITR (it's bit 31 PF and VF) 1669 q_vector->rx.target_itr);
1670 q_vector->rx.current_itr = q_vector->rx.target_itr;
1671 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1672 } else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) ||
1673 ((q_vector->rx.target_itr - q_vector->rx.current_itr) <
1674 (q_vector->tx.target_itr - q_vector->tx.current_itr))) {
1675 /* Tx ITR needs to be reduced, this is second priority
1676 * Tx ITR needs to be increased more than Rx, fourth priority
1548 */ 1677 */
1549 rxval |= BIT(31); 1678 intval = i40e_buildreg_itr(I40E_TX_ITR,
1550 /* don't check _DOWN because interrupt isn't being enabled */ 1679 q_vector->tx.target_itr);
1551 wr32(hw, INTREG(q_vector->reg_idx), rxval); 1680 q_vector->tx.current_itr = q_vector->tx.target_itr;
1681 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1682 } else if (q_vector->rx.current_itr != q_vector->rx.target_itr) {
1683 /* Rx ITR needs to be increased, third priority */
1684 intval = i40e_buildreg_itr(I40E_RX_ITR,
1685 q_vector->rx.target_itr);
1686 q_vector->rx.current_itr = q_vector->rx.target_itr;
1687 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1688 } else {
1689 /* No ITR update, lowest priority */
1690 intval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
1691 if (q_vector->itr_countdown)
1692 q_vector->itr_countdown--;
1552 } 1693 }
1553 1694
1554enable_int:
1555 if (!test_bit(__I40E_VSI_DOWN, vsi->state)) 1695 if (!test_bit(__I40E_VSI_DOWN, vsi->state))
1556 wr32(hw, INTREG(q_vector->reg_idx), txval); 1696 wr32(hw, INTREG(q_vector->reg_idx), intval);
1557
1558 if (q_vector->itr_countdown)
1559 q_vector->itr_countdown--;
1560 else
1561 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1562} 1697}
1563 1698
1564/** 1699/**
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
index 7798a6645c3f..9129447d079b 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
@@ -28,31 +28,35 @@
28#define _I40E_TXRX_H_ 28#define _I40E_TXRX_H_
29 29
30/* Interrupt Throttling and Rate Limiting Goodies */ 30/* Interrupt Throttling and Rate Limiting Goodies */
31
32#define I40E_MAX_ITR 0x0FF0 /* reg uses 2 usec resolution */
33#define I40E_MIN_ITR 0x0001 /* reg uses 2 usec resolution */
34#define I40E_ITR_100K 0x0005
35#define I40E_ITR_50K 0x000A
36#define I40E_ITR_20K 0x0019
37#define I40E_ITR_18K 0x001B
38#define I40E_ITR_8K 0x003E
39#define I40E_ITR_4K 0x007A
40#define I40E_MAX_INTRL 0x3B /* reg uses 4 usec resolution */
41#define I40E_ITR_RX_DEF (ITR_REG_TO_USEC(I40E_ITR_20K) | \
42 I40E_ITR_DYNAMIC)
43#define I40E_ITR_TX_DEF (ITR_REG_TO_USEC(I40E_ITR_20K) | \
44 I40E_ITR_DYNAMIC)
45#define I40E_ITR_DYNAMIC 0x8000 /* use top bit as a flag */
46#define I40E_MIN_INT_RATE 250 /* ~= 1000000 / (I40E_MAX_ITR * 2) */
47#define I40E_MAX_INT_RATE 500000 /* == 1000000 / (I40E_MIN_ITR * 2) */
48#define I40E_DEFAULT_IRQ_WORK 256 31#define I40E_DEFAULT_IRQ_WORK 256
49#define ITR_TO_REG(setting) ((setting & ~I40E_ITR_DYNAMIC) >> 1) 32
50#define ITR_IS_DYNAMIC(setting) (!!(setting & I40E_ITR_DYNAMIC)) 33/* The datasheet for the X710 and XL710 indicate that the maximum value for
51#define ITR_REG_TO_USEC(itr_reg) (itr_reg << 1) 34 * the ITR is 8160usec which is then called out as 0xFF0 with a 2usec
35 * resolution. 8160 is 0x1FE0 when written out in hex. So instead of storing
36 * the register value which is divided by 2 lets use the actual values and
37 * avoid an excessive amount of translation.
38 */
39#define I40E_ITR_DYNAMIC 0x8000 /* use top bit as a flag */
40#define I40E_ITR_MASK 0x1FFE /* mask for ITR register value */
41#define I40E_MIN_ITR 2 /* reg uses 2 usec resolution */
42#define I40E_ITR_100K 10 /* all values below must be even */
43#define I40E_ITR_50K 20
44#define I40E_ITR_20K 50
45#define I40E_ITR_18K 60
46#define I40E_ITR_8K 122
47#define I40E_MAX_ITR 8160 /* maximum value as per datasheet */
48#define ITR_TO_REG(setting) ((setting) & ~I40E_ITR_DYNAMIC)
49#define ITR_REG_ALIGN(setting) __ALIGN_MASK(setting, ~I40E_ITR_MASK)
50#define ITR_IS_DYNAMIC(setting) (!!((setting) & I40E_ITR_DYNAMIC))
51
52#define I40E_ITR_RX_DEF (I40E_ITR_20K | I40E_ITR_DYNAMIC)
53#define I40E_ITR_TX_DEF (I40E_ITR_20K | I40E_ITR_DYNAMIC)
54
52/* 0x40 is the enable bit for interrupt rate limiting, and must be set if 55/* 0x40 is the enable bit for interrupt rate limiting, and must be set if
53 * the value of the rate limit is non-zero 56 * the value of the rate limit is non-zero
54 */ 57 */
55#define INTRL_ENA BIT(6) 58#define INTRL_ENA BIT(6)
59#define I40E_MAX_INTRL 0x3B /* reg uses 4 usec resolution */
56#define INTRL_REG_TO_USEC(intrl) ((intrl & ~INTRL_ENA) << 2) 60#define INTRL_REG_TO_USEC(intrl) ((intrl & ~INTRL_ENA) << 2)
57#define INTRL_USEC_TO_REG(set) ((set) ? ((set) >> 2) | INTRL_ENA : 0) 61#define INTRL_USEC_TO_REG(set) ((set) ? ((set) >> 2) | INTRL_ENA : 0)
58#define I40E_INTRL_8K 125 /* 8000 ints/sec */ 62#define I40E_INTRL_8K 125 /* 8000 ints/sec */
@@ -362,8 +366,7 @@ struct i40e_ring {
362 * these values always store the USER setting, and must be converted 366 * these values always store the USER setting, and must be converted
363 * before programming to a register. 367 * before programming to a register.
364 */ 368 */
365 u16 rx_itr_setting; 369 u16 itr_setting;
366 u16 tx_itr_setting;
367 370
368 u16 count; /* Number of descriptors */ 371 u16 count; /* Number of descriptors */
369 u16 reg_idx; /* HW register index of the ring */ 372 u16 reg_idx; /* HW register index of the ring */
@@ -425,21 +428,21 @@ static inline void clear_ring_build_skb_enabled(struct i40e_ring *ring)
425 ring->flags &= ~I40E_RXR_FLAGS_BUILD_SKB_ENABLED; 428 ring->flags &= ~I40E_RXR_FLAGS_BUILD_SKB_ENABLED;
426} 429}
427 430
428enum i40e_latency_range { 431#define I40E_ITR_ADAPTIVE_MIN_INC 0x0002
429 I40E_LOWEST_LATENCY = 0, 432#define I40E_ITR_ADAPTIVE_MIN_USECS 0x0002
430 I40E_LOW_LATENCY = 1, 433#define I40E_ITR_ADAPTIVE_MAX_USECS 0x007e
431 I40E_BULK_LATENCY = 2, 434#define I40E_ITR_ADAPTIVE_LATENCY 0x8000
432}; 435#define I40E_ITR_ADAPTIVE_BULK 0x0000
436#define ITR_IS_BULK(x) (!((x) & I40E_ITR_ADAPTIVE_LATENCY))
433 437
434struct i40e_ring_container { 438struct i40e_ring_container {
435 /* array of pointers to rings */ 439 struct i40e_ring *ring; /* pointer to linked list of ring(s) */
436 struct i40e_ring *ring; 440 unsigned long next_update; /* jiffies value of next update */
437 unsigned int total_bytes; /* total bytes processed this int */ 441 unsigned int total_bytes; /* total bytes processed this int */
438 unsigned int total_packets; /* total packets processed this int */ 442 unsigned int total_packets; /* total packets processed this int */
439 unsigned long last_itr_update; /* jiffies of last ITR update */
440 u16 count; 443 u16 count;
441 enum i40e_latency_range latency_range; 444 u16 target_itr; /* target ITR setting for ring(s) */
442 u16 itr; 445 u16 current_itr; /* current ITR setting for ring(s) */
443}; 446};
444 447
445/* iterator for handling rings in ring container */ 448/* iterator for handling rings in ring container */
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf.h b/drivers/net/ethernet/intel/i40evf/i40evf.h
index 9690c1ea019e..b6991e8014d8 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf.h
+++ b/drivers/net/ethernet/intel/i40evf/i40evf.h
@@ -117,9 +117,8 @@ struct i40e_q_vector {
117 struct i40e_ring_container rx; 117 struct i40e_ring_container rx;
118 struct i40e_ring_container tx; 118 struct i40e_ring_container tx;
119 u32 ring_mask; 119 u32 ring_mask;
120 u8 itr_countdown; /* when 0 should adjust adaptive ITR */
120 u8 num_ringpairs; /* total number of ring pairs in vector */ 121 u8 num_ringpairs; /* total number of ring pairs in vector */
121#define ITR_COUNTDOWN_START 100
122 u8 itr_countdown; /* when 0 or 1 update ITR */
123 u16 v_idx; /* index in the vsi->q_vector array. */ 122 u16 v_idx; /* index in the vsi->q_vector array. */
124 u16 reg_idx; /* register index of the interrupt */ 123 u16 reg_idx; /* register index of the interrupt */
125 char name[IFNAMSIZ + 15]; 124 char name[IFNAMSIZ + 15];
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
index e2d8aa19d205..aded3ad7763e 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
@@ -457,14 +457,14 @@ static int __i40evf_get_coalesce(struct net_device *netdev,
457 rx_ring = &adapter->rx_rings[queue]; 457 rx_ring = &adapter->rx_rings[queue];
458 tx_ring = &adapter->tx_rings[queue]; 458 tx_ring = &adapter->tx_rings[queue];
459 459
460 if (ITR_IS_DYNAMIC(rx_ring->rx_itr_setting)) 460 if (ITR_IS_DYNAMIC(rx_ring->itr_setting))
461 ec->use_adaptive_rx_coalesce = 1; 461 ec->use_adaptive_rx_coalesce = 1;
462 462
463 if (ITR_IS_DYNAMIC(tx_ring->tx_itr_setting)) 463 if (ITR_IS_DYNAMIC(tx_ring->itr_setting))
464 ec->use_adaptive_tx_coalesce = 1; 464 ec->use_adaptive_tx_coalesce = 1;
465 465
466 ec->rx_coalesce_usecs = rx_ring->rx_itr_setting & ~I40E_ITR_DYNAMIC; 466 ec->rx_coalesce_usecs = rx_ring->itr_setting & ~I40E_ITR_DYNAMIC;
467 ec->tx_coalesce_usecs = tx_ring->tx_itr_setting & ~I40E_ITR_DYNAMIC; 467 ec->tx_coalesce_usecs = tx_ring->itr_setting & ~I40E_ITR_DYNAMIC;
468 468
469 return 0; 469 return 0;
470} 470}
@@ -502,7 +502,7 @@ static int i40evf_get_per_queue_coalesce(struct net_device *netdev,
502 502
503/** 503/**
504 * i40evf_set_itr_per_queue - set ITR values for specific queue 504 * i40evf_set_itr_per_queue - set ITR values for specific queue
505 * @vsi: the VSI to set values for 505 * @adapter: the VF adapter struct to set values for
506 * @ec: coalesce settings from ethtool 506 * @ec: coalesce settings from ethtool
507 * @queue: the queue to modify 507 * @queue: the queue to modify
508 * 508 *
@@ -514,33 +514,29 @@ static void i40evf_set_itr_per_queue(struct i40evf_adapter *adapter,
514{ 514{
515 struct i40e_ring *rx_ring = &adapter->rx_rings[queue]; 515 struct i40e_ring *rx_ring = &adapter->rx_rings[queue];
516 struct i40e_ring *tx_ring = &adapter->tx_rings[queue]; 516 struct i40e_ring *tx_ring = &adapter->tx_rings[queue];
517 struct i40e_vsi *vsi = &adapter->vsi;
518 struct i40e_hw *hw = &adapter->hw;
519 struct i40e_q_vector *q_vector; 517 struct i40e_q_vector *q_vector;
520 u16 vector;
521 518
522 rx_ring->rx_itr_setting = ec->rx_coalesce_usecs; 519 rx_ring->itr_setting = ITR_REG_ALIGN(ec->rx_coalesce_usecs);
523 tx_ring->tx_itr_setting = ec->tx_coalesce_usecs; 520 tx_ring->itr_setting = ITR_REG_ALIGN(ec->tx_coalesce_usecs);
524 521
525 rx_ring->rx_itr_setting |= I40E_ITR_DYNAMIC; 522 rx_ring->itr_setting |= I40E_ITR_DYNAMIC;
526 if (!ec->use_adaptive_rx_coalesce) 523 if (!ec->use_adaptive_rx_coalesce)
527 rx_ring->rx_itr_setting ^= I40E_ITR_DYNAMIC; 524 rx_ring->itr_setting ^= I40E_ITR_DYNAMIC;
528 525
529 tx_ring->tx_itr_setting |= I40E_ITR_DYNAMIC; 526 tx_ring->itr_setting |= I40E_ITR_DYNAMIC;
530 if (!ec->use_adaptive_tx_coalesce) 527 if (!ec->use_adaptive_tx_coalesce)
531 tx_ring->tx_itr_setting ^= I40E_ITR_DYNAMIC; 528 tx_ring->itr_setting ^= I40E_ITR_DYNAMIC;
532 529
533 q_vector = rx_ring->q_vector; 530 q_vector = rx_ring->q_vector;
534 q_vector->rx.itr = ITR_TO_REG(rx_ring->rx_itr_setting); 531 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
535 vector = vsi->base_vector + q_vector->v_idx;
536 wr32(hw, I40E_VFINT_ITRN1(I40E_RX_ITR, vector - 1), q_vector->rx.itr);
537 532
538 q_vector = tx_ring->q_vector; 533 q_vector = tx_ring->q_vector;
539 q_vector->tx.itr = ITR_TO_REG(tx_ring->tx_itr_setting); 534 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
540 vector = vsi->base_vector + q_vector->v_idx;
541 wr32(hw, I40E_VFINT_ITRN1(I40E_TX_ITR, vector - 1), q_vector->tx.itr);
542 535
543 i40e_flush(hw); 536 /* The interrupt handler itself will take care of programming
537 * the Tx and Rx ITR values based on the values we have entered
538 * into the q_vector, no need to write the values now.
539 */
544} 540}
545 541
546/** 542/**
@@ -565,8 +561,8 @@ static int __i40evf_set_coalesce(struct net_device *netdev,
565 if (ec->rx_coalesce_usecs == 0) { 561 if (ec->rx_coalesce_usecs == 0) {
566 if (ec->use_adaptive_rx_coalesce) 562 if (ec->use_adaptive_rx_coalesce)
567 netif_info(adapter, drv, netdev, "rx-usecs=0, need to disable adaptive-rx for a complete disable\n"); 563 netif_info(adapter, drv, netdev, "rx-usecs=0, need to disable adaptive-rx for a complete disable\n");
568 } else if ((ec->rx_coalesce_usecs < (I40E_MIN_ITR << 1)) || 564 } else if ((ec->rx_coalesce_usecs < I40E_MIN_ITR) ||
569 (ec->rx_coalesce_usecs > (I40E_MAX_ITR << 1))) { 565 (ec->rx_coalesce_usecs > I40E_MAX_ITR)) {
570 netif_info(adapter, drv, netdev, "Invalid value, rx-usecs range is 0-8160\n"); 566 netif_info(adapter, drv, netdev, "Invalid value, rx-usecs range is 0-8160\n");
571 return -EINVAL; 567 return -EINVAL;
572 } 568 }
@@ -575,8 +571,8 @@ static int __i40evf_set_coalesce(struct net_device *netdev,
575 if (ec->tx_coalesce_usecs == 0) { 571 if (ec->tx_coalesce_usecs == 0) {
576 if (ec->use_adaptive_tx_coalesce) 572 if (ec->use_adaptive_tx_coalesce)
577 netif_info(adapter, drv, netdev, "tx-usecs=0, need to disable adaptive-tx for a complete disable\n"); 573 netif_info(adapter, drv, netdev, "tx-usecs=0, need to disable adaptive-tx for a complete disable\n");
578 } else if ((ec->tx_coalesce_usecs < (I40E_MIN_ITR << 1)) || 574 } else if ((ec->tx_coalesce_usecs < I40E_MIN_ITR) ||
579 (ec->tx_coalesce_usecs > (I40E_MAX_ITR << 1))) { 575 (ec->tx_coalesce_usecs > I40E_MAX_ITR)) {
580 netif_info(adapter, drv, netdev, "Invalid value, tx-usecs range is 0-8160\n"); 576 netif_info(adapter, drv, netdev, "Invalid value, tx-usecs range is 0-8160\n");
581 return -EINVAL; 577 return -EINVAL;
582 } 578 }
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_main.c b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
index 16989ad2ca90..6fd09926181a 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_main.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
@@ -353,11 +353,12 @@ i40evf_map_vector_to_rxq(struct i40evf_adapter *adapter, int v_idx, int r_idx)
353 rx_ring->vsi = &adapter->vsi; 353 rx_ring->vsi = &adapter->vsi;
354 q_vector->rx.ring = rx_ring; 354 q_vector->rx.ring = rx_ring;
355 q_vector->rx.count++; 355 q_vector->rx.count++;
356 q_vector->rx.latency_range = I40E_LOW_LATENCY; 356 q_vector->rx.next_update = jiffies + 1;
357 q_vector->rx.itr = ITR_TO_REG(rx_ring->rx_itr_setting); 357 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
358 q_vector->ring_mask |= BIT(r_idx); 358 q_vector->ring_mask |= BIT(r_idx);
359 q_vector->itr_countdown = ITR_COUNTDOWN_START; 359 wr32(hw, I40E_VFINT_ITRN1(I40E_RX_ITR, q_vector->reg_idx),
360 wr32(hw, I40E_VFINT_ITRN1(I40E_RX_ITR, v_idx - 1), q_vector->rx.itr); 360 q_vector->rx.current_itr);
361 q_vector->rx.current_itr = q_vector->rx.target_itr;
361} 362}
362 363
363/** 364/**
@@ -378,11 +379,12 @@ i40evf_map_vector_to_txq(struct i40evf_adapter *adapter, int v_idx, int t_idx)
378 tx_ring->vsi = &adapter->vsi; 379 tx_ring->vsi = &adapter->vsi;
379 q_vector->tx.ring = tx_ring; 380 q_vector->tx.ring = tx_ring;
380 q_vector->tx.count++; 381 q_vector->tx.count++;
381 q_vector->tx.latency_range = I40E_LOW_LATENCY; 382 q_vector->tx.next_update = jiffies + 1;
382 q_vector->tx.itr = ITR_TO_REG(tx_ring->tx_itr_setting); 383 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
383 q_vector->itr_countdown = ITR_COUNTDOWN_START;
384 q_vector->num_ringpairs++; 384 q_vector->num_ringpairs++;
385 wr32(hw, I40E_VFINT_ITRN1(I40E_TX_ITR, v_idx - 1), q_vector->tx.itr); 385 wr32(hw, I40E_VFINT_ITRN1(I40E_TX_ITR, q_vector->reg_idx),
386 q_vector->tx.target_itr);
387 q_vector->tx.current_itr = q_vector->tx.target_itr;
386} 388}
387 389
388/** 390/**
@@ -1169,7 +1171,7 @@ static int i40evf_alloc_queues(struct i40evf_adapter *adapter)
1169 tx_ring->netdev = adapter->netdev; 1171 tx_ring->netdev = adapter->netdev;
1170 tx_ring->dev = &adapter->pdev->dev; 1172 tx_ring->dev = &adapter->pdev->dev;
1171 tx_ring->count = adapter->tx_desc_count; 1173 tx_ring->count = adapter->tx_desc_count;
1172 tx_ring->tx_itr_setting = I40E_ITR_TX_DEF; 1174 tx_ring->itr_setting = I40E_ITR_TX_DEF;
1173 if (adapter->flags & I40EVF_FLAG_WB_ON_ITR_CAPABLE) 1175 if (adapter->flags & I40EVF_FLAG_WB_ON_ITR_CAPABLE)
1174 tx_ring->flags |= I40E_TXR_FLAGS_WB_ON_ITR; 1176 tx_ring->flags |= I40E_TXR_FLAGS_WB_ON_ITR;
1175 1177
@@ -1178,7 +1180,7 @@ static int i40evf_alloc_queues(struct i40evf_adapter *adapter)
1178 rx_ring->netdev = adapter->netdev; 1180 rx_ring->netdev = adapter->netdev;
1179 rx_ring->dev = &adapter->pdev->dev; 1181 rx_ring->dev = &adapter->pdev->dev;
1180 rx_ring->count = adapter->rx_desc_count; 1182 rx_ring->count = adapter->rx_desc_count;
1181 rx_ring->rx_itr_setting = I40E_ITR_RX_DEF; 1183 rx_ring->itr_setting = I40E_ITR_RX_DEF;
1182 } 1184 }
1183 1185
1184 adapter->num_active_queues = num_active_queues; 1186 adapter->num_active_queues = num_active_queues;
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
index 50ce0d6c09ef..d57a67285505 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
@@ -344,6 +344,7 @@ void i40evf_disable_queues(struct i40evf_adapter *adapter)
344void i40evf_map_queues(struct i40evf_adapter *adapter) 344void i40evf_map_queues(struct i40evf_adapter *adapter)
345{ 345{
346 struct virtchnl_irq_map_info *vimi; 346 struct virtchnl_irq_map_info *vimi;
347 struct virtchnl_vector_map *vecmap;
347 int v_idx, q_vectors, len; 348 int v_idx, q_vectors, len;
348 struct i40e_q_vector *q_vector; 349 struct i40e_q_vector *q_vector;
349 350
@@ -367,17 +368,22 @@ void i40evf_map_queues(struct i40evf_adapter *adapter)
367 vimi->num_vectors = adapter->num_msix_vectors; 368 vimi->num_vectors = adapter->num_msix_vectors;
368 /* Queue vectors first */ 369 /* Queue vectors first */
369 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 370 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
370 q_vector = adapter->q_vectors + v_idx; 371 q_vector = &adapter->q_vectors[v_idx];
371 vimi->vecmap[v_idx].vsi_id = adapter->vsi_res->vsi_id; 372 vecmap = &vimi->vecmap[v_idx];
372 vimi->vecmap[v_idx].vector_id = v_idx + NONQ_VECS; 373
373 vimi->vecmap[v_idx].txq_map = q_vector->ring_mask; 374 vecmap->vsi_id = adapter->vsi_res->vsi_id;
374 vimi->vecmap[v_idx].rxq_map = q_vector->ring_mask; 375 vecmap->vector_id = v_idx + NONQ_VECS;
376 vecmap->txq_map = q_vector->ring_mask;
377 vecmap->rxq_map = q_vector->ring_mask;
378 vecmap->rxitr_idx = I40E_RX_ITR;
379 vecmap->txitr_idx = I40E_TX_ITR;
375 } 380 }
376 /* Misc vector last - this is only for AdminQ messages */ 381 /* Misc vector last - this is only for AdminQ messages */
377 vimi->vecmap[v_idx].vsi_id = adapter->vsi_res->vsi_id; 382 vecmap = &vimi->vecmap[v_idx];
378 vimi->vecmap[v_idx].vector_id = 0; 383 vecmap->vsi_id = adapter->vsi_res->vsi_id;
379 vimi->vecmap[v_idx].txq_map = 0; 384 vecmap->vector_id = 0;
380 vimi->vecmap[v_idx].rxq_map = 0; 385 vecmap->txq_map = 0;
386 vecmap->rxq_map = 0;
381 387
382 adapter->aq_required &= ~I40EVF_FLAG_AQ_MAP_VECTORS; 388 adapter->aq_required &= ~I40EVF_FLAG_AQ_MAP_VECTORS;
383 i40evf_send_pf_msg(adapter, VIRTCHNL_OP_CONFIG_IRQ_MAP, 389 i40evf_send_pf_msg(adapter, VIRTCHNL_OP_CONFIG_IRQ_MAP,
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-01 17:56:14 -0400