diff options
author | Eric Dumazet <edumazet@google.com> | 2014-10-05 05:35:15 -0400 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2014-10-06 01:04:15 -0400 |
commit | fb1843ee72c7b2ed784bf88ad79c6ccc2f652499 (patch) | |
tree | 139741643875150cd8cc289b92790b63dcd73f3c /drivers/net | |
parent | 29d40c903247596c954446639116467eb6689bb7 (diff) |
net/mlx4_en: Avoid false sharing in mlx4_en_en_process_tx_cq()
mlx4_en_process_tx_cq() carefully fetches and writes ring->last_nr_txbb
and ring->cons only one time to avoid false sharing
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Amir Vadai <amirv@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers/net')
-rw-r--r-- | drivers/net/ethernet/mellanox/mlx4/en_tx.c | 22 |
1 files changed, 15 insertions, 7 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index b96627c4972f..d9aaeb2d019f 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c | |||
@@ -387,6 +387,8 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev, | |||
387 | u64 timestamp = 0; | 387 | u64 timestamp = 0; |
388 | int done = 0; | 388 | int done = 0; |
389 | int budget = priv->tx_work_limit; | 389 | int budget = priv->tx_work_limit; |
390 | u32 last_nr_txbb; | ||
391 | u32 ring_cons; | ||
390 | 392 | ||
391 | if (!priv->port_up) | 393 | if (!priv->port_up) |
392 | return true; | 394 | return true; |
@@ -394,7 +396,9 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev, | |||
394 | prefetchw(&ring->tx_queue->dql.limit); | 396 | prefetchw(&ring->tx_queue->dql.limit); |
395 | index = cons_index & size_mask; | 397 | index = cons_index & size_mask; |
396 | cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; | 398 | cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; |
397 | ring_index = ring->cons & size_mask; | 399 | last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb); |
400 | ring_cons = ACCESS_ONCE(ring->cons); | ||
401 | ring_index = ring_cons & size_mask; | ||
398 | stamp_index = ring_index; | 402 | stamp_index = ring_index; |
399 | 403 | ||
400 | /* Process all completed CQEs */ | 404 | /* Process all completed CQEs */ |
@@ -419,19 +423,19 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev, | |||
419 | new_index = be16_to_cpu(cqe->wqe_index) & size_mask; | 423 | new_index = be16_to_cpu(cqe->wqe_index) & size_mask; |
420 | 424 | ||
421 | do { | 425 | do { |
422 | txbbs_skipped += ring->last_nr_txbb; | 426 | txbbs_skipped += last_nr_txbb; |
423 | ring_index = (ring_index + ring->last_nr_txbb) & size_mask; | 427 | ring_index = (ring_index + last_nr_txbb) & size_mask; |
424 | if (ring->tx_info[ring_index].ts_requested) | 428 | if (ring->tx_info[ring_index].ts_requested) |
425 | timestamp = mlx4_en_get_cqe_ts(cqe); | 429 | timestamp = mlx4_en_get_cqe_ts(cqe); |
426 | 430 | ||
427 | /* free next descriptor */ | 431 | /* free next descriptor */ |
428 | ring->last_nr_txbb = mlx4_en_free_tx_desc( | 432 | last_nr_txbb = mlx4_en_free_tx_desc( |
429 | priv, ring, ring_index, | 433 | priv, ring, ring_index, |
430 | !!((ring->cons + txbbs_skipped) & | 434 | !!((ring_cons + txbbs_skipped) & |
431 | ring->size), timestamp); | 435 | ring->size), timestamp); |
432 | 436 | ||
433 | mlx4_en_stamp_wqe(priv, ring, stamp_index, | 437 | mlx4_en_stamp_wqe(priv, ring, stamp_index, |
434 | !!((ring->cons + txbbs_stamp) & | 438 | !!((ring_cons + txbbs_stamp) & |
435 | ring->size)); | 439 | ring->size)); |
436 | stamp_index = ring_index; | 440 | stamp_index = ring_index; |
437 | txbbs_stamp = txbbs_skipped; | 441 | txbbs_stamp = txbbs_skipped; |
@@ -452,7 +456,11 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev, | |||
452 | mcq->cons_index = cons_index; | 456 | mcq->cons_index = cons_index; |
453 | mlx4_cq_set_ci(mcq); | 457 | mlx4_cq_set_ci(mcq); |
454 | wmb(); | 458 | wmb(); |
455 | ring->cons += txbbs_skipped; | 459 | |
460 | /* we want to dirty this cache line once */ | ||
461 | ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb; | ||
462 | ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped; | ||
463 | |||
456 | netdev_tx_completed_queue(ring->tx_queue, packets, bytes); | 464 | netdev_tx_completed_queue(ring->tx_queue, packets, bytes); |
457 | 465 | ||
458 | /* | 466 | /* |