aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2018-03-31 22:31:43 -0400
committerDavid S. Miller <davem@davemloft.net>2018-03-31 22:31:43 -0400
commit8bde261e535257e81087d39ff808414e2f5aa39d (patch)
tree9e252927912c6fa28db2c004e71a3f7f54e4e6c7
parente2e80c027f5adab3cc44c3d07c4484291384d278 (diff)
parentab966d7e4ff988a48b3ad72e7abf903aa840afd1 (diff)
Merge tag 'mlx5-updates-2018-03-30' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux
Saeed Mahameed says: ==================== mlx5-updates-2018-03-30 This series contains updates to mlx5 core and mlx5e netdev drivers. The main highlight of this series is the RX optimizations for striding RQ path, introduced by Tariq. First Four patches are trivial misc cleanups. - Spelling mistake fix - Dead code removal - Warning messages RX optimizations for striding RQ: 1) RX refactoring, cleanups and micro optimizations - MTU calculation simplifications, obsoletes some WQEs-to-packets translation functions and helps delete ~60 LOC. - Do not busy-wait a pending UMR completion. - post the new values of UMR WQE inline, instead of using a data pointer. - use pre-initialized structures to save calculations in datapath. 2) Use linear SKB in Striding RQ "build_skb", (Using linear SKB has many advantages): - Saves a memcpy of the headers. - No page-boundary checks in datapath. - No filler CQEs. - Significantly smaller CQ. - SKB data continuously resides in linear part, and not split to small amount (linear part) and large amount (fragment). This saves datapath cycles in driver and improves utilization of SKB fragments in GRO. - The fragments of a resulting GRO SKB follow the IP forwarding assumption of equal-size fragments. implementation details: HW writes the packets to the beginning of a stride, i.e. does not keep headroom. To overcome this we make sure we can extend backwards and use the last bytes of stride i-1. Extra care is needed for stride 0 as it has no preceding stride. We make sure headroom bytes are available by shifting the buffer pointer passed to HW by headroom bytes. This configuration now becomes default, whenever capable. Of course, this implies turning LRO off. Performance testing: ConnectX-5, single core, single RX ring, default MTU. UDP packet rate, early drop in TC layer: -------------------------------------------- | pkt size | before | after | ratio | -------------------------------------------- | 1500byte | 4.65 Mpps | 5.96 Mpps | 1.28x | | 500byte | 5.23 Mpps | 5.97 Mpps | 1.14x | | 64byte | 5.94 Mpps | 5.96 Mpps | 1.00x | -------------------------------------------- TCP streams: ~20% gain 3) Support XDP over Striding RQ: Now that linear SKB is supported over Striding RQ, we can support XDP by setting stride size to PAGE_SIZE and headroom to XDP_PACKET_HEADROOM. Striding RQ is capable of a higher packet-rate than conventional RQ. Performance testing: ConnectX-5, 24 rings, default MTU. CQE compression ON (to reduce completions BW in PCI). XDP_DROP packet rate: -------------------------------------------------- | pkt size | XDP rate | 100GbE linerate | pct% | -------------------------------------------------- | 64byte | 126.2 Mpps | 148.0 Mpps | 85% | | 128byte | 80.0 Mpps | 84.8 Mpps | 94% | | 256byte | 42.7 Mpps | 42.7 Mpps | 100% | | 512byte | 23.4 Mpps | 23.4 Mpps | 100% | -------------------------------------------------- 4) Remove mlx5 page_ref bulking in Striding RQ and use page_ref_inc only when needed. Without this bulking, we have: - no atomic ops on WQE allocation or free - one atomic op per SKB - In the default MTU configuration (1500, stride size is 2K), the non-bulking method execute 2 atomic ops as before - For larger MTUs with stride size of 4K, non-bulking method executes only a single op. - For XDP (stride size of 4K, no SKBs), non-bulking have no atomic ops per packet at all. Performance testing: ConnectX-5, Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz. Single core packet rate (64 bytes). Early drop in TC: no degradation. XDP_DROP: before: 14,270,188 pps after: 20,503,603 pps, 43% improvement. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en.h115
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c79
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_main.c310
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rep.c5
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rx.c284
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fw.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c27
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/transobj.c21
-rw-r--r--include/linux/mlx5/device.h3
-rw-r--r--include/linux/mlx5/mlx5_ifc.h7
-rw-r--r--include/linux/mlx5/transobj.h1
12 files changed, 409 insertions, 447 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 353ac6daa3dc..30cad07be2b5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -57,24 +57,12 @@
57 57
58#define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN) 58#define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
59 59
60#define MLX5E_HW2SW_MTU(priv, hwmtu) ((hwmtu) - ((priv)->hard_mtu)) 60#define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu))
61#define MLX5E_SW2HW_MTU(priv, swmtu) ((swmtu) + ((priv)->hard_mtu)) 61#define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu))
62 62
63#define MLX5E_MAX_DSCP 64 63#define MLX5E_MAX_DSCP 64
64#define MLX5E_MAX_NUM_TC 8 64#define MLX5E_MAX_NUM_TC 8
65 65
66#define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x6
67#define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa
68#define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd
69
70#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE 0x1
71#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa
72#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE 0xd
73
74#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2
75#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW 0x3
76#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW 0x6
77
78#define MLX5_RX_HEADROOM NET_SKB_PAD 66#define MLX5_RX_HEADROOM NET_SKB_PAD
79#define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \ 67#define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
80 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 68 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
@@ -95,11 +83,29 @@
95#define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER) 83#define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER)
96 84
97#define MLX5_MTT_OCTW(npages) (ALIGN(npages, 8) / 2) 85#define MLX5_MTT_OCTW(npages) (ALIGN(npages, 8) / 2)
98#define MLX5E_REQUIRED_MTTS(wqes) \ 86#define MLX5E_REQUIRED_WQE_MTTS (ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8))
99 (wqes * ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8)) 87#define MLX5E_LOG_ALIGNED_MPWQE_PPW (ilog2(MLX5E_REQUIRED_WQE_MTTS))
100#define MLX5E_VALID_NUM_MTTS(num_mtts) (MLX5_MTT_OCTW(num_mtts) - 1 <= U16_MAX) 88#define MLX5E_REQUIRED_MTTS(wqes) (wqes * MLX5E_REQUIRED_WQE_MTTS)
89#define MLX5E_MAX_RQ_NUM_MTTS \
90 ((1 << 16) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */
91#define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
92#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW \
93 (ilog2(MLX5E_MAX_RQ_NUM_MTTS / MLX5E_REQUIRED_WQE_MTTS))
94#define MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW \
95 (MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW + \
96 (MLX5_MPWRQ_LOG_WQE_SZ - MLX5E_ORDER2_MAX_PACKET_MTU))
97
98#define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x6
99#define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa
100#define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd
101
102#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE 0x1
103#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa
104#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE min_t(u8, 0xd, \
105 MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW)
106
107#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2
101 108
102#define MLX5_UMR_ALIGN (2048)
103#define MLX5_MPWRQ_SMALL_PACKET_THRESHOLD (256) 109#define MLX5_MPWRQ_SMALL_PACKET_THRESHOLD (256)
104 110
105#define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024) 111#define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024)
@@ -124,8 +130,13 @@
124#define MLX5E_UPDATE_STATS_INTERVAL 200 /* msecs */ 130#define MLX5E_UPDATE_STATS_INTERVAL 200 /* msecs */
125#define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */ 131#define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */
126 132
127#define MLX5E_ICOSQ_MAX_WQEBBS \ 133#define MLX5E_UMR_WQE_INLINE_SZ \
128 (DIV_ROUND_UP(sizeof(struct mlx5e_umr_wqe), MLX5_SEND_WQE_BB)) 134 (sizeof(struct mlx5e_umr_wqe) + \
135 ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(struct mlx5_mtt), \
136 MLX5_UMR_MTT_ALIGNMENT))
137#define MLX5E_UMR_WQEBBS \
138 (DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))
139#define MLX5E_ICOSQ_MAX_WQEBBS MLX5E_UMR_WQEBBS
129 140
130#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN) 141#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
131#define MLX5E_XDP_TX_DS_COUNT \ 142#define MLX5E_XDP_TX_DS_COUNT \
@@ -155,26 +166,6 @@ static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
155 } 166 }
156} 167}
157 168
158static inline int mlx5_min_log_rq_size(int wq_type)
159{
160 switch (wq_type) {
161 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
162 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
163 default:
164 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE;
165 }
166}
167
168static inline int mlx5_max_log_rq_size(int wq_type)
169{
170 switch (wq_type) {
171 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
172 return MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW;
173 default:
174 return MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE;
175 }
176}
177
178static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) 169static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
179{ 170{
180 return is_kdump_kernel() ? 171 return is_kdump_kernel() ?
@@ -197,7 +188,7 @@ struct mlx5e_umr_wqe {
197 struct mlx5_wqe_ctrl_seg ctrl; 188 struct mlx5_wqe_ctrl_seg ctrl;
198 struct mlx5_wqe_umr_ctrl_seg uctrl; 189 struct mlx5_wqe_umr_ctrl_seg uctrl;
199 struct mlx5_mkey_seg mkc; 190 struct mlx5_mkey_seg mkc;
200 struct mlx5_wqe_data_seg data; 191 struct mlx5_mtt inline_mtts[0];
201}; 192};
202 193
203extern const char mlx5e_self_tests[][ETH_GSTRING_LEN]; 194extern const char mlx5e_self_tests[][ETH_GSTRING_LEN];
@@ -233,7 +224,7 @@ enum mlx5e_priv_flag {
233struct mlx5e_params { 224struct mlx5e_params {
234 u8 log_sq_size; 225 u8 log_sq_size;
235 u8 rq_wq_type; 226 u8 rq_wq_type;
236 u8 log_rq_size; 227 u8 log_rq_mtu_frames;
237 u16 num_channels; 228 u16 num_channels;
238 u8 num_tc; 229 u8 num_tc;
239 bool rx_cqe_compress_def; 230 bool rx_cqe_compress_def;
@@ -251,6 +242,8 @@ struct mlx5e_params {
251 u32 lro_timeout; 242 u32 lro_timeout;
252 u32 pflags; 243 u32 pflags;
253 struct bpf_prog *xdp_prog; 244 struct bpf_prog *xdp_prog;
245 unsigned int sw_mtu;
246 int hard_mtu;
254}; 247};
255 248
256#ifdef CONFIG_MLX5_CORE_EN_DCB 249#ifdef CONFIG_MLX5_CORE_EN_DCB
@@ -433,7 +426,6 @@ struct mlx5e_icosq {
433 void __iomem *uar_map; 426 void __iomem *uar_map;
434 u32 sqn; 427 u32 sqn;
435 u16 edge; 428 u16 edge;
436 __be32 mkey_be;
437 unsigned long state; 429 unsigned long state;
438 430
439 /* control path */ 431 /* control path */
@@ -458,16 +450,13 @@ struct mlx5e_wqe_frag_info {
458}; 450};
459 451
460struct mlx5e_umr_dma_info { 452struct mlx5e_umr_dma_info {
461 __be64 *mtt;
462 dma_addr_t mtt_addr;
463 struct mlx5e_dma_info dma_info[MLX5_MPWRQ_PAGES_PER_WQE]; 453 struct mlx5e_dma_info dma_info[MLX5_MPWRQ_PAGES_PER_WQE];
464 struct mlx5e_umr_wqe wqe;
465}; 454};
466 455
467struct mlx5e_mpw_info { 456struct mlx5e_mpw_info {
468 struct mlx5e_umr_dma_info umr; 457 struct mlx5e_umr_dma_info umr;
469 u16 consumed_strides; 458 u16 consumed_strides;
470 u16 skbs_frags[MLX5_MPWRQ_PAGES_PER_WQE]; 459 DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
471}; 460};
472 461
473/* a single cache unit is capable to serve one napi call (for non-striding rq) 462/* a single cache unit is capable to serve one napi call (for non-striding rq)
@@ -484,9 +473,16 @@ struct mlx5e_page_cache {
484 473
485struct mlx5e_rq; 474struct mlx5e_rq;
486typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*); 475typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
476typedef struct sk_buff *
477(*mlx5e_fp_skb_from_cqe_mpwrq)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
478 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
487typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq); 479typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
488typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16); 480typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
489 481
482enum mlx5e_rq_flag {
483 MLX5E_RQ_FLAG_XDP_XMIT = BIT(0),
484};
485
490struct mlx5e_rq { 486struct mlx5e_rq {
491 /* data path */ 487 /* data path */
492 struct mlx5_wq_ll wq; 488 struct mlx5_wq_ll wq;
@@ -497,12 +493,12 @@ struct mlx5e_rq {
497 u32 frag_sz; /* max possible skb frag_sz */ 493 u32 frag_sz; /* max possible skb frag_sz */
498 union { 494 union {
499 bool page_reuse; 495 bool page_reuse;
500 bool xdp_xmit;
501 }; 496 };
502 } wqe; 497 } wqe;
503 struct { 498 struct {
499 struct mlx5e_umr_wqe umr_wqe;
504 struct mlx5e_mpw_info *info; 500 struct mlx5e_mpw_info *info;
505 void *mtt_no_align; 501 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
506 u16 num_strides; 502 u16 num_strides;
507 u8 log_stride_sz; 503 u8 log_stride_sz;
508 bool umr_in_progress; 504 bool umr_in_progress;
@@ -534,7 +530,9 @@ struct mlx5e_rq {
534 530
535 /* XDP */ 531 /* XDP */
536 struct bpf_prog *xdp_prog; 532 struct bpf_prog *xdp_prog;
533 unsigned int hw_mtu;
537 struct mlx5e_xdpsq xdpsq; 534 struct mlx5e_xdpsq xdpsq;
535 DECLARE_BITMAP(flags, 8);
538 536
539 /* control */ 537 /* control */
540 struct mlx5_wq_ctrl wq_ctrl; 538 struct mlx5_wq_ctrl wq_ctrl;
@@ -767,7 +765,6 @@ struct mlx5e_priv {
767 struct mlx5e_tir inner_indir_tir[MLX5E_NUM_INDIR_TIRS]; 765 struct mlx5e_tir inner_indir_tir[MLX5E_NUM_INDIR_TIRS];
768 struct mlx5e_tir direct_tir[MLX5E_MAX_NUM_CHANNELS]; 766 struct mlx5e_tir direct_tir[MLX5E_MAX_NUM_CHANNELS];
769 u32 tx_rates[MLX5E_MAX_NUM_SQS]; 767 u32 tx_rates[MLX5E_MAX_NUM_SQS];
770 int hard_mtu;
771 768
772 struct mlx5e_flow_steering fs; 769 struct mlx5e_flow_steering fs;
773 struct mlx5e_vxlan_db vxlan; 770 struct mlx5e_vxlan_db vxlan;
@@ -846,11 +843,12 @@ bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
846void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix); 843void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
847void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix); 844void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
848void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi); 845void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
849 846struct sk_buff *
850u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, 847mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
851 struct mlx5e_params *params); 848 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
852u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, 849struct sk_buff *
853 struct mlx5e_params *params); 850mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
851 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
854 852
855void mlx5e_update_stats(struct mlx5e_priv *priv); 853void mlx5e_update_stats(struct mlx5e_priv *priv);
856 854
@@ -981,11 +979,6 @@ static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
981 mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc); 979 mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
982} 980}
983 981
984static inline u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix)
985{
986 return wqe_ix * ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8);
987}
988
989extern const struct ethtool_ops mlx5e_ethtool_ops; 982extern const struct ethtool_ops mlx5e_ethtool_ops;
990#ifdef CONFIG_MLX5_CORE_EN_DCB 983#ifdef CONFIG_MLX5_CORE_EN_DCB
991extern const struct dcbnl_rtnl_ops mlx5e_dcbnl_ops; 984extern const struct dcbnl_rtnl_ops mlx5e_dcbnl_ops;
@@ -1111,7 +1104,7 @@ void mlx5e_detach_netdev(struct mlx5e_priv *priv);
1111void mlx5e_destroy_netdev(struct mlx5e_priv *priv); 1104void mlx5e_destroy_netdev(struct mlx5e_priv *priv);
1112void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, 1105void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
1113 struct mlx5e_params *params, 1106 struct mlx5e_params *params,
1114 u16 max_channels); 1107 u16 max_channels, u16 mtu);
1115u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev); 1108u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev);
1116void mlx5e_rx_dim_work(struct work_struct *work); 1109void mlx5e_rx_dim_work(struct work_struct *work);
1117#endif /* __MLX5_EN_H__ */ 1110#endif /* __MLX5_EN_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index c57c929d7973..a87d46bc2299 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -220,60 +220,12 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev,
220 mlx5e_ethtool_get_ethtool_stats(priv, stats, data); 220 mlx5e_ethtool_get_ethtool_stats(priv, stats, data);
221} 221}
222 222
223static u32 mlx5e_rx_wqes_to_packets(struct mlx5e_priv *priv, int rq_wq_type,
224 int num_wqe)
225{
226 int packets_per_wqe;
227 int stride_size;
228 int num_strides;
229 int wqe_size;
230
231 if (rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
232 return num_wqe;
233
234 stride_size = 1 << mlx5e_mpwqe_get_log_stride_size(priv->mdev, &priv->channels.params);
235 num_strides = 1 << mlx5e_mpwqe_get_log_num_strides(priv->mdev, &priv->channels.params);
236 wqe_size = stride_size * num_strides;
237
238 packets_per_wqe = wqe_size /
239 ALIGN(ETH_DATA_LEN, stride_size);
240 return (1 << (order_base_2(num_wqe * packets_per_wqe) - 1));
241}
242
243static u32 mlx5e_packets_to_rx_wqes(struct mlx5e_priv *priv, int rq_wq_type,
244 int num_packets)
245{
246 int packets_per_wqe;
247 int stride_size;
248 int num_strides;
249 int wqe_size;
250 int num_wqes;
251
252 if (rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
253 return num_packets;
254
255 stride_size = 1 << mlx5e_mpwqe_get_log_stride_size(priv->mdev, &priv->channels.params);
256 num_strides = 1 << mlx5e_mpwqe_get_log_num_strides(priv->mdev, &priv->channels.params);
257 wqe_size = stride_size * num_strides;
258
259 num_packets = (1 << order_base_2(num_packets));
260
261 packets_per_wqe = wqe_size /
262 ALIGN(ETH_DATA_LEN, stride_size);
263 num_wqes = DIV_ROUND_UP(num_packets, packets_per_wqe);
264 return 1 << (order_base_2(num_wqes));
265}
266
267void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, 223void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv,
268 struct ethtool_ringparam *param) 224 struct ethtool_ringparam *param)
269{ 225{
270 int rq_wq_type = priv->channels.params.rq_wq_type; 226 param->rx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE;
271
272 param->rx_max_pending = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
273 1 << mlx5_max_log_rq_size(rq_wq_type));
274 param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; 227 param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
275 param->rx_pending = mlx5e_rx_wqes_to_packets(priv, rq_wq_type, 228 param->rx_pending = 1 << priv->channels.params.log_rq_mtu_frames;
276 1 << priv->channels.params.log_rq_size);
277 param->tx_pending = 1 << priv->channels.params.log_sq_size; 229 param->tx_pending = 1 << priv->channels.params.log_sq_size;
278} 230}
279 231
@@ -288,13 +240,9 @@ static void mlx5e_get_ringparam(struct net_device *dev,
288int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv, 240int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
289 struct ethtool_ringparam *param) 241 struct ethtool_ringparam *param)
290{ 242{
291 int rq_wq_type = priv->channels.params.rq_wq_type;
292 struct mlx5e_channels new_channels = {}; 243 struct mlx5e_channels new_channels = {};
293 u32 rx_pending_wqes;
294 u32 min_rq_size;
295 u8 log_rq_size; 244 u8 log_rq_size;
296 u8 log_sq_size; 245 u8 log_sq_size;
297 u32 num_mtts;
298 int err = 0; 246 int err = 0;
299 247
300 if (param->rx_jumbo_pending) { 248 if (param->rx_jumbo_pending) {
@@ -308,23 +256,10 @@ int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
308 return -EINVAL; 256 return -EINVAL;
309 } 257 }
310 258
311 min_rq_size = mlx5e_rx_wqes_to_packets(priv, rq_wq_type, 259 if (param->rx_pending < (1 << MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE)) {
312 1 << mlx5_min_log_rq_size(rq_wq_type));
313 rx_pending_wqes = mlx5e_packets_to_rx_wqes(priv, rq_wq_type,
314 param->rx_pending);
315
316 if (param->rx_pending < min_rq_size) {
317 netdev_info(priv->netdev, "%s: rx_pending (%d) < min (%d)\n", 260 netdev_info(priv->netdev, "%s: rx_pending (%d) < min (%d)\n",
318 __func__, param->rx_pending, 261 __func__, param->rx_pending,
319 min_rq_size); 262 1 << MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE);
320 return -EINVAL;
321 }
322
323 num_mtts = MLX5E_REQUIRED_MTTS(rx_pending_wqes);
324 if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
325 !MLX5E_VALID_NUM_MTTS(num_mtts)) {
326 netdev_info(priv->netdev, "%s: rx_pending (%d) request can't be satisfied, try to reduce.\n",
327 __func__, param->rx_pending);
328 return -EINVAL; 263 return -EINVAL;
329 } 264 }
330 265
@@ -335,17 +270,17 @@ int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
335 return -EINVAL; 270 return -EINVAL;
336 } 271 }
337 272
338 log_rq_size = order_base_2(rx_pending_wqes); 273 log_rq_size = order_base_2(param->rx_pending);
339 log_sq_size = order_base_2(param->tx_pending); 274 log_sq_size = order_base_2(param->tx_pending);
340 275
341 if (log_rq_size == priv->channels.params.log_rq_size && 276 if (log_rq_size == priv->channels.params.log_rq_mtu_frames &&
342 log_sq_size == priv->channels.params.log_sq_size) 277 log_sq_size == priv->channels.params.log_sq_size)
343 return 0; 278 return 0;
344 279
345 mutex_lock(&priv->state_lock); 280 mutex_lock(&priv->state_lock);
346 281
347 new_channels.params = priv->channels.params; 282 new_channels.params = priv->channels.params;
348 new_channels.params.log_rq_size = log_rq_size; 283 new_channels.params.log_rq_mtu_frames = log_rq_size;
349 new_channels.params.log_sq_size = log_sq_size; 284 new_channels.params.log_sq_size = log_sq_size;
350 285
351 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 286 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 1b48dec67abf..0339609cfa56 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -73,26 +73,89 @@ struct mlx5e_channel_param {
73 73
74bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) 74bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
75{ 75{
76 return MLX5_CAP_GEN(mdev, striding_rq) && 76 bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) &&
77 MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 77 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
78 MLX5_CAP_ETH(mdev, reg_umr_sq); 78 MLX5_CAP_ETH(mdev, reg_umr_sq);
79 u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq);
80 bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap;
81
82 if (!striding_rq_umr)
83 return false;
84 if (!inline_umr) {
85 mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n",
86 (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap);
87 return false;
88 }
89 return true;
90}
91
92static u32 mlx5e_mpwqe_get_linear_frag_sz(struct mlx5e_params *params)
93{
94 if (!params->xdp_prog) {
95 u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
96 u16 rq_headroom = MLX5_RX_HEADROOM + NET_IP_ALIGN;
97
98 return MLX5_SKB_FRAG_SZ(rq_headroom + hw_mtu);
99 }
100
101 return PAGE_SIZE;
79} 102}
80 103
81u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, 104static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
82 struct mlx5e_params *params)
83{ 105{
106 u32 linear_frag_sz = mlx5e_mpwqe_get_linear_frag_sz(params);
107
108 return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
109}
110
111static bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
112 struct mlx5e_params *params)
113{
114 u32 frag_sz = mlx5e_mpwqe_get_linear_frag_sz(params);
115 s8 signed_log_num_strides_param;
116 u8 log_num_strides;
117
118 if (params->lro_en || frag_sz > PAGE_SIZE)
119 return false;
120
121 if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
122 return true;
123
124 log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);
125 signed_log_num_strides_param =
126 (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
127
128 return signed_log_num_strides_param >= 0;
129}
130
131static u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)
132{
133 if (params->log_rq_mtu_frames <
134 mlx5e_mpwqe_log_pkts_per_wqe(params) + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
135 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
136
137 return params->log_rq_mtu_frames - mlx5e_mpwqe_log_pkts_per_wqe(params);
138}
139
140static u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
141 struct mlx5e_params *params)
142{
143 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
144 return order_base_2(mlx5e_mpwqe_get_linear_frag_sz(params));
145
84 return MLX5E_MPWQE_STRIDE_SZ(mdev, 146 return MLX5E_MPWQE_STRIDE_SZ(mdev,
85 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); 147 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
86} 148}
87 149
88u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, 150static u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
89 struct mlx5e_params *params) 151 struct mlx5e_params *params)
90{ 152{
91 return MLX5_MPWRQ_LOG_WQE_SZ - 153 return MLX5_MPWRQ_LOG_WQE_SZ -
92 mlx5e_mpwqe_get_log_stride_size(mdev, params); 154 mlx5e_mpwqe_get_log_stride_size(mdev, params);
93} 155}
94 156
95static u16 mlx5e_get_rq_headroom(struct mlx5e_params *params) 157static u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
158 struct mlx5e_params *params)
96{ 159{
97 u16 linear_rq_headroom = params->xdp_prog ? 160 u16 linear_rq_headroom = params->xdp_prog ?
98 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM; 161 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
@@ -102,6 +165,9 @@ static u16 mlx5e_get_rq_headroom(struct mlx5e_params *params)
102 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST) 165 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST)
103 return linear_rq_headroom; 166 return linear_rq_headroom;
104 167
168 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
169 return linear_rq_headroom;
170
105 return 0; 171 return 0;
106} 172}
107 173
@@ -109,25 +175,23 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
109 struct mlx5e_params *params) 175 struct mlx5e_params *params)
110{ 176{
111 params->lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ; 177 params->lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
178 params->log_rq_mtu_frames = is_kdump_kernel() ?
179 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
180 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
112 switch (params->rq_wq_type) { 181 switch (params->rq_wq_type) {
113 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 182 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
114 params->log_rq_size = is_kdump_kernel() ?
115 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW :
116 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
117 break; 183 break;
118 default: /* MLX5_WQ_TYPE_LINKED_LIST */ 184 default: /* MLX5_WQ_TYPE_LINKED_LIST */
119 params->log_rq_size = is_kdump_kernel() ?
120 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
121 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
122
123 /* Extra room needed for build_skb */ 185 /* Extra room needed for build_skb */
124 params->lro_wqe_sz -= mlx5e_get_rq_headroom(params) + 186 params->lro_wqe_sz -= mlx5e_get_rq_headroom(mdev, params) +
125 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 187 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
126 } 188 }
127 189
128 mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n", 190 mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
129 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, 191 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
130 BIT(params->log_rq_size), 192 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
193 BIT(mlx5e_mpwqe_get_log_rq_size(params)) :
194 BIT(params->log_rq_mtu_frames),
131 BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params)), 195 BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params)),
132 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); 196 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
133} 197}
@@ -136,7 +200,8 @@ bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
136 struct mlx5e_params *params) 200 struct mlx5e_params *params)
137{ 201{
138 return mlx5e_check_fragmented_striding_rq_cap(mdev) && 202 return mlx5e_check_fragmented_striding_rq_cap(mdev) &&
139 !params->xdp_prog && !MLX5_IPSEC_DEV(mdev); 203 !MLX5_IPSEC_DEV(mdev) &&
204 !(params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params));
140} 205}
141 206
142void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 207void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
@@ -239,107 +304,38 @@ static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
239 synchronize_irq(pci_irq_vector(priv->mdev->pdev, MLX5_EQ_VEC_ASYNC)); 304 synchronize_irq(pci_irq_vector(priv->mdev->pdev, MLX5_EQ_VEC_ASYNC));
240} 305}
241 306
242static inline int mlx5e_get_wqe_mtt_sz(void)
243{
244 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
245 * To avoid copying garbage after the mtt array, we allocate
246 * a little more.
247 */
248 return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
249 MLX5_UMR_MTT_ALIGNMENT);
250}
251
252static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 307static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
253 struct mlx5e_icosq *sq, 308 struct mlx5e_icosq *sq,
254 struct mlx5e_umr_wqe *wqe, 309 struct mlx5e_umr_wqe *wqe)
255 u16 ix)
256{ 310{
257 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 311 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
258 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; 312 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
259 struct mlx5_wqe_data_seg *dseg = &wqe->data; 313 u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS);
260 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
261 u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
262 u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
263 314
264 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 315 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
265 ds_cnt); 316 ds_cnt);
266 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 317 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
267 cseg->imm = rq->mkey_be; 318 cseg->imm = rq->mkey_be;
268 319
269 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN; 320 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
270 ucseg->xlt_octowords = 321 ucseg->xlt_octowords =
271 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE)); 322 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
272 ucseg->bsf_octowords =
273 cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
274 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 323 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
275
276 dseg->lkey = sq->mkey_be;
277 dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
278} 324}
279 325
280static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, 326static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
281 struct mlx5e_channel *c) 327 struct mlx5e_channel *c)
282{ 328{
283 int wq_sz = mlx5_wq_ll_get_size(&rq->wq); 329 int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
284 int mtt_sz = mlx5e_get_wqe_mtt_sz();
285 int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
286 int i;
287 330
288 rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info), 331 rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info),
289 GFP_KERNEL, cpu_to_node(c->cpu)); 332 GFP_KERNEL, cpu_to_node(c->cpu));
290 if (!rq->mpwqe.info) 333 if (!rq->mpwqe.info)
291 goto err_out; 334 return -ENOMEM;
292
293 /* We allocate more than mtt_sz as we will align the pointer */
294 rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
295 cpu_to_node(c->cpu));
296 if (unlikely(!rq->mpwqe.mtt_no_align))
297 goto err_free_wqe_info;
298
299 for (i = 0; i < wq_sz; i++) {
300 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
301
302 wi->umr.mtt = PTR_ALIGN(rq->mpwqe.mtt_no_align + i * mtt_alloc,
303 MLX5_UMR_ALIGN);
304 wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
305 PCI_DMA_TODEVICE);
306 if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
307 goto err_unmap_mtts;
308 335
309 mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i); 336 mlx5e_build_umr_wqe(rq, &c->icosq, &rq->mpwqe.umr_wqe);
310 }
311 337
312 return 0; 338 return 0;
313
314err_unmap_mtts:
315 while (--i >= 0) {
316 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
317
318 dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
319 PCI_DMA_TODEVICE);
320 }
321 kfree(rq->mpwqe.mtt_no_align);
322err_free_wqe_info:
323 kfree(rq->mpwqe.info);
324
325err_out:
326 return -ENOMEM;
327}
328
329static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
330{
331 int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
332 int mtt_sz = mlx5e_get_wqe_mtt_sz();
333 int i;
334
335 for (i = 0; i < wq_sz; i++) {
336 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
337
338 dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
339 PCI_DMA_TODEVICE);
340 }
341 kfree(rq->mpwqe.mtt_no_align);
342 kfree(rq->mpwqe.info);
343} 339}
344 340
345static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 341static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
@@ -351,9 +347,6 @@ static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
351 u32 *in; 347 u32 *in;
352 int err; 348 int err;
353 349
354 if (!MLX5E_VALID_NUM_MTTS(npages))
355 return -EINVAL;
356
357 in = kvzalloc(inlen, GFP_KERNEL); 350 in = kvzalloc(inlen, GFP_KERNEL);
358 if (!in) 351 if (!in)
359 return -ENOMEM; 352 return -ENOMEM;
@@ -386,6 +379,11 @@ static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq
386 return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey); 379 return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey);
387} 380}
388 381
382static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix)
383{
384 return (wqe_ix << MLX5E_LOG_ALIGNED_MPWQE_PPW) << PAGE_SHIFT;
385}
386
389static int mlx5e_alloc_rq(struct mlx5e_channel *c, 387static int mlx5e_alloc_rq(struct mlx5e_channel *c,
390 struct mlx5e_params *params, 388 struct mlx5e_params *params,
391 struct mlx5e_rq_param *rqp, 389 struct mlx5e_rq_param *rqp,
@@ -419,6 +417,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
419 rq->channel = c; 417 rq->channel = c;
420 rq->ix = c->ix; 418 rq->ix = c->ix;
421 rq->mdev = mdev; 419 rq->mdev = mdev;
420 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
422 421
423 rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL; 422 rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
424 if (IS_ERR(rq->xdp_prog)) { 423 if (IS_ERR(rq->xdp_prog)) {
@@ -432,11 +431,10 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
432 goto err_rq_wq_destroy; 431 goto err_rq_wq_destroy;
433 432
434 rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 433 rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
435 rq->buff.headroom = mlx5e_get_rq_headroom(params); 434 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params);
436 435
437 switch (rq->wq_type) { 436 switch (rq->wq_type) {
438 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 437 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
439
440 rq->post_wqes = mlx5e_post_rx_mpwqes; 438 rq->post_wqes = mlx5e_post_rx_mpwqes;
441 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe; 439 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
442 440
@@ -454,6 +452,10 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
454 goto err_rq_wq_destroy; 452 goto err_rq_wq_destroy;
455 } 453 }
456 454
455 rq->mpwqe.skb_from_cqe_mpwrq =
456 mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ?
457 mlx5e_skb_from_cqe_mpwrq_linear :
458 mlx5e_skb_from_cqe_mpwrq_nonlinear;
457 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params); 459 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params);
458 rq->mpwqe.num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params)); 460 rq->mpwqe.num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params));
459 461
@@ -494,7 +496,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
494 496
495 byte_count = params->lro_en ? 497 byte_count = params->lro_en ?
496 params->lro_wqe_sz : 498 params->lro_wqe_sz :
497 MLX5E_SW2HW_MTU(c->priv, c->netdev->mtu); 499 MLX5E_SW2HW_MTU(params, params->sw_mtu);
498#ifdef CONFIG_MLX5_EN_IPSEC 500#ifdef CONFIG_MLX5_EN_IPSEC
499 if (MLX5_IPSEC_DEV(mdev)) 501 if (MLX5_IPSEC_DEV(mdev))
500 byte_count += MLX5E_METADATA_ETHER_LEN; 502 byte_count += MLX5E_METADATA_ETHER_LEN;
@@ -514,9 +516,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
514 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i); 516 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
515 517
516 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 518 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
517 u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, i) << PAGE_SHIFT; 519 u64 dma_offset = mlx5e_get_mpwqe_offset(rq, i);
518 520
519 wqe->data.addr = cpu_to_be64(dma_offset); 521 wqe->data.addr = cpu_to_be64(dma_offset + rq->buff.headroom);
520 } 522 }
521 523
522 wqe->data.byte_count = cpu_to_be32(byte_count); 524 wqe->data.byte_count = cpu_to_be32(byte_count);
@@ -562,7 +564,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
562 564
563 switch (rq->wq_type) { 565 switch (rq->wq_type) {
564 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 566 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
565 mlx5e_rq_free_mpwqe_info(rq); 567 kfree(rq->mpwqe.info);
566 mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey); 568 mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
567 break; 569 break;
568 default: /* MLX5_WQ_TYPE_LINKED_LIST */ 570 default: /* MLX5_WQ_TYPE_LINKED_LIST */
@@ -901,7 +903,6 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
901 struct mlx5_core_dev *mdev = c->mdev; 903 struct mlx5_core_dev *mdev = c->mdev;
902 int err; 904 int err;
903 905
904 sq->mkey_be = c->mkey_be;
905 sq->channel = c; 906 sq->channel = c;
906 sq->uar_map = mdev->mlx5e_res.bfreg.map; 907 sq->uar_map = mdev->mlx5e_res.bfreg.map;
907 908
@@ -1867,18 +1868,21 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
1867 switch (params->rq_wq_type) { 1868 switch (params->rq_wq_type) {
1868 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1869 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1869 MLX5_SET(wq, wq, log_wqe_num_of_strides, 1870 MLX5_SET(wq, wq, log_wqe_num_of_strides,
1870 mlx5e_mpwqe_get_log_num_strides(mdev, params) - 9); 1871 mlx5e_mpwqe_get_log_num_strides(mdev, params) -
1872 MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
1871 MLX5_SET(wq, wq, log_wqe_stride_size, 1873 MLX5_SET(wq, wq, log_wqe_stride_size,
1872 mlx5e_mpwqe_get_log_stride_size(mdev, params) - 6); 1874 mlx5e_mpwqe_get_log_stride_size(mdev, params) -
1875 MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
1873 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ); 1876 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
1877 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params));
1874 break; 1878 break;
1875 default: /* MLX5_WQ_TYPE_LINKED_LIST */ 1879 default: /* MLX5_WQ_TYPE_LINKED_LIST */
1876 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST); 1880 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1881 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
1877 } 1882 }
1878 1883
1879 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN); 1884 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1880 MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe))); 1885 MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
1881 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_size);
1882 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.pdn); 1886 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.pdn);
1883 MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter); 1887 MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
1884 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); 1888 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable);
@@ -1938,16 +1942,17 @@ static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1938 struct mlx5e_params *params, 1942 struct mlx5e_params *params,
1939 struct mlx5e_cq_param *param) 1943 struct mlx5e_cq_param *param)
1940{ 1944{
1945 struct mlx5_core_dev *mdev = priv->mdev;
1941 void *cqc = param->cqc; 1946 void *cqc = param->cqc;
1942 u8 log_cq_size; 1947 u8 log_cq_size;
1943 1948
1944 switch (params->rq_wq_type) { 1949 switch (params->rq_wq_type) {
1945 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1950 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1946 log_cq_size = params->log_rq_size + 1951 log_cq_size = mlx5e_mpwqe_get_log_rq_size(params) +
1947 mlx5e_mpwqe_get_log_num_strides(priv->mdev, params); 1952 mlx5e_mpwqe_get_log_num_strides(mdev, params);
1948 break; 1953 break;
1949 default: /* MLX5_WQ_TYPE_LINKED_LIST */ 1954 default: /* MLX5_WQ_TYPE_LINKED_LIST */
1950 log_cq_size = params->log_rq_size; 1955 log_cq_size = params->log_rq_mtu_frames;
1951 } 1956 }
1952 1957
1953 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size); 1958 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
@@ -2498,10 +2503,10 @@ static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv,
2498 mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, true); 2503 mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, true);
2499} 2504}
2500 2505
2501static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu) 2506static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2507 struct mlx5e_params *params, u16 mtu)
2502{ 2508{
2503 struct mlx5_core_dev *mdev = priv->mdev; 2509 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
2504 u16 hw_mtu = MLX5E_SW2HW_MTU(priv, mtu);
2505 int err; 2510 int err;
2506 2511
2507 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 2512 err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
@@ -2513,9 +2518,9 @@ static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
2513 return 0; 2518 return 0;
2514} 2519}
2515 2520
2516static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu) 2521static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2522 struct mlx5e_params *params, u16 *mtu)
2517{ 2523{
2518 struct mlx5_core_dev *mdev = priv->mdev;
2519 u16 hw_mtu = 0; 2524 u16 hw_mtu = 0;
2520 int err; 2525 int err;
2521 2526
@@ -2523,25 +2528,27 @@ static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
2523 if (err || !hw_mtu) /* fallback to port oper mtu */ 2528 if (err || !hw_mtu) /* fallback to port oper mtu */
2524 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 2529 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2525 2530
2526 *mtu = MLX5E_HW2SW_MTU(priv, hw_mtu); 2531 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
2527} 2532}
2528 2533
2529static int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 2534static int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
2530{ 2535{
2536 struct mlx5e_params *params = &priv->channels.params;
2531 struct net_device *netdev = priv->netdev; 2537 struct net_device *netdev = priv->netdev;
2538 struct mlx5_core_dev *mdev = priv->mdev;
2532 u16 mtu; 2539 u16 mtu;
2533 int err; 2540 int err;
2534 2541
2535 err = mlx5e_set_mtu(priv, netdev->mtu); 2542 err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
2536 if (err) 2543 if (err)
2537 return err; 2544 return err;
2538 2545
2539 mlx5e_query_mtu(priv, &mtu); 2546 mlx5e_query_mtu(mdev, params, &mtu);
2540 if (mtu != netdev->mtu) 2547 if (mtu != params->sw_mtu)
2541 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 2548 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2542 __func__, mtu, netdev->mtu); 2549 __func__, mtu, params->sw_mtu);
2543 2550
2544 netdev->mtu = mtu; 2551 params->sw_mtu = mtu;
2545 return 0; 2552 return 0;
2546} 2553}
2547 2554
@@ -3222,20 +3229,28 @@ typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3222static int set_feature_lro(struct net_device *netdev, bool enable) 3229static int set_feature_lro(struct net_device *netdev, bool enable)
3223{ 3230{
3224 struct mlx5e_priv *priv = netdev_priv(netdev); 3231 struct mlx5e_priv *priv = netdev_priv(netdev);
3232 struct mlx5_core_dev *mdev = priv->mdev;
3225 struct mlx5e_channels new_channels = {}; 3233 struct mlx5e_channels new_channels = {};
3234 struct mlx5e_params *old_params;
3226 int err = 0; 3235 int err = 0;
3227 bool reset; 3236 bool reset;
3228 3237
3229 mutex_lock(&priv->state_lock); 3238 mutex_lock(&priv->state_lock);
3230 3239
3231 reset = (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST); 3240 old_params = &priv->channels.params;
3232 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state); 3241 reset = test_bit(MLX5E_STATE_OPENED, &priv->state);
3233 3242
3234 new_channels.params = priv->channels.params; 3243 new_channels.params = *old_params;
3235 new_channels.params.lro_en = enable; 3244 new_channels.params.lro_en = enable;
3236 3245
3246 if (old_params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST) {
3247 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, old_params) ==
3248 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params))
3249 reset = false;
3250 }
3251
3237 if (!reset) { 3252 if (!reset) {
3238 priv->channels.params = new_channels.params; 3253 *old_params = new_channels.params;
3239 err = mlx5e_modify_tirs_lro(priv); 3254 err = mlx5e_modify_tirs_lro(priv);
3240 goto out; 3255 goto out;
3241 } 3256 }
@@ -3411,34 +3426,40 @@ static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
3411{ 3426{
3412 struct mlx5e_priv *priv = netdev_priv(netdev); 3427 struct mlx5e_priv *priv = netdev_priv(netdev);
3413 struct mlx5e_channels new_channels = {}; 3428 struct mlx5e_channels new_channels = {};
3414 int curr_mtu; 3429 struct mlx5e_params *params;
3415 int err = 0; 3430 int err = 0;
3416 bool reset; 3431 bool reset;
3417 3432
3418 mutex_lock(&priv->state_lock); 3433 mutex_lock(&priv->state_lock);
3419 3434
3420 reset = !priv->channels.params.lro_en && 3435 params = &priv->channels.params;
3421 (priv->channels.params.rq_wq_type !=
3422 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
3423 3436
3437 reset = !params->lro_en;
3424 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state); 3438 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state);
3425 3439
3426 curr_mtu = netdev->mtu; 3440 new_channels.params = *params;
3427 netdev->mtu = new_mtu; 3441 new_channels.params.sw_mtu = new_mtu;
3442
3443 if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST) {
3444 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params);
3445 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params);
3446
3447 reset = reset && (ppw_old != ppw_new);
3448 }
3428 3449
3429 if (!reset) { 3450 if (!reset) {
3451 params->sw_mtu = new_mtu;
3430 mlx5e_set_dev_port_mtu(priv); 3452 mlx5e_set_dev_port_mtu(priv);
3453 netdev->mtu = params->sw_mtu;
3431 goto out; 3454 goto out;
3432 } 3455 }
3433 3456
3434 new_channels.params = priv->channels.params;
3435 err = mlx5e_open_channels(priv, &new_channels); 3457 err = mlx5e_open_channels(priv, &new_channels);
3436 if (err) { 3458 if (err)
3437 netdev->mtu = curr_mtu;
3438 goto out; 3459 goto out;
3439 }
3440 3460
3441 mlx5e_switch_priv_channels(priv, &new_channels, mlx5e_set_dev_port_mtu); 3461 mlx5e_switch_priv_channels(priv, &new_channels, mlx5e_set_dev_port_mtu);
3462 netdev->mtu = new_channels.params.sw_mtu;
3442 3463
3443out: 3464out:
3444 mutex_unlock(&priv->state_lock); 3465 mutex_unlock(&priv->state_lock);
@@ -3728,21 +3749,11 @@ static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
3728static bool mlx5e_tx_timeout_eq_recover(struct net_device *dev, 3749static bool mlx5e_tx_timeout_eq_recover(struct net_device *dev,
3729 struct mlx5e_txqsq *sq) 3750 struct mlx5e_txqsq *sq)
3730{ 3751{
3731 struct mlx5e_priv *priv = netdev_priv(dev); 3752 struct mlx5_eq *eq = sq->cq.mcq.eq;
3732 struct mlx5_core_dev *mdev = priv->mdev;
3733 int irqn_not_used, eqn;
3734 struct mlx5_eq *eq;
3735 u32 eqe_count; 3753 u32 eqe_count;
3736 3754
3737 if (mlx5_vector2eqn(mdev, sq->cq.mcq.vector, &eqn, &irqn_not_used))
3738 return false;
3739
3740 eq = mlx5_eqn2eq(mdev, eqn);
3741 if (IS_ERR(eq))
3742 return false;
3743
3744 netdev_err(dev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n", 3755 netdev_err(dev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n",
3745 eqn, eq->cons_index, eq->irqn); 3756 eq->eqn, eq->cons_index, eq->irqn);
3746 3757
3747 eqe_count = mlx5_eq_poll_irq_disabled(eq); 3758 eqe_count = mlx5_eq_poll_irq_disabled(eq);
3748 if (!eqe_count) 3759 if (!eqe_count)
@@ -4121,10 +4132,12 @@ static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeo
4121 4132
4122void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, 4133void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
4123 struct mlx5e_params *params, 4134 struct mlx5e_params *params,
4124 u16 max_channels) 4135 u16 max_channels, u16 mtu)
4125{ 4136{
4126 u8 cq_period_mode = 0; 4137 u8 cq_period_mode = 0;
4127 4138
4139 params->sw_mtu = mtu;
4140 params->hard_mtu = MLX5E_ETH_HARD_MTU;
4128 params->num_channels = max_channels; 4141 params->num_channels = max_channels;
4129 params->num_tc = 1; 4142 params->num_tc = 1;
4130 4143
@@ -4152,7 +4165,8 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
4152 4165
4153 /* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */ 4166 /* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */
4154 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 4167 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
4155 params->lro_en = !slow_pci_heuristic(mdev); 4168 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
4169 params->lro_en = !slow_pci_heuristic(mdev);
4156 params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 4170 params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
4157 4171
4158 /* CQ moderation params */ 4172 /* CQ moderation params */
@@ -4185,9 +4199,9 @@ static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
4185 priv->profile = profile; 4199 priv->profile = profile;
4186 priv->ppriv = ppriv; 4200 priv->ppriv = ppriv;
4187 priv->msglevel = MLX5E_MSG_LEVEL; 4201 priv->msglevel = MLX5E_MSG_LEVEL;
4188 priv->hard_mtu = MLX5E_ETH_HARD_MTU;
4189 4202
4190 mlx5e_build_nic_params(mdev, &priv->channels.params, profile->max_nch(mdev)); 4203 mlx5e_build_nic_params(mdev, &priv->channels.params,
4204 profile->max_nch(mdev), netdev->mtu);
4191 4205
4192 mutex_init(&priv->state_lock); 4206 mutex_init(&priv->state_lock);
4193 4207
@@ -4464,7 +4478,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)
4464 /* MTU range: 68 - hw-specific max */ 4478 /* MTU range: 68 - hw-specific max */
4465 netdev->min_mtu = ETH_MIN_MTU; 4479 netdev->min_mtu = ETH_MIN_MTU;
4466 mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1); 4480 mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1);
4467 netdev->max_mtu = MLX5E_HW2SW_MTU(priv, max_mtu); 4481 netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu);
4468 mlx5e_set_dev_port_mtu(priv); 4482 mlx5e_set_dev_port_mtu(priv);
4469 4483
4470 mlx5_lag_add(mdev, netdev); 4484 mlx5_lag_add(mdev, netdev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index dd32f3e390ff..8e70fa9ef39a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -877,9 +877,10 @@ static void mlx5e_build_rep_params(struct mlx5_core_dev *mdev,
877 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 877 MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
878 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 878 MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
879 879
880 params->hard_mtu = MLX5E_ETH_HARD_MTU;
880 params->log_sq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; 881 params->log_sq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
881 params->rq_wq_type = MLX5_WQ_TYPE_LINKED_LIST; 882 params->rq_wq_type = MLX5_WQ_TYPE_LINKED_LIST;
882 params->log_rq_size = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE; 883 params->log_rq_mtu_frames = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE;
883 884
884 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 885 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
885 mlx5e_set_rx_cq_mode_params(params, cq_period_mode); 886 mlx5e_set_rx_cq_mode_params(params, cq_period_mode);
@@ -926,8 +927,6 @@ static void mlx5e_init_rep(struct mlx5_core_dev *mdev,
926 927
927 priv->channels.params.num_channels = profile->max_nch(mdev); 928 priv->channels.params.num_channels = profile->max_nch(mdev);
928 929
929 priv->hard_mtu = MLX5E_ETH_HARD_MTU;
930
931 mlx5e_build_rep_params(mdev, &priv->channels.params); 930 mlx5e_build_rep_params(mdev, &priv->channels.params);
932 mlx5e_build_rep_netdev(netdev); 931 mlx5e_build_rep_netdev(netdev);
933 932
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 781b8f21d6d1..176645762e49 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -296,37 +296,28 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
296 mlx5e_free_rx_wqe(rq, wi); 296 mlx5e_free_rx_wqe(rq, wi);
297} 297}
298 298
299static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq)
300{
301 return rq->mpwqe.num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
302}
303
304static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq, 299static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
305 struct sk_buff *skb, 300 struct sk_buff *skb,
306 struct mlx5e_mpw_info *wi, 301 struct mlx5e_dma_info *di,
307 u32 page_idx, u32 frag_offset, 302 u32 frag_offset, u32 len)
308 u32 len)
309{ 303{
310 unsigned int truesize = ALIGN(len, BIT(rq->mpwqe.log_stride_sz)); 304 unsigned int truesize = ALIGN(len, BIT(rq->mpwqe.log_stride_sz));
311 305
312 dma_sync_single_for_cpu(rq->pdev, 306 dma_sync_single_for_cpu(rq->pdev,
313 wi->umr.dma_info[page_idx].addr + frag_offset, 307 di->addr + frag_offset,
314 len, DMA_FROM_DEVICE); 308 len, DMA_FROM_DEVICE);
315 wi->skbs_frags[page_idx]++; 309 page_ref_inc(di->page);
316 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 310 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
317 wi->umr.dma_info[page_idx].page, frag_offset, 311 di->page, frag_offset, len, truesize);
318 len, truesize);
319} 312}
320 313
321static inline void 314static inline void
322mlx5e_copy_skb_header_mpwqe(struct device *pdev, 315mlx5e_copy_skb_header_mpwqe(struct device *pdev,
323 struct sk_buff *skb, 316 struct sk_buff *skb,
324 struct mlx5e_mpw_info *wi, 317 struct mlx5e_dma_info *dma_info,
325 u32 page_idx, u32 offset, 318 u32 offset, u32 headlen)
326 u32 headlen)
327{ 319{
328 u16 headlen_pg = min_t(u32, headlen, PAGE_SIZE - offset); 320 u16 headlen_pg = min_t(u32, headlen, PAGE_SIZE - offset);
329 struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[page_idx];
330 unsigned int len; 321 unsigned int len;
331 322
332 /* Aligning len to sizeof(long) optimizes memcpy performance */ 323 /* Aligning len to sizeof(long) optimizes memcpy performance */
@@ -347,14 +338,49 @@ mlx5e_copy_skb_header_mpwqe(struct device *pdev,
347 } 338 }
348} 339}
349 340
350static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix) 341void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
342{
343 const bool no_xdp_xmit =
344 bitmap_empty(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
345 struct mlx5e_dma_info *dma_info = wi->umr.dma_info;
346 int i;
347
348 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
349 if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
350 mlx5e_page_release(rq, &dma_info[i], true);
351}
352
353static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
354{
355 struct mlx5_wq_ll *wq = &rq->wq;
356 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
357
358 rq->mpwqe.umr_in_progress = false;
359
360 mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
361
362 /* ensure wqes are visible to device before updating doorbell record */
363 dma_wmb();
364
365 mlx5_wq_ll_update_db_record(wq);
366}
367
368static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq)
369{
370 return sq->pc >> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
371}
372
373static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
351{ 374{
352 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix]; 375 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
376 struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
353 struct mlx5e_icosq *sq = &rq->channel->icosq; 377 struct mlx5e_icosq *sq = &rq->channel->icosq;
354 struct mlx5_wq_cyc *wq = &sq->wq; 378 struct mlx5_wq_cyc *wq = &sq->wq;
355 struct mlx5e_umr_wqe *wqe; 379 struct mlx5e_umr_wqe *umr_wqe;
356 u8 num_wqebbs = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_BB); 380 u16 xlt_offset = ix << (MLX5E_LOG_ALIGNED_MPWQE_PPW - 1);
381 int err;
357 u16 pi; 382 u16 pi;
383 int i;
358 384
359 /* fill sq edge with nops to avoid wqe wrap around */ 385 /* fill sq edge with nops to avoid wqe wrap around */
360 while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) { 386 while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
@@ -362,90 +388,44 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
362 mlx5e_post_nop(wq, sq->sqn, &sq->pc); 388 mlx5e_post_nop(wq, sq->sqn, &sq->pc);
363 } 389 }
364 390
365 wqe = mlx5_wq_cyc_get_wqe(wq, pi); 391 umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
366 memcpy(wqe, &wi->umr.wqe, sizeof(*wqe)); 392 if (unlikely(mlx5e_icosq_wrap_cnt(sq) < 2))
367 wqe->ctrl.opmod_idx_opcode = 393 memcpy(umr_wqe, &rq->mpwqe.umr_wqe,
368 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | 394 offsetof(struct mlx5e_umr_wqe, inline_mtts));
369 MLX5_OPCODE_UMR);
370
371 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
372 sq->pc += num_wqebbs;
373 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl);
374}
375
376static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
377 u16 ix)
378{
379 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
380 int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
381 struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
382 int err;
383 int i;
384 395
385 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) { 396 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
386 err = mlx5e_page_alloc_mapped(rq, dma_info); 397 err = mlx5e_page_alloc_mapped(rq, dma_info);
387 if (unlikely(err)) 398 if (unlikely(err))
388 goto err_unmap; 399 goto err_unmap;
389 wi->umr.mtt[i] = cpu_to_be64(dma_info->addr | MLX5_EN_WR); 400 umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
390 page_ref_add(dma_info->page, pg_strides);
391 } 401 }
392 402
393 memset(wi->skbs_frags, 0, sizeof(*wi->skbs_frags) * MLX5_MPWRQ_PAGES_PER_WQE); 403 bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
394 wi->consumed_strides = 0; 404 wi->consumed_strides = 0;
395 405
406 rq->mpwqe.umr_in_progress = true;
407
408 umr_wqe->ctrl.opmod_idx_opcode =
409 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
410 MLX5_OPCODE_UMR);
411 umr_wqe->uctrl.xlt_offset = cpu_to_be16(xlt_offset);
412
413 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
414 sq->pc += MLX5E_UMR_WQEBBS;
415 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &umr_wqe->ctrl);
416
396 return 0; 417 return 0;
397 418
398err_unmap: 419err_unmap:
399 while (--i >= 0) { 420 while (--i >= 0) {
400 dma_info--; 421 dma_info--;
401 page_ref_sub(dma_info->page, pg_strides);
402 mlx5e_page_release(rq, dma_info, true); 422 mlx5e_page_release(rq, dma_info, true);
403 } 423 }
424 rq->stats.buff_alloc_err++;
404 425
405 return err; 426 return err;
406} 427}
407 428
408void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
409{
410 int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
411 struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
412 int i;
413
414 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
415 page_ref_sub(dma_info->page, pg_strides - wi->skbs_frags[i]);
416 mlx5e_page_release(rq, dma_info, true);
417 }
418}
419
420static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
421{
422 struct mlx5_wq_ll *wq = &rq->wq;
423 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
424
425 rq->mpwqe.umr_in_progress = false;
426
427 mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
428
429 /* ensure wqes are visible to device before updating doorbell record */
430 dma_wmb();
431
432 mlx5_wq_ll_update_db_record(wq);
433}
434
435static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
436{
437 int err;
438
439 err = mlx5e_alloc_rx_umr_mpwqe(rq, ix);
440 if (unlikely(err)) {
441 rq->stats.buff_alloc_err++;
442 return err;
443 }
444 rq->mpwqe.umr_in_progress = true;
445 mlx5e_post_umr_wqe(rq, ix);
446 return 0;
447}
448
449void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 429void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
450{ 430{
451 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix]; 431 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
@@ -544,7 +524,7 @@ bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
544 if (!rq->mpwqe.umr_in_progress) 524 if (!rq->mpwqe.umr_in_progress)
545 mlx5e_alloc_rx_mpwqe(rq, wq->head); 525 mlx5e_alloc_rx_mpwqe(rq, wq->head);
546 526
547 return true; 527 return false;
548} 528}
549 529
550static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64 *cqe, struct tcphdr *tcp) 530static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64 *cqe, struct tcphdr *tcp)
@@ -766,8 +746,7 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
766 746
767 prefetchw(wqe); 747 prefetchw(wqe);
768 748
769 if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || 749 if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || rq->hw_mtu < dma_len)) {
770 MLX5E_SW2HW_MTU(rq->channel->priv, rq->netdev->mtu) < dma_len)) {
771 rq->stats.xdp_drop++; 750 rq->stats.xdp_drop++;
772 return false; 751 return false;
773 } 752 }
@@ -806,7 +785,7 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
806 /* move page to reference to sq responsibility, 785 /* move page to reference to sq responsibility,
807 * and mark so it's not put back in page-cache. 786 * and mark so it's not put back in page-cache.
808 */ 787 */
809 rq->wqe.xdp_xmit = true; 788 __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
810 sq->db.di[pi] = *di; 789 sq->db.di[pi] = *di;
811 sq->pc++; 790 sq->pc++;
812 791
@@ -855,6 +834,24 @@ static inline int mlx5e_xdp_handle(struct mlx5e_rq *rq,
855} 834}
856 835
857static inline 836static inline
837struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
838 u32 frag_size, u16 headroom,
839 u32 cqe_bcnt)
840{
841 struct sk_buff *skb = build_skb(va, frag_size);
842
843 if (unlikely(!skb)) {
844 rq->stats.buff_alloc_err++;
845 return NULL;
846 }
847
848 skb_reserve(skb, headroom);
849 skb_put(skb, cqe_bcnt);
850
851 return skb;
852}
853
854static inline
858struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, 855struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
859 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) 856 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
860{ 857{
@@ -885,18 +882,13 @@ struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
885 if (consumed) 882 if (consumed)
886 return NULL; /* page/packet was consumed by XDP */ 883 return NULL; /* page/packet was consumed by XDP */
887 884
888 skb = build_skb(va, frag_size); 885 skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt);
889 if (unlikely(!skb)) { 886 if (unlikely(!skb))
890 rq->stats.buff_alloc_err++;
891 return NULL; 887 return NULL;
892 }
893 888
894 /* queue up for recycling/reuse */ 889 /* queue up for recycling/reuse */
895 page_ref_inc(di->page); 890 page_ref_inc(di->page);
896 891
897 skb_reserve(skb, rx_headroom);
898 skb_put(skb, cqe_bcnt);
899
900 return skb; 892 return skb;
901} 893}
902 894
@@ -918,9 +910,8 @@ void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
918 skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt); 910 skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt);
919 if (!skb) { 911 if (!skb) {
920 /* probably for XDP */ 912 /* probably for XDP */
921 if (rq->wqe.xdp_xmit) { 913 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
922 wi->di.page = NULL; 914 wi->di.page = NULL;
923 rq->wqe.xdp_xmit = false;
924 /* do not return page to cache, it will be returned on XDP_TX completion */ 915 /* do not return page to cache, it will be returned on XDP_TX completion */
925 goto wq_ll_pop; 916 goto wq_ll_pop;
926 } 917 }
@@ -960,9 +951,8 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
960 951
961 skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt); 952 skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt);
962 if (!skb) { 953 if (!skb) {
963 if (rq->wqe.xdp_xmit) { 954 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
964 wi->di.page = NULL; 955 wi->di.page = NULL;
965 rq->wqe.xdp_xmit = false;
966 /* do not return page to cache, it will be returned on XDP_TX completion */ 956 /* do not return page to cache, it will be returned on XDP_TX completion */
967 goto wq_ll_pop; 957 goto wq_ll_pop;
968 } 958 }
@@ -985,23 +975,28 @@ wq_ll_pop:
985} 975}
986#endif 976#endif
987 977
988static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq, 978struct sk_buff *
989 struct mlx5_cqe64 *cqe, 979mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
990 struct mlx5e_mpw_info *wi, 980 u16 cqe_bcnt, u32 head_offset, u32 page_idx)
991 u32 cqe_bcnt,
992 struct sk_buff *skb)
993{ 981{
994 u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
995 u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz;
996 u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
997 u32 page_idx = wqe_offset >> PAGE_SHIFT;
998 u32 head_page_idx = page_idx;
999 u16 headlen = min_t(u16, MLX5_MPWRQ_SMALL_PACKET_THRESHOLD, cqe_bcnt); 982 u16 headlen = min_t(u16, MLX5_MPWRQ_SMALL_PACKET_THRESHOLD, cqe_bcnt);
983 struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
1000 u32 frag_offset = head_offset + headlen; 984 u32 frag_offset = head_offset + headlen;
1001 u16 byte_cnt = cqe_bcnt - headlen; 985 u32 byte_cnt = cqe_bcnt - headlen;
986 struct mlx5e_dma_info *head_di = di;
987 struct sk_buff *skb;
988
989 skb = napi_alloc_skb(rq->cq.napi,
990 ALIGN(MLX5_MPWRQ_SMALL_PACKET_THRESHOLD, sizeof(long)));
991 if (unlikely(!skb)) {
992 rq->stats.buff_alloc_err++;
993 return NULL;
994 }
995
996 prefetchw(skb->data);
1002 997
1003 if (unlikely(frag_offset >= PAGE_SIZE)) { 998 if (unlikely(frag_offset >= PAGE_SIZE)) {
1004 page_idx++; 999 di++;
1005 frag_offset -= PAGE_SIZE; 1000 frag_offset -= PAGE_SIZE;
1006 } 1001 }
1007 1002
@@ -1009,18 +1004,59 @@ static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
1009 u32 pg_consumed_bytes = 1004 u32 pg_consumed_bytes =
1010 min_t(u32, PAGE_SIZE - frag_offset, byte_cnt); 1005 min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
1011 1006
1012 mlx5e_add_skb_frag_mpwqe(rq, skb, wi, page_idx, frag_offset, 1007 mlx5e_add_skb_frag_mpwqe(rq, skb, di, frag_offset,
1013 pg_consumed_bytes); 1008 pg_consumed_bytes);
1014 byte_cnt -= pg_consumed_bytes; 1009 byte_cnt -= pg_consumed_bytes;
1015 frag_offset = 0; 1010 frag_offset = 0;
1016 page_idx++; 1011 di++;
1017 } 1012 }
1018 /* copy header */ 1013 /* copy header */
1019 mlx5e_copy_skb_header_mpwqe(rq->pdev, skb, wi, head_page_idx, 1014 mlx5e_copy_skb_header_mpwqe(rq->pdev, skb, head_di,
1020 head_offset, headlen); 1015 head_offset, headlen);
1021 /* skb linear part was allocated with headlen and aligned to long */ 1016 /* skb linear part was allocated with headlen and aligned to long */
1022 skb->tail += headlen; 1017 skb->tail += headlen;
1023 skb->len += headlen; 1018 skb->len += headlen;
1019
1020 return skb;
1021}
1022
1023struct sk_buff *
1024mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1025 u16 cqe_bcnt, u32 head_offset, u32 page_idx)
1026{
1027 struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
1028 u16 rx_headroom = rq->buff.headroom;
1029 u32 cqe_bcnt32 = cqe_bcnt;
1030 struct sk_buff *skb;
1031 void *va, *data;
1032 u32 frag_size;
1033 bool consumed;
1034
1035 va = page_address(di->page) + head_offset;
1036 data = va + rx_headroom;
1037 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt32);
1038
1039 dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
1040 frag_size, DMA_FROM_DEVICE);
1041 prefetch(data);
1042
1043 rcu_read_lock();
1044 consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32);
1045 rcu_read_unlock();
1046 if (consumed) {
1047 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
1048 __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
1049 return NULL; /* page/packet was consumed by XDP */
1050 }
1051
1052 skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt32);
1053 if (unlikely(!skb))
1054 return NULL;
1055
1056 /* queue up for recycling/reuse */
1057 page_ref_inc(di->page);
1058
1059 return skb;
1024} 1060}
1025 1061
1026void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) 1062void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
@@ -1028,7 +1064,11 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1028 u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); 1064 u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
1029 u16 wqe_id = be16_to_cpu(cqe->wqe_id); 1065 u16 wqe_id = be16_to_cpu(cqe->wqe_id);
1030 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id]; 1066 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
1031 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_id); 1067 u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
1068 u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz;
1069 u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
1070 u32 page_idx = wqe_offset >> PAGE_SHIFT;
1071 struct mlx5e_rx_wqe *wqe;
1032 struct sk_buff *skb; 1072 struct sk_buff *skb;
1033 u16 cqe_bcnt; 1073 u16 cqe_bcnt;
1034 1074
@@ -1044,18 +1084,13 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1044 goto mpwrq_cqe_out; 1084 goto mpwrq_cqe_out;
1045 } 1085 }
1046 1086
1047 skb = napi_alloc_skb(rq->cq.napi,
1048 ALIGN(MLX5_MPWRQ_SMALL_PACKET_THRESHOLD,
1049 sizeof(long)));
1050 if (unlikely(!skb)) {
1051 rq->stats.buff_alloc_err++;
1052 goto mpwrq_cqe_out;
1053 }
1054
1055 prefetchw(skb->data);
1056 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe); 1087 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
1057 1088
1058 mlx5e_mpwqe_fill_rx_skb(rq, cqe, wi, cqe_bcnt, skb); 1089 skb = rq->mpwqe.skb_from_cqe_mpwrq(rq, wi, cqe_bcnt, head_offset,
1090 page_idx);
1091 if (!skb)
1092 goto mpwrq_cqe_out;
1093
1059 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); 1094 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1060 napi_gro_receive(rq->cq.napi, skb); 1095 napi_gro_receive(rq->cq.napi, skb);
1061 1096
@@ -1063,6 +1098,7 @@ mpwrq_cqe_out:
1063 if (likely(wi->consumed_strides < rq->mpwqe.num_strides)) 1098 if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
1064 return; 1099 return;
1065 1100
1101 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_id);
1066 mlx5e_free_rx_mpwqe(rq, wi); 1102 mlx5e_free_rx_mpwqe(rq, wi);
1067 mlx5_wq_ll_pop(&rq->wq, cqe->wqe_id, &wqe->next.next_wqe_index); 1103 mlx5_wq_ll_pop(&rq->wq, cqe->wqe_id, &wqe->next.next_wqe_index);
1068} 1104}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index d7bb10ab2173..70066975f1b5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -245,7 +245,7 @@ int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev)
245 245
246 force_state = MLX5_GET(teardown_hca_out, out, force_state); 246 force_state = MLX5_GET(teardown_hca_out, out, force_state);
247 if (force_state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) { 247 if (force_state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) {
248 mlx5_core_err(dev, "teardown with force mode failed\n"); 248 mlx5_core_warn(dev, "teardown with force mode failed, doing normal teardown\n");
249 return -EIO; 249 return -EIO;
250 } 250 }
251 251
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
index a35608faf8d2..af3bb2f7a504 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
@@ -61,11 +61,12 @@ static void mlx5i_build_nic_params(struct mlx5_core_dev *mdev,
61 mlx5e_init_rq_type_params(mdev, params); 61 mlx5e_init_rq_type_params(mdev, params);
62 62
63 /* RQ size in ipoib by default is 512 */ 63 /* RQ size in ipoib by default is 512 */
64 params->log_rq_size = is_kdump_kernel() ? 64 params->log_rq_mtu_frames = is_kdump_kernel() ?
65 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE : 65 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
66 MLX5I_PARAMS_DEFAULT_LOG_RQ_SIZE; 66 MLX5I_PARAMS_DEFAULT_LOG_RQ_SIZE;
67 67
68 params->lro_en = false; 68 params->lro_en = false;
69 params->hard_mtu = MLX5_IB_GRH_BYTES + MLX5_IPOIB_HARD_LEN;
69} 70}
70 71
71/* Called directly after IPoIB netdevice was created to initialize SW structs */ 72/* Called directly after IPoIB netdevice was created to initialize SW structs */
@@ -81,10 +82,10 @@ void mlx5i_init(struct mlx5_core_dev *mdev,
81 priv->netdev = netdev; 82 priv->netdev = netdev;
82 priv->profile = profile; 83 priv->profile = profile;
83 priv->ppriv = ppriv; 84 priv->ppriv = ppriv;
84 priv->hard_mtu = MLX5_IB_GRH_BYTES + MLX5_IPOIB_HARD_LEN;
85 mutex_init(&priv->state_lock); 85 mutex_init(&priv->state_lock);
86 86
87 mlx5e_build_nic_params(mdev, &priv->channels.params, profile->max_nch(mdev)); 87 mlx5e_build_nic_params(mdev, &priv->channels.params,
88 profile->max_nch(mdev), netdev->mtu);
88 mlx5i_build_nic_params(mdev, &priv->channels.params); 89 mlx5i_build_nic_params(mdev, &priv->channels.params);
89 90
90 mlx5e_timestamp_init(priv); 91 mlx5e_timestamp_init(priv);
@@ -368,25 +369,27 @@ static int mlx5i_change_mtu(struct net_device *netdev, int new_mtu)
368{ 369{
369 struct mlx5e_priv *priv = mlx5i_epriv(netdev); 370 struct mlx5e_priv *priv = mlx5i_epriv(netdev);
370 struct mlx5e_channels new_channels = {}; 371 struct mlx5e_channels new_channels = {};
371 int curr_mtu; 372 struct mlx5e_params *params;
372 int err = 0; 373 int err = 0;
373 374
374 mutex_lock(&priv->state_lock); 375 mutex_lock(&priv->state_lock);
375 376
376 curr_mtu = netdev->mtu; 377 params = &priv->channels.params;
377 netdev->mtu = new_mtu;
378 378
379 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 379 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
380 params->sw_mtu = new_mtu;
381 netdev->mtu = params->sw_mtu;
380 goto out; 382 goto out;
383 }
381 384
382 new_channels.params = priv->channels.params; 385 new_channels.params = *params;
386 new_channels.params.sw_mtu = new_mtu;
383 err = mlx5e_open_channels(priv, &new_channels); 387 err = mlx5e_open_channels(priv, &new_channels);
384 if (err) { 388 if (err)
385 netdev->mtu = curr_mtu;
386 goto out; 389 goto out;
387 }
388 390
389 mlx5e_switch_priv_channels(priv, &new_channels, NULL); 391 mlx5e_switch_priv_channels(priv, &new_channels, NULL);
392 netdev->mtu = new_channels.params.sw_mtu;
390 393
391out: 394out:
392 mutex_unlock(&priv->state_lock); 395 mutex_unlock(&priv->state_lock);
@@ -540,7 +543,7 @@ static int mlx5i_detach_mcast(struct net_device *netdev, struct ib_device *hca,
540 543
541 err = mlx5_core_detach_mcg(mdev, gid, ipriv->qp.qpn); 544 err = mlx5_core_detach_mcg(mdev, gid, ipriv->qp.qpn);
542 if (err) 545 if (err)
543 mlx5_core_dbg(mdev, "failed dettaching QPN 0x%x, MGID %pI6\n", 546 mlx5_core_dbg(mdev, "failed detaching QPN 0x%x, MGID %pI6\n",
544 ipriv->qp.qpn, gid->raw); 547 ipriv->qp.qpn, gid->raw);
545 548
546 return err; 549 return err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
index b69e9d847a6b..54a188f41f90 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
@@ -290,7 +290,7 @@ static void mlx5i_pkey_init(struct mlx5_core_dev *mdev,
290 netdev->ethtool_ops = &mlx5i_pkey_ethtool_ops; 290 netdev->ethtool_ops = &mlx5i_pkey_ethtool_ops;
291 291
292 /* Use dummy rqs */ 292 /* Use dummy rqs */
293 priv->channels.params.log_rq_size = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE; 293 priv->channels.params.log_rq_mtu_frames = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE;
294} 294}
295 295
296/* Called directly before IPoIB netdevice is destroyed to cleanup SW structs */ 296/* Called directly before IPoIB netdevice is destroyed to cleanup SW structs */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
index c64957b5ef47..dae1c5c5d27c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
@@ -354,27 +354,6 @@ int mlx5_core_destroy_xsrq(struct mlx5_core_dev *dev, u32 xsrqn)
354 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 354 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
355} 355}
356 356
357int mlx5_core_query_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u32 *out)
358{
359 u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {0};
360 void *srqc;
361 void *xrc_srqc;
362 int err;
363
364 MLX5_SET(query_xrc_srq_in, in, opcode, MLX5_CMD_OP_QUERY_XRC_SRQ);
365 MLX5_SET(query_xrc_srq_in, in, xrc_srqn, xsrqn);
366 err = mlx5_cmd_exec(dev, in, sizeof(in), out,
367 MLX5_ST_SZ_BYTES(query_xrc_srq_out));
368 if (!err) {
369 xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, out,
370 xrc_srq_context_entry);
371 srqc = MLX5_ADDR_OF(query_srq_out, out, srq_context_entry);
372 memcpy(srqc, xrc_srqc, MLX5_ST_SZ_BYTES(srqc));
373 }
374
375 return err;
376}
377
378int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u16 lwm) 357int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u16 lwm)
379{ 358{
380 u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {0}; 359 u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {0};
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 4b5939c78cdd..12758595459b 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -782,6 +782,9 @@ static inline u64 get_cqe_ts(struct mlx5_cqe64 *cqe)
782 return (u64)lo | ((u64)hi << 32); 782 return (u64)lo | ((u64)hi << 32);
783} 783}
784 784
785#define MLX5_MPWQE_LOG_NUM_STRIDES_BASE (9)
786#define MLX5_MPWQE_LOG_STRIDE_SZ_BASE (6)
787
785struct mpwrq_cqe_bc { 788struct mpwrq_cqe_bc {
786 __be16 filler_consumed_strides; 789 __be16 filler_consumed_strides;
787 __be16 byte_cnt; 790 __be16 byte_cnt;
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index c19e611d2782..d25011f84815 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -1038,7 +1038,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
1038 u8 reserved_at_398[0x3]; 1038 u8 reserved_at_398[0x3];
1039 u8 log_max_tis_per_sq[0x5]; 1039 u8 log_max_tis_per_sq[0x5];
1040 1040
1041 u8 reserved_at_3a0[0x3]; 1041 u8 ext_stride_num_range[0x1];
1042 u8 reserved_at_3a1[0x2];
1042 u8 log_max_stride_sz_rq[0x5]; 1043 u8 log_max_stride_sz_rq[0x5];
1043 u8 reserved_at_3a8[0x3]; 1044 u8 reserved_at_3a8[0x3];
1044 u8 log_min_stride_sz_rq[0x5]; 1045 u8 log_min_stride_sz_rq[0x5];
@@ -1205,9 +1206,9 @@ struct mlx5_ifc_wq_bits {
1205 u8 log_hairpin_num_packets[0x5]; 1206 u8 log_hairpin_num_packets[0x5];
1206 u8 reserved_at_128[0x3]; 1207 u8 reserved_at_128[0x3];
1207 u8 log_hairpin_data_sz[0x5]; 1208 u8 log_hairpin_data_sz[0x5];
1208 u8 reserved_at_130[0x5];
1209 1209
1210 u8 log_wqe_num_of_strides[0x3]; 1210 u8 reserved_at_130[0x4];
1211 u8 log_wqe_num_of_strides[0x4];
1211 u8 two_byte_shift_en[0x1]; 1212 u8 two_byte_shift_en[0x1];
1212 u8 reserved_at_139[0x4]; 1213 u8 reserved_at_139[0x4];
1213 u8 log_wqe_stride_size[0x3]; 1214 u8 log_wqe_stride_size[0x3];
diff --git a/include/linux/mlx5/transobj.h b/include/linux/mlx5/transobj.h
index 80d7aa8b2831..83a33a1873a6 100644
--- a/include/linux/mlx5/transobj.h
+++ b/include/linux/mlx5/transobj.h
@@ -67,7 +67,6 @@ int mlx5_core_arm_rmp(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm);
67int mlx5_core_create_xsrq(struct mlx5_core_dev *dev, u32 *in, int inlen, 67int mlx5_core_create_xsrq(struct mlx5_core_dev *dev, u32 *in, int inlen,
68 u32 *rmpn); 68 u32 *rmpn);
69int mlx5_core_destroy_xsrq(struct mlx5_core_dev *dev, u32 rmpn); 69int mlx5_core_destroy_xsrq(struct mlx5_core_dev *dev, u32 rmpn);
70int mlx5_core_query_xsrq(struct mlx5_core_dev *dev, u32 rmpn, u32 *out);
71int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm); 70int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm);
72 71
73int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen, 72int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-23 06:20:39 -0400