aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2018-01-21 18:13:23 -0500
committerDavid S. Miller <davem@davemloft.net>2018-01-21 18:13:23 -0500
commitf9b6ae29ae738c8ed06b8a869628b1d98d8e7624 (patch)
treee36149a676efb65e4d359e79c7340c8dfb39ad63
parent01c15e93a78cfcf45cc32d07aa38bdc84250f569 (diff)
parent63a612f984a1fae040ab6f1c6a0f1fdcdf1954b8 (diff)
Merge tag 'mlx5-updates-2018-01-19' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux
Saeed Mahameed says: ==================== mlx5-updates-2018-01-19 From: Or Gerlitz <ogerlitz@mellanox.com> ======= First six patches of this series further enhances the mlx5 hairpin support. The first two patches deal with using different hairpin instances for flows whose packets have different priorities to align with the port TX QoS model. The next four patches allow us to do HW spreading of flows over a set of hairpin pairs using RSS. The last two patches change the driver to also set the size of the HW hairpin queues. ======== Next four patches from Eran Ben Elisha <eranbe@mellanox.com>: Add more debug data for TX timeout handling, and further enhance and optimize TX timeout handling upon lost interrupts, which adds a mechanism for explicitly polling EQ in case of a TX timeout in order to recover from a lost interrupt. If this is not the case (no pending EQEs), perform a channels full recovery as usual. From Kamal Heib <kamalh@mellanox.com>, Two patches to extend the stats group API to have an update_stats() callback which will be used to fetch the hardware or software counters data, this will improve the current API and reduce code duplication. From Gal Pressman <galp@mellanox.com>, Last patch, Add likely to the common RX checksum flow. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en.h34
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_fs.c110
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_main.c233
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rx.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_stats.c282
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_stats.h12
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_tc.c292
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eq.c18
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fs_core.c5
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c22
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h1
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/transobj.c96
-rw-r--r--include/linux/mlx5/mlx5_ifc.h8
-rw-r--r--include/linux/mlx5/transobj.h8
15 files changed, 821 insertions, 304 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 7b988595ac5f..4c9360b25532 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -44,6 +44,7 @@
44#include <linux/mlx5/port.h> 44#include <linux/mlx5/port.h>
45#include <linux/mlx5/vport.h> 45#include <linux/mlx5/vport.h>
46#include <linux/mlx5/transobj.h> 46#include <linux/mlx5/transobj.h>
47#include <linux/mlx5/fs.h>
47#include <linux/rhashtable.h> 48#include <linux/rhashtable.h>
48#include <net/switchdev.h> 49#include <net/switchdev.h>
49#include <net/xdp.h> 50#include <net/xdp.h>
@@ -560,6 +561,7 @@ struct mlx5e_channel {
560 561
561 /* data path - accessed per napi poll */ 562 /* data path - accessed per napi poll */
562 struct irq_desc *irq_desc; 563 struct irq_desc *irq_desc;
564 struct mlx5e_ch_stats stats;
563 565
564 /* control */ 566 /* control */
565 struct mlx5e_priv *priv; 567 struct mlx5e_priv *priv;
@@ -696,6 +698,11 @@ enum {
696 MLX5E_ARFS_FT_LEVEL 698 MLX5E_ARFS_FT_LEVEL
697}; 699};
698 700
701enum {
702 MLX5E_TC_FT_LEVEL = 0,
703 MLX5E_TC_TTC_FT_LEVEL,
704};
705
699struct mlx5e_ethtool_table { 706struct mlx5e_ethtool_table {
700 struct mlx5_flow_table *ft; 707 struct mlx5_flow_table *ft;
701 int num_rules; 708 int num_rules;
@@ -834,7 +841,7 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
834void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix); 841void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
835void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi); 842void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
836 843
837void mlx5e_update_stats(struct mlx5e_priv *priv, bool full); 844void mlx5e_update_stats(struct mlx5e_priv *priv);
838 845
839int mlx5e_create_flow_steering(struct mlx5e_priv *priv); 846int mlx5e_create_flow_steering(struct mlx5e_priv *priv);
840void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv); 847void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv);
@@ -1024,11 +1031,26 @@ int mlx5e_create_direct_tirs(struct mlx5e_priv *priv);
1024void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv); 1031void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv);
1025void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt); 1032void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt);
1026 1033
1027int mlx5e_create_ttc_table(struct mlx5e_priv *priv); 1034struct ttc_params {
1028void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv); 1035 struct mlx5_flow_table_attr ft_attr;
1036 u32 any_tt_tirn;
1037 u32 indir_tirn[MLX5E_NUM_INDIR_TIRS];
1038 struct mlx5e_ttc_table *inner_ttc;
1039};
1029 1040
1030int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv); 1041void mlx5e_set_ttc_basic_params(struct mlx5e_priv *priv, struct ttc_params *ttc_params);
1031void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv); 1042void mlx5e_set_ttc_ft_params(struct ttc_params *ttc_params);
1043void mlx5e_set_inner_ttc_ft_params(struct ttc_params *ttc_params);
1044
1045int mlx5e_create_ttc_table(struct mlx5e_priv *priv, struct ttc_params *params,
1046 struct mlx5e_ttc_table *ttc);
1047void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv,
1048 struct mlx5e_ttc_table *ttc);
1049
1050int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv, struct ttc_params *params,
1051 struct mlx5e_ttc_table *ttc);
1052void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv,
1053 struct mlx5e_ttc_table *ttc);
1032 1054
1033int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc, 1055int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc,
1034 u32 underlay_qpn, u32 *tisn); 1056 u32 underlay_qpn, u32 *tisn);
@@ -1041,6 +1063,8 @@ int mlx5e_open(struct net_device *netdev);
1041void mlx5e_update_stats_work(struct work_struct *work); 1063void mlx5e_update_stats_work(struct work_struct *work);
1042u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout); 1064u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout);
1043 1065
1066int mlx5e_bits_invert(unsigned long a, int size);
1067
1044/* ethtool helpers */ 1068/* ethtool helpers */
1045void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv, 1069void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
1046 struct ethtool_drvinfo *drvinfo); 1070 struct ethtool_drvinfo *drvinfo);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 2d1395015ab5..cc8048f68f11 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -207,7 +207,7 @@ void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv,
207 return; 207 return;
208 208
209 mutex_lock(&priv->state_lock); 209 mutex_lock(&priv->state_lock);
210 mlx5e_update_stats(priv, true); 210 mlx5e_update_stats(priv);
211 mutex_unlock(&priv->state_lock); 211 mutex_unlock(&priv->state_lock);
212 212
213 for (i = 0; i < mlx5e_num_stats_grps; i++) 213 for (i = 0; i < mlx5e_num_stats_grps; i++)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index def513484845..f64dda2bed31 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -806,25 +806,25 @@ mlx5e_generate_ttc_rule(struct mlx5e_priv *priv,
806 return err ? ERR_PTR(err) : rule; 806 return err ? ERR_PTR(err) : rule;
807} 807}
808 808
809static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv) 809static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv,
810 struct ttc_params *params,
811 struct mlx5e_ttc_table *ttc)
810{ 812{
811 struct mlx5_flow_destination dest = {}; 813 struct mlx5_flow_destination dest = {};
812 struct mlx5e_ttc_table *ttc;
813 struct mlx5_flow_handle **rules; 814 struct mlx5_flow_handle **rules;
814 struct mlx5_flow_table *ft; 815 struct mlx5_flow_table *ft;
815 int tt; 816 int tt;
816 int err; 817 int err;
817 818
818 ttc = &priv->fs.ttc;
819 ft = ttc->ft.t; 819 ft = ttc->ft.t;
820 rules = ttc->rules; 820 rules = ttc->rules;
821 821
822 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 822 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
823 for (tt = 0; tt < MLX5E_NUM_TT; tt++) { 823 for (tt = 0; tt < MLX5E_NUM_TT; tt++) {
824 if (tt == MLX5E_TT_ANY) 824 if (tt == MLX5E_TT_ANY)
825 dest.tir_num = priv->direct_tir[0].tirn; 825 dest.tir_num = params->any_tt_tirn;
826 else 826 else
827 dest.tir_num = priv->indir_tir[tt].tirn; 827 dest.tir_num = params->indir_tirn[tt];
828 rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest, 828 rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest,
829 ttc_rules[tt].etype, 829 ttc_rules[tt].etype,
830 ttc_rules[tt].proto); 830 ttc_rules[tt].proto);
@@ -832,12 +832,12 @@ static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv)
832 goto del_rules; 832 goto del_rules;
833 } 833 }
834 834
835 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev)) 835 if (!params->inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev))
836 return 0; 836 return 0;
837 837
838 rules = ttc->tunnel_rules; 838 rules = ttc->tunnel_rules;
839 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 839 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
840 dest.ft = priv->fs.inner_ttc.ft.t; 840 dest.ft = params->inner_ttc->ft.t;
841 for (tt = 0; tt < MLX5E_NUM_TUNNEL_TT; tt++) { 841 for (tt = 0; tt < MLX5E_NUM_TUNNEL_TT; tt++) {
842 rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest, 842 rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest,
843 ttc_tunnel_rules[tt].etype, 843 ttc_tunnel_rules[tt].etype,
@@ -977,25 +977,25 @@ mlx5e_generate_inner_ttc_rule(struct mlx5e_priv *priv,
977 return err ? ERR_PTR(err) : rule; 977 return err ? ERR_PTR(err) : rule;
978} 978}
979 979
980static int mlx5e_generate_inner_ttc_table_rules(struct mlx5e_priv *priv) 980static int mlx5e_generate_inner_ttc_table_rules(struct mlx5e_priv *priv,
981 struct ttc_params *params,
982 struct mlx5e_ttc_table *ttc)
981{ 983{
982 struct mlx5_flow_destination dest = {}; 984 struct mlx5_flow_destination dest = {};
983 struct mlx5_flow_handle **rules; 985 struct mlx5_flow_handle **rules;
984 struct mlx5e_ttc_table *ttc;
985 struct mlx5_flow_table *ft; 986 struct mlx5_flow_table *ft;
986 int err; 987 int err;
987 int tt; 988 int tt;
988 989
989 ttc = &priv->fs.inner_ttc;
990 ft = ttc->ft.t; 990 ft = ttc->ft.t;
991 rules = ttc->rules; 991 rules = ttc->rules;
992 992
993 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 993 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
994 for (tt = 0; tt < MLX5E_NUM_TT; tt++) { 994 for (tt = 0; tt < MLX5E_NUM_TT; tt++) {
995 if (tt == MLX5E_TT_ANY) 995 if (tt == MLX5E_TT_ANY)
996 dest.tir_num = priv->direct_tir[0].tirn; 996 dest.tir_num = params->any_tt_tirn;
997 else 997 else
998 dest.tir_num = priv->inner_indir_tir[tt].tirn; 998 dest.tir_num = params->indir_tirn[tt];
999 999
1000 rules[tt] = mlx5e_generate_inner_ttc_rule(priv, ft, &dest, 1000 rules[tt] = mlx5e_generate_inner_ttc_rule(priv, ft, &dest,
1001 ttc_rules[tt].etype, 1001 ttc_rules[tt].etype,
@@ -1075,21 +1075,42 @@ err:
1075 return err; 1075 return err;
1076} 1076}
1077 1077
1078int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv) 1078void mlx5e_set_ttc_basic_params(struct mlx5e_priv *priv,
1079 struct ttc_params *ttc_params)
1080{
1081 ttc_params->any_tt_tirn = priv->direct_tir[0].tirn;
1082 ttc_params->inner_ttc = &priv->fs.inner_ttc;
1083}
1084
1085void mlx5e_set_inner_ttc_ft_params(struct ttc_params *ttc_params)
1086{
1087 struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
1088
1089 ft_attr->max_fte = MLX5E_INNER_TTC_TABLE_SIZE;
1090 ft_attr->level = MLX5E_INNER_TTC_FT_LEVEL;
1091 ft_attr->prio = MLX5E_NIC_PRIO;
1092}
1093
1094void mlx5e_set_ttc_ft_params(struct ttc_params *ttc_params)
1095
1096{
1097 struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
1098
1099 ft_attr->max_fte = MLX5E_TTC_TABLE_SIZE;
1100 ft_attr->level = MLX5E_TTC_FT_LEVEL;
1101 ft_attr->prio = MLX5E_NIC_PRIO;
1102}
1103
1104int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv, struct ttc_params *params,
1105 struct mlx5e_ttc_table *ttc)
1079{ 1106{
1080 struct mlx5e_ttc_table *ttc = &priv->fs.inner_ttc;
1081 struct mlx5_flow_table_attr ft_attr = {};
1082 struct mlx5e_flow_table *ft = &ttc->ft; 1107 struct mlx5e_flow_table *ft = &ttc->ft;
1083 int err; 1108 int err;
1084 1109
1085 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev)) 1110 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
1086 return 0; 1111 return 0;
1087 1112
1088 ft_attr.max_fte = MLX5E_INNER_TTC_TABLE_SIZE; 1113 ft->t = mlx5_create_flow_table(priv->fs.ns, &params->ft_attr);
1089 ft_attr.level = MLX5E_INNER_TTC_FT_LEVEL;
1090 ft_attr.prio = MLX5E_NIC_PRIO;
1091
1092 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
1093 if (IS_ERR(ft->t)) { 1114 if (IS_ERR(ft->t)) {
1094 err = PTR_ERR(ft->t); 1115 err = PTR_ERR(ft->t);
1095 ft->t = NULL; 1116 ft->t = NULL;
@@ -1100,7 +1121,7 @@ int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv)
1100 if (err) 1121 if (err)
1101 goto err; 1122 goto err;
1102 1123
1103 err = mlx5e_generate_inner_ttc_table_rules(priv); 1124 err = mlx5e_generate_inner_ttc_table_rules(priv, params, ttc);
1104 if (err) 1125 if (err)
1105 goto err; 1126 goto err;
1106 1127
@@ -1111,10 +1132,9 @@ err:
1111 return err; 1132 return err;
1112} 1133}
1113 1134
1114void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv) 1135void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv,
1136 struct mlx5e_ttc_table *ttc)
1115{ 1137{
1116 struct mlx5e_ttc_table *ttc = &priv->fs.inner_ttc;
1117
1118 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev)) 1138 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
1119 return; 1139 return;
1120 1140
@@ -1122,27 +1142,21 @@ void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv)
1122 mlx5e_destroy_flow_table(&ttc->ft); 1142 mlx5e_destroy_flow_table(&ttc->ft);
1123} 1143}
1124 1144
1125void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv) 1145void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv,
1146 struct mlx5e_ttc_table *ttc)
1126{ 1147{
1127 struct mlx5e_ttc_table *ttc = &priv->fs.ttc;
1128
1129 mlx5e_cleanup_ttc_rules(ttc); 1148 mlx5e_cleanup_ttc_rules(ttc);
1130 mlx5e_destroy_flow_table(&ttc->ft); 1149 mlx5e_destroy_flow_table(&ttc->ft);
1131} 1150}
1132 1151
1133int mlx5e_create_ttc_table(struct mlx5e_priv *priv) 1152int mlx5e_create_ttc_table(struct mlx5e_priv *priv, struct ttc_params *params,
1153 struct mlx5e_ttc_table *ttc)
1134{ 1154{
1135 bool match_ipv_outer = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version); 1155 bool match_ipv_outer = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version);
1136 struct mlx5e_ttc_table *ttc = &priv->fs.ttc;
1137 struct mlx5_flow_table_attr ft_attr = {};
1138 struct mlx5e_flow_table *ft = &ttc->ft; 1156 struct mlx5e_flow_table *ft = &ttc->ft;
1139 int err; 1157 int err;
1140 1158
1141 ft_attr.max_fte = MLX5E_TTC_TABLE_SIZE; 1159 ft->t = mlx5_create_flow_table(priv->fs.ns, &params->ft_attr);
1142 ft_attr.level = MLX5E_TTC_FT_LEVEL;
1143 ft_attr.prio = MLX5E_NIC_PRIO;
1144
1145 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
1146 if (IS_ERR(ft->t)) { 1160 if (IS_ERR(ft->t)) {
1147 err = PTR_ERR(ft->t); 1161 err = PTR_ERR(ft->t);
1148 ft->t = NULL; 1162 ft->t = NULL;
@@ -1153,7 +1167,7 @@ int mlx5e_create_ttc_table(struct mlx5e_priv *priv)
1153 if (err) 1167 if (err)
1154 goto err; 1168 goto err;
1155 1169
1156 err = mlx5e_generate_ttc_table_rules(priv); 1170 err = mlx5e_generate_ttc_table_rules(priv, params, ttc);
1157 if (err) 1171 if (err)
1158 goto err; 1172 goto err;
1159 1173
@@ -1474,7 +1488,8 @@ static void mlx5e_destroy_vlan_table(struct mlx5e_priv *priv)
1474 1488
1475int mlx5e_create_flow_steering(struct mlx5e_priv *priv) 1489int mlx5e_create_flow_steering(struct mlx5e_priv *priv)
1476{ 1490{
1477 int err; 1491 struct ttc_params ttc_params = {};
1492 int tt, err;
1478 1493
1479 priv->fs.ns = mlx5_get_flow_namespace(priv->mdev, 1494 priv->fs.ns = mlx5_get_flow_namespace(priv->mdev,
1480 MLX5_FLOW_NAMESPACE_KERNEL); 1495 MLX5_FLOW_NAMESPACE_KERNEL);
@@ -1489,14 +1504,23 @@ int mlx5e_create_flow_steering(struct mlx5e_priv *priv)
1489 priv->netdev->hw_features &= ~NETIF_F_NTUPLE; 1504 priv->netdev->hw_features &= ~NETIF_F_NTUPLE;
1490 } 1505 }
1491 1506
1492 err = mlx5e_create_inner_ttc_table(priv); 1507 mlx5e_set_ttc_basic_params(priv, &ttc_params);
1508 mlx5e_set_inner_ttc_ft_params(&ttc_params);
1509 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
1510 ttc_params.indir_tirn[tt] = priv->inner_indir_tir[tt].tirn;
1511
1512 err = mlx5e_create_inner_ttc_table(priv, &ttc_params, &priv->fs.inner_ttc);
1493 if (err) { 1513 if (err) {
1494 netdev_err(priv->netdev, "Failed to create inner ttc table, err=%d\n", 1514 netdev_err(priv->netdev, "Failed to create inner ttc table, err=%d\n",
1495 err); 1515 err);
1496 goto err_destroy_arfs_tables; 1516 goto err_destroy_arfs_tables;
1497 } 1517 }
1498 1518
1499 err = mlx5e_create_ttc_table(priv); 1519 mlx5e_set_ttc_ft_params(&ttc_params);
1520 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
1521 ttc_params.indir_tirn[tt] = priv->indir_tir[tt].tirn;
1522
1523 err = mlx5e_create_ttc_table(priv, &ttc_params, &priv->fs.ttc);
1500 if (err) { 1524 if (err) {
1501 netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n", 1525 netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n",
1502 err); 1526 err);
@@ -1524,9 +1548,9 @@ int mlx5e_create_flow_steering(struct mlx5e_priv *priv)
1524err_destroy_l2_table: 1548err_destroy_l2_table:
1525 mlx5e_destroy_l2_table(priv); 1549 mlx5e_destroy_l2_table(priv);
1526err_destroy_ttc_table: 1550err_destroy_ttc_table:
1527 mlx5e_destroy_ttc_table(priv); 1551 mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
1528err_destroy_inner_ttc_table: 1552err_destroy_inner_ttc_table:
1529 mlx5e_destroy_inner_ttc_table(priv); 1553 mlx5e_destroy_inner_ttc_table(priv, &priv->fs.inner_ttc);
1530err_destroy_arfs_tables: 1554err_destroy_arfs_tables:
1531 mlx5e_arfs_destroy_tables(priv); 1555 mlx5e_arfs_destroy_tables(priv);
1532 1556
@@ -1537,8 +1561,8 @@ void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv)
1537{ 1561{
1538 mlx5e_destroy_vlan_table(priv); 1562 mlx5e_destroy_vlan_table(priv);
1539 mlx5e_destroy_l2_table(priv); 1563 mlx5e_destroy_l2_table(priv);
1540 mlx5e_destroy_ttc_table(priv); 1564 mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
1541 mlx5e_destroy_inner_ttc_table(priv); 1565 mlx5e_destroy_inner_ttc_table(priv, &priv->fs.inner_ttc);
1542 mlx5e_arfs_destroy_tables(priv); 1566 mlx5e_arfs_destroy_tables(priv);
1543 mlx5e_ethtool_cleanup_steering(priv); 1567 mlx5e_ethtool_cleanup_steering(priv);
1544} 1568}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 466a4e1244d7..8530c770c873 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -173,182 +173,23 @@ unlock:
173 rtnl_unlock(); 173 rtnl_unlock();
174} 174}
175 175
176static void mlx5e_update_sw_counters(struct mlx5e_priv *priv) 176void mlx5e_update_stats(struct mlx5e_priv *priv)
177{ 177{
178 struct mlx5e_sw_stats temp, *s = &temp; 178 int i;
179 struct mlx5e_rq_stats *rq_stats;
180 struct mlx5e_sq_stats *sq_stats;
181 int i, j;
182
183 memset(s, 0, sizeof(*s));
184 for (i = 0; i < priv->channels.num; i++) {
185 struct mlx5e_channel *c = priv->channels.c[i];
186
187 rq_stats = &c->rq.stats;
188
189 s->rx_packets += rq_stats->packets;
190 s->rx_bytes += rq_stats->bytes;
191 s->rx_lro_packets += rq_stats->lro_packets;
192 s->rx_lro_bytes += rq_stats->lro_bytes;
193 s->rx_removed_vlan_packets += rq_stats->removed_vlan_packets;
194 s->rx_csum_none += rq_stats->csum_none;
195 s->rx_csum_complete += rq_stats->csum_complete;
196 s->rx_csum_unnecessary += rq_stats->csum_unnecessary;
197 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
198 s->rx_xdp_drop += rq_stats->xdp_drop;
199 s->rx_xdp_tx += rq_stats->xdp_tx;
200 s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
201 s->rx_wqe_err += rq_stats->wqe_err;
202 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
203 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
204 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
205 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
206 s->rx_page_reuse += rq_stats->page_reuse;
207 s->rx_cache_reuse += rq_stats->cache_reuse;
208 s->rx_cache_full += rq_stats->cache_full;
209 s->rx_cache_empty += rq_stats->cache_empty;
210 s->rx_cache_busy += rq_stats->cache_busy;
211 s->rx_cache_waive += rq_stats->cache_waive;
212
213 for (j = 0; j < priv->channels.params.num_tc; j++) {
214 sq_stats = &c->sq[j].stats;
215
216 s->tx_packets += sq_stats->packets;
217 s->tx_bytes += sq_stats->bytes;
218 s->tx_tso_packets += sq_stats->tso_packets;
219 s->tx_tso_bytes += sq_stats->tso_bytes;
220 s->tx_tso_inner_packets += sq_stats->tso_inner_packets;
221 s->tx_tso_inner_bytes += sq_stats->tso_inner_bytes;
222 s->tx_added_vlan_packets += sq_stats->added_vlan_packets;
223 s->tx_queue_stopped += sq_stats->stopped;
224 s->tx_queue_wake += sq_stats->wake;
225 s->tx_queue_dropped += sq_stats->dropped;
226 s->tx_xmit_more += sq_stats->xmit_more;
227 s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
228 s->tx_csum_none += sq_stats->csum_none;
229 s->tx_csum_partial += sq_stats->csum_partial;
230 }
231 }
232
233 s->link_down_events_phy = MLX5_GET(ppcnt_reg,
234 priv->stats.pport.phy_counters,
235 counter_set.phys_layer_cntrs.link_down_events);
236 memcpy(&priv->stats.sw, s, sizeof(*s));
237}
238
239static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
240{
241 int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
242 u32 *out = (u32 *)priv->stats.vport.query_vport_out;
243 u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
244 struct mlx5_core_dev *mdev = priv->mdev;
245
246 MLX5_SET(query_vport_counter_in, in, opcode,
247 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
248 MLX5_SET(query_vport_counter_in, in, op_mod, 0);
249 MLX5_SET(query_vport_counter_in, in, other_vport, 0);
250
251 mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
252}
253
254static void mlx5e_update_pport_counters(struct mlx5e_priv *priv, bool full)
255{
256 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
257 struct mlx5_core_dev *mdev = priv->mdev;
258 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
259 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
260 int prio;
261 void *out;
262
263 MLX5_SET(ppcnt_reg, in, local_port, 1);
264
265 out = pstats->IEEE_802_3_counters;
266 MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
267 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
268
269 if (!full)
270 return;
271
272 out = pstats->RFC_2863_counters;
273 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
274 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
275
276 out = pstats->RFC_2819_counters;
277 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
278 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
279
280 out = pstats->phy_counters;
281 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
282 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
283
284 if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) {
285 out = pstats->phy_statistical_counters;
286 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
287 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
288 }
289
290 if (MLX5_CAP_PCAM_FEATURE(mdev, rx_buffer_fullness_counters)) {
291 out = pstats->eth_ext_counters;
292 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
293 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
294 }
295
296 MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
297 for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
298 out = pstats->per_prio_counters[prio];
299 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
300 mlx5_core_access_reg(mdev, in, sz, out, sz,
301 MLX5_REG_PPCNT, 0, 0);
302 }
303}
304
305static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
306{
307 struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
308 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)];
309 int err;
310
311 if (!priv->q_counter)
312 return;
313
314 err = mlx5_core_query_q_counter(priv->mdev, priv->q_counter, 0, out, sizeof(out));
315 if (err)
316 return;
317
318 qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out, out, out_of_buffer);
319}
320
321static void mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
322{
323 struct mlx5e_pcie_stats *pcie_stats = &priv->stats.pcie;
324 struct mlx5_core_dev *mdev = priv->mdev;
325 u32 in[MLX5_ST_SZ_DW(mpcnt_reg)] = {0};
326 int sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
327 void *out;
328
329 if (!MLX5_CAP_MCAM_FEATURE(mdev, pcie_performance_group))
330 return;
331
332 out = pcie_stats->pcie_perf_counters;
333 MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
334 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
335}
336 179
337void mlx5e_update_stats(struct mlx5e_priv *priv, bool full) 180 for (i = mlx5e_num_stats_grps - 1; i >= 0; i--)
338{ 181 if (mlx5e_stats_grps[i].update_stats)
339 if (full) { 182 mlx5e_stats_grps[i].update_stats(priv);
340 mlx5e_update_pcie_counters(priv);
341 mlx5e_ipsec_update_stats(priv);
342 }
343 mlx5e_update_pport_counters(priv, full);
344 mlx5e_update_vport_counters(priv);
345 mlx5e_update_q_counter(priv);
346 mlx5e_update_sw_counters(priv);
347} 183}
348 184
349static void mlx5e_update_ndo_stats(struct mlx5e_priv *priv) 185static void mlx5e_update_ndo_stats(struct mlx5e_priv *priv)
350{ 186{
351 mlx5e_update_stats(priv, false); 187 int i;
188
189 for (i = mlx5e_num_stats_grps - 1; i >= 0; i--)
190 if (mlx5e_stats_grps[i].update_stats_mask &
191 MLX5E_NDO_UPDATE_STATS)
192 mlx5e_stats_grps[i].update_stats(priv);
352} 193}
353 194
354void mlx5e_update_stats_work(struct work_struct *work) 195void mlx5e_update_stats_work(struct work_struct *work)
@@ -2219,7 +2060,7 @@ static int mlx5e_rx_hash_fn(int hfunc)
2219 MLX5_RX_HASH_FN_INVERTED_XOR8; 2060 MLX5_RX_HASH_FN_INVERTED_XOR8;
2220} 2061}
2221 2062
2222static int mlx5e_bits_invert(unsigned long a, int size) 2063int mlx5e_bits_invert(unsigned long a, int size)
2223{ 2064{
2224 int inv = 0; 2065 int inv = 0;
2225 int i; 2066 int i;
@@ -3757,26 +3598,62 @@ static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
3757 return features; 3598 return features;
3758} 3599}
3759 3600
3601static bool mlx5e_tx_timeout_eq_recover(struct net_device *dev,
3602 struct mlx5e_txqsq *sq)
3603{
3604 struct mlx5e_priv *priv = netdev_priv(dev);
3605 struct mlx5_core_dev *mdev = priv->mdev;
3606 int irqn_not_used, eqn;
3607 struct mlx5_eq *eq;
3608 u32 eqe_count;
3609
3610 if (mlx5_vector2eqn(mdev, sq->cq.mcq.vector, &eqn, &irqn_not_used))
3611 return false;
3612
3613 eq = mlx5_eqn2eq(mdev, eqn);
3614 if (IS_ERR(eq))
3615 return false;
3616
3617 netdev_err(dev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n",
3618 eqn, eq->cons_index, eq->irqn);
3619
3620 eqe_count = mlx5_eq_poll_irq_disabled(eq);
3621 if (!eqe_count)
3622 return false;
3623
3624 netdev_err(dev, "Recover %d eqes on EQ 0x%x\n", eqe_count, eq->eqn);
3625 sq->channel->stats.eq_rearm++;
3626 return true;
3627}
3628
3760static void mlx5e_tx_timeout(struct net_device *dev) 3629static void mlx5e_tx_timeout(struct net_device *dev)
3761{ 3630{
3762 struct mlx5e_priv *priv = netdev_priv(dev); 3631 struct mlx5e_priv *priv = netdev_priv(dev);
3763 bool sched_work = false; 3632 bool reopen_channels = false;
3764 int i; 3633 int i;
3765 3634
3766 netdev_err(dev, "TX timeout detected\n"); 3635 netdev_err(dev, "TX timeout detected\n");
3767 3636
3768 for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) { 3637 for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) {
3638 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, i);
3769 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 3639 struct mlx5e_txqsq *sq = priv->txq2sq[i];
3770 3640
3771 if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i))) 3641 if (!netif_xmit_stopped(dev_queue))
3772 continue; 3642 continue;
3773 sched_work = true; 3643 netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u\n",
3774 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 3644 i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc,
3775 netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n", 3645 jiffies_to_usecs(jiffies - dev_queue->trans_start));
3776 i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc); 3646
3647 /* If we recover a lost interrupt, most likely TX timeout will
3648 * be resolved, skip reopening channels
3649 */
3650 if (!mlx5e_tx_timeout_eq_recover(dev, sq)) {
3651 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
3652 reopen_channels = true;
3653 }
3777 } 3654 }
3778 3655
3779 if (sched_work && test_bit(MLX5E_STATE_OPENED, &priv->state)) 3656 if (reopen_channels && test_bit(MLX5E_STATE_OPENED, &priv->state))
3780 schedule_work(&priv->tx_timeout_work); 3657 schedule_work(&priv->tx_timeout_work);
3781} 3658}
3782 3659
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index ff234dfefc27..0d4bb0688faa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -631,7 +631,7 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
631 return; 631 return;
632 } 632 }
633 633
634 if (is_last_ethertype_ip(skb, &network_depth)) { 634 if (likely(is_last_ethertype_ip(skb, &network_depth))) {
635 skb->ip_summed = CHECKSUM_COMPLETE; 635 skb->ip_summed = CHECKSUM_COMPLETE;
636 skb->csum = csum_unfold((__force __sum16)cqe->check_sum); 636 skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
637 if (network_depth > ETH_HLEN) 637 if (network_depth > ETH_HLEN)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
index b74ddc7984bc..5f0f3493d747 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
@@ -71,6 +71,7 @@ static const struct counter_desc sw_stats_desc[] = {
71 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) }, 71 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
72 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) }, 72 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
73 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) }, 73 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
74 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_eq_rearm) },
74 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) }, 75 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) },
75}; 76};
76 77
@@ -99,6 +100,72 @@ static int mlx5e_grp_sw_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
99 return idx; 100 return idx;
100} 101}
101 102
103static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
104{
105 struct mlx5e_sw_stats temp, *s = &temp;
106 struct mlx5e_rq_stats *rq_stats;
107 struct mlx5e_sq_stats *sq_stats;
108 struct mlx5e_ch_stats *ch_stats;
109 int i, j;
110
111 memset(s, 0, sizeof(*s));
112 for (i = 0; i < priv->channels.num; i++) {
113 struct mlx5e_channel *c = priv->channels.c[i];
114
115 rq_stats = &c->rq.stats;
116 ch_stats = &c->stats;
117
118 s->rx_packets += rq_stats->packets;
119 s->rx_bytes += rq_stats->bytes;
120 s->rx_lro_packets += rq_stats->lro_packets;
121 s->rx_lro_bytes += rq_stats->lro_bytes;
122 s->rx_removed_vlan_packets += rq_stats->removed_vlan_packets;
123 s->rx_csum_none += rq_stats->csum_none;
124 s->rx_csum_complete += rq_stats->csum_complete;
125 s->rx_csum_unnecessary += rq_stats->csum_unnecessary;
126 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
127 s->rx_xdp_drop += rq_stats->xdp_drop;
128 s->rx_xdp_tx += rq_stats->xdp_tx;
129 s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
130 s->rx_wqe_err += rq_stats->wqe_err;
131 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
132 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
133 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
134 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
135 s->rx_page_reuse += rq_stats->page_reuse;
136 s->rx_cache_reuse += rq_stats->cache_reuse;
137 s->rx_cache_full += rq_stats->cache_full;
138 s->rx_cache_empty += rq_stats->cache_empty;
139 s->rx_cache_busy += rq_stats->cache_busy;
140 s->rx_cache_waive += rq_stats->cache_waive;
141 s->ch_eq_rearm += ch_stats->eq_rearm;
142
143 for (j = 0; j < priv->channels.params.num_tc; j++) {
144 sq_stats = &c->sq[j].stats;
145
146 s->tx_packets += sq_stats->packets;
147 s->tx_bytes += sq_stats->bytes;
148 s->tx_tso_packets += sq_stats->tso_packets;
149 s->tx_tso_bytes += sq_stats->tso_bytes;
150 s->tx_tso_inner_packets += sq_stats->tso_inner_packets;
151 s->tx_tso_inner_bytes += sq_stats->tso_inner_bytes;
152 s->tx_added_vlan_packets += sq_stats->added_vlan_packets;
153 s->tx_queue_stopped += sq_stats->stopped;
154 s->tx_queue_wake += sq_stats->wake;
155 s->tx_queue_dropped += sq_stats->dropped;
156 s->tx_xmit_more += sq_stats->xmit_more;
157 s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
158 s->tx_csum_none += sq_stats->csum_none;
159 s->tx_csum_partial += sq_stats->csum_partial;
160 }
161 }
162
163 s->link_down_events_phy = MLX5_GET(ppcnt_reg,
164 priv->stats.pport.phy_counters,
165 counter_set.phys_layer_cntrs.link_down_events);
166 memcpy(&priv->stats.sw, s, sizeof(*s));
167}
168
102static const struct counter_desc q_stats_desc[] = { 169static const struct counter_desc q_stats_desc[] = {
103 { MLX5E_DECLARE_STAT(struct mlx5e_qcounter_stats, rx_out_of_buffer) }, 170 { MLX5E_DECLARE_STAT(struct mlx5e_qcounter_stats, rx_out_of_buffer) },
104}; 171};
@@ -128,6 +195,22 @@ static int mlx5e_grp_q_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
128 return idx; 195 return idx;
129} 196}
130 197
198static void mlx5e_grp_q_update_stats(struct mlx5e_priv *priv)
199{
200 struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
201 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)];
202 int err;
203
204 if (!priv->q_counter)
205 return;
206
207 err = mlx5_core_query_q_counter(priv->mdev, priv->q_counter, 0, out, sizeof(out));
208 if (err)
209 return;
210
211 qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out, out, out_of_buffer);
212}
213
131#define VPORT_COUNTER_OFF(c) MLX5_BYTE_OFF(query_vport_counter_out, c) 214#define VPORT_COUNTER_OFF(c) MLX5_BYTE_OFF(query_vport_counter_out, c)
132static const struct counter_desc vport_stats_desc[] = { 215static const struct counter_desc vport_stats_desc[] = {
133 { "rx_vport_unicast_packets", 216 { "rx_vport_unicast_packets",
@@ -200,6 +283,19 @@ static int mlx5e_grp_vport_fill_stats(struct mlx5e_priv *priv, u64 *data,
200 return idx; 283 return idx;
201} 284}
202 285
286static void mlx5e_grp_vport_update_stats(struct mlx5e_priv *priv)
287{
288 int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
289 u32 *out = (u32 *)priv->stats.vport.query_vport_out;
290 u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
291 struct mlx5_core_dev *mdev = priv->mdev;
292
293 MLX5_SET(query_vport_counter_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_COUNTER);
294 MLX5_SET(query_vport_counter_in, in, op_mod, 0);
295 MLX5_SET(query_vport_counter_in, in, other_vport, 0);
296 mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
297}
298
203#define PPORT_802_3_OFF(c) \ 299#define PPORT_802_3_OFF(c) \
204 MLX5_BYTE_OFF(ppcnt_reg, \ 300 MLX5_BYTE_OFF(ppcnt_reg, \
205 counter_set.eth_802_3_cntrs_grp_data_layout.c##_high) 301 counter_set.eth_802_3_cntrs_grp_data_layout.c##_high)
@@ -252,6 +348,20 @@ static int mlx5e_grp_802_3_fill_stats(struct mlx5e_priv *priv, u64 *data,
252 return idx; 348 return idx;
253} 349}
254 350
351static void mlx5e_grp_802_3_update_stats(struct mlx5e_priv *priv)
352{
353 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
354 struct mlx5_core_dev *mdev = priv->mdev;
355 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
356 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
357 void *out;
358
359 MLX5_SET(ppcnt_reg, in, local_port, 1);
360 out = pstats->IEEE_802_3_counters;
361 MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
362 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
363}
364
255#define PPORT_2863_OFF(c) \ 365#define PPORT_2863_OFF(c) \
256 MLX5_BYTE_OFF(ppcnt_reg, \ 366 MLX5_BYTE_OFF(ppcnt_reg, \
257 counter_set.eth_2863_cntrs_grp_data_layout.c##_high) 367 counter_set.eth_2863_cntrs_grp_data_layout.c##_high)
@@ -289,6 +399,20 @@ static int mlx5e_grp_2863_fill_stats(struct mlx5e_priv *priv, u64 *data,
289 return idx; 399 return idx;
290} 400}
291 401
402static void mlx5e_grp_2863_update_stats(struct mlx5e_priv *priv)
403{
404 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
405 struct mlx5_core_dev *mdev = priv->mdev;
406 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
407 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
408 void *out;
409
410 MLX5_SET(ppcnt_reg, in, local_port, 1);
411 out = pstats->RFC_2863_counters;
412 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
413 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
414}
415
292#define PPORT_2819_OFF(c) \ 416#define PPORT_2819_OFF(c) \
293 MLX5_BYTE_OFF(ppcnt_reg, \ 417 MLX5_BYTE_OFF(ppcnt_reg, \
294 counter_set.eth_2819_cntrs_grp_data_layout.c##_high) 418 counter_set.eth_2819_cntrs_grp_data_layout.c##_high)
@@ -336,6 +460,20 @@ static int mlx5e_grp_2819_fill_stats(struct mlx5e_priv *priv, u64 *data,
336 return idx; 460 return idx;
337} 461}
338 462
463static void mlx5e_grp_2819_update_stats(struct mlx5e_priv *priv)
464{
465 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
466 struct mlx5_core_dev *mdev = priv->mdev;
467 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
468 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
469 void *out;
470
471 MLX5_SET(ppcnt_reg, in, local_port, 1);
472 out = pstats->RFC_2819_counters;
473 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
474 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
475}
476
339#define PPORT_PHY_STATISTICAL_OFF(c) \ 477#define PPORT_PHY_STATISTICAL_OFF(c) \
340 MLX5_BYTE_OFF(ppcnt_reg, \ 478 MLX5_BYTE_OFF(ppcnt_reg, \
341 counter_set.phys_layer_statistical_cntrs.c##_high) 479 counter_set.phys_layer_statistical_cntrs.c##_high)
@@ -376,6 +514,27 @@ static int mlx5e_grp_phy_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
376 return idx; 514 return idx;
377} 515}
378 516
517static void mlx5e_grp_phy_update_stats(struct mlx5e_priv *priv)
518{
519 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
520 struct mlx5_core_dev *mdev = priv->mdev;
521 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
522 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
523 void *out;
524
525 MLX5_SET(ppcnt_reg, in, local_port, 1);
526 out = pstats->phy_counters;
527 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
528 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
529
530 if (!MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group))
531 return;
532
533 out = pstats->phy_statistical_counters;
534 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
535 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
536}
537
379#define PPORT_ETH_EXT_OFF(c) \ 538#define PPORT_ETH_EXT_OFF(c) \
380 MLX5_BYTE_OFF(ppcnt_reg, \ 539 MLX5_BYTE_OFF(ppcnt_reg, \
381 counter_set.eth_extended_cntrs_grp_data_layout.c##_high) 540 counter_set.eth_extended_cntrs_grp_data_layout.c##_high)
@@ -418,6 +577,23 @@ static int mlx5e_grp_eth_ext_fill_stats(struct mlx5e_priv *priv, u64 *data,
418 return idx; 577 return idx;
419} 578}
420 579
580static void mlx5e_grp_eth_ext_update_stats(struct mlx5e_priv *priv)
581{
582 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
583 struct mlx5_core_dev *mdev = priv->mdev;
584 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
585 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
586 void *out;
587
588 if (!MLX5_CAP_PCAM_FEATURE(mdev, rx_buffer_fullness_counters))
589 return;
590
591 MLX5_SET(ppcnt_reg, in, local_port, 1);
592 out = pstats->eth_ext_counters;
593 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
594 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
595}
596
421#define PCIE_PERF_OFF(c) \ 597#define PCIE_PERF_OFF(c) \
422 MLX5_BYTE_OFF(mpcnt_reg, counter_set.pcie_perf_cntrs_grp_data_layout.c) 598 MLX5_BYTE_OFF(mpcnt_reg, counter_set.pcie_perf_cntrs_grp_data_layout.c)
423static const struct counter_desc pcie_perf_stats_desc[] = { 599static const struct counter_desc pcie_perf_stats_desc[] = {
@@ -505,6 +681,22 @@ static int mlx5e_grp_pcie_fill_stats(struct mlx5e_priv *priv, u64 *data,
505 return idx; 681 return idx;
506} 682}
507 683
684static void mlx5e_grp_pcie_update_stats(struct mlx5e_priv *priv)
685{
686 struct mlx5e_pcie_stats *pcie_stats = &priv->stats.pcie;
687 struct mlx5_core_dev *mdev = priv->mdev;
688 u32 in[MLX5_ST_SZ_DW(mpcnt_reg)] = {0};
689 int sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
690 void *out;
691
692 if (!MLX5_CAP_MCAM_FEATURE(mdev, pcie_performance_group))
693 return;
694
695 out = pcie_stats->pcie_perf_counters;
696 MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
697 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
698}
699
508#define PPORT_PER_PRIO_OFF(c) \ 700#define PPORT_PER_PRIO_OFF(c) \
509 MLX5_BYTE_OFF(ppcnt_reg, \ 701 MLX5_BYTE_OFF(ppcnt_reg, \
510 counter_set.eth_per_prio_grp_data_layout.c##_high) 702 counter_set.eth_per_prio_grp_data_layout.c##_high)
@@ -656,6 +848,47 @@ static int mlx5e_grp_per_prio_pfc_fill_stats(struct mlx5e_priv *priv,
656 return idx; 848 return idx;
657} 849}
658 850
851static int mlx5e_grp_per_prio_get_num_stats(struct mlx5e_priv *priv)
852{
853 return mlx5e_grp_per_prio_traffic_get_num_stats(priv) +
854 mlx5e_grp_per_prio_pfc_get_num_stats(priv);
855}
856
857static int mlx5e_grp_per_prio_fill_strings(struct mlx5e_priv *priv, u8 *data,
858 int idx)
859{
860 idx = mlx5e_grp_per_prio_traffic_fill_strings(priv, data, idx);
861 idx = mlx5e_grp_per_prio_pfc_fill_strings(priv, data, idx);
862 return idx;
863}
864
865static int mlx5e_grp_per_prio_fill_stats(struct mlx5e_priv *priv, u64 *data,
866 int idx)
867{
868 idx = mlx5e_grp_per_prio_traffic_fill_stats(priv, data, idx);
869 idx = mlx5e_grp_per_prio_pfc_fill_stats(priv, data, idx);
870 return idx;
871}
872
873static void mlx5e_grp_per_prio_update_stats(struct mlx5e_priv *priv)
874{
875 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
876 struct mlx5_core_dev *mdev = priv->mdev;
877 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
878 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
879 int prio;
880 void *out;
881
882 MLX5_SET(ppcnt_reg, in, local_port, 1);
883 MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
884 for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
885 out = pstats->per_prio_counters[prio];
886 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
887 mlx5_core_access_reg(mdev, in, sz, out, sz,
888 MLX5_REG_PPCNT, 0, 0);
889 }
890}
891
659static const struct counter_desc mlx5e_pme_status_desc[] = { 892static const struct counter_desc mlx5e_pme_status_desc[] = {
660 { "module_unplug", 8 }, 893 { "module_unplug", 8 },
661}; 894};
@@ -723,6 +956,11 @@ static int mlx5e_grp_ipsec_fill_stats(struct mlx5e_priv *priv, u64 *data,
723 return idx + mlx5e_ipsec_get_stats(priv, data + idx); 956 return idx + mlx5e_ipsec_get_stats(priv, data + idx);
724} 957}
725 958
959static void mlx5e_grp_ipsec_update_stats(struct mlx5e_priv *priv)
960{
961 mlx5e_ipsec_update_stats(priv);
962}
963
726static const struct counter_desc rq_stats_desc[] = { 964static const struct counter_desc rq_stats_desc[] = {
727 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, packets) }, 965 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, packets) },
728 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, bytes) }, 966 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, bytes) },
@@ -767,12 +1005,18 @@ static const struct counter_desc sq_stats_desc[] = {
767 { MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, xmit_more) }, 1005 { MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, xmit_more) },
768}; 1006};
769 1007
1008static const struct counter_desc ch_stats_desc[] = {
1009 { MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, eq_rearm) },
1010};
1011
770#define NUM_RQ_STATS ARRAY_SIZE(rq_stats_desc) 1012#define NUM_RQ_STATS ARRAY_SIZE(rq_stats_desc)
771#define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc) 1013#define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc)
1014#define NUM_CH_STATS ARRAY_SIZE(ch_stats_desc)
772 1015
773static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv) 1016static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)
774{ 1017{
775 return (NUM_RQ_STATS * priv->channels.num) + 1018 return (NUM_RQ_STATS * priv->channels.num) +
1019 (NUM_CH_STATS * priv->channels.num) +
776 (NUM_SQ_STATS * priv->channels.num * priv->channels.params.num_tc); 1020 (NUM_SQ_STATS * priv->channels.num * priv->channels.params.num_tc);
777} 1021}
778 1022
@@ -785,6 +1029,11 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data,
785 return idx; 1029 return idx;
786 1030
787 for (i = 0; i < priv->channels.num; i++) 1031 for (i = 0; i < priv->channels.num; i++)
1032 for (j = 0; j < NUM_CH_STATS; j++)
1033 sprintf(data + (idx++) * ETH_GSTRING_LEN,
1034 ch_stats_desc[j].format, i);
1035
1036 for (i = 0; i < priv->channels.num; i++)
788 for (j = 0; j < NUM_RQ_STATS; j++) 1037 for (j = 0; j < NUM_RQ_STATS; j++)
789 sprintf(data + (idx++) * ETH_GSTRING_LEN, rq_stats_desc[j].format, i); 1038 sprintf(data + (idx++) * ETH_GSTRING_LEN, rq_stats_desc[j].format, i);
790 1039
@@ -808,6 +1057,12 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv *priv, u64 *data,
808 return idx; 1057 return idx;
809 1058
810 for (i = 0; i < channels->num; i++) 1059 for (i = 0; i < channels->num; i++)
1060 for (j = 0; j < NUM_CH_STATS; j++)
1061 data[idx++] =
1062 MLX5E_READ_CTR64_CPU(&channels->c[i]->stats,
1063 ch_stats_desc, j);
1064
1065 for (i = 0; i < channels->num; i++)
811 for (j = 0; j < NUM_RQ_STATS; j++) 1066 for (j = 0; j < NUM_RQ_STATS; j++)
812 data[idx++] = 1067 data[idx++] =
813 MLX5E_READ_CTR64_CPU(&channels->c[i]->rq.stats, 1068 MLX5E_READ_CTR64_CPU(&channels->c[i]->rq.stats,
@@ -823,61 +1078,71 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv *priv, u64 *data,
823 return idx; 1078 return idx;
824} 1079}
825 1080
1081/* The stats groups order is opposite to the update_stats() order calls */
826const struct mlx5e_stats_grp mlx5e_stats_grps[] = { 1082const struct mlx5e_stats_grp mlx5e_stats_grps[] = {
827 { 1083 {
828 .get_num_stats = mlx5e_grp_sw_get_num_stats, 1084 .get_num_stats = mlx5e_grp_sw_get_num_stats,
829 .fill_strings = mlx5e_grp_sw_fill_strings, 1085 .fill_strings = mlx5e_grp_sw_fill_strings,
830 .fill_stats = mlx5e_grp_sw_fill_stats, 1086 .fill_stats = mlx5e_grp_sw_fill_stats,
1087 .update_stats_mask = MLX5E_NDO_UPDATE_STATS,
1088 .update_stats = mlx5e_grp_sw_update_stats,
831 }, 1089 },
832 { 1090 {
833 .get_num_stats = mlx5e_grp_q_get_num_stats, 1091 .get_num_stats = mlx5e_grp_q_get_num_stats,
834 .fill_strings = mlx5e_grp_q_fill_strings, 1092 .fill_strings = mlx5e_grp_q_fill_strings,
835 .fill_stats = mlx5e_grp_q_fill_stats, 1093 .fill_stats = mlx5e_grp_q_fill_stats,
1094 .update_stats_mask = MLX5E_NDO_UPDATE_STATS,
1095 .update_stats = mlx5e_grp_q_update_stats,
836 }, 1096 },
837 { 1097 {
838 .get_num_stats = mlx5e_grp_vport_get_num_stats, 1098 .get_num_stats = mlx5e_grp_vport_get_num_stats,
839 .fill_strings = mlx5e_grp_vport_fill_strings, 1099 .fill_strings = mlx5e_grp_vport_fill_strings,
840 .fill_stats = mlx5e_grp_vport_fill_stats, 1100 .fill_stats = mlx5e_grp_vport_fill_stats,
1101 .update_stats_mask = MLX5E_NDO_UPDATE_STATS,
1102 .update_stats = mlx5e_grp_vport_update_stats,
841 }, 1103 },
842 { 1104 {
843 .get_num_stats = mlx5e_grp_802_3_get_num_stats, 1105 .get_num_stats = mlx5e_grp_802_3_get_num_stats,
844 .fill_strings = mlx5e_grp_802_3_fill_strings, 1106 .fill_strings = mlx5e_grp_802_3_fill_strings,
845 .fill_stats = mlx5e_grp_802_3_fill_stats, 1107 .fill_stats = mlx5e_grp_802_3_fill_stats,
1108 .update_stats_mask = MLX5E_NDO_UPDATE_STATS,
1109 .update_stats = mlx5e_grp_802_3_update_stats,
846 }, 1110 },
847 { 1111 {
848 .get_num_stats = mlx5e_grp_2863_get_num_stats, 1112 .get_num_stats = mlx5e_grp_2863_get_num_stats,
849 .fill_strings = mlx5e_grp_2863_fill_strings, 1113 .fill_strings = mlx5e_grp_2863_fill_strings,
850 .fill_stats = mlx5e_grp_2863_fill_stats, 1114 .fill_stats = mlx5e_grp_2863_fill_stats,
1115 .update_stats = mlx5e_grp_2863_update_stats,
851 }, 1116 },
852 { 1117 {
853 .get_num_stats = mlx5e_grp_2819_get_num_stats, 1118 .get_num_stats = mlx5e_grp_2819_get_num_stats,
854 .fill_strings = mlx5e_grp_2819_fill_strings, 1119 .fill_strings = mlx5e_grp_2819_fill_strings,
855 .fill_stats = mlx5e_grp_2819_fill_stats, 1120 .fill_stats = mlx5e_grp_2819_fill_stats,
1121 .update_stats = mlx5e_grp_2819_update_stats,
856 }, 1122 },
857 { 1123 {
858 .get_num_stats = mlx5e_grp_phy_get_num_stats, 1124 .get_num_stats = mlx5e_grp_phy_get_num_stats,
859 .fill_strings = mlx5e_grp_phy_fill_strings, 1125 .fill_strings = mlx5e_grp_phy_fill_strings,
860 .fill_stats = mlx5e_grp_phy_fill_stats, 1126 .fill_stats = mlx5e_grp_phy_fill_stats,
1127 .update_stats = mlx5e_grp_phy_update_stats,
861 }, 1128 },
862 { 1129 {
863 .get_num_stats = mlx5e_grp_eth_ext_get_num_stats, 1130 .get_num_stats = mlx5e_grp_eth_ext_get_num_stats,
864 .fill_strings = mlx5e_grp_eth_ext_fill_strings, 1131 .fill_strings = mlx5e_grp_eth_ext_fill_strings,
865 .fill_stats = mlx5e_grp_eth_ext_fill_stats, 1132 .fill_stats = mlx5e_grp_eth_ext_fill_stats,
1133 .update_stats = mlx5e_grp_eth_ext_update_stats,
866 }, 1134 },
867 { 1135 {
868 .get_num_stats = mlx5e_grp_pcie_get_num_stats, 1136 .get_num_stats = mlx5e_grp_pcie_get_num_stats,
869 .fill_strings = mlx5e_grp_pcie_fill_strings, 1137 .fill_strings = mlx5e_grp_pcie_fill_strings,
870 .fill_stats = mlx5e_grp_pcie_fill_stats, 1138 .fill_stats = mlx5e_grp_pcie_fill_stats,
1139 .update_stats = mlx5e_grp_pcie_update_stats,
871 }, 1140 },
872 { 1141 {
873 .get_num_stats = mlx5e_grp_per_prio_traffic_get_num_stats, 1142 .get_num_stats = mlx5e_grp_per_prio_get_num_stats,
874 .fill_strings = mlx5e_grp_per_prio_traffic_fill_strings, 1143 .fill_strings = mlx5e_grp_per_prio_fill_strings,
875 .fill_stats = mlx5e_grp_per_prio_traffic_fill_stats, 1144 .fill_stats = mlx5e_grp_per_prio_fill_stats,
876 }, 1145 .update_stats = mlx5e_grp_per_prio_update_stats,
877 {
878 .get_num_stats = mlx5e_grp_per_prio_pfc_get_num_stats,
879 .fill_strings = mlx5e_grp_per_prio_pfc_fill_strings,
880 .fill_stats = mlx5e_grp_per_prio_pfc_fill_stats,
881 }, 1146 },
882 { 1147 {
883 .get_num_stats = mlx5e_grp_pme_get_num_stats, 1148 .get_num_stats = mlx5e_grp_pme_get_num_stats,
@@ -888,6 +1153,7 @@ const struct mlx5e_stats_grp mlx5e_stats_grps[] = {
888 .get_num_stats = mlx5e_grp_ipsec_get_num_stats, 1153 .get_num_stats = mlx5e_grp_ipsec_get_num_stats,
889 .fill_strings = mlx5e_grp_ipsec_fill_strings, 1154 .fill_strings = mlx5e_grp_ipsec_fill_strings,
890 .fill_stats = mlx5e_grp_ipsec_fill_stats, 1155 .fill_stats = mlx5e_grp_ipsec_fill_stats,
1156 .update_stats = mlx5e_grp_ipsec_update_stats,
891 }, 1157 },
892 { 1158 {
893 .get_num_stats = mlx5e_grp_channels_get_num_stats, 1159 .get_num_stats = mlx5e_grp_channels_get_num_stats,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index d679e21f686e..0b3320a2b072 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -44,6 +44,7 @@
44#define MLX5E_DECLARE_STAT(type, fld) #fld, offsetof(type, fld) 44#define MLX5E_DECLARE_STAT(type, fld) #fld, offsetof(type, fld)
45#define MLX5E_DECLARE_RX_STAT(type, fld) "rx%d_"#fld, offsetof(type, fld) 45#define MLX5E_DECLARE_RX_STAT(type, fld) "rx%d_"#fld, offsetof(type, fld)
46#define MLX5E_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld) 46#define MLX5E_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld)
47#define MLX5E_DECLARE_CH_STAT(type, fld) "ch%d_"#fld, offsetof(type, fld)
47 48
48struct counter_desc { 49struct counter_desc {
49 char format[ETH_GSTRING_LEN]; 50 char format[ETH_GSTRING_LEN];
@@ -88,6 +89,7 @@ struct mlx5e_sw_stats {
88 u64 rx_cache_empty; 89 u64 rx_cache_empty;
89 u64 rx_cache_busy; 90 u64 rx_cache_busy;
90 u64 rx_cache_waive; 91 u64 rx_cache_waive;
92 u64 ch_eq_rearm;
91 93
92 /* Special handling counters */ 94 /* Special handling counters */
93 u64 link_down_events_phy; 95 u64 link_down_events_phy;
@@ -192,6 +194,10 @@ struct mlx5e_sq_stats {
192 u64 dropped; 194 u64 dropped;
193}; 195};
194 196
197struct mlx5e_ch_stats {
198 u64 eq_rearm;
199};
200
195struct mlx5e_stats { 201struct mlx5e_stats {
196 struct mlx5e_sw_stats sw; 202 struct mlx5e_sw_stats sw;
197 struct mlx5e_qcounter_stats qcnt; 203 struct mlx5e_qcounter_stats qcnt;
@@ -201,11 +207,17 @@ struct mlx5e_stats {
201 struct mlx5e_pcie_stats pcie; 207 struct mlx5e_pcie_stats pcie;
202}; 208};
203 209
210enum {
211 MLX5E_NDO_UPDATE_STATS = BIT(0x1),
212};
213
204struct mlx5e_priv; 214struct mlx5e_priv;
205struct mlx5e_stats_grp { 215struct mlx5e_stats_grp {
216 u16 update_stats_mask;
206 int (*get_num_stats)(struct mlx5e_priv *priv); 217 int (*get_num_stats)(struct mlx5e_priv *priv);
207 int (*fill_strings)(struct mlx5e_priv *priv, u8 *data, int idx); 218 int (*fill_strings)(struct mlx5e_priv *priv, u8 *data, int idx);
208 int (*fill_stats)(struct mlx5e_priv *priv, u64 *data, int idx); 219 int (*fill_stats)(struct mlx5e_priv *priv, u64 *data, int idx);
220 void (*update_stats)(struct mlx5e_priv *priv);
209}; 221};
210 222
211extern const struct mlx5e_stats_grp mlx5e_stats_grps[]; 223extern const struct mlx5e_stats_grp mlx5e_stats_grps[];
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index cf528da51243..fd98b0dc610f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -51,12 +51,14 @@
51#include "en_tc.h" 51#include "en_tc.h"
52#include "eswitch.h" 52#include "eswitch.h"
53#include "vxlan.h" 53#include "vxlan.h"
54#include "fs_core.h"
54 55
55struct mlx5_nic_flow_attr { 56struct mlx5_nic_flow_attr {
56 u32 action; 57 u32 action;
57 u32 flow_tag; 58 u32 flow_tag;
58 u32 mod_hdr_id; 59 u32 mod_hdr_id;
59 u32 hairpin_tirn; 60 u32 hairpin_tirn;
61 struct mlx5_flow_table *hairpin_ft;
60}; 62};
61 63
62enum { 64enum {
@@ -64,6 +66,7 @@ enum {
64 MLX5E_TC_FLOW_NIC = BIT(1), 66 MLX5E_TC_FLOW_NIC = BIT(1),
65 MLX5E_TC_FLOW_OFFLOADED = BIT(2), 67 MLX5E_TC_FLOW_OFFLOADED = BIT(2),
66 MLX5E_TC_FLOW_HAIRPIN = BIT(3), 68 MLX5E_TC_FLOW_HAIRPIN = BIT(3),
69 MLX5E_TC_FLOW_HAIRPIN_RSS = BIT(4),
67}; 70};
68 71
69struct mlx5e_tc_flow { 72struct mlx5e_tc_flow {
@@ -100,8 +103,14 @@ struct mlx5e_hairpin {
100 struct mlx5_hairpin *pair; 103 struct mlx5_hairpin *pair;
101 104
102 struct mlx5_core_dev *func_mdev; 105 struct mlx5_core_dev *func_mdev;
106 struct mlx5e_priv *func_priv;
103 u32 tdn; 107 u32 tdn;
104 u32 tirn; 108 u32 tirn;
109
110 int num_channels;
111 struct mlx5e_rqt indir_rqt;
112 u32 indir_tirn[MLX5E_NUM_INDIR_TIRS];
113 struct mlx5e_ttc_table ttc;
105}; 114};
106 115
107struct mlx5e_hairpin_entry { 116struct mlx5e_hairpin_entry {
@@ -111,7 +120,8 @@ struct mlx5e_hairpin_entry {
111 /* flows sharing the same hairpin */ 120 /* flows sharing the same hairpin */
112 struct list_head flows; 121 struct list_head flows;
113 122
114 int peer_ifindex; 123 u16 peer_vhca_id;
124 u8 prio;
115 struct mlx5e_hairpin *hp; 125 struct mlx5e_hairpin *hp;
116}; 126};
117 127
@@ -268,7 +278,7 @@ static int mlx5e_hairpin_create_transport(struct mlx5e_hairpin *hp)
268 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 278 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
269 279
270 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT); 280 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
271 MLX5_SET(tirc, tirc, inline_rqn, hp->pair->rqn); 281 MLX5_SET(tirc, tirc, inline_rqn, hp->pair->rqn[0]);
272 MLX5_SET(tirc, tirc, transport_domain, hp->tdn); 282 MLX5_SET(tirc, tirc, transport_domain, hp->tdn);
273 283
274 err = mlx5_core_create_tir(hp->func_mdev, in, MLX5_ST_SZ_BYTES(create_tir_in), &hp->tirn); 284 err = mlx5_core_create_tir(hp->func_mdev, in, MLX5_ST_SZ_BYTES(create_tir_in), &hp->tirn);
@@ -289,6 +299,151 @@ static void mlx5e_hairpin_destroy_transport(struct mlx5e_hairpin *hp)
289 mlx5_core_dealloc_transport_domain(hp->func_mdev, hp->tdn); 299 mlx5_core_dealloc_transport_domain(hp->func_mdev, hp->tdn);
290} 300}
291 301
302static void mlx5e_hairpin_fill_rqt_rqns(struct mlx5e_hairpin *hp, void *rqtc)
303{
304 u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE], rqn;
305 struct mlx5e_priv *priv = hp->func_priv;
306 int i, ix, sz = MLX5E_INDIR_RQT_SIZE;
307
308 mlx5e_build_default_indir_rqt(indirection_rqt, sz,
309 hp->num_channels);
310
311 for (i = 0; i < sz; i++) {
312 ix = i;
313 if (priv->channels.params.rss_hfunc == ETH_RSS_HASH_XOR)
314 ix = mlx5e_bits_invert(i, ilog2(sz));
315 ix = indirection_rqt[ix];
316 rqn = hp->pair->rqn[ix];
317 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
318 }
319}
320
321static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin *hp)
322{
323 int inlen, err, sz = MLX5E_INDIR_RQT_SIZE;
324 struct mlx5e_priv *priv = hp->func_priv;
325 struct mlx5_core_dev *mdev = priv->mdev;
326 void *rqtc;
327 u32 *in;
328
329 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
330 in = kvzalloc(inlen, GFP_KERNEL);
331 if (!in)
332 return -ENOMEM;
333
334 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
335
336 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
337 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
338
339 mlx5e_hairpin_fill_rqt_rqns(hp, rqtc);
340
341 err = mlx5_core_create_rqt(mdev, in, inlen, &hp->indir_rqt.rqtn);
342 if (!err)
343 hp->indir_rqt.enabled = true;
344
345 kvfree(in);
346 return err;
347}
348
349static int mlx5e_hairpin_create_indirect_tirs(struct mlx5e_hairpin *hp)
350{
351 struct mlx5e_priv *priv = hp->func_priv;
352 u32 in[MLX5_ST_SZ_DW(create_tir_in)];
353 int tt, i, err;
354 void *tirc;
355
356 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
357 memset(in, 0, MLX5_ST_SZ_BYTES(create_tir_in));
358 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
359
360 MLX5_SET(tirc, tirc, transport_domain, hp->tdn);
361 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
362 MLX5_SET(tirc, tirc, indirect_table, hp->indir_rqt.rqtn);
363 mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, false);
364
365 err = mlx5_core_create_tir(hp->func_mdev, in,
366 MLX5_ST_SZ_BYTES(create_tir_in), &hp->indir_tirn[tt]);
367 if (err) {
368 mlx5_core_warn(hp->func_mdev, "create indirect tirs failed, %d\n", err);
369 goto err_destroy_tirs;
370 }
371 }
372 return 0;
373
374err_destroy_tirs:
375 for (i = 0; i < tt; i++)
376 mlx5_core_destroy_tir(hp->func_mdev, hp->indir_tirn[i]);
377 return err;
378}
379
380static void mlx5e_hairpin_destroy_indirect_tirs(struct mlx5e_hairpin *hp)
381{
382 int tt;
383
384 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
385 mlx5_core_destroy_tir(hp->func_mdev, hp->indir_tirn[tt]);
386}
387
388static void mlx5e_hairpin_set_ttc_params(struct mlx5e_hairpin *hp,
389 struct ttc_params *ttc_params)
390{
391 struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
392 int tt;
393
394 memset(ttc_params, 0, sizeof(*ttc_params));
395
396 ttc_params->any_tt_tirn = hp->tirn;
397
398 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
399 ttc_params->indir_tirn[tt] = hp->indir_tirn[tt];
400
401 ft_attr->max_fte = MLX5E_NUM_TT;
402 ft_attr->level = MLX5E_TC_TTC_FT_LEVEL;
403 ft_attr->prio = MLX5E_TC_PRIO;
404}
405
406static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp)
407{
408 struct mlx5e_priv *priv = hp->func_priv;
409 struct ttc_params ttc_params;
410 int err;
411
412 err = mlx5e_hairpin_create_indirect_rqt(hp);
413 if (err)
414 return err;
415
416 err = mlx5e_hairpin_create_indirect_tirs(hp);
417 if (err)
418 goto err_create_indirect_tirs;
419
420 mlx5e_hairpin_set_ttc_params(hp, &ttc_params);
421 err = mlx5e_create_ttc_table(priv, &ttc_params, &hp->ttc);
422 if (err)
423 goto err_create_ttc_table;
424
425 netdev_dbg(priv->netdev, "add hairpin: using %d channels rss ttc table id %x\n",
426 hp->num_channels, hp->ttc.ft.t->id);
427
428 return 0;
429
430err_create_ttc_table:
431 mlx5e_hairpin_destroy_indirect_tirs(hp);
432err_create_indirect_tirs:
433 mlx5e_destroy_rqt(priv, &hp->indir_rqt);
434
435 return err;
436}
437
438static void mlx5e_hairpin_rss_cleanup(struct mlx5e_hairpin *hp)
439{
440 struct mlx5e_priv *priv = hp->func_priv;
441
442 mlx5e_destroy_ttc_table(priv, &hp->ttc);
443 mlx5e_hairpin_destroy_indirect_tirs(hp);
444 mlx5e_destroy_rqt(priv, &hp->indir_rqt);
445}
446
292static struct mlx5e_hairpin * 447static struct mlx5e_hairpin *
293mlx5e_hairpin_create(struct mlx5e_priv *priv, struct mlx5_hairpin_params *params, 448mlx5e_hairpin_create(struct mlx5e_priv *priv, struct mlx5_hairpin_params *params,
294 int peer_ifindex) 449 int peer_ifindex)
@@ -312,13 +467,23 @@ mlx5e_hairpin_create(struct mlx5e_priv *priv, struct mlx5_hairpin_params *params
312 } 467 }
313 hp->pair = pair; 468 hp->pair = pair;
314 hp->func_mdev = func_mdev; 469 hp->func_mdev = func_mdev;
470 hp->func_priv = priv;
471 hp->num_channels = params->num_channels;
315 472
316 err = mlx5e_hairpin_create_transport(hp); 473 err = mlx5e_hairpin_create_transport(hp);
317 if (err) 474 if (err)
318 goto create_transport_err; 475 goto create_transport_err;
319 476
477 if (hp->num_channels > 1) {
478 err = mlx5e_hairpin_rss_init(hp);
479 if (err)
480 goto rss_init_err;
481 }
482
320 return hp; 483 return hp;
321 484
485rss_init_err:
486 mlx5e_hairpin_destroy_transport(hp);
322create_transport_err: 487create_transport_err:
323 mlx5_core_hairpin_destroy(hp->pair); 488 mlx5_core_hairpin_destroy(hp->pair);
324create_pair_err: 489create_pair_err:
@@ -328,41 +493,96 @@ create_pair_err:
328 493
329static void mlx5e_hairpin_destroy(struct mlx5e_hairpin *hp) 494static void mlx5e_hairpin_destroy(struct mlx5e_hairpin *hp)
330{ 495{
496 if (hp->num_channels > 1)
497 mlx5e_hairpin_rss_cleanup(hp);
331 mlx5e_hairpin_destroy_transport(hp); 498 mlx5e_hairpin_destroy_transport(hp);
332 mlx5_core_hairpin_destroy(hp->pair); 499 mlx5_core_hairpin_destroy(hp->pair);
333 kvfree(hp); 500 kvfree(hp);
334} 501}
335 502
503static inline u32 hash_hairpin_info(u16 peer_vhca_id, u8 prio)
504{
505 return (peer_vhca_id << 16 | prio);
506}
507
336static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv, 508static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv,
337 int peer_ifindex) 509 u16 peer_vhca_id, u8 prio)
338{ 510{
339 struct mlx5e_hairpin_entry *hpe; 511 struct mlx5e_hairpin_entry *hpe;
512 u32 hash_key = hash_hairpin_info(peer_vhca_id, prio);
340 513
341 hash_for_each_possible(priv->fs.tc.hairpin_tbl, hpe, 514 hash_for_each_possible(priv->fs.tc.hairpin_tbl, hpe,
342 hairpin_hlist, peer_ifindex) { 515 hairpin_hlist, hash_key) {
343 if (hpe->peer_ifindex == peer_ifindex) 516 if (hpe->peer_vhca_id == peer_vhca_id && hpe->prio == prio)
344 return hpe; 517 return hpe;
345 } 518 }
346 519
347 return NULL; 520 return NULL;
348} 521}
349 522
523#define UNKNOWN_MATCH_PRIO 8
524
525static int mlx5e_hairpin_get_prio(struct mlx5e_priv *priv,
526 struct mlx5_flow_spec *spec, u8 *match_prio)
527{
528 void *headers_c, *headers_v;
529 u8 prio_val, prio_mask = 0;
530 bool vlan_present;
531
532#ifdef CONFIG_MLX5_CORE_EN_DCB
533 if (priv->dcbx_dp.trust_state != MLX5_QPTS_TRUST_PCP) {
534 netdev_warn(priv->netdev,
535 "only PCP trust state supported for hairpin\n");
536 return -EOPNOTSUPP;
537 }
538#endif
539 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers);
540 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers);
541
542 vlan_present = MLX5_GET(fte_match_set_lyr_2_4, headers_v, cvlan_tag);
543 if (vlan_present) {
544 prio_mask = MLX5_GET(fte_match_set_lyr_2_4, headers_c, first_prio);
545 prio_val = MLX5_GET(fte_match_set_lyr_2_4, headers_v, first_prio);
546 }
547
548 if (!vlan_present || !prio_mask) {
549 prio_val = UNKNOWN_MATCH_PRIO;
550 } else if (prio_mask != 0x7) {
551 netdev_warn(priv->netdev,
552 "masked priority match not supported for hairpin\n");
553 return -EOPNOTSUPP;
554 }
555
556 *match_prio = prio_val;
557 return 0;
558}
559
350static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, 560static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
351 struct mlx5e_tc_flow *flow, 561 struct mlx5e_tc_flow *flow,
352 struct mlx5e_tc_flow_parse_attr *parse_attr) 562 struct mlx5e_tc_flow_parse_attr *parse_attr)
353{ 563{
354 int peer_ifindex = parse_attr->mirred_ifindex; 564 int peer_ifindex = parse_attr->mirred_ifindex;
355 struct mlx5_hairpin_params params; 565 struct mlx5_hairpin_params params;
566 struct mlx5_core_dev *peer_mdev;
356 struct mlx5e_hairpin_entry *hpe; 567 struct mlx5e_hairpin_entry *hpe;
357 struct mlx5e_hairpin *hp; 568 struct mlx5e_hairpin *hp;
569 u64 link_speed64;
570 u32 link_speed;
571 u8 match_prio;
572 u16 peer_id;
358 int err; 573 int err;
359 574
360 if (!MLX5_CAP_GEN(priv->mdev, hairpin)) { 575 peer_mdev = mlx5e_hairpin_get_mdev(dev_net(priv->netdev), peer_ifindex);
576 if (!MLX5_CAP_GEN(priv->mdev, hairpin) || !MLX5_CAP_GEN(peer_mdev, hairpin)) {
361 netdev_warn(priv->netdev, "hairpin is not supported\n"); 577 netdev_warn(priv->netdev, "hairpin is not supported\n");
362 return -EOPNOTSUPP; 578 return -EOPNOTSUPP;
363 } 579 }
364 580
365 hpe = mlx5e_hairpin_get(priv, peer_ifindex); 581 peer_id = MLX5_CAP_GEN(peer_mdev, vhca_id);
582 err = mlx5e_hairpin_get_prio(priv, &parse_attr->spec, &match_prio);
583 if (err)
584 return err;
585 hpe = mlx5e_hairpin_get(priv, peer_id, match_prio);
366 if (hpe) 586 if (hpe)
367 goto attach_flow; 587 goto attach_flow;
368 588
@@ -371,14 +591,27 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
371 return -ENOMEM; 591 return -ENOMEM;
372 592
373 INIT_LIST_HEAD(&hpe->flows); 593 INIT_LIST_HEAD(&hpe->flows);
374 hpe->peer_ifindex = peer_ifindex; 594 hpe->peer_vhca_id = peer_id;
595 hpe->prio = match_prio;
375 596
376 params.log_data_size = 15; 597 params.log_data_size = 15;
377 params.log_data_size = min_t(u8, params.log_data_size, 598 params.log_data_size = min_t(u8, params.log_data_size,
378 MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz)); 599 MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz));
379 params.log_data_size = max_t(u8, params.log_data_size, 600 params.log_data_size = max_t(u8, params.log_data_size,
380 MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz)); 601 MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz));
602
603 params.log_num_packets = params.log_data_size -
604 MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(priv->mdev);
605 params.log_num_packets = min_t(u8, params.log_num_packets,
606 MLX5_CAP_GEN(priv->mdev, log_max_hairpin_num_packets));
607
381 params.q_counter = priv->q_counter; 608 params.q_counter = priv->q_counter;
609 /* set hairpin pair per each 50Gbs share of the link */
610 mlx5e_get_max_linkspeed(priv->mdev, &link_speed);
611 link_speed = max_t(u32, link_speed, 50000);
612 link_speed64 = link_speed;
613 do_div(link_speed64, 50000);
614 params.num_channels = link_speed64;
382 615
383 hp = mlx5e_hairpin_create(priv, &params, peer_ifindex); 616 hp = mlx5e_hairpin_create(priv, &params, peer_ifindex);
384 if (IS_ERR(hp)) { 617 if (IS_ERR(hp)) {
@@ -386,16 +619,23 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
386 goto create_hairpin_err; 619 goto create_hairpin_err;
387 } 620 }
388 621
389 netdev_dbg(priv->netdev, "add hairpin: tirn %x rqn %x peer %s sqn %x log data size %d\n", 622 netdev_dbg(priv->netdev, "add hairpin: tirn %x rqn %x peer %s sqn %x prio %d (log) data %d packets %d\n",
390 hp->tirn, hp->pair->rqn, hp->pair->peer_mdev->priv.name, 623 hp->tirn, hp->pair->rqn[0], hp->pair->peer_mdev->priv.name,
391 hp->pair->sqn, params.log_data_size); 624 hp->pair->sqn[0], match_prio, params.log_data_size, params.log_num_packets);
392 625
393 hpe->hp = hp; 626 hpe->hp = hp;
394 hash_add(priv->fs.tc.hairpin_tbl, &hpe->hairpin_hlist, peer_ifindex); 627 hash_add(priv->fs.tc.hairpin_tbl, &hpe->hairpin_hlist,
628 hash_hairpin_info(peer_id, match_prio));
395 629
396attach_flow: 630attach_flow:
397 flow->nic_attr->hairpin_tirn = hpe->hp->tirn; 631 if (hpe->hp->num_channels > 1) {
632 flow->flags |= MLX5E_TC_FLOW_HAIRPIN_RSS;
633 flow->nic_attr->hairpin_ft = hpe->hp->ttc.ft.t;
634 } else {
635 flow->nic_attr->hairpin_tirn = hpe->hp->tirn;
636 }
398 list_add(&flow->hairpin, &hpe->flows); 637 list_add(&flow->hairpin, &hpe->flows);
638
399 return 0; 639 return 0;
400 640
401create_hairpin_err: 641create_hairpin_err:
@@ -443,20 +683,24 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
443 bool table_created = false; 683 bool table_created = false;
444 int err, dest_ix = 0; 684 int err, dest_ix = 0;
445 685
446 if (attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 686 if (flow->flags & MLX5E_TC_FLOW_HAIRPIN) {
447 if (flow->flags & MLX5E_TC_FLOW_HAIRPIN) { 687 err = mlx5e_hairpin_flow_add(priv, flow, parse_attr);
448 err = mlx5e_hairpin_flow_add(priv, flow, parse_attr); 688 if (err) {
449 if (err) { 689 rule = ERR_PTR(err);
450 rule = ERR_PTR(err); 690 goto err_add_hairpin_flow;
451 goto err_add_hairpin_flow; 691 }
452 } 692 if (flow->flags & MLX5E_TC_FLOW_HAIRPIN_RSS) {
693 dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
694 dest[dest_ix].ft = attr->hairpin_ft;
695 } else {
453 dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 696 dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
454 dest[dest_ix].tir_num = attr->hairpin_tirn; 697 dest[dest_ix].tir_num = attr->hairpin_tirn;
455 } else {
456 dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
457 dest[dest_ix].ft = priv->fs.vlan.ft.t;
458 } 698 }
459 dest_ix++; 699 dest_ix++;
700 } else if (attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
701 dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
702 dest[dest_ix].ft = priv->fs.vlan.ft.t;
703 dest_ix++;
460 } 704 }
461 705
462 if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 706 if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
@@ -497,7 +741,7 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
497 MLX5E_TC_PRIO, 741 MLX5E_TC_PRIO,
498 tc_tbl_size, 742 tc_tbl_size,
499 MLX5E_TC_TABLE_NUM_GROUPS, 743 MLX5E_TC_TABLE_NUM_GROUPS,
500 0, 0); 744 MLX5E_TC_FT_LEVEL, 0);
501 if (IS_ERR(priv->fs.tc.t)) { 745 if (IS_ERR(priv->fs.tc.t)) {
502 netdev_err(priv->netdev, 746 netdev_err(priv->netdev,
503 "Failed to create tc offload table\n"); 747 "Failed to create tc offload table\n");
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index e7e7cef2bde4..4d98ce0901af 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -530,6 +530,24 @@ static irqreturn_t mlx5_eq_int(int irq, void *eq_ptr)
530 return IRQ_HANDLED; 530 return IRQ_HANDLED;
531} 531}
532 532
533/* Some architectures don't latch interrupts when they are disabled, so using
534 * mlx5_eq_poll_irq_disabled could end up losing interrupts while trying to
535 * avoid losing them. It is not recommended to use it, unless this is the last
536 * resort.
537 */
538u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq *eq)
539{
540 u32 count_eqe;
541
542 disable_irq(eq->irqn);
543 count_eqe = eq->cons_index;
544 mlx5_eq_int(eq->irqn, eq);
545 count_eqe = eq->cons_index - count_eqe;
546 enable_irq(eq->irqn);
547
548 return count_eqe;
549}
550
533static void init_eq_buf(struct mlx5_eq *eq) 551static void init_eq_buf(struct mlx5_eq *eq)
534{ 552{
535 struct mlx5_eqe *eqe; 553 struct mlx5_eqe *eqe;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index cc4f6ab9374a..c025c98700e4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -89,6 +89,9 @@
89/* One more level for tc */ 89/* One more level for tc */
90#define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1) 90#define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
91 91
92#define KERNEL_NIC_TC_NUM_PRIOS 1
93#define KERNEL_NIC_TC_NUM_LEVELS 2
94
92#define ANCHOR_NUM_LEVELS 1 95#define ANCHOR_NUM_LEVELS 1
93#define ANCHOR_NUM_PRIOS 1 96#define ANCHOR_NUM_PRIOS 1
94#define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1) 97#define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
@@ -134,7 +137,7 @@ static struct init_tree_node {
134 ADD_NS(ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS, 137 ADD_NS(ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS,
135 ETHTOOL_PRIO_NUM_LEVELS))), 138 ETHTOOL_PRIO_NUM_LEVELS))),
136 ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {}, 139 ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {},
137 ADD_NS(ADD_MULTIPLE_PRIO(1, 1), 140 ADD_NS(ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS, KERNEL_NIC_TC_NUM_LEVELS),
138 ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS, 141 ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS,
139 KERNEL_NIC_PRIO_NUM_LEVELS))), 142 KERNEL_NIC_PRIO_NUM_LEVELS))),
140 ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, 143 ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
index 2743525a40a0..264504a990ca 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
@@ -241,7 +241,8 @@ static void mlx5i_cleanup_tx(struct mlx5e_priv *priv)
241 241
242static int mlx5i_create_flow_steering(struct mlx5e_priv *priv) 242static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
243{ 243{
244 int err; 244 struct ttc_params ttc_params = {};
245 int tt, err;
245 246
246 priv->fs.ns = mlx5_get_flow_namespace(priv->mdev, 247 priv->fs.ns = mlx5_get_flow_namespace(priv->mdev,
247 MLX5_FLOW_NAMESPACE_KERNEL); 248 MLX5_FLOW_NAMESPACE_KERNEL);
@@ -256,14 +257,23 @@ static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
256 priv->netdev->hw_features &= ~NETIF_F_NTUPLE; 257 priv->netdev->hw_features &= ~NETIF_F_NTUPLE;
257 } 258 }
258 259
259 err = mlx5e_create_inner_ttc_table(priv); 260 mlx5e_set_ttc_basic_params(priv, &ttc_params);
261 mlx5e_set_inner_ttc_ft_params(&ttc_params);
262 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
263 ttc_params.indir_tirn[tt] = priv->inner_indir_tir[tt].tirn;
264
265 err = mlx5e_create_inner_ttc_table(priv, &ttc_params, &priv->fs.inner_ttc);
260 if (err) { 266 if (err) {
261 netdev_err(priv->netdev, "Failed to create inner ttc table, err=%d\n", 267 netdev_err(priv->netdev, "Failed to create inner ttc table, err=%d\n",
262 err); 268 err);
263 goto err_destroy_arfs_tables; 269 goto err_destroy_arfs_tables;
264 } 270 }
265 271
266 err = mlx5e_create_ttc_table(priv); 272 mlx5e_set_ttc_ft_params(&ttc_params);
273 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
274 ttc_params.indir_tirn[tt] = priv->inner_indir_tir[tt].tirn;
275
276 err = mlx5e_create_ttc_table(priv, &ttc_params, &priv->fs.ttc);
267 if (err) { 277 if (err) {
268 netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n", 278 netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n",
269 err); 279 err);
@@ -273,7 +283,7 @@ static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
273 return 0; 283 return 0;
274 284
275err_destroy_inner_ttc_table: 285err_destroy_inner_ttc_table:
276 mlx5e_destroy_inner_ttc_table(priv); 286 mlx5e_destroy_inner_ttc_table(priv, &priv->fs.inner_ttc);
277err_destroy_arfs_tables: 287err_destroy_arfs_tables:
278 mlx5e_arfs_destroy_tables(priv); 288 mlx5e_arfs_destroy_tables(priv);
279 289
@@ -282,8 +292,8 @@ err_destroy_arfs_tables:
282 292
283static void mlx5i_destroy_flow_steering(struct mlx5e_priv *priv) 293static void mlx5i_destroy_flow_steering(struct mlx5e_priv *priv)
284{ 294{
285 mlx5e_destroy_ttc_table(priv); 295 mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
286 mlx5e_destroy_inner_ttc_table(priv); 296 mlx5e_destroy_inner_ttc_table(priv, &priv->fs.inner_ttc);
287 mlx5e_arfs_destroy_tables(priv); 297 mlx5e_arfs_destroy_tables(priv);
288} 298}
289 299
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index ff4a0b889a6f..b5a46c128b28 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -116,6 +116,7 @@ int mlx5_destroy_scheduling_element_cmd(struct mlx5_core_dev *dev, u8 hierarchy,
116int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev); 116int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev);
117u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev); 117u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev);
118struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn); 118struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn);
119u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq *eq);
119void mlx5_cq_tasklet_cb(unsigned long data); 120void mlx5_cq_tasklet_cb(unsigned long data);
120 121
121int mlx5_query_pcam_reg(struct mlx5_core_dev *dev, u32 *pcam, u8 feature_group, 122int mlx5_query_pcam_reg(struct mlx5_core_dev *dev, u32 *pcam, u8 feature_group,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
index a09ebbaf3b68..9e38343a951f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
@@ -413,6 +413,7 @@ static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
413 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter); 413 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);
414 414
415 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size); 415 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
416 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
416 417
417 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn); 418 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
418} 419}
@@ -430,6 +431,7 @@ static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
430 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 431 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
431 432
432 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size); 433 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
434 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
433 435
434 return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn); 436 return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
435} 437}
@@ -437,28 +439,40 @@ static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
437static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp, 439static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
438 struct mlx5_hairpin_params *params) 440 struct mlx5_hairpin_params *params)
439{ 441{
440 int err; 442 int i, j, err;
441 443
442 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn); 444 for (i = 0; i < hp->num_channels; i++) {
443 if (err) 445 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);
444 goto out_err_rq; 446 if (err)
447 goto out_err_rq;
448 }
445 449
446 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn); 450 for (i = 0; i < hp->num_channels; i++) {
447 if (err) 451 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);
448 goto out_err_sq; 452 if (err)
453 goto out_err_sq;
454 }
449 455
450 return 0; 456 return 0;
451 457
452out_err_sq: 458out_err_sq:
453 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn); 459 for (j = 0; j < i; j++)
460 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);
461 i = hp->num_channels;
454out_err_rq: 462out_err_rq:
463 for (j = 0; j < i; j++)
464 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
455 return err; 465 return err;
456} 466}
457 467
458static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp) 468static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
459{ 469{
460 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn); 470 int i;
461 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn); 471
472 for (i = 0; i < hp->num_channels; i++) {
473 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
474 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
475 }
462} 476}
463 477
464static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn, 478static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
@@ -505,41 +519,53 @@ static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
505 519
506static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp) 520static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
507{ 521{
508 int err; 522 int i, j, err;
509 523
510 /* set peer SQ */ 524 /* set peer SQs */
511 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn, 525 for (i = 0; i < hp->num_channels; i++) {
512 MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY, 526 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],
513 MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn); 527 MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
514 if (err) 528 MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);
515 goto err_modify_sq; 529 if (err)
516 530 goto err_modify_sq;
517 /* set func RQ */ 531 }
518 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn,
519 MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
520 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn);
521 532
522 if (err) 533 /* set func RQs */
523 goto err_modify_rq; 534 for (i = 0; i < hp->num_channels; i++) {
535 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],
536 MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
537 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);
538 if (err)
539 goto err_modify_rq;
540 }
524 541
525 return 0; 542 return 0;
526 543
527err_modify_rq: 544err_modify_rq:
528 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn, MLX5_SQC_STATE_RDY, 545 for (j = 0; j < i; j++)
529 MLX5_SQC_STATE_RST, 0, 0); 546 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,
547 MLX5_RQC_STATE_RST, 0, 0);
548 i = hp->num_channels;
530err_modify_sq: 549err_modify_sq:
550 for (j = 0; j < i; j++)
551 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,
552 MLX5_SQC_STATE_RST, 0, 0);
531 return err; 553 return err;
532} 554}
533 555
534static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp) 556static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
535{ 557{
536 /* unset func RQ */ 558 int i;
537 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn, MLX5_RQC_STATE_RDY, 559
538 MLX5_RQC_STATE_RST, 0, 0); 560 /* unset func RQs */
561 for (i = 0; i < hp->num_channels; i++)
562 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,
563 MLX5_RQC_STATE_RST, 0, 0);
539 564
540 /* unset peer SQ */ 565 /* unset peer SQs */
541 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn, MLX5_SQC_STATE_RDY, 566 for (i = 0; i < hp->num_channels; i++)
542 MLX5_SQC_STATE_RST, 0, 0); 567 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,
568 MLX5_SQC_STATE_RST, 0, 0);
543} 569}
544 570
545struct mlx5_hairpin * 571struct mlx5_hairpin *
@@ -550,13 +576,17 @@ mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
550 struct mlx5_hairpin *hp; 576 struct mlx5_hairpin *hp;
551 int size, err; 577 int size, err;
552 578
553 size = sizeof(*hp); 579 size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);
554 hp = kzalloc(size, GFP_KERNEL); 580 hp = kzalloc(size, GFP_KERNEL);
555 if (!hp) 581 if (!hp)
556 return ERR_PTR(-ENOMEM); 582 return ERR_PTR(-ENOMEM);
557 583
558 hp->func_mdev = func_mdev; 584 hp->func_mdev = func_mdev;
559 hp->peer_mdev = peer_mdev; 585 hp->peer_mdev = peer_mdev;
586 hp->num_channels = params->num_channels;
587
588 hp->rqn = (void *)hp + sizeof(*hp);
589 hp->sqn = hp->rqn + params->num_channels;
560 590
561 /* alloc and pair func --> peer hairpin */ 591 /* alloc and pair func --> peer hairpin */
562 err = mlx5_hairpin_create_queues(hp, params); 592 err = mlx5_hairpin_create_queues(hp, params);
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index acd829d8613b..199bfcd2f2ce 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -1031,7 +1031,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
1031 u8 log_max_hairpin_queues[0x5]; 1031 u8 log_max_hairpin_queues[0x5];
1032 u8 reserved_at_3c8[0x3]; 1032 u8 reserved_at_3c8[0x3];
1033 u8 log_max_hairpin_wq_data_sz[0x5]; 1033 u8 log_max_hairpin_wq_data_sz[0x5];
1034 u8 reserved_at_3d0[0xb]; 1034 u8 reserved_at_3d0[0x3];
1035 u8 log_max_hairpin_num_packets[0x5];
1036 u8 reserved_at_3d8[0x3];
1035 u8 log_max_wq_sz[0x5]; 1037 u8 log_max_wq_sz[0x5];
1036 1038
1037 u8 nic_vport_change_event[0x1]; 1039 u8 nic_vport_change_event[0x1];
@@ -1172,7 +1174,9 @@ struct mlx5_ifc_wq_bits {
1172 u8 reserved_at_118[0x3]; 1174 u8 reserved_at_118[0x3];
1173 u8 log_wq_sz[0x5]; 1175 u8 log_wq_sz[0x5];
1174 1176
1175 u8 reserved_at_120[0xb]; 1177 u8 reserved_at_120[0x3];
1178 u8 log_hairpin_num_packets[0x5];
1179 u8 reserved_at_128[0x3];
1176 u8 log_hairpin_data_sz[0x5]; 1180 u8 log_hairpin_data_sz[0x5];
1177 u8 reserved_at_130[0x5]; 1181 u8 reserved_at_130[0x5];
1178 1182
diff --git a/include/linux/mlx5/transobj.h b/include/linux/mlx5/transobj.h
index a228310c1968..7e8f281f8c00 100644
--- a/include/linux/mlx5/transobj.h
+++ b/include/linux/mlx5/transobj.h
@@ -77,15 +77,19 @@ void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn);
77 77
78struct mlx5_hairpin_params { 78struct mlx5_hairpin_params {
79 u8 log_data_size; 79 u8 log_data_size;
80 u8 log_num_packets;
80 u16 q_counter; 81 u16 q_counter;
82 int num_channels;
81}; 83};
82 84
83struct mlx5_hairpin { 85struct mlx5_hairpin {
84 struct mlx5_core_dev *func_mdev; 86 struct mlx5_core_dev *func_mdev;
85 struct mlx5_core_dev *peer_mdev; 87 struct mlx5_core_dev *peer_mdev;
86 88
87 u32 rqn; 89 int num_channels;
88 u32 sqn; 90
91 u32 *rqn;
92 u32 *sqn;
89}; 93};
90 94
91struct mlx5_hairpin * 95struct mlx5_hairpin *
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-16 05:22:41 -0400