summaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
authorMatan Barak <matanb@mellanox.com>2015-05-31 02:30:16 -0400
committerDavid S. Miller <davem@davemloft.net>2015-05-31 02:35:34 -0400
commitc66fa19c405a36673d4aab13658c8246413d5c0f (patch)
treed2958abf16fbeaeda35ec26bc0f887e2dcde5354 /drivers
parent48564135cba806bd0d6d1704c0ea317318966d9f (diff)
net/mlx4: Add EQ pool
Previously, mlx4_en allocated EQs and used them exclusively. This affected RoCE performance, as applications which are events sensitive were limited to use only the legacy EQs. Change that by introducing an EQ pool. This pool is managed by mlx4_core. EQs are assigned to ports (when there are limited number of EQs, multiple ports could be assigned to the same EQs). An exception to this rule is the ASYNC EQ which handles various events. Legacy EQs are completely removed as all EQs could be shared. When a consumer (mlx4_ib/mlx4_en) requests an EQ, it asks for EQ serving on a specific port. The core driver calculates which EQ should be assigned to that request. Because IRQs are shared between IB and Ethernet modules, their names only include the PCI device BDF address. Signed-off-by: Matan Barak <matanb@mellanox.com> Signed-off-by: Ido Shamay <idos@mellanox.com> Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers')
-rw-r--r--drivers/infiniband/hw/mlx4/main.c71
-rw-r--r--drivers/infiniband/hw/mlx4/mlx4_ib.h1
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/cq.c10
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_cq.c48
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_netdev.c7
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_rx.c13
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/eq.c353
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/main.c74
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/mlx4.h11
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/mlx4_en.h2
10 files changed, 335 insertions, 255 deletions
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 8c96c71e7bab..024b0f745035 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -2041,77 +2041,52 @@ static void init_pkeys(struct mlx4_ib_dev *ibdev)
2041 2041
2042static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) 2042static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
2043{ 2043{
2044 char name[80]; 2044 int i, j, eq = 0, total_eqs = 0;
2045 int eq_per_port = 0;
2046 int added_eqs = 0;
2047 int total_eqs = 0;
2048 int i, j, eq;
2049
2050 /* Legacy mode or comp_pool is not large enough */
2051 if (dev->caps.comp_pool == 0 ||
2052 dev->caps.num_ports > dev->caps.comp_pool)
2053 return;
2054
2055 eq_per_port = dev->caps.comp_pool / dev->caps.num_ports;
2056
2057 /* Init eq table */
2058 added_eqs = 0;
2059 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
2060 added_eqs += eq_per_port;
2061
2062 total_eqs = dev->caps.num_comp_vectors + added_eqs;
2063 2045
2064 ibdev->eq_table = kzalloc(total_eqs * sizeof(int), GFP_KERNEL); 2046 ibdev->eq_table = kcalloc(dev->caps.num_comp_vectors,
2047 sizeof(ibdev->eq_table[0]), GFP_KERNEL);
2065 if (!ibdev->eq_table) 2048 if (!ibdev->eq_table)
2066 return; 2049 return;
2067 2050
2068 ibdev->eq_added = added_eqs; 2051 for (i = 1; i <= dev->caps.num_ports; i++) {
2069 2052 for (j = 0; j < mlx4_get_eqs_per_port(dev, i);
2070 eq = 0; 2053 j++, total_eqs++) {
2071 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) { 2054 if (i > 1 && mlx4_is_eq_shared(dev, total_eqs))
2072 for (j = 0; j < eq_per_port; j++) { 2055 continue;
2073 snprintf(name, sizeof(name), "mlx4-ib-%d-%d@%s", 2056 ibdev->eq_table[eq] = total_eqs;
2074 i, j, dev->persist->pdev->bus->name); 2057 if (!mlx4_assign_eq(dev, i,
2075 /* Set IRQ for specific name (per ring) */ 2058 &ibdev->eq_table[eq]))
2076 if (mlx4_assign_eq(dev, name, NULL, 2059 eq++;
2077 &ibdev->eq_table[eq])) { 2060 else
2078 /* Use legacy (same as mlx4_en driver) */ 2061 ibdev->eq_table[eq] = -1;
2079 pr_warn("Can't allocate EQ %d; reverting to legacy\n", eq);
2080 ibdev->eq_table[eq] =
2081 (eq % dev->caps.num_comp_vectors);
2082 }
2083 eq++;
2084 } 2062 }
2085 } 2063 }
2086 2064
2087 /* Fill the reset of the vector with legacy EQ */ 2065 for (i = eq; i < dev->caps.num_comp_vectors;
2088 for (i = 0, eq = added_eqs; i < dev->caps.num_comp_vectors; i++) 2066 ibdev->eq_table[i++] = -1)
2089 ibdev->eq_table[eq++] = i; 2067 ;
2090 2068
2091 /* Advertise the new number of EQs to clients */ 2069 /* Advertise the new number of EQs to clients */
2092 ibdev->ib_dev.num_comp_vectors = total_eqs; 2070 ibdev->ib_dev.num_comp_vectors = eq;
2093} 2071}
2094 2072
2095static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) 2073static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
2096{ 2074{
2097 int i; 2075 int i;
2076 int total_eqs = ibdev->ib_dev.num_comp_vectors;
2098 2077
2099 /* no additional eqs were added */ 2078 /* no eqs were allocated */
2100 if (!ibdev->eq_table) 2079 if (!ibdev->eq_table)
2101 return; 2080 return;
2102 2081
2103 /* Reset the advertised EQ number */ 2082 /* Reset the advertised EQ number */
2104 ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors; 2083 ibdev->ib_dev.num_comp_vectors = 0;
2105 2084
2106 /* Free only the added eqs */ 2085 for (i = 0; i < total_eqs; i++)
2107 for (i = 0; i < ibdev->eq_added; i++) {
2108 /* Don't free legacy eqs if used */
2109 if (ibdev->eq_table[i] <= dev->caps.num_comp_vectors)
2110 continue;
2111 mlx4_release_eq(dev, ibdev->eq_table[i]); 2086 mlx4_release_eq(dev, ibdev->eq_table[i]);
2112 }
2113 2087
2114 kfree(ibdev->eq_table); 2088 kfree(ibdev->eq_table);
2089 ibdev->eq_table = NULL;
2115} 2090}
2116 2091
2117static void *mlx4_ib_add(struct mlx4_dev *dev) 2092static void *mlx4_ib_add(struct mlx4_dev *dev)
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index fce3934372a1..ef80e6c99a68 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -523,7 +523,6 @@ struct mlx4_ib_dev {
523 struct mlx4_ib_iboe iboe; 523 struct mlx4_ib_iboe iboe;
524 int counters[MLX4_MAX_PORTS]; 524 int counters[MLX4_MAX_PORTS];
525 int *eq_table; 525 int *eq_table;
526 int eq_added;
527 struct kobject *iov_parent; 526 struct kobject *iov_parent;
528 struct kobject *ports_parent; 527 struct kobject *ports_parent;
529 struct kobject *dev_ports_parent[MLX4_MFUNC_MAX]; 528 struct kobject *dev_ports_parent[MLX4_MFUNC_MAX];
diff --git a/drivers/net/ethernet/mellanox/mlx4/cq.c b/drivers/net/ethernet/mellanox/mlx4/cq.c
index e71f31387ac6..7431cd4d7390 100644
--- a/drivers/net/ethernet/mellanox/mlx4/cq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/cq.c
@@ -292,7 +292,7 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
292 u64 mtt_addr; 292 u64 mtt_addr;
293 int err; 293 int err;
294 294
295 if (vector > dev->caps.num_comp_vectors + dev->caps.comp_pool) 295 if (vector >= dev->caps.num_comp_vectors)
296 return -EINVAL; 296 return -EINVAL;
297 297
298 cq->vector = vector; 298 cq->vector = vector;
@@ -319,7 +319,7 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
319 cq_context->flags |= cpu_to_be32(1 << 19); 319 cq_context->flags |= cpu_to_be32(1 << 19);
320 320
321 cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index); 321 cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index);
322 cq_context->comp_eqn = priv->eq_table.eq[vector].eqn; 322 cq_context->comp_eqn = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].eqn;
323 cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; 323 cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
324 324
325 mtt_addr = mlx4_mtt_addr(dev, mtt); 325 mtt_addr = mlx4_mtt_addr(dev, mtt);
@@ -339,11 +339,11 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
339 init_completion(&cq->free); 339 init_completion(&cq->free);
340 cq->comp = mlx4_add_cq_to_tasklet; 340 cq->comp = mlx4_add_cq_to_tasklet;
341 cq->tasklet_ctx.priv = 341 cq->tasklet_ctx.priv =
342 &priv->eq_table.eq[cq->vector].tasklet_ctx; 342 &priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].tasklet_ctx;
343 INIT_LIST_HEAD(&cq->tasklet_ctx.list); 343 INIT_LIST_HEAD(&cq->tasklet_ctx.list);
344 344
345 345
346 cq->irq = priv->eq_table.eq[cq->vector].irq; 346 cq->irq = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].irq;
347 return 0; 347 return 0;
348 348
349err_radix: 349err_radix:
@@ -368,7 +368,7 @@ void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq)
368 if (err) 368 if (err)
369 mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn); 369 mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn);
370 370
371 synchronize_irq(priv->eq_table.eq[cq->vector].irq); 371 synchronize_irq(priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq);
372 372
373 spin_lock_irq(&cq_table->lock); 373 spin_lock_irq(&cq_table->lock);
374 radix_tree_delete(&cq_table->tree, cq->cqn); 374 radix_tree_delete(&cq_table->tree, cq->cqn);
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_cq.c b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
index 22da4d0d0f05..d71c567eb076 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_cq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
@@ -66,6 +66,7 @@ int mlx4_en_create_cq(struct mlx4_en_priv *priv,
66 66
67 cq->ring = ring; 67 cq->ring = ring;
68 cq->is_tx = mode; 68 cq->is_tx = mode;
69 cq->vector = mdev->dev->caps.num_comp_vectors;
69 70
70 /* Allocate HW buffers on provided NUMA node. 71 /* Allocate HW buffers on provided NUMA node.
71 * dev->numa_node is used in mtt range allocation flow. 72 * dev->numa_node is used in mtt range allocation flow.
@@ -101,12 +102,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
101 int err = 0; 102 int err = 0;
102 char name[25]; 103 char name[25];
103 int timestamp_en = 0; 104 int timestamp_en = 0;
104 struct cpu_rmap *rmap = 105 bool assigned_eq = false;
105#ifdef CONFIG_RFS_ACCEL
106 priv->dev->rx_cpu_rmap;
107#else
108 NULL;
109#endif
110 106
111 cq->dev = mdev->pndev[priv->port]; 107 cq->dev = mdev->pndev[priv->port];
112 cq->mcq.set_ci_db = cq->wqres.db.db; 108 cq->mcq.set_ci_db = cq->wqres.db.db;
@@ -116,23 +112,19 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
116 memset(cq->buf, 0, cq->buf_size); 112 memset(cq->buf, 0, cq->buf_size);
117 113
118 if (cq->is_tx == RX) { 114 if (cq->is_tx == RX) {
119 if (mdev->dev->caps.comp_pool) { 115 if (!mlx4_is_eq_vector_valid(mdev->dev, priv->port,
120 if (!cq->vector) { 116 cq->vector)) {
121 sprintf(name, "%s-%d", priv->dev->name, 117 cq->vector = cq_idx;
122 cq->ring); 118
123 /* Set IRQ for specific name (per ring) */ 119 err = mlx4_assign_eq(mdev->dev, priv->port,
124 if (mlx4_assign_eq(mdev->dev, name, rmap, 120 &cq->vector);
125 &cq->vector)) { 121 if (err) {
126 cq->vector = (cq->ring + 1 + priv->port) 122 mlx4_err(mdev, "Failed assigning an EQ to %s\n",
127 % mdev->dev->caps.num_comp_vectors; 123 name);
128 mlx4_warn(mdev, "Failed assigning an EQ to %s, falling back to legacy EQ's\n", 124 goto free_eq;
129 name);
130 }
131
132 } 125 }
133 } else { 126
134 cq->vector = (cq->ring + 1 + priv->port) % 127 assigned_eq = true;
135 mdev->dev->caps.num_comp_vectors;
136 } 128 }
137 129
138 cq->irq_desc = 130 cq->irq_desc =
@@ -159,7 +151,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
159 &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq, 151 &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq,
160 cq->vector, 0, timestamp_en); 152 cq->vector, 0, timestamp_en);
161 if (err) 153 if (err)
162 return err; 154 goto free_eq;
163 155
164 cq->mcq.comp = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq; 156 cq->mcq.comp = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq;
165 cq->mcq.event = mlx4_en_cq_event; 157 cq->mcq.event = mlx4_en_cq_event;
@@ -182,6 +174,12 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
182 napi_enable(&cq->napi); 174 napi_enable(&cq->napi);
183 175
184 return 0; 176 return 0;
177
178free_eq:
179 if (assigned_eq)
180 mlx4_release_eq(mdev->dev, cq->vector);
181 cq->vector = mdev->dev->caps.num_comp_vectors;
182 return err;
185} 183}
186 184
187void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq) 185void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq)
@@ -191,9 +189,9 @@ void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq)
191 189
192 mlx4_en_unmap_buffer(&cq->wqres.buf); 190 mlx4_en_unmap_buffer(&cq->wqres.buf);
193 mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size); 191 mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size);
194 if (priv->mdev->dev->caps.comp_pool && cq->vector) { 192 if (mlx4_is_eq_vector_valid(mdev->dev, priv->port, cq->vector) &&
193 cq->is_tx == RX)
195 mlx4_release_eq(priv->mdev->dev, cq->vector); 194 mlx4_release_eq(priv->mdev->dev, cq->vector);
196 }
197 cq->vector = 0; 195 cq->vector = 0;
198 cq->buf_size = 0; 196 cq->buf_size = 0;
199 cq->buf = NULL; 197 cq->buf = NULL;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index 32f5ec737472..455cecae5aa4 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -1958,7 +1958,6 @@ void mlx4_en_free_resources(struct mlx4_en_priv *priv)
1958 int i; 1958 int i;
1959 1959
1960#ifdef CONFIG_RFS_ACCEL 1960#ifdef CONFIG_RFS_ACCEL
1961 free_irq_cpu_rmap(priv->dev->rx_cpu_rmap);
1962 priv->dev->rx_cpu_rmap = NULL; 1961 priv->dev->rx_cpu_rmap = NULL;
1963#endif 1962#endif
1964 1963
@@ -2016,11 +2015,7 @@ int mlx4_en_alloc_resources(struct mlx4_en_priv *priv)
2016 } 2015 }
2017 2016
2018#ifdef CONFIG_RFS_ACCEL 2017#ifdef CONFIG_RFS_ACCEL
2019 if (priv->mdev->dev->caps.comp_pool) { 2018 priv->dev->rx_cpu_rmap = mlx4_get_cpu_rmap(priv->mdev->dev, priv->port);
2020 priv->dev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->mdev->dev->caps.comp_pool);
2021 if (!priv->dev->rx_cpu_rmap)
2022 goto err;
2023 }
2024#endif 2019#endif
2025 2020
2026 return 0; 2021 return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index 2a77a6b19121..35f726c17e48 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -337,15 +337,10 @@ void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev)
337 struct mlx4_dev *dev = mdev->dev; 337 struct mlx4_dev *dev = mdev->dev;
338 338
339 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { 339 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) {
340 if (!dev->caps.comp_pool) 340 num_of_eqs = max_t(int, MIN_RX_RINGS,
341 num_of_eqs = max_t(int, MIN_RX_RINGS, 341 min_t(int,
342 min_t(int, 342 mlx4_get_eqs_per_port(mdev->dev, i),
343 dev->caps.num_comp_vectors, 343 DEF_RX_RINGS));
344 DEF_RX_RINGS));
345 else
346 num_of_eqs = min_t(int, MAX_MSIX_P_PORT,
347 dev->caps.comp_pool/
348 dev->caps.num_ports) - 1;
349 344
350 num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS : 345 num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS :
351 min_t(int, num_of_eqs, 346 min_t(int, num_of_eqs,
diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c
index 80bcd648c5e0..2e6fc6a860a7 100644
--- a/drivers/net/ethernet/mellanox/mlx4/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/eq.c
@@ -895,8 +895,8 @@ static int mlx4_num_eq_uar(struct mlx4_dev *dev)
895 * we need to map, take the difference of highest index and 895 * we need to map, take the difference of highest index and
896 * the lowest index we'll use and add 1. 896 * the lowest index we'll use and add 1.
897 */ 897 */
898 return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs + 898 return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs) / 4 -
899 dev->caps.comp_pool)/4 - dev->caps.reserved_eqs/4 + 1; 899 dev->caps.reserved_eqs / 4 + 1;
900} 900}
901 901
902static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq) 902static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq)
@@ -1085,8 +1085,7 @@ static void mlx4_free_eq(struct mlx4_dev *dev,
1085static void mlx4_free_irqs(struct mlx4_dev *dev) 1085static void mlx4_free_irqs(struct mlx4_dev *dev)
1086{ 1086{
1087 struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table; 1087 struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table;
1088 struct mlx4_priv *priv = mlx4_priv(dev); 1088 int i;
1089 int i, vec;
1090 1089
1091 if (eq_table->have_irq) 1090 if (eq_table->have_irq)
1092 free_irq(dev->persist->pdev->irq, dev); 1091 free_irq(dev->persist->pdev->irq, dev);
@@ -1097,20 +1096,6 @@ static void mlx4_free_irqs(struct mlx4_dev *dev)
1097 eq_table->eq[i].have_irq = 0; 1096 eq_table->eq[i].have_irq = 0;
1098 } 1097 }
1099 1098
1100 for (i = 0; i < dev->caps.comp_pool; i++) {
1101 /*
1102 * Freeing the assigned irq's
1103 * all bits should be 0, but we need to validate
1104 */
1105 if (priv->msix_ctl.pool_bm & 1ULL << i) {
1106 /* NO need protecting*/
1107 vec = dev->caps.num_comp_vectors + 1 + i;
1108 free_irq(priv->eq_table.eq[vec].irq,
1109 &priv->eq_table.eq[vec]);
1110 }
1111 }
1112
1113
1114 kfree(eq_table->irq_names); 1099 kfree(eq_table->irq_names);
1115} 1100}
1116 1101
@@ -1191,76 +1176,73 @@ int mlx4_init_eq_table(struct mlx4_dev *dev)
1191 } 1176 }
1192 1177
1193 priv->eq_table.irq_names = 1178 priv->eq_table.irq_names =
1194 kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1 + 1179 kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1),
1195 dev->caps.comp_pool),
1196 GFP_KERNEL); 1180 GFP_KERNEL);
1197 if (!priv->eq_table.irq_names) { 1181 if (!priv->eq_table.irq_names) {
1198 err = -ENOMEM; 1182 err = -ENOMEM;
1199 goto err_out_bitmap; 1183 goto err_out_clr_int;
1200 } 1184 }
1201 1185
1202 for (i = 0; i < dev->caps.num_comp_vectors; ++i) { 1186 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) {
1203 err = mlx4_create_eq(dev, dev->caps.num_cqs - 1187 if (i == MLX4_EQ_ASYNC) {
1204 dev->caps.reserved_cqs + 1188 err = mlx4_create_eq(dev,
1205 MLX4_NUM_SPARE_EQE, 1189 MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE,
1206 (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, 1190 0, &priv->eq_table.eq[MLX4_EQ_ASYNC]);
1207 &priv->eq_table.eq[i]); 1191 } else {
1208 if (err) { 1192#ifdef CONFIG_RFS_ACCEL
1209 --i; 1193 struct mlx4_eq *eq = &priv->eq_table.eq[i];
1210 goto err_out_unmap; 1194 int port = find_first_bit(eq->actv_ports.ports,
1211 } 1195 dev->caps.num_ports) + 1;
1212 } 1196
1213 1197 if (port <= dev->caps.num_ports) {
1214 err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE, 1198 struct mlx4_port_info *info =
1215 (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0, 1199 &mlx4_priv(dev)->port[port];
1216 &priv->eq_table.eq[dev->caps.num_comp_vectors]); 1200
1217 if (err) 1201 if (!info->rmap) {
1218 goto err_out_comp; 1202 info->rmap = alloc_irq_cpu_rmap(
1219 1203 mlx4_get_eqs_per_port(dev, port));
1220 /*if additional completion vectors poolsize is 0 this loop will not run*/ 1204 if (!info->rmap) {
1221 for (i = dev->caps.num_comp_vectors + 1; 1205 mlx4_warn(dev, "Failed to allocate cpu rmap\n");
1222 i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) { 1206 err = -ENOMEM;
1207 goto err_out_unmap;
1208 }
1209 }
1223 1210
1224 err = mlx4_create_eq(dev, dev->caps.num_cqs - 1211 err = irq_cpu_rmap_add(
1225 dev->caps.reserved_cqs + 1212 info->rmap, eq->irq);
1226 MLX4_NUM_SPARE_EQE, 1213 if (err)
1227 (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, 1214 mlx4_warn(dev, "Failed adding irq rmap\n");
1228 &priv->eq_table.eq[i]); 1215 }
1229 if (err) { 1216#endif
1230 --i; 1217 err = mlx4_create_eq(dev, dev->caps.num_cqs -
1231 goto err_out_unmap; 1218 dev->caps.reserved_cqs +
1219 MLX4_NUM_SPARE_EQE,
1220 (dev->flags & MLX4_FLAG_MSI_X) ?
1221 i + 1 - !!(i > MLX4_EQ_ASYNC) : 0,
1222 eq);
1232 } 1223 }
1224 if (err)
1225 goto err_out_unmap;
1233 } 1226 }
1234 1227
1235
1236 if (dev->flags & MLX4_FLAG_MSI_X) { 1228 if (dev->flags & MLX4_FLAG_MSI_X) {
1237 const char *eq_name; 1229 const char *eq_name;
1238 1230
1239 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) { 1231 snprintf(priv->eq_table.irq_names +
1240 if (i < dev->caps.num_comp_vectors) { 1232 MLX4_EQ_ASYNC * MLX4_IRQNAME_SIZE,
1241 snprintf(priv->eq_table.irq_names + 1233 MLX4_IRQNAME_SIZE,
1242 i * MLX4_IRQNAME_SIZE, 1234 "mlx4-async@pci:%s",
1243 MLX4_IRQNAME_SIZE, 1235 pci_name(dev->persist->pdev));
1244 "mlx4-comp-%d@pci:%s", i, 1236 eq_name = priv->eq_table.irq_names +
1245 pci_name(dev->persist->pdev)); 1237 MLX4_EQ_ASYNC * MLX4_IRQNAME_SIZE;
1246 } else {
1247 snprintf(priv->eq_table.irq_names +
1248 i * MLX4_IRQNAME_SIZE,
1249 MLX4_IRQNAME_SIZE,
1250 "mlx4-async@pci:%s",
1251 pci_name(dev->persist->pdev));
1252 }
1253 1238
1254 eq_name = priv->eq_table.irq_names + 1239 err = request_irq(priv->eq_table.eq[MLX4_EQ_ASYNC].irq,
1255 i * MLX4_IRQNAME_SIZE; 1240 mlx4_msi_x_interrupt, 0, eq_name,
1256 err = request_irq(priv->eq_table.eq[i].irq, 1241 priv->eq_table.eq + MLX4_EQ_ASYNC);
1257 mlx4_msi_x_interrupt, 0, eq_name, 1242 if (err)
1258 priv->eq_table.eq + i); 1243 goto err_out_unmap;
1259 if (err)
1260 goto err_out_async;
1261 1244
1262 priv->eq_table.eq[i].have_irq = 1; 1245 priv->eq_table.eq[MLX4_EQ_ASYNC].have_irq = 1;
1263 }
1264 } else { 1246 } else {
1265 snprintf(priv->eq_table.irq_names, 1247 snprintf(priv->eq_table.irq_names,
1266 MLX4_IRQNAME_SIZE, 1248 MLX4_IRQNAME_SIZE,
@@ -1269,36 +1251,38 @@ int mlx4_init_eq_table(struct mlx4_dev *dev)
1269 err = request_irq(dev->persist->pdev->irq, mlx4_interrupt, 1251 err = request_irq(dev->persist->pdev->irq, mlx4_interrupt,
1270 IRQF_SHARED, priv->eq_table.irq_names, dev); 1252 IRQF_SHARED, priv->eq_table.irq_names, dev);
1271 if (err) 1253 if (err)
1272 goto err_out_async; 1254 goto err_out_unmap;
1273 1255
1274 priv->eq_table.have_irq = 1; 1256 priv->eq_table.have_irq = 1;
1275 } 1257 }
1276 1258
1277 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, 1259 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1278 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 1260 priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
1279 if (err) 1261 if (err)
1280 mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n", 1262 mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
1281 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn, err); 1263 priv->eq_table.eq[MLX4_EQ_ASYNC].eqn, err);
1282 1264
1283 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) 1265 /* arm ASYNC eq */
1284 eq_set_ci(&priv->eq_table.eq[i], 1); 1266 eq_set_ci(&priv->eq_table.eq[MLX4_EQ_ASYNC], 1);
1285 1267
1286 return 0; 1268 return 0;
1287 1269
1288err_out_async:
1289 mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]);
1290
1291err_out_comp:
1292 i = dev->caps.num_comp_vectors - 1;
1293
1294err_out_unmap: 1270err_out_unmap:
1295 while (i >= 0) { 1271 while (i >= 0)
1296 mlx4_free_eq(dev, &priv->eq_table.eq[i]); 1272 mlx4_free_eq(dev, &priv->eq_table.eq[i--]);
1297 --i; 1273#ifdef CONFIG_RFS_ACCEL
1274 for (i = 1; i <= dev->caps.num_ports; i++) {
1275 if (mlx4_priv(dev)->port[i].rmap) {
1276 free_irq_cpu_rmap(mlx4_priv(dev)->port[i].rmap);
1277 mlx4_priv(dev)->port[i].rmap = NULL;
1278 }
1298 } 1279 }
1280#endif
1281 mlx4_free_irqs(dev);
1282
1283err_out_clr_int:
1299 if (!mlx4_is_slave(dev)) 1284 if (!mlx4_is_slave(dev))
1300 mlx4_unmap_clr_int(dev); 1285 mlx4_unmap_clr_int(dev);
1301 mlx4_free_irqs(dev);
1302 1286
1303err_out_bitmap: 1287err_out_bitmap:
1304 mlx4_unmap_uar(dev); 1288 mlx4_unmap_uar(dev);
@@ -1316,11 +1300,19 @@ void mlx4_cleanup_eq_table(struct mlx4_dev *dev)
1316 int i; 1300 int i;
1317 1301
1318 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1, 1302 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1,
1319 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 1303 priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
1320 1304
1305#ifdef CONFIG_RFS_ACCEL
1306 for (i = 1; i <= dev->caps.num_ports; i++) {
1307 if (mlx4_priv(dev)->port[i].rmap) {
1308 free_irq_cpu_rmap(mlx4_priv(dev)->port[i].rmap);
1309 mlx4_priv(dev)->port[i].rmap = NULL;
1310 }
1311 }
1312#endif
1321 mlx4_free_irqs(dev); 1313 mlx4_free_irqs(dev);
1322 1314
1323 for (i = 0; i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) 1315 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
1324 mlx4_free_eq(dev, &priv->eq_table.eq[i]); 1316 mlx4_free_eq(dev, &priv->eq_table.eq[i]);
1325 1317
1326 if (!mlx4_is_slave(dev)) 1318 if (!mlx4_is_slave(dev))
@@ -1371,87 +1363,166 @@ int mlx4_test_interrupts(struct mlx4_dev *dev)
1371 1363
1372 /* Return to default */ 1364 /* Return to default */
1373 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, 1365 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1374 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 1366 priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
1375 return err; 1367 return err;
1376} 1368}
1377EXPORT_SYMBOL(mlx4_test_interrupts); 1369EXPORT_SYMBOL(mlx4_test_interrupts);
1378 1370
1379int mlx4_assign_eq(struct mlx4_dev *dev, char *name, struct cpu_rmap *rmap, 1371bool mlx4_is_eq_vector_valid(struct mlx4_dev *dev, u8 port, int vector)
1380 int *vector)
1381{ 1372{
1373 struct mlx4_priv *priv = mlx4_priv(dev);
1382 1374
1375 vector = MLX4_CQ_TO_EQ_VECTOR(vector);
1376 if (vector < 0 || (vector >= dev->caps.num_comp_vectors + 1) ||
1377 (vector == MLX4_EQ_ASYNC))
1378 return false;
1379
1380 return test_bit(port - 1, priv->eq_table.eq[vector].actv_ports.ports);
1381}
1382EXPORT_SYMBOL(mlx4_is_eq_vector_valid);
1383
1384u32 mlx4_get_eqs_per_port(struct mlx4_dev *dev, u8 port)
1385{
1386 struct mlx4_priv *priv = mlx4_priv(dev);
1387 unsigned int i;
1388 unsigned int sum = 0;
1389
1390 for (i = 0; i < dev->caps.num_comp_vectors + 1; i++)
1391 sum += !!test_bit(port - 1,
1392 priv->eq_table.eq[i].actv_ports.ports);
1393
1394 return sum;
1395}
1396EXPORT_SYMBOL(mlx4_get_eqs_per_port);
1397
1398int mlx4_is_eq_shared(struct mlx4_dev *dev, int vector)
1399{
1400 struct mlx4_priv *priv = mlx4_priv(dev);
1401
1402 vector = MLX4_CQ_TO_EQ_VECTOR(vector);
1403 if (vector <= 0 || (vector >= dev->caps.num_comp_vectors + 1))
1404 return -EINVAL;
1405
1406 return !!(bitmap_weight(priv->eq_table.eq[vector].actv_ports.ports,
1407 dev->caps.num_ports) > 1);
1408}
1409EXPORT_SYMBOL(mlx4_is_eq_shared);
1410
1411struct cpu_rmap *mlx4_get_cpu_rmap(struct mlx4_dev *dev, int port)
1412{
1413 return mlx4_priv(dev)->port[port].rmap;
1414}
1415EXPORT_SYMBOL(mlx4_get_cpu_rmap);
1416
1417int mlx4_assign_eq(struct mlx4_dev *dev, u8 port, int *vector)
1418{
1383 struct mlx4_priv *priv = mlx4_priv(dev); 1419 struct mlx4_priv *priv = mlx4_priv(dev);
1384 int vec = 0, err = 0, i; 1420 int err = 0, i = 0;
1421 u32 min_ref_count_val = (u32)-1;
1422 int requested_vector = MLX4_CQ_TO_EQ_VECTOR(*vector);
1423 int *prequested_vector = NULL;
1424
1385 1425
1386 mutex_lock(&priv->msix_ctl.pool_lock); 1426 mutex_lock(&priv->msix_ctl.pool_lock);
1387 for (i = 0; !vec && i < dev->caps.comp_pool; i++) { 1427 if (requested_vector < (dev->caps.num_comp_vectors + 1) &&
1388 if (~priv->msix_ctl.pool_bm & 1ULL << i) { 1428 (requested_vector >= 0) &&
1389 priv->msix_ctl.pool_bm |= 1ULL << i; 1429 (requested_vector != MLX4_EQ_ASYNC)) {
1390 vec = dev->caps.num_comp_vectors + 1 + i; 1430 if (test_bit(port - 1,
1391 snprintf(priv->eq_table.irq_names + 1431 priv->eq_table.eq[requested_vector].actv_ports.ports)) {
1392 vec * MLX4_IRQNAME_SIZE, 1432 prequested_vector = &requested_vector;
1393 MLX4_IRQNAME_SIZE, "%s", name); 1433 } else {
1394#ifdef CONFIG_RFS_ACCEL 1434 struct mlx4_eq *eq;
1395 if (rmap) { 1435
1396 err = irq_cpu_rmap_add(rmap, 1436 for (i = 1; i < port;
1397 priv->eq_table.eq[vec].irq); 1437 requested_vector += mlx4_get_eqs_per_port(dev, i++))
1398 if (err) 1438 ;
1399 mlx4_warn(dev, "Failed adding irq rmap\n"); 1439
1440 eq = &priv->eq_table.eq[requested_vector];
1441 if (requested_vector < dev->caps.num_comp_vectors + 1 &&
1442 test_bit(port - 1, eq->actv_ports.ports)) {
1443 prequested_vector = &requested_vector;
1400 } 1444 }
1401#endif 1445 }
1402 err = request_irq(priv->eq_table.eq[vec].irq, 1446 }
1403 mlx4_msi_x_interrupt, 0, 1447
1404 &priv->eq_table.irq_names[vec<<5], 1448 if (!prequested_vector) {
1405 priv->eq_table.eq + vec); 1449 requested_vector = -1;
1406 if (err) { 1450 for (i = 0; min_ref_count_val && i < dev->caps.num_comp_vectors + 1;
1407 /*zero out bit by fliping it*/ 1451 i++) {
1408 priv->msix_ctl.pool_bm ^= 1 << i; 1452 struct mlx4_eq *eq = &priv->eq_table.eq[i];
1409 vec = 0; 1453
1410 continue; 1454 if (min_ref_count_val > eq->ref_count &&
1411 /*we dont want to break here*/ 1455 test_bit(port - 1, eq->actv_ports.ports)) {
1456 min_ref_count_val = eq->ref_count;
1457 requested_vector = i;
1412 } 1458 }
1459 }
1413 1460
1414 eq_set_ci(&priv->eq_table.eq[vec], 1); 1461 if (requested_vector < 0) {
1462 err = -ENOSPC;
1463 goto err_unlock;
1415 } 1464 }
1465
1466 prequested_vector = &requested_vector;
1416 } 1467 }
1468
1469 if (!test_bit(*prequested_vector, priv->msix_ctl.pool_bm) &&
1470 dev->flags & MLX4_FLAG_MSI_X) {
1471 set_bit(*prequested_vector, priv->msix_ctl.pool_bm);
1472 snprintf(priv->eq_table.irq_names +
1473 *prequested_vector * MLX4_IRQNAME_SIZE,
1474 MLX4_IRQNAME_SIZE, "mlx4-%d@%s",
1475 *prequested_vector, dev_name(&dev->persist->pdev->dev));
1476
1477 err = request_irq(priv->eq_table.eq[*prequested_vector].irq,
1478 mlx4_msi_x_interrupt, 0,
1479 &priv->eq_table.irq_names[*prequested_vector << 5],
1480 priv->eq_table.eq + *prequested_vector);
1481
1482 if (err) {
1483 clear_bit(*prequested_vector, priv->msix_ctl.pool_bm);
1484 *prequested_vector = -1;
1485 } else {
1486 eq_set_ci(&priv->eq_table.eq[*prequested_vector], 1);
1487 priv->eq_table.eq[*prequested_vector].have_irq = 1;
1488 }
1489 }
1490
1491 if (!err && *prequested_vector >= 0)
1492 priv->eq_table.eq[*prequested_vector].ref_count++;
1493
1494err_unlock:
1417 mutex_unlock(&priv->msix_ctl.pool_lock); 1495 mutex_unlock(&priv->msix_ctl.pool_lock);
1418 1496
1419 if (vec) { 1497 if (!err && *prequested_vector >= 0)
1420 *vector = vec; 1498 *vector = MLX4_EQ_TO_CQ_VECTOR(*prequested_vector);
1421 } else { 1499 else
1422 *vector = 0; 1500 *vector = 0;
1423 err = (i == dev->caps.comp_pool) ? -ENOSPC : err; 1501
1424 }
1425 return err; 1502 return err;
1426} 1503}
1427EXPORT_SYMBOL(mlx4_assign_eq); 1504EXPORT_SYMBOL(mlx4_assign_eq);
1428 1505
1429int mlx4_eq_get_irq(struct mlx4_dev *dev, int vec) 1506int mlx4_eq_get_irq(struct mlx4_dev *dev, int cq_vec)
1430{ 1507{
1431 struct mlx4_priv *priv = mlx4_priv(dev); 1508 struct mlx4_priv *priv = mlx4_priv(dev);
1432 1509
1433 return priv->eq_table.eq[vec].irq; 1510 return priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq_vec)].irq;
1434} 1511}
1435EXPORT_SYMBOL(mlx4_eq_get_irq); 1512EXPORT_SYMBOL(mlx4_eq_get_irq);
1436 1513
1437void mlx4_release_eq(struct mlx4_dev *dev, int vec) 1514void mlx4_release_eq(struct mlx4_dev *dev, int vec)
1438{ 1515{
1439 struct mlx4_priv *priv = mlx4_priv(dev); 1516 struct mlx4_priv *priv = mlx4_priv(dev);
1440 /*bm index*/ 1517 int eq_vec = MLX4_CQ_TO_EQ_VECTOR(vec);
1441 int i = vec - dev->caps.num_comp_vectors - 1; 1518
1442 1519 mutex_lock(&priv->msix_ctl.pool_lock);
1443 if (likely(i >= 0)) { 1520 priv->eq_table.eq[eq_vec].ref_count--;
1444 /*sanity check , making sure were not trying to free irq's
1445 Belonging to a legacy EQ*/
1446 mutex_lock(&priv->msix_ctl.pool_lock);
1447 if (priv->msix_ctl.pool_bm & 1ULL << i) {
1448 free_irq(priv->eq_table.eq[vec].irq,
1449 &priv->eq_table.eq[vec]);
1450 priv->msix_ctl.pool_bm &= ~(1ULL << i);
1451 }
1452 mutex_unlock(&priv->msix_ctl.pool_lock);
1453 }
1454 1521
1522 /* once we allocated EQ, we don't release it because it might be binded
1523 * to cpu_rmap.
1524 */
1525 mutex_unlock(&priv->msix_ctl.pool_lock);
1455} 1526}
1456EXPORT_SYMBOL(mlx4_release_eq); 1527EXPORT_SYMBOL(mlx4_release_eq);
1457 1528
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 70d33f6e2a41..3ec5113c5a33 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -2364,11 +2364,11 @@ static int mlx4_setup_hca(struct mlx4_dev *dev)
2364 if (err) { 2364 if (err) {
2365 if (dev->flags & MLX4_FLAG_MSI_X) { 2365 if (dev->flags & MLX4_FLAG_MSI_X) {
2366 mlx4_warn(dev, "NOP command failed to generate MSI-X interrupt IRQ %d)\n", 2366 mlx4_warn(dev, "NOP command failed to generate MSI-X interrupt IRQ %d)\n",
2367 priv->eq_table.eq[dev->caps.num_comp_vectors].irq); 2367 priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
2368 mlx4_warn(dev, "Trying again without MSI-X\n"); 2368 mlx4_warn(dev, "Trying again without MSI-X\n");
2369 } else { 2369 } else {
2370 mlx4_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting\n", 2370 mlx4_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting\n",
2371 priv->eq_table.eq[dev->caps.num_comp_vectors].irq); 2371 priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
2372 mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n"); 2372 mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n");
2373 } 2373 }
2374 2374
@@ -2486,9 +2486,10 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
2486 struct mlx4_priv *priv = mlx4_priv(dev); 2486 struct mlx4_priv *priv = mlx4_priv(dev);
2487 struct msix_entry *entries; 2487 struct msix_entry *entries;
2488 int i; 2488 int i;
2489 int port = 0;
2489 2490
2490 if (msi_x) { 2491 if (msi_x) {
2491 int nreq = dev->caps.num_ports * num_online_cpus() + MSIX_LEGACY_SZ; 2492 int nreq = dev->caps.num_ports * num_online_cpus() + 1;
2492 2493
2493 nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs, 2494 nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs,
2494 nreq); 2495 nreq);
@@ -2503,20 +2504,49 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
2503 nreq = pci_enable_msix_range(dev->persist->pdev, entries, 2, 2504 nreq = pci_enable_msix_range(dev->persist->pdev, entries, 2,
2504 nreq); 2505 nreq);
2505 2506
2506 if (nreq < 0) { 2507 if (nreq < 0 || nreq < MLX4_EQ_ASYNC) {
2507 kfree(entries); 2508 kfree(entries);
2508 goto no_msi; 2509 goto no_msi;
2509 } else if (nreq < MSIX_LEGACY_SZ +
2510 dev->caps.num_ports * MIN_MSIX_P_PORT) {
2511 /*Working in legacy mode , all EQ's shared*/
2512 dev->caps.comp_pool = 0;
2513 dev->caps.num_comp_vectors = nreq - 1;
2514 } else {
2515 dev->caps.comp_pool = nreq - MSIX_LEGACY_SZ;
2516 dev->caps.num_comp_vectors = MSIX_LEGACY_SZ - 1;
2517 } 2510 }
2518 for (i = 0; i < nreq; ++i) 2511 /* 1 is reserved for events (asyncrounous EQ) */
2519 priv->eq_table.eq[i].irq = entries[i].vector; 2512 dev->caps.num_comp_vectors = nreq - 1;
2513
2514 priv->eq_table.eq[MLX4_EQ_ASYNC].irq = entries[0].vector;
2515 bitmap_zero(priv->eq_table.eq[MLX4_EQ_ASYNC].actv_ports.ports,
2516 dev->caps.num_ports);
2517
2518 for (i = 0; i < dev->caps.num_comp_vectors + 1; i++) {
2519 if (i == MLX4_EQ_ASYNC)
2520 continue;
2521
2522 priv->eq_table.eq[i].irq =
2523 entries[i + 1 - !!(i > MLX4_EQ_ASYNC)].vector;
2524
2525 if (MLX4_IS_LEGACY_EQ_MODE(dev->caps)) {
2526 bitmap_fill(priv->eq_table.eq[i].actv_ports.ports,
2527 dev->caps.num_ports);
2528 } else {
2529 set_bit(port,
2530 priv->eq_table.eq[i].actv_ports.ports);
2531 }
2532 /* We divide the Eqs evenly between the two ports.
2533 * (dev->caps.num_comp_vectors / dev->caps.num_ports)
2534 * refers to the number of Eqs per port
2535 * (i.e eqs_per_port). Theoretically, we would like to
2536 * write something like (i + 1) % eqs_per_port == 0.
2537 * However, since there's an asynchronous Eq, we have
2538 * to skip over it by comparing this condition to
2539 * !!((i + 1) > MLX4_EQ_ASYNC).
2540 */
2541 if ((dev->caps.num_comp_vectors > dev->caps.num_ports) &&
2542 ((i + 1) %
2543 (dev->caps.num_comp_vectors / dev->caps.num_ports)) ==
2544 !!((i + 1) > MLX4_EQ_ASYNC))
2545 /* If dev->caps.num_comp_vectors < dev->caps.num_ports,
2546 * everything is shared anyway.
2547 */
2548 port++;
2549 }
2520 2550
2521 dev->flags |= MLX4_FLAG_MSI_X; 2551 dev->flags |= MLX4_FLAG_MSI_X;
2522 2552
@@ -2526,10 +2556,15 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
2526 2556
2527no_msi: 2557no_msi:
2528 dev->caps.num_comp_vectors = 1; 2558 dev->caps.num_comp_vectors = 1;
2529 dev->caps.comp_pool = 0;
2530 2559
2531 for (i = 0; i < 2; ++i) 2560 BUG_ON(MLX4_EQ_ASYNC >= 2);
2561 for (i = 0; i < 2; ++i) {
2532 priv->eq_table.eq[i].irq = dev->persist->pdev->irq; 2562 priv->eq_table.eq[i].irq = dev->persist->pdev->irq;
2563 if (i != MLX4_EQ_ASYNC) {
2564 bitmap_fill(priv->eq_table.eq[i].actv_ports.ports,
2565 dev->caps.num_ports);
2566 }
2567 }
2533} 2568}
2534 2569
2535static int mlx4_init_port_info(struct mlx4_dev *dev, int port) 2570static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
@@ -2594,6 +2629,10 @@ static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
2594 device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr); 2629 device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr);
2595 device_remove_file(&info->dev->persist->pdev->dev, 2630 device_remove_file(&info->dev->persist->pdev->dev,
2596 &info->port_mtu_attr); 2631 &info->port_mtu_attr);
2632#ifdef CONFIG_RFS_ACCEL
2633 free_irq_cpu_rmap(info->rmap);
2634 info->rmap = NULL;
2635#endif
2597} 2636}
2598 2637
2599static int mlx4_init_steering(struct mlx4_dev *dev) 2638static int mlx4_init_steering(struct mlx4_dev *dev)
@@ -3024,7 +3063,7 @@ slave_start:
3024 if (err) 3063 if (err)
3025 goto err_master_mfunc; 3064 goto err_master_mfunc;
3026 3065
3027 priv->msix_ctl.pool_bm = 0; 3066 bitmap_zero(priv->msix_ctl.pool_bm, MAX_MSIX);
3028 mutex_init(&priv->msix_ctl.pool_lock); 3067 mutex_init(&priv->msix_ctl.pool_lock);
3029 3068
3030 mlx4_enable_msi_x(dev); 3069 mlx4_enable_msi_x(dev);
@@ -3046,7 +3085,6 @@ slave_start:
3046 !mlx4_is_mfunc(dev)) { 3085 !mlx4_is_mfunc(dev)) {
3047 dev->flags &= ~MLX4_FLAG_MSI_X; 3086 dev->flags &= ~MLX4_FLAG_MSI_X;
3048 dev->caps.num_comp_vectors = 1; 3087 dev->caps.num_comp_vectors = 1;
3049 dev->caps.comp_pool = 0;
3050 pci_disable_msix(pdev); 3088 pci_disable_msix(pdev);
3051 err = mlx4_setup_hca(dev); 3089 err = mlx4_setup_hca(dev);
3052 } 3090 }
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
index 502d3dd2c888..ff40098eaf4c 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
@@ -287,6 +287,12 @@ struct mlx4_icm_table {
287#define MLX4_CQE_SIZE_MASK_STRIDE 0x3 287#define MLX4_CQE_SIZE_MASK_STRIDE 0x3
288#define MLX4_EQE_SIZE_MASK_STRIDE 0x30 288#define MLX4_EQE_SIZE_MASK_STRIDE 0x30
289 289
290#define MLX4_EQ_ASYNC 0
291#define MLX4_EQ_TO_CQ_VECTOR(vector) ((vector) - \
292 !!((int)(vector) >= MLX4_EQ_ASYNC))
293#define MLX4_CQ_TO_EQ_VECTOR(vector) ((vector) + \
294 !!((int)(vector) >= MLX4_EQ_ASYNC))
295
290/* 296/*
291 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. 297 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
292 */ 298 */
@@ -391,6 +397,8 @@ struct mlx4_eq {
391 struct mlx4_buf_list *page_list; 397 struct mlx4_buf_list *page_list;
392 struct mlx4_mtt mtt; 398 struct mlx4_mtt mtt;
393 struct mlx4_eq_tasklet tasklet_ctx; 399 struct mlx4_eq_tasklet tasklet_ctx;
400 struct mlx4_active_ports actv_ports;
401 u32 ref_count;
394}; 402};
395 403
396struct mlx4_slave_eqe { 404struct mlx4_slave_eqe {
@@ -808,6 +816,7 @@ struct mlx4_port_info {
808 struct mlx4_vlan_table vlan_table; 816 struct mlx4_vlan_table vlan_table;
809 struct mlx4_roce_gid_table gid_table; 817 struct mlx4_roce_gid_table gid_table;
810 int base_qpn; 818 int base_qpn;
819 struct cpu_rmap *rmap;
811}; 820};
812 821
813struct mlx4_sense { 822struct mlx4_sense {
@@ -818,7 +827,7 @@ struct mlx4_sense {
818}; 827};
819 828
820struct mlx4_msix_ctl { 829struct mlx4_msix_ctl {
821 u64 pool_bm; 830 DECLARE_BITMAP(pool_bm, MAX_MSIX);
822 struct mutex pool_lock; 831 struct mutex pool_lock;
823}; 832};
824 833
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index d021f079f181..edd8fd69ec9a 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -338,7 +338,7 @@ struct mlx4_en_cq {
338 struct napi_struct napi; 338 struct napi_struct napi;
339 int size; 339 int size;
340 int buf_size; 340 int buf_size;
341 unsigned vector; 341 int vector;
342 enum cq_type is_tx; 342 enum cq_type is_tx;
343 u16 moder_time; 343 u16 moder_time;
344 u16 moder_cnt; 344 u16 moder_cnt;
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-30 18:45:00 -0500