mlx4: Modify proxy/tunnel QP mechanism so that guests do no calculations

Previously, the structure of a guest's proxy QPs followed the
structure of the PPF special qps (qp0 port 1, qp0 port 2, qp1 port 1,
qp1 port 2, ...).  The guest then did offset calculations on the
sqp_base qp number that the PPF passed to it in QUERY_FUNC_CAP().

This is now changed so that the guest does no offset calculations
regarding proxy or tunnel QPs to use.  This change frees the PPF from
needing to adhere to a specific order in allocating proxy and tunnel
QPs.

Now QUERY_FUNC_CAP provides each port individually with its proxy
qp0, proxy qp1, tunnel qp0, and tunnel qp1 QP numbers, and these are
used directly where required (with no offset calculations).

To accomplish this change, several fields were added to the phys_caps
structure for use by the PPF and by non-SR-IOV mode:

    base_sqpn -- in non-sriov mode, this was formerly sqp_start.
    base_proxy_sqpn -- the first physical proxy qp number -- used by PPF
    base_tunnel_sqpn -- the first physical tunnel qp number -- used by PPF.

The current code in the PPF still adheres to the previous layout of
sqps, proxy-sqps and tunnel-sqps.  However, the PPF can change this
layout without affecting VF or (paravirtualized) PF code.

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Roland Dreier <roland@purestorage.com>

1 files changed, 48 insertions, 13 deletions

diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 76f69fdd01d5..1e2ab9d00f1c 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -410,15 +410,16 @@ static int mlx4_how_many_lives_vf(struct mlx4_dev *dev)
 int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey)
 {
         u32 qk = MLX4_RESERVED_QKEY_BASE;
-        if (qpn >= dev->caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX ||
-            qpn < dev->caps.sqp_start)
+
+        if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX ||
+            qpn < dev->phys_caps.base_proxy_sqpn)
                 return -EINVAL;
 
-        if (qpn >= dev->caps.base_tunnel_sqpn)
+        if (qpn >= dev->phys_caps.base_tunnel_sqpn)
                 /* tunnel qp */
-                qk += qpn - dev->caps.base_tunnel_sqpn;
+                qk += qpn - dev->phys_caps.base_tunnel_sqpn;
         else
-                qk += qpn - dev->caps.sqp_start;
+                qk += qpn - dev->phys_caps.base_proxy_sqpn;
         *qkey = qk;
         return 0;
 }
@@ -527,9 +528,10 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
         }
 
         memset(&func_cap, 0, sizeof(func_cap));
-        err = mlx4_QUERY_FUNC_CAP(dev, &func_cap);
+        err = mlx4_QUERY_FUNC_CAP(dev, 0, &func_cap);
         if (err) {
-                mlx4_err(dev, "QUERY_FUNC_CAP command failed, aborting.\n");
+                mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d).\n",
+                          err);
                 return err;
         }
 
@@ -557,12 +559,33 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
                 return -ENODEV;
         }
 
+        dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
+        dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
+        dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
+        dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
+
+        if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy ||
+            !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) {
+                err = -ENOMEM;
+                goto err_mem;
+        }
+
         for (i = 1; i <= dev->caps.num_ports; ++i) {
+                err = mlx4_QUERY_FUNC_CAP(dev, (u32) i, &func_cap);
+                if (err) {
+                        mlx4_err(dev, "QUERY_FUNC_CAP port command failed for"
+                                 " port %d, aborting (%d).\n", i, err);
+                        goto err_mem;
+                }
+                dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn;
+                dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn;
+                dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn;
+                dev->caps.qp1_proxy[i - 1] = func_cap.qp1_proxy_qpn;
                 dev->caps.port_mask[i] = dev->caps.port_type[i];
                 if (mlx4_get_slave_pkey_gid_tbl_len(dev, i,
                                                     &dev->caps.gid_table_len[i],
                                                     &dev->caps.pkey_table_len[i]))
-                        return -ENODEV;
+                        goto err_mem;
         }
 
         if (dev->caps.uar_page_size * (dev->caps.num_uars -
@@ -572,14 +595,20 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
                          "PCI resource 2 size of 0x%llx, aborting.\n",
                          dev->caps.uar_page_size * dev->caps.num_uars,
                          (unsigned long long) pci_resource_len(dev->pdev, 2));
-                return -ENODEV;
+                goto err_mem;
         }
 
-        /* Calculate our sqp_start */
-        dev->caps.sqp_start = func_cap.base_proxy_qpn;
-        dev->caps.base_tunnel_sqpn = func_cap.base_tunnel_qpn;
-
         return 0;
+
+err_mem:
+        kfree(dev->caps.qp0_tunnel);
+        kfree(dev->caps.qp0_proxy);
+        kfree(dev->caps.qp1_tunnel);
+        kfree(dev->caps.qp1_proxy);
+        dev->caps.qp0_tunnel = dev->caps.qp0_proxy =
+                dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL;
+
+        return err;
 }
 
 /*
@@ -2261,6 +2290,12 @@ static void mlx4_remove_one(struct pci_dev *pdev)
 
                 if (!mlx4_is_slave(dev))
                         mlx4_free_ownership(dev);
+
+                kfree(dev->caps.qp0_tunnel);
+                kfree(dev->caps.qp0_proxy);
+                kfree(dev->caps.qp1_tunnel);
+                kfree(dev->caps.qp1_proxy);
+
                 kfree(priv);
                 pci_release_regions(pdev);
                 pci_disable_device(pdev);