mlx4: In RoCE allow guests to have multiple GIDS

The GIDs are statically distributed, as follows:
PF: gets 16 GIDs
VFs:  Remaining GIDS are divided evenly between VFs activated by the driver.
      If the division is not even, lower-numbered VFs get an extra GID.

For an IB interface, the number of gids per guest remains as before: one gid per guest.

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

7 files changed, 192 insertions, 38 deletions

diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index c2e9879a5a34..c5bca0f0da4a 100644
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -511,9 +511,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
         memset(&attr, 0, sizeof attr);
         attr.port_num = port;
         if (is_eth) {
-                ret = mlx4_get_roce_gid_from_slave(dev->dev, port, slave, attr.grh.dgid.raw);
-                if (ret)
-                        return ret;
+                memcpy(&attr.grh.dgid.raw[0], &grh->dgid.raw[0], 16);
                 attr.ah_flags = IB_AH_GRH;
         }
         ah = ib_create_ah(tun_ctx->pd, &attr);
@@ -1216,6 +1214,34 @@ out:
         return ret;
 }
 
+static int get_slave_base_gid_ix(struct mlx4_ib_dev *dev, int slave, int port)
+{
+        int gids;
+        int vfs;
+
+        if (rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_INFINIBAND)
+                return slave;
+
+        gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS;
+        vfs = dev->dev->num_vfs;
+
+        if (slave == 0)
+                return 0;
+        if (slave <= gids % vfs)
+                return MLX4_ROCE_PF_GIDS + ((gids / vfs) + 1) * (slave - 1);
+
+        return MLX4_ROCE_PF_GIDS + (gids % vfs) + ((gids / vfs) * (slave - 1));
+}
+
+static void fill_in_real_sgid_index(struct mlx4_ib_dev *dev, int slave, int port,
+                                    struct ib_ah_attr *ah_attr)
+{
+        if (rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_INFINIBAND)
+                ah_attr->grh.sgid_index = slave;
+        else
+                ah_attr->grh.sgid_index += get_slave_base_gid_ix(dev, slave, port);
+}
+
 static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc *wc)
 {
         struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev);
@@ -1303,7 +1329,7 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc
         ah.ibah.device = ctx->ib_dev;
         mlx4_ib_query_ah(&ah.ibah, &ah_attr);
         if (ah_attr.ah_flags & IB_AH_GRH)
-                ah_attr.grh.sgid_index = slave;
+                fill_in_real_sgid_index(dev, slave, ctx->port, &ah_attr);
 
         mlx4_ib_send_to_wire(dev, slave, ctx->port,
                              is_proxy_qp0(dev, wc->src_qp, slave) ?
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index 9cdf452140da..6e1ee2170a39 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -934,7 +934,10 @@ int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave,
                 MLX4_PUT(outbox->buf, port_type,
                          QUERY_PORT_SUPPORTED_TYPE_OFFSET);
 
-                short_field = 1; /* slave max gids */
+                if (dev->caps.port_type[vhcr->in_modifier] == MLX4_PORT_TYPE_ETH)
+                        short_field = mlx4_get_slave_num_gids(dev, slave);
+                else
+                        short_field = 1; /* slave max gids */
                 MLX4_PUT(outbox->buf, short_field,
                          QUERY_PORT_CUR_MAX_GID_OFFSET);
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 979ea4364efb..4c441aa83016 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -1462,7 +1462,11 @@ static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev)
         int i;
 
         for (i = 1; i <= dev->caps.num_ports; i++) {
-                dev->caps.gid_table_len[i] = 1;
+                if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH)
+                        dev->caps.gid_table_len[i] =
+                                mlx4_get_slave_num_gids(dev, 0);
+                else
+                        dev->caps.gid_table_len[i] = 1;
                 dev->caps.pkey_table_len[i] =
                         dev->phys_caps.pkey_phys_table_len[i] - 1;
         }
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
index da829f4ef938..6ba38c98c492 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
@@ -1287,4 +1287,7 @@ void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work);
 
 void mlx4_init_quotas(struct mlx4_dev *dev);
 
+int mlx4_get_slave_num_gids(struct mlx4_dev *dev, int slave);
+int mlx4_get_base_gid_ix(struct mlx4_dev *dev, int slave);
+
 #endif /* MLX4_H */
diff --git a/drivers/net/ethernet/mellanox/mlx4/port.c b/drivers/net/ethernet/mellanox/mlx4/port.c
index 591740b06043..ece328166e94 100644
--- a/drivers/net/ethernet/mellanox/mlx4/port.c
+++ b/drivers/net/ethernet/mellanox/mlx4/port.c
@@ -507,6 +507,31 @@ int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps)
 }
 static struct mlx4_roce_gid_entry zgid_entry;
 
+int mlx4_get_slave_num_gids(struct mlx4_dev *dev, int slave)
+{
+        if (slave == 0)
+                return MLX4_ROCE_PF_GIDS;
+        if (slave <= ((MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) % dev->num_vfs))
+                return ((MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) / dev->num_vfs) + 1;
+        return (MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) / dev->num_vfs;
+}
+
+int mlx4_get_base_gid_ix(struct mlx4_dev *dev, int slave)
+{
+        int gids;
+        int vfs;
+
+        gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS;
+        vfs = dev->num_vfs;
+
+        if (slave == 0)
+                return 0;
+        if (slave <= gids % vfs)
+                return MLX4_ROCE_PF_GIDS + ((gids / vfs) + 1) * (slave - 1);
+
+        return MLX4_ROCE_PF_GIDS + (gids % vfs) + ((gids / vfs) * (slave - 1));
+}
+
 static int mlx4_common_set_port(struct mlx4_dev *dev, int slave, u32 in_mod,
                                 u8 op_mod, struct mlx4_cmd_mailbox *inbox)
 {
@@ -516,15 +541,18 @@ static int mlx4_common_set_port(struct mlx4_dev *dev, int slave, u32 in_mod,
         struct mlx4_slave_state *slave_st = &master->slave_state[slave];
         struct mlx4_set_port_rqp_calc_context *qpn_context;
         struct mlx4_set_port_general_context *gen_context;
-        struct mlx4_roce_gid_entry *gid_entry;
+        struct mlx4_roce_gid_entry *gid_entry_tbl, *gid_entry_mbox, *gid_entry_mb1;
         int reset_qkey_viols;
         int port;
         int is_eth;
+        int num_gids;
+        int base;
         u32 in_modifier;
         u32 promisc;
         u16 mtu, prev_mtu;
         int err;
-        int i;
+        int i, j;
+        int offset;
         __be32 agg_cap_mask;
         __be32 slave_cap_mask;