mlx4: MAD_IFC paravirtualization

The MAD_IFC firmware command fulfills two functions.

First, it is used in the QP0/QP1 MAD-handling flow to obtain
information from the FW (for answering queries), and for setting
variables in the HCA (MAD SET packets).

For this, MAD_IFC should provide the FW (physical) view of the data.
This is the view that OpenSM needs.  We call this the "network view".

In the second case, MAD_IFC is used by various verbs to obtain data
regarding the local HCA (e.g., ib_query_device()).  We call this the
"host view".

This data needs to be paravirtualized.

MAD_IFC therefore needs a wrapper function, and also needs another
flag indicating whether it should provide the network view (when it is
called by ib_process_mad in special-qp packet handling), or the host
view (when it is called while implementing a verb).

There are currently 2 flag parameters in mlx4_MAD_IFC already:
ignore_bkey and ignore_mkey.  These two parameters are replaced by a
single "mad_ifc_flags" parameter, with different bits set for each
flag.  A third flag is added: "network-view/host-view".

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Roland Dreier <roland@purestorage.com>

1 files changed, 162 insertions, 0 deletions

diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c
index 662a3c5da73..a13d8a69b3b 100644
--- a/drivers/net/ethernet/mellanox/mlx4/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c
@@ -40,6 +40,7 @@
 
 #include <linux/mlx4/cmd.h>
 #include <linux/semaphore.h>
+#include <rdma/ib_smi.h>
 
 #include <asm/io.h>
 
@@ -627,6 +628,149 @@ static int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr,
                             MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
 }
 
+static int query_pkey_block(struct mlx4_dev *dev, u8 port, u16 index, u16 *pkey,
+                               struct mlx4_cmd_mailbox *inbox,
+                               struct mlx4_cmd_mailbox *outbox)
+{
+        struct ib_smp *in_mad = (struct ib_smp *)(inbox->buf);
+        struct ib_smp *out_mad = (struct ib_smp *)(outbox->buf);
+        int err;
+        int i;
+
+        if (index & 0x1f)
+                return -EINVAL;
+
+        in_mad->attr_mod = cpu_to_be32(index / 32);
+
+        err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, 3,
+                           MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C,
+                           MLX4_CMD_NATIVE);
+        if (err)
+                return err;
+
+        for (i = 0; i < 32; ++i)
+                pkey[i] = be16_to_cpu(((__be16 *) out_mad->data)[i]);
+
+        return err;
+}
+
+static int get_full_pkey_table(struct mlx4_dev *dev, u8 port, u16 *table,
+                               struct mlx4_cmd_mailbox *inbox,
+                               struct mlx4_cmd_mailbox *outbox)
+{
+        int i;
+        int err;
+
+        for (i = 0; i < dev->caps.pkey_table_len[port]; i += 32) {
+                err = query_pkey_block(dev, port, i, table + i, inbox, outbox);
+                if (err)
+                        return err;
+        }
+
+        return 0;
+}
+#define PORT_CAPABILITY_LOCATION_IN_SMP 20
+#define PORT_STATE_OFFSET 32
+
+static enum ib_port_state vf_port_state(struct mlx4_dev *dev, int port, int vf)
+{
+        /* will be modified when add alias_guid feature */
+        return IB_PORT_DOWN;
+}
+
+static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave,
+                                struct mlx4_vhcr *vhcr,
+                                struct mlx4_cmd_mailbox *inbox,
+                                struct mlx4_cmd_mailbox *outbox,
+                                struct mlx4_cmd_info *cmd)
+{
+        struct ib_smp *smp = inbox->buf;
+        u32 index;
+        u8 port;
+        u16 *table;
+        int err;
+        int vidx, pidx;
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        struct ib_smp *outsmp = outbox->buf;
+        __be16 *outtab = (__be16 *)(outsmp->data);
+        __be32 slave_cap_mask;
+        port = vhcr->in_modifier;
+
+        if (smp->base_version == 1 &&
+            smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED &&
+            smp->class_version == 1) {
+                if (smp->method == IB_MGMT_METHOD_GET) {
+                        if (smp->attr_id == IB_SMP_ATTR_PKEY_TABLE) {
+                                index = be32_to_cpu(smp->attr_mod);
+                                if (port < 1 || port > dev->caps.num_ports)
+                                        return -EINVAL;
+                                table = kcalloc(dev->caps.pkey_table_len[port], sizeof *table, GFP_KERNEL);
+                                if (!table)
+                                        return -ENOMEM;
+                                /* need to get the full pkey table because the paravirtualized
+                                 * pkeys may be scattered among several pkey blocks.
+                                 */
+                                err = get_full_pkey_table(dev, port, table, inbox, outbox);
+                                if (!err) {
+                                        for (vidx = index * 32; vidx < (index + 1) * 32; ++vidx) {
+                                                pidx = priv->virt2phys_pkey[slave][port - 1][vidx];
+                                                outtab[vidx % 32] = cpu_to_be16(table[pidx]);
+                                        }
+                                }
+                                kfree(table);
+                                return err;
+                        }
+                        if (smp->attr_id == IB_SMP_ATTR_PORT_INFO) {
+                                /*get the slave specific caps:*/
+                                /*do the command */
+                                err = mlx4_cmd_box(dev, inbox->dma, outbox->dma,
+                                            vhcr->in_modifier, vhcr->op_modifier,
+                                            vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE);
+                                /* modify the response for slaves */
+                                if (!err && slave != mlx4_master_func_num(dev)) {
+                                        u8 *state = outsmp->data + PORT_STATE_OFFSET;
+
+                                        *state = (*state & 0xf0) | vf_port_state(dev, port, slave);
+                                        slave_cap_mask = priv->mfunc.master.slave_state[slave].ib_cap_mask[port];
+                                        memcpy(outsmp->data + PORT_CAPABILITY_LOCATION_IN_SMP, &slave_cap_mask, 4);
+                                }
+                                return err;
+                        }
+                        if (smp->attr_id == IB_SMP_ATTR_GUID_INFO) {
+                                /* compute slave's gid block */
+                                smp->attr_mod = cpu_to_be32(slave / 8);
+                                /* execute cmd */
+                                err = mlx4_cmd_box(dev, inbox->dma, outbox->dma,
+                                             vhcr->in_modifier, vhcr->op_modifier,
+                                             vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE);
+                                if (!err) {
+                                        /* if needed, move slave gid to index 0 */
+                                        if (slave % 8)
+                                                memcpy(outsmp->data,
+                                                       outsmp->data + (slave % 8) * 8, 8);
+                                        /* delete all other gids */
+                                        memset(outsmp->data + 8, 0, 56);
+                                }
+                                return err;
+                        }
+                }
+        }
+        if (slave != mlx4_master_func_num(dev) &&
+            ((smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) ||
+             (smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED &&
+              smp->method == IB_MGMT_METHOD_SET))) {
+                mlx4_err(dev, "slave %d is trying to execute a Subnet MGMT MAD, "
+                         "class 0x%x, method 0x%x for attr 0x%x. Rejecting\n",
+                         slave, smp->method, smp->mgmt_class,
+                         be16_to_cpu(smp->attr_id));
+                return -EPERM;
+        }
+        /*default:*/
+        return mlx4_cmd_box(dev, inbox->dma, outbox->dma,
+                                    vhcr->in_modifier, vhcr->op_modifier,
+                                    vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE);
+}
+
 int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave,
                      struct mlx4_vhcr *vhcr,
                      struct mlx4_cmd_mailbox *inbox,
@@ -1061,6 +1205,24 @@ static struct mlx4_cmd_info cmd_info[] = {
                 .wrapper = mlx4_GEN_QP_wrapper
         },
         {
+                .opcode = MLX4_CMD_CONF_SPECIAL_QP,
+                .has_inbox = false,
+                .has_outbox = false,
+                .out_is_imm = false,
+                .encode_slave_id = false,
+                .verify = NULL, /* XXX verify: only demux can do this */
+                .wrapper = NULL
+        },
+        {
+                .opcode = MLX4_CMD_MAD_IFC,
+                .has_inbox = true,
+                .has_outbox = true,
+                .out_is_imm = false,
+                .encode_slave_id = false,
+                .verify = NULL,
+                .wrapper = mlx4_MAD_IFC_wrapper
+        },
+        {
                 .opcode = MLX4_CMD_QUERY_IF_STAT,
                 .has_inbox = false,
                 .has_outbox = true,