IB/mlx4: Handle FW command interface rev 3

Upcoming firmware introduces command interface revision 3, which
changes the way port capabilities are queried and set.  Update the
driver to handle both the new and old command interfaces by adding a
new MLX4_FLAG_OLD_PORT_CMDS that it is set after querying the firmware
interface revision and then using the correct interface based on the
setting of the flag.

Signed-off-by: Roland Dreier <rolandd@cisco.com>

5 files changed, 113 insertions, 81 deletions

diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 402f3a20ec0a..1095c82b38c2 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -125,7 +125,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
         props->local_ca_ack_delay  = dev->dev->caps.local_ca_ack_delay;
         props->atomic_cap          = dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_ATOMIC ?
                 IB_ATOMIC_HCA : IB_ATOMIC_NONE;
-        props->max_pkeys           = dev->dev->caps.pkey_table_len;
+        props->max_pkeys           = dev->dev->caps.pkey_table_len[1];
         props->max_mcast_grp       = dev->dev->caps.num_mgms + dev->dev->caps.num_amgms;
         props->max_mcast_qp_attach = dev->dev->caps.num_qp_per_mgm;
         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
@@ -168,9 +168,9 @@ static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port,
         props->state            = out_mad->data[32] & 0xf;
         props->phys_state       = out_mad->data[33] >> 4;
         props->port_cap_flags   = be32_to_cpup((__be32 *) (out_mad->data + 20));
-        props->gid_tbl_len      = to_mdev(ibdev)->dev->caps.gid_table_len;
+        props->gid_tbl_len      = to_mdev(ibdev)->dev->caps.gid_table_len[port];
         props->max_msg_sz       = 0x80000000;
-        props->pkey_tbl_len     = to_mdev(ibdev)->dev->caps.pkey_table_len;
+        props->pkey_tbl_len     = to_mdev(ibdev)->dev->caps.pkey_table_len[port];
         props->bad_pkey_cntr    = be16_to_cpup((__be16 *) (out_mad->data + 46));
         props->qkey_viol_cntr   = be16_to_cpup((__be16 *) (out_mad->data + 48));
         props->active_width     = out_mad->data[31] & 0xf;
@@ -280,8 +280,14 @@ static int mlx4_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols,
                 return PTR_ERR(mailbox);
 
         memset(mailbox->buf, 0, 256);
-        *(u8 *) mailbox->buf         = !!reset_qkey_viols << 6;
-        ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask);
+
+        if (dev->dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
+                *(u8 *) mailbox->buf         = !!reset_qkey_viols << 6;
+                ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask);
+        } else {
+                ((u8 *) mailbox->buf)[3]     = !!reset_qkey_viols;
+                ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask);
+        }
 
         err = mlx4_cmd(dev->dev, mailbox->dma, port, 0, MLX4_CMD_SET_PORT,
                        MLX4_CMD_TIME_CLASS_B);
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 95d4a9d6994c..355a31f9c03c 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -603,24 +603,6 @@ int mlx4_ib_destroy_qp(struct ib_qp *qp)
         return 0;
 }
 
-static void init_port(struct mlx4_ib_dev *dev, int port)
-{
-        struct mlx4_init_port_param param;
-        int err;
-
-        memset(&param, 0, sizeof param);
-
-        param.port_width_cap = dev->dev->caps.port_width_cap;
-        param.vl_cap         = dev->dev->caps.vl_cap;
-        param.mtu            = ib_mtu_enum_to_int(dev->dev->caps.mtu_cap);
-        param.max_gid        = dev->dev->caps.gid_table_len;
-        param.max_pkey       = dev->dev->caps.pkey_table_len;
-
-        err = mlx4_INIT_PORT(dev->dev, &param, port);
-        if (err)
-                printk(KERN_WARNING "INIT_PORT failed, return code %d.\n", err);
-}
-
 static int to_mlx4_st(enum ib_qp_type type)
 {
         switch (type) {
@@ -694,9 +676,9 @@ static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,
         path->counter_index = 0xff;
 
         if (ah->ah_flags & IB_AH_GRH) {
-                if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len) {
+                if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {
                         printk(KERN_ERR "sgid_index (%u) too large. max is %d\n",
-                               ah->grh.sgid_index, dev->dev->caps.gid_table_len - 1);
+                               ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);
                         return -1;
                 }
 
@@ -812,13 +794,14 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
         }
 
         if (attr_mask & IB_QP_ALT_PATH) {
-                if (attr->alt_pkey_index >= dev->dev->caps.pkey_table_len)
-                        return -EINVAL;
-
                 if (attr->alt_port_num == 0 ||
                     attr->alt_port_num > dev->dev->caps.num_ports)
                         return -EINVAL;
 
+                if (attr->alt_pkey_index >=
+                    dev->dev->caps.pkey_table_len[attr->alt_port_num])
+                        return -EINVAL;
+
                 if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
                                   attr->alt_port_num))
                         return -EINVAL;
@@ -949,7 +932,9 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
          */
         if (is_qp0(dev, qp)) {
                 if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
-                        init_port(dev, qp->port);
+                        if (mlx4_INIT_PORT(dev->dev, qp->port))
+                                printk(KERN_WARNING "INIT_PORT failed for port %d\n",
+                                       qp->port);
 
                 if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
                     (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
@@ -1012,16 +997,17 @@ int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
         if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask))
                 goto out;
 
-        if ((attr_mask & IB_QP_PKEY_INDEX) &&
-             attr->pkey_index >= dev->dev->caps.pkey_table_len) {
-                goto out;
-        }
-
         if ((attr_mask & IB_QP_PORT) &&
             (attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) {
                 goto out;
         }
 
+        if (attr_mask & IB_QP_PKEY_INDEX) {
+                int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
+                if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])
+                        goto out;
+        }
+
         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
             attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
                 goto out;
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index 81fc546a1c44..d2b065351e45 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -38,7 +38,9 @@
 #include "icm.h"
 
 enum {
-        MLX4_COMMAND_INTERFACE_REV      = 2,
+        MLX4_COMMAND_INTERFACE_MIN_REV          = 2,
+        MLX4_COMMAND_INTERFACE_MAX_REV          = 3,
+        MLX4_COMMAND_INTERFACE_NEW_PORT_CMDS    = 3,
 };
 
 extern void __buggy_use_of_MLX4_GET(void);
@@ -107,6 +109,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
         u16 size;
         u16 stat_rate;
         int err;
+        int i;
 
 #define QUERY_DEV_CAP_OUT_SIZE                 0x100
 #define QUERY_DEV_CAP_MAX_SRQ_SZ_OFFSET         0x10
@@ -176,7 +179,6 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 
         err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_DEV_CAP,
                            MLX4_CMD_TIME_CLASS_A);
-
         if (err)
                 goto out;
 
@@ -216,18 +218,10 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
         dev_cap->max_rdma_global = 1 << (field & 0x3f);
         MLX4_GET(field, outbox, QUERY_DEV_CAP_ACK_DELAY_OFFSET);
         dev_cap->local_ca_ack_delay = field & 0x1f;
-        MLX4_GET(field, outbox, QUERY_DEV_CAP_MTU_WIDTH_OFFSET);
-        dev_cap->max_mtu        = field >> 4;
-        dev_cap->max_port_width = field & 0xf;
         MLX4_GET(field, outbox, QUERY_DEV_CAP_VL_PORT_OFFSET);
-        dev_cap->max_vl    = field >> 4;
         dev_cap->num_ports = field & 0xf;
-        MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_GID_OFFSET);
-        dev_cap->max_gids = 1 << (field & 0xf);
         MLX4_GET(stat_rate, outbox, QUERY_DEV_CAP_RATE_SUPPORT_OFFSET);
         dev_cap->stat_rate_support = stat_rate;
-        MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_PKEY_OFFSET);
-        dev_cap->max_pkeys = 1 << (field & 0xf);
         MLX4_GET(dev_cap->flags, outbox, QUERY_DEV_CAP_FLAGS_OFFSET);
         MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_UAR_OFFSET);
         dev_cap->reserved_uars = field >> 4;
@@ -304,6 +298,42 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
         MLX4_GET(dev_cap->max_icm_sz, outbox,
                  QUERY_DEV_CAP_MAX_ICM_SZ_OFFSET);
 
+        if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
+                for (i = 1; i <= dev_cap->num_ports; ++i) {
+                        MLX4_GET(field, outbox, QUERY_DEV_CAP_VL_PORT_OFFSET);
+                        dev_cap->max_vl[i]         = field >> 4;
+                        MLX4_GET(field, outbox, QUERY_DEV_CAP_MTU_WIDTH_OFFSET);
+                        dev_cap->max_mtu[i]        = field >> 4;
+                        dev_cap->max_port_width[i] = field & 0xf;
+                        MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_GID_OFFSET);
+                        dev_cap->max_gids[i]       = 1 << (field & 0xf);
+                        MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_PKEY_OFFSET);
+                        dev_cap->max_pkeys[i]      = 1 << (field & 0xf);
+                }
+        } else {
+#define QUERY_PORT_MTU_OFFSET                   0x01
+#define QUERY_PORT_WIDTH_OFFSET                 0x06
+#define QUERY_PORT_MAX_GID_PKEY_OFFSET          0x07
+#define QUERY_PORT_MAX_VL_OFFSET                0x0b
+
+                for (i = 1; i <= dev_cap->num_ports; ++i) {
+                        err = mlx4_cmd_box(dev, 0, mailbox->dma, i, 0, MLX4_CMD_QUERY_PORT,
+                                           MLX4_CMD_TIME_CLASS_B);
+                        if (err)
+                                goto out;
+
+                        MLX4_GET(field, outbox, QUERY_PORT_MTU_OFFSET);
+                        dev_cap->max_mtu[i]        = field & 0xf;
+                        MLX4_GET(field, outbox, QUERY_PORT_WIDTH_OFFSET);
+                        dev_cap->max_port_width[i] = field & 0xf;
+                        MLX4_GET(field, outbox, QUERY_PORT_MAX_GID_PKEY_OFFSET);
+                        dev_cap->max_gids[i]       = 1 << (field >> 4);
+                        dev_cap->max_pkeys[i]      = 1 << (field & 0xf);
+                        MLX4_GET(field, outbox, QUERY_PORT_MAX_VL_OFFSET);
+                        dev_cap->max_vl[i]         = field & 0xf;
+                }
+        }
+
         if (dev_cap->bmme_flags & 1)
                 mlx4_dbg(dev, "Base MM extensions: yes "
                          "(flags %d, rsvd L_Key %08x)\n",
@@ -338,8 +368,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
         mlx4_dbg(dev, "Max CQEs: %d, max WQEs: %d, max SRQ WQEs: %d\n",
                  dev_cap->max_cq_sz, dev_cap->max_qp_sz, dev_cap->max_srq_sz);
         mlx4_dbg(dev, "Local CA ACK delay: %d, max MTU: %d, port width cap: %d\n",
-                 dev_cap->local_ca_ack_delay, 128 << dev_cap->max_mtu,
-                 dev_cap->max_port_width);
+                 dev_cap->local_ca_ack_delay, 128 << dev_cap->max_mtu[1],
+                 dev_cap->max_port_width[1]);
         mlx4_dbg(dev, "Max SQ desc size: %d, max SQ S/G: %d\n",
                  dev_cap->max_sq_desc_sz, dev_cap->max_sq_sg);
         mlx4_dbg(dev, "Max RQ desc size: %d, max RQ S/G: %d\n",
@@ -491,7 +521,8 @@ int mlx4_QUERY_FW(struct mlx4_dev *dev)
                 ((fw_ver & 0x0000ffffull) << 16);
 
         MLX4_GET(cmd_if_rev, outbox, QUERY_FW_CMD_IF_REV_OFFSET);
-        if (cmd_if_rev != MLX4_COMMAND_INTERFACE_REV) {
+        if (cmd_if_rev < MLX4_COMMAND_INTERFACE_MIN_REV ||
+            cmd_if_rev > MLX4_COMMAND_INTERFACE_MAX_REV) {
                 mlx4_err(dev, "Installed FW has unsupported "
                          "command interface revision %d.\n",
                          cmd_if_rev);
@@ -499,12 +530,15 @@ int mlx4_QUERY_FW(struct mlx4_dev *dev)
                          (int) (dev->caps.fw_ver >> 32),
                          (int) (dev->caps.fw_ver >> 16) & 0xffff,
                          (int) dev->caps.fw_ver & 0xffff);
-                mlx4_err(dev, "This driver version supports only revision %d.\n",
-                         MLX4_COMMAND_INTERFACE_REV);
+                mlx4_err(dev, "This driver version supports only revisions %d to %d.\n",
+                         MLX4_COMMAND_INTERFACE_MIN_REV, MLX4_COMMAND_INTERFACE_MAX_REV);
                 err = -ENODEV;
                 goto out;
         }
 
+        if (cmd_if_rev < MLX4_COMMAND_INTERFACE_NEW_PORT_CMDS)
+                dev->flags |= MLX4_FLAG_OLD_PORT_CMDS;
+
         MLX4_GET(lg, outbox, QUERY_FW_MAX_CMD_OFFSET);
         cmd->max_cmds = 1 << lg;
 
@@ -708,13 +742,15 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
         return err;
 }
 
-int mlx4_INIT_PORT(struct mlx4_dev *dev, struct mlx4_init_port_param *param, int port)
+int mlx4_INIT_PORT(struct mlx4_dev *dev, int port)
 {
         struct mlx4_cmd_mailbox *mailbox;
         u32 *inbox;
         int err;
         u32 flags;
+        u16 field;
 
+        if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
 #define INIT_PORT_IN_SIZE          256
 #define INIT_PORT_FLAGS_OFFSET     0x00
 #define INIT_PORT_FLAG_SIG         (1 << 18)
@@ -729,32 +765,32 @@ int mlx4_INIT_PORT(struct mlx4_dev *dev, struct mlx4_init_port_param *param, int
 #define INIT_PORT_NODE_GUID_OFFSET 0x18
 #define INIT_PORT_SI_GUID_OFFSET   0x20
 
-        mailbox = mlx4_alloc_cmd_mailbox(dev);
-        if (IS_ERR(mailbox))
-                return PTR_ERR(mailbox);
-        inbox = mailbox->buf;
+                mailbox = mlx4_alloc_cmd_mailbox(dev);
+                if (IS_ERR(mailbox))
+                        return PTR_ERR(mailbox);
+                inbox = mailbox->buf;
 
-        memset(inbox, 0, INIT_PORT_IN_SIZE);
+                memset(inbox, 0, INIT_PORT_IN_SIZE);
 
-        flags = 0;
-        flags |= param->set_guid0     ? INIT_PORT_FLAG_G0  : 0;
-        flags |= param->set_node_guid ? INIT_PORT_FLAG_NG  : 0;
-        flags |= param->set_si_guid   ? INIT_PORT_FLAG_SIG : 0;
-        flags |= (param->vl_cap & 0xf) << INIT_PORT_VL_SHIFT;
-        flags |= (param->port_width_cap & 0xf) << INIT_PORT_PORT_WIDTH_SHIFT;
-        MLX4_PUT(inbox, flags,            INIT_PORT_FLAGS_OFFSET);
+                flags = 0;
+                flags |= (dev->caps.vl_cap[port] & 0xf) << INIT_PORT_VL_SHIFT;
+                flags |= (dev->caps.port_width_cap[port] & 0xf) << INIT_PORT_PORT_WIDTH_SHIFT;
+                MLX4_PUT(inbox, flags,            INIT_PORT_FLAGS_OFFSET);
 
-        MLX4_PUT(inbox, param->mtu,       INIT_PORT_MTU_OFFSET);
-        MLX4_PUT(inbox, param->max_gid,   INIT_PORT_MAX_GID_OFFSET);
-        MLX4_PUT(inbox, param->max_pkey,  INIT_PORT_MAX_PKEY_OFFSET);
-        MLX4_PUT(inbox, param->guid0,     INIT_PORT_GUID0_OFFSET);
-        MLX4_PUT(inbox, param->node_guid, INIT_PORT_NODE_GUID_OFFSET);
-        MLX4_PUT(inbox, param->si_guid,   INIT_PORT_SI_GUID_OFFSET);
+                field = 128 << dev->caps.mtu_cap[port];
+                MLX4_PUT(inbox, field, INIT_PORT_MTU_OFFSET);
+                field = dev->caps.gid_table_len[port];
+                MLX4_PUT(inbox, field, INIT_PORT_MAX_GID_OFFSET);
+                field = dev->caps.pkey_table_len[port];
+                MLX4_PUT(inbox, field, INIT_PORT_MAX_PKEY_OFFSET);
 
-        err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_INIT_PORT,
-                       MLX4_CMD_TIME_CLASS_A);
+                err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_INIT_PORT,
+                               MLX4_CMD_TIME_CLASS_A);
 
-        mlx4_free_cmd_mailbox(dev, mailbox);
+                mlx4_free_cmd_mailbox(dev, mailbox);
+        } else
+                err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT,
+                               MLX4_CMD_TIME_CLASS_A);
 
         return err;
 }
diff --git a/drivers/net/mlx4/fw.h b/drivers/net/mlx4/fw.h
index 2616fa53d4d0..296254ac27c1 100644
--- a/drivers/net/mlx4/fw.h
+++ b/drivers/net/mlx4/fw.h
@@ -59,13 +59,13 @@ struct mlx4_dev_cap {
         int max_responder_per_qp;
         int max_rdma_global;
         int local_ca_ack_delay;
-        int max_mtu;
-        int max_port_width;
-        int max_vl;
         int num_ports;
-        int max_gids;
+        int max_mtu[MLX4_MAX_PORTS + 1];
+        int max_port_width[MLX4_MAX_PORTS + 1];
+        int max_vl[MLX4_MAX_PORTS + 1];
+        int max_gids[MLX4_MAX_PORTS + 1];
+        int max_pkeys[MLX4_MAX_PORTS + 1];
         u16 stat_rate_support;
-        int max_pkeys;
         u32 flags;
         int reserved_uars;
         int uar_size;
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index d4172937025b..41eafebf5823 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -88,6 +88,7 @@ static struct mlx4_profile default_profile = {
 static int __devinit mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 {
         int err;
+        int i;
 
         err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
         if (err) {
@@ -117,11 +118,15 @@ static int __devinit mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev
         }
 
         dev->caps.num_ports          = dev_cap->num_ports;
+        for (i = 1; i <= dev->caps.num_ports; ++i) {
+                dev->caps.vl_cap[i]         = dev_cap->max_vl[i];
+                dev->caps.mtu_cap[i]        = dev_cap->max_mtu[i];
+                dev->caps.gid_table_len[i]  = dev_cap->max_gids[i];
+                dev->caps.pkey_table_len[i] = dev_cap->max_pkeys[i];
+                dev->caps.port_width_cap[i] = dev_cap->max_port_width[i];
+        }
+
         dev->caps.num_uars           = dev_cap->uar_size / PAGE_SIZE;
-        dev->caps.vl_cap             = dev_cap->max_vl;
-        dev->caps.mtu_cap            = dev_cap->max_mtu;
-        dev->caps.gid_table_len      = dev_cap->max_gids;
-        dev->caps.pkey_table_len     = dev_cap->max_pkeys;
         dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay;
         dev->caps.bf_reg_size        = dev_cap->bf_reg_size;
         dev->caps.bf_regs_per_page   = dev_cap->bf_regs_per_page;
@@ -148,7 +153,6 @@ static int __devinit mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev
         dev->caps.reserved_mrws      = dev_cap->reserved_mrws;
         dev->caps.reserved_uars      = dev_cap->reserved_uars;
         dev->caps.reserved_pds       = dev_cap->reserved_pds;
-        dev->caps.port_width_cap     = dev_cap->max_port_width;
         dev->caps.mtt_entry_sz       = MLX4_MTT_ENTRY_PER_SEG * dev_cap->mtt_entry_sz;
         dev->caps.page_size_cap      = ~(u32) (dev_cap->min_page_sz - 1);
         dev->caps.flags              = dev_cap->flags;