IB/mthca: Make all device methods truly reentrant

Documentation/infiniband/core_locking.txt says:

  All of the methods in struct ib_device exported by a low-level
  driver must be fully reentrant.  The low-level driver is required to
  perform all synchronization necessary to maintain consistency, even
  if multiple function calls using the same object are run
  simultaneously.

However, mthca's modify_qp, modify_srq and resize_cq methods are
currently not reentrant.  Add a mutex to the QP, SRQ and CQ structures
so that these calls can be properly serialized.

Signed-off-by: Roland Dreier <rolandd@cisco.com>

1 files changed, 23 insertions, 15 deletions

diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 322bc32a93f6..16c387d8170c 100644
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -536,6 +536,8 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
         u8 status;
         int err = -EINVAL;
 
+        mutex_lock(&qp->mutex);
+
         if (attr_mask & IB_QP_CUR_STATE) {
                 cur_state = attr->cur_qp_state;
         } else {
@@ -553,39 +555,41 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
                           "%d->%d with attr 0x%08x\n",
                           qp->transport, cur_state, new_state,
                           attr_mask);
-                return -EINVAL;
+                goto out;
         }
 
         if ((attr_mask & IB_QP_PKEY_INDEX) &&
              attr->pkey_index >= dev->limits.pkey_table_len) {
                 mthca_dbg(dev, "P_Key index (%u) too large. max is %d\n",
                           attr->pkey_index, dev->limits.pkey_table_len-1);
-                return -EINVAL;
+                goto out;
         }
 
         if ((attr_mask & IB_QP_PORT) &&
             (attr->port_num == 0 || attr->port_num > dev->limits.num_ports)) {
                 mthca_dbg(dev, "Port number (%u) is invalid\n", attr->port_num);
-                return -EINVAL;
+                goto out;
         }
 
         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
             attr->max_rd_atomic > dev->limits.max_qp_init_rdma) {
                 mthca_dbg(dev, "Max rdma_atomic as initiator %u too large (max is %d)\n",
                           attr->max_rd_atomic, dev->limits.max_qp_init_rdma);
-                return -EINVAL;
+                goto out;
         }
 
         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
             attr->max_dest_rd_atomic > 1 << dev->qp_table.rdb_shift) {
                 mthca_dbg(dev, "Max rdma_atomic as responder %u too large (max %d)\n",
                           attr->max_dest_rd_atomic, 1 << dev->qp_table.rdb_shift);
-                return -EINVAL;
+                goto out;
         }
 
         mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
-        if (IS_ERR(mailbox))
-                return PTR_ERR(mailbox);
+        if (IS_ERR(mailbox)) {
+                err = PTR_ERR(mailbox);
+                goto out;
+        }
         qp_param = mailbox->buf;
         qp_context = &qp_param->context;
         memset(qp_param, 0, sizeof *qp_param);
@@ -618,7 +622,7 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
                 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_2048) {
                         mthca_dbg(dev, "path MTU (%u) is invalid\n",
                                   attr->path_mtu);
-                        goto out;
+                        goto out_mailbox;
                 }
                 qp_context->mtu_msgmax = (attr->path_mtu << 5) | 31;
         }
@@ -672,7 +676,7 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
         if (attr_mask & IB_QP_AV) {
                 if (mthca_path_set(dev, &attr->ah_attr, &qp_context->pri_path,
                                    attr_mask & IB_QP_PORT ? attr->port_num : qp->port))
-                        goto out;
+                        goto out_mailbox;
 
                 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH);
         }
@@ -686,18 +690,18 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
                 if (attr->alt_pkey_index >= dev->limits.pkey_table_len) {
                         mthca_dbg(dev, "Alternate P_Key index (%u) too large. max is %d\n",
                                   attr->alt_pkey_index, dev->limits.pkey_table_len-1);
-                        goto out;
+                        goto out_mailbox;
                 }
 
                 if (attr->alt_port_num == 0 || attr->alt_port_num > dev->limits.num_ports) {
                         mthca_dbg(dev, "Alternate port number (%u) is invalid\n",
                                 attr->alt_port_num);
-                        goto out;
+                        goto out_mailbox;
                 }
 
                 if (mthca_path_set(dev, &attr->alt_ah_attr, &qp_context->alt_path,
                                    attr->alt_ah_attr.port_num))
-                        goto out;
+                        goto out_mailbox;
 
                 qp_context->alt_path.port_pkey |= cpu_to_be32(attr->alt_pkey_index |
                                                               attr->alt_port_num << 24);
@@ -793,12 +797,12 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
         err = mthca_MODIFY_QP(dev, cur_state, new_state, qp->qpn, 0,
                               mailbox, sqd_event, &status);
         if (err)
-                goto out;
+                goto out_mailbox;
         if (status) {
                 mthca_warn(dev, "modify QP %d->%d returned status %02x.\n",
                            cur_state, new_state, status);
                 err = -EINVAL;
-                goto out;
+                goto out_mailbox;
         }
 
         qp->state = new_state;
@@ -853,8 +857,11 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
                 }
         }
 
-out:
+out_mailbox:
         mthca_free_mailbox(dev, mailbox);
+
+out:
+        mutex_unlock(&qp->mutex);
         return err;
 }
 
@@ -1100,6 +1107,7 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
 
         qp->refcount = 1;
         init_waitqueue_head(&qp->wait);
+        mutex_init(&qp->mutex);
         qp->state        = IB_QPS_RESET;
         qp->atomic_rd_en = 0;
         qp->resp_depth   = 0;