1 files changed, 154 insertions, 5 deletions

diff --git a/drivers/infiniband/hw/ehca/ehca_qp.c b/drivers/infiniband/hw/ehca/ehca_qp.c
index 26c6a945459f..8d3c35fa051b 100644
--- a/drivers/infiniband/hw/ehca/ehca_qp.c
+++ b/drivers/infiniband/hw/ehca/ehca_qp.c
@@ -729,12 +729,31 @@ static struct ehca_qp *internal_create_qp(
         init_attr->cap.max_send_wr = parms.squeue.act_nr_wqes;
         my_qp->init_attr = *init_attr;
 
+        if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
+                shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
+                        &my_qp->ib_qp;
+                if (ehca_nr_ports < 0) {
+                        /* alloc array to cache subsequent modify qp parms
+                         * for autodetect mode
+                         */
+                        my_qp->mod_qp_parm =
+                                kzalloc(EHCA_MOD_QP_PARM_MAX *
+                                        sizeof(*my_qp->mod_qp_parm),
+                                        GFP_KERNEL);
+                        if (!my_qp->mod_qp_parm) {
+                                ehca_err(pd->device,
+                                         "Could not alloc mod_qp_parm");
+                                goto create_qp_exit4;
+                        }
+                }
+        }
+
         /* NOTE: define_apq0() not supported yet */
         if (qp_type == IB_QPT_GSI) {
                 h_ret = ehca_define_sqp(shca, my_qp, init_attr);
                 if (h_ret != H_SUCCESS) {
                         ret = ehca2ib_return_code(h_ret);
-                        goto create_qp_exit4;
+                        goto create_qp_exit5;
                 }
         }
 
@@ -743,7 +762,7 @@ static struct ehca_qp *internal_create_qp(
                 if (ret) {
                         ehca_err(pd->device,
                                  "Couldn't assign qp to send_cq ret=%i", ret);
-                        goto create_qp_exit4;
+                        goto create_qp_exit5;
                 }
         }
 
@@ -769,15 +788,18 @@ static struct ehca_qp *internal_create_qp(
                 if (ib_copy_to_udata(udata, &resp, sizeof resp)) {
                         ehca_err(pd->device, "Copy to udata failed");
                         ret = -EINVAL;
-                        goto create_qp_exit5;
+                        goto create_qp_exit6;
                 }
         }
 
         return my_qp;
 
-create_qp_exit5:
+create_qp_exit6:
         ehca_cq_unassign_qp(my_qp->send_cq, my_qp->real_qp_num);
 
+create_qp_exit5:
+        kfree(my_qp->mod_qp_parm);
+
 create_qp_exit4:
         if (HAS_RQ(my_qp))
                 ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
@@ -995,7 +1017,7 @@ static int internal_modify_qp(struct ib_qp *ibqp,
         unsigned long flags = 0;
 
         /* do query_qp to obtain current attr values */
-        mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+        mqpcb = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
         if (!mqpcb) {
                 ehca_err(ibqp->device, "Could not get zeroed page for mqpcb "
                          "ehca_qp=%p qp_num=%x ", my_qp, ibqp->qp_num);
@@ -1183,6 +1205,8 @@ static int internal_modify_qp(struct ib_qp *ibqp,
                 update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_P_KEY_IDX, 1);
         }
         if (attr_mask & IB_QP_PORT) {
+                struct ehca_sport *sport;
+                struct ehca_qp *aqp1;
                 if (attr->port_num < 1 || attr->port_num > shca->num_ports) {
                         ret = -EINVAL;
                         ehca_err(ibqp->device, "Invalid port=%x. "
@@ -1191,6 +1215,29 @@ static int internal_modify_qp(struct ib_qp *ibqp,
                                  shca->num_ports);
                         goto modify_qp_exit2;
                 }
+                sport = &shca->sport[attr->port_num - 1];
+                if (!sport->ibqp_sqp[IB_QPT_GSI]) {
+                        /* should not occur */
+                        ret = -EFAULT;
+                        ehca_err(ibqp->device, "AQP1 was not created for "
+                                 "port=%x", attr->port_num);
+                        goto modify_qp_exit2;
+                }
+                aqp1 = container_of(sport->ibqp_sqp[IB_QPT_GSI],
+                                    struct ehca_qp, ib_qp);
+                if (ibqp->qp_type != IB_QPT_GSI &&
+                    ibqp->qp_type != IB_QPT_SMI &&
+                    aqp1->mod_qp_parm) {
+                        /*
+                         * firmware will reject this modify_qp() because
+                         * port is not activated/initialized fully
+                         */
+                        ret = -EFAULT;
+                        ehca_warn(ibqp->device, "Couldn't modify qp port=%x: "
+                                  "either port is being activated (try again) "
+                                  "or cabling issue", attr->port_num);
+                        goto modify_qp_exit2;
+                }
                 mqpcb->prim_phys_port = attr->port_num;
                 update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_PHYS_PORT, 1);
         }
@@ -1470,6 +1517,8 @@ modify_qp_exit1:
 int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
                    struct ib_udata *udata)
 {
+        struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
+                                              ib_device);
         struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
         struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
                                              ib_pd);
@@ -1482,9 +1531,100 @@ int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
                 return -EINVAL;
         }
 
+        /* The if-block below caches qp_attr to be modified for GSI and SMI
+         * qps during the initialization by ib_mad. When the respective port
+         * is activated, ie we got an event PORT_ACTIVE, we'll replay the
+         * cached modify calls sequence, see ehca_recover_sqs() below.
+         * Why that is required:
+         * 1) If one port is connected, older code requires that port one
+         *    to be connected and module option nr_ports=1 to be given by
+         *    user, which is very inconvenient for end user.
+         * 2) Firmware accepts modify_qp() only if respective port has become
+         *    active. Older code had a wait loop of 30sec create_qp()/
+         *    define_aqp1(), which is not appropriate in practice. This
+         *    code now removes that wait loop, see define_aqp1(), and always
+         *    reports all ports to ib_mad resp. users. Only activated ports
+         *    will then usable for the users.
+         */
+        if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) {
+                int port = my_qp->init_attr.port_num;
+                struct ehca_sport *sport = &shca->sport[port - 1];
+                unsigned long flags;
+                spin_lock_irqsave(&sport->mod_sqp_lock, flags);
+                /* cache qp_attr only during init */
+                if (my_qp->mod_qp_parm) {
+                        struct ehca_mod_qp_parm *p;
+                        if (my_qp->mod_qp_parm_idx >= EHCA_MOD_QP_PARM_MAX) {
+                                ehca_err(&shca->ib_device,
+                                         "mod_qp_parm overflow state=%x port=%x"
+                                         " type=%x", attr->qp_state,
+                                         my_qp->init_attr.port_num,
+                                         ibqp->qp_type);
+                                spin_unlock_irqrestore(&sport->mod_sqp_lock,
+                                                       flags);
+                                return -EINVAL;
+                        }
+                        p = &my_qp->mod_qp_parm[my_qp->mod_qp_parm_idx];
+                        p->mask = attr_mask;
+                        p->attr = *attr;
+                        my_qp->mod_qp_parm_idx++;
+                        ehca_dbg(&shca->ib_device,
+                                 "Saved qp_attr for state=%x port=%x type=%x",
+                                 attr->qp_state, my_qp->init_attr.port_num,
+                                 ibqp->qp_type);
+                        spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+                        return 0;
+                }
+                spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+        }
+
         return internal_modify_qp(ibqp, attr, attr_mask, 0);
 }
 
+void ehca_recover_sqp(struct ib_qp *sqp)
+{
+        struct ehca_qp *my_sqp = container_of(sqp, struct ehca_qp, ib_qp);
+        int port = my_sqp->init_attr.port_num;
+        struct ib_qp_attr attr;
+        struct ehca_mod_qp_parm *qp_parm;
+        int i, qp_parm_idx, ret;
+        unsigned long flags, wr_cnt;
+
+        if (!my_sqp->mod_qp_parm)
+                return;
+        ehca_dbg(sqp->device, "SQP port=%x qp_num=%x", port, sqp->qp_num);
+
+        qp_parm = my_sqp->mod_qp_parm;
+        qp_parm_idx = my_sqp->mod_qp_parm_idx;
+        for (i = 0; i < qp_parm_idx; i++) {
+                attr = qp_parm[i].attr;
+                ret = internal_modify_qp(sqp, &attr, qp_parm[i].mask, 0);
+                if (ret) {
+                        ehca_err(sqp->device, "Could not modify SQP port=%x "
+                                 "qp_num=%x ret=%x", port, sqp->qp_num, ret);
+                        goto free_qp_parm;
+                }
+                ehca_dbg(sqp->device, "SQP port=%x qp_num=%x in state=%x",
+                         port, sqp->qp_num, attr.qp_state);
+        }
+
+        /* re-trigger posted recv wrs */
+        wr_cnt =  my_sqp->ipz_rqueue.current_q_offset /
+                my_sqp->ipz_rqueue.qe_size;
+        if (wr_cnt) {
+                spin_lock_irqsave(&my_sqp->spinlock_r, flags);
+                hipz_update_rqa(my_sqp, wr_cnt);
+                spin_unlock_irqrestore(&my_sqp->spinlock_r, flags);
+                ehca_dbg(sqp->device, "doorbell port=%x qp_num=%x wr_cnt=%lx",
+                         port, sqp->qp_num, wr_cnt);
+        }
+
+free_qp_parm:
+        kfree(qp_parm);
+        /* this prevents subsequent calls to modify_qp() to cache qp_attr */
+        my_sqp->mod_qp_parm = NULL;
+}
+
 int ehca_query_qp(struct ib_qp *qp,
                   struct ib_qp_attr *qp_attr,
                   int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
@@ -1772,6 +1912,7 @@ static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
         struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device);
         struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
                                              ib_pd);
+        struct ehca_sport *sport = &shca->sport[my_qp->init_attr.port_num - 1];
         u32 cur_pid = current->tgid;
         u32 qp_num = my_qp->real_qp_num;
         int ret;
@@ -1818,6 +1959,14 @@ static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
         port_num = my_qp->init_attr.port_num;
         qp_type  = my_qp->init_attr.qp_type;
 
+        if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
+                spin_lock_irqsave(&sport->mod_sqp_lock, flags);
+                kfree(my_qp->mod_qp_parm);
+                my_qp->mod_qp_parm = NULL;
+                shca->sport[port_num - 1].ibqp_sqp[qp_type] = NULL;
+                spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+        }
+
         /* no support for IB_QPT_SMI yet */
         if (qp_type == IB_QPT_GSI) {
                 struct ib_event event;