1 files changed, 199 insertions, 185 deletions

diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 43d08b5e9085..f05500bcdcf1 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -284,7 +284,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 {
         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
         struct cmdq_create_qp1 req;
-        struct creq_create_qp1_resp *resp;
+        struct creq_create_qp1_resp resp;
         struct bnxt_qplib_pbl *pbl;
         struct bnxt_qplib_q *sq = &qp->sq;
         struct bnxt_qplib_q *rq = &qp->rq;
@@ -394,31 +394,12 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
         req.pd_id = cpu_to_le32(qp->pd->id);
 
-        resp = (struct creq_create_qp1_resp *)
-                        bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-                                                     NULL, 0);
-        if (!resp) {
-                dev_err(&res->pdev->dev, "QPLIB: FP: CREATE_QP1 send failed");
-                rc = -EINVAL;
-                goto fail;
-        }
-        if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
-                /* Cmd timed out */
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 timed out");
-                rc = -ETIMEDOUT;
-                goto fail;
-        }
-        if (resp->status ||
-            le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 failed ");
-                dev_err(&rcfw->pdev->dev,
-                        "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
-                        resp->status, le16_to_cpu(req.cookie),
-                        le16_to_cpu(resp->cookie));
-                rc = -EINVAL;
+        rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+                                          (void *)&resp, NULL, 0);
+        if (rc)
                 goto fail;
-        }
-        qp->id = le32_to_cpu(resp->xid);
+
+        qp->id = le32_to_cpu(resp.xid);
         qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
         sq->flush_in_progress = false;
         rq->flush_in_progress = false;
@@ -442,7 +423,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
         struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
         struct cmdq_create_qp req;
-        struct creq_create_qp_resp *resp;
+        struct creq_create_qp_resp resp;
         struct bnxt_qplib_pbl *pbl;
         struct sq_psn_search **psn_search_ptr;
         unsigned long int psn_search, poff = 0;
@@ -627,31 +608,12 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
         }
         req.pd_id = cpu_to_le32(qp->pd->id);
 
-        resp = (struct creq_create_qp_resp *)
-                        bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-                                                     NULL, 0);
-        if (!resp) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP send failed");
-                rc = -EINVAL;
-                goto fail;
-        }
-        if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
-                /* Cmd timed out */
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP timed out");
-                rc = -ETIMEDOUT;
-                goto fail;
-        }
-        if (resp->status ||
-            le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP failed ");
-                dev_err(&rcfw->pdev->dev,
-                        "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
-                        resp->status, le16_to_cpu(req.cookie),
-                        le16_to_cpu(resp->cookie));
-                rc = -EINVAL;
+        rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+                                          (void *)&resp, NULL, 0);
+        if (rc)
                 goto fail;
-        }
-        qp->id = le32_to_cpu(resp->xid);
+
+        qp->id = le32_to_cpu(resp.xid);
         qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
         sq->flush_in_progress = false;
         rq->flush_in_progress = false;
@@ -769,10 +731,11 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 {
         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
         struct cmdq_modify_qp req;
-        struct creq_modify_qp_resp *resp;
+        struct creq_modify_qp_resp resp;
         u16 cmd_flags = 0, pkey;
         u32 temp32[4];
         u32 bmask;
+        int rc;
 
         RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
 
@@ -862,27 +825,10 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
         req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
 
-        resp = (struct creq_modify_qp_resp *)
-                        bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-                                                     NULL, 0);
-        if (!resp) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP send failed");
-                return -EINVAL;
-        }
-        if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
-                /* Cmd timed out */
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP timed out");
-                return -ETIMEDOUT;
-        }
-        if (resp->status ||
-            le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP failed ");
-                dev_err(&rcfw->pdev->dev,
-                        "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
-                        resp->status, le16_to_cpu(req.cookie),
-                        le16_to_cpu(resp->cookie));
-                return -EINVAL;
-        }
+        rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+                                          (void *)&resp, NULL, 0);
+        if (rc)
+                return rc;
         qp->cur_qp_state = qp->state;
         return 0;
 }
@@ -891,37 +837,26 @@ int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 {
         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
         struct cmdq_query_qp req;
-        struct creq_query_qp_resp *resp;
+        struct creq_query_qp_resp resp;
+        struct bnxt_qplib_rcfw_sbuf *sbuf;
         struct creq_query_qp_resp_sb *sb;
         u16 cmd_flags = 0;
         u32 temp32[4];
-        int i;
+        int i, rc = 0;
 
         RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
 
+        sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
+        if (!sbuf)
+                return -ENOMEM;
+        sb = sbuf->sb;
+
         req.qp_cid = cpu_to_le32(qp->id);
         req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
-        resp = (struct creq_query_qp_resp *)
-                        bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-                                                     (void **)&sb, 0);
-        if (!resp) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP send failed");
-                return -EINVAL;
-        }
-        if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
-                /* Cmd timed out */
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP timed out");
-                return -ETIMEDOUT;
-        }
-        if (resp->status ||
-            le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP failed ");
-                dev_err(&rcfw->pdev->dev,
-                        "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
-                        resp->status, le16_to_cpu(req.cookie),
-                        le16_to_cpu(resp->cookie));
-                return -EINVAL;
-        }
+        rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
+                                          (void *)sbuf, 0);
+        if (rc)
+                goto bail;
         /* Extract the context from the side buffer */
         qp->state = sb->en_sqd_async_notify_state &
                         CREQ_QUERY_QP_RESP_SB_STATE_MASK;
@@ -976,7 +911,9 @@ int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
         qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
         memcpy(qp->smac, sb->src_mac, 6);
         qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
-        return 0;
+bail:
+        bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
+        return rc;
 }
 
 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
@@ -1021,34 +958,18 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 {
         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
         struct cmdq_destroy_qp req;
-        struct creq_destroy_qp_resp *resp;
+        struct creq_destroy_qp_resp resp;
         unsigned long flags;
         u16 cmd_flags = 0;
+        int rc;
 
         RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
 
         req.qp_cid = cpu_to_le32(qp->id);
-        resp = (struct creq_destroy_qp_resp *)
-                        bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-                                                     NULL, 0);
-        if (!resp) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP send failed");
-                return -EINVAL;
-        }
-        if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
-                /* Cmd timed out */
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP timed out");
-                return -ETIMEDOUT;
-        }
-        if (resp->status ||
-            le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP failed ");
-                dev_err(&rcfw->pdev->dev,
-                        "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
-                        resp->status, le16_to_cpu(req.cookie),
-                        le16_to_cpu(resp->cookie));
-                return -EINVAL;
-        }
+        rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+                                          (void *)&resp, NULL, 0);
+        if (rc)
+                return rc;
 
         /* Must walk the associated CQs to nullified the QP ptr */
         spin_lock_irqsave(&qp->scq->hwq.lock, flags);
@@ -1162,8 +1083,12 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
                 rc = -EINVAL;
                 goto done;
         }
-        if (HWQ_CMP((sq->hwq.prod + 1), &sq->hwq) ==
-            HWQ_CMP(sq->hwq.cons, &sq->hwq)) {
+
+        if (bnxt_qplib_queue_full(sq)) {
+                dev_err(&sq->hwq.pdev->dev,
+                        "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
+                        sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
+                        sq->q_full_delta);
                 rc = -ENOMEM;
                 goto done;
         }
@@ -1373,6 +1298,9 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
         }
 
         sq->hwq.prod++;
+
+        qp->wqe_cnt++;
+
 done:
         return rc;
 }
@@ -1411,8 +1339,7 @@ int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
                 rc = -EINVAL;
                 goto done;
         }
-        if (HWQ_CMP((rq->hwq.prod + 1), &rq->hwq) ==
-            HWQ_CMP(rq->hwq.cons, &rq->hwq)) {
+        if (bnxt_qplib_queue_full(rq)) {
                 dev_err(&rq->hwq.pdev->dev,
                         "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
                 rc = -EINVAL;
@@ -1483,7 +1410,7 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
 {
         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
         struct cmdq_create_cq req;
-        struct creq_create_cq_resp *resp;
+        struct creq_create_cq_resp resp;
         struct bnxt_qplib_pbl *pbl;
         u16 cmd_flags = 0;
         int rc;
@@ -1525,30 +1452,12 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
                         (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
                          CMDQ_CREATE_CQ_CNQ_ID_SFT);
 
-        resp = (struct creq_create_cq_resp *)
-                        bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-                                                     NULL, 0);
-        if (!resp) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ send failed");
-                return -EINVAL;
-        }
-        if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
-                /* Cmd timed out */
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ timed out");
-                rc = -ETIMEDOUT;
-                goto fail;
-        }
-        if (resp->status ||
-            le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ failed ");
-                dev_err(&rcfw->pdev->dev,
-                        "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
-                        resp->status, le16_to_cpu(req.cookie),
-                        le16_to_cpu(resp->cookie));
-                rc = -EINVAL;
+        rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+                                          (void *)&resp, NULL, 0);
+        if (rc)
                 goto fail;
-        }
-        cq->id = le32_to_cpu(resp->xid);
+
+        cq->id = le32_to_cpu(resp.xid);
         cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
         cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
         init_waitqueue_head(&cq->waitq);
@@ -1566,33 +1475,17 @@ int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
 {
         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
         struct cmdq_destroy_cq req;
-        struct creq_destroy_cq_resp *resp;
+        struct creq_destroy_cq_resp resp;
         u16 cmd_flags = 0;
+        int rc;
 
         RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
 
         req.cq_cid = cpu_to_le32(cq->id);
-        resp = (struct creq_destroy_cq_resp *)
-                        bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-                                                     NULL, 0);
-        if (!resp) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ send failed");
-                return -EINVAL;
-        }
-        if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
-                /* Cmd timed out */
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ timed out");
-                return -ETIMEDOUT;
-        }
-        if (resp->status ||
-            le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
-                dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ failed ");
-                dev_err(&rcfw->pdev->dev,
-                        "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
-                        resp->status, le16_to_cpu(req.cookie),
-                        le16_to_cpu(resp->cookie));
-                return -EINVAL;
-        }
+        rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+                                          (void *)&resp, NULL, 0);
+        if (rc)
+                return rc;
         bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
         return 0;
 }
@@ -1664,14 +1557,113 @@ static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
         return rc;
 }
 
+/* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
+ *       CQE is track from sw_cq_cons to max_element but valid only if VALID=1
+ */
+static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
+                     u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
+{
+        struct bnxt_qplib_q *sq = &qp->sq;
+        struct bnxt_qplib_swq *swq;
+        u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
+        struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr;
+        struct cq_req *peek_req_hwcqe;
+        struct bnxt_qplib_qp *peek_qp;
+        struct bnxt_qplib_q *peek_sq;
+        int i, rc = 0;
+
+        /* Normal mode */
+        /* Check for the psn_search marking before completing */
+        swq = &sq->swq[sw_sq_cons];
+        if (swq->psn_search &&
+            le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
+                /* Unmark */
+                swq->psn_search->flags_next_psn = cpu_to_le32
+                        (le32_to_cpu(swq->psn_search->flags_next_psn)
+                                     & ~0x80000000);
+                dev_dbg(&cq->hwq.pdev->dev,
+                        "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
+                        cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
+                sq->condition = true;
+                sq->send_phantom = true;
+
+                /* TODO: Only ARM if the previous SQE is ARMALL */
+                bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL);
+
+                rc = -EAGAIN;
+                goto out;
+        }
+        if (sq->condition) {
+                /* Peek at the completions */
+                peek_raw_cq_cons = cq->hwq.cons;
+                peek_sw_cq_cons = cq_cons;
+                i = cq->hwq.max_elements;
+                while (i--) {
+                        peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
+                        peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
+                        peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
+                                                     [CQE_IDX(peek_sw_cq_cons)];
+                        /* If the next hwcqe is VALID */
+                        if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
+                                          cq->hwq.max_elements)) {
+                                /* If the next hwcqe is a REQ */
+                                if ((peek_hwcqe->cqe_type_toggle &
+                                    CQ_BASE_CQE_TYPE_MASK) ==
+                                    CQ_BASE_CQE_TYPE_REQ) {
+                                        peek_req_hwcqe = (struct cq_req *)
+                                                         peek_hwcqe;
+                                        peek_qp = (struct bnxt_qplib_qp *)
+                                                ((unsigned long)
+                                                 le64_to_cpu
+                                                 (peek_req_hwcqe->qp_handle));
+                                        peek_sq = &peek_qp->sq;
+                                        peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
+                                                peek_req_hwcqe->sq_cons_idx) - 1
+                                                , &sq->hwq);
+                                        /* If the hwcqe's sq's wr_id matches */
+                                        if (peek_sq == sq &&
+                                            sq->swq[peek_sq_cons_idx].wr_id ==
+                                            BNXT_QPLIB_FENCE_WRID) {
+                                                /*
+                                                 *  Unbreak only if the phantom
+                                                 *  comes back
+                                                 */
+                                                dev_dbg(&cq->hwq.pdev->dev,
+                                                        "FP:Got Phantom CQE");
+                                                sq->condition = false;
+                                                sq->single = true;
+                                                rc = 0;
+                                                goto out;
+                                        }
+                                }
+                                /* Valid but not the phantom, so keep looping */
+                        } else {
+                                /* Not valid yet, just exit and wait */
+                                rc = -EINVAL;
+                                goto out;
+                        }
+                        peek_sw_cq_cons++;
+                        peek_raw_cq_cons++;
+                }
+                dev_err(&cq->hwq.pdev->dev,
+                        "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
+                        cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
+                rc = -EINVAL;
+        }
+out:
+        return rc;
+}
+
 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
                                      struct cq_req *hwcqe,
-                                     struct bnxt_qplib_cqe **pcqe, int *budget)
+                                     struct bnxt_qplib_cqe **pcqe, int *budget,
+                                     u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
 {
         struct bnxt_qplib_qp *qp;
         struct bnxt_qplib_q *sq;
         struct bnxt_qplib_cqe *cqe;
-        u32 sw_cons, cqe_cons;
+        u32 sw_sq_cons, cqe_sq_cons;
+        struct bnxt_qplib_swq *swq;
         int rc = 0;
 
         qp = (struct bnxt_qplib_qp *)((unsigned long)
@@ -1683,13 +1675,13 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
         }
         sq = &qp->sq;
 
-        cqe_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
-        if (cqe_cons > sq->hwq.max_elements) {
+        cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
+        if (cqe_sq_cons > sq->hwq.max_elements) {
                 dev_err(&cq->hwq.pdev->dev,
                         "QPLIB: FP: CQ Process req reported ");
                 dev_err(&cq->hwq.pdev->dev,
                         "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
-                        cqe_cons, sq->hwq.max_elements);
+                        cqe_sq_cons, sq->hwq.max_elements);
                 return -EINVAL;
         }
         /* If we were in the middle of flushing the SQ, continue */
@@ -1698,53 +1690,74 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
 
         /* Require to walk the sq's swq to fabricate CQEs for all previously
          * signaled SWQEs due to CQE aggregation from the current sq cons
-         * to the cqe_cons
+         * to the cqe_sq_cons
          */
         cqe = *pcqe;
         while (*budget) {
-                sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
-                if (sw_cons == cqe_cons)
+                sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
+                if (sw_sq_cons == cqe_sq_cons)
+                        /* Done */
                         break;
+
+                swq = &sq->swq[sw_sq_cons];
                 memset(cqe, 0, sizeof(*cqe));
                 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
                 cqe->qp_handle = (u64)(unsigned long)qp;
                 cqe->src_qp = qp->id;
-                cqe->wr_id = sq->swq[sw_cons].wr_id;
-                cqe->type = sq->swq[sw_cons].type;
+                cqe->wr_id = swq->wr_id;
+                if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
+                        goto skip;
+                cqe->type = swq->type;
 
                 /* For the last CQE, check for status.  For errors, regardless
                  * of the request being signaled or not, it must complete with
                  * the hwcqe error status
                  */
-                if (HWQ_CMP((sw_cons + 1), &sq->hwq) == cqe_cons &&
+                if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
                     hwcqe->status != CQ_REQ_STATUS_OK) {
                         cqe->status = hwcqe->status;
                         dev_err(&cq->hwq.pdev->dev,
                                 "QPLIB: FP: CQ Processed Req ");
                         dev_err(&cq->hwq.pdev->dev,
                                 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
-                                sw_cons, cqe->wr_id, cqe->status);
+                                sw_sq_cons, cqe->wr_id, cqe->status);
                         cqe++;
                         (*budget)--;
                         sq->flush_in_progress = true;
                         /* Must block new posting of SQ and RQ */
                         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
+                        sq->condition = false;
+                        sq->single = false;
                 } else {
-                        if (sq->swq[sw_cons].flags &
-                            SQ_SEND_FLAGS_SIGNAL_COMP) {
+                        if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
+                                /* Before we complete, do WA 9060 */
+                                if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
+                                              cqe_sq_cons)) {
+                                        *lib_qp = qp;
+                                        goto out;
+                                }
                                 cqe->status = CQ_REQ_STATUS_OK;
                                 cqe++;
                                 (*budget)--;
                         }
                 }
+skip:
                 sq->hwq.cons++;
+                if (sq->single)
+                        break;
         }
+out:
         *pcqe = cqe;
-        if (!*budget && HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_cons) {
+        if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
                 /* Out of budget */
                 rc = -EAGAIN;
                 goto done;
         }
+        /*
+         * Back to normal completion mode only after it has completed all of
+         * the WC for this CQE
+         */
+        sq->single = false;
         if (!sq->flush_in_progress)
                 goto done;
 flush:
@@ -2074,7 +2087,7 @@ static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
 }
 
 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
-                       int num_cqes)
+                       int num_cqes, struct bnxt_qplib_qp **lib_qp)
 {
         struct cq_base *hw_cqe, **hw_cqe_ptr;
         unsigned long flags;
@@ -2099,7 +2112,8 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
                 case CQ_BASE_CQE_TYPE_REQ:
                         rc = bnxt_qplib_cq_process_req(cq,
                                                        (struct cq_req *)hw_cqe,
-                                                       &cqe, &budget);
+                                                       &cqe, &budget,
+                                                       sw_cons, lib_qp);
                         break;
                 case CQ_BASE_CQE_TYPE_RES_RC:
                         rc = bnxt_qplib_cq_process_res_rc(cq,