1 files changed, 81 insertions, 175 deletions

diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 5687c3014522..8600b6c3e0c2 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -2,6 +2,8 @@
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
  * Copyright (c) 2005 Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -37,7 +39,7 @@
 #include <linux/init.h>
 #include <linux/hardirq.h>
 
-#include <ib_pack.h>
+#include <rdma/ib_pack.h>
 
 #include "mthca_dev.h"
 #include "mthca_cmd.h"
@@ -55,21 +57,21 @@ enum {
  * Must be packed because start is 64 bits but only aligned to 32 bits.
  */
 struct mthca_cq_context {
-        u32 flags;
-        u64 start;
-        u32 logsize_usrpage;
-        u32 error_eqn;          /* Tavor only */
-        u32 comp_eqn;
-        u32 pd;
-        u32 lkey;
-        u32 last_notified_index;
-        u32 solicit_producer_index;
-        u32 consumer_index;
-        u32 producer_index;
-        u32 cqn;
-        u32 ci_db;              /* Arbel only */
-        u32 state_db;           /* Arbel only */
-        u32 reserved;
+        __be32 flags;
+        __be64 start;
+        __be32 logsize_usrpage;
+        __be32 error_eqn;       /* Tavor only */
+        __be32 comp_eqn;
+        __be32 pd;
+        __be32 lkey;
+        __be32 last_notified_index;
+        __be32 solicit_producer_index;
+        __be32 consumer_index;
+        __be32 producer_index;
+        __be32 cqn;
+        __be32 ci_db;           /* Arbel only */
+        __be32 state_db;        /* Arbel only */
+        u32    reserved;
 } __attribute__((packed));
 
 #define MTHCA_CQ_STATUS_OK          ( 0 << 28)
@@ -108,31 +110,31 @@ enum {
 };
 
 struct mthca_cqe {
-        u32 my_qpn;
-        u32 my_ee;
-        u32 rqpn;
-        u16 sl_g_mlpath;
-        u16 rlid;
-        u32 imm_etype_pkey_eec;
-        u32 byte_cnt;
-        u32 wqe;
-        u8  opcode;
-        u8  is_send;
-        u8  reserved;
-        u8  owner;
+        __be32 my_qpn;
+        __be32 my_ee;
+        __be32 rqpn;
+        __be16 sl_g_mlpath;
+        __be16 rlid;
+        __be32 imm_etype_pkey_eec;
+        __be32 byte_cnt;
+        __be32 wqe;
+        u8     opcode;
+        u8     is_send;
+        u8     reserved;
+        u8     owner;
 };
 
 struct mthca_err_cqe {
-        u32 my_qpn;
-        u32 reserved1[3];
-        u8  syndrome;
-        u8  reserved2;
-        u16 db_cnt;
-        u32 reserved3;
-        u32 wqe;
-        u8  opcode;
-        u8  reserved4[2];
-        u8  owner;
+        __be32 my_qpn;
+        u32    reserved1[3];
+        u8     syndrome;
+        u8     reserved2;
+        __be16 db_cnt;
+        u32    reserved3;
+        __be32 wqe;
+        u8     opcode;
+        u8     reserved4[2];
+        u8     owner;
 };
 
 #define MTHCA_CQ_ENTRY_OWNER_SW      (0 << 7)
@@ -191,7 +193,7 @@ static void dump_cqe(struct mthca_dev *dev, void *cqe_ptr)
 static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,
                                      int incr)
 {
-        u32 doorbell[2];
+        __be32 doorbell[2];
 
         if (mthca_is_memfree(dev)) {
                 *cq->set_ci_db = cpu_to_be32(cq->cons_index);
@@ -222,7 +224,8 @@ void mthca_cq_event(struct mthca_dev *dev, u32 cqn)
         cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
 
-void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn)
+void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn,
+                    struct mthca_srq *srq)
 {
         struct mthca_cq *cq;
         struct mthca_cqe *cqe;
@@ -263,8 +266,11 @@ void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn)
          */
         while (prod_index > cq->cons_index) {
                 cqe = get_cqe(cq, (prod_index - 1) & cq->ibcq.cqe);
-                if (cqe->my_qpn == cpu_to_be32(qpn))
+                if (cqe->my_qpn == cpu_to_be32(qpn)) {
+                        if (srq)
+                                mthca_free_srq_wqe(srq, be32_to_cpu(cqe->wqe));
                         ++nfreed;
+                }
                 else if (nfreed)
                         memcpy(get_cqe(cq, (prod_index - 1 + nfreed) &
                                        cq->ibcq.cqe),
@@ -291,7 +297,7 @@ static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
 {
         int err;
         int dbd;
-        u32 new_wqe;
+        __be32 new_wqe;
 
         if (cqe->syndrome == SYNDROME_LOCAL_QP_OP_ERR) {
                 mthca_dbg(dev, "local QP operation err "
@@ -365,6 +371,13 @@ static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
                 break;
         }
 
+        /*
+         * Mem-free HCAs always generate one CQE per WQE, even in the
+         * error case, so we don't have to check the doorbell count, etc.
+         */
+        if (mthca_is_memfree(dev))
+                return 0;
+
         err = mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe);
         if (err)
                 return err;
@@ -373,12 +386,8 @@ static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
          * If we're at the end of the WQE chain, or we've used up our
          * doorbell count, free the CQE.  Otherwise just update it for
          * the next poll operation.
-         *
-         * This does not apply to mem-free HCAs: they don't use the
-         * doorbell count field, and so we should always free the CQE.
          */
-        if (mthca_is_memfree(dev) ||
-            !(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd))
+        if (!(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd))
                 return 0;
 
         cqe->db_cnt   = cpu_to_be16(be16_to_cpu(cqe->db_cnt) - dbd);
@@ -450,23 +459,27 @@ static inline int mthca_poll_one(struct mthca_dev *dev,
                              >> wq->wqe_shift);
                 entry->wr_id = (*cur_qp)->wrid[wqe_index +
                                                (*cur_qp)->rq.max];
+        } else if ((*cur_qp)->ibqp.srq) {
+                struct mthca_srq *srq = to_msrq((*cur_qp)->ibqp.srq);
+                u32 wqe = be32_to_cpu(cqe->wqe);
+                wq = NULL;
+                wqe_index = wqe >> srq->wqe_shift;
+                entry->wr_id = srq->wrid[wqe_index];
+                mthca_free_srq_wqe(srq, wqe);
         } else {
                 wq = &(*cur_qp)->rq;
                 wqe_index = be32_to_cpu(cqe->wqe) >> wq->wqe_shift;
                 entry->wr_id = (*cur_qp)->wrid[wqe_index];
         }
 
-        if (wq->last_comp < wqe_index)
-                wq->tail += wqe_index - wq->last_comp;
-        else
-                wq->tail += wqe_index + wq->max - wq->last_comp;
-
-        wq->last_comp = wqe_index;
+        if (wq) {
+                if (wq->last_comp < wqe_index)
+                        wq->tail += wqe_index - wq->last_comp;
+                else
+                        wq->tail += wqe_index + wq->max - wq->last_comp;
 
-        if (0)
-                mthca_dbg(dev, "%s completion for QP %06x, index %d (nr %d)\n",
-                          is_send ? "Send" : "Receive",
-                          (*cur_qp)->qpn, wqe_index, wq->max);
+                wq->last_comp = wqe_index;
+        }
 
         if (is_error) {
                 err = handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send,
@@ -584,13 +597,13 @@ int mthca_poll_cq(struct ib_cq *ibcq, int num_entries,
 
 int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify)
 {
-        u32 doorbell[2];
+        __be32 doorbell[2];
 
         doorbell[0] = cpu_to_be32((notify == IB_CQ_SOLICITED ?
                                    MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL :
                                    MTHCA_TAVOR_CQ_DB_REQ_NOT)      |
                                   to_mcq(cq)->cqn);
-        doorbell[1] = 0xffffffff;
+        doorbell[1] = (__force __be32) 0xffffffff;
 
         mthca_write64(doorbell,
                       to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL,
@@ -602,9 +615,9 @@ int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify)
 int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
 {
         struct mthca_cq *cq = to_mcq(ibcq);
-        u32 doorbell[2];
+        __be32 doorbell[2];
         u32 sn;
-        u32 ci;
+        __be32 ci;
 
         sn = cq->arm_sn & 3;
         ci = cpu_to_be32(cq->cons_index);
@@ -637,113 +650,8 @@ int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
 
 static void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq *cq)
 {
-        int i;
-        int size;
-
-        if (cq->is_direct)
-                dma_free_coherent(&dev->pdev->dev,
-                                  (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE,
-                                  cq->queue.direct.buf,
-                                  pci_unmap_addr(&cq->queue.direct,
-                                                 mapping));
-        else {
-                size = (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE;
-                for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i)
-                        if (cq->queue.page_list[i].buf)
-                                dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
-                                                  cq->queue.page_list[i].buf,
-                                                  pci_unmap_addr(&cq->queue.page_list[i],
-                                                                 mapping));
-
-                kfree(cq->queue.page_list);
-        }
-}
-
-static int mthca_alloc_cq_buf(struct mthca_dev *dev, int size,
-                              struct mthca_cq *cq)
-{
-        int err = -ENOMEM;
-        int npages, shift;
-        u64 *dma_list = NULL;
-        dma_addr_t t;
-        int i;
-
-        if (size <= MTHCA_MAX_DIRECT_CQ_SIZE) {
-                cq->is_direct = 1;
-                npages        = 1;
-                shift         = get_order(size) + PAGE_SHIFT;
-
-                cq->queue.direct.buf = dma_alloc_coherent(&dev->pdev->dev,
-                                                          size, &t, GFP_KERNEL);
-                if (!cq->queue.direct.buf)
-                        return -ENOMEM;
-
-                pci_unmap_addr_set(&cq->queue.direct, mapping, t);
-
-                memset(cq->queue.direct.buf, 0, size);
-
-                while (t & ((1 << shift) - 1)) {
-                        --shift;
-                        npages *= 2;
-                }
-
-                dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
-                if (!dma_list)
-                        goto err_free;
-
-                for (i = 0; i < npages; ++i)
-                        dma_list[i] = t + i * (1 << shift);
-        } else {
-                cq->is_direct = 0;
-                npages        = (size + PAGE_SIZE - 1) / PAGE_SIZE;
-                shift         = PAGE_SHIFT;
-
-                dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
-                if (!dma_list)
-                        return -ENOMEM;
-
-                cq->queue.page_list = kmalloc(npages * sizeof *cq->queue.page_list,
-                                              GFP_KERNEL);
-                if (!cq->queue.page_list)
-                        goto err_out;
-
-                for (i = 0; i < npages; ++i)
-                        cq->queue.page_list[i].buf = NULL;
-
-                for (i = 0; i < npages; ++i) {
-                        cq->queue.page_list[i].buf =
-                                dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
-                                                   &t, GFP_KERNEL);
-                        if (!cq->queue.page_list[i].buf)
-                                goto err_free;
-
-                        dma_list[i] = t;
-                        pci_unmap_addr_set(&cq->queue.page_list[i], mapping, t);
-
-                        memset(cq->queue.page_list[i].buf, 0, PAGE_SIZE);
-                }
-        }
-
-        err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,
-                                  dma_list, shift, npages,
-                                  0, size,
-                                  MTHCA_MPT_FLAG_LOCAL_WRITE |
-                                  MTHCA_MPT_FLAG_LOCAL_READ,
-                                  &cq->mr);
-        if (err)
-                goto err_free;
-
-        kfree(dma_list);
-
-        return 0;
-
-err_free:
-        mthca_free_cq_buf(dev, cq);
-
-err_out:
-        kfree(dma_list);
-
-        return err;
+        mthca_buf_free(dev, (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE,
+                       &cq->queue, cq->is_direct, &cq->mr);
 }
 
 int mthca_init_cq(struct mthca_dev *dev, int nent,
@@ -795,7 +703,9 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
         cq_context = mailbox->buf;
 
         if (cq->is_kernel) {
-                err = mthca_alloc_cq_buf(dev, size, cq);
+                err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_CQ_SIZE,
+                                      &cq->queue, &cq->is_direct,
+                                      &dev->driver_pd, 1, &cq->mr);
                 if (err)
                         goto err_out_mailbox;
 
@@ -811,7 +721,6 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
         cq_context->flags           = cpu_to_be32(MTHCA_CQ_STATUS_OK      |
                                                   MTHCA_CQ_STATE_DISARMED |
                                                   MTHCA_CQ_FLAG_TR);
-        cq_context->start           = cpu_to_be64(0);
         cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24);
         if (ctx)
                 cq_context->logsize_usrpage |= cpu_to_be32(ctx->uar.index);
@@ -857,10 +766,8 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
         return 0;
 
 err_out_free_mr:
-        if (cq->is_kernel) {
-                mthca_free_mr(dev, &cq->mr);
+        if (cq->is_kernel)
                 mthca_free_cq_buf(dev, cq);
-        }
 
 err_out_mailbox:
         mthca_free_mailbox(dev, mailbox);
@@ -904,7 +811,7 @@ void mthca_free_cq(struct mthca_dev *dev,
                 mthca_warn(dev, "HW2SW_CQ returned status 0x%02x\n", status);
 
         if (0) {
-                u32 *ctx = mailbox->buf;
+                __be32 *ctx = mailbox->buf;
                 int j;
 
                 printk(KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n",
@@ -928,7 +835,6 @@ void mthca_free_cq(struct mthca_dev *dev,
         wait_event(cq->wait, !atomic_read(&cq->refcount));
 
         if (cq->is_kernel) {
-                mthca_free_mr(dev, &cq->mr);
                 mthca_free_cq_buf(dev, cq);
                 if (mthca_is_memfree(dev)) {
                         mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM,    cq->arm_db_index);