1 files changed, 108 insertions, 200 deletions

diff --git a/drivers/infiniband/hw/ipath/ipath_ruc.c b/drivers/infiniband/hw/ipath/ipath_ruc.c
index c69c25239443..4b6b7ee8e5c1 100644
--- a/drivers/infiniband/hw/ipath/ipath_ruc.c
+++ b/drivers/infiniband/hw/ipath/ipath_ruc.c
@@ -31,6 +31,8 @@
  * SOFTWARE.
  */
 
+#include <linux/spinlock.h>
+
 #include "ipath_verbs.h"
 #include "ipath_kernel.h"
 
@@ -106,27 +108,30 @@ void ipath_insert_rnr_queue(struct ipath_qp *qp)
         spin_unlock_irqrestore(&dev->pending_lock, flags);
 }
 
-static int init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe)
+/**
+ * ipath_init_sge - Validate a RWQE and fill in the SGE state
+ * @qp: the QP
+ *
+ * Return 1 if OK.
+ */
+int ipath_init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe,
+                   u32 *lengthp, struct ipath_sge_state *ss)
 {
-        int user = to_ipd(qp->ibqp.pd)->user;
         int i, j, ret;
         struct ib_wc wc;
 
-        qp->r_len = 0;
+        *lengthp = 0;
         for (i = j = 0; i < wqe->num_sge; i++) {
                 if (wqe->sg_list[i].length == 0)
                         continue;
                 /* Check LKEY */
-                if ((user && wqe->sg_list[i].lkey == 0) ||
-                    !ipath_lkey_ok(qp, &qp->r_sg_list[j], &wqe->sg_list[i],
-                                   IB_ACCESS_LOCAL_WRITE))
+                if (!ipath_lkey_ok(qp, j ? &ss->sg_list[j - 1] : &ss->sge,
+                                   &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE))
                         goto bad_lkey;
-                qp->r_len += wqe->sg_list[i].length;
+                *lengthp += wqe->sg_list[i].length;
                 j++;
         }
-        qp->r_sge.sge = qp->r_sg_list[0];
-        qp->r_sge.sg_list = qp->r_sg_list + 1;
-        qp->r_sge.num_sge = j;
+        ss->num_sge = j;
         ret = 1;
         goto bail;
 
@@ -172,6 +177,8 @@ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
         u32 tail;
         int ret;
 
+        qp->r_sge.sg_list = qp->r_sg_list;
+
         if (qp->ibqp.srq) {
                 srq = to_isrq(qp->ibqp.srq);
                 handler = srq->ibsrq.event_handler;
@@ -199,7 +206,8 @@ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
                 wqe = get_rwqe_ptr(rq, tail);
                 if (++tail >= rq->size)
                         tail = 0;
-        } while (!wr_id_only && !init_sge(qp, wqe));
+        } while (!wr_id_only && !ipath_init_sge(qp, wqe, &qp->r_len,
+                                                &qp->r_sge));
         qp->r_wr_id = wqe->wr_id;
         wq->tail = tail;
 
@@ -239,9 +247,9 @@ bail:
 
 /**
  * ipath_ruc_loopback - handle UC and RC lookback requests
- * @sqp: the loopback QP
+ * @sqp: the sending QP
  *
- * This is called from ipath_do_uc_send() or ipath_do_rc_send() to
+ * This is called from ipath_do_send() to
  * forward a WQE addressed to the same HCA.
  * Note that although we are single threaded due to the tasklet, we still
  * have to protect against post_send().  We don't have to worry about
@@ -450,40 +458,18 @@ again:
         wc.byte_len = wqe->length;
         wc.qp = &qp->ibqp;
         wc.src_qp = qp->remote_qpn;
-        /* XXX do we know which pkey matched? Only needed for GSI. */
         wc.pkey_index = 0;
         wc.slid = qp->remote_ah_attr.dlid;
         wc.sl = qp->remote_ah_attr.sl;
         wc.dlid_path_bits = 0;
+        wc.port_num = 1;
         /* Signal completion event if the solicited bit is set. */
         ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
                        wqe->wr.send_flags & IB_SEND_SOLICITED);
 
 send_comp:
         sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
-
-        if (!(sqp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||
-            (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
-                wc.wr_id = wqe->wr.wr_id;
-                wc.status = IB_WC_SUCCESS;
-                wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
-                wc.vendor_err = 0;
-                wc.byte_len = wqe->length;
-                wc.qp = &sqp->ibqp;
-                wc.src_qp = 0;
-                wc.pkey_index = 0;
-                wc.slid = 0;
-                wc.sl = 0;
-                wc.dlid_path_bits = 0;
-                wc.port_num = 0;
-                ipath_cq_enter(to_icq(sqp->ibqp.send_cq), &wc, 0);
-        }
-
-        /* Update s_last now that we are finished with the SWQE */
-        spin_lock_irqsave(&sqp->s_lock, flags);
-        if (++sqp->s_last >= sqp->s_size)
-                sqp->s_last = 0;
-        spin_unlock_irqrestore(&sqp->s_lock, flags);
+        ipath_send_complete(sqp, wqe, IB_WC_SUCCESS);
         goto again;
 
 done:
@@ -491,13 +477,11 @@ done:
                 wake_up(&qp->wait);
 }
 
-static int want_buffer(struct ipath_devdata *dd)
+static void want_buffer(struct ipath_devdata *dd)
 {
         set_bit(IPATH_S_PIOINTBUFAVAIL, &dd->ipath_sendctrl);
         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
                          dd->ipath_sendctrl);
-
-        return 0;
 }
 
 /**
@@ -507,14 +491,11 @@ static int want_buffer(struct ipath_devdata *dd)
  *
  * Called when we run out of PIO buffers.
  */
-static void ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev)
+static void ipath_no_bufs_available(struct ipath_qp *qp,
+                                    struct ipath_ibdev *dev)
 {
         unsigned long flags;
 
-        spin_lock_irqsave(&dev->pending_lock, flags);
-        if (list_empty(&qp->piowait))
-                list_add_tail(&qp->piowait, &dev->piowait);
-        spin_unlock_irqrestore(&dev->pending_lock, flags);
         /*
          * Note that as soon as want_buffer() is called and
          * possibly before it returns, ipath_ib_piobufavail()
@@ -524,101 +505,14 @@ static void ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev
          * We leave the busy flag set so that another post send doesn't
          * try to put the same QP on the piowait list again.
          */
+        spin_lock_irqsave(&dev->pending_lock, flags);
+        list_add_tail(&qp->piowait, &dev->piowait);
+        spin_unlock_irqrestore(&dev->pending_lock, flags);
         want_buffer(dev->dd);
         dev->n_piowait++;
 }
 
 /**
- * ipath_post_ruc_send - post RC and UC sends
- * @qp: the QP to post on
- * @wr: the work request to send
- */
-int ipath_post_ruc_send(struct ipath_qp *qp, struct ib_send_wr *wr)
-{
-        struct ipath_swqe *wqe;
-        unsigned long flags;
-        u32 next;
-        int i, j;
-        int acc;
-        int ret;
-
-        /*
-         * Don't allow RDMA reads or atomic operations on UC or
-         * undefined operations.
-         * Make sure buffer is large enough to hold the result for atomics.
-         */
-        if (qp->ibqp.qp_type == IB_QPT_UC) {
-                if ((unsigned) wr->opcode >= IB_WR_RDMA_READ) {
-                        ret = -EINVAL;
-                        goto bail;
-                }
-        } else if ((unsigned) wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) {
-                ret = -EINVAL;
-                goto bail;
-        } else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP &&
-                   (wr->num_sge == 0 ||
-                    wr->sg_list[0].length < sizeof(u64) ||
-                    wr->sg_list[0].addr & (sizeof(u64) - 1))) {
-                ret = -EINVAL;
-                goto bail;
-        } else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic) {
-                ret = -EINVAL;
-                goto bail;
-        }
-        /* IB spec says that num_sge == 0 is OK. */
-        if (wr->num_sge > qp->s_max_sge) {
-                ret = -ENOMEM;
-                goto bail;
-        }
-        spin_lock_irqsave(&qp->s_lock, flags);
-        next = qp->s_head + 1;
-        if (next >= qp->s_size)
-                next = 0;
-        if (next == qp->s_last) {
-                spin_unlock_irqrestore(&qp->s_lock, flags);
-                ret = -EINVAL;
-                goto bail;
-        }
-
-        wqe = get_swqe_ptr(qp, qp->s_head);
-        wqe->wr = *wr;
-        wqe->ssn = qp->s_ssn++;
-        wqe->sg_list[0].mr = NULL;
-        wqe->sg_list[0].vaddr = NULL;
-        wqe->sg_list[0].length = 0;
-        wqe->sg_list[0].sge_length = 0;
-        wqe->length = 0;
-        acc = wr->opcode >= IB_WR_RDMA_READ ? IB_ACCESS_LOCAL_WRITE : 0;
-        for (i = 0, j = 0; i < wr->num_sge; i++) {
-                if (to_ipd(qp->ibqp.pd)->user && wr->sg_list[i].lkey == 0) {
-                        spin_unlock_irqrestore(&qp->s_lock, flags);
-                        ret = -EINVAL;
-                        goto bail;
-                }
-                if (wr->sg_list[i].length == 0)
-                        continue;
-                if (!ipath_lkey_ok(qp, &wqe->sg_list[j], &wr->sg_list[i],
-                                   acc)) {
-                        spin_unlock_irqrestore(&qp->s_lock, flags);
-                        ret = -EINVAL;
-                        goto bail;
-                }
-                wqe->length += wr->sg_list[i].length;
-                j++;
-        }
-        wqe->wr.num_sge = j;
-        qp->s_head = next;
-        spin_unlock_irqrestore(&qp->s_lock, flags);
-
-        ipath_do_ruc_send((unsigned long) qp);
-
-        ret = 0;
-
-bail:
-        return ret;
-}
-
-/**
  * ipath_make_grh - construct a GRH header
  * @dev: a pointer to the ipath device
  * @hdr: a pointer to the GRH header being constructed
@@ -648,39 +542,66 @@ u32 ipath_make_grh(struct ipath_ibdev *dev, struct ib_grh *hdr,
         return sizeof(struct ib_grh) / sizeof(u32);
 }
 
+void ipath_make_ruc_header(struct ipath_ibdev *dev, struct ipath_qp *qp,
+                           struct ipath_other_headers *ohdr,
+                           u32 bth0, u32 bth2)
+{
+        u16 lrh0;
+        u32 nwords;
+        u32 extra_bytes;
+
+        /* Construct the header. */
+        extra_bytes = -qp->s_cur_size & 3;
+        nwords = (qp->s_cur_size + extra_bytes) >> 2;
+        lrh0 = IPATH_LRH_BTH;
+        if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+                qp->s_hdrwords += ipath_make_grh(dev, &qp->s_hdr.u.l.grh,
+                                                 &qp->remote_ah_attr.grh,
+                                                 qp->s_hdrwords, nwords);
+                lrh0 = IPATH_LRH_GRH;
+        }
+        lrh0 |= qp->remote_ah_attr.sl << 4;
+        qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+        qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+        qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
+        qp->s_hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid);
+        bth0 |= ipath_get_pkey(dev->dd, qp->s_pkey_index);
+        bth0 |= extra_bytes << 20;
+        ohdr->bth[0] = cpu_to_be32(bth0 | (1 << 22));
+        ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+        ohdr->bth[2] = cpu_to_be32(bth2);
+}
+
 /**
- * ipath_do_ruc_send - perform a send on an RC or UC QP
+ * ipath_do_send - perform a send on a QP
  * @data: contains a pointer to the QP
  *
  * Process entries in the send work queue until credit or queue is
  * exhausted.  Only allow one CPU to send a packet per QP (tasklet).
- * Otherwise, after we drop the QP s_lock, two threads could send
- * packets out of order.
+ * Otherwise, two threads could send packets out of order.
  */
-void ipath_do_ruc_send(unsigned long data)
+void ipath_do_send(unsigned long data)
 {
         struct ipath_qp *qp = (struct ipath_qp *)data;
         struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
-        unsigned long flags;
-        u16 lrh0;
-        u32 nwords;
-        u32 extra_bytes;
-        u32 bth0;
-        u32 bth2;
-        u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
-        struct ipath_other_headers *ohdr;
+        int (*make_req)(struct ipath_qp *qp);
 
         if (test_and_set_bit(IPATH_S_BUSY, &qp->s_busy))
                 goto bail;
 
-        if (unlikely(qp->remote_ah_attr.dlid == dev->dd->ipath_lid)) {
+        if ((qp->ibqp.qp_type == IB_QPT_RC ||
+             qp->ibqp.qp_type == IB_QPT_UC) &&
+            qp->remote_ah_attr.dlid == dev->dd->ipath_lid) {
                 ipath_ruc_loopback(qp);
                 goto clear;
         }
 
-        ohdr = &qp->s_hdr.u.oth;
-        if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
-                ohdr = &qp->s_hdr.u.l.oth;
+        if (qp->ibqp.qp_type == IB_QPT_RC)
+               make_req = ipath_make_rc_req;
+        else if (qp->ibqp.qp_type == IB_QPT_UC)
+               make_req = ipath_make_uc_req;
+        else
+               make_req = ipath_make_ud_req;
 
 again:
         /* Check for a constructed packet to be sent. */
@@ -689,9 +610,8 @@ again:
                  * If no PIO bufs are available, return.  An interrupt will
                  * call ipath_ib_piobufavail() when one is available.
                  */
-                if (ipath_verbs_send(dev->dd, qp->s_hdrwords,
-                                     (u32 *) &qp->s_hdr, qp->s_cur_size,
-                                     qp->s_cur_sge)) {
+                if (ipath_verbs_send(qp, &qp->s_hdr, qp->s_hdrwords,
+                                     qp->s_cur_sge, qp->s_cur_size)) {
                         ipath_no_bufs_available(qp, dev);
                         goto bail;
                 }
@@ -700,54 +620,42 @@ again:
                 qp->s_hdrwords = 0;
         }
 
-        /*
-         * The lock is needed to synchronize between setting
-         * qp->s_ack_state, resend timer, and post_send().
-         */
-        spin_lock_irqsave(&qp->s_lock, flags);
-
-        if (!((qp->ibqp.qp_type == IB_QPT_RC) ?
-               ipath_make_rc_req(qp, ohdr, pmtu, &bth0, &bth2) :
-               ipath_make_uc_req(qp, ohdr, pmtu, &bth0, &bth2))) {
-                /*
-                 * Clear the busy bit before unlocking to avoid races with
-                 * adding new work queue items and then failing to process
-                 * them.
-                 */
-                clear_bit(IPATH_S_BUSY, &qp->s_busy);
-                spin_unlock_irqrestore(&qp->s_lock, flags);
-                goto bail;
-        }
+        if (make_req(qp))
+                goto again;
+clear:
+        clear_bit(IPATH_S_BUSY, &qp->s_busy);
+bail:;
+}
 
-        spin_unlock_irqrestore(&qp->s_lock, flags);
+void ipath_send_complete(struct ipath_qp *qp, struct ipath_swqe *wqe,
+                         enum ib_wc_status status)
+{
+        u32 last = qp->s_last;
 
-        /* Construct the header. */
-        extra_bytes = (4 - qp->s_cur_size) & 3;
-        nwords = (qp->s_cur_size + extra_bytes) >> 2;
-        lrh0 = IPATH_LRH_BTH;
-        if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
-                qp->s_hdrwords += ipath_make_grh(dev, &qp->s_hdr.u.l.grh,
-                                                 &qp->remote_ah_attr.grh,
-                                                 qp->s_hdrwords, nwords);
-                lrh0 = IPATH_LRH_GRH;
-        }
-        lrh0 |= qp->remote_ah_attr.sl << 4;
-        qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
-        qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
-        qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords +
-                                       SIZE_OF_CRC);
-        qp->s_hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid);
-        bth0 |= ipath_get_pkey(dev->dd, qp->s_pkey_index);
-        bth0 |= extra_bytes << 20;
-        ohdr->bth[0] = cpu_to_be32(bth0);
-        ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
-        ohdr->bth[2] = cpu_to_be32(bth2);
+        if (++last == qp->s_size)
+                last = 0;
+        qp->s_last = last;
 
-        /* Check for more work to do. */
-        goto again;
+        /* See ch. 11.2.4.1 and 10.7.3.1 */
+        if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||
+            (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
+            status != IB_WC_SUCCESS) {
+                struct ib_wc wc;
 
-clear:
-        clear_bit(IPATH_S_BUSY, &qp->s_busy);
-bail:
-        return;
+                wc.wr_id = wqe->wr.wr_id;
+                wc.status = status;
+                wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
+                wc.vendor_err = 0;
+                wc.byte_len = wqe->length;
+                wc.imm_data = 0;
+                wc.qp = &qp->ibqp;
+                wc.src_qp = 0;
+                wc.wc_flags = 0;
+                wc.pkey_index = 0;
+                wc.slid = 0;
+                wc.sl = 0;
+                wc.dlid_path_bits = 0;
+                wc.port_num = 0;
+                ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0);
+        }
 }