9 files changed, 512 insertions, 116 deletions
diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c
index 3f441fc57c17..f6d5747153a5 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_hal.c
+++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c
@@ -145,7 +145,9 @@ static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)
        }
        wqe = (struct t3_modify_qp_wr *) skb_put(skb, sizeof(*wqe));
        memset(wqe, 0, sizeof(*wqe));
-        build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0, qpid, 7);
+        build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD,
+                       T3_COMPLETION_FLAG | T3_NOTIFY_FLAG, 0, qpid, 7,
+                       T3_SOPEOP);
        wqe->flags = cpu_to_be32(MODQP_WRITE_EC);
        sge_cmd = qpid << 8 | 3;
        wqe->sge_cmd = cpu_to_be64(sge_cmd);
@@ -276,7 +278,7 @@ int cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,
        if (!wq->qpid)
                return -ENOMEM;
-        wq->rq = kzalloc(depth * sizeof(u64), GFP_KERNEL);
+        wq->rq = kzalloc(depth * sizeof(struct t3_swrq), GFP_KERNEL);
        if (!wq->rq)
                goto err1;
@@ -300,6 +302,7 @@ int cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,
        if (!kernel_domain)
                wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +
                                        (wq->qpid << rdev_p->qpshift);
+        wq->rdev = rdev_p;
        PDBG("%s qpid 0x%x doorbell 0x%p udb 0x%llx\n", __func__,
             wq->qpid, wq->doorbell, (unsigned long long) wq->udb);
        return 0;
@@ -558,7 +561,7 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
        wqe = (struct t3_modify_qp_wr *) skb_put(skb, sizeof(*wqe));
        memset(wqe, 0, sizeof(*wqe));
        build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 0, 0,
-                       T3_CTL_QP_TID, 7);
+                       T3_CTL_QP_TID, 7, T3_SOPEOP);
        wqe->flags = cpu_to_be32(MODQP_WRITE_EC);
        sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3;
        wqe->sge_cmd = cpu_to_be64(sge_cmd);
@@ -674,7 +677,7 @@ static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
                build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,
                               Q_GENBIT(rdev_p->ctrl_qp.wptr,
                                        T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID,
-                               wr_len);
+                               wr_len, T3_SOPEOP);
                if (flag == T3_COMPLETION_FLAG)
                        ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);
                len -= 96;
@@ -816,6 +819,13 @@ int cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)
                             0, 0);
 }
+int cxio_allocate_stag(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr)
+{
+        *stag = T3_STAG_UNSET;
+        return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_NON_SHARED_MR,
+                             0, 0, 0ULL, 0, 0, pbl_size, pbl_addr);
+}
 int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
 {
        struct t3_rdma_init_wr *wqe;
@@ -1257,13 +1267,16 @@ proc_cqe:
                wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
                PDBG("%s completing sq idx %ld\n", __func__,
                     Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2));
-                *cookie = (wq->sq +
+                *cookie = wq->sq[Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2)].wr_id;
-                           Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2))->wr_id;
                wq->sq_rptr++;
        } else {
                PDBG("%s completing rq idx %ld\n", __func__,
                     Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
-                *cookie = *(wq->rq + Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
+                *cookie = wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].wr_id;
+                if (wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].pbl_addr)
+                        cxio_hal_pblpool_free(wq->rdev,
+                                wq->rq[Q_PTR2IDX(wq->rq_rptr,
+                                wq->rq_size_log2)].pbl_addr, T3_STAG0_PBL_SIZE);
                wq->rq_rptr++;
        }
diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.h b/drivers/infiniband/hw/cxgb3/cxio_hal.h
index 6e128f6bab05..656fe47bc84f 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_hal.h
+++ b/drivers/infiniband/hw/cxgb3/cxio_hal.h
@@ -45,15 +45,17 @@
 #define T3_CTRL_QP_SIZE_LOG2  8
 #define T3_CTRL_CQ_ID    0
-/* TBD */
 #define T3_MAX_NUM_RI (1<<15)
 #define T3_MAX_NUM_QP (1<<15)
 #define T3_MAX_NUM_CQ (1<<15)
 #define T3_MAX_NUM_PD (1<<15)
 #define T3_MAX_PBL_SIZE 256
 #define T3_MAX_RQ_SIZE 1024
+#define T3_MAX_QP_DEPTH (T3_MAX_RQ_SIZE-1)
+#define T3_MAX_CQ_DEPTH 8192
 #define T3_MAX_NUM_STAG (1<<15)
 #define T3_MAX_MR_SIZE 0x100000000ULL
+#define T3_PAGESIZE_MASK 0xffff000  /* 4KB-128MB */
 #define T3_STAG_UNSET 0xffffffff
@@ -165,6 +167,7 @@ int cxio_reregister_phys_mem(struct cxio_rdev *rdev, u32 * stag, u32 pdid,
 int cxio_dereg_mem(struct cxio_rdev *rdev, u32 stag, u32 pbl_size,
                   u32 pbl_addr);
 int cxio_allocate_window(struct cxio_rdev *rdev, u32 * stag, u32 pdid);
+int cxio_allocate_stag(struct cxio_rdev *rdev, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr);
 int cxio_deallocate_window(struct cxio_rdev *rdev, u32 stag);
 int cxio_rdma_init(struct cxio_rdev *rdev, struct t3_rdma_init_attr *attr);
 void cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb);
diff --git a/drivers/infiniband/hw/cxgb3/cxio_wr.h b/drivers/infiniband/hw/cxgb3/cxio_wr.h
index f1a25a821a45..04618f7bfbb3 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_wr.h
+++ b/drivers/infiniband/hw/cxgb3/cxio_wr.h
@@ -39,6 +39,9 @@
 #define T3_MAX_SGE      4
 #define T3_MAX_INLINE   64
+#define T3_STAG0_PBL_SIZE (2 * T3_MAX_SGE << 3)
+#define T3_STAG0_MAX_PBE_LEN (128 * 1024 * 1024)
+#define T3_STAG0_PAGE_SHIFT 15
 #define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
 #define Q_FULL(rptr,wptr,size_log2)  ( (((wptr)-(rptr))>>(size_log2)) && \
@@ -72,7 +75,8 @@ enum t3_wr_opcode {
        T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
        T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
        T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
-        T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
+        T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP,
+        T3_WR_FASTREG = FW_WROPCODE_RI_FASTREGISTER_MR
 } __attribute__ ((packed));
 enum t3_rdma_opcode {
@@ -89,7 +93,8 @@ enum t3_rdma_opcode {
        T3_FAST_REGISTER,
        T3_LOCAL_INV,
        T3_QP_MOD,
-        T3_BYPASS
+        T3_BYPASS,
+        T3_RDMA_READ_REQ_WITH_INV,
 } __attribute__ ((packed));
 static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
@@ -103,6 +108,7 @@ static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
                case T3_WR_BIND: return T3_BIND_MW;
                case T3_WR_INIT: return T3_RDMA_INIT;
                case T3_WR_QP_MOD: return T3_QP_MOD;
+                case T3_WR_FASTREG: return T3_FAST_REGISTER;
                default: break;
        }
        return -1;
@@ -170,11 +176,54 @@ struct t3_send_wr {
        struct t3_sge sgl[T3_MAX_SGE];  /* 4+ */
 };
+#define T3_MAX_FASTREG_DEPTH 24
+#define T3_MAX_FASTREG_FRAG 10
+struct t3_fastreg_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        __be32 stag;            /* 2 */
+        __be32 len;
+        __be32 va_base_hi;      /* 3 */
+        __be32 va_base_lo_fbo;
+        __be32 page_type_perms; /* 4 */
+        __be32 reserved1;
+        __be64 pbl_addrs[0];    /* 5+ */
+};
+/*
+ * If a fastreg wr spans multiple wqes, then the 2nd fragment look like this.
+ */
+struct t3_pbl_frag {
+        struct fw_riwrh wrh;    /* 0 */
+        __be64 pbl_addrs[14];   /* 1..14 */
+};
+#define S_FR_PAGE_COUNT         24
+#define M_FR_PAGE_COUNT         0xff
+#define V_FR_PAGE_COUNT(x)      ((x) << S_FR_PAGE_COUNT)
+#define G_FR_PAGE_COUNT(x)      ((((x) >> S_FR_PAGE_COUNT)) & M_FR_PAGE_COUNT)
+#define S_FR_PAGE_SIZE          16
+#define M_FR_PAGE_SIZE          0x1f
+#define V_FR_PAGE_SIZE(x)       ((x) << S_FR_PAGE_SIZE)
+#define G_FR_PAGE_SIZE(x)       ((((x) >> S_FR_PAGE_SIZE)) & M_FR_PAGE_SIZE)
+#define S_FR_TYPE               8
+#define M_FR_TYPE               0x1
+#define V_FR_TYPE(x)            ((x) << S_FR_TYPE)
+#define G_FR_TYPE(x)            ((((x) >> S_FR_TYPE)) & M_FR_TYPE)
+#define S_FR_PERMS              0
+#define M_FR_PERMS              0xff
+#define V_FR_PERMS(x)           ((x) << S_FR_PERMS)
+#define G_FR_PERMS(x)           ((((x) >> S_FR_PERMS)) & M_FR_PERMS)
 struct t3_local_inv_wr {
        struct fw_riwrh wrh;    /* 0 */
        union t3_wrid wrid;     /* 1 */
        __be32 stag;            /* 2 */
-        __be32 reserved3;
+        __be32 reserved;
 };
 struct t3_rdma_write_wr {
@@ -193,7 +242,8 @@ struct t3_rdma_read_wr {
        struct fw_riwrh wrh;    /* 0 */
        union t3_wrid wrid;     /* 1 */
        u8 rdmaop;              /* 2 */
-        u8 reserved[3];
+        u8 local_inv;
+        u8 reserved[2];
        __be32 rem_stag;
        __be64 rem_to;          /* 3 */
        __be32 local_stag;      /* 4 */
@@ -201,18 +251,6 @@ struct t3_rdma_read_wr {
        __be64 local_to;        /* 5 */
 };
-enum t3_addr_type {
-        T3_VA_BASED_TO = 0x0,
-        T3_ZERO_BASED_TO = 0x1
-} __attribute__ ((packed));
-enum t3_mem_perms {
-        T3_MEM_ACCESS_LOCAL_READ = 0x1,
-        T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
-        T3_MEM_ACCESS_REM_READ = 0x4,
-        T3_MEM_ACCESS_REM_WRITE = 0x8
-} __attribute__ ((packed));
 struct t3_bind_mw_wr {
        struct fw_riwrh wrh;    /* 0 */
        union t3_wrid wrid;     /* 1 */
@@ -336,6 +374,11 @@ struct t3_genbit {
        __be64 genbit;
 };
+struct t3_wq_in_err {
+        u64 flit[13];
+        u64 err;
+};
 enum rdma_init_wr_flags {
        MPA_INITIATOR = (1<<0),
        PRIV_QP = (1<<1),
@@ -346,13 +389,16 @@ union t3_wr {
        struct t3_rdma_write_wr write;
        struct t3_rdma_read_wr read;
        struct t3_receive_wr recv;
+        struct t3_fastreg_wr fastreg;
+        struct t3_pbl_frag pbl_frag;
        struct t3_local_inv_wr local_inv;
        struct t3_bind_mw_wr bind;
        struct t3_bypass_wr bypass;
        struct t3_rdma_init_wr init;
        struct t3_modify_qp_wr qp_mod;
        struct t3_genbit genbit;
-        u64 flit[16];
+        struct t3_wq_in_err wq_in_err;
+        __be64 flit[16];
 };
 #define T3_SQ_CQE_FLIT    13
@@ -366,12 +412,18 @@ static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)
        return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));
 }
+enum t3_wr_hdr_bits {
+        T3_EOP = 1,
+        T3_SOP = 2,
+        T3_SOPEOP = T3_EOP|T3_SOP,
+};
 static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
                                  enum t3_wr_flags flags, u8 genbit, u32 tid,
-                                  u8 len)
+                                  u8 len, u8 sopeop)
 {
        wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |
-                                         V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
+                                         V_FW_RIWR_SOPEOP(sopeop) |
                                         V_FW_RIWR_FLAGS(flags));
        wmb();
        wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
@@ -404,6 +456,7 @@ enum tpt_addr_type {
 };
 enum tpt_mem_perm {
+        TPT_MW_BIND = 0x10,
        TPT_LOCAL_READ = 0x8,
        TPT_LOCAL_WRITE = 0x4,
        TPT_REMOTE_READ = 0x2,
@@ -615,6 +668,11 @@ struct t3_swsq {
        int                     signaled;
 };
+struct t3_swrq {
+        __u64                   wr_id;
+        __u32                   pbl_addr;
+};
 /*
 * A T3 WQ implements both the SQ and RQ.
 */
@@ -631,14 +689,15 @@ struct t3_wq {
        u32 sq_wptr;                    /* sq_wptr - sq_rptr == count of */
        u32 sq_rptr;                    /* pending wrs */
        u32 sq_size_log2;               /* sq size */
-        u64 *rq;                        /* SW RQ (holds consumer wr_ids */
+        struct t3_swrq *rq;             /* SW RQ (holds consumer wr_ids */
        u32 rq_wptr;                    /* rq_wptr - rq_rptr == count of */
        u32 rq_rptr;                    /* pending wrs */
-        u64 *rq_oldest_wr;              /* oldest wr on the SW RQ */
+        struct t3_swrq *rq_oldest_wr;   /* oldest wr on the SW RQ */
        u32 rq_size_log2;               /* rq size */
        u32 rq_addr;                    /* rq adapter address */
        void __iomem *doorbell;         /* kernel db */
        u64 udb;                        /* user db if any */
+        struct cxio_rdev *rdev;
 };
 struct t3_cq {
@@ -659,7 +718,7 @@ struct t3_cq {
 static inline void cxio_set_wq_in_error(struct t3_wq *wq)
 {
-        wq->queue->flit[13] = 1;
+        wq->queue->wq_in_err.err = 1;
 }
 static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)
diff --git a/drivers/infiniband/hw/cxgb3/iwch.c b/drivers/infiniband/hw/cxgb3/iwch.c
index 71554eacb13c..4489c89d6710 100644
--- a/drivers/infiniband/hw/cxgb3/iwch.c
+++ b/drivers/infiniband/hw/cxgb3/iwch.c
@@ -71,18 +71,16 @@ static void rnic_init(struct iwch_dev *rnicp)
        idr_init(&rnicp->mmidr);
        spin_lock_init(&rnicp->lock);
-        rnicp->attr.vendor_id = 0x168;
-        rnicp->attr.vendor_part_id = 7;
        rnicp->attr.max_qps = T3_MAX_NUM_QP - 32;
-        rnicp->attr.max_wrs = (1UL << 24) - 1;
+        rnicp->attr.max_wrs = T3_MAX_QP_DEPTH;
        rnicp->attr.max_sge_per_wr = T3_MAX_SGE;
        rnicp->attr.max_sge_per_rdma_write_wr = T3_MAX_SGE;
        rnicp->attr.max_cqs = T3_MAX_NUM_CQ - 1;
-        rnicp->attr.max_cqes_per_cq = (1UL << 24) - 1;
+        rnicp->attr.max_cqes_per_cq = T3_MAX_CQ_DEPTH;
        rnicp->attr.max_mem_regs = cxio_num_stags(&rnicp->rdev);
        rnicp->attr.max_phys_buf_entries = T3_MAX_PBL_SIZE;
        rnicp->attr.max_pds = T3_MAX_NUM_PD - 1;
-        rnicp->attr.mem_pgsizes_bitmask = 0x7FFF;       /* 4KB-128MB */
+        rnicp->attr.mem_pgsizes_bitmask = T3_PAGESIZE_MASK;
        rnicp->attr.max_mr_size = T3_MAX_MR_SIZE;
        rnicp->attr.can_resize_wq = 0;
        rnicp->attr.max_rdma_reads_per_qp = 8;
diff --git a/drivers/infiniband/hw/cxgb3/iwch.h b/drivers/infiniband/hw/cxgb3/iwch.h
index d2409a505e8d..3773453b2cf0 100644
--- a/drivers/infiniband/hw/cxgb3/iwch.h
+++ b/drivers/infiniband/hw/cxgb3/iwch.h
@@ -48,8 +48,6 @@ struct iwch_qp;
 struct iwch_mr;
 struct iwch_rnic_attributes {
-        u32 vendor_id;
-        u32 vendor_part_id;
        u32 max_qps;
        u32 max_wrs;                            /* Max for any SQ/RQ */
        u32 max_sge_per_wr;
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cq.c b/drivers/infiniband/hw/cxgb3/iwch_cq.c
index 4ee8ccd0a9e5..cf5474ae68ff 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cq.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_cq.c
@@ -81,6 +81,7 @@ static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
        wc->wr_id = cookie;
        wc->qp = &qhp->ibqp;
        wc->vendor_err = CQE_STATUS(cqe);
+        wc->wc_flags = 0;
        PDBG("%s qpid 0x%x type %d opcode %d status 0x%x wrid hi 0x%x "
             "lo 0x%x cookie 0x%llx\n", __func__,
@@ -94,6 +95,11 @@ static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
                else
                        wc->byte_len = 0;
                wc->opcode = IB_WC_RECV;
+                if (CQE_OPCODE(cqe) == T3_SEND_WITH_INV ||
+                    CQE_OPCODE(cqe) == T3_SEND_WITH_SE_INV) {
+                        wc->ex.invalidate_rkey = CQE_WRID_STAG(cqe);
+                        wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+                }
        } else {
                switch (CQE_OPCODE(cqe)) {
                case T3_RDMA_WRITE:
@@ -105,17 +111,20 @@ static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
                        break;
                case T3_SEND:
                case T3_SEND_WITH_SE:
+                case T3_SEND_WITH_INV:
+                case T3_SEND_WITH_SE_INV:
                        wc->opcode = IB_WC_SEND;
                        break;
                case T3_BIND_MW:
                        wc->opcode = IB_WC_BIND_MW;
                        break;
-                /* these aren't supported yet */
-                case T3_SEND_WITH_INV:
-                case T3_SEND_WITH_SE_INV:
                case T3_LOCAL_INV:
+                        wc->opcode = IB_WC_LOCAL_INV;
+                        break;
                case T3_FAST_REGISTER:
+                        wc->opcode = IB_WC_FAST_REG_MR;
+                        break;
                default:
                        printk(KERN_ERR MOD "Unexpected opcode %d "
                               "in the CQE received for QPID=0x%0x\n",
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 95f82cfb6c54..b89640aa6e10 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -56,6 +56,7 @@
 #include "iwch_provider.h"
 #include "iwch_cm.h"
 #include "iwch_user.h"
+#include "common.h"
 static int iwch_modify_port(struct ib_device *ibdev,
                            u8 port, int port_modify_mask,
@@ -747,6 +748,7 @@ static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd)
        mhp->attr.type = TPT_MW;
        mhp->attr.stag = stag;
        mmid = (stag) >> 8;
+        mhp->ibmw.rkey = stag;
        insert_handle(rhp, &rhp->mmidr, mhp, mmid);
        PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
        return &(mhp->ibmw);
@@ -768,6 +770,68 @@ static int iwch_dealloc_mw(struct ib_mw *mw)
        return 0;
 }
+static struct ib_mr *iwch_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth)
+{
+        struct iwch_dev *rhp;
+        struct iwch_pd *php;
+        struct iwch_mr *mhp;
+        u32 mmid;
+        u32 stag = 0;
+        int ret;
+        php = to_iwch_pd(pd);
+        rhp = php->rhp;
+        mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
+        if (!mhp)
+                return ERR_PTR(-ENOMEM);
+        mhp->rhp = rhp;
+        ret = iwch_alloc_pbl(mhp, pbl_depth);
+        if (ret) {
+                kfree(mhp);
+                return ERR_PTR(ret);
+        }
+        mhp->attr.pbl_size = pbl_depth;
+        ret = cxio_allocate_stag(&rhp->rdev, &stag, php->pdid,
+                                 mhp->attr.pbl_size, mhp->attr.pbl_addr);
+        if (ret) {
+                iwch_free_pbl(mhp);
+                kfree(mhp);
+                return ERR_PTR(ret);
+        }
+        mhp->attr.pdid = php->pdid;
+        mhp->attr.type = TPT_NON_SHARED_MR;
+        mhp->attr.stag = stag;
+        mhp->attr.state = 1;
+        mmid = (stag) >> 8;
+        mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
+        insert_handle(rhp, &rhp->mmidr, mhp, mmid);
+        PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
+        return &(mhp->ibmr);
+}
+static struct ib_fast_reg_page_list *iwch_alloc_fastreg_pbl(
+                                        struct ib_device *device,
+                                        int page_list_len)
+{
+        struct ib_fast_reg_page_list *page_list;
+        page_list = kmalloc(sizeof *page_list + page_list_len * sizeof(u64),
+                            GFP_KERNEL);
+        if (!page_list)
+                return ERR_PTR(-ENOMEM);
+        page_list->page_list = (u64 *)(page_list + 1);
+        page_list->max_page_list_len = page_list_len;
+        return page_list;
+}
+static void iwch_free_fastreg_pbl(struct ib_fast_reg_page_list *page_list)
+{
+        kfree(page_list);
+}
 static int iwch_destroy_qp(struct ib_qp *ib_qp)
 {
        struct iwch_dev *rhp;
@@ -843,6 +907,15 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
         */
        sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
        wqsize = roundup_pow_of_two(rqsize + sqsize);
+        /*
+         * Kernel users need more wq space for fastreg WRs which can take
+         * 2 WR fragments.
+         */
+        ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
+        if (!ucontext && wqsize < (rqsize + (2 * sqsize)))
+                wqsize = roundup_pow_of_two(rqsize +
+                                roundup_pow_of_two(attrs->cap.max_send_wr * 2));
        PDBG("%s wqsize %d sqsize %d rqsize %d\n", __func__,
             wqsize, sqsize, rqsize);
        qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
@@ -851,7 +924,6 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
        qhp->wq.size_log2 = ilog2(wqsize);
        qhp->wq.rq_size_log2 = ilog2(rqsize);
        qhp->wq.sq_size_log2 = ilog2(sqsize);
-        ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
        if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
                           ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
                kfree(qhp);
@@ -935,10 +1007,10 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
        qhp->ibqp.qp_num = qhp->wq.qpid;
        init_timer(&(qhp->timer));
        PDBG("%s sq_num_entries %d, rq_num_entries %d "
-             "qpid 0x%0x qhp %p dma_addr 0x%llx size %d\n",
+             "qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
             __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
             qhp->wq.qpid, qhp, (unsigned long long) qhp->wq.dma_addr,
-             1 << qhp->wq.size_log2);
+             1 << qhp->wq.size_log2, qhp->wq.rq_addr);
        return &qhp->ibqp;
 }
@@ -1023,6 +1095,29 @@ static int iwch_query_gid(struct ib_device *ibdev, u8 port,
        return 0;
 }
+static u64 fw_vers_string_to_u64(struct iwch_dev *iwch_dev)
+{
+        struct ethtool_drvinfo info;
+        struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
+        char *cp, *next;
+        unsigned fw_maj, fw_min, fw_mic;
+        rtnl_lock();
+        lldev->ethtool_ops->get_drvinfo(lldev, &info);
+        rtnl_unlock();
+        next = info.fw_version + 1;
+        cp = strsep(&next, ".");
+        sscanf(cp, "%i", &fw_maj);
+        cp = strsep(&next, ".");
+        sscanf(cp, "%i", &fw_min);
+        cp = strsep(&next, ".");
+        sscanf(cp, "%i", &fw_mic);
+        return (((u64)fw_maj & 0xffff) << 32) | ((fw_min & 0xffff) << 16) |
+               (fw_mic & 0xffff);
+}
 static int iwch_query_device(struct ib_device *ibdev,
                             struct ib_device_attr *props)
 {
@@ -1033,7 +1128,10 @@ static int iwch_query_device(struct ib_device *ibdev,
        dev = to_iwch_dev(ibdev);
        memset(props, 0, sizeof *props);
        memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
+        props->hw_ver = dev->rdev.t3cdev_p->type;
+        props->fw_ver = fw_vers_string_to_u64(dev);
        props->device_cap_flags = dev->device_cap_flags;
+        props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
        props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
        props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
        props->max_mr_size = dev->attr.max_mr_size;
@@ -1048,6 +1146,7 @@ static int iwch_query_device(struct ib_device *ibdev,
        props->max_mr = dev->attr.max_mem_regs;
        props->max_pd = dev->attr.max_pds;
        props->local_ca_ack_delay = 0;
+        props->max_fast_reg_page_list_len = T3_MAX_FASTREG_DEPTH;
        return 0;
 }
@@ -1088,6 +1187,28 @@ static ssize_t show_rev(struct device *dev, struct device_attribute *attr,
        return sprintf(buf, "%d\n", iwch_dev->rdev.t3cdev_p->type);
 }
+static int fw_supports_fastreg(struct iwch_dev *iwch_dev)
+{
+        struct ethtool_drvinfo info;
+        struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
+        char *cp, *next;
+        unsigned fw_maj, fw_min;
+        rtnl_lock();
+        lldev->ethtool_ops->get_drvinfo(lldev, &info);
+        rtnl_unlock();
+        next = info.fw_version+1;
+        cp = strsep(&next, ".");
+        sscanf(cp, "%i", &fw_maj);
+        cp = strsep(&next, ".");
+        sscanf(cp, "%i", &fw_min);
+        PDBG("%s maj %u min %u\n", __func__, fw_maj, fw_min);
+        return fw_maj > 6 || (fw_maj == 6 && fw_min > 0);
+}
 static ssize_t show_fw_ver(struct device *dev, struct device_attribute *attr, char *buf)
 {
        struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
@@ -1127,6 +1248,61 @@ static ssize_t show_board(struct device *dev, struct device_attribute *attr,
                       iwch_dev->rdev.rnic_info.pdev->device);
 }
+static int iwch_get_mib(struct ib_device *ibdev,
+                        union rdma_protocol_stats *stats)
+{
+        struct iwch_dev *dev;
+        struct tp_mib_stats m;
+        int ret;
+        PDBG("%s ibdev %p\n", __func__, ibdev);
+        dev = to_iwch_dev(ibdev);
+        ret = dev->rdev.t3cdev_p->ctl(dev->rdev.t3cdev_p, RDMA_GET_MIB, &m);
+        if (ret)
+                return -ENOSYS;
+        memset(stats, 0, sizeof *stats);
+        stats->iw.ipInReceives = ((u64) m.ipInReceive_hi << 32) +
+                                m.ipInReceive_lo;
+        stats->iw.ipInHdrErrors = ((u64) m.ipInHdrErrors_hi << 32) +
+                                  m.ipInHdrErrors_lo;
+        stats->iw.ipInAddrErrors = ((u64) m.ipInAddrErrors_hi << 32) +
+                                   m.ipInAddrErrors_lo;
+        stats->iw.ipInUnknownProtos = ((u64) m.ipInUnknownProtos_hi << 32) +
+                                      m.ipInUnknownProtos_lo;
+        stats->iw.ipInDiscards = ((u64) m.ipInDiscards_hi << 32) +
+                                 m.ipInDiscards_lo;
+        stats->iw.ipInDelivers = ((u64) m.ipInDelivers_hi << 32) +
+                                 m.ipInDelivers_lo;
+        stats->iw.ipOutRequests = ((u64) m.ipOutRequests_hi << 32) +
+                                  m.ipOutRequests_lo;
+        stats->iw.ipOutDiscards = ((u64) m.ipOutDiscards_hi << 32) +
+                                  m.ipOutDiscards_lo;
+        stats->iw.ipOutNoRoutes = ((u64) m.ipOutNoRoutes_hi << 32) +
+                                  m.ipOutNoRoutes_lo;
+        stats->iw.ipReasmTimeout = (u64) m.ipReasmTimeout;
+        stats->iw.ipReasmReqds = (u64) m.ipReasmReqds;
+        stats->iw.ipReasmOKs = (u64) m.ipReasmOKs;
+        stats->iw.ipReasmFails = (u64) m.ipReasmFails;
+        stats->iw.tcpActiveOpens = (u64) m.tcpActiveOpens;
+        stats->iw.tcpPassiveOpens = (u64) m.tcpPassiveOpens;
+        stats->iw.tcpAttemptFails = (u64) m.tcpAttemptFails;
+        stats->iw.tcpEstabResets = (u64) m.tcpEstabResets;
+        stats->iw.tcpOutRsts = (u64) m.tcpOutRsts;
+        stats->iw.tcpCurrEstab = (u64) m.tcpCurrEstab;
+        stats->iw.tcpInSegs = ((u64) m.tcpInSegs_hi << 32) +
+                              m.tcpInSegs_lo;
+        stats->iw.tcpOutSegs = ((u64) m.tcpOutSegs_hi << 32) +
+                               m.tcpOutSegs_lo;
+        stats->iw.tcpRetransSegs = ((u64) m.tcpRetransSeg_hi << 32) +
+                                  m.tcpRetransSeg_lo;
+        stats->iw.tcpInErrs = ((u64) m.tcpInErrs_hi << 32) +
+                              m.tcpInErrs_lo;
+        stats->iw.tcpRtoMin = (u64) m.tcpRtoMin;
+        stats->iw.tcpRtoMax = (u64) m.tcpRtoMax;
+        return 0;
+}
 static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
 static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
@@ -1136,7 +1312,7 @@ static struct device_attribute *iwch_class_attributes[] = {
        &dev_attr_hw_rev,
        &dev_attr_fw_ver,
        &dev_attr_hca_type,
-        &dev_attr_board_id
+        &dev_attr_board_id,
 };
 int iwch_register_device(struct iwch_dev *dev)
@@ -1149,8 +1325,12 @@ int iwch_register_device(struct iwch_dev *dev)
        memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
        memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
        dev->ibdev.owner = THIS_MODULE;
-        dev->device_cap_flags =
+        dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
-            (IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW);
+        /* cxgb3 supports STag 0. */
+        dev->ibdev.local_dma_lkey = 0;
+        if (fw_supports_fastreg(dev))
+                dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
        dev->ibdev.uverbs_cmd_mask =
            (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
@@ -1202,15 +1382,16 @@ int iwch_register_device(struct iwch_dev *dev)
        dev->ibdev.alloc_mw = iwch_alloc_mw;
        dev->ibdev.bind_mw = iwch_bind_mw;
        dev->ibdev.dealloc_mw = iwch_dealloc_mw;
+        dev->ibdev.alloc_fast_reg_mr = iwch_alloc_fast_reg_mr;
+        dev->ibdev.alloc_fast_reg_page_list = iwch_alloc_fastreg_pbl;
+        dev->ibdev.free_fast_reg_page_list = iwch_free_fastreg_pbl;
        dev->ibdev.attach_mcast = iwch_multicast_attach;
        dev->ibdev.detach_mcast = iwch_multicast_detach;
        dev->ibdev.process_mad = iwch_process_mad;
        dev->ibdev.req_notify_cq = iwch_arm_cq;
        dev->ibdev.post_send = iwch_post_send;
        dev->ibdev.post_recv = iwch_post_receive;
+        dev->ibdev.get_protocol_stats = iwch_get_mib;
        dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
        if (!dev->ibdev.iwcm)
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.h b/drivers/infiniband/hw/cxgb3/iwch_provider.h
index 836163fc5429..f5ceca05c435 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.h
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.h
@@ -296,14 +296,6 @@ static inline u32 iwch_ib_to_tpt_access(int acc)
               TPT_LOCAL_READ;
 }
-static inline u32 iwch_ib_to_mwbind_access(int acc)
-{
-        return (acc & IB_ACCESS_REMOTE_WRITE ? T3_MEM_ACCESS_REM_WRITE : 0) |
-               (acc & IB_ACCESS_REMOTE_READ ? T3_MEM_ACCESS_REM_READ : 0) |
-               (acc & IB_ACCESS_LOCAL_WRITE ? T3_MEM_ACCESS_LOCAL_WRITE : 0) |
-               T3_MEM_ACCESS_LOCAL_READ;
-}
 enum iwch_mmid_state {
        IWCH_STAG_STATE_VALID,
        IWCH_STAG_STATE_INVALID
diff --git a/drivers/infiniband/hw/cxgb3/iwch_qp.c b/drivers/infiniband/hw/cxgb3/iwch_qp.c
index 992613799228..9a3be3a9d5dc 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_qp.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_qp.c
@@ -33,10 +33,11 @@
 #include "iwch.h"
 #include "iwch_cm.h"
 #include "cxio_hal.h"
+#include "cxio_resource.h"
 #define NO_SUPPORT -1
-static int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr,
+static int build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr,
                                u8 * flit_cnt)
 {
        int i;
@@ -44,59 +45,44 @@ static int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr,
        switch (wr->opcode) {
        case IB_WR_SEND:
-        case IB_WR_SEND_WITH_IMM:
                if (wr->send_flags & IB_SEND_SOLICITED)
                        wqe->send.rdmaop = T3_SEND_WITH_SE;
                else
                        wqe->send.rdmaop = T3_SEND;
                wqe->send.rem_stag = 0;
                break;
-#if 0                           /* Not currently supported */
+        case IB_WR_SEND_WITH_INV:
-        case TYPE_SEND_INVALIDATE:
+                if (wr->send_flags & IB_SEND_SOLICITED)
-        case TYPE_SEND_INVALIDATE_IMMEDIATE:
+                        wqe->send.rdmaop = T3_SEND_WITH_SE_INV;
-                wqe->send.rdmaop = T3_SEND_WITH_INV;
+                else
-                wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey);
+                        wqe->send.rdmaop = T3_SEND_WITH_INV;
-                break;
+                wqe->send.rem_stag = cpu_to_be32(wr->ex.invalidate_rkey);
-        case TYPE_SEND_SE_INVALIDATE:
-                wqe->send.rdmaop = T3_SEND_WITH_SE_INV;
-                wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey);
                break;
-#endif
        default:
-                break;
+                return -EINVAL;
        }
        if (wr->num_sge > T3_MAX_SGE)
                return -EINVAL;
        wqe->send.reserved[0] = 0;
        wqe->send.reserved[1] = 0;
        wqe->send.reserved[2] = 0;
-        if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+        plen = 0;
-                plen = 4;
+        for (i = 0; i < wr->num_sge; i++) {
-                wqe->send.sgl[0].stag = wr->ex.imm_data;
+                if ((plen + wr->sg_list[i].length) < plen)
-                wqe->send.sgl[0].len = __constant_cpu_to_be32(0);
+                        return -EMSGSIZE;
-                wqe->send.num_sgle = __constant_cpu_to_be32(0);
-                *flit_cnt = 5;
+                plen += wr->sg_list[i].length;
-        } else {
+                wqe->send.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey);
-                plen = 0;
+                wqe->send.sgl[i].len = cpu_to_be32(wr->sg_list[i].length);
-                for (i = 0; i < wr->num_sge; i++) {
+                wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr);
-                        if ((plen + wr->sg_list[i].length) < plen) {
-                                return -EMSGSIZE;
-                        }
-                        plen += wr->sg_list[i].length;
-                        wqe->send.sgl[i].stag =
-                            cpu_to_be32(wr->sg_list[i].lkey);
-                        wqe->send.sgl[i].len =
-                            cpu_to_be32(wr->sg_list[i].length);
-                        wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr);
-                }
-                wqe->send.num_sgle = cpu_to_be32(wr->num_sge);
-                *flit_cnt = 4 + ((wr->num_sge) << 1);
        }
+        wqe->send.num_sgle = cpu_to_be32(wr->num_sge);
+        *flit_cnt = 4 + ((wr->num_sge) << 1);
        wqe->send.plen = cpu_to_be32(plen);
        return 0;
 }
-static int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr,
+static int build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr,
                                 u8 *flit_cnt)
 {
        int i;
@@ -137,15 +123,18 @@ static int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr,
        return 0;
 }
-static int iwch_build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr,
+static int build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr,
                                u8 *flit_cnt)
 {
        if (wr->num_sge > 1)
                return -EINVAL;
        wqe->read.rdmaop = T3_READ_REQ;
+        if (wr->opcode == IB_WR_RDMA_READ_WITH_INV)
+                wqe->read.local_inv = 1;
+        else
+                wqe->read.local_inv = 0;
        wqe->read.reserved[0] = 0;
        wqe->read.reserved[1] = 0;
-        wqe->read.reserved[2] = 0;
        wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey);
        wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr);
        wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey);
@@ -155,6 +144,57 @@ static int iwch_build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr,
        return 0;
 }
+static int build_fastreg(union t3_wr *wqe, struct ib_send_wr *wr,
+                                u8 *flit_cnt, int *wr_cnt, struct t3_wq *wq)
+{
+        int i;
+        __be64 *p;
+        if (wr->wr.fast_reg.page_list_len > T3_MAX_FASTREG_DEPTH)
+                return -EINVAL;
+        *wr_cnt = 1;
+        wqe->fastreg.stag = cpu_to_be32(wr->wr.fast_reg.rkey);
+        wqe->fastreg.len = cpu_to_be32(wr->wr.fast_reg.length);
+        wqe->fastreg.va_base_hi = cpu_to_be32(wr->wr.fast_reg.iova_start >> 32);
+        wqe->fastreg.va_base_lo_fbo =
+                                cpu_to_be32(wr->wr.fast_reg.iova_start & 0xffffffff);
+        wqe->fastreg.page_type_perms = cpu_to_be32(
+                V_FR_PAGE_COUNT(wr->wr.fast_reg.page_list_len) |
+                V_FR_PAGE_SIZE(wr->wr.fast_reg.page_shift-12) |
+                V_FR_TYPE(TPT_VATO) |
+                V_FR_PERMS(iwch_ib_to_tpt_access(wr->wr.fast_reg.access_flags)));
+        p = &wqe->fastreg.pbl_addrs[0];
+        for (i = 0; i < wr->wr.fast_reg.page_list_len; i++, p++) {
+                /* If we need a 2nd WR, then set it up */
+                if (i == T3_MAX_FASTREG_FRAG) {
+                        *wr_cnt = 2;
+                        wqe = (union t3_wr *)(wq->queue +
+                                Q_PTR2IDX((wq->wptr+1), wq->size_log2));
+                        build_fw_riwrh((void *)wqe, T3_WR_FASTREG, 0,
+                               Q_GENBIT(wq->wptr + 1, wq->size_log2),
+                               0, 1 + wr->wr.fast_reg.page_list_len - T3_MAX_FASTREG_FRAG,
+                               T3_EOP);
+                        p = &wqe->pbl_frag.pbl_addrs[0];
+                }
+                *p = cpu_to_be64((u64)wr->wr.fast_reg.page_list->page_list[i]);
+        }
+        *flit_cnt = 5 + wr->wr.fast_reg.page_list_len;
+        if (*flit_cnt > 15)
+                *flit_cnt = 15;
+        return 0;
+}
+static int build_inv_stag(union t3_wr *wqe, struct ib_send_wr *wr,
+                                u8 *flit_cnt)
+{
+        wqe->local_inv.stag = cpu_to_be32(wr->ex.invalidate_rkey);
+        wqe->local_inv.reserved = 0;
+        *flit_cnt = sizeof(struct t3_local_inv_wr) >> 3;
+        return 0;
+}
 /*
 * TBD: this is going to be moved to firmware. Missing pdid/qpid check for now.
 */
@@ -205,23 +245,106 @@ static int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list,
        return 0;
 }
-static int iwch_build_rdma_recv(struct iwch_dev *rhp, union t3_wr *wqe,
+static int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe,
                                struct ib_recv_wr *wr)
 {
-        int i;
+        int i, err = 0;
-        if (wr->num_sge > T3_MAX_SGE)
+        u32 pbl_addr[T3_MAX_SGE];
-                return -EINVAL;
+        u8 page_size[T3_MAX_SGE];
+        err = iwch_sgl2pbl_map(qhp->rhp, wr->sg_list, wr->num_sge, pbl_addr,
+                               page_size);
+        if (err)
+                return err;
+        wqe->recv.pagesz[0] = page_size[0];
+        wqe->recv.pagesz[1] = page_size[1];
+        wqe->recv.pagesz[2] = page_size[2];
+        wqe->recv.pagesz[3] = page_size[3];
        wqe->recv.num_sgle = cpu_to_be32(wr->num_sge);
        for (i = 0; i < wr->num_sge; i++) {
                wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey);
                wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length);
+                /* to in the WQE == the offset into the page */
+                wqe->recv.sgl[i].to = cpu_to_be64(((u32) wr->sg_list[i].addr) %
+                                (1UL << (12 + page_size[i])));
+                /* pbl_addr is the adapters address in the PBL */
+                wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]);
+        }
+        for (; i < T3_MAX_SGE; i++) {
+                wqe->recv.sgl[i].stag = 0;
+                wqe->recv.sgl[i].len = 0;
+                wqe->recv.sgl[i].to = 0;
+                wqe->recv.pbl_addr[i] = 0;
+        }
+        qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,
+                             qhp->wq.rq_size_log2)].wr_id = wr->wr_id;
+        qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,
+                             qhp->wq.rq_size_log2)].pbl_addr = 0;
+        return 0;
+}
+static int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe,
+                                struct ib_recv_wr *wr)
+{
+        int i;
+        u32 pbl_addr;
+        u32 pbl_offset;
+        /*
+         * The T3 HW requires the PBL in the HW recv descriptor to reference
+         * a PBL entry.  So we allocate the max needed PBL memory here and pass
+         * it to the uP in the recv WR.  The uP will build the PBL and setup
+         * the HW recv descriptor.
+         */
+        pbl_addr = cxio_hal_pblpool_alloc(&qhp->rhp->rdev, T3_STAG0_PBL_SIZE);
+        if (!pbl_addr)
+                return -ENOMEM;
+        /*
+         * Compute the 8B aligned offset.
+         */
+        pbl_offset = (pbl_addr - qhp->rhp->rdev.rnic_info.pbl_base) >> 3;
+        wqe->recv.num_sgle = cpu_to_be32(wr->num_sge);
+        for (i = 0; i < wr->num_sge; i++) {
+                /*
+                 * Use a 128MB page size. This and an imposed 128MB
+                 * sge length limit allows us to require only a 2-entry HW
+                 * PBL for each SGE.  This restriction is acceptable since
+                 * since it is not possible to allocate 128MB of contiguous
+                 * DMA coherent memory!
+                 */
+                if (wr->sg_list[i].length > T3_STAG0_MAX_PBE_LEN)
+                        return -EINVAL;
+                wqe->recv.pagesz[i] = T3_STAG0_PAGE_SHIFT;
+                /*
+                 * T3 restricts a recv to all zero-stag or all non-zero-stag.
+                 */
+                if (wr->sg_list[i].lkey != 0)
+                        return -EINVAL;
+                wqe->recv.sgl[i].stag = 0;
+                wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length);
                wqe->recv.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr);
+                wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_offset);
+                pbl_offset += 2;
        }
        for (; i < T3_MAX_SGE; i++) {
+                wqe->recv.pagesz[i] = 0;
                wqe->recv.sgl[i].stag = 0;
                wqe->recv.sgl[i].len = 0;
                wqe->recv.sgl[i].to = 0;
+                wqe->recv.pbl_addr[i] = 0;
        }
+        qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,
+                             qhp->wq.rq_size_log2)].wr_id = wr->wr_id;
+        qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,
+                             qhp->wq.rq_size_log2)].pbl_addr = pbl_addr;
        return 0;
 }
@@ -238,6 +361,7 @@ int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
        u32 num_wrs;
        unsigned long flag;
        struct t3_swsq *sqp;
+        int wr_cnt = 1;
        qhp = to_iwch_qp(ibqp);
        spin_lock_irqsave(&qhp->lock, flag);
@@ -262,33 +386,45 @@ int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                t3_wr_flags = 0;
                if (wr->send_flags & IB_SEND_SOLICITED)
                        t3_wr_flags |= T3_SOLICITED_EVENT_FLAG;
-                if (wr->send_flags & IB_SEND_FENCE)
-                        t3_wr_flags |= T3_READ_FENCE_FLAG;
                if (wr->send_flags & IB_SEND_SIGNALED)
                        t3_wr_flags |= T3_COMPLETION_FLAG;
                sqp = qhp->wq.sq +
                      Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);
                switch (wr->opcode) {
                case IB_WR_SEND:
-                case IB_WR_SEND_WITH_IMM:
+                case IB_WR_SEND_WITH_INV:
+                        if (wr->send_flags & IB_SEND_FENCE)
+                                t3_wr_flags |= T3_READ_FENCE_FLAG;
                        t3_wr_opcode = T3_WR_SEND;
-                        err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt);
+                        err = build_rdma_send(wqe, wr, &t3_wr_flit_cnt);
                        break;
                case IB_WR_RDMA_WRITE:
                case IB_WR_RDMA_WRITE_WITH_IMM:
                        t3_wr_opcode = T3_WR_WRITE;
-                        err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt);
+                        err = build_rdma_write(wqe, wr, &t3_wr_flit_cnt);
                        break;
                case IB_WR_RDMA_READ:
+                case IB_WR_RDMA_READ_WITH_INV:
                        t3_wr_opcode = T3_WR_READ;
                        t3_wr_flags = 0; /* T3 reads are always signaled */
-                        err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt);
+                        err = build_rdma_read(wqe, wr, &t3_wr_flit_cnt);
                        if (err)
                                break;
                        sqp->read_len = wqe->read.local_len;
                        if (!qhp->wq.oldest_read)
                                qhp->wq.oldest_read = sqp;
                        break;
+                case IB_WR_FAST_REG_MR:
+                        t3_wr_opcode = T3_WR_FASTREG;
+                        err = build_fastreg(wqe, wr, &t3_wr_flit_cnt,
+                                                 &wr_cnt, &qhp->wq);
+                        break;
+                case IB_WR_LOCAL_INV:
+                        if (wr->send_flags & IB_SEND_FENCE)
+                                t3_wr_flags |= T3_LOCAL_FENCE_FLAG;
+                        t3_wr_opcode = T3_WR_INV_STAG;
+                        err = build_inv_stag(wqe, wr, &t3_wr_flit_cnt);
+                        break;
                default:
                        PDBG("%s post of type=%d TBD!\n", __func__,
                             wr->opcode);
@@ -307,14 +443,15 @@ int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags,
                               Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),
-                               0, t3_wr_flit_cnt);
+                               0, t3_wr_flit_cnt,
+                               (wr_cnt == 1) ? T3_SOPEOP : T3_SOP);
                PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n",
                     __func__, (unsigned long long) wr->wr_id, idx,
                     Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2),
                     sqp->opcode);
                wr = wr->next;
                num_wrs--;
-                ++(qhp->wq.wptr);
+                qhp->wq.wptr += wr_cnt;
                ++(qhp->wq.sq_wptr);
        }
        spin_unlock_irqrestore(&qhp->lock, flag);
@@ -345,21 +482,27 @@ int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
                return -EINVAL;
        }
        while (wr) {
+                if (wr->num_sge > T3_MAX_SGE) {
+                        err = -EINVAL;
+                        *bad_wr = wr;
+                        break;
+                }
                idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
                wqe = (union t3_wr *) (qhp->wq.queue + idx);
                if (num_wrs)
-                        err = iwch_build_rdma_recv(qhp->rhp, wqe, wr);
+                        if (wr->sg_list[0].lkey)
+                                err = build_rdma_recv(qhp, wqe, wr);
+                        else
+                                err = build_zero_stag_recv(qhp, wqe, wr);
                else
                        err = -ENOMEM;
                if (err) {
                        *bad_wr = wr;
                        break;
                }
-                qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] =
-                        wr->wr_id;
                build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG,
                               Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),
-                               0, sizeof(struct t3_receive_wr) >> 3);
+                               0, sizeof(struct t3_receive_wr) >> 3, T3_SOPEOP);
                PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x "
                     "wqe %p \n", __func__, (unsigned long long) wr->wr_id,
                     idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe);
@@ -419,10 +562,10 @@ int iwch_bind_mw(struct ib_qp *qp,
        sgl.lkey = mw_bind->mr->lkey;
        sgl.length = mw_bind->length;
        wqe->bind.reserved = 0;
-        wqe->bind.type = T3_VA_BASED_TO;
+        wqe->bind.type = TPT_VATO;
        /* TBD: check perms */
-        wqe->bind.perms = iwch_ib_to_mwbind_access(mw_bind->mw_access_flags);
+        wqe->bind.perms = iwch_ib_to_tpt_access(mw_bind->mw_access_flags);
        wqe->bind.mr_stag = cpu_to_be32(mw_bind->mr->lkey);
        wqe->bind.mw_stag = cpu_to_be32(mw->rkey);
        wqe->bind.mw_len = cpu_to_be32(mw_bind->length);
@@ -430,7 +573,7 @@ int iwch_bind_mw(struct ib_qp *qp,
        err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size);
        if (err) {
                spin_unlock_irqrestore(&qhp->lock, flag);
-                return err;
+                return err;
        }
        wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;
        sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);
@@ -441,10 +584,9 @@ int iwch_bind_mw(struct ib_qp *qp,
        sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED);
        wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr);
        wqe->bind.mr_pagesz = page_size;
-        wqe->flit[T3_SQ_COOKIE_FLIT] = mw_bind->wr_id;
        build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags,
                       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0,
-                                sizeof(struct t3_bind_mw_wr) >> 3);
+                       sizeof(struct t3_bind_mw_wr) >> 3, T3_SOPEOP);
        ++(qhp->wq.wptr);
        ++(qhp->wq.sq_wptr);
        spin_unlock_irqrestore(&qhp->lock, flag);
@@ -758,7 +900,8 @@ static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp,
        init_attr.qp_dma_size = (1UL << qhp->wq.size_log2);
        init_attr.rqe_count = iwch_rqes_posted(qhp);
        init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0;
-        init_attr.flags |= capable(CAP_NET_BIND_SERVICE) ? PRIV_QP : 0;
+        if (!qhp->ibqp.uobject)
+                init_attr.flags |= PRIV_QP;
        if (peer2peer) {
                init_attr.rtr_type = RTR_READ;
                if (init_attr.ord == 0 && qhp->attr.mpa_attr.initiator)