RPC/RDMA: support FRMR client memory registration.

Configure, detect and use "fastreg" support from IB/iWARP verbs
layer to perform RPC/RDMA memory registration.

Make FRMR the default memreg mode (will fall back if not supported
by the selected RDMA adapter).

This allows full and optimal operation over the cxgb3 adapter, and others.

Signed-off-by: Tom Talpey <talpey@netapp.com>
Acked-by: Tom Tucker <tom@opengridcomputing.com>
Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

2 files changed, 167 insertions, 6 deletions

diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
index a564c1a39ec5..89970b0a4cc9 100644
--- a/net/sunrpc/xprtrdma/transport.c
+++ b/net/sunrpc/xprtrdma/transport.c
@@ -70,11 +70,7 @@ static unsigned int xprt_rdma_slot_table_entries = RPCRDMA_DEF_SLOT_TABLE;
 static unsigned int xprt_rdma_max_inline_read = RPCRDMA_DEF_INLINE;
 static unsigned int xprt_rdma_max_inline_write = RPCRDMA_DEF_INLINE;
 static unsigned int xprt_rdma_inline_write_padding;
-#if !RPCRDMA_PERSISTENT_REGISTRATION
-static unsigned int xprt_rdma_memreg_strategy = RPCRDMA_REGISTER; /* FMR? */
-#else
-static unsigned int xprt_rdma_memreg_strategy = RPCRDMA_ALLPHYSICAL;
-#endif
+static unsigned int xprt_rdma_memreg_strategy = RPCRDMA_FRMR;
 
 #ifdef RPC_DEBUG
 
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 0f3b43148b7f..39a165202d8f 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -488,6 +488,26 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
 #endif
                 }
                 break;
+        case RPCRDMA_FRMR:
+                /* Requires both frmr reg and local dma lkey */
+                if ((devattr.device_cap_flags &
+                     (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) !=
+                    (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) {
+#if RPCRDMA_PERSISTENT_REGISTRATION
+                        dprintk("RPC:       %s: FRMR registration "
+                                "specified but not supported by adapter, "
+                                "using riskier RPCRDMA_ALLPHYSICAL\n",
+                                __func__);
+                        memreg = RPCRDMA_ALLPHYSICAL;
+#else
+                        dprintk("RPC:       %s: FRMR registration "
+                                "specified but not supported by adapter, "
+                                "using slower RPCRDMA_REGISTER\n",
+                                __func__);
+                        memreg = RPCRDMA_REGISTER;
+#endif
+                }
+                break;
         }
 
         /*
@@ -501,6 +521,7 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
         switch (memreg) {
         case RPCRDMA_BOUNCEBUFFERS:
         case RPCRDMA_REGISTER:
+        case RPCRDMA_FRMR:
                 break;
 #if RPCRDMA_PERSISTENT_REGISTRATION
         case RPCRDMA_ALLPHYSICAL:
@@ -602,6 +623,12 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
         ep->rep_attr.srq = NULL;
         ep->rep_attr.cap.max_send_wr = cdata->max_requests;
         switch (ia->ri_memreg_strategy) {
+        case RPCRDMA_FRMR:
+                /* Add room for frmr register and invalidate WRs */
+                ep->rep_attr.cap.max_send_wr *= 3;
+                if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr)
+                        return -EINVAL;
+                break;
         case RPCRDMA_MEMWINDOWS_ASYNC:
         case RPCRDMA_MEMWINDOWS:
                 /* Add room for mw_binds+unbinds - overkill! */
@@ -684,6 +711,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
                 break;
         case RPCRDMA_MTHCAFMR:
         case RPCRDMA_REGISTER:
+        case RPCRDMA_FRMR:
                 ep->rep_remote_cma.responder_resources = cdata->max_requests *
                                 (RPCRDMA_MAX_DATA_SEGS / 8);
                 break;
@@ -935,7 +963,7 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
          *   2.  arrays of struct rpcrdma_req to fill in pointers
          *   3.  array of struct rpcrdma_rep for replies
          *   4.  padding, if any
-         *   5.  mw's or fmr's, if any
+         *   5.  mw's, fmr's or frmr's, if any
          * Send/recv buffers in req/rep need to be registered
          */
 
@@ -943,6 +971,10 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
                 (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *));
         len += cdata->padding;
         switch (ia->ri_memreg_strategy) {
+        case RPCRDMA_FRMR:
+                len += buf->rb_max_requests * RPCRDMA_MAX_SEGS *
+                                sizeof(struct rpcrdma_mw);
+                break;
         case RPCRDMA_MTHCAFMR:
                 /* TBD we are perhaps overallocating here */
                 len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS *
@@ -991,6 +1023,30 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
         INIT_LIST_HEAD(&buf->rb_mws);
         r = (struct rpcrdma_mw *)p;
         switch (ia->ri_memreg_strategy) {
+        case RPCRDMA_FRMR:
+                for (i = buf->rb_max_requests * RPCRDMA_MAX_SEGS; i; i--) {
+                        r->r.frmr.fr_mr = ib_alloc_fast_reg_mr(ia->ri_pd,
+                                                         RPCRDMA_MAX_SEGS);
+                        if (IS_ERR(r->r.frmr.fr_mr)) {
+                                rc = PTR_ERR(r->r.frmr.fr_mr);
+                                dprintk("RPC:       %s: ib_alloc_fast_reg_mr"
+                                        " failed %i\n", __func__, rc);
+                                goto out;
+                        }
+                        r->r.frmr.fr_pgl =
+                                ib_alloc_fast_reg_page_list(ia->ri_id->device,
+                                                            RPCRDMA_MAX_SEGS);
+                        if (IS_ERR(r->r.frmr.fr_pgl)) {
+                                rc = PTR_ERR(r->r.frmr.fr_pgl);
+                                dprintk("RPC:       %s: "
+                                        "ib_alloc_fast_reg_page_list "
+                                        "failed %i\n", __func__, rc);
+                                goto out;
+                        }
+                        list_add(&r->mw_list, &buf->rb_mws);
+                        ++r;
+                }
+                break;
         case RPCRDMA_MTHCAFMR:
                 /* TBD we are perhaps overallocating here */
                 for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
@@ -1126,6 +1182,15 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
                                         struct rpcrdma_mw, mw_list);
                                 list_del(&r->mw_list);
                                 switch (ia->ri_memreg_strategy) {
+                                case RPCRDMA_FRMR:
+                                        rc = ib_dereg_mr(r->r.frmr.fr_mr);
+                                        if (rc)
+                                                dprintk("RPC:       %s:"
+                                                        " ib_dereg_mr"
+                                                        " failed %i\n",
+                                                        __func__, rc);
+                                        ib_free_fast_reg_page_list(r->r.frmr.fr_pgl);
+                                        break;
                                 case RPCRDMA_MTHCAFMR:
                                         rc = ib_dealloc_fmr(r->r.fmr);
                                         if (rc)
@@ -1228,6 +1293,7 @@ rpcrdma_buffer_put(struct rpcrdma_req *req)
                 req->rl_reply = NULL;
         }
         switch (ia->ri_memreg_strategy) {
+        case RPCRDMA_FRMR:
         case RPCRDMA_MTHCAFMR:
         case RPCRDMA_MEMWINDOWS_ASYNC:
         case RPCRDMA_MEMWINDOWS:
@@ -1391,6 +1457,96 @@ rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg)
 }
 
 static int
+rpcrdma_register_frmr_external(struct rpcrdma_mr_seg *seg,
+                        int *nsegs, int writing, struct rpcrdma_ia *ia,
+                        struct rpcrdma_xprt *r_xprt)
+{
+        struct rpcrdma_mr_seg *seg1 = seg;
+        struct ib_send_wr frmr_wr, *bad_wr;
+        u8 key;
+        int len, pageoff;
+        int i, rc;
+
+        pageoff = offset_in_page(seg1->mr_offset);
+        seg1->mr_offset -= pageoff;     /* start of page */
+        seg1->mr_len += pageoff;
+        len = -pageoff;
+        if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
+                *nsegs = RPCRDMA_MAX_DATA_SEGS;
+        for (i = 0; i < *nsegs;) {
+                rpcrdma_map_one(ia, seg, writing);
+                seg1->mr_chunk.rl_mw->r.frmr.fr_pgl->page_list[i] = seg->mr_dma;
+                len += seg->mr_len;
+                ++seg;
+                ++i;
+                /* Check for holes */
+                if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
+                    offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
+                        break;
+        }
+        dprintk("RPC:       %s: Using frmr %p to map %d segments\n",
+                __func__, seg1->mr_chunk.rl_mw, i);
+
+        /* Bump the key */
+        key = (u8)(seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey & 0x000000FF);
+        ib_update_fast_reg_key(seg1->mr_chunk.rl_mw->r.frmr.fr_mr, ++key);
+
+        /* Prepare FRMR WR */
+        memset(&frmr_wr, 0, sizeof frmr_wr);
+        frmr_wr.opcode = IB_WR_FAST_REG_MR;
+        frmr_wr.send_flags = 0;                 /* unsignaled */
+        frmr_wr.wr.fast_reg.iova_start = (unsigned long)seg1->mr_dma;
+        frmr_wr.wr.fast_reg.page_list = seg1->mr_chunk.rl_mw->r.frmr.fr_pgl;
+        frmr_wr.wr.fast_reg.page_list_len = i;
+        frmr_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
+        frmr_wr.wr.fast_reg.length = i << PAGE_SHIFT;
+        frmr_wr.wr.fast_reg.access_flags = (writing ?
+                                IB_ACCESS_REMOTE_WRITE : IB_ACCESS_REMOTE_READ);
+        frmr_wr.wr.fast_reg.rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
+        DECR_CQCOUNT(&r_xprt->rx_ep);
+
+        rc = ib_post_send(ia->ri_id->qp, &frmr_wr, &bad_wr);
+
+        if (rc) {
+                dprintk("RPC:       %s: failed ib_post_send for register,"
+                        " status %i\n", __func__, rc);
+                while (i--)
+                        rpcrdma_unmap_one(ia, --seg);
+        } else {
+                seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
+                seg1->mr_base = seg1->mr_dma + pageoff;
+                seg1->mr_nsegs = i;
+                seg1->mr_len = len;
+        }
+        *nsegs = i;
+        return rc;
+}
+
+static int
+rpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg,
+                        struct rpcrdma_ia *ia, struct rpcrdma_xprt *r_xprt)
+{
+        struct rpcrdma_mr_seg *seg1 = seg;
+        struct ib_send_wr invalidate_wr, *bad_wr;
+        int rc;
+
+        while (seg1->mr_nsegs--)
+                rpcrdma_unmap_one(ia, seg++);
+
+        memset(&invalidate_wr, 0, sizeof invalidate_wr);
+        invalidate_wr.opcode = IB_WR_LOCAL_INV;
+        invalidate_wr.send_flags = 0;                   /* unsignaled */
+        invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
+        DECR_CQCOUNT(&r_xprt->rx_ep);
+
+        rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr);
+        if (rc)
+                dprintk("RPC:       %s: failed ib_post_send for invalidate,"
+                        " status %i\n", __func__, rc);
+        return rc;
+}
+
+static int
 rpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg,
                         int *nsegs, int writing, struct rpcrdma_ia *ia)
 {
@@ -1600,6 +1756,11 @@ rpcrdma_register_external(struct rpcrdma_mr_seg *seg,
                 break;
 #endif
 
+        /* Registration using frmr registration */
+        case RPCRDMA_FRMR:
+                rc = rpcrdma_register_frmr_external(seg, &nsegs, writing, ia, r_xprt);
+                break;
+
         /* Registration using fmr memory registration */
         case RPCRDMA_MTHCAFMR:
                 rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia);
@@ -1639,6 +1800,10 @@ rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg,
                 break;
 #endif
 
+        case RPCRDMA_FRMR:
+                rc = rpcrdma_deregister_frmr_external(seg, ia, r_xprt);
+                break;
+
         case RPCRDMA_MTHCAFMR:
                 rc = rpcrdma_deregister_fmr_external(seg, ia);
                 break;