iser-target: introduce fast memory registration mode (FRWR)

This model was introduced in 00f7ec36c "RDMA/core: Add memory
management extensions support" and works when the IB device
supports the IB_DEVICE_MEM_MGT_EXTENSIONS capability.

Upon creating the isert device, ib_isert will test whether the HCA
supports FRWR. If supported then set the flag and assign
function pointers that handle fast registration and deregistration
of appropriate resources (fast_reg descriptors).

When new connection coming in, ib_isert will check frwr flag and
create frwr resouces, if fail to do it will switch back to
old model of using global dma key and turn off the frwr support.

Registration is done using posting IB_WR_FAST_REG_MR to the QP and
invalidations using posting IB_WR_LOCAL_INV.

Signed-off-by: Vu Pham <vu@mellanox.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>

2 files changed, 371 insertions, 13 deletions

diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 4c3d66009237..51c3bed6c12b 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -45,6 +45,11 @@ isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
 static int
 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
                struct isert_rdma_wr *wr);
+static void
+isert_unreg_rdma_frwr(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
+static int
+isert_reg_rdma_frwr(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
+                    struct isert_rdma_wr *wr);
 
 static void
 isert_qp_event_callback(struct ib_event *e, void *context)
@@ -85,14 +90,8 @@ isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
 {
         struct isert_device *device = isert_conn->conn_device;
         struct ib_qp_init_attr attr;
-        struct ib_device_attr devattr;
         int ret, index, min_index = 0;
 
-        memset(&devattr, 0, sizeof(struct ib_device_attr));
-        ret = isert_query_device(cma_id->device, &devattr);
-        if (ret)
-                return ret;
-
         mutex_lock(&device_list_mutex);
         for (index = 0; index < device->cqs_used; index++)
                 if (device->cq_active_qps[index] <
@@ -113,7 +112,7 @@ isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
          * FIXME: Use devattr.max_sge - 2 for max_send_sge as
          * work-around for RDMA_READ..
          */
-        attr.cap.max_send_sge = devattr.max_sge - 2;
+        attr.cap.max_send_sge = device->dev_attr.max_sge - 2;
         isert_conn->max_sge = attr.cap.max_send_sge;
 
         attr.cap.max_recv_sge = 1;
@@ -215,18 +214,31 @@ isert_create_device_ib_res(struct isert_device *device)
 {
         struct ib_device *ib_dev = device->ib_device;
         struct isert_cq_desc *cq_desc;
+        struct ib_device_attr *dev_attr;
         int ret = 0, i, j;
 
+        dev_attr = &device->dev_attr;
+        ret = isert_query_device(ib_dev, dev_attr);
+        if (ret)
+                return ret;
+
         /* asign function handlers */
-        device->reg_rdma_mem = isert_map_rdma;
-        device->unreg_rdma_mem = isert_unmap_cmd;
+        if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {
+                device->use_frwr = 1;
+                device->reg_rdma_mem = isert_reg_rdma_frwr;
+                device->unreg_rdma_mem = isert_unreg_rdma_frwr;
+        } else {
+                device->use_frwr = 0;
+                device->reg_rdma_mem = isert_map_rdma;
+                device->unreg_rdma_mem = isert_unmap_cmd;
+        }
 
         device->cqs_used = min_t(int, num_online_cpus(),
                                  device->ib_device->num_comp_vectors);
         device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used);
-        pr_debug("Using %d CQs, device %s supports %d vectors\n",
+        pr_debug("Using %d CQs, device %s supports %d vectors support FRWR %d\n",
                  device->cqs_used, device->ib_device->name,
-                 device->ib_device->num_comp_vectors);
+                 device->ib_device->num_comp_vectors, device->use_frwr);
         device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) *
                                 device->cqs_used, GFP_KERNEL);
         if (!device->cq_desc) {
@@ -372,6 +384,85 @@ isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id)
         return device;
 }
 
+static void
+isert_conn_free_frwr_pool(struct isert_conn *isert_conn)
+{
+        struct fast_reg_descriptor *fr_desc, *tmp;
+        int i = 0;
+
+        if (list_empty(&isert_conn->conn_frwr_pool))
+                return;
+
+        pr_debug("Freeing conn %p frwr pool", isert_conn);
+
+        list_for_each_entry_safe(fr_desc, tmp,
+                                 &isert_conn->conn_frwr_pool, list) {
+                list_del(&fr_desc->list);
+                ib_free_fast_reg_page_list(fr_desc->data_frpl);
+                ib_dereg_mr(fr_desc->data_mr);
+                kfree(fr_desc);
+                ++i;
+        }
+
+        if (i < isert_conn->conn_frwr_pool_size)
+                pr_warn("Pool still has %d regions registered\n",
+                        isert_conn->conn_frwr_pool_size - i);
+}
+
+static int
+isert_conn_create_frwr_pool(struct isert_conn *isert_conn)
+{
+        struct fast_reg_descriptor *fr_desc;
+        struct isert_device *device = isert_conn->conn_device;
+        int i, ret;
+
+        INIT_LIST_HEAD(&isert_conn->conn_frwr_pool);
+        isert_conn->conn_frwr_pool_size = 0;
+        for (i = 0; i < ISCSI_DEF_XMIT_CMDS_MAX; i++) {
+                fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL);
+                if (!fr_desc) {
+                        pr_err("Failed to allocate fast_reg descriptor\n");
+                        ret = -ENOMEM;
+                        goto err;
+                }
+
+                fr_desc->data_frpl =
+                        ib_alloc_fast_reg_page_list(device->ib_device,
+                                                    ISCSI_ISER_SG_TABLESIZE);
+                if (IS_ERR(fr_desc->data_frpl)) {
+                        pr_err("Failed to allocate fr_pg_list err=%ld\n",
+                               PTR_ERR(fr_desc->data_frpl));
+                        ret = PTR_ERR(fr_desc->data_frpl);
+                        goto err;
+                }
+
+                fr_desc->data_mr = ib_alloc_fast_reg_mr(device->dev_pd,
+                                        ISCSI_ISER_SG_TABLESIZE);
+                if (IS_ERR(fr_desc->data_mr)) {
+                        pr_err("Failed to allocate frmr err=%ld\n",
+                               PTR_ERR(fr_desc->data_mr));
+                        ret = PTR_ERR(fr_desc->data_mr);
+                        ib_free_fast_reg_page_list(fr_desc->data_frpl);
+                        goto err;
+                }
+                pr_debug("Create fr_desc %p page_list %p\n",
+                         fr_desc, fr_desc->data_frpl->page_list);
+
+                fr_desc->valid = true;
+                list_add_tail(&fr_desc->list, &isert_conn->conn_frwr_pool);
+                isert_conn->conn_frwr_pool_size++;
+        }
+
+        pr_debug("Creating conn %p frwr pool size=%d",
+                 isert_conn, isert_conn->conn_frwr_pool_size);
+
+        return 0;
+
+err:
+        isert_conn_free_frwr_pool(isert_conn);
+        return ret;
+}
+
 static int
 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 {
@@ -398,6 +489,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)