IB/mlx5: Add DEVX support for memory registration

Add support to register a memory with the firmware via the DEVX
interface.

The driver translates a given user address to ib_umem then it will
register the physical addresses with the firmware and get a unique id
for this registration to be used for this virtual address.

Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

3 files changed, 217 insertions, 1 deletions

diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 9b1804eb9924..162321f486eb 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -24,6 +24,22 @@ struct devx_obj {
         u32                     dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
 };
 
+struct devx_umem {
+        struct mlx5_core_dev            *mdev;
+        struct ib_umem                  *umem;
+        u32                             page_offset;
+        int                             page_shift;
+        int                             ncont;
+        u32                             dinlen;
+        u32                             dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
+};
+
+struct devx_umem_reg_cmd {
+        void                            *in;
+        u32                             inlen;
+        u32                             out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+};
+
 static struct mlx5_ib_ucontext *devx_ufile2uctx(struct ib_uverbs_file *file)
 {
         return to_mucontext(ib_uverbs_get_ucontext(file));
@@ -788,6 +804,181 @@ other_cmd_free:
         return err;
 }
 
+static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
+                         struct uverbs_attr_bundle *attrs,
+                         struct devx_umem *obj)
+{
+        u64 addr;
+        size_t size;
+        int access;
+        int npages;
+        int err;
+        u32 page_mask;
+
+        if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
+            uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN) ||
+            uverbs_copy_from(&access, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS))
+                return -EFAULT;
+
+        err = ib_check_mr_access(access);
+        if (err)
+                return err;
+
+        obj->umem = ib_umem_get(ucontext, addr, size, access, 0);
+        if (IS_ERR(obj->umem))
+                return PTR_ERR(obj->umem);
+
+        mlx5_ib_cont_pages(obj->umem, obj->umem->address,
+                           MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
+                           &obj->page_shift, &obj->ncont, NULL);
+
+        if (!npages) {
+                ib_umem_release(obj->umem);
+                return -EINVAL;
+        }
+
+        page_mask = (1 << obj->page_shift) - 1;
+        obj->page_offset = obj->umem->address & page_mask;
+
+        return 0;
+}
+
+static int devx_umem_reg_cmd_alloc(struct devx_umem *obj,
+                                   struct devx_umem_reg_cmd *cmd)
+{
+        cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
+                    (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
+        cmd->in = kvzalloc(cmd->inlen, GFP_KERNEL);
+        return cmd->in ? 0 : -ENOMEM;
+}
+
+static void devx_umem_reg_cmd_free(struct devx_umem_reg_cmd *cmd)
+{
+        kvfree(cmd->in);
+}
+
+static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
+                                    struct devx_umem *obj,
+                                    struct devx_umem_reg_cmd *cmd)
+{
+        void *umem;
+        __be64 *mtt;
+
+        umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
+        mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
+
+        MLX5_SET(general_obj_in_cmd_hdr, cmd->in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+        MLX5_SET(general_obj_in_cmd_hdr, cmd->in, obj_type, MLX5_OBJ_TYPE_UMEM);
+        MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
+        MLX5_SET(umem, umem, log_page_size, obj->page_shift -
+                                            MLX5_ADAPTER_PAGE_SHIFT);
+        MLX5_SET(umem, umem, page_offset, obj->page_offset);
+        mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
+                             (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
+                             MLX5_IB_MTT_READ);
+}
+
+static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(struct ib_device *ib_dev,
+                                 struct ib_uverbs_file *file,
+                                 struct uverbs_attr_bundle *attrs)
+{
+        struct mlx5_ib_ucontext *c = devx_ufile2uctx(file);
+        struct mlx5_ib_dev *dev = to_mdev(ib_dev);
+        struct devx_umem_reg_cmd cmd;
+        struct devx_umem *obj;
+        struct ib_uobject *uobj;
+        u32 obj_id;
+        int err;
+
+        if (!c->devx_uid)
+                return -EPERM;
+
+        uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
+        obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
+        if (!obj)
+                return -ENOMEM;
+
+        err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
+        if (err)
+                goto err_obj_free;
+
+        err = devx_umem_reg_cmd_alloc(obj, &cmd);
+        if (err)
+                goto err_umem_release;
+
+        devx_umem_reg_cmd_build(dev, obj, &cmd);
+
+        MLX5_SET(general_obj_in_cmd_hdr, cmd.in, uid, c->devx_uid);
+        err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
+                            sizeof(cmd.out));
+        if (err)
+                goto err_umem_reg_cmd_free;
+
+        obj->mdev = dev->mdev;
+        uobj->object = obj;
+        devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
+        err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id));
+        if (err)
+                goto err_umem_destroy;
+
+        devx_umem_reg_cmd_free(&cmd);
+
+        return 0;
+
+err_umem_destroy:
+        mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out));
+err_umem_reg_cmd_free:
+        devx_umem_reg_cmd_free(&cmd);
+err_umem_release:
+        ib_umem_release(obj->umem);
+err_obj_free:
+        kfree(obj);
+        return err;
+}
+
+static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_DEREG)(struct ib_device *ib_dev,
+                                   struct ib_uverbs_file *file,
+                                   struct uverbs_attr_bundle *attrs)
+{
+        return 0;
+}
+
+static int devx_umem_cleanup(struct ib_uobject *uobject,
+                             enum rdma_remove_reason why)
+{
+        struct devx_umem *obj = uobject->object;
+        u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+        int err;
+
+        err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
+        if (err && why == RDMA_REMOVE_DESTROY)
+                return err;
+
+        ib_umem_release(obj->umem);
+        kfree(obj);
+        return 0;
+}
+
+static DECLARE_UVERBS_NAMED_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG,
+        &UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
+                         MLX5_IB_OBJECT_DEVX_UMEM,
+                         UVERBS_ACCESS_NEW,
+                         UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+        &UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR, UVERBS_ATTR_TYPE(u64),
+                            UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+        &UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN, UVERBS_ATTR_TYPE(u64),
+                            UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+        &UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, UVERBS_ATTR_TYPE(u32),
+                            UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+        &UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, UVERBS_ATTR_TYPE(u32),
+                             UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)));
+
+static DECLARE_UVERBS_NAMED_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG,
+        &UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
+                         MLX5_IB_OBJECT_DEVX_UMEM,
+                         UVERBS_ACCESS_DESTROY,
+                         UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)));
+
 static DECLARE_UVERBS_NAMED_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR,
         &UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX, UVERBS_ATTR_TYPE(u32),
                             UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
@@ -868,6 +1059,12 @@ static DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
                 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
                 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY));
 
+static DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
+        &UVERBS_TYPE_ALLOC_IDR(0, devx_umem_cleanup),
+        &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
+        &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
+
 static DECLARE_UVERBS_OBJECT_TREE(devx_objects,
         &UVERBS_OBJECT(MLX5_IB_OBJECT_DEVX),
-        &UVERBS_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ));
+        &UVERBS_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ),
+        &UVERBS_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM));
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index ac24ed87c67e..00b539303f5d 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -82,6 +82,7 @@ enum {
 
 enum {
         MLX5_OBJ_TYPE_UCTX = 0x0004,
+        MLX5_OBJ_TYPE_UMEM = 0x0005,
 };
 
 enum {
diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
index 1252695cd94b..dbc549a7bf50 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -89,9 +89,27 @@ enum mlx5_ib_devx_obj_methods {
         MLX5_IB_METHOD_DEVX_OBJ_QUERY,
 };
 
+enum mlx5_ib_devx_umem_reg_attrs {
+        MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+        MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
+        MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
+        MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
+        MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
+};
+
+enum mlx5_ib_devx_umem_dereg_attrs {
+        MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+enum mlx5_ib_devx_umem_methods {
+        MLX5_IB_METHOD_DEVX_UMEM_REG = (1U << UVERBS_ID_NS_SHIFT),
+        MLX5_IB_METHOD_DEVX_UMEM_DEREG,
+};
+
 enum mlx5_ib_devx_objects {
         MLX5_IB_OBJECT_DEVX = (1U << UVERBS_ID_NS_SHIFT),
         MLX5_IB_OBJECT_DEVX_OBJ,
+        MLX5_IB_OBJECT_DEVX_UMEM,
 };
 
 #endif