6 files changed, 106 insertions, 54 deletions

diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 62bb6b49dc1d..8ae4f896cb41 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -32,6 +32,7 @@
 
 #include <linux/kref.h>
 #include <rdma/ib_umem.h>
+#include <rdma/ib_user_verbs.h>
 #include "mlx5_ib.h"
 #include "user.h"
 
@@ -602,14 +603,24 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
                           int *cqe_size, int *index, int *inlen)
 {
         struct mlx5_ib_create_cq ucmd;
+        size_t ucmdlen;
         int page_shift;
         int npages;
         int ncont;
         int err;
 
-        if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
+        ucmdlen =
+                (udata->inlen - sizeof(struct ib_uverbs_cmd_hdr) <
+                 sizeof(ucmd)) ? (sizeof(ucmd) -
+                                  sizeof(ucmd.reserved)) : sizeof(ucmd);
+
+        if (ib_copy_from_udata(&ucmd, udata, ucmdlen))
                 return -EFAULT;
 
+        if (ucmdlen == sizeof(ucmd) &&
+            ucmd.reserved != 0)
+                return -EINVAL;
+
         if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128)
                 return -EINVAL;
 
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 50541586e0a6..f2ccf1a5a291 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -264,8 +264,6 @@ struct mlx5_ib_mr {
         __be64                  *pas;
         dma_addr_t              dma;
         int                     npages;
-        struct completion       done;
-        enum ib_wc_status       status;
         struct mlx5_ib_dev     *dev;
         struct mlx5_create_mkey_mbox_out out;
         struct mlx5_core_sig_ctx    *sig;
@@ -277,6 +275,17 @@ struct mlx5_ib_fast_reg_page_list {
         dma_addr_t                      map;
 };
 
+struct mlx5_ib_umr_context {
+        enum ib_wc_status       status;
+        struct completion       done;
+};
+
+static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
+{
+        context->status = -1;
+        init_completion(&context->done);
+}
+
 struct umr_common {
         struct ib_pd    *pd;
         struct ib_cq    *cq;
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 81392b26d078..afa873bd028e 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -73,6 +73,8 @@ static void reg_mr_callback(int status, void *context)
         struct mlx5_cache_ent *ent = &cache->ent[c];
         u8 key;
         unsigned long flags;
+        struct mlx5_mr_table *table = &dev->mdev.priv.mr_table;
+        int err;
 
         spin_lock_irqsave(&ent->lock, flags);
         ent->pending--;
@@ -107,6 +109,13 @@ static void reg_mr_callback(int status, void *context)
         ent->cur++;
         ent->size++;
         spin_unlock_irqrestore(&ent->lock, flags);
+
+        write_lock_irqsave(&table->lock, flags);
+        err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmr.key),
+                                &mr->mmr);
+        if (err)
+                pr_err("Error inserting to mr tree. 0x%x\n", -err);
+        write_unlock_irqrestore(&table->lock, flags);
 }
 
 static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
@@ -699,7 +708,7 @@ static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
 
 void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context)
 {
-        struct mlx5_ib_mr *mr;
+        struct mlx5_ib_umr_context *context;
         struct ib_wc wc;
         int err;
 
@@ -712,9 +721,9 @@ void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context)
                 if (err == 0)
                         break;
 
-                mr = (struct mlx5_ib_mr *)(unsigned long)wc.wr_id;
-                mr->status = wc.status;
-                complete(&mr->done);
+                context = (struct mlx5_ib_umr_context *) (unsigned long) wc.wr_id;
+                context->status = wc.status;
+                complete(&context->done);
         }
         ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
 }
@@ -726,11 +735,12 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
         struct mlx5_ib_dev *dev = to_mdev(pd->device);
         struct device *ddev = dev->ib_dev.dma_device;
         struct umr_common *umrc = &dev->umrc;
+        struct mlx5_ib_umr_context umr_context;
         struct ib_send_wr wr, *bad;
         struct mlx5_ib_mr *mr;
         struct ib_sge sg;
         int size = sizeof(u64) * npages;
-        int err;
+        int err = 0;
         int i;
 
         for (i = 0; i < 1; i++) {
@@ -751,7 +761,7 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
         mr->pas = kmalloc(size + MLX5_UMR_ALIGN - 1, GFP_KERNEL);
         if (!mr->pas) {
                 err = -ENOMEM;
-                goto error;
+                goto free_mr;
         }
 
         mlx5_ib_populate_pas(dev, umem, page_shift,
@@ -760,44 +770,46 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
         mr->dma = dma_map_single(ddev, mr_align(mr->pas, MLX5_UMR_ALIGN), size,
                                  DMA_TO_DEVICE);
         if (dma_mapping_error(ddev, mr->dma)) {
-                kfree(mr->pas);
                 err = -ENOMEM;
-                goto error;
+                goto free_pas;
         }
 
         memset(&wr, 0, sizeof(wr));
-        wr.wr_id = (u64)(unsigned long)mr;
+        wr.wr_id = (u64)(unsigned long)&umr_context;
         prep_umr_reg_wqe(pd, &wr, &sg, mr->dma, npages, mr->mmr.key, page_shift, virt_addr, len, access_flags);
 
-        /* We serialize polls so one process does not kidnap another's
-         * completion. This is not a problem since wr is completed in
-         * around 1 usec
-         */
+        mlx5_ib_init_umr_context(&umr_context);
         down(&umrc->sem);
-        init_completion(&mr->done);
         err = ib_post_send(umrc->qp, &wr, &bad);
         if (err) {
                 mlx5_ib_warn(dev, "post send failed, err %d\n", err);
-                up(&umrc->sem);
-                goto error;
+                goto unmap_dma;
+        } else {
+                wait_for_completion(&umr_context.done);
+                if (umr_context.status != IB_WC_SUCCESS) {
+                        mlx5_ib_warn(dev, "reg umr failed\n");
+                        err = -EFAULT;
+                }
         }
-        wait_for_completion(&mr->done);
-        up(&umrc->sem);
 
+        mr->mmr.iova = virt_addr;
+        mr->mmr.size = len;
+        mr->mmr.pd = to_mpd(pd)->pdn;
+
+unmap_dma:
+        up(&umrc->sem);
         dma_unmap_single(ddev, mr->dma, size, DMA_TO_DEVICE);
+
+free_pas:
         kfree(mr->pas);
 
-        if (mr->status != IB_WC_SUCCESS) {
-                mlx5_ib_warn(dev, "reg umr failed\n");
-                err = -EFAULT;
-                goto error;
+free_mr:
+        if (err) {
+                free_cached_mr(dev, mr);
+                return ERR_PTR(err);
         }
 
         return mr;
-
-error:
-        free_cached_mr(dev, mr);
-        return ERR_PTR(err);
 }
 
 static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr,
@@ -926,24 +938,26 @@ error:
 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 {
         struct umr_common *umrc = &dev->umrc;
+        struct mlx5_ib_umr_context umr_context;
         struct ib_send_wr wr, *bad;
         int err;
 
         memset(&wr, 0, sizeof(wr));
-        wr.wr_id = (u64)(unsigned long)mr;
+        wr.wr_id = (u64)(unsigned long)&umr_context;
         prep_umr_unreg_wqe(dev, &wr, mr->mmr.key);
 
+        mlx5_ib_init_umr_context(&umr_context);
         down(&umrc->sem);
-        init_completion(&mr->done);
         err = ib_post_send(umrc->qp, &wr, &bad);
         if (err) {
                 up(&umrc->sem);
                 mlx5_ib_dbg(dev, "err %d\n", err);
                 goto error;
+        } else {
+                wait_for_completion(&umr_context.done);
+                up(&umrc->sem);
         }
-        wait_for_completion(&mr->done);
-        up(&umrc->sem);
-        if (mr->status != IB_WC_SUCCESS) {
+        if (umr_context.status != IB_WC_SUCCESS) {
                 mlx5_ib_warn(dev, "unreg umr failed\n");
                 err = -EFAULT;
                 goto error;
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index dc930ed21eca..d13ddf1c0033 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -574,6 +574,10 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
         uar_index = uuarn_to_uar_index(&context->uuari, uuarn);
         mlx5_ib_dbg(dev, "uuarn 0x%x, uar_index 0x%x\n", uuarn, uar_index);
 
+        qp->rq.offset = 0;
+        qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
+        qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
+
         err = set_user_buf_size(dev, qp, &ucmd);
         if (err)
                 goto err_uuar;
@@ -2078,6 +2082,7 @@ static int mlx5_set_bsf(struct ib_mr *sig_mr,
         struct ib_sig_domain *wire = &sig_attrs->wire;
         int ret, selector;
 
+        memset(bsf, 0, sizeof(*bsf));
         switch (sig_attrs->mem.sig_type) {
         case IB_SIG_TYPE_T10_DIF:
                 if (sig_attrs->wire.sig_type != IB_SIG_TYPE_T10_DIF)
@@ -2090,9 +2095,11 @@ static int mlx5_set_bsf(struct ib_mr *sig_mr,
                         /* Same block structure */
                         basic->bsf_size_sbs = 1 << 4;
                         if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
-                                basic->wire.copy_byte_mask = 0xff;
-                        else
-                                basic->wire.copy_byte_mask = 0x3f;
+                                basic->wire.copy_byte_mask |= 0xc0;
+                        if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
+                                basic->wire.copy_byte_mask |= 0x30;
+                        if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
+                                basic->wire.copy_byte_mask |= 0x0f;
                 } else
                         basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
 
@@ -2131,9 +2138,13 @@ static int set_sig_data_segment(struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
         int ret;
         int wqe_size;
 
-        if (!wr->wr.sig_handover.prot) {
+        if (!wr->wr.sig_handover.prot ||
+            (data_key == wr->wr.sig_handover.prot->lkey &&
+             data_va == wr->wr.sig_handover.prot->addr &&
+             data_len == wr->wr.sig_handover.prot->length)) {
                 /**
                  * Source domain doesn't contain signature information
+                 * or data and protection are interleaved in memory.
                  * So need construct:
                  *                  ------------------
                  *                 |     data_klm     |
@@ -2187,23 +2198,13 @@ static int set_sig_data_segment(struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
                 data_sentry->bcount = cpu_to_be16(block_size);
                 data_sentry->key = cpu_to_be32(data_key);
                 data_sentry->va = cpu_to_be64(data_va);
+                data_sentry->stride = cpu_to_be16(block_size);
+
                 prot_sentry->bcount = cpu_to_be16(prot_size);
                 prot_sentry->key = cpu_to_be32(prot_key);
+                prot_sentry->va = cpu_to_be64(prot_va);
+                prot_sentry->stride = cpu_to_be16(prot_size);
 
-                if (prot_key == data_key && prot_va == data_va) {
-                        /**
-                         * The data and protection are interleaved
-                         * in a single memory region
-                         **/
-                        prot_sentry->va = cpu_to_be64(data_va + block_size);
-                        prot_sentry->stride = cpu_to_be16(block_size + prot_size);
-                        data_sentry->stride = prot_sentry->stride;
-                } else {
-                        /* The data and protection are two different buffers */
-                        prot_sentry->va = cpu_to_be64(prot_va);
-                        data_sentry->stride = cpu_to_be16(block_size);
-                        prot_sentry->stride = cpu_to_be16(prot_size);
-                }
                 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
                                  sizeof(*prot_sentry), 64);
         }
@@ -2275,7 +2276,10 @@ static int set_sig_umr_wr(struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
 
         /* length of the protected region, data + protection */
         region_len = wr->sg_list->length;
-        if (wr->wr.sig_handover.prot)
+        if (wr->wr.sig_handover.prot &&
+            (wr->wr.sig_handover.prot->lkey != wr->sg_list->lkey  ||
+             wr->wr.sig_handover.prot->addr != wr->sg_list->addr  ||
+             wr->wr.sig_handover.prot->length != wr->sg_list->length))
                 region_len += wr->wr.sig_handover.prot->length;
 
         /**
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index 210b3eaf188a..384af6dec5eb 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -35,6 +35,7 @@
 #include <linux/mlx5/srq.h>
 #include <linux/slab.h>
 #include <rdma/ib_umem.h>
+#include <rdma/ib_user_verbs.h>
 
 #include "mlx5_ib.h"
 #include "user.h"
@@ -78,16 +79,27 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
 {
         struct mlx5_ib_dev *dev = to_mdev(pd->device);
         struct mlx5_ib_create_srq ucmd;
+        size_t ucmdlen;
         int err;
         int npages;
         int page_shift;
         int ncont;
         u32 offset;
 
-        if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
+        ucmdlen =
+                (udata->inlen - sizeof(struct ib_uverbs_cmd_hdr) <
+                 sizeof(ucmd)) ? (sizeof(ucmd) -
+                                  sizeof(ucmd.reserved)) : sizeof(ucmd);
+
+        if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) {
                 mlx5_ib_dbg(dev, "failed copy udata\n");
                 return -EFAULT;
         }
+
+        if (ucmdlen == sizeof(ucmd) &&
+            ucmd.reserved != 0)
+                return -EINVAL;
+
         srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE);
 
         srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size,
diff --git a/drivers/infiniband/hw/mlx5/user.h b/drivers/infiniband/hw/mlx5/user.h
index 0f4f8e42a17f..d0ba264ac1ed 100644
--- a/drivers/infiniband/hw/mlx5/user.h
+++ b/drivers/infiniband/hw/mlx5/user.h
@@ -91,6 +91,7 @@ struct mlx5_ib_create_cq {
         __u64   buf_addr;
         __u64   db_addr;
         __u32   cqe_size;
+        __u32   reserved; /* explicit padding (optional on i386) */
 };
 
 struct mlx5_ib_create_cq_resp {
@@ -109,6 +110,7 @@ struct mlx5_ib_create_srq {
         __u64   buf_addr;
         __u64   db_addr;
         __u32   flags;
+        __u32   reserved; /* explicit padding (optional on i386) */
 };
 
 struct mlx5_ib_create_srq_resp {