aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/infiniband
diff options
context:
space:
mode:
authorNicholas Bellinger <nab@linux-iscsi.org>2014-03-13 15:01:58 -0400
committerNicholas Bellinger <nab@linux-iscsi.org>2014-03-13 15:01:58 -0400
commit5aad2145ac42a12ae8bf8383d2e0319c172d99a7 (patch)
tree2c4a38a9e13222aed73b14c05c159955a88c6b1f /drivers/infiniband
parentfa389e220254c69ffae0d403eac4146171062d08 (diff)
parent2dea909444c294f55316c068906945ef38980ef3 (diff)
Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband into for-next
Diffstat (limited to 'drivers/infiniband')
-rw-r--r--drivers/infiniband/core/umem.c120
-rw-r--r--drivers/infiniband/core/verbs.c47
-rw-r--r--drivers/infiniband/hw/amso1100/c2_provider.c23
-rw-r--r--drivers/infiniband/hw/cxgb3/iwch_provider.c19
-rw-r--r--drivers/infiniband/hw/cxgb4/mem.c39
-rw-r--r--drivers/infiniband/hw/ehca/ehca_classes.h2
-rw-r--r--drivers/infiniband/hw/ehca/ehca_mrmw.c245
-rw-r--r--drivers/infiniband/hw/ipath/ipath_mr.c39
-rw-r--r--drivers/infiniband/hw/mlx4/doorbell.c4
-rw-r--r--drivers/infiniband/hw/mlx4/mr.c39
-rw-r--r--drivers/infiniband/hw/mlx5/cq.c62
-rw-r--r--drivers/infiniband/hw/mlx5/doorbell.c4
-rw-r--r--drivers/infiniband/hw/mlx5/main.c12
-rw-r--r--drivers/infiniband/hw/mlx5/mem.c80
-rw-r--r--drivers/infiniband/hw/mlx5/mlx5_ib.h14
-rw-r--r--drivers/infiniband/hw/mlx5/mr.c157
-rw-r--r--drivers/infiniband/hw/mlx5/qp.c540
-rw-r--r--drivers/infiniband/hw/mthca/mthca_provider.c42
-rw-r--r--drivers/infiniband/hw/nes/nes_verbs.c253
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.c66
-rw-r--r--drivers/infiniband/hw/qib/qib_mr.c14
21 files changed, 1236 insertions, 585 deletions
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index a84112322071..a3a2e9c1639b 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -42,29 +42,29 @@
42 42
43#include "uverbs.h" 43#include "uverbs.h"
44 44
45#define IB_UMEM_MAX_PAGE_CHUNK \
46 ((PAGE_SIZE - offsetof(struct ib_umem_chunk, page_list)) / \
47 ((void *) &((struct ib_umem_chunk *) 0)->page_list[1] - \
48 (void *) &((struct ib_umem_chunk *) 0)->page_list[0]))
49 45
50static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty) 46static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)
51{ 47{
52 struct ib_umem_chunk *chunk, *tmp; 48 struct scatterlist *sg;
49 struct page *page;
53 int i; 50 int i;
54 51
55 list_for_each_entry_safe(chunk, tmp, &umem->chunk_list, list) { 52 if (umem->nmap > 0)
56 ib_dma_unmap_sg(dev, chunk->page_list, 53 ib_dma_unmap_sg(dev, umem->sg_head.sgl,
57 chunk->nents, DMA_BIDIRECTIONAL); 54 umem->nmap,
58 for (i = 0; i < chunk->nents; ++i) { 55 DMA_BIDIRECTIONAL);
59 struct page *page = sg_page(&chunk->page_list[i]);
60 56
61 if (umem->writable && dirty) 57 for_each_sg(umem->sg_head.sgl, sg, umem->npages, i) {
62 set_page_dirty_lock(page);
63 put_page(page);
64 }
65 58
66 kfree(chunk); 59 page = sg_page(sg);
60 if (umem->writable && dirty)
61 set_page_dirty_lock(page);
62 put_page(page);
67 } 63 }
64
65 sg_free_table(&umem->sg_head);
66 return;
67
68} 68}
69 69
70/** 70/**
@@ -81,15 +81,15 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
81 struct ib_umem *umem; 81 struct ib_umem *umem;
82 struct page **page_list; 82 struct page **page_list;
83 struct vm_area_struct **vma_list; 83 struct vm_area_struct **vma_list;
84 struct ib_umem_chunk *chunk;
85 unsigned long locked; 84 unsigned long locked;
86 unsigned long lock_limit; 85 unsigned long lock_limit;
87 unsigned long cur_base; 86 unsigned long cur_base;
88 unsigned long npages; 87 unsigned long npages;
89 int ret; 88 int ret;
90 int off;
91 int i; 89 int i;
92 DEFINE_DMA_ATTRS(attrs); 90 DEFINE_DMA_ATTRS(attrs);
91 struct scatterlist *sg, *sg_list_start;
92 int need_release = 0;
93 93
94 if (dmasync) 94 if (dmasync)
95 dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs); 95 dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs);
@@ -97,7 +97,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
97 if (!can_do_mlock()) 97 if (!can_do_mlock())
98 return ERR_PTR(-EPERM); 98 return ERR_PTR(-EPERM);
99 99
100 umem = kmalloc(sizeof *umem, GFP_KERNEL); 100 umem = kzalloc(sizeof *umem, GFP_KERNEL);
101 if (!umem) 101 if (!umem)
102 return ERR_PTR(-ENOMEM); 102 return ERR_PTR(-ENOMEM);
103 103
@@ -117,8 +117,6 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
117 /* We assume the memory is from hugetlb until proved otherwise */ 117 /* We assume the memory is from hugetlb until proved otherwise */
118 umem->hugetlb = 1; 118 umem->hugetlb = 1;
119 119
120 INIT_LIST_HEAD(&umem->chunk_list);
121
122 page_list = (struct page **) __get_free_page(GFP_KERNEL); 120 page_list = (struct page **) __get_free_page(GFP_KERNEL);
123 if (!page_list) { 121 if (!page_list) {
124 kfree(umem); 122 kfree(umem);
@@ -147,7 +145,18 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
147 145
148 cur_base = addr & PAGE_MASK; 146 cur_base = addr & PAGE_MASK;
149 147
150 ret = 0; 148 if (npages == 0) {
149 ret = -EINVAL;
150 goto out;
151 }
152
153 ret = sg_alloc_table(&umem->sg_head, npages, GFP_KERNEL);
154 if (ret)
155 goto out;
156
157 need_release = 1;
158 sg_list_start = umem->sg_head.sgl;
159
151 while (npages) { 160 while (npages) {
152 ret = get_user_pages(current, current->mm, cur_base, 161 ret = get_user_pages(current, current->mm, cur_base,
153 min_t(unsigned long, npages, 162 min_t(unsigned long, npages,
@@ -157,54 +166,38 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
157 if (ret < 0) 166 if (ret < 0)
158 goto out; 167 goto out;
159 168
169 umem->npages += ret;
160 cur_base += ret * PAGE_SIZE; 170 cur_base += ret * PAGE_SIZE;
161 npages -= ret; 171 npages -= ret;
162 172
163 off = 0; 173 for_each_sg(sg_list_start, sg, ret, i) {
164 174 if (vma_list && !is_vm_hugetlb_page(vma_list[i]))
165 while (ret) { 175 umem->hugetlb = 0;
166 chunk = kmalloc(sizeof *chunk + sizeof (struct scatterlist) * 176
167 min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK), 177 sg_set_page(sg, page_list[i], PAGE_SIZE, 0);
168 GFP_KERNEL);
169 if (!chunk) {
170 ret = -ENOMEM;
171 goto out;
172 }
173
174 chunk->nents = min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK);
175 sg_init_table(chunk->page_list, chunk->nents);
176 for (i = 0; i < chunk->nents; ++i) {
177 if (vma_list &&
178 !is_vm_hugetlb_page(vma_list[i + off]))
179 umem->hugetlb = 0;
180 sg_set_page(&chunk->page_list[i], page_list[i + off], PAGE_SIZE, 0);
181 }
182
183 chunk->nmap = ib_dma_map_sg_attrs(context->device,
184 &chunk->page_list[0],
185 chunk->nents,
186 DMA_BIDIRECTIONAL,
187 &attrs);
188 if (chunk->nmap <= 0) {
189 for (i = 0; i < chunk->nents; ++i)
190 put_page(sg_page(&chunk->page_list[i]));
191 kfree(chunk);
192
193 ret = -ENOMEM;
194 goto out;
195 }
196
197 ret -= chunk->nents;
198 off += chunk->nents;
199 list_add_tail(&chunk->list, &umem->chunk_list);
200 } 178 }
201 179
202 ret = 0; 180 /* preparing for next loop */
181 sg_list_start = sg;
203 } 182 }
204 183
184 umem->nmap = ib_dma_map_sg_attrs(context->device,
185 umem->sg_head.sgl,
186 umem->npages,
187 DMA_BIDIRECTIONAL,
188 &attrs);
189
190 if (umem->nmap <= 0) {
191 ret = -ENOMEM;
192 goto out;
193 }
194
195 ret = 0;
196
205out: 197out:
206 if (ret < 0) { 198 if (ret < 0) {
207 __ib_umem_release(context->device, umem, 0); 199 if (need_release)
200 __ib_umem_release(context->device, umem, 0);
208 kfree(umem); 201 kfree(umem);
209 } else 202 } else
210 current->mm->pinned_vm = locked; 203 current->mm->pinned_vm = locked;
@@ -278,17 +271,16 @@ EXPORT_SYMBOL(ib_umem_release);
278 271
279int ib_umem_page_count(struct ib_umem *umem) 272int ib_umem_page_count(struct ib_umem *umem)
280{ 273{
281 struct ib_umem_chunk *chunk;
282 int shift; 274 int shift;
283 int i; 275 int i;
284 int n; 276 int n;
277 struct scatterlist *sg;
285 278
286 shift = ilog2(umem->page_size); 279 shift = ilog2(umem->page_size);
287 280
288 n = 0; 281 n = 0;
289 list_for_each_entry(chunk, &umem->chunk_list, list) 282 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i)
290 for (i = 0; i < chunk->nmap; ++i) 283 n += sg_dma_len(sg) >> shift;
291 n += sg_dma_len(&chunk->page_list[i]) >> shift;
292 284
293 return n; 285 return n;
294} 286}
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 3ac795115438..92525f855d82 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -1169,6 +1169,45 @@ int ib_dereg_mr(struct ib_mr *mr)
1169} 1169}
1170EXPORT_SYMBOL(ib_dereg_mr); 1170EXPORT_SYMBOL(ib_dereg_mr);
1171 1171
1172struct ib_mr *ib_create_mr(struct ib_pd *pd,
1173 struct ib_mr_init_attr *mr_init_attr)
1174{
1175 struct ib_mr *mr;
1176
1177 if (!pd->device->create_mr)
1178 return ERR_PTR(-ENOSYS);
1179
1180 mr = pd->device->create_mr(pd, mr_init_attr);
1181
1182 if (!IS_ERR(mr)) {
1183 mr->device = pd->device;
1184 mr->pd = pd;
1185 mr->uobject = NULL;
1186 atomic_inc(&pd->usecnt);
1187 atomic_set(&mr->usecnt, 0);
1188 }
1189
1190 return mr;
1191}
1192EXPORT_SYMBOL(ib_create_mr);
1193
1194int ib_destroy_mr(struct ib_mr *mr)
1195{
1196 struct ib_pd *pd;
1197 int ret;
1198
1199 if (atomic_read(&mr->usecnt))
1200 return -EBUSY;
1201
1202 pd = mr->pd;
1203 ret = mr->device->destroy_mr(mr);
1204 if (!ret)
1205 atomic_dec(&pd->usecnt);
1206
1207 return ret;
1208}
1209EXPORT_SYMBOL(ib_destroy_mr);
1210
1172struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len) 1211struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
1173{ 1212{
1174 struct ib_mr *mr; 1213 struct ib_mr *mr;
@@ -1398,3 +1437,11 @@ int ib_destroy_flow(struct ib_flow *flow_id)
1398 return err; 1437 return err;
1399} 1438}
1400EXPORT_SYMBOL(ib_destroy_flow); 1439EXPORT_SYMBOL(ib_destroy_flow);
1440
1441int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
1442 struct ib_mr_status *mr_status)
1443{
1444 return mr->device->check_mr_status ?
1445 mr->device->check_mr_status(mr, check_mask, mr_status) : -ENOSYS;
1446}
1447EXPORT_SYMBOL(ib_check_mr_status);
diff --git a/drivers/infiniband/hw/amso1100/c2_provider.c b/drivers/infiniband/hw/amso1100/c2_provider.c
index 07eb3a8067d8..8af33cf1fc4e 100644
--- a/drivers/infiniband/hw/amso1100/c2_provider.c
+++ b/drivers/infiniband/hw/amso1100/c2_provider.c
@@ -431,9 +431,9 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
431 u64 *pages; 431 u64 *pages;
432 u64 kva = 0; 432 u64 kva = 0;
433 int shift, n, len; 433 int shift, n, len;
434 int i, j, k; 434 int i, k, entry;
435 int err = 0; 435 int err = 0;
436 struct ib_umem_chunk *chunk; 436 struct scatterlist *sg;
437 struct c2_pd *c2pd = to_c2pd(pd); 437 struct c2_pd *c2pd = to_c2pd(pd);
438 struct c2_mr *c2mr; 438 struct c2_mr *c2mr;
439 439
@@ -452,10 +452,7 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
452 } 452 }
453 453
454 shift = ffs(c2mr->umem->page_size) - 1; 454 shift = ffs(c2mr->umem->page_size) - 1;
455 455 n = c2mr->umem->nmap;
456 n = 0;
457 list_for_each_entry(chunk, &c2mr->umem->chunk_list, list)
458 n += chunk->nents;
459 456
460 pages = kmalloc(n * sizeof(u64), GFP_KERNEL); 457 pages = kmalloc(n * sizeof(u64), GFP_KERNEL);
461 if (!pages) { 458 if (!pages) {
@@ -464,14 +461,12 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
464 } 461 }
465 462
466 i = 0; 463 i = 0;
467 list_for_each_entry(chunk, &c2mr->umem->chunk_list, list) { 464 for_each_sg(c2mr->umem->sg_head.sgl, sg, c2mr->umem->nmap, entry) {
468 for (j = 0; j < chunk->nmap; ++j) { 465 len = sg_dma_len(sg) >> shift;
469 len = sg_dma_len(&chunk->page_list[j]) >> shift; 466 for (k = 0; k < len; ++k) {
470 for (k = 0; k < len; ++k) { 467 pages[i++] =
471 pages[i++] = 468 sg_dma_address(sg) +
472 sg_dma_address(&chunk->page_list[j]) + 469 (c2mr->umem->page_size * k);
473 (c2mr->umem->page_size * k);
474 }
475 } 470 }
476 } 471 }
477 472
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index d2283837d451..811b24a539c0 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -618,14 +618,13 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
618{ 618{
619 __be64 *pages; 619 __be64 *pages;
620 int shift, n, len; 620 int shift, n, len;
621 int i, j, k; 621 int i, k, entry;
622 int err = 0; 622 int err = 0;
623 struct ib_umem_chunk *chunk;
624 struct iwch_dev *rhp; 623 struct iwch_dev *rhp;
625 struct iwch_pd *php; 624 struct iwch_pd *php;
626 struct iwch_mr *mhp; 625 struct iwch_mr *mhp;
627 struct iwch_reg_user_mr_resp uresp; 626 struct iwch_reg_user_mr_resp uresp;
628 627 struct scatterlist *sg;
629 PDBG("%s ib_pd %p\n", __func__, pd); 628 PDBG("%s ib_pd %p\n", __func__, pd);
630 629
631 php = to_iwch_pd(pd); 630 php = to_iwch_pd(pd);
@@ -645,9 +644,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
645 644
646 shift = ffs(mhp->umem->page_size) - 1; 645 shift = ffs(mhp->umem->page_size) - 1;
647 646
648 n = 0; 647 n = mhp->umem->nmap;
649 list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
650 n += chunk->nents;
651 648
652 err = iwch_alloc_pbl(mhp, n); 649 err = iwch_alloc_pbl(mhp, n);
653 if (err) 650 if (err)
@@ -661,12 +658,10 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
661 658
662 i = n = 0; 659 i = n = 0;
663 660
664 list_for_each_entry(chunk, &mhp->umem->chunk_list, list) 661 for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
665 for (j = 0; j < chunk->nmap; ++j) { 662 len = sg_dma_len(sg) >> shift;
666 len = sg_dma_len(&chunk->page_list[j]) >> shift;
667 for (k = 0; k < len; ++k) { 663 for (k = 0; k < len; ++k) {
668 pages[i++] = cpu_to_be64(sg_dma_address( 664 pages[i++] = cpu_to_be64(sg_dma_address(sg) +
669 &chunk->page_list[j]) +
670 mhp->umem->page_size * k); 665 mhp->umem->page_size * k);
671 if (i == PAGE_SIZE / sizeof *pages) { 666 if (i == PAGE_SIZE / sizeof *pages) {
672 err = iwch_write_pbl(mhp, pages, i, n); 667 err = iwch_write_pbl(mhp, pages, i, n);
@@ -676,7 +671,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
676 i = 0; 671 i = 0;
677 } 672 }
678 } 673 }
679 } 674 }
680 675
681 if (i) 676 if (i)
682 err = iwch_write_pbl(mhp, pages, i, n); 677 err = iwch_write_pbl(mhp, pages, i, n);
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index 41b11951a30a..392d422b00cb 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -678,9 +678,9 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
678{ 678{
679 __be64 *pages; 679 __be64 *pages;
680 int shift, n, len; 680 int shift, n, len;
681 int i, j, k; 681 int i, k, entry;
682 int err = 0; 682 int err = 0;
683 struct ib_umem_chunk *chunk; 683 struct scatterlist *sg;
684 struct c4iw_dev *rhp; 684 struct c4iw_dev *rhp;
685 struct c4iw_pd *php; 685 struct c4iw_pd *php;
686 struct c4iw_mr *mhp; 686 struct c4iw_mr *mhp;
@@ -710,10 +710,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
710 710
711 shift = ffs(mhp->umem->page_size) - 1; 711 shift = ffs(mhp->umem->page_size) - 1;
712 712
713 n = 0; 713 n = mhp->umem->nmap;
714 list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
715 n += chunk->nents;
716
717 err = alloc_pbl(mhp, n); 714 err = alloc_pbl(mhp, n);
718 if (err) 715 if (err)
719 goto err; 716 goto err;
@@ -726,24 +723,22 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
726 723
727 i = n = 0; 724 i = n = 0;
728 725
729 list_for_each_entry(chunk, &mhp->umem->chunk_list, list) 726 for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
730 for (j = 0; j < chunk->nmap; ++j) { 727 len = sg_dma_len(sg) >> shift;
731 len = sg_dma_len(&chunk->page_list[j]) >> shift; 728 for (k = 0; k < len; ++k) {
732 for (k = 0; k < len; ++k) { 729 pages[i++] = cpu_to_be64(sg_dma_address(sg) +
733 pages[i++] = cpu_to_be64(sg_dma_address( 730 mhp->umem->page_size * k);
734 &chunk->page_list[j]) + 731 if (i == PAGE_SIZE / sizeof *pages) {
735 mhp->umem->page_size * k); 732 err = write_pbl(&mhp->rhp->rdev,
736 if (i == PAGE_SIZE / sizeof *pages) { 733 pages,
737 err = write_pbl(&mhp->rhp->rdev, 734 mhp->attr.pbl_addr + (n << 3), i);
738 pages, 735 if (err)
739 mhp->attr.pbl_addr + (n << 3), i); 736 goto pbl_done;
740 if (err) 737 n += i;
741 goto pbl_done; 738 i = 0;
742 n += i;
743 i = 0;
744 }
745 } 739 }
746 } 740 }
741 }
747 742
748 if (i) 743 if (i)
749 err = write_pbl(&mhp->rhp->rdev, pages, 744 err = write_pbl(&mhp->rhp->rdev, pages,
diff --git a/drivers/infiniband/hw/ehca/ehca_classes.h b/drivers/infiniband/hw/ehca/ehca_classes.h
index f08f6eaf3fa8..bd45e0f3923f 100644
--- a/drivers/infiniband/hw/ehca/ehca_classes.h
+++ b/drivers/infiniband/hw/ehca/ehca_classes.h
@@ -322,7 +322,7 @@ struct ehca_mr_pginfo {
322 } phy; 322 } phy;
323 struct { /* type EHCA_MR_PGI_USER section */ 323 struct { /* type EHCA_MR_PGI_USER section */
324 struct ib_umem *region; 324 struct ib_umem *region;
325 struct ib_umem_chunk *next_chunk; 325 struct scatterlist *next_sg;
326 u64 next_nmap; 326 u64 next_nmap;
327 } usr; 327 } usr;
328 struct { /* type EHCA_MR_PGI_FMR section */ 328 struct { /* type EHCA_MR_PGI_FMR section */
diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.c b/drivers/infiniband/hw/ehca/ehca_mrmw.c
index bcfb0c183620..7168f594d457 100644
--- a/drivers/infiniband/hw/ehca/ehca_mrmw.c
+++ b/drivers/infiniband/hw/ehca/ehca_mrmw.c
@@ -400,10 +400,7 @@ reg_user_mr_fallback:
400 pginfo.num_hwpages = num_hwpages; 400 pginfo.num_hwpages = num_hwpages;
401 pginfo.u.usr.region = e_mr->umem; 401 pginfo.u.usr.region = e_mr->umem;
402 pginfo.next_hwpage = e_mr->umem->offset / hwpage_size; 402 pginfo.next_hwpage = e_mr->umem->offset / hwpage_size;
403 pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk, 403 pginfo.u.usr.next_sg = pginfo.u.usr.region->sg_head.sgl;
404 (&e_mr->umem->chunk_list),
405 list);
406
407 ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags, 404 ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
408 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, 405 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
409 &e_mr->ib.ib_mr.rkey, EHCA_REG_MR); 406 &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
@@ -1858,61 +1855,39 @@ static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1858 u64 *kpage) 1855 u64 *kpage)
1859{ 1856{
1860 int ret = 0; 1857 int ret = 0;
1861 struct ib_umem_chunk *prev_chunk;
1862 struct ib_umem_chunk *chunk;
1863 u64 pgaddr; 1858 u64 pgaddr;
1864 u32 i = 0;
1865 u32 j = 0; 1859 u32 j = 0;
1866 int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size; 1860 int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;
1867 1861 struct scatterlist **sg = &pginfo->u.usr.next_sg;
1868 /* loop over desired chunk entries */ 1862
1869 chunk = pginfo->u.usr.next_chunk; 1863 while (*sg != NULL) {
1870 prev_chunk = pginfo->u.usr.next_chunk; 1864 pgaddr = page_to_pfn(sg_page(*sg))
1871 list_for_each_entry_continue( 1865 << PAGE_SHIFT;
1872 chunk, (&(pginfo->u.usr.region->chunk_list)), list) { 1866 *kpage = pgaddr + (pginfo->next_hwpage *
1873 for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) { 1867 pginfo->hwpage_size);
1874 pgaddr = page_to_pfn(sg_page(&chunk->page_list[i])) 1868 if (!(*kpage)) {
1875 << PAGE_SHIFT ; 1869 ehca_gen_err("pgaddr=%llx "
1876 *kpage = pgaddr + (pginfo->next_hwpage * 1870 "sg_dma_address=%llx "
1877 pginfo->hwpage_size); 1871 "entry=%llx next_hwpage=%llx",
1878 if ( !(*kpage) ) { 1872 pgaddr, (u64)sg_dma_address(*sg),
1879 ehca_gen_err("pgaddr=%llx " 1873 pginfo->u.usr.next_nmap,
1880 "chunk->page_list[i]=%llx " 1874 pginfo->next_hwpage);
1881 "i=%x next_hwpage=%llx", 1875 return -EFAULT;
1882 pgaddr, (u64)sg_dma_address(
1883 &chunk->page_list[i]),
1884 i, pginfo->next_hwpage);
1885 return -EFAULT;
1886 }
1887 (pginfo->hwpage_cnt)++;
1888 (pginfo->next_hwpage)++;
1889 kpage++;
1890 if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
1891 (pginfo->kpage_cnt)++;
1892 (pginfo->u.usr.next_nmap)++;
1893 pginfo->next_hwpage = 0;
1894 i++;
1895 }
1896 j++;
1897 if (j >= number) break;
1898 } 1876 }
1899 if ((pginfo->u.usr.next_nmap >= chunk->nmap) && 1877 (pginfo->hwpage_cnt)++;
1900 (j >= number)) { 1878 (pginfo->next_hwpage)++;
1901 pginfo->u.usr.next_nmap = 0; 1879 kpage++;
1902 prev_chunk = chunk; 1880 if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
1903 break; 1881 (pginfo->kpage_cnt)++;
1904 } else if (pginfo->u.usr.next_nmap >= chunk->nmap) { 1882 (pginfo->u.usr.next_nmap)++;
1905 pginfo->u.usr.next_nmap = 0; 1883 pginfo->next_hwpage = 0;
1906 prev_chunk = chunk; 1884 *sg = sg_next(*sg);
1907 } else if (j >= number) 1885 }
1886 j++;
1887 if (j >= number)
1908 break; 1888 break;
1909 else
1910 prev_chunk = chunk;
1911 } 1889 }
1912 pginfo->u.usr.next_chunk = 1890
1913 list_prepare_entry(prev_chunk,
1914 (&(pginfo->u.usr.region->chunk_list)),
1915 list);
1916 return ret; 1891 return ret;
1917} 1892}
1918 1893
@@ -1920,20 +1895,19 @@ static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1920 * check given pages for contiguous layout 1895 * check given pages for contiguous layout
1921 * last page addr is returned in prev_pgaddr for further check 1896 * last page addr is returned in prev_pgaddr for further check
1922 */ 1897 */
1923static int ehca_check_kpages_per_ate(struct scatterlist *page_list, 1898static int ehca_check_kpages_per_ate(struct scatterlist **sg,
1924 int start_idx, int end_idx, 1899 int num_pages,
1925 u64 *prev_pgaddr) 1900 u64 *prev_pgaddr)
1926{ 1901{
1927 int t; 1902 for (; *sg && num_pages > 0; *sg = sg_next(*sg), num_pages--) {
1928 for (t = start_idx; t <= end_idx; t++) { 1903 u64 pgaddr = page_to_pfn(sg_page(*sg)) << PAGE_SHIFT;
1929 u64 pgaddr = page_to_pfn(sg_page(&page_list[t])) << PAGE_SHIFT;
1930 if (ehca_debug_level >= 3) 1904 if (ehca_debug_level >= 3)
1931 ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr, 1905 ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr,
1932 *(u64 *)__va(pgaddr)); 1906 *(u64 *)__va(pgaddr));
1933 if (pgaddr - PAGE_SIZE != *prev_pgaddr) { 1907 if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
1934 ehca_gen_err("uncontiguous page found pgaddr=%llx " 1908 ehca_gen_err("uncontiguous page found pgaddr=%llx "
1935 "prev_pgaddr=%llx page_list_i=%x", 1909 "prev_pgaddr=%llx entries_left_in_hwpage=%x",
1936 pgaddr, *prev_pgaddr, t); 1910 pgaddr, *prev_pgaddr, num_pages);
1937 return -EINVAL; 1911 return -EINVAL;
1938 } 1912 }
1939 *prev_pgaddr = pgaddr; 1913 *prev_pgaddr = pgaddr;
@@ -1947,111 +1921,80 @@ static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
1947 u64 *kpage) 1921 u64 *kpage)
1948{ 1922{
1949 int ret = 0; 1923 int ret = 0;
1950 struct ib_umem_chunk *prev_chunk;
1951 struct ib_umem_chunk *chunk;
1952 u64 pgaddr, prev_pgaddr; 1924 u64 pgaddr, prev_pgaddr;
1953 u32 i = 0;
1954 u32 j = 0; 1925 u32 j = 0;
1955 int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE; 1926 int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;
1956 int nr_kpages = kpages_per_hwpage; 1927 int nr_kpages = kpages_per_hwpage;
1928 struct scatterlist **sg = &pginfo->u.usr.next_sg;
1929
1930 while (*sg != NULL) {
1957 1931
1958 /* loop over desired chunk entries */ 1932 if (nr_kpages == kpages_per_hwpage) {
1959 chunk = pginfo->u.usr.next_chunk; 1933 pgaddr = (page_to_pfn(sg_page(*sg))
1960 prev_chunk = pginfo->u.usr.next_chunk; 1934 << PAGE_SHIFT);
1961 list_for_each_entry_continue( 1935 *kpage = pgaddr;
1962 chunk, (&(pginfo->u.usr.region->chunk_list)), list) { 1936 if (!(*kpage)) {
1963 for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) { 1937 ehca_gen_err("pgaddr=%llx entry=%llx",
1964 if (nr_kpages == kpages_per_hwpage) { 1938 pgaddr, pginfo->u.usr.next_nmap);
1965 pgaddr = ( page_to_pfn(sg_page(&chunk->page_list[i])) 1939 ret = -EFAULT;
1966 << PAGE_SHIFT ); 1940 return ret;
1967 *kpage = pgaddr; 1941 }
1968 if ( !(*kpage) ) { 1942 /*
1969 ehca_gen_err("pgaddr=%llx i=%x", 1943 * The first page in a hwpage must be aligned;
1970 pgaddr, i); 1944 * the first MR page is exempt from this rule.
1945 */
1946 if (pgaddr & (pginfo->hwpage_size - 1)) {
1947 if (pginfo->hwpage_cnt) {
1948 ehca_gen_err(
1949 "invalid alignment "
1950 "pgaddr=%llx entry=%llx "
1951 "mr_pgsize=%llx",
1952 pgaddr, pginfo->u.usr.next_nmap,
1953 pginfo->hwpage_size);
1971 ret = -EFAULT; 1954 ret = -EFAULT;
1972 return ret; 1955 return ret;
1973 } 1956 }
1974 /* 1957 /* first MR page */
1975 * The first page in a hwpage must be aligned; 1958 pginfo->kpage_cnt =
1976 * the first MR page is exempt from this rule. 1959 (pgaddr &
1977 */ 1960 (pginfo->hwpage_size - 1)) >>
1978 if (pgaddr & (pginfo->hwpage_size - 1)) { 1961 PAGE_SHIFT;
1979 if (pginfo->hwpage_cnt) { 1962 nr_kpages -= pginfo->kpage_cnt;
1980 ehca_gen_err( 1963 *kpage = pgaddr &
1981 "invalid alignment " 1964 ~(pginfo->hwpage_size - 1);
1982 "pgaddr=%llx i=%x "
1983 "mr_pgsize=%llx",
1984 pgaddr, i,
1985 pginfo->hwpage_size);
1986 ret = -EFAULT;
1987 return ret;
1988 }
1989 /* first MR page */
1990 pginfo->kpage_cnt =
1991 (pgaddr &
1992 (pginfo->hwpage_size - 1)) >>
1993 PAGE_SHIFT;
1994 nr_kpages -= pginfo->kpage_cnt;
1995 *kpage = pgaddr &
1996 ~(pginfo->hwpage_size - 1);
1997 }
1998 if (ehca_debug_level >= 3) {
1999 u64 val = *(u64 *)__va(pgaddr);
2000 ehca_gen_dbg("kpage=%llx chunk_page=%llx "
2001 "value=%016llx",
2002 *kpage, pgaddr, val);
2003 }
2004 prev_pgaddr = pgaddr;
2005 i++;
2006 pginfo->kpage_cnt++;
2007 pginfo->u.usr.next_nmap++;
2008 nr_kpages--;
2009 if (!nr_kpages)
2010 goto next_kpage;
2011 continue;
2012 } 1965 }
2013 if (i + nr_kpages > chunk->nmap) { 1966 if (ehca_debug_level >= 3) {
2014 ret = ehca_check_kpages_per_ate( 1967 u64 val = *(u64 *)__va(pgaddr);
2015 chunk->page_list, i, 1968 ehca_gen_dbg("kpage=%llx page=%llx "
2016 chunk->nmap - 1, &prev_pgaddr); 1969 "value=%016llx",
2017 if (ret) return ret; 1970 *kpage, pgaddr, val);
2018 pginfo->kpage_cnt += chunk->nmap - i;
2019 pginfo->u.usr.next_nmap += chunk->nmap - i;
2020 nr_kpages -= chunk->nmap - i;
2021 break;
2022 } 1971 }
1972 prev_pgaddr = pgaddr;
1973 *sg = sg_next(*sg);
1974 pginfo->kpage_cnt++;
1975 pginfo->u.usr.next_nmap++;
1976 nr_kpages--;
1977 if (!nr_kpages)
1978 goto next_kpage;
1979 continue;
1980 }
1981
1982 ret = ehca_check_kpages_per_ate(sg, nr_kpages,
1983 &prev_pgaddr);
1984 if (ret)
1985 return ret;
1986 pginfo->kpage_cnt += nr_kpages;
1987 pginfo->u.usr.next_nmap += nr_kpages;
2023 1988
2024 ret = ehca_check_kpages_per_ate(chunk->page_list, i,
2025 i + nr_kpages - 1,
2026 &prev_pgaddr);
2027 if (ret) return ret;
2028 i += nr_kpages;
2029 pginfo->kpage_cnt += nr_kpages;
2030 pginfo->u.usr.next_nmap += nr_kpages;
2031next_kpage: 1989next_kpage:
2032 nr_kpages = kpages_per_hwpage; 1990 nr_kpages = kpages_per_hwpage;
2033 (pginfo->hwpage_cnt)++; 1991 (pginfo->hwpage_cnt)++;
2034 kpage++; 1992 kpage++;
2035 j++; 1993 j++;
2036 if (j >= number) break; 1994 if (j >= number)
2037 }
2038 if ((pginfo->u.usr.next_nmap >= chunk->nmap) &&
2039 (j >= number)) {
2040 pginfo->u.usr.next_nmap = 0;
2041 prev_chunk = chunk;
2042 break;
2043 } else if (pginfo->u.usr.next_nmap >= chunk->nmap) {
2044 pginfo->u.usr.next_nmap = 0;
2045 prev_chunk = chunk;
2046 } else if (j >= number)
2047 break; 1995 break;
2048 else
2049 prev_chunk = chunk;
2050 } 1996 }
2051 pginfo->u.usr.next_chunk = 1997
2052 list_prepare_entry(prev_chunk,
2053 (&(pginfo->u.usr.region->chunk_list)),
2054 list);
2055 return ret; 1998 return ret;
2056} 1999}
2057 2000
diff --git a/drivers/infiniband/hw/ipath/ipath_mr.c b/drivers/infiniband/hw/ipath/ipath_mr.c
index e346d3890a0e..5e61e9bff697 100644
--- a/drivers/infiniband/hw/ipath/ipath_mr.c
+++ b/drivers/infiniband/hw/ipath/ipath_mr.c
@@ -188,8 +188,8 @@ struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
188{ 188{
189 struct ipath_mr *mr; 189 struct ipath_mr *mr;
190 struct ib_umem *umem; 190 struct ib_umem *umem;
191 struct ib_umem_chunk *chunk; 191 int n, m, entry;
192 int n, m, i; 192 struct scatterlist *sg;
193 struct ib_mr *ret; 193 struct ib_mr *ret;
194 194
195 if (length == 0) { 195 if (length == 0) {
@@ -202,10 +202,7 @@ struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
202 if (IS_ERR(umem)) 202 if (IS_ERR(umem))
203 return (void *) umem; 203 return (void *) umem;
204 204
205 n = 0; 205 n = umem->nmap;
206 list_for_each_entry(chunk, &umem->chunk_list, list)
207 n += chunk->nents;
208
209 mr = alloc_mr(n, &to_idev(pd->device)->lk_table); 206 mr = alloc_mr(n, &to_idev(pd->device)->lk_table);
210 if (!mr) { 207 if (!mr) {
211 ret = ERR_PTR(-ENOMEM); 208 ret = ERR_PTR(-ENOMEM);
@@ -224,22 +221,20 @@ struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
224 221
225 m = 0; 222 m = 0;
226 n = 0; 223 n = 0;
227 list_for_each_entry(chunk, &umem->chunk_list, list) { 224 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
228 for (i = 0; i < chunk->nents; i++) { 225 void *vaddr;
229 void *vaddr; 226
230 227 vaddr = page_address(sg_page(sg));
231 vaddr = page_address(sg_page(&chunk->page_list[i])); 228 if (!vaddr) {
232 if (!vaddr) { 229 ret = ERR_PTR(-EINVAL);
233 ret = ERR_PTR(-EINVAL); 230 goto bail;
234 goto bail; 231 }
235 } 232 mr->mr.map[m]->segs[n].vaddr = vaddr;
236 mr->mr.map[m]->segs[n].vaddr = vaddr; 233 mr->mr.map[m]->segs[n].length = umem->page_size;
237 mr->mr.map[m]->segs[n].length = umem->page_size; 234 n++;
238 n++; 235 if (n == IPATH_SEGSZ) {
239 if (n == IPATH_SEGSZ) { 236 m++;
240 m++; 237 n = 0;
241 n = 0;
242 }
243 } 238 }
244 } 239 }
245 ret = &mr->ibmr; 240 ret = &mr->ibmr;
diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c
index 8aee4233b388..c51740986367 100644
--- a/drivers/infiniband/hw/mlx4/doorbell.c
+++ b/drivers/infiniband/hw/mlx4/doorbell.c
@@ -45,7 +45,6 @@ int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt,
45 struct mlx4_db *db) 45 struct mlx4_db *db)
46{ 46{
47 struct mlx4_ib_user_db_page *page; 47 struct mlx4_ib_user_db_page *page;
48 struct ib_umem_chunk *chunk;
49 int err = 0; 48 int err = 0;
50 49
51 mutex_lock(&context->db_page_mutex); 50 mutex_lock(&context->db_page_mutex);
@@ -73,8 +72,7 @@ int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt,
73 list_add(&page->list, &context->db_page_list); 72 list_add(&page->list, &context->db_page_list);
74 73
75found: 74found:
76 chunk = list_entry(page->umem->chunk_list.next, struct ib_umem_chunk, list); 75 db->dma = sg_dma_address(page->umem->sg_head.sgl) + (virt & ~PAGE_MASK);
77 db->dma = sg_dma_address(chunk->page_list) + (virt & ~PAGE_MASK);
78 db->u.user_page = page; 76 db->u.user_page = page;
79 ++page->refcnt; 77 ++page->refcnt;
80 78
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index e471f089ff00..cb2a8727f3fb 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -90,11 +90,11 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
90 struct ib_umem *umem) 90 struct ib_umem *umem)
91{ 91{
92 u64 *pages; 92 u64 *pages;
93 struct ib_umem_chunk *chunk; 93 int i, k, entry;
94 int i, j, k;
95 int n; 94 int n;
96 int len; 95 int len;
97 int err = 0; 96 int err = 0;
97 struct scatterlist *sg;
98 98
99 pages = (u64 *) __get_free_page(GFP_KERNEL); 99 pages = (u64 *) __get_free_page(GFP_KERNEL);
100 if (!pages) 100 if (!pages)
@@ -102,26 +102,25 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
102 102
103 i = n = 0; 103 i = n = 0;
104 104
105 list_for_each_entry(chunk, &umem->chunk_list, list) 105 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
106 for (j = 0; j < chunk->nmap; ++j) { 106 len = sg_dma_len(sg) >> mtt->page_shift;
107 len = sg_dma_len(&chunk->page_list[j]) >> mtt->page_shift; 107 for (k = 0; k < len; ++k) {
108 for (k = 0; k < len; ++k) { 108 pages[i++] = sg_dma_address(sg) +
109 pages[i++] = sg_dma_address(&chunk->page_list[j]) + 109 umem->page_size * k;
110 umem->page_size * k; 110 /*
111 /* 111 * Be friendly to mlx4_write_mtt() and
112 * Be friendly to mlx4_write_mtt() and 112 * pass it chunks of appropriate size.
113 * pass it chunks of appropriate size. 113 */
114 */ 114 if (i == PAGE_SIZE / sizeof (u64)) {
115 if (i == PAGE_SIZE / sizeof (u64)) { 115 err = mlx4_write_mtt(dev->dev, mtt, n,
116 err = mlx4_write_mtt(dev->dev, mtt, n, 116 i, pages);
117 i, pages); 117 if (err)
118 if (err) 118 goto out;
119 goto out; 119 n += i;
120 n += i; 120 i = 0;
121 i = 0;
122 }
123 } 121 }
124 } 122 }
123 }
125 124
126 if (i) 125 if (i)
127 err = mlx4_write_mtt(dev->dev, mtt, n, i, pages); 126 err = mlx4_write_mtt(dev->dev, mtt, n, i, pages);
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index b1705ce6eb88..62bb6b49dc1d 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -366,6 +366,38 @@ static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf)
366 mlx5_buf_free(&dev->mdev, &buf->buf); 366 mlx5_buf_free(&dev->mdev, &buf->buf);
367} 367}
368 368
369static void get_sig_err_item(struct mlx5_sig_err_cqe *cqe,
370 struct ib_sig_err *item)
371{
372 u16 syndrome = be16_to_cpu(cqe->syndrome);
373
374#define GUARD_ERR (1 << 13)
375#define APPTAG_ERR (1 << 12)
376#define REFTAG_ERR (1 << 11)
377
378 if (syndrome & GUARD_ERR) {
379 item->err_type = IB_SIG_BAD_GUARD;
380 item->expected = be32_to_cpu(cqe->expected_trans_sig) >> 16;
381 item->actual = be32_to_cpu(cqe->actual_trans_sig) >> 16;
382 } else
383 if (syndrome & REFTAG_ERR) {
384 item->err_type = IB_SIG_BAD_REFTAG;
385 item->expected = be32_to_cpu(cqe->expected_reftag);
386 item->actual = be32_to_cpu(cqe->actual_reftag);
387 } else
388 if (syndrome & APPTAG_ERR) {
389 item->err_type = IB_SIG_BAD_APPTAG;
390 item->expected = be32_to_cpu(cqe->expected_trans_sig) & 0xffff;
391 item->actual = be32_to_cpu(cqe->actual_trans_sig) & 0xffff;
392 } else {
393 pr_err("Got signature completion error with bad syndrome %04x\n",
394 syndrome);
395 }
396
397 item->sig_err_offset = be64_to_cpu(cqe->err_offset);
398 item->key = be32_to_cpu(cqe->mkey);
399}
400
369static int mlx5_poll_one(struct mlx5_ib_cq *cq, 401static int mlx5_poll_one(struct mlx5_ib_cq *cq,
370 struct mlx5_ib_qp **cur_qp, 402 struct mlx5_ib_qp **cur_qp,
371 struct ib_wc *wc) 403 struct ib_wc *wc)
@@ -375,6 +407,9 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
375 struct mlx5_cqe64 *cqe64; 407 struct mlx5_cqe64 *cqe64;
376 struct mlx5_core_qp *mqp; 408 struct mlx5_core_qp *mqp;
377 struct mlx5_ib_wq *wq; 409 struct mlx5_ib_wq *wq;
410 struct mlx5_sig_err_cqe *sig_err_cqe;
411 struct mlx5_core_mr *mmr;
412 struct mlx5_ib_mr *mr;
378 uint8_t opcode; 413 uint8_t opcode;
379 uint32_t qpn; 414 uint32_t qpn;
380 u16 wqe_ctr; 415 u16 wqe_ctr;
@@ -475,6 +510,33 @@ repoll:
475 } 510 }
476 } 511 }
477 break; 512 break;
513 case MLX5_CQE_SIG_ERR:
514 sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64;
515
516 read_lock(&dev->mdev.priv.mr_table.lock);
517 mmr = __mlx5_mr_lookup(&dev->mdev,
518 mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
519 if (unlikely(!mmr)) {
520 read_unlock(&dev->mdev.priv.mr_table.lock);
521 mlx5_ib_warn(dev, "CQE@CQ %06x for unknown MR %6x\n",
522 cq->mcq.cqn, be32_to_cpu(sig_err_cqe->mkey));
523 return -EINVAL;
524 }
525
526 mr = to_mibmr(mmr);
527 get_sig_err_item(sig_err_cqe, &mr->sig->err_item);
528 mr->sig->sig_err_exists = true;
529 mr->sig->sigerr_count++;
530
531 mlx5_ib_warn(dev, "CQN: 0x%x Got SIGERR on key: 0x%x err_type %x err_offset %llx expected %x actual %x\n",
532 cq->mcq.cqn, mr->sig->err_item.key,
533 mr->sig->err_item.err_type,
534 mr->sig->err_item.sig_err_offset,
535 mr->sig->err_item.expected,
536 mr->sig->err_item.actual);
537
538 read_unlock(&dev->mdev.priv.mr_table.lock);
539 goto repoll;
478 } 540 }
479 541
480 return 0; 542 return 0;
diff --git a/drivers/infiniband/hw/mlx5/doorbell.c b/drivers/infiniband/hw/mlx5/doorbell.c
index 256a23344f28..ece028fc47d6 100644
--- a/drivers/infiniband/hw/mlx5/doorbell.c
+++ b/drivers/infiniband/hw/mlx5/doorbell.c
@@ -47,7 +47,6 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
47 struct mlx5_db *db) 47 struct mlx5_db *db)
48{ 48{
49 struct mlx5_ib_user_db_page *page; 49 struct mlx5_ib_user_db_page *page;
50 struct ib_umem_chunk *chunk;
51 int err = 0; 50 int err = 0;
52 51
53 mutex_lock(&context->db_page_mutex); 52 mutex_lock(&context->db_page_mutex);
@@ -75,8 +74,7 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
75 list_add(&page->list, &context->db_page_list); 74 list_add(&page->list, &context->db_page_list);
76 75
77found: 76found:
78 chunk = list_entry(page->umem->chunk_list.next, struct ib_umem_chunk, list); 77 db->dma = sg_dma_address(page->umem->sg_head.sgl) + (virt & ~PAGE_MASK);
79 db->dma = sg_dma_address(chunk->page_list) + (virt & ~PAGE_MASK);
80 db->u.user_page = page; 78 db->u.user_page = page;
81 ++page->refcnt; 79 ++page->refcnt;
82 80
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index bf900579ac08..fa6dc870adae 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -273,6 +273,15 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
273 if (flags & MLX5_DEV_CAP_FLAG_XRC) 273 if (flags & MLX5_DEV_CAP_FLAG_XRC)
274 props->device_cap_flags |= IB_DEVICE_XRC; 274 props->device_cap_flags |= IB_DEVICE_XRC;
275 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; 275 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
276 if (flags & MLX5_DEV_CAP_FLAG_SIG_HAND_OVER) {
277 props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
278 /* At this stage no support for signature handover */
279 props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
280 IB_PROT_T10DIF_TYPE_2 |
281 IB_PROT_T10DIF_TYPE_3;
282 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
283 IB_GUARD_T10DIF_CSUM;
284 }
276 285
277 props->vendor_id = be32_to_cpup((__be32 *)(out_mad->data + 36)) & 286 props->vendor_id = be32_to_cpup((__be32 *)(out_mad->data + 36)) &
278 0xffffff; 287 0xffffff;
@@ -1423,12 +1432,15 @@ static int init_one(struct pci_dev *pdev,
1423 dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr; 1432 dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr;
1424 dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr; 1433 dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr;
1425 dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr; 1434 dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr;
1435 dev->ib_dev.destroy_mr = mlx5_ib_destroy_mr;
1426 dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach; 1436 dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach;
1427 dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach; 1437 dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach;
1428 dev->ib_dev.process_mad = mlx5_ib_process_mad; 1438 dev->ib_dev.process_mad = mlx5_ib_process_mad;
1439 dev->ib_dev.create_mr = mlx5_ib_create_mr;
1429 dev->ib_dev.alloc_fast_reg_mr = mlx5_ib_alloc_fast_reg_mr; 1440 dev->ib_dev.alloc_fast_reg_mr = mlx5_ib_alloc_fast_reg_mr;
1430 dev->ib_dev.alloc_fast_reg_page_list = mlx5_ib_alloc_fast_reg_page_list; 1441 dev->ib_dev.alloc_fast_reg_page_list = mlx5_ib_alloc_fast_reg_page_list;
1431 dev->ib_dev.free_fast_reg_page_list = mlx5_ib_free_fast_reg_page_list; 1442 dev->ib_dev.free_fast_reg_page_list = mlx5_ib_free_fast_reg_page_list;
1443 dev->ib_dev.check_mr_status = mlx5_ib_check_mr_status;
1432 1444
1433 if (mdev->caps.flags & MLX5_DEV_CAP_FLAG_XRC) { 1445 if (mdev->caps.flags & MLX5_DEV_CAP_FLAG_XRC) {
1434 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd; 1446 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c
index 3a5322870b96..8499aec94db6 100644
--- a/drivers/infiniband/hw/mlx5/mem.c
+++ b/drivers/infiniband/hw/mlx5/mem.c
@@ -44,16 +44,17 @@
44void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift, 44void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
45 int *ncont, int *order) 45 int *ncont, int *order)
46{ 46{
47 struct ib_umem_chunk *chunk;
48 unsigned long tmp; 47 unsigned long tmp;
49 unsigned long m; 48 unsigned long m;
50 int i, j, k; 49 int i, k;
51 u64 base = 0; 50 u64 base = 0;
52 int p = 0; 51 int p = 0;
53 int skip; 52 int skip;
54 int mask; 53 int mask;
55 u64 len; 54 u64 len;
56 u64 pfn; 55 u64 pfn;
56 struct scatterlist *sg;
57 int entry;
57 58
58 addr = addr >> PAGE_SHIFT; 59 addr = addr >> PAGE_SHIFT;
59 tmp = (unsigned long)addr; 60 tmp = (unsigned long)addr;
@@ -61,32 +62,31 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
61 skip = 1 << m; 62 skip = 1 << m;
62 mask = skip - 1; 63 mask = skip - 1;
63 i = 0; 64 i = 0;
64 list_for_each_entry(chunk, &umem->chunk_list, list) 65 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
65 for (j = 0; j < chunk->nmap; j++) { 66 len = sg_dma_len(sg) >> PAGE_SHIFT;
66 len = sg_dma_len(&chunk->page_list[j]) >> PAGE_SHIFT; 67 pfn = sg_dma_address(sg) >> PAGE_SHIFT;
67 pfn = sg_dma_address(&chunk->page_list[j]) >> PAGE_SHIFT; 68 for (k = 0; k < len; k++) {
68 for (k = 0; k < len; k++) { 69 if (!(i & mask)) {
69 if (!(i & mask)) { 70 tmp = (unsigned long)pfn;
70 tmp = (unsigned long)pfn; 71 m = min(m, find_first_bit(&tmp, sizeof(tmp)));
71 m = min(m, find_first_bit(&tmp, sizeof(tmp))); 72 skip = 1 << m;
73 mask = skip - 1;
74 base = pfn;
75 p = 0;
76 } else {
77 if (base + p != pfn) {
78 tmp = (unsigned long)p;
79 m = find_first_bit(&tmp, sizeof(tmp));
72 skip = 1 << m; 80 skip = 1 << m;
73 mask = skip - 1; 81 mask = skip - 1;
74 base = pfn; 82 base = pfn;
75 p = 0; 83 p = 0;
76 } else {
77 if (base + p != pfn) {
78 tmp = (unsigned long)p;
79 m = find_first_bit(&tmp, sizeof(tmp));
80 skip = 1 << m;
81 mask = skip - 1;
82 base = pfn;
83 p = 0;
84 }
85 } 84 }
86 p++;
87 i++;
88 } 85 }
86 p++;
87 i++;
89 } 88 }
89 }
90 90
91 if (i) { 91 if (i) {
92 m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m); 92 m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m);
@@ -112,32 +112,32 @@ void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
112{ 112{
113 int shift = page_shift - PAGE_SHIFT; 113 int shift = page_shift - PAGE_SHIFT;
114 int mask = (1 << shift) - 1; 114 int mask = (1 << shift) - 1;
115 struct ib_umem_chunk *chunk; 115 int i, k;
116 int i, j, k;
117 u64 cur = 0; 116 u64 cur = 0;
118 u64 base; 117 u64 base;
119 int len; 118 int len;
119 struct scatterlist *sg;
120 int entry;
120 121
121 i = 0; 122 i = 0;
122 list_for_each_entry(chunk, &umem->chunk_list, list) 123 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
123 for (j = 0; j < chunk->nmap; j++) { 124 len = sg_dma_len(sg) >> PAGE_SHIFT;
124 len = sg_dma_len(&chunk->page_list[j]) >> PAGE_SHIFT; 125 base = sg_dma_address(sg);
125 base = sg_dma_address(&chunk->page_list[j]); 126 for (k = 0; k < len; k++) {
126 for (k = 0; k < len; k++) { 127 if (!(i & mask)) {
127 if (!(i & mask)) { 128 cur = base + (k << PAGE_SHIFT);
128 cur = base + (k << PAGE_SHIFT); 129 if (umr)
129 if (umr) 130 cur |= 3;
130 cur |= 3;
131 131
132 pas[i >> shift] = cpu_to_be64(cur); 132 pas[i >> shift] = cpu_to_be64(cur);
133 mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n", 133 mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n",
134 i >> shift, be64_to_cpu(pas[i >> shift])); 134 i >> shift, be64_to_cpu(pas[i >> shift]));
135 } else 135 } else
136 mlx5_ib_dbg(dev, "=====> 0x%llx\n", 136 mlx5_ib_dbg(dev, "=====> 0x%llx\n",
137 base + (k << PAGE_SHIFT)); 137 base + (k << PAGE_SHIFT));
138 i++; 138 i++;
139 }
140 } 139 }
140 }
141} 141}
142 142
143int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset) 143int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset)
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 389e31965773..50541586e0a6 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -189,6 +189,9 @@ struct mlx5_ib_qp {
189 189
190 int create_type; 190 int create_type;
191 u32 pa_lkey; 191 u32 pa_lkey;
192
193 /* Store signature errors */
194 bool signature_en;
192}; 195};
193 196
194struct mlx5_ib_cq_buf { 197struct mlx5_ib_cq_buf {
@@ -265,6 +268,7 @@ struct mlx5_ib_mr {
265 enum ib_wc_status status; 268 enum ib_wc_status status;
266 struct mlx5_ib_dev *dev; 269 struct mlx5_ib_dev *dev;
267 struct mlx5_create_mkey_mbox_out out; 270 struct mlx5_create_mkey_mbox_out out;
271 struct mlx5_core_sig_ctx *sig;
268}; 272};
269 273
270struct mlx5_ib_fast_reg_page_list { 274struct mlx5_ib_fast_reg_page_list {
@@ -396,6 +400,11 @@ static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
396 return container_of(mqp, struct mlx5_ib_qp, mqp); 400 return container_of(mqp, struct mlx5_ib_qp, mqp);
397} 401}
398 402
403static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mr *mmr)
404{
405 return container_of(mmr, struct mlx5_ib_mr, mmr);
406}
407
399static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd) 408static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
400{ 409{
401 return container_of(ibpd, struct mlx5_ib_pd, ibpd); 410 return container_of(ibpd, struct mlx5_ib_pd, ibpd);
@@ -495,6 +504,9 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
495 u64 virt_addr, int access_flags, 504 u64 virt_addr, int access_flags,
496 struct ib_udata *udata); 505 struct ib_udata *udata);
497int mlx5_ib_dereg_mr(struct ib_mr *ibmr); 506int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
507int mlx5_ib_destroy_mr(struct ib_mr *ibmr);
508struct ib_mr *mlx5_ib_create_mr(struct ib_pd *pd,
509 struct ib_mr_init_attr *mr_init_attr);
498struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd, 510struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd,
499 int max_page_list_len); 511 int max_page_list_len);
500struct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, 512struct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,
@@ -530,6 +542,8 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
530int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev); 542int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
531int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift); 543int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift);
532void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context); 544void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context);
545int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
546 struct ib_mr_status *mr_status);
533 547
534static inline void init_query_mad(struct ib_smp *mad) 548static inline void init_query_mad(struct ib_smp *mad)
535{ 549{
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 7c95ca1f0c25..81392b26d078 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -992,6 +992,122 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
992 return 0; 992 return 0;
993} 993}
994 994
995struct ib_mr *mlx5_ib_create_mr(struct ib_pd *pd,
996 struct ib_mr_init_attr *mr_init_attr)
997{
998 struct mlx5_ib_dev *dev = to_mdev(pd->device);
999 struct mlx5_create_mkey_mbox_in *in;
1000 struct mlx5_ib_mr *mr;
1001 int access_mode, err;
1002 int ndescs = roundup(mr_init_attr->max_reg_descriptors, 4);
1003
1004 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1005 if (!mr)
1006 return ERR_PTR(-ENOMEM);
1007
1008 in = kzalloc(sizeof(*in), GFP_KERNEL);
1009 if (!in) {
1010 err = -ENOMEM;
1011 goto err_free;
1012 }
1013
1014 in->seg.status = 1 << 6; /* free */
1015 in->seg.xlt_oct_size = cpu_to_be32(ndescs);
1016 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
1017 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
1018 access_mode = MLX5_ACCESS_MODE_MTT;
1019
1020 if (mr_init_attr->flags & IB_MR_SIGNATURE_EN) {
1021 u32 psv_index[2];
1022
1023 in->seg.flags_pd = cpu_to_be32(be32_to_cpu(in->seg.flags_pd) |
1024 MLX5_MKEY_BSF_EN);
1025 in->seg.bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
1026 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
1027 if (!mr->sig) {
1028 err = -ENOMEM;
1029 goto err_free_in;
1030 }
1031
1032 /* create mem & wire PSVs */
1033 err = mlx5_core_create_psv(&dev->mdev, to_mpd(pd)->pdn,
1034 2, psv_index);
1035 if (err)
1036 goto err_free_sig;
1037
1038 access_mode = MLX5_ACCESS_MODE_KLM;
1039 mr->sig->psv_memory.psv_idx = psv_index[0];
1040 mr->sig->psv_wire.psv_idx = psv_index[1];
1041
1042 mr->sig->sig_status_checked = true;
1043 mr->sig->sig_err_exists = false;
1044 /* Next UMR, Arm SIGERR */
1045 ++mr->sig->sigerr_count;
1046 }
1047
1048 in->seg.flags = MLX5_PERM_UMR_EN | access_mode;
1049 err = mlx5_core_create_mkey(&dev->mdev, &mr->mmr, in, sizeof(*in),
1050 NULL, NULL, NULL);
1051 if (err)
1052 goto err_destroy_psv;
1053
1054 mr->ibmr.lkey = mr->mmr.key;
1055 mr->ibmr.rkey = mr->mmr.key;
1056 mr->umem = NULL;
1057 kfree(in);
1058
1059 return &mr->ibmr;
1060
1061err_destroy_psv:
1062 if (mr->sig) {
1063 if (mlx5_core_destroy_psv(&dev->mdev,
1064 mr->sig->psv_memory.psv_idx))
1065 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1066 mr->sig->psv_memory.psv_idx);
1067 if (mlx5_core_destroy_psv(&dev->mdev,
1068 mr->sig->psv_wire.psv_idx))
1069 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1070 mr->sig->psv_wire.psv_idx);
1071 }
1072err_free_sig:
1073 kfree(mr->sig);
1074err_free_in:
1075 kfree(in);
1076err_free:
1077 kfree(mr);
1078 return ERR_PTR(err);
1079}
1080
1081int mlx5_ib_destroy_mr(struct ib_mr *ibmr)
1082{
1083 struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
1084 struct mlx5_ib_mr *mr = to_mmr(ibmr);
1085 int err;
1086
1087 if (mr->sig) {
1088 if (mlx5_core_destroy_psv(&dev->mdev,
1089 mr->sig->psv_memory.psv_idx))
1090 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1091 mr->sig->psv_memory.psv_idx);
1092 if (mlx5_core_destroy_psv(&dev->mdev,
1093 mr->sig->psv_wire.psv_idx))
1094 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1095 mr->sig->psv_wire.psv_idx);
1096 kfree(mr->sig);
1097 }
1098
1099 err = mlx5_core_destroy_mkey(&dev->mdev, &mr->mmr);
1100 if (err) {
1101 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
1102 mr->mmr.key, err);
1103 return err;
1104 }
1105
1106 kfree(mr);
1107
1108 return err;
1109}
1110
995struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd, 1111struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd,
996 int max_page_list_len) 1112 int max_page_list_len)
997{ 1113{
@@ -1077,3 +1193,44 @@ void mlx5_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1077 kfree(mfrpl->ibfrpl.page_list); 1193 kfree(mfrpl->ibfrpl.page_list);
1078 kfree(mfrpl); 1194 kfree(mfrpl);
1079} 1195}
1196
1197int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1198 struct ib_mr_status *mr_status)
1199{
1200 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
1201 int ret = 0;
1202
1203 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
1204 pr_err("Invalid status check mask\n");
1205 ret = -EINVAL;
1206 goto done;
1207 }
1208
1209 mr_status->fail_status = 0;
1210 if (check_mask & IB_MR_CHECK_SIG_STATUS) {
1211 if (!mmr->sig) {
1212 ret = -EINVAL;
1213 pr_err("signature status check requested on a non-signature enabled MR\n");
1214 goto done;
1215 }
1216
1217 mmr->sig->sig_status_checked = true;
1218 if (!mmr->sig->sig_err_exists)
1219 goto done;
1220
1221 if (ibmr->lkey == mmr->sig->err_item.key)
1222 memcpy(&mr_status->sig_err, &mmr->sig->err_item,
1223 sizeof(mr_status->sig_err));
1224 else {
1225 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
1226 mr_status->sig_err.sig_err_offset = 0;
1227 mr_status->sig_err.key = mmr->sig->err_item.key;
1228 }
1229
1230 mmr->sig->sig_err_exists = false;
1231 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
1232 }
1233
1234done:
1235 return ret;
1236}
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 7dfe8a1c84cf..ae788d27b93f 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -256,8 +256,11 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr)
256 } 256 }
257 257
258 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg); 258 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
259 259 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN &&
260 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB); 260 ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
261 return MLX5_SIG_WQE_SIZE;
262 else
263 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
261} 264}
262 265
263static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, 266static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
@@ -284,6 +287,9 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
284 sizeof(struct mlx5_wqe_inline_seg); 287 sizeof(struct mlx5_wqe_inline_seg);
285 attr->cap.max_inline_data = qp->max_inline_data; 288 attr->cap.max_inline_data = qp->max_inline_data;
286 289
290 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
291 qp->signature_en = true;
292
287 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size); 293 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
288 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; 294 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
289 if (qp->sq.wqe_cnt > dev->mdev.caps.max_wqes) { 295 if (qp->sq.wqe_cnt > dev->mdev.caps.max_wqes) {
@@ -665,7 +671,7 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
665 int err; 671 int err;
666 672
667 uuari = &dev->mdev.priv.uuari; 673 uuari = &dev->mdev.priv.uuari;
668 if (init_attr->create_flags) 674 if (init_attr->create_flags & ~IB_QP_CREATE_SIGNATURE_EN)
669 return -EINVAL; 675 return -EINVAL;
670 676
671 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR) 677 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
@@ -1771,6 +1777,27 @@ static __be64 frwr_mkey_mask(void)
1771 return cpu_to_be64(result); 1777 return cpu_to_be64(result);
1772} 1778}
1773 1779
1780static __be64 sig_mkey_mask(void)
1781{
1782 u64 result;
1783
1784 result = MLX5_MKEY_MASK_LEN |
1785 MLX5_MKEY_MASK_PAGE_SIZE |
1786 MLX5_MKEY_MASK_START_ADDR |
1787 MLX5_MKEY_MASK_EN_SIGERR |
1788 MLX5_MKEY_MASK_EN_RINVAL |
1789 MLX5_MKEY_MASK_KEY |
1790 MLX5_MKEY_MASK_LR |
1791 MLX5_MKEY_MASK_LW |
1792 MLX5_MKEY_MASK_RR |
1793 MLX5_MKEY_MASK_RW |
1794 MLX5_MKEY_MASK_SMALL_FENCE |
1795 MLX5_MKEY_MASK_FREE |
1796 MLX5_MKEY_MASK_BSF_EN;
1797
1798 return cpu_to_be64(result);
1799}
1800
1774static void set_frwr_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, 1801static void set_frwr_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
1775 struct ib_send_wr *wr, int li) 1802 struct ib_send_wr *wr, int li)
1776{ 1803{
@@ -1826,7 +1853,7 @@ static u8 get_umr_flags(int acc)
1826 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) | 1853 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
1827 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) | 1854 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
1828 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) | 1855 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
1829 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN | MLX5_ACCESS_MODE_MTT; 1856 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
1830} 1857}
1831 1858
1832static void set_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr, 1859static void set_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr,
@@ -1838,7 +1865,8 @@ static void set_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr,
1838 return; 1865 return;
1839 } 1866 }
1840 1867
1841 seg->flags = get_umr_flags(wr->wr.fast_reg.access_flags); 1868 seg->flags = get_umr_flags(wr->wr.fast_reg.access_flags) |
1869 MLX5_ACCESS_MODE_MTT;
1842 *writ = seg->flags & (MLX5_PERM_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE); 1870 *writ = seg->flags & (MLX5_PERM_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE);
1843 seg->qpn_mkey7_0 = cpu_to_be32((wr->wr.fast_reg.rkey & 0xff) | 0xffffff00); 1871 seg->qpn_mkey7_0 = cpu_to_be32((wr->wr.fast_reg.rkey & 0xff) | 0xffffff00);
1844 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL); 1872 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
@@ -1954,6 +1982,342 @@ static int set_data_inl_seg(struct mlx5_ib_qp *qp, struct ib_send_wr *wr,
1954 return 0; 1982 return 0;
1955} 1983}
1956 1984
1985static u16 prot_field_size(enum ib_signature_type type)
1986{
1987 switch (type) {
1988 case IB_SIG_TYPE_T10_DIF:
1989 return MLX5_DIF_SIZE;
1990 default:
1991 return 0;
1992 }
1993}
1994
1995static u8 bs_selector(int block_size)
1996{
1997 switch (block_size) {
1998 case 512: return 0x1;
1999 case 520: return 0x2;
2000 case 4096: return 0x3;
2001 case 4160: return 0x4;
2002 case 1073741824: return 0x5;
2003 default: return 0;
2004 }
2005}
2006
2007static int format_selector(struct ib_sig_attrs *attr,
2008 struct ib_sig_domain *domain,
2009 int *selector)
2010{
2011
2012#define FORMAT_DIF_NONE 0
2013#define FORMAT_DIF_CRC_INC 8
2014#define FORMAT_DIF_CRC_NO_INC 12
2015#define FORMAT_DIF_CSUM_INC 13
2016#define FORMAT_DIF_CSUM_NO_INC 14
2017
2018 switch (domain->sig.dif.type) {
2019 case IB_T10DIF_NONE:
2020 /* No DIF */
2021 *selector = FORMAT_DIF_NONE;
2022 break;
2023 case IB_T10DIF_TYPE1: /* Fall through */
2024 case IB_T10DIF_TYPE2:
2025 switch (domain->sig.dif.bg_type) {
2026 case IB_T10DIF_CRC:
2027 *selector = FORMAT_DIF_CRC_INC;
2028 break;
2029 case IB_T10DIF_CSUM:
2030 *selector = FORMAT_DIF_CSUM_INC;
2031 break;
2032 default:
2033 return 1;
2034 }
2035 break;
2036 case IB_T10DIF_TYPE3:
2037 switch (domain->sig.dif.bg_type) {
2038 case IB_T10DIF_CRC:
2039 *selector = domain->sig.dif.type3_inc_reftag ?
2040 FORMAT_DIF_CRC_INC :
2041 FORMAT_DIF_CRC_NO_INC;
2042 break;
2043 case IB_T10DIF_CSUM:
2044 *selector = domain->sig.dif.type3_inc_reftag ?
2045 FORMAT_DIF_CSUM_INC :
2046 FORMAT_DIF_CSUM_NO_INC;
2047 break;
2048 default:
2049 return 1;
2050 }
2051 break;
2052 default:
2053 return 1;
2054 }
2055
2056 return 0;
2057}
2058
2059static int mlx5_set_bsf(struct ib_mr *sig_mr,
2060 struct ib_sig_attrs *sig_attrs,
2061 struct mlx5_bsf *bsf, u32 data_size)
2062{
2063 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
2064 struct mlx5_bsf_basic *basic = &bsf->basic;
2065 struct ib_sig_domain *mem = &sig_attrs->mem;
2066 struct ib_sig_domain *wire = &sig_attrs->wire;
2067 int ret, selector;
2068
2069 switch (sig_attrs->mem.sig_type) {
2070 case IB_SIG_TYPE_T10_DIF:
2071 if (sig_attrs->wire.sig_type != IB_SIG_TYPE_T10_DIF)
2072 return -EINVAL;
2073
2074 /* Input domain check byte mask */
2075 basic->check_byte_mask = sig_attrs->check_mask;
2076 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
2077 mem->sig.dif.type == wire->sig.dif.type) {
2078 /* Same block structure */
2079 basic->bsf_size_sbs = 1 << 4;
2080 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
2081 basic->wire.copy_byte_mask = 0xff;
2082 else
2083 basic->wire.copy_byte_mask = 0x3f;
2084 } else
2085 basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
2086
2087 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
2088 basic->raw_data_size = cpu_to_be32(data_size);
2089
2090 ret = format_selector(sig_attrs, mem, &selector);
2091 if (ret)
2092 return -EINVAL;
2093 basic->m_bfs_psv = cpu_to_be32(selector << 24 |
2094 msig->psv_memory.psv_idx);
2095
2096 ret = format_selector(sig_attrs, wire, &selector);
2097 if (ret)
2098 return -EINVAL;
2099 basic->w_bfs_psv = cpu_to_be32(selector << 24 |
2100 msig->psv_wire.psv_idx);
2101 break;
2102
2103 default:
2104 return -EINVAL;
2105 }
2106
2107 return 0;
2108}
2109
2110static int set_sig_data_segment(struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
2111 void **seg, int *size)
2112{
2113 struct ib_sig_attrs *sig_attrs = wr->wr.sig_handover.sig_attrs;
2114 struct ib_mr *sig_mr = wr->wr.sig_handover.sig_mr;
2115 struct mlx5_bsf *bsf;
2116 u32 data_len = wr->sg_list->length;
2117 u32 data_key = wr->sg_list->lkey;
2118 u64 data_va = wr->sg_list->addr;
2119 int ret;
2120 int wqe_size;
2121
2122 if (!wr->wr.sig_handover.prot) {
2123 /**
2124 * Source domain doesn't contain signature information
2125 * So need construct:
2126 * ------------------
2127 * | data_klm |
2128 * ------------------
2129 * | BSF |
2130 * ------------------
2131 **/
2132 struct mlx5_klm *data_klm = *seg;
2133
2134 data_klm->bcount = cpu_to_be32(data_len);
2135 data_klm->key = cpu_to_be32(data_key);
2136 data_klm->va = cpu_to_be64(data_va);
2137 wqe_size = ALIGN(sizeof(*data_klm), 64);
2138 } else {
2139 /**
2140 * Source domain contains signature information
2141 * So need construct a strided block format:
2142 * ---------------------------
2143 * | stride_block_ctrl |
2144 * ---------------------------
2145 * | data_klm |
2146 * ---------------------------
2147 * | prot_klm |
2148 * ---------------------------
2149 * | BSF |
2150 * ---------------------------
2151 **/
2152 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
2153 struct mlx5_stride_block_entry *data_sentry;
2154 struct mlx5_stride_block_entry *prot_sentry;
2155 u32 prot_key = wr->wr.sig_handover.prot->lkey;
2156 u64 prot_va = wr->wr.sig_handover.prot->addr;
2157 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
2158 int prot_size;
2159
2160 sblock_ctrl = *seg;
2161 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
2162 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
2163
2164 prot_size = prot_field_size(sig_attrs->mem.sig_type);
2165 if (!prot_size) {
2166 pr_err("Bad block size given: %u\n", block_size);
2167 return -EINVAL;
2168 }
2169 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
2170 prot_size);
2171 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
2172 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
2173 sblock_ctrl->num_entries = cpu_to_be16(2);
2174
2175 data_sentry->bcount = cpu_to_be16(block_size);
2176 data_sentry->key = cpu_to_be32(data_key);
2177 data_sentry->va = cpu_to_be64(data_va);
2178 prot_sentry->bcount = cpu_to_be16(prot_size);
2179 prot_sentry->key = cpu_to_be32(prot_key);
2180
2181 if (prot_key == data_key && prot_va == data_va) {
2182 /**
2183 * The data and protection are interleaved
2184 * in a single memory region
2185 **/
2186 prot_sentry->va = cpu_to_be64(data_va + block_size);
2187 prot_sentry->stride = cpu_to_be16(block_size + prot_size);
2188 data_sentry->stride = prot_sentry->stride;
2189 } else {
2190 /* The data and protection are two different buffers */
2191 prot_sentry->va = cpu_to_be64(prot_va);
2192 data_sentry->stride = cpu_to_be16(block_size);
2193 prot_sentry->stride = cpu_to_be16(prot_size);
2194 }
2195 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
2196 sizeof(*prot_sentry), 64);
2197 }
2198
2199 *seg += wqe_size;
2200 *size += wqe_size / 16;
2201 if (unlikely((*seg == qp->sq.qend)))
2202 *seg = mlx5_get_send_wqe(qp, 0);
2203
2204 bsf = *seg;
2205 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
2206 if (ret)
2207 return -EINVAL;
2208
2209 *seg += sizeof(*bsf);
2210 *size += sizeof(*bsf) / 16;
2211 if (unlikely((*seg == qp->sq.qend)))
2212 *seg = mlx5_get_send_wqe(qp, 0);
2213
2214 return 0;
2215}
2216
2217static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
2218 struct ib_send_wr *wr, u32 nelements,
2219 u32 length, u32 pdn)
2220{
2221 struct ib_mr *sig_mr = wr->wr.sig_handover.sig_mr;
2222 u32 sig_key = sig_mr->rkey;
2223 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
2224
2225 memset(seg, 0, sizeof(*seg));
2226
2227 seg->flags = get_umr_flags(wr->wr.sig_handover.access_flags) |
2228 MLX5_ACCESS_MODE_KLM;
2229 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
2230 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
2231 MLX5_MKEY_BSF_EN | pdn);
2232 seg->len = cpu_to_be64(length);
2233 seg->xlt_oct_size = cpu_to_be32(be16_to_cpu(get_klm_octo(nelements)));
2234 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
2235}
2236
2237static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
2238 struct ib_send_wr *wr, u32 nelements)
2239{
2240 memset(umr, 0, sizeof(*umr));
2241
2242 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
2243 umr->klm_octowords = get_klm_octo(nelements);
2244 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
2245 umr->mkey_mask = sig_mkey_mask();
2246}
2247
2248
2249static int set_sig_umr_wr(struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
2250 void **seg, int *size)
2251{
2252 struct mlx5_ib_mr *sig_mr = to_mmr(wr->wr.sig_handover.sig_mr);
2253 u32 pdn = get_pd(qp)->pdn;
2254 u32 klm_oct_size;
2255 int region_len, ret;
2256
2257 if (unlikely(wr->num_sge != 1) ||
2258 unlikely(wr->wr.sig_handover.access_flags &
2259 IB_ACCESS_REMOTE_ATOMIC) ||
2260 unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) ||
2261 unlikely(!sig_mr->sig->sig_status_checked))
2262 return -EINVAL;
2263
2264 /* length of the protected region, data + protection */
2265 region_len = wr->sg_list->length;
2266 if (wr->wr.sig_handover.prot)
2267 region_len += wr->wr.sig_handover.prot->length;
2268
2269 /**
2270 * KLM octoword size - if protection was provided
2271 * then we use strided block format (3 octowords),
2272 * else we use single KLM (1 octoword)
2273 **/
2274 klm_oct_size = wr->wr.sig_handover.prot ? 3 : 1;
2275
2276 set_sig_umr_segment(*seg, wr, klm_oct_size);
2277 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
2278 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
2279 if (unlikely((*seg == qp->sq.qend)))
2280 *seg = mlx5_get_send_wqe(qp, 0);
2281
2282 set_sig_mkey_segment(*seg, wr, klm_oct_size, region_len, pdn);
2283 *seg += sizeof(struct mlx5_mkey_seg);
2284 *size += sizeof(struct mlx5_mkey_seg) / 16;
2285 if (unlikely((*seg == qp->sq.qend)))
2286 *seg = mlx5_get_send_wqe(qp, 0);
2287
2288 ret = set_sig_data_segment(wr, qp, seg, size);
2289 if (ret)
2290 return ret;
2291
2292 sig_mr->sig->sig_status_checked = false;
2293 return 0;
2294}
2295
2296static int set_psv_wr(struct ib_sig_domain *domain,
2297 u32 psv_idx, void **seg, int *size)
2298{
2299 struct mlx5_seg_set_psv *psv_seg = *seg;
2300
2301 memset(psv_seg, 0, sizeof(*psv_seg));
2302 psv_seg->psv_num = cpu_to_be32(psv_idx);
2303 switch (domain->sig_type) {
2304 case IB_SIG_TYPE_T10_DIF:
2305 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
2306 domain->sig.dif.app_tag);
2307 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
2308
2309 *seg += sizeof(*psv_seg);
2310 *size += sizeof(*psv_seg) / 16;
2311 break;
2312
2313 default:
2314 pr_err("Bad signature type given.\n");
2315 return 1;
2316 }
2317
2318 return 0;
2319}
2320
1957static int set_frwr_li_wr(void **seg, struct ib_send_wr *wr, int *size, 2321static int set_frwr_li_wr(void **seg, struct ib_send_wr *wr, int *size,
1958 struct mlx5_core_dev *mdev, struct mlx5_ib_pd *pd, struct mlx5_ib_qp *qp) 2322 struct mlx5_core_dev *mdev, struct mlx5_ib_pd *pd, struct mlx5_ib_qp *qp)
1959{ 2323{
@@ -2041,6 +2405,59 @@ static u8 get_fence(u8 fence, struct ib_send_wr *wr)
2041 } 2405 }
2042} 2406}
2043 2407
2408static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
2409 struct mlx5_wqe_ctrl_seg **ctrl,
2410 struct ib_send_wr *wr, int *idx,
2411 int *size, int nreq)
2412{
2413 int err = 0;
2414
2415 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq))) {
2416 err = -ENOMEM;
2417 return err;
2418 }
2419
2420 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
2421 *seg = mlx5_get_send_wqe(qp, *idx);
2422 *ctrl = *seg;
2423 *(uint32_t *)(*seg + 8) = 0;
2424 (*ctrl)->imm = send_ieth(wr);
2425 (*ctrl)->fm_ce_se = qp->sq_signal_bits |
2426 (wr->send_flags & IB_SEND_SIGNALED ?
2427 MLX5_WQE_CTRL_CQ_UPDATE : 0) |
2428 (wr->send_flags & IB_SEND_SOLICITED ?
2429 MLX5_WQE_CTRL_SOLICITED : 0);
2430
2431 *seg += sizeof(**ctrl);
2432 *size = sizeof(**ctrl) / 16;
2433
2434 return err;
2435}
2436
2437static void finish_wqe(struct mlx5_ib_qp *qp,
2438 struct mlx5_wqe_ctrl_seg *ctrl,
2439 u8 size, unsigned idx, u64 wr_id,
2440 int nreq, u8 fence, u8 next_fence,
2441 u32 mlx5_opcode)
2442{
2443 u8 opmod = 0;
2444
2445 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
2446 mlx5_opcode | ((u32)opmod << 24));
2447 ctrl->qpn_ds = cpu_to_be32(size | (qp->mqp.qpn << 8));
2448 ctrl->fm_ce_se |= fence;
2449 qp->fm_cache = next_fence;
2450 if (unlikely(qp->wq_sig))
2451 ctrl->signature = wq_sig(ctrl);
2452
2453 qp->sq.wrid[idx] = wr_id;
2454 qp->sq.w_list[idx].opcode = mlx5_opcode;
2455 qp->sq.wqe_head[idx] = qp->sq.head + nreq;
2456 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
2457 qp->sq.w_list[idx].next = qp->sq.cur_post;
2458}
2459
2460
2044int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 2461int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2045 struct ib_send_wr **bad_wr) 2462 struct ib_send_wr **bad_wr)
2046{ 2463{
@@ -2048,13 +2465,13 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2048 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 2465 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2049 struct mlx5_core_dev *mdev = &dev->mdev; 2466 struct mlx5_core_dev *mdev = &dev->mdev;
2050 struct mlx5_ib_qp *qp = to_mqp(ibqp); 2467 struct mlx5_ib_qp *qp = to_mqp(ibqp);
2468 struct mlx5_ib_mr *mr;
2051 struct mlx5_wqe_data_seg *dpseg; 2469 struct mlx5_wqe_data_seg *dpseg;
2052 struct mlx5_wqe_xrc_seg *xrc; 2470 struct mlx5_wqe_xrc_seg *xrc;
2053 struct mlx5_bf *bf = qp->bf; 2471 struct mlx5_bf *bf = qp->bf;
2054 int uninitialized_var(size); 2472 int uninitialized_var(size);
2055 void *qend = qp->sq.qend; 2473 void *qend = qp->sq.qend;
2056 unsigned long flags; 2474 unsigned long flags;
2057 u32 mlx5_opcode;
2058 unsigned idx; 2475 unsigned idx;
2059 int err = 0; 2476 int err = 0;
2060 int inl = 0; 2477 int inl = 0;
@@ -2063,7 +2480,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2063 int nreq; 2480 int nreq;
2064 int i; 2481 int i;
2065 u8 next_fence = 0; 2482 u8 next_fence = 0;
2066 u8 opmod = 0;
2067 u8 fence; 2483 u8 fence;
2068 2484
2069 spin_lock_irqsave(&qp->sq.lock, flags); 2485 spin_lock_irqsave(&qp->sq.lock, flags);
@@ -2076,36 +2492,23 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2076 goto out; 2492 goto out;
2077 } 2493 }
2078 2494
2079 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq))) { 2495 fence = qp->fm_cache;
2496 num_sge = wr->num_sge;
2497 if (unlikely(num_sge > qp->sq.max_gs)) {
2080 mlx5_ib_warn(dev, "\n"); 2498 mlx5_ib_warn(dev, "\n");
2081 err = -ENOMEM; 2499 err = -ENOMEM;
2082 *bad_wr = wr; 2500 *bad_wr = wr;
2083 goto out; 2501 goto out;
2084 } 2502 }
2085 2503
2086 fence = qp->fm_cache; 2504 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq);
2087 num_sge = wr->num_sge; 2505 if (err) {
2088 if (unlikely(num_sge > qp->sq.max_gs)) {
2089 mlx5_ib_warn(dev, "\n"); 2506 mlx5_ib_warn(dev, "\n");
2090 err = -ENOMEM; 2507 err = -ENOMEM;
2091 *bad_wr = wr; 2508 *bad_wr = wr;
2092 goto out; 2509 goto out;
2093 } 2510 }
2094 2511
2095 idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
2096 seg = mlx5_get_send_wqe(qp, idx);
2097 ctrl = seg;
2098 *(uint32_t *)(seg + 8) = 0;
2099 ctrl->imm = send_ieth(wr);
2100 ctrl->fm_ce_se = qp->sq_signal_bits |
2101 (wr->send_flags & IB_SEND_SIGNALED ?
2102 MLX5_WQE_CTRL_CQ_UPDATE : 0) |
2103 (wr->send_flags & IB_SEND_SOLICITED ?
2104 MLX5_WQE_CTRL_SOLICITED : 0);
2105
2106 seg += sizeof(*ctrl);
2107 size = sizeof(*ctrl) / 16;
2108
2109 switch (ibqp->qp_type) { 2512 switch (ibqp->qp_type) {
2110 case IB_QPT_XRC_INI: 2513 case IB_QPT_XRC_INI:
2111 xrc = seg; 2514 xrc = seg;
@@ -2158,6 +2561,73 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2158 num_sge = 0; 2561 num_sge = 0;
2159 break; 2562 break;
2160 2563
2564 case IB_WR_REG_SIG_MR:
2565 qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR;
2566 mr = to_mmr(wr->wr.sig_handover.sig_mr);
2567
2568 ctrl->imm = cpu_to_be32(mr->ibmr.rkey);
2569 err = set_sig_umr_wr(wr, qp, &seg, &size);
2570 if (err) {
2571 mlx5_ib_warn(dev, "\n");
2572 *bad_wr = wr;
2573 goto out;
2574 }
2575
2576 finish_wqe(qp, ctrl, size, idx, wr->wr_id,
2577 nreq, get_fence(fence, wr),
2578 next_fence, MLX5_OPCODE_UMR);
2579 /*
2580 * SET_PSV WQEs are not signaled and solicited
2581 * on error
2582 */
2583 wr->send_flags &= ~IB_SEND_SIGNALED;
2584 wr->send_flags |= IB_SEND_SOLICITED;
2585 err = begin_wqe(qp, &seg, &ctrl, wr,
2586 &idx, &size, nreq);
2587 if (err) {
2588 mlx5_ib_warn(dev, "\n");
2589 err = -ENOMEM;
2590 *bad_wr = wr;
2591 goto out;
2592 }
2593
2594 err = set_psv_wr(&wr->wr.sig_handover.sig_attrs->mem,
2595 mr->sig->psv_memory.psv_idx, &seg,
2596 &size);
2597 if (err) {
2598 mlx5_ib_warn(dev, "\n");
2599 *bad_wr = wr;
2600 goto out;
2601 }
2602
2603 finish_wqe(qp, ctrl, size, idx, wr->wr_id,
2604 nreq, get_fence(fence, wr),
2605 next_fence, MLX5_OPCODE_SET_PSV);
2606 err = begin_wqe(qp, &seg, &ctrl, wr,
2607 &idx, &size, nreq);
2608 if (err) {
2609 mlx5_ib_warn(dev, "\n");
2610 err = -ENOMEM;
2611 *bad_wr = wr;
2612 goto out;
2613 }
2614
2615 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
2616 err = set_psv_wr(&wr->wr.sig_handover.sig_attrs->wire,
2617 mr->sig->psv_wire.psv_idx, &seg,
2618 &size);
2619 if (err) {
2620 mlx5_ib_warn(dev, "\n");
2621 *bad_wr = wr;
2622 goto out;
2623 }
2624
2625 finish_wqe(qp, ctrl, size, idx, wr->wr_id,
2626 nreq, get_fence(fence, wr),
2627 next_fence, MLX5_OPCODE_SET_PSV);
2628 num_sge = 0;
2629 goto skip_psv;
2630
2161 default: 2631 default:
2162 break; 2632 break;
2163 } 2633 }
@@ -2238,22 +2708,10 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2238 } 2708 }
2239 } 2709 }
2240 2710
2241 mlx5_opcode = mlx5_ib_opcode[wr->opcode]; 2711 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
2242 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) | 2712 get_fence(fence, wr), next_fence,
2243 mlx5_opcode | 2713 mlx5_ib_opcode[wr->opcode]);
2244 ((u32)opmod << 24)); 2714skip_psv:
2245 ctrl->qpn_ds = cpu_to_be32(size | (qp->mqp.qpn << 8));
2246 ctrl->fm_ce_se |= get_fence(fence, wr);
2247 qp->fm_cache = next_fence;
2248 if (unlikely(qp->wq_sig))
2249 ctrl->signature = wq_sig(ctrl);
2250
2251 qp->sq.wrid[idx] = wr->wr_id;
2252 qp->sq.w_list[idx].opcode = mlx5_opcode;
2253 qp->sq.wqe_head[idx] = qp->sq.head + nreq;
2254 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
2255 qp->sq.w_list[idx].next = qp->sq.cur_post;
2256
2257 if (0) 2715 if (0)
2258 dump_wqe(qp, idx, size); 2716 dump_wqe(qp, idx, size);
2259 } 2717 }
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 5b71d43bd89c..64408000f1c7 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -976,12 +976,12 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
976 u64 virt, int acc, struct ib_udata *udata) 976 u64 virt, int acc, struct ib_udata *udata)
977{ 977{
978 struct mthca_dev *dev = to_mdev(pd->device); 978 struct mthca_dev *dev = to_mdev(pd->device);
979 struct ib_umem_chunk *chunk; 979 struct scatterlist *sg;
980 struct mthca_mr *mr; 980 struct mthca_mr *mr;
981 struct mthca_reg_mr ucmd; 981 struct mthca_reg_mr ucmd;
982 u64 *pages; 982 u64 *pages;
983 int shift, n, len; 983 int shift, n, len;
984 int i, j, k; 984 int i, k, entry;
985 int err = 0; 985 int err = 0;
986 int write_mtt_size; 986 int write_mtt_size;
987 987
@@ -1009,10 +1009,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1009 } 1009 }
1010 1010
1011 shift = ffs(mr->umem->page_size) - 1; 1011 shift = ffs(mr->umem->page_size) - 1;
1012 1012 n = mr->umem->nmap;
1013 n = 0;
1014 list_for_each_entry(chunk, &mr->umem->chunk_list, list)
1015 n += chunk->nents;
1016 1013
1017 mr->mtt = mthca_alloc_mtt(dev, n); 1014 mr->mtt = mthca_alloc_mtt(dev, n);
1018 if (IS_ERR(mr->mtt)) { 1015 if (IS_ERR(mr->mtt)) {
@@ -1030,25 +1027,24 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1030 1027
1031 write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); 1028 write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages));
1032 1029
1033 list_for_each_entry(chunk, &mr->umem->chunk_list, list) 1030 for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1034 for (j = 0; j < chunk->nmap; ++j) { 1031 len = sg_dma_len(sg) >> shift;
1035 len = sg_dma_len(&chunk->page_list[j]) >> shift; 1032 for (k = 0; k < len; ++k) {
1036 for (k = 0; k < len; ++k) { 1033 pages[i++] = sg_dma_address(sg) +
1037 pages[i++] = sg_dma_address(&chunk->page_list[j]) + 1034 mr->umem->page_size * k;
1038 mr->umem->page_size * k; 1035 /*
1039 /* 1036 * Be friendly to write_mtt and pass it chunks
1040 * Be friendly to write_mtt and pass it chunks 1037 * of appropriate size.
1041 * of appropriate size. 1038 */
1042 */ 1039 if (i == write_mtt_size) {
1043 if (i == write_mtt_size) { 1040 err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
1044 err = mthca_write_mtt(dev, mr->mtt, n, pages, i); 1041 if (err)
1045 if (err) 1042 goto mtt_done;
1046 goto mtt_done; 1043 n += i;
1047 n += i; 1044 i = 0;
1048 i = 0;
1049 }
1050 } 1045 }
1051 } 1046 }
1047 }
1052 1048
1053 if (i) 1049 if (i)
1054 err = mthca_write_mtt(dev, mr->mtt, n, pages, i); 1050 err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 8308e3634767..32d3682daaf5 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -2307,7 +2307,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2307 struct nes_device *nesdev = nesvnic->nesdev; 2307 struct nes_device *nesdev = nesvnic->nesdev;
2308 struct nes_adapter *nesadapter = nesdev->nesadapter; 2308 struct nes_adapter *nesadapter = nesdev->nesadapter;
2309 struct ib_mr *ibmr = ERR_PTR(-EINVAL); 2309 struct ib_mr *ibmr = ERR_PTR(-EINVAL);
2310 struct ib_umem_chunk *chunk; 2310 struct scatterlist *sg;
2311 struct nes_ucontext *nes_ucontext; 2311 struct nes_ucontext *nes_ucontext;
2312 struct nes_pbl *nespbl; 2312 struct nes_pbl *nespbl;
2313 struct nes_mr *nesmr; 2313 struct nes_mr *nesmr;
@@ -2315,7 +2315,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2315 struct nes_mem_reg_req req; 2315 struct nes_mem_reg_req req;
2316 struct nes_vpbl vpbl; 2316 struct nes_vpbl vpbl;
2317 struct nes_root_vpbl root_vpbl; 2317 struct nes_root_vpbl root_vpbl;
2318 int nmap_index, page_index; 2318 int entry, page_index;
2319 int page_count = 0; 2319 int page_count = 0;
2320 int err, pbl_depth = 0; 2320 int err, pbl_depth = 0;
2321 int chunk_pages; 2321 int chunk_pages;
@@ -2330,6 +2330,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2330 u16 pbl_count; 2330 u16 pbl_count;
2331 u8 single_page = 1; 2331 u8 single_page = 1;
2332 u8 stag_key; 2332 u8 stag_key;
2333 int first_page = 1;
2333 2334
2334 region = ib_umem_get(pd->uobject->context, start, length, acc, 0); 2335 region = ib_umem_get(pd->uobject->context, start, length, acc, 0);
2335 if (IS_ERR(region)) { 2336 if (IS_ERR(region)) {
@@ -2380,128 +2381,125 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2380 } 2381 }
2381 nesmr->region = region; 2382 nesmr->region = region;
2382 2383
2383 list_for_each_entry(chunk, &region->chunk_list, list) { 2384 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
2384 nes_debug(NES_DBG_MR, "Chunk: nents = %u, nmap = %u .\n", 2385 if (sg_dma_address(sg) & ~PAGE_MASK) {
2385 chunk->nents, chunk->nmap); 2386 ib_umem_release(region);
2386 for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) { 2387 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2387 if (sg_dma_address(&chunk->page_list[nmap_index]) & ~PAGE_MASK) { 2388 nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n",
2388 ib_umem_release(region); 2389 (unsigned int) sg_dma_address(sg));
2389 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2390 ibmr = ERR_PTR(-EINVAL);
2390 nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n", 2391 kfree(nesmr);
2391 (unsigned int) sg_dma_address(&chunk->page_list[nmap_index])); 2392 goto reg_user_mr_err;
2392 ibmr = ERR_PTR(-EINVAL); 2393 }
2393 kfree(nesmr);
2394 goto reg_user_mr_err;
2395 }
2396 2394
2397 if (!sg_dma_len(&chunk->page_list[nmap_index])) { 2395 if (!sg_dma_len(sg)) {
2398 ib_umem_release(region); 2396 ib_umem_release(region);
2399 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2397 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2400 stag_index); 2398 stag_index);
2401 nes_debug(NES_DBG_MR, "Invalid Buffer Size\n"); 2399 nes_debug(NES_DBG_MR, "Invalid Buffer Size\n");
2402 ibmr = ERR_PTR(-EINVAL); 2400 ibmr = ERR_PTR(-EINVAL);
2403 kfree(nesmr); 2401 kfree(nesmr);
2404 goto reg_user_mr_err; 2402 goto reg_user_mr_err;
2405 } 2403 }
2406 2404
2407 region_length += sg_dma_len(&chunk->page_list[nmap_index]); 2405 region_length += sg_dma_len(sg);
2408 chunk_pages = sg_dma_len(&chunk->page_list[nmap_index]) >> 12; 2406 chunk_pages = sg_dma_len(sg) >> 12;
2409 region_length -= skip_pages << 12; 2407 region_length -= skip_pages << 12;
2410 for (page_index=skip_pages; page_index < chunk_pages; page_index++) { 2408 for (page_index = skip_pages; page_index < chunk_pages; page_index++) {
2411 skip_pages = 0; 2409 skip_pages = 0;
2412 if ((page_count!=0)&&(page_count<<12)-(region->offset&(4096-1))>=region->length) 2410 if ((page_count != 0) && (page_count<<12)-(region->offset&(4096-1)) >= region->length)
2413 goto enough_pages; 2411 goto enough_pages;
2414 if ((page_count&0x01FF) == 0) { 2412 if ((page_count&0x01FF) == 0) {
2415 if (page_count >= 1024 * 512) { 2413 if (page_count >= 1024 * 512) {
2414 ib_umem_release(region);
2415 nes_free_resource(nesadapter,
2416 nesadapter->allocated_mrs, stag_index);
2417 kfree(nesmr);
2418 ibmr = ERR_PTR(-E2BIG);
2419 goto reg_user_mr_err;
2420 }
2421 if (root_pbl_index == 1) {
2422 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
2423 8192, &root_vpbl.pbl_pbase);
2424 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2425 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
2426 if (!root_vpbl.pbl_vbase) {
2416 ib_umem_release(region); 2427 ib_umem_release(region);
2417 nes_free_resource(nesadapter, 2428 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2418 nesadapter->allocated_mrs, stag_index); 2429 vpbl.pbl_pbase);
2430 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2431 stag_index);
2419 kfree(nesmr); 2432 kfree(nesmr);
2420 ibmr = ERR_PTR(-E2BIG); 2433 ibmr = ERR_PTR(-ENOMEM);
2421 goto reg_user_mr_err; 2434 goto reg_user_mr_err;
2422 } 2435 }
2423 if (root_pbl_index == 1) { 2436 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024,
2424 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 2437 GFP_KERNEL);
2425 8192, &root_vpbl.pbl_pbase); 2438 if (!root_vpbl.leaf_vpbl) {
2426 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2427 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
2428 if (!root_vpbl.pbl_vbase) {
2429 ib_umem_release(region);
2430 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2431 vpbl.pbl_pbase);
2432 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2433 stag_index);
2434 kfree(nesmr);
2435 ibmr = ERR_PTR(-ENOMEM);
2436 goto reg_user_mr_err;
2437 }
2438 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024,
2439 GFP_KERNEL);
2440 if (!root_vpbl.leaf_vpbl) {
2441 ib_umem_release(region);
2442 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
2443 root_vpbl.pbl_pbase);
2444 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2445 vpbl.pbl_pbase);
2446 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2447 stag_index);
2448 kfree(nesmr);
2449 ibmr = ERR_PTR(-ENOMEM);
2450 goto reg_user_mr_err;
2451 }
2452 root_vpbl.pbl_vbase[0].pa_low =
2453 cpu_to_le32((u32)vpbl.pbl_pbase);
2454 root_vpbl.pbl_vbase[0].pa_high =
2455 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
2456 root_vpbl.leaf_vpbl[0] = vpbl;
2457 }
2458 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
2459 &vpbl.pbl_pbase);
2460 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
2461 vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase);
2462 if (!vpbl.pbl_vbase) {
2463 ib_umem_release(region); 2439 ib_umem_release(region);
2464 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2440 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
2465 ibmr = ERR_PTR(-ENOMEM); 2441 root_vpbl.pbl_pbase);
2442 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2443 vpbl.pbl_pbase);
2444 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2445 stag_index);
2466 kfree(nesmr); 2446 kfree(nesmr);
2447 ibmr = ERR_PTR(-ENOMEM);
2467 goto reg_user_mr_err; 2448 goto reg_user_mr_err;
2468 } 2449 }
2469 if (1 <= root_pbl_index) { 2450 root_vpbl.pbl_vbase[0].pa_low =
2470 root_vpbl.pbl_vbase[root_pbl_index].pa_low = 2451 cpu_to_le32((u32)vpbl.pbl_pbase);
2471 cpu_to_le32((u32)vpbl.pbl_pbase); 2452 root_vpbl.pbl_vbase[0].pa_high =
2472 root_vpbl.pbl_vbase[root_pbl_index].pa_high = 2453 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
2473 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32))); 2454 root_vpbl.leaf_vpbl[0] = vpbl;
2474 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
2475 }
2476 root_pbl_index++;
2477 cur_pbl_index = 0;
2478 } 2455 }
2479 if (single_page) { 2456 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
2480 if (page_count != 0) { 2457 &vpbl.pbl_pbase);
2481 if ((last_dma_addr+4096) != 2458 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
2482 (sg_dma_address(&chunk->page_list[nmap_index])+ 2459 vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase);
2483 (page_index*4096))) 2460 if (!vpbl.pbl_vbase) {
2484 single_page = 0; 2461 ib_umem_release(region);
2485 last_dma_addr = sg_dma_address(&chunk->page_list[nmap_index])+ 2462 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2486 (page_index*4096); 2463 ibmr = ERR_PTR(-ENOMEM);
2487 } else { 2464 kfree(nesmr);
2488 first_dma_addr = sg_dma_address(&chunk->page_list[nmap_index])+ 2465 goto reg_user_mr_err;
2489 (page_index*4096); 2466 }
2490 last_dma_addr = first_dma_addr; 2467 if (1 <= root_pbl_index) {
2491 } 2468 root_vpbl.pbl_vbase[root_pbl_index].pa_low =
2469 cpu_to_le32((u32)vpbl.pbl_pbase);
2470 root_vpbl.pbl_vbase[root_pbl_index].pa_high =
2471 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32)));
2472 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
2473 }
2474 root_pbl_index++;
2475 cur_pbl_index = 0;
2476 }
2477 if (single_page) {
2478 if (page_count != 0) {
2479 if ((last_dma_addr+4096) !=
2480 (sg_dma_address(sg)+
2481 (page_index*4096)))
2482 single_page = 0;
2483 last_dma_addr = sg_dma_address(sg)+
2484 (page_index*4096);
2485 } else {
2486 first_dma_addr = sg_dma_address(sg)+
2487 (page_index*4096);
2488 last_dma_addr = first_dma_addr;
2492 } 2489 }
2493
2494 vpbl.pbl_vbase[cur_pbl_index].pa_low =
2495 cpu_to_le32((u32)(sg_dma_address(&chunk->page_list[nmap_index])+
2496 (page_index*4096)));
2497 vpbl.pbl_vbase[cur_pbl_index].pa_high =
2498 cpu_to_le32((u32)((((u64)(sg_dma_address(&chunk->page_list[nmap_index])+
2499 (page_index*4096))) >> 32)));
2500 cur_pbl_index++;
2501 page_count++;
2502 } 2490 }
2491
2492 vpbl.pbl_vbase[cur_pbl_index].pa_low =
2493 cpu_to_le32((u32)(sg_dma_address(sg)+
2494 (page_index*4096)));
2495 vpbl.pbl_vbase[cur_pbl_index].pa_high =
2496 cpu_to_le32((u32)((((u64)(sg_dma_address(sg)+
2497 (page_index*4096))) >> 32)));
2498 cur_pbl_index++;
2499 page_count++;
2503 } 2500 }
2504 } 2501 }
2502
2505 enough_pages: 2503 enough_pages:
2506 nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x," 2504 nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x,"
2507 " stag_key=0x%08x\n", 2505 " stag_key=0x%08x\n",
@@ -2613,25 +2611,28 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2613 nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase, 2611 nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase,
2614 (void *) nespbl->pbl_vbase, nespbl->user_base); 2612 (void *) nespbl->pbl_vbase, nespbl->user_base);
2615 2613
2616 list_for_each_entry(chunk, &region->chunk_list, list) { 2614 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
2617 for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) { 2615 chunk_pages = sg_dma_len(sg) >> 12;
2618 chunk_pages = sg_dma_len(&chunk->page_list[nmap_index]) >> 12; 2616 chunk_pages += (sg_dma_len(sg) & (4096-1)) ? 1 : 0;
2619 chunk_pages += (sg_dma_len(&chunk->page_list[nmap_index]) & (4096-1)) ? 1 : 0; 2617 if (first_page) {
2620 nespbl->page = sg_page(&chunk->page_list[0]); 2618 nespbl->page = sg_page(sg);
2621 for (page_index=0; page_index<chunk_pages; page_index++) { 2619 first_page = 0;
2622 ((__le32 *)pbl)[0] = cpu_to_le32((u32) 2620 }
2623 (sg_dma_address(&chunk->page_list[nmap_index])+ 2621
2624 (page_index*4096))); 2622 for (page_index = 0; page_index < chunk_pages; page_index++) {
2625 ((__le32 *)pbl)[1] = cpu_to_le32(((u64) 2623 ((__le32 *)pbl)[0] = cpu_to_le32((u32)
2626 (sg_dma_address(&chunk->page_list[nmap_index])+ 2624 (sg_dma_address(sg)+
2627 (page_index*4096)))>>32); 2625 (page_index*4096)));
2628 nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl, 2626 ((__le32 *)pbl)[1] = cpu_to_le32(((u64)
2629 (unsigned long long)*pbl, 2627 (sg_dma_address(sg)+
2630 le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0])); 2628 (page_index*4096)))>>32);
2631 pbl++; 2629 nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl,
2632 } 2630 (unsigned long long)*pbl,
2631 le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0]));
2632 pbl++;
2633 } 2633 }
2634 } 2634 }
2635
2635 if (req.reg_type == IWNES_MEMREG_TYPE_QP) { 2636 if (req.reg_type == IWNES_MEMREG_TYPE_QP) {
2636 list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list); 2637 list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list);
2637 } else { 2638 } else {
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index e0cc201be41a..0de3473fa7d9 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -726,10 +726,10 @@ static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
726 u32 num_pbes) 726 u32 num_pbes)
727{ 727{
728 struct ocrdma_pbe *pbe; 728 struct ocrdma_pbe *pbe;
729 struct ib_umem_chunk *chunk; 729 struct scatterlist *sg;
730 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 730 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
731 struct ib_umem *umem = mr->umem; 731 struct ib_umem *umem = mr->umem;
732 int i, shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 732 int shift, pg_cnt, pages, pbe_cnt, entry, total_num_pbes = 0;
733 733
734 if (!mr->hwmr.num_pbes) 734 if (!mr->hwmr.num_pbes)
735 return; 735 return;
@@ -739,39 +739,37 @@ static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
739 739
740 shift = ilog2(umem->page_size); 740 shift = ilog2(umem->page_size);
741 741
742 list_for_each_entry(chunk, &umem->chunk_list, list) { 742 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
743 /* get all the dma regions from the chunk. */ 743 pages = sg_dma_len(sg) >> shift;
744 for (i = 0; i < chunk->nmap; i++) { 744 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
745 pages = sg_dma_len(&chunk->page_list[i]) >> shift; 745 /* store the page address in pbe */
746 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 746 pbe->pa_lo =
747 /* store the page address in pbe */ 747 cpu_to_le32(sg_dma_address
748 pbe->pa_lo = 748 (sg) +
749 cpu_to_le32(sg_dma_address 749 (umem->page_size * pg_cnt));
750 (&chunk->page_list[i]) + 750 pbe->pa_hi =
751 (umem->page_size * pg_cnt)); 751 cpu_to_le32(upper_32_bits
752 pbe->pa_hi = 752 ((sg_dma_address
753 cpu_to_le32(upper_32_bits 753 (sg) +
754 ((sg_dma_address 754 umem->page_size * pg_cnt)));
755 (&chunk->page_list[i]) + 755 pbe_cnt += 1;
756 umem->page_size * pg_cnt))); 756 total_num_pbes += 1;
757 pbe_cnt += 1; 757 pbe++;
758 total_num_pbes += 1; 758
759 pbe++; 759 /* if done building pbes, issue the mbx cmd. */
760 760 if (total_num_pbes == num_pbes)
761 /* if done building pbes, issue the mbx cmd. */ 761 return;
762 if (total_num_pbes == num_pbes) 762
763 return; 763 /* if the given pbl is full storing the pbes,
764 764 * move to next pbl.
765 /* if the given pbl is full storing the pbes, 765 */
766 * move to next pbl. 766 if (pbe_cnt ==
767 */ 767 (mr->hwmr.pbl_size / sizeof(u64))) {
768 if (pbe_cnt == 768 pbl_tbl++;
769 (mr->hwmr.pbl_size / sizeof(u64))) { 769 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
770 pbl_tbl++; 770 pbe_cnt = 0;
771 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
772 pbe_cnt = 0;
773 }
774 } 771 }
772
775 } 773 }
776 } 774 }
777} 775}
diff --git a/drivers/infiniband/hw/qib/qib_mr.c b/drivers/infiniband/hw/qib/qib_mr.c
index e6687ded8210..9bbb55347cc1 100644
--- a/drivers/infiniband/hw/qib/qib_mr.c
+++ b/drivers/infiniband/hw/qib/qib_mr.c
@@ -232,8 +232,8 @@ struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
232{ 232{
233 struct qib_mr *mr; 233 struct qib_mr *mr;
234 struct ib_umem *umem; 234 struct ib_umem *umem;
235 struct ib_umem_chunk *chunk; 235 struct scatterlist *sg;
236 int n, m, i; 236 int n, m, entry;
237 struct ib_mr *ret; 237 struct ib_mr *ret;
238 238
239 if (length == 0) { 239 if (length == 0) {
@@ -246,9 +246,7 @@ struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
246 if (IS_ERR(umem)) 246 if (IS_ERR(umem))
247 return (void *) umem; 247 return (void *) umem;
248 248
249 n = 0; 249 n = umem->nmap;
250 list_for_each_entry(chunk, &umem->chunk_list, list)
251 n += chunk->nents;
252 250
253 mr = alloc_mr(n, pd); 251 mr = alloc_mr(n, pd);
254 if (IS_ERR(mr)) { 252 if (IS_ERR(mr)) {
@@ -268,11 +266,10 @@ struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
268 mr->mr.page_shift = ilog2(umem->page_size); 266 mr->mr.page_shift = ilog2(umem->page_size);
269 m = 0; 267 m = 0;
270 n = 0; 268 n = 0;
271 list_for_each_entry(chunk, &umem->chunk_list, list) { 269 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
272 for (i = 0; i < chunk->nents; i++) {
273 void *vaddr; 270 void *vaddr;
274 271
275 vaddr = page_address(sg_page(&chunk->page_list[i])); 272 vaddr = page_address(sg_page(sg));
276 if (!vaddr) { 273 if (!vaddr) {
277 ret = ERR_PTR(-EINVAL); 274 ret = ERR_PTR(-EINVAL);
278 goto bail; 275 goto bail;
@@ -284,7 +281,6 @@ struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
284 m++; 281 m++;
285 n = 0; 282 n = 0;
286 } 283 }
287 }
288 } 284 }
289 ret = &mr->ibmr; 285 ret = &mr->ibmr;
290 286
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-14 09:38:43 -0500