aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYishai Hadas <yishaih@mellanox.com>2014-01-28 06:40:15 -0500
committerRoland Dreier <roland@purestorage.com>2014-03-04 13:34:28 -0500
commiteeb8461e36c99fdf2d058751be924a2aab215005 (patch)
treec92498349f842be5985194c840e2dd12201df861
parentcfbf8d4857c26a8a307fb7cd258074c9dcd8c691 (diff)
IB: Refactor umem to use linear SG table
This patch refactors the IB core umem code and vendor drivers to use a linear (chained) SG table instead of chunk list. With this change the relevant code becomes clearer—no need for nested loops to build and use umem. Signed-off-by: Shachar Raindel <raindel@mellanox.com> Signed-off-by: Yishai Hadas <yishaih@mellanox.com> Signed-off-by: Roland Dreier <roland@purestorage.com>
Diffstat
-rw-r--r--drivers/infiniband/core/umem.c120
-rw-r--r--drivers/infiniband/hw/amso1100/c2_provider.c23
-rw-r--r--drivers/infiniband/hw/cxgb3/iwch_provider.c19
-rw-r--r--drivers/infiniband/hw/cxgb4/mem.c39
-rw-r--r--drivers/infiniband/hw/ehca/ehca_classes.h2
-rw-r--r--drivers/infiniband/hw/ehca/ehca_mrmw.c245
-rw-r--r--drivers/infiniband/hw/ipath/ipath_mr.c39
-rw-r--r--drivers/infiniband/hw/mlx4/doorbell.c4
-rw-r--r--drivers/infiniband/hw/mlx4/mr.c39
-rw-r--r--drivers/infiniband/hw/mlx5/doorbell.c4
-rw-r--r--drivers/infiniband/hw/mlx5/mem.c80
-rw-r--r--drivers/infiniband/hw/mthca/mthca_provider.c42
-rw-r--r--drivers/infiniband/hw/nes/nes_verbs.c253
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.c66
-rw-r--r--drivers/infiniband/hw/qib/qib_mr.c14
-rw-r--r--include/rdma/ib_umem.h11
16 files changed, 448 insertions, 552 deletions
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index a84112322071..a3a2e9c1639b 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -42,29 +42,29 @@
42 42
43#include "uverbs.h" 43#include "uverbs.h"
44 44
45#define IB_UMEM_MAX_PAGE_CHUNK \
46 ((PAGE_SIZE - offsetof(struct ib_umem_chunk, page_list)) / \
47 ((void *) &((struct ib_umem_chunk *) 0)->page_list[1] - \
48 (void *) &((struct ib_umem_chunk *) 0)->page_list[0]))
49 45
50static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty) 46static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)
51{ 47{
52 struct ib_umem_chunk *chunk, *tmp; 48 struct scatterlist *sg;
49 struct page *page;
53 int i; 50 int i;
54 51
55 list_for_each_entry_safe(chunk, tmp, &umem->chunk_list, list) { 52 if (umem->nmap > 0)
56 ib_dma_unmap_sg(dev, chunk->page_list, 53 ib_dma_unmap_sg(dev, umem->sg_head.sgl,
57 chunk->nents, DMA_BIDIRECTIONAL); 54 umem->nmap,
58 for (i = 0; i < chunk->nents; ++i) { 55 DMA_BIDIRECTIONAL);
59 struct page *page = sg_page(&chunk->page_list[i]);
60 56
61 if (umem->writable && dirty) 57 for_each_sg(umem->sg_head.sgl, sg, umem->npages, i) {
62 set_page_dirty_lock(page);
63 put_page(page);
64 }
65 58
66 kfree(chunk); 59 page = sg_page(sg);
60 if (umem->writable && dirty)
61 set_page_dirty_lock(page);
62 put_page(page);
67 } 63 }
64
65 sg_free_table(&umem->sg_head);
66 return;
67
68} 68}
69 69
70/** 70/**
@@ -81,15 +81,15 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
81 struct ib_umem *umem; 81 struct ib_umem *umem;
82 struct page **page_list; 82 struct page **page_list;
83 struct vm_area_struct **vma_list; 83 struct vm_area_struct **vma_list;
84 struct ib_umem_chunk *chunk;
85 unsigned long locked; 84 unsigned long locked;
86 unsigned long lock_limit; 85 unsigned long lock_limit;
87 unsigned long cur_base; 86 unsigned long cur_base;
88 unsigned long npages; 87 unsigned long npages;
89 int ret; 88 int ret;
90 int off;
91 int i; 89 int i;
92 DEFINE_DMA_ATTRS(attrs); 90 DEFINE_DMA_ATTRS(attrs);
91 struct scatterlist *sg, *sg_list_start;
92 int need_release = 0;
93 93
94 if (dmasync) 94 if (dmasync)
95 dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs); 95 dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs);
@@ -97,7 +97,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
97 if (!can_do_mlock()) 97 if (!can_do_mlock())
98 return ERR_PTR(-EPERM); 98 return ERR_PTR(-EPERM);
99 99
100 umem = kmalloc(sizeof *umem, GFP_KERNEL); 100 umem = kzalloc(sizeof *umem, GFP_KERNEL);
101 if (!umem) 101 if (!umem)
102 return ERR_PTR(-ENOMEM); 102 return ERR_PTR(-ENOMEM);
103 103
@@ -117,8 +117,6 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
117 /* We assume the memory is from hugetlb until proved otherwise */ 117 /* We assume the memory is from hugetlb until proved otherwise */
118 umem->hugetlb = 1; 118 umem->hugetlb = 1;
119 119
120 INIT_LIST_HEAD(&umem->chunk_list);
121
122 page_list = (struct page **) __get_free_page(GFP_KERNEL); 120 page_list = (struct page **) __get_free_page(GFP_KERNEL);
123 if (!page_list) { 121 if (!page_list) {
124 kfree(umem); 122 kfree(umem);
@@ -147,7 +145,18 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
147 145
148 cur_base = addr & PAGE_MASK; 146 cur_base = addr & PAGE_MASK;
149 147
150 ret = 0; 148 if (npages == 0) {
149 ret = -EINVAL;
150 goto out;
151 }
152
153 ret = sg_alloc_table(&umem->sg_head, npages, GFP_KERNEL);
154 if (ret)
155 goto out;
156
157 need_release = 1;
158 sg_list_start = umem->sg_head.sgl;
159
151 while (npages) { 160 while (npages) {
152 ret = get_user_pages(current, current->mm, cur_base, 161 ret = get_user_pages(current, current->mm, cur_base,
153 min_t(unsigned long, npages, 162 min_t(unsigned long, npages,
@@ -157,54 +166,38 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
157 if (ret < 0) 166 if (ret < 0)
158 goto out; 167 goto out;
159 168
169 umem->npages += ret;
160 cur_base += ret * PAGE_SIZE; 170 cur_base += ret * PAGE_SIZE;
161 npages -= ret; 171 npages -= ret;
162 172
163 off = 0; 173 for_each_sg(sg_list_start, sg, ret, i) {
164 174 if (vma_list && !is_vm_hugetlb_page(vma_list[i]))
165 while (ret) { 175 umem->hugetlb = 0;
166 chunk = kmalloc(sizeof *chunk + sizeof (struct scatterlist) * 176
167 min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK), 177 sg_set_page(sg, page_list[i], PAGE_SIZE, 0);
168 GFP_KERNEL);
169 if (!chunk) {
170 ret = -ENOMEM;
171 goto out;
172 }
173
174 chunk->nents = min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK);
175 sg_init_table(chunk->page_list, chunk->nents);
176 for (i = 0; i < chunk->nents; ++i) {
177 if (vma_list &&
178 !is_vm_hugetlb_page(vma_list[i + off]))
179 umem->hugetlb = 0;
180 sg_set_page(&chunk->page_list[i], page_list[i + off], PAGE_SIZE, 0);
181 }
182
183 chunk->nmap = ib_dma_map_sg_attrs(context->device,
184 &chunk->page_list[0],
185 chunk->nents,
186 DMA_BIDIRECTIONAL,
187 &attrs);
188 if (chunk->nmap <= 0) {
189 for (i = 0; i < chunk->nents; ++i)
190 put_page(sg_page(&chunk->page_list[i]));
191 kfree(chunk);
192
193 ret = -ENOMEM;
194 goto out;
195 }
196
197 ret -= chunk->nents;
198 off += chunk->nents;
199 list_add_tail(&chunk->list, &umem->chunk_list);
200 } 178 }
201 179
202 ret = 0; 180 /* preparing for next loop */
181 sg_list_start = sg;
203 } 182 }
204 183
184 umem->nmap = ib_dma_map_sg_attrs(context->device,
185 umem->sg_head.sgl,
186 umem->npages,
187 DMA_BIDIRECTIONAL,
188 &attrs);
189
190 if (umem->nmap <= 0) {
191 ret = -ENOMEM;
192 goto out;
193 }
194
195 ret = 0;
196
205out: 197out:
206 if (ret < 0) { 198 if (ret < 0) {
207 __ib_umem_release(context->device, umem, 0); 199 if (need_release)
200 __ib_umem_release(context->device, umem, 0);
208 kfree(umem); 201 kfree(umem);
209 } else 202 } else
210 current->mm->pinned_vm = locked; 203 current->mm->pinned_vm = locked;
@@ -278,17 +271,16 @@ EXPORT_SYMBOL(ib_umem_release);
278 271
279int ib_umem_page_count(struct ib_umem *umem) 272int ib_umem_page_count(struct ib_umem *umem)
280{ 273{
281 struct ib_umem_chunk *chunk;
282 int shift; 274 int shift;
283 int i; 275 int i;
284 int n; 276 int n;
277 struct scatterlist *sg;
285 278
286 shift = ilog2(umem->page_size); 279 shift = ilog2(umem->page_size);
287 280
288 n = 0; 281 n = 0;
289 list_for_each_entry(chunk, &umem->chunk_list, list) 282 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i)
290 for (i = 0; i < chunk->nmap; ++i) 283 n += sg_dma_len(sg) >> shift;
291 n += sg_dma_len(&chunk->page_list[i]) >> shift;
292 284
293 return n; 285 return n;
294} 286}
diff --git a/drivers/infiniband/hw/amso1100/c2_provider.c b/drivers/infiniband/hw/amso1100/c2_provider.c
index 07eb3a8067d8..8af33cf1fc4e 100644
--- a/drivers/infiniband/hw/amso1100/c2_provider.c
+++ b/drivers/infiniband/hw/amso1100/c2_provider.c
@@ -431,9 +431,9 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
431 u64 *pages; 431 u64 *pages;
432 u64 kva = 0; 432 u64 kva = 0;
433 int shift, n, len; 433 int shift, n, len;
434 int i, j, k; 434 int i, k, entry;
435 int err = 0; 435 int err = 0;
436 struct ib_umem_chunk *chunk; 436 struct scatterlist *sg;
437 struct c2_pd *c2pd = to_c2pd(pd); 437 struct c2_pd *c2pd = to_c2pd(pd);
438 struct c2_mr *c2mr; 438 struct c2_mr *c2mr;
439 439
@@ -452,10 +452,7 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
452 } 452 }
453 453
454 shift = ffs(c2mr->umem->page_size) - 1; 454 shift = ffs(c2mr->umem->page_size) - 1;
455 455 n = c2mr->umem->nmap;
456 n = 0;
457 list_for_each_entry(chunk, &c2mr->umem->chunk_list, list)
458 n += chunk->nents;
459 456
460 pages = kmalloc(n * sizeof(u64), GFP_KERNEL); 457 pages = kmalloc(n * sizeof(u64), GFP_KERNEL);
461 if (!pages) { 458 if (!pages) {
@@ -464,14 +461,12 @@ static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
464 } 461 }
465 462
466 i = 0; 463 i = 0;
467 list_for_each_entry(chunk, &c2mr->umem->chunk_list, list) { 464 for_each_sg(c2mr->umem->sg_head.sgl, sg, c2mr->umem->nmap, entry) {
468 for (j = 0; j < chunk->nmap; ++j) { 465 len = sg_dma_len(sg) >> shift;
469 len = sg_dma_len(&chunk->page_list[j]) >> shift; 466 for (k = 0; k < len; ++k) {
470 for (k = 0; k < len; ++k) { 467 pages[i++] =
471 pages[i++] = 468 sg_dma_address(sg) +
472 sg_dma_address(&chunk->page_list[j]) + 469 (c2mr->umem->page_size * k);
473 (c2mr->umem->page_size * k);
474 }
475 } 470 }
476 } 471 }
477 472
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index d2283837d451..811b24a539c0 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -618,14 +618,13 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
618{ 618{
619 __be64 *pages; 619 __be64 *pages;
620 int shift, n, len; 620 int shift, n, len;
621 int i, j, k; 621 int i, k, entry;
622 int err = 0; 622 int err = 0;
623 struct ib_umem_chunk *chunk;
624 struct iwch_dev *rhp; 623 struct iwch_dev *rhp;
625 struct iwch_pd *php; 624 struct iwch_pd *php;
626 struct iwch_mr *mhp; 625 struct iwch_mr *mhp;
627 struct iwch_reg_user_mr_resp uresp; 626 struct iwch_reg_user_mr_resp uresp;
628 627 struct scatterlist *sg;
629 PDBG("%s ib_pd %p\n", __func__, pd); 628 PDBG("%s ib_pd %p\n", __func__, pd);
630 629
631 php = to_iwch_pd(pd); 630 php = to_iwch_pd(pd);
@@ -645,9 +644,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
645 644
646 shift = ffs(mhp->umem->page_size) - 1; 645 shift = ffs(mhp->umem->page_size) - 1;
647 646
648 n = 0; 647 n = mhp->umem->nmap;
649 list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
650 n += chunk->nents;
651 648
652 err = iwch_alloc_pbl(mhp, n); 649 err = iwch_alloc_pbl(mhp, n);
653 if (err) 650 if (err)
@@ -661,12 +658,10 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
661 658
662 i = n = 0; 659 i = n = 0;
663 660
664 list_for_each_entry(chunk, &mhp->umem->chunk_list, list) 661 for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
665 for (j = 0; j < chunk->nmap; ++j) { 662 len = sg_dma_len(sg) >> shift;
666 len = sg_dma_len(&chunk->page_list[j]) >> shift;
667 for (k = 0; k < len; ++k) { 663 for (k = 0; k < len; ++k) {
668 pages[i++] = cpu_to_be64(sg_dma_address( 664 pages[i++] = cpu_to_be64(sg_dma_address(sg) +
669 &chunk->page_list[j]) +
670 mhp->umem->page_size * k); 665 mhp->umem->page_size * k);
671 if (i == PAGE_SIZE / sizeof *pages) { 666 if (i == PAGE_SIZE / sizeof *pages) {
672 err = iwch_write_pbl(mhp, pages, i, n); 667 err = iwch_write_pbl(mhp, pages, i, n);
@@ -676,7 +671,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
676 i = 0; 671 i = 0;
677 } 672 }
678 } 673 }
679 } 674 }
680 675
681 if (i) 676 if (i)
682 err = iwch_write_pbl(mhp, pages, i, n); 677 err = iwch_write_pbl(mhp, pages, i, n);
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index 41b11951a30a..392d422b00cb 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -678,9 +678,9 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
678{ 678{
679 __be64 *pages; 679 __be64 *pages;
680 int shift, n, len; 680 int shift, n, len;
681 int i, j, k; 681 int i, k, entry;
682 int err = 0; 682 int err = 0;
683 struct ib_umem_chunk *chunk; 683 struct scatterlist *sg;
684 struct c4iw_dev *rhp; 684 struct c4iw_dev *rhp;
685 struct c4iw_pd *php; 685 struct c4iw_pd *php;
686 struct c4iw_mr *mhp; 686 struct c4iw_mr *mhp;
@@ -710,10 +710,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
710 710
711 shift = ffs(mhp->umem->page_size) - 1; 711 shift = ffs(mhp->umem->page_size) - 1;
712 712
713 n = 0; 713 n = mhp->umem->nmap;
714 list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
715 n += chunk->nents;
716
717 err = alloc_pbl(mhp, n); 714 err = alloc_pbl(mhp, n);
718 if (err) 715 if (err)
719 goto err; 716 goto err;
@@ -726,24 +723,22 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
726 723
727 i = n = 0; 724 i = n = 0;
728 725
729 list_for_each_entry(chunk, &mhp->umem->chunk_list, list) 726 for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
730 for (j = 0; j < chunk->nmap; ++j) { 727 len = sg_dma_len(sg) >> shift;
731 len = sg_dma_len(&chunk->page_list[j]) >> shift; 728 for (k = 0; k < len; ++k) {
732 for (k = 0; k < len; ++k) { 729 pages[i++] = cpu_to_be64(sg_dma_address(sg) +
733 pages[i++] = cpu_to_be64(sg_dma_address( 730 mhp->umem->page_size * k);
734 &chunk->page_list[j]) + 731 if (i == PAGE_SIZE / sizeof *pages) {
735 mhp->umem->page_size * k); 732 err = write_pbl(&mhp->rhp->rdev,
736 if (i == PAGE_SIZE / sizeof *pages) { 733 pages,
737 err = write_pbl(&mhp->rhp->rdev, 734 mhp->attr.pbl_addr + (n << 3), i);
738 pages, 735 if (err)
739 mhp->attr.pbl_addr + (n << 3), i); 736 goto pbl_done;
740 if (err) 737 n += i;
741 goto pbl_done; 738 i = 0;
742 n += i;
743 i = 0;
744 }
745 } 739 }
746 } 740 }
741 }
747 742
748 if (i) 743 if (i)
749 err = write_pbl(&mhp->rhp->rdev, pages, 744 err = write_pbl(&mhp->rhp->rdev, pages,
diff --git a/drivers/infiniband/hw/ehca/ehca_classes.h b/drivers/infiniband/hw/ehca/ehca_classes.h
index f08f6eaf3fa8..bd45e0f3923f 100644
--- a/drivers/infiniband/hw/ehca/ehca_classes.h
+++ b/drivers/infiniband/hw/ehca/ehca_classes.h
@@ -322,7 +322,7 @@ struct ehca_mr_pginfo {
322 } phy; 322 } phy;
323 struct { /* type EHCA_MR_PGI_USER section */ 323 struct { /* type EHCA_MR_PGI_USER section */
324 struct ib_umem *region; 324 struct ib_umem *region;
325 struct ib_umem_chunk *next_chunk; 325 struct scatterlist *next_sg;
326 u64 next_nmap; 326 u64 next_nmap;
327 } usr; 327 } usr;
328 struct { /* type EHCA_MR_PGI_FMR section */ 328 struct { /* type EHCA_MR_PGI_FMR section */
diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.c b/drivers/infiniband/hw/ehca/ehca_mrmw.c
index bcfb0c183620..7168f594d457 100644
--- a/drivers/infiniband/hw/ehca/ehca_mrmw.c
+++ b/drivers/infiniband/hw/ehca/ehca_mrmw.c
@@ -400,10 +400,7 @@ reg_user_mr_fallback:
400 pginfo.num_hwpages = num_hwpages; 400 pginfo.num_hwpages = num_hwpages;
401 pginfo.u.usr.region = e_mr->umem; 401 pginfo.u.usr.region = e_mr->umem;
402 pginfo.next_hwpage = e_mr->umem->offset / hwpage_size; 402 pginfo.next_hwpage = e_mr->umem->offset / hwpage_size;
403 pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk, 403 pginfo.u.usr.next_sg = pginfo.u.usr.region->sg_head.sgl;
404 (&e_mr->umem->chunk_list),
405 list);
406
407 ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags, 404 ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
408 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, 405 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
409 &e_mr->ib.ib_mr.rkey, EHCA_REG_MR); 406 &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
@@ -1858,61 +1855,39 @@ static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1858 u64 *kpage) 1855 u64 *kpage)
1859{ 1856{
1860 int ret = 0; 1857 int ret = 0;
1861 struct ib_umem_chunk *prev_chunk;
1862 struct ib_umem_chunk *chunk;
1863 u64 pgaddr; 1858 u64 pgaddr;
1864 u32 i = 0;
1865 u32 j = 0; 1859 u32 j = 0;
1866 int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size; 1860 int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;
1867 1861 struct scatterlist **sg = &pginfo->u.usr.next_sg;
1868 /* loop over desired chunk entries */ 1862
1869 chunk = pginfo->u.usr.next_chunk; 1863 while (*sg != NULL) {
1870 prev_chunk = pginfo->u.usr.next_chunk; 1864 pgaddr = page_to_pfn(sg_page(*sg))
1871 list_for_each_entry_continue( 1865 << PAGE_SHIFT;
1872 chunk, (&(pginfo->u.usr.region->chunk_list)), list) { 1866 *kpage = pgaddr + (pginfo->next_hwpage *
1873 for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) { 1867 pginfo->hwpage_size);
1874 pgaddr = page_to_pfn(sg_page(&chunk->page_list[i])) 1868 if (!(*kpage)) {
1875 << PAGE_SHIFT ; 1869 ehca_gen_err("pgaddr=%llx "
1876 *kpage = pgaddr + (pginfo->next_hwpage * 1870 "sg_dma_address=%llx "
1877 pginfo->hwpage_size); 1871 "entry=%llx next_hwpage=%llx",
1878 if ( !(*kpage) ) { 1872 pgaddr, (u64)sg_dma_address(*sg),
1879 ehca_gen_err("pgaddr=%llx " 1873 pginfo->u.usr.next_nmap,
1880 "chunk->page_list[i]=%llx " 1874 pginfo->next_hwpage);
1881 "i=%x next_hwpage=%llx", 1875 return -EFAULT;
1882 pgaddr, (u64)sg_dma_address(
1883 &chunk->page_list[i]),
1884 i, pginfo->next_hwpage);
1885 return -EFAULT;
1886 }
1887 (pginfo->hwpage_cnt)++;
1888 (pginfo->next_hwpage)++;
1889 kpage++;
1890 if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
1891 (pginfo->kpage_cnt)++;
1892 (pginfo->u.usr.next_nmap)++;
1893 pginfo->next_hwpage = 0;
1894 i++;
1895 }
1896 j++;
1897 if (j >= number) break;
1898 } 1876 }
1899 if ((pginfo->u.usr.next_nmap >= chunk->nmap) && 1877 (pginfo->hwpage_cnt)++;
1900 (j >= number)) { 1878 (pginfo->next_hwpage)++;
1901 pginfo->u.usr.next_nmap = 0; 1879 kpage++;
1902 prev_chunk = chunk; 1880 if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
1903 break; 1881 (pginfo->kpage_cnt)++;
1904 } else if (pginfo->u.usr.next_nmap >= chunk->nmap) { 1882 (pginfo->u.usr.next_nmap)++;
1905 pginfo->u.usr.next_nmap = 0; 1883 pginfo->next_hwpage = 0;
1906 prev_chunk = chunk; 1884 *sg = sg_next(*sg);
1907 } else if (j >= number) 1885 }
1886 j++;
1887 if (j >= number)
1908 break; 1888 break;
1909 else
1910 prev_chunk = chunk;
1911 } 1889 }
1912 pginfo->u.usr.next_chunk = 1890
1913 list_prepare_entry(prev_chunk,
1914 (&(pginfo->u.usr.region->chunk_list)),
1915 list);
1916 return ret; 1891 return ret;
1917} 1892}
1918 1893
@@ -1920,20 +1895,19 @@ static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1920 * check given pages for contiguous layout 1895 * check given pages for contiguous layout
1921 * last page addr is returned in prev_pgaddr for further check 1896 * last page addr is returned in prev_pgaddr for further check
1922 */ 1897 */
1923static int ehca_check_kpages_per_ate(struct scatterlist *page_list, 1898static int ehca_check_kpages_per_ate(struct scatterlist **sg,
1924 int start_idx, int end_idx, 1899 int num_pages,
1925 u64 *prev_pgaddr) 1900 u64 *prev_pgaddr)
1926{ 1901{
1927 int t; 1902 for (; *sg && num_pages > 0; *sg = sg_next(*sg), num_pages--) {
1928 for (t = start_idx; t <= end_idx; t++) { 1903 u64 pgaddr = page_to_pfn(sg_page(*sg)) << PAGE_SHIFT;
1929 u64 pgaddr = page_to_pfn(sg_page(&page_list[t])) << PAGE_SHIFT;
1930 if (ehca_debug_level >= 3) 1904 if (ehca_debug_level >= 3)
1931 ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr, 1905 ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr,
1932 *(u64 *)__va(pgaddr)); 1906 *(u64 *)__va(pgaddr));
1933 if (pgaddr - PAGE_SIZE != *prev_pgaddr) { 1907 if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
1934 ehca_gen_err("uncontiguous page found pgaddr=%llx " 1908 ehca_gen_err("uncontiguous page found pgaddr=%llx "
1935 "prev_pgaddr=%llx page_list_i=%x", 1909 "prev_pgaddr=%llx entries_left_in_hwpage=%x",
1936 pgaddr, *prev_pgaddr, t); 1910 pgaddr, *prev_pgaddr, num_pages);
1937 return -EINVAL; 1911 return -EINVAL;
1938 } 1912 }
1939 *prev_pgaddr = pgaddr; 1913 *prev_pgaddr = pgaddr;
@@ -1947,111 +1921,80 @@ static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
1947 u64 *kpage) 1921 u64 *kpage)
1948{ 1922{
1949 int ret = 0; 1923 int ret = 0;
1950 struct ib_umem_chunk *prev_chunk;
1951 struct ib_umem_chunk *chunk;
1952 u64 pgaddr, prev_pgaddr; 1924 u64 pgaddr, prev_pgaddr;
1953 u32 i = 0;
1954 u32 j = 0; 1925 u32 j = 0;
1955 int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE; 1926 int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;
1956 int nr_kpages = kpages_per_hwpage; 1927 int nr_kpages = kpages_per_hwpage;
1928 struct scatterlist **sg = &pginfo->u.usr.next_sg;
1929
1930 while (*sg != NULL) {
1957 1931
1958 /* loop over desired chunk entries */ 1932 if (nr_kpages == kpages_per_hwpage) {
1959 chunk = pginfo->u.usr.next_chunk; 1933 pgaddr = (page_to_pfn(sg_page(*sg))
1960 prev_chunk = pginfo->u.usr.next_chunk; 1934 << PAGE_SHIFT);
1961 list_for_each_entry_continue( 1935 *kpage = pgaddr;
1962 chunk, (&(pginfo->u.usr.region->chunk_list)), list) { 1936 if (!(*kpage)) {
1963 for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) { 1937 ehca_gen_err("pgaddr=%llx entry=%llx",
1964 if (nr_kpages == kpages_per_hwpage) { 1938 pgaddr, pginfo->u.usr.next_nmap);
1965 pgaddr = ( page_to_pfn(sg_page(&chunk->page_list[i])) 1939 ret = -EFAULT;
1966 << PAGE_SHIFT ); 1940 return ret;
1967 *kpage = pgaddr; 1941 }
1968 if ( !(*kpage) ) { 1942 /*
1969 ehca_gen_err("pgaddr=%llx i=%x", 1943 * The first page in a hwpage must be aligned;
1970 pgaddr, i); 1944 * the first MR page is exempt from this rule.
1945 */
1946 if (pgaddr & (pginfo->hwpage_size - 1)) {
1947 if (pginfo->hwpage_cnt) {
1948 ehca_gen_err(
1949 "invalid alignment "
1950 "pgaddr=%llx entry=%llx "
1951 "mr_pgsize=%llx",
1952 pgaddr, pginfo->u.usr.next_nmap,
1953 pginfo->hwpage_size);
1971 ret = -EFAULT; 1954 ret = -EFAULT;
1972 return ret; 1955 return ret;
1973 } 1956 }
1974 /* 1957 /* first MR page */
1975 * The first page in a hwpage must be aligned; 1958 pginfo->kpage_cnt =
1976 * the first MR page is exempt from this rule. 1959 (pgaddr &
1977 */ 1960 (pginfo->hwpage_size - 1)) >>
1978 if (pgaddr & (pginfo->hwpage_size - 1)) { 1961 PAGE_SHIFT;
1979 if (pginfo->hwpage_cnt) { 1962 nr_kpages -= pginfo->kpage_cnt;
1980 ehca_gen_err( 1963 *kpage = pgaddr &
1981 "invalid alignment " 1964 ~(pginfo->hwpage_size - 1);
1982 "pgaddr=%llx i=%x "
1983 "mr_pgsize=%llx",
1984 pgaddr, i,
1985 pginfo->hwpage_size);
1986 ret = -EFAULT;
1987 return ret;
1988 }
1989 /* first MR page */
1990 pginfo->kpage_cnt =
1991 (pgaddr &
1992 (pginfo->hwpage_size - 1)) >>
1993 PAGE_SHIFT;
1994 nr_kpages -= pginfo->kpage_cnt;
1995 *kpage = pgaddr &
1996 ~(pginfo->hwpage_size - 1);
1997 }
1998 if (ehca_debug_level >= 3) {
1999 u64 val = *(u64 *)__va(pgaddr);
2000 ehca_gen_dbg("kpage=%llx chunk_page=%llx "
2001 "value=%016llx",
2002 *kpage, pgaddr, val);
2003 }
2004 prev_pgaddr = pgaddr;
2005 i++;
2006 pginfo->kpage_cnt++;
2007 pginfo->u.usr.next_nmap++;
2008 nr_kpages--;
2009 if (!nr_kpages)
2010 goto next_kpage;
2011 continue;
2012 } 1965 }
2013 if (i + nr_kpages > chunk->nmap) { 1966 if (ehca_debug_level >= 3) {
2014 ret = ehca_check_kpages_per_ate( 1967 u64 val = *(u64 *)__va(pgaddr);
2015 chunk->page_list, i, 1968 ehca_gen_dbg("kpage=%llx page=%llx "
2016 chunk->nmap - 1, &prev_pgaddr); 1969 "value=%016llx",
2017 if (ret) return ret; 1970 *kpage, pgaddr, val);
2018 pginfo->kpage_cnt += chunk->nmap - i;
2019 pginfo->u.usr.next_nmap += chunk->nmap - i;
2020 nr_kpages -= chunk->nmap - i;
2021 break;
2022 } 1971 }
1972 prev_pgaddr = pgaddr;
1973 *sg = sg_next(*sg);
1974 pginfo->kpage_cnt++;
1975 pginfo->u.usr.next_nmap++;
1976 nr_kpages--;
1977 if (!nr_kpages)
1978 goto next_kpage;
1979 continue;
1980 }
1981
1982 ret = ehca_check_kpages_per_ate(sg, nr_kpages,
1983 &prev_pgaddr);
1984 if (ret)
1985 return ret;
1986 pginfo->kpage_cnt += nr_kpages;
1987 pginfo->u.usr.next_nmap += nr_kpages;
2023 1988
2024 ret = ehca_check_kpages_per_ate(chunk->page_list, i,
2025 i + nr_kpages - 1,
2026 &prev_pgaddr);
2027 if (ret) return ret;
2028 i += nr_kpages;
2029 pginfo->kpage_cnt += nr_kpages;
2030 pginfo->u.usr.next_nmap += nr_kpages;
2031next_kpage: 1989next_kpage:
2032 nr_kpages = kpages_per_hwpage; 1990 nr_kpages = kpages_per_hwpage;
2033 (pginfo->hwpage_cnt)++; 1991 (pginfo->hwpage_cnt)++;
2034 kpage++; 1992 kpage++;
2035 j++; 1993 j++;
2036 if (j >= number) break; 1994 if (j >= number)
2037 }
2038 if ((pginfo->u.usr.next_nmap >= chunk->nmap) &&
2039 (j >= number)) {
2040 pginfo->u.usr.next_nmap = 0;
2041 prev_chunk = chunk;
2042 break;
2043 } else if (pginfo->u.usr.next_nmap >= chunk->nmap) {
2044 pginfo->u.usr.next_nmap = 0;
2045 prev_chunk = chunk;
2046 } else if (j >= number)
2047 break; 1995 break;
2048 else
2049 prev_chunk = chunk;
2050 } 1996 }
2051 pginfo->u.usr.next_chunk = 1997
2052 list_prepare_entry(prev_chunk,
2053 (&(pginfo->u.usr.region->chunk_list)),
2054 list);
2055 return ret; 1998 return ret;
2056} 1999}
2057 2000
diff --git a/drivers/infiniband/hw/ipath/ipath_mr.c b/drivers/infiniband/hw/ipath/ipath_mr.c
index e346d3890a0e..5e61e9bff697 100644
--- a/drivers/infiniband/hw/ipath/ipath_mr.c
+++ b/drivers/infiniband/hw/ipath/ipath_mr.c
@@ -188,8 +188,8 @@ struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
188{ 188{
189 struct ipath_mr *mr; 189 struct ipath_mr *mr;
190 struct ib_umem *umem; 190 struct ib_umem *umem;
191 struct ib_umem_chunk *chunk; 191 int n, m, entry;
192 int n, m, i; 192 struct scatterlist *sg;
193 struct ib_mr *ret; 193 struct ib_mr *ret;
194 194
195 if (length == 0) { 195 if (length == 0) {
@@ -202,10 +202,7 @@ struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
202 if (IS_ERR(umem)) 202 if (IS_ERR(umem))
203 return (void *) umem; 203 return (void *) umem;
204 204
205 n = 0; 205 n = umem->nmap;
206 list_for_each_entry(chunk, &umem->chunk_list, list)
207 n += chunk->nents;
208
209 mr = alloc_mr(n, &to_idev(pd->device)->lk_table); 206 mr = alloc_mr(n, &to_idev(pd->device)->lk_table);
210 if (!mr) { 207 if (!mr) {
211 ret = ERR_PTR(-ENOMEM); 208 ret = ERR_PTR(-ENOMEM);
@@ -224,22 +221,20 @@ struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
224 221
225 m = 0; 222 m = 0;
226 n = 0; 223 n = 0;
227 list_for_each_entry(chunk, &umem->chunk_list, list) { 224 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
228 for (i = 0; i < chunk->nents; i++) { 225 void *vaddr;
229 void *vaddr; 226
230 227 vaddr = page_address(sg_page(sg));
231 vaddr = page_address(sg_page(&chunk->page_list[i])); 228 if (!vaddr) {
232 if (!vaddr) { 229 ret = ERR_PTR(-EINVAL);
233 ret = ERR_PTR(-EINVAL); 230 goto bail;
234 goto bail; 231 }
235 } 232 mr->mr.map[m]->segs[n].vaddr = vaddr;
236 mr->mr.map[m]->segs[n].vaddr = vaddr; 233 mr->mr.map[m]->segs[n].length = umem->page_size;
237 mr->mr.map[m]->segs[n].length = umem->page_size; 234 n++;
238 n++; 235 if (n == IPATH_SEGSZ) {
239 if (n == IPATH_SEGSZ) { 236 m++;
240 m++; 237 n = 0;
241 n = 0;
242 }
243 } 238 }
244 } 239 }
245 ret = &mr->ibmr; 240 ret = &mr->ibmr;
diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c
index 8aee4233b388..c51740986367 100644
--- a/drivers/infiniband/hw/mlx4/doorbell.c
+++ b/drivers/infiniband/hw/mlx4/doorbell.c
@@ -45,7 +45,6 @@ int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt,
45 struct mlx4_db *db) 45 struct mlx4_db *db)
46{ 46{
47 struct mlx4_ib_user_db_page *page; 47 struct mlx4_ib_user_db_page *page;
48 struct ib_umem_chunk *chunk;
49 int err = 0; 48 int err = 0;
50 49
51 mutex_lock(&context->db_page_mutex); 50 mutex_lock(&context->db_page_mutex);
@@ -73,8 +72,7 @@ int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt,
73 list_add(&page->list, &context->db_page_list); 72 list_add(&page->list, &context->db_page_list);
74 73
75found: 74found:
76 chunk = list_entry(page->umem->chunk_list.next, struct ib_umem_chunk, list); 75 db->dma = sg_dma_address(page->umem->sg_head.sgl) + (virt & ~PAGE_MASK);
77 db->dma = sg_dma_address(chunk->page_list) + (virt & ~PAGE_MASK);
78 db->u.user_page = page; 76 db->u.user_page = page;
79 ++page->refcnt; 77 ++page->refcnt;
80 78
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index e471f089ff00..cb2a8727f3fb 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -90,11 +90,11 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
90 struct ib_umem *umem) 90 struct ib_umem *umem)
91{ 91{
92 u64 *pages; 92 u64 *pages;
93 struct ib_umem_chunk *chunk; 93 int i, k, entry;
94 int i, j, k;
95 int n; 94 int n;
96 int len; 95 int len;
97 int err = 0; 96 int err = 0;
97 struct scatterlist *sg;
98 98
99 pages = (u64 *) __get_free_page(GFP_KERNEL); 99 pages = (u64 *) __get_free_page(GFP_KERNEL);
100 if (!pages) 100 if (!pages)
@@ -102,26 +102,25 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
102 102
103 i = n = 0; 103 i = n = 0;
104 104
105 list_for_each_entry(chunk, &umem->chunk_list, list) 105 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
106 for (j = 0; j < chunk->nmap; ++j) { 106 len = sg_dma_len(sg) >> mtt->page_shift;
107 len = sg_dma_len(&chunk->page_list[j]) >> mtt->page_shift; 107 for (k = 0; k < len; ++k) {
108 for (k = 0; k < len; ++k) { 108 pages[i++] = sg_dma_address(sg) +
109 pages[i++] = sg_dma_address(&chunk->page_list[j]) + 109 umem->page_size * k;
110 umem->page_size * k; 110 /*
111 /* 111 * Be friendly to mlx4_write_mtt() and
112 * Be friendly to mlx4_write_mtt() and 112 * pass it chunks of appropriate size.
113 * pass it chunks of appropriate size. 113 */
114 */ 114 if (i == PAGE_SIZE / sizeof (u64)) {
115 if (i == PAGE_SIZE / sizeof (u64)) { 115 err = mlx4_write_mtt(dev->dev, mtt, n,
116 err = mlx4_write_mtt(dev->dev, mtt, n, 116 i, pages);
117 i, pages); 117 if (err)
118 if (err) 118 goto out;
119 goto out; 119 n += i;
120 n += i; 120 i = 0;
121 i = 0;
122 }
123 } 121 }
124 } 122 }
123 }
125 124
126 if (i) 125 if (i)
127 err = mlx4_write_mtt(dev->dev, mtt, n, i, pages); 126 err = mlx4_write_mtt(dev->dev, mtt, n, i, pages);
diff --git a/drivers/infiniband/hw/mlx5/doorbell.c b/drivers/infiniband/hw/mlx5/doorbell.c
index 256a23344f28..ece028fc47d6 100644
--- a/drivers/infiniband/hw/mlx5/doorbell.c
+++ b/drivers/infiniband/hw/mlx5/doorbell.c
@@ -47,7 +47,6 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
47 struct mlx5_db *db) 47 struct mlx5_db *db)
48{ 48{
49 struct mlx5_ib_user_db_page *page; 49 struct mlx5_ib_user_db_page *page;
50 struct ib_umem_chunk *chunk;
51 int err = 0; 50 int err = 0;
52 51
53 mutex_lock(&context->db_page_mutex); 52 mutex_lock(&context->db_page_mutex);
@@ -75,8 +74,7 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
75 list_add(&page->list, &context->db_page_list); 74 list_add(&page->list, &context->db_page_list);
76 75
77found: 76found:
78 chunk = list_entry(page->umem->chunk_list.next, struct ib_umem_chunk, list); 77 db->dma = sg_dma_address(page->umem->sg_head.sgl) + (virt & ~PAGE_MASK);
79 db->dma = sg_dma_address(chunk->page_list) + (virt & ~PAGE_MASK);
80 db->u.user_page = page; 78 db->u.user_page = page;
81 ++page->refcnt; 79 ++page->refcnt;
82 80
diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c
index 3a5322870b96..8499aec94db6 100644
--- a/drivers/infiniband/hw/mlx5/mem.c
+++ b/drivers/infiniband/hw/mlx5/mem.c
@@ -44,16 +44,17 @@
44void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift, 44void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
45 int *ncont, int *order) 45 int *ncont, int *order)
46{ 46{
47 struct ib_umem_chunk *chunk;
48 unsigned long tmp; 47 unsigned long tmp;
49 unsigned long m; 48 unsigned long m;
50 int i, j, k; 49 int i, k;
51 u64 base = 0; 50 u64 base = 0;
52 int p = 0; 51 int p = 0;
53 int skip; 52 int skip;
54 int mask; 53 int mask;
55 u64 len; 54 u64 len;
56 u64 pfn; 55 u64 pfn;
56 struct scatterlist *sg;
57 int entry;
57 58
58 addr = addr >> PAGE_SHIFT; 59 addr = addr >> PAGE_SHIFT;
59 tmp = (unsigned long)addr; 60 tmp = (unsigned long)addr;
@@ -61,32 +62,31 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
61 skip = 1 << m; 62 skip = 1 << m;
62 mask = skip - 1; 63 mask = skip - 1;
63 i = 0; 64 i = 0;
64 list_for_each_entry(chunk, &umem->chunk_list, list) 65 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
65 for (j = 0; j < chunk->nmap; j++) { 66 len = sg_dma_len(sg) >> PAGE_SHIFT;
66 len = sg_dma_len(&chunk->page_list[j]) >> PAGE_SHIFT; 67 pfn = sg_dma_address(sg) >> PAGE_SHIFT;
67 pfn = sg_dma_address(&chunk->page_list[j]) >> PAGE_SHIFT; 68 for (k = 0; k < len; k++) {
68 for (k = 0; k < len; k++) { 69 if (!(i & mask)) {
69 if (!(i & mask)) { 70 tmp = (unsigned long)pfn;
70 tmp = (unsigned long)pfn; 71 m = min(m, find_first_bit(&tmp, sizeof(tmp)));
71 m = min(m, find_first_bit(&tmp, sizeof(tmp))); 72 skip = 1 << m;
73 mask = skip - 1;
74 base = pfn;
75 p = 0;
76 } else {
77 if (base + p != pfn) {
78 tmp = (unsigned long)p;
79 m = find_first_bit(&tmp, sizeof(tmp));
72 skip = 1 << m; 80 skip = 1 << m;
73 mask = skip - 1; 81 mask = skip - 1;
74 base = pfn; 82 base = pfn;
75 p = 0; 83 p = 0;
76 } else {
77 if (base + p != pfn) {
78 tmp = (unsigned long)p;
79 m = find_first_bit(&tmp, sizeof(tmp));
80 skip = 1 << m;
81 mask = skip - 1;
82 base = pfn;
83 p = 0;
84 }
85 } 84 }
86 p++;
87 i++;
88 } 85 }
86 p++;
87 i++;
89 } 88 }
89 }
90 90
91 if (i) { 91 if (i) {
92 m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m); 92 m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m);
@@ -112,32 +112,32 @@ void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
112{ 112{
113 int shift = page_shift - PAGE_SHIFT; 113 int shift = page_shift - PAGE_SHIFT;
114 int mask = (1 << shift) - 1; 114 int mask = (1 << shift) - 1;
115 struct ib_umem_chunk *chunk; 115 int i, k;
116 int i, j, k;
117 u64 cur = 0; 116 u64 cur = 0;
118 u64 base; 117 u64 base;
119 int len; 118 int len;
119 struct scatterlist *sg;
120 int entry;
120 121
121 i = 0; 122 i = 0;
122 list_for_each_entry(chunk, &umem->chunk_list, list) 123 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
123 for (j = 0; j < chunk->nmap; j++) { 124 len = sg_dma_len(sg) >> PAGE_SHIFT;
124 len = sg_dma_len(&chunk->page_list[j]) >> PAGE_SHIFT; 125 base = sg_dma_address(sg);
125 base = sg_dma_address(&chunk->page_list[j]); 126 for (k = 0; k < len; k++) {
126 for (k = 0; k < len; k++) { 127 if (!(i & mask)) {
127 if (!(i & mask)) { 128 cur = base + (k << PAGE_SHIFT);
128 cur = base + (k << PAGE_SHIFT); 129 if (umr)
129 if (umr) 130 cur |= 3;
130 cur |= 3;
131 131
132 pas[i >> shift] = cpu_to_be64(cur); 132 pas[i >> shift] = cpu_to_be64(cur);
133 mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n", 133 mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n",
134 i >> shift, be64_to_cpu(pas[i >> shift])); 134 i >> shift, be64_to_cpu(pas[i >> shift]));
135 } else 135 } else
136 mlx5_ib_dbg(dev, "=====> 0x%llx\n", 136 mlx5_ib_dbg(dev, "=====> 0x%llx\n",
137 base + (k << PAGE_SHIFT)); 137 base + (k << PAGE_SHIFT));
138 i++; 138 i++;
139 }
140 } 139 }
140 }
141} 141}
142 142
143int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset) 143int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset)
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 5b71d43bd89c..64408000f1c7 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -976,12 +976,12 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
976 u64 virt, int acc, struct ib_udata *udata) 976 u64 virt, int acc, struct ib_udata *udata)
977{ 977{
978 struct mthca_dev *dev = to_mdev(pd->device); 978 struct mthca_dev *dev = to_mdev(pd->device);
979 struct ib_umem_chunk *chunk; 979 struct scatterlist *sg;
980 struct mthca_mr *mr; 980 struct mthca_mr *mr;
981 struct mthca_reg_mr ucmd; 981 struct mthca_reg_mr ucmd;
982 u64 *pages; 982 u64 *pages;
983 int shift, n, len; 983 int shift, n, len;
984 int i, j, k; 984 int i, k, entry;
985 int err = 0; 985 int err = 0;
986 int write_mtt_size; 986 int write_mtt_size;
987 987
@@ -1009,10 +1009,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1009 } 1009 }
1010 1010
1011 shift = ffs(mr->umem->page_size) - 1; 1011 shift = ffs(mr->umem->page_size) - 1;
1012 1012 n = mr->umem->nmap;
1013 n = 0;
1014 list_for_each_entry(chunk, &mr->umem->chunk_list, list)
1015 n += chunk->nents;
1016 1013
1017 mr->mtt = mthca_alloc_mtt(dev, n); 1014 mr->mtt = mthca_alloc_mtt(dev, n);
1018 if (IS_ERR(mr->mtt)) { 1015 if (IS_ERR(mr->mtt)) {
@@ -1030,25 +1027,24 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1030 1027
1031 write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); 1028 write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages));
1032 1029
1033 list_for_each_entry(chunk, &mr->umem->chunk_list, list) 1030 for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1034 for (j = 0; j < chunk->nmap; ++j) { 1031 len = sg_dma_len(sg) >> shift;
1035 len = sg_dma_len(&chunk->page_list[j]) >> shift; 1032 for (k = 0; k < len; ++k) {
1036 for (k = 0; k < len; ++k) { 1033 pages[i++] = sg_dma_address(sg) +
1037 pages[i++] = sg_dma_address(&chunk->page_list[j]) + 1034 mr->umem->page_size * k;
1038 mr->umem->page_size * k; 1035 /*
1039 /* 1036 * Be friendly to write_mtt and pass it chunks
1040 * Be friendly to write_mtt and pass it chunks 1037 * of appropriate size.
1041 * of appropriate size. 1038 */
1042 */ 1039 if (i == write_mtt_size) {
1043 if (i == write_mtt_size) { 1040 err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
1044 err = mthca_write_mtt(dev, mr->mtt, n, pages, i); 1041 if (err)
1045 if (err) 1042 goto mtt_done;
1046 goto mtt_done; 1043 n += i;
1047 n += i; 1044 i = 0;
1048 i = 0;
1049 }
1050 } 1045 }
1051 } 1046 }
1047 }
1052 1048
1053 if (i) 1049 if (i)
1054 err = mthca_write_mtt(dev, mr->mtt, n, pages, i); 1050 err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 8308e3634767..32d3682daaf5 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -2307,7 +2307,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2307 struct nes_device *nesdev = nesvnic->nesdev; 2307 struct nes_device *nesdev = nesvnic->nesdev;
2308 struct nes_adapter *nesadapter = nesdev->nesadapter; 2308 struct nes_adapter *nesadapter = nesdev->nesadapter;
2309 struct ib_mr *ibmr = ERR_PTR(-EINVAL); 2309 struct ib_mr *ibmr = ERR_PTR(-EINVAL);
2310 struct ib_umem_chunk *chunk; 2310 struct scatterlist *sg;
2311 struct nes_ucontext *nes_ucontext; 2311 struct nes_ucontext *nes_ucontext;
2312 struct nes_pbl *nespbl; 2312 struct nes_pbl *nespbl;
2313 struct nes_mr *nesmr; 2313 struct nes_mr *nesmr;
@@ -2315,7 +2315,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2315 struct nes_mem_reg_req req; 2315 struct nes_mem_reg_req req;
2316 struct nes_vpbl vpbl; 2316 struct nes_vpbl vpbl;
2317 struct nes_root_vpbl root_vpbl; 2317 struct nes_root_vpbl root_vpbl;
2318 int nmap_index, page_index; 2318 int entry, page_index;
2319 int page_count = 0; 2319 int page_count = 0;
2320 int err, pbl_depth = 0; 2320 int err, pbl_depth = 0;
2321 int chunk_pages; 2321 int chunk_pages;
@@ -2330,6 +2330,7 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2330 u16 pbl_count; 2330 u16 pbl_count;
2331 u8 single_page = 1; 2331 u8 single_page = 1;
2332 u8 stag_key; 2332 u8 stag_key;
2333 int first_page = 1;
2333 2334
2334 region = ib_umem_get(pd->uobject->context, start, length, acc, 0); 2335 region = ib_umem_get(pd->uobject->context, start, length, acc, 0);
2335 if (IS_ERR(region)) { 2336 if (IS_ERR(region)) {
@@ -2380,128 +2381,125 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2380 } 2381 }
2381 nesmr->region = region; 2382 nesmr->region = region;
2382 2383
2383 list_for_each_entry(chunk, &region->chunk_list, list) { 2384 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
2384 nes_debug(NES_DBG_MR, "Chunk: nents = %u, nmap = %u .\n", 2385 if (sg_dma_address(sg) & ~PAGE_MASK) {
2385 chunk->nents, chunk->nmap); 2386 ib_umem_release(region);
2386 for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) { 2387 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2387 if (sg_dma_address(&chunk->page_list[nmap_index]) & ~PAGE_MASK) { 2388 nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n",
2388 ib_umem_release(region); 2389 (unsigned int) sg_dma_address(sg));
2389 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2390 ibmr = ERR_PTR(-EINVAL);
2390 nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n", 2391 kfree(nesmr);
2391 (unsigned int) sg_dma_address(&chunk->page_list[nmap_index])); 2392 goto reg_user_mr_err;
2392 ibmr = ERR_PTR(-EINVAL); 2393 }
2393 kfree(nesmr);
2394 goto reg_user_mr_err;
2395 }
2396 2394
2397 if (!sg_dma_len(&chunk->page_list[nmap_index])) { 2395 if (!sg_dma_len(sg)) {
2398 ib_umem_release(region); 2396 ib_umem_release(region);
2399 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2397 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2400 stag_index); 2398 stag_index);
2401 nes_debug(NES_DBG_MR, "Invalid Buffer Size\n"); 2399 nes_debug(NES_DBG_MR, "Invalid Buffer Size\n");
2402 ibmr = ERR_PTR(-EINVAL); 2400 ibmr = ERR_PTR(-EINVAL);
2403 kfree(nesmr); 2401 kfree(nesmr);
2404 goto reg_user_mr_err; 2402 goto reg_user_mr_err;
2405 } 2403 }
2406 2404
2407 region_length += sg_dma_len(&chunk->page_list[nmap_index]); 2405 region_length += sg_dma_len(sg);
2408 chunk_pages = sg_dma_len(&chunk->page_list[nmap_index]) >> 12; 2406 chunk_pages = sg_dma_len(sg) >> 12;
2409 region_length -= skip_pages << 12; 2407 region_length -= skip_pages << 12;
2410 for (page_index=skip_pages; page_index < chunk_pages; page_index++) { 2408 for (page_index = skip_pages; page_index < chunk_pages; page_index++) {
2411 skip_pages = 0; 2409 skip_pages = 0;
2412 if ((page_count!=0)&&(page_count<<12)-(region->offset&(4096-1))>=region->length) 2410 if ((page_count != 0) && (page_count<<12)-(region->offset&(4096-1)) >= region->length)
2413 goto enough_pages; 2411 goto enough_pages;
2414 if ((page_count&0x01FF) == 0) { 2412 if ((page_count&0x01FF) == 0) {
2415 if (page_count >= 1024 * 512) { 2413 if (page_count >= 1024 * 512) {
2414 ib_umem_release(region);
2415 nes_free_resource(nesadapter,
2416 nesadapter->allocated_mrs, stag_index);
2417 kfree(nesmr);
2418 ibmr = ERR_PTR(-E2BIG);
2419 goto reg_user_mr_err;
2420 }
2421 if (root_pbl_index == 1) {
2422 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
2423 8192, &root_vpbl.pbl_pbase);
2424 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2425 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
2426 if (!root_vpbl.pbl_vbase) {
2416 ib_umem_release(region); 2427 ib_umem_release(region);
2417 nes_free_resource(nesadapter, 2428 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2418 nesadapter->allocated_mrs, stag_index); 2429 vpbl.pbl_pbase);
2430 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2431 stag_index);
2419 kfree(nesmr); 2432 kfree(nesmr);
2420 ibmr = ERR_PTR(-E2BIG); 2433 ibmr = ERR_PTR(-ENOMEM);
2421 goto reg_user_mr_err; 2434 goto reg_user_mr_err;
2422 } 2435 }
2423 if (root_pbl_index == 1) { 2436 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024,
2424 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 2437 GFP_KERNEL);
2425 8192, &root_vpbl.pbl_pbase); 2438 if (!root_vpbl.leaf_vpbl) {
2426 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2427 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
2428 if (!root_vpbl.pbl_vbase) {
2429 ib_umem_release(region);
2430 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2431 vpbl.pbl_pbase);
2432 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2433 stag_index);
2434 kfree(nesmr);
2435 ibmr = ERR_PTR(-ENOMEM);
2436 goto reg_user_mr_err;
2437 }
2438 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024,
2439 GFP_KERNEL);
2440 if (!root_vpbl.leaf_vpbl) {
2441 ib_umem_release(region);
2442 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
2443 root_vpbl.pbl_pbase);
2444 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2445 vpbl.pbl_pbase);
2446 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2447 stag_index);
2448 kfree(nesmr);
2449 ibmr = ERR_PTR(-ENOMEM);
2450 goto reg_user_mr_err;
2451 }
2452 root_vpbl.pbl_vbase[0].pa_low =
2453 cpu_to_le32((u32)vpbl.pbl_pbase);
2454 root_vpbl.pbl_vbase[0].pa_high =
2455 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
2456 root_vpbl.leaf_vpbl[0] = vpbl;
2457 }
2458 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
2459 &vpbl.pbl_pbase);
2460 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
2461 vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase);
2462 if (!vpbl.pbl_vbase) {
2463 ib_umem_release(region); 2439 ib_umem_release(region);
2464 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2440 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
2465 ibmr = ERR_PTR(-ENOMEM); 2441 root_vpbl.pbl_pbase);
2442 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2443 vpbl.pbl_pbase);
2444 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2445 stag_index);
2466 kfree(nesmr); 2446 kfree(nesmr);
2447 ibmr = ERR_PTR(-ENOMEM);
2467 goto reg_user_mr_err; 2448 goto reg_user_mr_err;
2468 } 2449 }
2469 if (1 <= root_pbl_index) { 2450 root_vpbl.pbl_vbase[0].pa_low =
2470 root_vpbl.pbl_vbase[root_pbl_index].pa_low = 2451 cpu_to_le32((u32)vpbl.pbl_pbase);
2471 cpu_to_le32((u32)vpbl.pbl_pbase); 2452 root_vpbl.pbl_vbase[0].pa_high =
2472 root_vpbl.pbl_vbase[root_pbl_index].pa_high = 2453 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
2473 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32))); 2454 root_vpbl.leaf_vpbl[0] = vpbl;
2474 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
2475 }
2476 root_pbl_index++;
2477 cur_pbl_index = 0;
2478 } 2455 }
2479 if (single_page) { 2456 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
2480 if (page_count != 0) { 2457 &vpbl.pbl_pbase);
2481 if ((last_dma_addr+4096) != 2458 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
2482 (sg_dma_address(&chunk->page_list[nmap_index])+ 2459 vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase);
2483 (page_index*4096))) 2460 if (!vpbl.pbl_vbase) {
2484 single_page = 0; 2461 ib_umem_release(region);
2485 last_dma_addr = sg_dma_address(&chunk->page_list[nmap_index])+ 2462 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2486 (page_index*4096); 2463 ibmr = ERR_PTR(-ENOMEM);
2487 } else { 2464 kfree(nesmr);
2488 first_dma_addr = sg_dma_address(&chunk->page_list[nmap_index])+ 2465 goto reg_user_mr_err;
2489 (page_index*4096); 2466 }
2490 last_dma_addr = first_dma_addr; 2467 if (1 <= root_pbl_index) {
2491 } 2468 root_vpbl.pbl_vbase[root_pbl_index].pa_low =
2469 cpu_to_le32((u32)vpbl.pbl_pbase);
2470 root_vpbl.pbl_vbase[root_pbl_index].pa_high =
2471 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32)));
2472 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
2473 }
2474 root_pbl_index++;
2475 cur_pbl_index = 0;
2476 }
2477 if (single_page) {
2478 if (page_count != 0) {
2479 if ((last_dma_addr+4096) !=
2480 (sg_dma_address(sg)+
2481 (page_index*4096)))
2482 single_page = 0;
2483 last_dma_addr = sg_dma_address(sg)+
2484 (page_index*4096);
2485 } else {
2486 first_dma_addr = sg_dma_address(sg)+
2487 (page_index*4096);
2488 last_dma_addr = first_dma_addr;
2492 } 2489 }
2493
2494 vpbl.pbl_vbase[cur_pbl_index].pa_low =
2495 cpu_to_le32((u32)(sg_dma_address(&chunk->page_list[nmap_index])+
2496 (page_index*4096)));
2497 vpbl.pbl_vbase[cur_pbl_index].pa_high =
2498 cpu_to_le32((u32)((((u64)(sg_dma_address(&chunk->page_list[nmap_index])+
2499 (page_index*4096))) >> 32)));
2500 cur_pbl_index++;
2501 page_count++;
2502 } 2490 }
2491
2492 vpbl.pbl_vbase[cur_pbl_index].pa_low =
2493 cpu_to_le32((u32)(sg_dma_address(sg)+
2494 (page_index*4096)));
2495 vpbl.pbl_vbase[cur_pbl_index].pa_high =
2496 cpu_to_le32((u32)((((u64)(sg_dma_address(sg)+
2497 (page_index*4096))) >> 32)));
2498 cur_pbl_index++;
2499 page_count++;
2503 } 2500 }
2504 } 2501 }
2502
2505 enough_pages: 2503 enough_pages:
2506 nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x," 2504 nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x,"
2507 " stag_key=0x%08x\n", 2505 " stag_key=0x%08x\n",
@@ -2613,25 +2611,28 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2613 nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase, 2611 nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase,
2614 (void *) nespbl->pbl_vbase, nespbl->user_base); 2612 (void *) nespbl->pbl_vbase, nespbl->user_base);
2615 2613
2616 list_for_each_entry(chunk, &region->chunk_list, list) { 2614 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
2617 for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) { 2615 chunk_pages = sg_dma_len(sg) >> 12;
2618 chunk_pages = sg_dma_len(&chunk->page_list[nmap_index]) >> 12; 2616 chunk_pages += (sg_dma_len(sg) & (4096-1)) ? 1 : 0;
2619 chunk_pages += (sg_dma_len(&chunk->page_list[nmap_index]) & (4096-1)) ? 1 : 0; 2617 if (first_page) {
2620 nespbl->page = sg_page(&chunk->page_list[0]); 2618 nespbl->page = sg_page(sg);
2621 for (page_index=0; page_index<chunk_pages; page_index++) { 2619 first_page = 0;
2622 ((__le32 *)pbl)[0] = cpu_to_le32((u32) 2620 }
2623 (sg_dma_address(&chunk->page_list[nmap_index])+ 2621
2624 (page_index*4096))); 2622 for (page_index = 0; page_index < chunk_pages; page_index++) {
2625 ((__le32 *)pbl)[1] = cpu_to_le32(((u64) 2623 ((__le32 *)pbl)[0] = cpu_to_le32((u32)
2626 (sg_dma_address(&chunk->page_list[nmap_index])+ 2624 (sg_dma_address(sg)+
2627 (page_index*4096)))>>32); 2625 (page_index*4096)));
2628 nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl, 2626 ((__le32 *)pbl)[1] = cpu_to_le32(((u64)
2629 (unsigned long long)*pbl, 2627 (sg_dma_address(sg)+
2630 le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0])); 2628 (page_index*4096)))>>32);
2631 pbl++; 2629 nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl,
2632 } 2630 (unsigned long long)*pbl,
2631 le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0]));
2632 pbl++;
2633 } 2633 }
2634 } 2634 }
2635
2635 if (req.reg_type == IWNES_MEMREG_TYPE_QP) { 2636 if (req.reg_type == IWNES_MEMREG_TYPE_QP) {
2636 list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list); 2637 list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list);
2637 } else { 2638 } else {
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index e0cc201be41a..0de3473fa7d9 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -726,10 +726,10 @@ static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
726 u32 num_pbes) 726 u32 num_pbes)
727{ 727{
728 struct ocrdma_pbe *pbe; 728 struct ocrdma_pbe *pbe;
729 struct ib_umem_chunk *chunk; 729 struct scatterlist *sg;
730 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 730 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
731 struct ib_umem *umem = mr->umem; 731 struct ib_umem *umem = mr->umem;
732 int i, shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 732 int shift, pg_cnt, pages, pbe_cnt, entry, total_num_pbes = 0;
733 733
734 if (!mr->hwmr.num_pbes) 734 if (!mr->hwmr.num_pbes)
735 return; 735 return;
@@ -739,39 +739,37 @@ static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
739 739
740 shift = ilog2(umem->page_size); 740 shift = ilog2(umem->page_size);
741 741
742 list_for_each_entry(chunk, &umem->chunk_list, list) { 742 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
743 /* get all the dma regions from the chunk. */ 743 pages = sg_dma_len(sg) >> shift;
744 for (i = 0; i < chunk->nmap; i++) { 744 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
745 pages = sg_dma_len(&chunk->page_list[i]) >> shift; 745 /* store the page address in pbe */
746 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 746 pbe->pa_lo =
747 /* store the page address in pbe */ 747 cpu_to_le32(sg_dma_address
748 pbe->pa_lo = 748 (sg) +
749 cpu_to_le32(sg_dma_address 749 (umem->page_size * pg_cnt));
750 (&chunk->page_list[i]) + 750 pbe->pa_hi =
751 (umem->page_size * pg_cnt)); 751 cpu_to_le32(upper_32_bits
752 pbe->pa_hi = 752 ((sg_dma_address
753 cpu_to_le32(upper_32_bits 753 (sg) +
754 ((sg_dma_address 754 umem->page_size * pg_cnt)));
755 (&chunk->page_list[i]) + 755 pbe_cnt += 1;
756 umem->page_size * pg_cnt))); 756 total_num_pbes += 1;
757 pbe_cnt += 1; 757 pbe++;
758 total_num_pbes += 1; 758
759 pbe++; 759 /* if done building pbes, issue the mbx cmd. */
760 760 if (total_num_pbes == num_pbes)
761 /* if done building pbes, issue the mbx cmd. */ 761 return;
762 if (total_num_pbes == num_pbes) 762
763 return; 763 /* if the given pbl is full storing the pbes,
764 764 * move to next pbl.
765 /* if the given pbl is full storing the pbes, 765 */
766 * move to next pbl. 766 if (pbe_cnt ==
767 */ 767 (mr->hwmr.pbl_size / sizeof(u64))) {
768 if (pbe_cnt == 768 pbl_tbl++;
769 (mr->hwmr.pbl_size / sizeof(u64))) { 769 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
770 pbl_tbl++; 770 pbe_cnt = 0;
771 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
772 pbe_cnt = 0;
773 }
774 } 771 }
772
775 } 773 }
776 } 774 }
777} 775}
diff --git a/drivers/infiniband/hw/qib/qib_mr.c b/drivers/infiniband/hw/qib/qib_mr.c
index e6687ded8210..9bbb55347cc1 100644
--- a/drivers/infiniband/hw/qib/qib_mr.c
+++ b/drivers/infiniband/hw/qib/qib_mr.c
@@ -232,8 +232,8 @@ struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
232{ 232{
233 struct qib_mr *mr; 233 struct qib_mr *mr;
234 struct ib_umem *umem; 234 struct ib_umem *umem;
235 struct ib_umem_chunk *chunk; 235 struct scatterlist *sg;
236 int n, m, i; 236 int n, m, entry;
237 struct ib_mr *ret; 237 struct ib_mr *ret;
238 238
239 if (length == 0) { 239 if (length == 0) {
@@ -246,9 +246,7 @@ struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
246 if (IS_ERR(umem)) 246 if (IS_ERR(umem))
247 return (void *) umem; 247 return (void *) umem;
248 248
249 n = 0; 249 n = umem->nmap;
250 list_for_each_entry(chunk, &umem->chunk_list, list)
251 n += chunk->nents;
252 250
253 mr = alloc_mr(n, pd); 251 mr = alloc_mr(n, pd);
254 if (IS_ERR(mr)) { 252 if (IS_ERR(mr)) {
@@ -268,11 +266,10 @@ struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
268 mr->mr.page_shift = ilog2(umem->page_size); 266 mr->mr.page_shift = ilog2(umem->page_size);
269 m = 0; 267 m = 0;
270 n = 0; 268 n = 0;
271 list_for_each_entry(chunk, &umem->chunk_list, list) { 269 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
272 for (i = 0; i < chunk->nents; i++) {
273 void *vaddr; 270 void *vaddr;
274 271
275 vaddr = page_address(sg_page(&chunk->page_list[i])); 272 vaddr = page_address(sg_page(sg));
276 if (!vaddr) { 273 if (!vaddr) {
277 ret = ERR_PTR(-EINVAL); 274 ret = ERR_PTR(-EINVAL);
278 goto bail; 275 goto bail;
@@ -284,7 +281,6 @@ struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
284 m++; 281 m++;
285 n = 0; 282 n = 0;
286 } 283 }
287 }
288 } 284 }
289 ret = &mr->ibmr; 285 ret = &mr->ibmr;
290 286
diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h
index 9ee0d2e51b16..1ea0b65c4cfb 100644
--- a/include/rdma/ib_umem.h
+++ b/include/rdma/ib_umem.h
@@ -46,17 +46,12 @@ struct ib_umem {
46 int page_size; 46 int page_size;
47 int writable; 47 int writable;
48 int hugetlb; 48 int hugetlb;
49 struct list_head chunk_list;
50 struct work_struct work; 49 struct work_struct work;
51 struct mm_struct *mm; 50 struct mm_struct *mm;
52 unsigned long diff; 51 unsigned long diff;
53}; 52 struct sg_table sg_head;
54 53 int nmap;
55struct ib_umem_chunk { 54 int npages;
56 struct list_head list;
57 int nents;
58 int nmap;
59 struct scatterlist page_list[0];
60}; 55};
61 56
62#ifdef CONFIG_INFINIBAND_USER_MEM 57#ifdef CONFIG_INFINIBAND_USER_MEM
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-07 13:45:48 -0500