diff options
author | Hoang-Nam Nguyen <hnguyen@de.ibm.com> | 2007-07-12 11:53:11 -0400 |
---|---|---|
committer | Roland Dreier <rolandd@cisco.com> | 2007-07-17 21:37:40 -0400 |
commit | 187c72e31f92791ec70395b80aa9883f2edad97f (patch) | |
tree | 74863c647f46f1282d9b8c9b0ebd7a3ea33fca74 /drivers/infiniband | |
parent | df17bfd4a030f7d986de14210f4b21876a7a2989 (diff) |
IB/ehca: Restructure ehca_set_pagebuf()
Split ehca_set_pagebuf() into three functions depending on MR type
(phys/user/fast) and remove superfluous ehca_set_pagebuf_1().
Signed-off-by: Joachim Fenkes <fenkes@de.ibm.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
Diffstat (limited to 'drivers/infiniband')
-rw-r--r-- | drivers/infiniband/hw/ehca/ehca_mrmw.c | 531 |
1 files changed, 200 insertions, 331 deletions
diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.c b/drivers/infiniband/hw/ehca/ehca_mrmw.c index 53b334b3c325..93c26ccfc3c5 100644 --- a/drivers/infiniband/hw/ehca/ehca_mrmw.c +++ b/drivers/infiniband/hw/ehca/ehca_mrmw.c | |||
@@ -824,6 +824,7 @@ int ehca_map_phys_fmr(struct ib_fmr *fmr, | |||
824 | pginfo.u.fmr.page_list = page_list; | 824 | pginfo.u.fmr.page_list = page_list; |
825 | pginfo.next_hwpage = ((iova & (e_fmr->fmr_page_size-1)) / | 825 | pginfo.next_hwpage = ((iova & (e_fmr->fmr_page_size-1)) / |
826 | EHCA_PAGESIZE); | 826 | EHCA_PAGESIZE); |
827 | pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size; | ||
827 | 828 | ||
828 | ret = ehca_rereg_mr(shca, e_fmr, (u64*)iova, | 829 | ret = ehca_rereg_mr(shca, e_fmr, (u64*)iova, |
829 | list_len * e_fmr->fmr_page_size, | 830 | list_len * e_fmr->fmr_page_size, |
@@ -1044,15 +1045,15 @@ int ehca_reg_mr_rpages(struct ehca_shca *shca, | |||
1044 | } else | 1045 | } else |
1045 | rnum = MAX_RPAGES; | 1046 | rnum = MAX_RPAGES; |
1046 | 1047 | ||
1047 | if (rnum > 1) { | 1048 | ret = ehca_set_pagebuf(pginfo, rnum, kpage); |
1048 | ret = ehca_set_pagebuf(e_mr, pginfo, rnum, kpage); | 1049 | if (ret) { |
1049 | if (ret) { | 1050 | ehca_err(&shca->ib_device, "ehca_set_pagebuf " |
1050 | ehca_err(&shca->ib_device, "ehca_set_pagebuf " | ||
1051 | "bad rc, ret=%x rnum=%x kpage=%p", | 1051 | "bad rc, ret=%x rnum=%x kpage=%p", |
1052 | ret, rnum, kpage); | 1052 | ret, rnum, kpage); |
1053 | ret = -EFAULT; | 1053 | goto ehca_reg_mr_rpages_exit1; |
1054 | goto ehca_reg_mr_rpages_exit1; | 1054 | } |
1055 | } | 1055 | |
1056 | if (rnum > 1) { | ||
1056 | rpage = virt_to_abs(kpage); | 1057 | rpage = virt_to_abs(kpage); |
1057 | if (!rpage) { | 1058 | if (!rpage) { |
1058 | ehca_err(&shca->ib_device, "kpage=%p i=%x", | 1059 | ehca_err(&shca->ib_device, "kpage=%p i=%x", |
@@ -1060,15 +1061,8 @@ int ehca_reg_mr_rpages(struct ehca_shca *shca, | |||
1060 | ret = -EFAULT; | 1061 | ret = -EFAULT; |
1061 | goto ehca_reg_mr_rpages_exit1; | 1062 | goto ehca_reg_mr_rpages_exit1; |
1062 | } | 1063 | } |
1063 | } else { /* rnum==1 */ | 1064 | } else |
1064 | ret = ehca_set_pagebuf_1(e_mr, pginfo, &rpage); | 1065 | rpage = *kpage; |
1065 | if (ret) { | ||
1066 | ehca_err(&shca->ib_device, "ehca_set_pagebuf_1 " | ||
1067 | "bad rc, ret=%x i=%x", ret, i); | ||
1068 | ret = -EFAULT; | ||
1069 | goto ehca_reg_mr_rpages_exit1; | ||
1070 | } | ||
1071 | } | ||
1072 | 1066 | ||
1073 | h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr, | 1067 | h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr, |
1074 | 0, /* pagesize 4k */ | 1068 | 0, /* pagesize 4k */ |
@@ -1146,7 +1140,7 @@ inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca, | |||
1146 | } | 1140 | } |
1147 | 1141 | ||
1148 | pginfo_save = *pginfo; | 1142 | pginfo_save = *pginfo; |
1149 | ret = ehca_set_pagebuf(e_mr, pginfo, pginfo->num_hwpages, kpage); | 1143 | ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage); |
1150 | if (ret) { | 1144 | if (ret) { |
1151 | ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p " | 1145 | ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p " |
1152 | "pginfo=%p type=%x num_kpages=%lx num_hwpages=%lx " | 1146 | "pginfo=%p type=%x num_kpages=%lx num_hwpages=%lx " |
@@ -1306,98 +1300,86 @@ int ehca_unmap_one_fmr(struct ehca_shca *shca, | |||
1306 | { | 1300 | { |
1307 | int ret = 0; | 1301 | int ret = 0; |
1308 | u64 h_ret; | 1302 | u64 h_ret; |
1309 | int rereg_1_hcall = 1; /* 1: use hipz_mr_reregister directly */ | ||
1310 | int rereg_3_hcall = 0; /* 1: use 3 hipz calls for unmapping */ | ||
1311 | struct ehca_pd *e_pd = | 1303 | struct ehca_pd *e_pd = |
1312 | container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd); | 1304 | container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd); |
1313 | struct ehca_mr save_fmr; | 1305 | struct ehca_mr save_fmr; |
1314 | u32 tmp_lkey, tmp_rkey; | 1306 | u32 tmp_lkey, tmp_rkey; |
1315 | struct ehca_mr_pginfo pginfo; | 1307 | struct ehca_mr_pginfo pginfo; |
1316 | struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0}; | 1308 | struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0}; |
1309 | struct ehca_mr save_mr; | ||
1317 | 1310 | ||
1318 | /* first check if reregistration hCall can be used for unmap */ | 1311 | if (e_fmr->fmr_max_pages <= MAX_RPAGES) { |
1319 | if (e_fmr->fmr_max_pages > MAX_RPAGES) { | ||
1320 | rereg_1_hcall = 0; | ||
1321 | rereg_3_hcall = 1; | ||
1322 | } | ||
1323 | |||
1324 | if (rereg_1_hcall) { | ||
1325 | /* | 1312 | /* |
1326 | * note: after using rereg hcall with len=0, | 1313 | * note: after using rereg hcall with len=0, |
1327 | * rereg hcall must be used again for registering pages | 1314 | * rereg hcall must be used again for registering pages |
1328 | */ | 1315 | */ |
1329 | h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0, | 1316 | h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0, |
1330 | 0, 0, e_pd->fw_pd, 0, &hipzout); | 1317 | 0, 0, e_pd->fw_pd, 0, &hipzout); |
1331 | if (h_ret != H_SUCCESS) { | 1318 | if (h_ret == H_SUCCESS) { |
1332 | /* | ||
1333 | * should not happen, because length checked above, | ||
1334 | * FMRs are not shared and no MW bound to FMRs | ||
1335 | */ | ||
1336 | ehca_err(&shca->ib_device, "hipz_reregister_pmr failed " | ||
1337 | "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx " | ||
1338 | "mr_hndl=%lx lkey=%x lkey_out=%x", | ||
1339 | h_ret, e_fmr, shca->ipz_hca_handle.handle, | ||
1340 | e_fmr->ipz_mr_handle.handle, | ||
1341 | e_fmr->ib.ib_fmr.lkey, hipzout.lkey); | ||
1342 | rereg_3_hcall = 1; | ||
1343 | } else { | ||
1344 | /* successful reregistration */ | 1319 | /* successful reregistration */ |
1345 | e_fmr->start = NULL; | 1320 | e_fmr->start = NULL; |
1346 | e_fmr->size = 0; | 1321 | e_fmr->size = 0; |
1347 | tmp_lkey = hipzout.lkey; | 1322 | tmp_lkey = hipzout.lkey; |
1348 | tmp_rkey = hipzout.rkey; | 1323 | tmp_rkey = hipzout.rkey; |
1324 | return 0; | ||
1349 | } | 1325 | } |
1326 | /* | ||
1327 | * should not happen, because length checked above, | ||
1328 | * FMRs are not shared and no MW bound to FMRs | ||
1329 | */ | ||
1330 | ehca_err(&shca->ib_device, "hipz_reregister_pmr failed " | ||
1331 | "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx " | ||
1332 | "mr_hndl=%lx lkey=%x lkey_out=%x", | ||
1333 | h_ret, e_fmr, shca->ipz_hca_handle.handle, | ||
1334 | e_fmr->ipz_mr_handle.handle, | ||
1335 | e_fmr->ib.ib_fmr.lkey, hipzout.lkey); | ||
1336 | /* try free and rereg */ | ||
1350 | } | 1337 | } |
1351 | 1338 | ||
1352 | if (rereg_3_hcall) { | 1339 | /* first free old FMR */ |
1353 | struct ehca_mr save_mr; | 1340 | h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr); |
1354 | 1341 | if (h_ret != H_SUCCESS) { | |
1355 | /* first free old FMR */ | 1342 | ehca_err(&shca->ib_device, "hipz_free_mr failed, " |
1356 | h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr); | 1343 | "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx " |
1357 | if (h_ret != H_SUCCESS) { | 1344 | "lkey=%x", |
1358 | ehca_err(&shca->ib_device, "hipz_free_mr failed, " | 1345 | h_ret, e_fmr, shca->ipz_hca_handle.handle, |
1359 | "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx " | 1346 | e_fmr->ipz_mr_handle.handle, |
1360 | "lkey=%x", | 1347 | e_fmr->ib.ib_fmr.lkey); |
1361 | h_ret, e_fmr, shca->ipz_hca_handle.handle, | 1348 | ret = ehca2ib_return_code(h_ret); |
1362 | e_fmr->ipz_mr_handle.handle, | 1349 | goto ehca_unmap_one_fmr_exit0; |
1363 | e_fmr->ib.ib_fmr.lkey); | 1350 | } |
1364 | ret = ehca2ib_return_code(h_ret); | 1351 | /* clean ehca_mr_t, without changing lock */ |
1365 | goto ehca_unmap_one_fmr_exit0; | 1352 | save_fmr = *e_fmr; |
1366 | } | 1353 | ehca_mr_deletenew(e_fmr); |
1367 | /* clean ehca_mr_t, without changing lock */ | ||
1368 | save_fmr = *e_fmr; | ||
1369 | ehca_mr_deletenew(e_fmr); | ||
1370 | 1354 | ||
1371 | /* set some MR values */ | 1355 | /* set some MR values */ |
1372 | e_fmr->flags = save_fmr.flags; | 1356 | e_fmr->flags = save_fmr.flags; |
1373 | e_fmr->fmr_page_size = save_fmr.fmr_page_size; | 1357 | e_fmr->fmr_page_size = save_fmr.fmr_page_size; |
1374 | e_fmr->fmr_max_pages = save_fmr.fmr_max_pages; | 1358 | e_fmr->fmr_max_pages = save_fmr.fmr_max_pages; |
1375 | e_fmr->fmr_max_maps = save_fmr.fmr_max_maps; | 1359 | e_fmr->fmr_max_maps = save_fmr.fmr_max_maps; |
1376 | e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt; | 1360 | e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt; |
1377 | e_fmr->acl = save_fmr.acl; | 1361 | e_fmr->acl = save_fmr.acl; |
1378 | 1362 | ||
1379 | memset(&pginfo, 0, sizeof(pginfo)); | 1363 | memset(&pginfo, 0, sizeof(pginfo)); |
1380 | pginfo.type = EHCA_MR_PGI_FMR; | 1364 | pginfo.type = EHCA_MR_PGI_FMR; |
1381 | pginfo.num_kpages = 0; | 1365 | pginfo.num_kpages = 0; |
1382 | pginfo.num_hwpages = 0; | 1366 | pginfo.num_hwpages = 0; |
1383 | ret = ehca_reg_mr(shca, e_fmr, NULL, | 1367 | ret = ehca_reg_mr(shca, e_fmr, NULL, |
1384 | (e_fmr->fmr_max_pages * e_fmr->fmr_page_size), | 1368 | (e_fmr->fmr_max_pages * e_fmr->fmr_page_size), |
1385 | e_fmr->acl, e_pd, &pginfo, &tmp_lkey, | 1369 | e_fmr->acl, e_pd, &pginfo, &tmp_lkey, |
1386 | &tmp_rkey); | 1370 | &tmp_rkey); |
1387 | if (ret) { | 1371 | if (ret) { |
1388 | u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr; | 1372 | u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr; |
1389 | memcpy(&e_fmr->flags, &(save_mr.flags), | 1373 | memcpy(&e_fmr->flags, &(save_mr.flags), |
1390 | sizeof(struct ehca_mr) - offset); | 1374 | sizeof(struct ehca_mr) - offset); |
1391 | goto ehca_unmap_one_fmr_exit0; | 1375 | goto ehca_unmap_one_fmr_exit0; |
1392 | } | ||
1393 | } | 1376 | } |
1394 | 1377 | ||
1395 | ehca_unmap_one_fmr_exit0: | 1378 | ehca_unmap_one_fmr_exit0: |
1396 | if (ret) | 1379 | if (ret) |
1397 | ehca_err(&shca->ib_device, "ret=%x tmp_lkey=%x tmp_rkey=%x " | 1380 | ehca_err(&shca->ib_device, "ret=%x tmp_lkey=%x tmp_rkey=%x " |
1398 | "fmr_max_pages=%x rereg_1_hcall=%x rereg_3_hcall=%x", | 1381 | "fmr_max_pages=%x", |
1399 | ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages, | 1382 | ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages); |
1400 | rereg_1_hcall, rereg_3_hcall); | ||
1401 | return ret; | 1383 | return ret; |
1402 | } /* end ehca_unmap_one_fmr() */ | 1384 | } /* end ehca_unmap_one_fmr() */ |
1403 | 1385 | ||
@@ -1690,300 +1672,187 @@ int ehca_fmr_check_page_list(struct ehca_mr *e_fmr, | |||
1690 | 1672 | ||
1691 | /*----------------------------------------------------------------------*/ | 1673 | /*----------------------------------------------------------------------*/ |
1692 | 1674 | ||
1693 | /* setup page buffer from page info */ | 1675 | /* PAGE_SIZE >= pginfo->hwpage_size */ |
1694 | int ehca_set_pagebuf(struct ehca_mr *e_mr, | 1676 | static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo, |
1695 | struct ehca_mr_pginfo *pginfo, | 1677 | u32 number, |
1696 | u32 number, | 1678 | u64 *kpage) |
1697 | u64 *kpage) | ||
1698 | { | 1679 | { |
1699 | int ret = 0; | 1680 | int ret = 0; |
1700 | struct ib_umem_chunk *prev_chunk; | 1681 | struct ib_umem_chunk *prev_chunk; |
1701 | struct ib_umem_chunk *chunk; | 1682 | struct ib_umem_chunk *chunk; |
1702 | struct ib_phys_buf *pbuf; | 1683 | u64 pgaddr; |
1703 | u64 *fmrlist; | ||
1704 | u64 num_hw, pgaddr, offs_hw; | ||
1705 | u32 i = 0; | 1684 | u32 i = 0; |
1706 | u32 j = 0; | 1685 | u32 j = 0; |
1707 | 1686 | ||
1708 | if (pginfo->type == EHCA_MR_PGI_PHYS) { | 1687 | /* loop over desired chunk entries */ |
1709 | /* loop over desired phys_buf_array entries */ | 1688 | chunk = pginfo->u.usr.next_chunk; |
1710 | while (i < number) { | 1689 | prev_chunk = pginfo->u.usr.next_chunk; |
1711 | pbuf = pginfo->u.phy.phys_buf_array | 1690 | list_for_each_entry_continue( |
1712 | + pginfo->u.phy.next_buf; | 1691 | chunk, (&(pginfo->u.usr.region->chunk_list)), list) { |
1713 | num_hw = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) + | 1692 | for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) { |
1714 | pbuf->size, EHCA_PAGESIZE); | 1693 | pgaddr = page_to_pfn(chunk->page_list[i].page) |
1715 | offs_hw = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE; | 1694 | << PAGE_SHIFT ; |
1716 | while (pginfo->next_hwpage < offs_hw + num_hw) { | 1695 | *kpage = phys_to_abs(pgaddr + |
1717 | /* sanity check */ | 1696 | (pginfo->next_hwpage * |
1718 | if ((pginfo->kpage_cnt >= pginfo->num_kpages) || | 1697 | EHCA_PAGESIZE)); |
1719 | (pginfo->hwpage_cnt >= pginfo->num_hwpages)) { | ||
1720 | ehca_gen_err("kpage_cnt >= num_kpages, " | ||
1721 | "kpage_cnt=%lx " | ||
1722 | "num_kpages=%lx " | ||
1723 | "hwpage_cnt=%lx " | ||
1724 | "num_hwpages=%lx i=%x", | ||
1725 | pginfo->kpage_cnt, | ||
1726 | pginfo->num_kpages, | ||
1727 | pginfo->hwpage_cnt, | ||
1728 | pginfo->num_hwpages, i); | ||
1729 | ret = -EFAULT; | ||
1730 | goto ehca_set_pagebuf_exit0; | ||
1731 | } | ||
1732 | *kpage = phys_to_abs( | ||
1733 | (pbuf->addr & EHCA_PAGEMASK) | ||
1734 | + (pginfo->next_hwpage * EHCA_PAGESIZE)); | ||
1735 | if ( !(*kpage) && pbuf->addr ) { | ||
1736 | ehca_gen_err("pbuf->addr=%lx " | ||
1737 | "pbuf->size=%lx " | ||
1738 | "next_hwpage=%lx", pbuf->addr, | ||
1739 | pbuf->size, | ||
1740 | pginfo->next_hwpage); | ||
1741 | ret = -EFAULT; | ||
1742 | goto ehca_set_pagebuf_exit0; | ||
1743 | } | ||
1744 | (pginfo->hwpage_cnt)++; | ||
1745 | (pginfo->next_hwpage)++; | ||
1746 | if (pginfo->next_hwpage % | ||
1747 | (PAGE_SIZE / EHCA_PAGESIZE) == 0) | ||
1748 | (pginfo->kpage_cnt)++; | ||
1749 | kpage++; | ||
1750 | i++; | ||
1751 | if (i >= number) break; | ||
1752 | } | ||
1753 | if (pginfo->next_hwpage >= offs_hw + num_hw) { | ||
1754 | (pginfo->u.phy.next_buf)++; | ||
1755 | pginfo->next_hwpage = 0; | ||
1756 | } | ||
1757 | } | ||
1758 | } else if (pginfo->type == EHCA_MR_PGI_USER) { | ||
1759 | /* loop over desired chunk entries */ | ||
1760 | chunk = pginfo->u.usr.next_chunk; | ||
1761 | prev_chunk = pginfo->u.usr.next_chunk; | ||
1762 | list_for_each_entry_continue(chunk, | ||
1763 | (&(pginfo->u.usr.region->chunk_list)), | ||
1764 | list) { | ||
1765 | for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) { | ||
1766 | pgaddr = ( page_to_pfn(chunk->page_list[i].page) | ||
1767 | << PAGE_SHIFT ); | ||
1768 | *kpage = phys_to_abs(pgaddr + | ||
1769 | (pginfo->next_hwpage * | ||
1770 | EHCA_PAGESIZE)); | ||
1771 | if ( !(*kpage) ) { | ||
1772 | ehca_gen_err("pgaddr=%lx " | ||
1773 | "chunk->page_list[i]=%lx " | ||
1774 | "i=%x next_hwpage=%lx mr=%p", | ||
1775 | pgaddr, | ||
1776 | (u64)sg_dma_address( | ||
1777 | &chunk-> | ||
1778 | page_list[i]), | ||
1779 | i, pginfo->next_hwpage, e_mr); | ||
1780 | ret = -EFAULT; | ||
1781 | goto ehca_set_pagebuf_exit0; | ||
1782 | } | ||
1783 | (pginfo->hwpage_cnt)++; | ||
1784 | (pginfo->next_hwpage)++; | ||
1785 | kpage++; | ||
1786 | if (pginfo->next_hwpage % | ||
1787 | (PAGE_SIZE / EHCA_PAGESIZE) == 0) { | ||
1788 | (pginfo->kpage_cnt)++; | ||
1789 | (pginfo->u.usr.next_nmap)++; | ||
1790 | pginfo->next_hwpage = 0; | ||
1791 | i++; | ||
1792 | } | ||
1793 | j++; | ||
1794 | if (j >= number) break; | ||
1795 | } | ||
1796 | if ((pginfo->u.usr.next_nmap >= chunk->nmap) && | ||
1797 | (j >= number)) { | ||
1798 | pginfo->u.usr.next_nmap = 0; | ||
1799 | prev_chunk = chunk; | ||
1800 | break; | ||
1801 | } else if (pginfo->u.usr.next_nmap >= chunk->nmap) { | ||
1802 | pginfo->u.usr.next_nmap = 0; | ||
1803 | prev_chunk = chunk; | ||
1804 | } else if (j >= number) | ||
1805 | break; | ||
1806 | else | ||
1807 | prev_chunk = chunk; | ||
1808 | } | ||
1809 | pginfo->u.usr.next_chunk = | ||
1810 | list_prepare_entry(prev_chunk, | ||
1811 | (&(pginfo->u.usr.region->chunk_list)), | ||
1812 | list); | ||
1813 | } else if (pginfo->type == EHCA_MR_PGI_FMR) { | ||
1814 | /* loop over desired page_list entries */ | ||
1815 | fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem; | ||
1816 | for (i = 0; i < number; i++) { | ||
1817 | *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) + | ||
1818 | pginfo->next_hwpage * EHCA_PAGESIZE); | ||
1819 | if ( !(*kpage) ) { | 1698 | if ( !(*kpage) ) { |
1820 | ehca_gen_err("*fmrlist=%lx fmrlist=%p " | 1699 | ehca_gen_err("pgaddr=%lx " |
1821 | "next_listelem=%lx next_hwpage=%lx", | 1700 | "chunk->page_list[i]=%lx " |
1822 | *fmrlist, fmrlist, | 1701 | "i=%x next_hwpage=%lx", |
1823 | pginfo->u.fmr.next_listelem, | 1702 | pgaddr, (u64)sg_dma_address( |
1824 | pginfo->next_hwpage); | 1703 | &chunk->page_list[i]), |
1825 | ret = -EFAULT; | 1704 | i, pginfo->next_hwpage); |
1826 | goto ehca_set_pagebuf_exit0; | 1705 | return -EFAULT; |
1827 | } | 1706 | } |
1828 | (pginfo->hwpage_cnt)++; | 1707 | (pginfo->hwpage_cnt)++; |
1829 | (pginfo->next_hwpage)++; | 1708 | (pginfo->next_hwpage)++; |
1830 | kpage++; | 1709 | kpage++; |
1831 | if (pginfo->next_hwpage % | 1710 | if (pginfo->next_hwpage % |
1832 | (e_mr->fmr_page_size / EHCA_PAGESIZE) == 0) { | 1711 | (PAGE_SIZE / EHCA_PAGESIZE) == 0) { |
1833 | (pginfo->kpage_cnt)++; | 1712 | (pginfo->kpage_cnt)++; |
1834 | (pginfo->u.fmr.next_listelem)++; | 1713 | (pginfo->u.usr.next_nmap)++; |
1835 | fmrlist++; | ||
1836 | pginfo->next_hwpage = 0; | 1714 | pginfo->next_hwpage = 0; |
1715 | i++; | ||
1837 | } | 1716 | } |
1717 | j++; | ||
1718 | if (j >= number) break; | ||
1838 | } | 1719 | } |
1839 | } else { | 1720 | if ((pginfo->u.usr.next_nmap >= chunk->nmap) && |
1840 | ehca_gen_err("bad pginfo->type=%x", pginfo->type); | 1721 | (j >= number)) { |
1841 | ret = -EFAULT; | 1722 | pginfo->u.usr.next_nmap = 0; |
1842 | goto ehca_set_pagebuf_exit0; | 1723 | prev_chunk = chunk; |
1724 | break; | ||
1725 | } else if (pginfo->u.usr.next_nmap >= chunk->nmap) { | ||
1726 | pginfo->u.usr.next_nmap = 0; | ||
1727 | prev_chunk = chunk; | ||
1728 | } else if (j >= number) | ||
1729 | break; | ||
1730 | else | ||
1731 | prev_chunk = chunk; | ||
1843 | } | 1732 | } |
1844 | 1733 | pginfo->u.usr.next_chunk = | |
1845 | ehca_set_pagebuf_exit0: | 1734 | list_prepare_entry(prev_chunk, |
1846 | if (ret) | 1735 | (&(pginfo->u.usr.region->chunk_list)), |
1847 | ehca_gen_err("ret=%x e_mr=%p pginfo=%p type=%x num_kpages=%lx " | 1736 | list); |
1848 | "num_hwpages=%lx next_buf=%lx next_hwpage=%lx number=%x " | ||
1849 | "kpage=%p kpage_cnt=%lx hwpage_cnt=%lx i=%x " | ||
1850 | "next_listelem=%lx region=%p next_chunk=%p " | ||
1851 | "next_nmap=%lx", ret, e_mr, pginfo, pginfo->type, | ||
1852 | pginfo->num_kpages, pginfo->num_hwpages, | ||
1853 | pginfo->u.phy.next_buf, pginfo->next_hwpage, number, kpage, | ||
1854 | pginfo->kpage_cnt, pginfo->hwpage_cnt, i, | ||
1855 | pginfo->u.fmr.next_listelem, pginfo->u.usr.region, | ||
1856 | pginfo->u.usr.next_chunk, pginfo->u.usr.next_nmap); | ||
1857 | return ret; | 1737 | return ret; |
1858 | } /* end ehca_set_pagebuf() */ | 1738 | } |
1859 | |||
1860 | /*----------------------------------------------------------------------*/ | ||
1861 | 1739 | ||
1862 | /* setup 1 page from page info page buffer */ | 1740 | int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo, |
1863 | int ehca_set_pagebuf_1(struct ehca_mr *e_mr, | 1741 | u32 number, |
1864 | struct ehca_mr_pginfo *pginfo, | 1742 | u64 *kpage) |
1865 | u64 *rpage) | ||
1866 | { | 1743 | { |
1867 | int ret = 0; | 1744 | int ret = 0; |
1868 | struct ib_phys_buf *tmp_pbuf; | 1745 | struct ib_phys_buf *pbuf; |
1869 | u64 *fmrlist; | 1746 | u64 num_hw, offs_hw; |
1870 | struct ib_umem_chunk *chunk; | 1747 | u32 i = 0; |
1871 | struct ib_umem_chunk *prev_chunk; | 1748 | |
1872 | u64 pgaddr, num_hw, offs_hw; | 1749 | /* loop over desired phys_buf_array entries */ |
1873 | 1750 | while (i < number) { | |
1874 | if (pginfo->type == EHCA_MR_PGI_PHYS) { | 1751 | pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf; |
1875 | /* sanity check */ | 1752 | num_hw = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) + |
1876 | if ((pginfo->kpage_cnt >= pginfo->num_kpages) || | 1753 | pbuf->size, EHCA_PAGESIZE); |
1877 | (pginfo->hwpage_cnt >= pginfo->num_hwpages)) { | 1754 | offs_hw = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE; |
1878 | ehca_gen_err("kpage_cnt >= num_hwpages, kpage_cnt=%lx " | 1755 | while (pginfo->next_hwpage < offs_hw + num_hw) { |
1879 | "num_hwpages=%lx hwpage_cnt=%lx num_hwpages=%lx", | 1756 | /* sanity check */ |
1880 | pginfo->kpage_cnt, pginfo->num_kpages, | 1757 | if ((pginfo->kpage_cnt >= pginfo->num_kpages) || |
1881 | pginfo->hwpage_cnt, pginfo->num_hwpages); | 1758 | (pginfo->hwpage_cnt >= pginfo->num_hwpages)) { |
1882 | ret = -EFAULT; | 1759 | ehca_gen_err("kpage_cnt >= num_kpages, " |
1883 | goto ehca_set_pagebuf_1_exit0; | 1760 | "kpage_cnt=%lx num_kpages=%lx " |
1884 | } | 1761 | "hwpage_cnt=%lx " |
1885 | tmp_pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf; | 1762 | "num_hwpages=%lx i=%x", |
1886 | num_hw = NUM_CHUNKS((tmp_pbuf->addr % EHCA_PAGESIZE) + | 1763 | pginfo->kpage_cnt, |
1887 | tmp_pbuf->size, EHCA_PAGESIZE); | 1764 | pginfo->num_kpages, |
1888 | offs_hw = (tmp_pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE; | 1765 | pginfo->hwpage_cnt, |
1889 | *rpage = phys_to_abs((tmp_pbuf->addr & EHCA_PAGEMASK) + | 1766 | pginfo->num_hwpages, i); |
1890 | (pginfo->next_hwpage * EHCA_PAGESIZE)); | 1767 | return -EFAULT; |
1891 | if ( !(*rpage) && tmp_pbuf->addr ) { | 1768 | } |
1892 | ehca_gen_err("tmp_pbuf->addr=%lx" | 1769 | *kpage = phys_to_abs( |
1893 | " tmp_pbuf->size=%lx next_hwpage=%lx", | 1770 | (pbuf->addr & EHCA_PAGEMASK) |
1894 | tmp_pbuf->addr, tmp_pbuf->size, | 1771 | + (pginfo->next_hwpage * EHCA_PAGESIZE)); |
1895 | pginfo->next_hwpage); | 1772 | if ( !(*kpage) && pbuf->addr ) { |
1896 | ret = -EFAULT; | 1773 | ehca_gen_err("pbuf->addr=%lx " |
1897 | goto ehca_set_pagebuf_1_exit0; | 1774 | "pbuf->size=%lx " |
1898 | } | 1775 | "next_hwpage=%lx", pbuf->addr, |
1899 | (pginfo->hwpage_cnt)++; | 1776 | pbuf->size, |
1900 | (pginfo->next_hwpage)++; | 1777 | pginfo->next_hwpage); |
1901 | if (pginfo->next_hwpage % (PAGE_SIZE / EHCA_PAGESIZE) == 0) | 1778 | return -EFAULT; |
1902 | (pginfo->kpage_cnt)++; | ||
1903 | if (pginfo->next_hwpage >= offs_hw + num_hw) { | ||
1904 | (pginfo->u.phy.next_buf)++; | ||
1905 | pginfo->next_hwpage = 0; | ||
1906 | } | ||
1907 | } else if (pginfo->type == EHCA_MR_PGI_USER) { | ||
1908 | chunk = pginfo->u.usr.next_chunk; | ||
1909 | prev_chunk = pginfo->u.usr.next_chunk; | ||
1910 | list_for_each_entry_continue(chunk, | ||
1911 | (&(pginfo->u.usr.region->chunk_list)), | ||
1912 | list) { | ||
1913 | pgaddr = ( page_to_pfn(chunk->page_list[ | ||
1914 | pginfo->u.usr.next_nmap].page) | ||
1915 | << PAGE_SHIFT); | ||
1916 | *rpage = phys_to_abs(pgaddr + | ||
1917 | (pginfo->next_hwpage * EHCA_PAGESIZE)); | ||
1918 | if ( !(*rpage) ) { | ||
1919 | ehca_gen_err("pgaddr=%lx chunk->page_list[]=%lx" | ||
1920 | " next_nmap=%lx next_hwpage=%lx mr=%p", | ||
1921 | pgaddr, (u64)sg_dma_address( | ||
1922 | &chunk->page_list[ | ||
1923 | pginfo->u.usr. | ||
1924 | next_nmap]), | ||
1925 | pginfo->u.usr.next_nmap, pginfo->next_hwpage, | ||
1926 | e_mr); | ||
1927 | ret = -EFAULT; | ||
1928 | goto ehca_set_pagebuf_1_exit0; | ||
1929 | } | 1779 | } |
1930 | (pginfo->hwpage_cnt)++; | 1780 | (pginfo->hwpage_cnt)++; |
1931 | (pginfo->next_hwpage)++; | 1781 | (pginfo->next_hwpage)++; |
1932 | if (pginfo->next_hwpage % | 1782 | if (pginfo->next_hwpage % |
1933 | (PAGE_SIZE / EHCA_PAGESIZE) == 0) { | 1783 | (PAGE_SIZE / EHCA_PAGESIZE) == 0) |
1934 | (pginfo->kpage_cnt)++; | 1784 | (pginfo->kpage_cnt)++; |
1935 | (pginfo->u.usr.next_nmap)++; | 1785 | kpage++; |
1936 | pginfo->next_hwpage = 0; | 1786 | i++; |
1937 | } | 1787 | if (i >= number) break; |
1938 | if (pginfo->u.usr.next_nmap >= chunk->nmap) { | ||
1939 | pginfo->u.usr.next_nmap = 0; | ||
1940 | prev_chunk = chunk; | ||
1941 | } | ||
1942 | break; | ||
1943 | } | 1788 | } |
1944 | pginfo->u.usr.next_chunk = | 1789 | if (pginfo->next_hwpage >= offs_hw + num_hw) { |
1945 | list_prepare_entry(prev_chunk, | 1790 | (pginfo->u.phy.next_buf)++; |
1946 | (&(pginfo->u.usr.region->chunk_list)), | 1791 | pginfo->next_hwpage = 0; |
1947 | list); | 1792 | } |
1948 | } else if (pginfo->type == EHCA_MR_PGI_FMR) { | 1793 | } |
1949 | fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem; | 1794 | return ret; |
1950 | *rpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) + | 1795 | } |
1796 | |||
1797 | int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo, | ||
1798 | u32 number, | ||
1799 | u64 *kpage) | ||
1800 | { | ||
1801 | int ret = 0; | ||
1802 | u64 *fmrlist; | ||
1803 | u32 i; | ||
1804 | |||
1805 | /* loop over desired page_list entries */ | ||
1806 | fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem; | ||
1807 | for (i = 0; i < number; i++) { | ||
1808 | *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) + | ||
1951 | pginfo->next_hwpage * EHCA_PAGESIZE); | 1809 | pginfo->next_hwpage * EHCA_PAGESIZE); |
1952 | if ( !(*rpage) ) { | 1810 | if ( !(*kpage) ) { |
1953 | ehca_gen_err("*fmrlist=%lx fmrlist=%p " | 1811 | ehca_gen_err("*fmrlist=%lx fmrlist=%p " |
1954 | "next_listelem=%lx next_hwpage=%lx", | 1812 | "next_listelem=%lx next_hwpage=%lx", |
1955 | *fmrlist, fmrlist, pginfo->u.fmr.next_listelem, | 1813 | *fmrlist, fmrlist, |
1814 | pginfo->u.fmr.next_listelem, | ||
1956 | pginfo->next_hwpage); | 1815 | pginfo->next_hwpage); |
1957 | ret = -EFAULT; | 1816 | return -EFAULT; |
1958 | goto ehca_set_pagebuf_1_exit0; | ||
1959 | } | 1817 | } |
1960 | (pginfo->hwpage_cnt)++; | 1818 | (pginfo->hwpage_cnt)++; |
1961 | (pginfo->next_hwpage)++; | 1819 | (pginfo->next_hwpage)++; |
1820 | kpage++; | ||
1962 | if (pginfo->next_hwpage % | 1821 | if (pginfo->next_hwpage % |
1963 | (e_mr->fmr_page_size / EHCA_PAGESIZE) == 0) { | 1822 | (pginfo->u.fmr.fmr_pgsize / EHCA_PAGESIZE) == 0) { |
1964 | (pginfo->kpage_cnt)++; | 1823 | (pginfo->kpage_cnt)++; |
1965 | (pginfo->u.fmr.next_listelem)++; | 1824 | (pginfo->u.fmr.next_listelem)++; |
1825 | fmrlist++; | ||
1966 | pginfo->next_hwpage = 0; | 1826 | pginfo->next_hwpage = 0; |
1967 | } | 1827 | } |
1968 | } else { | 1828 | } |
1829 | return ret; | ||
1830 | } | ||
1831 | |||
1832 | /* setup page buffer from page info */ | ||
1833 | int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo, | ||
1834 | u32 number, | ||
1835 | u64 *kpage) | ||
1836 | { | ||
1837 | int ret; | ||
1838 | |||
1839 | switch (pginfo->type) { | ||
1840 | case EHCA_MR_PGI_PHYS: | ||
1841 | ret = ehca_set_pagebuf_phys(pginfo, number, kpage); | ||
1842 | break; | ||
1843 | case EHCA_MR_PGI_USER: | ||
1844 | ret = ehca_set_pagebuf_user1(pginfo, number, kpage); | ||
1845 | break; | ||
1846 | case EHCA_MR_PGI_FMR: | ||
1847 | ret = ehca_set_pagebuf_fmr(pginfo, number, kpage); | ||
1848 | break; | ||
1849 | default: | ||
1969 | ehca_gen_err("bad pginfo->type=%x", pginfo->type); | 1850 | ehca_gen_err("bad pginfo->type=%x", pginfo->type); |
1970 | ret = -EFAULT; | 1851 | ret = -EFAULT; |
1971 | goto ehca_set_pagebuf_1_exit0; | 1852 | break; |
1972 | } | 1853 | } |
1973 | |||
1974 | ehca_set_pagebuf_1_exit0: | ||
1975 | if (ret) | ||
1976 | ehca_gen_err("ret=%x e_mr=%p pginfo=%p type=%x num_kpages=%lx " | ||
1977 | "num_hwpages=%lx next_buf=%lx next_hwpage=%lx rpage=%p " | ||
1978 | "kpage_cnt=%lx hwpage_cnt=%lx next_listelem=%lx " | ||
1979 | "region=%p next_chunk=%p next_nmap=%lx", ret, e_mr, | ||
1980 | pginfo, pginfo->type, pginfo->num_kpages, | ||
1981 | pginfo->num_hwpages, pginfo->u.phy.next_buf, pginfo->next_hwpage, | ||
1982 | rpage, pginfo->kpage_cnt, pginfo->hwpage_cnt, | ||
1983 | pginfo->u.fmr.next_listelem, pginfo->u.usr.region, | ||
1984 | pginfo->u.usr.next_chunk, pginfo->u.usr.next_nmap); | ||
1985 | return ret; | 1854 | return ret; |
1986 | } /* end ehca_set_pagebuf_1() */ | 1855 | } /* end ehca_set_pagebuf() */ |
1987 | 1856 | ||
1988 | /*----------------------------------------------------------------------*/ | 1857 | /*----------------------------------------------------------------------*/ |
1989 | 1858 | ||