aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/infiniband/hw/nes/nes_verbs.c191
1 files changed, 134 insertions, 57 deletions
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index b42b17ac7712..96d953540a2c 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -1887,21 +1887,75 @@ static int nes_destroy_cq(struct ib_cq *ib_cq)
1887 return ret; 1887 return ret;
1888} 1888}
1889 1889
1890/**
1891 * root_256
1892 */
1893static u32 root_256(struct nes_device *nesdev,
1894 struct nes_root_vpbl *root_vpbl,
1895 struct nes_root_vpbl *new_root,
1896 u16 pbl_count_4k,
1897 u16 pbl_count_256)
1898{
1899 u64 leaf_pbl;
1900 int i, j, k;
1901
1902 if (pbl_count_4k == 1) {
1903 new_root->pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
1904 512, &new_root->pbl_pbase);
1905
1906 if (new_root->pbl_vbase == NULL)
1907 return 0;
1908
1909 leaf_pbl = (u64)root_vpbl->pbl_pbase;
1910 for (i = 0; i < 16; i++) {
1911 new_root->pbl_vbase[i].pa_low =
1912 cpu_to_le32((u32)leaf_pbl);
1913 new_root->pbl_vbase[i].pa_high =
1914 cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
1915 leaf_pbl += 256;
1916 }
1917 } else {
1918 for (i = 3; i >= 0; i--) {
1919 j = i * 16;
1920 root_vpbl->pbl_vbase[j] = root_vpbl->pbl_vbase[i];
1921 leaf_pbl = le32_to_cpu(root_vpbl->pbl_vbase[j].pa_low) +
1922 (((u64)le32_to_cpu(root_vpbl->pbl_vbase[j].pa_high))
1923 << 32);
1924 for (k = 1; k < 16; k++) {
1925 leaf_pbl += 256;
1926 root_vpbl->pbl_vbase[j + k].pa_low =
1927 cpu_to_le32((u32)leaf_pbl);
1928 root_vpbl->pbl_vbase[j + k].pa_high =
1929 cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
1930 }
1931 }
1932 }
1933
1934 return 1;
1935}
1936
1890 1937
1891/** 1938/**
1892 * nes_reg_mr 1939 * nes_reg_mr
1893 */ 1940 */
1894static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, 1941static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
1895 u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl, 1942 u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl,
1896 dma_addr_t single_buffer, u16 pbl_count, u16 residual_page_count, 1943 dma_addr_t single_buffer, u16 pbl_count_4k,
1897 int acc, u64 *iova_start) 1944 u16 residual_page_count_4k, int acc, u64 *iova_start,
1945 u16 *actual_pbl_cnt, u8 *used_4k_pbls)
1898{ 1946{
1899 struct nes_hw_cqp_wqe *cqp_wqe; 1947 struct nes_hw_cqp_wqe *cqp_wqe;
1900 struct nes_cqp_request *cqp_request; 1948 struct nes_cqp_request *cqp_request;
1901 unsigned long flags; 1949 unsigned long flags;
1902 int ret; 1950 int ret;
1903 struct nes_adapter *nesadapter = nesdev->nesadapter; 1951 struct nes_adapter *nesadapter = nesdev->nesadapter;
1904 /* int count; */ 1952 uint pg_cnt = 0;
1953 u16 pbl_count_256;
1954 u16 pbl_count = 0;
1955 u8 use_256_pbls = 0;
1956 u8 use_4k_pbls = 0;
1957 u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0;
1958 struct nes_root_vpbl new_root = {0, 0, 0};
1905 u32 opcode = 0; 1959 u32 opcode = 0;
1906 u16 major_code; 1960 u16 major_code;
1907 1961
@@ -1914,41 +1968,70 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
1914 cqp_request->waiting = 1; 1968 cqp_request->waiting = 1;
1915 cqp_wqe = &cqp_request->cqp_wqe; 1969 cqp_wqe = &cqp_request->cqp_wqe;
1916 1970
1917 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1971 if (pbl_count_4k) {
1918 /* track PBL resources */ 1972 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
1919 if (pbl_count != 0) { 1973
1920 if (pbl_count > 1) { 1974 pg_cnt = ((pbl_count_4k - 1) * 512) + residual_page_count_4k;
1921 /* Two level PBL */ 1975 pbl_count_256 = (pg_cnt + 31) / 32;
1922 if ((pbl_count+1) > nesadapter->free_4kpbl) { 1976 if (pg_cnt <= 32) {
1923 nes_debug(NES_DBG_MR, "Out of 4KB Pbls for two level request.\n"); 1977 if (pbl_count_256 <= nesadapter->free_256pbl)
1924 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1978 use_256_pbls = 1;
1925 nes_free_cqp_request(nesdev, cqp_request); 1979 else if (pbl_count_4k <= nesadapter->free_4kpbl)
1926 return -ENOMEM; 1980 use_4k_pbls = 1;
1927 } else { 1981 } else if (pg_cnt <= 2048) {
1928 nesadapter->free_4kpbl -= pbl_count+1; 1982 if (((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) &&
1929 } 1983 (nesadapter->free_4kpbl > (nesadapter->max_4kpbl >> 1))) {
1930 } else if (residual_page_count > 32) { 1984 use_4k_pbls = 1;
1931 if (pbl_count > nesadapter->free_4kpbl) { 1985 } else if ((pbl_count_256 + 1) <= nesadapter->free_256pbl) {
1932 nes_debug(NES_DBG_MR, "Out of 4KB Pbls.\n"); 1986 use_256_pbls = 1;
1933 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1987 use_two_level = 1;
1934 nes_free_cqp_request(nesdev, cqp_request); 1988 } else if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
1935 return -ENOMEM; 1989 use_4k_pbls = 1;
1936 } else {
1937 nesadapter->free_4kpbl -= pbl_count;
1938 } 1990 }
1939 } else { 1991 } else {
1940 if (pbl_count > nesadapter->free_256pbl) { 1992 if ((pbl_count_4k + 1) <= nesadapter->free_4kpbl)
1941 nes_debug(NES_DBG_MR, "Out of 256B Pbls.\n"); 1993 use_4k_pbls = 1;
1942 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1994 }
1943 nes_free_cqp_request(nesdev, cqp_request); 1995
1944 return -ENOMEM; 1996 if (use_256_pbls) {
1945 } else { 1997 pbl_count = pbl_count_256;
1946 nesadapter->free_256pbl -= pbl_count; 1998 nesadapter->free_256pbl -= pbl_count + use_two_level;
1947 } 1999 } else if (use_4k_pbls) {
2000 pbl_count = pbl_count_4k;
2001 nesadapter->free_4kpbl -= pbl_count + use_two_level;
2002 } else {
2003 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
2004 nes_debug(NES_DBG_MR, "Out of Pbls\n");
2005 nes_free_cqp_request(nesdev, cqp_request);
2006 return -ENOMEM;
1948 } 2007 }
2008
2009 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
1949 } 2010 }
1950 2011
1951 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2012 if (use_256_pbls && use_two_level) {
2013 if (root_256(nesdev, root_vpbl, &new_root, pbl_count_4k, pbl_count_256) == 1) {
2014 if (new_root.pbl_pbase != 0)
2015 root_vpbl = &new_root;
2016 } else {
2017 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
2018 nesadapter->free_256pbl += pbl_count_256 + use_two_level;
2019 use_256_pbls = 0;
2020
2021 if (pbl_count_4k == 1)
2022 use_two_level = 0;
2023 pbl_count = pbl_count_4k;
2024
2025 if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
2026 nesadapter->free_4kpbl -= pbl_count + use_two_level;
2027 use_4k_pbls = 1;
2028 }
2029 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
2030
2031 if (use_4k_pbls == 0)
2032 return -ENOMEM;
2033 }
2034 }
1952 2035
1953 opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ | 2036 opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ |
1954 NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR; 2037 NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR;
@@ -1977,10 +2060,9 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
1977 } else { 2060 } else {
1978 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase); 2061 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase);
1979 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count); 2062 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count);
1980 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, 2063 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (pg_cnt * 8));
1981 (((pbl_count - 1) * 4096) + (residual_page_count*8)));
1982 2064
1983 if ((pbl_count > 1) || (residual_page_count > 32)) 2065 if (use_4k_pbls)
1984 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE); 2066 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
1985 } 2067 }
1986 barrier(); 2068 barrier();
@@ -1996,23 +2078,26 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
1996 stag, ret, cqp_request->major_code, cqp_request->minor_code); 2078 stag, ret, cqp_request->major_code, cqp_request->minor_code);
1997 major_code = cqp_request->major_code; 2079 major_code = cqp_request->major_code;
1998 nes_put_cqp_request(nesdev, cqp_request); 2080 nes_put_cqp_request(nesdev, cqp_request);
2081
1999 if ((!ret || major_code) && pbl_count != 0) { 2082 if ((!ret || major_code) && pbl_count != 0) {
2000 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 2083 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
2001 if (pbl_count > 1) 2084 if (use_256_pbls)
2002 nesadapter->free_4kpbl += pbl_count+1; 2085 nesadapter->free_256pbl += pbl_count + use_two_level;
2003 else if (residual_page_count > 32) 2086 else if (use_4k_pbls)
2004 nesadapter->free_4kpbl += pbl_count; 2087 nesadapter->free_4kpbl += pbl_count + use_two_level;
2005 else
2006 nesadapter->free_256pbl += pbl_count;
2007 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2088 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
2008 } 2089 }
2090 if (new_root.pbl_pbase)
2091 pci_free_consistent(nesdev->pcidev, 512, new_root.pbl_vbase,
2092 new_root.pbl_pbase);
2093
2009 if (!ret) 2094 if (!ret)
2010 return -ETIME; 2095 return -ETIME;
2011 else if (major_code) 2096 else if (major_code)
2012 return -EIO; 2097 return -EIO;
2013 else
2014 return 0;
2015 2098
2099 *actual_pbl_cnt = pbl_count + use_two_level;
2100 *used_4k_pbls = use_4k_pbls;
2016 return 0; 2101 return 0;
2017} 2102}
2018 2103
@@ -2177,18 +2262,14 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
2177 pbl_count = root_pbl_index; 2262 pbl_count = root_pbl_index;
2178 } 2263 }
2179 ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl, 2264 ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl,
2180 buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start); 2265 buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start,
2266 &nesmr->pbls_used, &nesmr->pbl_4k);
2181 2267
2182 if (ret == 0) { 2268 if (ret == 0) {
2183 nesmr->ibmr.rkey = stag; 2269 nesmr->ibmr.rkey = stag;
2184 nesmr->ibmr.lkey = stag; 2270 nesmr->ibmr.lkey = stag;
2185 nesmr->mode = IWNES_MEMREG_TYPE_MEM; 2271 nesmr->mode = IWNES_MEMREG_TYPE_MEM;
2186 ibmr = &nesmr->ibmr; 2272 ibmr = &nesmr->ibmr;
2187 nesmr->pbl_4k = ((pbl_count > 1) || (cur_pbl_index > 32)) ? 1 : 0;
2188 nesmr->pbls_used = pbl_count;
2189 if (pbl_count > 1) {
2190 nesmr->pbls_used++;
2191 }
2192 } else { 2273 } else {
2193 kfree(nesmr); 2274 kfree(nesmr);
2194 ibmr = ERR_PTR(-ENOMEM); 2275 ibmr = ERR_PTR(-ENOMEM);
@@ -2466,8 +2547,9 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2466 stag, (unsigned int)iova_start, 2547 stag, (unsigned int)iova_start,
2467 (unsigned int)region_length, stag_index, 2548 (unsigned int)region_length, stag_index,
2468 (unsigned long long)region->length, pbl_count); 2549 (unsigned long long)region->length, pbl_count);
2469 ret = nes_reg_mr( nesdev, nespd, stag, region->length, &root_vpbl, 2550 ret = nes_reg_mr(nesdev, nespd, stag, region->length, &root_vpbl,
2470 first_dma_addr, pbl_count, (u16)cur_pbl_index, acc, &iova_start); 2551 first_dma_addr, pbl_count, (u16)cur_pbl_index, acc,
2552 &iova_start, &nesmr->pbls_used, &nesmr->pbl_4k);
2471 2553
2472 nes_debug(NES_DBG_MR, "ret=%d\n", ret); 2554 nes_debug(NES_DBG_MR, "ret=%d\n", ret);
2473 2555
@@ -2476,11 +2558,6 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2476 nesmr->ibmr.lkey = stag; 2558 nesmr->ibmr.lkey = stag;
2477 nesmr->mode = IWNES_MEMREG_TYPE_MEM; 2559 nesmr->mode = IWNES_MEMREG_TYPE_MEM;
2478 ibmr = &nesmr->ibmr; 2560 ibmr = &nesmr->ibmr;
2479 nesmr->pbl_4k = ((pbl_count > 1) || (cur_pbl_index > 32)) ? 1 : 0;
2480 nesmr->pbls_used = pbl_count;
2481 if (pbl_count > 1) {
2482 nesmr->pbls_used++;
2483 }
2484 } else { 2561 } else {
2485 ib_umem_release(region); 2562 ib_umem_release(region);
2486 kfree(nesmr); 2563 kfree(nesmr);
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-16 05:44:41 -0500