aboutsummaryrefslogtreecommitdiffstats
path: root/net
diff options
context:
space:
mode:
authorChuck Lever <chuck.lever@oracle.com>2015-09-24 03:34:21 -0400
committerDoug Ledford <dledford@redhat.com>2015-09-25 10:46:51 -0400
commitbb6c96d72879fe1f674a804eb95b891def4ace61 (patch)
tree5377d9e4b36358f4ddb6e4f43909005c2a9a5b85 /net
parent310b7cec8ea32dcd4e9978423717ce78dd89d45d (diff)
xprtrdma: Replace global lkey with lkey local to PD
The core API has changed so that devices that do not have a global DMA lkey automatically create an mr, per-PD, and make that lkey available. The global DMA lkey interface is going away in favor of the per-PD DMA lkey. The per-PD DMA lkey is always available. Convert xprtrdma to use the device's per-PD DMA lkey for regbufs, no matter which memory registration scheme is in use. Signed-off-by: Chuck Lever <chuck.lever@oracle.com> Signed-off-by: Sagi Grimberg <sagig@mellanox.com> Cc: linux-nfs <linux-nfs@vger.kernel.org> Acked-by: Anna Schumaker <Anna.Schumaker@Netapp.com> Signed-off-by: Doug Ledford <dledford@redhat.com>
Diffstat (limited to 'net')
-rw-r--r--net/sunrpc/xprtrdma/fmr_ops.c19
-rw-r--r--net/sunrpc/xprtrdma/frwr_ops.c5
-rw-r--r--net/sunrpc/xprtrdma/physical_ops.c10
-rw-r--r--net/sunrpc/xprtrdma/verbs.c2
-rw-r--r--net/sunrpc/xprtrdma/xprt_rdma.h1
5 files changed, 2 insertions, 35 deletions
diff --git a/net/sunrpc/xprtrdma/fmr_ops.c b/net/sunrpc/xprtrdma/fmr_ops.c
index cb25c89da623..f1e8dafbd507 100644
--- a/net/sunrpc/xprtrdma/fmr_ops.c
+++ b/net/sunrpc/xprtrdma/fmr_ops.c
@@ -39,25 +39,6 @@ static int
39fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, 39fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
40 struct rpcrdma_create_data_internal *cdata) 40 struct rpcrdma_create_data_internal *cdata)
41{ 41{
42 struct ib_device_attr *devattr = &ia->ri_devattr;
43 struct ib_mr *mr;
44
45 /* Obtain an lkey to use for the regbufs, which are
46 * protected from remote access.
47 */
48 if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {
49 ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
50 } else {
51 mr = ib_get_dma_mr(ia->ri_pd, IB_ACCESS_LOCAL_WRITE);
52 if (IS_ERR(mr)) {
53 pr_err("%s: ib_get_dma_mr for failed with %lX\n",
54 __func__, PTR_ERR(mr));
55 return -ENOMEM;
56 }
57 ia->ri_dma_lkey = ia->ri_dma_mr->lkey;
58 ia->ri_dma_mr = mr;
59 }
60
61 return 0; 42 return 0;
62} 43}
63 44
diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index d6653f5d0830..5318951b3b53 100644
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -189,11 +189,6 @@ frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
189 struct ib_device_attr *devattr = &ia->ri_devattr; 189 struct ib_device_attr *devattr = &ia->ri_devattr;
190 int depth, delta; 190 int depth, delta;
191 191
192 /* Obtain an lkey to use for the regbufs, which are
193 * protected from remote access.
194 */
195 ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
196
197 ia->ri_max_frmr_depth = 192 ia->ri_max_frmr_depth =
198 min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS, 193 min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS,
199 devattr->max_fast_reg_page_list_len); 194 devattr->max_fast_reg_page_list_len);
diff --git a/net/sunrpc/xprtrdma/physical_ops.c b/net/sunrpc/xprtrdma/physical_ops.c
index 72cf8b15bbb4..617b76f22154 100644
--- a/net/sunrpc/xprtrdma/physical_ops.c
+++ b/net/sunrpc/xprtrdma/physical_ops.c
@@ -23,7 +23,6 @@ static int
23physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, 23physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
24 struct rpcrdma_create_data_internal *cdata) 24 struct rpcrdma_create_data_internal *cdata)
25{ 25{
26 struct ib_device_attr *devattr = &ia->ri_devattr;
27 struct ib_mr *mr; 26 struct ib_mr *mr;
28 27
29 /* Obtain an rkey to use for RPC data payloads. 28 /* Obtain an rkey to use for RPC data payloads.
@@ -37,15 +36,8 @@ physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
37 __func__, PTR_ERR(mr)); 36 __func__, PTR_ERR(mr));
38 return -ENOMEM; 37 return -ENOMEM;
39 } 38 }
40 ia->ri_dma_mr = mr;
41
42 /* Obtain an lkey to use for regbufs.
43 */
44 if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
45 ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
46 else
47 ia->ri_dma_lkey = ia->ri_dma_mr->lkey;
48 39
40 ia->ri_dma_mr = mr;
49 return 0; 41 return 0;
50} 42}
51 43
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 682996779970..eb081ad05e33 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -1252,7 +1252,7 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)
1252 goto out_free; 1252 goto out_free;
1253 1253
1254 iov->length = size; 1254 iov->length = size;
1255 iov->lkey = ia->ri_dma_lkey; 1255 iov->lkey = ia->ri_pd->local_dma_lkey;
1256 rb->rg_size = size; 1256 rb->rg_size = size;
1257 rb->rg_owner = NULL; 1257 rb->rg_owner = NULL;
1258 return rb; 1258 return rb;
diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 02512221b8bc..c09414e6f91b 100644
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -65,7 +65,6 @@ struct rpcrdma_ia {
65 struct rdma_cm_id *ri_id; 65 struct rdma_cm_id *ri_id;
66 struct ib_pd *ri_pd; 66 struct ib_pd *ri_pd;
67 struct ib_mr *ri_dma_mr; 67 struct ib_mr *ri_dma_mr;
68 u32 ri_dma_lkey;
69 struct completion ri_done; 68 struct completion ri_done;
70 int ri_async_rc; 69 int ri_async_rc;
71 unsigned int ri_max_frmr_depth; 70 unsigned int ri_max_frmr_depth;
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-29 09:16:40 -0400