diff options
author | Nicholas Bellinger <nab@linux-iscsi.org> | 2013-03-07 03:56:19 -0500 |
---|---|---|
committer | Nicholas Bellinger <nab@linux-iscsi.org> | 2013-04-25 04:09:41 -0400 |
commit | b8d26b3be8b33682cf163274ed07479a70554633 (patch) | |
tree | b15a4f8a36a5f0b834729347d993a661d1934246 /drivers/infiniband/ulp | |
parent | 04b59babc0ddd2adaebe6ee3064b4d1b4eeecb7f (diff) |
iser-target: Add iSCSI Extensions for RDMA (iSER) target driver
This patch adds support for iSCSI Extensions for RDMA target mode,
and includes CQ pooling per isert_device context distributed across
multiple active iser target sessions.
It also uses cmwq process context for RX / TX ib_post_cq() polling
via isert_cq_desc->cq_[rx,tx]_work invoked by isert_cq_[rx,tx]_callback()
hardIRQ context callbacks.
v5 changes:
- Use ISER_RECV_DATA_SEG_LEN instead of hardcoded value in ISER_RX_PAD_SIZE (Or)
- Fix make W=1 warnings (Or)
- Add missing depends on NET && INFINIBAND_ADDR_TRANS in Kconfig (Randy + Or)
- Make isert_device_find_by_ib_dev() return proper ERR_PTR (Wei Yongjun)
- Properly setup iscsi_np->np_sockaddr in isert_setup_np() (Shlomi + nab)
- Add special case for early ISCSI_OP_SCSI_CMD exception handling (nab)
v4 changes:
- Mark isert_cq_rx_work as static (Or)
- Drop unnecessary ib_dma_sync_single_for_cpu + ib_dma_sync_single_for_device
calls for isert_cmd->sense_buf_dma from isert_put_response (Or)
- Use 12288 for ISER_RX_PAD_SIZE base to save extra page per
struct iser_rx_desc (Or + nab)
- Drop now unnecessary isert_rx_desc usage, and convert RX users to
iser_rx_desc (Or + nab)
- Move isert_[alloc,free]_rx_descriptors() ahead of
isert_create_device_ib_res() usage (nab)
- Mark isert_cq_[rx,tx]_callback() + prototypes as static
- Fix 'warning: 'ret' may be used uninitialized' warning for
isert_create_device_ib_res on powerpc allmodconfig (fengguang + nab)
- Fix 'warning: 'ret' may be used uninitialized' warning for
isert_connect_request on i386 allyesconfig (fengguang + nab)
- Fix pr_debug conversion specification in isert_rx_completion()
(fengguang + nab)
- Drop unnecessary isert_conn->conn_cm_id != NULL check in
isert_connect_release causing the build warning:
"variable dereferenced before check 'isert_conn->conn_cm_id'"
- Fix isert_lid + isert_np leak in isert_setup_np failure path
- Add isert_conn->conn_wait_comp_err usage in isert_free_conn()
for isert_cq_comp_err completion path
- Add isert_conn->logout_posted bit to determine decrement of
isert_conn->post_send_buf_count from logout response completion
- Always set ISER_CONN_DOWN from isert_disconnect_work() callback
v3 changes:
- Convert to use per isert_cq_desc->cq_[rx,tx]_work + drop tasklets (Or + nab)
- Move IB_EVENT_QP_LAST_WQE_REACHED warn into correct
isert_qp_event_callback (Or)
- Drop unnecessary IB_ACCESS_REMOTE_* access flag usage in
isert_create_device_ib_res (Or)
- Add common isert_init_send_wr(), and convert isert_put_* calls (Or)
- Move to verbs+core logic to single ib_isert.[c,h] (Or + nab)
- Add kmem_cache isert_cmd_cache usage for descriptor allocation (nab)
- Move common ib_post_send() logic used by isert_put_*() to
isert_post_response() (nab)
- Add isert_put_reject call in isert_response_queue() for posting
ISCSI_REJECT response. (nab)
- Add ISTATE_SEND_REJECT checking in isert_do_control_comp. (nab)
v2 changes:
- Drop unused ISERT_ADDR_ROUTE_TIMEOUT define
- Add rdma_notify() call for IB_EVENT_COMM_EST in isert_qp_event_callback()
- Make isert_query_device() less verbose
- Drop unused RDMA_CM_EVENT_ADDR_ERROR and RDMA_CM_EVENT_ROUTE_ERROR
cases from isert_cma_handler()
- Drop unused rdma/ib_fmr_pool.h include
- Update isert_conn_setup_qp() to assign cq based upon least used
- Add isert_create_device_ib_res() to setup PD, CQs and MRs for each
underlying struct ib_device, instead of using per isert_conn resources.
- Add isert_free_device_ib_res() to release PD, CQs and MRs for each
underlying struct ib_device.
- Add isert_device_find_by_ib_dev()
- Change isert_connect_request() to drop PD, CQs and MRs allocation,
and use isert_device_find_by_ib_dev() instead.
- Add isert_device_try_release()
- Change isert_connect_release() to decrement cq_active_qps, and drop
PD, CQs and MRs resource release.
- Update isert_connect_release() to call isert_device_try_release()
- Make isert_create_device_ib_res() determine device->cqs_used based
upon num_online_cpus()
- Drop misleading isert_dump_ib_wc() usage
- Drop unused rdma/ib_fmr_pool.h include
- Use proper xfer_len for login PDUs in isert_rx_completion()
- Add isert_release_cmd() usage
- Change isert_alloc_cmd() to setup iscsi_cmd.release_cmd() pointer
- Change isert_put_cmd() to perform per iscsi_opcode specific release
logic
- Add isert_unmap_cmd() call for ISCSI_OP_SCSI_CMD from isert_put_cmd()
- Change isert_send_completion() to call
atomic_dec(&isert_conn->post_send_buf_count)
based upon per iscsi_opcode logic
- Drop ISTATE_REMOVE processing from isert_immediate_queue()
- Drop ISTATE_SEND_DATAIN processing from isert_response_queue()
- Drop ISTATE_SEND_STATUS processing from isert_response_queue()
- Drop iscsit_transport->iscsit_unmap_cmd() and ->iscsit_free_cmd()
- Convert iser_cq_tx_tasklet() to use struct isert_cq_desc pooling logic
- Convert isert_cq_tx_callback() to use struct isert_cq_desc pooling
logic
- Convert iser_cq_rx_tasklet() to use struct isert_cq_desc pooling logic
- Convert isert_cq_rx_callback() to use struct isert_cq_desc pooling
logic
- Add explict iscsit_stop_dataout_timer() call to
isert_do_rdma_read_comp()
- Use isert_get_dataout() for iscsit_transport->iscsit_get_dataout()
caller
- Drop ISTATE_SEND_R2T processing from isert_immediate_queue()
- Drop unused rdma/ib_fmr_pool.h include
- Drop isert_cmd->cmd_kref in favor of se_cmd->cmd_kref usage
- Add struct isert_device in order to support multiple EQs + CQ pooling
- Add struct isert_cq_desc
- Drop tasklets and cqs from isert_conn
- Bump ISERT_MAX_CQ to 64
- Various minor checkpatch fixes
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
Diffstat (limited to 'drivers/infiniband/ulp')
-rw-r--r-- | drivers/infiniband/ulp/isert/Kconfig | 5 | ||||
-rw-r--r-- | drivers/infiniband/ulp/isert/Makefile | 2 | ||||
-rw-r--r-- | drivers/infiniband/ulp/isert/ib_isert.c | 2281 | ||||
-rw-r--r-- | drivers/infiniband/ulp/isert/ib_isert.h | 138 | ||||
-rw-r--r-- | drivers/infiniband/ulp/isert/isert_proto.h | 47 |
5 files changed, 2473 insertions, 0 deletions
diff --git a/drivers/infiniband/ulp/isert/Kconfig b/drivers/infiniband/ulp/isert/Kconfig new file mode 100644 index 000000000000..ce3fd32167dc --- /dev/null +++ b/drivers/infiniband/ulp/isert/Kconfig | |||
@@ -0,0 +1,5 @@ | |||
1 | config INFINIBAND_ISERT | ||
2 | tristate "iSCSI Extentions for RDMA (iSER) target support" | ||
3 | depends on INET && INFINIBAND_ADDR_TRANS && TARGET_CORE && ISCSI_TARGET | ||
4 | ---help--- | ||
5 | Support for iSCSI Extentions for RDMA (iSER) Target on Infiniband fabrics. | ||
diff --git a/drivers/infiniband/ulp/isert/Makefile b/drivers/infiniband/ulp/isert/Makefile new file mode 100644 index 000000000000..c8bf2421f5bc --- /dev/null +++ b/drivers/infiniband/ulp/isert/Makefile | |||
@@ -0,0 +1,2 @@ | |||
1 | ccflags-y := -Idrivers/target -Idrivers/target/iscsi | ||
2 | obj-$(CONFIG_INFINIBAND_ISERT) += ib_isert.o | ||
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c new file mode 100644 index 000000000000..41712f096515 --- /dev/null +++ b/drivers/infiniband/ulp/isert/ib_isert.c | |||
@@ -0,0 +1,2281 @@ | |||
1 | /******************************************************************************* | ||
2 | * This file contains iSCSI extentions for RDMA (iSER) Verbs | ||
3 | * | ||
4 | * (c) Copyright 2013 RisingTide Systems LLC. | ||
5 | * | ||
6 | * Nicholas A. Bellinger <nab@linux-iscsi.org> | ||
7 | * | ||
8 | * This program is free software; you can redistribute it and/or modify | ||
9 | * it under the terms of the GNU General Public License as published by | ||
10 | * the Free Software Foundation; either version 2 of the License, or | ||
11 | * (at your option) any later version. | ||
12 | * | ||
13 | * This program is distributed in the hope that it will be useful, | ||
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
16 | * GNU General Public License for more details. | ||
17 | ****************************************************************************/ | ||
18 | |||
19 | #include <linux/string.h> | ||
20 | #include <linux/module.h> | ||
21 | #include <linux/scatterlist.h> | ||
22 | #include <linux/socket.h> | ||
23 | #include <linux/in.h> | ||
24 | #include <linux/in6.h> | ||
25 | #include <rdma/ib_verbs.h> | ||
26 | #include <rdma/rdma_cm.h> | ||
27 | #include <target/target_core_base.h> | ||
28 | #include <target/target_core_fabric.h> | ||
29 | #include <target/iscsi/iscsi_transport.h> | ||
30 | |||
31 | #include "isert_proto.h" | ||
32 | #include "ib_isert.h" | ||
33 | |||
34 | #define ISERT_MAX_CONN 8 | ||
35 | #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) | ||
36 | #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) | ||
37 | |||
38 | static DEFINE_MUTEX(device_list_mutex); | ||
39 | static LIST_HEAD(device_list); | ||
40 | static struct workqueue_struct *isert_rx_wq; | ||
41 | static struct workqueue_struct *isert_comp_wq; | ||
42 | static struct kmem_cache *isert_cmd_cache; | ||
43 | |||
44 | static void | ||
45 | isert_qp_event_callback(struct ib_event *e, void *context) | ||
46 | { | ||
47 | struct isert_conn *isert_conn = (struct isert_conn *)context; | ||
48 | |||
49 | pr_err("isert_qp_event_callback event: %d\n", e->event); | ||
50 | switch (e->event) { | ||
51 | case IB_EVENT_COMM_EST: | ||
52 | rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); | ||
53 | break; | ||
54 | case IB_EVENT_QP_LAST_WQE_REACHED: | ||
55 | pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); | ||
56 | break; | ||
57 | default: | ||
58 | break; | ||
59 | } | ||
60 | } | ||
61 | |||
62 | static int | ||
63 | isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) | ||
64 | { | ||
65 | int ret; | ||
66 | |||
67 | ret = ib_query_device(ib_dev, devattr); | ||
68 | if (ret) { | ||
69 | pr_err("ib_query_device() failed: %d\n", ret); | ||
70 | return ret; | ||
71 | } | ||
72 | pr_debug("devattr->max_sge: %d\n", devattr->max_sge); | ||
73 | pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); | ||
74 | |||
75 | return 0; | ||
76 | } | ||
77 | |||
78 | static int | ||
79 | isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) | ||
80 | { | ||
81 | struct isert_device *device = isert_conn->conn_device; | ||
82 | struct ib_qp_init_attr attr; | ||
83 | struct ib_device_attr devattr; | ||
84 | int ret, index, min_index = 0; | ||
85 | |||
86 | memset(&devattr, 0, sizeof(struct ib_device_attr)); | ||
87 | ret = isert_query_device(cma_id->device, &devattr); | ||
88 | if (ret) | ||
89 | return ret; | ||
90 | |||
91 | mutex_lock(&device_list_mutex); | ||
92 | for (index = 0; index < device->cqs_used; index++) | ||
93 | if (device->cq_active_qps[index] < | ||
94 | device->cq_active_qps[min_index]) | ||
95 | min_index = index; | ||
96 | device->cq_active_qps[min_index]++; | ||
97 | pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); | ||
98 | mutex_unlock(&device_list_mutex); | ||
99 | |||
100 | memset(&attr, 0, sizeof(struct ib_qp_init_attr)); | ||
101 | attr.event_handler = isert_qp_event_callback; | ||
102 | attr.qp_context = isert_conn; | ||
103 | attr.send_cq = device->dev_tx_cq[min_index]; | ||
104 | attr.recv_cq = device->dev_rx_cq[min_index]; | ||
105 | attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; | ||
106 | attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; | ||
107 | /* | ||
108 | * FIXME: Use devattr.max_sge - 2 for max_send_sge as | ||
109 | * work-around for RDMA_READ.. | ||
110 | */ | ||
111 | attr.cap.max_send_sge = devattr.max_sge - 2; | ||
112 | isert_conn->max_sge = attr.cap.max_send_sge; | ||
113 | |||
114 | attr.cap.max_recv_sge = 1; | ||
115 | attr.sq_sig_type = IB_SIGNAL_REQ_WR; | ||
116 | attr.qp_type = IB_QPT_RC; | ||
117 | |||
118 | pr_debug("isert_conn_setup_qp cma_id->device: %p\n", | ||
119 | cma_id->device); | ||
120 | pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", | ||
121 | isert_conn->conn_pd->device); | ||
122 | |||
123 | ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); | ||
124 | if (ret) { | ||
125 | pr_err("rdma_create_qp failed for cma_id %d\n", ret); | ||
126 | return ret; | ||
127 | } | ||
128 | isert_conn->conn_qp = cma_id->qp; | ||
129 | pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); | ||
130 | |||
131 | return 0; | ||
132 | } | ||
133 | |||
134 | static void | ||
135 | isert_cq_event_callback(struct ib_event *e, void *context) | ||
136 | { | ||
137 | pr_debug("isert_cq_event_callback event: %d\n", e->event); | ||
138 | } | ||
139 | |||
140 | static int | ||
141 | isert_alloc_rx_descriptors(struct isert_conn *isert_conn) | ||
142 | { | ||
143 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
144 | struct iser_rx_desc *rx_desc; | ||
145 | struct ib_sge *rx_sg; | ||
146 | u64 dma_addr; | ||
147 | int i, j; | ||
148 | |||
149 | isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * | ||
150 | sizeof(struct iser_rx_desc), GFP_KERNEL); | ||
151 | if (!isert_conn->conn_rx_descs) | ||
152 | goto fail; | ||
153 | |||
154 | rx_desc = isert_conn->conn_rx_descs; | ||
155 | |||
156 | for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { | ||
157 | dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, | ||
158 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | ||
159 | if (ib_dma_mapping_error(ib_dev, dma_addr)) | ||
160 | goto dma_map_fail; | ||
161 | |||
162 | rx_desc->dma_addr = dma_addr; | ||
163 | |||
164 | rx_sg = &rx_desc->rx_sg; | ||
165 | rx_sg->addr = rx_desc->dma_addr; | ||
166 | rx_sg->length = ISER_RX_PAYLOAD_SIZE; | ||
167 | rx_sg->lkey = isert_conn->conn_mr->lkey; | ||
168 | } | ||
169 | |||
170 | isert_conn->conn_rx_desc_head = 0; | ||
171 | return 0; | ||
172 | |||
173 | dma_map_fail: | ||
174 | rx_desc = isert_conn->conn_rx_descs; | ||
175 | for (j = 0; j < i; j++, rx_desc++) { | ||
176 | ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, | ||
177 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | ||
178 | } | ||
179 | kfree(isert_conn->conn_rx_descs); | ||
180 | isert_conn->conn_rx_descs = NULL; | ||
181 | fail: | ||
182 | return -ENOMEM; | ||
183 | } | ||
184 | |||
185 | static void | ||
186 | isert_free_rx_descriptors(struct isert_conn *isert_conn) | ||
187 | { | ||
188 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
189 | struct iser_rx_desc *rx_desc; | ||
190 | int i; | ||
191 | |||
192 | if (!isert_conn->conn_rx_descs) | ||
193 | return; | ||
194 | |||
195 | rx_desc = isert_conn->conn_rx_descs; | ||
196 | for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { | ||
197 | ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, | ||
198 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | ||
199 | } | ||
200 | |||
201 | kfree(isert_conn->conn_rx_descs); | ||
202 | isert_conn->conn_rx_descs = NULL; | ||
203 | } | ||
204 | |||
205 | static void isert_cq_tx_callback(struct ib_cq *, void *); | ||
206 | static void isert_cq_rx_callback(struct ib_cq *, void *); | ||
207 | |||
208 | static int | ||
209 | isert_create_device_ib_res(struct isert_device *device) | ||
210 | { | ||
211 | struct ib_device *ib_dev = device->ib_device; | ||
212 | struct isert_cq_desc *cq_desc; | ||
213 | int ret = 0, i, j; | ||
214 | |||
215 | device->cqs_used = min_t(int, num_online_cpus(), | ||
216 | device->ib_device->num_comp_vectors); | ||
217 | device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); | ||
218 | pr_debug("Using %d CQs, device %s supports %d vectors\n", | ||
219 | device->cqs_used, device->ib_device->name, | ||
220 | device->ib_device->num_comp_vectors); | ||
221 | device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * | ||
222 | device->cqs_used, GFP_KERNEL); | ||
223 | if (!device->cq_desc) { | ||
224 | pr_err("Unable to allocate device->cq_desc\n"); | ||
225 | return -ENOMEM; | ||
226 | } | ||
227 | cq_desc = device->cq_desc; | ||
228 | |||
229 | device->dev_pd = ib_alloc_pd(ib_dev); | ||
230 | if (IS_ERR(device->dev_pd)) { | ||
231 | ret = PTR_ERR(device->dev_pd); | ||
232 | pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret); | ||
233 | goto out_cq_desc; | ||
234 | } | ||
235 | |||
236 | for (i = 0; i < device->cqs_used; i++) { | ||
237 | cq_desc[i].device = device; | ||
238 | cq_desc[i].cq_index = i; | ||
239 | |||
240 | device->dev_rx_cq[i] = ib_create_cq(device->ib_device, | ||
241 | isert_cq_rx_callback, | ||
242 | isert_cq_event_callback, | ||
243 | (void *)&cq_desc[i], | ||
244 | ISER_MAX_RX_CQ_LEN, i); | ||
245 | if (IS_ERR(device->dev_rx_cq[i])) | ||
246 | goto out_cq; | ||
247 | |||
248 | device->dev_tx_cq[i] = ib_create_cq(device->ib_device, | ||
249 | isert_cq_tx_callback, | ||
250 | isert_cq_event_callback, | ||
251 | (void *)&cq_desc[i], | ||
252 | ISER_MAX_TX_CQ_LEN, i); | ||
253 | if (IS_ERR(device->dev_tx_cq[i])) | ||
254 | goto out_cq; | ||
255 | |||
256 | if (ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP)) | ||
257 | goto out_cq; | ||
258 | |||
259 | if (ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP)) | ||
260 | goto out_cq; | ||
261 | } | ||
262 | |||
263 | device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE); | ||
264 | if (IS_ERR(device->dev_mr)) { | ||
265 | ret = PTR_ERR(device->dev_mr); | ||
266 | pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret); | ||
267 | goto out_cq; | ||
268 | } | ||
269 | |||
270 | return 0; | ||
271 | |||
272 | out_cq: | ||
273 | for (j = 0; j < i; j++) { | ||
274 | cq_desc = &device->cq_desc[j]; | ||
275 | |||
276 | if (device->dev_rx_cq[j]) { | ||
277 | cancel_work_sync(&cq_desc->cq_rx_work); | ||
278 | ib_destroy_cq(device->dev_rx_cq[j]); | ||
279 | } | ||
280 | if (device->dev_tx_cq[j]) { | ||
281 | cancel_work_sync(&cq_desc->cq_tx_work); | ||
282 | ib_destroy_cq(device->dev_tx_cq[j]); | ||
283 | } | ||
284 | } | ||
285 | ib_dealloc_pd(device->dev_pd); | ||
286 | |||
287 | out_cq_desc: | ||
288 | kfree(device->cq_desc); | ||
289 | |||
290 | return ret; | ||
291 | } | ||
292 | |||
293 | static void | ||
294 | isert_free_device_ib_res(struct isert_device *device) | ||
295 | { | ||
296 | struct isert_cq_desc *cq_desc; | ||
297 | int i; | ||
298 | |||
299 | for (i = 0; i < device->cqs_used; i++) { | ||
300 | cq_desc = &device->cq_desc[i]; | ||
301 | |||
302 | cancel_work_sync(&cq_desc->cq_rx_work); | ||
303 | cancel_work_sync(&cq_desc->cq_tx_work); | ||
304 | ib_destroy_cq(device->dev_rx_cq[i]); | ||
305 | ib_destroy_cq(device->dev_tx_cq[i]); | ||
306 | device->dev_rx_cq[i] = NULL; | ||
307 | device->dev_tx_cq[i] = NULL; | ||
308 | } | ||
309 | |||
310 | ib_dereg_mr(device->dev_mr); | ||
311 | ib_dealloc_pd(device->dev_pd); | ||
312 | kfree(device->cq_desc); | ||
313 | } | ||
314 | |||
315 | static void | ||
316 | isert_device_try_release(struct isert_device *device) | ||
317 | { | ||
318 | mutex_lock(&device_list_mutex); | ||
319 | device->refcount--; | ||
320 | if (!device->refcount) { | ||
321 | isert_free_device_ib_res(device); | ||
322 | list_del(&device->dev_node); | ||
323 | kfree(device); | ||
324 | } | ||
325 | mutex_unlock(&device_list_mutex); | ||
326 | } | ||
327 | |||
328 | static struct isert_device * | ||
329 | isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) | ||
330 | { | ||
331 | struct isert_device *device; | ||
332 | int ret; | ||
333 | |||
334 | mutex_lock(&device_list_mutex); | ||
335 | list_for_each_entry(device, &device_list, dev_node) { | ||
336 | if (device->ib_device->node_guid == cma_id->device->node_guid) { | ||
337 | device->refcount++; | ||
338 | mutex_unlock(&device_list_mutex); | ||
339 | return device; | ||
340 | } | ||
341 | } | ||
342 | |||
343 | device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); | ||
344 | if (!device) { | ||
345 | mutex_unlock(&device_list_mutex); | ||
346 | return ERR_PTR(-ENOMEM); | ||
347 | } | ||
348 | |||
349 | INIT_LIST_HEAD(&device->dev_node); | ||
350 | |||
351 | device->ib_device = cma_id->device; | ||
352 | ret = isert_create_device_ib_res(device); | ||
353 | if (ret) { | ||
354 | kfree(device); | ||
355 | mutex_unlock(&device_list_mutex); | ||
356 | return ERR_PTR(ret); | ||
357 | } | ||
358 | |||
359 | device->refcount++; | ||
360 | list_add_tail(&device->dev_node, &device_list); | ||
361 | mutex_unlock(&device_list_mutex); | ||
362 | |||
363 | return device; | ||
364 | } | ||
365 | |||
366 | static int | ||
367 | isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) | ||
368 | { | ||
369 | struct iscsi_np *np = cma_id->context; | ||
370 | struct isert_np *isert_np = np->np_context; | ||
371 | struct isert_conn *isert_conn; | ||
372 | struct isert_device *device; | ||
373 | struct ib_device *ib_dev = cma_id->device; | ||
374 | int ret = 0; | ||
375 | |||
376 | pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", | ||
377 | cma_id, cma_id->context); | ||
378 | |||
379 | isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); | ||
380 | if (!isert_conn) { | ||
381 | pr_err("Unable to allocate isert_conn\n"); | ||
382 | return -ENOMEM; | ||
383 | } | ||
384 | isert_conn->state = ISER_CONN_INIT; | ||
385 | INIT_LIST_HEAD(&isert_conn->conn_accept_node); | ||
386 | init_completion(&isert_conn->conn_login_comp); | ||
387 | init_waitqueue_head(&isert_conn->conn_wait); | ||
388 | init_waitqueue_head(&isert_conn->conn_wait_comp_err); | ||
389 | kref_init(&isert_conn->conn_kref); | ||
390 | kref_get(&isert_conn->conn_kref); | ||
391 | |||
392 | cma_id->context = isert_conn; | ||
393 | isert_conn->conn_cm_id = cma_id; | ||
394 | isert_conn->responder_resources = event->param.conn.responder_resources; | ||
395 | isert_conn->initiator_depth = event->param.conn.initiator_depth; | ||
396 | pr_debug("Using responder_resources: %u initiator_depth: %u\n", | ||
397 | isert_conn->responder_resources, isert_conn->initiator_depth); | ||
398 | |||
399 | isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + | ||
400 | ISER_RX_LOGIN_SIZE, GFP_KERNEL); | ||
401 | if (!isert_conn->login_buf) { | ||
402 | pr_err("Unable to allocate isert_conn->login_buf\n"); | ||
403 | ret = -ENOMEM; | ||
404 | goto out; | ||
405 | } | ||
406 | |||
407 | isert_conn->login_req_buf = isert_conn->login_buf; | ||
408 | isert_conn->login_rsp_buf = isert_conn->login_buf + | ||
409 | ISCSI_DEF_MAX_RECV_SEG_LEN; | ||
410 | pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", | ||
411 | isert_conn->login_buf, isert_conn->login_req_buf, | ||
412 | isert_conn->login_rsp_buf); | ||
413 | |||
414 | isert_conn->login_req_dma = ib_dma_map_single(ib_dev, | ||
415 | (void *)isert_conn->login_req_buf, | ||
416 | ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); | ||
417 | |||
418 | ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); | ||
419 | if (ret) { | ||
420 | pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", | ||
421 | ret); | ||
422 | isert_conn->login_req_dma = 0; | ||
423 | goto out_login_buf; | ||
424 | } | ||
425 | |||
426 | isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, | ||
427 | (void *)isert_conn->login_rsp_buf, | ||
428 | ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); | ||
429 | |||
430 | ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); | ||
431 | if (ret) { | ||
432 | pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", | ||
433 | ret); | ||
434 | isert_conn->login_rsp_dma = 0; | ||
435 | goto out_req_dma_map; | ||
436 | } | ||
437 | |||
438 | device = isert_device_find_by_ib_dev(cma_id); | ||
439 | if (IS_ERR(device)) { | ||
440 | ret = PTR_ERR(device); | ||
441 | goto out_rsp_dma_map; | ||
442 | } | ||
443 | |||
444 | isert_conn->conn_device = device; | ||
445 | isert_conn->conn_pd = device->dev_pd; | ||
446 | isert_conn->conn_mr = device->dev_mr; | ||
447 | |||
448 | ret = isert_conn_setup_qp(isert_conn, cma_id); | ||
449 | if (ret) | ||
450 | goto out_conn_dev; | ||
451 | |||
452 | mutex_lock(&isert_np->np_accept_mutex); | ||
453 | list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); | ||
454 | mutex_unlock(&isert_np->np_accept_mutex); | ||
455 | |||
456 | pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); | ||
457 | wake_up(&isert_np->np_accept_wq); | ||
458 | return 0; | ||
459 | |||
460 | out_conn_dev: | ||
461 | isert_device_try_release(device); | ||
462 | out_rsp_dma_map: | ||
463 | ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, | ||
464 | ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); | ||
465 | out_req_dma_map: | ||
466 | ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, | ||
467 | ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); | ||
468 | out_login_buf: | ||
469 | kfree(isert_conn->login_buf); | ||
470 | out: | ||
471 | kfree(isert_conn); | ||
472 | return ret; | ||
473 | } | ||
474 | |||
475 | static void | ||
476 | isert_connect_release(struct isert_conn *isert_conn) | ||
477 | { | ||
478 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
479 | struct isert_device *device = isert_conn->conn_device; | ||
480 | int cq_index; | ||
481 | |||
482 | pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); | ||
483 | |||
484 | if (isert_conn->conn_qp) { | ||
485 | cq_index = ((struct isert_cq_desc *) | ||
486 | isert_conn->conn_qp->recv_cq->cq_context)->cq_index; | ||
487 | pr_debug("isert_connect_release: cq_index: %d\n", cq_index); | ||
488 | isert_conn->conn_device->cq_active_qps[cq_index]--; | ||
489 | |||
490 | rdma_destroy_qp(isert_conn->conn_cm_id); | ||
491 | } | ||
492 | |||
493 | isert_free_rx_descriptors(isert_conn); | ||
494 | rdma_destroy_id(isert_conn->conn_cm_id); | ||
495 | |||
496 | if (isert_conn->login_buf) { | ||
497 | ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, | ||
498 | ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); | ||
499 | ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, | ||
500 | ISCSI_DEF_MAX_RECV_SEG_LEN, | ||
501 | DMA_FROM_DEVICE); | ||
502 | kfree(isert_conn->login_buf); | ||
503 | } | ||
504 | kfree(isert_conn); | ||
505 | |||
506 | if (device) | ||
507 | isert_device_try_release(device); | ||
508 | |||
509 | pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); | ||
510 | } | ||
511 | |||
512 | static void | ||
513 | isert_connected_handler(struct rdma_cm_id *cma_id) | ||
514 | { | ||
515 | return; | ||
516 | } | ||
517 | |||
518 | static void | ||
519 | isert_release_conn_kref(struct kref *kref) | ||
520 | { | ||
521 | struct isert_conn *isert_conn = container_of(kref, | ||
522 | struct isert_conn, conn_kref); | ||
523 | |||
524 | pr_debug("Calling isert_connect_release for final kref %s/%d\n", | ||
525 | current->comm, current->pid); | ||
526 | |||
527 | isert_connect_release(isert_conn); | ||
528 | } | ||
529 | |||
530 | static void | ||
531 | isert_put_conn(struct isert_conn *isert_conn) | ||
532 | { | ||
533 | kref_put(&isert_conn->conn_kref, isert_release_conn_kref); | ||
534 | } | ||
535 | |||
536 | static void | ||
537 | isert_disconnect_work(struct work_struct *work) | ||
538 | { | ||
539 | struct isert_conn *isert_conn = container_of(work, | ||
540 | struct isert_conn, conn_logout_work); | ||
541 | |||
542 | pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); | ||
543 | |||
544 | isert_conn->state = ISER_CONN_DOWN; | ||
545 | |||
546 | if (isert_conn->post_recv_buf_count == 0 && | ||
547 | atomic_read(&isert_conn->post_send_buf_count) == 0) { | ||
548 | pr_debug("Calling wake_up(&isert_conn->conn_wait);\n"); | ||
549 | wake_up(&isert_conn->conn_wait); | ||
550 | } | ||
551 | |||
552 | isert_put_conn(isert_conn); | ||
553 | } | ||
554 | |||
555 | static void | ||
556 | isert_disconnected_handler(struct rdma_cm_id *cma_id) | ||
557 | { | ||
558 | struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; | ||
559 | |||
560 | INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); | ||
561 | schedule_work(&isert_conn->conn_logout_work); | ||
562 | } | ||
563 | |||
564 | static int | ||
565 | isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) | ||
566 | { | ||
567 | int ret = 0; | ||
568 | |||
569 | pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", | ||
570 | event->event, event->status, cma_id->context, cma_id); | ||
571 | |||
572 | switch (event->event) { | ||
573 | case RDMA_CM_EVENT_CONNECT_REQUEST: | ||
574 | pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); | ||
575 | ret = isert_connect_request(cma_id, event); | ||
576 | break; | ||
577 | case RDMA_CM_EVENT_ESTABLISHED: | ||
578 | pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); | ||
579 | isert_connected_handler(cma_id); | ||
580 | break; | ||
581 | case RDMA_CM_EVENT_DISCONNECTED: | ||
582 | pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); | ||
583 | isert_disconnected_handler(cma_id); | ||
584 | break; | ||
585 | case RDMA_CM_EVENT_DEVICE_REMOVAL: | ||
586 | case RDMA_CM_EVENT_ADDR_CHANGE: | ||
587 | break; | ||
588 | case RDMA_CM_EVENT_CONNECT_ERROR: | ||
589 | default: | ||
590 | pr_err("Unknown RDMA CMA event: %d\n", event->event); | ||
591 | break; | ||
592 | } | ||
593 | |||
594 | if (ret != 0) { | ||
595 | pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", | ||
596 | event->event, ret); | ||
597 | dump_stack(); | ||
598 | } | ||
599 | |||
600 | return ret; | ||
601 | } | ||
602 | |||
603 | static int | ||
604 | isert_post_recv(struct isert_conn *isert_conn, u32 count) | ||
605 | { | ||
606 | struct ib_recv_wr *rx_wr, *rx_wr_failed; | ||
607 | int i, ret; | ||
608 | unsigned int rx_head = isert_conn->conn_rx_desc_head; | ||
609 | struct iser_rx_desc *rx_desc; | ||
610 | |||
611 | for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { | ||
612 | rx_desc = &isert_conn->conn_rx_descs[rx_head]; | ||
613 | rx_wr->wr_id = (unsigned long)rx_desc; | ||
614 | rx_wr->sg_list = &rx_desc->rx_sg; | ||
615 | rx_wr->num_sge = 1; | ||
616 | rx_wr->next = rx_wr + 1; | ||
617 | rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); | ||
618 | } | ||
619 | |||
620 | rx_wr--; | ||
621 | rx_wr->next = NULL; /* mark end of work requests list */ | ||
622 | |||
623 | isert_conn->post_recv_buf_count += count; | ||
624 | ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, | ||
625 | &rx_wr_failed); | ||
626 | if (ret) { | ||
627 | pr_err("ib_post_recv() failed with ret: %d\n", ret); | ||
628 | isert_conn->post_recv_buf_count -= count; | ||
629 | } else { | ||
630 | pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); | ||
631 | isert_conn->conn_rx_desc_head = rx_head; | ||
632 | } | ||
633 | return ret; | ||
634 | } | ||
635 | |||
636 | static int | ||
637 | isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) | ||
638 | { | ||
639 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
640 | struct ib_send_wr send_wr, *send_wr_failed; | ||
641 | int ret; | ||
642 | |||
643 | ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, | ||
644 | ISER_HEADERS_LEN, DMA_TO_DEVICE); | ||
645 | |||
646 | send_wr.next = NULL; | ||
647 | send_wr.wr_id = (unsigned long)tx_desc; | ||
648 | send_wr.sg_list = tx_desc->tx_sg; | ||
649 | send_wr.num_sge = tx_desc->num_sge; | ||
650 | send_wr.opcode = IB_WR_SEND; | ||
651 | send_wr.send_flags = IB_SEND_SIGNALED; | ||
652 | |||
653 | atomic_inc(&isert_conn->post_send_buf_count); | ||
654 | |||
655 | ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); | ||
656 | if (ret) { | ||
657 | pr_err("ib_post_send() failed, ret: %d\n", ret); | ||
658 | atomic_dec(&isert_conn->post_send_buf_count); | ||
659 | } | ||
660 | |||
661 | return ret; | ||
662 | } | ||
663 | |||
664 | static void | ||
665 | isert_create_send_desc(struct isert_conn *isert_conn, | ||
666 | struct isert_cmd *isert_cmd, | ||
667 | struct iser_tx_desc *tx_desc) | ||
668 | { | ||
669 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
670 | |||
671 | ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, | ||
672 | ISER_HEADERS_LEN, DMA_TO_DEVICE); | ||
673 | |||
674 | memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); | ||
675 | tx_desc->iser_header.flags = ISER_VER; | ||
676 | |||
677 | tx_desc->num_sge = 1; | ||
678 | tx_desc->isert_cmd = isert_cmd; | ||
679 | |||
680 | if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { | ||
681 | tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; | ||
682 | pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); | ||
683 | } | ||
684 | } | ||
685 | |||
686 | static int | ||
687 | isert_init_tx_hdrs(struct isert_conn *isert_conn, | ||
688 | struct iser_tx_desc *tx_desc) | ||
689 | { | ||
690 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
691 | u64 dma_addr; | ||
692 | |||
693 | dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, | ||
694 | ISER_HEADERS_LEN, DMA_TO_DEVICE); | ||
695 | if (ib_dma_mapping_error(ib_dev, dma_addr)) { | ||
696 | pr_err("ib_dma_mapping_error() failed\n"); | ||
697 | return -ENOMEM; | ||
698 | } | ||
699 | |||
700 | tx_desc->dma_addr = dma_addr; | ||
701 | tx_desc->tx_sg[0].addr = tx_desc->dma_addr; | ||
702 | tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; | ||
703 | tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; | ||
704 | |||
705 | pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" | ||
706 | " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, | ||
707 | tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); | ||
708 | |||
709 | return 0; | ||
710 | } | ||
711 | |||
712 | static void | ||
713 | isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr) | ||
714 | { | ||
715 | isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; | ||
716 | send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; | ||
717 | send_wr->opcode = IB_WR_SEND; | ||
718 | send_wr->send_flags = IB_SEND_SIGNALED; | ||
719 | send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0]; | ||
720 | send_wr->num_sge = isert_cmd->tx_desc.num_sge; | ||
721 | } | ||
722 | |||
723 | static int | ||
724 | isert_rdma_post_recvl(struct isert_conn *isert_conn) | ||
725 | { | ||
726 | struct ib_recv_wr rx_wr, *rx_wr_fail; | ||
727 | struct ib_sge sge; | ||
728 | int ret; | ||
729 | |||
730 | memset(&sge, 0, sizeof(struct ib_sge)); | ||
731 | sge.addr = isert_conn->login_req_dma; | ||
732 | sge.length = ISER_RX_LOGIN_SIZE; | ||
733 | sge.lkey = isert_conn->conn_mr->lkey; | ||
734 | |||
735 | pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", | ||
736 | sge.addr, sge.length, sge.lkey); | ||
737 | |||
738 | memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); | ||
739 | rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; | ||
740 | rx_wr.sg_list = &sge; | ||
741 | rx_wr.num_sge = 1; | ||
742 | |||
743 | isert_conn->post_recv_buf_count++; | ||
744 | ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); | ||
745 | if (ret) { | ||
746 | pr_err("ib_post_recv() failed: %d\n", ret); | ||
747 | isert_conn->post_recv_buf_count--; | ||
748 | } | ||
749 | |||
750 | pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); | ||
751 | return ret; | ||
752 | } | ||
753 | |||
754 | static int | ||
755 | isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, | ||
756 | u32 length) | ||
757 | { | ||
758 | struct isert_conn *isert_conn = conn->context; | ||
759 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
760 | struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; | ||
761 | int ret; | ||
762 | |||
763 | isert_create_send_desc(isert_conn, NULL, tx_desc); | ||
764 | |||
765 | memcpy(&tx_desc->iscsi_header, &login->rsp[0], | ||
766 | sizeof(struct iscsi_hdr)); | ||
767 | |||
768 | isert_init_tx_hdrs(isert_conn, tx_desc); | ||
769 | |||
770 | if (length > 0) { | ||
771 | struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; | ||
772 | |||
773 | ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, | ||
774 | length, DMA_TO_DEVICE); | ||
775 | |||
776 | memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); | ||
777 | |||
778 | ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, | ||
779 | length, DMA_TO_DEVICE); | ||
780 | |||
781 | tx_dsg->addr = isert_conn->login_rsp_dma; | ||
782 | tx_dsg->length = length; | ||
783 | tx_dsg->lkey = isert_conn->conn_mr->lkey; | ||
784 | tx_desc->num_sge = 2; | ||
785 | } | ||
786 | if (!login->login_failed) { | ||
787 | if (login->login_complete) { | ||
788 | ret = isert_alloc_rx_descriptors(isert_conn); | ||
789 | if (ret) | ||
790 | return ret; | ||
791 | |||
792 | ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); | ||
793 | if (ret) | ||
794 | return ret; | ||
795 | |||
796 | isert_conn->state = ISER_CONN_UP; | ||
797 | goto post_send; | ||
798 | } | ||
799 | |||
800 | ret = isert_rdma_post_recvl(isert_conn); | ||
801 | if (ret) | ||
802 | return ret; | ||
803 | } | ||
804 | post_send: | ||
805 | ret = isert_post_send(isert_conn, tx_desc); | ||
806 | if (ret) | ||
807 | return ret; | ||
808 | |||
809 | return 0; | ||
810 | } | ||
811 | |||
812 | static void | ||
813 | isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, | ||
814 | struct isert_conn *isert_conn) | ||
815 | { | ||
816 | struct iscsi_conn *conn = isert_conn->conn; | ||
817 | struct iscsi_login *login = conn->conn_login; | ||
818 | int size; | ||
819 | |||
820 | if (!login) { | ||
821 | pr_err("conn->conn_login is NULL\n"); | ||
822 | dump_stack(); | ||
823 | return; | ||
824 | } | ||
825 | |||
826 | if (login->first_request) { | ||
827 | struct iscsi_login_req *login_req = | ||
828 | (struct iscsi_login_req *)&rx_desc->iscsi_header; | ||
829 | /* | ||
830 | * Setup the initial iscsi_login values from the leading | ||
831 | * login request PDU. | ||
832 | */ | ||
833 | login->leading_connection = (!login_req->tsih) ? 1 : 0; | ||
834 | login->current_stage = | ||
835 | (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) | ||
836 | >> 2; | ||
837 | login->version_min = login_req->min_version; | ||
838 | login->version_max = login_req->max_version; | ||
839 | memcpy(login->isid, login_req->isid, 6); | ||
840 | login->cmd_sn = be32_to_cpu(login_req->cmdsn); | ||
841 | login->init_task_tag = login_req->itt; | ||
842 | login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); | ||
843 | login->cid = be16_to_cpu(login_req->cid); | ||
844 | login->tsih = be16_to_cpu(login_req->tsih); | ||
845 | } | ||
846 | |||
847 | memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); | ||
848 | |||
849 | size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); | ||
850 | pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", | ||
851 | size, rx_buflen, MAX_KEY_VALUE_PAIRS); | ||
852 | memcpy(login->req_buf, &rx_desc->data[0], size); | ||
853 | |||
854 | complete(&isert_conn->conn_login_comp); | ||
855 | } | ||
856 | |||
857 | static void | ||
858 | isert_release_cmd(struct iscsi_cmd *cmd) | ||
859 | { | ||
860 | struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd, | ||
861 | iscsi_cmd); | ||
862 | |||
863 | pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd); | ||
864 | |||
865 | kfree(cmd->buf_ptr); | ||
866 | kfree(cmd->tmr_req); | ||
867 | |||
868 | kmem_cache_free(isert_cmd_cache, isert_cmd); | ||
869 | } | ||
870 | |||
871 | static struct iscsi_cmd | ||
872 | *isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp) | ||
873 | { | ||
874 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
875 | struct isert_cmd *isert_cmd; | ||
876 | |||
877 | isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp); | ||
878 | if (!isert_cmd) { | ||
879 | pr_err("Unable to allocate isert_cmd\n"); | ||
880 | return NULL; | ||
881 | } | ||
882 | isert_cmd->conn = isert_conn; | ||
883 | isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd; | ||
884 | |||
885 | return &isert_cmd->iscsi_cmd; | ||
886 | } | ||
887 | |||
888 | static int | ||
889 | isert_handle_scsi_cmd(struct isert_conn *isert_conn, | ||
890 | struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc, | ||
891 | unsigned char *buf) | ||
892 | { | ||
893 | struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; | ||
894 | struct iscsi_conn *conn = isert_conn->conn; | ||
895 | struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; | ||
896 | struct scatterlist *sg; | ||
897 | int imm_data, imm_data_len, unsol_data, sg_nents, rc; | ||
898 | bool dump_payload = false; | ||
899 | |||
900 | rc = iscsit_setup_scsi_cmd(conn, cmd, buf); | ||
901 | if (rc < 0) | ||
902 | return rc; | ||
903 | |||
904 | imm_data = cmd->immediate_data; | ||
905 | imm_data_len = cmd->first_burst_len; | ||
906 | unsol_data = cmd->unsolicited_data; | ||
907 | |||
908 | rc = iscsit_process_scsi_cmd(conn, cmd, hdr); | ||
909 | if (rc < 0) { | ||
910 | return 0; | ||
911 | } else if (rc > 0) { | ||
912 | dump_payload = true; | ||
913 | goto sequence_cmd; | ||
914 | } | ||
915 | |||
916 | if (!imm_data) | ||
917 | return 0; | ||
918 | |||
919 | sg = &cmd->se_cmd.t_data_sg[0]; | ||
920 | sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); | ||
921 | |||
922 | pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", | ||
923 | sg, sg_nents, &rx_desc->data[0], imm_data_len); | ||
924 | |||
925 | sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); | ||
926 | |||
927 | cmd->write_data_done += imm_data_len; | ||
928 | |||
929 | if (cmd->write_data_done == cmd->se_cmd.data_length) { | ||
930 | spin_lock_bh(&cmd->istate_lock); | ||
931 | cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; | ||
932 | cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; | ||
933 | spin_unlock_bh(&cmd->istate_lock); | ||
934 | } | ||
935 | |||
936 | sequence_cmd: | ||
937 | rc = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); | ||
938 | |||
939 | if (!rc && dump_payload == false && unsol_data) | ||
940 | iscsit_set_unsoliticed_dataout(cmd); | ||
941 | |||
942 | if (rc == CMDSN_ERROR_CANNOT_RECOVER) | ||
943 | return iscsit_add_reject_from_cmd( | ||
944 | ISCSI_REASON_PROTOCOL_ERROR, | ||
945 | 1, 0, (unsigned char *)hdr, cmd); | ||
946 | |||
947 | return 0; | ||
948 | } | ||
949 | |||
950 | static int | ||
951 | isert_handle_iscsi_dataout(struct isert_conn *isert_conn, | ||
952 | struct iser_rx_desc *rx_desc, unsigned char *buf) | ||
953 | { | ||
954 | struct scatterlist *sg_start; | ||
955 | struct iscsi_conn *conn = isert_conn->conn; | ||
956 | struct iscsi_cmd *cmd = NULL; | ||
957 | struct iscsi_data *hdr = (struct iscsi_data *)buf; | ||
958 | u32 unsol_data_len = ntoh24(hdr->dlength); | ||
959 | int rc, sg_nents, sg_off, page_off; | ||
960 | |||
961 | rc = iscsit_check_dataout_hdr(conn, buf, &cmd); | ||
962 | if (rc < 0) | ||
963 | return rc; | ||
964 | else if (!cmd) | ||
965 | return 0; | ||
966 | /* | ||
967 | * FIXME: Unexpected unsolicited_data out | ||
968 | */ | ||
969 | if (!cmd->unsolicited_data) { | ||
970 | pr_err("Received unexpected solicited data payload\n"); | ||
971 | dump_stack(); | ||
972 | return -1; | ||
973 | } | ||
974 | |||
975 | pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", | ||
976 | unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); | ||
977 | |||
978 | sg_off = cmd->write_data_done / PAGE_SIZE; | ||
979 | sg_start = &cmd->se_cmd.t_data_sg[sg_off]; | ||
980 | sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); | ||
981 | page_off = cmd->write_data_done % PAGE_SIZE; | ||
982 | /* | ||
983 | * FIXME: Non page-aligned unsolicited_data out | ||
984 | */ | ||
985 | if (page_off) { | ||
986 | pr_err("Received unexpected non-page aligned data payload\n"); | ||
987 | dump_stack(); | ||
988 | return -1; | ||
989 | } | ||
990 | pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", | ||
991 | sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); | ||
992 | |||
993 | sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], | ||
994 | unsol_data_len); | ||
995 | |||
996 | rc = iscsit_check_dataout_payload(cmd, hdr, false); | ||
997 | if (rc < 0) | ||
998 | return rc; | ||
999 | |||
1000 | return 0; | ||
1001 | } | ||
1002 | |||
1003 | static int | ||
1004 | isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, | ||
1005 | uint32_t read_stag, uint64_t read_va, | ||
1006 | uint32_t write_stag, uint64_t write_va) | ||
1007 | { | ||
1008 | struct iscsi_hdr *hdr = &rx_desc->iscsi_header; | ||
1009 | struct iscsi_conn *conn = isert_conn->conn; | ||
1010 | struct iscsi_cmd *cmd; | ||
1011 | struct isert_cmd *isert_cmd; | ||
1012 | int ret = -EINVAL; | ||
1013 | u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); | ||
1014 | |||
1015 | switch (opcode) { | ||
1016 | case ISCSI_OP_SCSI_CMD: | ||
1017 | cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); | ||
1018 | if (!cmd) | ||
1019 | break; | ||
1020 | |||
1021 | isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd); | ||
1022 | isert_cmd->read_stag = read_stag; | ||
1023 | isert_cmd->read_va = read_va; | ||
1024 | isert_cmd->write_stag = write_stag; | ||
1025 | isert_cmd->write_va = write_va; | ||
1026 | |||
1027 | ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, | ||
1028 | rx_desc, (unsigned char *)hdr); | ||
1029 | break; | ||
1030 | case ISCSI_OP_NOOP_OUT: | ||
1031 | cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); | ||
1032 | if (!cmd) | ||
1033 | break; | ||
1034 | |||
1035 | ret = iscsit_handle_nop_out(conn, cmd, (unsigned char *)hdr); | ||
1036 | break; | ||
1037 | case ISCSI_OP_SCSI_DATA_OUT: | ||
1038 | ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, | ||
1039 | (unsigned char *)hdr); | ||
1040 | break; | ||
1041 | case ISCSI_OP_SCSI_TMFUNC: | ||
1042 | cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); | ||
1043 | if (!cmd) | ||
1044 | break; | ||
1045 | |||
1046 | ret = iscsit_handle_task_mgt_cmd(conn, cmd, | ||
1047 | (unsigned char *)hdr); | ||
1048 | break; | ||
1049 | case ISCSI_OP_LOGOUT: | ||
1050 | cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); | ||
1051 | if (!cmd) | ||
1052 | break; | ||
1053 | |||
1054 | ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); | ||
1055 | if (ret > 0) | ||
1056 | wait_for_completion_timeout(&conn->conn_logout_comp, | ||
1057 | SECONDS_FOR_LOGOUT_COMP * | ||
1058 | HZ); | ||
1059 | break; | ||
1060 | default: | ||
1061 | pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); | ||
1062 | dump_stack(); | ||
1063 | break; | ||
1064 | } | ||
1065 | |||
1066 | return ret; | ||
1067 | } | ||
1068 | |||
1069 | static void | ||
1070 | isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) | ||
1071 | { | ||
1072 | struct iser_hdr *iser_hdr = &rx_desc->iser_header; | ||
1073 | uint64_t read_va = 0, write_va = 0; | ||
1074 | uint32_t read_stag = 0, write_stag = 0; | ||
1075 | int rc; | ||
1076 | |||
1077 | switch (iser_hdr->flags & 0xF0) { | ||
1078 | case ISCSI_CTRL: | ||
1079 | if (iser_hdr->flags & ISER_RSV) { | ||
1080 | read_stag = be32_to_cpu(iser_hdr->read_stag); | ||
1081 | read_va = be64_to_cpu(iser_hdr->read_va); | ||
1082 | pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", | ||
1083 | read_stag, (unsigned long long)read_va); | ||
1084 | } | ||
1085 | if (iser_hdr->flags & ISER_WSV) { | ||
1086 | write_stag = be32_to_cpu(iser_hdr->write_stag); | ||
1087 | write_va = be64_to_cpu(iser_hdr->write_va); | ||
1088 | pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", | ||
1089 | write_stag, (unsigned long long)write_va); | ||
1090 | } | ||
1091 | |||
1092 | pr_debug("ISER ISCSI_CTRL PDU\n"); | ||
1093 | break; | ||
1094 | case ISER_HELLO: | ||
1095 | pr_err("iSER Hello message\n"); | ||
1096 | break; | ||
1097 | default: | ||
1098 | pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); | ||
1099 | break; | ||
1100 | } | ||
1101 | |||
1102 | rc = isert_rx_opcode(isert_conn, rx_desc, | ||
1103 | read_stag, read_va, write_stag, write_va); | ||
1104 | } | ||
1105 | |||
1106 | static void | ||
1107 | isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, | ||
1108 | unsigned long xfer_len) | ||
1109 | { | ||
1110 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1111 | struct iscsi_hdr *hdr; | ||
1112 | u64 rx_dma; | ||
1113 | int rx_buflen, outstanding; | ||
1114 | |||
1115 | if ((char *)desc == isert_conn->login_req_buf) { | ||
1116 | rx_dma = isert_conn->login_req_dma; | ||
1117 | rx_buflen = ISER_RX_LOGIN_SIZE; | ||
1118 | pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", | ||
1119 | rx_dma, rx_buflen); | ||
1120 | } else { | ||
1121 | rx_dma = desc->dma_addr; | ||
1122 | rx_buflen = ISER_RX_PAYLOAD_SIZE; | ||
1123 | pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", | ||
1124 | rx_dma, rx_buflen); | ||
1125 | } | ||
1126 | |||
1127 | ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); | ||
1128 | |||
1129 | hdr = &desc->iscsi_header; | ||
1130 | pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", | ||
1131 | hdr->opcode, hdr->itt, hdr->flags, | ||
1132 | (int)(xfer_len - ISER_HEADERS_LEN)); | ||
1133 | |||
1134 | if ((char *)desc == isert_conn->login_req_buf) | ||
1135 | isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, | ||
1136 | isert_conn); | ||
1137 | else | ||
1138 | isert_rx_do_work(desc, isert_conn); | ||
1139 | |||
1140 | ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, | ||
1141 | DMA_FROM_DEVICE); | ||
1142 | |||
1143 | isert_conn->post_recv_buf_count--; | ||
1144 | pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", | ||
1145 | isert_conn->post_recv_buf_count); | ||
1146 | |||
1147 | if ((char *)desc == isert_conn->login_req_buf) | ||
1148 | return; | ||
1149 | |||
1150 | outstanding = isert_conn->post_recv_buf_count; | ||
1151 | if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { | ||
1152 | int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, | ||
1153 | ISERT_MIN_POSTED_RX); | ||
1154 | err = isert_post_recv(isert_conn, count); | ||
1155 | if (err) { | ||
1156 | pr_err("isert_post_recv() count: %d failed, %d\n", | ||
1157 | count, err); | ||
1158 | } | ||
1159 | } | ||
1160 | } | ||
1161 | |||
1162 | static void | ||
1163 | isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) | ||
1164 | { | ||
1165 | struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; | ||
1166 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1167 | |||
1168 | pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n"); | ||
1169 | |||
1170 | if (wr->sge) { | ||
1171 | ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE); | ||
1172 | wr->sge = NULL; | ||
1173 | } | ||
1174 | |||
1175 | kfree(wr->send_wr); | ||
1176 | wr->send_wr = NULL; | ||
1177 | |||
1178 | kfree(isert_cmd->ib_sge); | ||
1179 | isert_cmd->ib_sge = NULL; | ||
1180 | } | ||
1181 | |||
1182 | static void | ||
1183 | isert_put_cmd(struct isert_cmd *isert_cmd) | ||
1184 | { | ||
1185 | struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; | ||
1186 | struct isert_conn *isert_conn = isert_cmd->conn; | ||
1187 | struct iscsi_conn *conn; | ||
1188 | |||
1189 | pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); | ||
1190 | |||
1191 | switch (cmd->iscsi_opcode) { | ||
1192 | case ISCSI_OP_SCSI_CMD: | ||
1193 | conn = isert_conn->conn; | ||
1194 | |||
1195 | spin_lock_bh(&conn->cmd_lock); | ||
1196 | if (!list_empty(&cmd->i_conn_node)) | ||
1197 | list_del(&cmd->i_conn_node); | ||
1198 | spin_unlock_bh(&conn->cmd_lock); | ||
1199 | |||
1200 | if (cmd->data_direction == DMA_TO_DEVICE) | ||
1201 | iscsit_stop_dataout_timer(cmd); | ||
1202 | |||
1203 | isert_unmap_cmd(isert_cmd, isert_conn); | ||
1204 | /* | ||
1205 | * Fall-through | ||
1206 | */ | ||
1207 | case ISCSI_OP_SCSI_TMFUNC: | ||
1208 | transport_generic_free_cmd(&cmd->se_cmd, 0); | ||
1209 | break; | ||
1210 | case ISCSI_OP_REJECT: | ||
1211 | case ISCSI_OP_NOOP_OUT: | ||
1212 | conn = isert_conn->conn; | ||
1213 | |||
1214 | spin_lock_bh(&conn->cmd_lock); | ||
1215 | if (!list_empty(&cmd->i_conn_node)) | ||
1216 | list_del(&cmd->i_conn_node); | ||
1217 | spin_unlock_bh(&conn->cmd_lock); | ||
1218 | |||
1219 | /* | ||
1220 | * Handle special case for REJECT when iscsi_add_reject*() has | ||
1221 | * overwritten the original iscsi_opcode assignment, and the | ||
1222 | * associated cmd->se_cmd needs to be released. | ||
1223 | */ | ||
1224 | if (cmd->se_cmd.se_tfo != NULL) { | ||
1225 | transport_generic_free_cmd(&cmd->se_cmd, 0); | ||
1226 | break; | ||
1227 | } | ||
1228 | /* | ||
1229 | * Fall-through | ||
1230 | */ | ||
1231 | default: | ||
1232 | isert_release_cmd(cmd); | ||
1233 | break; | ||
1234 | } | ||
1235 | } | ||
1236 | |||
1237 | static void | ||
1238 | isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) | ||
1239 | { | ||
1240 | if (tx_desc->dma_addr != 0) { | ||
1241 | pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); | ||
1242 | ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, | ||
1243 | ISER_HEADERS_LEN, DMA_TO_DEVICE); | ||
1244 | tx_desc->dma_addr = 0; | ||
1245 | } | ||
1246 | } | ||
1247 | |||
1248 | static void | ||
1249 | isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, | ||
1250 | struct ib_device *ib_dev) | ||
1251 | { | ||
1252 | if (isert_cmd->sense_buf_dma != 0) { | ||
1253 | pr_debug("Calling ib_dma_unmap_single for isert_cmd->sense_buf_dma\n"); | ||
1254 | ib_dma_unmap_single(ib_dev, isert_cmd->sense_buf_dma, | ||
1255 | isert_cmd->sense_buf_len, DMA_TO_DEVICE); | ||
1256 | isert_cmd->sense_buf_dma = 0; | ||
1257 | } | ||
1258 | |||
1259 | isert_unmap_tx_desc(tx_desc, ib_dev); | ||
1260 | isert_put_cmd(isert_cmd); | ||
1261 | } | ||
1262 | |||
1263 | static void | ||
1264 | isert_completion_rdma_read(struct iser_tx_desc *tx_desc, | ||
1265 | struct isert_cmd *isert_cmd) | ||
1266 | { | ||
1267 | struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; | ||
1268 | struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; | ||
1269 | struct se_cmd *se_cmd = &cmd->se_cmd; | ||
1270 | struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device; | ||
1271 | |||
1272 | iscsit_stop_dataout_timer(cmd); | ||
1273 | |||
1274 | if (wr->sge) { | ||
1275 | pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n"); | ||
1276 | ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE); | ||
1277 | wr->sge = NULL; | ||
1278 | } | ||
1279 | |||
1280 | if (isert_cmd->ib_sge) { | ||
1281 | pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n"); | ||
1282 | kfree(isert_cmd->ib_sge); | ||
1283 | isert_cmd->ib_sge = NULL; | ||
1284 | } | ||
1285 | |||
1286 | cmd->write_data_done = se_cmd->data_length; | ||
1287 | |||
1288 | pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n"); | ||
1289 | spin_lock_bh(&cmd->istate_lock); | ||
1290 | cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; | ||
1291 | cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; | ||
1292 | spin_unlock_bh(&cmd->istate_lock); | ||
1293 | |||
1294 | target_execute_cmd(se_cmd); | ||
1295 | } | ||
1296 | |||
1297 | static void | ||
1298 | isert_do_control_comp(struct work_struct *work) | ||
1299 | { | ||
1300 | struct isert_cmd *isert_cmd = container_of(work, | ||
1301 | struct isert_cmd, comp_work); | ||
1302 | struct isert_conn *isert_conn = isert_cmd->conn; | ||
1303 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1304 | struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; | ||
1305 | |||
1306 | switch (cmd->i_state) { | ||
1307 | case ISTATE_SEND_TASKMGTRSP: | ||
1308 | pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); | ||
1309 | |||
1310 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1311 | iscsit_tmr_post_handler(cmd, cmd->conn); | ||
1312 | |||
1313 | cmd->i_state = ISTATE_SENT_STATUS; | ||
1314 | isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); | ||
1315 | break; | ||
1316 | case ISTATE_SEND_REJECT: | ||
1317 | pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); | ||
1318 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1319 | |||
1320 | cmd->i_state = ISTATE_SENT_STATUS; | ||
1321 | complete(&cmd->reject_comp); | ||
1322 | isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); | ||
1323 | case ISTATE_SEND_LOGOUTRSP: | ||
1324 | pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); | ||
1325 | /* | ||
1326 | * Call atomic_dec(&isert_conn->post_send_buf_count) | ||
1327 | * from isert_free_conn() | ||
1328 | */ | ||
1329 | isert_conn->logout_posted = true; | ||
1330 | iscsit_logout_post_handler(cmd, cmd->conn); | ||
1331 | break; | ||
1332 | default: | ||
1333 | pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); | ||
1334 | dump_stack(); | ||
1335 | break; | ||
1336 | } | ||
1337 | } | ||
1338 | |||
1339 | static void | ||
1340 | isert_response_completion(struct iser_tx_desc *tx_desc, | ||
1341 | struct isert_cmd *isert_cmd, | ||
1342 | struct isert_conn *isert_conn, | ||
1343 | struct ib_device *ib_dev) | ||
1344 | { | ||
1345 | struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; | ||
1346 | |||
1347 | if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || | ||
1348 | cmd->i_state == ISTATE_SEND_LOGOUTRSP) { | ||
1349 | isert_unmap_tx_desc(tx_desc, ib_dev); | ||
1350 | |||
1351 | INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); | ||
1352 | queue_work(isert_comp_wq, &isert_cmd->comp_work); | ||
1353 | return; | ||
1354 | } | ||
1355 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1356 | |||
1357 | cmd->i_state = ISTATE_SENT_STATUS; | ||
1358 | isert_completion_put(tx_desc, isert_cmd, ib_dev); | ||
1359 | } | ||
1360 | |||
1361 | static void | ||
1362 | isert_send_completion(struct iser_tx_desc *tx_desc, | ||
1363 | struct isert_conn *isert_conn) | ||
1364 | { | ||
1365 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1366 | struct isert_cmd *isert_cmd = tx_desc->isert_cmd; | ||
1367 | struct isert_rdma_wr *wr; | ||
1368 | |||
1369 | if (!isert_cmd) { | ||
1370 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1371 | isert_unmap_tx_desc(tx_desc, ib_dev); | ||
1372 | return; | ||
1373 | } | ||
1374 | wr = &isert_cmd->rdma_wr; | ||
1375 | |||
1376 | switch (wr->iser_ib_op) { | ||
1377 | case ISER_IB_RECV: | ||
1378 | pr_err("isert_send_completion: Got ISER_IB_RECV\n"); | ||
1379 | dump_stack(); | ||
1380 | break; | ||
1381 | case ISER_IB_SEND: | ||
1382 | pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); | ||
1383 | isert_response_completion(tx_desc, isert_cmd, | ||
1384 | isert_conn, ib_dev); | ||
1385 | break; | ||
1386 | case ISER_IB_RDMA_WRITE: | ||
1387 | pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); | ||
1388 | dump_stack(); | ||
1389 | break; | ||
1390 | case ISER_IB_RDMA_READ: | ||
1391 | pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); | ||
1392 | |||
1393 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1394 | isert_completion_rdma_read(tx_desc, isert_cmd); | ||
1395 | break; | ||
1396 | default: | ||
1397 | pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); | ||
1398 | dump_stack(); | ||
1399 | break; | ||
1400 | } | ||
1401 | } | ||
1402 | |||
1403 | static void | ||
1404 | isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) | ||
1405 | { | ||
1406 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1407 | |||
1408 | if (tx_desc) { | ||
1409 | struct isert_cmd *isert_cmd = tx_desc->isert_cmd; | ||
1410 | |||
1411 | if (!isert_cmd) | ||
1412 | isert_unmap_tx_desc(tx_desc, ib_dev); | ||
1413 | else | ||
1414 | isert_completion_put(tx_desc, isert_cmd, ib_dev); | ||
1415 | } | ||
1416 | |||
1417 | if (isert_conn->post_recv_buf_count == 0 && | ||
1418 | atomic_read(&isert_conn->post_send_buf_count) == 0) { | ||
1419 | pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); | ||
1420 | pr_debug("Calling wake_up from isert_cq_comp_err\n"); | ||
1421 | |||
1422 | isert_conn->state = ISER_CONN_TERMINATING; | ||
1423 | wake_up(&isert_conn->conn_wait_comp_err); | ||
1424 | } | ||
1425 | } | ||
1426 | |||
1427 | static void | ||
1428 | isert_cq_tx_work(struct work_struct *work) | ||
1429 | { | ||
1430 | struct isert_cq_desc *cq_desc = container_of(work, | ||
1431 | struct isert_cq_desc, cq_tx_work); | ||
1432 | struct isert_device *device = cq_desc->device; | ||
1433 | int cq_index = cq_desc->cq_index; | ||
1434 | struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; | ||
1435 | struct isert_conn *isert_conn; | ||
1436 | struct iser_tx_desc *tx_desc; | ||
1437 | struct ib_wc wc; | ||
1438 | |||
1439 | while (ib_poll_cq(tx_cq, 1, &wc) == 1) { | ||
1440 | tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; | ||
1441 | isert_conn = wc.qp->qp_context; | ||
1442 | |||
1443 | if (wc.status == IB_WC_SUCCESS) { | ||
1444 | isert_send_completion(tx_desc, isert_conn); | ||
1445 | } else { | ||
1446 | pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); | ||
1447 | pr_debug("TX wc.status: 0x%08x\n", wc.status); | ||
1448 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1449 | isert_cq_comp_err(tx_desc, isert_conn); | ||
1450 | } | ||
1451 | } | ||
1452 | |||
1453 | ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); | ||
1454 | } | ||
1455 | |||
1456 | static void | ||
1457 | isert_cq_tx_callback(struct ib_cq *cq, void *context) | ||
1458 | { | ||
1459 | struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; | ||
1460 | |||
1461 | INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work); | ||
1462 | queue_work(isert_comp_wq, &cq_desc->cq_tx_work); | ||
1463 | } | ||
1464 | |||
1465 | static void | ||
1466 | isert_cq_rx_work(struct work_struct *work) | ||
1467 | { | ||
1468 | struct isert_cq_desc *cq_desc = container_of(work, | ||
1469 | struct isert_cq_desc, cq_rx_work); | ||
1470 | struct isert_device *device = cq_desc->device; | ||
1471 | int cq_index = cq_desc->cq_index; | ||
1472 | struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; | ||
1473 | struct isert_conn *isert_conn; | ||
1474 | struct iser_rx_desc *rx_desc; | ||
1475 | struct ib_wc wc; | ||
1476 | unsigned long xfer_len; | ||
1477 | |||
1478 | while (ib_poll_cq(rx_cq, 1, &wc) == 1) { | ||
1479 | rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; | ||
1480 | isert_conn = wc.qp->qp_context; | ||
1481 | |||
1482 | if (wc.status == IB_WC_SUCCESS) { | ||
1483 | xfer_len = (unsigned long)wc.byte_len; | ||
1484 | isert_rx_completion(rx_desc, isert_conn, xfer_len); | ||
1485 | } else { | ||
1486 | pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); | ||
1487 | if (wc.status != IB_WC_WR_FLUSH_ERR) | ||
1488 | pr_debug("RX wc.status: 0x%08x\n", wc.status); | ||
1489 | |||
1490 | isert_conn->post_recv_buf_count--; | ||
1491 | isert_cq_comp_err(NULL, isert_conn); | ||
1492 | } | ||
1493 | } | ||
1494 | |||
1495 | ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); | ||
1496 | } | ||
1497 | |||
1498 | static void | ||
1499 | isert_cq_rx_callback(struct ib_cq *cq, void *context) | ||
1500 | { | ||
1501 | struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; | ||
1502 | |||
1503 | INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work); | ||
1504 | queue_work(isert_rx_wq, &cq_desc->cq_rx_work); | ||
1505 | } | ||
1506 | |||
1507 | static int | ||
1508 | isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) | ||
1509 | { | ||
1510 | struct ib_send_wr *wr_failed; | ||
1511 | int ret; | ||
1512 | |||
1513 | atomic_inc(&isert_conn->post_send_buf_count); | ||
1514 | |||
1515 | ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, | ||
1516 | &wr_failed); | ||
1517 | if (ret) { | ||
1518 | pr_err("ib_post_send failed with %d\n", ret); | ||
1519 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1520 | return ret; | ||
1521 | } | ||
1522 | return ret; | ||
1523 | } | ||
1524 | |||
1525 | static int | ||
1526 | isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) | ||
1527 | { | ||
1528 | struct isert_cmd *isert_cmd = container_of(cmd, | ||
1529 | struct isert_cmd, iscsi_cmd); | ||
1530 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
1531 | struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; | ||
1532 | struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) | ||
1533 | &isert_cmd->tx_desc.iscsi_header; | ||
1534 | |||
1535 | isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); | ||
1536 | iscsit_build_rsp_pdu(cmd, conn, true, hdr); | ||
1537 | isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); | ||
1538 | /* | ||
1539 | * Attach SENSE DATA payload to iSCSI Response PDU | ||
1540 | */ | ||
1541 | if (cmd->se_cmd.sense_buffer && | ||
1542 | ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || | ||
1543 | (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { | ||
1544 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1545 | struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; | ||
1546 | u32 padding, sense_len; | ||
1547 | |||
1548 | put_unaligned_be16(cmd->se_cmd.scsi_sense_length, | ||
1549 | cmd->sense_buffer); | ||
1550 | cmd->se_cmd.scsi_sense_length += sizeof(__be16); | ||
1551 | |||
1552 | padding = -(cmd->se_cmd.scsi_sense_length) & 3; | ||
1553 | hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); | ||
1554 | sense_len = cmd->se_cmd.scsi_sense_length + padding; | ||
1555 | |||
1556 | isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev, | ||
1557 | (void *)cmd->sense_buffer, sense_len, | ||
1558 | DMA_TO_DEVICE); | ||
1559 | |||
1560 | isert_cmd->sense_buf_len = sense_len; | ||
1561 | tx_dsg->addr = isert_cmd->sense_buf_dma; | ||
1562 | tx_dsg->length = sense_len; | ||
1563 | tx_dsg->lkey = isert_conn->conn_mr->lkey; | ||
1564 | isert_cmd->tx_desc.num_sge = 2; | ||
1565 | } | ||
1566 | |||
1567 | isert_init_send_wr(isert_cmd, send_wr); | ||
1568 | |||
1569 | pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); | ||
1570 | |||
1571 | return isert_post_response(isert_conn, isert_cmd); | ||
1572 | } | ||
1573 | |||
1574 | static int | ||
1575 | isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, | ||
1576 | bool nopout_response) | ||
1577 | { | ||
1578 | struct isert_cmd *isert_cmd = container_of(cmd, | ||
1579 | struct isert_cmd, iscsi_cmd); | ||
1580 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
1581 | struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; | ||
1582 | |||
1583 | isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); | ||
1584 | iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) | ||
1585 | &isert_cmd->tx_desc.iscsi_header, | ||
1586 | nopout_response); | ||
1587 | isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); | ||
1588 | isert_init_send_wr(isert_cmd, send_wr); | ||
1589 | |||
1590 | pr_debug("Posting NOPIN Reponse IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); | ||
1591 | |||
1592 | return isert_post_response(isert_conn, isert_cmd); | ||
1593 | } | ||
1594 | |||
1595 | static int | ||
1596 | isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) | ||
1597 | { | ||
1598 | struct isert_cmd *isert_cmd = container_of(cmd, | ||
1599 | struct isert_cmd, iscsi_cmd); | ||
1600 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
1601 | struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; | ||
1602 | |||
1603 | isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); | ||
1604 | iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) | ||
1605 | &isert_cmd->tx_desc.iscsi_header); | ||
1606 | isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); | ||
1607 | isert_init_send_wr(isert_cmd, send_wr); | ||
1608 | |||
1609 | pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); | ||
1610 | |||
1611 | return isert_post_response(isert_conn, isert_cmd); | ||
1612 | } | ||
1613 | |||
1614 | static int | ||
1615 | isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) | ||
1616 | { | ||
1617 | struct isert_cmd *isert_cmd = container_of(cmd, | ||
1618 | struct isert_cmd, iscsi_cmd); | ||
1619 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
1620 | struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; | ||
1621 | |||
1622 | isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); | ||
1623 | iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) | ||
1624 | &isert_cmd->tx_desc.iscsi_header); | ||
1625 | isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); | ||
1626 | isert_init_send_wr(isert_cmd, send_wr); | ||
1627 | |||
1628 | pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); | ||
1629 | |||
1630 | return isert_post_response(isert_conn, isert_cmd); | ||
1631 | } | ||
1632 | |||
1633 | static int | ||
1634 | isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) | ||
1635 | { | ||
1636 | struct isert_cmd *isert_cmd = container_of(cmd, | ||
1637 | struct isert_cmd, iscsi_cmd); | ||
1638 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
1639 | struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; | ||
1640 | |||
1641 | isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); | ||
1642 | iscsit_build_reject(cmd, conn, (struct iscsi_reject *) | ||
1643 | &isert_cmd->tx_desc.iscsi_header); | ||
1644 | isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); | ||
1645 | isert_init_send_wr(isert_cmd, send_wr); | ||
1646 | |||
1647 | pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); | ||
1648 | |||
1649 | return isert_post_response(isert_conn, isert_cmd); | ||
1650 | } | ||
1651 | |||
1652 | static int | ||
1653 | isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, | ||
1654 | struct ib_sge *ib_sge, struct ib_send_wr *send_wr, | ||
1655 | u32 data_left, u32 offset) | ||
1656 | { | ||
1657 | struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; | ||
1658 | struct scatterlist *sg_start, *tmp_sg; | ||
1659 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1660 | u32 sg_off, page_off; | ||
1661 | int i = 0, sg_nents; | ||
1662 | |||
1663 | sg_off = offset / PAGE_SIZE; | ||
1664 | sg_start = &cmd->se_cmd.t_data_sg[sg_off]; | ||
1665 | sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); | ||
1666 | page_off = offset % PAGE_SIZE; | ||
1667 | |||
1668 | send_wr->sg_list = ib_sge; | ||
1669 | send_wr->num_sge = sg_nents; | ||
1670 | send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; | ||
1671 | /* | ||
1672 | * Perform mapping of TCM scatterlist memory ib_sge dma_addr. | ||
1673 | */ | ||
1674 | for_each_sg(sg_start, tmp_sg, sg_nents, i) { | ||
1675 | pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", | ||
1676 | (unsigned long long)tmp_sg->dma_address, | ||
1677 | tmp_sg->length, page_off); | ||
1678 | |||
1679 | ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; | ||
1680 | ib_sge->length = min_t(u32, data_left, | ||
1681 | ib_sg_dma_len(ib_dev, tmp_sg) - page_off); | ||
1682 | ib_sge->lkey = isert_conn->conn_mr->lkey; | ||
1683 | |||
1684 | pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u\n", | ||
1685 | ib_sge->addr, ib_sge->length); | ||
1686 | page_off = 0; | ||
1687 | data_left -= ib_sge->length; | ||
1688 | ib_sge++; | ||
1689 | pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); | ||
1690 | } | ||
1691 | |||
1692 | pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", | ||
1693 | send_wr->sg_list, send_wr->num_sge); | ||
1694 | |||
1695 | return sg_nents; | ||
1696 | } | ||
1697 | |||
1698 | static int | ||
1699 | isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) | ||
1700 | { | ||
1701 | struct se_cmd *se_cmd = &cmd->se_cmd; | ||
1702 | struct isert_cmd *isert_cmd = container_of(cmd, | ||
1703 | struct isert_cmd, iscsi_cmd); | ||
1704 | struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; | ||
1705 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
1706 | struct ib_send_wr *wr_failed, *send_wr; | ||
1707 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1708 | struct ib_sge *ib_sge; | ||
1709 | struct scatterlist *sg; | ||
1710 | u32 offset = 0, data_len, data_left, rdma_write_max; | ||
1711 | int rc, ret = 0, count, sg_nents, i, ib_sge_cnt; | ||
1712 | |||
1713 | pr_debug("RDMA_WRITE: data_length: %u\n", se_cmd->data_length); | ||
1714 | |||
1715 | sg = &se_cmd->t_data_sg[0]; | ||
1716 | sg_nents = se_cmd->t_data_nents; | ||
1717 | |||
1718 | count = ib_dma_map_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE); | ||
1719 | if (unlikely(!count)) { | ||
1720 | pr_err("Unable to map put_datain SGs\n"); | ||
1721 | return -EINVAL; | ||
1722 | } | ||
1723 | wr->sge = sg; | ||
1724 | wr->num_sge = sg_nents; | ||
1725 | pr_debug("Mapped IB count: %u sg: %p sg_nents: %u for RDMA_WRITE\n", | ||
1726 | count, sg, sg_nents); | ||
1727 | |||
1728 | ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); | ||
1729 | if (!ib_sge) { | ||
1730 | pr_warn("Unable to allocate datain ib_sge\n"); | ||
1731 | ret = -ENOMEM; | ||
1732 | goto unmap_sg; | ||
1733 | } | ||
1734 | isert_cmd->ib_sge = ib_sge; | ||
1735 | |||
1736 | pr_debug("Allocated ib_sge: %p from t_data_ents: %d for RDMA_WRITE\n", | ||
1737 | ib_sge, se_cmd->t_data_nents); | ||
1738 | |||
1739 | wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); | ||
1740 | wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, | ||
1741 | GFP_KERNEL); | ||
1742 | if (!wr->send_wr) { | ||
1743 | pr_err("Unable to allocate wr->send_wr\n"); | ||
1744 | ret = -ENOMEM; | ||
1745 | goto unmap_sg; | ||
1746 | } | ||
1747 | pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n", | ||
1748 | wr->send_wr, wr->send_wr_num); | ||
1749 | |||
1750 | iscsit_increment_maxcmdsn(cmd, conn->sess); | ||
1751 | cmd->stat_sn = conn->stat_sn++; | ||
1752 | |||
1753 | wr->isert_cmd = isert_cmd; | ||
1754 | rdma_write_max = isert_conn->max_sge * PAGE_SIZE; | ||
1755 | data_left = se_cmd->data_length; | ||
1756 | |||
1757 | for (i = 0; i < wr->send_wr_num; i++) { | ||
1758 | send_wr = &isert_cmd->rdma_wr.send_wr[i]; | ||
1759 | data_len = min(data_left, rdma_write_max); | ||
1760 | |||
1761 | send_wr->opcode = IB_WR_RDMA_WRITE; | ||
1762 | send_wr->send_flags = 0; | ||
1763 | send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; | ||
1764 | send_wr->wr.rdma.rkey = isert_cmd->read_stag; | ||
1765 | |||
1766 | ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, | ||
1767 | send_wr, data_len, offset); | ||
1768 | ib_sge += ib_sge_cnt; | ||
1769 | |||
1770 | if (i + 1 == wr->send_wr_num) | ||
1771 | send_wr->next = &isert_cmd->tx_desc.send_wr; | ||
1772 | else | ||
1773 | send_wr->next = &wr->send_wr[i + 1]; | ||
1774 | |||
1775 | offset += data_len; | ||
1776 | data_left -= data_len; | ||
1777 | } | ||
1778 | /* | ||
1779 | * Build isert_conn->tx_desc for iSCSI response PDU and attach | ||
1780 | */ | ||
1781 | isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); | ||
1782 | iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *) | ||
1783 | &isert_cmd->tx_desc.iscsi_header); | ||
1784 | isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); | ||
1785 | isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr); | ||
1786 | |||
1787 | atomic_inc(&isert_conn->post_send_buf_count); | ||
1788 | |||
1789 | rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); | ||
1790 | if (rc) { | ||
1791 | pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); | ||
1792 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1793 | } | ||
1794 | pr_debug("Posted RDMA_WRITE + Response for iSER Data READ\n"); | ||
1795 | return 1; | ||
1796 | |||
1797 | unmap_sg: | ||
1798 | ib_dma_unmap_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE); | ||
1799 | return ret; | ||
1800 | } | ||
1801 | |||
1802 | static int | ||
1803 | isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) | ||
1804 | { | ||
1805 | struct se_cmd *se_cmd = &cmd->se_cmd; | ||
1806 | struct isert_cmd *isert_cmd = container_of(cmd, | ||
1807 | struct isert_cmd, iscsi_cmd); | ||
1808 | struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; | ||
1809 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
1810 | struct ib_send_wr *wr_failed, *send_wr; | ||
1811 | struct ib_sge *ib_sge; | ||
1812 | struct ib_device *ib_dev = isert_conn->conn_cm_id->device; | ||
1813 | struct scatterlist *sg_start; | ||
1814 | u32 sg_off, sg_nents, page_off, va_offset = 0; | ||
1815 | u32 offset = 0, data_len, data_left, rdma_write_max; | ||
1816 | int rc, ret = 0, count, i, ib_sge_cnt; | ||
1817 | |||
1818 | pr_debug("RDMA_READ: data_length: %u write_data_done: %u\n", | ||
1819 | se_cmd->data_length, cmd->write_data_done); | ||
1820 | |||
1821 | sg_off = cmd->write_data_done / PAGE_SIZE; | ||
1822 | sg_start = &cmd->se_cmd.t_data_sg[sg_off]; | ||
1823 | page_off = cmd->write_data_done % PAGE_SIZE; | ||
1824 | |||
1825 | pr_debug("RDMA_READ: sg_off: %d, sg_start: %p page_off: %d\n", | ||
1826 | sg_off, sg_start, page_off); | ||
1827 | |||
1828 | data_left = se_cmd->data_length - cmd->write_data_done; | ||
1829 | sg_nents = se_cmd->t_data_nents - sg_off; | ||
1830 | |||
1831 | pr_debug("RDMA_READ: data_left: %d, sg_nents: %d\n", | ||
1832 | data_left, sg_nents); | ||
1833 | |||
1834 | count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE); | ||
1835 | if (unlikely(!count)) { | ||
1836 | pr_err("Unable to map get_dataout SGs\n"); | ||
1837 | return -EINVAL; | ||
1838 | } | ||
1839 | wr->sge = sg_start; | ||
1840 | wr->num_sge = sg_nents; | ||
1841 | pr_debug("Mapped IB count: %u sg_start: %p sg_nents: %u for RDMA_READ\n", | ||
1842 | count, sg_start, sg_nents); | ||
1843 | |||
1844 | ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); | ||
1845 | if (!ib_sge) { | ||
1846 | pr_warn("Unable to allocate dataout ib_sge\n"); | ||
1847 | ret = -ENOMEM; | ||
1848 | goto unmap_sg; | ||
1849 | } | ||
1850 | isert_cmd->ib_sge = ib_sge; | ||
1851 | |||
1852 | pr_debug("Using ib_sge: %p from sg_ents: %d for RDMA_READ\n", | ||
1853 | ib_sge, sg_nents); | ||
1854 | |||
1855 | wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); | ||
1856 | wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, | ||
1857 | GFP_KERNEL); | ||
1858 | if (!wr->send_wr) { | ||
1859 | pr_debug("Unable to allocate wr->send_wr\n"); | ||
1860 | ret = -ENOMEM; | ||
1861 | goto unmap_sg; | ||
1862 | } | ||
1863 | pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n", | ||
1864 | wr->send_wr, wr->send_wr_num); | ||
1865 | |||
1866 | isert_cmd->tx_desc.isert_cmd = isert_cmd; | ||
1867 | |||
1868 | wr->iser_ib_op = ISER_IB_RDMA_READ; | ||
1869 | wr->isert_cmd = isert_cmd; | ||
1870 | rdma_write_max = isert_conn->max_sge * PAGE_SIZE; | ||
1871 | offset = cmd->write_data_done; | ||
1872 | |||
1873 | for (i = 0; i < wr->send_wr_num; i++) { | ||
1874 | send_wr = &isert_cmd->rdma_wr.send_wr[i]; | ||
1875 | data_len = min(data_left, rdma_write_max); | ||
1876 | |||
1877 | send_wr->opcode = IB_WR_RDMA_READ; | ||
1878 | send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; | ||
1879 | send_wr->wr.rdma.rkey = isert_cmd->write_stag; | ||
1880 | |||
1881 | ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, | ||
1882 | send_wr, data_len, offset); | ||
1883 | ib_sge += ib_sge_cnt; | ||
1884 | |||
1885 | if (i + 1 == wr->send_wr_num) | ||
1886 | send_wr->send_flags = IB_SEND_SIGNALED; | ||
1887 | else | ||
1888 | send_wr->next = &wr->send_wr[i + 1]; | ||
1889 | |||
1890 | offset += data_len; | ||
1891 | va_offset += data_len; | ||
1892 | data_left -= data_len; | ||
1893 | } | ||
1894 | |||
1895 | atomic_inc(&isert_conn->post_send_buf_count); | ||
1896 | |||
1897 | rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); | ||
1898 | if (rc) { | ||
1899 | pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); | ||
1900 | atomic_dec(&isert_conn->post_send_buf_count); | ||
1901 | } | ||
1902 | pr_debug("Posted RDMA_READ memory for ISER Data WRITE\n"); | ||
1903 | return 0; | ||
1904 | |||
1905 | unmap_sg: | ||
1906 | ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE); | ||
1907 | return ret; | ||
1908 | } | ||
1909 | |||
1910 | static int | ||
1911 | isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) | ||
1912 | { | ||
1913 | int ret; | ||
1914 | |||
1915 | switch (state) { | ||
1916 | case ISTATE_SEND_NOPIN_WANT_RESPONSE: | ||
1917 | ret = isert_put_nopin(cmd, conn, false); | ||
1918 | break; | ||
1919 | default: | ||
1920 | pr_err("Unknown immediate state: 0x%02x\n", state); | ||
1921 | ret = -EINVAL; | ||
1922 | break; | ||
1923 | } | ||
1924 | |||
1925 | return ret; | ||
1926 | } | ||
1927 | |||
1928 | static int | ||
1929 | isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) | ||
1930 | { | ||
1931 | int ret; | ||
1932 | |||
1933 | switch (state) { | ||
1934 | case ISTATE_SEND_LOGOUTRSP: | ||
1935 | ret = isert_put_logout_rsp(cmd, conn); | ||
1936 | if (!ret) { | ||
1937 | pr_debug("Returning iSER Logout -EAGAIN\n"); | ||
1938 | ret = -EAGAIN; | ||
1939 | } | ||
1940 | break; | ||
1941 | case ISTATE_SEND_NOPIN: | ||
1942 | ret = isert_put_nopin(cmd, conn, true); | ||
1943 | break; | ||
1944 | case ISTATE_SEND_TASKMGTRSP: | ||
1945 | ret = isert_put_tm_rsp(cmd, conn); | ||
1946 | break; | ||
1947 | case ISTATE_SEND_REJECT: | ||
1948 | ret = isert_put_reject(cmd, conn); | ||
1949 | break; | ||
1950 | case ISTATE_SEND_STATUS: | ||
1951 | /* | ||
1952 | * Special case for sending non GOOD SCSI status from TX thread | ||
1953 | * context during pre se_cmd excecution failure. | ||
1954 | */ | ||
1955 | ret = isert_put_response(conn, cmd); | ||
1956 | break; | ||
1957 | default: | ||
1958 | pr_err("Unknown response state: 0x%02x\n", state); | ||
1959 | ret = -EINVAL; | ||
1960 | break; | ||
1961 | } | ||
1962 | |||
1963 | return ret; | ||
1964 | } | ||
1965 | |||
1966 | static int | ||
1967 | isert_setup_np(struct iscsi_np *np, | ||
1968 | struct __kernel_sockaddr_storage *ksockaddr) | ||
1969 | { | ||
1970 | struct isert_np *isert_np; | ||
1971 | struct rdma_cm_id *isert_lid; | ||
1972 | struct sockaddr *sa; | ||
1973 | int ret; | ||
1974 | |||
1975 | isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); | ||
1976 | if (!isert_np) { | ||
1977 | pr_err("Unable to allocate struct isert_np\n"); | ||
1978 | return -ENOMEM; | ||
1979 | } | ||
1980 | init_waitqueue_head(&isert_np->np_accept_wq); | ||
1981 | mutex_init(&isert_np->np_accept_mutex); | ||
1982 | INIT_LIST_HEAD(&isert_np->np_accept_list); | ||
1983 | init_completion(&isert_np->np_login_comp); | ||
1984 | |||
1985 | sa = (struct sockaddr *)ksockaddr; | ||
1986 | pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); | ||
1987 | /* | ||
1988 | * Setup the np->np_sockaddr from the passed sockaddr setup | ||
1989 | * in iscsi_target_configfs.c code.. | ||
1990 | */ | ||
1991 | memcpy(&np->np_sockaddr, ksockaddr, | ||
1992 | sizeof(struct __kernel_sockaddr_storage)); | ||
1993 | |||
1994 | isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, | ||
1995 | IB_QPT_RC); | ||
1996 | if (IS_ERR(isert_lid)) { | ||
1997 | pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", | ||
1998 | PTR_ERR(isert_lid)); | ||
1999 | ret = PTR_ERR(isert_lid); | ||
2000 | goto out; | ||
2001 | } | ||
2002 | |||
2003 | ret = rdma_bind_addr(isert_lid, sa); | ||
2004 | if (ret) { | ||
2005 | pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); | ||
2006 | goto out_lid; | ||
2007 | } | ||
2008 | |||
2009 | ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); | ||
2010 | if (ret) { | ||
2011 | pr_err("rdma_listen() for isert_lid failed: %d\n", ret); | ||
2012 | goto out_lid; | ||
2013 | } | ||
2014 | |||
2015 | isert_np->np_cm_id = isert_lid; | ||
2016 | np->np_context = isert_np; | ||
2017 | pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); | ||
2018 | |||
2019 | return 0; | ||
2020 | |||
2021 | out_lid: | ||
2022 | rdma_destroy_id(isert_lid); | ||
2023 | out: | ||
2024 | kfree(isert_np); | ||
2025 | return ret; | ||
2026 | } | ||
2027 | |||
2028 | static int | ||
2029 | isert_check_accept_queue(struct isert_np *isert_np) | ||
2030 | { | ||
2031 | int empty; | ||
2032 | |||
2033 | mutex_lock(&isert_np->np_accept_mutex); | ||
2034 | empty = list_empty(&isert_np->np_accept_list); | ||
2035 | mutex_unlock(&isert_np->np_accept_mutex); | ||
2036 | |||
2037 | return empty; | ||
2038 | } | ||
2039 | |||
2040 | static int | ||
2041 | isert_rdma_accept(struct isert_conn *isert_conn) | ||
2042 | { | ||
2043 | struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; | ||
2044 | struct rdma_conn_param cp; | ||
2045 | int ret; | ||
2046 | |||
2047 | memset(&cp, 0, sizeof(struct rdma_conn_param)); | ||
2048 | cp.responder_resources = isert_conn->responder_resources; | ||
2049 | cp.initiator_depth = isert_conn->initiator_depth; | ||
2050 | cp.retry_count = 7; | ||
2051 | cp.rnr_retry_count = 7; | ||
2052 | |||
2053 | pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); | ||
2054 | |||
2055 | ret = rdma_accept(cm_id, &cp); | ||
2056 | if (ret) { | ||
2057 | pr_err("rdma_accept() failed with: %d\n", ret); | ||
2058 | return ret; | ||
2059 | } | ||
2060 | |||
2061 | pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); | ||
2062 | |||
2063 | return 0; | ||
2064 | } | ||
2065 | |||
2066 | static int | ||
2067 | isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) | ||
2068 | { | ||
2069 | struct isert_conn *isert_conn = (struct isert_conn *)conn->context; | ||
2070 | int ret; | ||
2071 | |||
2072 | pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); | ||
2073 | |||
2074 | ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); | ||
2075 | if (ret) | ||
2076 | return ret; | ||
2077 | |||
2078 | pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); | ||
2079 | return 0; | ||
2080 | } | ||
2081 | |||
2082 | static void | ||
2083 | isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, | ||
2084 | struct isert_conn *isert_conn) | ||
2085 | { | ||
2086 | struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; | ||
2087 | struct rdma_route *cm_route = &cm_id->route; | ||
2088 | struct sockaddr_in *sock_in; | ||
2089 | struct sockaddr_in6 *sock_in6; | ||
2090 | |||
2091 | conn->login_family = np->np_sockaddr.ss_family; | ||
2092 | |||
2093 | if (np->np_sockaddr.ss_family == AF_INET6) { | ||
2094 | sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; | ||
2095 | snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", | ||
2096 | &sock_in6->sin6_addr.in6_u); | ||
2097 | conn->login_port = ntohs(sock_in6->sin6_port); | ||
2098 | |||
2099 | sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; | ||
2100 | snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", | ||
2101 | &sock_in6->sin6_addr.in6_u); | ||
2102 | conn->local_port = ntohs(sock_in6->sin6_port); | ||
2103 | } else { | ||
2104 | sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; | ||
2105 | sprintf(conn->login_ip, "%pI4", | ||
2106 | &sock_in->sin_addr.s_addr); | ||
2107 | conn->login_port = ntohs(sock_in->sin_port); | ||
2108 | |||
2109 | sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; | ||
2110 | sprintf(conn->local_ip, "%pI4", | ||
2111 | &sock_in->sin_addr.s_addr); | ||
2112 | conn->local_port = ntohs(sock_in->sin_port); | ||
2113 | } | ||
2114 | } | ||
2115 | |||
2116 | static int | ||
2117 | isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) | ||
2118 | { | ||
2119 | struct isert_np *isert_np = (struct isert_np *)np->np_context; | ||
2120 | struct isert_conn *isert_conn; | ||
2121 | int max_accept = 0, ret; | ||
2122 | |||
2123 | accept_wait: | ||
2124 | ret = wait_event_interruptible(isert_np->np_accept_wq, | ||
2125 | !isert_check_accept_queue(isert_np) || | ||
2126 | np->np_thread_state == ISCSI_NP_THREAD_RESET); | ||
2127 | if (max_accept > 5) | ||
2128 | return -ENODEV; | ||
2129 | |||
2130 | spin_lock_bh(&np->np_thread_lock); | ||
2131 | if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { | ||
2132 | spin_unlock_bh(&np->np_thread_lock); | ||
2133 | pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); | ||
2134 | return -ENODEV; | ||
2135 | } | ||
2136 | spin_unlock_bh(&np->np_thread_lock); | ||
2137 | |||
2138 | mutex_lock(&isert_np->np_accept_mutex); | ||
2139 | if (list_empty(&isert_np->np_accept_list)) { | ||
2140 | mutex_unlock(&isert_np->np_accept_mutex); | ||
2141 | max_accept++; | ||
2142 | goto accept_wait; | ||
2143 | } | ||
2144 | isert_conn = list_first_entry(&isert_np->np_accept_list, | ||
2145 | struct isert_conn, conn_accept_node); | ||
2146 | list_del_init(&isert_conn->conn_accept_node); | ||
2147 | mutex_unlock(&isert_np->np_accept_mutex); | ||
2148 | |||
2149 | conn->context = isert_conn; | ||
2150 | isert_conn->conn = conn; | ||
2151 | max_accept = 0; | ||
2152 | |||
2153 | ret = isert_rdma_post_recvl(isert_conn); | ||
2154 | if (ret) | ||
2155 | return ret; | ||
2156 | |||
2157 | ret = isert_rdma_accept(isert_conn); | ||
2158 | if (ret) | ||
2159 | return ret; | ||
2160 | |||
2161 | isert_set_conn_info(np, conn, isert_conn); | ||
2162 | |||
2163 | pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); | ||
2164 | return 0; | ||
2165 | } | ||
2166 | |||
2167 | static void | ||
2168 | isert_free_np(struct iscsi_np *np) | ||
2169 | { | ||
2170 | struct isert_np *isert_np = (struct isert_np *)np->np_context; | ||
2171 | |||
2172 | rdma_destroy_id(isert_np->np_cm_id); | ||
2173 | |||
2174 | np->np_context = NULL; | ||
2175 | kfree(isert_np); | ||
2176 | } | ||
2177 | |||
2178 | static void isert_free_conn(struct iscsi_conn *conn) | ||
2179 | { | ||
2180 | struct isert_conn *isert_conn = conn->context; | ||
2181 | |||
2182 | pr_debug("isert_free_conn: Starting \n"); | ||
2183 | /* | ||
2184 | * Decrement post_send_buf_count for special case when called | ||
2185 | * from isert_do_control_comp() -> iscsit_logout_post_handler() | ||
2186 | */ | ||
2187 | if (isert_conn->logout_posted) | ||
2188 | atomic_dec(&isert_conn->post_send_buf_count); | ||
2189 | |||
2190 | if (isert_conn->conn_cm_id) | ||
2191 | rdma_disconnect(isert_conn->conn_cm_id); | ||
2192 | /* | ||
2193 | * Only wait for conn_wait_comp_err if the isert_conn made it | ||
2194 | * into full feature phase.. | ||
2195 | */ | ||
2196 | if (isert_conn->state > ISER_CONN_INIT) { | ||
2197 | pr_debug("isert_free_conn: Before wait_event comp_err %d\n", | ||
2198 | isert_conn->state); | ||
2199 | wait_event(isert_conn->conn_wait_comp_err, | ||
2200 | isert_conn->state == ISER_CONN_TERMINATING); | ||
2201 | pr_debug("isert_free_conn: After wait_event #1 >>>>>>>>>>>>\n"); | ||
2202 | } | ||
2203 | |||
2204 | pr_debug("isert_free_conn: wait_event conn_wait %d\n", isert_conn->state); | ||
2205 | wait_event(isert_conn->conn_wait, isert_conn->state == ISER_CONN_DOWN); | ||
2206 | pr_debug("isert_free_conn: After wait_event #2 >>>>>>>>>>>>>>>>>>>>\n"); | ||
2207 | |||
2208 | isert_put_conn(isert_conn); | ||
2209 | } | ||
2210 | |||
2211 | static struct iscsit_transport iser_target_transport = { | ||
2212 | .name = "IB/iSER", | ||
2213 | .transport_type = ISCSI_INFINIBAND, | ||
2214 | .owner = THIS_MODULE, | ||
2215 | .iscsit_setup_np = isert_setup_np, | ||
2216 | .iscsit_accept_np = isert_accept_np, | ||
2217 | .iscsit_free_np = isert_free_np, | ||
2218 | .iscsit_free_conn = isert_free_conn, | ||
2219 | .iscsit_alloc_cmd = isert_alloc_cmd, | ||
2220 | .iscsit_get_login_rx = isert_get_login_rx, | ||
2221 | .iscsit_put_login_tx = isert_put_login_tx, | ||
2222 | .iscsit_immediate_queue = isert_immediate_queue, | ||
2223 | .iscsit_response_queue = isert_response_queue, | ||
2224 | .iscsit_get_dataout = isert_get_dataout, | ||
2225 | .iscsit_queue_data_in = isert_put_datain, | ||
2226 | .iscsit_queue_status = isert_put_response, | ||
2227 | }; | ||
2228 | |||
2229 | static int __init isert_init(void) | ||
2230 | { | ||
2231 | int ret; | ||
2232 | |||
2233 | isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); | ||
2234 | if (!isert_rx_wq) { | ||
2235 | pr_err("Unable to allocate isert_rx_wq\n"); | ||
2236 | return -ENOMEM; | ||
2237 | } | ||
2238 | |||
2239 | isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); | ||
2240 | if (!isert_comp_wq) { | ||
2241 | pr_err("Unable to allocate isert_comp_wq\n"); | ||
2242 | ret = -ENOMEM; | ||
2243 | goto destroy_rx_wq; | ||
2244 | } | ||
2245 | |||
2246 | isert_cmd_cache = kmem_cache_create("isert_cmd_cache", | ||
2247 | sizeof(struct isert_cmd), __alignof__(struct isert_cmd), | ||
2248 | 0, NULL); | ||
2249 | if (!isert_cmd_cache) { | ||
2250 | pr_err("Unable to create isert_cmd_cache\n"); | ||
2251 | ret = -ENOMEM; | ||
2252 | goto destroy_tx_cq; | ||
2253 | } | ||
2254 | |||
2255 | iscsit_register_transport(&iser_target_transport); | ||
2256 | pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); | ||
2257 | return 0; | ||
2258 | |||
2259 | destroy_tx_cq: | ||
2260 | destroy_workqueue(isert_comp_wq); | ||
2261 | destroy_rx_wq: | ||
2262 | destroy_workqueue(isert_rx_wq); | ||
2263 | return ret; | ||
2264 | } | ||
2265 | |||
2266 | static void __exit isert_exit(void) | ||
2267 | { | ||
2268 | kmem_cache_destroy(isert_cmd_cache); | ||
2269 | destroy_workqueue(isert_comp_wq); | ||
2270 | destroy_workqueue(isert_rx_wq); | ||
2271 | iscsit_unregister_transport(&iser_target_transport); | ||
2272 | pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); | ||
2273 | } | ||
2274 | |||
2275 | MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); | ||
2276 | MODULE_VERSION("0.1"); | ||
2277 | MODULE_AUTHOR("nab@Linux-iSCSI.org"); | ||
2278 | MODULE_LICENSE("GPL"); | ||
2279 | |||
2280 | module_init(isert_init); | ||
2281 | module_exit(isert_exit); | ||
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h new file mode 100644 index 000000000000..b104f4c2cd38 --- /dev/null +++ b/drivers/infiniband/ulp/isert/ib_isert.h | |||
@@ -0,0 +1,138 @@ | |||
1 | #include <linux/socket.h> | ||
2 | #include <linux/in.h> | ||
3 | #include <linux/in6.h> | ||
4 | #include <rdma/ib_verbs.h> | ||
5 | #include <rdma/rdma_cm.h> | ||
6 | |||
7 | #define ISERT_RDMA_LISTEN_BACKLOG 10 | ||
8 | |||
9 | enum isert_desc_type { | ||
10 | ISCSI_TX_CONTROL, | ||
11 | ISCSI_TX_DATAIN | ||
12 | }; | ||
13 | |||
14 | enum iser_ib_op_code { | ||
15 | ISER_IB_RECV, | ||
16 | ISER_IB_SEND, | ||
17 | ISER_IB_RDMA_WRITE, | ||
18 | ISER_IB_RDMA_READ, | ||
19 | }; | ||
20 | |||
21 | enum iser_conn_state { | ||
22 | ISER_CONN_INIT, | ||
23 | ISER_CONN_UP, | ||
24 | ISER_CONN_TERMINATING, | ||
25 | ISER_CONN_DOWN, | ||
26 | }; | ||
27 | |||
28 | struct iser_rx_desc { | ||
29 | struct iser_hdr iser_header; | ||
30 | struct iscsi_hdr iscsi_header; | ||
31 | char data[ISER_RECV_DATA_SEG_LEN]; | ||
32 | u64 dma_addr; | ||
33 | struct ib_sge rx_sg; | ||
34 | char pad[ISER_RX_PAD_SIZE]; | ||
35 | } __packed; | ||
36 | |||
37 | struct iser_tx_desc { | ||
38 | struct iser_hdr iser_header; | ||
39 | struct iscsi_hdr iscsi_header; | ||
40 | enum isert_desc_type type; | ||
41 | u64 dma_addr; | ||
42 | struct ib_sge tx_sg[2]; | ||
43 | int num_sge; | ||
44 | struct isert_cmd *isert_cmd; | ||
45 | struct ib_send_wr send_wr; | ||
46 | } __packed; | ||
47 | |||
48 | struct isert_rdma_wr { | ||
49 | struct list_head wr_list; | ||
50 | struct isert_cmd *isert_cmd; | ||
51 | enum iser_ib_op_code iser_ib_op; | ||
52 | struct ib_sge *ib_sge; | ||
53 | int num_sge; | ||
54 | struct scatterlist *sge; | ||
55 | int send_wr_num; | ||
56 | struct ib_send_wr *send_wr; | ||
57 | }; | ||
58 | |||
59 | struct isert_cmd { | ||
60 | uint32_t read_stag; | ||
61 | uint32_t write_stag; | ||
62 | uint64_t read_va; | ||
63 | uint64_t write_va; | ||
64 | u64 sense_buf_dma; | ||
65 | u32 sense_buf_len; | ||
66 | u32 read_va_off; | ||
67 | u32 write_va_off; | ||
68 | u32 rdma_wr_num; | ||
69 | struct isert_conn *conn; | ||
70 | struct iscsi_cmd iscsi_cmd; | ||
71 | struct ib_sge *ib_sge; | ||
72 | struct iser_tx_desc tx_desc; | ||
73 | struct isert_rdma_wr rdma_wr; | ||
74 | struct work_struct comp_work; | ||
75 | }; | ||
76 | |||
77 | struct isert_device; | ||
78 | |||
79 | struct isert_conn { | ||
80 | enum iser_conn_state state; | ||
81 | bool logout_posted; | ||
82 | int post_recv_buf_count; | ||
83 | atomic_t post_send_buf_count; | ||
84 | u32 responder_resources; | ||
85 | u32 initiator_depth; | ||
86 | u32 max_sge; | ||
87 | char *login_buf; | ||
88 | char *login_req_buf; | ||
89 | char *login_rsp_buf; | ||
90 | u64 login_req_dma; | ||
91 | u64 login_rsp_dma; | ||
92 | unsigned int conn_rx_desc_head; | ||
93 | struct iser_rx_desc *conn_rx_descs; | ||
94 | struct ib_recv_wr conn_rx_wr[ISERT_MIN_POSTED_RX]; | ||
95 | struct iscsi_conn *conn; | ||
96 | struct list_head conn_accept_node; | ||
97 | struct completion conn_login_comp; | ||
98 | struct iser_tx_desc conn_login_tx_desc; | ||
99 | struct rdma_cm_id *conn_cm_id; | ||
100 | struct ib_pd *conn_pd; | ||
101 | struct ib_mr *conn_mr; | ||
102 | struct ib_qp *conn_qp; | ||
103 | struct isert_device *conn_device; | ||
104 | struct work_struct conn_logout_work; | ||
105 | wait_queue_head_t conn_wait; | ||
106 | wait_queue_head_t conn_wait_comp_err; | ||
107 | struct kref conn_kref; | ||
108 | }; | ||
109 | |||
110 | #define ISERT_MAX_CQ 64 | ||
111 | |||
112 | struct isert_cq_desc { | ||
113 | struct isert_device *device; | ||
114 | int cq_index; | ||
115 | struct work_struct cq_rx_work; | ||
116 | struct work_struct cq_tx_work; | ||
117 | }; | ||
118 | |||
119 | struct isert_device { | ||
120 | int cqs_used; | ||
121 | int refcount; | ||
122 | int cq_active_qps[ISERT_MAX_CQ]; | ||
123 | struct ib_device *ib_device; | ||
124 | struct ib_pd *dev_pd; | ||
125 | struct ib_mr *dev_mr; | ||
126 | struct ib_cq *dev_rx_cq[ISERT_MAX_CQ]; | ||
127 | struct ib_cq *dev_tx_cq[ISERT_MAX_CQ]; | ||
128 | struct isert_cq_desc *cq_desc; | ||
129 | struct list_head dev_node; | ||
130 | }; | ||
131 | |||
132 | struct isert_np { | ||
133 | wait_queue_head_t np_accept_wq; | ||
134 | struct rdma_cm_id *np_cm_id; | ||
135 | struct mutex np_accept_mutex; | ||
136 | struct list_head np_accept_list; | ||
137 | struct completion np_login_comp; | ||
138 | }; | ||
diff --git a/drivers/infiniband/ulp/isert/isert_proto.h b/drivers/infiniband/ulp/isert/isert_proto.h new file mode 100644 index 000000000000..4dccd313b777 --- /dev/null +++ b/drivers/infiniband/ulp/isert/isert_proto.h | |||
@@ -0,0 +1,47 @@ | |||
1 | /* From iscsi_iser.h */ | ||
2 | |||
3 | struct iser_hdr { | ||
4 | u8 flags; | ||
5 | u8 rsvd[3]; | ||
6 | __be32 write_stag; /* write rkey */ | ||
7 | __be64 write_va; | ||
8 | __be32 read_stag; /* read rkey */ | ||
9 | __be64 read_va; | ||
10 | } __packed; | ||
11 | |||
12 | /*Constant PDU lengths calculations */ | ||
13 | #define ISER_HEADERS_LEN (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr)) | ||
14 | |||
15 | #define ISER_RECV_DATA_SEG_LEN 8192 | ||
16 | #define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN) | ||
17 | #define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN) | ||
18 | |||
19 | /* QP settings */ | ||
20 | /* Maximal bounds on received asynchronous PDUs */ | ||
21 | #define ISERT_MAX_TX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */ | ||
22 | |||
23 | #define ISERT_MAX_RX_MISC_PDUS 6 /* NOOP_OUT(2), TEXT(1), * | ||
24 | * SCSI_TMFUNC(2), LOGOUT(1) */ | ||
25 | |||
26 | #define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */ | ||
27 | |||
28 | #define ISERT_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX) | ||
29 | |||
30 | #define ISERT_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2) | ||
31 | |||
32 | #define ISERT_INFLIGHT_DATAOUTS 8 | ||
33 | |||
34 | #define ISERT_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX * \ | ||
35 | (1 + ISERT_INFLIGHT_DATAOUTS) + \ | ||
36 | ISERT_MAX_TX_MISC_PDUS + \ | ||
37 | ISERT_MAX_RX_MISC_PDUS) | ||
38 | |||
39 | #define ISER_RX_PAD_SIZE (ISER_RECV_DATA_SEG_LEN + 4096 - \ | ||
40 | (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge))) | ||
41 | |||
42 | #define ISER_VER 0x10 | ||
43 | #define ISER_WSV 0x08 | ||
44 | #define ISER_RSV 0x04 | ||
45 | #define ISCSI_CTRL 0x10 | ||
46 | #define ISER_HELLO 0x20 | ||
47 | #define ISER_HELLORPLY 0x30 | ||