diff options
Diffstat (limited to 'drivers/infiniband/ulp/iser/iser_initiator.c')
-rw-r--r-- | drivers/infiniband/ulp/iser/iser_initiator.c | 506 |
1 files changed, 163 insertions, 343 deletions
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c index 9de640200ad3..0b9ef0716588 100644 --- a/drivers/infiniband/ulp/iser/iser_initiator.c +++ b/drivers/infiniband/ulp/iser/iser_initiator.c | |||
@@ -39,29 +39,6 @@ | |||
39 | 39 | ||
40 | #include "iscsi_iser.h" | 40 | #include "iscsi_iser.h" |
41 | 41 | ||
42 | /* Constant PDU lengths calculations */ | ||
43 | #define ISER_TOTAL_HEADERS_LEN (sizeof (struct iser_hdr) + \ | ||
44 | sizeof (struct iscsi_hdr)) | ||
45 | |||
46 | /* iser_dto_add_regd_buff - increments the reference count for * | ||
47 | * the registered buffer & adds it to the DTO object */ | ||
48 | static void iser_dto_add_regd_buff(struct iser_dto *dto, | ||
49 | struct iser_regd_buf *regd_buf, | ||
50 | unsigned long use_offset, | ||
51 | unsigned long use_size) | ||
52 | { | ||
53 | int add_idx; | ||
54 | |||
55 | atomic_inc(®d_buf->ref_count); | ||
56 | |||
57 | add_idx = dto->regd_vector_len; | ||
58 | dto->regd[add_idx] = regd_buf; | ||
59 | dto->used_sz[add_idx] = use_size; | ||
60 | dto->offset[add_idx] = use_offset; | ||
61 | |||
62 | dto->regd_vector_len++; | ||
63 | } | ||
64 | |||
65 | /* Register user buffer memory and initialize passive rdma | 42 | /* Register user buffer memory and initialize passive rdma |
66 | * dto descriptor. Total data size is stored in | 43 | * dto descriptor. Total data size is stored in |
67 | * iser_task->data[ISER_DIR_IN].data_len | 44 | * iser_task->data[ISER_DIR_IN].data_len |
@@ -122,9 +99,9 @@ iser_prepare_write_cmd(struct iscsi_task *task, | |||
122 | struct iscsi_iser_task *iser_task = task->dd_data; | 99 | struct iscsi_iser_task *iser_task = task->dd_data; |
123 | struct iser_regd_buf *regd_buf; | 100 | struct iser_regd_buf *regd_buf; |
124 | int err; | 101 | int err; |
125 | struct iser_dto *send_dto = &iser_task->desc.dto; | ||
126 | struct iser_hdr *hdr = &iser_task->desc.iser_header; | 102 | struct iser_hdr *hdr = &iser_task->desc.iser_header; |
127 | struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT]; | 103 | struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT]; |
104 | struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1]; | ||
128 | 105 | ||
129 | err = iser_dma_map_task_data(iser_task, | 106 | err = iser_dma_map_task_data(iser_task, |
130 | buf_out, | 107 | buf_out, |
@@ -163,135 +140,100 @@ iser_prepare_write_cmd(struct iscsi_task *task, | |||
163 | if (imm_sz > 0) { | 140 | if (imm_sz > 0) { |
164 | iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", | 141 | iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", |
165 | task->itt, imm_sz); | 142 | task->itt, imm_sz); |
166 | iser_dto_add_regd_buff(send_dto, | 143 | tx_dsg->addr = regd_buf->reg.va; |
167 | regd_buf, | 144 | tx_dsg->length = imm_sz; |
168 | 0, | 145 | tx_dsg->lkey = regd_buf->reg.lkey; |
169 | imm_sz); | 146 | iser_task->desc.num_sge = 2; |
170 | } | 147 | } |
171 | 148 | ||
172 | return 0; | 149 | return 0; |
173 | } | 150 | } |
174 | 151 | ||
175 | /** | 152 | /* creates a new tx descriptor and adds header regd buffer */ |
176 | * iser_post_receive_control - allocates, initializes and posts receive DTO. | 153 | static void iser_create_send_desc(struct iser_conn *ib_conn, |
177 | */ | 154 | struct iser_tx_desc *tx_desc) |
178 | static int iser_post_receive_control(struct iscsi_conn *conn) | ||
179 | { | 155 | { |
180 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | 156 | struct iser_device *device = ib_conn->device; |
181 | struct iser_desc *rx_desc; | ||
182 | struct iser_regd_buf *regd_hdr; | ||
183 | struct iser_regd_buf *regd_data; | ||
184 | struct iser_dto *recv_dto = NULL; | ||
185 | struct iser_device *device = iser_conn->ib_conn->device; | ||
186 | int rx_data_size, err; | ||
187 | int posts, outstanding_unexp_pdus; | ||
188 | |||
189 | /* for the login sequence we must support rx of upto 8K; login is done | ||
190 | * after conn create/bind (connect) and conn stop/bind (reconnect), | ||
191 | * what's common for both schemes is that the connection is not started | ||
192 | */ | ||
193 | if (conn->c_stage != ISCSI_CONN_STARTED) | ||
194 | rx_data_size = ISCSI_DEF_MAX_RECV_SEG_LEN; | ||
195 | else /* FIXME till user space sets conn->max_recv_dlength correctly */ | ||
196 | rx_data_size = 128; | ||
197 | |||
198 | outstanding_unexp_pdus = | ||
199 | atomic_xchg(&iser_conn->ib_conn->unexpected_pdu_count, 0); | ||
200 | |||
201 | /* | ||
202 | * in addition to the response buffer, replace those consumed by | ||
203 | * unexpected pdus. | ||
204 | */ | ||
205 | for (posts = 0; posts < 1 + outstanding_unexp_pdus; posts++) { | ||
206 | rx_desc = kmem_cache_alloc(ig.desc_cache, GFP_NOIO); | ||
207 | if (rx_desc == NULL) { | ||
208 | iser_err("Failed to alloc desc for post recv %d\n", | ||
209 | posts); | ||
210 | err = -ENOMEM; | ||
211 | goto post_rx_cache_alloc_failure; | ||
212 | } | ||
213 | rx_desc->type = ISCSI_RX; | ||
214 | rx_desc->data = kmalloc(rx_data_size, GFP_NOIO); | ||
215 | if (rx_desc->data == NULL) { | ||
216 | iser_err("Failed to alloc data buf for post recv %d\n", | ||
217 | posts); | ||
218 | err = -ENOMEM; | ||
219 | goto post_rx_kmalloc_failure; | ||
220 | } | ||
221 | |||
222 | recv_dto = &rx_desc->dto; | ||
223 | recv_dto->ib_conn = iser_conn->ib_conn; | ||
224 | recv_dto->regd_vector_len = 0; | ||
225 | 157 | ||
226 | regd_hdr = &rx_desc->hdr_regd_buf; | 158 | ib_dma_sync_single_for_cpu(device->ib_device, |
227 | memset(regd_hdr, 0, sizeof(struct iser_regd_buf)); | 159 | tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); |
228 | regd_hdr->device = device; | ||
229 | regd_hdr->virt_addr = rx_desc; /* == &rx_desc->iser_header */ | ||
230 | regd_hdr->data_size = ISER_TOTAL_HEADERS_LEN; | ||
231 | 160 | ||
232 | iser_reg_single(device, regd_hdr, DMA_FROM_DEVICE); | 161 | memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); |
233 | 162 | tx_desc->iser_header.flags = ISER_VER; | |
234 | iser_dto_add_regd_buff(recv_dto, regd_hdr, 0, 0); | ||
235 | 163 | ||
236 | regd_data = &rx_desc->data_regd_buf; | 164 | tx_desc->num_sge = 1; |
237 | memset(regd_data, 0, sizeof(struct iser_regd_buf)); | ||
238 | regd_data->device = device; | ||
239 | regd_data->virt_addr = rx_desc->data; | ||
240 | regd_data->data_size = rx_data_size; | ||
241 | 165 | ||
242 | iser_reg_single(device, regd_data, DMA_FROM_DEVICE); | 166 | if (tx_desc->tx_sg[0].lkey != device->mr->lkey) { |
167 | tx_desc->tx_sg[0].lkey = device->mr->lkey; | ||
168 | iser_dbg("sdesc %p lkey mismatch, fixing\n", tx_desc); | ||
169 | } | ||
170 | } | ||
243 | 171 | ||
244 | iser_dto_add_regd_buff(recv_dto, regd_data, 0, 0); | ||
245 | 172 | ||
246 | err = iser_post_recv(rx_desc); | 173 | int iser_alloc_rx_descriptors(struct iser_conn *ib_conn) |
247 | if (err) { | 174 | { |
248 | iser_err("Failed iser_post_recv for post %d\n", posts); | 175 | int i, j; |
249 | goto post_rx_post_recv_failure; | 176 | u64 dma_addr; |
250 | } | 177 | struct iser_rx_desc *rx_desc; |
178 | struct ib_sge *rx_sg; | ||
179 | struct iser_device *device = ib_conn->device; | ||
180 | |||
181 | ib_conn->rx_descs = kmalloc(ISER_QP_MAX_RECV_DTOS * | ||
182 | sizeof(struct iser_rx_desc), GFP_KERNEL); | ||
183 | if (!ib_conn->rx_descs) | ||
184 | goto rx_desc_alloc_fail; | ||
185 | |||
186 | rx_desc = ib_conn->rx_descs; | ||
187 | |||
188 | for (i = 0; i < ISER_QP_MAX_RECV_DTOS; i++, rx_desc++) { | ||
189 | dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, | ||
190 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | ||
191 | if (ib_dma_mapping_error(device->ib_device, dma_addr)) | ||
192 | goto rx_desc_dma_map_failed; | ||
193 | |||
194 | rx_desc->dma_addr = dma_addr; | ||
195 | |||
196 | rx_sg = &rx_desc->rx_sg; | ||
197 | rx_sg->addr = rx_desc->dma_addr; | ||
198 | rx_sg->length = ISER_RX_PAYLOAD_SIZE; | ||
199 | rx_sg->lkey = device->mr->lkey; | ||
251 | } | 200 | } |
252 | /* all posts successful */ | ||
253 | return 0; | ||
254 | 201 | ||
255 | post_rx_post_recv_failure: | 202 | ib_conn->rx_desc_head = 0; |
256 | iser_dto_buffs_release(recv_dto); | 203 | return 0; |
257 | kfree(rx_desc->data); | ||
258 | post_rx_kmalloc_failure: | ||
259 | kmem_cache_free(ig.desc_cache, rx_desc); | ||
260 | post_rx_cache_alloc_failure: | ||
261 | if (posts > 0) { | ||
262 | /* | ||
263 | * response buffer posted, but did not replace all unexpected | ||
264 | * pdu recv bufs. Ignore error, retry occurs next send | ||
265 | */ | ||
266 | outstanding_unexp_pdus -= (posts - 1); | ||
267 | err = 0; | ||
268 | } | ||
269 | atomic_add(outstanding_unexp_pdus, | ||
270 | &iser_conn->ib_conn->unexpected_pdu_count); | ||
271 | 204 | ||
272 | return err; | 205 | rx_desc_dma_map_failed: |
206 | rx_desc = ib_conn->rx_descs; | ||
207 | for (j = 0; j < i; j++, rx_desc++) | ||
208 | ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, | ||
209 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | ||
210 | kfree(ib_conn->rx_descs); | ||
211 | ib_conn->rx_descs = NULL; | ||
212 | rx_desc_alloc_fail: | ||
213 | iser_err("failed allocating rx descriptors / data buffers\n"); | ||
214 | return -ENOMEM; | ||
273 | } | 215 | } |
274 | 216 | ||
275 | /* creates a new tx descriptor and adds header regd buffer */ | 217 | void iser_free_rx_descriptors(struct iser_conn *ib_conn) |
276 | static void iser_create_send_desc(struct iscsi_iser_conn *iser_conn, | ||
277 | struct iser_desc *tx_desc) | ||
278 | { | 218 | { |
279 | struct iser_regd_buf *regd_hdr = &tx_desc->hdr_regd_buf; | 219 | int i; |
280 | struct iser_dto *send_dto = &tx_desc->dto; | 220 | struct iser_rx_desc *rx_desc; |
221 | struct iser_device *device = ib_conn->device; | ||
281 | 222 | ||
282 | memset(regd_hdr, 0, sizeof(struct iser_regd_buf)); | 223 | if (ib_conn->login_buf) { |
283 | regd_hdr->device = iser_conn->ib_conn->device; | 224 | ib_dma_unmap_single(device->ib_device, ib_conn->login_dma, |
284 | regd_hdr->virt_addr = tx_desc; /* == &tx_desc->iser_header */ | 225 | ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); |
285 | regd_hdr->data_size = ISER_TOTAL_HEADERS_LEN; | 226 | kfree(ib_conn->login_buf); |
227 | } | ||
286 | 228 | ||
287 | send_dto->ib_conn = iser_conn->ib_conn; | 229 | if (!ib_conn->rx_descs) |
288 | send_dto->notify_enable = 1; | 230 | return; |
289 | send_dto->regd_vector_len = 0; | ||
290 | 231 | ||
291 | memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); | 232 | rx_desc = ib_conn->rx_descs; |
292 | tx_desc->iser_header.flags = ISER_VER; | 233 | for (i = 0; i < ISER_QP_MAX_RECV_DTOS; i++, rx_desc++) |
293 | 234 | ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, | |
294 | iser_dto_add_regd_buff(send_dto, regd_hdr, 0, 0); | 235 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); |
236 | kfree(ib_conn->rx_descs); | ||
295 | } | 237 | } |
296 | 238 | ||
297 | /** | 239 | /** |
@@ -301,46 +243,23 @@ int iser_conn_set_full_featured_mode(struct iscsi_conn *conn) | |||
301 | { | 243 | { |
302 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | 244 | struct iscsi_iser_conn *iser_conn = conn->dd_data; |
303 | 245 | ||
304 | int i; | 246 | iser_dbg("Initially post: %d\n", ISER_MIN_POSTED_RX); |
305 | /* | ||
306 | * FIXME this value should be declared to the target during login with | ||
307 | * the MaxOutstandingUnexpectedPDUs key when supported | ||
308 | */ | ||
309 | int initial_post_recv_bufs_num = ISER_MAX_RX_MISC_PDUS; | ||
310 | |||
311 | iser_dbg("Initially post: %d\n", initial_post_recv_bufs_num); | ||
312 | 247 | ||
313 | /* Check that there is no posted recv or send buffers left - */ | 248 | /* Check that there is no posted recv or send buffers left - */ |
314 | /* they must be consumed during the login phase */ | 249 | /* they must be consumed during the login phase */ |
315 | BUG_ON(atomic_read(&iser_conn->ib_conn->post_recv_buf_count) != 0); | 250 | BUG_ON(iser_conn->ib_conn->post_recv_buf_count != 0); |
316 | BUG_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0); | 251 | BUG_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0); |
317 | 252 | ||
318 | /* Initial post receive buffers */ | 253 | if (iser_alloc_rx_descriptors(iser_conn->ib_conn)) |
319 | for (i = 0; i < initial_post_recv_bufs_num; i++) { | 254 | return -ENOMEM; |
320 | if (iser_post_receive_control(conn) != 0) { | ||
321 | iser_err("Failed to post recv bufs at:%d conn:0x%p\n", | ||
322 | i, conn); | ||
323 | return -ENOMEM; | ||
324 | } | ||
325 | } | ||
326 | iser_dbg("Posted %d post recv bufs, conn:0x%p\n", i, conn); | ||
327 | return 0; | ||
328 | } | ||
329 | 255 | ||
330 | static int | 256 | /* Initial post receive buffers */ |
331 | iser_check_xmit(struct iscsi_conn *conn, void *task) | 257 | if (iser_post_recvm(iser_conn->ib_conn, ISER_MIN_POSTED_RX)) |
332 | { | 258 | return -ENOMEM; |
333 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | ||
334 | 259 | ||
335 | if (atomic_read(&iser_conn->ib_conn->post_send_buf_count) == | ||
336 | ISER_QP_MAX_REQ_DTOS) { | ||
337 | iser_dbg("%ld can't xmit task %p\n",jiffies,task); | ||
338 | return -ENOBUFS; | ||
339 | } | ||
340 | return 0; | 260 | return 0; |
341 | } | 261 | } |
342 | 262 | ||
343 | |||
344 | /** | 263 | /** |
345 | * iser_send_command - send command PDU | 264 | * iser_send_command - send command PDU |
346 | */ | 265 | */ |
@@ -349,27 +268,18 @@ int iser_send_command(struct iscsi_conn *conn, | |||
349 | { | 268 | { |
350 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | 269 | struct iscsi_iser_conn *iser_conn = conn->dd_data; |
351 | struct iscsi_iser_task *iser_task = task->dd_data; | 270 | struct iscsi_iser_task *iser_task = task->dd_data; |
352 | struct iser_dto *send_dto = NULL; | ||
353 | unsigned long edtl; | 271 | unsigned long edtl; |
354 | int err = 0; | 272 | int err; |
355 | struct iser_data_buf *data_buf; | 273 | struct iser_data_buf *data_buf; |
356 | struct iscsi_cmd *hdr = (struct iscsi_cmd *)task->hdr; | 274 | struct iscsi_cmd *hdr = (struct iscsi_cmd *)task->hdr; |
357 | struct scsi_cmnd *sc = task->sc; | 275 | struct scsi_cmnd *sc = task->sc; |
358 | 276 | struct iser_tx_desc *tx_desc = &iser_task->desc; | |
359 | if (!iser_conn_state_comp(iser_conn->ib_conn, ISER_CONN_UP)) { | ||
360 | iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn->ib_conn); | ||
361 | return -EPERM; | ||
362 | } | ||
363 | if (iser_check_xmit(conn, task)) | ||
364 | return -ENOBUFS; | ||
365 | 277 | ||
366 | edtl = ntohl(hdr->data_length); | 278 | edtl = ntohl(hdr->data_length); |
367 | 279 | ||
368 | /* build the tx desc regd header and add it to the tx desc dto */ | 280 | /* build the tx desc regd header and add it to the tx desc dto */ |
369 | iser_task->desc.type = ISCSI_TX_SCSI_COMMAND; | 281 | tx_desc->type = ISCSI_TX_SCSI_COMMAND; |
370 | send_dto = &iser_task->desc.dto; | 282 | iser_create_send_desc(iser_conn->ib_conn, tx_desc); |
371 | send_dto->task = iser_task; | ||
372 | iser_create_send_desc(iser_conn, &iser_task->desc); | ||
373 | 283 | ||
374 | if (hdr->flags & ISCSI_FLAG_CMD_READ) | 284 | if (hdr->flags & ISCSI_FLAG_CMD_READ) |
375 | data_buf = &iser_task->data[ISER_DIR_IN]; | 285 | data_buf = &iser_task->data[ISER_DIR_IN]; |
@@ -398,23 +308,13 @@ int iser_send_command(struct iscsi_conn *conn, | |||
398 | goto send_command_error; | 308 | goto send_command_error; |
399 | } | 309 | } |
400 | 310 | ||
401 | iser_reg_single(iser_conn->ib_conn->device, | ||
402 | send_dto->regd[0], DMA_TO_DEVICE); | ||
403 | |||
404 | if (iser_post_receive_control(conn) != 0) { | ||
405 | iser_err("post_recv failed!\n"); | ||
406 | err = -ENOMEM; | ||
407 | goto send_command_error; | ||
408 | } | ||
409 | |||
410 | iser_task->status = ISER_TASK_STATUS_STARTED; | 311 | iser_task->status = ISER_TASK_STATUS_STARTED; |
411 | 312 | ||
412 | err = iser_post_send(&iser_task->desc); | 313 | err = iser_post_send(iser_conn->ib_conn, tx_desc); |
413 | if (!err) | 314 | if (!err) |
414 | return 0; | 315 | return 0; |
415 | 316 | ||
416 | send_command_error: | 317 | send_command_error: |
417 | iser_dto_buffs_release(send_dto); | ||
418 | iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err); | 318 | iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err); |
419 | return err; | 319 | return err; |
420 | } | 320 | } |
@@ -428,20 +328,13 @@ int iser_send_data_out(struct iscsi_conn *conn, | |||
428 | { | 328 | { |
429 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | 329 | struct iscsi_iser_conn *iser_conn = conn->dd_data; |
430 | struct iscsi_iser_task *iser_task = task->dd_data; | 330 | struct iscsi_iser_task *iser_task = task->dd_data; |
431 | struct iser_desc *tx_desc = NULL; | 331 | struct iser_tx_desc *tx_desc = NULL; |
432 | struct iser_dto *send_dto = NULL; | 332 | struct iser_regd_buf *regd_buf; |
433 | unsigned long buf_offset; | 333 | unsigned long buf_offset; |
434 | unsigned long data_seg_len; | 334 | unsigned long data_seg_len; |
435 | uint32_t itt; | 335 | uint32_t itt; |
436 | int err = 0; | 336 | int err = 0; |
437 | 337 | struct ib_sge *tx_dsg; | |
438 | if (!iser_conn_state_comp(iser_conn->ib_conn, ISER_CONN_UP)) { | ||
439 | iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn->ib_conn); | ||
440 | return -EPERM; | ||
441 | } | ||
442 | |||
443 | if (iser_check_xmit(conn, task)) | ||
444 | return -ENOBUFS; | ||
445 | 338 | ||
446 | itt = (__force uint32_t)hdr->itt; | 339 | itt = (__force uint32_t)hdr->itt; |
447 | data_seg_len = ntoh24(hdr->dlength); | 340 | data_seg_len = ntoh24(hdr->dlength); |
@@ -450,28 +343,25 @@ int iser_send_data_out(struct iscsi_conn *conn, | |||
450 | iser_dbg("%s itt %d dseg_len %d offset %d\n", | 343 | iser_dbg("%s itt %d dseg_len %d offset %d\n", |
451 | __func__,(int)itt,(int)data_seg_len,(int)buf_offset); | 344 | __func__,(int)itt,(int)data_seg_len,(int)buf_offset); |
452 | 345 | ||
453 | tx_desc = kmem_cache_alloc(ig.desc_cache, GFP_NOIO); | 346 | tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC); |
454 | if (tx_desc == NULL) { | 347 | if (tx_desc == NULL) { |
455 | iser_err("Failed to alloc desc for post dataout\n"); | 348 | iser_err("Failed to alloc desc for post dataout\n"); |
456 | return -ENOMEM; | 349 | return -ENOMEM; |
457 | } | 350 | } |
458 | 351 | ||
459 | tx_desc->type = ISCSI_TX_DATAOUT; | 352 | tx_desc->type = ISCSI_TX_DATAOUT; |
353 | tx_desc->iser_header.flags = ISER_VER; | ||
460 | memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); | 354 | memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); |
461 | 355 | ||
462 | /* build the tx desc regd header and add it to the tx desc dto */ | 356 | /* build the tx desc */ |
463 | send_dto = &tx_desc->dto; | 357 | iser_initialize_task_headers(task, tx_desc); |
464 | send_dto->task = iser_task; | ||
465 | iser_create_send_desc(iser_conn, tx_desc); | ||
466 | |||
467 | iser_reg_single(iser_conn->ib_conn->device, | ||
468 | send_dto->regd[0], DMA_TO_DEVICE); | ||
469 | 358 | ||
470 | /* all data was registered for RDMA, we can use the lkey */ | 359 | regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT]; |
471 | iser_dto_add_regd_buff(send_dto, | 360 | tx_dsg = &tx_desc->tx_sg[1]; |
472 | &iser_task->rdma_regd[ISER_DIR_OUT], | 361 | tx_dsg->addr = regd_buf->reg.va + buf_offset; |
473 | buf_offset, | 362 | tx_dsg->length = data_seg_len; |
474 | data_seg_len); | 363 | tx_dsg->lkey = regd_buf->reg.lkey; |
364 | tx_desc->num_sge = 2; | ||
475 | 365 | ||
476 | if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) { | 366 | if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) { |
477 | iser_err("Offset:%ld & DSL:%ld in Data-Out " | 367 | iser_err("Offset:%ld & DSL:%ld in Data-Out " |
@@ -485,12 +375,11 @@ int iser_send_data_out(struct iscsi_conn *conn, | |||
485 | itt, buf_offset, data_seg_len); | 375 | itt, buf_offset, data_seg_len); |
486 | 376 | ||
487 | 377 | ||
488 | err = iser_post_send(tx_desc); | 378 | err = iser_post_send(iser_conn->ib_conn, tx_desc); |
489 | if (!err) | 379 | if (!err) |
490 | return 0; | 380 | return 0; |
491 | 381 | ||
492 | send_data_out_error: | 382 | send_data_out_error: |
493 | iser_dto_buffs_release(send_dto); | ||
494 | kmem_cache_free(ig.desc_cache, tx_desc); | 383 | kmem_cache_free(ig.desc_cache, tx_desc); |
495 | iser_err("conn %p failed err %d\n",conn, err); | 384 | iser_err("conn %p failed err %d\n",conn, err); |
496 | return err; | 385 | return err; |
@@ -501,64 +390,44 @@ int iser_send_control(struct iscsi_conn *conn, | |||
501 | { | 390 | { |
502 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | 391 | struct iscsi_iser_conn *iser_conn = conn->dd_data; |
503 | struct iscsi_iser_task *iser_task = task->dd_data; | 392 | struct iscsi_iser_task *iser_task = task->dd_data; |
504 | struct iser_desc *mdesc = &iser_task->desc; | 393 | struct iser_tx_desc *mdesc = &iser_task->desc; |
505 | struct iser_dto *send_dto = NULL; | ||
506 | unsigned long data_seg_len; | 394 | unsigned long data_seg_len; |
507 | int err = 0; | 395 | int err = 0; |
508 | struct iser_regd_buf *regd_buf; | ||
509 | struct iser_device *device; | 396 | struct iser_device *device; |
510 | unsigned char opcode; | ||
511 | |||
512 | if (!iser_conn_state_comp(iser_conn->ib_conn, ISER_CONN_UP)) { | ||
513 | iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn->ib_conn); | ||
514 | return -EPERM; | ||
515 | } | ||
516 | |||
517 | if (iser_check_xmit(conn, task)) | ||
518 | return -ENOBUFS; | ||
519 | 397 | ||
520 | /* build the tx desc regd header and add it to the tx desc dto */ | 398 | /* build the tx desc regd header and add it to the tx desc dto */ |
521 | mdesc->type = ISCSI_TX_CONTROL; | 399 | mdesc->type = ISCSI_TX_CONTROL; |
522 | send_dto = &mdesc->dto; | 400 | iser_create_send_desc(iser_conn->ib_conn, mdesc); |
523 | send_dto->task = NULL; | ||
524 | iser_create_send_desc(iser_conn, mdesc); | ||
525 | 401 | ||
526 | device = iser_conn->ib_conn->device; | 402 | device = iser_conn->ib_conn->device; |
527 | 403 | ||
528 | iser_reg_single(device, send_dto->regd[0], DMA_TO_DEVICE); | ||
529 | |||
530 | data_seg_len = ntoh24(task->hdr->dlength); | 404 | data_seg_len = ntoh24(task->hdr->dlength); |
531 | 405 | ||
532 | if (data_seg_len > 0) { | 406 | if (data_seg_len > 0) { |
533 | regd_buf = &mdesc->data_regd_buf; | 407 | struct ib_sge *tx_dsg = &mdesc->tx_sg[1]; |
534 | memset(regd_buf, 0, sizeof(struct iser_regd_buf)); | 408 | if (task != conn->login_task) { |
535 | regd_buf->device = device; | 409 | iser_err("data present on non login task!!!\n"); |
536 | regd_buf->virt_addr = task->data; | 410 | goto send_control_error; |
537 | regd_buf->data_size = task->data_count; | 411 | } |
538 | iser_reg_single(device, regd_buf, | 412 | memcpy(iser_conn->ib_conn->login_buf, task->data, |
539 | DMA_TO_DEVICE); | 413 | task->data_count); |
540 | iser_dto_add_regd_buff(send_dto, regd_buf, | 414 | tx_dsg->addr = iser_conn->ib_conn->login_dma; |
541 | 0, | 415 | tx_dsg->length = data_seg_len; |
542 | data_seg_len); | 416 | tx_dsg->lkey = device->mr->lkey; |
417 | mdesc->num_sge = 2; | ||
543 | } | 418 | } |
544 | 419 | ||
545 | opcode = task->hdr->opcode & ISCSI_OPCODE_MASK; | 420 | if (task == conn->login_task) { |
546 | 421 | err = iser_post_recvl(iser_conn->ib_conn); | |
547 | /* post recv buffer for response if one is expected */ | 422 | if (err) |
548 | if (!(opcode == ISCSI_OP_NOOP_OUT && task->hdr->itt == RESERVED_ITT)) { | ||
549 | if (iser_post_receive_control(conn) != 0) { | ||
550 | iser_err("post_rcv_buff failed!\n"); | ||
551 | err = -ENOMEM; | ||
552 | goto send_control_error; | 423 | goto send_control_error; |
553 | } | ||
554 | } | 424 | } |
555 | 425 | ||
556 | err = iser_post_send(mdesc); | 426 | err = iser_post_send(iser_conn->ib_conn, mdesc); |
557 | if (!err) | 427 | if (!err) |
558 | return 0; | 428 | return 0; |
559 | 429 | ||
560 | send_control_error: | 430 | send_control_error: |
561 | iser_dto_buffs_release(send_dto); | ||
562 | iser_err("conn %p failed err %d\n",conn, err); | 431 | iser_err("conn %p failed err %d\n",conn, err); |
563 | return err; | 432 | return err; |
564 | } | 433 | } |
@@ -566,104 +435,71 @@ send_control_error: | |||
566 | /** | 435 | /** |
567 | * iser_rcv_dto_completion - recv DTO completion | 436 | * iser_rcv_dto_completion - recv DTO completion |
568 | */ | 437 | */ |
569 | void iser_rcv_completion(struct iser_desc *rx_desc, | 438 | void iser_rcv_completion(struct iser_rx_desc *rx_desc, |
570 | unsigned long dto_xfer_len) | 439 | unsigned long rx_xfer_len, |
440 | struct iser_conn *ib_conn) | ||
571 | { | 441 | { |
572 | struct iser_dto *dto = &rx_desc->dto; | 442 | struct iscsi_iser_conn *conn = ib_conn->iser_conn; |
573 | struct iscsi_iser_conn *conn = dto->ib_conn->iser_conn; | ||
574 | struct iscsi_task *task; | ||
575 | struct iscsi_iser_task *iser_task; | ||
576 | struct iscsi_hdr *hdr; | 443 | struct iscsi_hdr *hdr; |
577 | char *rx_data = NULL; | 444 | u64 rx_dma; |
578 | int rx_data_len = 0; | 445 | int rx_buflen, outstanding, count, err; |
579 | unsigned char opcode; | 446 | |
580 | 447 | /* differentiate between login to all other PDUs */ | |
581 | hdr = &rx_desc->iscsi_header; | 448 | if ((char *)rx_desc == ib_conn->login_buf) { |
449 | rx_dma = ib_conn->login_dma; | ||
450 | rx_buflen = ISER_RX_LOGIN_SIZE; | ||
451 | } else { | ||
452 | rx_dma = rx_desc->dma_addr; | ||
453 | rx_buflen = ISER_RX_PAYLOAD_SIZE; | ||
454 | } | ||
582 | 455 | ||
583 | iser_dbg("op 0x%x itt 0x%x\n", hdr->opcode,hdr->itt); | 456 | ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma, |
457 | rx_buflen, DMA_FROM_DEVICE); | ||
584 | 458 | ||
585 | if (dto_xfer_len > ISER_TOTAL_HEADERS_LEN) { /* we have data */ | 459 | hdr = &rx_desc->iscsi_header; |
586 | rx_data_len = dto_xfer_len - ISER_TOTAL_HEADERS_LEN; | ||
587 | rx_data = dto->regd[1]->virt_addr; | ||
588 | rx_data += dto->offset[1]; | ||
589 | } | ||
590 | 460 | ||
591 | opcode = hdr->opcode & ISCSI_OPCODE_MASK; | 461 | iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, |
592 | 462 | hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN)); | |
593 | if (opcode == ISCSI_OP_SCSI_CMD_RSP) { | ||
594 | spin_lock(&conn->iscsi_conn->session->lock); | ||
595 | task = iscsi_itt_to_ctask(conn->iscsi_conn, hdr->itt); | ||
596 | if (task) | ||
597 | __iscsi_get_task(task); | ||
598 | spin_unlock(&conn->iscsi_conn->session->lock); | ||
599 | |||
600 | if (!task) | ||
601 | iser_err("itt can't be matched to task!!! " | ||
602 | "conn %p opcode %d itt %d\n", | ||
603 | conn->iscsi_conn, opcode, hdr->itt); | ||
604 | else { | ||
605 | iser_task = task->dd_data; | ||
606 | iser_dbg("itt %d task %p\n",hdr->itt, task); | ||
607 | iser_task->status = ISER_TASK_STATUS_COMPLETED; | ||
608 | iser_task_rdma_finalize(iser_task); | ||
609 | iscsi_put_task(task); | ||
610 | } | ||
611 | } | ||
612 | iser_dto_buffs_release(dto); | ||
613 | 463 | ||
614 | iscsi_iser_recv(conn->iscsi_conn, hdr, rx_data, rx_data_len); | 464 | iscsi_iser_recv(conn->iscsi_conn, hdr, |
465 | rx_desc->data, rx_xfer_len - ISER_HEADERS_LEN); | ||
615 | 466 | ||
616 | kfree(rx_desc->data); | 467 | ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma, |
617 | kmem_cache_free(ig.desc_cache, rx_desc); | 468 | rx_buflen, DMA_FROM_DEVICE); |
618 | 469 | ||
619 | /* decrementing conn->post_recv_buf_count only --after-- freeing the * | 470 | /* decrementing conn->post_recv_buf_count only --after-- freeing the * |
620 | * task eliminates the need to worry on tasks which are completed in * | 471 | * task eliminates the need to worry on tasks which are completed in * |
621 | * parallel to the execution of iser_conn_term. So the code that waits * | 472 | * parallel to the execution of iser_conn_term. So the code that waits * |
622 | * for the posted rx bufs refcount to become zero handles everything */ | 473 | * for the posted rx bufs refcount to become zero handles everything */ |
623 | atomic_dec(&conn->ib_conn->post_recv_buf_count); | 474 | conn->ib_conn->post_recv_buf_count--; |
624 | 475 | ||
625 | /* | 476 | if (rx_dma == ib_conn->login_dma) |
626 | * if an unexpected PDU was received then the recv wr consumed must | 477 | return; |
627 | * be replaced, this is done in the next send of a control-type PDU | 478 | |
628 | */ | 479 | outstanding = ib_conn->post_recv_buf_count; |
629 | if (opcode == ISCSI_OP_NOOP_IN && hdr->itt == RESERVED_ITT) { | 480 | if (outstanding + ISER_MIN_POSTED_RX <= ISER_QP_MAX_RECV_DTOS) { |
630 | /* nop-in with itt = 0xffffffff */ | 481 | count = min(ISER_QP_MAX_RECV_DTOS - outstanding, |
631 | atomic_inc(&conn->ib_conn->unexpected_pdu_count); | 482 | ISER_MIN_POSTED_RX); |
632 | } | 483 | err = iser_post_recvm(ib_conn, count); |
633 | else if (opcode == ISCSI_OP_ASYNC_EVENT) { | 484 | if (err) |
634 | /* asyncronous message */ | 485 | iser_err("posting %d rx bufs err %d\n", count, err); |
635 | atomic_inc(&conn->ib_conn->unexpected_pdu_count); | ||
636 | } | 486 | } |
637 | /* a reject PDU consumes the recv buf posted for the response */ | ||
638 | } | 487 | } |
639 | 488 | ||
640 | void iser_snd_completion(struct iser_desc *tx_desc) | 489 | void iser_snd_completion(struct iser_tx_desc *tx_desc, |
490 | struct iser_conn *ib_conn) | ||
641 | { | 491 | { |
642 | struct iser_dto *dto = &tx_desc->dto; | ||
643 | struct iser_conn *ib_conn = dto->ib_conn; | ||
644 | struct iscsi_iser_conn *iser_conn = ib_conn->iser_conn; | ||
645 | struct iscsi_conn *conn = iser_conn->iscsi_conn; | ||
646 | struct iscsi_task *task; | 492 | struct iscsi_task *task; |
647 | int resume_tx = 0; | 493 | struct iser_device *device = ib_conn->device; |
648 | |||
649 | iser_dbg("Initiator, Data sent dto=0x%p\n", dto); | ||
650 | |||
651 | iser_dto_buffs_release(dto); | ||
652 | 494 | ||
653 | if (tx_desc->type == ISCSI_TX_DATAOUT) | 495 | if (tx_desc->type == ISCSI_TX_DATAOUT) { |
496 | ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr, | ||
497 | ISER_HEADERS_LEN, DMA_TO_DEVICE); | ||
654 | kmem_cache_free(ig.desc_cache, tx_desc); | 498 | kmem_cache_free(ig.desc_cache, tx_desc); |
655 | 499 | } | |
656 | if (atomic_read(&iser_conn->ib_conn->post_send_buf_count) == | ||
657 | ISER_QP_MAX_REQ_DTOS) | ||
658 | resume_tx = 1; | ||
659 | 500 | ||
660 | atomic_dec(&ib_conn->post_send_buf_count); | 501 | atomic_dec(&ib_conn->post_send_buf_count); |
661 | 502 | ||
662 | if (resume_tx) { | ||
663 | iser_dbg("%ld resuming tx\n",jiffies); | ||
664 | iscsi_conn_queue_work(conn); | ||
665 | } | ||
666 | |||
667 | if (tx_desc->type == ISCSI_TX_CONTROL) { | 503 | if (tx_desc->type == ISCSI_TX_CONTROL) { |
668 | /* this arithmetic is legal by libiscsi dd_data allocation */ | 504 | /* this arithmetic is legal by libiscsi dd_data allocation */ |
669 | task = (void *) ((long)(void *)tx_desc - | 505 | task = (void *) ((long)(void *)tx_desc - |
@@ -692,7 +528,6 @@ void iser_task_rdma_init(struct iscsi_iser_task *iser_task) | |||
692 | 528 | ||
693 | void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) | 529 | void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) |
694 | { | 530 | { |
695 | int deferred; | ||
696 | int is_rdma_aligned = 1; | 531 | int is_rdma_aligned = 1; |
697 | struct iser_regd_buf *regd; | 532 | struct iser_regd_buf *regd; |
698 | 533 | ||
@@ -710,32 +545,17 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) | |||
710 | 545 | ||
711 | if (iser_task->dir[ISER_DIR_IN]) { | 546 | if (iser_task->dir[ISER_DIR_IN]) { |
712 | regd = &iser_task->rdma_regd[ISER_DIR_IN]; | 547 | regd = &iser_task->rdma_regd[ISER_DIR_IN]; |
713 | deferred = iser_regd_buff_release(regd); | 548 | if (regd->reg.is_fmr) |
714 | if (deferred) { | 549 | iser_unreg_mem(®d->reg); |
715 | iser_err("%d references remain for BUF-IN rdma reg\n", | ||
716 | atomic_read(®d->ref_count)); | ||
717 | } | ||
718 | } | 550 | } |
719 | 551 | ||
720 | if (iser_task->dir[ISER_DIR_OUT]) { | 552 | if (iser_task->dir[ISER_DIR_OUT]) { |
721 | regd = &iser_task->rdma_regd[ISER_DIR_OUT]; | 553 | regd = &iser_task->rdma_regd[ISER_DIR_OUT]; |
722 | deferred = iser_regd_buff_release(regd); | 554 | if (regd->reg.is_fmr) |
723 | if (deferred) { | 555 | iser_unreg_mem(®d->reg); |
724 | iser_err("%d references remain for BUF-OUT rdma reg\n", | ||
725 | atomic_read(®d->ref_count)); | ||
726 | } | ||
727 | } | 556 | } |
728 | 557 | ||
729 | /* if the data was unaligned, it was already unmapped and then copied */ | 558 | /* if the data was unaligned, it was already unmapped and then copied */ |
730 | if (is_rdma_aligned) | 559 | if (is_rdma_aligned) |
731 | iser_dma_unmap_task_data(iser_task); | 560 | iser_dma_unmap_task_data(iser_task); |
732 | } | 561 | } |
733 | |||
734 | void iser_dto_buffs_release(struct iser_dto *dto) | ||
735 | { | ||
736 | int i; | ||
737 | |||
738 | for (i = 0; i < dto->regd_vector_len; i++) | ||
739 | iser_regd_buff_release(dto->regd[i]); | ||
740 | } | ||
741 | |||