aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi')
-rw-r--r--drivers/scsi/iscsi_tcp.c3627
1 files changed, 3627 insertions, 0 deletions
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
new file mode 100644
index 000000000000..810e5e59658f
--- /dev/null
+++ b/drivers/scsi/iscsi_tcp.c
@@ -0,0 +1,3627 @@
1/*
2 * iSCSI Initiator over TCP/IP Data-Path
3 *
4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 Mike Christie
7 * maintained by open-iscsi@googlegroups.com
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published
11 * by the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
18 *
19 * See the file COPYING included with this distribution for more details.
20 *
21 * Credits:
22 * Christoph Hellwig
23 * FUJITA Tomonori
24 * Arne Redlich
25 * Zhenyu Wang
26 */
27
28#include <linux/types.h>
29#include <linux/list.h>
30#include <linux/inet.h>
31#include <linux/blkdev.h>
32#include <linux/crypto.h>
33#include <linux/delay.h>
34#include <linux/kfifo.h>
35#include <linux/scatterlist.h>
36#include <net/tcp.h>
37#include <scsi/scsi_cmnd.h>
38#include <scsi/scsi_device.h>
39#include <scsi/scsi_eh.h>
40#include <scsi/scsi_request.h>
41#include <scsi/scsi_tcq.h>
42#include <scsi/scsi_host.h>
43#include <scsi/scsi.h>
44#include <scsi/scsi_transport_iscsi.h>
45
46#include "iscsi_tcp.h"
47
48MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
49 "Alex Aizman <itn780@yahoo.com>");
50MODULE_DESCRIPTION("iSCSI/TCP data-path");
51MODULE_LICENSE("GPL");
52
53/* #define DEBUG_TCP */
54/* #define DEBUG_SCSI */
55#define DEBUG_ASSERT
56
57#ifdef DEBUG_TCP
58#define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
59#else
60#define debug_tcp(fmt...)
61#endif
62
63#ifdef DEBUG_SCSI
64#define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
65#else
66#define debug_scsi(fmt...)
67#endif
68
69#ifndef DEBUG_ASSERT
70#ifdef BUG_ON
71#undef BUG_ON
72#endif
73#define BUG_ON(expr)
74#endif
75
76#define INVALID_SN_DELTA 0xffff
77
78static unsigned int iscsi_max_lun = 512;
79module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
80
81/* global data */
82static kmem_cache_t *taskcache;
83
84static inline void
85iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
86{
87 sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
88 ibuf->sent = 0;
89}
90
91static inline void
92iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
93{
94 ibuf->sg.page = (void*)vbuf;
95 ibuf->sg.offset = (unsigned int)-1;
96 ibuf->sg.length = size;
97 ibuf->sent = 0;
98}
99
100static inline void*
101iscsi_buf_iov_base(struct iscsi_buf *ibuf)
102{
103 return (char*)ibuf->sg.page + ibuf->sent;
104}
105
106static inline void
107iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
108{
109 /*
110 * Fastpath: sg element fits into single page
111 */
112 if (sg->length + sg->offset <= PAGE_SIZE && page_count(sg->page) >= 2) {
113 ibuf->sg.page = sg->page;
114 ibuf->sg.offset = sg->offset;
115 ibuf->sg.length = sg->length;
116 } else
117 iscsi_buf_init_iov(ibuf, page_address(sg->page), sg->length);
118 ibuf->sent = 0;
119}
120
121static inline int
122iscsi_buf_left(struct iscsi_buf *ibuf)
123{
124 int rc;
125
126 rc = ibuf->sg.length - ibuf->sent;
127 BUG_ON(rc < 0);
128 return rc;
129}
130
131static inline void
132iscsi_buf_init_hdr(struct iscsi_conn *conn, struct iscsi_buf *ibuf,
133 char *vbuf, u8 *crc)
134{
135 iscsi_buf_init_virt(ibuf, vbuf, sizeof(struct iscsi_hdr));
136 if (conn->hdrdgst_en) {
137 crypto_digest_digest(conn->tx_tfm, &ibuf->sg, 1, crc);
138 ibuf->sg.length += sizeof(uint32_t);
139 }
140}
141
142static void
143iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
144{
145 struct iscsi_session *session = conn->session;
146 unsigned long flags;
147
148 spin_lock_irqsave(&session->lock, flags);
149 if (session->conn_cnt == 1 || session->leadconn == conn)
150 session->state = ISCSI_STATE_FAILED;
151 spin_unlock_irqrestore(&session->lock, flags);
152 set_bit(SUSPEND_BIT, &conn->suspend_tx);
153 set_bit(SUSPEND_BIT, &conn->suspend_rx);
154 iscsi_conn_error(iscsi_handle(conn), err);
155}
156
157static inline int
158iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
159{
160 uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
161 uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
162
163 if (max_cmdsn < exp_cmdsn -1 &&
164 max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
165 return ISCSI_ERR_MAX_CMDSN;
166 if (max_cmdsn > session->max_cmdsn ||
167 max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
168 session->max_cmdsn = max_cmdsn;
169 if (exp_cmdsn > session->exp_cmdsn ||
170 exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
171 session->exp_cmdsn = exp_cmdsn;
172
173 return 0;
174}
175
176static inline int
177iscsi_hdr_extract(struct iscsi_conn *conn)
178{
179 struct sk_buff *skb = conn->in.skb;
180
181 if (conn->in.copy >= conn->hdr_size &&
182 conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
183 /*
184 * Zero-copy PDU Header: using connection context
185 * to store header pointer.
186 */
187 if (skb_shinfo(skb)->frag_list == NULL &&
188 !skb_shinfo(skb)->nr_frags)
189 conn->in.hdr = (struct iscsi_hdr *)
190 ((char*)skb->data + conn->in.offset);
191 else {
192 /* ignoring return code since we checked
193 * in.copy before */
194 skb_copy_bits(skb, conn->in.offset,
195 &conn->hdr, conn->hdr_size);
196 conn->in.hdr = &conn->hdr;
197 }
198 conn->in.offset += conn->hdr_size;
199 conn->in.copy -= conn->hdr_size;
200 } else {
201 int hdr_remains;
202 int copylen;
203
204 /*
205 * PDU header scattered across SKB's,
206 * copying it... This'll happen quite rarely.
207 */
208
209 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
210 conn->in.hdr_offset = 0;
211
212 hdr_remains = conn->hdr_size - conn->in.hdr_offset;
213 BUG_ON(hdr_remains <= 0);
214
215 copylen = min(conn->in.copy, hdr_remains);
216 skb_copy_bits(skb, conn->in.offset,
217 (char*)&conn->hdr + conn->in.hdr_offset, copylen);
218
219 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
220 "in.copy %d\n", conn->in.hdr_offset, copylen,
221 conn->in.offset, conn->in.copy);
222
223 conn->in.offset += copylen;
224 conn->in.copy -= copylen;
225 if (copylen < hdr_remains) {
226 conn->in_progress = IN_PROGRESS_HEADER_GATHER;
227 conn->in.hdr_offset += copylen;
228 return -EAGAIN;
229 }
230 conn->in.hdr = &conn->hdr;
231 conn->discontiguous_hdr_cnt++;
232 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
233 }
234
235 return 0;
236}
237
238static inline void
239iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
240{
241 struct scsi_cmnd *sc = ctask->sc;
242 struct iscsi_session *session = conn->session;
243
244 spin_lock(&session->lock);
245 if (unlikely(!sc)) {
246 spin_unlock(&session->lock);
247 return;
248 }
249 if (sc->sc_data_direction == DMA_TO_DEVICE) {
250 struct iscsi_data_task *dtask, *n;
251 /* WRITE: cleanup Data-Out's if any */
252 spin_lock(&conn->lock);
253 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
254 list_del(&dtask->item);
255 mempool_free(dtask, ctask->datapool);
256 }
257 spin_unlock(&conn->lock);
258 }
259 ctask->xmstate = XMSTATE_IDLE;
260 ctask->r2t = NULL;
261 ctask->sc = NULL;
262 __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
263 spin_unlock(&session->lock);
264}
265
266/**
267 * iscsi_cmd_rsp - SCSI Command Response processing
268 * @conn: iscsi connection
269 * @ctask: scsi command task
270 **/
271static int
272iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
273{
274 int rc;
275 struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
276 struct iscsi_session *session = conn->session;
277 struct scsi_cmnd *sc = ctask->sc;
278
279 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
280 if (rc) {
281 sc->result = (DID_ERROR << 16);
282 goto out;
283 }
284
285 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
286
287 sc->result = (DID_OK << 16) | rhdr->cmd_status;
288
289 if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
290 sc->result = (DID_ERROR << 16);
291 goto out;
292 }
293
294 if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
295 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
296
297 memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
298 debug_scsi("copied %d bytes of sense\n", sensecopy);
299 }
300
301 if (sc->sc_data_direction == DMA_TO_DEVICE)
302 goto out;
303
304 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
305 int res_count = be32_to_cpu(rhdr->residual_count);
306
307 if (res_count > 0 && res_count <= sc->request_bufflen)
308 sc->resid = res_count;
309 else
310 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
311 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
312 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
313 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
314 sc->resid = be32_to_cpu(rhdr->residual_count);
315
316out:
317 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
318 (long)sc, sc->result, ctask->itt);
319 conn->scsirsp_pdus_cnt++;
320 iscsi_ctask_cleanup(conn, ctask);
321 sc->scsi_done(sc);
322 return rc;
323}
324
325/**
326 * iscsi_data_rsp - SCSI Data-In Response processing
327 * @conn: iscsi connection
328 * @ctask: scsi command task
329 **/
330static int
331iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
332{
333 int rc;
334 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
335 struct iscsi_session *session = conn->session;
336 int datasn = be32_to_cpu(rhdr->datasn);
337
338 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
339 if (rc)
340 return rc;
341 /*
342 * setup Data-In byte counter (gets decremented..)
343 */
344 ctask->data_count = conn->in.datalen;
345
346 if (conn->in.datalen == 0)
347 return 0;
348
349 if (ctask->datasn != datasn)
350 return ISCSI_ERR_DATASN;
351
352 ctask->datasn++;
353
354 ctask->data_offset = be32_to_cpu(rhdr->offset);
355 if (ctask->data_offset + conn->in.datalen > ctask->total_length)
356 return ISCSI_ERR_DATA_OFFSET;
357
358 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
359 struct scsi_cmnd *sc = ctask->sc;
360
361 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
362 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
363 int res_count = be32_to_cpu(rhdr->residual_count);
364
365 if (res_count > 0 &&
366 res_count <= sc->request_bufflen) {
367 sc->resid = res_count;
368 sc->result = (DID_OK << 16) | rhdr->cmd_status;
369 } else
370 sc->result = (DID_BAD_TARGET << 16) |
371 rhdr->cmd_status;
372 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
373 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
374 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW) {
375 sc->resid = be32_to_cpu(rhdr->residual_count);
376 sc->result = (DID_OK << 16) | rhdr->cmd_status;
377 } else
378 sc->result = (DID_OK << 16) | rhdr->cmd_status;
379 }
380
381 conn->datain_pdus_cnt++;
382 return 0;
383}
384
385/**
386 * iscsi_solicit_data_init - initialize first Data-Out
387 * @conn: iscsi connection
388 * @ctask: scsi command task
389 * @r2t: R2T info
390 *
391 * Notes:
392 * Initialize first Data-Out within this R2T sequence and finds
393 * proper data_offset within this SCSI command.
394 *
395 * This function is called with connection lock taken.
396 **/
397static void
398iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
399 struct iscsi_r2t_info *r2t)
400{
401 struct iscsi_data *hdr;
402 struct iscsi_data_task *dtask;
403 struct scsi_cmnd *sc = ctask->sc;
404
405 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
406 BUG_ON(!dtask);
407 hdr = &dtask->hdr;
408 memset(hdr, 0, sizeof(struct iscsi_data));
409 hdr->ttt = r2t->ttt;
410 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
411 r2t->solicit_datasn++;
412 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
413 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
414 hdr->itt = ctask->hdr.itt;
415 hdr->exp_statsn = r2t->exp_statsn;
416 hdr->offset = cpu_to_be32(r2t->data_offset);
417 if (r2t->data_length > conn->max_xmit_dlength) {
418 hton24(hdr->dlength, conn->max_xmit_dlength);
419 r2t->data_count = conn->max_xmit_dlength;
420 hdr->flags = 0;
421 } else {
422 hton24(hdr->dlength, r2t->data_length);
423 r2t->data_count = r2t->data_length;
424 hdr->flags = ISCSI_FLAG_CMD_FINAL;
425 }
426 conn->dataout_pdus_cnt++;
427
428 r2t->sent = 0;
429
430 iscsi_buf_init_hdr(conn, &r2t->headbuf, (char*)hdr,
431 (u8 *)dtask->hdrext);
432
433 r2t->dtask = dtask;
434
435 if (sc->use_sg) {
436 int i, sg_count = 0;
437 struct scatterlist *sg = sc->request_buffer;
438
439 r2t->sg = NULL;
440 for (i = 0; i < sc->use_sg; i++, sg += 1) {
441 /* FIXME: prefetch ? */
442 if (sg_count + sg->length > r2t->data_offset) {
443 int page_offset;
444
445 /* sg page found! */
446
447 /* offset within this page */
448 page_offset = r2t->data_offset - sg_count;
449
450 /* fill in this buffer */
451 iscsi_buf_init_sg(&r2t->sendbuf, sg);
452 r2t->sendbuf.sg.offset += page_offset;
453 r2t->sendbuf.sg.length -= page_offset;
454
455 /* xmit logic will continue with next one */
456 r2t->sg = sg + 1;
457 break;
458 }
459 sg_count += sg->length;
460 }
461 BUG_ON(r2t->sg == NULL);
462 } else
463 iscsi_buf_init_iov(&ctask->sendbuf,
464 (char*)sc->request_buffer + r2t->data_offset,
465 r2t->data_count);
466
467 list_add(&dtask->item, &ctask->dataqueue);
468}
469
470/**
471 * iscsi_r2t_rsp - iSCSI R2T Response processing
472 * @conn: iscsi connection
473 * @ctask: scsi command task
474 **/
475static int
476iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
477{
478 struct iscsi_r2t_info *r2t;
479 struct iscsi_session *session = conn->session;
480 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
481 int r2tsn = be32_to_cpu(rhdr->r2tsn);
482 int rc;
483
484 if (conn->in.ahslen)
485 return ISCSI_ERR_AHSLEN;
486
487 if (conn->in.datalen)
488 return ISCSI_ERR_DATALEN;
489
490 if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
491 return ISCSI_ERR_R2TSN;
492
493 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
494 if (rc)
495 return rc;
496
497 /* FIXME: use R2TSN to detect missing R2T */
498
499 /* fill-in new R2T associated with the task */
500 spin_lock(&session->lock);
501 if (!ctask->sc || ctask->mtask ||
502 session->state != ISCSI_STATE_LOGGED_IN) {
503 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
504 "recovery...\n", ctask->itt);
505 spin_unlock(&session->lock);
506 return 0;
507 }
508 rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
509 BUG_ON(!rc);
510
511 r2t->exp_statsn = rhdr->statsn;
512 r2t->data_length = be32_to_cpu(rhdr->data_length);
513 if (r2t->data_length == 0 ||
514 r2t->data_length > session->max_burst) {
515 spin_unlock(&session->lock);
516 return ISCSI_ERR_DATALEN;
517 }
518
519 r2t->data_offset = be32_to_cpu(rhdr->data_offset);
520 if (r2t->data_offset + r2t->data_length > ctask->total_length) {
521 spin_unlock(&session->lock);
522 return ISCSI_ERR_DATALEN;
523 }
524
525 r2t->ttt = rhdr->ttt; /* no flip */
526 r2t->solicit_datasn = 0;
527
528 iscsi_solicit_data_init(conn, ctask, r2t);
529
530 ctask->exp_r2tsn = r2tsn + 1;
531 ctask->xmstate |= XMSTATE_SOL_HDR;
532 __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
533 __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));
534
535 schedule_work(&conn->xmitwork);
536 conn->r2t_pdus_cnt++;
537 spin_unlock(&session->lock);
538
539 return 0;
540}
541
542static int
543iscsi_hdr_recv(struct iscsi_conn *conn)
544{
545 int rc = 0;
546 struct iscsi_hdr *hdr;
547 struct iscsi_cmd_task *ctask;
548 struct iscsi_session *session = conn->session;
549 uint32_t cdgst, rdgst = 0;
550
551 hdr = conn->in.hdr;
552
553 /* verify PDU length */
554 conn->in.datalen = ntoh24(hdr->dlength);
555 if (conn->in.datalen > conn->max_recv_dlength) {
556 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
557 conn->in.datalen, conn->max_recv_dlength);
558 return ISCSI_ERR_DATALEN;
559 }
560 conn->data_copied = 0;
561
562 /* read AHS */
563 conn->in.ahslen = hdr->hlength*(4*sizeof(__u16));
564 conn->in.offset += conn->in.ahslen;
565 conn->in.copy -= conn->in.ahslen;
566 if (conn->in.copy < 0) {
567 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
568 "%d bytes\n", conn->in.ahslen);
569 return ISCSI_ERR_AHSLEN;
570 }
571
572 /* calculate read padding */
573 conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
574 if (conn->in.padding) {
575 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
576 debug_scsi("read padding %d bytes\n", conn->in.padding);
577 }
578
579 if (conn->hdrdgst_en) {
580 struct scatterlist sg;
581
582 sg_init_one(&sg, (u8 *)hdr,
583 sizeof(struct iscsi_hdr) + conn->in.ahslen);
584 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
585 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
586 conn->in.ahslen);
587 }
588
589 /* save opcode for later */
590 conn->in.opcode = hdr->opcode;
591
592 /* verify itt (itt encoding: age+cid+itt) */
593 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
594 if ((hdr->itt & AGE_MASK) !=
595 (session->age << AGE_SHIFT)) {
596 printk(KERN_ERR "iscsi_tcp: received itt %x expected "
597 "session age (%x)\n", hdr->itt,
598 session->age & AGE_MASK);
599 return ISCSI_ERR_BAD_ITT;
600 }
601
602 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
603 printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
604 "CID (%x)\n", hdr->itt, conn->id);
605 return ISCSI_ERR_BAD_ITT;
606 }
607 conn->in.itt = hdr->itt & ITT_MASK;
608 } else
609 conn->in.itt = hdr->itt;
610
611 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
612 hdr->opcode, conn->in.offset, conn->in.copy,
613 conn->in.ahslen, conn->in.datalen);
614
615 if (conn->in.itt < session->cmds_max) {
616 if (conn->hdrdgst_en && cdgst != rdgst) {
617 printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
618 "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
619 cdgst);
620 return ISCSI_ERR_HDR_DGST;
621 }
622
623 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
624
625 if (!ctask->sc) {
626 printk(KERN_INFO "iscsi_tcp: dropping ctask with "
627 "itt 0x%x\n", ctask->itt);
628 conn->in.datalen = 0; /* force drop */
629 return 0;
630 }
631
632 if (ctask->sc->SCp.phase != session->age) {
633 printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
634 "expected %d\n", ctask->sc->SCp.phase,
635 session->age);
636 return ISCSI_ERR_SESSION_FAILED;
637 }
638
639 conn->in.ctask = ctask;
640
641 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
642 hdr->opcode, conn->id, (long)ctask->sc,
643 ctask->itt, conn->in.datalen);
644
645 switch(conn->in.opcode) {
646 case ISCSI_OP_SCSI_CMD_RSP:
647 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
648 if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE)
649 rc = iscsi_cmd_rsp(conn, ctask);
650 else if (!conn->in.datalen)
651 rc = iscsi_cmd_rsp(conn, ctask);
652 else
653 /*
654 * got sense or response data; copying PDU
655 * Header to the connection's header
656 * placeholder
657 */
658 memcpy(&conn->hdr, hdr,
659 sizeof(struct iscsi_hdr));
660 break;
661 case ISCSI_OP_SCSI_DATA_IN:
662 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
663 /* save flags for non-exceptional status */
664 conn->in.flags = hdr->flags;
665 /* save cmd_status for sense data */
666 conn->in.cmd_status =
667 ((struct iscsi_data_rsp*)hdr)->cmd_status;
668 rc = iscsi_data_rsp(conn, ctask);
669 break;
670 case ISCSI_OP_R2T:
671 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
672 if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE &&
673 ctask->sc->sc_data_direction == DMA_TO_DEVICE)
674 rc = iscsi_r2t_rsp(conn, ctask);
675 else
676 rc = ISCSI_ERR_PROTO;
677 break;
678 case ISCSI_OP_NOOP_IN:
679 case ISCSI_OP_TEXT_RSP:
680 case ISCSI_OP_LOGOUT_RSP:
681 case ISCSI_OP_ASYNC_EVENT:
682 case ISCSI_OP_REJECT:
683 rc = iscsi_check_assign_cmdsn(session,
684 (struct iscsi_nopin*)hdr);
685 if (rc)
686 break;
687
688 /* update ExpStatSN */
689 conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
690 if (!conn->in.datalen) {
691 struct iscsi_mgmt_task *mtask;
692
693 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
694 NULL, 0);
695 mtask = (struct iscsi_mgmt_task *)
696 session->mgmt_cmds[conn->in.itt -
697 ISCSI_MGMT_ITT_OFFSET];
698 if (conn->login_mtask != mtask) {
699 spin_lock(&session->lock);
700 __kfifo_put(session->mgmtpool.queue,
701 (void*)&mtask, sizeof(void*));
702 spin_unlock(&session->lock);
703 }
704 }
705 break;
706 default:
707 rc = ISCSI_ERR_BAD_OPCODE;
708 break;
709 }
710 } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
711 conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
712 session->mgmtpool_max) {
713 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
714 session->mgmt_cmds[conn->in.itt -
715 ISCSI_MGMT_ITT_OFFSET];
716
717 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
718 conn->in.opcode, conn->id, mtask->itt,
719 conn->in.datalen);
720
721 switch(conn->in.opcode) {
722 case ISCSI_OP_LOGIN_RSP:
723 case ISCSI_OP_TEXT_RSP:
724 rc = iscsi_check_assign_cmdsn(session,
725 (struct iscsi_nopin*)hdr);
726 if (rc)
727 break;
728
729 if (!conn->in.datalen) {
730 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
731 NULL, 0);
732 if (conn->login_mtask != mtask) {
733 spin_lock(&session->lock);
734 __kfifo_put(session->mgmtpool.queue,
735 (void*)&mtask, sizeof(void*));
736 spin_unlock(&session->lock);
737 }
738 }
739 break;
740 case ISCSI_OP_SCSI_TMFUNC_RSP:
741 rc = iscsi_check_assign_cmdsn(session,
742 (struct iscsi_nopin*)hdr);
743 if (rc)
744 break;
745
746 if (conn->in.datalen || conn->in.ahslen) {
747 rc = ISCSI_ERR_PROTO;
748 break;
749 }
750 conn->tmfrsp_pdus_cnt++;
751 spin_lock(&session->lock);
752 if (conn->tmabort_state == TMABORT_INITIAL) {
753 __kfifo_put(session->mgmtpool.queue,
754 (void*)&mtask, sizeof(void*));
755 conn->tmabort_state =
756 ((struct iscsi_tm_rsp *)hdr)->
757 response == SCSI_TCP_TM_RESP_COMPLETE ?
758 TMABORT_SUCCESS:TMABORT_FAILED;
759 /* unblock eh_abort() */
760 wake_up(&conn->ehwait);
761 }
762 spin_unlock(&session->lock);
763 break;
764 default:
765 rc = ISCSI_ERR_BAD_OPCODE;
766 break;
767 }
768 } else if (conn->in.itt == ISCSI_RESERVED_TAG) {
769 if (conn->in.opcode == ISCSI_OP_NOOP_IN && !conn->in.datalen) {
770 rc = iscsi_check_assign_cmdsn(session,
771 (struct iscsi_nopin*)hdr);
772 if (!rc)
773 rc = iscsi_recv_pdu(iscsi_handle(conn),
774 hdr, NULL, 0);
775 }
776 else
777 rc = ISCSI_ERR_BAD_OPCODE;
778 } else
779 rc = ISCSI_ERR_BAD_ITT;
780
781 return rc;
782}
783
784/**
785 * iscsi_ctask_copy - copy skb bits to the destanation cmd task
786 * @conn: iscsi connection
787 * @ctask: scsi command task
788 * @buf: buffer to copy to
789 * @buf_size: size of buffer
790 * @offset: offset within the buffer
791 *
792 * Notes:
793 * The function calls skb_copy_bits() and updates per-connection and
794 * per-cmd byte counters.
795 *
796 * Read counters (in bytes):
797 *
798 * conn->in.offset offset within in progress SKB
799 * conn->in.copy left to copy from in progress SKB
800 * including padding
801 * conn->in.copied copied already from in progress SKB
802 * conn->data_copied copied already from in progress buffer
803 * ctask->sent total bytes sent up to the MidLayer
804 * ctask->data_count left to copy from in progress Data-In
805 * buf_left left to copy from in progress buffer
806 **/
807static inline int
808iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
809 void *buf, int buf_size, int offset)
810{
811 int buf_left = buf_size - (conn->data_copied + offset);
812 int size = min(conn->in.copy, buf_left);
813 int rc;
814
815 size = min(size, ctask->data_count);
816
817 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
818 size, conn->in.offset, conn->in.copied);
819
820 BUG_ON(size <= 0);
821 BUG_ON(ctask->sent + size > ctask->total_length);
822
823 rc = skb_copy_bits(conn->in.skb, conn->in.offset,
824 (char*)buf + (offset + conn->data_copied), size);
825 /* must fit into skb->len */
826 BUG_ON(rc);
827
828 conn->in.offset += size;
829 conn->in.copy -= size;
830 conn->in.copied += size;
831 conn->data_copied += size;
832 ctask->sent += size;
833 ctask->data_count -= size;
834
835 BUG_ON(conn->in.copy < 0);
836 BUG_ON(ctask->data_count < 0);
837
838 if (buf_size != (conn->data_copied + offset)) {
839 if (!ctask->data_count) {
840 BUG_ON(buf_size - conn->data_copied < 0);
841 /* done with this PDU */
842 return buf_size - conn->data_copied;
843 }
844 return -EAGAIN;
845 }
846
847 /* done with this buffer or with both - PDU and buffer */
848 conn->data_copied = 0;
849 return 0;
850}
851
852/**
853 * iscsi_tcp_copy - copy skb bits to the destanation buffer
854 * @conn: iscsi connection
855 * @buf: buffer to copy to
856 * @buf_size: number of bytes to copy
857 *
858 * Notes:
859 * The function calls skb_copy_bits() and updates per-connection
860 * byte counters.
861 **/
862static inline int
863iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
864{
865 int buf_left = buf_size - conn->data_copied;
866 int size = min(conn->in.copy, buf_left);
867 int rc;
868
869 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
870 size, conn->in.offset, conn->data_copied);
871 BUG_ON(size <= 0);
872
873 rc = skb_copy_bits(conn->in.skb, conn->in.offset,
874 (char*)buf + conn->data_copied, size);
875 BUG_ON(rc);
876
877 conn->in.offset += size;
878 conn->in.copy -= size;
879 conn->in.copied += size;
880 conn->data_copied += size;
881
882 if (buf_size != conn->data_copied)
883 return -EAGAIN;
884
885 return 0;
886}
887
888static inline void
889partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
890 int offset, int length)
891{
892 struct scatterlist temp;
893
894 memcpy(&temp, sg, sizeof(struct scatterlist));
895 temp.offset = offset;
896 temp.length = length;
897 crypto_digest_update(conn->data_rx_tfm, &temp, 1);
898}
899
900static int iscsi_scsi_data_in(struct iscsi_conn *conn)
901{
902 struct iscsi_cmd_task *ctask = conn->in.ctask;
903 struct scsi_cmnd *sc = ctask->sc;
904 struct scatterlist tmp, *sg;
905 int i, offset, rc = 0;
906
907 BUG_ON((void*)ctask != sc->SCp.ptr);
908
909 /*
910 * copying Data-In into the Scsi_Cmnd
911 */
912 if (!sc->use_sg) {
913 i = ctask->data_count;
914 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
915 sc->request_bufflen, ctask->data_offset);
916 if (rc == -EAGAIN)
917 return rc;
918 if (conn->datadgst_en) {
919 sg_init_one(&tmp, sc->request_buffer, i);
920 crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
921 }
922 rc = 0;
923 goto done;
924 }
925
926 offset = ctask->data_offset;
927 sg = sc->request_buffer;
928
929 if (ctask->data_offset)
930 for (i = 0; i < ctask->sg_count; i++)
931 offset -= sg[i].length;
932 /* we've passed through partial sg*/
933 if (offset < 0)
934 offset = 0;
935
936 for (i = ctask->sg_count; i < sc->use_sg; i++) {
937 char *dest;
938
939 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
940 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
941 sg[i].length, offset);
942 kunmap_atomic(dest, KM_SOFTIRQ0);
943 if (rc == -EAGAIN)
944 /* continue with the next SKB/PDU */
945 return rc;
946 if (!rc) {
947 if (conn->datadgst_en) {
948 if (!offset)
949 crypto_digest_update(conn->data_rx_tfm,
950 &sg[i], 1);
951 else
952 partial_sg_digest_update(conn, &sg[i],
953 sg[i].offset + offset,
954 sg[i].length - offset);
955 }
956 offset = 0;
957 ctask->sg_count++;
958 }
959
960 if (!ctask->data_count) {
961 if (rc && conn->datadgst_en)
962 /*
963 * data-in is complete, but buffer not...
964 */
965 partial_sg_digest_update(conn, &sg[i],
966 sg[i].offset, sg[i].length-rc);
967 rc = 0;
968 break;
969 }
970
971 if (!conn->in.copy)
972 return -EAGAIN;
973 }
974 BUG_ON(ctask->data_count);
975
976done:
977 /* check for non-exceptional status */
978 if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
979 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
980 (long)sc, sc->result, ctask->itt);
981 conn->scsirsp_pdus_cnt++;
982 iscsi_ctask_cleanup(conn, ctask);
983 sc->scsi_done(sc);
984 }
985
986 return rc;
987}
988
989static int
990iscsi_data_recv(struct iscsi_conn *conn)
991{
992 struct iscsi_session *session = conn->session;
993 int rc = 0;
994
995 switch(conn->in.opcode) {
996 case ISCSI_OP_SCSI_DATA_IN:
997 rc = iscsi_scsi_data_in(conn);
998 break;
999 case ISCSI_OP_SCSI_CMD_RSP: {
1000 /*
1001 * SCSI Sense Data:
1002 * copying the entire Data Segment.
1003 */
1004 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1005 rc = -EAGAIN;
1006 goto exit;
1007 }
1008
1009 /*
1010 * check for sense
1011 */
1012 conn->in.hdr = &conn->hdr;
1013 conn->senselen = (conn->data[0] << 8) | conn->data[1];
1014 rc = iscsi_cmd_rsp(conn, conn->in.ctask);
1015 }
1016 break;
1017 case ISCSI_OP_TEXT_RSP:
1018 case ISCSI_OP_LOGIN_RSP:
1019 case ISCSI_OP_NOOP_IN: {
1020 struct iscsi_mgmt_task *mtask = NULL;
1021
1022 if (conn->in.itt != ISCSI_RESERVED_TAG)
1023 mtask = (struct iscsi_mgmt_task *)
1024 session->mgmt_cmds[conn->in.itt -
1025 ISCSI_MGMT_ITT_OFFSET];
1026
1027 /*
1028 * Collect data segment to the connection's data
1029 * placeholder
1030 */
1031 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1032 rc = -EAGAIN;
1033 goto exit;
1034 }
1035
1036 rc = iscsi_recv_pdu(iscsi_handle(conn), conn->in.hdr,
1037 conn->data, conn->in.datalen);
1038
1039 if (mtask && conn->login_mtask != mtask) {
1040 spin_lock(&session->lock);
1041 __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
1042 sizeof(void*));
1043 spin_unlock(&session->lock);
1044 }
1045 }
1046 break;
1047 default:
1048 BUG_ON(1);
1049 }
1050exit:
1051 return rc;
1052}
1053
1054/**
1055 * iscsi_tcp_data_recv - TCP receive in sendfile fashion
1056 * @rd_desc: read descriptor
1057 * @skb: socket buffer
1058 * @offset: offset in skb
1059 * @len: skb->len - offset
1060 **/
1061static int
1062iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1063 unsigned int offset, size_t len)
1064{
1065 int rc;
1066 struct iscsi_conn *conn = rd_desc->arg.data;
1067 int start = skb_headlen(skb);
1068 int processed;
1069 char pad[ISCSI_PAD_LEN];
1070 struct scatterlist sg;
1071
1072 /*
1073 * Save current SKB and its offset in the corresponding
1074 * connection context.
1075 */
1076 conn->in.copy = start - offset;
1077 conn->in.offset = offset;
1078 conn->in.skb = skb;
1079 conn->in.len = conn->in.copy;
1080 BUG_ON(conn->in.copy <= 0);
1081 debug_tcp("in %d bytes\n", conn->in.copy);
1082
1083more:
1084 conn->in.copied = 0;
1085 rc = 0;
1086
1087 if (unlikely(conn->suspend_rx)) {
1088 debug_tcp("conn %d Rx suspended!\n", conn->id);
1089 return 0;
1090 }
1091
1092 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
1093 conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
1094 rc = iscsi_hdr_extract(conn);
1095 if (rc) {
1096 if (rc == -EAGAIN)
1097 goto nomore;
1098 else {
1099 iscsi_conn_failure(conn, rc);
1100 return 0;
1101 }
1102 }
1103
1104 /*
1105 * Verify and process incoming PDU header.
1106 */
1107 rc = iscsi_hdr_recv(conn);
1108 if (!rc && conn->in.datalen) {
1109 if (conn->datadgst_en &&
1110 conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1111 BUG_ON(!conn->data_rx_tfm);
1112 crypto_digest_init(conn->data_rx_tfm);
1113 }
1114 conn->in_progress = IN_PROGRESS_DATA_RECV;
1115 } else if (rc) {
1116 iscsi_conn_failure(conn, rc);
1117 return 0;
1118 }
1119 }
1120
1121 if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
1122 debug_tcp("extra data_recv offset %d copy %d\n",
1123 conn->in.offset, conn->in.copy);
1124 if (conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1125 uint32_t recv_digest;
1126 skb_copy_bits(conn->in.skb, conn->in.offset,
1127 &recv_digest, 4);
1128 conn->in.offset += 4;
1129 conn->in.copy -= 4;
1130 if (recv_digest != conn->in.datadgst) {
1131 debug_tcp("iscsi_tcp: data digest error!"
1132 "0x%x != 0x%x\n", recv_digest,
1133 conn->in.datadgst);
1134 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1135 return 0;
1136 } else {
1137 debug_tcp("iscsi_tcp: data digest match!"
1138 "0x%x == 0x%x\n", recv_digest,
1139 conn->in.datadgst);
1140 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1141 }
1142 }
1143 }
1144
1145 if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) {
1146
1147 debug_tcp("data_recv offset %d copy %d\n",
1148 conn->in.offset, conn->in.copy);
1149
1150 rc = iscsi_data_recv(conn);
1151 if (rc) {
1152 if (rc == -EAGAIN) {
1153 rd_desc->count = conn->in.datalen -
1154 conn->in.ctask->data_count;
1155 goto again;
1156 }
1157 iscsi_conn_failure(conn, rc);
1158 return 0;
1159 }
1160 conn->in.copy -= conn->in.padding;
1161 conn->in.offset += conn->in.padding;
1162 if (conn->datadgst_en &&
1163 conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1164 if (conn->in.padding) {
1165 debug_tcp("padding -> %d\n", conn->in.padding);
1166 memset(pad, 0, conn->in.padding);
1167 sg_init_one(&sg, pad, conn->in.padding);
1168 crypto_digest_update(conn->data_rx_tfm, &sg, 1);
1169 }
1170 crypto_digest_final(conn->data_rx_tfm,
1171 (u8 *) & conn->in.datadgst);
1172 debug_tcp("rx digest 0x%x\n", conn->in.datadgst);
1173 conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
1174 } else
1175 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1176 }
1177
1178 debug_tcp("f, processed %d from out of %d padding %d\n",
1179 conn->in.offset - offset, (int)len, conn->in.padding);
1180 BUG_ON(conn->in.offset - offset > len);
1181
1182 if (conn->in.offset - offset != len) {
1183 debug_tcp("continue to process %d bytes\n",
1184 (int)len - (conn->in.offset - offset));
1185 goto more;
1186 }
1187
1188nomore:
1189 processed = conn->in.offset - offset;
1190 BUG_ON(processed == 0);
1191 return processed;
1192
1193again:
1194 processed = conn->in.offset - offset;
1195 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
1196 processed, (int)len, (int)rd_desc->count);
1197 BUG_ON(processed == 0);
1198 BUG_ON(processed > len);
1199
1200 conn->rxdata_octets += processed;
1201 return processed;
1202}
1203
1204static void
1205iscsi_tcp_data_ready(struct sock *sk, int flag)
1206{
1207 struct iscsi_conn *conn = sk->sk_user_data;
1208 read_descriptor_t rd_desc;
1209
1210 read_lock(&sk->sk_callback_lock);
1211
1212 /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */
1213 rd_desc.arg.data = conn;
1214 rd_desc.count = 0;
1215 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
1216
1217 read_unlock(&sk->sk_callback_lock);
1218}
1219
1220static void
1221iscsi_tcp_state_change(struct sock *sk)
1222{
1223 struct iscsi_conn *conn;
1224 struct iscsi_session *session;
1225 void (*old_state_change)(struct sock *);
1226
1227 read_lock(&sk->sk_callback_lock);
1228
1229 conn = (struct iscsi_conn*)sk->sk_user_data;
1230 session = conn->session;
1231
1232 if (sk->sk_state == TCP_CLOSE_WAIT ||
1233 sk->sk_state == TCP_CLOSE) {
1234 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1235 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1236 }
1237
1238 old_state_change = conn->old_state_change;
1239
1240 read_unlock(&sk->sk_callback_lock);
1241
1242 old_state_change(sk);
1243}
1244
1245/**
1246 * iscsi_write_space - Called when more output buffer space is available
1247 * @sk: socket space is available for
1248 **/
1249static void
1250iscsi_write_space(struct sock *sk)
1251{
1252 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1253 conn->old_write_space(sk);
1254 debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1255 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
1256 schedule_work(&conn->xmitwork);
1257}
1258
1259static void
1260iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1261{
1262 struct sock *sk = conn->sock->sk;
1263
1264 /* assign new callbacks */
1265 write_lock_bh(&sk->sk_callback_lock);
1266 sk->sk_user_data = conn;
1267 conn->old_data_ready = sk->sk_data_ready;
1268 conn->old_state_change = sk->sk_state_change;
1269 conn->old_write_space = sk->sk_write_space;
1270 sk->sk_data_ready = iscsi_tcp_data_ready;
1271 sk->sk_state_change = iscsi_tcp_state_change;
1272 sk->sk_write_space = iscsi_write_space;
1273 write_unlock_bh(&sk->sk_callback_lock);
1274}
1275
1276static void
1277iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1278{
1279 struct sock *sk = conn->sock->sk;
1280
1281 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1282 write_lock_bh(&sk->sk_callback_lock);
1283 sk->sk_user_data = NULL;
1284 sk->sk_data_ready = conn->old_data_ready;
1285 sk->sk_state_change = conn->old_state_change;
1286 sk->sk_write_space = conn->old_write_space;
1287 sk->sk_no_check = 0;
1288 write_unlock_bh(&sk->sk_callback_lock);
1289}
1290
1291/**
1292 * iscsi_send - generic send routine
1293 * @sk: kernel's socket
1294 * @buf: buffer to write from
1295 * @size: actual size to write
1296 * @flags: socket's flags
1297 *
1298 * Notes:
1299 * depending on buffer will use tcp_sendpage() or tcp_sendmsg().
1300 * buf->sg.offset == -1 tells us that buffer is non S/G and forces
1301 * to use tcp_sendmsg().
1302 */
1303static inline int
1304iscsi_send(struct socket *sk, struct iscsi_buf *buf, int size, int flags)
1305{
1306 int res;
1307
1308 if ((int)buf->sg.offset >= 0) {
1309 int offset = buf->sg.offset + buf->sent;
1310
1311 /* tcp_sendpage */
1312 res = sk->ops->sendpage(sk, buf->sg.page, offset, size, flags);
1313 } else {
1314 struct msghdr msg;
1315
1316 buf->iov.iov_base = iscsi_buf_iov_base(buf);
1317 buf->iov.iov_len = size;
1318
1319 memset(&msg, 0, sizeof(struct msghdr));
1320
1321 /* tcp_sendmsg */
1322 res = kernel_sendmsg(sk, &msg, &buf->iov, 1, size);
1323 }
1324
1325 return res;
1326}
1327
1328/**
1329 * iscsi_sendhdr - send PDU Header via tcp_sendpage()
1330 * @conn: iscsi connection
1331 * @buf: buffer to write from
1332 * @datalen: lenght of data to be sent after the header
1333 *
1334 * Notes:
1335 * (Tx, Fast Path)
1336 **/
1337static inline int
1338iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1339{
1340 struct socket *sk = conn->sock;
1341 int flags = 0; /* MSG_DONTWAIT; */
1342 int res, size;
1343
1344 size = buf->sg.length - buf->sent;
1345 BUG_ON(buf->sent + size > buf->sg.length);
1346 if (buf->sent + size != buf->sg.length || datalen)
1347 flags |= MSG_MORE;
1348
1349 res = iscsi_send(sk, buf, size, flags);
1350 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1351 if (res >= 0) {
1352 conn->txdata_octets += res;
1353 buf->sent += res;
1354 if (size != res)
1355 return -EAGAIN;
1356 return 0;
1357 } else if (res == -EAGAIN) {
1358 conn->sendpage_failures_cnt++;
1359 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1360 } else if (res == -EPIPE)
1361 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1362
1363 return res;
1364}
1365
1366/**
1367 * iscsi_sendpage - send one page of iSCSI Data-Out.
1368 * @conn: iscsi connection
1369 * @buf: buffer to write from
1370 * @count: remaining data
1371 * @sent: number of bytes sent
1372 *
1373 * Notes:
1374 * (Tx, Fast Path)
1375 **/
1376static inline int
1377iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1378 int *count, int *sent)
1379{
1380 struct socket *sk = conn->sock;
1381 int flags = 0; /* MSG_DONTWAIT; */
1382 int res, size;
1383
1384 size = buf->sg.length - buf->sent;
1385 BUG_ON(buf->sent + size > buf->sg.length);
1386 if (size > *count)
1387 size = *count;
1388 if (buf->sent + size != buf->sg.length)
1389 flags |= MSG_MORE;
1390
1391 res = iscsi_send(sk, buf, size, flags);
1392 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1393 size, buf->sent, *count, *sent, res);
1394 if (res >= 0) {
1395 conn->txdata_octets += res;
1396 buf->sent += res;
1397 *count -= res;
1398 *sent += res;
1399 if (size != res)
1400 return -EAGAIN;
1401 return 0;
1402 } else if (res == -EAGAIN) {
1403 conn->sendpage_failures_cnt++;
1404 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1405 } else if (res == -EPIPE)
1406 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1407
1408 return res;
1409}
1410
1411static inline void
1412iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1413{
1414 BUG_ON(!conn->data_tx_tfm);
1415 crypto_digest_init(conn->data_tx_tfm);
1416 ctask->digest_count = 4;
1417}
1418
1419static inline void
1420iscsi_buf_data_digest_update(struct iscsi_conn *conn, struct iscsi_buf *buf)
1421{
1422 struct scatterlist sg;
1423
1424 if (buf->sg.offset != -1)
1425 crypto_digest_update(conn->data_tx_tfm, &buf->sg, 1);
1426 else {
1427 sg_init_one(&sg, (char *)buf->sg.page, buf->sg.length);
1428 crypto_digest_update(conn->data_tx_tfm, &sg, 1);
1429 }
1430}
1431
1432static inline int
1433iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1434 struct iscsi_buf *buf, uint32_t *digest, int final)
1435{
1436 int rc = 0;
1437 int sent = 0;
1438
1439 if (final)
1440 crypto_digest_final(conn->data_tx_tfm, (u8*)digest);
1441
1442 iscsi_buf_init_virt(buf, (char*)digest, 4);
1443 rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent);
1444 if (rc) {
1445 ctask->datadigest = *digest;
1446 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1447 } else
1448 ctask->digest_count = 4;
1449 return rc;
1450}
1451
1452/**
1453 * iscsi_solicit_data_cont - initialize next Data-Out
1454 * @conn: iscsi connection
1455 * @ctask: scsi command task
1456 * @r2t: R2T info
1457 * @left: bytes left to transfer
1458 *
1459 * Notes:
1460 * Initialize next Data-Out within this R2T sequence and continue
1461 * to process next Scatter-Gather element(if any) of this SCSI command.
1462 *
1463 * Called under connection lock.
1464 **/
1465static void
1466iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1467 struct iscsi_r2t_info *r2t, int left)
1468{
1469 struct iscsi_data *hdr;
1470 struct iscsi_data_task *dtask;
1471 struct scsi_cmnd *sc = ctask->sc;
1472 int new_offset;
1473
1474 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1475 BUG_ON(!dtask);
1476 hdr = &dtask->hdr;
1477 memset(hdr, 0, sizeof(struct iscsi_data));
1478 hdr->ttt = r2t->ttt;
1479 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1480 r2t->solicit_datasn++;
1481 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1482 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1483 hdr->itt = ctask->hdr.itt;
1484 hdr->exp_statsn = r2t->exp_statsn;
1485 new_offset = r2t->data_offset + r2t->sent;
1486 hdr->offset = cpu_to_be32(new_offset);
1487 if (left > conn->max_xmit_dlength) {
1488 hton24(hdr->dlength, conn->max_xmit_dlength);
1489 r2t->data_count = conn->max_xmit_dlength;
1490 } else {
1491 hton24(hdr->dlength, left);
1492 r2t->data_count = left;
1493 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1494 }
1495 conn->dataout_pdus_cnt++;
1496
1497 iscsi_buf_init_hdr(conn, &r2t->headbuf, (char*)hdr,
1498 (u8 *)dtask->hdrext);
1499
1500 r2t->dtask = dtask;
1501
1502 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1503 BUG_ON(ctask->bad_sg == r2t->sg);
1504 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1505 r2t->sg += 1;
1506 } else
1507 iscsi_buf_init_iov(&ctask->sendbuf,
1508 (char*)sc->request_buffer + new_offset,
1509 r2t->data_count);
1510
1511 list_add(&dtask->item, &ctask->dataqueue);
1512}
1513
1514static void
1515iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1516{
1517 struct iscsi_data *hdr;
1518 struct iscsi_data_task *dtask;
1519
1520 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1521 BUG_ON(!dtask);
1522 hdr = &dtask->hdr;
1523 memset(hdr, 0, sizeof(struct iscsi_data));
1524 hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1525 hdr->datasn = cpu_to_be32(ctask->unsol_datasn);
1526 ctask->unsol_datasn++;
1527 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1528 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1529 hdr->itt = ctask->hdr.itt;
1530 hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
1531 hdr->offset = cpu_to_be32(ctask->total_length -
1532 ctask->r2t_data_count -
1533 ctask->unsol_count);
1534 if (ctask->unsol_count > conn->max_xmit_dlength) {
1535 hton24(hdr->dlength, conn->max_xmit_dlength);
1536 ctask->data_count = conn->max_xmit_dlength;
1537 hdr->flags = 0;
1538 } else {
1539 hton24(hdr->dlength, ctask->unsol_count);
1540 ctask->data_count = ctask->unsol_count;
1541 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1542 }
1543
1544 iscsi_buf_init_hdr(conn, &ctask->headbuf, (char*)hdr,
1545 (u8 *)dtask->hdrext);
1546
1547 list_add(&dtask->item, &ctask->dataqueue);
1548
1549 ctask->dtask = dtask;
1550}
1551
1552/**
1553 * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1554 * @conn: iscsi connection
1555 * @ctask: scsi command task
1556 * @sc: scsi command
1557 **/
1558static void
1559iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1560 struct scsi_cmnd *sc)
1561{
1562 struct iscsi_session *session = conn->session;
1563
1564 BUG_ON(__kfifo_len(ctask->r2tqueue));
1565
1566 ctask->sc = sc;
1567 ctask->conn = conn;
1568 ctask->hdr.opcode = ISCSI_OP_SCSI_CMD;
1569 ctask->hdr.flags = ISCSI_ATTR_SIMPLE;
1570 int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun);
1571 ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) |
1572 (session->age << AGE_SHIFT);
1573 ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen);
1574 ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++;
1575 ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn);
1576 memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len);
1577 memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
1578
1579 ctask->mtask = NULL;
1580 ctask->sent = 0;
1581 ctask->sg_count = 0;
1582
1583 ctask->total_length = sc->request_bufflen;
1584
1585 if (sc->sc_data_direction == DMA_TO_DEVICE) {
1586 ctask->exp_r2tsn = 0;
1587 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE;
1588 BUG_ON(ctask->total_length == 0);
1589 if (sc->use_sg) {
1590 struct scatterlist *sg = sc->request_buffer;
1591
1592 iscsi_buf_init_sg(&ctask->sendbuf,
1593 &sg[ctask->sg_count++]);
1594 ctask->sg = sg;
1595 ctask->bad_sg = sg + sc->use_sg;
1596 } else {
1597 iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer,
1598 sc->request_bufflen);
1599 }
1600
1601 /*
1602 * Write counters:
1603 *
1604 * imm_count bytes to be sent right after
1605 * SCSI PDU Header
1606 *
1607 * unsol_count bytes(as Data-Out) to be sent
1608 * without R2T ack right after
1609 * immediate data
1610 *
1611 * r2t_data_count bytes to be sent via R2T ack's
1612 *
1613 * pad_count bytes to be sent as zero-padding
1614 */
1615 ctask->imm_count = 0;
1616 ctask->unsol_count = 0;
1617 ctask->unsol_datasn = 0;
1618 ctask->xmstate = XMSTATE_W_HDR;
1619 /* calculate write padding */
1620 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1621 if (ctask->pad_count) {
1622 ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count;
1623 debug_scsi("write padding %d bytes\n",
1624 ctask->pad_count);
1625 ctask->xmstate |= XMSTATE_W_PAD;
1626 }
1627 if (session->imm_data_en) {
1628 if (ctask->total_length >= session->first_burst)
1629 ctask->imm_count = min(session->first_burst,
1630 conn->max_xmit_dlength);
1631 else
1632 ctask->imm_count = min(ctask->total_length,
1633 conn->max_xmit_dlength);
1634 hton24(ctask->hdr.dlength, ctask->imm_count);
1635 ctask->xmstate |= XMSTATE_IMM_DATA;
1636 } else
1637 zero_data(ctask->hdr.dlength);
1638
1639 if (!session->initial_r2t_en)
1640 ctask->unsol_count = min(session->first_burst,
1641 ctask->total_length) - ctask->imm_count;
1642 if (!ctask->unsol_count)
1643 /* No unsolicit Data-Out's */
1644 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1645 else
1646 ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1647
1648 ctask->r2t_data_count = ctask->total_length -
1649 ctask->imm_count -
1650 ctask->unsol_count;
1651
1652 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1653 "r2t_data %d]\n",
1654 ctask->itt, ctask->total_length, ctask->imm_count,
1655 ctask->unsol_count, ctask->r2t_data_count);
1656 } else {
1657 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1658 if (sc->sc_data_direction == DMA_FROM_DEVICE)
1659 ctask->hdr.flags |= ISCSI_FLAG_CMD_READ;
1660 ctask->datasn = 0;
1661 ctask->xmstate = XMSTATE_R_HDR;
1662 zero_data(ctask->hdr.dlength);
1663 }
1664
1665 iscsi_buf_init_hdr(conn, &ctask->headbuf, (char*)&ctask->hdr,
1666 (u8 *)ctask->hdrext);
1667 conn->scsicmd_pdus_cnt++;
1668}
1669
1670/**
1671 * iscsi_mtask_xmit - xmit management(immediate) task
1672 * @conn: iscsi connection
1673 * @mtask: task management task
1674 *
1675 * Notes:
1676 * The function can return -EAGAIN in which case caller must
1677 * call it again later, or recover. '0' return code means successful
1678 * xmit.
1679 *
1680 * Management xmit state machine consists of two states:
1681 * IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
1682 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1683 **/
1684static int
1685iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1686{
1687
1688 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1689 conn->id, mtask->xmstate, mtask->itt);
1690
1691 if (mtask->xmstate & XMSTATE_IMM_HDR) {
1692 mtask->xmstate &= ~XMSTATE_IMM_HDR;
1693 if (mtask->data_count)
1694 mtask->xmstate |= XMSTATE_IMM_DATA;
1695 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) {
1696 mtask->xmstate |= XMSTATE_IMM_HDR;
1697 if (mtask->data_count)
1698 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1699 return -EAGAIN;
1700 }
1701 }
1702
1703 if (mtask->xmstate & XMSTATE_IMM_DATA) {
1704 BUG_ON(!mtask->data_count);
1705 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1706 /* FIXME: implement.
1707 * Virtual buffer could be spreaded across multiple pages...
1708 */
1709 do {
1710 if (iscsi_sendpage(conn, &mtask->sendbuf,
1711 &mtask->data_count, &mtask->sent)) {
1712 mtask->xmstate |= XMSTATE_IMM_DATA;
1713 return -EAGAIN;
1714 }
1715 } while (mtask->data_count);
1716 }
1717
1718 BUG_ON(mtask->xmstate != XMSTATE_IDLE);
1719 return 0;
1720}
1721
1722static inline int
1723handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1724{
1725 ctask->xmstate &= ~XMSTATE_R_HDR;
1726 if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) {
1727 BUG_ON(ctask->xmstate != XMSTATE_IDLE);
1728 return 0; /* wait for Data-In */
1729 }
1730 ctask->xmstate |= XMSTATE_R_HDR;
1731 return -EAGAIN;
1732}
1733
1734static inline int
1735handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1736{
1737 ctask->xmstate &= ~XMSTATE_W_HDR;
1738 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) {
1739 ctask->xmstate |= XMSTATE_W_HDR;
1740 return -EAGAIN;
1741 }
1742 return 0;
1743}
1744
1745static inline int
1746handle_xmstate_data_digest(struct iscsi_conn *conn,
1747 struct iscsi_cmd_task *ctask)
1748{
1749 ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1750 debug_tcp("resent data digest 0x%x\n", ctask->datadigest);
1751 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1752 &ctask->datadigest, 0)) {
1753 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1754 debug_tcp("resent data digest 0x%x fail!\n",
1755 ctask->datadigest);
1756 return -EAGAIN;
1757 }
1758 return 0;
1759}
1760
1761static inline int
1762handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1763{
1764 BUG_ON(!ctask->imm_count);
1765 ctask->xmstate &= ~XMSTATE_IMM_DATA;
1766
1767 if (conn->datadgst_en) {
1768 iscsi_data_digest_init(conn, ctask);
1769 ctask->immdigest = 0;
1770 }
1771
1772 for (;;) {
1773 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count,
1774 &ctask->sent)) {
1775 ctask->xmstate |= XMSTATE_IMM_DATA;
1776 if (conn->datadgst_en) {
1777 crypto_digest_final(conn->data_tx_tfm,
1778 (u8*)&ctask->immdigest);
1779 debug_tcp("tx imm sendpage fail 0x%x\n",
1780 ctask->datadigest);
1781 }
1782 return -EAGAIN;
1783 }
1784 if (conn->datadgst_en)
1785 iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
1786
1787 if (!ctask->imm_count)
1788 break;
1789 iscsi_buf_init_sg(&ctask->sendbuf,
1790 &ctask->sg[ctask->sg_count++]);
1791 }
1792
1793 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1794 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1795 &ctask->immdigest, 1)) {
1796 debug_tcp("sending imm digest 0x%x fail!\n",
1797 ctask->immdigest);
1798 return -EAGAIN;
1799 }
1800 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest);
1801 }
1802
1803 return 0;
1804}
1805
1806static inline int
1807handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1808{
1809 struct iscsi_data_task *dtask;
1810
1811 ctask->xmstate |= XMSTATE_UNS_DATA;
1812 if (ctask->xmstate & XMSTATE_UNS_INIT) {
1813 iscsi_unsolicit_data_init(conn, ctask);
1814 BUG_ON(!ctask->dtask);
1815 dtask = ctask->dtask;
1816
1817 ctask->xmstate &= ~XMSTATE_UNS_INIT;
1818 }
1819 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) {
1820 ctask->xmstate &= ~XMSTATE_UNS_DATA;
1821 ctask->xmstate |= XMSTATE_UNS_HDR;
1822 return -EAGAIN;
1823 }
1824
1825 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1826 ctask->itt, ctask->unsol_count, ctask->sent);
1827 return 0;
1828}
1829
1830static inline int
1831handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1832{
1833 struct iscsi_data_task *dtask = ctask->dtask;
1834
1835 BUG_ON(!ctask->data_count);
1836 ctask->xmstate &= ~XMSTATE_UNS_DATA;
1837
1838 if (conn->datadgst_en) {
1839 iscsi_data_digest_init(conn, ctask);
1840 dtask->digest = 0;
1841 }
1842
1843 for (;;) {
1844 int start = ctask->sent;
1845
1846 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count,
1847 &ctask->sent)) {
1848 ctask->unsol_count -= ctask->sent - start;
1849 ctask->xmstate |= XMSTATE_UNS_DATA;
1850 /* will continue with this ctask later.. */
1851 if (conn->datadgst_en) {
1852 crypto_digest_final(conn->data_tx_tfm,
1853 (u8 *)&dtask->digest);
1854 debug_tcp("tx uns data fail 0x%x\n",
1855 dtask->digest);
1856 }
1857 return -EAGAIN;
1858 }
1859
1860 BUG_ON(ctask->sent > ctask->total_length);
1861 ctask->unsol_count -= ctask->sent - start;
1862
1863 /*
1864 * XXX:we may run here with un-initial sendbuf.
1865 * so pass it
1866 */
1867 if (conn->datadgst_en && ctask->sent - start > 0)
1868 iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
1869
1870 if (!ctask->data_count)
1871 break;
1872 iscsi_buf_init_sg(&ctask->sendbuf,
1873 &ctask->sg[ctask->sg_count++]);
1874 }
1875 BUG_ON(ctask->unsol_count < 0);
1876
1877 /*
1878 * Done with the Data-Out. Next, check if we need
1879 * to send another unsolicited Data-Out.
1880 */
1881 if (ctask->unsol_count) {
1882 if (conn->datadgst_en) {
1883 if (iscsi_digest_final_send(conn, ctask,
1884 &dtask->digestbuf,
1885 &dtask->digest, 1)) {
1886 debug_tcp("send uns digest 0x%x fail\n",
1887 dtask->digest);
1888 return -EAGAIN;
1889 }
1890 debug_tcp("sending uns digest 0x%x, more uns\n",
1891 dtask->digest);
1892 }
1893 ctask->xmstate |= XMSTATE_UNS_INIT;
1894 return 1;
1895 }
1896
1897 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1898 if (iscsi_digest_final_send(conn, ctask,
1899 &dtask->digestbuf,
1900 &dtask->digest, 1)) {
1901 debug_tcp("send last uns digest 0x%x fail\n",
1902 dtask->digest);
1903 return -EAGAIN;
1904 }
1905 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1906 }
1907
1908 return 0;
1909}
1910
1911static inline int
1912handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1913{
1914 struct iscsi_session *session = conn->session;
1915 struct iscsi_r2t_info *r2t = ctask->r2t;
1916 struct iscsi_data_task *dtask = r2t->dtask;
1917 int left;
1918
1919 ctask->xmstate &= ~XMSTATE_SOL_DATA;
1920 ctask->dtask = dtask;
1921
1922 if (conn->datadgst_en) {
1923 iscsi_data_digest_init(conn, ctask);
1924 dtask->digest = 0;
1925 }
1926solicit_again:
1927 /*
1928 * send Data-Out whitnin this R2T sequence.
1929 */
1930 if (!r2t->data_count)
1931 goto data_out_done;
1932
1933 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
1934 ctask->xmstate |= XMSTATE_SOL_DATA;
1935 /* will continue with this ctask later.. */
1936 if (conn->datadgst_en) {
1937 crypto_digest_final(conn->data_tx_tfm,
1938 (u8 *)&dtask->digest);
1939 debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1940 }
1941 return -EAGAIN;
1942 }
1943
1944 BUG_ON(r2t->data_count < 0);
1945 if (conn->datadgst_en)
1946 iscsi_buf_data_digest_update(conn, &r2t->sendbuf);
1947
1948 if (r2t->data_count) {
1949 BUG_ON(ctask->sc->use_sg == 0);
1950 if (!iscsi_buf_left(&r2t->sendbuf)) {
1951 BUG_ON(ctask->bad_sg == r2t->sg);
1952 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1953 r2t->sg += 1;
1954 }
1955 goto solicit_again;
1956 }
1957
1958data_out_done:
1959 /*
1960 * Done with this Data-Out. Next, check if we have
1961 * to send another Data-Out for this R2T.
1962 */
1963 BUG_ON(r2t->data_length - r2t->sent < 0);
1964 left = r2t->data_length - r2t->sent;
1965 if (left) {
1966 if (conn->datadgst_en) {
1967 if (iscsi_digest_final_send(conn, ctask,
1968 &dtask->digestbuf,
1969 &dtask->digest, 1)) {
1970 debug_tcp("send r2t data digest 0x%x"
1971 "fail\n", dtask->digest);
1972 return -EAGAIN;
1973 }
1974 debug_tcp("r2t data send digest 0x%x\n",
1975 dtask->digest);
1976 }
1977 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1978 ctask->xmstate |= XMSTATE_SOL_DATA;
1979 ctask->xmstate &= ~XMSTATE_SOL_HDR;
1980 return 1;
1981 }
1982
1983 /*
1984 * Done with this R2T. Check if there are more
1985 * outstanding R2Ts ready to be processed.
1986 */
1987 BUG_ON(ctask->r2t_data_count - r2t->data_length < 0);
1988 if (conn->datadgst_en) {
1989 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1990 &dtask->digest, 1)) {
1991 debug_tcp("send last r2t data digest 0x%x"
1992 "fail\n", dtask->digest);
1993 return -EAGAIN;
1994 }
1995 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1996 }
1997
1998 ctask->r2t_data_count -= r2t->data_length;
1999 ctask->r2t = NULL;
2000 spin_lock_bh(&session->lock);
2001 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
2002 spin_unlock_bh(&session->lock);
2003 if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
2004 ctask->r2t = r2t;
2005 ctask->xmstate |= XMSTATE_SOL_DATA;
2006 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2007 return 1;
2008 }
2009
2010 return 0;
2011}
2012
2013static inline int
2014handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2015{
2016 struct iscsi_data_task *dtask = ctask->dtask;
2017 int sent;
2018
2019 ctask->xmstate &= ~XMSTATE_W_PAD;
2020 iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad,
2021 ctask->pad_count);
2022 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) {
2023 ctask->xmstate |= XMSTATE_W_PAD;
2024 return -EAGAIN;
2025 }
2026
2027 if (conn->datadgst_en) {
2028 iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
2029 /* imm data? */
2030 if (!dtask) {
2031 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
2032 &ctask->immdigest, 1)) {
2033 debug_tcp("send padding digest 0x%x"
2034 "fail!\n", ctask->immdigest);
2035 return -EAGAIN;
2036 }
2037 debug_tcp("done with padding, digest 0x%x\n",
2038 ctask->datadigest);
2039 } else {
2040 if (iscsi_digest_final_send(conn, ctask,
2041 &dtask->digestbuf,
2042 &dtask->digest, 1)) {
2043 debug_tcp("send padding digest 0x%x"
2044 "fail\n", dtask->digest);
2045 return -EAGAIN;
2046 }
2047 debug_tcp("done with padding, digest 0x%x\n",
2048 dtask->digest);
2049 }
2050 }
2051
2052 return 0;
2053}
2054
2055static int
2056iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2057{
2058 int rc = 0;
2059
2060 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
2061 conn->id, ctask->xmstate, ctask->itt);
2062
2063 /*
2064 * serialize with TMF AbortTask
2065 */
2066 if (ctask->mtask)
2067 return rc;
2068
2069 if (ctask->xmstate & XMSTATE_R_HDR) {
2070 rc = handle_xmstate_r_hdr(conn, ctask);
2071 return rc;
2072 }
2073
2074 if (ctask->xmstate & XMSTATE_W_HDR) {
2075 rc = handle_xmstate_w_hdr(conn, ctask);
2076 if (rc)
2077 return rc;
2078 }
2079
2080 /* XXX: for data digest xmit recover */
2081 if (ctask->xmstate & XMSTATE_DATA_DIGEST) {
2082 rc = handle_xmstate_data_digest(conn, ctask);
2083 if (rc)
2084 return rc;
2085 }
2086
2087 if (ctask->xmstate & XMSTATE_IMM_DATA) {
2088 rc = handle_xmstate_imm_data(conn, ctask);
2089 if (rc)
2090 return rc;
2091 }
2092
2093 if (ctask->xmstate & XMSTATE_UNS_HDR) {
2094 BUG_ON(!ctask->unsol_count);
2095 ctask->xmstate &= ~XMSTATE_UNS_HDR;
2096unsolicit_head_again:
2097 rc = handle_xmstate_uns_hdr(conn, ctask);
2098 if (rc)
2099 return rc;
2100 }
2101
2102 if (ctask->xmstate & XMSTATE_UNS_DATA) {
2103 rc = handle_xmstate_uns_data(conn, ctask);
2104 if (rc == 1)
2105 goto unsolicit_head_again;
2106 else if (rc)
2107 return rc;
2108 goto done;
2109 }
2110
2111 if (ctask->xmstate & XMSTATE_SOL_HDR) {
2112 struct iscsi_r2t_info *r2t;
2113
2114 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2115 ctask->xmstate |= XMSTATE_SOL_DATA;
2116 if (!ctask->r2t)
2117 __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t,
2118 sizeof(void*));
2119solicit_head_again:
2120 r2t = ctask->r2t;
2121
2122 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
2123 ctask->xmstate &= ~XMSTATE_SOL_DATA;
2124 ctask->xmstate |= XMSTATE_SOL_HDR;
2125 return -EAGAIN;
2126 }
2127
2128 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
2129 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
2130 r2t->sent);
2131 }
2132
2133 if (ctask->xmstate & XMSTATE_SOL_DATA) {
2134 rc = handle_xmstate_sol_data(conn, ctask);
2135 if (rc == 1)
2136 goto solicit_head_again;
2137 if (rc)
2138 return rc;
2139 }
2140
2141done:
2142 /*
2143 * Last thing to check is whether we need to send write
2144 * padding. Note that we check for xmstate equality, not just the bit.
2145 */
2146 if (ctask->xmstate == XMSTATE_W_PAD)
2147 rc = handle_xmstate_w_pad(conn, ctask);
2148
2149 return rc;
2150}
2151
2152/**
2153 * iscsi_data_xmit - xmit any command into the scheduled connection
2154 * @conn: iscsi connection
2155 *
2156 * Notes:
2157 * The function can return -EAGAIN in which case the caller must
2158 * re-schedule it again later or recover. '0' return code means
2159 * successful xmit.
2160 **/
2161static int
2162iscsi_data_xmit(struct iscsi_conn *conn)
2163{
2164 if (unlikely(conn->suspend_tx)) {
2165 debug_tcp("conn %d Tx suspended!\n", conn->id);
2166 return 0;
2167 }
2168
2169 /*
2170 * Transmit in the following order:
2171 *
2172 * 1) un-finished xmit (ctask or mtask)
2173 * 2) immediate control PDUs
2174 * 3) write data
2175 * 4) SCSI commands
2176 * 5) non-immediate control PDUs
2177 *
2178 * No need to lock around __kfifo_get as long as
2179 * there's one producer and one consumer.
2180 */
2181
2182 BUG_ON(conn->ctask && conn->mtask);
2183
2184 if (conn->ctask) {
2185 if (iscsi_ctask_xmit(conn, conn->ctask))
2186 goto again;
2187 /* done with this in-progress ctask */
2188 conn->ctask = NULL;
2189 }
2190 if (conn->mtask) {
2191 if (iscsi_mtask_xmit(conn, conn->mtask))
2192 goto again;
2193 /* done with this in-progress mtask */
2194 conn->mtask = NULL;
2195 }
2196
2197 /* process immediate first */
2198 if (unlikely(__kfifo_len(conn->immqueue))) {
2199 struct iscsi_session *session = conn->session;
2200 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask,
2201 sizeof(void*))) {
2202 if (iscsi_mtask_xmit(conn, conn->mtask))
2203 goto again;
2204
2205 if (conn->mtask->hdr.itt ==
2206 cpu_to_be32(ISCSI_RESERVED_TAG)) {
2207 spin_lock_bh(&session->lock);
2208 __kfifo_put(session->mgmtpool.queue,
2209 (void*)&conn->mtask, sizeof(void*));
2210 spin_unlock_bh(&session->lock);
2211 }
2212 }
2213 /* done with this mtask */
2214 conn->mtask = NULL;
2215 }
2216
2217 /* process write queue */
2218 while (__kfifo_get(conn->writequeue, (void*)&conn->ctask,
2219 sizeof(void*))) {
2220 if (iscsi_ctask_xmit(conn, conn->ctask))
2221 goto again;
2222 }
2223
2224 /* process command queue */
2225 while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask,
2226 sizeof(void*))) {
2227 if (iscsi_ctask_xmit(conn, conn->ctask))
2228 goto again;
2229 }
2230 /* done with this ctask */
2231 conn->ctask = NULL;
2232
2233 /* process the rest control plane PDUs, if any */
2234 if (unlikely(__kfifo_len(conn->mgmtqueue))) {
2235 struct iscsi_session *session = conn->session;
2236
2237 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask,
2238 sizeof(void*))) {
2239 if (iscsi_mtask_xmit(conn, conn->mtask))
2240 goto again;
2241
2242 if (conn->mtask->hdr.itt ==
2243 cpu_to_be32(ISCSI_RESERVED_TAG)) {
2244 spin_lock_bh(&session->lock);
2245 __kfifo_put(session->mgmtpool.queue,
2246 (void*)&conn->mtask,
2247 sizeof(void*));
2248 spin_unlock_bh(&session->lock);
2249 }
2250 }
2251 /* done with this mtask */
2252 conn->mtask = NULL;
2253 }
2254
2255 return 0;
2256
2257again:
2258 if (unlikely(conn->suspend_tx))
2259 return 0;
2260
2261 return -EAGAIN;
2262}
2263
2264static void
2265iscsi_xmitworker(void *data)
2266{
2267 struct iscsi_conn *conn = data;
2268
2269 /*
2270 * serialize Xmit worker on a per-connection basis.
2271 */
2272 down(&conn->xmitsema);
2273 if (iscsi_data_xmit(conn))
2274 schedule_work(&conn->xmitwork);
2275 up(&conn->xmitsema);
2276}
2277
2278#define FAILURE_BAD_HOST 1
2279#define FAILURE_SESSION_FAILED 2
2280#define FAILURE_SESSION_FREED 3
2281#define FAILURE_WINDOW_CLOSED 4
2282#define FAILURE_SESSION_TERMINATE 5
2283
2284static int
2285iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
2286{
2287 struct Scsi_Host *host;
2288 int reason = 0;
2289 struct iscsi_session *session;
2290 struct iscsi_conn *conn = NULL;
2291 struct iscsi_cmd_task *ctask = NULL;
2292
2293 sc->scsi_done = done;
2294 sc->result = 0;
2295
2296 host = sc->device->host;
2297 session = iscsi_hostdata(host->hostdata);
2298 BUG_ON(host != session->host);
2299
2300 spin_lock(&session->lock);
2301
2302 if (session->state != ISCSI_STATE_LOGGED_IN) {
2303 if (session->state == ISCSI_STATE_FAILED) {
2304 reason = FAILURE_SESSION_FAILED;
2305 goto reject;
2306 } else if (session->state == ISCSI_STATE_TERMINATE) {
2307 reason = FAILURE_SESSION_TERMINATE;
2308 goto fault;
2309 }
2310 reason = FAILURE_SESSION_FREED;
2311 goto fault;
2312 }
2313
2314 /*
2315 * Check for iSCSI window and take care of CmdSN wrap-around
2316 */
2317 if ((int)(session->max_cmdsn - session->cmdsn) < 0) {
2318 reason = FAILURE_WINDOW_CLOSED;
2319 goto reject;
2320 }
2321
2322 conn = session->leadconn;
2323
2324 __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
2325 BUG_ON(ctask->sc);
2326
2327 sc->SCp.phase = session->age;
2328 sc->SCp.ptr = (char*)ctask;
2329 iscsi_cmd_init(conn, ctask, sc);
2330
2331 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
2332 debug_scsi(
2333 "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
2334 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
2335 conn->id, (long)sc, ctask->itt, sc->request_bufflen,
2336 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
2337 spin_unlock(&session->lock);
2338
2339 if (!in_interrupt() && !down_trylock(&conn->xmitsema)) {
2340 spin_unlock_irq(host->host_lock);
2341 if (iscsi_data_xmit(conn))
2342 schedule_work(&conn->xmitwork);
2343 up(&conn->xmitsema);
2344 spin_lock_irq(host->host_lock);
2345 } else
2346 schedule_work(&conn->xmitwork);
2347
2348 return 0;
2349
2350reject:
2351 spin_unlock(&session->lock);
2352 debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
2353 return SCSI_MLQUEUE_HOST_BUSY;
2354
2355fault:
2356 spin_unlock(&session->lock);
2357 printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n",
2358 sc->cmnd[0], reason);
2359 sc->sense_buffer[0] = 0x70;
2360 sc->sense_buffer[2] = NOT_READY;
2361 sc->sense_buffer[7] = 0x6;
2362 sc->sense_buffer[12] = 0x08;
2363 sc->sense_buffer[13] = 0x00;
2364 sc->result = (DID_NO_CONNECT << 16);
2365 sc->resid = sc->request_bufflen;
2366 sc->scsi_done(sc);
2367 return 0;
2368}
2369
2370static int
2371iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size)
2372{
2373 int i;
2374
2375 *items = kmalloc(max * sizeof(void*), GFP_KERNEL);
2376 if (*items == NULL)
2377 return -ENOMEM;
2378
2379 q->max = max;
2380 q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL);
2381 if (q->pool == NULL) {
2382 kfree(*items);
2383 return -ENOMEM;
2384 }
2385
2386 q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2387 GFP_KERNEL, NULL);
2388 if (q->queue == ERR_PTR(-ENOMEM)) {
2389 kfree(q->pool);
2390 kfree(*items);
2391 return -ENOMEM;
2392 }
2393
2394 for (i = 0; i < max; i++) {
2395 q->pool[i] = kmalloc(item_size, GFP_KERNEL);
2396 if (q->pool[i] == NULL) {
2397 int j;
2398
2399 for (j = 0; j < i; j++)
2400 kfree(q->pool[j]);
2401
2402 kfifo_free(q->queue);
2403 kfree(q->pool);
2404 kfree(*items);
2405 return -ENOMEM;
2406 }
2407 memset(q->pool[i], 0, item_size);
2408 (*items)[i] = q->pool[i];
2409 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2410 }
2411 return 0;
2412}
2413
2414static void
2415iscsi_pool_free(struct iscsi_queue *q, void **items)
2416{
2417 int i;
2418
2419 for (i = 0; i < q->max; i++)
2420 kfree(items[i]);
2421 kfree(q->pool);
2422 kfree(items);
2423}
2424
2425static iscsi_connh_t
2426iscsi_conn_create(iscsi_sessionh_t sessionh, uint32_t conn_idx)
2427{
2428 struct iscsi_session *session = iscsi_ptr(sessionh);
2429 struct iscsi_conn *conn = NULL;
2430
2431 conn = kmalloc(sizeof(struct iscsi_conn), GFP_KERNEL);
2432 if (conn == NULL)
2433 goto conn_alloc_fail;
2434 memset(conn, 0, sizeof(struct iscsi_conn));
2435
2436 conn->c_stage = ISCSI_CONN_INITIAL_STAGE;
2437 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2438 conn->id = conn_idx;
2439 conn->exp_statsn = 0;
2440 conn->tmabort_state = TMABORT_INITIAL;
2441
2442 /* initial operational parameters */
2443 conn->hdr_size = sizeof(struct iscsi_hdr);
2444 conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2445 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2446
2447 spin_lock_init(&conn->lock);
2448
2449 /* initialize general xmit PDU commands queue */
2450 conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*),
2451 GFP_KERNEL, NULL);
2452 if (conn->xmitqueue == ERR_PTR(-ENOMEM))
2453 goto xmitqueue_alloc_fail;
2454
2455 /* initialize write response PDU commands queue */
2456 conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*),
2457 GFP_KERNEL, NULL);
2458 if (conn->writequeue == ERR_PTR(-ENOMEM))
2459 goto writequeue_alloc_fail;
2460
2461 /* initialize general immediate & non-immediate PDU commands queue */
2462 conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2463 GFP_KERNEL, NULL);
2464 if (conn->immqueue == ERR_PTR(-ENOMEM))
2465 goto immqueue_alloc_fail;
2466
2467 conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2468 GFP_KERNEL, NULL);
2469 if (conn->mgmtqueue == ERR_PTR(-ENOMEM))
2470 goto mgmtqueue_alloc_fail;
2471
2472 INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn);
2473
2474 /* allocate login_mtask used for the login/text sequences */
2475 spin_lock_bh(&session->lock);
2476 if (!__kfifo_get(session->mgmtpool.queue,
2477 (void*)&conn->login_mtask,
2478 sizeof(void*))) {
2479 spin_unlock_bh(&session->lock);
2480 goto login_mtask_alloc_fail;
2481 }
2482 spin_unlock_bh(&session->lock);
2483
2484 /* allocate initial PDU receive place holder */
2485 if (conn->data_size <= PAGE_SIZE)
2486 conn->data = kmalloc(conn->data_size, GFP_KERNEL);
2487 else
2488 conn->data = (void*)__get_free_pages(GFP_KERNEL,
2489 get_order(conn->data_size));
2490 if (!conn->data)
2491 goto max_recv_dlenght_alloc_fail;
2492
2493 init_timer(&conn->tmabort_timer);
2494 init_MUTEX(&conn->xmitsema);
2495 init_waitqueue_head(&conn->ehwait);
2496
2497 return iscsi_handle(conn);
2498
2499max_recv_dlenght_alloc_fail:
2500 spin_lock_bh(&session->lock);
2501 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2502 sizeof(void*));
2503 spin_unlock_bh(&session->lock);
2504login_mtask_alloc_fail:
2505 kfifo_free(conn->mgmtqueue);
2506mgmtqueue_alloc_fail:
2507 kfifo_free(conn->immqueue);
2508immqueue_alloc_fail:
2509 kfifo_free(conn->writequeue);
2510writequeue_alloc_fail:
2511 kfifo_free(conn->xmitqueue);
2512xmitqueue_alloc_fail:
2513 kfree(conn);
2514conn_alloc_fail:
2515 return iscsi_handle(NULL);
2516}
2517
2518static void
2519iscsi_conn_destroy(iscsi_connh_t connh)
2520{
2521 struct iscsi_conn *conn = iscsi_ptr(connh);
2522 struct iscsi_session *session = conn->session;
2523
2524 down(&conn->xmitsema);
2525 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2526 if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) {
2527 struct sock *sk = conn->sock->sk;
2528
2529 /*
2530 * conn_start() has never been called!
2531 * need to cleanup the socket.
2532 */
2533 write_lock_bh(&sk->sk_callback_lock);
2534 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2535 write_unlock_bh(&sk->sk_callback_lock);
2536
2537 sock_hold(conn->sock->sk);
2538 iscsi_conn_restore_callbacks(conn);
2539 sock_put(conn->sock->sk);
2540 sock_release(conn->sock);
2541 conn->sock = NULL;
2542 }
2543
2544 spin_lock_bh(&session->lock);
2545 conn->c_stage = ISCSI_CONN_CLEANUP_WAIT;
2546 if (session->leadconn == conn) {
2547 /*
2548 * leading connection? then give up on recovery.
2549 */
2550 session->state = ISCSI_STATE_TERMINATE;
2551 wake_up(&conn->ehwait);
2552 }
2553 spin_unlock_bh(&session->lock);
2554
2555 up(&conn->xmitsema);
2556
2557 /*
2558 * Block until all in-progress commands for this connection
2559 * time out or fail.
2560 */
2561 for (;;) {
2562 spin_lock_bh(&conn->lock);
2563 if (!session->host->host_busy) { /* OK for ERL == 0 */
2564 spin_unlock_bh(&conn->lock);
2565 break;
2566 }
2567 spin_unlock_bh(&conn->lock);
2568 msleep_interruptible(500);
2569 printk("conn_destroy(): host_busy %d host_failed %d\n",
2570 session->host->host_busy, session->host->host_failed);
2571 /*
2572 * force eh_abort() to unblock
2573 */
2574 wake_up(&conn->ehwait);
2575 }
2576
2577 /* now free crypto */
2578 if (conn->hdrdgst_en || conn->datadgst_en) {
2579 if (conn->tx_tfm)
2580 crypto_free_tfm(conn->tx_tfm);
2581 if (conn->rx_tfm)
2582 crypto_free_tfm(conn->rx_tfm);
2583 if (conn->data_tx_tfm)
2584 crypto_free_tfm(conn->data_tx_tfm);
2585 if (conn->data_rx_tfm)
2586 crypto_free_tfm(conn->data_rx_tfm);
2587 }
2588
2589 /* free conn->data, size = MaxRecvDataSegmentLength */
2590 if (conn->data_size <= PAGE_SIZE)
2591 kfree(conn->data);
2592 else
2593 free_pages((unsigned long)conn->data,
2594 get_order(conn->data_size));
2595
2596 spin_lock_bh(&session->lock);
2597 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2598 sizeof(void*));
2599 list_del(&conn->item);
2600 if (list_empty(&session->connections))
2601 session->leadconn = NULL;
2602 if (session->leadconn && session->leadconn == conn)
2603 session->leadconn = container_of(session->connections.next,
2604 struct iscsi_conn, item);
2605
2606 if (session->leadconn == NULL)
2607 /* none connections exits.. reset sequencing */
2608 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1;
2609 spin_unlock_bh(&session->lock);
2610
2611 kfifo_free(conn->xmitqueue);
2612 kfifo_free(conn->writequeue);
2613 kfifo_free(conn->immqueue);
2614 kfifo_free(conn->mgmtqueue);
2615 kfree(conn);
2616}
2617
2618static int
2619iscsi_conn_bind(iscsi_sessionh_t sessionh, iscsi_connh_t connh,
2620 uint32_t transport_fd, int is_leading)
2621{
2622 struct iscsi_session *session = iscsi_ptr(sessionh);
2623 struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = iscsi_ptr(connh);
2624 struct sock *sk;
2625 struct socket *sock;
2626 int err;
2627
2628 /* lookup for existing socket */
2629 sock = sockfd_lookup(transport_fd, &err);
2630 if (!sock) {
2631 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
2632 return -EEXIST;
2633 }
2634
2635 /* lookup for existing connection */
2636 spin_lock_bh(&session->lock);
2637 list_for_each_entry(tmp, &session->connections, item) {
2638 if (tmp == conn) {
2639 if (conn->c_stage != ISCSI_CONN_STOPPED ||
2640 conn->stop_stage == STOP_CONN_TERM) {
2641 printk(KERN_ERR "iscsi_tcp: can't bind "
2642 "non-stopped connection (%d:%d)\n",
2643 conn->c_stage, conn->stop_stage);
2644 spin_unlock_bh(&session->lock);
2645 return -EIO;
2646 }
2647 break;
2648 }
2649 }
2650 if (tmp != conn) {
2651 /* bind new iSCSI connection to session */
2652 conn->session = session;
2653
2654 list_add(&conn->item, &session->connections);
2655 }
2656 spin_unlock_bh(&session->lock);
2657
2658 if (conn->stop_stage != STOP_CONN_SUSPEND) {
2659 /* bind iSCSI connection and socket */
2660 conn->sock = sock;
2661
2662 /* setup Socket parameters */
2663 sk = sock->sk;
2664 sk->sk_reuse = 1;
2665 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2666 sk->sk_allocation = GFP_ATOMIC;
2667
2668 /* FIXME: disable Nagle's algorithm */
2669
2670 /*
2671 * Intercept TCP callbacks for sendfile like receive
2672 * processing.
2673 */
2674 iscsi_conn_set_callbacks(conn);
2675
2676 /*
2677 * set receive state machine into initial state
2678 */
2679 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2680 }
2681
2682 if (is_leading)
2683 session->leadconn = conn;
2684
2685 /*
2686 * Unblock xmitworker(), Login Phase will pass through.
2687 */
2688 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2689 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2690
2691 return 0;
2692}
2693
2694static int
2695iscsi_conn_start(iscsi_connh_t connh)
2696{
2697 struct iscsi_conn *conn = iscsi_ptr(connh);
2698 struct iscsi_session *session = conn->session;
2699 struct sock *sk;
2700
2701 /* FF phase warming up... */
2702
2703 if (session == NULL) {
2704 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n");
2705 return -EPERM;
2706 }
2707
2708 sk = conn->sock->sk;
2709
2710 write_lock_bh(&sk->sk_callback_lock);
2711 spin_lock_bh(&session->lock);
2712 conn->c_stage = ISCSI_CONN_STARTED;
2713 session->state = ISCSI_STATE_LOGGED_IN;
2714
2715 switch(conn->stop_stage) {
2716 case STOP_CONN_RECOVER:
2717 /*
2718 * unblock eh_abort() if it is blocked. re-try all
2719 * commands after successful recovery
2720 */
2721 session->conn_cnt++;
2722 conn->stop_stage = 0;
2723 conn->tmabort_state = TMABORT_INITIAL;
2724 session->age++;
2725 wake_up(&conn->ehwait);
2726 break;
2727 case STOP_CONN_TERM:
2728 session->conn_cnt++;
2729 conn->stop_stage = 0;
2730 break;
2731 case STOP_CONN_SUSPEND:
2732 conn->stop_stage = 0;
2733 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2734 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2735 break;
2736 default:
2737 break;
2738 }
2739 spin_unlock_bh(&session->lock);
2740 write_unlock_bh(&sk->sk_callback_lock);
2741
2742 return 0;
2743}
2744
2745static void
2746iscsi_conn_stop(iscsi_connh_t connh, int flag)
2747{
2748 struct iscsi_conn *conn = iscsi_ptr(connh);
2749 struct iscsi_session *session = conn->session;
2750 struct sock *sk;
2751 unsigned long flags;
2752
2753 BUG_ON(!conn->sock);
2754 sk = conn->sock->sk;
2755 write_lock_bh(&sk->sk_callback_lock);
2756 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2757 write_unlock_bh(&sk->sk_callback_lock);
2758
2759 down(&conn->xmitsema);
2760
2761 spin_lock_irqsave(session->host->host_lock, flags);
2762 spin_lock(&session->lock);
2763 conn->stop_stage = flag;
2764 conn->c_stage = ISCSI_CONN_STOPPED;
2765 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2766
2767 if (flag != STOP_CONN_SUSPEND)
2768 session->conn_cnt--;
2769
2770 if (session->conn_cnt == 0 || session->leadconn == conn)
2771 session->state = ISCSI_STATE_FAILED;
2772
2773 spin_unlock(&session->lock);
2774 spin_unlock_irqrestore(session->host->host_lock, flags);
2775
2776 if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) {
2777 struct iscsi_cmd_task *ctask;
2778 struct iscsi_mgmt_task *mtask;
2779
2780 /*
2781 * Socket must go now.
2782 */
2783 sock_hold(conn->sock->sk);
2784 iscsi_conn_restore_callbacks(conn);
2785 sock_put(conn->sock->sk);
2786
2787 /*
2788 * flush xmit queues.
2789 */
2790 spin_lock_bh(&session->lock);
2791 while (__kfifo_get(conn->writequeue, (void*)&ctask,
2792 sizeof(void*)) ||
2793 __kfifo_get(conn->xmitqueue, (void*)&ctask,
2794 sizeof(void*))) {
2795 struct iscsi_r2t_info *r2t;
2796
2797 /*
2798 * flush ctask's r2t queues
2799 */
2800 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
2801 sizeof(void*)))
2802 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
2803 sizeof(void*));
2804
2805 spin_unlock_bh(&session->lock);
2806 local_bh_disable();
2807 iscsi_ctask_cleanup(conn, ctask);
2808 local_bh_enable();
2809 spin_lock_bh(&session->lock);
2810 }
2811 conn->ctask = NULL;
2812 while (__kfifo_get(conn->immqueue, (void*)&mtask,
2813 sizeof(void*)) ||
2814 __kfifo_get(conn->mgmtqueue, (void*)&mtask,
2815 sizeof(void*))) {
2816 __kfifo_put(session->mgmtpool.queue,
2817 (void*)&mtask, sizeof(void*));
2818 }
2819 conn->mtask = NULL;
2820 spin_unlock_bh(&session->lock);
2821
2822 /*
2823 * release socket only after we stopped data_xmit()
2824 * activity and flushed all outstandings
2825 */
2826 sock_release(conn->sock);
2827 conn->sock = NULL;
2828
2829 /*
2830 * for connection level recovery we should not calculate
2831 * header digest. conn->hdr_size used for optimization
2832 * in hdr_extract() and will be re-negotiated at
2833 * set_param() time.
2834 */
2835 if (flag == STOP_CONN_RECOVER)
2836 conn->hdr_size = sizeof(struct iscsi_hdr);
2837 }
2838 up(&conn->xmitsema);
2839}
2840
2841static int
2842iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
2843 char *data, uint32_t data_size)
2844{
2845 struct iscsi_session *session = conn->session;
2846 struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
2847 struct iscsi_mgmt_task *mtask;
2848
2849 spin_lock_bh(&session->lock);
2850 if (session->state == ISCSI_STATE_TERMINATE) {
2851 spin_unlock_bh(&session->lock);
2852 return -EPERM;
2853 }
2854 if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
2855 hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
2856 /*
2857 * Login and Text are sent serially, in
2858 * request-followed-by-response sequence.
2859 * Same mtask can be used. Same ITT must be used.
2860 * Note that login_mtask is preallocated at conn_create().
2861 */
2862 mtask = conn->login_mtask;
2863 else {
2864 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
2865 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
2866
2867 if (!__kfifo_get(session->mgmtpool.queue,
2868 (void*)&mtask, sizeof(void*))) {
2869 spin_unlock_bh(&session->lock);
2870 return -ENOSPC;
2871 }
2872 }
2873
2874 /*
2875 * pre-format CmdSN and ExpStatSN for outgoing PDU.
2876 */
2877 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2878 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) |
2879 (session->age << AGE_SHIFT);
2880 nop->cmdsn = cpu_to_be32(session->cmdsn);
2881 if (conn->c_stage == ISCSI_CONN_STARTED &&
2882 !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2883 session->cmdsn++;
2884 } else
2885 /* do not advance CmdSN */
2886 nop->cmdsn = cpu_to_be32(session->cmdsn);
2887
2888 nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
2889
2890 memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr));
2891
2892 if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE ||
2893 conn->stop_stage == STOP_CONN_RECOVER)
2894 iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr,
2895 sizeof(struct iscsi_hdr));
2896 else
2897 /* this will update header digest */
2898 iscsi_buf_init_hdr(conn, &mtask->headbuf, (char*)&mtask->hdr,
2899 (u8 *)mtask->hdrext);
2900
2901 spin_unlock_bh(&session->lock);
2902
2903 if (data_size) {
2904 memcpy(mtask->data, data, data_size);
2905 mtask->data_count = data_size;
2906 } else
2907 mtask->data_count = 0;
2908
2909 mtask->xmstate = XMSTATE_IMM_HDR;
2910
2911 if (mtask->data_count) {
2912 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data,
2913 mtask->data_count);
2914 }
2915
2916 debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n",
2917 hdr->opcode, hdr->itt, data_size);
2918
2919 /*
2920 * since send_pdu() could be called at least from two contexts,
2921 * we need to serialize __kfifo_put, so we don't have to take
2922 * additional lock on fast data-path
2923 */
2924 if (hdr->opcode & ISCSI_OP_IMMEDIATE)
2925 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
2926 else
2927 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*));
2928
2929 schedule_work(&conn->xmitwork);
2930
2931 return 0;
2932}
2933
2934static int
2935iscsi_eh_host_reset(struct scsi_cmnd *sc)
2936{
2937 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2938 struct iscsi_conn *conn = ctask->conn;
2939 struct iscsi_session *session = conn->session;
2940
2941 spin_lock_bh(&session->lock);
2942 if (session->state == ISCSI_STATE_TERMINATE) {
2943 debug_scsi("failing host reset: session terminated "
2944 "[CID %d age %d]", conn->id, session->age);
2945 spin_unlock_bh(&session->lock);
2946 return FAILED;
2947 }
2948 spin_unlock_bh(&session->lock);
2949
2950 debug_scsi("failing connection CID %d due to SCSI host reset "
2951 "[itt 0x%x age %d]", conn->id, ctask->itt,
2952 session->age);
2953 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
2954
2955 return SUCCESS;
2956}
2957
2958static void
2959iscsi_tmabort_timedout(unsigned long data)
2960{
2961 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data;
2962 struct iscsi_conn *conn = ctask->conn;
2963 struct iscsi_session *session = conn->session;
2964
2965 spin_lock(&session->lock);
2966 if (conn->tmabort_state == TMABORT_INITIAL) {
2967 __kfifo_put(session->mgmtpool.queue,
2968 (void*)&ctask->mtask, sizeof(void*));
2969 conn->tmabort_state = TMABORT_TIMEDOUT;
2970 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n",
2971 (long)ctask->sc, ctask->itt);
2972 /* unblock eh_abort() */
2973 wake_up(&conn->ehwait);
2974 }
2975 spin_unlock(&session->lock);
2976}
2977
2978static int
2979iscsi_eh_abort(struct scsi_cmnd *sc)
2980{
2981 int rc;
2982 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2983 struct iscsi_conn *conn = ctask->conn;
2984 struct iscsi_session *session = conn->session;
2985
2986 conn->eh_abort_cnt++;
2987 debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
2988
2989 /*
2990 * two cases for ERL=0 here:
2991 *
2992 * 1) connection-level failure;
2993 * 2) recovery due protocol error;
2994 */
2995 down(&conn->xmitsema);
2996 spin_lock_bh(&session->lock);
2997 if (session->state != ISCSI_STATE_LOGGED_IN) {
2998 if (session->state == ISCSI_STATE_TERMINATE) {
2999 spin_unlock_bh(&session->lock);
3000 up(&conn->xmitsema);
3001 goto failed;
3002 }
3003 spin_unlock_bh(&session->lock);
3004 } else {
3005 struct iscsi_tm *hdr = &conn->tmhdr;
3006
3007 /*
3008 * Still LOGGED_IN...
3009 */
3010
3011 if (!ctask->sc || sc->SCp.phase != session->age) {
3012 /*
3013 * 1) ctask completed before time out. But session
3014 * is still ok => Happy Retry.
3015 * 2) session was re-open during time out of ctask.
3016 */
3017 spin_unlock_bh(&session->lock);
3018 up(&conn->xmitsema);
3019 goto success;
3020 }
3021 conn->tmabort_state = TMABORT_INITIAL;
3022 spin_unlock_bh(&session->lock);
3023
3024 /*
3025 * ctask timed out but session is OK
3026 * ERL=0 requires task mgmt abort to be issued on each
3027 * failed command. requests must be serialized.
3028 */
3029 memset(hdr, 0, sizeof(struct iscsi_tm));
3030 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
3031 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK;
3032 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3033 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
3034 hdr->rtt = ctask->hdr.itt;
3035 hdr->refcmdsn = ctask->hdr.cmdsn;
3036
3037 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr,
3038 NULL, 0);
3039 if (rc) {
3040 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3041 debug_scsi("abort sent failure [itt 0x%x]", ctask->itt);
3042 } else {
3043 struct iscsi_r2t_info *r2t;
3044
3045 /*
3046 * TMF abort vs. TMF response race logic
3047 */
3048 spin_lock_bh(&session->lock);
3049 ctask->mtask = (struct iscsi_mgmt_task *)
3050 session->mgmt_cmds[(hdr->itt & ITT_MASK) -
3051 ISCSI_MGMT_ITT_OFFSET];
3052 /*
3053 * have to flush r2tqueue to avoid r2t leaks
3054 */
3055 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
3056 sizeof(void*))) {
3057 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
3058 sizeof(void*));
3059 }
3060 if (conn->tmabort_state == TMABORT_INITIAL) {
3061 conn->tmfcmd_pdus_cnt++;
3062 conn->tmabort_timer.expires = 3*HZ + jiffies;
3063 conn->tmabort_timer.function =
3064 iscsi_tmabort_timedout;
3065 conn->tmabort_timer.data = (unsigned long)ctask;
3066 add_timer(&conn->tmabort_timer);
3067 debug_scsi("abort sent [itt 0x%x]", ctask->itt);
3068 } else {
3069 if (!ctask->sc ||
3070 conn->tmabort_state == TMABORT_SUCCESS) {
3071 conn->tmabort_state = TMABORT_INITIAL;
3072 spin_unlock_bh(&session->lock);
3073 up(&conn->xmitsema);
3074 goto success;
3075 }
3076 conn->tmabort_state = TMABORT_INITIAL;
3077 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3078 }
3079 spin_unlock_bh(&session->lock);
3080 }
3081 }
3082 up(&conn->xmitsema);
3083
3084
3085 /*
3086 * block eh thread until:
3087 *
3088 * 1) abort response;
3089 * 2) abort timeout;
3090 * 3) session re-opened;
3091 * 4) session terminated;
3092 */
3093 for (;;) {
3094 int p_state = session->state;
3095
3096 rc = wait_event_interruptible(conn->ehwait,
3097 (p_state == ISCSI_STATE_LOGGED_IN ?
3098 (session->state == ISCSI_STATE_TERMINATE ||
3099 conn->tmabort_state != TMABORT_INITIAL) :
3100 (session->state == ISCSI_STATE_TERMINATE ||
3101 session->state == ISCSI_STATE_LOGGED_IN)));
3102 if (rc) {
3103 /* shutdown.. */
3104 session->state = ISCSI_STATE_TERMINATE;
3105 goto failed;
3106 }
3107
3108 if (signal_pending(current))
3109 flush_signals(current);
3110
3111 if (session->state == ISCSI_STATE_TERMINATE)
3112 goto failed;
3113
3114 spin_lock_bh(&session->lock);
3115 if (sc->SCp.phase == session->age &&
3116 (conn->tmabort_state == TMABORT_TIMEDOUT ||
3117 conn->tmabort_state == TMABORT_FAILED)) {
3118 conn->tmabort_state = TMABORT_INITIAL;
3119 if (!ctask->sc) {
3120 /*
3121 * ctask completed before tmf abort response or
3122 * time out.
3123 * But session is still ok => Happy Retry.
3124 */
3125 spin_unlock_bh(&session->lock);
3126 break;
3127 }
3128 spin_unlock_bh(&session->lock);
3129 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3130 continue;
3131 }
3132 spin_unlock_bh(&session->lock);
3133 break;
3134 }
3135
3136success:
3137 debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3138 rc = SUCCESS;
3139 goto exit;
3140
3141failed:
3142 debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3143 rc = FAILED;
3144
3145exit:
3146 del_timer_sync(&conn->tmabort_timer);
3147
3148 down(&conn->xmitsema);
3149 if (conn->sock) {
3150 struct sock *sk = conn->sock->sk;
3151
3152 write_lock_bh(&sk->sk_callback_lock);
3153 iscsi_ctask_cleanup(conn, ctask);
3154 write_unlock_bh(&sk->sk_callback_lock);
3155 }
3156 up(&conn->xmitsema);
3157 return rc;
3158}
3159
3160static int
3161iscsi_r2tpool_alloc(struct iscsi_session *session)
3162{
3163 int i;
3164 int cmd_i;
3165
3166 /*
3167 * initialize per-task: R2T pool and xmit queue
3168 */
3169 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3170 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3171
3172 /*
3173 * pre-allocated x4 as much r2ts to handle race when
3174 * target acks DataOut faster than we data_xmit() queues
3175 * could replenish r2tqueue.
3176 */
3177
3178 /* R2T pool */
3179 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4,
3180 (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) {
3181 goto r2t_alloc_fail;
3182 }
3183
3184 /* R2T xmit queue */
3185 ctask->r2tqueue = kfifo_alloc(
3186 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
3187 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
3188 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3189 goto r2t_alloc_fail;
3190 }
3191
3192 /*
3193 * number of
3194 * Data-Out PDU's within R2T-sequence can be quite big;
3195 * using mempool
3196 */
3197 ctask->datapool = mempool_create(ISCSI_DTASK_DEFAULT_MAX,
3198 mempool_alloc_slab, mempool_free_slab, taskcache);
3199 if (ctask->datapool == NULL) {
3200 kfifo_free(ctask->r2tqueue);
3201 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3202 goto r2t_alloc_fail;
3203 }
3204 INIT_LIST_HEAD(&ctask->dataqueue);
3205 }
3206
3207 return 0;
3208
3209r2t_alloc_fail:
3210 for (i = 0; i < cmd_i; i++) {
3211 mempool_destroy(session->cmds[i]->datapool);
3212 kfifo_free(session->cmds[i]->r2tqueue);
3213 iscsi_pool_free(&session->cmds[i]->r2tpool,
3214 (void**)session->cmds[i]->r2ts);
3215 }
3216 return -ENOMEM;
3217}
3218
3219static void
3220iscsi_r2tpool_free(struct iscsi_session *session)
3221{
3222 int i;
3223
3224 for (i = 0; i < session->cmds_max; i++) {
3225 mempool_destroy(session->cmds[i]->datapool);
3226 kfifo_free(session->cmds[i]->r2tqueue);
3227 iscsi_pool_free(&session->cmds[i]->r2tpool,
3228 (void**)session->cmds[i]->r2ts);
3229 }
3230}
3231
3232static struct scsi_host_template iscsi_sht = {
3233 .name = "iSCSI Initiator over TCP/IP, v."
3234 ISCSI_VERSION_STR,
3235 .queuecommand = iscsi_queuecommand,
3236 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
3237 .sg_tablesize = ISCSI_SG_TABLESIZE,
3238 .cmd_per_lun = ISCSI_CMD_PER_LUN,
3239 .eh_abort_handler = iscsi_eh_abort,
3240 .eh_host_reset_handler = iscsi_eh_host_reset,
3241 .use_clustering = DISABLE_CLUSTERING,
3242 .proc_name = "iscsi_tcp",
3243 .this_id = -1,
3244};
3245
3246static iscsi_sessionh_t
3247iscsi_session_create(uint32_t initial_cmdsn, struct Scsi_Host *host)
3248{
3249 int cmd_i;
3250 struct iscsi_session *session;
3251
3252 session = iscsi_hostdata(host->hostdata);
3253 memset(session, 0, sizeof(struct iscsi_session));
3254
3255 session->host = host;
3256 session->id = host->host_no;
3257 session->state = ISCSI_STATE_LOGGED_IN;
3258 session->mgmtpool_max = ISCSI_MGMT_CMDS_MAX;
3259 session->cmds_max = ISCSI_XMIT_CMDS_MAX;
3260 session->cmdsn = initial_cmdsn;
3261 session->exp_cmdsn = initial_cmdsn + 1;
3262 session->max_cmdsn = initial_cmdsn + 1;
3263 session->max_r2t = 1;
3264
3265 /* initialize SCSI PDU commands pool */
3266 if (iscsi_pool_init(&session->cmdpool, session->cmds_max,
3267 (void***)&session->cmds, sizeof(struct iscsi_cmd_task)))
3268 goto cmdpool_alloc_fail;
3269
3270 /* pre-format cmds pool with ITT */
3271 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++)
3272 session->cmds[cmd_i]->itt = cmd_i;
3273
3274 spin_lock_init(&session->lock);
3275 INIT_LIST_HEAD(&session->connections);
3276
3277 /* initialize immediate command pool */
3278 if (iscsi_pool_init(&session->mgmtpool, session->mgmtpool_max,
3279 (void***)&session->mgmt_cmds, sizeof(struct iscsi_mgmt_task)))
3280 goto mgmtpool_alloc_fail;
3281
3282
3283 /* pre-format immediate cmds pool with ITT */
3284 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
3285 session->mgmt_cmds[cmd_i]->itt = ISCSI_MGMT_ITT_OFFSET + cmd_i;
3286 session->mgmt_cmds[cmd_i]->data = kmalloc(
3287 DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, GFP_KERNEL);
3288 if (!session->mgmt_cmds[cmd_i]->data) {
3289 int j;
3290
3291 for (j = 0; j < cmd_i; j++)
3292 kfree(session->mgmt_cmds[j]->data);
3293 goto immdata_alloc_fail;
3294 }
3295 }
3296
3297 if (iscsi_r2tpool_alloc(session))
3298 goto r2tpool_alloc_fail;
3299
3300 return iscsi_handle(session);
3301
3302r2tpool_alloc_fail:
3303 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3304 kfree(session->mgmt_cmds[cmd_i]->data);
3305 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3306immdata_alloc_fail:
3307mgmtpool_alloc_fail:
3308 iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3309cmdpool_alloc_fail:
3310 return iscsi_handle(NULL);
3311}
3312
3313static void
3314iscsi_session_destroy(iscsi_sessionh_t sessionh)
3315{
3316 int cmd_i;
3317 struct iscsi_data_task *dtask, *n;
3318 struct iscsi_session *session = iscsi_ptr(sessionh);
3319
3320 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3321 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3322 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
3323 list_del(&dtask->item);
3324 mempool_free(dtask, ctask->datapool);
3325 }
3326 }
3327
3328 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3329 kfree(session->mgmt_cmds[cmd_i]->data);
3330
3331 iscsi_r2tpool_free(session);
3332 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3333 iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3334}
3335
3336static int
3337iscsi_conn_set_param(iscsi_connh_t connh, enum iscsi_param param,
3338 uint32_t value)
3339{
3340 struct iscsi_conn *conn = iscsi_ptr(connh);
3341 struct iscsi_session *session = conn->session;
3342
3343 spin_lock_bh(&session->lock);
3344 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
3345 conn->stop_stage != STOP_CONN_RECOVER) {
3346 printk(KERN_ERR "iscsi_tcp: can not change parameter [%d]\n",
3347 param);
3348 spin_unlock_bh(&session->lock);
3349 return 0;
3350 }
3351 spin_unlock_bh(&session->lock);
3352
3353 switch(param) {
3354 case ISCSI_PARAM_MAX_RECV_DLENGTH: {
3355 char *saveptr = conn->data;
3356 int flags = GFP_KERNEL;
3357
3358 if (conn->data_size >= value) {
3359 conn->max_recv_dlength = value;
3360 break;
3361 }
3362
3363 spin_lock_bh(&session->lock);
3364 if (conn->stop_stage == STOP_CONN_RECOVER)
3365 flags = GFP_ATOMIC;
3366 spin_unlock_bh(&session->lock);
3367
3368 if (value <= PAGE_SIZE)
3369 conn->data = kmalloc(value, flags);
3370 else
3371 conn->data = (void*)__get_free_pages(flags,
3372 get_order(value));
3373 if (conn->data == NULL) {
3374 conn->data = saveptr;
3375 return -ENOMEM;
3376 }
3377 if (conn->data_size <= PAGE_SIZE)
3378 kfree(saveptr);
3379 else
3380 free_pages((unsigned long)saveptr,
3381 get_order(conn->data_size));
3382 conn->max_recv_dlength = value;
3383 conn->data_size = value;
3384 }
3385 break;
3386 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3387 conn->max_xmit_dlength = value;
3388 break;
3389 case ISCSI_PARAM_HDRDGST_EN:
3390 conn->hdrdgst_en = value;
3391 conn->hdr_size = sizeof(struct iscsi_hdr);
3392 if (conn->hdrdgst_en) {
3393 conn->hdr_size += sizeof(__u32);
3394 if (!conn->tx_tfm)
3395 conn->tx_tfm = crypto_alloc_tfm("crc32c", 0);
3396 if (!conn->tx_tfm)
3397 return -ENOMEM;
3398 if (!conn->rx_tfm)
3399 conn->rx_tfm = crypto_alloc_tfm("crc32c", 0);
3400 if (!conn->rx_tfm) {
3401 crypto_free_tfm(conn->tx_tfm);
3402 return -ENOMEM;
3403 }
3404 } else {
3405 if (conn->tx_tfm)
3406 crypto_free_tfm(conn->tx_tfm);
3407 if (conn->rx_tfm)
3408 crypto_free_tfm(conn->rx_tfm);
3409 }
3410 break;
3411 case ISCSI_PARAM_DATADGST_EN:
3412 conn->datadgst_en = value;
3413 if (conn->datadgst_en) {
3414 if (!conn->data_tx_tfm)
3415 conn->data_tx_tfm =
3416 crypto_alloc_tfm("crc32c", 0);
3417 if (!conn->data_tx_tfm)
3418 return -ENOMEM;
3419 if (!conn->data_rx_tfm)
3420 conn->data_rx_tfm =
3421 crypto_alloc_tfm("crc32c", 0);
3422 if (!conn->data_rx_tfm) {
3423 crypto_free_tfm(conn->data_tx_tfm);
3424 return -ENOMEM;
3425 }
3426 } else {
3427 if (conn->data_tx_tfm)
3428 crypto_free_tfm(conn->data_tx_tfm);
3429 if (conn->data_rx_tfm)
3430 crypto_free_tfm(conn->data_rx_tfm);
3431 }
3432 break;
3433 case ISCSI_PARAM_INITIAL_R2T_EN:
3434 session->initial_r2t_en = value;
3435 break;
3436 case ISCSI_PARAM_MAX_R2T:
3437 if (session->max_r2t == roundup_pow_of_two(value))
3438 break;
3439 iscsi_r2tpool_free(session);
3440 session->max_r2t = value;
3441 if (session->max_r2t & (session->max_r2t - 1))
3442 session->max_r2t = roundup_pow_of_two(session->max_r2t);
3443 if (iscsi_r2tpool_alloc(session))
3444 return -ENOMEM;
3445 break;
3446 case ISCSI_PARAM_IMM_DATA_EN:
3447 session->imm_data_en = value;
3448 break;
3449 case ISCSI_PARAM_FIRST_BURST:
3450 session->first_burst = value;
3451 break;
3452 case ISCSI_PARAM_MAX_BURST:
3453 session->max_burst = value;
3454 break;
3455 case ISCSI_PARAM_PDU_INORDER_EN:
3456 session->pdu_inorder_en = value;
3457 break;
3458 case ISCSI_PARAM_DATASEQ_INORDER_EN:
3459 session->dataseq_inorder_en = value;
3460 break;
3461 case ISCSI_PARAM_ERL:
3462 session->erl = value;
3463 break;
3464 case ISCSI_PARAM_IFMARKER_EN:
3465 BUG_ON(value);
3466 session->ifmarker_en = value;
3467 break;
3468 case ISCSI_PARAM_OFMARKER_EN:
3469 BUG_ON(value);
3470 session->ofmarker_en = value;
3471 break;
3472 default:
3473 break;
3474 }
3475
3476 return 0;
3477}
3478
3479static int
3480iscsi_conn_get_param(iscsi_connh_t connh, enum iscsi_param param,
3481 uint32_t *value)
3482{
3483 struct iscsi_conn *conn = iscsi_ptr(connh);
3484 struct iscsi_session *session = conn->session;
3485
3486 switch(param) {
3487 case ISCSI_PARAM_MAX_RECV_DLENGTH:
3488 *value = conn->max_recv_dlength;
3489 break;
3490 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3491 *value = conn->max_xmit_dlength;
3492 break;
3493 case ISCSI_PARAM_HDRDGST_EN:
3494 *value = conn->hdrdgst_en;
3495 break;
3496 case ISCSI_PARAM_DATADGST_EN:
3497 *value = conn->datadgst_en;
3498 break;
3499 case ISCSI_PARAM_INITIAL_R2T_EN:
3500 *value = session->initial_r2t_en;
3501 break;
3502 case ISCSI_PARAM_MAX_R2T:
3503 *value = session->max_r2t;
3504 break;
3505 case ISCSI_PARAM_IMM_DATA_EN:
3506 *value = session->imm_data_en;
3507 break;
3508 case ISCSI_PARAM_FIRST_BURST:
3509 *value = session->first_burst;
3510 break;
3511 case ISCSI_PARAM_MAX_BURST:
3512 *value = session->max_burst;
3513 break;
3514 case ISCSI_PARAM_PDU_INORDER_EN:
3515 *value = session->pdu_inorder_en;
3516 break;
3517 case ISCSI_PARAM_DATASEQ_INORDER_EN:
3518 *value = session->dataseq_inorder_en;
3519 break;
3520 case ISCSI_PARAM_ERL:
3521 *value = session->erl;
3522 break;
3523 case ISCSI_PARAM_IFMARKER_EN:
3524 *value = session->ifmarker_en;
3525 break;
3526 case ISCSI_PARAM_OFMARKER_EN:
3527 *value = session->ofmarker_en;
3528 break;
3529 default:
3530 return ISCSI_ERR_PARAM_NOT_FOUND;
3531 }
3532
3533 return 0;
3534}
3535
3536static void
3537iscsi_conn_get_stats(iscsi_connh_t connh, struct iscsi_stats *stats)
3538{
3539 struct iscsi_conn *conn = iscsi_ptr(connh);
3540
3541 stats->txdata_octets = conn->txdata_octets;
3542 stats->rxdata_octets = conn->rxdata_octets;
3543 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
3544 stats->dataout_pdus = conn->dataout_pdus_cnt;
3545 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
3546 stats->datain_pdus = conn->datain_pdus_cnt;
3547 stats->r2t_pdus = conn->r2t_pdus_cnt;
3548 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
3549 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
3550 stats->custom_length = 3;
3551 strcpy(stats->custom[0].desc, "tx_sendpage_failures");
3552 stats->custom[0].value = conn->sendpage_failures_cnt;
3553 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
3554 stats->custom[1].value = conn->discontiguous_hdr_cnt;
3555 strcpy(stats->custom[2].desc, "eh_abort_cnt");
3556 stats->custom[2].value = conn->eh_abort_cnt;
3557}
3558
3559static int
3560iscsi_conn_send_pdu(iscsi_connh_t connh, struct iscsi_hdr *hdr, char *data,
3561 uint32_t data_size)
3562{
3563 struct iscsi_conn *conn = iscsi_ptr(connh);
3564 int rc;
3565
3566 down(&conn->xmitsema);
3567 rc = iscsi_conn_send_generic(conn, hdr, data, data_size);
3568 up(&conn->xmitsema);
3569
3570 return rc;
3571}
3572
3573static struct iscsi_transport iscsi_tcp_transport = {
3574 .owner = THIS_MODULE,
3575 .name = "tcp",
3576 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
3577 | CAP_DATADGST,
3578 .host_template = &iscsi_sht,
3579 .hostdata_size = sizeof(struct iscsi_session),
3580 .max_conn = 1,
3581 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN,
3582 .create_session = iscsi_session_create,
3583 .destroy_session = iscsi_session_destroy,
3584 .create_conn = iscsi_conn_create,
3585 .bind_conn = iscsi_conn_bind,
3586 .destroy_conn = iscsi_conn_destroy,
3587 .set_param = iscsi_conn_set_param,
3588 .get_param = iscsi_conn_get_param,
3589 .start_conn = iscsi_conn_start,
3590 .stop_conn = iscsi_conn_stop,
3591 .send_pdu = iscsi_conn_send_pdu,
3592 .get_stats = iscsi_conn_get_stats,
3593};
3594
3595static int __init
3596iscsi_tcp_init(void)
3597{
3598 int error;
3599
3600 if (iscsi_max_lun < 1) {
3601 printk(KERN_ERR "Invalid max_lun value of %u\n", iscsi_max_lun);
3602 return -EINVAL;
3603 }
3604 iscsi_tcp_transport.max_lun = iscsi_max_lun;
3605
3606 taskcache = kmem_cache_create("iscsi_taskcache",
3607 sizeof(struct iscsi_data_task), 0,
3608 SLAB_HWCACHE_ALIGN | SLAB_NO_REAP, NULL, NULL);
3609 if (!taskcache)
3610 return -ENOMEM;
3611
3612 error = iscsi_register_transport(&iscsi_tcp_transport);
3613 if (error)
3614 kmem_cache_destroy(taskcache);
3615
3616 return error;
3617}
3618
3619static void __exit
3620iscsi_tcp_exit(void)
3621{
3622 iscsi_unregister_transport(&iscsi_tcp_transport);
3623 kmem_cache_destroy(taskcache);
3624}
3625
3626module_init(iscsi_tcp_init);
3627module_exit(iscsi_tcp_exit);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-21 22:03:57 -0500