aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi/iscsi_tcp.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/iscsi_tcp.c')
-rw-r--r--drivers/scsi/iscsi_tcp.c1134
1 files changed, 402 insertions, 732 deletions
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index b4743a9ecc80..0a9dbc59663f 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -43,13 +43,10 @@
43 43
44#include "iscsi_tcp.h" 44#include "iscsi_tcp.h"
45 45
46#define ISCSI_TCP_VERSION "1.0-595"
47
48MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, " 46MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
49 "Alex Aizman <itn780@yahoo.com>"); 47 "Alex Aizman <itn780@yahoo.com>");
50MODULE_DESCRIPTION("iSCSI/TCP data-path"); 48MODULE_DESCRIPTION("iSCSI/TCP data-path");
51MODULE_LICENSE("GPL"); 49MODULE_LICENSE("GPL");
52MODULE_VERSION(ISCSI_TCP_VERSION);
53/* #define DEBUG_TCP */ 50/* #define DEBUG_TCP */
54#define DEBUG_ASSERT 51#define DEBUG_ASSERT
55 52
@@ -111,8 +108,8 @@ iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
111{ 108{
112 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 109 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
113 110
114 crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc); 111 crypto_hash_digest(&tcp_conn->tx_hash, &buf->sg, buf->sg.length, crc);
115 buf->sg.length += sizeof(uint32_t); 112 buf->sg.length = tcp_conn->hdr_size;
116} 113}
117 114
118static inline int 115static inline int
@@ -185,11 +182,19 @@ iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
185 * must be called with session lock 182 * must be called with session lock
186 */ 183 */
187static void 184static void
188__iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 185iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
189{ 186{
190 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 187 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
188 struct iscsi_r2t_info *r2t;
191 struct scsi_cmnd *sc; 189 struct scsi_cmnd *sc;
192 190
191 /* flush ctask's r2t queues */
192 while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
193 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
194 sizeof(void*));
195 debug_scsi("iscsi_tcp_cleanup_ctask pending r2t dropped\n");
196 }
197
193 sc = ctask->sc; 198 sc = ctask->sc;
194 if (unlikely(!sc)) 199 if (unlikely(!sc))
195 return; 200 return;
@@ -276,7 +281,6 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
276{ 281{
277 struct iscsi_data *hdr; 282 struct iscsi_data *hdr;
278 struct scsi_cmnd *sc = ctask->sc; 283 struct scsi_cmnd *sc = ctask->sc;
279 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
280 284
281 hdr = &r2t->dtask.hdr; 285 hdr = &r2t->dtask.hdr;
282 memset(hdr, 0, sizeof(struct iscsi_data)); 286 memset(hdr, 0, sizeof(struct iscsi_data));
@@ -331,10 +335,12 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
331 sg_count += sg->length; 335 sg_count += sg->length;
332 } 336 }
333 BUG_ON(r2t->sg == NULL); 337 BUG_ON(r2t->sg == NULL);
334 } else 338 } else {
335 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 339 iscsi_buf_init_iov(&r2t->sendbuf,
336 (char*)sc->request_buffer + r2t->data_offset, 340 (char*)sc->request_buffer + r2t->data_offset,
337 r2t->data_count); 341 r2t->data_count);
342 r2t->sg = NULL;
343 }
338} 344}
339 345
340/** 346/**
@@ -353,8 +359,11 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
353 int r2tsn = be32_to_cpu(rhdr->r2tsn); 359 int r2tsn = be32_to_cpu(rhdr->r2tsn);
354 int rc; 360 int rc;
355 361
356 if (tcp_conn->in.datalen) 362 if (tcp_conn->in.datalen) {
363 printk(KERN_ERR "iscsi_tcp: invalid R2t with datalen %d\n",
364 tcp_conn->in.datalen);
357 return ISCSI_ERR_DATALEN; 365 return ISCSI_ERR_DATALEN;
366 }
358 367
359 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn) 368 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
360 return ISCSI_ERR_R2TSN; 369 return ISCSI_ERR_R2TSN;
@@ -374,20 +383,29 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
374 spin_unlock(&session->lock); 383 spin_unlock(&session->lock);
375 return 0; 384 return 0;
376 } 385 }
386
377 rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 387 rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
378 BUG_ON(!rc); 388 BUG_ON(!rc);
379 389
380 r2t->exp_statsn = rhdr->statsn; 390 r2t->exp_statsn = rhdr->statsn;
381 r2t->data_length = be32_to_cpu(rhdr->data_length); 391 r2t->data_length = be32_to_cpu(rhdr->data_length);
382 if (r2t->data_length == 0 || 392 if (r2t->data_length == 0) {
383 r2t->data_length > session->max_burst) { 393 printk(KERN_ERR "iscsi_tcp: invalid R2T with zero data len\n");
384 spin_unlock(&session->lock); 394 spin_unlock(&session->lock);
385 return ISCSI_ERR_DATALEN; 395 return ISCSI_ERR_DATALEN;
386 } 396 }
387 397
398 if (r2t->data_length > session->max_burst)
399 debug_scsi("invalid R2T with data len %u and max burst %u."
400 "Attempting to execute request.\n",
401 r2t->data_length, session->max_burst);
402
388 r2t->data_offset = be32_to_cpu(rhdr->data_offset); 403 r2t->data_offset = be32_to_cpu(rhdr->data_offset);
389 if (r2t->data_offset + r2t->data_length > ctask->total_length) { 404 if (r2t->data_offset + r2t->data_length > ctask->total_length) {
390 spin_unlock(&session->lock); 405 spin_unlock(&session->lock);
406 printk(KERN_ERR "iscsi_tcp: invalid R2T with data len %u at "
407 "offset %u and total length %d\n", r2t->data_length,
408 r2t->data_offset, ctask->total_length);
391 return ISCSI_ERR_DATALEN; 409 return ISCSI_ERR_DATALEN;
392 } 410 }
393 411
@@ -399,7 +417,7 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
399 tcp_ctask->exp_r2tsn = r2tsn + 1; 417 tcp_ctask->exp_r2tsn = r2tsn + 1;
400 tcp_ctask->xmstate |= XMSTATE_SOL_HDR; 418 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
401 __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*)); 419 __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
402 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*)); 420 list_move_tail(&ctask->running, &conn->xmitqueue);
403 421
404 scsi_queue_work(session->host, &conn->xmitwork); 422 scsi_queue_work(session->host, &conn->xmitwork);
405 conn->r2t_pdus_cnt++; 423 conn->r2t_pdus_cnt++;
@@ -450,7 +468,8 @@ iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
450 468
451 sg_init_one(&sg, (u8 *)hdr, 469 sg_init_one(&sg, (u8 *)hdr,
452 sizeof(struct iscsi_hdr) + ahslen); 470 sizeof(struct iscsi_hdr) + ahslen);
453 crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst); 471 crypto_hash_digest(&tcp_conn->rx_hash, &sg, sg.length,
472 (u8 *)&cdgst);
454 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) + 473 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
455 ahslen); 474 ahslen);
456 if (cdgst != rdgst) { 475 if (cdgst != rdgst) {
@@ -477,6 +496,8 @@ iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
477 case ISCSI_OP_SCSI_DATA_IN: 496 case ISCSI_OP_SCSI_DATA_IN:
478 tcp_conn->in.ctask = session->cmds[itt]; 497 tcp_conn->in.ctask = session->cmds[itt];
479 rc = iscsi_data_rsp(conn, tcp_conn->in.ctask); 498 rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
499 if (rc)
500 return rc;
480 /* fall through */ 501 /* fall through */
481 case ISCSI_OP_SCSI_CMD_RSP: 502 case ISCSI_OP_SCSI_CMD_RSP:
482 tcp_conn->in.ctask = session->cmds[itt]; 503 tcp_conn->in.ctask = session->cmds[itt];
@@ -484,7 +505,6 @@ iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
484 goto copy_hdr; 505 goto copy_hdr;
485 506
486 spin_lock(&session->lock); 507 spin_lock(&session->lock);
487 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
488 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0); 508 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
489 spin_unlock(&session->lock); 509 spin_unlock(&session->lock);
490 break; 510 break;
@@ -500,13 +520,28 @@ iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
500 break; 520 break;
501 case ISCSI_OP_LOGIN_RSP: 521 case ISCSI_OP_LOGIN_RSP:
502 case ISCSI_OP_TEXT_RSP: 522 case ISCSI_OP_TEXT_RSP:
503 case ISCSI_OP_LOGOUT_RSP:
504 case ISCSI_OP_NOOP_IN:
505 case ISCSI_OP_REJECT: 523 case ISCSI_OP_REJECT:
506 case ISCSI_OP_ASYNC_EVENT: 524 case ISCSI_OP_ASYNC_EVENT:
525 /*
526 * It is possible that we could get a PDU with a buffer larger
527 * than 8K, but there are no targets that currently do this.
528 * For now we fail until we find a vendor that needs it
529 */
530 if (DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH <
531 tcp_conn->in.datalen) {
532 printk(KERN_ERR "iscsi_tcp: received buffer of len %u "
533 "but conn buffer is only %u (opcode %0x)\n",
534 tcp_conn->in.datalen,
535 DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, opcode);
536 rc = ISCSI_ERR_PROTO;
537 break;
538 }
539
507 if (tcp_conn->in.datalen) 540 if (tcp_conn->in.datalen)
508 goto copy_hdr; 541 goto copy_hdr;
509 /* fall through */ 542 /* fall through */
543 case ISCSI_OP_LOGOUT_RSP:
544 case ISCSI_OP_NOOP_IN:
510 case ISCSI_OP_SCSI_TMFUNC_RSP: 545 case ISCSI_OP_SCSI_TMFUNC_RSP:
511 rc = iscsi_complete_pdu(conn, hdr, NULL, 0); 546 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
512 break; 547 break;
@@ -523,7 +558,7 @@ copy_hdr:
523 * skbs to complete the command then we have to copy the header 558 * skbs to complete the command then we have to copy the header
524 * for later use 559 * for later use
525 */ 560 */
526 if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy < 561 if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <=
527 (tcp_conn->in.datalen + tcp_conn->in.padding + 562 (tcp_conn->in.datalen + tcp_conn->in.padding +
528 (conn->datadgst_en ? 4 : 0))) { 563 (conn->datadgst_en ? 4 : 0))) {
529 debug_tcp("Copying header for later use. in.copy %d in.datalen" 564 debug_tcp("Copying header for later use. in.copy %d in.datalen"
@@ -614,10 +649,9 @@ iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
614 * byte counters. 649 * byte counters.
615 **/ 650 **/
616static inline int 651static inline int
617iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn) 652iscsi_tcp_copy(struct iscsi_conn *conn, int buf_size)
618{ 653{
619 void *buf = tcp_conn->data; 654 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
620 int buf_size = tcp_conn->in.datalen;
621 int buf_left = buf_size - tcp_conn->data_copied; 655 int buf_left = buf_size - tcp_conn->data_copied;
622 int size = min(tcp_conn->in.copy, buf_left); 656 int size = min(tcp_conn->in.copy, buf_left);
623 int rc; 657 int rc;
@@ -627,7 +661,7 @@ iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn)
627 BUG_ON(size <= 0); 661 BUG_ON(size <= 0);
628 662
629 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset, 663 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
630 (char*)buf + tcp_conn->data_copied, size); 664 (char*)conn->data + tcp_conn->data_copied, size);
631 BUG_ON(rc); 665 BUG_ON(rc);
632 666
633 tcp_conn->in.offset += size; 667 tcp_conn->in.offset += size;
@@ -642,15 +676,15 @@ iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn)
642} 676}
643 677
644static inline void 678static inline void
645partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn, 679partial_sg_digest_update(struct hash_desc *desc, struct scatterlist *sg,
646 struct scatterlist *sg, int offset, int length) 680 int offset, int length)
647{ 681{
648 struct scatterlist temp; 682 struct scatterlist temp;
649 683
650 memcpy(&temp, sg, sizeof(struct scatterlist)); 684 memcpy(&temp, sg, sizeof(struct scatterlist));
651 temp.offset = offset; 685 temp.offset = offset;
652 temp.length = length; 686 temp.length = length;
653 crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1); 687 crypto_hash_update(desc, &temp, length);
654} 688}
655 689
656static void 690static void
@@ -659,7 +693,7 @@ iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
659 struct scatterlist tmp; 693 struct scatterlist tmp;
660 694
661 sg_init_one(&tmp, buf, len); 695 sg_init_one(&tmp, buf, len);
662 crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1); 696 crypto_hash_update(&tcp_conn->rx_hash, &tmp, len);
663} 697}
664 698
665static int iscsi_scsi_data_in(struct iscsi_conn *conn) 699static int iscsi_scsi_data_in(struct iscsi_conn *conn)
@@ -713,11 +747,12 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
713 if (!rc) { 747 if (!rc) {
714 if (conn->datadgst_en) { 748 if (conn->datadgst_en) {
715 if (!offset) 749 if (!offset)
716 crypto_digest_update( 750 crypto_hash_update(
717 tcp_conn->data_rx_tfm, 751 &tcp_conn->rx_hash,
718 &sg[i], 1); 752 &sg[i], 1);
719 else 753 else
720 partial_sg_digest_update(tcp_conn, 754 partial_sg_digest_update(
755 &tcp_conn->rx_hash,
721 &sg[i], 756 &sg[i],
722 sg[i].offset + offset, 757 sg[i].offset + offset,
723 sg[i].length - offset); 758 sg[i].length - offset);
@@ -731,8 +766,10 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
731 /* 766 /*
732 * data-in is complete, but buffer not... 767 * data-in is complete, but buffer not...
733 */ 768 */
734 partial_sg_digest_update(tcp_conn, &sg[i], 769 partial_sg_digest_update(&tcp_conn->rx_hash,
735 sg[i].offset, sg[i].length-rc); 770 &sg[i],
771 sg[i].offset,
772 sg[i].length-rc);
736 rc = 0; 773 rc = 0;
737 break; 774 break;
738 } 775 }
@@ -745,10 +782,10 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
745done: 782done:
746 /* check for non-exceptional status */ 783 /* check for non-exceptional status */
747 if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) { 784 if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
748 debug_scsi("done [sc %lx res %d itt 0x%x]\n", 785 debug_scsi("done [sc %lx res %d itt 0x%x flags 0x%x]\n",
749 (long)sc, sc->result, ctask->itt); 786 (long)sc, sc->result, ctask->itt,
787 tcp_conn->in.hdr->flags);
750 spin_lock(&conn->session->lock); 788 spin_lock(&conn->session->lock);
751 __iscsi_ctask_cleanup(conn, ctask);
752 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0); 789 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
753 spin_unlock(&conn->session->lock); 790 spin_unlock(&conn->session->lock);
754 } 791 }
@@ -768,27 +805,23 @@ iscsi_data_recv(struct iscsi_conn *conn)
768 rc = iscsi_scsi_data_in(conn); 805 rc = iscsi_scsi_data_in(conn);
769 break; 806 break;
770 case ISCSI_OP_SCSI_CMD_RSP: 807 case ISCSI_OP_SCSI_CMD_RSP:
771 spin_lock(&conn->session->lock);
772 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
773 spin_unlock(&conn->session->lock);
774 case ISCSI_OP_TEXT_RSP: 808 case ISCSI_OP_TEXT_RSP:
775 case ISCSI_OP_LOGIN_RSP: 809 case ISCSI_OP_LOGIN_RSP:
776 case ISCSI_OP_NOOP_IN:
777 case ISCSI_OP_ASYNC_EVENT: 810 case ISCSI_OP_ASYNC_EVENT:
778 case ISCSI_OP_REJECT: 811 case ISCSI_OP_REJECT:
779 /* 812 /*
780 * Collect data segment to the connection's data 813 * Collect data segment to the connection's data
781 * placeholder 814 * placeholder
782 */ 815 */
783 if (iscsi_tcp_copy(tcp_conn)) { 816 if (iscsi_tcp_copy(conn, tcp_conn->in.datalen)) {
784 rc = -EAGAIN; 817 rc = -EAGAIN;
785 goto exit; 818 goto exit;
786 } 819 }
787 820
788 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, tcp_conn->data, 821 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, conn->data,
789 tcp_conn->in.datalen); 822 tcp_conn->in.datalen);
790 if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP) 823 if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP)
791 iscsi_recv_digest_update(tcp_conn, tcp_conn->data, 824 iscsi_recv_digest_update(tcp_conn, conn->data,
792 tcp_conn->in.datalen); 825 tcp_conn->in.datalen);
793 break; 826 break;
794 default: 827 default:
@@ -843,7 +876,7 @@ more:
843 if (rc == -EAGAIN) 876 if (rc == -EAGAIN)
844 goto nomore; 877 goto nomore;
845 else { 878 else {
846 iscsi_conn_failure(conn, rc); 879 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
847 return 0; 880 return 0;
848 } 881 }
849 } 882 }
@@ -853,10 +886,8 @@ more:
853 */ 886 */
854 rc = iscsi_tcp_hdr_recv(conn); 887 rc = iscsi_tcp_hdr_recv(conn);
855 if (!rc && tcp_conn->in.datalen) { 888 if (!rc && tcp_conn->in.datalen) {
856 if (conn->datadgst_en) { 889 if (conn->datadgst_en)
857 BUG_ON(!tcp_conn->data_rx_tfm); 890 crypto_hash_init(&tcp_conn->rx_hash);
858 crypto_digest_init(tcp_conn->data_rx_tfm);
859 }
860 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV; 891 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
861 } else if (rc) { 892 } else if (rc) {
862 iscsi_conn_failure(conn, rc); 893 iscsi_conn_failure(conn, rc);
@@ -869,10 +900,15 @@ more:
869 900
870 debug_tcp("extra data_recv offset %d copy %d\n", 901 debug_tcp("extra data_recv offset %d copy %d\n",
871 tcp_conn->in.offset, tcp_conn->in.copy); 902 tcp_conn->in.offset, tcp_conn->in.copy);
872 skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset, 903 rc = iscsi_tcp_copy(conn, sizeof(uint32_t));
873 &recv_digest, 4); 904 if (rc) {
874 tcp_conn->in.offset += 4; 905 if (rc == -EAGAIN)
875 tcp_conn->in.copy -= 4; 906 goto again;
907 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
908 return 0;
909 }
910
911 memcpy(&recv_digest, conn->data, sizeof(uint32_t));
876 if (recv_digest != tcp_conn->in.datadgst) { 912 if (recv_digest != tcp_conn->in.datadgst) {
877 debug_tcp("iscsi_tcp: data digest error!" 913 debug_tcp("iscsi_tcp: data digest error!"
878 "0x%x != 0x%x\n", recv_digest, 914 "0x%x != 0x%x\n", recv_digest,
@@ -897,7 +933,7 @@ more:
897 if (rc) { 933 if (rc) {
898 if (rc == -EAGAIN) 934 if (rc == -EAGAIN)
899 goto again; 935 goto again;
900 iscsi_conn_failure(conn, rc); 936 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
901 return 0; 937 return 0;
902 } 938 }
903 tcp_conn->in.copy -= tcp_conn->in.padding; 939 tcp_conn->in.copy -= tcp_conn->in.padding;
@@ -908,13 +944,14 @@ more:
908 tcp_conn->in.padding); 944 tcp_conn->in.padding);
909 memset(pad, 0, tcp_conn->in.padding); 945 memset(pad, 0, tcp_conn->in.padding);
910 sg_init_one(&sg, pad, tcp_conn->in.padding); 946 sg_init_one(&sg, pad, tcp_conn->in.padding);
911 crypto_digest_update(tcp_conn->data_rx_tfm, 947 crypto_hash_update(&tcp_conn->rx_hash,
912 &sg, 1); 948 &sg, sg.length);
913 } 949 }
914 crypto_digest_final(tcp_conn->data_rx_tfm, 950 crypto_hash_final(&tcp_conn->rx_hash,
915 (u8 *) & tcp_conn->in.datadgst); 951 (u8 *) &tcp_conn->in.datadgst);
916 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst); 952 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
917 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV; 953 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
954 tcp_conn->data_copied = 0;
918 } else 955 } else
919 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER; 956 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
920 } 957 }
@@ -1028,9 +1065,8 @@ iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1028} 1065}
1029 1066
1030static void 1067static void
1031iscsi_conn_restore_callbacks(struct iscsi_conn *conn) 1068iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn)
1032{ 1069{
1033 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1034 struct sock *sk = tcp_conn->sock->sk; 1070 struct sock *sk = tcp_conn->sock->sk;
1035 1071
1036 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 1072 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
@@ -1155,37 +1191,12 @@ iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1155 1191
1156static inline void 1192static inline void
1157iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn, 1193iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
1158 struct iscsi_cmd_task *ctask) 1194 struct iscsi_tcp_cmd_task *tcp_ctask)
1159{ 1195{
1160 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1196 crypto_hash_init(&tcp_conn->tx_hash);
1161
1162 BUG_ON(!tcp_conn->data_tx_tfm);
1163 crypto_digest_init(tcp_conn->data_tx_tfm);
1164 tcp_ctask->digest_count = 4; 1197 tcp_ctask->digest_count = 4;
1165} 1198}
1166 1199
1167static int
1168iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1169 struct iscsi_buf *buf, uint32_t *digest, int final)
1170{
1171 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1172 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1173 int rc = 0;
1174 int sent = 0;
1175
1176 if (final)
1177 crypto_digest_final(tcp_conn->data_tx_tfm, (u8*)digest);
1178
1179 iscsi_buf_init_iov(buf, (char*)digest, 4);
1180 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
1181 if (rc) {
1182 tcp_ctask->datadigest = *digest;
1183 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
1184 } else
1185 tcp_ctask->digest_count = 4;
1186 return rc;
1187}
1188
1189/** 1200/**
1190 * iscsi_solicit_data_cont - initialize next Data-Out 1201 * iscsi_solicit_data_cont - initialize next Data-Out
1191 * @conn: iscsi connection 1202 * @conn: iscsi connection
@@ -1203,7 +1214,6 @@ static void
1203iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1214iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1204 struct iscsi_r2t_info *r2t, int left) 1215 struct iscsi_r2t_info *r2t, int left)
1205{ 1216{
1206 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1207 struct iscsi_data *hdr; 1217 struct iscsi_data *hdr;
1208 struct scsi_cmnd *sc = ctask->sc; 1218 struct scsi_cmnd *sc = ctask->sc;
1209 int new_offset; 1219 int new_offset;
@@ -1232,27 +1242,30 @@ iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1232 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr, 1242 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
1233 sizeof(struct iscsi_hdr)); 1243 sizeof(struct iscsi_hdr));
1234 1244
1235 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) { 1245 if (iscsi_buf_left(&r2t->sendbuf))
1236 BUG_ON(tcp_ctask->bad_sg == r2t->sg); 1246 return;
1247
1248 if (sc->use_sg) {
1237 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1249 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1238 r2t->sg += 1; 1250 r2t->sg += 1;
1239 } else 1251 } else {
1240 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 1252 iscsi_buf_init_iov(&r2t->sendbuf,
1241 (char*)sc->request_buffer + new_offset, 1253 (char*)sc->request_buffer + new_offset,
1242 r2t->data_count); 1254 r2t->data_count);
1255 r2t->sg = NULL;
1256 }
1243} 1257}
1244 1258
1245static void 1259static void iscsi_set_padding(struct iscsi_tcp_cmd_task *tcp_ctask,
1246iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1260 unsigned long len)
1247{ 1261{
1248 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1262 tcp_ctask->pad_count = len & (ISCSI_PAD_LEN - 1);
1249 struct iscsi_data_task *dtask; 1263 if (!tcp_ctask->pad_count)
1264 return;
1250 1265
1251 dtask = tcp_ctask->dtask = &tcp_ctask->unsol_dtask; 1266 tcp_ctask->pad_count = ISCSI_PAD_LEN - tcp_ctask->pad_count;
1252 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr, 1267 debug_scsi("write padding %d bytes\n", tcp_ctask->pad_count);
1253 tcp_ctask->r2t_data_count); 1268 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1254 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
1255 sizeof(struct iscsi_hdr));
1256} 1269}
1257 1270
1258/** 1271/**
@@ -1280,38 +1293,20 @@ iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
1280 if (sc->use_sg) { 1293 if (sc->use_sg) {
1281 struct scatterlist *sg = sc->request_buffer; 1294 struct scatterlist *sg = sc->request_buffer;
1282 1295
1283 iscsi_buf_init_sg(&tcp_ctask->sendbuf, 1296 iscsi_buf_init_sg(&tcp_ctask->sendbuf, sg);
1284 &sg[tcp_ctask->sg_count++]); 1297 tcp_ctask->sg = sg + 1;
1285 tcp_ctask->sg = sg;
1286 tcp_ctask->bad_sg = sg + sc->use_sg; 1298 tcp_ctask->bad_sg = sg + sc->use_sg;
1287 } else 1299 } else {
1288 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 1300 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1289 sc->request_buffer, 1301 sc->request_buffer,
1290 sc->request_bufflen); 1302 sc->request_bufflen);
1291 1303 tcp_ctask->sg = NULL;
1292 if (ctask->imm_count) 1304 tcp_ctask->bad_sg = NULL;
1293 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1294
1295 tcp_ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1296 if (tcp_ctask->pad_count) {
1297 tcp_ctask->pad_count = ISCSI_PAD_LEN -
1298 tcp_ctask->pad_count;
1299 debug_scsi("write padding %d bytes\n",
1300 tcp_ctask->pad_count);
1301 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1302 } 1305 }
1303 1306 debug_scsi("cmd [itt 0x%x total %d imm_data %d "
1304 if (ctask->unsol_count) 1307 "unsol count %d, unsol offset %d]\n",
1305 tcp_ctask->xmstate |= XMSTATE_UNS_HDR |
1306 XMSTATE_UNS_INIT;
1307 tcp_ctask->r2t_data_count = ctask->total_length -
1308 ctask->imm_count -
1309 ctask->unsol_count;
1310
1311 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1312 "r2t_data %d]\n",
1313 ctask->itt, ctask->total_length, ctask->imm_count, 1308 ctask->itt, ctask->total_length, ctask->imm_count,
1314 ctask->unsol_count, tcp_ctask->r2t_data_count); 1309 ctask->unsol_count, ctask->unsol_offset);
1315 } else 1310 } else
1316 tcp_ctask->xmstate = XMSTATE_R_HDR; 1311 tcp_ctask->xmstate = XMSTATE_R_HDR;
1317 1312
@@ -1393,8 +1388,8 @@ iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1393} 1388}
1394 1389
1395static inline int 1390static inline int
1396handle_xmstate_r_hdr(struct iscsi_conn *conn, 1391iscsi_send_read_hdr(struct iscsi_conn *conn,
1397 struct iscsi_tcp_cmd_task *tcp_ctask) 1392 struct iscsi_tcp_cmd_task *tcp_ctask)
1398{ 1393{
1399 int rc; 1394 int rc;
1400 1395
@@ -1412,7 +1407,7 @@ handle_xmstate_r_hdr(struct iscsi_conn *conn,
1412} 1407}
1413 1408
1414static inline int 1409static inline int
1415handle_xmstate_w_hdr(struct iscsi_conn *conn, 1410iscsi_send_write_hdr(struct iscsi_conn *conn,
1416 struct iscsi_cmd_task *ctask) 1411 struct iscsi_cmd_task *ctask)
1417{ 1412{
1418 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1413 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
@@ -1423,85 +1418,126 @@ handle_xmstate_w_hdr(struct iscsi_conn *conn,
1423 iscsi_hdr_digest(conn, &tcp_ctask->headbuf, 1418 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1424 (u8*)tcp_ctask->hdrext); 1419 (u8*)tcp_ctask->hdrext);
1425 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count); 1420 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count);
1426 if (rc) 1421 if (rc) {
1427 tcp_ctask->xmstate |= XMSTATE_W_HDR; 1422 tcp_ctask->xmstate |= XMSTATE_W_HDR;
1428 return rc; 1423 return rc;
1424 }
1425
1426 if (ctask->imm_count) {
1427 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1428 iscsi_set_padding(tcp_ctask, ctask->imm_count);
1429
1430 if (ctask->conn->datadgst_en) {
1431 iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
1432 tcp_ctask->immdigest = 0;
1433 }
1434 }
1435
1436 if (ctask->unsol_count)
1437 tcp_ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1438 return 0;
1429} 1439}
1430 1440
1431static inline int 1441static int
1432handle_xmstate_data_digest(struct iscsi_conn *conn, 1442iscsi_send_padding(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1433 struct iscsi_cmd_task *ctask)
1434{ 1443{
1435 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1444 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1436 int rc; 1445 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1446 int sent = 0, rc;
1437 1447
1438 tcp_ctask->xmstate &= ~XMSTATE_DATA_DIGEST; 1448 if (tcp_ctask->xmstate & XMSTATE_W_PAD) {
1439 debug_tcp("resent data digest 0x%x\n", tcp_ctask->datadigest); 1449 iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
1440 rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf, 1450 tcp_ctask->pad_count);
1441 &tcp_ctask->datadigest, 0); 1451 if (conn->datadgst_en)
1452 crypto_hash_update(&tcp_conn->tx_hash,
1453 &tcp_ctask->sendbuf.sg,
1454 tcp_ctask->sendbuf.sg.length);
1455 } else if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_PAD))
1456 return 0;
1457
1458 tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
1459 tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_PAD;
1460 debug_scsi("sending %d pad bytes for itt 0x%x\n",
1461 tcp_ctask->pad_count, ctask->itt);
1462 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
1463 &sent);
1442 if (rc) { 1464 if (rc) {
1443 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST; 1465 debug_scsi("padding send failed %d\n", rc);
1444 debug_tcp("resent data digest 0x%x fail!\n", 1466 tcp_ctask->xmstate |= XMSTATE_W_RESEND_PAD;
1445 tcp_ctask->datadigest);
1446 } 1467 }
1447
1448 return rc; 1468 return rc;
1449} 1469}
1450 1470
1451static inline int 1471static int
1452handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1472iscsi_send_digest(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1473 struct iscsi_buf *buf, uint32_t *digest)
1453{ 1474{
1454 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1475 struct iscsi_tcp_cmd_task *tcp_ctask;
1455 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1476 struct iscsi_tcp_conn *tcp_conn;
1456 int rc; 1477 int rc, sent = 0;
1457 1478
1458 BUG_ON(!ctask->imm_count); 1479 if (!conn->datadgst_en)
1459 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA; 1480 return 0;
1460 1481
1461 if (conn->datadgst_en) { 1482 tcp_ctask = ctask->dd_data;
1462 iscsi_data_digest_init(tcp_conn, ctask); 1483 tcp_conn = conn->dd_data;
1463 tcp_ctask->immdigest = 0;
1464 }
1465 1484
1466 for (;;) { 1485 if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_DATA_DIGEST)) {
1467 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, 1486 crypto_hash_final(&tcp_conn->tx_hash, (u8*)digest);
1468 &ctask->imm_count, &tcp_ctask->sent); 1487 iscsi_buf_init_iov(buf, (char*)digest, 4);
1469 if (rc) { 1488 }
1470 tcp_ctask->xmstate |= XMSTATE_IMM_DATA; 1489 tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_DATA_DIGEST;
1471 if (conn->datadgst_en) {
1472 crypto_digest_final(tcp_conn->data_tx_tfm,
1473 (u8*)&tcp_ctask->immdigest);
1474 debug_tcp("tx imm sendpage fail 0x%x\n",
1475 tcp_ctask->datadigest);
1476 }
1477 return rc;
1478 }
1479 if (conn->datadgst_en)
1480 crypto_digest_update(tcp_conn->data_tx_tfm,
1481 &tcp_ctask->sendbuf.sg, 1);
1482 1490
1483 if (!ctask->imm_count) 1491 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
1484 break; 1492 if (!rc)
1485 iscsi_buf_init_sg(&tcp_ctask->sendbuf, 1493 debug_scsi("sent digest 0x%x for itt 0x%x\n", *digest,
1486 &tcp_ctask->sg[tcp_ctask->sg_count++]); 1494 ctask->itt);
1495 else {
1496 debug_scsi("sending digest 0x%x failed for itt 0x%x!\n",
1497 *digest, ctask->itt);
1498 tcp_ctask->xmstate |= XMSTATE_W_RESEND_DATA_DIGEST;
1487 } 1499 }
1500 return rc;
1501}
1488 1502
1489 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) { 1503static int
1490 rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf, 1504iscsi_send_data(struct iscsi_cmd_task *ctask, struct iscsi_buf *sendbuf,
1491 &tcp_ctask->immdigest, 1); 1505 struct scatterlist **sg, int *sent, int *count,
1492 if (rc) { 1506 struct iscsi_buf *digestbuf, uint32_t *digest)
1493 debug_tcp("sending imm digest 0x%x fail!\n", 1507{
1494 tcp_ctask->immdigest); 1508 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1495 return rc; 1509 struct iscsi_conn *conn = ctask->conn;
1510 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1511 int rc, buf_sent, offset;
1512
1513 while (*count) {
1514 buf_sent = 0;
1515 offset = sendbuf->sent;
1516
1517 rc = iscsi_sendpage(conn, sendbuf, count, &buf_sent);
1518 *sent = *sent + buf_sent;
1519 if (buf_sent && conn->datadgst_en)
1520 partial_sg_digest_update(&tcp_conn->tx_hash,
1521 &sendbuf->sg, sendbuf->sg.offset + offset,
1522 buf_sent);
1523 if (!iscsi_buf_left(sendbuf) && *sg != tcp_ctask->bad_sg) {
1524 iscsi_buf_init_sg(sendbuf, *sg);
1525 *sg = *sg + 1;
1496 } 1526 }
1497 debug_tcp("sending imm digest 0x%x\n", tcp_ctask->immdigest); 1527
1528 if (rc)
1529 return rc;
1498 } 1530 }
1499 1531
1500 return 0; 1532 rc = iscsi_send_padding(conn, ctask);
1533 if (rc)
1534 return rc;
1535
1536 return iscsi_send_digest(conn, ctask, digestbuf, digest);
1501} 1537}
1502 1538
1503static inline int 1539static int
1504handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1540iscsi_send_unsol_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1505{ 1541{
1506 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1542 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1507 struct iscsi_data_task *dtask; 1543 struct iscsi_data_task *dtask;
@@ -1509,12 +1545,17 @@ handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1509 1545
1510 tcp_ctask->xmstate |= XMSTATE_UNS_DATA; 1546 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1511 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) { 1547 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
1512 iscsi_unsolicit_data_init(conn, ctask); 1548 dtask = &tcp_ctask->unsol_dtask;
1513 dtask = tcp_ctask->dtask; 1549
1550 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr);
1551 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
1552 sizeof(struct iscsi_hdr));
1514 if (conn->hdrdgst_en) 1553 if (conn->hdrdgst_en)
1515 iscsi_hdr_digest(conn, &tcp_ctask->headbuf, 1554 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1516 (u8*)dtask->hdrext); 1555 (u8*)dtask->hdrext);
1556
1517 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT; 1557 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
1558 iscsi_set_padding(tcp_ctask, ctask->data_count);
1518 } 1559 }
1519 1560
1520 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count); 1561 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count);
@@ -1524,254 +1565,138 @@ handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1524 return rc; 1565 return rc;
1525 } 1566 }
1526 1567
1568 if (conn->datadgst_en) {
1569 dtask = &tcp_ctask->unsol_dtask;
1570 iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
1571 dtask->digest = 0;
1572 }
1573
1527 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n", 1574 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1528 ctask->itt, ctask->unsol_count, tcp_ctask->sent); 1575 ctask->itt, ctask->unsol_count, tcp_ctask->sent);
1529 return 0; 1576 return 0;
1530} 1577}
1531 1578
1532static inline int 1579static int
1533handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1580iscsi_send_unsol_pdu(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1534{ 1581{
1535 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1582 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1536 struct iscsi_data_task *dtask = tcp_ctask->dtask;
1537 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1538 int rc; 1583 int rc;
1539 1584
1540 BUG_ON(!ctask->data_count); 1585 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
1541 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA; 1586 BUG_ON(!ctask->unsol_count);
1542 1587 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
1543 if (conn->datadgst_en) { 1588send_hdr:
1544 iscsi_data_digest_init(tcp_conn, ctask); 1589 rc = iscsi_send_unsol_hdr(conn, ctask);
1545 dtask->digest = 0; 1590 if (rc)
1591 return rc;
1546 } 1592 }
1547 1593
1548 for (;;) { 1594 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
1595 struct iscsi_data_task *dtask = &tcp_ctask->unsol_dtask;
1549 int start = tcp_ctask->sent; 1596 int start = tcp_ctask->sent;
1550 1597
1551 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, 1598 rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
1552 &ctask->data_count, &tcp_ctask->sent); 1599 &tcp_ctask->sent, &ctask->data_count,
1553 if (rc) { 1600 &dtask->digestbuf, &dtask->digest);
1554 ctask->unsol_count -= tcp_ctask->sent - start;
1555 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1556 /* will continue with this ctask later.. */
1557 if (conn->datadgst_en) {
1558 crypto_digest_final(tcp_conn->data_tx_tfm,
1559 (u8 *)&dtask->digest);
1560 debug_tcp("tx uns data fail 0x%x\n",
1561 dtask->digest);
1562 }
1563 return rc;
1564 }
1565
1566 BUG_ON(tcp_ctask->sent > ctask->total_length);
1567 ctask->unsol_count -= tcp_ctask->sent - start; 1601 ctask->unsol_count -= tcp_ctask->sent - start;
1568 1602 if (rc)
1603 return rc;
1604 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1569 /* 1605 /*
1570 * XXX:we may run here with un-initial sendbuf. 1606 * Done with the Data-Out. Next, check if we need
1571 * so pass it 1607 * to send another unsolicited Data-Out.
1572 */ 1608 */
1573 if (conn->datadgst_en && tcp_ctask->sent - start > 0) 1609 if (ctask->unsol_count) {
1574 crypto_digest_update(tcp_conn->data_tx_tfm, 1610 debug_scsi("sending more uns\n");
1575 &tcp_ctask->sendbuf.sg, 1); 1611 tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
1576 1612 goto send_hdr;
1577 if (!ctask->data_count)
1578 break;
1579 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1580 &tcp_ctask->sg[tcp_ctask->sg_count++]);
1581 }
1582 BUG_ON(ctask->unsol_count < 0);
1583
1584 /*
1585 * Done with the Data-Out. Next, check if we need
1586 * to send another unsolicited Data-Out.
1587 */
1588 if (ctask->unsol_count) {
1589 if (conn->datadgst_en) {
1590 rc = iscsi_digest_final_send(conn, ctask,
1591 &dtask->digestbuf,
1592 &dtask->digest, 1);
1593 if (rc) {
1594 debug_tcp("send uns digest 0x%x fail\n",
1595 dtask->digest);
1596 return rc;
1597 }
1598 debug_tcp("sending uns digest 0x%x, more uns\n",
1599 dtask->digest);
1600 } 1613 }
1601 tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
1602 return 1;
1603 } 1614 }
1604
1605 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1606 rc = iscsi_digest_final_send(conn, ctask,
1607 &dtask->digestbuf,
1608 &dtask->digest, 1);
1609 if (rc) {
1610 debug_tcp("send last uns digest 0x%x fail\n",
1611 dtask->digest);
1612 return rc;
1613 }
1614 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1615 }
1616
1617 return 0; 1615 return 0;
1618} 1616}
1619 1617
1620static inline int 1618static int iscsi_send_sol_pdu(struct iscsi_conn *conn,
1621handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1619 struct iscsi_cmd_task *ctask)
1622{ 1620{
1623 struct iscsi_session *session = conn->session;
1624 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1625 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1621 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1626 struct iscsi_r2t_info *r2t = tcp_ctask->r2t; 1622 struct iscsi_session *session = conn->session;
1627 struct iscsi_data_task *dtask = &r2t->dtask; 1623 struct iscsi_r2t_info *r2t;
1624 struct iscsi_data_task *dtask;
1628 int left, rc; 1625 int left, rc;
1629 1626
1630 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA; 1627 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
1631 tcp_ctask->dtask = dtask; 1628 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1632
1633 if (conn->datadgst_en) {
1634 iscsi_data_digest_init(tcp_conn, ctask);
1635 dtask->digest = 0;
1636 }
1637solicit_again:
1638 /*
1639 * send Data-Out whitnin this R2T sequence.
1640 */
1641 if (!r2t->data_count)
1642 goto data_out_done;
1643
1644 rc = iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent);
1645 if (rc) {
1646 tcp_ctask->xmstate |= XMSTATE_SOL_DATA; 1629 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1647 /* will continue with this ctask later.. */ 1630 if (!tcp_ctask->r2t)
1648 if (conn->datadgst_en) { 1631 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
1649 crypto_digest_final(tcp_conn->data_tx_tfm, 1632 sizeof(void*));
1650 (u8 *)&dtask->digest); 1633send_hdr:
1651 debug_tcp("r2t data send fail 0x%x\n", dtask->digest); 1634 r2t = tcp_ctask->r2t;
1652 } 1635 dtask = &r2t->dtask;
1653 return rc;
1654 }
1655 1636
1656 BUG_ON(r2t->data_count < 0); 1637 if (conn->hdrdgst_en)
1657 if (conn->datadgst_en) 1638 iscsi_hdr_digest(conn, &r2t->headbuf,
1658 crypto_digest_update(tcp_conn->data_tx_tfm, &r2t->sendbuf.sg, 1639 (u8*)dtask->hdrext);
1659 1); 1640 rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
1660 1641 if (rc) {
1661 if (r2t->data_count) { 1642 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1662 BUG_ON(ctask->sc->use_sg == 0); 1643 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
1663 if (!iscsi_buf_left(&r2t->sendbuf)) { 1644 return rc;
1664 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
1665 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1666 r2t->sg += 1;
1667 } 1645 }
1668 goto solicit_again;
1669 }
1670 1646
1671data_out_done:
1672 /*
1673 * Done with this Data-Out. Next, check if we have
1674 * to send another Data-Out for this R2T.
1675 */
1676 BUG_ON(r2t->data_length - r2t->sent < 0);
1677 left = r2t->data_length - r2t->sent;
1678 if (left) {
1679 if (conn->datadgst_en) { 1647 if (conn->datadgst_en) {
1680 rc = iscsi_digest_final_send(conn, ctask, 1648 iscsi_data_digest_init(conn->dd_data, tcp_ctask);
1681 &dtask->digestbuf, 1649 dtask->digest = 0;
1682 &dtask->digest, 1);
1683 if (rc) {
1684 debug_tcp("send r2t data digest 0x%x"
1685 "fail\n", dtask->digest);
1686 return rc;
1687 }
1688 debug_tcp("r2t data send digest 0x%x\n",
1689 dtask->digest);
1690 } 1650 }
1691 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1692 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1693 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1694 return 1;
1695 }
1696
1697 /*
1698 * Done with this R2T. Check if there are more
1699 * outstanding R2Ts ready to be processed.
1700 */
1701 BUG_ON(tcp_ctask->r2t_data_count - r2t->data_length < 0);
1702 if (conn->datadgst_en) {
1703 rc = iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1704 &dtask->digest, 1);
1705 if (rc) {
1706 debug_tcp("send last r2t data digest 0x%x"
1707 "fail\n", dtask->digest);
1708 return rc;
1709 }
1710 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1711 }
1712 1651
1713 tcp_ctask->r2t_data_count -= r2t->data_length; 1652 iscsi_set_padding(tcp_ctask, r2t->data_count);
1714 tcp_ctask->r2t = NULL; 1653 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
1715 spin_lock_bh(&session->lock); 1654 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
1716 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 1655 r2t->sent);
1717 spin_unlock_bh(&session->lock);
1718 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
1719 tcp_ctask->r2t = r2t;
1720 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1721 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1722 return 1;
1723 } 1656 }
1724 1657
1725 return 0; 1658 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
1726} 1659 r2t = tcp_ctask->r2t;
1660 dtask = &r2t->dtask;
1727 1661
1728static inline int 1662 rc = iscsi_send_data(ctask, &r2t->sendbuf, &r2t->sg,
1729handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1663 &r2t->sent, &r2t->data_count,
1730{ 1664 &dtask->digestbuf, &dtask->digest);
1731 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1665 if (rc)
1732 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1666 return rc;
1733 struct iscsi_data_task *dtask = tcp_ctask->dtask; 1667 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1734 int sent, rc;
1735 1668
1736 tcp_ctask->xmstate &= ~XMSTATE_W_PAD; 1669 /*
1737 iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad, 1670 * Done with this Data-Out. Next, check if we have
1738 tcp_ctask->pad_count); 1671 * to send another Data-Out for this R2T.
1739 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count, 1672 */
1740 &sent); 1673 BUG_ON(r2t->data_length - r2t->sent < 0);
1741 if (rc) { 1674 left = r2t->data_length - r2t->sent;
1742 tcp_ctask->xmstate |= XMSTATE_W_PAD; 1675 if (left) {
1743 return rc; 1676 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1744 } 1677 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1678 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1679 goto send_hdr;
1680 }
1745 1681
1746 if (conn->datadgst_en) { 1682 /*
1747 crypto_digest_update(tcp_conn->data_tx_tfm, 1683 * Done with this R2T. Check if there are more
1748 &tcp_ctask->sendbuf.sg, 1); 1684 * outstanding R2Ts ready to be processed.
1749 /* imm data? */ 1685 */
1750 if (!dtask) { 1686 spin_lock_bh(&session->lock);
1751 rc = iscsi_digest_final_send(conn, ctask, 1687 tcp_ctask->r2t = NULL;
1752 &tcp_ctask->immbuf, 1688 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
1753 &tcp_ctask->immdigest, 1); 1689 sizeof(void*));
1754 if (rc) { 1690 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t,
1755 debug_tcp("send padding digest 0x%x" 1691 sizeof(void*))) {
1756 "fail!\n", tcp_ctask->immdigest); 1692 tcp_ctask->r2t = r2t;
1757 return rc; 1693 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1758 } 1694 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1759 debug_tcp("done with padding, digest 0x%x\n", 1695 spin_unlock_bh(&session->lock);
1760 tcp_ctask->datadigest); 1696 goto send_hdr;
1761 } else {
1762 rc = iscsi_digest_final_send(conn, ctask,
1763 &dtask->digestbuf,
1764 &dtask->digest, 1);
1765 if (rc) {
1766 debug_tcp("send padding digest 0x%x"
1767 "fail\n", dtask->digest);
1768 return rc;
1769 }
1770 debug_tcp("done with padding, digest 0x%x\n",
1771 dtask->digest);
1772 } 1697 }
1698 spin_unlock_bh(&session->lock);
1773 } 1699 }
1774
1775 return 0; 1700 return 0;
1776} 1701}
1777 1702
@@ -1791,85 +1716,30 @@ iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1791 return rc; 1716 return rc;
1792 1717
1793 if (tcp_ctask->xmstate & XMSTATE_R_HDR) 1718 if (tcp_ctask->xmstate & XMSTATE_R_HDR)
1794 return handle_xmstate_r_hdr(conn, tcp_ctask); 1719 return iscsi_send_read_hdr(conn, tcp_ctask);
1795 1720
1796 if (tcp_ctask->xmstate & XMSTATE_W_HDR) { 1721 if (tcp_ctask->xmstate & XMSTATE_W_HDR) {
1797 rc = handle_xmstate_w_hdr(conn, ctask); 1722 rc = iscsi_send_write_hdr(conn, ctask);
1798 if (rc)
1799 return rc;
1800 }
1801
1802 /* XXX: for data digest xmit recover */
1803 if (tcp_ctask->xmstate & XMSTATE_DATA_DIGEST) {
1804 rc = handle_xmstate_data_digest(conn, ctask);
1805 if (rc) 1723 if (rc)
1806 return rc; 1724 return rc;
1807 } 1725 }
1808 1726
1809 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) { 1727 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
1810 rc = handle_xmstate_imm_data(conn, ctask); 1728 rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
1729 &tcp_ctask->sent, &ctask->imm_count,
1730 &tcp_ctask->immbuf, &tcp_ctask->immdigest);
1811 if (rc) 1731 if (rc)
1812 return rc; 1732 return rc;
1733 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
1813 } 1734 }
1814 1735
1815 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) { 1736 rc = iscsi_send_unsol_pdu(conn, ctask);
1816 BUG_ON(!ctask->unsol_count); 1737 if (rc)
1817 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR; 1738 return rc;
1818unsolicit_head_again:
1819 rc = handle_xmstate_uns_hdr(conn, ctask);
1820 if (rc)
1821 return rc;
1822 }
1823
1824 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
1825 rc = handle_xmstate_uns_data(conn, ctask);
1826 if (rc == 1)
1827 goto unsolicit_head_again;
1828 else if (rc)
1829 return rc;
1830 goto done;
1831 }
1832
1833 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
1834 struct iscsi_r2t_info *r2t;
1835
1836 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1837 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1838 if (!tcp_ctask->r2t)
1839 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
1840 sizeof(void*));
1841solicit_head_again:
1842 r2t = tcp_ctask->r2t;
1843 if (conn->hdrdgst_en)
1844 iscsi_hdr_digest(conn, &r2t->headbuf,
1845 (u8*)r2t->dtask.hdrext);
1846 rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
1847 if (rc) {
1848 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1849 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
1850 return rc;
1851 }
1852
1853 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
1854 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
1855 r2t->sent);
1856 }
1857
1858 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
1859 rc = handle_xmstate_sol_data(conn, ctask);
1860 if (rc == 1)
1861 goto solicit_head_again;
1862 if (rc)
1863 return rc;
1864 }
1865 1739
1866done: 1740 rc = iscsi_send_sol_pdu(conn, ctask);
1867 /* 1741 if (rc)
1868 * Last thing to check is whether we need to send write 1742 return rc;
1869 * padding. Note that we check for xmstate equality, not just the bit.
1870 */
1871 if (tcp_ctask->xmstate == XMSTATE_W_PAD)
1872 rc = handle_xmstate_w_pad(conn, ctask);
1873 1743
1874 return rc; 1744 return rc;
1875} 1745}
@@ -1900,20 +1770,24 @@ iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
1900 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER; 1770 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1901 /* initial operational parameters */ 1771 /* initial operational parameters */
1902 tcp_conn->hdr_size = sizeof(struct iscsi_hdr); 1772 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1903 tcp_conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
1904 1773
1905 /* allocate initial PDU receive place holder */ 1774 tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1906 if (tcp_conn->data_size <= PAGE_SIZE) 1775 CRYPTO_ALG_ASYNC);
1907 tcp_conn->data = kmalloc(tcp_conn->data_size, GFP_KERNEL); 1776 tcp_conn->tx_hash.flags = 0;
1908 else 1777 if (!tcp_conn->tx_hash.tfm)
1909 tcp_conn->data = (void*)__get_free_pages(GFP_KERNEL, 1778 goto free_tcp_conn;
1910 get_order(tcp_conn->data_size)); 1779
1911 if (!tcp_conn->data) 1780 tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1912 goto max_recv_dlenght_alloc_fail; 1781 CRYPTO_ALG_ASYNC);
1782 tcp_conn->rx_hash.flags = 0;
1783 if (!tcp_conn->rx_hash.tfm)
1784 goto free_tx_tfm;
1913 1785
1914 return cls_conn; 1786 return cls_conn;
1915 1787
1916max_recv_dlenght_alloc_fail: 1788free_tx_tfm:
1789 crypto_free_hash(tcp_conn->tx_hash.tfm);
1790free_tcp_conn:
1917 kfree(tcp_conn); 1791 kfree(tcp_conn);
1918tcp_conn_alloc_fail: 1792tcp_conn_alloc_fail:
1919 iscsi_conn_teardown(cls_conn); 1793 iscsi_conn_teardown(cls_conn);
@@ -1921,6 +1795,23 @@ tcp_conn_alloc_fail:
1921} 1795}
1922 1796
1923static void 1797static void
1798iscsi_tcp_release_conn(struct iscsi_conn *conn)
1799{
1800 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1801
1802 if (!tcp_conn->sock)
1803 return;
1804
1805 sock_hold(tcp_conn->sock->sk);
1806 iscsi_conn_restore_callbacks(tcp_conn);
1807 sock_put(tcp_conn->sock->sk);
1808
1809 sock_release(tcp_conn->sock);
1810 tcp_conn->sock = NULL;
1811 conn->recv_lock = NULL;
1812}
1813
1814static void
1924iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn) 1815iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1925{ 1816{
1926 struct iscsi_conn *conn = cls_conn->dd_data; 1817 struct iscsi_conn *conn = cls_conn->dd_data;
@@ -1930,29 +1821,31 @@ iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1930 if (conn->hdrdgst_en || conn->datadgst_en) 1821 if (conn->hdrdgst_en || conn->datadgst_en)
1931 digest = 1; 1822 digest = 1;
1932 1823
1824 iscsi_tcp_release_conn(conn);
1933 iscsi_conn_teardown(cls_conn); 1825 iscsi_conn_teardown(cls_conn);
1934 1826
1935 /* now free tcp_conn */ 1827 /* now free tcp_conn */
1936 if (digest) { 1828 if (digest) {
1937 if (tcp_conn->tx_tfm) 1829 if (tcp_conn->tx_hash.tfm)
1938 crypto_free_tfm(tcp_conn->tx_tfm); 1830 crypto_free_hash(tcp_conn->tx_hash.tfm);
1939 if (tcp_conn->rx_tfm) 1831 if (tcp_conn->rx_hash.tfm)
1940 crypto_free_tfm(tcp_conn->rx_tfm); 1832 crypto_free_hash(tcp_conn->rx_hash.tfm);
1941 if (tcp_conn->data_tx_tfm)
1942 crypto_free_tfm(tcp_conn->data_tx_tfm);
1943 if (tcp_conn->data_rx_tfm)
1944 crypto_free_tfm(tcp_conn->data_rx_tfm);
1945 } 1833 }
1946 1834
1947 /* free conn->data, size = MaxRecvDataSegmentLength */
1948 if (tcp_conn->data_size <= PAGE_SIZE)
1949 kfree(tcp_conn->data);
1950 else
1951 free_pages((unsigned long)tcp_conn->data,
1952 get_order(tcp_conn->data_size));
1953 kfree(tcp_conn); 1835 kfree(tcp_conn);
1954} 1836}
1955 1837
1838static void
1839iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
1840{
1841 struct iscsi_conn *conn = cls_conn->dd_data;
1842 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1843
1844 iscsi_conn_stop(cls_conn, flag);
1845 iscsi_tcp_release_conn(conn);
1846 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1847}
1848
1956static int 1849static int
1957iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session, 1850iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1958 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, 1851 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
@@ -2001,52 +1894,6 @@ iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
2001 return 0; 1894 return 0;
2002} 1895}
2003 1896
2004static void
2005iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2006{
2007 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2008 struct iscsi_r2t_info *r2t;
2009
2010 /* flush ctask's r2t queues */
2011 while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*)))
2012 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
2013 sizeof(void*));
2014
2015 __iscsi_ctask_cleanup(conn, ctask);
2016}
2017
2018static void
2019iscsi_tcp_suspend_conn_rx(struct iscsi_conn *conn)
2020{
2021 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2022 struct sock *sk;
2023
2024 if (!tcp_conn->sock)
2025 return;
2026
2027 sk = tcp_conn->sock->sk;
2028 write_lock_bh(&sk->sk_callback_lock);
2029 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
2030 write_unlock_bh(&sk->sk_callback_lock);
2031}
2032
2033static void
2034iscsi_tcp_terminate_conn(struct iscsi_conn *conn)
2035{
2036 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2037
2038 if (!tcp_conn->sock)
2039 return;
2040
2041 sock_hold(tcp_conn->sock->sk);
2042 iscsi_conn_restore_callbacks(conn);
2043 sock_put(tcp_conn->sock->sk);
2044
2045 sock_release(tcp_conn->sock);
2046 tcp_conn->sock = NULL;
2047 conn->recv_lock = NULL;
2048}
2049
2050/* called with host lock */ 1897/* called with host lock */
2051static void 1898static void
2052iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask, 1899iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask,
@@ -2057,6 +1904,7 @@ iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask,
2057 iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr, 1904 iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr,
2058 sizeof(struct iscsi_hdr)); 1905 sizeof(struct iscsi_hdr));
2059 tcp_mtask->xmstate = XMSTATE_IMM_HDR; 1906 tcp_mtask->xmstate = XMSTATE_IMM_HDR;
1907 tcp_mtask->sent = 0;
2060 1908
2061 if (mtask->data_count) 1909 if (mtask->data_count)
2062 iscsi_buf_init_iov(&tcp_mtask->sendbuf, (char*)mtask->data, 1910 iscsi_buf_init_iov(&tcp_mtask->sendbuf, (char*)mtask->data,
@@ -2130,211 +1978,55 @@ iscsi_r2tpool_free(struct iscsi_session *session)
2130 1978
2131static int 1979static int
2132iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param, 1980iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
2133 uint32_t value) 1981 char *buf, int buflen)
2134{ 1982{
2135 struct iscsi_conn *conn = cls_conn->dd_data; 1983 struct iscsi_conn *conn = cls_conn->dd_data;
2136 struct iscsi_session *session = conn->session; 1984 struct iscsi_session *session = conn->session;
2137 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1985 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1986 int value;
2138 1987
2139 switch(param) { 1988 switch(param) {
2140 case ISCSI_PARAM_MAX_RECV_DLENGTH: {
2141 char *saveptr = tcp_conn->data;
2142 gfp_t flags = GFP_KERNEL;
2143
2144 if (tcp_conn->data_size >= value) {
2145 conn->max_recv_dlength = value;
2146 break;
2147 }
2148
2149 spin_lock_bh(&session->lock);
2150 if (conn->stop_stage == STOP_CONN_RECOVER)
2151 flags = GFP_ATOMIC;
2152 spin_unlock_bh(&session->lock);
2153
2154 if (value <= PAGE_SIZE)
2155 tcp_conn->data = kmalloc(value, flags);
2156 else
2157 tcp_conn->data = (void*)__get_free_pages(flags,
2158 get_order(value));
2159 if (tcp_conn->data == NULL) {
2160 tcp_conn->data = saveptr;
2161 return -ENOMEM;
2162 }
2163 if (tcp_conn->data_size <= PAGE_SIZE)
2164 kfree(saveptr);
2165 else
2166 free_pages((unsigned long)saveptr,
2167 get_order(tcp_conn->data_size));
2168 conn->max_recv_dlength = value;
2169 tcp_conn->data_size = value;
2170 }
2171 break;
2172 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2173 conn->max_xmit_dlength = value;
2174 break;
2175 case ISCSI_PARAM_HDRDGST_EN: 1989 case ISCSI_PARAM_HDRDGST_EN:
2176 conn->hdrdgst_en = value; 1990 iscsi_set_param(cls_conn, param, buf, buflen);
2177 tcp_conn->hdr_size = sizeof(struct iscsi_hdr); 1991 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
2178 if (conn->hdrdgst_en) { 1992 if (conn->hdrdgst_en)
2179 tcp_conn->hdr_size += sizeof(__u32); 1993 tcp_conn->hdr_size += sizeof(__u32);
2180 if (!tcp_conn->tx_tfm)
2181 tcp_conn->tx_tfm = crypto_alloc_tfm("crc32c",
2182 0);
2183 if (!tcp_conn->tx_tfm)
2184 return -ENOMEM;
2185 if (!tcp_conn->rx_tfm)
2186 tcp_conn->rx_tfm = crypto_alloc_tfm("crc32c",
2187 0);
2188 if (!tcp_conn->rx_tfm) {
2189 crypto_free_tfm(tcp_conn->tx_tfm);
2190 return -ENOMEM;
2191 }
2192 } else {
2193 if (tcp_conn->tx_tfm)
2194 crypto_free_tfm(tcp_conn->tx_tfm);
2195 if (tcp_conn->rx_tfm)
2196 crypto_free_tfm(tcp_conn->rx_tfm);
2197 }
2198 break; 1994 break;
2199 case ISCSI_PARAM_DATADGST_EN: 1995 case ISCSI_PARAM_DATADGST_EN:
2200 conn->datadgst_en = value; 1996 iscsi_set_param(cls_conn, param, buf, buflen);
2201 if (conn->datadgst_en) {
2202 if (!tcp_conn->data_tx_tfm)
2203 tcp_conn->data_tx_tfm =
2204 crypto_alloc_tfm("crc32c", 0);
2205 if (!tcp_conn->data_tx_tfm)
2206 return -ENOMEM;
2207 if (!tcp_conn->data_rx_tfm)
2208 tcp_conn->data_rx_tfm =
2209 crypto_alloc_tfm("crc32c", 0);
2210 if (!tcp_conn->data_rx_tfm) {
2211 crypto_free_tfm(tcp_conn->data_tx_tfm);
2212 return -ENOMEM;
2213 }
2214 } else {
2215 if (tcp_conn->data_tx_tfm)
2216 crypto_free_tfm(tcp_conn->data_tx_tfm);
2217 if (tcp_conn->data_rx_tfm)
2218 crypto_free_tfm(tcp_conn->data_rx_tfm);
2219 }
2220 tcp_conn->sendpage = conn->datadgst_en ? 1997 tcp_conn->sendpage = conn->datadgst_en ?
2221 sock_no_sendpage : tcp_conn->sock->ops->sendpage; 1998 sock_no_sendpage : tcp_conn->sock->ops->sendpage;
2222 break; 1999 break;
2223 case ISCSI_PARAM_INITIAL_R2T_EN:
2224 session->initial_r2t_en = value;
2225 break;
2226 case ISCSI_PARAM_MAX_R2T: 2000 case ISCSI_PARAM_MAX_R2T:
2001 sscanf(buf, "%d", &value);
2227 if (session->max_r2t == roundup_pow_of_two(value)) 2002 if (session->max_r2t == roundup_pow_of_two(value))
2228 break; 2003 break;
2229 iscsi_r2tpool_free(session); 2004 iscsi_r2tpool_free(session);
2230 session->max_r2t = value; 2005 iscsi_set_param(cls_conn, param, buf, buflen);
2231 if (session->max_r2t & (session->max_r2t - 1)) 2006 if (session->max_r2t & (session->max_r2t - 1))
2232 session->max_r2t = roundup_pow_of_two(session->max_r2t); 2007 session->max_r2t = roundup_pow_of_two(session->max_r2t);
2233 if (iscsi_r2tpool_alloc(session)) 2008 if (iscsi_r2tpool_alloc(session))
2234 return -ENOMEM; 2009 return -ENOMEM;
2235 break; 2010 break;
2236 case ISCSI_PARAM_IMM_DATA_EN:
2237 session->imm_data_en = value;
2238 break;
2239 case ISCSI_PARAM_FIRST_BURST:
2240 session->first_burst = value;
2241 break;
2242 case ISCSI_PARAM_MAX_BURST:
2243 session->max_burst = value;
2244 break;
2245 case ISCSI_PARAM_PDU_INORDER_EN:
2246 session->pdu_inorder_en = value;
2247 break;
2248 case ISCSI_PARAM_DATASEQ_INORDER_EN:
2249 session->dataseq_inorder_en = value;
2250 break;
2251 case ISCSI_PARAM_ERL:
2252 session->erl = value;
2253 break;
2254 case ISCSI_PARAM_IFMARKER_EN:
2255 BUG_ON(value);
2256 session->ifmarker_en = value;
2257 break;
2258 case ISCSI_PARAM_OFMARKER_EN:
2259 BUG_ON(value);
2260 session->ofmarker_en = value;
2261 break;
2262 case ISCSI_PARAM_EXP_STATSN:
2263 conn->exp_statsn = value;
2264 break;
2265 default:
2266 break;
2267 }
2268
2269 return 0;
2270}
2271
2272static int
2273iscsi_session_get_param(struct iscsi_cls_session *cls_session,
2274 enum iscsi_param param, uint32_t *value)
2275{
2276 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
2277 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
2278
2279 switch(param) {
2280 case ISCSI_PARAM_INITIAL_R2T_EN:
2281 *value = session->initial_r2t_en;
2282 break;
2283 case ISCSI_PARAM_MAX_R2T:
2284 *value = session->max_r2t;
2285 break;
2286 case ISCSI_PARAM_IMM_DATA_EN:
2287 *value = session->imm_data_en;
2288 break;
2289 case ISCSI_PARAM_FIRST_BURST:
2290 *value = session->first_burst;
2291 break;
2292 case ISCSI_PARAM_MAX_BURST:
2293 *value = session->max_burst;
2294 break;
2295 case ISCSI_PARAM_PDU_INORDER_EN:
2296 *value = session->pdu_inorder_en;
2297 break;
2298 case ISCSI_PARAM_DATASEQ_INORDER_EN:
2299 *value = session->dataseq_inorder_en;
2300 break;
2301 case ISCSI_PARAM_ERL:
2302 *value = session->erl;
2303 break;
2304 case ISCSI_PARAM_IFMARKER_EN:
2305 *value = session->ifmarker_en;
2306 break;
2307 case ISCSI_PARAM_OFMARKER_EN:
2308 *value = session->ofmarker_en;
2309 break;
2310 default: 2011 default:
2311 return -EINVAL; 2012 return iscsi_set_param(cls_conn, param, buf, buflen);
2312 } 2013 }
2313 2014
2314 return 0; 2015 return 0;
2315} 2016}
2316 2017
2317static int 2018static int
2318iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn, 2019iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
2319 enum iscsi_param param, uint32_t *value) 2020 enum iscsi_param param, char *buf)
2320{ 2021{
2321 struct iscsi_conn *conn = cls_conn->dd_data; 2022 struct iscsi_conn *conn = cls_conn->dd_data;
2322 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2023 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2323 struct inet_sock *inet; 2024 struct inet_sock *inet;
2025 struct ipv6_pinfo *np;
2026 struct sock *sk;
2027 int len;
2324 2028
2325 switch(param) { 2029 switch(param) {
2326 case ISCSI_PARAM_MAX_RECV_DLENGTH:
2327 *value = conn->max_recv_dlength;
2328 break;
2329 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2330 *value = conn->max_xmit_dlength;
2331 break;
2332 case ISCSI_PARAM_HDRDGST_EN:
2333 *value = conn->hdrdgst_en;
2334 break;
2335 case ISCSI_PARAM_DATADGST_EN:
2336 *value = conn->datadgst_en;
2337 break;
2338 case ISCSI_PARAM_CONN_PORT: 2030 case ISCSI_PARAM_CONN_PORT:
2339 mutex_lock(&conn->xmitmutex); 2031 mutex_lock(&conn->xmitmutex);
2340 if (!tcp_conn->sock) { 2032 if (!tcp_conn->sock) {
@@ -2343,30 +2035,9 @@ iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn,
2343 } 2035 }
2344 2036
2345 inet = inet_sk(tcp_conn->sock->sk); 2037 inet = inet_sk(tcp_conn->sock->sk);
2346 *value = be16_to_cpu(inet->dport); 2038 len = sprintf(buf, "%hu\n", be16_to_cpu(inet->dport));
2347 mutex_unlock(&conn->xmitmutex); 2039 mutex_unlock(&conn->xmitmutex);
2348 case ISCSI_PARAM_EXP_STATSN:
2349 *value = conn->exp_statsn;
2350 break; 2040 break;
2351 default:
2352 return -EINVAL;
2353 }
2354
2355 return 0;
2356}
2357
2358static int
2359iscsi_conn_get_str_param(struct iscsi_cls_conn *cls_conn,
2360 enum iscsi_param param, char *buf)
2361{
2362 struct iscsi_conn *conn = cls_conn->dd_data;
2363 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2364 struct sock *sk;
2365 struct inet_sock *inet;
2366 struct ipv6_pinfo *np;
2367 int len = 0;
2368
2369 switch (param) {
2370 case ISCSI_PARAM_CONN_ADDRESS: 2041 case ISCSI_PARAM_CONN_ADDRESS:
2371 mutex_lock(&conn->xmitmutex); 2042 mutex_lock(&conn->xmitmutex);
2372 if (!tcp_conn->sock) { 2043 if (!tcp_conn->sock) {
@@ -2388,7 +2059,7 @@ iscsi_conn_get_str_param(struct iscsi_cls_conn *cls_conn,
2388 mutex_unlock(&conn->xmitmutex); 2059 mutex_unlock(&conn->xmitmutex);
2389 break; 2060 break;
2390 default: 2061 default:
2391 return -EINVAL; 2062 return iscsi_conn_get_param(cls_conn, param, buf);
2392 } 2063 }
2393 2064
2394 return len; 2065 return len;
@@ -2468,8 +2139,7 @@ static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
2468} 2139}
2469 2140
2470static struct scsi_host_template iscsi_sht = { 2141static struct scsi_host_template iscsi_sht = {
2471 .name = "iSCSI Initiator over TCP/IP, v" 2142 .name = "iSCSI Initiator over TCP/IP",
2472 ISCSI_TCP_VERSION,
2473 .queuecommand = iscsi_queuecommand, 2143 .queuecommand = iscsi_queuecommand,
2474 .change_queue_depth = iscsi_change_queue_depth, 2144 .change_queue_depth = iscsi_change_queue_depth,
2475 .can_queue = ISCSI_XMIT_CMDS_MAX - 1, 2145 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
@@ -2501,7 +2171,11 @@ static struct iscsi_transport iscsi_tcp_transport = {
2501 ISCSI_ERL | 2171 ISCSI_ERL |
2502 ISCSI_CONN_PORT | 2172 ISCSI_CONN_PORT |
2503 ISCSI_CONN_ADDRESS | 2173 ISCSI_CONN_ADDRESS |
2504 ISCSI_EXP_STATSN, 2174 ISCSI_EXP_STATSN |
2175 ISCSI_PERSISTENT_PORT |
2176 ISCSI_PERSISTENT_ADDRESS |
2177 ISCSI_TARGET_NAME |
2178 ISCSI_TPGT,
2505 .host_template = &iscsi_sht, 2179 .host_template = &iscsi_sht,
2506 .conndata_size = sizeof(struct iscsi_conn), 2180 .conndata_size = sizeof(struct iscsi_conn),
2507 .max_conn = 1, 2181 .max_conn = 1,
@@ -2514,14 +2188,10 @@ static struct iscsi_transport iscsi_tcp_transport = {
2514 .bind_conn = iscsi_tcp_conn_bind, 2188 .bind_conn = iscsi_tcp_conn_bind,
2515 .destroy_conn = iscsi_tcp_conn_destroy, 2189 .destroy_conn = iscsi_tcp_conn_destroy,
2516 .set_param = iscsi_conn_set_param, 2190 .set_param = iscsi_conn_set_param,
2517 .get_conn_param = iscsi_conn_get_param, 2191 .get_conn_param = iscsi_tcp_conn_get_param,
2518 .get_conn_str_param = iscsi_conn_get_str_param,
2519 .get_session_param = iscsi_session_get_param, 2192 .get_session_param = iscsi_session_get_param,
2520 .start_conn = iscsi_conn_start, 2193 .start_conn = iscsi_conn_start,
2521 .stop_conn = iscsi_conn_stop, 2194 .stop_conn = iscsi_tcp_conn_stop,
2522 /* these are called as part of conn recovery */
2523 .suspend_conn_recv = iscsi_tcp_suspend_conn_rx,
2524 .terminate_conn = iscsi_tcp_terminate_conn,
2525 /* IO */ 2195 /* IO */
2526 .send_pdu = iscsi_conn_send_pdu, 2196 .send_pdu = iscsi_conn_send_pdu,
2527 .get_stats = iscsi_conn_get_stats, 2197 .get_stats = iscsi_conn_get_stats,
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-14 21:22:15 -0400