aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi/iscsi_tcp.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/iscsi_tcp.c')
-rw-r--r--drivers/scsi/iscsi_tcp.c801
1 files changed, 313 insertions, 488 deletions
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 058f094f945a..0a9dbc59663f 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -108,8 +108,8 @@ iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
108{ 108{
109 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 109 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
110 110
111 crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc); 111 crypto_hash_digest(&tcp_conn->tx_hash, &buf->sg, buf->sg.length, crc);
112 buf->sg.length += sizeof(uint32_t); 112 buf->sg.length = tcp_conn->hdr_size;
113} 113}
114 114
115static inline int 115static inline int
@@ -281,7 +281,6 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
281{ 281{
282 struct iscsi_data *hdr; 282 struct iscsi_data *hdr;
283 struct scsi_cmnd *sc = ctask->sc; 283 struct scsi_cmnd *sc = ctask->sc;
284 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
285 284
286 hdr = &r2t->dtask.hdr; 285 hdr = &r2t->dtask.hdr;
287 memset(hdr, 0, sizeof(struct iscsi_data)); 286 memset(hdr, 0, sizeof(struct iscsi_data));
@@ -336,10 +335,12 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
336 sg_count += sg->length; 335 sg_count += sg->length;
337 } 336 }
338 BUG_ON(r2t->sg == NULL); 337 BUG_ON(r2t->sg == NULL);
339 } else 338 } else {
340 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 339 iscsi_buf_init_iov(&r2t->sendbuf,
341 (char*)sc->request_buffer + r2t->data_offset, 340 (char*)sc->request_buffer + r2t->data_offset,
342 r2t->data_count); 341 r2t->data_count);
342 r2t->sg = NULL;
343 }
343} 344}
344 345
345/** 346/**
@@ -358,8 +359,11 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
358 int r2tsn = be32_to_cpu(rhdr->r2tsn); 359 int r2tsn = be32_to_cpu(rhdr->r2tsn);
359 int rc; 360 int rc;
360 361
361 if (tcp_conn->in.datalen) 362 if (tcp_conn->in.datalen) {
363 printk(KERN_ERR "iscsi_tcp: invalid R2t with datalen %d\n",
364 tcp_conn->in.datalen);
362 return ISCSI_ERR_DATALEN; 365 return ISCSI_ERR_DATALEN;
366 }
363 367
364 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn) 368 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
365 return ISCSI_ERR_R2TSN; 369 return ISCSI_ERR_R2TSN;
@@ -385,15 +389,23 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
385 389
386 r2t->exp_statsn = rhdr->statsn; 390 r2t->exp_statsn = rhdr->statsn;
387 r2t->data_length = be32_to_cpu(rhdr->data_length); 391 r2t->data_length = be32_to_cpu(rhdr->data_length);
388 if (r2t->data_length == 0 || 392 if (r2t->data_length == 0) {
389 r2t->data_length > session->max_burst) { 393 printk(KERN_ERR "iscsi_tcp: invalid R2T with zero data len\n");
390 spin_unlock(&session->lock); 394 spin_unlock(&session->lock);
391 return ISCSI_ERR_DATALEN; 395 return ISCSI_ERR_DATALEN;
392 } 396 }
393 397
398 if (r2t->data_length > session->max_burst)
399 debug_scsi("invalid R2T with data len %u and max burst %u."
400 "Attempting to execute request.\n",
401 r2t->data_length, session->max_burst);
402
394 r2t->data_offset = be32_to_cpu(rhdr->data_offset); 403 r2t->data_offset = be32_to_cpu(rhdr->data_offset);
395 if (r2t->data_offset + r2t->data_length > ctask->total_length) { 404 if (r2t->data_offset + r2t->data_length > ctask->total_length) {
396 spin_unlock(&session->lock); 405 spin_unlock(&session->lock);
406 printk(KERN_ERR "iscsi_tcp: invalid R2T with data len %u at "
407 "offset %u and total length %d\n", r2t->data_length,
408 r2t->data_offset, ctask->total_length);
397 return ISCSI_ERR_DATALEN; 409 return ISCSI_ERR_DATALEN;
398 } 410 }
399 411
@@ -456,7 +468,8 @@ iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
456 468
457 sg_init_one(&sg, (u8 *)hdr, 469 sg_init_one(&sg, (u8 *)hdr,
458 sizeof(struct iscsi_hdr) + ahslen); 470 sizeof(struct iscsi_hdr) + ahslen);
459 crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst); 471 crypto_hash_digest(&tcp_conn->rx_hash, &sg, sg.length,
472 (u8 *)&cdgst);
460 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) + 473 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
461 ahslen); 474 ahslen);
462 if (cdgst != rdgst) { 475 if (cdgst != rdgst) {
@@ -492,7 +505,6 @@ iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
492 goto copy_hdr; 505 goto copy_hdr;
493 506
494 spin_lock(&session->lock); 507 spin_lock(&session->lock);
495 iscsi_tcp_cleanup_ctask(conn, tcp_conn->in.ctask);
496 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0); 508 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
497 spin_unlock(&session->lock); 509 spin_unlock(&session->lock);
498 break; 510 break;
@@ -637,10 +649,9 @@ iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
637 * byte counters. 649 * byte counters.
638 **/ 650 **/
639static inline int 651static inline int
640iscsi_tcp_copy(struct iscsi_conn *conn) 652iscsi_tcp_copy(struct iscsi_conn *conn, int buf_size)
641{ 653{
642 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 654 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
643 int buf_size = tcp_conn->in.datalen;
644 int buf_left = buf_size - tcp_conn->data_copied; 655 int buf_left = buf_size - tcp_conn->data_copied;
645 int size = min(tcp_conn->in.copy, buf_left); 656 int size = min(tcp_conn->in.copy, buf_left);
646 int rc; 657 int rc;
@@ -665,15 +676,15 @@ iscsi_tcp_copy(struct iscsi_conn *conn)
665} 676}
666 677
667static inline void 678static inline void
668partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn, 679partial_sg_digest_update(struct hash_desc *desc, struct scatterlist *sg,
669 struct scatterlist *sg, int offset, int length) 680 int offset, int length)
670{ 681{
671 struct scatterlist temp; 682 struct scatterlist temp;
672 683
673 memcpy(&temp, sg, sizeof(struct scatterlist)); 684 memcpy(&temp, sg, sizeof(struct scatterlist));
674 temp.offset = offset; 685 temp.offset = offset;
675 temp.length = length; 686 temp.length = length;
676 crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1); 687 crypto_hash_update(desc, &temp, length);
677} 688}
678 689
679static void 690static void
@@ -682,7 +693,7 @@ iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
682 struct scatterlist tmp; 693 struct scatterlist tmp;
683 694
684 sg_init_one(&tmp, buf, len); 695 sg_init_one(&tmp, buf, len);
685 crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1); 696 crypto_hash_update(&tcp_conn->rx_hash, &tmp, len);
686} 697}
687 698
688static int iscsi_scsi_data_in(struct iscsi_conn *conn) 699static int iscsi_scsi_data_in(struct iscsi_conn *conn)
@@ -736,11 +747,12 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
736 if (!rc) { 747 if (!rc) {
737 if (conn->datadgst_en) { 748 if (conn->datadgst_en) {
738 if (!offset) 749 if (!offset)
739 crypto_digest_update( 750 crypto_hash_update(
740 tcp_conn->data_rx_tfm, 751 &tcp_conn->rx_hash,
741 &sg[i], 1); 752 &sg[i], 1);
742 else 753 else
743 partial_sg_digest_update(tcp_conn, 754 partial_sg_digest_update(
755 &tcp_conn->rx_hash,
744 &sg[i], 756 &sg[i],
745 sg[i].offset + offset, 757 sg[i].offset + offset,
746 sg[i].length - offset); 758 sg[i].length - offset);
@@ -754,8 +766,10 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
754 /* 766 /*
755 * data-in is complete, but buffer not... 767 * data-in is complete, but buffer not...
756 */ 768 */
757 partial_sg_digest_update(tcp_conn, &sg[i], 769 partial_sg_digest_update(&tcp_conn->rx_hash,
758 sg[i].offset, sg[i].length-rc); 770 &sg[i],
771 sg[i].offset,
772 sg[i].length-rc);
759 rc = 0; 773 rc = 0;
760 break; 774 break;
761 } 775 }
@@ -772,7 +786,6 @@ done:
772 (long)sc, sc->result, ctask->itt, 786 (long)sc, sc->result, ctask->itt,
773 tcp_conn->in.hdr->flags); 787 tcp_conn->in.hdr->flags);
774 spin_lock(&conn->session->lock); 788 spin_lock(&conn->session->lock);
775 iscsi_tcp_cleanup_ctask(conn, ctask);
776 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0); 789 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
777 spin_unlock(&conn->session->lock); 790 spin_unlock(&conn->session->lock);
778 } 791 }
@@ -792,9 +805,6 @@ iscsi_data_recv(struct iscsi_conn *conn)
792 rc = iscsi_scsi_data_in(conn); 805 rc = iscsi_scsi_data_in(conn);
793 break; 806 break;
794 case ISCSI_OP_SCSI_CMD_RSP: 807 case ISCSI_OP_SCSI_CMD_RSP:
795 spin_lock(&conn->session->lock);
796 iscsi_tcp_cleanup_ctask(conn, tcp_conn->in.ctask);
797 spin_unlock(&conn->session->lock);
798 case ISCSI_OP_TEXT_RSP: 808 case ISCSI_OP_TEXT_RSP:
799 case ISCSI_OP_LOGIN_RSP: 809 case ISCSI_OP_LOGIN_RSP:
800 case ISCSI_OP_ASYNC_EVENT: 810 case ISCSI_OP_ASYNC_EVENT:
@@ -803,7 +813,7 @@ iscsi_data_recv(struct iscsi_conn *conn)
803 * Collect data segment to the connection's data 813 * Collect data segment to the connection's data
804 * placeholder 814 * placeholder
805 */ 815 */
806 if (iscsi_tcp_copy(conn)) { 816 if (iscsi_tcp_copy(conn, tcp_conn->in.datalen)) {
807 rc = -EAGAIN; 817 rc = -EAGAIN;
808 goto exit; 818 goto exit;
809 } 819 }
@@ -876,10 +886,8 @@ more:
876 */ 886 */
877 rc = iscsi_tcp_hdr_recv(conn); 887 rc = iscsi_tcp_hdr_recv(conn);
878 if (!rc && tcp_conn->in.datalen) { 888 if (!rc && tcp_conn->in.datalen) {
879 if (conn->datadgst_en) { 889 if (conn->datadgst_en)
880 BUG_ON(!tcp_conn->data_rx_tfm); 890 crypto_hash_init(&tcp_conn->rx_hash);
881 crypto_digest_init(tcp_conn->data_rx_tfm);
882 }
883 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV; 891 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
884 } else if (rc) { 892 } else if (rc) {
885 iscsi_conn_failure(conn, rc); 893 iscsi_conn_failure(conn, rc);
@@ -892,10 +900,15 @@ more:
892 900
893 debug_tcp("extra data_recv offset %d copy %d\n", 901 debug_tcp("extra data_recv offset %d copy %d\n",
894 tcp_conn->in.offset, tcp_conn->in.copy); 902 tcp_conn->in.offset, tcp_conn->in.copy);
895 skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset, 903 rc = iscsi_tcp_copy(conn, sizeof(uint32_t));
896 &recv_digest, 4); 904 if (rc) {
897 tcp_conn->in.offset += 4; 905 if (rc == -EAGAIN)
898 tcp_conn->in.copy -= 4; 906 goto again;
907 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
908 return 0;
909 }
910
911 memcpy(&recv_digest, conn->data, sizeof(uint32_t));
899 if (recv_digest != tcp_conn->in.datadgst) { 912 if (recv_digest != tcp_conn->in.datadgst) {
900 debug_tcp("iscsi_tcp: data digest error!" 913 debug_tcp("iscsi_tcp: data digest error!"
901 "0x%x != 0x%x\n", recv_digest, 914 "0x%x != 0x%x\n", recv_digest,
@@ -931,13 +944,14 @@ more:
931 tcp_conn->in.padding); 944 tcp_conn->in.padding);
932 memset(pad, 0, tcp_conn->in.padding); 945 memset(pad, 0, tcp_conn->in.padding);
933 sg_init_one(&sg, pad, tcp_conn->in.padding); 946 sg_init_one(&sg, pad, tcp_conn->in.padding);
934 crypto_digest_update(tcp_conn->data_rx_tfm, 947 crypto_hash_update(&tcp_conn->rx_hash,
935 &sg, 1); 948 &sg, sg.length);
936 } 949 }
937 crypto_digest_final(tcp_conn->data_rx_tfm, 950 crypto_hash_final(&tcp_conn->rx_hash,
938 (u8 *) & tcp_conn->in.datadgst); 951 (u8 *) &tcp_conn->in.datadgst);
939 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst); 952 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
940 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV; 953 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
954 tcp_conn->data_copied = 0;
941 } else 955 } else
942 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER; 956 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
943 } 957 }
@@ -1177,37 +1191,12 @@ iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1177 1191
1178static inline void 1192static inline void
1179iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn, 1193iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
1180 struct iscsi_cmd_task *ctask) 1194 struct iscsi_tcp_cmd_task *tcp_ctask)
1181{ 1195{
1182 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1196 crypto_hash_init(&tcp_conn->tx_hash);
1183
1184 BUG_ON(!tcp_conn->data_tx_tfm);
1185 crypto_digest_init(tcp_conn->data_tx_tfm);
1186 tcp_ctask->digest_count = 4; 1197 tcp_ctask->digest_count = 4;
1187} 1198}
1188 1199
1189static int
1190iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1191 struct iscsi_buf *buf, uint32_t *digest, int final)
1192{
1193 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1194 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1195 int rc = 0;
1196 int sent = 0;
1197
1198 if (final)
1199 crypto_digest_final(tcp_conn->data_tx_tfm, (u8*)digest);
1200
1201 iscsi_buf_init_iov(buf, (char*)digest, 4);
1202 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
1203 if (rc) {
1204 tcp_ctask->datadigest = *digest;
1205 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
1206 } else
1207 tcp_ctask->digest_count = 4;
1208 return rc;
1209}
1210
1211/** 1200/**
1212 * iscsi_solicit_data_cont - initialize next Data-Out 1201 * iscsi_solicit_data_cont - initialize next Data-Out
1213 * @conn: iscsi connection 1202 * @conn: iscsi connection
@@ -1225,7 +1214,6 @@ static void
1225iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1214iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1226 struct iscsi_r2t_info *r2t, int left) 1215 struct iscsi_r2t_info *r2t, int left)
1227{ 1216{
1228 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1229 struct iscsi_data *hdr; 1217 struct iscsi_data *hdr;
1230 struct scsi_cmnd *sc = ctask->sc; 1218 struct scsi_cmnd *sc = ctask->sc;
1231 int new_offset; 1219 int new_offset;
@@ -1254,27 +1242,30 @@ iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1254 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr, 1242 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
1255 sizeof(struct iscsi_hdr)); 1243 sizeof(struct iscsi_hdr));
1256 1244
1257 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) { 1245 if (iscsi_buf_left(&r2t->sendbuf))
1258 BUG_ON(tcp_ctask->bad_sg == r2t->sg); 1246 return;
1247
1248 if (sc->use_sg) {
1259 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1249 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1260 r2t->sg += 1; 1250 r2t->sg += 1;
1261 } else 1251 } else {
1262 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 1252 iscsi_buf_init_iov(&r2t->sendbuf,
1263 (char*)sc->request_buffer + new_offset, 1253 (char*)sc->request_buffer + new_offset,
1264 r2t->data_count); 1254 r2t->data_count);
1255 r2t->sg = NULL;
1256 }
1265} 1257}
1266 1258
1267static void 1259static void iscsi_set_padding(struct iscsi_tcp_cmd_task *tcp_ctask,
1268iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1260 unsigned long len)
1269{ 1261{
1270 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1262 tcp_ctask->pad_count = len & (ISCSI_PAD_LEN - 1);
1271 struct iscsi_data_task *dtask; 1263 if (!tcp_ctask->pad_count)
1264 return;
1272 1265
1273 dtask = tcp_ctask->dtask = &tcp_ctask->unsol_dtask; 1266 tcp_ctask->pad_count = ISCSI_PAD_LEN - tcp_ctask->pad_count;
1274 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr, 1267 debug_scsi("write padding %d bytes\n", tcp_ctask->pad_count);
1275 tcp_ctask->r2t_data_count); 1268 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1276 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
1277 sizeof(struct iscsi_hdr));
1278} 1269}
1279 1270
1280/** 1271/**
@@ -1302,38 +1293,20 @@ iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
1302 if (sc->use_sg) { 1293 if (sc->use_sg) {
1303 struct scatterlist *sg = sc->request_buffer; 1294 struct scatterlist *sg = sc->request_buffer;
1304 1295
1305 iscsi_buf_init_sg(&tcp_ctask->sendbuf, 1296 iscsi_buf_init_sg(&tcp_ctask->sendbuf, sg);
1306 &sg[tcp_ctask->sg_count++]); 1297 tcp_ctask->sg = sg + 1;
1307 tcp_ctask->sg = sg;
1308 tcp_ctask->bad_sg = sg + sc->use_sg; 1298 tcp_ctask->bad_sg = sg + sc->use_sg;
1309 } else 1299 } else {
1310 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 1300 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1311 sc->request_buffer, 1301 sc->request_buffer,
1312 sc->request_bufflen); 1302 sc->request_bufflen);
1313 1303 tcp_ctask->sg = NULL;
1314 if (ctask->imm_count) 1304 tcp_ctask->bad_sg = NULL;
1315 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1316
1317 tcp_ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1318 if (tcp_ctask->pad_count) {
1319 tcp_ctask->pad_count = ISCSI_PAD_LEN -
1320 tcp_ctask->pad_count;
1321 debug_scsi("write padding %d bytes\n",
1322 tcp_ctask->pad_count);
1323 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1324 } 1305 }
1325 1306 debug_scsi("cmd [itt 0x%x total %d imm_data %d "
1326 if (ctask->unsol_count) 1307 "unsol count %d, unsol offset %d]\n",
1327 tcp_ctask->xmstate |= XMSTATE_UNS_HDR |
1328 XMSTATE_UNS_INIT;
1329 tcp_ctask->r2t_data_count = ctask->total_length -
1330 ctask->imm_count -
1331 ctask->unsol_count;
1332
1333 debug_scsi("cmd [itt 0x%x total %d imm %d imm_data %d "
1334 "r2t_data %d]\n",
1335 ctask->itt, ctask->total_length, ctask->imm_count, 1308 ctask->itt, ctask->total_length, ctask->imm_count,
1336 ctask->unsol_count, tcp_ctask->r2t_data_count); 1309 ctask->unsol_count, ctask->unsol_offset);
1337 } else 1310 } else
1338 tcp_ctask->xmstate = XMSTATE_R_HDR; 1311 tcp_ctask->xmstate = XMSTATE_R_HDR;
1339 1312
@@ -1415,8 +1388,8 @@ iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1415} 1388}
1416 1389
1417static inline int 1390static inline int
1418handle_xmstate_r_hdr(struct iscsi_conn *conn, 1391iscsi_send_read_hdr(struct iscsi_conn *conn,
1419 struct iscsi_tcp_cmd_task *tcp_ctask) 1392 struct iscsi_tcp_cmd_task *tcp_ctask)
1420{ 1393{
1421 int rc; 1394 int rc;
1422 1395
@@ -1434,7 +1407,7 @@ handle_xmstate_r_hdr(struct iscsi_conn *conn,
1434} 1407}
1435 1408
1436static inline int 1409static inline int
1437handle_xmstate_w_hdr(struct iscsi_conn *conn, 1410iscsi_send_write_hdr(struct iscsi_conn *conn,
1438 struct iscsi_cmd_task *ctask) 1411 struct iscsi_cmd_task *ctask)
1439{ 1412{
1440 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1413 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
@@ -1445,85 +1418,126 @@ handle_xmstate_w_hdr(struct iscsi_conn *conn,
1445 iscsi_hdr_digest(conn, &tcp_ctask->headbuf, 1418 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1446 (u8*)tcp_ctask->hdrext); 1419 (u8*)tcp_ctask->hdrext);
1447 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count); 1420 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count);
1448 if (rc) 1421 if (rc) {
1449 tcp_ctask->xmstate |= XMSTATE_W_HDR; 1422 tcp_ctask->xmstate |= XMSTATE_W_HDR;
1450 return rc; 1423 return rc;
1424 }
1425
1426 if (ctask->imm_count) {
1427 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1428 iscsi_set_padding(tcp_ctask, ctask->imm_count);
1429
1430 if (ctask->conn->datadgst_en) {
1431 iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
1432 tcp_ctask->immdigest = 0;
1433 }
1434 }
1435
1436 if (ctask->unsol_count)
1437 tcp_ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1438 return 0;
1451} 1439}
1452 1440
1453static inline int 1441static int
1454handle_xmstate_data_digest(struct iscsi_conn *conn, 1442iscsi_send_padding(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1455 struct iscsi_cmd_task *ctask)
1456{ 1443{
1457 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1444 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1458 int rc; 1445 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1446 int sent = 0, rc;
1459 1447
1460 tcp_ctask->xmstate &= ~XMSTATE_DATA_DIGEST; 1448 if (tcp_ctask->xmstate & XMSTATE_W_PAD) {
1461 debug_tcp("resent data digest 0x%x\n", tcp_ctask->datadigest); 1449 iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
1462 rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf, 1450 tcp_ctask->pad_count);
1463 &tcp_ctask->datadigest, 0); 1451 if (conn->datadgst_en)
1452 crypto_hash_update(&tcp_conn->tx_hash,
1453 &tcp_ctask->sendbuf.sg,
1454 tcp_ctask->sendbuf.sg.length);
1455 } else if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_PAD))
1456 return 0;
1457
1458 tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
1459 tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_PAD;
1460 debug_scsi("sending %d pad bytes for itt 0x%x\n",
1461 tcp_ctask->pad_count, ctask->itt);
1462 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
1463 &sent);
1464 if (rc) { 1464 if (rc) {
1465 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST; 1465 debug_scsi("padding send failed %d\n", rc);
1466 debug_tcp("resent data digest 0x%x fail!\n", 1466 tcp_ctask->xmstate |= XMSTATE_W_RESEND_PAD;
1467 tcp_ctask->datadigest);
1468 } 1467 }
1469
1470 return rc; 1468 return rc;
1471} 1469}
1472 1470
1473static inline int 1471static int
1474handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1472iscsi_send_digest(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1473 struct iscsi_buf *buf, uint32_t *digest)
1475{ 1474{
1476 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1475 struct iscsi_tcp_cmd_task *tcp_ctask;
1477 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1476 struct iscsi_tcp_conn *tcp_conn;
1478 int rc; 1477 int rc, sent = 0;
1479 1478
1480 BUG_ON(!ctask->imm_count); 1479 if (!conn->datadgst_en)
1481 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA; 1480 return 0;
1482 1481
1483 if (conn->datadgst_en) { 1482 tcp_ctask = ctask->dd_data;
1484 iscsi_data_digest_init(tcp_conn, ctask); 1483 tcp_conn = conn->dd_data;
1485 tcp_ctask->immdigest = 0;
1486 }
1487 1484
1488 for (;;) { 1485 if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_DATA_DIGEST)) {
1489 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, 1486 crypto_hash_final(&tcp_conn->tx_hash, (u8*)digest);
1490 &ctask->imm_count, &tcp_ctask->sent); 1487 iscsi_buf_init_iov(buf, (char*)digest, 4);
1491 if (rc) { 1488 }
1492 tcp_ctask->xmstate |= XMSTATE_IMM_DATA; 1489 tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_DATA_DIGEST;
1493 if (conn->datadgst_en) {
1494 crypto_digest_final(tcp_conn->data_tx_tfm,
1495 (u8*)&tcp_ctask->immdigest);
1496 debug_tcp("tx imm sendpage fail 0x%x\n",
1497 tcp_ctask->datadigest);
1498 }
1499 return rc;
1500 }
1501 if (conn->datadgst_en)
1502 crypto_digest_update(tcp_conn->data_tx_tfm,
1503 &tcp_ctask->sendbuf.sg, 1);
1504 1490
1505 if (!ctask->imm_count) 1491 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
1506 break; 1492 if (!rc)
1507 iscsi_buf_init_sg(&tcp_ctask->sendbuf, 1493 debug_scsi("sent digest 0x%x for itt 0x%x\n", *digest,
1508 &tcp_ctask->sg[tcp_ctask->sg_count++]); 1494 ctask->itt);
1495 else {
1496 debug_scsi("sending digest 0x%x failed for itt 0x%x!\n",
1497 *digest, ctask->itt);
1498 tcp_ctask->xmstate |= XMSTATE_W_RESEND_DATA_DIGEST;
1509 } 1499 }
1500 return rc;
1501}
1510 1502
1511 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) { 1503static int
1512 rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf, 1504iscsi_send_data(struct iscsi_cmd_task *ctask, struct iscsi_buf *sendbuf,
1513 &tcp_ctask->immdigest, 1); 1505 struct scatterlist **sg, int *sent, int *count,
1514 if (rc) { 1506 struct iscsi_buf *digestbuf, uint32_t *digest)
1515 debug_tcp("sending imm digest 0x%x fail!\n", 1507{
1516 tcp_ctask->immdigest); 1508 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1517 return rc; 1509 struct iscsi_conn *conn = ctask->conn;
1510 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1511 int rc, buf_sent, offset;
1512
1513 while (*count) {
1514 buf_sent = 0;
1515 offset = sendbuf->sent;
1516
1517 rc = iscsi_sendpage(conn, sendbuf, count, &buf_sent);
1518 *sent = *sent + buf_sent;
1519 if (buf_sent && conn->datadgst_en)
1520 partial_sg_digest_update(&tcp_conn->tx_hash,
1521 &sendbuf->sg, sendbuf->sg.offset + offset,
1522 buf_sent);
1523 if (!iscsi_buf_left(sendbuf) && *sg != tcp_ctask->bad_sg) {
1524 iscsi_buf_init_sg(sendbuf, *sg);
1525 *sg = *sg + 1;
1518 } 1526 }
1519 debug_tcp("sending imm digest 0x%x\n", tcp_ctask->immdigest); 1527
1528 if (rc)
1529 return rc;
1520 } 1530 }
1521 1531
1522 return 0; 1532 rc = iscsi_send_padding(conn, ctask);
1533 if (rc)
1534 return rc;
1535
1536 return iscsi_send_digest(conn, ctask, digestbuf, digest);
1523} 1537}
1524 1538
1525static inline int 1539static int
1526handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1540iscsi_send_unsol_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1527{ 1541{
1528 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1542 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1529 struct iscsi_data_task *dtask; 1543 struct iscsi_data_task *dtask;
@@ -1531,12 +1545,17 @@ handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1531 1545
1532 tcp_ctask->xmstate |= XMSTATE_UNS_DATA; 1546 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1533 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) { 1547 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
1534 iscsi_unsolicit_data_init(conn, ctask); 1548 dtask = &tcp_ctask->unsol_dtask;
1535 dtask = tcp_ctask->dtask; 1549
1550 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr);
1551 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
1552 sizeof(struct iscsi_hdr));
1536 if (conn->hdrdgst_en) 1553 if (conn->hdrdgst_en)
1537 iscsi_hdr_digest(conn, &tcp_ctask->headbuf, 1554 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1538 (u8*)dtask->hdrext); 1555 (u8*)dtask->hdrext);
1556
1539 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT; 1557 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
1558 iscsi_set_padding(tcp_ctask, ctask->data_count);
1540 } 1559 }
1541 1560
1542 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count); 1561 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count);
@@ -1546,254 +1565,138 @@ handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1546 return rc; 1565 return rc;
1547 } 1566 }
1548 1567
1568 if (conn->datadgst_en) {
1569 dtask = &tcp_ctask->unsol_dtask;
1570 iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
1571 dtask->digest = 0;
1572 }
1573
1549 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n", 1574 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1550 ctask->itt, ctask->unsol_count, tcp_ctask->sent); 1575 ctask->itt, ctask->unsol_count, tcp_ctask->sent);
1551 return 0; 1576 return 0;
1552} 1577}
1553 1578
1554static inline int 1579static int
1555handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1580iscsi_send_unsol_pdu(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1556{ 1581{
1557 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1582 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1558 struct iscsi_data_task *dtask = tcp_ctask->dtask;
1559 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1560 int rc; 1583 int rc;
1561 1584
1562 BUG_ON(!ctask->data_count); 1585 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
1563 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA; 1586 BUG_ON(!ctask->unsol_count);
1564 1587 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
1565 if (conn->datadgst_en) { 1588send_hdr:
1566 iscsi_data_digest_init(tcp_conn, ctask); 1589 rc = iscsi_send_unsol_hdr(conn, ctask);
1567 dtask->digest = 0; 1590 if (rc)
1591 return rc;
1568 } 1592 }
1569 1593
1570 for (;;) { 1594 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
1595 struct iscsi_data_task *dtask = &tcp_ctask->unsol_dtask;
1571 int start = tcp_ctask->sent; 1596 int start = tcp_ctask->sent;
1572 1597
1573 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, 1598 rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
1574 &ctask->data_count, &tcp_ctask->sent); 1599 &tcp_ctask->sent, &ctask->data_count,
1575 if (rc) { 1600 &dtask->digestbuf, &dtask->digest);
1576 ctask->unsol_count -= tcp_ctask->sent - start;
1577 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1578 /* will continue with this ctask later.. */
1579 if (conn->datadgst_en) {
1580 crypto_digest_final(tcp_conn->data_tx_tfm,
1581 (u8 *)&dtask->digest);
1582 debug_tcp("tx uns data fail 0x%x\n",
1583 dtask->digest);
1584 }
1585 return rc;
1586 }
1587
1588 BUG_ON(tcp_ctask->sent > ctask->total_length);
1589 ctask->unsol_count -= tcp_ctask->sent - start; 1601 ctask->unsol_count -= tcp_ctask->sent - start;
1590 1602 if (rc)
1603 return rc;
1604 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1591 /* 1605 /*
1592 * XXX:we may run here with un-initial sendbuf. 1606 * Done with the Data-Out. Next, check if we need
1593 * so pass it 1607 * to send another unsolicited Data-Out.
1594 */ 1608 */
1595 if (conn->datadgst_en && tcp_ctask->sent - start > 0) 1609 if (ctask->unsol_count) {
1596 crypto_digest_update(tcp_conn->data_tx_tfm, 1610 debug_scsi("sending more uns\n");
1597 &tcp_ctask->sendbuf.sg, 1); 1611 tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
1598 1612 goto send_hdr;
1599 if (!ctask->data_count)
1600 break;
1601 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1602 &tcp_ctask->sg[tcp_ctask->sg_count++]);
1603 }
1604 BUG_ON(ctask->unsol_count < 0);
1605
1606 /*
1607 * Done with the Data-Out. Next, check if we need
1608 * to send another unsolicited Data-Out.
1609 */
1610 if (ctask->unsol_count) {
1611 if (conn->datadgst_en) {
1612 rc = iscsi_digest_final_send(conn, ctask,
1613 &dtask->digestbuf,
1614 &dtask->digest, 1);
1615 if (rc) {
1616 debug_tcp("send uns digest 0x%x fail\n",
1617 dtask->digest);
1618 return rc;
1619 }
1620 debug_tcp("sending uns digest 0x%x, more uns\n",
1621 dtask->digest);
1622 } 1613 }
1623 tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
1624 return 1;
1625 } 1614 }
1626
1627 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1628 rc = iscsi_digest_final_send(conn, ctask,
1629 &dtask->digestbuf,
1630 &dtask->digest, 1);
1631 if (rc) {
1632 debug_tcp("send last uns digest 0x%x fail\n",
1633 dtask->digest);
1634 return rc;
1635 }
1636 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1637 }
1638
1639 return 0; 1615 return 0;
1640} 1616}
1641 1617
1642static inline int 1618static int iscsi_send_sol_pdu(struct iscsi_conn *conn,
1643handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1619 struct iscsi_cmd_task *ctask)
1644{ 1620{
1645 struct iscsi_session *session = conn->session;
1646 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1647 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1621 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1648 struct iscsi_r2t_info *r2t = tcp_ctask->r2t; 1622 struct iscsi_session *session = conn->session;
1649 struct iscsi_data_task *dtask = &r2t->dtask; 1623 struct iscsi_r2t_info *r2t;
1624 struct iscsi_data_task *dtask;
1650 int left, rc; 1625 int left, rc;
1651 1626
1652 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA; 1627 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
1653 tcp_ctask->dtask = dtask; 1628 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1654
1655 if (conn->datadgst_en) {
1656 iscsi_data_digest_init(tcp_conn, ctask);
1657 dtask->digest = 0;
1658 }
1659solicit_again:
1660 /*
1661 * send Data-Out within this R2T sequence.
1662 */
1663 if (!r2t->data_count)
1664 goto data_out_done;
1665
1666 rc = iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent);
1667 if (rc) {
1668 tcp_ctask->xmstate |= XMSTATE_SOL_DATA; 1629 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1669 /* will continue with this ctask later.. */ 1630 if (!tcp_ctask->r2t)
1670 if (conn->datadgst_en) { 1631 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
1671 crypto_digest_final(tcp_conn->data_tx_tfm, 1632 sizeof(void*));
1672 (u8 *)&dtask->digest); 1633send_hdr:
1673 debug_tcp("r2t data send fail 0x%x\n", dtask->digest); 1634 r2t = tcp_ctask->r2t;
1674 } 1635 dtask = &r2t->dtask;
1675 return rc;
1676 }
1677 1636
1678 BUG_ON(r2t->data_count < 0); 1637 if (conn->hdrdgst_en)
1679 if (conn->datadgst_en) 1638 iscsi_hdr_digest(conn, &r2t->headbuf,
1680 crypto_digest_update(tcp_conn->data_tx_tfm, &r2t->sendbuf.sg, 1639 (u8*)dtask->hdrext);
1681 1); 1640 rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
1682 1641 if (rc) {
1683 if (r2t->data_count) { 1642 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1684 BUG_ON(ctask->sc->use_sg == 0); 1643 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
1685 if (!iscsi_buf_left(&r2t->sendbuf)) { 1644 return rc;
1686 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
1687 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1688 r2t->sg += 1;
1689 } 1645 }
1690 goto solicit_again;
1691 }
1692 1646
1693data_out_done:
1694 /*
1695 * Done with this Data-Out. Next, check if we have
1696 * to send another Data-Out for this R2T.
1697 */
1698 BUG_ON(r2t->data_length - r2t->sent < 0);
1699 left = r2t->data_length - r2t->sent;
1700 if (left) {
1701 if (conn->datadgst_en) { 1647 if (conn->datadgst_en) {
1702 rc = iscsi_digest_final_send(conn, ctask, 1648 iscsi_data_digest_init(conn->dd_data, tcp_ctask);
1703 &dtask->digestbuf, 1649 dtask->digest = 0;
1704 &dtask->digest, 1);
1705 if (rc) {
1706 debug_tcp("send r2t data digest 0x%x"
1707 "fail\n", dtask->digest);
1708 return rc;
1709 }
1710 debug_tcp("r2t data send digest 0x%x\n",
1711 dtask->digest);
1712 } 1650 }
1713 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1714 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1715 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1716 return 1;
1717 }
1718 1651
1719 /* 1652 iscsi_set_padding(tcp_ctask, r2t->data_count);
1720 * Done with this R2T. Check if there are more 1653 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
1721 * outstanding R2Ts ready to be processed. 1654 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
1722 */ 1655 r2t->sent);
1723 BUG_ON(tcp_ctask->r2t_data_count - r2t->data_length < 0);
1724 if (conn->datadgst_en) {
1725 rc = iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1726 &dtask->digest, 1);
1727 if (rc) {
1728 debug_tcp("send last r2t data digest 0x%x"
1729 "fail\n", dtask->digest);
1730 return rc;
1731 }
1732 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1733 }
1734
1735 tcp_ctask->r2t_data_count -= r2t->data_length;
1736 tcp_ctask->r2t = NULL;
1737 spin_lock_bh(&session->lock);
1738 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
1739 spin_unlock_bh(&session->lock);
1740 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
1741 tcp_ctask->r2t = r2t;
1742 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1743 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1744 return 1;
1745 } 1656 }
1746 1657
1747 return 0; 1658 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
1748} 1659 r2t = tcp_ctask->r2t;
1660 dtask = &r2t->dtask;
1749 1661
1750static inline int 1662 rc = iscsi_send_data(ctask, &r2t->sendbuf, &r2t->sg,
1751handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1663 &r2t->sent, &r2t->data_count,
1752{ 1664 &dtask->digestbuf, &dtask->digest);
1753 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1665 if (rc)
1754 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1666 return rc;
1755 struct iscsi_data_task *dtask = tcp_ctask->dtask; 1667 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1756 int sent = 0, rc;
1757 1668
1758 tcp_ctask->xmstate &= ~XMSTATE_W_PAD; 1669 /*
1759 iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad, 1670 * Done with this Data-Out. Next, check if we have
1760 tcp_ctask->pad_count); 1671 * to send another Data-Out for this R2T.
1761 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count, 1672 */
1762 &sent); 1673 BUG_ON(r2t->data_length - r2t->sent < 0);
1763 if (rc) { 1674 left = r2t->data_length - r2t->sent;
1764 tcp_ctask->xmstate |= XMSTATE_W_PAD; 1675 if (left) {
1765 return rc; 1676 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1766 } 1677 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1678 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1679 goto send_hdr;
1680 }
1767 1681
1768 if (conn->datadgst_en) { 1682 /*
1769 crypto_digest_update(tcp_conn->data_tx_tfm, 1683 * Done with this R2T. Check if there are more
1770 &tcp_ctask->sendbuf.sg, 1); 1684 * outstanding R2Ts ready to be processed.
1771 /* imm data? */ 1685 */
1772 if (!dtask) { 1686 spin_lock_bh(&session->lock);
1773 rc = iscsi_digest_final_send(conn, ctask, 1687 tcp_ctask->r2t = NULL;
1774 &tcp_ctask->immbuf, 1688 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
1775 &tcp_ctask->immdigest, 1); 1689 sizeof(void*));
1776 if (rc) { 1690 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t,
1777 debug_tcp("send padding digest 0x%x" 1691 sizeof(void*))) {
1778 "fail!\n", tcp_ctask->immdigest); 1692 tcp_ctask->r2t = r2t;
1779 return rc; 1693 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1780 } 1694 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1781 debug_tcp("done with padding, digest 0x%x\n", 1695 spin_unlock_bh(&session->lock);
1782 tcp_ctask->datadigest); 1696 goto send_hdr;
1783 } else {
1784 rc = iscsi_digest_final_send(conn, ctask,
1785 &dtask->digestbuf,
1786 &dtask->digest, 1);
1787 if (rc) {
1788 debug_tcp("send padding digest 0x%x"
1789 "fail\n", dtask->digest);
1790 return rc;
1791 }
1792 debug_tcp("done with padding, digest 0x%x\n",
1793 dtask->digest);
1794 } 1697 }
1698 spin_unlock_bh(&session->lock);
1795 } 1699 }
1796
1797 return 0; 1700 return 0;
1798} 1701}
1799 1702
@@ -1813,85 +1716,30 @@ iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1813 return rc; 1716 return rc;
1814 1717
1815 if (tcp_ctask->xmstate & XMSTATE_R_HDR) 1718 if (tcp_ctask->xmstate & XMSTATE_R_HDR)
1816 return handle_xmstate_r_hdr(conn, tcp_ctask); 1719 return iscsi_send_read_hdr(conn, tcp_ctask);
1817 1720
1818 if (tcp_ctask->xmstate & XMSTATE_W_HDR) { 1721 if (tcp_ctask->xmstate & XMSTATE_W_HDR) {
1819 rc = handle_xmstate_w_hdr(conn, ctask); 1722 rc = iscsi_send_write_hdr(conn, ctask);
1820 if (rc)
1821 return rc;
1822 }
1823
1824 /* XXX: for data digest xmit recover */
1825 if (tcp_ctask->xmstate & XMSTATE_DATA_DIGEST) {
1826 rc = handle_xmstate_data_digest(conn, ctask);
1827 if (rc) 1723 if (rc)
1828 return rc; 1724 return rc;
1829 } 1725 }
1830 1726
1831 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) { 1727 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
1832 rc = handle_xmstate_imm_data(conn, ctask); 1728 rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
1729 &tcp_ctask->sent, &ctask->imm_count,
1730 &tcp_ctask->immbuf, &tcp_ctask->immdigest);
1833 if (rc) 1731 if (rc)
1834 return rc; 1732 return rc;
1733 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
1835 } 1734 }
1836 1735
1837 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) { 1736 rc = iscsi_send_unsol_pdu(conn, ctask);
1838 BUG_ON(!ctask->unsol_count); 1737 if (rc)
1839 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR; 1738 return rc;
1840unsolicit_head_again:
1841 rc = handle_xmstate_uns_hdr(conn, ctask);
1842 if (rc)
1843 return rc;
1844 }
1845
1846 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
1847 rc = handle_xmstate_uns_data(conn, ctask);
1848 if (rc == 1)
1849 goto unsolicit_head_again;
1850 else if (rc)
1851 return rc;
1852 goto done;
1853 }
1854
1855 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
1856 struct iscsi_r2t_info *r2t;
1857
1858 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1859 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1860 if (!tcp_ctask->r2t)
1861 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
1862 sizeof(void*));
1863solicit_head_again:
1864 r2t = tcp_ctask->r2t;
1865 if (conn->hdrdgst_en)
1866 iscsi_hdr_digest(conn, &r2t->headbuf,
1867 (u8*)r2t->dtask.hdrext);
1868 rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
1869 if (rc) {
1870 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1871 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
1872 return rc;
1873 }
1874
1875 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
1876 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
1877 r2t->sent);
1878 }
1879
1880 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
1881 rc = handle_xmstate_sol_data(conn, ctask);
1882 if (rc == 1)
1883 goto solicit_head_again;
1884 if (rc)
1885 return rc;
1886 }
1887 1739
1888done: 1740 rc = iscsi_send_sol_pdu(conn, ctask);
1889 /* 1741 if (rc)
1890 * Last thing to check is whether we need to send write 1742 return rc;
1891 * padding. Note that we check for xmstate equality, not just the bit.
1892 */
1893 if (tcp_ctask->xmstate == XMSTATE_W_PAD)
1894 rc = handle_xmstate_w_pad(conn, ctask);
1895 1743
1896 return rc; 1744 return rc;
1897} 1745}
@@ -1923,8 +1771,24 @@ iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
1923 /* initial operational parameters */ 1771 /* initial operational parameters */
1924 tcp_conn->hdr_size = sizeof(struct iscsi_hdr); 1772 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1925 1773
1774 tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1775 CRYPTO_ALG_ASYNC);
1776 tcp_conn->tx_hash.flags = 0;
1777 if (!tcp_conn->tx_hash.tfm)
1778 goto free_tcp_conn;
1779
1780 tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1781 CRYPTO_ALG_ASYNC);
1782 tcp_conn->rx_hash.flags = 0;
1783 if (!tcp_conn->rx_hash.tfm)
1784 goto free_tx_tfm;
1785
1926 return cls_conn; 1786 return cls_conn;
1927 1787
1788free_tx_tfm:
1789 crypto_free_hash(tcp_conn->tx_hash.tfm);
1790free_tcp_conn:
1791 kfree(tcp_conn);
1928tcp_conn_alloc_fail: 1792tcp_conn_alloc_fail:
1929 iscsi_conn_teardown(cls_conn); 1793 iscsi_conn_teardown(cls_conn);
1930 return NULL; 1794 return NULL;
@@ -1962,14 +1826,10 @@ iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1962 1826
1963 /* now free tcp_conn */ 1827 /* now free tcp_conn */
1964 if (digest) { 1828 if (digest) {
1965 if (tcp_conn->tx_tfm) 1829 if (tcp_conn->tx_hash.tfm)
1966 crypto_free_tfm(tcp_conn->tx_tfm); 1830 crypto_free_hash(tcp_conn->tx_hash.tfm);
1967 if (tcp_conn->rx_tfm) 1831 if (tcp_conn->rx_hash.tfm)
1968 crypto_free_tfm(tcp_conn->rx_tfm); 1832 crypto_free_hash(tcp_conn->rx_hash.tfm);
1969 if (tcp_conn->data_tx_tfm)
1970 crypto_free_tfm(tcp_conn->data_tx_tfm);
1971 if (tcp_conn->data_rx_tfm)
1972 crypto_free_tfm(tcp_conn->data_rx_tfm);
1973 } 1833 }
1974 1834
1975 kfree(tcp_conn); 1835 kfree(tcp_conn);
@@ -1979,9 +1839,11 @@ static void
1979iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) 1839iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
1980{ 1840{
1981 struct iscsi_conn *conn = cls_conn->dd_data; 1841 struct iscsi_conn *conn = cls_conn->dd_data;
1842 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1982 1843
1983 iscsi_conn_stop(cls_conn, flag); 1844 iscsi_conn_stop(cls_conn, flag);
1984 iscsi_tcp_release_conn(conn); 1845 iscsi_tcp_release_conn(conn);
1846 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1985} 1847}
1986 1848
1987static int 1849static int
@@ -2127,48 +1989,11 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
2127 case ISCSI_PARAM_HDRDGST_EN: 1989 case ISCSI_PARAM_HDRDGST_EN:
2128 iscsi_set_param(cls_conn, param, buf, buflen); 1990 iscsi_set_param(cls_conn, param, buf, buflen);
2129 tcp_conn->hdr_size = sizeof(struct iscsi_hdr); 1991 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
2130 if (conn->hdrdgst_en) { 1992 if (conn->hdrdgst_en)
2131 tcp_conn->hdr_size += sizeof(__u32); 1993 tcp_conn->hdr_size += sizeof(__u32);
2132 if (!tcp_conn->tx_tfm)
2133 tcp_conn->tx_tfm = crypto_alloc_tfm("crc32c",
2134 0);
2135 if (!tcp_conn->tx_tfm)
2136 return -ENOMEM;
2137 if (!tcp_conn->rx_tfm)
2138 tcp_conn->rx_tfm = crypto_alloc_tfm("crc32c",
2139 0);
2140 if (!tcp_conn->rx_tfm) {
2141 crypto_free_tfm(tcp_conn->tx_tfm);
2142 return -ENOMEM;
2143 }
2144 } else {
2145 if (tcp_conn->tx_tfm)
2146 crypto_free_tfm(tcp_conn->tx_tfm);
2147 if (tcp_conn->rx_tfm)
2148 crypto_free_tfm(tcp_conn->rx_tfm);
2149 }
2150 break; 1994 break;
2151 case ISCSI_PARAM_DATADGST_EN: 1995 case ISCSI_PARAM_DATADGST_EN:
2152 iscsi_set_param(cls_conn, param, buf, buflen); 1996 iscsi_set_param(cls_conn, param, buf, buflen);
2153 if (conn->datadgst_en) {
2154 if (!tcp_conn->data_tx_tfm)
2155 tcp_conn->data_tx_tfm =
2156 crypto_alloc_tfm("crc32c", 0);
2157 if (!tcp_conn->data_tx_tfm)
2158 return -ENOMEM;
2159 if (!tcp_conn->data_rx_tfm)
2160 tcp_conn->data_rx_tfm =
2161 crypto_alloc_tfm("crc32c", 0);
2162 if (!tcp_conn->data_rx_tfm) {
2163 crypto_free_tfm(tcp_conn->data_tx_tfm);
2164 return -ENOMEM;
2165 }
2166 } else {
2167 if (tcp_conn->data_tx_tfm)
2168 crypto_free_tfm(tcp_conn->data_tx_tfm);
2169 if (tcp_conn->data_rx_tfm)
2170 crypto_free_tfm(tcp_conn->data_rx_tfm);
2171 }
2172 tcp_conn->sendpage = conn->datadgst_en ? 1997 tcp_conn->sendpage = conn->datadgst_en ?
2173 sock_no_sendpage : tcp_conn->sock->ops->sendpage; 1998 sock_no_sendpage : tcp_conn->sock->ops->sendpage;
2174 break; 1999 break;
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-15 01:58:53 -0500