aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi/iscsi_tcp.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/iscsi_tcp.c')
-rw-r--r--drivers/scsi/iscsi_tcp.c1621
1 files changed, 272 insertions, 1349 deletions
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index e11bce6ab63c..23808dfe22ba 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -27,7 +27,6 @@
27 */ 27 */
28 28
29#include <linux/types.h> 29#include <linux/types.h>
30#include <linux/list.h>
31#include <linux/inet.h> 30#include <linux/inet.h>
32#include <linux/file.h> 31#include <linux/file.h>
33#include <linux/blkdev.h> 32#include <linux/blkdev.h>
@@ -44,12 +43,12 @@
44 43
45#include "iscsi_tcp.h" 44#include "iscsi_tcp.h"
46 45
47MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, " 46MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, "
47 "Dmitry Yusupov <dmitry_yus@yahoo.com>, "
48 "Alex Aizman <itn780@yahoo.com>"); 48 "Alex Aizman <itn780@yahoo.com>");
49MODULE_DESCRIPTION("iSCSI/TCP data-path"); 49MODULE_DESCRIPTION("iSCSI/TCP data-path");
50MODULE_LICENSE("GPL"); 50MODULE_LICENSE("GPL");
51#undef DEBUG_TCP 51#undef DEBUG_TCP
52#define DEBUG_ASSERT
53 52
54#ifdef DEBUG_TCP 53#ifdef DEBUG_TCP
55#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt) 54#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
@@ -57,934 +56,41 @@ MODULE_LICENSE("GPL");
57#define debug_tcp(fmt...) 56#define debug_tcp(fmt...)
58#endif 57#endif
59 58
60#ifndef DEBUG_ASSERT 59static struct scsi_transport_template *iscsi_sw_tcp_scsi_transport;
61#ifdef BUG_ON 60static struct scsi_host_template iscsi_sw_tcp_sht;
62#undef BUG_ON 61static struct iscsi_transport iscsi_sw_tcp_transport;
63#endif
64#define BUG_ON(expr)
65#endif
66
67static struct scsi_transport_template *iscsi_tcp_scsi_transport;
68static struct scsi_host_template iscsi_sht;
69static struct iscsi_transport iscsi_tcp_transport;
70 62
71static unsigned int iscsi_max_lun = 512; 63static unsigned int iscsi_max_lun = 512;
72module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 64module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
73 65
74static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
75 struct iscsi_segment *segment);
76
77/*
78 * Scatterlist handling: inside the iscsi_segment, we
79 * remember an index into the scatterlist, and set data/size
80 * to the current scatterlist entry. For highmem pages, we
81 * kmap as needed.
82 *
83 * Note that the page is unmapped when we return from
84 * TCP's data_ready handler, so we may end up mapping and
85 * unmapping the same page repeatedly. The whole reason
86 * for this is that we shouldn't keep the page mapped
87 * outside the softirq.
88 */
89
90/**
91 * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
92 * @segment: the buffer object
93 * @sg: scatterlist
94 * @offset: byte offset into that sg entry
95 *
96 * This function sets up the segment so that subsequent
97 * data is copied to the indicated sg entry, at the given
98 * offset.
99 */
100static inline void
101iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,
102 struct scatterlist *sg, unsigned int offset)
103{
104 segment->sg = sg;
105 segment->sg_offset = offset;
106 segment->size = min(sg->length - offset,
107 segment->total_size - segment->total_copied);
108 segment->data = NULL;
109}
110
111/**
112 * iscsi_tcp_segment_map - map the current S/G page
113 * @segment: iscsi_segment
114 * @recv: 1 if called from recv path
115 *
116 * We only need to possibly kmap data if scatter lists are being used,
117 * because the iscsi passthrough and internal IO paths will never use high
118 * mem pages.
119 */
120static inline void
121iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)
122{
123 struct scatterlist *sg;
124
125 if (segment->data != NULL || !segment->sg)
126 return;
127
128 sg = segment->sg;
129 BUG_ON(segment->sg_mapped);
130 BUG_ON(sg->length == 0);
131
132 /*
133 * If the page count is greater than one it is ok to send
134 * to the network layer's zero copy send path. If not we
135 * have to go the slow sendmsg path. We always map for the
136 * recv path.
137 */
138 if (page_count(sg_page(sg)) >= 1 && !recv)
139 return;
140
141 debug_tcp("iscsi_tcp_segment_map %s %p\n", recv ? "recv" : "xmit",
142 segment);
143 segment->sg_mapped = kmap_atomic(sg_page(sg), KM_SOFTIRQ0);
144 segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;
145}
146
147static inline void
148iscsi_tcp_segment_unmap(struct iscsi_segment *segment)
149{
150 debug_tcp("iscsi_tcp_segment_unmap %p\n", segment);
151
152 if (segment->sg_mapped) {
153 debug_tcp("iscsi_tcp_segment_unmap valid\n");
154 kunmap_atomic(segment->sg_mapped, KM_SOFTIRQ0);
155 segment->sg_mapped = NULL;
156 segment->data = NULL;
157 }
158}
159
160/*
161 * Splice the digest buffer into the buffer
162 */
163static inline void
164iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)
165{
166 segment->data = digest;
167 segment->digest_len = ISCSI_DIGEST_SIZE;
168 segment->total_size += ISCSI_DIGEST_SIZE;
169 segment->size = ISCSI_DIGEST_SIZE;
170 segment->copied = 0;
171 segment->sg = NULL;
172 segment->hash = NULL;
173}
174
175/**
176 * iscsi_tcp_segment_done - check whether the segment is complete
177 * @segment: iscsi segment to check
178 * @recv: set to one of this is called from the recv path
179 * @copied: number of bytes copied
180 *
181 * Check if we're done receiving this segment. If the receive
182 * buffer is full but we expect more data, move on to the
183 * next entry in the scatterlist.
184 *
185 * If the amount of data we received isn't a multiple of 4,
186 * we will transparently receive the pad bytes, too.
187 *
188 * This function must be re-entrant.
189 */
190static inline int
191iscsi_tcp_segment_done(struct iscsi_segment *segment, int recv, unsigned copied)
192{
193 static unsigned char padbuf[ISCSI_PAD_LEN];
194 struct scatterlist sg;
195 unsigned int pad;
196
197 debug_tcp("copied %u %u size %u %s\n", segment->copied, copied,
198 segment->size, recv ? "recv" : "xmit");
199 if (segment->hash && copied) {
200 /*
201 * If a segment is kmapd we must unmap it before sending
202 * to the crypto layer since that will try to kmap it again.
203 */
204 iscsi_tcp_segment_unmap(segment);
205
206 if (!segment->data) {
207 sg_init_table(&sg, 1);
208 sg_set_page(&sg, sg_page(segment->sg), copied,
209 segment->copied + segment->sg_offset +
210 segment->sg->offset);
211 } else
212 sg_init_one(&sg, segment->data + segment->copied,
213 copied);
214 crypto_hash_update(segment->hash, &sg, copied);
215 }
216
217 segment->copied += copied;
218 if (segment->copied < segment->size) {
219 iscsi_tcp_segment_map(segment, recv);
220 return 0;
221 }
222
223 segment->total_copied += segment->copied;
224 segment->copied = 0;
225 segment->size = 0;
226
227 /* Unmap the current scatterlist page, if there is one. */
228 iscsi_tcp_segment_unmap(segment);
229
230 /* Do we have more scatterlist entries? */
231 debug_tcp("total copied %u total size %u\n", segment->total_copied,
232 segment->total_size);
233 if (segment->total_copied < segment->total_size) {
234 /* Proceed to the next entry in the scatterlist. */
235 iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),
236 0);
237 iscsi_tcp_segment_map(segment, recv);
238 BUG_ON(segment->size == 0);
239 return 0;
240 }
241
242 /* Do we need to handle padding? */
243 pad = iscsi_padding(segment->total_copied);
244 if (pad != 0) {
245 debug_tcp("consume %d pad bytes\n", pad);
246 segment->total_size += pad;
247 segment->size = pad;
248 segment->data = padbuf;
249 return 0;
250 }
251
252 /*
253 * Set us up for transferring the data digest. hdr digest
254 * is completely handled in hdr done function.
255 */
256 if (segment->hash) {
257 crypto_hash_final(segment->hash, segment->digest);
258 iscsi_tcp_segment_splice_digest(segment,
259 recv ? segment->recv_digest : segment->digest);
260 return 0;
261 }
262
263 return 1;
264}
265
266/**
267 * iscsi_tcp_xmit_segment - transmit segment
268 * @tcp_conn: the iSCSI TCP connection
269 * @segment: the buffer to transmnit
270 *
271 * This function transmits as much of the buffer as
272 * the network layer will accept, and returns the number of
273 * bytes transmitted.
274 *
275 * If CRC hashing is enabled, the function will compute the
276 * hash as it goes. When the entire segment has been transmitted,
277 * it will retrieve the hash value and send it as well.
278 */
279static int
280iscsi_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
281 struct iscsi_segment *segment)
282{
283 struct socket *sk = tcp_conn->sock;
284 unsigned int copied = 0;
285 int r = 0;
286
287 while (!iscsi_tcp_segment_done(segment, 0, r)) {
288 struct scatterlist *sg;
289 unsigned int offset, copy;
290 int flags = 0;
291
292 r = 0;
293 offset = segment->copied;
294 copy = segment->size - offset;
295
296 if (segment->total_copied + segment->size < segment->total_size)
297 flags |= MSG_MORE;
298
299 /* Use sendpage if we can; else fall back to sendmsg */
300 if (!segment->data) {
301 sg = segment->sg;
302 offset += segment->sg_offset + sg->offset;
303 r = tcp_conn->sendpage(sk, sg_page(sg), offset, copy,
304 flags);
305 } else {
306 struct msghdr msg = { .msg_flags = flags };
307 struct kvec iov = {
308 .iov_base = segment->data + offset,
309 .iov_len = copy
310 };
311
312 r = kernel_sendmsg(sk, &msg, &iov, 1, copy);
313 }
314
315 if (r < 0) {
316 iscsi_tcp_segment_unmap(segment);
317 if (copied || r == -EAGAIN)
318 break;
319 return r;
320 }
321 copied += r;
322 }
323 return copied;
324}
325
326/**
327 * iscsi_tcp_segment_recv - copy data to segment
328 * @tcp_conn: the iSCSI TCP connection
329 * @segment: the buffer to copy to
330 * @ptr: data pointer
331 * @len: amount of data available
332 *
333 * This function copies up to @len bytes to the
334 * given buffer, and returns the number of bytes
335 * consumed, which can actually be less than @len.
336 *
337 * If hash digest is enabled, the function will update the
338 * hash while copying.
339 * Combining these two operations doesn't buy us a lot (yet),
340 * but in the future we could implement combined copy+crc,
341 * just way we do for network layer checksums.
342 */
343static int
344iscsi_tcp_segment_recv(struct iscsi_tcp_conn *tcp_conn,
345 struct iscsi_segment *segment, const void *ptr,
346 unsigned int len)
347{
348 unsigned int copy = 0, copied = 0;
349
350 while (!iscsi_tcp_segment_done(segment, 1, copy)) {
351 if (copied == len) {
352 debug_tcp("iscsi_tcp_segment_recv copied %d bytes\n",
353 len);
354 break;
355 }
356
357 copy = min(len - copied, segment->size - segment->copied);
358 debug_tcp("iscsi_tcp_segment_recv copying %d\n", copy);
359 memcpy(segment->data + segment->copied, ptr + copied, copy);
360 copied += copy;
361 }
362 return copied;
363}
364
365static inline void
366iscsi_tcp_dgst_header(struct hash_desc *hash, const void *hdr, size_t hdrlen,
367 unsigned char digest[ISCSI_DIGEST_SIZE])
368{
369 struct scatterlist sg;
370
371 sg_init_one(&sg, hdr, hdrlen);
372 crypto_hash_digest(hash, &sg, hdrlen, digest);
373}
374
375static inline int
376iscsi_tcp_dgst_verify(struct iscsi_tcp_conn *tcp_conn,
377 struct iscsi_segment *segment)
378{
379 if (!segment->digest_len)
380 return 1;
381
382 if (memcmp(segment->recv_digest, segment->digest,
383 segment->digest_len)) {
384 debug_scsi("digest mismatch\n");
385 return 0;
386 }
387
388 return 1;
389}
390
391/*
392 * Helper function to set up segment buffer
393 */
394static inline void
395__iscsi_segment_init(struct iscsi_segment *segment, size_t size,
396 iscsi_segment_done_fn_t *done, struct hash_desc *hash)
397{
398 memset(segment, 0, sizeof(*segment));
399 segment->total_size = size;
400 segment->done = done;
401
402 if (hash) {
403 segment->hash = hash;
404 crypto_hash_init(hash);
405 }
406}
407
408static inline void
409iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,
410 size_t size, iscsi_segment_done_fn_t *done,
411 struct hash_desc *hash)
412{
413 __iscsi_segment_init(segment, size, done, hash);
414 segment->data = data;
415 segment->size = size;
416}
417
418static inline int
419iscsi_segment_seek_sg(struct iscsi_segment *segment,
420 struct scatterlist *sg_list, unsigned int sg_count,
421 unsigned int offset, size_t size,
422 iscsi_segment_done_fn_t *done, struct hash_desc *hash)
423{
424 struct scatterlist *sg;
425 unsigned int i;
426
427 debug_scsi("iscsi_segment_seek_sg offset %u size %llu\n",
428 offset, size);
429 __iscsi_segment_init(segment, size, done, hash);
430 for_each_sg(sg_list, sg, sg_count, i) {
431 debug_scsi("sg %d, len %u offset %u\n", i, sg->length,
432 sg->offset);
433 if (offset < sg->length) {
434 iscsi_tcp_segment_init_sg(segment, sg, offset);
435 return 0;
436 }
437 offset -= sg->length;
438 }
439
440 return ISCSI_ERR_DATA_OFFSET;
441}
442
443/**
444 * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
445 * @tcp_conn: iscsi connection to prep for
446 *
447 * This function always passes NULL for the hash argument, because when this
448 * function is called we do not yet know the final size of the header and want
449 * to delay the digest processing until we know that.
450 */
451static void
452iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn *tcp_conn)
453{
454 debug_tcp("iscsi_tcp_hdr_recv_prep(%p%s)\n", tcp_conn,
455 tcp_conn->iscsi_conn->hdrdgst_en ? ", digest enabled" : "");
456 iscsi_segment_init_linear(&tcp_conn->in.segment,
457 tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr),
458 iscsi_tcp_hdr_recv_done, NULL);
459}
460
461/*
462 * Handle incoming reply to any other type of command
463 */
464static int
465iscsi_tcp_data_recv_done(struct iscsi_tcp_conn *tcp_conn,
466 struct iscsi_segment *segment)
467{
468 struct iscsi_conn *conn = tcp_conn->iscsi_conn;
469 int rc = 0;
470
471 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
472 return ISCSI_ERR_DATA_DGST;
473
474 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr,
475 conn->data, tcp_conn->in.datalen);
476 if (rc)
477 return rc;
478
479 iscsi_tcp_hdr_recv_prep(tcp_conn);
480 return 0;
481}
482
483static void
484iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn)
485{
486 struct iscsi_conn *conn = tcp_conn->iscsi_conn;
487 struct hash_desc *rx_hash = NULL;
488
489 if (conn->datadgst_en)
490 rx_hash = &tcp_conn->rx_hash;
491
492 iscsi_segment_init_linear(&tcp_conn->in.segment,
493 conn->data, tcp_conn->in.datalen,
494 iscsi_tcp_data_recv_done, rx_hash);
495}
496
497/*
498 * must be called with session lock
499 */
500static void
501iscsi_tcp_cleanup_task(struct iscsi_conn *conn, struct iscsi_task *task)
502{
503 struct iscsi_tcp_task *tcp_task = task->dd_data;
504 struct iscsi_r2t_info *r2t;
505
506 /* nothing to do for mgmt tasks */
507 if (!task->sc)
508 return;
509
510 /* flush task's r2t queues */
511 while (__kfifo_get(tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) {
512 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
513 sizeof(void*));
514 debug_scsi("iscsi_tcp_cleanup_task pending r2t dropped\n");
515 }
516
517 r2t = tcp_task->r2t;
518 if (r2t != NULL) {
519 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
520 sizeof(void*));
521 tcp_task->r2t = NULL;
522 }
523}
524
525/**
526 * iscsi_data_in - SCSI Data-In Response processing
527 * @conn: iscsi connection
528 * @task: scsi command task
529 **/
530static int
531iscsi_data_in(struct iscsi_conn *conn, struct iscsi_task *task)
532{
533 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
534 struct iscsi_tcp_task *tcp_task = task->dd_data;
535 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
536 int datasn = be32_to_cpu(rhdr->datasn);
537 unsigned total_in_length = scsi_in(task->sc)->length;
538
539 iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr);
540 if (tcp_conn->in.datalen == 0)
541 return 0;
542
543 if (tcp_task->exp_datasn != datasn) {
544 debug_tcp("%s: task->exp_datasn(%d) != rhdr->datasn(%d)\n",
545 __func__, tcp_task->exp_datasn, datasn);
546 return ISCSI_ERR_DATASN;
547 }
548
549 tcp_task->exp_datasn++;
550
551 tcp_task->data_offset = be32_to_cpu(rhdr->offset);
552 if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) {
553 debug_tcp("%s: data_offset(%d) + data_len(%d) > total_length_in(%d)\n",
554 __func__, tcp_task->data_offset,
555 tcp_conn->in.datalen, total_in_length);
556 return ISCSI_ERR_DATA_OFFSET;
557 }
558
559 conn->datain_pdus_cnt++;
560 return 0;
561}
562
563/**
564 * iscsi_solicit_data_init - initialize first Data-Out
565 * @conn: iscsi connection
566 * @task: scsi command task
567 * @r2t: R2T info
568 *
569 * Notes:
570 * Initialize first Data-Out within this R2T sequence and finds
571 * proper data_offset within this SCSI command.
572 *
573 * This function is called with connection lock taken.
574 **/
575static void
576iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_task *task,
577 struct iscsi_r2t_info *r2t)
578{
579 struct iscsi_data *hdr;
580
581 hdr = &r2t->dtask.hdr;
582 memset(hdr, 0, sizeof(struct iscsi_data));
583 hdr->ttt = r2t->ttt;
584 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
585 r2t->solicit_datasn++;
586 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
587 memcpy(hdr->lun, task->hdr->lun, sizeof(hdr->lun));
588 hdr->itt = task->hdr->itt;
589 hdr->exp_statsn = r2t->exp_statsn;
590 hdr->offset = cpu_to_be32(r2t->data_offset);
591 if (r2t->data_length > conn->max_xmit_dlength) {
592 hton24(hdr->dlength, conn->max_xmit_dlength);
593 r2t->data_count = conn->max_xmit_dlength;
594 hdr->flags = 0;
595 } else {
596 hton24(hdr->dlength, r2t->data_length);
597 r2t->data_count = r2t->data_length;
598 hdr->flags = ISCSI_FLAG_CMD_FINAL;
599 }
600 conn->dataout_pdus_cnt++;
601
602 r2t->sent = 0;
603}
604
605/**
606 * iscsi_r2t_rsp - iSCSI R2T Response processing
607 * @conn: iscsi connection
608 * @task: scsi command task
609 **/
610static int
611iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task)
612{
613 struct iscsi_r2t_info *r2t;
614 struct iscsi_session *session = conn->session;
615 struct iscsi_tcp_task *tcp_task = task->dd_data;
616 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
617 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
618 int r2tsn = be32_to_cpu(rhdr->r2tsn);
619 int rc;
620
621 if (tcp_conn->in.datalen) {
622 iscsi_conn_printk(KERN_ERR, conn,
623 "invalid R2t with datalen %d\n",
624 tcp_conn->in.datalen);
625 return ISCSI_ERR_DATALEN;
626 }
627
628 if (tcp_task->exp_datasn != r2tsn){
629 debug_tcp("%s: task->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
630 __func__, tcp_task->exp_datasn, r2tsn);
631 return ISCSI_ERR_R2TSN;
632 }
633
634 /* fill-in new R2T associated with the task */
635 iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
636
637 if (!task->sc || session->state != ISCSI_STATE_LOGGED_IN) {
638 iscsi_conn_printk(KERN_INFO, conn,
639 "dropping R2T itt %d in recovery.\n",
640 task->itt);
641 return 0;
642 }
643
644 rc = __kfifo_get(tcp_task->r2tpool.queue, (void*)&r2t, sizeof(void*));
645 BUG_ON(!rc);
646
647 r2t->exp_statsn = rhdr->statsn;
648 r2t->data_length = be32_to_cpu(rhdr->data_length);
649 if (r2t->data_length == 0) {
650 iscsi_conn_printk(KERN_ERR, conn,
651 "invalid R2T with zero data len\n");
652 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
653 sizeof(void*));
654 return ISCSI_ERR_DATALEN;
655 }
656
657 if (r2t->data_length > session->max_burst)
658 debug_scsi("invalid R2T with data len %u and max burst %u."
659 "Attempting to execute request.\n",
660 r2t->data_length, session->max_burst);
661
662 r2t->data_offset = be32_to_cpu(rhdr->data_offset);
663 if (r2t->data_offset + r2t->data_length > scsi_out(task->sc)->length) {
664 iscsi_conn_printk(KERN_ERR, conn,
665 "invalid R2T with data len %u at offset %u "
666 "and total length %d\n", r2t->data_length,
667 r2t->data_offset, scsi_out(task->sc)->length);
668 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
669 sizeof(void*));
670 return ISCSI_ERR_DATALEN;
671 }
672
673 r2t->ttt = rhdr->ttt; /* no flip */
674 r2t->solicit_datasn = 0;
675
676 iscsi_solicit_data_init(conn, task, r2t);
677
678 tcp_task->exp_datasn = r2tsn + 1;
679 __kfifo_put(tcp_task->r2tqueue, (void*)&r2t, sizeof(void*));
680 conn->r2t_pdus_cnt++;
681
682 iscsi_requeue_task(task);
683 return 0;
684}
685
686/*
687 * Handle incoming reply to DataIn command
688 */
689static int
690iscsi_tcp_process_data_in(struct iscsi_tcp_conn *tcp_conn,
691 struct iscsi_segment *segment)
692{
693 struct iscsi_conn *conn = tcp_conn->iscsi_conn;
694 struct iscsi_hdr *hdr = tcp_conn->in.hdr;
695 int rc;
696
697 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
698 return ISCSI_ERR_DATA_DGST;
699
700 /* check for non-exceptional status */
701 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
702 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
703 if (rc)
704 return rc;
705 }
706
707 iscsi_tcp_hdr_recv_prep(tcp_conn);
708 return 0;
709}
710
711/**
712 * iscsi_tcp_hdr_dissect - process PDU header
713 * @conn: iSCSI connection
714 * @hdr: PDU header
715 *
716 * This function analyzes the header of the PDU received,
717 * and performs several sanity checks. If the PDU is accompanied
718 * by data, the receive buffer is set up to copy the incoming data
719 * to the correct location.
720 */
721static int
722iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
723{
724 int rc = 0, opcode, ahslen;
725 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
726 struct iscsi_task *task;
727
728 /* verify PDU length */
729 tcp_conn->in.datalen = ntoh24(hdr->dlength);
730 if (tcp_conn->in.datalen > conn->max_recv_dlength) {
731 iscsi_conn_printk(KERN_ERR, conn,
732 "iscsi_tcp: datalen %d > %d\n",
733 tcp_conn->in.datalen, conn->max_recv_dlength);
734 return ISCSI_ERR_DATALEN;
735 }
736
737 /* Additional header segments. So far, we don't
738 * process additional headers.
739 */
740 ahslen = hdr->hlength << 2;
741
742 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
743 /* verify itt (itt encoding: age+cid+itt) */
744 rc = iscsi_verify_itt(conn, hdr->itt);
745 if (rc)
746 return rc;
747
748 debug_tcp("opcode 0x%x ahslen %d datalen %d\n",
749 opcode, ahslen, tcp_conn->in.datalen);
750
751 switch(opcode) {
752 case ISCSI_OP_SCSI_DATA_IN:
753 spin_lock(&conn->session->lock);
754 task = iscsi_itt_to_ctask(conn, hdr->itt);
755 if (!task)
756 rc = ISCSI_ERR_BAD_ITT;
757 else
758 rc = iscsi_data_in(conn, task);
759 if (rc) {
760 spin_unlock(&conn->session->lock);
761 break;
762 }
763
764 if (tcp_conn->in.datalen) {
765 struct iscsi_tcp_task *tcp_task = task->dd_data;
766 struct hash_desc *rx_hash = NULL;
767 struct scsi_data_buffer *sdb = scsi_in(task->sc);
768
769 /*
770 * Setup copy of Data-In into the Scsi_Cmnd
771 * Scatterlist case:
772 * We set up the iscsi_segment to point to the next
773 * scatterlist entry to copy to. As we go along,
774 * we move on to the next scatterlist entry and
775 * update the digest per-entry.
776 */
777 if (conn->datadgst_en)
778 rx_hash = &tcp_conn->rx_hash;
779
780 debug_tcp("iscsi_tcp_begin_data_in(%p, offset=%d, "
781 "datalen=%d)\n", tcp_conn,
782 tcp_task->data_offset,
783 tcp_conn->in.datalen);
784 rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,
785 sdb->table.sgl,
786 sdb->table.nents,
787 tcp_task->data_offset,
788 tcp_conn->in.datalen,
789 iscsi_tcp_process_data_in,
790 rx_hash);
791 spin_unlock(&conn->session->lock);
792 return rc;
793 }
794 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
795 spin_unlock(&conn->session->lock);
796 break;
797 case ISCSI_OP_SCSI_CMD_RSP:
798 if (tcp_conn->in.datalen) {
799 iscsi_tcp_data_recv_prep(tcp_conn);
800 return 0;
801 }
802 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
803 break;
804 case ISCSI_OP_R2T:
805 spin_lock(&conn->session->lock);
806 task = iscsi_itt_to_ctask(conn, hdr->itt);
807 if (!task)
808 rc = ISCSI_ERR_BAD_ITT;
809 else if (ahslen)
810 rc = ISCSI_ERR_AHSLEN;
811 else if (task->sc->sc_data_direction == DMA_TO_DEVICE)
812 rc = iscsi_r2t_rsp(conn, task);
813 else
814 rc = ISCSI_ERR_PROTO;
815 spin_unlock(&conn->session->lock);
816 break;
817 case ISCSI_OP_LOGIN_RSP:
818 case ISCSI_OP_TEXT_RSP:
819 case ISCSI_OP_REJECT:
820 case ISCSI_OP_ASYNC_EVENT:
821 /*
822 * It is possible that we could get a PDU with a buffer larger
823 * than 8K, but there are no targets that currently do this.
824 * For now we fail until we find a vendor that needs it
825 */
826 if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) {
827 iscsi_conn_printk(KERN_ERR, conn,
828 "iscsi_tcp: received buffer of "
829 "len %u but conn buffer is only %u "
830 "(opcode %0x)\n",
831 tcp_conn->in.datalen,
832 ISCSI_DEF_MAX_RECV_SEG_LEN, opcode);
833 rc = ISCSI_ERR_PROTO;
834 break;
835 }
836
837 /* If there's data coming in with the response,
838 * receive it to the connection's buffer.
839 */
840 if (tcp_conn->in.datalen) {
841 iscsi_tcp_data_recv_prep(tcp_conn);
842 return 0;
843 }
844 /* fall through */
845 case ISCSI_OP_LOGOUT_RSP:
846 case ISCSI_OP_NOOP_IN:
847 case ISCSI_OP_SCSI_TMFUNC_RSP:
848 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
849 break;
850 default:
851 rc = ISCSI_ERR_BAD_OPCODE;
852 break;
853 }
854
855 if (rc == 0) {
856 /* Anything that comes with data should have
857 * been handled above. */
858 if (tcp_conn->in.datalen)
859 return ISCSI_ERR_PROTO;
860 iscsi_tcp_hdr_recv_prep(tcp_conn);
861 }
862
863 return rc;
864}
865
866/** 66/**
867 * iscsi_tcp_hdr_recv_done - process PDU header 67 * iscsi_sw_tcp_recv - TCP receive in sendfile fashion
868 *
869 * This is the callback invoked when the PDU header has
870 * been received. If the header is followed by additional
871 * header segments, we go back for more data.
872 */
873static int
874iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
875 struct iscsi_segment *segment)
876{
877 struct iscsi_conn *conn = tcp_conn->iscsi_conn;
878 struct iscsi_hdr *hdr;
879
880 /* Check if there are additional header segments
881 * *prior* to computing the digest, because we
882 * may need to go back to the caller for more.
883 */
884 hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf;
885 if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {
886 /* Bump the header length - the caller will
887 * just loop around and get the AHS for us, and
888 * call again. */
889 unsigned int ahslen = hdr->hlength << 2;
890
891 /* Make sure we don't overflow */
892 if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))
893 return ISCSI_ERR_AHSLEN;
894
895 segment->total_size += ahslen;
896 segment->size += ahslen;
897 return 0;
898 }
899
900 /* We're done processing the header. See if we're doing
901 * header digests; if so, set up the recv_digest buffer
902 * and go back for more. */
903 if (conn->hdrdgst_en) {
904 if (segment->digest_len == 0) {
905 iscsi_tcp_segment_splice_digest(segment,
906 segment->recv_digest);
907 return 0;
908 }
909 iscsi_tcp_dgst_header(&tcp_conn->rx_hash, hdr,
910 segment->total_copied - ISCSI_DIGEST_SIZE,
911 segment->digest);
912
913 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
914 return ISCSI_ERR_HDR_DGST;
915 }
916
917 tcp_conn->in.hdr = hdr;
918 return iscsi_tcp_hdr_dissect(conn, hdr);
919}
920
921/**
922 * iscsi_tcp_recv - TCP receive in sendfile fashion
923 * @rd_desc: read descriptor 68 * @rd_desc: read descriptor
924 * @skb: socket buffer 69 * @skb: socket buffer
925 * @offset: offset in skb 70 * @offset: offset in skb
926 * @len: skb->len - offset 71 * @len: skb->len - offset
927 **/ 72 */
928static int 73static int iscsi_sw_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
929iscsi_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 74 unsigned int offset, size_t len)
930 unsigned int offset, size_t len)
931{ 75{
932 struct iscsi_conn *conn = rd_desc->arg.data; 76 struct iscsi_conn *conn = rd_desc->arg.data;
933 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 77 unsigned int consumed, total_consumed = 0;
934 struct iscsi_segment *segment = &tcp_conn->in.segment; 78 int status;
935 struct skb_seq_state seq;
936 unsigned int consumed = 0;
937 int rc = 0;
938 79
939 debug_tcp("in %d bytes\n", skb->len - offset); 80 debug_tcp("in %d bytes\n", skb->len - offset);
940 81
941 if (unlikely(conn->suspend_rx)) { 82 do {
942 debug_tcp("conn %d Rx suspended!\n", conn->id); 83 status = 0;
943 return 0; 84 consumed = iscsi_tcp_recv_skb(conn, skb, offset, 0, &status);
944 } 85 offset += consumed;
86 total_consumed += consumed;
87 } while (consumed != 0 && status != ISCSI_TCP_SKB_DONE);
945 88
946 skb_prepare_seq_read(skb, offset, skb->len, &seq); 89 debug_tcp("read %d bytes status %d\n", skb->len - offset, status);
947 while (1) { 90 return total_consumed;
948 unsigned int avail;
949 const u8 *ptr;
950
951 avail = skb_seq_read(consumed, &ptr, &seq);
952 if (avail == 0) {
953 debug_tcp("no more data avail. Consumed %d\n",
954 consumed);
955 break;
956 }
957 BUG_ON(segment->copied >= segment->size);
958
959 debug_tcp("skb %p ptr=%p avail=%u\n", skb, ptr, avail);
960 rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);
961 BUG_ON(rc == 0);
962 consumed += rc;
963
964 if (segment->total_copied >= segment->total_size) {
965 debug_tcp("segment done\n");
966 rc = segment->done(tcp_conn, segment);
967 if (rc != 0) {
968 skb_abort_seq_read(&seq);
969 goto error;
970 }
971
972 /* The done() functions sets up the
973 * next segment. */
974 }
975 }
976 skb_abort_seq_read(&seq);
977 conn->rxdata_octets += consumed;
978 return consumed;
979
980error:
981 debug_tcp("Error receiving PDU, errno=%d\n", rc);
982 iscsi_conn_failure(conn, rc);
983 return 0;
984} 91}
985 92
986static void 93static void iscsi_sw_tcp_data_ready(struct sock *sk, int flag)
987iscsi_tcp_data_ready(struct sock *sk, int flag)
988{ 94{
989 struct iscsi_conn *conn = sk->sk_user_data; 95 struct iscsi_conn *conn = sk->sk_user_data;
990 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 96 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
@@ -1000,7 +106,7 @@ iscsi_tcp_data_ready(struct sock *sk, int flag)
1000 */ 106 */
1001 rd_desc.arg.data = conn; 107 rd_desc.arg.data = conn;
1002 rd_desc.count = 1; 108 rd_desc.count = 1;
1003 tcp_read_sock(sk, &rd_desc, iscsi_tcp_recv); 109 tcp_read_sock(sk, &rd_desc, iscsi_sw_tcp_recv);
1004 110
1005 read_unlock(&sk->sk_callback_lock); 111 read_unlock(&sk->sk_callback_lock);
1006 112
@@ -1009,10 +115,10 @@ iscsi_tcp_data_ready(struct sock *sk, int flag)
1009 iscsi_tcp_segment_unmap(&tcp_conn->in.segment); 115 iscsi_tcp_segment_unmap(&tcp_conn->in.segment);
1010} 116}
1011 117
1012static void 118static void iscsi_sw_tcp_state_change(struct sock *sk)
1013iscsi_tcp_state_change(struct sock *sk)
1014{ 119{
1015 struct iscsi_tcp_conn *tcp_conn; 120 struct iscsi_tcp_conn *tcp_conn;
121 struct iscsi_sw_tcp_conn *tcp_sw_conn;
1016 struct iscsi_conn *conn; 122 struct iscsi_conn *conn;
1017 struct iscsi_session *session; 123 struct iscsi_session *session;
1018 void (*old_state_change)(struct sock *); 124 void (*old_state_change)(struct sock *);
@@ -1030,7 +136,8 @@ iscsi_tcp_state_change(struct sock *sk)
1030 } 136 }
1031 137
1032 tcp_conn = conn->dd_data; 138 tcp_conn = conn->dd_data;
1033 old_state_change = tcp_conn->old_state_change; 139 tcp_sw_conn = tcp_conn->dd_data;
140 old_state_change = tcp_sw_conn->old_state_change;
1034 141
1035 read_unlock(&sk->sk_callback_lock); 142 read_unlock(&sk->sk_callback_lock);
1036 143
@@ -1041,63 +148,123 @@ iscsi_tcp_state_change(struct sock *sk)
1041 * iscsi_write_space - Called when more output buffer space is available 148 * iscsi_write_space - Called when more output buffer space is available
1042 * @sk: socket space is available for 149 * @sk: socket space is available for
1043 **/ 150 **/
1044static void 151static void iscsi_sw_tcp_write_space(struct sock *sk)
1045iscsi_write_space(struct sock *sk)
1046{ 152{
1047 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data; 153 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1048 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 154 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
155 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1049 156
1050 tcp_conn->old_write_space(sk); 157 tcp_sw_conn->old_write_space(sk);
1051 debug_tcp("iscsi_write_space: cid %d\n", conn->id); 158 debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1052 scsi_queue_work(conn->session->host, &conn->xmitwork); 159 scsi_queue_work(conn->session->host, &conn->xmitwork);
1053} 160}
1054 161
1055static void 162static void iscsi_sw_tcp_conn_set_callbacks(struct iscsi_conn *conn)
1056iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1057{ 163{
1058 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 164 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1059 struct sock *sk = tcp_conn->sock->sk; 165 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
166 struct sock *sk = tcp_sw_conn->sock->sk;
1060 167
1061 /* assign new callbacks */ 168 /* assign new callbacks */
1062 write_lock_bh(&sk->sk_callback_lock); 169 write_lock_bh(&sk->sk_callback_lock);
1063 sk->sk_user_data = conn; 170 sk->sk_user_data = conn;
1064 tcp_conn->old_data_ready = sk->sk_data_ready; 171 tcp_sw_conn->old_data_ready = sk->sk_data_ready;
1065 tcp_conn->old_state_change = sk->sk_state_change; 172 tcp_sw_conn->old_state_change = sk->sk_state_change;
1066 tcp_conn->old_write_space = sk->sk_write_space; 173 tcp_sw_conn->old_write_space = sk->sk_write_space;
1067 sk->sk_data_ready = iscsi_tcp_data_ready; 174 sk->sk_data_ready = iscsi_sw_tcp_data_ready;
1068 sk->sk_state_change = iscsi_tcp_state_change; 175 sk->sk_state_change = iscsi_sw_tcp_state_change;
1069 sk->sk_write_space = iscsi_write_space; 176 sk->sk_write_space = iscsi_sw_tcp_write_space;
1070 write_unlock_bh(&sk->sk_callback_lock); 177 write_unlock_bh(&sk->sk_callback_lock);
1071} 178}
1072 179
1073static void 180static void
1074iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn) 181iscsi_sw_tcp_conn_restore_callbacks(struct iscsi_sw_tcp_conn *tcp_sw_conn)
1075{ 182{
1076 struct sock *sk = tcp_conn->sock->sk; 183 struct sock *sk = tcp_sw_conn->sock->sk;
1077 184
1078 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 185 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1079 write_lock_bh(&sk->sk_callback_lock); 186 write_lock_bh(&sk->sk_callback_lock);
1080 sk->sk_user_data = NULL; 187 sk->sk_user_data = NULL;
1081 sk->sk_data_ready = tcp_conn->old_data_ready; 188 sk->sk_data_ready = tcp_sw_conn->old_data_ready;
1082 sk->sk_state_change = tcp_conn->old_state_change; 189 sk->sk_state_change = tcp_sw_conn->old_state_change;
1083 sk->sk_write_space = tcp_conn->old_write_space; 190 sk->sk_write_space = tcp_sw_conn->old_write_space;
1084 sk->sk_no_check = 0; 191 sk->sk_no_check = 0;
1085 write_unlock_bh(&sk->sk_callback_lock); 192 write_unlock_bh(&sk->sk_callback_lock);
1086} 193}
1087 194
1088/** 195/**
1089 * iscsi_xmit - TCP transmit 196 * iscsi_sw_tcp_xmit_segment - transmit segment
197 * @tcp_conn: the iSCSI TCP connection
198 * @segment: the buffer to transmnit
199 *
200 * This function transmits as much of the buffer as
201 * the network layer will accept, and returns the number of
202 * bytes transmitted.
203 *
204 * If CRC hashing is enabled, the function will compute the
205 * hash as it goes. When the entire segment has been transmitted,
206 * it will retrieve the hash value and send it as well.
207 */
208static int iscsi_sw_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
209 struct iscsi_segment *segment)
210{
211 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
212 struct socket *sk = tcp_sw_conn->sock;
213 unsigned int copied = 0;
214 int r = 0;
215
216 while (!iscsi_tcp_segment_done(tcp_conn, segment, 0, r)) {
217 struct scatterlist *sg;
218 unsigned int offset, copy;
219 int flags = 0;
220
221 r = 0;
222 offset = segment->copied;
223 copy = segment->size - offset;
224
225 if (segment->total_copied + segment->size < segment->total_size)
226 flags |= MSG_MORE;
227
228 /* Use sendpage if we can; else fall back to sendmsg */
229 if (!segment->data) {
230 sg = segment->sg;
231 offset += segment->sg_offset + sg->offset;
232 r = tcp_sw_conn->sendpage(sk, sg_page(sg), offset,
233 copy, flags);
234 } else {
235 struct msghdr msg = { .msg_flags = flags };
236 struct kvec iov = {
237 .iov_base = segment->data + offset,
238 .iov_len = copy
239 };
240
241 r = kernel_sendmsg(sk, &msg, &iov, 1, copy);
242 }
243
244 if (r < 0) {
245 iscsi_tcp_segment_unmap(segment);
246 if (copied || r == -EAGAIN)
247 break;
248 return r;
249 }
250 copied += r;
251 }
252 return copied;
253}
254
255/**
256 * iscsi_sw_tcp_xmit - TCP transmit
1090 **/ 257 **/
1091static int 258static int iscsi_sw_tcp_xmit(struct iscsi_conn *conn)
1092iscsi_xmit(struct iscsi_conn *conn)
1093{ 259{
1094 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 260 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1095 struct iscsi_segment *segment = &tcp_conn->out.segment; 261 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
262 struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
1096 unsigned int consumed = 0; 263 unsigned int consumed = 0;
1097 int rc = 0; 264 int rc = 0;
1098 265
1099 while (1) { 266 while (1) {
1100 rc = iscsi_tcp_xmit_segment(tcp_conn, segment); 267 rc = iscsi_sw_tcp_xmit_segment(tcp_conn, segment);
1101 if (rc < 0) { 268 if (rc < 0) {
1102 rc = ISCSI_ERR_XMIT_FAILED; 269 rc = ISCSI_ERR_XMIT_FAILED;
1103 goto error; 270 goto error;
@@ -1132,22 +299,22 @@ error:
1132/** 299/**
1133 * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit 300 * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit
1134 */ 301 */
1135static inline int 302static inline int iscsi_sw_tcp_xmit_qlen(struct iscsi_conn *conn)
1136iscsi_tcp_xmit_qlen(struct iscsi_conn *conn)
1137{ 303{
1138 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 304 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1139 struct iscsi_segment *segment = &tcp_conn->out.segment; 305 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
306 struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
1140 307
1141 return segment->total_copied - segment->total_size; 308 return segment->total_copied - segment->total_size;
1142} 309}
1143 310
1144static inline int 311static int iscsi_sw_tcp_pdu_xmit(struct iscsi_task *task)
1145iscsi_tcp_flush(struct iscsi_conn *conn)
1146{ 312{
313 struct iscsi_conn *conn = task->conn;
1147 int rc; 314 int rc;
1148 315
1149 while (iscsi_tcp_xmit_qlen(conn)) { 316 while (iscsi_sw_tcp_xmit_qlen(conn)) {
1150 rc = iscsi_xmit(conn); 317 rc = iscsi_sw_tcp_xmit(conn);
1151 if (rc == 0) 318 if (rc == 0)
1152 return -EAGAIN; 319 return -EAGAIN;
1153 if (rc < 0) 320 if (rc < 0)
@@ -1161,27 +328,31 @@ iscsi_tcp_flush(struct iscsi_conn *conn)
1161 * This is called when we're done sending the header. 328 * This is called when we're done sending the header.
1162 * Simply copy the data_segment to the send segment, and return. 329 * Simply copy the data_segment to the send segment, and return.
1163 */ 330 */
1164static int 331static int iscsi_sw_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn,
1165iscsi_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn, 332 struct iscsi_segment *segment)
1166 struct iscsi_segment *segment)
1167{ 333{
1168 tcp_conn->out.segment = tcp_conn->out.data_segment; 334 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
335
336 tcp_sw_conn->out.segment = tcp_sw_conn->out.data_segment;
1169 debug_tcp("Header done. Next segment size %u total_size %u\n", 337 debug_tcp("Header done. Next segment size %u total_size %u\n",
1170 tcp_conn->out.segment.size, tcp_conn->out.segment.total_size); 338 tcp_sw_conn->out.segment.size,
339 tcp_sw_conn->out.segment.total_size);
1171 return 0; 340 return 0;
1172} 341}
1173 342
1174static void 343static void iscsi_sw_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr,
1175iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen) 344 size_t hdrlen)
1176{ 345{
1177 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 346 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
347 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1178 348
1179 debug_tcp("%s(%p%s)\n", __func__, tcp_conn, 349 debug_tcp("%s(%p%s)\n", __func__, tcp_conn,
1180 conn->hdrdgst_en? ", digest enabled" : ""); 350 conn->hdrdgst_en? ", digest enabled" : "");
1181 351
1182 /* Clear the data segment - needs to be filled in by the 352 /* Clear the data segment - needs to be filled in by the
1183 * caller using iscsi_tcp_send_data_prep() */ 353 * caller using iscsi_tcp_send_data_prep() */
1184 memset(&tcp_conn->out.data_segment, 0, sizeof(struct iscsi_segment)); 354 memset(&tcp_sw_conn->out.data_segment, 0,
355 sizeof(struct iscsi_segment));
1185 356
1186 /* If header digest is enabled, compute the CRC and 357 /* If header digest is enabled, compute the CRC and
1187 * place the digest into the same buffer. We make 358 * place the digest into the same buffer. We make
@@ -1189,7 +360,7 @@ iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen)
1189 * sufficient room. 360 * sufficient room.
1190 */ 361 */
1191 if (conn->hdrdgst_en) { 362 if (conn->hdrdgst_en) {
1192 iscsi_tcp_dgst_header(&tcp_conn->tx_hash, hdr, hdrlen, 363 iscsi_tcp_dgst_header(&tcp_sw_conn->tx_hash, hdr, hdrlen,
1193 hdr + hdrlen); 364 hdr + hdrlen);
1194 hdrlen += ISCSI_DIGEST_SIZE; 365 hdrlen += ISCSI_DIGEST_SIZE;
1195 } 366 }
@@ -1197,10 +368,10 @@ iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen)
1197 /* Remember header pointer for later, when we need 368 /* Remember header pointer for later, when we need
1198 * to decide whether there's a payload to go along 369 * to decide whether there's a payload to go along
1199 * with the header. */ 370 * with the header. */
1200 tcp_conn->out.hdr = hdr; 371 tcp_sw_conn->out.hdr = hdr;
1201 372
1202 iscsi_segment_init_linear(&tcp_conn->out.segment, hdr, hdrlen, 373 iscsi_segment_init_linear(&tcp_sw_conn->out.segment, hdr, hdrlen,
1203 iscsi_tcp_send_hdr_done, NULL); 374 iscsi_sw_tcp_send_hdr_done, NULL);
1204} 375}
1205 376
1206/* 377/*
@@ -1209,11 +380,12 @@ iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen)
1209 * of by the iscsi_segment routines. 380 * of by the iscsi_segment routines.
1210 */ 381 */
1211static int 382static int
1212iscsi_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg, 383iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
1213 unsigned int count, unsigned int offset, 384 unsigned int count, unsigned int offset,
1214 unsigned int len) 385 unsigned int len)
1215{ 386{
1216 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 387 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
388 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1217 struct hash_desc *tx_hash = NULL; 389 struct hash_desc *tx_hash = NULL;
1218 unsigned int hdr_spec_len; 390 unsigned int hdr_spec_len;
1219 391
@@ -1223,22 +395,23 @@ iscsi_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
1223 395
1224 /* Make sure the datalen matches what the caller 396 /* Make sure the datalen matches what the caller
1225 said he would send. */ 397 said he would send. */
1226 hdr_spec_len = ntoh24(tcp_conn->out.hdr->dlength); 398 hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
1227 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); 399 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
1228 400
1229 if (conn->datadgst_en) 401 if (conn->datadgst_en)
1230 tx_hash = &tcp_conn->tx_hash; 402 tx_hash = &tcp_sw_conn->tx_hash;
1231 403
1232 return iscsi_segment_seek_sg(&tcp_conn->out.data_segment, 404 return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment,
1233 sg, count, offset, len, 405 sg, count, offset, len,
1234 NULL, tx_hash); 406 NULL, tx_hash);
1235} 407}
1236 408
1237static void 409static void
1238iscsi_tcp_send_linear_data_prepare(struct iscsi_conn *conn, void *data, 410iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data,
1239 size_t len) 411 size_t len)
1240{ 412{
1241 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 413 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
414 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1242 struct hash_desc *tx_hash = NULL; 415 struct hash_desc *tx_hash = NULL;
1243 unsigned int hdr_spec_len; 416 unsigned int hdr_spec_len;
1244 417
@@ -1247,341 +420,160 @@ iscsi_tcp_send_linear_data_prepare(struct iscsi_conn *conn, void *data,
1247 420
1248 /* Make sure the datalen matches what the caller 421 /* Make sure the datalen matches what the caller
1249 said he would send. */ 422 said he would send. */
1250 hdr_spec_len = ntoh24(tcp_conn->out.hdr->dlength); 423 hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
1251 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); 424 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
1252 425
1253 if (conn->datadgst_en) 426 if (conn->datadgst_en)
1254 tx_hash = &tcp_conn->tx_hash; 427 tx_hash = &tcp_sw_conn->tx_hash;
1255 428
1256 iscsi_segment_init_linear(&tcp_conn->out.data_segment, 429 iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment,
1257 data, len, NULL, tx_hash); 430 data, len, NULL, tx_hash);
1258} 431}
1259 432
1260/** 433static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task,
1261 * iscsi_solicit_data_cont - initialize next Data-Out 434 unsigned int offset, unsigned int count)
1262 * @conn: iscsi connection
1263 * @task: scsi command task
1264 * @r2t: R2T info
1265 * @left: bytes left to transfer
1266 *
1267 * Notes:
1268 * Initialize next Data-Out within this R2T sequence and continue
1269 * to process next Scatter-Gather element(if any) of this SCSI command.
1270 *
1271 * Called under connection lock.
1272 **/
1273static int
1274iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_task *task,
1275 struct iscsi_r2t_info *r2t)
1276{ 435{
1277 struct iscsi_data *hdr;
1278 int new_offset, left;
1279
1280 BUG_ON(r2t->data_length - r2t->sent < 0);
1281 left = r2t->data_length - r2t->sent;
1282 if (left == 0)
1283 return 0;
1284
1285 hdr = &r2t->dtask.hdr;
1286 memset(hdr, 0, sizeof(struct iscsi_data));
1287 hdr->ttt = r2t->ttt;
1288 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1289 r2t->solicit_datasn++;
1290 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1291 memcpy(hdr->lun, task->hdr->lun, sizeof(hdr->lun));
1292 hdr->itt = task->hdr->itt;
1293 hdr->exp_statsn = r2t->exp_statsn;
1294 new_offset = r2t->data_offset + r2t->sent;
1295 hdr->offset = cpu_to_be32(new_offset);
1296 if (left > conn->max_xmit_dlength) {
1297 hton24(hdr->dlength, conn->max_xmit_dlength);
1298 r2t->data_count = conn->max_xmit_dlength;
1299 } else {
1300 hton24(hdr->dlength, left);
1301 r2t->data_count = left;
1302 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1303 }
1304
1305 conn->dataout_pdus_cnt++;
1306 return 1;
1307}
1308
1309/**
1310 * iscsi_tcp_task - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1311 * @conn: iscsi connection
1312 * @task: scsi command task
1313 * @sc: scsi command
1314 **/
1315static int
1316iscsi_tcp_task_init(struct iscsi_task *task)
1317{
1318 struct iscsi_tcp_task *tcp_task = task->dd_data;
1319 struct iscsi_conn *conn = task->conn; 436 struct iscsi_conn *conn = task->conn;
1320 struct scsi_cmnd *sc = task->sc; 437 int err = 0;
1321 int err;
1322 438
1323 if (!sc) { 439 iscsi_sw_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len);
1324 /*
1325 * mgmt tasks do not have a scatterlist since they come
1326 * in from the iscsi interface.
1327 */
1328 debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn->id,
1329 task->itt);
1330
1331 /* Prepare PDU, optionally w/ immediate data */
1332 iscsi_tcp_send_hdr_prep(conn, task->hdr, sizeof(*task->hdr));
1333
1334 /* If we have immediate data, attach a payload */
1335 if (task->data_count)
1336 iscsi_tcp_send_linear_data_prepare(conn, task->data,
1337 task->data_count);
1338 return 0;
1339 }
1340 440
1341 BUG_ON(__kfifo_len(tcp_task->r2tqueue)); 441 if (!count)
1342 tcp_task->sent = 0; 442 return 0;
1343 tcp_task->exp_datasn = 0;
1344 443
1345 /* Prepare PDU, optionally w/ immediate data */ 444 if (!task->sc)
1346 debug_scsi("task deq [cid %d itt 0x%x imm %d unsol %d]\n", 445 iscsi_sw_tcp_send_linear_data_prep(conn, task->data, count);
1347 conn->id, task->itt, task->imm_count, 446 else {
1348 task->unsol_count); 447 struct scsi_data_buffer *sdb = scsi_out(task->sc);
1349 iscsi_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len);
1350 448
1351 if (!task->imm_count) 449 err = iscsi_sw_tcp_send_data_prep(conn, sdb->table.sgl,
1352 return 0; 450 sdb->table.nents, offset,
451 count);
452 }
1353 453
1354 /* If we have immediate data, attach a payload */ 454 if (err) {
1355 err = iscsi_tcp_send_data_prep(conn, scsi_out(sc)->table.sgl, 455 iscsi_conn_failure(conn, err);
1356 scsi_out(sc)->table.nents, 456 return -EIO;
1357 0, task->imm_count); 457 }
1358 if (err)
1359 return err;
1360 tcp_task->sent += task->imm_count;
1361 task->imm_count = 0;
1362 return 0; 458 return 0;
1363} 459}
1364 460
1365/* 461static int iscsi_sw_tcp_pdu_alloc(struct iscsi_task *task, uint8_t opcode)
1366 * iscsi_tcp_task_xmit - xmit normal PDU task
1367 * @task: iscsi command task
1368 *
1369 * We're expected to return 0 when everything was transmitted succesfully,
1370 * -EAGAIN if there's still data in the queue, or != 0 for any other kind
1371 * of error.
1372 */
1373static int
1374iscsi_tcp_task_xmit(struct iscsi_task *task)
1375{ 462{
1376 struct iscsi_conn *conn = task->conn;
1377 struct iscsi_tcp_task *tcp_task = task->dd_data; 463 struct iscsi_tcp_task *tcp_task = task->dd_data;
1378 struct scsi_cmnd *sc = task->sc;
1379 struct scsi_data_buffer *sdb;
1380 int rc = 0;
1381
1382flush:
1383 /* Flush any pending data first. */
1384 rc = iscsi_tcp_flush(conn);
1385 if (rc < 0)
1386 return rc;
1387
1388 /* mgmt command */
1389 if (!sc) {
1390 if (task->hdr->itt == RESERVED_ITT)
1391 iscsi_put_task(task);
1392 return 0;
1393 }
1394
1395 /* Are we done already? */
1396 if (sc->sc_data_direction != DMA_TO_DEVICE)
1397 return 0;
1398 464
1399 sdb = scsi_out(sc); 465 task->hdr = task->dd_data + sizeof(*tcp_task);
1400 if (task->unsol_count != 0) { 466 task->hdr_max = sizeof(struct iscsi_sw_tcp_hdrbuf) - ISCSI_DIGEST_SIZE;
1401 struct iscsi_data *hdr = &tcp_task->unsol_dtask.hdr;
1402
1403 /* Prepare a header for the unsolicited PDU.
1404 * The amount of data we want to send will be
1405 * in task->data_count.
1406 * FIXME: return the data count instead.
1407 */
1408 iscsi_prep_unsolicit_data_pdu(task, hdr);
1409
1410 debug_tcp("unsol dout [itt 0x%x doff %d dlen %d]\n",
1411 task->itt, tcp_task->sent, task->data_count);
1412
1413 iscsi_tcp_send_hdr_prep(conn, hdr, sizeof(*hdr));
1414 rc = iscsi_tcp_send_data_prep(conn, sdb->table.sgl,
1415 sdb->table.nents, tcp_task->sent,
1416 task->data_count);
1417 if (rc)
1418 goto fail;
1419 tcp_task->sent += task->data_count;
1420 task->unsol_count -= task->data_count;
1421 goto flush;
1422 } else {
1423 struct iscsi_session *session = conn->session;
1424 struct iscsi_r2t_info *r2t;
1425
1426 /* All unsolicited PDUs sent. Check for solicited PDUs.
1427 */
1428 spin_lock_bh(&session->lock);
1429 r2t = tcp_task->r2t;
1430 if (r2t != NULL) {
1431 /* Continue with this R2T? */
1432 if (!iscsi_solicit_data_cont(conn, task, r2t)) {
1433 debug_scsi(" done with r2t %p\n", r2t);
1434
1435 __kfifo_put(tcp_task->r2tpool.queue,
1436 (void*)&r2t, sizeof(void*));
1437 tcp_task->r2t = r2t = NULL;
1438 }
1439 }
1440
1441 if (r2t == NULL) {
1442 __kfifo_get(tcp_task->r2tqueue, (void*)&tcp_task->r2t,
1443 sizeof(void*));
1444 r2t = tcp_task->r2t;
1445 }
1446 spin_unlock_bh(&session->lock);
1447
1448 /* Waiting for more R2Ts to arrive. */
1449 if (r2t == NULL) {
1450 debug_tcp("no R2Ts yet\n");
1451 return 0;
1452 }
1453
1454 debug_scsi("sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n",
1455 r2t, r2t->solicit_datasn - 1, task->itt,
1456 r2t->data_offset + r2t->sent, r2t->data_count);
1457
1458 iscsi_tcp_send_hdr_prep(conn, &r2t->dtask.hdr,
1459 sizeof(struct iscsi_hdr));
1460
1461 rc = iscsi_tcp_send_data_prep(conn, sdb->table.sgl,
1462 sdb->table.nents,
1463 r2t->data_offset + r2t->sent,
1464 r2t->data_count);
1465 if (rc)
1466 goto fail;
1467 tcp_task->sent += r2t->data_count;
1468 r2t->sent += r2t->data_count;
1469 goto flush;
1470 }
1471 return 0; 467 return 0;
1472fail:
1473 iscsi_conn_failure(conn, rc);
1474 return -EIO;
1475} 468}
1476 469
1477static struct iscsi_cls_conn * 470static struct iscsi_cls_conn *
1478iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx) 471iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session,
472 uint32_t conn_idx)
1479{ 473{
1480 struct iscsi_conn *conn; 474 struct iscsi_conn *conn;
1481 struct iscsi_cls_conn *cls_conn; 475 struct iscsi_cls_conn *cls_conn;
1482 struct iscsi_tcp_conn *tcp_conn; 476 struct iscsi_tcp_conn *tcp_conn;
477 struct iscsi_sw_tcp_conn *tcp_sw_conn;
1483 478
1484 cls_conn = iscsi_conn_setup(cls_session, sizeof(*tcp_conn), conn_idx); 479 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn),
480 conn_idx);
1485 if (!cls_conn) 481 if (!cls_conn)
1486 return NULL; 482 return NULL;
1487 conn = cls_conn->dd_data; 483 conn = cls_conn->dd_data;
1488 /*
1489 * due to strange issues with iser these are not set
1490 * in iscsi_conn_setup
1491 */
1492 conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;
1493
1494 tcp_conn = conn->dd_data; 484 tcp_conn = conn->dd_data;
1495 tcp_conn->iscsi_conn = conn; 485 tcp_sw_conn = tcp_conn->dd_data;
1496 486
1497 tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0, 487 tcp_sw_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1498 CRYPTO_ALG_ASYNC); 488 CRYPTO_ALG_ASYNC);
1499 tcp_conn->tx_hash.flags = 0; 489 tcp_sw_conn->tx_hash.flags = 0;
1500 if (IS_ERR(tcp_conn->tx_hash.tfm)) 490 if (IS_ERR(tcp_sw_conn->tx_hash.tfm))
1501 goto free_conn; 491 goto free_conn;
1502 492
1503 tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0, 493 tcp_sw_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1504 CRYPTO_ALG_ASYNC); 494 CRYPTO_ALG_ASYNC);
1505 tcp_conn->rx_hash.flags = 0; 495 tcp_sw_conn->rx_hash.flags = 0;
1506 if (IS_ERR(tcp_conn->rx_hash.tfm)) 496 if (IS_ERR(tcp_sw_conn->rx_hash.tfm))
1507 goto free_tx_tfm; 497 goto free_tx_tfm;
498 tcp_conn->rx_hash = &tcp_sw_conn->rx_hash;
1508 499
1509 return cls_conn; 500 return cls_conn;
1510 501
1511free_tx_tfm: 502free_tx_tfm:
1512 crypto_free_hash(tcp_conn->tx_hash.tfm); 503 crypto_free_hash(tcp_sw_conn->tx_hash.tfm);
1513free_conn: 504free_conn:
1514 iscsi_conn_printk(KERN_ERR, conn, 505 iscsi_conn_printk(KERN_ERR, conn,
1515 "Could not create connection due to crc32c " 506 "Could not create connection due to crc32c "
1516 "loading error. Make sure the crc32c " 507 "loading error. Make sure the crc32c "
1517 "module is built as a module or into the " 508 "module is built as a module or into the "
1518 "kernel\n"); 509 "kernel\n");
1519 iscsi_conn_teardown(cls_conn); 510 iscsi_tcp_conn_teardown(cls_conn);
1520 return NULL; 511 return NULL;
1521} 512}
1522 513
1523static void 514static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn)
1524iscsi_tcp_release_conn(struct iscsi_conn *conn)
1525{ 515{
1526 struct iscsi_session *session = conn->session; 516 struct iscsi_session *session = conn->session;
1527 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 517 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1528 struct socket *sock = tcp_conn->sock; 518 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
519 struct socket *sock = tcp_sw_conn->sock;
1529 520
1530 if (!sock) 521 if (!sock)
1531 return; 522 return;
1532 523
1533 sock_hold(sock->sk); 524 sock_hold(sock->sk);
1534 iscsi_conn_restore_callbacks(tcp_conn); 525 iscsi_sw_tcp_conn_restore_callbacks(tcp_sw_conn);
1535 sock_put(sock->sk); 526 sock_put(sock->sk);
1536 527
1537 spin_lock_bh(&session->lock); 528 spin_lock_bh(&session->lock);
1538 tcp_conn->sock = NULL; 529 tcp_sw_conn->sock = NULL;
1539 spin_unlock_bh(&session->lock); 530 spin_unlock_bh(&session->lock);
1540 sockfd_put(sock); 531 sockfd_put(sock);
1541} 532}
1542 533
1543static void 534static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1544iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1545{ 535{
1546 struct iscsi_conn *conn = cls_conn->dd_data; 536 struct iscsi_conn *conn = cls_conn->dd_data;
1547 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 537 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
538 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1548 539
1549 iscsi_tcp_release_conn(conn); 540 iscsi_sw_tcp_release_conn(conn);
1550 541
1551 if (tcp_conn->tx_hash.tfm) 542 if (tcp_sw_conn->tx_hash.tfm)
1552 crypto_free_hash(tcp_conn->tx_hash.tfm); 543 crypto_free_hash(tcp_sw_conn->tx_hash.tfm);
1553 if (tcp_conn->rx_hash.tfm) 544 if (tcp_sw_conn->rx_hash.tfm)
1554 crypto_free_hash(tcp_conn->rx_hash.tfm); 545 crypto_free_hash(tcp_sw_conn->rx_hash.tfm);
1555 546
1556 iscsi_conn_teardown(cls_conn); 547 iscsi_tcp_conn_teardown(cls_conn);
1557} 548}
1558 549
1559static void 550static void iscsi_sw_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
1560iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
1561{ 551{
1562 struct iscsi_conn *conn = cls_conn->dd_data; 552 struct iscsi_conn *conn = cls_conn->dd_data;
1563 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 553 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
554 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1564 555
1565 /* userspace may have goofed up and not bound us */ 556 /* userspace may have goofed up and not bound us */
1566 if (!tcp_conn->sock) 557 if (!tcp_sw_conn->sock)
1567 return; 558 return;
1568 /* 559 /*
1569 * Make sure our recv side is stopped. 560 * Make sure our recv side is stopped.
1570 * Older tools called conn stop before ep_disconnect 561 * Older tools called conn stop before ep_disconnect
1571 * so IO could still be coming in. 562 * so IO could still be coming in.
1572 */ 563 */
1573 write_lock_bh(&tcp_conn->sock->sk->sk_callback_lock); 564 write_lock_bh(&tcp_sw_conn->sock->sk->sk_callback_lock);
1574 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx); 565 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
1575 write_unlock_bh(&tcp_conn->sock->sk->sk_callback_lock); 566 write_unlock_bh(&tcp_sw_conn->sock->sk->sk_callback_lock);
1576 567
1577 iscsi_conn_stop(cls_conn, flag); 568 iscsi_conn_stop(cls_conn, flag);
1578 iscsi_tcp_release_conn(conn); 569 iscsi_sw_tcp_release_conn(conn);
1579} 570}
1580 571
1581static int iscsi_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock, 572static int iscsi_sw_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock,
1582 char *buf, int *port, 573 char *buf, int *port,
1583 int (*getname)(struct socket *, struct sockaddr *, 574 int (*getname)(struct socket *,
1584 int *addrlen)) 575 struct sockaddr *,
576 int *addrlen))
1585{ 577{
1586 struct sockaddr_storage *addr; 578 struct sockaddr_storage *addr;
1587 struct sockaddr_in6 *sin6; 579 struct sockaddr_in6 *sin6;
@@ -1619,14 +611,15 @@ free_addr:
1619} 611}
1620 612
1621static int 613static int
1622iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session, 614iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1623 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, 615 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
1624 int is_leading) 616 int is_leading)
1625{ 617{
1626 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 618 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
1627 struct iscsi_host *ihost = shost_priv(shost); 619 struct iscsi_host *ihost = shost_priv(shost);
1628 struct iscsi_conn *conn = cls_conn->dd_data; 620 struct iscsi_conn *conn = cls_conn->dd_data;
1629 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 621 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
622 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1630 struct sock *sk; 623 struct sock *sk;
1631 struct socket *sock; 624 struct socket *sock;
1632 int err; 625 int err;
@@ -1643,13 +636,13 @@ iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1643 * userspace may still want to query the values since we will 636 * userspace may still want to query the values since we will
1644 * be using them for the reconnect 637 * be using them for the reconnect
1645 */ 638 */
1646 err = iscsi_tcp_get_addr(conn, sock, conn->portal_address, 639 err = iscsi_sw_tcp_get_addr(conn, sock, conn->portal_address,
1647 &conn->portal_port, kernel_getpeername); 640 &conn->portal_port, kernel_getpeername);
1648 if (err) 641 if (err)
1649 goto free_socket; 642 goto free_socket;
1650 643
1651 err = iscsi_tcp_get_addr(conn, sock, ihost->local_address, 644 err = iscsi_sw_tcp_get_addr(conn, sock, ihost->local_address,
1652 &ihost->local_port, kernel_getsockname); 645 &ihost->local_port, kernel_getsockname);
1653 if (err) 646 if (err)
1654 goto free_socket; 647 goto free_socket;
1655 648
@@ -1658,7 +651,7 @@ iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1658 goto free_socket; 651 goto free_socket;
1659 652
1660 /* bind iSCSI connection and socket */ 653 /* bind iSCSI connection and socket */
1661 tcp_conn->sock = sock; 654 tcp_sw_conn->sock = sock;
1662 655
1663 /* setup Socket parameters */ 656 /* setup Socket parameters */
1664 sk = sock->sk; 657 sk = sock->sk;
@@ -1666,8 +659,8 @@ iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1666 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */ 659 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
1667 sk->sk_allocation = GFP_ATOMIC; 660 sk->sk_allocation = GFP_ATOMIC;
1668 661
1669 iscsi_conn_set_callbacks(conn); 662 iscsi_sw_tcp_conn_set_callbacks(conn);
1670 tcp_conn->sendpage = tcp_conn->sock->ops->sendpage; 663 tcp_sw_conn->sendpage = tcp_sw_conn->sock->ops->sendpage;
1671 /* 664 /*
1672 * set receive state machine into initial state 665 * set receive state machine into initial state
1673 */ 666 */
@@ -1679,74 +672,14 @@ free_socket:
1679 return err; 672 return err;
1680} 673}
1681 674
1682static int 675static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
1683iscsi_r2tpool_alloc(struct iscsi_session *session) 676 enum iscsi_param param, char *buf,
1684{ 677 int buflen)
1685 int i;
1686 int cmd_i;
1687
1688 /*
1689 * initialize per-task: R2T pool and xmit queue
1690 */
1691 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
1692 struct iscsi_task *task = session->cmds[cmd_i];
1693 struct iscsi_tcp_task *tcp_task = task->dd_data;
1694
1695 /*
1696 * pre-allocated x4 as much r2ts to handle race when
1697 * target acks DataOut faster than we data_xmit() queues
1698 * could replenish r2tqueue.
1699 */
1700
1701 /* R2T pool */
1702 if (iscsi_pool_init(&tcp_task->r2tpool, session->max_r2t * 4, NULL,
1703 sizeof(struct iscsi_r2t_info))) {
1704 goto r2t_alloc_fail;
1705 }
1706
1707 /* R2T xmit queue */
1708 tcp_task->r2tqueue = kfifo_alloc(
1709 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
1710 if (tcp_task->r2tqueue == ERR_PTR(-ENOMEM)) {
1711 iscsi_pool_free(&tcp_task->r2tpool);
1712 goto r2t_alloc_fail;
1713 }
1714 }
1715
1716 return 0;
1717
1718r2t_alloc_fail:
1719 for (i = 0; i < cmd_i; i++) {
1720 struct iscsi_task *task = session->cmds[i];
1721 struct iscsi_tcp_task *tcp_task = task->dd_data;
1722
1723 kfifo_free(tcp_task->r2tqueue);
1724 iscsi_pool_free(&tcp_task->r2tpool);
1725 }
1726 return -ENOMEM;
1727}
1728
1729static void
1730iscsi_r2tpool_free(struct iscsi_session *session)
1731{
1732 int i;
1733
1734 for (i = 0; i < session->cmds_max; i++) {
1735 struct iscsi_task *task = session->cmds[i];
1736 struct iscsi_tcp_task *tcp_task = task->dd_data;
1737
1738 kfifo_free(tcp_task->r2tqueue);
1739 iscsi_pool_free(&tcp_task->r2tpool);
1740 }
1741}
1742
1743static int
1744iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1745 char *buf, int buflen)
1746{ 678{
1747 struct iscsi_conn *conn = cls_conn->dd_data; 679 struct iscsi_conn *conn = cls_conn->dd_data;
1748 struct iscsi_session *session = conn->session; 680 struct iscsi_session *session = conn->session;
1749 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 681 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
682 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1750 int value; 683 int value;
1751 684
1752 switch(param) { 685 switch(param) {
@@ -1755,8 +688,8 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1755 break; 688 break;
1756 case ISCSI_PARAM_DATADGST_EN: 689 case ISCSI_PARAM_DATADGST_EN:
1757 iscsi_set_param(cls_conn, param, buf, buflen); 690 iscsi_set_param(cls_conn, param, buf, buflen);
1758 tcp_conn->sendpage = conn->datadgst_en ? 691 tcp_sw_conn->sendpage = conn->datadgst_en ?
1759 sock_no_sendpage : tcp_conn->sock->ops->sendpage; 692 sock_no_sendpage : tcp_sw_conn->sock->ops->sendpage;
1760 break; 693 break;
1761 case ISCSI_PARAM_MAX_R2T: 694 case ISCSI_PARAM_MAX_R2T:
1762 sscanf(buf, "%d", &value); 695 sscanf(buf, "%d", &value);
@@ -1764,9 +697,9 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1764 return -EINVAL; 697 return -EINVAL;
1765 if (session->max_r2t == value) 698 if (session->max_r2t == value)
1766 break; 699 break;
1767 iscsi_r2tpool_free(session); 700 iscsi_tcp_r2tpool_free(session);
1768 iscsi_set_param(cls_conn, param, buf, buflen); 701 iscsi_set_param(cls_conn, param, buf, buflen);
1769 if (iscsi_r2tpool_alloc(session)) 702 if (iscsi_tcp_r2tpool_alloc(session))
1770 return -ENOMEM; 703 return -ENOMEM;
1771 break; 704 break;
1772 default: 705 default:
@@ -1776,9 +709,8 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1776 return 0; 709 return 0;
1777} 710}
1778 711
1779static int 712static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
1780iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn, 713 enum iscsi_param param, char *buf)
1781 enum iscsi_param param, char *buf)
1782{ 714{
1783 struct iscsi_conn *conn = cls_conn->dd_data; 715 struct iscsi_conn *conn = cls_conn->dd_data;
1784 int len; 716 int len;
@@ -1802,48 +734,42 @@ iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
1802} 734}
1803 735
1804static void 736static void
1805iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) 737iscsi_sw_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn,
738 struct iscsi_stats *stats)
1806{ 739{
1807 struct iscsi_conn *conn = cls_conn->dd_data; 740 struct iscsi_conn *conn = cls_conn->dd_data;
1808 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 741 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
742 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1809 743
1810 stats->txdata_octets = conn->txdata_octets;
1811 stats->rxdata_octets = conn->rxdata_octets;
1812 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
1813 stats->dataout_pdus = conn->dataout_pdus_cnt;
1814 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
1815 stats->datain_pdus = conn->datain_pdus_cnt;
1816 stats->r2t_pdus = conn->r2t_pdus_cnt;
1817 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
1818 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
1819 stats->custom_length = 3; 744 stats->custom_length = 3;
1820 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 745 strcpy(stats->custom[0].desc, "tx_sendpage_failures");
1821 stats->custom[0].value = tcp_conn->sendpage_failures_cnt; 746 stats->custom[0].value = tcp_sw_conn->sendpage_failures_cnt;
1822 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 747 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
1823 stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt; 748 stats->custom[1].value = tcp_sw_conn->discontiguous_hdr_cnt;
1824 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 749 strcpy(stats->custom[2].desc, "eh_abort_cnt");
1825 stats->custom[2].value = conn->eh_abort_cnt; 750 stats->custom[2].value = conn->eh_abort_cnt;
751
752 iscsi_tcp_conn_get_stats(cls_conn, stats);
1826} 753}
1827 754
1828static struct iscsi_cls_session * 755static struct iscsi_cls_session *
1829iscsi_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max, 756iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
1830 uint16_t qdepth, uint32_t initial_cmdsn, 757 uint16_t qdepth, uint32_t initial_cmdsn,
1831 uint32_t *hostno) 758 uint32_t *hostno)
1832{ 759{
1833 struct iscsi_cls_session *cls_session; 760 struct iscsi_cls_session *cls_session;
1834 struct iscsi_session *session; 761 struct iscsi_session *session;
1835 struct Scsi_Host *shost; 762 struct Scsi_Host *shost;
1836 int cmd_i;
1837 763
1838 if (ep) { 764 if (ep) {
1839 printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep); 765 printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep);
1840 return NULL; 766 return NULL;
1841 } 767 }
1842 768
1843 shost = iscsi_host_alloc(&iscsi_sht, 0, qdepth); 769 shost = iscsi_host_alloc(&iscsi_sw_tcp_sht, 0, qdepth);
1844 if (!shost) 770 if (!shost)
1845 return NULL; 771 return NULL;
1846 shost->transportt = iscsi_tcp_scsi_transport; 772 shost->transportt = iscsi_sw_tcp_scsi_transport;
1847 shost->max_lun = iscsi_max_lun; 773 shost->max_lun = iscsi_max_lun;
1848 shost->max_id = 0; 774 shost->max_id = 0;
1849 shost->max_channel = 0; 775 shost->max_channel = 0;
@@ -1853,23 +779,17 @@ iscsi_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
1853 goto free_host; 779 goto free_host;
1854 *hostno = shost->host_no; 780 *hostno = shost->host_no;
1855 781
1856 cls_session = iscsi_session_setup(&iscsi_tcp_transport, shost, cmds_max, 782 cls_session = iscsi_session_setup(&iscsi_sw_tcp_transport, shost,
1857 sizeof(struct iscsi_tcp_task), 783 cmds_max,
784 sizeof(struct iscsi_tcp_task) +
785 sizeof(struct iscsi_sw_tcp_hdrbuf),
1858 initial_cmdsn, 0); 786 initial_cmdsn, 0);
1859 if (!cls_session) 787 if (!cls_session)
1860 goto remove_host; 788 goto remove_host;
1861 session = cls_session->dd_data; 789 session = cls_session->dd_data;
1862 790
1863 shost->can_queue = session->scsi_cmds_max; 791 shost->can_queue = session->scsi_cmds_max;
1864 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 792 if (iscsi_tcp_r2tpool_alloc(session))
1865 struct iscsi_task *task = session->cmds[cmd_i];
1866 struct iscsi_tcp_task *tcp_task = task->dd_data;
1867
1868 task->hdr = &tcp_task->hdr.cmd_hdr;
1869 task->hdr_max = sizeof(tcp_task->hdr) - ISCSI_DIGEST_SIZE;
1870 }
1871
1872 if (iscsi_r2tpool_alloc(session))
1873 goto remove_session; 793 goto remove_session;
1874 return cls_session; 794 return cls_session;
1875 795
@@ -1882,25 +802,25 @@ free_host:
1882 return NULL; 802 return NULL;
1883} 803}
1884 804
1885static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session) 805static void iscsi_sw_tcp_session_destroy(struct iscsi_cls_session *cls_session)
1886{ 806{
1887 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 807 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
1888 808
1889 iscsi_r2tpool_free(cls_session->dd_data); 809 iscsi_tcp_r2tpool_free(cls_session->dd_data);
1890 iscsi_session_teardown(cls_session); 810 iscsi_session_teardown(cls_session);
1891 811
1892 iscsi_host_remove(shost); 812 iscsi_host_remove(shost);
1893 iscsi_host_free(shost); 813 iscsi_host_free(shost);
1894} 814}
1895 815
1896static int iscsi_tcp_slave_configure(struct scsi_device *sdev) 816static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
1897{ 817{
1898 blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY); 818 blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
1899 blk_queue_dma_alignment(sdev->request_queue, 0); 819 blk_queue_dma_alignment(sdev->request_queue, 0);
1900 return 0; 820 return 0;
1901} 821}
1902 822
1903static struct scsi_host_template iscsi_sht = { 823static struct scsi_host_template iscsi_sw_tcp_sht = {
1904 .module = THIS_MODULE, 824 .module = THIS_MODULE,
1905 .name = "iSCSI Initiator over TCP/IP", 825 .name = "iSCSI Initiator over TCP/IP",
1906 .queuecommand = iscsi_queuecommand, 826 .queuecommand = iscsi_queuecommand,
@@ -1913,12 +833,12 @@ static struct scsi_host_template iscsi_sht = {
1913 .eh_device_reset_handler= iscsi_eh_device_reset, 833 .eh_device_reset_handler= iscsi_eh_device_reset,
1914 .eh_target_reset_handler= iscsi_eh_target_reset, 834 .eh_target_reset_handler= iscsi_eh_target_reset,
1915 .use_clustering = DISABLE_CLUSTERING, 835 .use_clustering = DISABLE_CLUSTERING,
1916 .slave_configure = iscsi_tcp_slave_configure, 836 .slave_configure = iscsi_sw_tcp_slave_configure,
1917 .proc_name = "iscsi_tcp", 837 .proc_name = "iscsi_tcp",
1918 .this_id = -1, 838 .this_id = -1,
1919}; 839};
1920 840
1921static struct iscsi_transport iscsi_tcp_transport = { 841static struct iscsi_transport iscsi_sw_tcp_transport = {
1922 .owner = THIS_MODULE, 842 .owner = THIS_MODULE,
1923 .name = "tcp", 843 .name = "tcp",
1924 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 844 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
@@ -1951,32 +871,36 @@ static struct iscsi_transport iscsi_tcp_transport = {
1951 ISCSI_HOST_INITIATOR_NAME | 871 ISCSI_HOST_INITIATOR_NAME |
1952 ISCSI_HOST_NETDEV_NAME, 872 ISCSI_HOST_NETDEV_NAME,
1953 /* session management */ 873 /* session management */
1954 .create_session = iscsi_tcp_session_create, 874 .create_session = iscsi_sw_tcp_session_create,
1955 .destroy_session = iscsi_tcp_session_destroy, 875 .destroy_session = iscsi_sw_tcp_session_destroy,
1956 /* connection management */ 876 /* connection management */
1957 .create_conn = iscsi_tcp_conn_create, 877 .create_conn = iscsi_sw_tcp_conn_create,
1958 .bind_conn = iscsi_tcp_conn_bind, 878 .bind_conn = iscsi_sw_tcp_conn_bind,
1959 .destroy_conn = iscsi_tcp_conn_destroy, 879 .destroy_conn = iscsi_sw_tcp_conn_destroy,
1960 .set_param = iscsi_conn_set_param, 880 .set_param = iscsi_sw_tcp_conn_set_param,
1961 .get_conn_param = iscsi_tcp_conn_get_param, 881 .get_conn_param = iscsi_sw_tcp_conn_get_param,
1962 .get_session_param = iscsi_session_get_param, 882 .get_session_param = iscsi_session_get_param,
1963 .start_conn = iscsi_conn_start, 883 .start_conn = iscsi_conn_start,
1964 .stop_conn = iscsi_tcp_conn_stop, 884 .stop_conn = iscsi_sw_tcp_conn_stop,
1965 /* iscsi host params */ 885 /* iscsi host params */
1966 .get_host_param = iscsi_host_get_param, 886 .get_host_param = iscsi_host_get_param,
1967 .set_host_param = iscsi_host_set_param, 887 .set_host_param = iscsi_host_set_param,
1968 /* IO */ 888 /* IO */
1969 .send_pdu = iscsi_conn_send_pdu, 889 .send_pdu = iscsi_conn_send_pdu,
1970 .get_stats = iscsi_conn_get_stats, 890 .get_stats = iscsi_sw_tcp_conn_get_stats,
891 /* iscsi task/cmd helpers */
1971 .init_task = iscsi_tcp_task_init, 892 .init_task = iscsi_tcp_task_init,
1972 .xmit_task = iscsi_tcp_task_xmit, 893 .xmit_task = iscsi_tcp_task_xmit,
1973 .cleanup_task = iscsi_tcp_cleanup_task, 894 .cleanup_task = iscsi_tcp_cleanup_task,
895 /* low level pdu helpers */
896 .xmit_pdu = iscsi_sw_tcp_pdu_xmit,
897 .init_pdu = iscsi_sw_tcp_pdu_init,
898 .alloc_pdu = iscsi_sw_tcp_pdu_alloc,
1974 /* recovery */ 899 /* recovery */
1975 .session_recovery_timedout = iscsi_session_recovery_timedout, 900 .session_recovery_timedout = iscsi_session_recovery_timedout,
1976}; 901};
1977 902
1978static int __init 903static int __init iscsi_sw_tcp_init(void)
1979iscsi_tcp_init(void)
1980{ 904{
1981 if (iscsi_max_lun < 1) { 905 if (iscsi_max_lun < 1) {
1982 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n", 906 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
@@ -1984,19 +908,18 @@ iscsi_tcp_init(void)
1984 return -EINVAL; 908 return -EINVAL;
1985 } 909 }
1986 910
1987 iscsi_tcp_scsi_transport = iscsi_register_transport( 911 iscsi_sw_tcp_scsi_transport = iscsi_register_transport(
1988 &iscsi_tcp_transport); 912 &iscsi_sw_tcp_transport);
1989 if (!iscsi_tcp_scsi_transport) 913 if (!iscsi_sw_tcp_scsi_transport)
1990 return -ENODEV; 914 return -ENODEV;
1991 915
1992 return 0; 916 return 0;
1993} 917}
1994 918
1995static void __exit 919static void __exit iscsi_sw_tcp_exit(void)
1996iscsi_tcp_exit(void)
1997{ 920{
1998 iscsi_unregister_transport(&iscsi_tcp_transport); 921 iscsi_unregister_transport(&iscsi_sw_tcp_transport);
1999} 922}
2000 923
2001module_init(iscsi_tcp_init); 924module_init(iscsi_sw_tcp_init);
2002module_exit(iscsi_tcp_exit); 925module_exit(iscsi_sw_tcp_exit);
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-11 19:12:45 -0500