aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi/iscsi_tcp.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/iscsi_tcp.c')
-rw-r--r--drivers/scsi/iscsi_tcp.c1524
1 files changed, 248 insertions, 1276 deletions
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 97f3f68aeb64..07d9eb14721b 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -27,7 +27,6 @@
27 */ 27 */
28 28
29#include <linux/types.h> 29#include <linux/types.h>
30#include <linux/list.h>
31#include <linux/inet.h> 30#include <linux/inet.h>
32#include <linux/file.h> 31#include <linux/file.h>
33#include <linux/blkdev.h> 32#include <linux/blkdev.h>
@@ -44,12 +43,12 @@
44 43
45#include "iscsi_tcp.h" 44#include "iscsi_tcp.h"
46 45
47MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, " 46MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, "
47 "Dmitry Yusupov <dmitry_yus@yahoo.com>, "
48 "Alex Aizman <itn780@yahoo.com>"); 48 "Alex Aizman <itn780@yahoo.com>");
49MODULE_DESCRIPTION("iSCSI/TCP data-path"); 49MODULE_DESCRIPTION("iSCSI/TCP data-path");
50MODULE_LICENSE("GPL"); 50MODULE_LICENSE("GPL");
51#undef DEBUG_TCP 51#undef DEBUG_TCP
52#define DEBUG_ASSERT
53 52
54#ifdef DEBUG_TCP 53#ifdef DEBUG_TCP
55#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt) 54#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
@@ -57,924 +56,22 @@ MODULE_LICENSE("GPL");
57#define debug_tcp(fmt...) 56#define debug_tcp(fmt...)
58#endif 57#endif
59 58
60static struct scsi_transport_template *iscsi_tcp_scsi_transport; 59static struct scsi_transport_template *iscsi_sw_tcp_scsi_transport;
61static struct scsi_host_template iscsi_sht; 60static struct scsi_host_template iscsi_sw_tcp_sht;
62static struct iscsi_transport iscsi_tcp_transport; 61static struct iscsi_transport iscsi_sw_tcp_transport;
63 62
64static unsigned int iscsi_max_lun = 512; 63static unsigned int iscsi_max_lun = 512;
65module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 64module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
66 65
67static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
68 struct iscsi_segment *segment);
69
70/*
71 * Scatterlist handling: inside the iscsi_segment, we
72 * remember an index into the scatterlist, and set data/size
73 * to the current scatterlist entry. For highmem pages, we
74 * kmap as needed.
75 *
76 * Note that the page is unmapped when we return from
77 * TCP's data_ready handler, so we may end up mapping and
78 * unmapping the same page repeatedly. The whole reason
79 * for this is that we shouldn't keep the page mapped
80 * outside the softirq.
81 */
82
83/**
84 * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
85 * @segment: the buffer object
86 * @sg: scatterlist
87 * @offset: byte offset into that sg entry
88 *
89 * This function sets up the segment so that subsequent
90 * data is copied to the indicated sg entry, at the given
91 * offset.
92 */
93static inline void
94iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,
95 struct scatterlist *sg, unsigned int offset)
96{
97 segment->sg = sg;
98 segment->sg_offset = offset;
99 segment->size = min(sg->length - offset,
100 segment->total_size - segment->total_copied);
101 segment->data = NULL;
102}
103
104/**
105 * iscsi_tcp_segment_map - map the current S/G page
106 * @segment: iscsi_segment
107 * @recv: 1 if called from recv path
108 *
109 * We only need to possibly kmap data if scatter lists are being used,
110 * because the iscsi passthrough and internal IO paths will never use high
111 * mem pages.
112 */
113static inline void
114iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)
115{
116 struct scatterlist *sg;
117
118 if (segment->data != NULL || !segment->sg)
119 return;
120
121 sg = segment->sg;
122 BUG_ON(segment->sg_mapped);
123 BUG_ON(sg->length == 0);
124
125 /*
126 * If the page count is greater than one it is ok to send
127 * to the network layer's zero copy send path. If not we
128 * have to go the slow sendmsg path. We always map for the
129 * recv path.
130 */
131 if (page_count(sg_page(sg)) >= 1 && !recv)
132 return;
133
134 debug_tcp("iscsi_tcp_segment_map %s %p\n", recv ? "recv" : "xmit",
135 segment);
136 segment->sg_mapped = kmap_atomic(sg_page(sg), KM_SOFTIRQ0);
137 segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;
138}
139
140static inline void
141iscsi_tcp_segment_unmap(struct iscsi_segment *segment)
142{
143 debug_tcp("iscsi_tcp_segment_unmap %p\n", segment);
144
145 if (segment->sg_mapped) {
146 debug_tcp("iscsi_tcp_segment_unmap valid\n");
147 kunmap_atomic(segment->sg_mapped, KM_SOFTIRQ0);
148 segment->sg_mapped = NULL;
149 segment->data = NULL;
150 }
151}
152
153/*
154 * Splice the digest buffer into the buffer
155 */
156static inline void
157iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)
158{
159 segment->data = digest;
160 segment->digest_len = ISCSI_DIGEST_SIZE;
161 segment->total_size += ISCSI_DIGEST_SIZE;
162 segment->size = ISCSI_DIGEST_SIZE;
163 segment->copied = 0;
164 segment->sg = NULL;
165 segment->hash = NULL;
166}
167
168/**
169 * iscsi_tcp_segment_done - check whether the segment is complete
170 * @segment: iscsi segment to check
171 * @recv: set to one of this is called from the recv path
172 * @copied: number of bytes copied
173 *
174 * Check if we're done receiving this segment. If the receive
175 * buffer is full but we expect more data, move on to the
176 * next entry in the scatterlist.
177 *
178 * If the amount of data we received isn't a multiple of 4,
179 * we will transparently receive the pad bytes, too.
180 *
181 * This function must be re-entrant.
182 */
183static inline int
184iscsi_tcp_segment_done(struct iscsi_segment *segment, int recv, unsigned copied)
185{
186 static unsigned char padbuf[ISCSI_PAD_LEN];
187 struct scatterlist sg;
188 unsigned int pad;
189
190 debug_tcp("copied %u %u size %u %s\n", segment->copied, copied,
191 segment->size, recv ? "recv" : "xmit");
192 if (segment->hash && copied) {
193 /*
194 * If a segment is kmapd we must unmap it before sending
195 * to the crypto layer since that will try to kmap it again.
196 */
197 iscsi_tcp_segment_unmap(segment);
198
199 if (!segment->data) {
200 sg_init_table(&sg, 1);
201 sg_set_page(&sg, sg_page(segment->sg), copied,
202 segment->copied + segment->sg_offset +
203 segment->sg->offset);
204 } else
205 sg_init_one(&sg, segment->data + segment->copied,
206 copied);
207 crypto_hash_update(segment->hash, &sg, copied);
208 }
209
210 segment->copied += copied;
211 if (segment->copied < segment->size) {
212 iscsi_tcp_segment_map(segment, recv);
213 return 0;
214 }
215
216 segment->total_copied += segment->copied;
217 segment->copied = 0;
218 segment->size = 0;
219
220 /* Unmap the current scatterlist page, if there is one. */
221 iscsi_tcp_segment_unmap(segment);
222
223 /* Do we have more scatterlist entries? */
224 debug_tcp("total copied %u total size %u\n", segment->total_copied,
225 segment->total_size);
226 if (segment->total_copied < segment->total_size) {
227 /* Proceed to the next entry in the scatterlist. */
228 iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),
229 0);
230 iscsi_tcp_segment_map(segment, recv);
231 BUG_ON(segment->size == 0);
232 return 0;
233 }
234
235 /* Do we need to handle padding? */
236 pad = iscsi_padding(segment->total_copied);
237 if (pad != 0) {
238 debug_tcp("consume %d pad bytes\n", pad);
239 segment->total_size += pad;
240 segment->size = pad;
241 segment->data = padbuf;
242 return 0;
243 }
244
245 /*
246 * Set us up for transferring the data digest. hdr digest
247 * is completely handled in hdr done function.
248 */
249 if (segment->hash) {
250 crypto_hash_final(segment->hash, segment->digest);
251 iscsi_tcp_segment_splice_digest(segment,
252 recv ? segment->recv_digest : segment->digest);
253 return 0;
254 }
255
256 return 1;
257}
258
259/** 66/**
260 * iscsi_tcp_xmit_segment - transmit segment 67 * iscsi_sw_tcp_recv - TCP receive in sendfile fashion
261 * @tcp_conn: the iSCSI TCP connection
262 * @segment: the buffer to transmnit
263 *
264 * This function transmits as much of the buffer as
265 * the network layer will accept, and returns the number of
266 * bytes transmitted.
267 *
268 * If CRC hashing is enabled, the function will compute the
269 * hash as it goes. When the entire segment has been transmitted,
270 * it will retrieve the hash value and send it as well.
271 */
272static int
273iscsi_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
274 struct iscsi_segment *segment)
275{
276 struct socket *sk = tcp_conn->sock;
277 unsigned int copied = 0;
278 int r = 0;
279
280 while (!iscsi_tcp_segment_done(segment, 0, r)) {
281 struct scatterlist *sg;
282 unsigned int offset, copy;
283 int flags = 0;
284
285 r = 0;
286 offset = segment->copied;
287 copy = segment->size - offset;
288
289 if (segment->total_copied + segment->size < segment->total_size)
290 flags |= MSG_MORE;
291
292 /* Use sendpage if we can; else fall back to sendmsg */
293 if (!segment->data) {
294 sg = segment->sg;
295 offset += segment->sg_offset + sg->offset;
296 r = tcp_conn->sendpage(sk, sg_page(sg), offset, copy,
297 flags);
298 } else {
299 struct msghdr msg = { .msg_flags = flags };
300 struct kvec iov = {
301 .iov_base = segment->data + offset,
302 .iov_len = copy
303 };
304
305 r = kernel_sendmsg(sk, &msg, &iov, 1, copy);
306 }
307
308 if (r < 0) {
309 iscsi_tcp_segment_unmap(segment);
310 if (copied || r == -EAGAIN)
311 break;
312 return r;
313 }
314 copied += r;
315 }
316 return copied;
317}
318
319/**
320 * iscsi_tcp_segment_recv - copy data to segment
321 * @tcp_conn: the iSCSI TCP connection
322 * @segment: the buffer to copy to
323 * @ptr: data pointer
324 * @len: amount of data available
325 *
326 * This function copies up to @len bytes to the
327 * given buffer, and returns the number of bytes
328 * consumed, which can actually be less than @len.
329 *
330 * If hash digest is enabled, the function will update the
331 * hash while copying.
332 * Combining these two operations doesn't buy us a lot (yet),
333 * but in the future we could implement combined copy+crc,
334 * just way we do for network layer checksums.
335 */
336static int
337iscsi_tcp_segment_recv(struct iscsi_tcp_conn *tcp_conn,
338 struct iscsi_segment *segment, const void *ptr,
339 unsigned int len)
340{
341 unsigned int copy = 0, copied = 0;
342
343 while (!iscsi_tcp_segment_done(segment, 1, copy)) {
344 if (copied == len) {
345 debug_tcp("iscsi_tcp_segment_recv copied %d bytes\n",
346 len);
347 break;
348 }
349
350 copy = min(len - copied, segment->size - segment->copied);
351 debug_tcp("iscsi_tcp_segment_recv copying %d\n", copy);
352 memcpy(segment->data + segment->copied, ptr + copied, copy);
353 copied += copy;
354 }
355 return copied;
356}
357
358static inline void
359iscsi_tcp_dgst_header(struct hash_desc *hash, const void *hdr, size_t hdrlen,
360 unsigned char digest[ISCSI_DIGEST_SIZE])
361{
362 struct scatterlist sg;
363
364 sg_init_one(&sg, hdr, hdrlen);
365 crypto_hash_digest(hash, &sg, hdrlen, digest);
366}
367
368static inline int
369iscsi_tcp_dgst_verify(struct iscsi_tcp_conn *tcp_conn,
370 struct iscsi_segment *segment)
371{
372 if (!segment->digest_len)
373 return 1;
374
375 if (memcmp(segment->recv_digest, segment->digest,
376 segment->digest_len)) {
377 debug_scsi("digest mismatch\n");
378 return 0;
379 }
380
381 return 1;
382}
383
384/*
385 * Helper function to set up segment buffer
386 */
387static inline void
388__iscsi_segment_init(struct iscsi_segment *segment, size_t size,
389 iscsi_segment_done_fn_t *done, struct hash_desc *hash)
390{
391 memset(segment, 0, sizeof(*segment));
392 segment->total_size = size;
393 segment->done = done;
394
395 if (hash) {
396 segment->hash = hash;
397 crypto_hash_init(hash);
398 }
399}
400
401static inline void
402iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,
403 size_t size, iscsi_segment_done_fn_t *done,
404 struct hash_desc *hash)
405{
406 __iscsi_segment_init(segment, size, done, hash);
407 segment->data = data;
408 segment->size = size;
409}
410
411static inline int
412iscsi_segment_seek_sg(struct iscsi_segment *segment,
413 struct scatterlist *sg_list, unsigned int sg_count,
414 unsigned int offset, size_t size,
415 iscsi_segment_done_fn_t *done, struct hash_desc *hash)
416{
417 struct scatterlist *sg;
418 unsigned int i;
419
420 debug_scsi("iscsi_segment_seek_sg offset %u size %llu\n",
421 offset, size);
422 __iscsi_segment_init(segment, size, done, hash);
423 for_each_sg(sg_list, sg, sg_count, i) {
424 debug_scsi("sg %d, len %u offset %u\n", i, sg->length,
425 sg->offset);
426 if (offset < sg->length) {
427 iscsi_tcp_segment_init_sg(segment, sg, offset);
428 return 0;
429 }
430 offset -= sg->length;
431 }
432
433 return ISCSI_ERR_DATA_OFFSET;
434}
435
436/**
437 * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
438 * @tcp_conn: iscsi connection to prep for
439 *
440 * This function always passes NULL for the hash argument, because when this
441 * function is called we do not yet know the final size of the header and want
442 * to delay the digest processing until we know that.
443 */
444static void
445iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn *tcp_conn)
446{
447 debug_tcp("iscsi_tcp_hdr_recv_prep(%p%s)\n", tcp_conn,
448 tcp_conn->iscsi_conn->hdrdgst_en ? ", digest enabled" : "");
449 iscsi_segment_init_linear(&tcp_conn->in.segment,
450 tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr),
451 iscsi_tcp_hdr_recv_done, NULL);
452}
453
454/*
455 * Handle incoming reply to any other type of command
456 */
457static int
458iscsi_tcp_data_recv_done(struct iscsi_tcp_conn *tcp_conn,
459 struct iscsi_segment *segment)
460{
461 struct iscsi_conn *conn = tcp_conn->iscsi_conn;
462 int rc = 0;
463
464 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
465 return ISCSI_ERR_DATA_DGST;
466
467 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr,
468 conn->data, tcp_conn->in.datalen);
469 if (rc)
470 return rc;
471
472 iscsi_tcp_hdr_recv_prep(tcp_conn);
473 return 0;
474}
475
476static void
477iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn)
478{
479 struct iscsi_conn *conn = tcp_conn->iscsi_conn;
480 struct hash_desc *rx_hash = NULL;
481
482 if (conn->datadgst_en &
483 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
484 rx_hash = &tcp_conn->rx_hash;
485
486 iscsi_segment_init_linear(&tcp_conn->in.segment,
487 conn->data, tcp_conn->in.datalen,
488 iscsi_tcp_data_recv_done, rx_hash);
489}
490
491/*
492 * must be called with session lock
493 */
494static void iscsi_tcp_cleanup_task(struct iscsi_task *task)
495{
496 struct iscsi_tcp_task *tcp_task = task->dd_data;
497 struct iscsi_r2t_info *r2t;
498
499 /* nothing to do for mgmt or pending tasks */
500 if (!task->sc || task->state == ISCSI_TASK_PENDING)
501 return;
502
503 /* flush task's r2t queues */
504 while (__kfifo_get(tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) {
505 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
506 sizeof(void*));
507 debug_scsi("iscsi_tcp_cleanup_task pending r2t dropped\n");
508 }
509
510 r2t = tcp_task->r2t;
511 if (r2t != NULL) {
512 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
513 sizeof(void*));
514 tcp_task->r2t = NULL;
515 }
516}
517
518/**
519 * iscsi_tcp_data_in - SCSI Data-In Response processing
520 * @conn: iscsi connection
521 * @task: scsi command task
522 */
523static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task)
524{
525 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
526 struct iscsi_tcp_task *tcp_task = task->dd_data;
527 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
528 int datasn = be32_to_cpu(rhdr->datasn);
529 unsigned total_in_length = scsi_in(task->sc)->length;
530
531 iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr);
532 if (tcp_conn->in.datalen == 0)
533 return 0;
534
535 if (tcp_task->exp_datasn != datasn) {
536 debug_tcp("%s: task->exp_datasn(%d) != rhdr->datasn(%d)\n",
537 __func__, tcp_task->exp_datasn, datasn);
538 return ISCSI_ERR_DATASN;
539 }
540
541 tcp_task->exp_datasn++;
542
543 tcp_task->data_offset = be32_to_cpu(rhdr->offset);
544 if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) {
545 debug_tcp("%s: data_offset(%d) + data_len(%d) > total_length_in(%d)\n",
546 __func__, tcp_task->data_offset,
547 tcp_conn->in.datalen, total_in_length);
548 return ISCSI_ERR_DATA_OFFSET;
549 }
550
551 conn->datain_pdus_cnt++;
552 return 0;
553}
554
555/**
556 * iscsi_tcp_r2t_rsp - iSCSI R2T Response processing
557 * @conn: iscsi connection
558 * @task: scsi command task
559 */
560static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task)
561{
562 struct iscsi_session *session = conn->session;
563 struct iscsi_tcp_task *tcp_task = task->dd_data;
564 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
565 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
566 struct iscsi_r2t_info *r2t;
567 int r2tsn = be32_to_cpu(rhdr->r2tsn);
568 int rc;
569
570 if (tcp_conn->in.datalen) {
571 iscsi_conn_printk(KERN_ERR, conn,
572 "invalid R2t with datalen %d\n",
573 tcp_conn->in.datalen);
574 return ISCSI_ERR_DATALEN;
575 }
576
577 if (tcp_task->exp_datasn != r2tsn){
578 debug_tcp("%s: task->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
579 __func__, tcp_task->exp_datasn, r2tsn);
580 return ISCSI_ERR_R2TSN;
581 }
582
583 /* fill-in new R2T associated with the task */
584 iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
585
586 if (!task->sc || session->state != ISCSI_STATE_LOGGED_IN) {
587 iscsi_conn_printk(KERN_INFO, conn,
588 "dropping R2T itt %d in recovery.\n",
589 task->itt);
590 return 0;
591 }
592
593 rc = __kfifo_get(tcp_task->r2tpool.queue, (void*)&r2t, sizeof(void*));
594 if (!rc) {
595 iscsi_conn_printk(KERN_ERR, conn, "Could not allocate R2T. "
596 "Target has sent more R2Ts than it "
597 "negotiated for or driver has has leaked.\n");
598 return ISCSI_ERR_PROTO;
599 }
600
601 r2t->exp_statsn = rhdr->statsn;
602 r2t->data_length = be32_to_cpu(rhdr->data_length);
603 if (r2t->data_length == 0) {
604 iscsi_conn_printk(KERN_ERR, conn,
605 "invalid R2T with zero data len\n");
606 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
607 sizeof(void*));
608 return ISCSI_ERR_DATALEN;
609 }
610
611 if (r2t->data_length > session->max_burst)
612 debug_scsi("invalid R2T with data len %u and max burst %u."
613 "Attempting to execute request.\n",
614 r2t->data_length, session->max_burst);
615
616 r2t->data_offset = be32_to_cpu(rhdr->data_offset);
617 if (r2t->data_offset + r2t->data_length > scsi_out(task->sc)->length) {
618 iscsi_conn_printk(KERN_ERR, conn,
619 "invalid R2T with data len %u at offset %u "
620 "and total length %d\n", r2t->data_length,
621 r2t->data_offset, scsi_out(task->sc)->length);
622 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
623 sizeof(void*));
624 return ISCSI_ERR_DATALEN;
625 }
626
627 r2t->ttt = rhdr->ttt; /* no flip */
628 r2t->datasn = 0;
629 r2t->sent = 0;
630
631 tcp_task->exp_datasn = r2tsn + 1;
632 __kfifo_put(tcp_task->r2tqueue, (void*)&r2t, sizeof(void*));
633 conn->r2t_pdus_cnt++;
634
635 iscsi_requeue_task(task);
636 return 0;
637}
638
639/*
640 * Handle incoming reply to DataIn command
641 */
642static int
643iscsi_tcp_process_data_in(struct iscsi_tcp_conn *tcp_conn,
644 struct iscsi_segment *segment)
645{
646 struct iscsi_conn *conn = tcp_conn->iscsi_conn;
647 struct iscsi_hdr *hdr = tcp_conn->in.hdr;
648 int rc;
649
650 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
651 return ISCSI_ERR_DATA_DGST;
652
653 /* check for non-exceptional status */
654 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
655 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
656 if (rc)
657 return rc;
658 }
659
660 iscsi_tcp_hdr_recv_prep(tcp_conn);
661 return 0;
662}
663
664/**
665 * iscsi_tcp_hdr_dissect - process PDU header
666 * @conn: iSCSI connection
667 * @hdr: PDU header
668 *
669 * This function analyzes the header of the PDU received,
670 * and performs several sanity checks. If the PDU is accompanied
671 * by data, the receive buffer is set up to copy the incoming data
672 * to the correct location.
673 */
674static int
675iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
676{
677 int rc = 0, opcode, ahslen;
678 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
679 struct iscsi_task *task;
680
681 /* verify PDU length */
682 tcp_conn->in.datalen = ntoh24(hdr->dlength);
683 if (tcp_conn->in.datalen > conn->max_recv_dlength) {
684 iscsi_conn_printk(KERN_ERR, conn,
685 "iscsi_tcp: datalen %d > %d\n",
686 tcp_conn->in.datalen, conn->max_recv_dlength);
687 return ISCSI_ERR_DATALEN;
688 }
689
690 /* Additional header segments. So far, we don't
691 * process additional headers.
692 */
693 ahslen = hdr->hlength << 2;
694
695 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
696 /* verify itt (itt encoding: age+cid+itt) */
697 rc = iscsi_verify_itt(conn, hdr->itt);
698 if (rc)
699 return rc;
700
701 debug_tcp("opcode 0x%x ahslen %d datalen %d\n",
702 opcode, ahslen, tcp_conn->in.datalen);
703
704 switch(opcode) {
705 case ISCSI_OP_SCSI_DATA_IN:
706 spin_lock(&conn->session->lock);
707 task = iscsi_itt_to_ctask(conn, hdr->itt);
708 if (!task)
709 rc = ISCSI_ERR_BAD_ITT;
710 else
711 rc = iscsi_tcp_data_in(conn, task);
712 if (rc) {
713 spin_unlock(&conn->session->lock);
714 break;
715 }
716
717 if (tcp_conn->in.datalen) {
718 struct iscsi_tcp_task *tcp_task = task->dd_data;
719 struct hash_desc *rx_hash = NULL;
720 struct scsi_data_buffer *sdb = scsi_in(task->sc);
721
722 /*
723 * Setup copy of Data-In into the Scsi_Cmnd
724 * Scatterlist case:
725 * We set up the iscsi_segment to point to the next
726 * scatterlist entry to copy to. As we go along,
727 * we move on to the next scatterlist entry and
728 * update the digest per-entry.
729 */
730 if (conn->datadgst_en &&
731 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
732 rx_hash = &tcp_conn->rx_hash;
733
734 debug_tcp("iscsi_tcp_begin_data_in(%p, offset=%d, "
735 "datalen=%d)\n", tcp_conn,
736 tcp_task->data_offset,
737 tcp_conn->in.datalen);
738 rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,
739 sdb->table.sgl,
740 sdb->table.nents,
741 tcp_task->data_offset,
742 tcp_conn->in.datalen,
743 iscsi_tcp_process_data_in,
744 rx_hash);
745 spin_unlock(&conn->session->lock);
746 return rc;
747 }
748 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
749 spin_unlock(&conn->session->lock);
750 break;
751 case ISCSI_OP_SCSI_CMD_RSP:
752 if (tcp_conn->in.datalen) {
753 iscsi_tcp_data_recv_prep(tcp_conn);
754 return 0;
755 }
756 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
757 break;
758 case ISCSI_OP_R2T:
759 spin_lock(&conn->session->lock);
760 task = iscsi_itt_to_ctask(conn, hdr->itt);
761 if (!task)
762 rc = ISCSI_ERR_BAD_ITT;
763 else if (ahslen)
764 rc = ISCSI_ERR_AHSLEN;
765 else if (task->sc->sc_data_direction == DMA_TO_DEVICE)
766 rc = iscsi_tcp_r2t_rsp(conn, task);
767 else
768 rc = ISCSI_ERR_PROTO;
769 spin_unlock(&conn->session->lock);
770 break;
771 case ISCSI_OP_LOGIN_RSP:
772 case ISCSI_OP_TEXT_RSP:
773 case ISCSI_OP_REJECT:
774 case ISCSI_OP_ASYNC_EVENT:
775 /*
776 * It is possible that we could get a PDU with a buffer larger
777 * than 8K, but there are no targets that currently do this.
778 * For now we fail until we find a vendor that needs it
779 */
780 if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) {
781 iscsi_conn_printk(KERN_ERR, conn,
782 "iscsi_tcp: received buffer of "
783 "len %u but conn buffer is only %u "
784 "(opcode %0x)\n",
785 tcp_conn->in.datalen,
786 ISCSI_DEF_MAX_RECV_SEG_LEN, opcode);
787 rc = ISCSI_ERR_PROTO;
788 break;
789 }
790
791 /* If there's data coming in with the response,
792 * receive it to the connection's buffer.
793 */
794 if (tcp_conn->in.datalen) {
795 iscsi_tcp_data_recv_prep(tcp_conn);
796 return 0;
797 }
798 /* fall through */
799 case ISCSI_OP_LOGOUT_RSP:
800 case ISCSI_OP_NOOP_IN:
801 case ISCSI_OP_SCSI_TMFUNC_RSP:
802 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
803 break;
804 default:
805 rc = ISCSI_ERR_BAD_OPCODE;
806 break;
807 }
808
809 if (rc == 0) {
810 /* Anything that comes with data should have
811 * been handled above. */
812 if (tcp_conn->in.datalen)
813 return ISCSI_ERR_PROTO;
814 iscsi_tcp_hdr_recv_prep(tcp_conn);
815 }
816
817 return rc;
818}
819
820/**
821 * iscsi_tcp_hdr_recv_done - process PDU header
822 *
823 * This is the callback invoked when the PDU header has
824 * been received. If the header is followed by additional
825 * header segments, we go back for more data.
826 */
827static int
828iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
829 struct iscsi_segment *segment)
830{
831 struct iscsi_conn *conn = tcp_conn->iscsi_conn;
832 struct iscsi_hdr *hdr;
833
834 /* Check if there are additional header segments
835 * *prior* to computing the digest, because we
836 * may need to go back to the caller for more.
837 */
838 hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf;
839 if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {
840 /* Bump the header length - the caller will
841 * just loop around and get the AHS for us, and
842 * call again. */
843 unsigned int ahslen = hdr->hlength << 2;
844
845 /* Make sure we don't overflow */
846 if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))
847 return ISCSI_ERR_AHSLEN;
848
849 segment->total_size += ahslen;
850 segment->size += ahslen;
851 return 0;
852 }
853
854 /* We're done processing the header. See if we're doing
855 * header digests; if so, set up the recv_digest buffer
856 * and go back for more. */
857 if (conn->hdrdgst_en) {
858 if (segment->digest_len == 0) {
859 /*
860 * Even if we offload the digest processing we
861 * splice it in so we can increment the skb/segment
862 * counters in preparation for the data segment.
863 */
864 iscsi_tcp_segment_splice_digest(segment,
865 segment->recv_digest);
866 return 0;
867 }
868
869 if (!(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) {
870 iscsi_tcp_dgst_header(&tcp_conn->rx_hash, hdr,
871 segment->total_copied - ISCSI_DIGEST_SIZE,
872 segment->digest);
873
874 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
875 return ISCSI_ERR_HDR_DGST;
876 }
877 }
878
879 tcp_conn->in.hdr = hdr;
880 return iscsi_tcp_hdr_dissect(conn, hdr);
881}
882
883inline int iscsi_tcp_recv_segment_is_hdr(struct iscsi_tcp_conn *tcp_conn)
884{
885 return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;
886}
887
888enum {
889 ISCSI_TCP_SEGMENT_DONE, /* curr seg has been processed */
890 ISCSI_TCP_SKB_DONE, /* skb is out of data */
891 ISCSI_TCP_CONN_ERR, /* iscsi layer has fired a conn err */
892 ISCSI_TCP_SUSPENDED, /* conn is suspended */
893};
894
895/**
896 * iscsi_tcp_recv_skb - Process skb
897 * @conn: iscsi connection
898 * @skb: network buffer with header and/or data segment
899 * @offset: offset in skb
900 * @offload: bool indicating if transfer was offloaded
901 */
902int iscsi_tcp_recv_skb(struct iscsi_conn *conn, struct sk_buff *skb,
903 unsigned int offset, bool offloaded, int *status)
904{
905 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
906 struct iscsi_segment *segment = &tcp_conn->in.segment;
907 struct skb_seq_state seq;
908 unsigned int consumed = 0;
909 int rc = 0;
910
911 debug_tcp("in %d bytes\n", skb->len - offset);
912
913 if (unlikely(conn->suspend_rx)) {
914 debug_tcp("conn %d Rx suspended!\n", conn->id);
915 *status = ISCSI_TCP_SUSPENDED;
916 return 0;
917 }
918
919 if (offloaded) {
920 segment->total_copied = segment->total_size;
921 goto segment_done;
922 }
923
924 skb_prepare_seq_read(skb, offset, skb->len, &seq);
925 while (1) {
926 unsigned int avail;
927 const u8 *ptr;
928
929 avail = skb_seq_read(consumed, &ptr, &seq);
930 if (avail == 0) {
931 debug_tcp("no more data avail. Consumed %d\n",
932 consumed);
933 *status = ISCSI_TCP_SKB_DONE;
934 skb_abort_seq_read(&seq);
935 goto skb_done;
936 }
937 BUG_ON(segment->copied >= segment->size);
938
939 debug_tcp("skb %p ptr=%p avail=%u\n", skb, ptr, avail);
940 rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);
941 BUG_ON(rc == 0);
942 consumed += rc;
943
944 if (segment->total_copied >= segment->total_size) {
945 skb_abort_seq_read(&seq);
946 goto segment_done;
947 }
948 }
949
950segment_done:
951 *status = ISCSI_TCP_SEGMENT_DONE;
952 debug_tcp("segment done\n");
953 rc = segment->done(tcp_conn, segment);
954 if (rc != 0) {
955 *status = ISCSI_TCP_CONN_ERR;
956 debug_tcp("Error receiving PDU, errno=%d\n", rc);
957 iscsi_conn_failure(conn, rc);
958 return 0;
959 }
960 /* The done() functions sets up the next segment. */
961
962skb_done:
963 conn->rxdata_octets += consumed;
964 return consumed;
965}
966EXPORT_SYMBOL_GPL(iscsi_tcp_recv_skb);
967
968/**
969 * iscsi_tcp_recv - TCP receive in sendfile fashion
970 * @rd_desc: read descriptor 68 * @rd_desc: read descriptor
971 * @skb: socket buffer 69 * @skb: socket buffer
972 * @offset: offset in skb 70 * @offset: offset in skb
973 * @len: skb->len - offset 71 * @len: skb->len - offset
974 **/ 72 */
975static int 73static int iscsi_sw_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
976iscsi_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 74 unsigned int offset, size_t len)
977 unsigned int offset, size_t len)
978{ 75{
979 struct iscsi_conn *conn = rd_desc->arg.data; 76 struct iscsi_conn *conn = rd_desc->arg.data;
980 unsigned int consumed, total_consumed = 0; 77 unsigned int consumed, total_consumed = 0;
@@ -993,8 +90,7 @@ iscsi_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
993 return total_consumed; 90 return total_consumed;
994} 91}
995 92
996static void 93static void iscsi_sw_tcp_data_ready(struct sock *sk, int flag)
997iscsi_tcp_data_ready(struct sock *sk, int flag)
998{ 94{
999 struct iscsi_conn *conn = sk->sk_user_data; 95 struct iscsi_conn *conn = sk->sk_user_data;
1000 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 96 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
@@ -1010,7 +106,7 @@ iscsi_tcp_data_ready(struct sock *sk, int flag)
1010 */ 106 */
1011 rd_desc.arg.data = conn; 107 rd_desc.arg.data = conn;
1012 rd_desc.count = 1; 108 rd_desc.count = 1;
1013 tcp_read_sock(sk, &rd_desc, iscsi_tcp_recv); 109 tcp_read_sock(sk, &rd_desc, iscsi_sw_tcp_recv);
1014 110
1015 read_unlock(&sk->sk_callback_lock); 111 read_unlock(&sk->sk_callback_lock);
1016 112
@@ -1019,10 +115,10 @@ iscsi_tcp_data_ready(struct sock *sk, int flag)
1019 iscsi_tcp_segment_unmap(&tcp_conn->in.segment); 115 iscsi_tcp_segment_unmap(&tcp_conn->in.segment);
1020} 116}
1021 117
1022static void 118static void iscsi_sw_tcp_state_change(struct sock *sk)
1023iscsi_tcp_state_change(struct sock *sk)
1024{ 119{
1025 struct iscsi_tcp_conn *tcp_conn; 120 struct iscsi_tcp_conn *tcp_conn;
121 struct iscsi_sw_tcp_conn *tcp_sw_conn;
1026 struct iscsi_conn *conn; 122 struct iscsi_conn *conn;
1027 struct iscsi_session *session; 123 struct iscsi_session *session;
1028 void (*old_state_change)(struct sock *); 124 void (*old_state_change)(struct sock *);
@@ -1040,7 +136,8 @@ iscsi_tcp_state_change(struct sock *sk)
1040 } 136 }
1041 137
1042 tcp_conn = conn->dd_data; 138 tcp_conn = conn->dd_data;
1043 old_state_change = tcp_conn->old_state_change; 139 tcp_sw_conn = tcp_conn->dd_data;
140 old_state_change = tcp_sw_conn->old_state_change;
1044 141
1045 read_unlock(&sk->sk_callback_lock); 142 read_unlock(&sk->sk_callback_lock);
1046 143
@@ -1051,59 +148,122 @@ iscsi_tcp_state_change(struct sock *sk)
1051 * iscsi_write_space - Called when more output buffer space is available 148 * iscsi_write_space - Called when more output buffer space is available
1052 * @sk: socket space is available for 149 * @sk: socket space is available for
1053 **/ 150 **/
1054static void iscsi_tcp_write_space(struct sock *sk) 151static void iscsi_sw_tcp_write_space(struct sock *sk)
1055{ 152{
1056 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data; 153 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1057 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 154 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
155 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1058 156
1059 tcp_conn->old_write_space(sk); 157 tcp_sw_conn->old_write_space(sk);
1060 debug_tcp("iscsi_write_space: cid %d\n", conn->id); 158 debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1061 scsi_queue_work(conn->session->host, &conn->xmitwork); 159 scsi_queue_work(conn->session->host, &conn->xmitwork);
1062} 160}
1063 161
1064static void iscsi_tcp_conn_set_callbacks(struct iscsi_conn *conn) 162static void iscsi_sw_tcp_conn_set_callbacks(struct iscsi_conn *conn)
1065{ 163{
1066 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 164 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1067 struct sock *sk = tcp_conn->sock->sk; 165 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
166 struct sock *sk = tcp_sw_conn->sock->sk;
1068 167
1069 /* assign new callbacks */ 168 /* assign new callbacks */
1070 write_lock_bh(&sk->sk_callback_lock); 169 write_lock_bh(&sk->sk_callback_lock);
1071 sk->sk_user_data = conn; 170 sk->sk_user_data = conn;
1072 tcp_conn->old_data_ready = sk->sk_data_ready; 171 tcp_sw_conn->old_data_ready = sk->sk_data_ready;
1073 tcp_conn->old_state_change = sk->sk_state_change; 172 tcp_sw_conn->old_state_change = sk->sk_state_change;
1074 tcp_conn->old_write_space = sk->sk_write_space; 173 tcp_sw_conn->old_write_space = sk->sk_write_space;
1075 sk->sk_data_ready = iscsi_tcp_data_ready; 174 sk->sk_data_ready = iscsi_sw_tcp_data_ready;
1076 sk->sk_state_change = iscsi_tcp_state_change; 175 sk->sk_state_change = iscsi_sw_tcp_state_change;
1077 sk->sk_write_space = iscsi_tcp_write_space; 176 sk->sk_write_space = iscsi_sw_tcp_write_space;
1078 write_unlock_bh(&sk->sk_callback_lock); 177 write_unlock_bh(&sk->sk_callback_lock);
1079} 178}
1080 179
1081static void iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn) 180static void
181iscsi_sw_tcp_conn_restore_callbacks(struct iscsi_sw_tcp_conn *tcp_sw_conn)
1082{ 182{
1083 struct sock *sk = tcp_conn->sock->sk; 183 struct sock *sk = tcp_sw_conn->sock->sk;
1084 184
1085 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 185 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1086 write_lock_bh(&sk->sk_callback_lock); 186 write_lock_bh(&sk->sk_callback_lock);
1087 sk->sk_user_data = NULL; 187 sk->sk_user_data = NULL;
1088 sk->sk_data_ready = tcp_conn->old_data_ready; 188 sk->sk_data_ready = tcp_sw_conn->old_data_ready;
1089 sk->sk_state_change = tcp_conn->old_state_change; 189 sk->sk_state_change = tcp_sw_conn->old_state_change;
1090 sk->sk_write_space = tcp_conn->old_write_space; 190 sk->sk_write_space = tcp_sw_conn->old_write_space;
1091 sk->sk_no_check = 0; 191 sk->sk_no_check = 0;
1092 write_unlock_bh(&sk->sk_callback_lock); 192 write_unlock_bh(&sk->sk_callback_lock);
1093} 193}
1094 194
1095/** 195/**
1096 * iscsi_tcp_xmit - TCP transmit 196 * iscsi_sw_tcp_xmit_segment - transmit segment
197 * @tcp_sw_conn: the iSCSI TCP connection
198 * @segment: the buffer to transmnit
199 *
200 * This function transmits as much of the buffer as
201 * the network layer will accept, and returns the number of
202 * bytes transmitted.
203 *
204 * If CRC hashing is enabled, the function will compute the
205 * hash as it goes. When the entire segment has been transmitted,
206 * it will retrieve the hash value and send it as well.
207 */
208static int iscsi_sw_tcp_xmit_segment(struct iscsi_sw_tcp_conn *tcp_sw_conn,
209 struct iscsi_segment *segment)
210{
211 struct socket *sk = tcp_sw_conn->sock;
212 unsigned int copied = 0;
213 int r = 0;
214
215 while (!iscsi_tcp_segment_done(segment, 0, r)) {
216 struct scatterlist *sg;
217 unsigned int offset, copy;
218 int flags = 0;
219
220 r = 0;
221 offset = segment->copied;
222 copy = segment->size - offset;
223
224 if (segment->total_copied + segment->size < segment->total_size)
225 flags |= MSG_MORE;
226
227 /* Use sendpage if we can; else fall back to sendmsg */
228 if (!segment->data) {
229 sg = segment->sg;
230 offset += segment->sg_offset + sg->offset;
231 r = tcp_sw_conn->sendpage(sk, sg_page(sg), offset,
232 copy, flags);
233 } else {
234 struct msghdr msg = { .msg_flags = flags };
235 struct kvec iov = {
236 .iov_base = segment->data + offset,
237 .iov_len = copy
238 };
239
240 r = kernel_sendmsg(sk, &msg, &iov, 1, copy);
241 }
242
243 if (r < 0) {
244 iscsi_tcp_segment_unmap(segment);
245 if (copied || r == -EAGAIN)
246 break;
247 return r;
248 }
249 copied += r;
250 }
251 return copied;
252}
253
254/**
255 * iscsi_sw_tcp_xmit - TCP transmit
1097 **/ 256 **/
1098static int iscsi_tcp_xmit(struct iscsi_conn *conn) 257static int iscsi_sw_tcp_xmit(struct iscsi_conn *conn)
1099{ 258{
1100 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 259 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1101 struct iscsi_segment *segment = &tcp_conn->out.segment; 260 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
261 struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
1102 unsigned int consumed = 0; 262 unsigned int consumed = 0;
1103 int rc = 0; 263 int rc = 0;
1104 264
1105 while (1) { 265 while (1) {
1106 rc = iscsi_tcp_xmit_segment(tcp_conn, segment); 266 rc = iscsi_sw_tcp_xmit_segment(tcp_sw_conn, segment);
1107 if (rc < 0) { 267 if (rc < 0) {
1108 rc = ISCSI_ERR_XMIT_FAILED; 268 rc = ISCSI_ERR_XMIT_FAILED;
1109 goto error; 269 goto error;
@@ -1138,22 +298,22 @@ error:
1138/** 298/**
1139 * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit 299 * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit
1140 */ 300 */
1141static inline int 301static inline int iscsi_sw_tcp_xmit_qlen(struct iscsi_conn *conn)
1142iscsi_tcp_xmit_qlen(struct iscsi_conn *conn)
1143{ 302{
1144 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 303 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1145 struct iscsi_segment *segment = &tcp_conn->out.segment; 304 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
305 struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
1146 306
1147 return segment->total_copied - segment->total_size; 307 return segment->total_copied - segment->total_size;
1148} 308}
1149 309
1150static int iscsi_tcp_flush(struct iscsi_task *task) 310static int iscsi_sw_tcp_pdu_xmit(struct iscsi_task *task)
1151{ 311{
1152 struct iscsi_conn *conn = task->conn; 312 struct iscsi_conn *conn = task->conn;
1153 int rc; 313 int rc;
1154 314
1155 while (iscsi_tcp_xmit_qlen(conn)) { 315 while (iscsi_sw_tcp_xmit_qlen(conn)) {
1156 rc = iscsi_tcp_xmit(conn); 316 rc = iscsi_sw_tcp_xmit(conn);
1157 if (rc == 0) 317 if (rc == 0)
1158 return -EAGAIN; 318 return -EAGAIN;
1159 if (rc < 0) 319 if (rc < 0)
@@ -1167,27 +327,31 @@ static int iscsi_tcp_flush(struct iscsi_task *task)
1167 * This is called when we're done sending the header. 327 * This is called when we're done sending the header.
1168 * Simply copy the data_segment to the send segment, and return. 328 * Simply copy the data_segment to the send segment, and return.
1169 */ 329 */
1170static int 330static int iscsi_sw_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn,
1171iscsi_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn, 331 struct iscsi_segment *segment)
1172 struct iscsi_segment *segment)
1173{ 332{
1174 tcp_conn->out.segment = tcp_conn->out.data_segment; 333 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
334
335 tcp_sw_conn->out.segment = tcp_sw_conn->out.data_segment;
1175 debug_tcp("Header done. Next segment size %u total_size %u\n", 336 debug_tcp("Header done. Next segment size %u total_size %u\n",
1176 tcp_conn->out.segment.size, tcp_conn->out.segment.total_size); 337 tcp_sw_conn->out.segment.size,
338 tcp_sw_conn->out.segment.total_size);
1177 return 0; 339 return 0;
1178} 340}
1179 341
1180static void 342static void iscsi_sw_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr,
1181iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen) 343 size_t hdrlen)
1182{ 344{
1183 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 345 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
346 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1184 347
1185 debug_tcp("%s(%p%s)\n", __func__, tcp_conn, 348 debug_tcp("%s(%p%s)\n", __func__, tcp_conn,
1186 conn->hdrdgst_en? ", digest enabled" : ""); 349 conn->hdrdgst_en? ", digest enabled" : "");
1187 350
1188 /* Clear the data segment - needs to be filled in by the 351 /* Clear the data segment - needs to be filled in by the
1189 * caller using iscsi_tcp_send_data_prep() */ 352 * caller using iscsi_tcp_send_data_prep() */
1190 memset(&tcp_conn->out.data_segment, 0, sizeof(struct iscsi_segment)); 353 memset(&tcp_sw_conn->out.data_segment, 0,
354 sizeof(struct iscsi_segment));
1191 355
1192 /* If header digest is enabled, compute the CRC and 356 /* If header digest is enabled, compute the CRC and
1193 * place the digest into the same buffer. We make 357 * place the digest into the same buffer. We make
@@ -1195,7 +359,7 @@ iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen)
1195 * sufficient room. 359 * sufficient room.
1196 */ 360 */
1197 if (conn->hdrdgst_en) { 361 if (conn->hdrdgst_en) {
1198 iscsi_tcp_dgst_header(&tcp_conn->tx_hash, hdr, hdrlen, 362 iscsi_tcp_dgst_header(&tcp_sw_conn->tx_hash, hdr, hdrlen,
1199 hdr + hdrlen); 363 hdr + hdrlen);
1200 hdrlen += ISCSI_DIGEST_SIZE; 364 hdrlen += ISCSI_DIGEST_SIZE;
1201 } 365 }
@@ -1203,10 +367,10 @@ iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen)
1203 /* Remember header pointer for later, when we need 367 /* Remember header pointer for later, when we need
1204 * to decide whether there's a payload to go along 368 * to decide whether there's a payload to go along
1205 * with the header. */ 369 * with the header. */
1206 tcp_conn->out.hdr = hdr; 370 tcp_sw_conn->out.hdr = hdr;
1207 371
1208 iscsi_segment_init_linear(&tcp_conn->out.segment, hdr, hdrlen, 372 iscsi_segment_init_linear(&tcp_sw_conn->out.segment, hdr, hdrlen,
1209 iscsi_tcp_send_hdr_done, NULL); 373 iscsi_sw_tcp_send_hdr_done, NULL);
1210} 374}
1211 375
1212/* 376/*
@@ -1215,11 +379,12 @@ iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen)
1215 * of by the iscsi_segment routines. 379 * of by the iscsi_segment routines.
1216 */ 380 */
1217static int 381static int
1218iscsi_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg, 382iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
1219 unsigned int count, unsigned int offset, 383 unsigned int count, unsigned int offset,
1220 unsigned int len) 384 unsigned int len)
1221{ 385{
1222 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 386 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
387 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1223 struct hash_desc *tx_hash = NULL; 388 struct hash_desc *tx_hash = NULL;
1224 unsigned int hdr_spec_len; 389 unsigned int hdr_spec_len;
1225 390
@@ -1229,22 +394,23 @@ iscsi_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
1229 394
1230 /* Make sure the datalen matches what the caller 395 /* Make sure the datalen matches what the caller
1231 said he would send. */ 396 said he would send. */
1232 hdr_spec_len = ntoh24(tcp_conn->out.hdr->dlength); 397 hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
1233 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); 398 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
1234 399
1235 if (conn->datadgst_en) 400 if (conn->datadgst_en)
1236 tx_hash = &tcp_conn->tx_hash; 401 tx_hash = &tcp_sw_conn->tx_hash;
1237 402
1238 return iscsi_segment_seek_sg(&tcp_conn->out.data_segment, 403 return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment,
1239 sg, count, offset, len, 404 sg, count, offset, len,
1240 NULL, tx_hash); 405 NULL, tx_hash);
1241} 406}
1242 407
1243static void 408static void
1244iscsi_tcp_send_linear_data_prepare(struct iscsi_conn *conn, void *data, 409iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data,
1245 size_t len) 410 size_t len)
1246{ 411{
1247 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 412 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
413 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1248 struct hash_desc *tx_hash = NULL; 414 struct hash_desc *tx_hash = NULL;
1249 unsigned int hdr_spec_len; 415 unsigned int hdr_spec_len;
1250 416
@@ -1253,34 +419,35 @@ iscsi_tcp_send_linear_data_prepare(struct iscsi_conn *conn, void *data,
1253 419
1254 /* Make sure the datalen matches what the caller 420 /* Make sure the datalen matches what the caller
1255 said he would send. */ 421 said he would send. */
1256 hdr_spec_len = ntoh24(tcp_conn->out.hdr->dlength); 422 hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
1257 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); 423 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
1258 424
1259 if (conn->datadgst_en) 425 if (conn->datadgst_en)
1260 tx_hash = &tcp_conn->tx_hash; 426 tx_hash = &tcp_sw_conn->tx_hash;
1261 427
1262 iscsi_segment_init_linear(&tcp_conn->out.data_segment, 428 iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment,
1263 data, len, NULL, tx_hash); 429 data, len, NULL, tx_hash);
1264} 430}
1265 431
1266static int iscsi_tcp_pdu_init(struct iscsi_task *task, 432static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task,
1267 unsigned int offset, unsigned int count) 433 unsigned int offset, unsigned int count)
1268{ 434{
1269 struct iscsi_conn *conn = task->conn; 435 struct iscsi_conn *conn = task->conn;
1270 int err = 0; 436 int err = 0;
1271 437
1272 iscsi_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len); 438 iscsi_sw_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len);
1273 439
1274 if (!count) 440 if (!count)
1275 return 0; 441 return 0;
1276 442
1277 if (!task->sc) 443 if (!task->sc)
1278 iscsi_tcp_send_linear_data_prepare(conn, task->data, count); 444 iscsi_sw_tcp_send_linear_data_prep(conn, task->data, count);
1279 else { 445 else {
1280 struct scsi_data_buffer *sdb = scsi_out(task->sc); 446 struct scsi_data_buffer *sdb = scsi_out(task->sc);
1281 447
1282 err = iscsi_tcp_send_data_prep(conn, sdb->table.sgl, 448 err = iscsi_sw_tcp_send_data_prep(conn, sdb->table.sgl,
1283 sdb->table.nents, offset, count); 449 sdb->table.nents, offset,
450 count);
1284 } 451 }
1285 452
1286 if (err) { 453 if (err) {
@@ -1290,251 +457,122 @@ static int iscsi_tcp_pdu_init(struct iscsi_task *task,
1290 return 0; 457 return 0;
1291} 458}
1292 459
1293static int iscsi_tcp_pdu_alloc(struct iscsi_task *task) 460static int iscsi_sw_tcp_pdu_alloc(struct iscsi_task *task)
1294{
1295 struct iscsi_tcp_task *tcp_task = task->dd_data;
1296
1297 task->hdr = &tcp_task->hdr.hdrbuf;
1298 task->hdr_max = sizeof(tcp_task->hdr) - ISCSI_DIGEST_SIZE;
1299 return 0;
1300}
1301
1302/**
1303 * iscsi_tcp_task - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1304 * @conn: iscsi connection
1305 * @task: scsi command task
1306 * @sc: scsi command
1307 **/
1308static int iscsi_tcp_task_init(struct iscsi_task *task)
1309{ 461{
1310 struct iscsi_tcp_task *tcp_task = task->dd_data; 462 struct iscsi_tcp_task *tcp_task = task->dd_data;
1311 struct iscsi_conn *conn = task->conn;
1312 struct scsi_cmnd *sc = task->sc;
1313 int err;
1314 463
1315 if (!sc) { 464 task->hdr = task->dd_data + sizeof(*tcp_task);
1316 /* 465 task->hdr_max = sizeof(struct iscsi_sw_tcp_hdrbuf) - ISCSI_DIGEST_SIZE;
1317 * mgmt tasks do not have a scatterlist since they come
1318 * in from the iscsi interface.
1319 */
1320 debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn->id,
1321 task->itt);
1322
1323 return conn->session->tt->init_pdu(task, 0, task->data_count);
1324 }
1325
1326 BUG_ON(__kfifo_len(tcp_task->r2tqueue));
1327 tcp_task->exp_datasn = 0;
1328
1329 /* Prepare PDU, optionally w/ immediate data */
1330 debug_scsi("task deq [cid %d itt 0x%x imm %d unsol %d]\n",
1331 conn->id, task->itt, task->imm_count,
1332 task->unsol_r2t.data_length);
1333
1334 err = conn->session->tt->init_pdu(task, 0, task->imm_count);
1335 if (err)
1336 return err;
1337 task->imm_count = 0;
1338 return 0; 466 return 0;
1339} 467}
1340 468
1341static struct iscsi_r2t_info *iscsi_tcp_get_curr_r2t(struct iscsi_task *task)
1342{
1343 struct iscsi_session *session = task->conn->session;
1344 struct iscsi_tcp_task *tcp_task = task->dd_data;
1345 struct iscsi_r2t_info *r2t = NULL;
1346
1347 if (iscsi_task_has_unsol_data(task))
1348 r2t = &task->unsol_r2t;
1349 else {
1350 spin_lock_bh(&session->lock);
1351 if (tcp_task->r2t) {
1352 r2t = tcp_task->r2t;
1353 /* Continue with this R2T? */
1354 if (r2t->data_length <= r2t->sent) {
1355 debug_scsi(" done with r2t %p\n", r2t);
1356 __kfifo_put(tcp_task->r2tpool.queue,
1357 (void *)&tcp_task->r2t,
1358 sizeof(void *));
1359 tcp_task->r2t = r2t = NULL;
1360 }
1361 }
1362
1363 if (r2t == NULL) {
1364 __kfifo_get(tcp_task->r2tqueue,
1365 (void *)&tcp_task->r2t, sizeof(void *));
1366 r2t = tcp_task->r2t;
1367 }
1368 spin_unlock_bh(&session->lock);
1369 }
1370
1371 return r2t;
1372}
1373
1374/*
1375 * iscsi_tcp_task_xmit - xmit normal PDU task
1376 * @task: iscsi command task
1377 *
1378 * We're expected to return 0 when everything was transmitted succesfully,
1379 * -EAGAIN if there's still data in the queue, or != 0 for any other kind
1380 * of error.
1381 */
1382static int iscsi_tcp_task_xmit(struct iscsi_task *task)
1383{
1384 struct iscsi_conn *conn = task->conn;
1385 struct iscsi_session *session = conn->session;
1386 struct iscsi_r2t_info *r2t;
1387 int rc = 0;
1388
1389flush:
1390 /* Flush any pending data first. */
1391 rc = session->tt->xmit_pdu(task);
1392 if (rc < 0)
1393 return rc;
1394
1395 /* mgmt command */
1396 if (!task->sc) {
1397 if (task->hdr->itt == RESERVED_ITT)
1398 iscsi_put_task(task);
1399 return 0;
1400 }
1401
1402 /* Are we done already? */
1403 if (task->sc->sc_data_direction != DMA_TO_DEVICE)
1404 return 0;
1405
1406 r2t = iscsi_tcp_get_curr_r2t(task);
1407 if (r2t == NULL) {
1408 /* Waiting for more R2Ts to arrive. */
1409 debug_tcp("no R2Ts yet\n");
1410 return 0;
1411 }
1412
1413 rc = conn->session->tt->alloc_pdu(task);
1414 if (rc)
1415 return rc;
1416 iscsi_prep_data_out_pdu(task, r2t, (struct iscsi_data *) task->hdr);
1417
1418 debug_scsi("sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n",
1419 r2t, r2t->datasn - 1, task->hdr->itt,
1420 r2t->data_offset + r2t->sent, r2t->data_count);
1421
1422 rc = conn->session->tt->init_pdu(task, r2t->data_offset + r2t->sent,
1423 r2t->data_count);
1424 if (rc)
1425 return rc;
1426 r2t->sent += r2t->data_count;
1427 goto flush;
1428}
1429
1430static struct iscsi_cls_conn * 469static struct iscsi_cls_conn *
1431iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx) 470iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session,
471 uint32_t conn_idx)
1432{ 472{
1433 struct iscsi_conn *conn; 473 struct iscsi_conn *conn;
1434 struct iscsi_cls_conn *cls_conn; 474 struct iscsi_cls_conn *cls_conn;
1435 struct iscsi_tcp_conn *tcp_conn; 475 struct iscsi_tcp_conn *tcp_conn;
476 struct iscsi_sw_tcp_conn *tcp_sw_conn;
1436 477
1437 cls_conn = iscsi_conn_setup(cls_session, sizeof(*tcp_conn), conn_idx); 478 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn),
479 conn_idx);
1438 if (!cls_conn) 480 if (!cls_conn)
1439 return NULL; 481 return NULL;
1440 conn = cls_conn->dd_data; 482 conn = cls_conn->dd_data;
1441 /*
1442 * due to strange issues with iser these are not set
1443 * in iscsi_conn_setup
1444 */
1445 conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;
1446
1447 tcp_conn = conn->dd_data; 483 tcp_conn = conn->dd_data;
1448 tcp_conn->iscsi_conn = conn; 484 tcp_sw_conn = tcp_conn->dd_data;
1449 485
1450 tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0, 486 tcp_sw_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1451 CRYPTO_ALG_ASYNC); 487 CRYPTO_ALG_ASYNC);
1452 tcp_conn->tx_hash.flags = 0; 488 tcp_sw_conn->tx_hash.flags = 0;
1453 if (IS_ERR(tcp_conn->tx_hash.tfm)) 489 if (IS_ERR(tcp_sw_conn->tx_hash.tfm))
1454 goto free_conn; 490 goto free_conn;
1455 491
1456 tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0, 492 tcp_sw_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1457 CRYPTO_ALG_ASYNC); 493 CRYPTO_ALG_ASYNC);
1458 tcp_conn->rx_hash.flags = 0; 494 tcp_sw_conn->rx_hash.flags = 0;
1459 if (IS_ERR(tcp_conn->rx_hash.tfm)) 495 if (IS_ERR(tcp_sw_conn->rx_hash.tfm))
1460 goto free_tx_tfm; 496 goto free_tx_tfm;
497 tcp_conn->rx_hash = &tcp_sw_conn->rx_hash;
1461 498
1462 return cls_conn; 499 return cls_conn;
1463 500
1464free_tx_tfm: 501free_tx_tfm:
1465 crypto_free_hash(tcp_conn->tx_hash.tfm); 502 crypto_free_hash(tcp_sw_conn->tx_hash.tfm);
1466free_conn: 503free_conn:
1467 iscsi_conn_printk(KERN_ERR, conn, 504 iscsi_conn_printk(KERN_ERR, conn,
1468 "Could not create connection due to crc32c " 505 "Could not create connection due to crc32c "
1469 "loading error. Make sure the crc32c " 506 "loading error. Make sure the crc32c "
1470 "module is built as a module or into the " 507 "module is built as a module or into the "
1471 "kernel\n"); 508 "kernel\n");
1472 iscsi_conn_teardown(cls_conn); 509 iscsi_tcp_conn_teardown(cls_conn);
1473 return NULL; 510 return NULL;
1474} 511}
1475 512
1476static void 513static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn)
1477iscsi_tcp_release_conn(struct iscsi_conn *conn)
1478{ 514{
1479 struct iscsi_session *session = conn->session; 515 struct iscsi_session *session = conn->session;
1480 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 516 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1481 struct socket *sock = tcp_conn->sock; 517 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
518 struct socket *sock = tcp_sw_conn->sock;
1482 519
1483 if (!sock) 520 if (!sock)
1484 return; 521 return;
1485 522
1486 sock_hold(sock->sk); 523 sock_hold(sock->sk);
1487 iscsi_conn_restore_callbacks(tcp_conn); 524 iscsi_sw_tcp_conn_restore_callbacks(tcp_sw_conn);
1488 sock_put(sock->sk); 525 sock_put(sock->sk);
1489 526
1490 spin_lock_bh(&session->lock); 527 spin_lock_bh(&session->lock);
1491 tcp_conn->sock = NULL; 528 tcp_sw_conn->sock = NULL;
1492 spin_unlock_bh(&session->lock); 529 spin_unlock_bh(&session->lock);
1493 sockfd_put(sock); 530 sockfd_put(sock);
1494} 531}
1495 532
1496static void 533static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1497iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1498{ 534{
1499 struct iscsi_conn *conn = cls_conn->dd_data; 535 struct iscsi_conn *conn = cls_conn->dd_data;
1500 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 536 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
537 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1501 538
1502 iscsi_tcp_release_conn(conn); 539 iscsi_sw_tcp_release_conn(conn);
1503 540
1504 if (tcp_conn->tx_hash.tfm) 541 if (tcp_sw_conn->tx_hash.tfm)
1505 crypto_free_hash(tcp_conn->tx_hash.tfm); 542 crypto_free_hash(tcp_sw_conn->tx_hash.tfm);
1506 if (tcp_conn->rx_hash.tfm) 543 if (tcp_sw_conn->rx_hash.tfm)
1507 crypto_free_hash(tcp_conn->rx_hash.tfm); 544 crypto_free_hash(tcp_sw_conn->rx_hash.tfm);
1508 545
1509 iscsi_conn_teardown(cls_conn); 546 iscsi_tcp_conn_teardown(cls_conn);
1510} 547}
1511 548
1512static void 549static void iscsi_sw_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
1513iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
1514{ 550{
1515 struct iscsi_conn *conn = cls_conn->dd_data; 551 struct iscsi_conn *conn = cls_conn->dd_data;
1516 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 552 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
553 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1517 554
1518 /* userspace may have goofed up and not bound us */ 555 /* userspace may have goofed up and not bound us */
1519 if (!tcp_conn->sock) 556 if (!tcp_sw_conn->sock)
1520 return; 557 return;
1521 /* 558 /*
1522 * Make sure our recv side is stopped. 559 * Make sure our recv side is stopped.
1523 * Older tools called conn stop before ep_disconnect 560 * Older tools called conn stop before ep_disconnect
1524 * so IO could still be coming in. 561 * so IO could still be coming in.
1525 */ 562 */
1526 write_lock_bh(&tcp_conn->sock->sk->sk_callback_lock); 563 write_lock_bh(&tcp_sw_conn->sock->sk->sk_callback_lock);
1527 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx); 564 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
1528 write_unlock_bh(&tcp_conn->sock->sk->sk_callback_lock); 565 write_unlock_bh(&tcp_sw_conn->sock->sk->sk_callback_lock);
1529 566
1530 iscsi_conn_stop(cls_conn, flag); 567 iscsi_conn_stop(cls_conn, flag);
1531 iscsi_tcp_release_conn(conn); 568 iscsi_sw_tcp_release_conn(conn);
1532} 569}
1533 570
1534static int iscsi_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock, 571static int iscsi_sw_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock,
1535 char *buf, int *port, 572 char *buf, int *port,
1536 int (*getname)(struct socket *, struct sockaddr *, 573 int (*getname)(struct socket *,
1537 int *addrlen)) 574 struct sockaddr *,
575 int *addrlen))
1538{ 576{
1539 struct sockaddr_storage *addr; 577 struct sockaddr_storage *addr;
1540 struct sockaddr_in6 *sin6; 578 struct sockaddr_in6 *sin6;
@@ -1572,14 +610,15 @@ free_addr:
1572} 610}
1573 611
1574static int 612static int
1575iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session, 613iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1576 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, 614 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
1577 int is_leading) 615 int is_leading)
1578{ 616{
1579 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 617 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
1580 struct iscsi_host *ihost = shost_priv(shost); 618 struct iscsi_host *ihost = shost_priv(shost);
1581 struct iscsi_conn *conn = cls_conn->dd_data; 619 struct iscsi_conn *conn = cls_conn->dd_data;
1582 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 620 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
621 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1583 struct sock *sk; 622 struct sock *sk;
1584 struct socket *sock; 623 struct socket *sock;
1585 int err; 624 int err;
@@ -1596,13 +635,13 @@ iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1596 * userspace may still want to query the values since we will 635 * userspace may still want to query the values since we will
1597 * be using them for the reconnect 636 * be using them for the reconnect
1598 */ 637 */
1599 err = iscsi_tcp_get_addr(conn, sock, conn->portal_address, 638 err = iscsi_sw_tcp_get_addr(conn, sock, conn->portal_address,
1600 &conn->portal_port, kernel_getpeername); 639 &conn->portal_port, kernel_getpeername);
1601 if (err) 640 if (err)
1602 goto free_socket; 641 goto free_socket;
1603 642
1604 err = iscsi_tcp_get_addr(conn, sock, ihost->local_address, 643 err = iscsi_sw_tcp_get_addr(conn, sock, ihost->local_address,
1605 &ihost->local_port, kernel_getsockname); 644 &ihost->local_port, kernel_getsockname);
1606 if (err) 645 if (err)
1607 goto free_socket; 646 goto free_socket;
1608 647
@@ -1611,7 +650,7 @@ iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1611 goto free_socket; 650 goto free_socket;
1612 651
1613 /* bind iSCSI connection and socket */ 652 /* bind iSCSI connection and socket */
1614 tcp_conn->sock = sock; 653 tcp_sw_conn->sock = sock;
1615 654
1616 /* setup Socket parameters */ 655 /* setup Socket parameters */
1617 sk = sock->sk; 656 sk = sock->sk;
@@ -1619,8 +658,8 @@ iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1619 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */ 658 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
1620 sk->sk_allocation = GFP_ATOMIC; 659 sk->sk_allocation = GFP_ATOMIC;
1621 660
1622 iscsi_tcp_conn_set_callbacks(conn); 661 iscsi_sw_tcp_conn_set_callbacks(conn);
1623 tcp_conn->sendpage = tcp_conn->sock->ops->sendpage; 662 tcp_sw_conn->sendpage = tcp_sw_conn->sock->ops->sendpage;
1624 /* 663 /*
1625 * set receive state machine into initial state 664 * set receive state machine into initial state
1626 */ 665 */
@@ -1632,75 +671,14 @@ free_socket:
1632 return err; 671 return err;
1633} 672}
1634 673
1635static int 674static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
1636iscsi_r2tpool_alloc(struct iscsi_session *session) 675 enum iscsi_param param, char *buf,
1637{ 676 int buflen)
1638 int i;
1639 int cmd_i;
1640
1641 /*
1642 * initialize per-task: R2T pool and xmit queue
1643 */
1644 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
1645 struct iscsi_task *task = session->cmds[cmd_i];
1646 struct iscsi_tcp_task *tcp_task = task->dd_data;
1647
1648 /*
1649 * pre-allocated x2 as much r2ts to handle race when
1650 * target acks DataOut faster than we data_xmit() queues
1651 * could replenish r2tqueue.
1652 */
1653
1654 /* R2T pool */
1655 if (iscsi_pool_init(&tcp_task->r2tpool,
1656 session->max_r2t * 2, NULL,
1657 sizeof(struct iscsi_r2t_info))) {
1658 goto r2t_alloc_fail;
1659 }
1660
1661 /* R2T xmit queue */
1662 tcp_task->r2tqueue = kfifo_alloc(
1663 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
1664 if (tcp_task->r2tqueue == ERR_PTR(-ENOMEM)) {
1665 iscsi_pool_free(&tcp_task->r2tpool);
1666 goto r2t_alloc_fail;
1667 }
1668 }
1669
1670 return 0;
1671
1672r2t_alloc_fail:
1673 for (i = 0; i < cmd_i; i++) {
1674 struct iscsi_task *task = session->cmds[i];
1675 struct iscsi_tcp_task *tcp_task = task->dd_data;
1676
1677 kfifo_free(tcp_task->r2tqueue);
1678 iscsi_pool_free(&tcp_task->r2tpool);
1679 }
1680 return -ENOMEM;
1681}
1682
1683static void
1684iscsi_r2tpool_free(struct iscsi_session *session)
1685{
1686 int i;
1687
1688 for (i = 0; i < session->cmds_max; i++) {
1689 struct iscsi_task *task = session->cmds[i];
1690 struct iscsi_tcp_task *tcp_task = task->dd_data;
1691
1692 kfifo_free(tcp_task->r2tqueue);
1693 iscsi_pool_free(&tcp_task->r2tpool);
1694 }
1695}
1696
1697static int
1698iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1699 char *buf, int buflen)
1700{ 677{
1701 struct iscsi_conn *conn = cls_conn->dd_data; 678 struct iscsi_conn *conn = cls_conn->dd_data;
1702 struct iscsi_session *session = conn->session; 679 struct iscsi_session *session = conn->session;
1703 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 680 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
681 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1704 int value; 682 int value;
1705 683
1706 switch(param) { 684 switch(param) {
@@ -1709,8 +687,8 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1709 break; 687 break;
1710 case ISCSI_PARAM_DATADGST_EN: 688 case ISCSI_PARAM_DATADGST_EN:
1711 iscsi_set_param(cls_conn, param, buf, buflen); 689 iscsi_set_param(cls_conn, param, buf, buflen);
1712 tcp_conn->sendpage = conn->datadgst_en ? 690 tcp_sw_conn->sendpage = conn->datadgst_en ?
1713 sock_no_sendpage : tcp_conn->sock->ops->sendpage; 691 sock_no_sendpage : tcp_sw_conn->sock->ops->sendpage;
1714 break; 692 break;
1715 case ISCSI_PARAM_MAX_R2T: 693 case ISCSI_PARAM_MAX_R2T:
1716 sscanf(buf, "%d", &value); 694 sscanf(buf, "%d", &value);
@@ -1718,9 +696,9 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1718 return -EINVAL; 696 return -EINVAL;
1719 if (session->max_r2t == value) 697 if (session->max_r2t == value)
1720 break; 698 break;
1721 iscsi_r2tpool_free(session); 699 iscsi_tcp_r2tpool_free(session);
1722 iscsi_set_param(cls_conn, param, buf, buflen); 700 iscsi_set_param(cls_conn, param, buf, buflen);
1723 if (iscsi_r2tpool_alloc(session)) 701 if (iscsi_tcp_r2tpool_alloc(session))
1724 return -ENOMEM; 702 return -ENOMEM;
1725 break; 703 break;
1726 default: 704 default:
@@ -1730,9 +708,8 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1730 return 0; 708 return 0;
1731} 709}
1732 710
1733static int 711static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
1734iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn, 712 enum iscsi_param param, char *buf)
1735 enum iscsi_param param, char *buf)
1736{ 713{
1737 struct iscsi_conn *conn = cls_conn->dd_data; 714 struct iscsi_conn *conn = cls_conn->dd_data;
1738 int len; 715 int len;
@@ -1756,33 +733,28 @@ iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
1756} 733}
1757 734
1758static void 735static void
1759iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) 736iscsi_sw_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn,
737 struct iscsi_stats *stats)
1760{ 738{
1761 struct iscsi_conn *conn = cls_conn->dd_data; 739 struct iscsi_conn *conn = cls_conn->dd_data;
1762 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 740 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
741 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
1763 742
1764 stats->txdata_octets = conn->txdata_octets;
1765 stats->rxdata_octets = conn->rxdata_octets;
1766 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
1767 stats->dataout_pdus = conn->dataout_pdus_cnt;
1768 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
1769 stats->datain_pdus = conn->datain_pdus_cnt;
1770 stats->r2t_pdus = conn->r2t_pdus_cnt;
1771 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
1772 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
1773 stats->custom_length = 3; 743 stats->custom_length = 3;
1774 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 744 strcpy(stats->custom[0].desc, "tx_sendpage_failures");
1775 stats->custom[0].value = tcp_conn->sendpage_failures_cnt; 745 stats->custom[0].value = tcp_sw_conn->sendpage_failures_cnt;
1776 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 746 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
1777 stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt; 747 stats->custom[1].value = tcp_sw_conn->discontiguous_hdr_cnt;
1778 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 748 strcpy(stats->custom[2].desc, "eh_abort_cnt");
1779 stats->custom[2].value = conn->eh_abort_cnt; 749 stats->custom[2].value = conn->eh_abort_cnt;
750
751 iscsi_tcp_conn_get_stats(cls_conn, stats);
1780} 752}
1781 753
1782static struct iscsi_cls_session * 754static struct iscsi_cls_session *
1783iscsi_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max, 755iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
1784 uint16_t qdepth, uint32_t initial_cmdsn, 756 uint16_t qdepth, uint32_t initial_cmdsn,
1785 uint32_t *hostno) 757 uint32_t *hostno)
1786{ 758{
1787 struct iscsi_cls_session *cls_session; 759 struct iscsi_cls_session *cls_session;
1788 struct iscsi_session *session; 760 struct iscsi_session *session;
@@ -1793,10 +765,10 @@ iscsi_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
1793 return NULL; 765 return NULL;
1794 } 766 }
1795 767
1796 shost = iscsi_host_alloc(&iscsi_sht, 0, qdepth); 768 shost = iscsi_host_alloc(&iscsi_sw_tcp_sht, 0, qdepth);
1797 if (!shost) 769 if (!shost)
1798 return NULL; 770 return NULL;
1799 shost->transportt = iscsi_tcp_scsi_transport; 771 shost->transportt = iscsi_sw_tcp_scsi_transport;
1800 shost->max_lun = iscsi_max_lun; 772 shost->max_lun = iscsi_max_lun;
1801 shost->max_id = 0; 773 shost->max_id = 0;
1802 shost->max_channel = 0; 774 shost->max_channel = 0;
@@ -1806,15 +778,17 @@ iscsi_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
1806 goto free_host; 778 goto free_host;
1807 *hostno = shost->host_no; 779 *hostno = shost->host_no;
1808 780
1809 cls_session = iscsi_session_setup(&iscsi_tcp_transport, shost, cmds_max, 781 cls_session = iscsi_session_setup(&iscsi_sw_tcp_transport, shost,
1810 sizeof(struct iscsi_tcp_task), 782 cmds_max,
783 sizeof(struct iscsi_tcp_task) +
784 sizeof(struct iscsi_sw_tcp_hdrbuf),
1811 initial_cmdsn, 0); 785 initial_cmdsn, 0);
1812 if (!cls_session) 786 if (!cls_session)
1813 goto remove_host; 787 goto remove_host;
1814 session = cls_session->dd_data; 788 session = cls_session->dd_data;
1815 789
1816 shost->can_queue = session->scsi_cmds_max; 790 shost->can_queue = session->scsi_cmds_max;
1817 if (iscsi_r2tpool_alloc(session)) 791 if (iscsi_tcp_r2tpool_alloc(session))
1818 goto remove_session; 792 goto remove_session;
1819 return cls_session; 793 return cls_session;
1820 794
@@ -1827,25 +801,25 @@ free_host:
1827 return NULL; 801 return NULL;
1828} 802}
1829 803
1830static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session) 804static void iscsi_sw_tcp_session_destroy(struct iscsi_cls_session *cls_session)
1831{ 805{
1832 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 806 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
1833 807
1834 iscsi_r2tpool_free(cls_session->dd_data); 808 iscsi_tcp_r2tpool_free(cls_session->dd_data);
1835 iscsi_session_teardown(cls_session); 809 iscsi_session_teardown(cls_session);
1836 810
1837 iscsi_host_remove(shost); 811 iscsi_host_remove(shost);
1838 iscsi_host_free(shost); 812 iscsi_host_free(shost);
1839} 813}
1840 814
1841static int iscsi_tcp_slave_configure(struct scsi_device *sdev) 815static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
1842{ 816{
1843 blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY); 817 blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
1844 blk_queue_dma_alignment(sdev->request_queue, 0); 818 blk_queue_dma_alignment(sdev->request_queue, 0);
1845 return 0; 819 return 0;
1846} 820}
1847 821
1848static struct scsi_host_template iscsi_sht = { 822static struct scsi_host_template iscsi_sw_tcp_sht = {
1849 .module = THIS_MODULE, 823 .module = THIS_MODULE,
1850 .name = "iSCSI Initiator over TCP/IP", 824 .name = "iSCSI Initiator over TCP/IP",
1851 .queuecommand = iscsi_queuecommand, 825 .queuecommand = iscsi_queuecommand,
@@ -1858,12 +832,12 @@ static struct scsi_host_template iscsi_sht = {
1858 .eh_device_reset_handler= iscsi_eh_device_reset, 832 .eh_device_reset_handler= iscsi_eh_device_reset,
1859 .eh_target_reset_handler= iscsi_eh_target_reset, 833 .eh_target_reset_handler= iscsi_eh_target_reset,
1860 .use_clustering = DISABLE_CLUSTERING, 834 .use_clustering = DISABLE_CLUSTERING,
1861 .slave_configure = iscsi_tcp_slave_configure, 835 .slave_configure = iscsi_sw_tcp_slave_configure,
1862 .proc_name = "iscsi_tcp", 836 .proc_name = "iscsi_tcp",
1863 .this_id = -1, 837 .this_id = -1,
1864}; 838};
1865 839
1866static struct iscsi_transport iscsi_tcp_transport = { 840static struct iscsi_transport iscsi_sw_tcp_transport = {
1867 .owner = THIS_MODULE, 841 .owner = THIS_MODULE,
1868 .name = "tcp", 842 .name = "tcp",
1869 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 843 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
@@ -1896,37 +870,36 @@ static struct iscsi_transport iscsi_tcp_transport = {
1896 ISCSI_HOST_INITIATOR_NAME | 870 ISCSI_HOST_INITIATOR_NAME |
1897 ISCSI_HOST_NETDEV_NAME, 871 ISCSI_HOST_NETDEV_NAME,
1898 /* session management */ 872 /* session management */
1899 .create_session = iscsi_tcp_session_create, 873 .create_session = iscsi_sw_tcp_session_create,
1900 .destroy_session = iscsi_tcp_session_destroy, 874 .destroy_session = iscsi_sw_tcp_session_destroy,
1901 /* connection management */ 875 /* connection management */
1902 .create_conn = iscsi_tcp_conn_create, 876 .create_conn = iscsi_sw_tcp_conn_create,
1903 .bind_conn = iscsi_tcp_conn_bind, 877 .bind_conn = iscsi_sw_tcp_conn_bind,
1904 .destroy_conn = iscsi_tcp_conn_destroy, 878 .destroy_conn = iscsi_sw_tcp_conn_destroy,
1905 .set_param = iscsi_conn_set_param, 879 .set_param = iscsi_sw_tcp_conn_set_param,
1906 .get_conn_param = iscsi_tcp_conn_get_param, 880 .get_conn_param = iscsi_sw_tcp_conn_get_param,
1907 .get_session_param = iscsi_session_get_param, 881 .get_session_param = iscsi_session_get_param,
1908 .start_conn = iscsi_conn_start, 882 .start_conn = iscsi_conn_start,
1909 .stop_conn = iscsi_tcp_conn_stop, 883 .stop_conn = iscsi_sw_tcp_conn_stop,
1910 /* iscsi host params */ 884 /* iscsi host params */
1911 .get_host_param = iscsi_host_get_param, 885 .get_host_param = iscsi_host_get_param,
1912 .set_host_param = iscsi_host_set_param, 886 .set_host_param = iscsi_host_set_param,
1913 /* IO */ 887 /* IO */
1914 .send_pdu = iscsi_conn_send_pdu, 888 .send_pdu = iscsi_conn_send_pdu,
1915 .get_stats = iscsi_conn_get_stats, 889 .get_stats = iscsi_sw_tcp_conn_get_stats,
1916 /* iscsi task/cmd helpers */ 890 /* iscsi task/cmd helpers */
1917 .init_task = iscsi_tcp_task_init, 891 .init_task = iscsi_tcp_task_init,
1918 .xmit_task = iscsi_tcp_task_xmit, 892 .xmit_task = iscsi_tcp_task_xmit,
1919 .cleanup_task = iscsi_tcp_cleanup_task, 893 .cleanup_task = iscsi_tcp_cleanup_task,
1920 /* low level pdu helpers */ 894 /* low level pdu helpers */
1921 .xmit_pdu = iscsi_tcp_flush, 895 .xmit_pdu = iscsi_sw_tcp_pdu_xmit,
1922 .init_pdu = iscsi_tcp_pdu_init, 896 .init_pdu = iscsi_sw_tcp_pdu_init,
1923 .alloc_pdu = iscsi_tcp_pdu_alloc, 897 .alloc_pdu = iscsi_sw_tcp_pdu_alloc,
1924 /* recovery */ 898 /* recovery */
1925 .session_recovery_timedout = iscsi_session_recovery_timedout, 899 .session_recovery_timedout = iscsi_session_recovery_timedout,
1926}; 900};
1927 901
1928static int __init 902static int __init iscsi_sw_tcp_init(void)
1929iscsi_tcp_init(void)
1930{ 903{
1931 if (iscsi_max_lun < 1) { 904 if (iscsi_max_lun < 1) {
1932 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n", 905 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
@@ -1934,19 +907,18 @@ iscsi_tcp_init(void)
1934 return -EINVAL; 907 return -EINVAL;
1935 } 908 }
1936 909
1937 iscsi_tcp_scsi_transport = iscsi_register_transport( 910 iscsi_sw_tcp_scsi_transport = iscsi_register_transport(
1938 &iscsi_tcp_transport); 911 &iscsi_sw_tcp_transport);
1939 if (!iscsi_tcp_scsi_transport) 912 if (!iscsi_sw_tcp_scsi_transport)
1940 return -ENODEV; 913 return -ENODEV;
1941 914
1942 return 0; 915 return 0;
1943} 916}
1944 917
1945static void __exit 918static void __exit iscsi_sw_tcp_exit(void)
1946iscsi_tcp_exit(void)
1947{ 919{
1948 iscsi_unregister_transport(&iscsi_tcp_transport); 920 iscsi_unregister_transport(&iscsi_sw_tcp_transport);
1949} 921}
1950 922
1951module_init(iscsi_tcp_init); 923module_init(iscsi_sw_tcp_init);
1952module_exit(iscsi_tcp_exit); 924module_exit(iscsi_sw_tcp_exit);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-12 15:00:27 -0500