aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi')
-rw-r--r--drivers/scsi/iscsi_tcp.c2837
-rw-r--r--drivers/scsi/iscsi_tcp.h182
2 files changed, 813 insertions, 2206 deletions
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index aca3e23b1b9e..fe00a3f6d204 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -3,7 +3,8 @@
3 * 3 *
4 * Copyright (C) 2004 Dmitry Yusupov 4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman 5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 Mike Christie 6 * Copyright (C) 2005 - 2006 Mike Christie
7 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
7 * maintained by open-iscsi@googlegroups.com 8 * maintained by open-iscsi@googlegroups.com
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
@@ -36,10 +37,6 @@
36#include <linux/mutex.h> 37#include <linux/mutex.h>
37#include <net/tcp.h> 38#include <net/tcp.h>
38#include <scsi/scsi_cmnd.h> 39#include <scsi/scsi_cmnd.h>
39#include <scsi/scsi_device.h>
40#include <scsi/scsi_eh.h>
41#include <scsi/scsi_request.h>
42#include <scsi/scsi_tcq.h>
43#include <scsi/scsi_host.h> 40#include <scsi/scsi_host.h>
44#include <scsi/scsi.h> 41#include <scsi/scsi.h>
45#include <scsi/scsi_transport_iscsi.h> 42#include <scsi/scsi_transport_iscsi.h>
@@ -52,21 +49,14 @@ MODULE_DESCRIPTION("iSCSI/TCP data-path");
52MODULE_LICENSE("GPL"); 49MODULE_LICENSE("GPL");
53MODULE_VERSION("0:4.445"); 50MODULE_VERSION("0:4.445");
54/* #define DEBUG_TCP */ 51/* #define DEBUG_TCP */
55/* #define DEBUG_SCSI */
56#define DEBUG_ASSERT 52#define DEBUG_ASSERT
57 53
58#ifdef DEBUG_TCP 54#ifdef DEBUG_TCP
59#define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt) 55#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
60#else 56#else
61#define debug_tcp(fmt...) 57#define debug_tcp(fmt...)
62#endif 58#endif
63 59
64#ifdef DEBUG_SCSI
65#define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
66#else
67#define debug_scsi(fmt...)
68#endif
69
70#ifndef DEBUG_ASSERT 60#ifndef DEBUG_ASSERT
71#ifdef BUG_ON 61#ifdef BUG_ON
72#undef BUG_ON 62#undef BUG_ON
@@ -74,17 +64,12 @@ MODULE_VERSION("0:4.445");
74#define BUG_ON(expr) 64#define BUG_ON(expr)
75#endif 65#endif
76 66
77#define INVALID_SN_DELTA 0xffff
78
79static unsigned int iscsi_max_lun = 512; 67static unsigned int iscsi_max_lun = 512;
80module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 68module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
81 69
82/* global data */ 70/* global data */
83static kmem_cache_t *taskcache; 71static kmem_cache_t *taskcache;
84 72
85#define session_to_cls(_sess) \
86 hostdata_session(_sess->host->hostdata)
87
88static inline void 73static inline void
89iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size) 74iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
90{ 75{
@@ -133,68 +118,39 @@ static inline void
133iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf, 118iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
134 u8* crc) 119 u8* crc)
135{ 120{
136 crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc); 121 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
137 buf->sg.length += sizeof(uint32_t);
138}
139 122
140static void 123 crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc);
141iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err) 124 buf->sg.length += sizeof(uint32_t);
142{
143 struct iscsi_session *session = conn->session;
144 unsigned long flags;
145
146 spin_lock_irqsave(&session->lock, flags);
147 if (session->conn_cnt == 1 || session->leadconn == conn)
148 session->state = ISCSI_STATE_FAILED;
149 spin_unlock_irqrestore(&session->lock, flags);
150 set_bit(SUSPEND_BIT, &conn->suspend_tx);
151 set_bit(SUSPEND_BIT, &conn->suspend_rx);
152 iscsi_conn_error(conn->cls_conn, err);
153} 125}
154 126
155static inline int 127static inline int
156iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr) 128iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
157{ 129{
158 uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn); 130 struct sk_buff *skb = tcp_conn->in.skb;
159 uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
160
161 if (max_cmdsn < exp_cmdsn -1 &&
162 max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
163 return ISCSI_ERR_MAX_CMDSN;
164 if (max_cmdsn > session->max_cmdsn ||
165 max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
166 session->max_cmdsn = max_cmdsn;
167 if (exp_cmdsn > session->exp_cmdsn ||
168 exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
169 session->exp_cmdsn = exp_cmdsn;
170 131
171 return 0; 132 tcp_conn->in.zero_copy_hdr = 0;
172}
173
174static inline int
175iscsi_hdr_extract(struct iscsi_conn *conn)
176{
177 struct sk_buff *skb = conn->in.skb;
178 133
179 if (conn->in.copy >= conn->hdr_size && 134 if (tcp_conn->in.copy >= tcp_conn->hdr_size &&
180 conn->in_progress == IN_PROGRESS_WAIT_HEADER) { 135 tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
181 /* 136 /*
182 * Zero-copy PDU Header: using connection context 137 * Zero-copy PDU Header: using connection context
183 * to store header pointer. 138 * to store header pointer.
184 */ 139 */
185 if (skb_shinfo(skb)->frag_list == NULL && 140 if (skb_shinfo(skb)->frag_list == NULL &&
186 !skb_shinfo(skb)->nr_frags) 141 !skb_shinfo(skb)->nr_frags) {
187 conn->in.hdr = (struct iscsi_hdr *) 142 tcp_conn->in.hdr = (struct iscsi_hdr *)
188 ((char*)skb->data + conn->in.offset); 143 ((char*)skb->data + tcp_conn->in.offset);
189 else { 144 tcp_conn->in.zero_copy_hdr = 1;
145 } else {
190 /* ignoring return code since we checked 146 /* ignoring return code since we checked
191 * in.copy before */ 147 * in.copy before */
192 skb_copy_bits(skb, conn->in.offset, 148 skb_copy_bits(skb, tcp_conn->in.offset,
193 &conn->hdr, conn->hdr_size); 149 &tcp_conn->hdr, tcp_conn->hdr_size);
194 conn->in.hdr = &conn->hdr; 150 tcp_conn->in.hdr = &tcp_conn->hdr;
195 } 151 }
196 conn->in.offset += conn->hdr_size; 152 tcp_conn->in.offset += tcp_conn->hdr_size;
197 conn->in.copy -= conn->hdr_size; 153 tcp_conn->in.copy -= tcp_conn->hdr_size;
198 } else { 154 } else {
199 int hdr_remains; 155 int hdr_remains;
200 int copylen; 156 int copylen;
@@ -204,30 +160,31 @@ iscsi_hdr_extract(struct iscsi_conn *conn)
204 * copying it... This'll happen quite rarely. 160 * copying it... This'll happen quite rarely.
205 */ 161 */
206 162
207 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER) 163 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER)
208 conn->in.hdr_offset = 0; 164 tcp_conn->in.hdr_offset = 0;
209 165
210 hdr_remains = conn->hdr_size - conn->in.hdr_offset; 166 hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset;
211 BUG_ON(hdr_remains <= 0); 167 BUG_ON(hdr_remains <= 0);
212 168
213 copylen = min(conn->in.copy, hdr_remains); 169 copylen = min(tcp_conn->in.copy, hdr_remains);
214 skb_copy_bits(skb, conn->in.offset, 170 skb_copy_bits(skb, tcp_conn->in.offset,
215 (char*)&conn->hdr + conn->in.hdr_offset, copylen); 171 (char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset,
172 copylen);
216 173
217 debug_tcp("PDU gather offset %d bytes %d in.offset %d " 174 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
218 "in.copy %d\n", conn->in.hdr_offset, copylen, 175 "in.copy %d\n", tcp_conn->in.hdr_offset, copylen,
219 conn->in.offset, conn->in.copy); 176 tcp_conn->in.offset, tcp_conn->in.copy);
220 177
221 conn->in.offset += copylen; 178 tcp_conn->in.offset += copylen;
222 conn->in.copy -= copylen; 179 tcp_conn->in.copy -= copylen;
223 if (copylen < hdr_remains) { 180 if (copylen < hdr_remains) {
224 conn->in_progress = IN_PROGRESS_HEADER_GATHER; 181 tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER;
225 conn->in.hdr_offset += copylen; 182 tcp_conn->in.hdr_offset += copylen;
226 return -EAGAIN; 183 return -EAGAIN;
227 } 184 }
228 conn->in.hdr = &conn->hdr; 185 tcp_conn->in.hdr = &tcp_conn->hdr;
229 conn->discontiguous_hdr_cnt++; 186 tcp_conn->discontiguous_hdr_cnt++;
230 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 187 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
231 } 188 }
232 189
233 return 0; 190 return 0;
@@ -239,7 +196,7 @@ iscsi_hdr_extract(struct iscsi_conn *conn)
239static void 196static void
240__iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 197__iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
241{ 198{
242 struct iscsi_session *session = conn->session; 199 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
243 struct scsi_cmnd *sc; 200 struct scsi_cmnd *sc;
244 201
245 sc = ctask->sc; 202 sc = ctask->sc;
@@ -248,88 +205,16 @@ __iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
248 205
249 if (sc->sc_data_direction == DMA_TO_DEVICE) { 206 if (sc->sc_data_direction == DMA_TO_DEVICE) {
250 struct iscsi_data_task *dtask, *n; 207 struct iscsi_data_task *dtask, *n;
208
251 /* WRITE: cleanup Data-Out's if any */ 209 /* WRITE: cleanup Data-Out's if any */
252 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) { 210 list_for_each_entry_safe(dtask, n, &tcp_ctask->dataqueue,
211 item) {
253 list_del(&dtask->item); 212 list_del(&dtask->item);
254 mempool_free(dtask, ctask->datapool); 213 mempool_free(dtask, tcp_ctask->datapool);
255 } 214 }
256 } 215 }
257 ctask->xmstate = XMSTATE_IDLE; 216 tcp_ctask->xmstate = XMSTATE_IDLE;
258 ctask->r2t = NULL; 217 tcp_ctask->r2t = NULL;
259 ctask->sc = NULL;
260 list_del(&ctask->running);
261
262 __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
263
264 sc->scsi_done(sc);
265}
266
267static void
268iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
269{
270 struct iscsi_session *session = conn->session;
271
272 spin_lock(&session->lock);
273 __iscsi_ctask_cleanup(conn, ctask);
274 spin_unlock(&session->lock);
275}
276
277/**
278 * iscsi_cmd_rsp - SCSI Command Response processing
279 * @conn: iscsi connection
280 * @ctask: scsi command task
281 **/
282static int
283iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
284{
285 int rc;
286 struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
287 struct iscsi_session *session = conn->session;
288 struct scsi_cmnd *sc = ctask->sc;
289
290 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
291 if (rc) {
292 sc->result = (DID_ERROR << 16);
293 goto out;
294 }
295
296 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
297
298 sc->result = (DID_OK << 16) | rhdr->cmd_status;
299
300 if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
301 sc->result = (DID_ERROR << 16);
302 goto out;
303 }
304
305 if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
306 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
307
308 memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
309 debug_scsi("copied %d bytes of sense\n", sensecopy);
310 }
311
312 if (sc->sc_data_direction == DMA_TO_DEVICE)
313 goto out;
314
315 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
316 int res_count = be32_to_cpu(rhdr->residual_count);
317
318 if (res_count > 0 && res_count <= sc->request_bufflen)
319 sc->resid = res_count;
320 else
321 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
322 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
323 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
324 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
325 sc->resid = be32_to_cpu(rhdr->residual_count);
326
327out:
328 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
329 (long)sc, sc->result, ctask->itt);
330 conn->scsirsp_pdus_cnt++;
331 iscsi_ctask_cleanup(conn, ctask);
332 return rc;
333} 218}
334 219
335/** 220/**
@@ -341,7 +226,9 @@ static int
341iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 226iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
342{ 227{
343 int rc; 228 int rc;
344 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr; 229 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
230 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
231 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
345 struct iscsi_session *session = conn->session; 232 struct iscsi_session *session = conn->session;
346 int datasn = be32_to_cpu(rhdr->datasn); 233 int datasn = be32_to_cpu(rhdr->datasn);
347 234
@@ -351,9 +238,9 @@ iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
351 /* 238 /*
352 * setup Data-In byte counter (gets decremented..) 239 * setup Data-In byte counter (gets decremented..)
353 */ 240 */
354 ctask->data_count = conn->in.datalen; 241 ctask->data_count = tcp_conn->in.datalen;
355 242
356 if (conn->in.datalen == 0) 243 if (tcp_conn->in.datalen == 0)
357 return 0; 244 return 0;
358 245
359 if (ctask->datasn != datasn) 246 if (ctask->datasn != datasn)
@@ -361,8 +248,8 @@ iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
361 248
362 ctask->datasn++; 249 ctask->datasn++;
363 250
364 ctask->data_offset = be32_to_cpu(rhdr->offset); 251 tcp_ctask->data_offset = be32_to_cpu(rhdr->offset);
365 if (ctask->data_offset + conn->in.datalen > ctask->total_length) 252 if (tcp_ctask->data_offset + tcp_conn->in.datalen > ctask->total_length)
366 return ISCSI_ERR_DATA_OFFSET; 253 return ISCSI_ERR_DATA_OFFSET;
367 254
368 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) { 255 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
@@ -409,8 +296,9 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
409 struct iscsi_data *hdr; 296 struct iscsi_data *hdr;
410 struct iscsi_data_task *dtask; 297 struct iscsi_data_task *dtask;
411 struct scsi_cmnd *sc = ctask->sc; 298 struct scsi_cmnd *sc = ctask->sc;
299 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
412 300
413 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 301 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
414 BUG_ON(!dtask); 302 BUG_ON(!dtask);
415 INIT_LIST_HEAD(&dtask->item); 303 INIT_LIST_HEAD(&dtask->item);
416 hdr = &dtask->hdr; 304 hdr = &dtask->hdr;
@@ -419,8 +307,8 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
419 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 307 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
420 r2t->solicit_datasn++; 308 r2t->solicit_datasn++;
421 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 309 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
422 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 310 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
423 hdr->itt = ctask->hdr.itt; 311 hdr->itt = ctask->hdr->itt;
424 hdr->exp_statsn = r2t->exp_statsn; 312 hdr->exp_statsn = r2t->exp_statsn;
425 hdr->offset = cpu_to_be32(r2t->data_offset); 313 hdr->offset = cpu_to_be32(r2t->data_offset);
426 if (r2t->data_length > conn->max_xmit_dlength) { 314 if (r2t->data_length > conn->max_xmit_dlength) {
@@ -469,11 +357,11 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
469 } 357 }
470 BUG_ON(r2t->sg == NULL); 358 BUG_ON(r2t->sg == NULL);
471 } else 359 } else
472 iscsi_buf_init_iov(&ctask->sendbuf, 360 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
473 (char*)sc->request_buffer + r2t->data_offset, 361 (char*)sc->request_buffer + r2t->data_offset,
474 r2t->data_count); 362 r2t->data_count);
475 363
476 list_add(&dtask->item, &ctask->dataqueue); 364 list_add(&dtask->item, &tcp_ctask->dataqueue);
477} 365}
478 366
479/** 367/**
@@ -486,17 +374,16 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
486{ 374{
487 struct iscsi_r2t_info *r2t; 375 struct iscsi_r2t_info *r2t;
488 struct iscsi_session *session = conn->session; 376 struct iscsi_session *session = conn->session;
489 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr; 377 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
378 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
379 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
490 int r2tsn = be32_to_cpu(rhdr->r2tsn); 380 int r2tsn = be32_to_cpu(rhdr->r2tsn);
491 int rc; 381 int rc;
492 382
493 if (conn->in.ahslen) 383 if (tcp_conn->in.datalen)
494 return ISCSI_ERR_AHSLEN;
495
496 if (conn->in.datalen)
497 return ISCSI_ERR_DATALEN; 384 return ISCSI_ERR_DATALEN;
498 385
499 if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn) 386 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
500 return ISCSI_ERR_R2TSN; 387 return ISCSI_ERR_R2TSN;
501 388
502 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 389 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
@@ -514,7 +401,7 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
514 spin_unlock(&session->lock); 401 spin_unlock(&session->lock);
515 return 0; 402 return 0;
516 } 403 }
517 rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 404 rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
518 BUG_ON(!rc); 405 BUG_ON(!rc);
519 406
520 r2t->exp_statsn = rhdr->statsn; 407 r2t->exp_statsn = rhdr->statsn;
@@ -536,10 +423,10 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
536 423
537 iscsi_solicit_data_init(conn, ctask, r2t); 424 iscsi_solicit_data_init(conn, ctask, r2t);
538 425
539 ctask->exp_r2tsn = r2tsn + 1; 426 tcp_ctask->exp_r2tsn = r2tsn + 1;
540 ctask->xmstate |= XMSTATE_SOL_HDR; 427 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
541 __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*)); 428 __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
542 __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*)); 429 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
543 430
544 scsi_queue_work(session->host, &conn->xmitwork); 431 scsi_queue_work(session->host, &conn->xmitwork);
545 conn->r2t_pdus_cnt++; 432 conn->r2t_pdus_cnt++;
@@ -549,256 +436,136 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
549} 436}
550 437
551static int 438static int
552iscsi_hdr_recv(struct iscsi_conn *conn) 439iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
553{ 440{
554 int rc = 0; 441 int rc = 0, opcode, ahslen;
555 struct iscsi_hdr *hdr; 442 struct iscsi_hdr *hdr;
556 struct iscsi_cmd_task *ctask;
557 struct iscsi_session *session = conn->session; 443 struct iscsi_session *session = conn->session;
558 uint32_t cdgst, rdgst = 0; 444 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
445 uint32_t cdgst, rdgst = 0, itt;
559 446
560 hdr = conn->in.hdr; 447 hdr = tcp_conn->in.hdr;
561 448
562 /* verify PDU length */ 449 /* verify PDU length */
563 conn->in.datalen = ntoh24(hdr->dlength); 450 tcp_conn->in.datalen = ntoh24(hdr->dlength);
564 if (conn->in.datalen > conn->max_recv_dlength) { 451 if (tcp_conn->in.datalen > conn->max_recv_dlength) {
565 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n", 452 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
566 conn->in.datalen, conn->max_recv_dlength); 453 tcp_conn->in.datalen, conn->max_recv_dlength);
567 return ISCSI_ERR_DATALEN; 454 return ISCSI_ERR_DATALEN;
568 } 455 }
569 conn->data_copied = 0; 456 tcp_conn->data_copied = 0;
570 457
571 /* read AHS */ 458 /* read AHS */
572 conn->in.ahslen = hdr->hlength * 4; 459 ahslen = hdr->hlength << 2;
573 conn->in.offset += conn->in.ahslen; 460 tcp_conn->in.offset += ahslen;
574 conn->in.copy -= conn->in.ahslen; 461 tcp_conn->in.copy -= ahslen;
575 if (conn->in.copy < 0) { 462 if (tcp_conn->in.copy < 0) {
576 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length " 463 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
577 "%d bytes\n", conn->in.ahslen); 464 "%d bytes\n", ahslen);
578 return ISCSI_ERR_AHSLEN; 465 return ISCSI_ERR_AHSLEN;
579 } 466 }
580 467
581 /* calculate read padding */ 468 /* calculate read padding */
582 conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1); 469 tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1);
583 if (conn->in.padding) { 470 if (tcp_conn->in.padding) {
584 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding; 471 tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding;
585 debug_scsi("read padding %d bytes\n", conn->in.padding); 472 debug_scsi("read padding %d bytes\n", tcp_conn->in.padding);
586 } 473 }
587 474
588 if (conn->hdrdgst_en) { 475 if (conn->hdrdgst_en) {
589 struct scatterlist sg; 476 struct scatterlist sg;
590 477
591 sg_init_one(&sg, (u8 *)hdr, 478 sg_init_one(&sg, (u8 *)hdr,
592 sizeof(struct iscsi_hdr) + conn->in.ahslen); 479 sizeof(struct iscsi_hdr) + ahslen);
593 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst); 480 crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
594 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) + 481 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
595 conn->in.ahslen); 482 ahslen);
596 if (cdgst != rdgst) { 483 if (cdgst != rdgst) {
597 printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error " 484 printk(KERN_ERR "iscsi_tcp: hdrdgst error "
598 "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst, 485 "recv 0x%x calc 0x%x\n", rdgst, cdgst);
599 cdgst);
600 return ISCSI_ERR_HDR_DGST; 486 return ISCSI_ERR_HDR_DGST;
601 } 487 }
602 } 488 }
603 489
604 /* save opcode for later */ 490 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
605 conn->in.opcode = hdr->opcode & ISCSI_OPCODE_MASK;
606
607 /* verify itt (itt encoding: age+cid+itt) */ 491 /* verify itt (itt encoding: age+cid+itt) */
608 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) { 492 rc = iscsi_verify_itt(conn, hdr, &itt);
609 if ((hdr->itt & AGE_MASK) != 493 if (rc == ISCSI_ERR_NO_SCSI_CMD) {
610 (session->age << AGE_SHIFT)) { 494 tcp_conn->in.datalen = 0; /* force drop */
611 printk(KERN_ERR "iscsi_tcp: received itt %x expected " 495 return 0;
612 "session age (%x)\n", hdr->itt, 496 } else if (rc)
613 session->age & AGE_MASK); 497 return rc;
614 return ISCSI_ERR_BAD_ITT;
615 }
616
617 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
618 printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
619 "CID (%x)\n", hdr->itt, conn->id);
620 return ISCSI_ERR_BAD_ITT;
621 }
622 conn->in.itt = hdr->itt & ITT_MASK;
623 } else
624 conn->in.itt = hdr->itt;
625 498
626 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n", 499 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
627 hdr->opcode, conn->in.offset, conn->in.copy, 500 opcode, tcp_conn->in.offset, tcp_conn->in.copy,
628 conn->in.ahslen, conn->in.datalen); 501 ahslen, tcp_conn->in.datalen);
629
630 if (conn->in.itt < session->cmds_max) {
631 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
632
633 if (!ctask->sc) {
634 printk(KERN_INFO "iscsi_tcp: dropping ctask with "
635 "itt 0x%x\n", ctask->itt);
636 conn->in.datalen = 0; /* force drop */
637 return 0;
638 }
639
640 if (ctask->sc->SCp.phase != session->age) {
641 printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
642 "expected %d\n", ctask->sc->SCp.phase,
643 session->age);
644 return ISCSI_ERR_SESSION_FAILED;
645 }
646
647 conn->in.ctask = ctask;
648
649 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
650 hdr->opcode, conn->id, (long)ctask->sc,
651 ctask->itt, conn->in.datalen);
652
653 switch(conn->in.opcode) {
654 case ISCSI_OP_SCSI_CMD_RSP:
655 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
656 if (!conn->in.datalen)
657 rc = iscsi_cmd_rsp(conn, ctask);
658 else
659 /*
660 * got sense or response data; copying PDU
661 * Header to the connection's header
662 * placeholder
663 */
664 memcpy(&conn->hdr, hdr,
665 sizeof(struct iscsi_hdr));
666 break;
667 case ISCSI_OP_SCSI_DATA_IN:
668 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
669 /* save flags for non-exceptional status */
670 conn->in.flags = hdr->flags;
671 /* save cmd_status for sense data */
672 conn->in.cmd_status =
673 ((struct iscsi_data_rsp*)hdr)->cmd_status;
674 rc = iscsi_data_rsp(conn, ctask);
675 break;
676 case ISCSI_OP_R2T:
677 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
678 if (ctask->sc->sc_data_direction == DMA_TO_DEVICE)
679 rc = iscsi_r2t_rsp(conn, ctask);
680 else
681 rc = ISCSI_ERR_PROTO;
682 break;
683 default:
684 rc = ISCSI_ERR_BAD_OPCODE;
685 break;
686 }
687 } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
688 conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
689 session->mgmtpool_max) {
690 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
691 session->mgmt_cmds[conn->in.itt -
692 ISCSI_MGMT_ITT_OFFSET];
693
694 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
695 conn->in.opcode, conn->id, mtask->itt,
696 conn->in.datalen);
697
698 switch(conn->in.opcode) {
699 case ISCSI_OP_LOGIN_RSP:
700 case ISCSI_OP_TEXT_RSP:
701 case ISCSI_OP_LOGOUT_RSP:
702 rc = iscsi_check_assign_cmdsn(session,
703 (struct iscsi_nopin*)hdr);
704 if (rc)
705 break;
706 502
707 if (!conn->in.datalen) { 503 switch(opcode) {
708 rc = iscsi_recv_pdu(conn->cls_conn, hdr, 504 case ISCSI_OP_SCSI_DATA_IN:
709 NULL, 0); 505 tcp_conn->in.ctask = session->cmds[itt];
710 if (conn->login_mtask != mtask) { 506 rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
711 spin_lock(&session->lock); 507 /* fall through */
712 __kfifo_put(session->mgmtpool.queue, 508 case ISCSI_OP_SCSI_CMD_RSP:
713 (void*)&mtask, sizeof(void*)); 509 tcp_conn->in.ctask = session->cmds[itt];
714 spin_unlock(&session->lock); 510 if (tcp_conn->in.datalen)
715 } 511 goto copy_hdr;
716 }
717 break;
718 case ISCSI_OP_SCSI_TMFUNC_RSP:
719 rc = iscsi_check_assign_cmdsn(session,
720 (struct iscsi_nopin*)hdr);
721 if (rc)
722 break;
723 512
724 if (conn->in.datalen || conn->in.ahslen) { 513 spin_lock(&session->lock);
725 rc = ISCSI_ERR_PROTO; 514 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
726 break; 515 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
727 } 516 spin_unlock(&session->lock);
728 conn->tmfrsp_pdus_cnt++; 517 break;
729 spin_lock(&session->lock); 518 case ISCSI_OP_R2T:
730 if (conn->tmabort_state == TMABORT_INITIAL) { 519 tcp_conn->in.ctask = session->cmds[itt];
731 conn->tmabort_state = 520 if (ahslen)
732 ((struct iscsi_tm_rsp *)hdr)-> 521 rc = ISCSI_ERR_AHSLEN;
733 response == ISCSI_TMF_RSP_COMPLETE ? 522 else if (tcp_conn->in.ctask->sc->sc_data_direction ==
734 TMABORT_SUCCESS:TMABORT_FAILED; 523 DMA_TO_DEVICE)
735 /* unblock eh_abort() */ 524 rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask);
736 wake_up(&conn->ehwait); 525 else
737 } 526 rc = ISCSI_ERR_PROTO;
738 spin_unlock(&session->lock); 527 break;
739 break; 528 case ISCSI_OP_LOGIN_RSP:
740 case ISCSI_OP_NOOP_IN: 529 case ISCSI_OP_TEXT_RSP:
741 if (hdr->ttt != ISCSI_RESERVED_TAG) { 530 case ISCSI_OP_LOGOUT_RSP:
742 rc = ISCSI_ERR_PROTO; 531 case ISCSI_OP_NOOP_IN:
743 break; 532 case ISCSI_OP_REJECT:
744 } 533 case ISCSI_OP_ASYNC_EVENT:
745 rc = iscsi_check_assign_cmdsn(session, 534 if (tcp_conn->in.datalen)
746 (struct iscsi_nopin*)hdr); 535 goto copy_hdr;
747 if (rc) 536 /* fall through */
748 break; 537 case ISCSI_OP_SCSI_TMFUNC_RSP:
749 conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1; 538 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
750 539 break;
751 if (!conn->in.datalen) { 540 default:
752 struct iscsi_mgmt_task *mtask; 541 rc = ISCSI_ERR_BAD_OPCODE;
753 542 break;
754 rc = iscsi_recv_pdu(conn->cls_conn, hdr, 543 }
755 NULL, 0);
756 mtask = (struct iscsi_mgmt_task *)
757 session->mgmt_cmds[conn->in.itt -
758 ISCSI_MGMT_ITT_OFFSET];
759 if (conn->login_mtask != mtask) {
760 spin_lock(&session->lock);
761 __kfifo_put(session->mgmtpool.queue,
762 (void*)&mtask, sizeof(void*));
763 spin_unlock(&session->lock);
764 }
765 }
766 break;
767 default:
768 rc = ISCSI_ERR_BAD_OPCODE;
769 break;
770 }
771 } else if (conn->in.itt == ISCSI_RESERVED_TAG) {
772 switch(conn->in.opcode) {
773 case ISCSI_OP_NOOP_IN:
774 if (!conn->in.datalen) {
775 rc = iscsi_check_assign_cmdsn(session,
776 (struct iscsi_nopin*)hdr);
777 if (!rc && hdr->ttt != ISCSI_RESERVED_TAG)
778 rc = iscsi_recv_pdu(conn->cls_conn,
779 hdr, NULL, 0);
780 } else
781 rc = ISCSI_ERR_PROTO;
782 break;
783 case ISCSI_OP_REJECT:
784 /* we need sth like iscsi_reject_rsp()*/
785 case ISCSI_OP_ASYNC_EVENT:
786 /* we need sth like iscsi_async_event_rsp() */
787 rc = ISCSI_ERR_BAD_OPCODE;
788 break;
789 default:
790 rc = ISCSI_ERR_BAD_OPCODE;
791 break;
792 }
793 } else
794 rc = ISCSI_ERR_BAD_ITT;
795 544
796 return rc; 545 return rc;
546
547copy_hdr:
548 /*
549 * if we did zero copy for the header but we will need multiple
550 * skbs to complete the command then we have to copy the header
551 * for later use
552 */
553 if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <
554 (tcp_conn->in.datalen + tcp_conn->in.padding +
555 (conn->datadgst_en ? 4 : 0))) {
556 debug_tcp("Copying header for later use. in.copy %d in.datalen"
557 " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen);
558 memcpy(&tcp_conn->hdr, tcp_conn->in.hdr,
559 sizeof(struct iscsi_hdr));
560 tcp_conn->in.hdr = &tcp_conn->hdr;
561 tcp_conn->in.zero_copy_hdr = 0;
562 }
563 return 0;
797} 564}
798 565
799/** 566/**
800 * iscsi_ctask_copy - copy skb bits to the destanation cmd task 567 * iscsi_ctask_copy - copy skb bits to the destanation cmd task
801 * @conn: iscsi connection 568 * @conn: iscsi tcp connection
802 * @ctask: scsi command task 569 * @ctask: scsi command task
803 * @buf: buffer to copy to 570 * @buf: buffer to copy to
804 * @buf_size: size of buffer 571 * @buf_size: size of buffer
@@ -820,110 +587,113 @@ iscsi_hdr_recv(struct iscsi_conn *conn)
820 * buf_left left to copy from in progress buffer 587 * buf_left left to copy from in progress buffer
821 **/ 588 **/
822static inline int 589static inline int
823iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 590iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
824 void *buf, int buf_size, int offset) 591 void *buf, int buf_size, int offset)
825{ 592{
826 int buf_left = buf_size - (conn->data_copied + offset); 593 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
827 int size = min(conn->in.copy, buf_left); 594 int buf_left = buf_size - (tcp_conn->data_copied + offset);
595 int size = min(tcp_conn->in.copy, buf_left);
828 int rc; 596 int rc;
829 597
830 size = min(size, ctask->data_count); 598 size = min(size, ctask->data_count);
831 599
832 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n", 600 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
833 size, conn->in.offset, conn->in.copied); 601 size, tcp_conn->in.offset, tcp_conn->in.copied);
834 602
835 BUG_ON(size <= 0); 603 BUG_ON(size <= 0);
836 BUG_ON(ctask->sent + size > ctask->total_length); 604 BUG_ON(tcp_ctask->sent + size > ctask->total_length);
837 605
838 rc = skb_copy_bits(conn->in.skb, conn->in.offset, 606 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
839 (char*)buf + (offset + conn->data_copied), size); 607 (char*)buf + (offset + tcp_conn->data_copied), size);
840 /* must fit into skb->len */ 608 /* must fit into skb->len */
841 BUG_ON(rc); 609 BUG_ON(rc);
842 610
843 conn->in.offset += size; 611 tcp_conn->in.offset += size;
844 conn->in.copy -= size; 612 tcp_conn->in.copy -= size;
845 conn->in.copied += size; 613 tcp_conn->in.copied += size;
846 conn->data_copied += size; 614 tcp_conn->data_copied += size;
847 ctask->sent += size; 615 tcp_ctask->sent += size;
848 ctask->data_count -= size; 616 ctask->data_count -= size;
849 617
850 BUG_ON(conn->in.copy < 0); 618 BUG_ON(tcp_conn->in.copy < 0);
851 BUG_ON(ctask->data_count < 0); 619 BUG_ON(ctask->data_count < 0);
852 620
853 if (buf_size != (conn->data_copied + offset)) { 621 if (buf_size != (tcp_conn->data_copied + offset)) {
854 if (!ctask->data_count) { 622 if (!ctask->data_count) {
855 BUG_ON(buf_size - conn->data_copied < 0); 623 BUG_ON(buf_size - tcp_conn->data_copied < 0);
856 /* done with this PDU */ 624 /* done with this PDU */
857 return buf_size - conn->data_copied; 625 return buf_size - tcp_conn->data_copied;
858 } 626 }
859 return -EAGAIN; 627 return -EAGAIN;
860 } 628 }
861 629
862 /* done with this buffer or with both - PDU and buffer */ 630 /* done with this buffer or with both - PDU and buffer */
863 conn->data_copied = 0; 631 tcp_conn->data_copied = 0;
864 return 0; 632 return 0;
865} 633}
866 634
867/** 635/**
868 * iscsi_tcp_copy - copy skb bits to the destanation buffer 636 * iscsi_tcp_copy - copy skb bits to the destanation buffer
869 * @conn: iscsi connection 637 * @conn: iscsi tcp connection
870 * @buf: buffer to copy to
871 * @buf_size: number of bytes to copy
872 * 638 *
873 * Notes: 639 * Notes:
874 * The function calls skb_copy_bits() and updates per-connection 640 * The function calls skb_copy_bits() and updates per-connection
875 * byte counters. 641 * byte counters.
876 **/ 642 **/
877static inline int 643static inline int
878iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size) 644iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn)
879{ 645{
880 int buf_left = buf_size - conn->data_copied; 646 void *buf = tcp_conn->data;
881 int size = min(conn->in.copy, buf_left); 647 int buf_size = tcp_conn->in.datalen;
648 int buf_left = buf_size - tcp_conn->data_copied;
649 int size = min(tcp_conn->in.copy, buf_left);
882 int rc; 650 int rc;
883 651
884 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n", 652 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
885 size, conn->in.offset, conn->data_copied); 653 size, tcp_conn->in.offset, tcp_conn->data_copied);
886 BUG_ON(size <= 0); 654 BUG_ON(size <= 0);
887 655
888 rc = skb_copy_bits(conn->in.skb, conn->in.offset, 656 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
889 (char*)buf + conn->data_copied, size); 657 (char*)buf + tcp_conn->data_copied, size);
890 BUG_ON(rc); 658 BUG_ON(rc);
891 659
892 conn->in.offset += size; 660 tcp_conn->in.offset += size;
893 conn->in.copy -= size; 661 tcp_conn->in.copy -= size;
894 conn->in.copied += size; 662 tcp_conn->in.copied += size;
895 conn->data_copied += size; 663 tcp_conn->data_copied += size;
896 664
897 if (buf_size != conn->data_copied) 665 if (buf_size != tcp_conn->data_copied)
898 return -EAGAIN; 666 return -EAGAIN;
899 667
900 return 0; 668 return 0;
901} 669}
902 670
903static inline void 671static inline void
904partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg, 672partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn,
905 int offset, int length) 673 struct scatterlist *sg, int offset, int length)
906{ 674{
907 struct scatterlist temp; 675 struct scatterlist temp;
908 676
909 memcpy(&temp, sg, sizeof(struct scatterlist)); 677 memcpy(&temp, sg, sizeof(struct scatterlist));
910 temp.offset = offset; 678 temp.offset = offset;
911 temp.length = length; 679 temp.length = length;
912 crypto_digest_update(conn->data_rx_tfm, &temp, 1); 680 crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1);
913} 681}
914 682
915static void 683static void
916iscsi_recv_digest_update(struct iscsi_conn *conn, char* buf, int len) 684iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
917{ 685{
918 struct scatterlist tmp; 686 struct scatterlist tmp;
919 687
920 sg_init_one(&tmp, buf, len); 688 sg_init_one(&tmp, buf, len);
921 crypto_digest_update(conn->data_rx_tfm, &tmp, 1); 689 crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1);
922} 690}
923 691
924static int iscsi_scsi_data_in(struct iscsi_conn *conn) 692static int iscsi_scsi_data_in(struct iscsi_conn *conn)
925{ 693{
926 struct iscsi_cmd_task *ctask = conn->in.ctask; 694 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
695 struct iscsi_cmd_task *ctask = tcp_conn->in.ctask;
696 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
927 struct scsi_cmnd *sc = ctask->sc; 697 struct scsi_cmnd *sc = ctask->sc;
928 struct scatterlist *sg; 698 struct scatterlist *sg;
929 int i, offset, rc = 0; 699 int i, offset, rc = 0;
@@ -935,31 +705,33 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
935 */ 705 */
936 if (!sc->use_sg) { 706 if (!sc->use_sg) {
937 i = ctask->data_count; 707 i = ctask->data_count;
938 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer, 708 rc = iscsi_ctask_copy(tcp_conn, ctask, sc->request_buffer,
939 sc->request_bufflen, ctask->data_offset); 709 sc->request_bufflen,
710 tcp_ctask->data_offset);
940 if (rc == -EAGAIN) 711 if (rc == -EAGAIN)
941 return rc; 712 return rc;
942 if (conn->datadgst_en) 713 if (conn->datadgst_en)
943 iscsi_recv_digest_update(conn, sc->request_buffer, i); 714 iscsi_recv_digest_update(tcp_conn, sc->request_buffer,
715 i);
944 rc = 0; 716 rc = 0;
945 goto done; 717 goto done;
946 } 718 }
947 719
948 offset = ctask->data_offset; 720 offset = tcp_ctask->data_offset;
949 sg = sc->request_buffer; 721 sg = sc->request_buffer;
950 722
951 if (ctask->data_offset) 723 if (tcp_ctask->data_offset)
952 for (i = 0; i < ctask->sg_count; i++) 724 for (i = 0; i < tcp_ctask->sg_count; i++)
953 offset -= sg[i].length; 725 offset -= sg[i].length;
954 /* we've passed through partial sg*/ 726 /* we've passed through partial sg*/
955 if (offset < 0) 727 if (offset < 0)
956 offset = 0; 728 offset = 0;
957 729
958 for (i = ctask->sg_count; i < sc->use_sg; i++) { 730 for (i = tcp_ctask->sg_count; i < sc->use_sg; i++) {
959 char *dest; 731 char *dest;
960 732
961 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0); 733 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
962 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset, 734 rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset,
963 sg[i].length, offset); 735 sg[i].length, offset);
964 kunmap_atomic(dest, KM_SOFTIRQ0); 736 kunmap_atomic(dest, KM_SOFTIRQ0);
965 if (rc == -EAGAIN) 737 if (rc == -EAGAIN)
@@ -968,15 +740,17 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
968 if (!rc) { 740 if (!rc) {
969 if (conn->datadgst_en) { 741 if (conn->datadgst_en) {
970 if (!offset) 742 if (!offset)
971 crypto_digest_update(conn->data_rx_tfm, 743 crypto_digest_update(
972 &sg[i], 1); 744 tcp_conn->data_rx_tfm,
745 &sg[i], 1);
973 else 746 else
974 partial_sg_digest_update(conn, &sg[i], 747 partial_sg_digest_update(tcp_conn,
748 &sg[i],
975 sg[i].offset + offset, 749 sg[i].offset + offset,
976 sg[i].length - offset); 750 sg[i].length - offset);
977 } 751 }
978 offset = 0; 752 offset = 0;
979 ctask->sg_count++; 753 tcp_ctask->sg_count++;
980 } 754 }
981 755
982 if (!ctask->data_count) { 756 if (!ctask->data_count) {
@@ -984,24 +758,26 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
984 /* 758 /*
985 * data-in is complete, but buffer not... 759 * data-in is complete, but buffer not...
986 */ 760 */
987 partial_sg_digest_update(conn, &sg[i], 761 partial_sg_digest_update(tcp_conn, &sg[i],
988 sg[i].offset, sg[i].length-rc); 762 sg[i].offset, sg[i].length-rc);
989 rc = 0; 763 rc = 0;
990 break; 764 break;
991 } 765 }
992 766
993 if (!conn->in.copy) 767 if (!tcp_conn->in.copy)
994 return -EAGAIN; 768 return -EAGAIN;
995 } 769 }
996 BUG_ON(ctask->data_count); 770 BUG_ON(ctask->data_count);
997 771
998done: 772done:
999 /* check for non-exceptional status */ 773 /* check for non-exceptional status */
1000 if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) { 774 if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
1001 debug_scsi("done [sc %lx res %d itt 0x%x]\n", 775 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
1002 (long)sc, sc->result, ctask->itt); 776 (long)sc, sc->result, ctask->itt);
1003 conn->scsirsp_pdus_cnt++; 777 spin_lock(&conn->session->lock);
1004 iscsi_ctask_cleanup(conn, ctask); 778 __iscsi_ctask_cleanup(conn, ctask);
779 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
780 spin_unlock(&conn->session->lock);
1005 } 781 }
1006 782
1007 return rc; 783 return rc;
@@ -1010,71 +786,38 @@ done:
1010static int 786static int
1011iscsi_data_recv(struct iscsi_conn *conn) 787iscsi_data_recv(struct iscsi_conn *conn)
1012{ 788{
1013 struct iscsi_session *session = conn->session; 789 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1014 int rc = 0; 790 int rc = 0, opcode;
1015 791
1016 switch(conn->in.opcode) { 792 opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
793 switch (opcode) {
1017 case ISCSI_OP_SCSI_DATA_IN: 794 case ISCSI_OP_SCSI_DATA_IN:
1018 rc = iscsi_scsi_data_in(conn); 795 rc = iscsi_scsi_data_in(conn);
1019 break; 796 break;
1020 case ISCSI_OP_SCSI_CMD_RSP: { 797 case ISCSI_OP_SCSI_CMD_RSP:
1021 /* 798 spin_lock(&conn->session->lock);
1022 * SCSI Sense Data: 799 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
1023 * copying the entire Data Segment. 800 spin_unlock(&conn->session->lock);
1024 */
1025 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1026 rc = -EAGAIN;
1027 goto exit;
1028 }
1029
1030 /*
1031 * check for sense
1032 */
1033 conn->in.hdr = &conn->hdr;
1034 conn->senselen = (conn->data[0] << 8) | conn->data[1];
1035 rc = iscsi_cmd_rsp(conn, conn->in.ctask);
1036 if (!rc && conn->datadgst_en)
1037 iscsi_recv_digest_update(conn, conn->data,
1038 conn->in.datalen);
1039 }
1040 break;
1041 case ISCSI_OP_TEXT_RSP: 801 case ISCSI_OP_TEXT_RSP:
1042 case ISCSI_OP_LOGIN_RSP: 802 case ISCSI_OP_LOGIN_RSP:
1043 case ISCSI_OP_NOOP_IN: { 803 case ISCSI_OP_NOOP_IN:
1044 struct iscsi_mgmt_task *mtask = NULL; 804 case ISCSI_OP_ASYNC_EVENT:
1045 805 case ISCSI_OP_REJECT:
1046 if (conn->in.itt != ISCSI_RESERVED_TAG)
1047 mtask = (struct iscsi_mgmt_task *)
1048 session->mgmt_cmds[conn->in.itt -
1049 ISCSI_MGMT_ITT_OFFSET];
1050
1051 /* 806 /*
1052 * Collect data segment to the connection's data 807 * Collect data segment to the connection's data
1053 * placeholder 808 * placeholder
1054 */ 809 */
1055 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) { 810 if (iscsi_tcp_copy(tcp_conn)) {
1056 rc = -EAGAIN; 811 rc = -EAGAIN;
1057 goto exit; 812 goto exit;
1058 } 813 }
1059 814
1060 rc = iscsi_recv_pdu(conn->cls_conn, conn->in.hdr, 815 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, tcp_conn->data,
1061 conn->data, conn->in.datalen); 816 tcp_conn->in.datalen);
1062 817 if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP)
1063 if (!rc && conn->datadgst_en && 818 iscsi_recv_digest_update(tcp_conn, tcp_conn->data,
1064 conn->in.opcode != ISCSI_OP_LOGIN_RSP) 819 tcp_conn->in.datalen);
1065 iscsi_recv_digest_update(conn, conn->data, 820 break;
1066 conn->in.datalen);
1067
1068 if (mtask && conn->login_mtask != mtask) {
1069 spin_lock(&session->lock);
1070 __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
1071 sizeof(void*));
1072 spin_unlock(&session->lock);
1073 }
1074 }
1075 break;
1076 case ISCSI_OP_ASYNC_EVENT:
1077 case ISCSI_OP_REJECT:
1078 default: 821 default:
1079 BUG_ON(1); 822 BUG_ON(1);
1080 } 823 }
@@ -1095,6 +838,7 @@ iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1095{ 838{
1096 int rc; 839 int rc;
1097 struct iscsi_conn *conn = rd_desc->arg.data; 840 struct iscsi_conn *conn = rd_desc->arg.data;
841 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1098 int processed; 842 int processed;
1099 char pad[ISCSI_PAD_LEN]; 843 char pad[ISCSI_PAD_LEN];
1100 struct scatterlist sg; 844 struct scatterlist sg;
@@ -1103,15 +847,15 @@ iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1103 * Save current SKB and its offset in the corresponding 847 * Save current SKB and its offset in the corresponding
1104 * connection context. 848 * connection context.
1105 */ 849 */
1106 conn->in.copy = skb->len - offset; 850 tcp_conn->in.copy = skb->len - offset;
1107 conn->in.offset = offset; 851 tcp_conn->in.offset = offset;
1108 conn->in.skb = skb; 852 tcp_conn->in.skb = skb;
1109 conn->in.len = conn->in.copy; 853 tcp_conn->in.len = tcp_conn->in.copy;
1110 BUG_ON(conn->in.copy <= 0); 854 BUG_ON(tcp_conn->in.copy <= 0);
1111 debug_tcp("in %d bytes\n", conn->in.copy); 855 debug_tcp("in %d bytes\n", tcp_conn->in.copy);
1112 856
1113more: 857more:
1114 conn->in.copied = 0; 858 tcp_conn->in.copied = 0;
1115 rc = 0; 859 rc = 0;
1116 860
1117 if (unlikely(conn->suspend_rx)) { 861 if (unlikely(conn->suspend_rx)) {
@@ -1119,9 +863,9 @@ more:
1119 return 0; 863 return 0;
1120 } 864 }
1121 865
1122 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER || 866 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
1123 conn->in_progress == IN_PROGRESS_HEADER_GATHER) { 867 tcp_conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
1124 rc = iscsi_hdr_extract(conn); 868 rc = iscsi_hdr_extract(tcp_conn);
1125 if (rc) { 869 if (rc) {
1126 if (rc == -EAGAIN) 870 if (rc == -EAGAIN)
1127 goto nomore; 871 goto nomore;
@@ -1134,90 +878,94 @@ more:
1134 /* 878 /*
1135 * Verify and process incoming PDU header. 879 * Verify and process incoming PDU header.
1136 */ 880 */
1137 rc = iscsi_hdr_recv(conn); 881 rc = iscsi_tcp_hdr_recv(conn);
1138 if (!rc && conn->in.datalen) { 882 if (!rc && tcp_conn->in.datalen) {
1139 if (conn->datadgst_en) { 883 if (conn->datadgst_en) {
1140 BUG_ON(!conn->data_rx_tfm); 884 BUG_ON(!tcp_conn->data_rx_tfm);
1141 crypto_digest_init(conn->data_rx_tfm); 885 crypto_digest_init(tcp_conn->data_rx_tfm);
1142 } 886 }
1143 conn->in_progress = IN_PROGRESS_DATA_RECV; 887 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
1144 } else if (rc) { 888 } else if (rc) {
1145 iscsi_conn_failure(conn, rc); 889 iscsi_conn_failure(conn, rc);
1146 return 0; 890 return 0;
1147 } 891 }
1148 } 892 }
1149 893
1150 if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) { 894 if (tcp_conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
1151 uint32_t recv_digest; 895 uint32_t recv_digest;
896
1152 debug_tcp("extra data_recv offset %d copy %d\n", 897 debug_tcp("extra data_recv offset %d copy %d\n",
1153 conn->in.offset, conn->in.copy); 898 tcp_conn->in.offset, tcp_conn->in.copy);
1154 skb_copy_bits(conn->in.skb, conn->in.offset, 899 skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
1155 &recv_digest, 4); 900 &recv_digest, 4);
1156 conn->in.offset += 4; 901 tcp_conn->in.offset += 4;
1157 conn->in.copy -= 4; 902 tcp_conn->in.copy -= 4;
1158 if (recv_digest != conn->in.datadgst) { 903 if (recv_digest != tcp_conn->in.datadgst) {
1159 debug_tcp("iscsi_tcp: data digest error!" 904 debug_tcp("iscsi_tcp: data digest error!"
1160 "0x%x != 0x%x\n", recv_digest, 905 "0x%x != 0x%x\n", recv_digest,
1161 conn->in.datadgst); 906 tcp_conn->in.datadgst);
1162 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 907 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1163 return 0; 908 return 0;
1164 } else { 909 } else {
1165 debug_tcp("iscsi_tcp: data digest match!" 910 debug_tcp("iscsi_tcp: data digest match!"
1166 "0x%x == 0x%x\n", recv_digest, 911 "0x%x == 0x%x\n", recv_digest,
1167 conn->in.datadgst); 912 tcp_conn->in.datadgst);
1168 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 913 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1169 } 914 }
1170 } 915 }
1171 916
1172 if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) { 917 if (tcp_conn->in_progress == IN_PROGRESS_DATA_RECV &&
918 tcp_conn->in.copy) {
1173 919
1174 debug_tcp("data_recv offset %d copy %d\n", 920 debug_tcp("data_recv offset %d copy %d\n",
1175 conn->in.offset, conn->in.copy); 921 tcp_conn->in.offset, tcp_conn->in.copy);
1176 922
1177 rc = iscsi_data_recv(conn); 923 rc = iscsi_data_recv(conn);
1178 if (rc) { 924 if (rc) {
1179 if (rc == -EAGAIN) { 925 if (rc == -EAGAIN) {
1180 rd_desc->count = conn->in.datalen - 926 rd_desc->count = tcp_conn->in.datalen -
1181 conn->in.ctask->data_count; 927 tcp_conn->in.ctask->data_count;
1182 goto again; 928 goto again;
1183 } 929 }
1184 iscsi_conn_failure(conn, rc); 930 iscsi_conn_failure(conn, rc);
1185 return 0; 931 return 0;
1186 } 932 }
1187 conn->in.copy -= conn->in.padding; 933 tcp_conn->in.copy -= tcp_conn->in.padding;
1188 conn->in.offset += conn->in.padding; 934 tcp_conn->in.offset += tcp_conn->in.padding;
1189 if (conn->datadgst_en) { 935 if (conn->datadgst_en) {
1190 if (conn->in.padding) { 936 if (tcp_conn->in.padding) {
1191 debug_tcp("padding -> %d\n", conn->in.padding); 937 debug_tcp("padding -> %d\n",
1192 memset(pad, 0, conn->in.padding); 938 tcp_conn->in.padding);
1193 sg_init_one(&sg, pad, conn->in.padding); 939 memset(pad, 0, tcp_conn->in.padding);
1194 crypto_digest_update(conn->data_rx_tfm, &sg, 1); 940 sg_init_one(&sg, pad, tcp_conn->in.padding);
941 crypto_digest_update(tcp_conn->data_rx_tfm,
942 &sg, 1);
1195 } 943 }
1196 crypto_digest_final(conn->data_rx_tfm, 944 crypto_digest_final(tcp_conn->data_rx_tfm,
1197 (u8 *) & conn->in.datadgst); 945 (u8 *) & tcp_conn->in.datadgst);
1198 debug_tcp("rx digest 0x%x\n", conn->in.datadgst); 946 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
1199 conn->in_progress = IN_PROGRESS_DDIGEST_RECV; 947 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
1200 } else 948 } else
1201 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 949 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1202 } 950 }
1203 951
1204 debug_tcp("f, processed %d from out of %d padding %d\n", 952 debug_tcp("f, processed %d from out of %d padding %d\n",
1205 conn->in.offset - offset, (int)len, conn->in.padding); 953 tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding);
1206 BUG_ON(conn->in.offset - offset > len); 954 BUG_ON(tcp_conn->in.offset - offset > len);
1207 955
1208 if (conn->in.offset - offset != len) { 956 if (tcp_conn->in.offset - offset != len) {
1209 debug_tcp("continue to process %d bytes\n", 957 debug_tcp("continue to process %d bytes\n",
1210 (int)len - (conn->in.offset - offset)); 958 (int)len - (tcp_conn->in.offset - offset));
1211 goto more; 959 goto more;
1212 } 960 }
1213 961
1214nomore: 962nomore:
1215 processed = conn->in.offset - offset; 963 processed = tcp_conn->in.offset - offset;
1216 BUG_ON(processed == 0); 964 BUG_ON(processed == 0);
1217 return processed; 965 return processed;
1218 966
1219again: 967again:
1220 processed = conn->in.offset - offset; 968 processed = tcp_conn->in.offset - offset;
1221 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n", 969 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
1222 processed, (int)len, (int)rd_desc->count); 970 processed, (int)len, (int)rd_desc->count);
1223 BUG_ON(processed == 0); 971 BUG_ON(processed == 0);
@@ -1246,6 +994,7 @@ iscsi_tcp_data_ready(struct sock *sk, int flag)
1246static void 994static void
1247iscsi_tcp_state_change(struct sock *sk) 995iscsi_tcp_state_change(struct sock *sk)
1248{ 996{
997 struct iscsi_tcp_conn *tcp_conn;
1249 struct iscsi_conn *conn; 998 struct iscsi_conn *conn;
1250 struct iscsi_session *session; 999 struct iscsi_session *session;
1251 void (*old_state_change)(struct sock *); 1000 void (*old_state_change)(struct sock *);
@@ -1262,7 +1011,8 @@ iscsi_tcp_state_change(struct sock *sk)
1262 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1011 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1263 } 1012 }
1264 1013
1265 old_state_change = conn->old_state_change; 1014 tcp_conn = conn->dd_data;
1015 old_state_change = tcp_conn->old_state_change;
1266 1016
1267 read_unlock(&sk->sk_callback_lock); 1017 read_unlock(&sk->sk_callback_lock);
1268 1018
@@ -1277,23 +1027,26 @@ static void
1277iscsi_write_space(struct sock *sk) 1027iscsi_write_space(struct sock *sk)
1278{ 1028{
1279 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data; 1029 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1280 conn->old_write_space(sk); 1030 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1031
1032 tcp_conn->old_write_space(sk);
1281 debug_tcp("iscsi_write_space: cid %d\n", conn->id); 1033 debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1282 clear_bit(SUSPEND_BIT, &conn->suspend_tx); 1034 clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1283 scsi_queue_work(conn->session->host, &conn->xmitwork); 1035 scsi_queue_work(conn->session->host, &conn->xmitwork);
1284} 1036}
1285 1037
1286static void 1038static void
1287iscsi_conn_set_callbacks(struct iscsi_conn *conn) 1039iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1288{ 1040{
1289 struct sock *sk = conn->sock->sk; 1041 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1042 struct sock *sk = tcp_conn->sock->sk;
1290 1043
1291 /* assign new callbacks */ 1044 /* assign new callbacks */
1292 write_lock_bh(&sk->sk_callback_lock); 1045 write_lock_bh(&sk->sk_callback_lock);
1293 sk->sk_user_data = conn; 1046 sk->sk_user_data = conn;
1294 conn->old_data_ready = sk->sk_data_ready; 1047 tcp_conn->old_data_ready = sk->sk_data_ready;
1295 conn->old_state_change = sk->sk_state_change; 1048 tcp_conn->old_state_change = sk->sk_state_change;
1296 conn->old_write_space = sk->sk_write_space; 1049 tcp_conn->old_write_space = sk->sk_write_space;
1297 sk->sk_data_ready = iscsi_tcp_data_ready; 1050 sk->sk_data_ready = iscsi_tcp_data_ready;
1298 sk->sk_state_change = iscsi_tcp_state_change; 1051 sk->sk_state_change = iscsi_tcp_state_change;
1299 sk->sk_write_space = iscsi_write_space; 1052 sk->sk_write_space = iscsi_write_space;
@@ -1303,14 +1056,15 @@ iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1303static void 1056static void
1304iscsi_conn_restore_callbacks(struct iscsi_conn *conn) 1057iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1305{ 1058{
1306 struct sock *sk = conn->sock->sk; 1059 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1060 struct sock *sk = tcp_conn->sock->sk;
1307 1061
1308 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 1062 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1309 write_lock_bh(&sk->sk_callback_lock); 1063 write_lock_bh(&sk->sk_callback_lock);
1310 sk->sk_user_data = NULL; 1064 sk->sk_user_data = NULL;
1311 sk->sk_data_ready = conn->old_data_ready; 1065 sk->sk_data_ready = tcp_conn->old_data_ready;
1312 sk->sk_state_change = conn->old_state_change; 1066 sk->sk_state_change = tcp_conn->old_state_change;
1313 sk->sk_write_space = conn->old_write_space; 1067 sk->sk_write_space = tcp_conn->old_write_space;
1314 sk->sk_no_check = 0; 1068 sk->sk_no_check = 0;
1315 write_unlock_bh(&sk->sk_callback_lock); 1069 write_unlock_bh(&sk->sk_callback_lock);
1316} 1070}
@@ -1325,7 +1079,8 @@ iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1325static inline int 1079static inline int
1326iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags) 1080iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
1327{ 1081{
1328 struct socket *sk = conn->sock; 1082 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1083 struct socket *sk = tcp_conn->sock;
1329 int offset = buf->sg.offset + buf->sent; 1084 int offset = buf->sg.offset + buf->sent;
1330 1085
1331 /* 1086 /*
@@ -1339,7 +1094,8 @@ iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
1339 if (buf->use_sendmsg) 1094 if (buf->use_sendmsg)
1340 return sock_no_sendpage(sk, buf->sg.page, offset, size, flags); 1095 return sock_no_sendpage(sk, buf->sg.page, offset, size, flags);
1341 else 1096 else
1342 return conn->sendpage(sk, buf->sg.page, offset, size, flags); 1097 return tcp_conn->sendpage(sk, buf->sg.page, offset, size,
1098 flags);
1343} 1099}
1344 1100
1345/** 1101/**
@@ -1354,6 +1110,7 @@ iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
1354static inline int 1110static inline int
1355iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen) 1111iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1356{ 1112{
1113 struct iscsi_tcp_conn *tcp_conn;
1357 int flags = 0; /* MSG_DONTWAIT; */ 1114 int flags = 0; /* MSG_DONTWAIT; */
1358 int res, size; 1115 int res, size;
1359 1116
@@ -1371,8 +1128,9 @@ iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1371 return -EAGAIN; 1128 return -EAGAIN;
1372 return 0; 1129 return 0;
1373 } else if (res == -EAGAIN) { 1130 } else if (res == -EAGAIN) {
1374 conn->sendpage_failures_cnt++; 1131 tcp_conn = conn->dd_data;
1375 set_bit(SUSPEND_BIT, &conn->suspend_tx); 1132 tcp_conn->sendpage_failures_cnt++;
1133 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1376 } else if (res == -EPIPE) 1134 } else if (res == -EPIPE)
1377 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1135 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1378 1136
@@ -1393,6 +1151,7 @@ static inline int
1393iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf, 1151iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1394 int *count, int *sent) 1152 int *count, int *sent)
1395{ 1153{
1154 struct iscsi_tcp_conn *tcp_conn;
1396 int flags = 0; /* MSG_DONTWAIT; */ 1155 int flags = 0; /* MSG_DONTWAIT; */
1397 int res, size; 1156 int res, size;
1398 1157
@@ -1415,8 +1174,9 @@ iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1415 return -EAGAIN; 1174 return -EAGAIN;
1416 return 0; 1175 return 0;
1417 } else if (res == -EAGAIN) { 1176 } else if (res == -EAGAIN) {
1418 conn->sendpage_failures_cnt++; 1177 tcp_conn = conn->dd_data;
1419 set_bit(SUSPEND_BIT, &conn->suspend_tx); 1178 tcp_conn->sendpage_failures_cnt++;
1179 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1420 } else if (res == -EPIPE) 1180 } else if (res == -EPIPE)
1421 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1181 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1422 1182
@@ -1424,30 +1184,35 @@ iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1424} 1184}
1425 1185
1426static inline void 1186static inline void
1427iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1187iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
1188 struct iscsi_cmd_task *ctask)
1428{ 1189{
1429 BUG_ON(!conn->data_tx_tfm); 1190 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1430 crypto_digest_init(conn->data_tx_tfm); 1191
1431 ctask->digest_count = 4; 1192 BUG_ON(!tcp_conn->data_tx_tfm);
1193 crypto_digest_init(tcp_conn->data_tx_tfm);
1194 tcp_ctask->digest_count = 4;
1432} 1195}
1433 1196
1434static int 1197static int
1435iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1198iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1436 struct iscsi_buf *buf, uint32_t *digest, int final) 1199 struct iscsi_buf *buf, uint32_t *digest, int final)
1437{ 1200{
1201 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1202 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1438 int rc = 0; 1203 int rc = 0;
1439 int sent = 0; 1204 int sent = 0;
1440 1205
1441 if (final) 1206 if (final)
1442 crypto_digest_final(conn->data_tx_tfm, (u8*)digest); 1207 crypto_digest_final(tcp_conn->data_tx_tfm, (u8*)digest);
1443 1208
1444 iscsi_buf_init_virt(buf, (char*)digest, 4); 1209 iscsi_buf_init_virt(buf, (char*)digest, 4);
1445 rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent); 1210 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
1446 if (rc) { 1211 if (rc) {
1447 ctask->datadigest = *digest; 1212 tcp_ctask->datadigest = *digest;
1448 ctask->xmstate |= XMSTATE_DATA_DIGEST; 1213 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
1449 } else 1214 } else
1450 ctask->digest_count = 4; 1215 tcp_ctask->digest_count = 4;
1451 return rc; 1216 return rc;
1452} 1217}
1453 1218
@@ -1468,12 +1233,13 @@ static void
1468iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1233iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1469 struct iscsi_r2t_info *r2t, int left) 1234 struct iscsi_r2t_info *r2t, int left)
1470{ 1235{
1236 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1471 struct iscsi_data *hdr; 1237 struct iscsi_data *hdr;
1472 struct iscsi_data_task *dtask; 1238 struct iscsi_data_task *dtask;
1473 struct scsi_cmnd *sc = ctask->sc; 1239 struct scsi_cmnd *sc = ctask->sc;
1474 int new_offset; 1240 int new_offset;
1475 1241
1476 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 1242 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
1477 BUG_ON(!dtask); 1243 BUG_ON(!dtask);
1478 INIT_LIST_HEAD(&dtask->item); 1244 INIT_LIST_HEAD(&dtask->item);
1479 hdr = &dtask->hdr; 1245 hdr = &dtask->hdr;
@@ -1482,8 +1248,8 @@ iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1482 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 1248 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1483 r2t->solicit_datasn++; 1249 r2t->solicit_datasn++;
1484 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 1250 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1485 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 1251 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
1486 hdr->itt = ctask->hdr.itt; 1252 hdr->itt = ctask->hdr->itt;
1487 hdr->exp_statsn = r2t->exp_statsn; 1253 hdr->exp_statsn = r2t->exp_statsn;
1488 new_offset = r2t->data_offset + r2t->sent; 1254 new_offset = r2t->data_offset + r2t->sent;
1489 hdr->offset = cpu_to_be32(new_offset); 1255 hdr->offset = cpu_to_be32(new_offset);
@@ -1503,177 +1269,102 @@ iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1503 r2t->dtask = dtask; 1269 r2t->dtask = dtask;
1504 1270
1505 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) { 1271 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1506 BUG_ON(ctask->bad_sg == r2t->sg); 1272 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
1507 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1273 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1508 r2t->sg += 1; 1274 r2t->sg += 1;
1509 } else 1275 } else
1510 iscsi_buf_init_iov(&ctask->sendbuf, 1276 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1511 (char*)sc->request_buffer + new_offset, 1277 (char*)sc->request_buffer + new_offset,
1512 r2t->data_count); 1278 r2t->data_count);
1513 1279
1514 list_add(&dtask->item, &ctask->dataqueue); 1280 list_add(&dtask->item, &tcp_ctask->dataqueue);
1515} 1281}
1516 1282
1517static void 1283static void
1518iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1284iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1519{ 1285{
1520 struct iscsi_data *hdr; 1286 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1521 struct iscsi_data_task *dtask; 1287 struct iscsi_data_task *dtask;
1522 1288
1523 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 1289 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
1524 BUG_ON(!dtask); 1290 BUG_ON(!dtask);
1525 INIT_LIST_HEAD(&dtask->item); 1291 INIT_LIST_HEAD(&dtask->item);
1526 hdr = &dtask->hdr;
1527 memset(hdr, 0, sizeof(struct iscsi_data));
1528 hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1529 hdr->datasn = cpu_to_be32(ctask->unsol_datasn);
1530 ctask->unsol_datasn++;
1531 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1532 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1533 hdr->itt = ctask->hdr.itt;
1534 hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
1535 hdr->offset = cpu_to_be32(ctask->total_length -
1536 ctask->r2t_data_count -
1537 ctask->unsol_count);
1538 if (ctask->unsol_count > conn->max_xmit_dlength) {
1539 hton24(hdr->dlength, conn->max_xmit_dlength);
1540 ctask->data_count = conn->max_xmit_dlength;
1541 hdr->flags = 0;
1542 } else {
1543 hton24(hdr->dlength, ctask->unsol_count);
1544 ctask->data_count = ctask->unsol_count;
1545 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1546 }
1547 1292
1548 iscsi_buf_init_virt(&ctask->headbuf, (char*)hdr, 1293 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr,
1294 tcp_ctask->r2t_data_count);
1295 iscsi_buf_init_virt(&tcp_ctask->headbuf, (char*)&dtask->hdr,
1549 sizeof(struct iscsi_hdr)); 1296 sizeof(struct iscsi_hdr));
1550 1297
1551 list_add(&dtask->item, &ctask->dataqueue); 1298 list_add(&dtask->item, &tcp_ctask->dataqueue);
1552 1299 tcp_ctask->dtask = dtask;
1553 ctask->dtask = dtask;
1554} 1300}
1555 1301
1556/** 1302/**
1557 * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands 1303 * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1558 * @conn: iscsi connection 1304 * @conn: iscsi connection
1559 * @ctask: scsi command task 1305 * @ctask: scsi command task
1560 * @sc: scsi command 1306 * @sc: scsi command
1561 **/ 1307 **/
1562static void 1308static void
1563iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1309iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
1564 struct scsi_cmnd *sc)
1565{ 1310{
1566 struct iscsi_session *session = conn->session; 1311 struct scsi_cmnd *sc = ctask->sc;
1567 1312 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1568 BUG_ON(__kfifo_len(ctask->r2tqueue));
1569
1570 INIT_LIST_HEAD(&ctask->running);
1571 ctask->sc = sc;
1572 ctask->conn = conn;
1573 ctask->hdr.opcode = ISCSI_OP_SCSI_CMD;
1574 ctask->hdr.flags = ISCSI_ATTR_SIMPLE;
1575 int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun);
1576 ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) |
1577 (session->age << AGE_SHIFT);
1578 ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen);
1579 ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++;
1580 ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn);
1581 memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len);
1582 memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
1583 1313
1584 ctask->mtask = NULL; 1314 BUG_ON(__kfifo_len(tcp_ctask->r2tqueue));
1585 ctask->sent = 0;
1586 ctask->sg_count = 0;
1587 1315
1588 ctask->total_length = sc->request_bufflen; 1316 tcp_ctask->sent = 0;
1317 tcp_ctask->sg_count = 0;
1589 1318
1590 if (sc->sc_data_direction == DMA_TO_DEVICE) { 1319 if (sc->sc_data_direction == DMA_TO_DEVICE) {
1591 ctask->exp_r2tsn = 0; 1320 tcp_ctask->xmstate = XMSTATE_W_HDR;
1592 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE; 1321 tcp_ctask->exp_r2tsn = 0;
1593 BUG_ON(ctask->total_length == 0); 1322 BUG_ON(ctask->total_length == 0);
1323
1594 if (sc->use_sg) { 1324 if (sc->use_sg) {
1595 struct scatterlist *sg = sc->request_buffer; 1325 struct scatterlist *sg = sc->request_buffer;
1596 1326
1597 iscsi_buf_init_sg(&ctask->sendbuf, 1327 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1598 &sg[ctask->sg_count++]); 1328 &sg[tcp_ctask->sg_count++]);
1599 ctask->sg = sg; 1329 tcp_ctask->sg = sg;
1600 ctask->bad_sg = sg + sc->use_sg; 1330 tcp_ctask->bad_sg = sg + sc->use_sg;
1601 } else { 1331 } else
1602 iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer, 1332 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1603 sc->request_bufflen); 1333 sc->request_buffer,
1604 } 1334 sc->request_bufflen);
1605 1335
1606 /* 1336 if (ctask->imm_count)
1607 * Write counters: 1337 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1608 * 1338
1609 * imm_count bytes to be sent right after 1339 tcp_ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1610 * SCSI PDU Header 1340 if (tcp_ctask->pad_count) {
1611 * 1341 tcp_ctask->pad_count = ISCSI_PAD_LEN -
1612 * unsol_count bytes(as Data-Out) to be sent 1342 tcp_ctask->pad_count;
1613 * without R2T ack right after
1614 * immediate data
1615 *
1616 * r2t_data_count bytes to be sent via R2T ack's
1617 *
1618 * pad_count bytes to be sent as zero-padding
1619 */
1620 ctask->imm_count = 0;
1621 ctask->unsol_count = 0;
1622 ctask->unsol_datasn = 0;
1623 ctask->xmstate = XMSTATE_W_HDR;
1624 /* calculate write padding */
1625 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1626 if (ctask->pad_count) {
1627 ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count;
1628 debug_scsi("write padding %d bytes\n", 1343 debug_scsi("write padding %d bytes\n",
1629 ctask->pad_count); 1344 tcp_ctask->pad_count);
1630 ctask->xmstate |= XMSTATE_W_PAD; 1345 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1631 } 1346 }
1632 if (session->imm_data_en) {
1633 if (ctask->total_length >= session->first_burst)
1634 ctask->imm_count = min(session->first_burst,
1635 conn->max_xmit_dlength);
1636 else
1637 ctask->imm_count = min(ctask->total_length,
1638 conn->max_xmit_dlength);
1639 hton24(ctask->hdr.dlength, ctask->imm_count);
1640 ctask->xmstate |= XMSTATE_IMM_DATA;
1641 } else
1642 zero_data(ctask->hdr.dlength);
1643
1644 if (!session->initial_r2t_en)
1645 ctask->unsol_count = min(session->first_burst,
1646 ctask->total_length) - ctask->imm_count;
1647 if (!ctask->unsol_count)
1648 /* No unsolicit Data-Out's */
1649 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1650 else
1651 ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1652 1347
1653 ctask->r2t_data_count = ctask->total_length - 1348 if (ctask->unsol_count)
1349 tcp_ctask->xmstate |= XMSTATE_UNS_HDR |
1350 XMSTATE_UNS_INIT;
1351 tcp_ctask->r2t_data_count = ctask->total_length -
1654 ctask->imm_count - 1352 ctask->imm_count -
1655 ctask->unsol_count; 1353 ctask->unsol_count;
1656 1354
1657 debug_scsi("cmd [itt %x total %d imm %d imm_data %d " 1355 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1658 "r2t_data %d]\n", 1356 "r2t_data %d]\n",
1659 ctask->itt, ctask->total_length, ctask->imm_count, 1357 ctask->itt, ctask->total_length, ctask->imm_count,
1660 ctask->unsol_count, ctask->r2t_data_count); 1358 ctask->unsol_count, tcp_ctask->r2t_data_count);
1661 } else { 1359 } else
1662 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL; 1360 tcp_ctask->xmstate = XMSTATE_R_HDR;
1663 if (sc->sc_data_direction == DMA_FROM_DEVICE)
1664 ctask->hdr.flags |= ISCSI_FLAG_CMD_READ;
1665 ctask->datasn = 0;
1666 ctask->xmstate = XMSTATE_R_HDR;
1667 zero_data(ctask->hdr.dlength);
1668 }
1669 1361
1670 iscsi_buf_init_virt(&ctask->headbuf, (char*)&ctask->hdr, 1362 iscsi_buf_init_virt(&tcp_ctask->headbuf, (char*)ctask->hdr,
1671 sizeof(struct iscsi_hdr)); 1363 sizeof(struct iscsi_hdr));
1672 conn->scsicmd_pdus_cnt++;
1673} 1364}
1674 1365
1675/** 1366/**
1676 * iscsi_mtask_xmit - xmit management(immediate) task 1367 * iscsi_tcp_mtask_xmit - xmit management(immediate) task
1677 * @conn: iscsi connection 1368 * @conn: iscsi connection
1678 * @mtask: task management task 1369 * @mtask: task management task
1679 * 1370 *
@@ -1687,70 +1378,87 @@ iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1687 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress 1378 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1688 **/ 1379 **/
1689static int 1380static int
1690iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask) 1381iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1691{ 1382{
1383 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
1692 1384
1693 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n", 1385 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1694 conn->id, mtask->xmstate, mtask->itt); 1386 conn->id, tcp_mtask->xmstate, mtask->itt);
1695 1387
1696 if (mtask->xmstate & XMSTATE_IMM_HDR) { 1388 if (tcp_mtask->xmstate & XMSTATE_IMM_HDR) {
1697 mtask->xmstate &= ~XMSTATE_IMM_HDR; 1389 tcp_mtask->xmstate &= ~XMSTATE_IMM_HDR;
1698 if (mtask->data_count) 1390 if (mtask->data_count)
1699 mtask->xmstate |= XMSTATE_IMM_DATA; 1391 tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
1700 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE && 1392 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
1701 conn->stop_stage != STOP_CONN_RECOVER && 1393 conn->stop_stage != STOP_CONN_RECOVER &&
1702 conn->hdrdgst_en) 1394 conn->hdrdgst_en)
1703 iscsi_hdr_digest(conn, &mtask->headbuf, 1395 iscsi_hdr_digest(conn, &tcp_mtask->headbuf,
1704 (u8*)mtask->hdrext); 1396 (u8*)tcp_mtask->hdrext);
1705 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) { 1397 if (iscsi_sendhdr(conn, &tcp_mtask->headbuf,
1706 mtask->xmstate |= XMSTATE_IMM_HDR; 1398 mtask->data_count)) {
1399 tcp_mtask->xmstate |= XMSTATE_IMM_HDR;
1707 if (mtask->data_count) 1400 if (mtask->data_count)
1708 mtask->xmstate &= ~XMSTATE_IMM_DATA; 1401 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
1709 return -EAGAIN; 1402 return -EAGAIN;
1710 } 1403 }
1711 } 1404 }
1712 1405
1713 if (mtask->xmstate & XMSTATE_IMM_DATA) { 1406 if (tcp_mtask->xmstate & XMSTATE_IMM_DATA) {
1714 BUG_ON(!mtask->data_count); 1407 BUG_ON(!mtask->data_count);
1715 mtask->xmstate &= ~XMSTATE_IMM_DATA; 1408 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
1716 /* FIXME: implement. 1409 /* FIXME: implement.
1717 * Virtual buffer could be spreaded across multiple pages... 1410 * Virtual buffer could be spreaded across multiple pages...
1718 */ 1411 */
1719 do { 1412 do {
1720 if (iscsi_sendpage(conn, &mtask->sendbuf, 1413 if (iscsi_sendpage(conn, &tcp_mtask->sendbuf,
1721 &mtask->data_count, &mtask->sent)) { 1414 &mtask->data_count, &tcp_mtask->sent)) {
1722 mtask->xmstate |= XMSTATE_IMM_DATA; 1415 tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
1723 return -EAGAIN; 1416 return -EAGAIN;
1724 } 1417 }
1725 } while (mtask->data_count); 1418 } while (mtask->data_count);
1726 } 1419 }
1727 1420
1728 BUG_ON(mtask->xmstate != XMSTATE_IDLE); 1421 BUG_ON(tcp_mtask->xmstate != XMSTATE_IDLE);
1422 if (mtask->hdr->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1423 struct iscsi_session *session = conn->session;
1424
1425 spin_lock_bh(&session->lock);
1426 list_del(&conn->mtask->running);
1427 __kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
1428 sizeof(void*));
1429 spin_unlock_bh(&session->lock);
1430 }
1729 return 0; 1431 return 0;
1730} 1432}
1731 1433
1732static inline int 1434static inline int
1733handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1435handle_xmstate_r_hdr(struct iscsi_conn *conn,
1436 struct iscsi_tcp_cmd_task *tcp_ctask)
1734{ 1437{
1735 ctask->xmstate &= ~XMSTATE_R_HDR; 1438 tcp_ctask->xmstate &= ~XMSTATE_R_HDR;
1736 if (conn->hdrdgst_en) 1439 if (conn->hdrdgst_en)
1737 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext); 1440 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1738 if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) { 1441 (u8*)tcp_ctask->hdrext);
1739 BUG_ON(ctask->xmstate != XMSTATE_IDLE); 1442 if (!iscsi_sendhdr(conn, &tcp_ctask->headbuf, 0)) {
1443 BUG_ON(tcp_ctask->xmstate != XMSTATE_IDLE);
1740 return 0; /* wait for Data-In */ 1444 return 0; /* wait for Data-In */
1741 } 1445 }
1742 ctask->xmstate |= XMSTATE_R_HDR; 1446 tcp_ctask->xmstate |= XMSTATE_R_HDR;
1743 return -EAGAIN; 1447 return -EAGAIN;
1744} 1448}
1745 1449
1746static inline int 1450static inline int
1747handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1451handle_xmstate_w_hdr(struct iscsi_conn *conn,
1452 struct iscsi_cmd_task *ctask)
1748{ 1453{
1749 ctask->xmstate &= ~XMSTATE_W_HDR; 1454 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1455
1456 tcp_ctask->xmstate &= ~XMSTATE_W_HDR;
1750 if (conn->hdrdgst_en) 1457 if (conn->hdrdgst_en)
1751 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext); 1458 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1752 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) { 1459 (u8*)tcp_ctask->hdrext);
1753 ctask->xmstate |= XMSTATE_W_HDR; 1460 if (iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count)) {
1461 tcp_ctask->xmstate |= XMSTATE_W_HDR;
1754 return -EAGAIN; 1462 return -EAGAIN;
1755 } 1463 }
1756 return 0; 1464 return 0;
@@ -1760,13 +1468,15 @@ static inline int
1760handle_xmstate_data_digest(struct iscsi_conn *conn, 1468handle_xmstate_data_digest(struct iscsi_conn *conn,
1761 struct iscsi_cmd_task *ctask) 1469 struct iscsi_cmd_task *ctask)
1762{ 1470{
1763 ctask->xmstate &= ~XMSTATE_DATA_DIGEST; 1471 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1764 debug_tcp("resent data digest 0x%x\n", ctask->datadigest); 1472
1765 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1473 tcp_ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1766 &ctask->datadigest, 0)) { 1474 debug_tcp("resent data digest 0x%x\n", tcp_ctask->datadigest);
1767 ctask->xmstate |= XMSTATE_DATA_DIGEST; 1475 if (iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
1476 &tcp_ctask->datadigest, 0)) {
1477 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
1768 debug_tcp("resent data digest 0x%x fail!\n", 1478 debug_tcp("resent data digest 0x%x fail!\n",
1769 ctask->datadigest); 1479 tcp_ctask->datadigest);
1770 return -EAGAIN; 1480 return -EAGAIN;
1771 } 1481 }
1772 return 0; 1482 return 0;
@@ -1775,44 +1485,47 @@ handle_xmstate_data_digest(struct iscsi_conn *conn,
1775static inline int 1485static inline int
1776handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1486handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1777{ 1487{
1488 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1489 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1490
1778 BUG_ON(!ctask->imm_count); 1491 BUG_ON(!ctask->imm_count);
1779 ctask->xmstate &= ~XMSTATE_IMM_DATA; 1492 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
1780 1493
1781 if (conn->datadgst_en) { 1494 if (conn->datadgst_en) {
1782 iscsi_data_digest_init(conn, ctask); 1495 iscsi_data_digest_init(tcp_conn, ctask);
1783 ctask->immdigest = 0; 1496 tcp_ctask->immdigest = 0;
1784 } 1497 }
1785 1498
1786 for (;;) { 1499 for (;;) {
1787 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count, 1500 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf, &ctask->imm_count,
1788 &ctask->sent)) { 1501 &tcp_ctask->sent)) {
1789 ctask->xmstate |= XMSTATE_IMM_DATA; 1502 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1790 if (conn->datadgst_en) { 1503 if (conn->datadgst_en) {
1791 crypto_digest_final(conn->data_tx_tfm, 1504 crypto_digest_final(tcp_conn->data_tx_tfm,
1792 (u8*)&ctask->immdigest); 1505 (u8*)&tcp_ctask->immdigest);
1793 debug_tcp("tx imm sendpage fail 0x%x\n", 1506 debug_tcp("tx imm sendpage fail 0x%x\n",
1794 ctask->datadigest); 1507 tcp_ctask->datadigest);
1795 } 1508 }
1796 return -EAGAIN; 1509 return -EAGAIN;
1797 } 1510 }
1798 if (conn->datadgst_en) 1511 if (conn->datadgst_en)
1799 crypto_digest_update(conn->data_tx_tfm, 1512 crypto_digest_update(tcp_conn->data_tx_tfm,
1800 &ctask->sendbuf.sg, 1); 1513 &tcp_ctask->sendbuf.sg, 1);
1801 1514
1802 if (!ctask->imm_count) 1515 if (!ctask->imm_count)
1803 break; 1516 break;
1804 iscsi_buf_init_sg(&ctask->sendbuf, 1517 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1805 &ctask->sg[ctask->sg_count++]); 1518 &tcp_ctask->sg[tcp_ctask->sg_count++]);
1806 } 1519 }
1807 1520
1808 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) { 1521 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1809 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1522 if (iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
1810 &ctask->immdigest, 1)) { 1523 &tcp_ctask->immdigest, 1)) {
1811 debug_tcp("sending imm digest 0x%x fail!\n", 1524 debug_tcp("sending imm digest 0x%x fail!\n",
1812 ctask->immdigest); 1525 tcp_ctask->immdigest);
1813 return -EAGAIN; 1526 return -EAGAIN;
1814 } 1527 }
1815 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest); 1528 debug_tcp("sending imm digest 0x%x\n", tcp_ctask->immdigest);
1816 } 1529 }
1817 1530
1818 return 0; 1531 return 0;
@@ -1821,52 +1534,55 @@ handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1821static inline int 1534static inline int
1822handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1535handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1823{ 1536{
1537 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1824 struct iscsi_data_task *dtask; 1538 struct iscsi_data_task *dtask;
1825 1539
1826 ctask->xmstate |= XMSTATE_UNS_DATA; 1540 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1827 if (ctask->xmstate & XMSTATE_UNS_INIT) { 1541 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
1828 iscsi_unsolicit_data_init(conn, ctask); 1542 iscsi_unsolicit_data_init(conn, ctask);
1829 BUG_ON(!ctask->dtask); 1543 BUG_ON(!tcp_ctask->dtask);
1830 dtask = ctask->dtask; 1544 dtask = tcp_ctask->dtask;
1831 if (conn->hdrdgst_en) 1545 if (conn->hdrdgst_en)
1832 iscsi_hdr_digest(conn, &ctask->headbuf, 1546 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1833 (u8*)dtask->hdrext); 1547 (u8*)dtask->hdrext);
1834 ctask->xmstate &= ~XMSTATE_UNS_INIT; 1548 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
1835 } 1549 }
1836 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) { 1550 if (iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count)) {
1837 ctask->xmstate &= ~XMSTATE_UNS_DATA; 1551 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1838 ctask->xmstate |= XMSTATE_UNS_HDR; 1552 tcp_ctask->xmstate |= XMSTATE_UNS_HDR;
1839 return -EAGAIN; 1553 return -EAGAIN;
1840 } 1554 }
1841 1555
1842 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n", 1556 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1843 ctask->itt, ctask->unsol_count, ctask->sent); 1557 ctask->itt, ctask->unsol_count, tcp_ctask->sent);
1844 return 0; 1558 return 0;
1845} 1559}
1846 1560
1847static inline int 1561static inline int
1848handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1562handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1849{ 1563{
1850 struct iscsi_data_task *dtask = ctask->dtask; 1564 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1565 struct iscsi_data_task *dtask = tcp_ctask->dtask;
1566 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1851 1567
1852 BUG_ON(!ctask->data_count); 1568 BUG_ON(!ctask->data_count);
1853 ctask->xmstate &= ~XMSTATE_UNS_DATA; 1569 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1854 1570
1855 if (conn->datadgst_en) { 1571 if (conn->datadgst_en) {
1856 iscsi_data_digest_init(conn, ctask); 1572 iscsi_data_digest_init(tcp_conn, ctask);
1857 dtask->digest = 0; 1573 dtask->digest = 0;
1858 } 1574 }
1859 1575
1860 for (;;) { 1576 for (;;) {
1861 int start = ctask->sent; 1577 int start = tcp_ctask->sent;
1862 1578
1863 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count, 1579 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf,
1864 &ctask->sent)) { 1580 &ctask->data_count, &tcp_ctask->sent)) {
1865 ctask->unsol_count -= ctask->sent - start; 1581 ctask->unsol_count -= tcp_ctask->sent - start;
1866 ctask->xmstate |= XMSTATE_UNS_DATA; 1582 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1867 /* will continue with this ctask later.. */ 1583 /* will continue with this ctask later.. */
1868 if (conn->datadgst_en) { 1584 if (conn->datadgst_en) {
1869 crypto_digest_final(conn->data_tx_tfm, 1585 crypto_digest_final(tcp_conn->data_tx_tfm,
1870 (u8 *)&dtask->digest); 1586 (u8 *)&dtask->digest);
1871 debug_tcp("tx uns data fail 0x%x\n", 1587 debug_tcp("tx uns data fail 0x%x\n",
1872 dtask->digest); 1588 dtask->digest);
@@ -1874,21 +1590,21 @@ handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1874 return -EAGAIN; 1590 return -EAGAIN;
1875 } 1591 }
1876 1592
1877 BUG_ON(ctask->sent > ctask->total_length); 1593 BUG_ON(tcp_ctask->sent > ctask->total_length);
1878 ctask->unsol_count -= ctask->sent - start; 1594 ctask->unsol_count -= tcp_ctask->sent - start;
1879 1595
1880 /* 1596 /*
1881 * XXX:we may run here with un-initial sendbuf. 1597 * XXX:we may run here with un-initial sendbuf.
1882 * so pass it 1598 * so pass it
1883 */ 1599 */
1884 if (conn->datadgst_en && ctask->sent - start > 0) 1600 if (conn->datadgst_en && tcp_ctask->sent - start > 0)
1885 crypto_digest_update(conn->data_tx_tfm, 1601 crypto_digest_update(tcp_conn->data_tx_tfm,
1886 &ctask->sendbuf.sg, 1); 1602 &tcp_ctask->sendbuf.sg, 1);
1887 1603
1888 if (!ctask->data_count) 1604 if (!ctask->data_count)
1889 break; 1605 break;
1890 iscsi_buf_init_sg(&ctask->sendbuf, 1606 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1891 &ctask->sg[ctask->sg_count++]); 1607 &tcp_ctask->sg[tcp_ctask->sg_count++]);
1892 } 1608 }
1893 BUG_ON(ctask->unsol_count < 0); 1609 BUG_ON(ctask->unsol_count < 0);
1894 1610
@@ -1908,11 +1624,11 @@ handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1908 debug_tcp("sending uns digest 0x%x, more uns\n", 1624 debug_tcp("sending uns digest 0x%x, more uns\n",
1909 dtask->digest); 1625 dtask->digest);
1910 } 1626 }
1911 ctask->xmstate |= XMSTATE_UNS_INIT; 1627 tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
1912 return 1; 1628 return 1;
1913 } 1629 }
1914 1630
1915 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) { 1631 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1916 if (iscsi_digest_final_send(conn, ctask, 1632 if (iscsi_digest_final_send(conn, ctask,
1917 &dtask->digestbuf, 1633 &dtask->digestbuf,
1918 &dtask->digest, 1)) { 1634 &dtask->digest, 1)) {
@@ -1930,15 +1646,17 @@ static inline int
1930handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1646handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1931{ 1647{
1932 struct iscsi_session *session = conn->session; 1648 struct iscsi_session *session = conn->session;
1933 struct iscsi_r2t_info *r2t = ctask->r2t; 1649 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1650 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1651 struct iscsi_r2t_info *r2t = tcp_ctask->r2t;
1934 struct iscsi_data_task *dtask = r2t->dtask; 1652 struct iscsi_data_task *dtask = r2t->dtask;
1935 int left; 1653 int left;
1936 1654
1937 ctask->xmstate &= ~XMSTATE_SOL_DATA; 1655 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1938 ctask->dtask = dtask; 1656 tcp_ctask->dtask = dtask;
1939 1657
1940 if (conn->datadgst_en) { 1658 if (conn->datadgst_en) {
1941 iscsi_data_digest_init(conn, ctask); 1659 iscsi_data_digest_init(tcp_conn, ctask);
1942 dtask->digest = 0; 1660 dtask->digest = 0;
1943 } 1661 }
1944solicit_again: 1662solicit_again:
@@ -1949,10 +1667,10 @@ solicit_again:
1949 goto data_out_done; 1667 goto data_out_done;
1950 1668
1951 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) { 1669 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
1952 ctask->xmstate |= XMSTATE_SOL_DATA; 1670 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1953 /* will continue with this ctask later.. */ 1671 /* will continue with this ctask later.. */
1954 if (conn->datadgst_en) { 1672 if (conn->datadgst_en) {
1955 crypto_digest_final(conn->data_tx_tfm, 1673 crypto_digest_final(tcp_conn->data_tx_tfm,
1956 (u8 *)&dtask->digest); 1674 (u8 *)&dtask->digest);
1957 debug_tcp("r2t data send fail 0x%x\n", dtask->digest); 1675 debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1958 } 1676 }
@@ -1961,12 +1679,13 @@ solicit_again:
1961 1679
1962 BUG_ON(r2t->data_count < 0); 1680 BUG_ON(r2t->data_count < 0);
1963 if (conn->datadgst_en) 1681 if (conn->datadgst_en)
1964 crypto_digest_update(conn->data_tx_tfm, &r2t->sendbuf.sg, 1); 1682 crypto_digest_update(tcp_conn->data_tx_tfm, &r2t->sendbuf.sg,
1683 1);
1965 1684
1966 if (r2t->data_count) { 1685 if (r2t->data_count) {
1967 BUG_ON(ctask->sc->use_sg == 0); 1686 BUG_ON(ctask->sc->use_sg == 0);
1968 if (!iscsi_buf_left(&r2t->sendbuf)) { 1687 if (!iscsi_buf_left(&r2t->sendbuf)) {
1969 BUG_ON(ctask->bad_sg == r2t->sg); 1688 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
1970 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1689 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1971 r2t->sg += 1; 1690 r2t->sg += 1;
1972 } 1691 }
@@ -1993,8 +1712,8 @@ data_out_done:
1993 dtask->digest); 1712 dtask->digest);
1994 } 1713 }
1995 iscsi_solicit_data_cont(conn, ctask, r2t, left); 1714 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1996 ctask->xmstate |= XMSTATE_SOL_DATA; 1715 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1997 ctask->xmstate &= ~XMSTATE_SOL_HDR; 1716 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1998 return 1; 1717 return 1;
1999 } 1718 }
2000 1719
@@ -2002,7 +1721,7 @@ data_out_done:
2002 * Done with this R2T. Check if there are more 1721 * Done with this R2T. Check if there are more
2003 * outstanding R2Ts ready to be processed. 1722 * outstanding R2Ts ready to be processed.
2004 */ 1723 */
2005 BUG_ON(ctask->r2t_data_count - r2t->data_length < 0); 1724 BUG_ON(tcp_ctask->r2t_data_count - r2t->data_length < 0);
2006 if (conn->datadgst_en) { 1725 if (conn->datadgst_en) {
2007 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf, 1726 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
2008 &dtask->digest, 1)) { 1727 &dtask->digest, 1)) {
@@ -2013,15 +1732,15 @@ data_out_done:
2013 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest); 1732 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
2014 } 1733 }
2015 1734
2016 ctask->r2t_data_count -= r2t->data_length; 1735 tcp_ctask->r2t_data_count -= r2t->data_length;
2017 ctask->r2t = NULL; 1736 tcp_ctask->r2t = NULL;
2018 spin_lock_bh(&session->lock); 1737 spin_lock_bh(&session->lock);
2019 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 1738 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
2020 spin_unlock_bh(&session->lock); 1739 spin_unlock_bh(&session->lock);
2021 if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) { 1740 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
2022 ctask->r2t = r2t; 1741 tcp_ctask->r2t = r2t;
2023 ctask->xmstate |= XMSTATE_SOL_DATA; 1742 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
2024 ctask->xmstate &= ~XMSTATE_SOL_HDR; 1743 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
2025 return 1; 1744 return 1;
2026 } 1745 }
2027 1746
@@ -2031,29 +1750,34 @@ data_out_done:
2031static inline int 1750static inline int
2032handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1751handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2033{ 1752{
2034 struct iscsi_data_task *dtask = ctask->dtask; 1753 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1754 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1755 struct iscsi_data_task *dtask = tcp_ctask->dtask;
2035 int sent; 1756 int sent;
2036 1757
2037 ctask->xmstate &= ~XMSTATE_W_PAD; 1758 tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
2038 iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad, 1759 iscsi_buf_init_virt(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
2039 ctask->pad_count); 1760 tcp_ctask->pad_count);
2040 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) { 1761 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
2041 ctask->xmstate |= XMSTATE_W_PAD; 1762 &sent)) {
1763 tcp_ctask->xmstate |= XMSTATE_W_PAD;
2042 return -EAGAIN; 1764 return -EAGAIN;
2043 } 1765 }
2044 1766
2045 if (conn->datadgst_en) { 1767 if (conn->datadgst_en) {
2046 crypto_digest_update(conn->data_tx_tfm, &ctask->sendbuf.sg, 1); 1768 crypto_digest_update(tcp_conn->data_tx_tfm,
1769 &tcp_ctask->sendbuf.sg, 1);
2047 /* imm data? */ 1770 /* imm data? */
2048 if (!dtask) { 1771 if (!dtask) {
2049 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1772 if (iscsi_digest_final_send(conn, ctask,
2050 &ctask->immdigest, 1)) { 1773 &tcp_ctask->immbuf,
1774 &tcp_ctask->immdigest, 1)) {
2051 debug_tcp("send padding digest 0x%x" 1775 debug_tcp("send padding digest 0x%x"
2052 "fail!\n", ctask->immdigest); 1776 "fail!\n", tcp_ctask->immdigest);
2053 return -EAGAIN; 1777 return -EAGAIN;
2054 } 1778 }
2055 debug_tcp("done with padding, digest 0x%x\n", 1779 debug_tcp("done with padding, digest 0x%x\n",
2056 ctask->datadigest); 1780 tcp_ctask->datadigest);
2057 } else { 1781 } else {
2058 if (iscsi_digest_final_send(conn, ctask, 1782 if (iscsi_digest_final_send(conn, ctask,
2059 &dtask->digestbuf, 1783 &dtask->digestbuf,
@@ -2071,12 +1795,13 @@ handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2071} 1795}
2072 1796
2073static int 1797static int
2074iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1798iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2075{ 1799{
1800 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2076 int rc = 0; 1801 int rc = 0;
2077 1802
2078 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n", 1803 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
2079 conn->id, ctask->xmstate, ctask->itt); 1804 conn->id, tcp_ctask->xmstate, ctask->itt);
2080 1805
2081 /* 1806 /*
2082 * serialize with TMF AbortTask 1807 * serialize with TMF AbortTask
@@ -2084,40 +1809,40 @@ iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2084 if (ctask->mtask) 1809 if (ctask->mtask)
2085 return rc; 1810 return rc;
2086 1811
2087 if (ctask->xmstate & XMSTATE_R_HDR) { 1812 if (tcp_ctask->xmstate & XMSTATE_R_HDR) {
2088 rc = handle_xmstate_r_hdr(conn, ctask); 1813 rc = handle_xmstate_r_hdr(conn, tcp_ctask);
2089 return rc; 1814 return rc;
2090 } 1815 }
2091 1816
2092 if (ctask->xmstate & XMSTATE_W_HDR) { 1817 if (tcp_ctask->xmstate & XMSTATE_W_HDR) {
2093 rc = handle_xmstate_w_hdr(conn, ctask); 1818 rc = handle_xmstate_w_hdr(conn, ctask);
2094 if (rc) 1819 if (rc)
2095 return rc; 1820 return rc;
2096 } 1821 }
2097 1822
2098 /* XXX: for data digest xmit recover */ 1823 /* XXX: for data digest xmit recover */
2099 if (ctask->xmstate & XMSTATE_DATA_DIGEST) { 1824 if (tcp_ctask->xmstate & XMSTATE_DATA_DIGEST) {
2100 rc = handle_xmstate_data_digest(conn, ctask); 1825 rc = handle_xmstate_data_digest(conn, ctask);
2101 if (rc) 1826 if (rc)
2102 return rc; 1827 return rc;
2103 } 1828 }
2104 1829
2105 if (ctask->xmstate & XMSTATE_IMM_DATA) { 1830 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
2106 rc = handle_xmstate_imm_data(conn, ctask); 1831 rc = handle_xmstate_imm_data(conn, ctask);
2107 if (rc) 1832 if (rc)
2108 return rc; 1833 return rc;
2109 } 1834 }
2110 1835
2111 if (ctask->xmstate & XMSTATE_UNS_HDR) { 1836 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
2112 BUG_ON(!ctask->unsol_count); 1837 BUG_ON(!ctask->unsol_count);
2113 ctask->xmstate &= ~XMSTATE_UNS_HDR; 1838 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
2114unsolicit_head_again: 1839unsolicit_head_again:
2115 rc = handle_xmstate_uns_hdr(conn, ctask); 1840 rc = handle_xmstate_uns_hdr(conn, ctask);
2116 if (rc) 1841 if (rc)
2117 return rc; 1842 return rc;
2118 } 1843 }
2119 1844
2120 if (ctask->xmstate & XMSTATE_UNS_DATA) { 1845 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
2121 rc = handle_xmstate_uns_data(conn, ctask); 1846 rc = handle_xmstate_uns_data(conn, ctask);
2122 if (rc == 1) 1847 if (rc == 1)
2123 goto unsolicit_head_again; 1848 goto unsolicit_head_again;
@@ -2126,22 +1851,22 @@ unsolicit_head_again:
2126 goto done; 1851 goto done;
2127 } 1852 }
2128 1853
2129 if (ctask->xmstate & XMSTATE_SOL_HDR) { 1854 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
2130 struct iscsi_r2t_info *r2t; 1855 struct iscsi_r2t_info *r2t;
2131 1856
2132 ctask->xmstate &= ~XMSTATE_SOL_HDR; 1857 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
2133 ctask->xmstate |= XMSTATE_SOL_DATA; 1858 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
2134 if (!ctask->r2t) 1859 if (!tcp_ctask->r2t)
2135 __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t, 1860 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
2136 sizeof(void*)); 1861 sizeof(void*));
2137solicit_head_again: 1862solicit_head_again:
2138 r2t = ctask->r2t; 1863 r2t = tcp_ctask->r2t;
2139 if (conn->hdrdgst_en) 1864 if (conn->hdrdgst_en)
2140 iscsi_hdr_digest(conn, &r2t->headbuf, 1865 iscsi_hdr_digest(conn, &r2t->headbuf,
2141 (u8*)r2t->dtask->hdrext); 1866 (u8*)r2t->dtask->hdrext);
2142 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) { 1867 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
2143 ctask->xmstate &= ~XMSTATE_SOL_DATA; 1868 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
2144 ctask->xmstate |= XMSTATE_SOL_HDR; 1869 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
2145 return -EAGAIN; 1870 return -EAGAIN;
2146 } 1871 }
2147 1872
@@ -2150,7 +1875,7 @@ solicit_head_again:
2150 r2t->sent); 1875 r2t->sent);
2151 } 1876 }
2152 1877
2153 if (ctask->xmstate & XMSTATE_SOL_DATA) { 1878 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
2154 rc = handle_xmstate_sol_data(conn, ctask); 1879 rc = handle_xmstate_sol_data(conn, ctask);
2155 if (rc == 1) 1880 if (rc == 1)
2156 goto solicit_head_again; 1881 goto solicit_head_again;
@@ -2163,492 +1888,98 @@ done:
2163 * Last thing to check is whether we need to send write 1888 * Last thing to check is whether we need to send write
2164 * padding. Note that we check for xmstate equality, not just the bit. 1889 * padding. Note that we check for xmstate equality, not just the bit.
2165 */ 1890 */
2166 if (ctask->xmstate == XMSTATE_W_PAD) 1891 if (tcp_ctask->xmstate == XMSTATE_W_PAD)
2167 rc = handle_xmstate_w_pad(conn, ctask); 1892 rc = handle_xmstate_w_pad(conn, ctask);
2168 1893
2169 return rc; 1894 return rc;
2170} 1895}
2171 1896
2172/**
2173 * iscsi_data_xmit - xmit any command into the scheduled connection
2174 * @conn: iscsi connection
2175 *
2176 * Notes:
2177 * The function can return -EAGAIN in which case the caller must
2178 * re-schedule it again later or recover. '0' return code means
2179 * successful xmit.
2180 **/
2181static int
2182iscsi_data_xmit(struct iscsi_conn *conn)
2183{
2184 if (unlikely(conn->suspend_tx)) {
2185 debug_tcp("conn %d Tx suspended!\n", conn->id);
2186 return 0;
2187 }
2188
2189 /*
2190 * Transmit in the following order:
2191 *
2192 * 1) un-finished xmit (ctask or mtask)
2193 * 2) immediate control PDUs
2194 * 3) write data
2195 * 4) SCSI commands
2196 * 5) non-immediate control PDUs
2197 *
2198 * No need to lock around __kfifo_get as long as
2199 * there's one producer and one consumer.
2200 */
2201
2202 BUG_ON(conn->ctask && conn->mtask);
2203
2204 if (conn->ctask) {
2205 if (iscsi_ctask_xmit(conn, conn->ctask))
2206 goto again;
2207 /* done with this in-progress ctask */
2208 conn->ctask = NULL;
2209 }
2210 if (conn->mtask) {
2211 if (iscsi_mtask_xmit(conn, conn->mtask))
2212 goto again;
2213 /* done with this in-progress mtask */
2214 conn->mtask = NULL;
2215 }
2216
2217 /* process immediate first */
2218 if (unlikely(__kfifo_len(conn->immqueue))) {
2219 struct iscsi_session *session = conn->session;
2220 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask,
2221 sizeof(void*))) {
2222 if (iscsi_mtask_xmit(conn, conn->mtask))
2223 goto again;
2224
2225 if (conn->mtask->hdr.itt ==
2226 cpu_to_be32(ISCSI_RESERVED_TAG)) {
2227 spin_lock_bh(&session->lock);
2228 __kfifo_put(session->mgmtpool.queue,
2229 (void*)&conn->mtask, sizeof(void*));
2230 spin_unlock_bh(&session->lock);
2231 }
2232 }
2233 /* done with this mtask */
2234 conn->mtask = NULL;
2235 }
2236
2237 /* process write queue */
2238 while (__kfifo_get(conn->writequeue, (void*)&conn->ctask,
2239 sizeof(void*))) {
2240 if (iscsi_ctask_xmit(conn, conn->ctask))
2241 goto again;
2242 }
2243
2244 /* process command queue */
2245 while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask,
2246 sizeof(void*))) {
2247 list_add_tail(&conn->ctask->running, &conn->run_list);
2248 if (iscsi_ctask_xmit(conn, conn->ctask))
2249 goto again;
2250 }
2251 /* done with this ctask */
2252 conn->ctask = NULL;
2253
2254 /* process the rest control plane PDUs, if any */
2255 if (unlikely(__kfifo_len(conn->mgmtqueue))) {
2256 struct iscsi_session *session = conn->session;
2257
2258 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask,
2259 sizeof(void*))) {
2260 if (iscsi_mtask_xmit(conn, conn->mtask))
2261 goto again;
2262
2263 if (conn->mtask->hdr.itt ==
2264 cpu_to_be32(ISCSI_RESERVED_TAG)) {
2265 spin_lock_bh(&session->lock);
2266 __kfifo_put(session->mgmtpool.queue,
2267 (void*)&conn->mtask,
2268 sizeof(void*));
2269 spin_unlock_bh(&session->lock);
2270 }
2271 }
2272 /* done with this mtask */
2273 conn->mtask = NULL;
2274 }
2275
2276 return 0;
2277
2278again:
2279 if (unlikely(conn->suspend_tx))
2280 return 0;
2281
2282 return -EAGAIN;
2283}
2284
2285static void
2286iscsi_xmitworker(void *data)
2287{
2288 struct iscsi_conn *conn = data;
2289
2290 /*
2291 * serialize Xmit worker on a per-connection basis.
2292 */
2293 mutex_lock(&conn->xmitmutex);
2294 if (iscsi_data_xmit(conn))
2295 scsi_queue_work(conn->session->host, &conn->xmitwork);
2296 mutex_unlock(&conn->xmitmutex);
2297}
2298
2299enum {
2300 FAILURE_BAD_HOST = 1,
2301 FAILURE_SESSION_FAILED,
2302 FAILURE_SESSION_FREED,
2303 FAILURE_WINDOW_CLOSED,
2304 FAILURE_SESSION_TERMINATE,
2305 FAILURE_SESSION_RECOVERY_TIMEOUT,
2306};
2307
2308static int
2309iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
2310{
2311 struct Scsi_Host *host;
2312 int reason = 0;
2313 struct iscsi_session *session;
2314 struct iscsi_conn *conn = NULL;
2315 struct iscsi_cmd_task *ctask = NULL;
2316
2317 sc->scsi_done = done;
2318 sc->result = 0;
2319
2320 host = sc->device->host;
2321 session = iscsi_hostdata(host->hostdata);
2322
2323 spin_lock(&session->lock);
2324
2325 if (session->state != ISCSI_STATE_LOGGED_IN) {
2326 if (session->recovery_failed) {
2327 reason = FAILURE_SESSION_RECOVERY_TIMEOUT;
2328 goto fault;
2329 } else if (session->state == ISCSI_STATE_FAILED) {
2330 reason = FAILURE_SESSION_FAILED;
2331 goto reject;
2332 } else if (session->state == ISCSI_STATE_TERMINATE) {
2333 reason = FAILURE_SESSION_TERMINATE;
2334 goto fault;
2335 }
2336 reason = FAILURE_SESSION_FREED;
2337 goto fault;
2338 }
2339
2340 /*
2341 * Check for iSCSI window and take care of CmdSN wrap-around
2342 */
2343 if ((int)(session->max_cmdsn - session->cmdsn) < 0) {
2344 reason = FAILURE_WINDOW_CLOSED;
2345 goto reject;
2346 }
2347
2348 conn = session->leadconn;
2349
2350 __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
2351 BUG_ON(ctask->sc);
2352
2353 sc->SCp.phase = session->age;
2354 sc->SCp.ptr = (char*)ctask;
2355 iscsi_cmd_init(conn, ctask, sc);
2356
2357 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
2358 debug_scsi(
2359 "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
2360 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
2361 conn->id, (long)sc, ctask->itt, sc->request_bufflen,
2362 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
2363 spin_unlock(&session->lock);
2364
2365 scsi_queue_work(host, &conn->xmitwork);
2366 return 0;
2367
2368reject:
2369 spin_unlock(&session->lock);
2370 debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
2371 return SCSI_MLQUEUE_HOST_BUSY;
2372
2373fault:
2374 spin_unlock(&session->lock);
2375 printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n",
2376 sc->cmnd[0], reason);
2377 sc->result = (DID_NO_CONNECT << 16);
2378 sc->resid = sc->request_bufflen;
2379 sc->scsi_done(sc);
2380 return 0;
2381}
2382
2383static int
2384iscsi_change_queue_depth(struct scsi_device *sdev, int depth)
2385{
2386 if (depth > ISCSI_MAX_CMD_PER_LUN)
2387 depth = ISCSI_MAX_CMD_PER_LUN;
2388 scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth);
2389 return sdev->queue_depth;
2390}
2391
2392static int
2393iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size)
2394{
2395 int i;
2396
2397 *items = kmalloc(max * sizeof(void*), GFP_KERNEL);
2398 if (*items == NULL)
2399 return -ENOMEM;
2400
2401 q->max = max;
2402 q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL);
2403 if (q->pool == NULL) {
2404 kfree(*items);
2405 return -ENOMEM;
2406 }
2407
2408 q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2409 GFP_KERNEL, NULL);
2410 if (q->queue == ERR_PTR(-ENOMEM)) {
2411 kfree(q->pool);
2412 kfree(*items);
2413 return -ENOMEM;
2414 }
2415
2416 for (i = 0; i < max; i++) {
2417 q->pool[i] = kmalloc(item_size, GFP_KERNEL);
2418 if (q->pool[i] == NULL) {
2419 int j;
2420
2421 for (j = 0; j < i; j++)
2422 kfree(q->pool[j]);
2423
2424 kfifo_free(q->queue);
2425 kfree(q->pool);
2426 kfree(*items);
2427 return -ENOMEM;
2428 }
2429 memset(q->pool[i], 0, item_size);
2430 (*items)[i] = q->pool[i];
2431 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2432 }
2433 return 0;
2434}
2435
2436static void
2437iscsi_pool_free(struct iscsi_queue *q, void **items)
2438{
2439 int i;
2440
2441 for (i = 0; i < q->max; i++)
2442 kfree(items[i]);
2443 kfree(q->pool);
2444 kfree(items);
2445}
2446
2447static struct iscsi_cls_conn * 1897static struct iscsi_cls_conn *
2448iscsi_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx) 1898iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
2449{ 1899{
2450 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
2451 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
2452 struct iscsi_conn *conn; 1900 struct iscsi_conn *conn;
2453 struct iscsi_cls_conn *cls_conn; 1901 struct iscsi_cls_conn *cls_conn;
1902 struct iscsi_tcp_conn *tcp_conn;
2454 1903
2455 cls_conn = iscsi_create_conn(cls_session, conn_idx); 1904 cls_conn = iscsi_conn_setup(cls_session, conn_idx);
2456 if (!cls_conn) 1905 if (!cls_conn)
2457 return NULL; 1906 return NULL;
2458 conn = cls_conn->dd_data; 1907 conn = cls_conn->dd_data;
2459 memset(conn, 0, sizeof(*conn)); 1908 /*
2460 1909 * due to strange issues with iser these are not set
2461 conn->cls_conn = cls_conn; 1910 * in iscsi_conn_setup
2462 conn->c_stage = ISCSI_CONN_INITIAL_STAGE; 1911 */
2463 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2464 conn->id = conn_idx;
2465 conn->exp_statsn = 0;
2466 conn->tmabort_state = TMABORT_INITIAL;
2467 INIT_LIST_HEAD(&conn->run_list);
2468
2469 /* initial operational parameters */
2470 conn->hdr_size = sizeof(struct iscsi_hdr);
2471 conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2472 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH; 1912 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2473 1913
2474 /* initialize general xmit PDU commands queue */ 1914 tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL);
2475 conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*), 1915 if (!tcp_conn)
2476 GFP_KERNEL, NULL); 1916 goto tcp_conn_alloc_fail;
2477 if (conn->xmitqueue == ERR_PTR(-ENOMEM))
2478 goto xmitqueue_alloc_fail;
2479 1917
2480 /* initialize write response PDU commands queue */ 1918 conn->dd_data = tcp_conn;
2481 conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*), 1919 tcp_conn->iscsi_conn = conn;
2482 GFP_KERNEL, NULL); 1920 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2483 if (conn->writequeue == ERR_PTR(-ENOMEM)) 1921 /* initial operational parameters */
2484 goto writequeue_alloc_fail; 1922 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
2485 1923 tcp_conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2486 /* initialize general immediate & non-immediate PDU commands queue */
2487 conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2488 GFP_KERNEL, NULL);
2489 if (conn->immqueue == ERR_PTR(-ENOMEM))
2490 goto immqueue_alloc_fail;
2491
2492 conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2493 GFP_KERNEL, NULL);
2494 if (conn->mgmtqueue == ERR_PTR(-ENOMEM))
2495 goto mgmtqueue_alloc_fail;
2496
2497 INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn);
2498
2499 /* allocate login_mtask used for the login/text sequences */
2500 spin_lock_bh(&session->lock);
2501 if (!__kfifo_get(session->mgmtpool.queue,
2502 (void*)&conn->login_mtask,
2503 sizeof(void*))) {
2504 spin_unlock_bh(&session->lock);
2505 goto login_mtask_alloc_fail;
2506 }
2507 spin_unlock_bh(&session->lock);
2508 1924
2509 /* allocate initial PDU receive place holder */ 1925 /* allocate initial PDU receive place holder */
2510 if (conn->data_size <= PAGE_SIZE) 1926 if (tcp_conn->data_size <= PAGE_SIZE)
2511 conn->data = kmalloc(conn->data_size, GFP_KERNEL); 1927 tcp_conn->data = kmalloc(tcp_conn->data_size, GFP_KERNEL);
2512 else 1928 else
2513 conn->data = (void*)__get_free_pages(GFP_KERNEL, 1929 tcp_conn->data = (void*)__get_free_pages(GFP_KERNEL,
2514 get_order(conn->data_size)); 1930 get_order(tcp_conn->data_size));
2515 if (!conn->data) 1931 if (!tcp_conn->data)
2516 goto max_recv_dlenght_alloc_fail; 1932 goto max_recv_dlenght_alloc_fail;
2517 1933
2518 init_timer(&conn->tmabort_timer);
2519 mutex_init(&conn->xmitmutex);
2520 init_waitqueue_head(&conn->ehwait);
2521
2522 return cls_conn; 1934 return cls_conn;
2523 1935
2524max_recv_dlenght_alloc_fail: 1936max_recv_dlenght_alloc_fail:
2525 spin_lock_bh(&session->lock); 1937 kfree(tcp_conn);
2526 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask, 1938tcp_conn_alloc_fail:
2527 sizeof(void*)); 1939 iscsi_conn_teardown(cls_conn);
2528 spin_unlock_bh(&session->lock);
2529login_mtask_alloc_fail:
2530 kfifo_free(conn->mgmtqueue);
2531mgmtqueue_alloc_fail:
2532 kfifo_free(conn->immqueue);
2533immqueue_alloc_fail:
2534 kfifo_free(conn->writequeue);
2535writequeue_alloc_fail:
2536 kfifo_free(conn->xmitqueue);
2537xmitqueue_alloc_fail:
2538 iscsi_destroy_conn(cls_conn);
2539 return NULL; 1940 return NULL;
2540} 1941}
2541 1942
2542static void 1943static void
2543iscsi_conn_destroy(struct iscsi_cls_conn *cls_conn) 1944iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
2544{ 1945{
2545 struct iscsi_conn *conn = cls_conn->dd_data; 1946 struct iscsi_conn *conn = cls_conn->dd_data;
2546 struct iscsi_session *session = conn->session; 1947 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2547 unsigned long flags; 1948 int digest = 0;
2548
2549 mutex_lock(&conn->xmitmutex);
2550 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2551 if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) {
2552 struct sock *sk = conn->sock->sk;
2553
2554 /*
2555 * conn_start() has never been called!
2556 * need to cleanup the socket.
2557 */
2558 write_lock_bh(&sk->sk_callback_lock);
2559 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2560 write_unlock_bh(&sk->sk_callback_lock);
2561
2562 sock_hold(conn->sock->sk);
2563 iscsi_conn_restore_callbacks(conn);
2564 sock_put(conn->sock->sk);
2565 sock_release(conn->sock);
2566 conn->sock = NULL;
2567 }
2568
2569 spin_lock_bh(&session->lock);
2570 conn->c_stage = ISCSI_CONN_CLEANUP_WAIT;
2571 if (session->leadconn == conn) {
2572 /*
2573 * leading connection? then give up on recovery.
2574 */
2575 session->state = ISCSI_STATE_TERMINATE;
2576 wake_up(&conn->ehwait);
2577 }
2578 spin_unlock_bh(&session->lock);
2579 1949
2580 mutex_unlock(&conn->xmitmutex); 1950 if (conn->hdrdgst_en || conn->datadgst_en)
1951 digest = 1;
2581 1952
2582 /* 1953 iscsi_conn_teardown(cls_conn);
2583 * Block until all in-progress commands for this connection
2584 * time out or fail.
2585 */
2586 for (;;) {
2587 spin_lock_irqsave(session->host->host_lock, flags);
2588 if (!session->host->host_busy) { /* OK for ERL == 0 */
2589 spin_unlock_irqrestore(session->host->host_lock, flags);
2590 break;
2591 }
2592 spin_unlock_irqrestore(session->host->host_lock, flags);
2593 msleep_interruptible(500);
2594 printk("conn_destroy(): host_busy %d host_failed %d\n",
2595 session->host->host_busy, session->host->host_failed);
2596 /*
2597 * force eh_abort() to unblock
2598 */
2599 wake_up(&conn->ehwait);
2600 }
2601 1954
2602 /* now free crypto */ 1955 /* now free tcp_conn */
2603 if (conn->hdrdgst_en || conn->datadgst_en) { 1956 if (digest) {
2604 if (conn->tx_tfm) 1957 if (tcp_conn->tx_tfm)
2605 crypto_free_tfm(conn->tx_tfm); 1958 crypto_free_tfm(tcp_conn->tx_tfm);
2606 if (conn->rx_tfm) 1959 if (tcp_conn->rx_tfm)
2607 crypto_free_tfm(conn->rx_tfm); 1960 crypto_free_tfm(tcp_conn->rx_tfm);
2608 if (conn->data_tx_tfm) 1961 if (tcp_conn->data_tx_tfm)
2609 crypto_free_tfm(conn->data_tx_tfm); 1962 crypto_free_tfm(tcp_conn->data_tx_tfm);
2610 if (conn->data_rx_tfm) 1963 if (tcp_conn->data_rx_tfm)
2611 crypto_free_tfm(conn->data_rx_tfm); 1964 crypto_free_tfm(tcp_conn->data_rx_tfm);
2612 } 1965 }
2613 1966
2614 /* free conn->data, size = MaxRecvDataSegmentLength */ 1967 /* free conn->data, size = MaxRecvDataSegmentLength */
2615 if (conn->data_size <= PAGE_SIZE) 1968 if (tcp_conn->data_size <= PAGE_SIZE)
2616 kfree(conn->data); 1969 kfree(tcp_conn->data);
2617 else 1970 else
2618 free_pages((unsigned long)conn->data, 1971 free_pages((unsigned long)tcp_conn->data,
2619 get_order(conn->data_size)); 1972 get_order(tcp_conn->data_size));
2620 1973 kfree(tcp_conn);
2621 spin_lock_bh(&session->lock);
2622 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2623 sizeof(void*));
2624 list_del(&conn->item);
2625 if (list_empty(&session->connections))
2626 session->leadconn = NULL;
2627 if (session->leadconn && session->leadconn == conn)
2628 session->leadconn = container_of(session->connections.next,
2629 struct iscsi_conn, item);
2630
2631 if (session->leadconn == NULL)
2632 /* none connections exits.. reset sequencing */
2633 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1;
2634 spin_unlock_bh(&session->lock);
2635
2636 kfifo_free(conn->xmitqueue);
2637 kfifo_free(conn->writequeue);
2638 kfifo_free(conn->immqueue);
2639 kfifo_free(conn->mgmtqueue);
2640
2641 iscsi_destroy_conn(cls_conn);
2642} 1974}
2643 1975
2644static int 1976static int
2645iscsi_conn_bind(struct iscsi_cls_session *cls_session, 1977iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
2646 struct iscsi_cls_conn *cls_conn, uint32_t transport_fd, 1978 struct iscsi_cls_conn *cls_conn, uint32_t transport_fd,
2647 int is_leading) 1979 int is_leading)
2648{ 1980{
2649 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 1981 struct iscsi_conn *conn = cls_conn->dd_data;
2650 struct iscsi_session *session = iscsi_hostdata(shost->hostdata); 1982 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2651 struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = cls_conn->dd_data;
2652 struct sock *sk; 1983 struct sock *sk;
2653 struct socket *sock; 1984 struct socket *sock;
2654 int err; 1985 int err;
@@ -2660,32 +1991,13 @@ iscsi_conn_bind(struct iscsi_cls_session *cls_session,
2660 return -EEXIST; 1991 return -EEXIST;
2661 } 1992 }
2662 1993
2663 /* lookup for existing connection */ 1994 err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
2664 spin_lock_bh(&session->lock); 1995 if (err)
2665 list_for_each_entry(tmp, &session->connections, item) { 1996 return err;
2666 if (tmp == conn) {
2667 if (conn->c_stage != ISCSI_CONN_STOPPED ||
2668 conn->stop_stage == STOP_CONN_TERM) {
2669 printk(KERN_ERR "iscsi_tcp: can't bind "
2670 "non-stopped connection (%d:%d)\n",
2671 conn->c_stage, conn->stop_stage);
2672 spin_unlock_bh(&session->lock);
2673 return -EIO;
2674 }
2675 break;
2676 }
2677 }
2678 if (tmp != conn) {
2679 /* bind new iSCSI connection to session */
2680 conn->session = session;
2681
2682 list_add(&conn->item, &session->connections);
2683 }
2684 spin_unlock_bh(&session->lock);
2685 1997
2686 if (conn->stop_stage != STOP_CONN_SUSPEND) { 1998 if (conn->stop_stage != STOP_CONN_SUSPEND) {
2687 /* bind iSCSI connection and socket */ 1999 /* bind iSCSI connection and socket */
2688 conn->sock = sock; 2000 tcp_conn->sock = sock;
2689 2001
2690 /* setup Socket parameters */ 2002 /* setup Socket parameters */
2691 sk = sock->sk; 2003 sk = sock->sk;
@@ -2699,608 +2011,78 @@ iscsi_conn_bind(struct iscsi_cls_session *cls_session,
2699 * Intercept TCP callbacks for sendfile like receive 2011 * Intercept TCP callbacks for sendfile like receive
2700 * processing. 2012 * processing.
2701 */ 2013 */
2014 conn->recv_lock = &sk->sk_callback_lock;
2702 iscsi_conn_set_callbacks(conn); 2015 iscsi_conn_set_callbacks(conn);
2703 2016 tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
2704 conn->sendpage = conn->sock->ops->sendpage;
2705
2706 /* 2017 /*
2707 * set receive state machine into initial state 2018 * set receive state machine into initial state
2708 */ 2019 */
2709 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 2020 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2710 } 2021 }
2711 2022
2712 if (is_leading)
2713 session->leadconn = conn;
2714
2715 /*
2716 * Unblock xmitworker(), Login Phase will pass through.
2717 */
2718 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2719 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2720
2721 return 0;
2722}
2723
2724static void
2725iscsi_session_recovery_timedout(struct iscsi_cls_session *csession)
2726{
2727 struct Scsi_Host *shost = iscsi_session_to_shost(csession);
2728 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
2729 struct iscsi_conn *conn = session->leadconn;
2730
2731 spin_lock_bh(&session->lock);
2732 if (session->state != ISCSI_STATE_LOGGED_IN) {
2733 session->recovery_failed = 1;
2734 if (conn)
2735 wake_up(&conn->ehwait);
2736 }
2737 spin_unlock_bh(&session->lock);
2738}
2739
2740static int
2741iscsi_conn_start(struct iscsi_cls_conn *cls_conn)
2742{
2743 struct iscsi_conn *conn = cls_conn->dd_data;
2744 struct iscsi_session *session = conn->session;
2745 struct sock *sk;
2746
2747 /* FF phase warming up... */
2748
2749 if (session == NULL) {
2750 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n");
2751 return -EPERM;
2752 }
2753
2754 sk = conn->sock->sk;
2755 write_lock_bh(&sk->sk_callback_lock);
2756 spin_lock_bh(&session->lock);
2757 conn->c_stage = ISCSI_CONN_STARTED;
2758 session->state = ISCSI_STATE_LOGGED_IN;
2759
2760 switch(conn->stop_stage) {
2761 case STOP_CONN_RECOVER:
2762 /*
2763 * unblock eh_abort() if it is blocked. re-try all
2764 * commands after successful recovery
2765 */
2766 session->conn_cnt++;
2767 conn->stop_stage = 0;
2768 conn->tmabort_state = TMABORT_INITIAL;
2769 session->age++;
2770 session->recovery_failed = 0;
2771 spin_unlock_bh(&session->lock);
2772 write_unlock_bh(&sk->sk_callback_lock);
2773
2774 iscsi_unblock_session(session_to_cls(session));
2775 wake_up(&conn->ehwait);
2776 return 0;
2777 case STOP_CONN_TERM:
2778 session->conn_cnt++;
2779 conn->stop_stage = 0;
2780 break;
2781 case STOP_CONN_SUSPEND:
2782 conn->stop_stage = 0;
2783 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2784 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2785 break;
2786 default:
2787 break;
2788 }
2789 spin_unlock_bh(&session->lock);
2790 write_unlock_bh(&sk->sk_callback_lock);
2791
2792 return 0;
2793}
2794
2795static int
2796iscsi_ctask_mtask_cleanup(struct iscsi_cmd_task *ctask)
2797{
2798 struct iscsi_conn *conn = ctask->conn;
2799 struct iscsi_session *session = conn->session;
2800 struct iscsi_mgmt_task *mtask;
2801
2802 if (!ctask->mtask)
2803 return -EINVAL;
2804
2805 while (__kfifo_get(conn->immqueue, (void*)&mtask, sizeof(void*))) {
2806 if (mtask == ctask->mtask)
2807 break;
2808 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
2809 }
2810
2811 __kfifo_put(session->mgmtpool.queue, (void*)&ctask->mtask,
2812 sizeof(void*));
2813 ctask->mtask = NULL;
2814 return 0; 2023 return 0;
2815} 2024}
2816 2025
2817static void 2026static void
2818fail_command(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, int err) 2027iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2819{ 2028{
2029 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2820 struct iscsi_r2t_info *r2t; 2030 struct iscsi_r2t_info *r2t;
2821 struct scsi_cmnd *sc;
2822 2031
2823 /* flush ctask's r2t queues */ 2032 /* flush ctask's r2t queues */
2824 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) 2033 while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*)))
2825 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 2034 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
2826
2827 iscsi_ctask_mtask_cleanup(ctask);
2828
2829 sc = ctask->sc;
2830 if (!sc)
2831 return;
2832
2833 sc->result = err;
2834 sc->resid = sc->request_bufflen;
2835
2836 __iscsi_ctask_cleanup(conn, ctask);
2837}
2838
2839/* Fail commands. Mutex and session lock held */
2840static void
2841fail_all_commands(struct iscsi_session *session, struct iscsi_conn *conn)
2842{
2843 struct iscsi_cmd_task *ctask, *tmp;
2844
2845 /* flush pending */
2846 while (__kfifo_get(conn->xmitqueue, (void*)&ctask, sizeof(void*))) {
2847 debug_scsi("failing pending sc %p itt 0x%x\n", ctask->sc,
2848 ctask->itt);
2849 fail_command(conn, ctask, DID_BUS_BUSY << 16);
2850 }
2851
2852 /* fail running */
2853 list_for_each_entry_safe(ctask, tmp, &conn->run_list, running) {
2854 debug_scsi("failing in progress sc %p itt 0x%x\n",
2855 ctask->sc, ctask->itt);
2856 fail_command(conn, ctask, DID_BUS_BUSY << 16);
2857 }
2858
2859 conn->ctask = NULL;
2860}
2861
2862static void
2863flush_control_queues(struct iscsi_session *session, struct iscsi_conn *conn)
2864{
2865 struct iscsi_mgmt_task *mtask;
2866
2867 /* TODO: handle running pdus */
2868 while (__kfifo_get(conn->immqueue, (void*)&mtask, sizeof(void*)) ||
2869 __kfifo_get(conn->mgmtqueue, (void*)&mtask, sizeof(void*))) {
2870 if (mtask == conn->login_mtask)
2871 continue;
2872 __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
2873 sizeof(void*)); 2035 sizeof(void*));
2874 }
2875 2036
2876 if (conn->mtask && conn->mtask != conn->login_mtask) 2037 __iscsi_ctask_cleanup(conn, ctask);
2877 __kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
2878 sizeof(void*));
2879 conn->mtask = NULL;
2880} 2038}
2881 2039
2882static void 2040static void
2883iscsi_suspend_conn_rx(struct iscsi_conn *conn) 2041iscsi_tcp_suspend_conn_rx(struct iscsi_conn *conn)
2884{ 2042{
2043 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2885 struct sock *sk; 2044 struct sock *sk;
2886 2045
2887 BUG_ON(!conn->sock); 2046 if (!tcp_conn->sock)
2888 sk = conn->sock->sk; 2047 return;
2048
2049 sk = tcp_conn->sock->sk;
2889 write_lock_bh(&sk->sk_callback_lock); 2050 write_lock_bh(&sk->sk_callback_lock);
2890 set_bit(SUSPEND_BIT, &conn->suspend_rx); 2051 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
2891 write_unlock_bh(&sk->sk_callback_lock); 2052 write_unlock_bh(&sk->sk_callback_lock);
2892} 2053}
2893 2054
2894static void 2055static void
2895iscsi_start_session_recovery(struct iscsi_session *session, 2056iscsi_tcp_terminate_conn(struct iscsi_conn *conn)
2896 struct iscsi_conn *conn, int flag)
2897{ 2057{
2898 spin_lock_bh(&session->lock); 2058 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2899 if (conn->stop_stage == STOP_CONN_RECOVER ||
2900 conn->stop_stage == STOP_CONN_TERM) {
2901 spin_unlock_bh(&session->lock);
2902 return;
2903 }
2904 conn->stop_stage = flag;
2905 spin_unlock_bh(&session->lock);
2906
2907 iscsi_suspend_conn_rx(conn);
2908 2059
2909 mutex_lock(&conn->xmitmutex); 2060 if (!tcp_conn->sock)
2910 spin_lock_bh(&session->lock); 2061 return;
2911 conn->c_stage = ISCSI_CONN_STOPPED;
2912 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2913
2914 session->conn_cnt--;
2915 if (session->conn_cnt == 0 || session->leadconn == conn)
2916 session->state = ISCSI_STATE_FAILED;
2917
2918 spin_unlock_bh(&session->lock);
2919 2062
2920 /* 2063 sock_hold(tcp_conn->sock->sk);
2921 * Socket must go now.
2922 */
2923 sock_hold(conn->sock->sk);
2924 iscsi_conn_restore_callbacks(conn); 2064 iscsi_conn_restore_callbacks(conn);
2925 sock_put(conn->sock->sk); 2065 sock_put(tcp_conn->sock->sk);
2926
2927 /*
2928 * flush queues.
2929 */
2930 spin_lock_bh(&session->lock);
2931 fail_all_commands(session, conn);
2932 flush_control_queues(session, conn);
2933 spin_unlock_bh(&session->lock);
2934
2935 /*
2936 * release socket only after we stopped data_xmit()
2937 * activity and flushed all outstandings
2938 */
2939 sock_release(conn->sock);
2940 conn->sock = NULL;
2941
2942 /*
2943 * for connection level recovery we should not calculate
2944 * header digest. conn->hdr_size used for optimization
2945 * in hdr_extract() and will be re-negotiated at
2946 * set_param() time.
2947 */
2948 if (flag == STOP_CONN_RECOVER) {
2949 conn->hdr_size = sizeof(struct iscsi_hdr);
2950 conn->hdrdgst_en = 0;
2951 conn->datadgst_en = 0;
2952 2066
2953 if (session->state == ISCSI_STATE_FAILED) 2067 sock_release(tcp_conn->sock);
2954 iscsi_block_session(session_to_cls(session)); 2068 tcp_conn->sock = NULL;
2955 } 2069 conn->recv_lock = NULL;
2956 mutex_unlock(&conn->xmitmutex);
2957} 2070}
2958 2071
2072/* called with host lock */
2959static void 2073static void
2960iscsi_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) 2074iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask,
2961{ 2075 char *data, uint32_t data_size)
2962 struct iscsi_conn *conn = cls_conn->dd_data;
2963 struct iscsi_session *session = conn->session;
2964
2965 switch (flag) {
2966 case STOP_CONN_RECOVER:
2967 case STOP_CONN_TERM:
2968 iscsi_start_session_recovery(session, conn, flag);
2969 return;
2970 case STOP_CONN_SUSPEND:
2971 iscsi_suspend_conn_rx(conn);
2972
2973 mutex_lock(&conn->xmitmutex);
2974 spin_lock_bh(&session->lock);
2975
2976 conn->stop_stage = flag;
2977 conn->c_stage = ISCSI_CONN_STOPPED;
2978 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2979
2980 spin_unlock_bh(&session->lock);
2981 mutex_unlock(&conn->xmitmutex);
2982 break;
2983 default:
2984 printk(KERN_ERR "invalid stop flag %d\n", flag);
2985 }
2986}
2987
2988static int
2989iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
2990 char *data, uint32_t data_size)
2991{ 2076{
2992 struct iscsi_session *session = conn->session; 2077 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
2993 struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
2994 struct iscsi_mgmt_task *mtask;
2995 2078
2996 spin_lock_bh(&session->lock); 2079 iscsi_buf_init_virt(&tcp_mtask->headbuf, (char*)mtask->hdr,
2997 if (session->state == ISCSI_STATE_TERMINATE) {
2998 spin_unlock_bh(&session->lock);
2999 return -EPERM;
3000 }
3001 if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
3002 hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
3003 /*
3004 * Login and Text are sent serially, in
3005 * request-followed-by-response sequence.
3006 * Same mtask can be used. Same ITT must be used.
3007 * Note that login_mtask is preallocated at conn_create().
3008 */
3009 mtask = conn->login_mtask;
3010 else {
3011 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
3012 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
3013
3014 if (!__kfifo_get(session->mgmtpool.queue,
3015 (void*)&mtask, sizeof(void*))) {
3016 spin_unlock_bh(&session->lock);
3017 return -ENOSPC;
3018 }
3019 }
3020
3021 /*
3022 * pre-format CmdSN and ExpStatSN for outgoing PDU.
3023 */
3024 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
3025 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) |
3026 (session->age << AGE_SHIFT);
3027 nop->cmdsn = cpu_to_be32(session->cmdsn);
3028 if (conn->c_stage == ISCSI_CONN_STARTED &&
3029 !(hdr->opcode & ISCSI_OP_IMMEDIATE))
3030 session->cmdsn++;
3031 } else
3032 /* do not advance CmdSN */
3033 nop->cmdsn = cpu_to_be32(session->cmdsn);
3034
3035 nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
3036
3037 memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr));
3038
3039 iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr,
3040 sizeof(struct iscsi_hdr)); 2080 sizeof(struct iscsi_hdr));
2081 tcp_mtask->xmstate = XMSTATE_IMM_HDR;
3041 2082
3042 spin_unlock_bh(&session->lock); 2083 if (mtask->data_count)
3043 2084 iscsi_buf_init_iov(&tcp_mtask->sendbuf, (char*)mtask->data,
3044 if (data_size) {
3045 memcpy(mtask->data, data, data_size);
3046 mtask->data_count = data_size;
3047 } else
3048 mtask->data_count = 0;
3049
3050 mtask->xmstate = XMSTATE_IMM_HDR;
3051
3052 if (mtask->data_count) {
3053 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data,
3054 mtask->data_count); 2085 mtask->data_count);
3055 }
3056
3057 debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n",
3058 hdr->opcode, hdr->itt, data_size);
3059
3060 /*
3061 * since send_pdu() could be called at least from two contexts,
3062 * we need to serialize __kfifo_put, so we don't have to take
3063 * additional lock on fast data-path
3064 */
3065 if (hdr->opcode & ISCSI_OP_IMMEDIATE)
3066 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
3067 else
3068 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*));
3069
3070 scsi_queue_work(session->host, &conn->xmitwork);
3071 return 0;
3072}
3073
3074static int
3075iscsi_eh_host_reset(struct scsi_cmnd *sc)
3076{
3077 struct Scsi_Host *host = sc->device->host;
3078 struct iscsi_session *session = iscsi_hostdata(host->hostdata);
3079 struct iscsi_conn *conn = session->leadconn;
3080 int fail_session = 0;
3081
3082 spin_lock_bh(&session->lock);
3083 if (session->state == ISCSI_STATE_TERMINATE) {
3084failed:
3085 debug_scsi("failing host reset: session terminated "
3086 "[CID %d age %d]", conn->id, session->age);
3087 spin_unlock_bh(&session->lock);
3088 return FAILED;
3089 }
3090
3091 if (sc->SCp.phase == session->age) {
3092 debug_scsi("failing connection CID %d due to SCSI host reset",
3093 conn->id);
3094 fail_session = 1;
3095 }
3096 spin_unlock_bh(&session->lock);
3097
3098 /*
3099 * we drop the lock here but the leadconn cannot be destoyed while
3100 * we are in the scsi eh
3101 */
3102 if (fail_session) {
3103 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3104 /*
3105 * if userspace cannot respond then we must kick this off
3106 * here for it
3107 */
3108 iscsi_start_session_recovery(session, conn, STOP_CONN_RECOVER);
3109 }
3110
3111 debug_scsi("iscsi_eh_host_reset wait for relogin\n");
3112 wait_event_interruptible(conn->ehwait,
3113 session->state == ISCSI_STATE_TERMINATE ||
3114 session->state == ISCSI_STATE_LOGGED_IN ||
3115 session->recovery_failed);
3116 if (signal_pending(current))
3117 flush_signals(current);
3118
3119 spin_lock_bh(&session->lock);
3120 if (session->state == ISCSI_STATE_LOGGED_IN)
3121 printk(KERN_INFO "host reset succeeded\n");
3122 else
3123 goto failed;
3124 spin_unlock_bh(&session->lock);
3125
3126 return SUCCESS;
3127}
3128
3129static void
3130iscsi_tmabort_timedout(unsigned long data)
3131{
3132 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data;
3133 struct iscsi_conn *conn = ctask->conn;
3134 struct iscsi_session *session = conn->session;
3135
3136 spin_lock(&session->lock);
3137 if (conn->tmabort_state == TMABORT_INITIAL) {
3138 conn->tmabort_state = TMABORT_TIMEDOUT;
3139 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n",
3140 (long)ctask->sc, ctask->itt);
3141 /* unblock eh_abort() */
3142 wake_up(&conn->ehwait);
3143 }
3144 spin_unlock(&session->lock);
3145}
3146
3147/* must be called with the mutex lock */
3148static int
3149iscsi_exec_abort_task(struct scsi_cmnd *sc, struct iscsi_cmd_task *ctask)
3150{
3151 struct iscsi_conn *conn = ctask->conn;
3152 struct iscsi_session *session = conn->session;
3153 struct iscsi_tm *hdr = &conn->tmhdr;
3154 int rc;
3155
3156 /*
3157 * ctask timed out but session is OK requests must be serialized.
3158 */
3159 memset(hdr, 0, sizeof(struct iscsi_tm));
3160 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
3161 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK;
3162 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3163 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
3164 hdr->rtt = ctask->hdr.itt;
3165 hdr->refcmdsn = ctask->hdr.cmdsn;
3166
3167 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr,
3168 NULL, 0);
3169 if (rc) {
3170 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3171 debug_scsi("abort sent failure [itt 0x%x] %d", ctask->itt, rc);
3172 return rc;
3173 }
3174
3175 debug_scsi("abort sent [itt 0x%x]\n", ctask->itt);
3176
3177 spin_lock_bh(&session->lock);
3178 ctask->mtask = (struct iscsi_mgmt_task *)
3179 session->mgmt_cmds[(hdr->itt & ITT_MASK) -
3180 ISCSI_MGMT_ITT_OFFSET];
3181
3182 if (conn->tmabort_state == TMABORT_INITIAL) {
3183 conn->tmfcmd_pdus_cnt++;
3184 conn->tmabort_timer.expires = 10*HZ + jiffies;
3185 conn->tmabort_timer.function = iscsi_tmabort_timedout;
3186 conn->tmabort_timer.data = (unsigned long)ctask;
3187 add_timer(&conn->tmabort_timer);
3188 debug_scsi("abort set timeout [itt 0x%x]", ctask->itt);
3189 }
3190 spin_unlock_bh(&session->lock);
3191 mutex_unlock(&conn->xmitmutex);
3192
3193 /*
3194 * block eh thread until:
3195 *
3196 * 1) abort response
3197 * 2) abort timeout
3198 * 3) session is terminated or restarted or userspace has
3199 * given up on recovery
3200 */
3201 wait_event_interruptible(conn->ehwait,
3202 sc->SCp.phase != session->age ||
3203 session->state != ISCSI_STATE_LOGGED_IN ||
3204 conn->tmabort_state != TMABORT_INITIAL ||
3205 session->recovery_failed);
3206 if (signal_pending(current))
3207 flush_signals(current);
3208 del_timer_sync(&conn->tmabort_timer);
3209
3210 mutex_lock(&conn->xmitmutex);
3211 return 0;
3212}
3213
3214static int
3215iscsi_eh_abort(struct scsi_cmnd *sc)
3216{
3217 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
3218 struct iscsi_cmd_task *tmp_ctask;
3219 struct iscsi_conn *conn = ctask->conn;
3220 struct iscsi_session *session = conn->session;
3221 int rc;
3222
3223 conn->eh_abort_cnt++;
3224 debug_scsi("aborting [sc %p itt 0x%x]\n", sc, ctask->itt);
3225
3226 mutex_lock(&conn->xmitmutex);
3227 spin_lock_bh(&session->lock);
3228
3229 /*
3230 * If we are not logged in or we have started a new session
3231 * then let the host reset code handle this
3232 */
3233 if (session->state != ISCSI_STATE_LOGGED_IN ||
3234 sc->SCp.phase != session->age)
3235 goto failed;
3236
3237 /* ctask completed before time out */
3238 if (!ctask->sc)
3239 goto success;
3240
3241 /* what should we do here ? */
3242 if (conn->ctask == ctask) {
3243 printk(KERN_INFO "sc %p itt 0x%x partially sent. Failing "
3244 "abort\n", sc, ctask->itt);
3245 goto failed;
3246 }
3247
3248 /* check for the easy pending cmd abort */
3249 while (__kfifo_get(conn->xmitqueue, (void*)&tmp_ctask, sizeof(void*))) {
3250 if (tmp_ctask->itt == ctask->itt) {
3251 debug_scsi("found pending task\n");
3252 goto success;
3253 }
3254 __kfifo_put(conn->xmitqueue, (void*)&tmp_ctask, sizeof(void*));
3255 }
3256
3257 conn->tmabort_state = TMABORT_INITIAL;
3258
3259 spin_unlock_bh(&session->lock);
3260 rc = iscsi_exec_abort_task(sc, ctask);
3261 spin_lock_bh(&session->lock);
3262
3263 iscsi_ctask_mtask_cleanup(ctask);
3264 if (rc || sc->SCp.phase != session->age ||
3265 session->state != ISCSI_STATE_LOGGED_IN)
3266 goto failed;
3267
3268 /* ctask completed before tmf abort response */
3269 if (!ctask->sc) {
3270 debug_scsi("sc completed while abort in progress\n");
3271 goto success;
3272 }
3273
3274 if (conn->tmabort_state != TMABORT_SUCCESS) {
3275 spin_unlock_bh(&session->lock);
3276 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3277 spin_lock_bh(&session->lock);
3278 goto failed;
3279 }
3280
3281success:
3282 debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3283 spin_unlock_bh(&session->lock);
3284
3285 /* clean up task if aborted */
3286 if (conn->sock) {
3287 struct sock *sk = conn->sock->sk;
3288
3289 write_lock_bh(&sk->sk_callback_lock);
3290 spin_lock(&session->lock);
3291 fail_command(conn, ctask, DRIVER_TIMEOUT << 24);
3292 spin_unlock(&session->lock);
3293 write_unlock_bh(&sk->sk_callback_lock);
3294 }
3295 mutex_unlock(&conn->xmitmutex);
3296 return SUCCESS;
3297
3298failed:
3299 spin_unlock_bh(&session->lock);
3300 mutex_unlock(&conn->xmitmutex);
3301
3302 debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3303 return FAILED;
3304} 2086}
3305 2087
3306static int 2088static int
@@ -3314,6 +2096,7 @@ iscsi_r2tpool_alloc(struct iscsi_session *session)
3314 */ 2096 */
3315 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 2097 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3316 struct iscsi_cmd_task *ctask = session->cmds[cmd_i]; 2098 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2099 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
3317 2100
3318 /* 2101 /*
3319 * pre-allocated x4 as much r2ts to handle race when 2102 * pre-allocated x4 as much r2ts to handle race when
@@ -3322,16 +2105,18 @@ iscsi_r2tpool_alloc(struct iscsi_session *session)
3322 */ 2105 */
3323 2106
3324 /* R2T pool */ 2107 /* R2T pool */
3325 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4, 2108 if (iscsi_pool_init(&tcp_ctask->r2tpool, session->max_r2t * 4,
3326 (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) { 2109 (void***)&tcp_ctask->r2ts,
2110 sizeof(struct iscsi_r2t_info))) {
3327 goto r2t_alloc_fail; 2111 goto r2t_alloc_fail;
3328 } 2112 }
3329 2113
3330 /* R2T xmit queue */ 2114 /* R2T xmit queue */
3331 ctask->r2tqueue = kfifo_alloc( 2115 tcp_ctask->r2tqueue = kfifo_alloc(
3332 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL); 2116 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
3333 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) { 2117 if (tcp_ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
3334 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts); 2118 iscsi_pool_free(&tcp_ctask->r2tpool,
2119 (void**)tcp_ctask->r2ts);
3335 goto r2t_alloc_fail; 2120 goto r2t_alloc_fail;
3336 } 2121 }
3337 2122
@@ -3340,24 +2125,28 @@ iscsi_r2tpool_alloc(struct iscsi_session *session)
3340 * Data-Out PDU's within R2T-sequence can be quite big; 2125 * Data-Out PDU's within R2T-sequence can be quite big;
3341 * using mempool 2126 * using mempool
3342 */ 2127 */
3343 ctask->datapool = mempool_create_slab_pool(ISCSI_DTASK_DEFAULT_MAX, 2128 tcp_ctask->datapool = mempool_create_slab_pool(ISCSI_DTASK_DEFAULT_MAX,
3344 taskcache); 2129 taskcache);
3345 if (ctask->datapool == NULL) { 2130 if (tcp_ctask->datapool == NULL) {
3346 kfifo_free(ctask->r2tqueue); 2131 kfifo_free(tcp_ctask->r2tqueue);
3347 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts); 2132 iscsi_pool_free(&tcp_ctask->r2tpool,
2133 (void**)tcp_ctask->r2ts);
3348 goto r2t_alloc_fail; 2134 goto r2t_alloc_fail;
3349 } 2135 }
3350 INIT_LIST_HEAD(&ctask->dataqueue); 2136 INIT_LIST_HEAD(&tcp_ctask->dataqueue);
3351 } 2137 }
3352 2138
3353 return 0; 2139 return 0;
3354 2140
3355r2t_alloc_fail: 2141r2t_alloc_fail:
3356 for (i = 0; i < cmd_i; i++) { 2142 for (i = 0; i < cmd_i; i++) {
3357 mempool_destroy(session->cmds[i]->datapool); 2143 struct iscsi_cmd_task *ctask = session->cmds[i];
3358 kfifo_free(session->cmds[i]->r2tqueue); 2144 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
3359 iscsi_pool_free(&session->cmds[i]->r2tpool, 2145
3360 (void**)session->cmds[i]->r2ts); 2146 mempool_destroy(tcp_ctask->datapool);
2147 kfifo_free(tcp_ctask->r2tqueue);
2148 iscsi_pool_free(&tcp_ctask->r2tpool,
2149 (void**)tcp_ctask->r2ts);
3361 } 2150 }
3362 return -ENOMEM; 2151 return -ENOMEM;
3363} 2152}
@@ -3368,128 +2157,14 @@ iscsi_r2tpool_free(struct iscsi_session *session)
3368 int i; 2157 int i;
3369 2158
3370 for (i = 0; i < session->cmds_max; i++) { 2159 for (i = 0; i < session->cmds_max; i++) {
3371 mempool_destroy(session->cmds[i]->datapool); 2160 struct iscsi_cmd_task *ctask = session->cmds[i];
3372 kfifo_free(session->cmds[i]->r2tqueue); 2161 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
3373 iscsi_pool_free(&session->cmds[i]->r2tpool,
3374 (void**)session->cmds[i]->r2ts);
3375 }
3376}
3377
3378static struct scsi_host_template iscsi_sht = {
3379 .name = "iSCSI Initiator over TCP/IP, v."
3380 ISCSI_VERSION_STR,
3381 .queuecommand = iscsi_queuecommand,
3382 .change_queue_depth = iscsi_change_queue_depth,
3383 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
3384 .sg_tablesize = ISCSI_SG_TABLESIZE,
3385 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
3386 .eh_abort_handler = iscsi_eh_abort,
3387 .eh_host_reset_handler = iscsi_eh_host_reset,
3388 .use_clustering = DISABLE_CLUSTERING,
3389 .proc_name = "iscsi_tcp",
3390 .this_id = -1,
3391};
3392
3393static struct iscsi_transport iscsi_tcp_transport;
3394 2162
3395static struct iscsi_cls_session * 2163 mempool_destroy(tcp_ctask->datapool);
3396iscsi_session_create(struct scsi_transport_template *scsit, 2164 kfifo_free(tcp_ctask->r2tqueue);
3397 uint32_t initial_cmdsn, uint32_t *hostno) 2165 iscsi_pool_free(&tcp_ctask->r2tpool,
3398{ 2166 (void**)tcp_ctask->r2ts);
3399 struct Scsi_Host *shost;
3400 struct iscsi_session *session;
3401 int cmd_i;
3402
3403 shost = iscsi_transport_create_session(scsit, &iscsi_tcp_transport);
3404 if (!shost)
3405 return NULL;
3406
3407 session = iscsi_hostdata(shost->hostdata);
3408 memset(session, 0, sizeof(struct iscsi_session));
3409 session->host = shost;
3410 session->state = ISCSI_STATE_FREE;
3411 session->mgmtpool_max = ISCSI_MGMT_CMDS_MAX;
3412 session->cmds_max = ISCSI_XMIT_CMDS_MAX;
3413 session->cmdsn = initial_cmdsn;
3414 session->exp_cmdsn = initial_cmdsn + 1;
3415 session->max_cmdsn = initial_cmdsn + 1;
3416 session->max_r2t = 1;
3417
3418 *hostno = shost->host_no;
3419
3420 /* initialize SCSI PDU commands pool */
3421 if (iscsi_pool_init(&session->cmdpool, session->cmds_max,
3422 (void***)&session->cmds, sizeof(struct iscsi_cmd_task)))
3423 goto cmdpool_alloc_fail;
3424
3425 /* pre-format cmds pool with ITT */
3426 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++)
3427 session->cmds[cmd_i]->itt = cmd_i;
3428
3429 spin_lock_init(&session->lock);
3430 INIT_LIST_HEAD(&session->connections);
3431
3432 /* initialize immediate command pool */
3433 if (iscsi_pool_init(&session->mgmtpool, session->mgmtpool_max,
3434 (void***)&session->mgmt_cmds, sizeof(struct iscsi_mgmt_task)))
3435 goto mgmtpool_alloc_fail;
3436
3437
3438 /* pre-format immediate cmds pool with ITT */
3439 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
3440 session->mgmt_cmds[cmd_i]->itt = ISCSI_MGMT_ITT_OFFSET + cmd_i;
3441 session->mgmt_cmds[cmd_i]->data = kmalloc(
3442 DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, GFP_KERNEL);
3443 if (!session->mgmt_cmds[cmd_i]->data) {
3444 int j;
3445
3446 for (j = 0; j < cmd_i; j++)
3447 kfree(session->mgmt_cmds[j]->data);
3448 goto immdata_alloc_fail;
3449 }
3450 }
3451
3452 if (iscsi_r2tpool_alloc(session))
3453 goto r2tpool_alloc_fail;
3454
3455 return hostdata_session(shost->hostdata);
3456
3457r2tpool_alloc_fail:
3458 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3459 kfree(session->mgmt_cmds[cmd_i]->data);
3460immdata_alloc_fail:
3461 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3462mgmtpool_alloc_fail:
3463 iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3464cmdpool_alloc_fail:
3465 iscsi_transport_destroy_session(shost);
3466 return NULL;
3467}
3468
3469static void
3470iscsi_session_destroy(struct iscsi_cls_session *cls_session)
3471{
3472 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
3473 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
3474 int cmd_i;
3475 struct iscsi_data_task *dtask, *n;
3476
3477 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3478 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3479 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
3480 list_del(&dtask->item);
3481 mempool_free(dtask, ctask->datapool);
3482 }
3483 } 2167 }
3484
3485 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3486 kfree(session->mgmt_cmds[cmd_i]->data);
3487
3488 iscsi_r2tpool_free(session);
3489 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3490 iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3491
3492 iscsi_transport_destroy_session(shost);
3493} 2168}
3494 2169
3495static int 2170static int
@@ -3498,13 +2173,14 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
3498{ 2173{
3499 struct iscsi_conn *conn = cls_conn->dd_data; 2174 struct iscsi_conn *conn = cls_conn->dd_data;
3500 struct iscsi_session *session = conn->session; 2175 struct iscsi_session *session = conn->session;
2176 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
3501 2177
3502 switch(param) { 2178 switch(param) {
3503 case ISCSI_PARAM_MAX_RECV_DLENGTH: { 2179 case ISCSI_PARAM_MAX_RECV_DLENGTH: {
3504 char *saveptr = conn->data; 2180 char *saveptr = tcp_conn->data;
3505 gfp_t flags = GFP_KERNEL; 2181 gfp_t flags = GFP_KERNEL;
3506 2182
3507 if (conn->data_size >= value) { 2183 if (tcp_conn->data_size >= value) {
3508 conn->max_recv_dlength = value; 2184 conn->max_recv_dlength = value;
3509 break; 2185 break;
3510 } 2186 }
@@ -3515,21 +2191,21 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
3515 spin_unlock_bh(&session->lock); 2191 spin_unlock_bh(&session->lock);
3516 2192
3517 if (value <= PAGE_SIZE) 2193 if (value <= PAGE_SIZE)
3518 conn->data = kmalloc(value, flags); 2194 tcp_conn->data = kmalloc(value, flags);
3519 else 2195 else
3520 conn->data = (void*)__get_free_pages(flags, 2196 tcp_conn->data = (void*)__get_free_pages(flags,
3521 get_order(value)); 2197 get_order(value));
3522 if (conn->data == NULL) { 2198 if (tcp_conn->data == NULL) {
3523 conn->data = saveptr; 2199 tcp_conn->data = saveptr;
3524 return -ENOMEM; 2200 return -ENOMEM;
3525 } 2201 }
3526 if (conn->data_size <= PAGE_SIZE) 2202 if (tcp_conn->data_size <= PAGE_SIZE)
3527 kfree(saveptr); 2203 kfree(saveptr);
3528 else 2204 else
3529 free_pages((unsigned long)saveptr, 2205 free_pages((unsigned long)saveptr,
3530 get_order(conn->data_size)); 2206 get_order(tcp_conn->data_size));
3531 conn->max_recv_dlength = value; 2207 conn->max_recv_dlength = value;
3532 conn->data_size = value; 2208 tcp_conn->data_size = value;
3533 } 2209 }
3534 break; 2210 break;
3535 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2211 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
@@ -3537,49 +2213,51 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
3537 break; 2213 break;
3538 case ISCSI_PARAM_HDRDGST_EN: 2214 case ISCSI_PARAM_HDRDGST_EN:
3539 conn->hdrdgst_en = value; 2215 conn->hdrdgst_en = value;
3540 conn->hdr_size = sizeof(struct iscsi_hdr); 2216 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
3541 if (conn->hdrdgst_en) { 2217 if (conn->hdrdgst_en) {
3542 conn->hdr_size += sizeof(__u32); 2218 tcp_conn->hdr_size += sizeof(__u32);
3543 if (!conn->tx_tfm) 2219 if (!tcp_conn->tx_tfm)
3544 conn->tx_tfm = crypto_alloc_tfm("crc32c", 0); 2220 tcp_conn->tx_tfm = crypto_alloc_tfm("crc32c",
3545 if (!conn->tx_tfm) 2221 0);
2222 if (!tcp_conn->tx_tfm)
3546 return -ENOMEM; 2223 return -ENOMEM;
3547 if (!conn->rx_tfm) 2224 if (!tcp_conn->rx_tfm)
3548 conn->rx_tfm = crypto_alloc_tfm("crc32c", 0); 2225 tcp_conn->rx_tfm = crypto_alloc_tfm("crc32c",
3549 if (!conn->rx_tfm) { 2226 0);
3550 crypto_free_tfm(conn->tx_tfm); 2227 if (!tcp_conn->rx_tfm) {
2228 crypto_free_tfm(tcp_conn->tx_tfm);
3551 return -ENOMEM; 2229 return -ENOMEM;
3552 } 2230 }
3553 } else { 2231 } else {
3554 if (conn->tx_tfm) 2232 if (tcp_conn->tx_tfm)
3555 crypto_free_tfm(conn->tx_tfm); 2233 crypto_free_tfm(tcp_conn->tx_tfm);
3556 if (conn->rx_tfm) 2234 if (tcp_conn->rx_tfm)
3557 crypto_free_tfm(conn->rx_tfm); 2235 crypto_free_tfm(tcp_conn->rx_tfm);
3558 } 2236 }
3559 break; 2237 break;
3560 case ISCSI_PARAM_DATADGST_EN: 2238 case ISCSI_PARAM_DATADGST_EN:
3561 conn->datadgst_en = value; 2239 conn->datadgst_en = value;
3562 if (conn->datadgst_en) { 2240 if (conn->datadgst_en) {
3563 if (!conn->data_tx_tfm) 2241 if (!tcp_conn->data_tx_tfm)
3564 conn->data_tx_tfm = 2242 tcp_conn->data_tx_tfm =
3565 crypto_alloc_tfm("crc32c", 0); 2243 crypto_alloc_tfm("crc32c", 0);
3566 if (!conn->data_tx_tfm) 2244 if (!tcp_conn->data_tx_tfm)
3567 return -ENOMEM; 2245 return -ENOMEM;
3568 if (!conn->data_rx_tfm) 2246 if (!tcp_conn->data_rx_tfm)
3569 conn->data_rx_tfm = 2247 tcp_conn->data_rx_tfm =
3570 crypto_alloc_tfm("crc32c", 0); 2248 crypto_alloc_tfm("crc32c", 0);
3571 if (!conn->data_rx_tfm) { 2249 if (!tcp_conn->data_rx_tfm) {
3572 crypto_free_tfm(conn->data_tx_tfm); 2250 crypto_free_tfm(tcp_conn->data_tx_tfm);
3573 return -ENOMEM; 2251 return -ENOMEM;
3574 } 2252 }
3575 } else { 2253 } else {
3576 if (conn->data_tx_tfm) 2254 if (tcp_conn->data_tx_tfm)
3577 crypto_free_tfm(conn->data_tx_tfm); 2255 crypto_free_tfm(tcp_conn->data_tx_tfm);
3578 if (conn->data_rx_tfm) 2256 if (tcp_conn->data_rx_tfm)
3579 crypto_free_tfm(conn->data_rx_tfm); 2257 crypto_free_tfm(tcp_conn->data_rx_tfm);
3580 } 2258 }
3581 conn->sendpage = conn->datadgst_en ? 2259 tcp_conn->sendpage = conn->datadgst_en ?
3582 sock_no_sendpage : conn->sock->ops->sendpage; 2260 sock_no_sendpage : tcp_conn->sock->ops->sendpage;
3583 break; 2261 break;
3584 case ISCSI_PARAM_INITIAL_R2T_EN: 2262 case ISCSI_PARAM_INITIAL_R2T_EN:
3585 session->initial_r2t_en = value; 2263 session->initial_r2t_en = value;
@@ -3677,6 +2355,7 @@ iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn,
3677 enum iscsi_param param, uint32_t *value) 2355 enum iscsi_param param, uint32_t *value)
3678{ 2356{
3679 struct iscsi_conn *conn = cls_conn->dd_data; 2357 struct iscsi_conn *conn = cls_conn->dd_data;
2358 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
3680 struct inet_sock *inet; 2359 struct inet_sock *inet;
3681 2360
3682 switch(param) { 2361 switch(param) {
@@ -3694,12 +2373,12 @@ iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn,
3694 break; 2373 break;
3695 case ISCSI_PARAM_CONN_PORT: 2374 case ISCSI_PARAM_CONN_PORT:
3696 mutex_lock(&conn->xmitmutex); 2375 mutex_lock(&conn->xmitmutex);
3697 if (!conn->sock) { 2376 if (!tcp_conn->sock) {
3698 mutex_unlock(&conn->xmitmutex); 2377 mutex_unlock(&conn->xmitmutex);
3699 return -EINVAL; 2378 return -EINVAL;
3700 } 2379 }
3701 2380
3702 inet = inet_sk(conn->sock->sk); 2381 inet = inet_sk(tcp_conn->sock->sk);
3703 *value = be16_to_cpu(inet->dport); 2382 *value = be16_to_cpu(inet->dport);
3704 mutex_unlock(&conn->xmitmutex); 2383 mutex_unlock(&conn->xmitmutex);
3705 default: 2384 default:
@@ -3714,6 +2393,7 @@ iscsi_conn_get_str_param(struct iscsi_cls_conn *cls_conn,
3714 enum iscsi_param param, char *buf) 2393 enum iscsi_param param, char *buf)
3715{ 2394{
3716 struct iscsi_conn *conn = cls_conn->dd_data; 2395 struct iscsi_conn *conn = cls_conn->dd_data;
2396 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
3717 struct sock *sk; 2397 struct sock *sk;
3718 struct inet_sock *inet; 2398 struct inet_sock *inet;
3719 struct ipv6_pinfo *np; 2399 struct ipv6_pinfo *np;
@@ -3722,12 +2402,12 @@ iscsi_conn_get_str_param(struct iscsi_cls_conn *cls_conn,
3722 switch (param) { 2402 switch (param) {
3723 case ISCSI_PARAM_CONN_ADDRESS: 2403 case ISCSI_PARAM_CONN_ADDRESS:
3724 mutex_lock(&conn->xmitmutex); 2404 mutex_lock(&conn->xmitmutex);
3725 if (!conn->sock) { 2405 if (!tcp_conn->sock) {
3726 mutex_unlock(&conn->xmitmutex); 2406 mutex_unlock(&conn->xmitmutex);
3727 return -EINVAL; 2407 return -EINVAL;
3728 } 2408 }
3729 2409
3730 sk = conn->sock->sk; 2410 sk = tcp_conn->sock->sk;
3731 if (sk->sk_family == PF_INET) { 2411 if (sk->sk_family == PF_INET) {
3732 inet = inet_sk(sk); 2412 inet = inet_sk(sk);
3733 len = sprintf(buf, "%u.%u.%u.%u\n", 2413 len = sprintf(buf, "%u.%u.%u.%u\n",
@@ -3751,6 +2431,7 @@ static void
3751iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) 2431iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
3752{ 2432{
3753 struct iscsi_conn *conn = cls_conn->dd_data; 2433 struct iscsi_conn *conn = cls_conn->dd_data;
2434 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
3754 2435
3755 stats->txdata_octets = conn->txdata_octets; 2436 stats->txdata_octets = conn->txdata_octets;
3756 stats->rxdata_octets = conn->rxdata_octets; 2437 stats->rxdata_octets = conn->rxdata_octets;
@@ -3763,27 +2444,92 @@ iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
3763 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 2444 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
3764 stats->custom_length = 3; 2445 stats->custom_length = 3;
3765 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 2446 strcpy(stats->custom[0].desc, "tx_sendpage_failures");
3766 stats->custom[0].value = conn->sendpage_failures_cnt; 2447 stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
3767 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 2448 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
3768 stats->custom[1].value = conn->discontiguous_hdr_cnt; 2449 stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
3769 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 2450 strcpy(stats->custom[2].desc, "eh_abort_cnt");
3770 stats->custom[2].value = conn->eh_abort_cnt; 2451 stats->custom[2].value = conn->eh_abort_cnt;
3771} 2452}
3772 2453
3773static int 2454static struct iscsi_cls_session *
3774iscsi_conn_send_pdu(struct iscsi_cls_conn *cls_conn, struct iscsi_hdr *hdr, 2455iscsi_tcp_session_create(struct iscsi_transport *iscsit,
3775 char *data, uint32_t data_size) 2456 struct scsi_transport_template *scsit,
2457 uint32_t initial_cmdsn, uint32_t *hostno)
3776{ 2458{
3777 struct iscsi_conn *conn = cls_conn->dd_data; 2459 struct iscsi_cls_session *cls_session;
3778 int rc; 2460 struct iscsi_session *session;
2461 uint32_t hn;
2462 int cmd_i;
3779 2463
3780 mutex_lock(&conn->xmitmutex); 2464 cls_session = iscsi_session_setup(iscsit, scsit,
3781 rc = iscsi_conn_send_generic(conn, hdr, data, data_size); 2465 sizeof(struct iscsi_tcp_cmd_task),
3782 mutex_unlock(&conn->xmitmutex); 2466 sizeof(struct iscsi_tcp_mgmt_task),
2467 initial_cmdsn, &hn);
2468 if (!cls_session)
2469 return NULL;
2470 *hostno = hn;
3783 2471
3784 return rc; 2472 session = class_to_transport_session(cls_session);
2473 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2474 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2475 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2476
2477 ctask->hdr = &tcp_ctask->hdr;
2478 }
2479
2480 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
2481 struct iscsi_mgmt_task *mtask = session->mgmt_cmds[cmd_i];
2482 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
2483
2484 mtask->hdr = &tcp_mtask->hdr;
2485 }
2486
2487 if (iscsi_r2tpool_alloc(class_to_transport_session(cls_session)))
2488 goto r2tpool_alloc_fail;
2489
2490 return cls_session;
2491
2492r2tpool_alloc_fail:
2493 iscsi_session_teardown(cls_session);
2494 return NULL;
3785} 2495}
3786 2496
2497static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
2498{
2499 struct iscsi_session *session = class_to_transport_session(cls_session);
2500 struct iscsi_data_task *dtask, *n;
2501 int cmd_i;
2502
2503 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2504 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2505 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2506
2507 list_for_each_entry_safe(dtask, n, &tcp_ctask->dataqueue,
2508 item) {
2509 list_del(&dtask->item);
2510 mempool_free(dtask, tcp_ctask->datapool);
2511 }
2512 }
2513
2514 iscsi_r2tpool_free(class_to_transport_session(cls_session));
2515 iscsi_session_teardown(cls_session);
2516}
2517
2518static struct scsi_host_template iscsi_sht = {
2519 .name = "iSCSI Initiator over TCP/IP, v."
2520 ISCSI_VERSION_STR,
2521 .queuecommand = iscsi_queuecommand,
2522 .change_queue_depth = iscsi_change_queue_depth,
2523 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
2524 .sg_tablesize = ISCSI_SG_TABLESIZE,
2525 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
2526 .eh_abort_handler = iscsi_eh_abort,
2527 .eh_host_reset_handler = iscsi_eh_host_reset,
2528 .use_clustering = DISABLE_CLUSTERING,
2529 .proc_name = "iscsi_tcp",
2530 .this_id = -1,
2531};
2532
3787static struct iscsi_transport iscsi_tcp_transport = { 2533static struct iscsi_transport iscsi_tcp_transport = {
3788 .owner = THIS_MODULE, 2534 .owner = THIS_MODULE,
3789 .name = "tcp", 2535 .name = "tcp",
@@ -3804,23 +2550,34 @@ static struct iscsi_transport iscsi_tcp_transport = {
3804 ISCSI_CONN_PORT | 2550 ISCSI_CONN_PORT |
3805 ISCSI_CONN_ADDRESS, 2551 ISCSI_CONN_ADDRESS,
3806 .host_template = &iscsi_sht, 2552 .host_template = &iscsi_sht,
3807 .hostdata_size = sizeof(struct iscsi_session),
3808 .conndata_size = sizeof(struct iscsi_conn), 2553 .conndata_size = sizeof(struct iscsi_conn),
3809 .max_conn = 1, 2554 .max_conn = 1,
3810 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN, 2555 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN,
3811 .create_session = iscsi_session_create, 2556 /* session management */
3812 .destroy_session = iscsi_session_destroy, 2557 .create_session = iscsi_tcp_session_create,
3813 .create_conn = iscsi_conn_create, 2558 .destroy_session = iscsi_tcp_session_destroy,
3814 .bind_conn = iscsi_conn_bind, 2559 /* connection management */
3815 .destroy_conn = iscsi_conn_destroy, 2560 .create_conn = iscsi_tcp_conn_create,
2561 .bind_conn = iscsi_tcp_conn_bind,
2562 .destroy_conn = iscsi_tcp_conn_destroy,
3816 .set_param = iscsi_conn_set_param, 2563 .set_param = iscsi_conn_set_param,
3817 .get_conn_param = iscsi_conn_get_param, 2564 .get_conn_param = iscsi_conn_get_param,
3818 .get_conn_str_param = iscsi_conn_get_str_param, 2565 .get_conn_str_param = iscsi_conn_get_str_param,
3819 .get_session_param = iscsi_session_get_param, 2566 .get_session_param = iscsi_session_get_param,
3820 .start_conn = iscsi_conn_start, 2567 .start_conn = iscsi_conn_start,
3821 .stop_conn = iscsi_conn_stop, 2568 .stop_conn = iscsi_conn_stop,
2569 /* these are called as part of conn recovery */
2570 .suspend_conn_recv = iscsi_tcp_suspend_conn_rx,
2571 .terminate_conn = iscsi_tcp_terminate_conn,
2572 /* IO */
3822 .send_pdu = iscsi_conn_send_pdu, 2573 .send_pdu = iscsi_conn_send_pdu,
3823 .get_stats = iscsi_conn_get_stats, 2574 .get_stats = iscsi_conn_get_stats,
2575 .init_cmd_task = iscsi_tcp_cmd_init,
2576 .init_mgmt_task = iscsi_tcp_mgmt_init,
2577 .xmit_cmd_task = iscsi_tcp_ctask_xmit,
2578 .xmit_mgmt_task = iscsi_tcp_mtask_xmit,
2579 .cleanup_cmd_task = iscsi_tcp_cleanup_ctask,
2580 /* recovery */
3824 .session_recovery_timedout = iscsi_session_recovery_timedout, 2581 .session_recovery_timedout = iscsi_session_recovery_timedout,
3825}; 2582};
3826 2583
diff --git a/drivers/scsi/iscsi_tcp.h b/drivers/scsi/iscsi_tcp.h
index 12ef64179b4c..c5918854d595 100644
--- a/drivers/scsi/iscsi_tcp.h
+++ b/drivers/scsi/iscsi_tcp.h
@@ -2,7 +2,8 @@
2 * iSCSI Initiator TCP Transport 2 * iSCSI Initiator TCP Transport
3 * Copyright (C) 2004 Dmitry Yusupov 3 * Copyright (C) 2004 Dmitry Yusupov
4 * Copyright (C) 2004 Alex Aizman 4 * Copyright (C) 2004 Alex Aizman
5 * Copyright (C) 2005 Mike Christie 5 * Copyright (C) 2005 - 2006 Mike Christie
6 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
6 * maintained by open-iscsi@googlegroups.com 7 * maintained by open-iscsi@googlegroups.com
7 * 8 *
8 * This program is free software; you can redistribute it and/or modify 9 * This program is free software; you can redistribute it and/or modify
@@ -21,20 +22,7 @@
21#ifndef ISCSI_TCP_H 22#ifndef ISCSI_TCP_H
22#define ISCSI_TCP_H 23#define ISCSI_TCP_H
23 24
24/* Session's states */ 25#include <scsi/libiscsi.h>
25#define ISCSI_STATE_FREE 1
26#define ISCSI_STATE_LOGGED_IN 2
27#define ISCSI_STATE_FAILED 3
28#define ISCSI_STATE_TERMINATE 4
29
30/* Connection's states */
31#define ISCSI_CONN_INITIAL_STAGE 0
32#define ISCSI_CONN_STARTED 1
33#define ISCSI_CONN_STOPPED 2
34#define ISCSI_CONN_CLEANUP_WAIT 3
35
36/* Connection suspend "bit" */
37#define SUSPEND_BIT 1
38 26
39/* Socket's Receive state machine */ 27/* Socket's Receive state machine */
40#define IN_PROGRESS_WAIT_HEADER 0x0 28#define IN_PROGRESS_WAIT_HEADER 0x0
@@ -42,12 +30,6 @@
42#define IN_PROGRESS_DATA_RECV 0x2 30#define IN_PROGRESS_DATA_RECV 0x2
43#define IN_PROGRESS_DDIGEST_RECV 0x3 31#define IN_PROGRESS_DDIGEST_RECV 0x3
44 32
45/* Task Mgmt states */
46#define TMABORT_INITIAL 0x0
47#define TMABORT_SUCCESS 0x1
48#define TMABORT_FAILED 0x2
49#define TMABORT_TIMEDOUT 0x3
50
51/* xmit state machine */ 33/* xmit state machine */
52#define XMSTATE_IDLE 0x0 34#define XMSTATE_IDLE 0x0
53#define XMSTATE_R_HDR 0x1 35#define XMSTATE_R_HDR 0x1
@@ -62,34 +44,14 @@
62#define XMSTATE_W_PAD 0x200 44#define XMSTATE_W_PAD 0x200
63#define XMSTATE_DATA_DIGEST 0x400 45#define XMSTATE_DATA_DIGEST 0x400
64 46
65#define ISCSI_CONN_MAX 1
66#define ISCSI_CONN_RCVBUF_MIN 262144 47#define ISCSI_CONN_RCVBUF_MIN 262144
67#define ISCSI_CONN_SNDBUF_MIN 262144 48#define ISCSI_CONN_SNDBUF_MIN 262144
68#define ISCSI_PAD_LEN 4 49#define ISCSI_PAD_LEN 4
69#define ISCSI_R2T_MAX 16 50#define ISCSI_R2T_MAX 16
70#define ISCSI_XMIT_CMDS_MAX 128 /* must be power of 2 */
71#define ISCSI_MGMT_CMDS_MAX 32 /* must be power of 2 */
72#define ISCSI_MGMT_ITT_OFFSET 0xa00
73#define ISCSI_SG_TABLESIZE SG_ALL 51#define ISCSI_SG_TABLESIZE SG_ALL
74#define ISCSI_DEF_CMD_PER_LUN 32
75#define ISCSI_MAX_CMD_PER_LUN 128
76#define ISCSI_TCP_MAX_CMD_LEN 16 52#define ISCSI_TCP_MAX_CMD_LEN 16
77 53
78#define ITT_MASK (0xfff) 54struct socket;
79#define CID_SHIFT 12
80#define CID_MASK (0xffff<<CID_SHIFT)
81#define AGE_SHIFT 28
82#define AGE_MASK (0xf<<AGE_SHIFT)
83
84struct iscsi_queue {
85 struct kfifo *queue; /* FIFO Queue */
86 void **pool; /* Pool of elements */
87 int max; /* Max number of elements */
88};
89
90struct iscsi_session;
91struct iscsi_cmd_task;
92struct iscsi_mgmt_task;
93 55
94/* Socket connection recieve helper */ 56/* Socket connection recieve helper */
95struct iscsi_tcp_recv { 57struct iscsi_tcp_recv {
@@ -104,48 +66,32 @@ struct iscsi_tcp_recv {
104 struct iscsi_cmd_task *ctask; /* current cmd in progress */ 66 struct iscsi_cmd_task *ctask; /* current cmd in progress */
105 67
106 /* copied and flipped values */ 68 /* copied and flipped values */
107 int opcode;
108 int flags;
109 int cmd_status;
110 int ahslen;
111 int datalen; 69 int datalen;
112 uint32_t itt;
113 int datadgst; 70 int datadgst;
71 char zero_copy_hdr;
114}; 72};
115 73
116struct iscsi_cls_conn; 74struct iscsi_tcp_conn {
117 75 struct iscsi_conn *iscsi_conn;
118struct iscsi_conn { 76 struct socket *sock;
119 struct iscsi_cls_conn *cls_conn; /* ptr to class connection */
120 struct iscsi_hdr hdr; /* header placeholder */ 77 struct iscsi_hdr hdr; /* header placeholder */
121 char hdrext[4*sizeof(__u16) + 78 char hdrext[4*sizeof(__u16) +
122 sizeof(__u32)]; 79 sizeof(__u32)];
123 int data_copied; 80 int data_copied;
124 char *data; /* data placeholder */ 81 char *data; /* data placeholder */
125 struct socket *sock; /* TCP socket */
126 int data_size; /* actual recv_dlength */ 82 int data_size; /* actual recv_dlength */
127 int stop_stage; /* conn_stop() flag: * 83 int stop_stage; /* conn_stop() flag: *
128 * stop to recover, * 84 * stop to recover, *
129 * stop to terminate */ 85 * stop to terminate */
130 /* iSCSI connection-wide sequencing */ 86 /* iSCSI connection-wide sequencing */
131 uint32_t exp_statsn;
132 int hdr_size; /* PDU header size */ 87 int hdr_size; /* PDU header size */
133 unsigned long suspend_rx; /* suspend Rx */
134 88
135 struct crypto_tfm *rx_tfm; /* CRC32C (Rx) */ 89 struct crypto_tfm *rx_tfm; /* CRC32C (Rx) */
136 struct crypto_tfm *data_rx_tfm; /* CRC32C (Rx) for data */ 90 struct crypto_tfm *data_rx_tfm; /* CRC32C (Rx) for data */
137 91
138 /* control data */ 92 /* control data */
139 int senselen; /* scsi sense length */
140 int id; /* CID */
141 struct iscsi_tcp_recv in; /* TCP receive context */ 93 struct iscsi_tcp_recv in; /* TCP receive context */
142 struct iscsi_session *session; /* parent session */
143 struct list_head item; /* maintains list of conns */
144 int in_progress; /* connection state machine */ 94 int in_progress; /* connection state machine */
145 int c_stage; /* connection state */
146 struct iscsi_mgmt_task *login_mtask; /* mtask used for login/text */
147 struct iscsi_mgmt_task *mtask; /* xmit mtask in progress */
148 struct iscsi_cmd_task *ctask; /* xmit ctask in progress */
149 95
150 /* old values for socket callbacks */ 96 /* old values for socket callbacks */
151 void (*old_data_ready)(struct sock *, int); 97 void (*old_data_ready)(struct sock *, int);
@@ -155,95 +101,14 @@ struct iscsi_conn {
155 /* xmit */ 101 /* xmit */
156 struct crypto_tfm *tx_tfm; /* CRC32C (Tx) */ 102 struct crypto_tfm *tx_tfm; /* CRC32C (Tx) */
157 struct crypto_tfm *data_tx_tfm; /* CRC32C (Tx) for data */ 103 struct crypto_tfm *data_tx_tfm; /* CRC32C (Tx) for data */
158 struct kfifo *writequeue; /* write cmds for Data-Outs */
159 struct kfifo *immqueue; /* immediate xmit queue */
160 struct kfifo *mgmtqueue; /* mgmt (control) xmit queue */
161 struct kfifo *xmitqueue; /* data-path cmd queue */
162 struct list_head run_list; /* list of cmds in progress */
163 struct work_struct xmitwork; /* per-conn. xmit workqueue */
164 struct mutex xmitmutex; /* serializes connection xmit,
165 * access to kfifos: *
166 * xmitqueue, writequeue, *
167 * immqueue, mgmtqueue */
168 unsigned long suspend_tx; /* suspend Tx */
169
170 /* abort */
171 wait_queue_head_t ehwait; /* used in eh_abort() */
172 struct iscsi_tm tmhdr;
173 struct timer_list tmabort_timer; /* abort timer */
174 int tmabort_state; /* see TMABORT_INITIAL, etc.*/
175
176 /* negotiated params */
177 int max_recv_dlength;
178 int max_xmit_dlength;
179 int hdrdgst_en;
180 int datadgst_en;
181 104
182 /* MIB-statistics */ 105 /* MIB custom statistics */
183 uint64_t txdata_octets;
184 uint64_t rxdata_octets;
185 uint32_t scsicmd_pdus_cnt;
186 uint32_t dataout_pdus_cnt;
187 uint32_t scsirsp_pdus_cnt;
188 uint32_t datain_pdus_cnt;
189 uint32_t r2t_pdus_cnt;
190 uint32_t tmfcmd_pdus_cnt;
191 int32_t tmfrsp_pdus_cnt;
192
193 /* custom statistics */
194 uint32_t sendpage_failures_cnt; 106 uint32_t sendpage_failures_cnt;
195 uint32_t discontiguous_hdr_cnt; 107 uint32_t discontiguous_hdr_cnt;
196 uint32_t eh_abort_cnt;
197 108
198 ssize_t (*sendpage)(struct socket *, struct page *, int, size_t, int); 109 ssize_t (*sendpage)(struct socket *, struct page *, int, size_t, int);
199}; 110};
200 111
201struct iscsi_session {
202 /* iSCSI session-wide sequencing */
203 uint32_t cmdsn;
204 uint32_t exp_cmdsn;
205 uint32_t max_cmdsn;
206
207 /* configuration */
208 int initial_r2t_en;
209 int max_r2t;
210 int imm_data_en;
211 int first_burst;
212 int max_burst;
213 int time2wait;
214 int time2retain;
215 int pdu_inorder_en;
216 int dataseq_inorder_en;
217 int erl;
218 int ifmarker_en;
219 int ofmarker_en;
220
221 /* control data */
222 struct Scsi_Host *host;
223 int id;
224 struct iscsi_conn *leadconn; /* leading connection */
225 spinlock_t lock; /* protects session state, *
226 * sequence numbers, *
227 * session resources: *
228 * - cmdpool, *
229 * - mgmtpool, *
230 * - r2tpool */
231 int state; /* session state */
232 int recovery_failed;
233 struct list_head item;
234 void *auth_client;
235 int conn_cnt;
236 int age; /* counts session re-opens */
237
238 struct list_head connections; /* list of connections */
239 int cmds_max; /* size of cmds array */
240 struct iscsi_cmd_task **cmds; /* Original Cmds arr */
241 struct iscsi_queue cmdpool; /* PDU's pool */
242 int mgmtpool_max; /* size of mgmt array */
243 struct iscsi_mgmt_task **mgmt_cmds; /* Original mgmt arr */
244 struct iscsi_queue mgmtpool; /* Mgmt PDU's pool */
245};
246
247struct iscsi_buf { 112struct iscsi_buf {
248 struct scatterlist sg; 113 struct scatterlist sg;
249 unsigned int sent; 114 unsigned int sent;
@@ -259,16 +124,13 @@ struct iscsi_data_task {
259}; 124};
260#define ISCSI_DTASK_DEFAULT_MAX ISCSI_SG_TABLESIZE * PAGE_SIZE / 512 125#define ISCSI_DTASK_DEFAULT_MAX ISCSI_SG_TABLESIZE * PAGE_SIZE / 512
261 126
262struct iscsi_mgmt_task { 127struct iscsi_tcp_mgmt_task {
263 struct iscsi_hdr hdr; /* mgmt. PDU */ 128 struct iscsi_hdr hdr;
264 char hdrext[sizeof(__u32)]; /* Header-Digest */ 129 char hdrext[sizeof(__u32)]; /* Header-Digest */
265 char *data; /* mgmt payload */
266 int xmstate; /* mgmt xmit progress */ 130 int xmstate; /* mgmt xmit progress */
267 int data_count; /* counts data to be sent */
268 struct iscsi_buf headbuf; /* header buffer */ 131 struct iscsi_buf headbuf; /* header buffer */
269 struct iscsi_buf sendbuf; /* in progress buffer */ 132 struct iscsi_buf sendbuf; /* in progress buffer */
270 int sent; 133 int sent;
271 uint32_t itt; /* this ITT */
272}; 134};
273 135
274struct iscsi_r2t_info { 136struct iscsi_r2t_info {
@@ -285,34 +147,22 @@ struct iscsi_r2t_info {
285 struct iscsi_data_task *dtask; /* which data task */ 147 struct iscsi_data_task *dtask; /* which data task */
286}; 148};
287 149
288struct iscsi_cmd_task { 150struct iscsi_tcp_cmd_task {
289 struct iscsi_cmd hdr; /* iSCSI PDU header */ 151 struct iscsi_cmd hdr;
290 char hdrext[4*sizeof(__u16)+ /* AHS */ 152 char hdrext[4*sizeof(__u16)+ /* AHS */
291 sizeof(__u32)]; /* HeaderDigest */ 153 sizeof(__u32)]; /* HeaderDigest */
292 char pad[ISCSI_PAD_LEN]; 154 char pad[ISCSI_PAD_LEN];
293 int itt; /* this ITT */ 155 int pad_count; /* padded bytes */
294 int datasn; /* DataSN */
295 struct iscsi_buf headbuf; /* header buf (xmit) */ 156 struct iscsi_buf headbuf; /* header buf (xmit) */
296 struct iscsi_buf sendbuf; /* in progress buffer*/ 157 struct iscsi_buf sendbuf; /* in progress buffer*/
158 int xmstate; /* xmit xtate machine */
297 int sent; 159 int sent;
298 struct scatterlist *sg; /* per-cmd SG list */ 160 struct scatterlist *sg; /* per-cmd SG list */
299 struct scatterlist *bad_sg; /* assert statement */ 161 struct scatterlist *bad_sg; /* assert statement */
300 int sg_count; /* SG's to process */ 162 int sg_count; /* SG's to process */
301 uint32_t unsol_datasn;
302 uint32_t exp_r2tsn; 163 uint32_t exp_r2tsn;
303 int xmstate; /* xmit xtate machine */
304 int imm_count; /* imm-data (bytes) */
305 int unsol_count; /* unsolicited (bytes)*/
306 int r2t_data_count; /* R2T Data-Out bytes */ 164 int r2t_data_count; /* R2T Data-Out bytes */
307 int data_count; /* remaining Data-Out */
308 int pad_count; /* padded bytes */
309 struct scsi_cmnd *sc; /* associated SCSI cmd*/
310 int total_length;
311 int data_offset; 165 int data_offset;
312 struct iscsi_conn *conn; /* used connection */
313 struct iscsi_mgmt_task *mtask; /* tmf mtask in progr */
314
315 struct list_head running; /* running cmd list */
316 struct iscsi_r2t_info *r2t; /* in progress R2T */ 166 struct iscsi_r2t_info *r2t; /* in progress R2T */
317 struct iscsi_queue r2tpool; 167 struct iscsi_queue r2tpool;
318 struct kfifo *r2tqueue; 168 struct kfifo *r2tqueue;
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-28 19:20:33 -0500