aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi/iscsi_tcp.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/iscsi_tcp.c')
-rw-r--r--drivers/scsi/iscsi_tcp.c3006
1 files changed, 953 insertions, 2053 deletions
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 2068b66822b7..b4743a9ecc80 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -3,7 +3,8 @@
3 * 3 *
4 * Copyright (C) 2004 Dmitry Yusupov 4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman 5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 Mike Christie 6 * Copyright (C) 2005 - 2006 Mike Christie
7 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
7 * maintained by open-iscsi@googlegroups.com 8 * maintained by open-iscsi@googlegroups.com
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
@@ -36,37 +37,28 @@
36#include <linux/mutex.h> 37#include <linux/mutex.h>
37#include <net/tcp.h> 38#include <net/tcp.h>
38#include <scsi/scsi_cmnd.h> 39#include <scsi/scsi_cmnd.h>
39#include <scsi/scsi_device.h>
40#include <scsi/scsi_eh.h>
41#include <scsi/scsi_request.h>
42#include <scsi/scsi_tcq.h>
43#include <scsi/scsi_host.h> 40#include <scsi/scsi_host.h>
44#include <scsi/scsi.h> 41#include <scsi/scsi.h>
45#include <scsi/scsi_transport_iscsi.h> 42#include <scsi/scsi_transport_iscsi.h>
46 43
47#include "iscsi_tcp.h" 44#include "iscsi_tcp.h"
48 45
46#define ISCSI_TCP_VERSION "1.0-595"
47
49MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, " 48MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
50 "Alex Aizman <itn780@yahoo.com>"); 49 "Alex Aizman <itn780@yahoo.com>");
51MODULE_DESCRIPTION("iSCSI/TCP data-path"); 50MODULE_DESCRIPTION("iSCSI/TCP data-path");
52MODULE_LICENSE("GPL"); 51MODULE_LICENSE("GPL");
53MODULE_VERSION("0:4.445"); 52MODULE_VERSION(ISCSI_TCP_VERSION);
54/* #define DEBUG_TCP */ 53/* #define DEBUG_TCP */
55/* #define DEBUG_SCSI */
56#define DEBUG_ASSERT 54#define DEBUG_ASSERT
57 55
58#ifdef DEBUG_TCP 56#ifdef DEBUG_TCP
59#define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt) 57#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
60#else 58#else
61#define debug_tcp(fmt...) 59#define debug_tcp(fmt...)
62#endif 60#endif
63 61
64#ifdef DEBUG_SCSI
65#define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
66#else
67#define debug_scsi(fmt...)
68#endif
69
70#ifndef DEBUG_ASSERT 62#ifndef DEBUG_ASSERT
71#ifdef BUG_ON 63#ifdef BUG_ON
72#undef BUG_ON 64#undef BUG_ON
@@ -74,22 +66,9 @@ MODULE_VERSION("0:4.445");
74#define BUG_ON(expr) 66#define BUG_ON(expr)
75#endif 67#endif
76 68
77#define INVALID_SN_DELTA 0xffff
78
79static unsigned int iscsi_max_lun = 512; 69static unsigned int iscsi_max_lun = 512;
80module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 70module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
81 71
82/* global data */
83static kmem_cache_t *taskcache;
84
85static inline void
86iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
87{
88 sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
89 ibuf->sent = 0;
90 ibuf->use_sendmsg = 0;
91}
92
93static inline void 72static inline void
94iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size) 73iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
95{ 74{
@@ -130,68 +109,39 @@ static inline void
130iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf, 109iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
131 u8* crc) 110 u8* crc)
132{ 111{
133 crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc); 112 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
134 buf->sg.length += sizeof(uint32_t);
135}
136 113
137static void 114 crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc);
138iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err) 115 buf->sg.length += sizeof(uint32_t);
139{
140 struct iscsi_session *session = conn->session;
141 unsigned long flags;
142
143 spin_lock_irqsave(&session->lock, flags);
144 if (session->conn_cnt == 1 || session->leadconn == conn)
145 session->state = ISCSI_STATE_FAILED;
146 spin_unlock_irqrestore(&session->lock, flags);
147 set_bit(SUSPEND_BIT, &conn->suspend_tx);
148 set_bit(SUSPEND_BIT, &conn->suspend_rx);
149 iscsi_conn_error(conn->cls_conn, err);
150} 116}
151 117
152static inline int 118static inline int
153iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr) 119iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
154{ 120{
155 uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn); 121 struct sk_buff *skb = tcp_conn->in.skb;
156 uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
157
158 if (max_cmdsn < exp_cmdsn -1 &&
159 max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
160 return ISCSI_ERR_MAX_CMDSN;
161 if (max_cmdsn > session->max_cmdsn ||
162 max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
163 session->max_cmdsn = max_cmdsn;
164 if (exp_cmdsn > session->exp_cmdsn ||
165 exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
166 session->exp_cmdsn = exp_cmdsn;
167 122
168 return 0; 123 tcp_conn->in.zero_copy_hdr = 0;
169}
170 124
171static inline int 125 if (tcp_conn->in.copy >= tcp_conn->hdr_size &&
172iscsi_hdr_extract(struct iscsi_conn *conn) 126 tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
173{
174 struct sk_buff *skb = conn->in.skb;
175
176 if (conn->in.copy >= conn->hdr_size &&
177 conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
178 /* 127 /*
179 * Zero-copy PDU Header: using connection context 128 * Zero-copy PDU Header: using connection context
180 * to store header pointer. 129 * to store header pointer.
181 */ 130 */
182 if (skb_shinfo(skb)->frag_list == NULL && 131 if (skb_shinfo(skb)->frag_list == NULL &&
183 !skb_shinfo(skb)->nr_frags) 132 !skb_shinfo(skb)->nr_frags) {
184 conn->in.hdr = (struct iscsi_hdr *) 133 tcp_conn->in.hdr = (struct iscsi_hdr *)
185 ((char*)skb->data + conn->in.offset); 134 ((char*)skb->data + tcp_conn->in.offset);
186 else { 135 tcp_conn->in.zero_copy_hdr = 1;
136 } else {
187 /* ignoring return code since we checked 137 /* ignoring return code since we checked
188 * in.copy before */ 138 * in.copy before */
189 skb_copy_bits(skb, conn->in.offset, 139 skb_copy_bits(skb, tcp_conn->in.offset,
190 &conn->hdr, conn->hdr_size); 140 &tcp_conn->hdr, tcp_conn->hdr_size);
191 conn->in.hdr = &conn->hdr; 141 tcp_conn->in.hdr = &tcp_conn->hdr;
192 } 142 }
193 conn->in.offset += conn->hdr_size; 143 tcp_conn->in.offset += tcp_conn->hdr_size;
194 conn->in.copy -= conn->hdr_size; 144 tcp_conn->in.copy -= tcp_conn->hdr_size;
195 } else { 145 } else {
196 int hdr_remains; 146 int hdr_remains;
197 int copylen; 147 int copylen;
@@ -201,118 +151,51 @@ iscsi_hdr_extract(struct iscsi_conn *conn)
201 * copying it... This'll happen quite rarely. 151 * copying it... This'll happen quite rarely.
202 */ 152 */
203 153
204 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER) 154 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER)
205 conn->in.hdr_offset = 0; 155 tcp_conn->in.hdr_offset = 0;
206 156
207 hdr_remains = conn->hdr_size - conn->in.hdr_offset; 157 hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset;
208 BUG_ON(hdr_remains <= 0); 158 BUG_ON(hdr_remains <= 0);
209 159
210 copylen = min(conn->in.copy, hdr_remains); 160 copylen = min(tcp_conn->in.copy, hdr_remains);
211 skb_copy_bits(skb, conn->in.offset, 161 skb_copy_bits(skb, tcp_conn->in.offset,
212 (char*)&conn->hdr + conn->in.hdr_offset, copylen); 162 (char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset,
163 copylen);
213 164
214 debug_tcp("PDU gather offset %d bytes %d in.offset %d " 165 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
215 "in.copy %d\n", conn->in.hdr_offset, copylen, 166 "in.copy %d\n", tcp_conn->in.hdr_offset, copylen,
216 conn->in.offset, conn->in.copy); 167 tcp_conn->in.offset, tcp_conn->in.copy);
217 168
218 conn->in.offset += copylen; 169 tcp_conn->in.offset += copylen;
219 conn->in.copy -= copylen; 170 tcp_conn->in.copy -= copylen;
220 if (copylen < hdr_remains) { 171 if (copylen < hdr_remains) {
221 conn->in_progress = IN_PROGRESS_HEADER_GATHER; 172 tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER;
222 conn->in.hdr_offset += copylen; 173 tcp_conn->in.hdr_offset += copylen;
223 return -EAGAIN; 174 return -EAGAIN;
224 } 175 }
225 conn->in.hdr = &conn->hdr; 176 tcp_conn->in.hdr = &tcp_conn->hdr;
226 conn->discontiguous_hdr_cnt++; 177 tcp_conn->discontiguous_hdr_cnt++;
227 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 178 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
228 } 179 }
229 180
230 return 0; 181 return 0;
231} 182}
232 183
233static inline void 184/*
234iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 185 * must be called with session lock
186 */
187static void
188__iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
235{ 189{
236 struct scsi_cmnd *sc = ctask->sc; 190 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
237 struct iscsi_session *session = conn->session; 191 struct scsi_cmnd *sc;
238 192
239 spin_lock(&session->lock); 193 sc = ctask->sc;
240 if (unlikely(!sc)) { 194 if (unlikely(!sc))
241 spin_unlock(&session->lock);
242 return; 195 return;
243 }
244 if (sc->sc_data_direction == DMA_TO_DEVICE) {
245 struct iscsi_data_task *dtask, *n;
246 /* WRITE: cleanup Data-Out's if any */
247 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
248 list_del(&dtask->item);
249 mempool_free(dtask, ctask->datapool);
250 }
251 }
252 ctask->xmstate = XMSTATE_IDLE;
253 ctask->r2t = NULL;
254 ctask->sc = NULL;
255 __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
256 spin_unlock(&session->lock);
257}
258
259/**
260 * iscsi_cmd_rsp - SCSI Command Response processing
261 * @conn: iscsi connection
262 * @ctask: scsi command task
263 **/
264static int
265iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
266{
267 int rc;
268 struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
269 struct iscsi_session *session = conn->session;
270 struct scsi_cmnd *sc = ctask->sc;
271
272 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
273 if (rc) {
274 sc->result = (DID_ERROR << 16);
275 goto out;
276 }
277 196
278 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1; 197 tcp_ctask->xmstate = XMSTATE_IDLE;
279 198 tcp_ctask->r2t = NULL;
280 sc->result = (DID_OK << 16) | rhdr->cmd_status;
281
282 if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
283 sc->result = (DID_ERROR << 16);
284 goto out;
285 }
286
287 if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
288 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
289
290 memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
291 debug_scsi("copied %d bytes of sense\n", sensecopy);
292 }
293
294 if (sc->sc_data_direction == DMA_TO_DEVICE)
295 goto out;
296
297 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
298 int res_count = be32_to_cpu(rhdr->residual_count);
299
300 if (res_count > 0 && res_count <= sc->request_bufflen)
301 sc->resid = res_count;
302 else
303 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
304 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
305 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
306 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
307 sc->resid = be32_to_cpu(rhdr->residual_count);
308
309out:
310 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
311 (long)sc, sc->result, ctask->itt);
312 conn->scsirsp_pdus_cnt++;
313 iscsi_ctask_cleanup(conn, ctask);
314 sc->scsi_done(sc);
315 return rc;
316} 199}
317 200
318/** 201/**
@@ -324,7 +207,9 @@ static int
324iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 207iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
325{ 208{
326 int rc; 209 int rc;
327 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr; 210 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
211 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
212 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
328 struct iscsi_session *session = conn->session; 213 struct iscsi_session *session = conn->session;
329 int datasn = be32_to_cpu(rhdr->datasn); 214 int datasn = be32_to_cpu(rhdr->datasn);
330 215
@@ -334,9 +219,9 @@ iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
334 /* 219 /*
335 * setup Data-In byte counter (gets decremented..) 220 * setup Data-In byte counter (gets decremented..)
336 */ 221 */
337 ctask->data_count = conn->in.datalen; 222 ctask->data_count = tcp_conn->in.datalen;
338 223
339 if (conn->in.datalen == 0) 224 if (tcp_conn->in.datalen == 0)
340 return 0; 225 return 0;
341 226
342 if (ctask->datasn != datasn) 227 if (ctask->datasn != datasn)
@@ -344,8 +229,8 @@ iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
344 229
345 ctask->datasn++; 230 ctask->datasn++;
346 231
347 ctask->data_offset = be32_to_cpu(rhdr->offset); 232 tcp_ctask->data_offset = be32_to_cpu(rhdr->offset);
348 if (ctask->data_offset + conn->in.datalen > ctask->total_length) 233 if (tcp_ctask->data_offset + tcp_conn->in.datalen > ctask->total_length)
349 return ISCSI_ERR_DATA_OFFSET; 234 return ISCSI_ERR_DATA_OFFSET;
350 235
351 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) { 236 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
@@ -390,19 +275,17 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
390 struct iscsi_r2t_info *r2t) 275 struct iscsi_r2t_info *r2t)
391{ 276{
392 struct iscsi_data *hdr; 277 struct iscsi_data *hdr;
393 struct iscsi_data_task *dtask;
394 struct scsi_cmnd *sc = ctask->sc; 278 struct scsi_cmnd *sc = ctask->sc;
279 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
395 280
396 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 281 hdr = &r2t->dtask.hdr;
397 BUG_ON(!dtask);
398 hdr = &dtask->hdr;
399 memset(hdr, 0, sizeof(struct iscsi_data)); 282 memset(hdr, 0, sizeof(struct iscsi_data));
400 hdr->ttt = r2t->ttt; 283 hdr->ttt = r2t->ttt;
401 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 284 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
402 r2t->solicit_datasn++; 285 r2t->solicit_datasn++;
403 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 286 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
404 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 287 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
405 hdr->itt = ctask->hdr.itt; 288 hdr->itt = ctask->hdr->itt;
406 hdr->exp_statsn = r2t->exp_statsn; 289 hdr->exp_statsn = r2t->exp_statsn;
407 hdr->offset = cpu_to_be32(r2t->data_offset); 290 hdr->offset = cpu_to_be32(r2t->data_offset);
408 if (r2t->data_length > conn->max_xmit_dlength) { 291 if (r2t->data_length > conn->max_xmit_dlength) {
@@ -418,11 +301,9 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
418 301
419 r2t->sent = 0; 302 r2t->sent = 0;
420 303
421 iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr, 304 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
422 sizeof(struct iscsi_hdr)); 305 sizeof(struct iscsi_hdr));
423 306
424 r2t->dtask = dtask;
425
426 if (sc->use_sg) { 307 if (sc->use_sg) {
427 int i, sg_count = 0; 308 int i, sg_count = 0;
428 struct scatterlist *sg = sc->request_buffer; 309 struct scatterlist *sg = sc->request_buffer;
@@ -451,11 +332,9 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
451 } 332 }
452 BUG_ON(r2t->sg == NULL); 333 BUG_ON(r2t->sg == NULL);
453 } else 334 } else
454 iscsi_buf_init_iov(&ctask->sendbuf, 335 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
455 (char*)sc->request_buffer + r2t->data_offset, 336 (char*)sc->request_buffer + r2t->data_offset,
456 r2t->data_count); 337 r2t->data_count);
457
458 list_add(&dtask->item, &ctask->dataqueue);
459} 338}
460 339
461/** 340/**
@@ -468,17 +347,16 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
468{ 347{
469 struct iscsi_r2t_info *r2t; 348 struct iscsi_r2t_info *r2t;
470 struct iscsi_session *session = conn->session; 349 struct iscsi_session *session = conn->session;
471 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr; 350 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
351 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
352 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
472 int r2tsn = be32_to_cpu(rhdr->r2tsn); 353 int r2tsn = be32_to_cpu(rhdr->r2tsn);
473 int rc; 354 int rc;
474 355
475 if (conn->in.ahslen) 356 if (tcp_conn->in.datalen)
476 return ISCSI_ERR_AHSLEN;
477
478 if (conn->in.datalen)
479 return ISCSI_ERR_DATALEN; 357 return ISCSI_ERR_DATALEN;
480 358
481 if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn) 359 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
482 return ISCSI_ERR_R2TSN; 360 return ISCSI_ERR_R2TSN;
483 361
484 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 362 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
@@ -496,7 +374,7 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
496 spin_unlock(&session->lock); 374 spin_unlock(&session->lock);
497 return 0; 375 return 0;
498 } 376 }
499 rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 377 rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
500 BUG_ON(!rc); 378 BUG_ON(!rc);
501 379
502 r2t->exp_statsn = rhdr->statsn; 380 r2t->exp_statsn = rhdr->statsn;
@@ -518,10 +396,10 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
518 396
519 iscsi_solicit_data_init(conn, ctask, r2t); 397 iscsi_solicit_data_init(conn, ctask, r2t);
520 398
521 ctask->exp_r2tsn = r2tsn + 1; 399 tcp_ctask->exp_r2tsn = r2tsn + 1;
522 ctask->xmstate |= XMSTATE_SOL_HDR; 400 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
523 __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*)); 401 __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
524 __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*)); 402 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
525 403
526 scsi_queue_work(session->host, &conn->xmitwork); 404 scsi_queue_work(session->host, &conn->xmitwork);
527 conn->r2t_pdus_cnt++; 405 conn->r2t_pdus_cnt++;
@@ -531,258 +409,136 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
531} 409}
532 410
533static int 411static int
534iscsi_hdr_recv(struct iscsi_conn *conn) 412iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
535{ 413{
536 int rc = 0; 414 int rc = 0, opcode, ahslen;
537 struct iscsi_hdr *hdr; 415 struct iscsi_hdr *hdr;
538 struct iscsi_cmd_task *ctask;
539 struct iscsi_session *session = conn->session; 416 struct iscsi_session *session = conn->session;
540 uint32_t cdgst, rdgst = 0; 417 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
418 uint32_t cdgst, rdgst = 0, itt;
541 419
542 hdr = conn->in.hdr; 420 hdr = tcp_conn->in.hdr;
543 421
544 /* verify PDU length */ 422 /* verify PDU length */
545 conn->in.datalen = ntoh24(hdr->dlength); 423 tcp_conn->in.datalen = ntoh24(hdr->dlength);
546 if (conn->in.datalen > conn->max_recv_dlength) { 424 if (tcp_conn->in.datalen > conn->max_recv_dlength) {
547 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n", 425 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
548 conn->in.datalen, conn->max_recv_dlength); 426 tcp_conn->in.datalen, conn->max_recv_dlength);
549 return ISCSI_ERR_DATALEN; 427 return ISCSI_ERR_DATALEN;
550 } 428 }
551 conn->data_copied = 0; 429 tcp_conn->data_copied = 0;
552 430
553 /* read AHS */ 431 /* read AHS */
554 conn->in.ahslen = hdr->hlength * 4; 432 ahslen = hdr->hlength << 2;
555 conn->in.offset += conn->in.ahslen; 433 tcp_conn->in.offset += ahslen;
556 conn->in.copy -= conn->in.ahslen; 434 tcp_conn->in.copy -= ahslen;
557 if (conn->in.copy < 0) { 435 if (tcp_conn->in.copy < 0) {
558 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length " 436 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
559 "%d bytes\n", conn->in.ahslen); 437 "%d bytes\n", ahslen);
560 return ISCSI_ERR_AHSLEN; 438 return ISCSI_ERR_AHSLEN;
561 } 439 }
562 440
563 /* calculate read padding */ 441 /* calculate read padding */
564 conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1); 442 tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1);
565 if (conn->in.padding) { 443 if (tcp_conn->in.padding) {
566 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding; 444 tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding;
567 debug_scsi("read padding %d bytes\n", conn->in.padding); 445 debug_scsi("read padding %d bytes\n", tcp_conn->in.padding);
568 } 446 }
569 447
570 if (conn->hdrdgst_en) { 448 if (conn->hdrdgst_en) {
571 struct scatterlist sg; 449 struct scatterlist sg;
572 450
573 sg_init_one(&sg, (u8 *)hdr, 451 sg_init_one(&sg, (u8 *)hdr,
574 sizeof(struct iscsi_hdr) + conn->in.ahslen); 452 sizeof(struct iscsi_hdr) + ahslen);
575 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst); 453 crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
576 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) + 454 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
577 conn->in.ahslen); 455 ahslen);
578 if (cdgst != rdgst) { 456 if (cdgst != rdgst) {
579 printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error " 457 printk(KERN_ERR "iscsi_tcp: hdrdgst error "
580 "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst, 458 "recv 0x%x calc 0x%x\n", rdgst, cdgst);
581 cdgst);
582 return ISCSI_ERR_HDR_DGST; 459 return ISCSI_ERR_HDR_DGST;
583 } 460 }
584 } 461 }
585 462
586 /* save opcode for later */ 463 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
587 conn->in.opcode = hdr->opcode & ISCSI_OPCODE_MASK;
588
589 /* verify itt (itt encoding: age+cid+itt) */ 464 /* verify itt (itt encoding: age+cid+itt) */
590 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) { 465 rc = iscsi_verify_itt(conn, hdr, &itt);
591 if ((hdr->itt & AGE_MASK) != 466 if (rc == ISCSI_ERR_NO_SCSI_CMD) {
592 (session->age << AGE_SHIFT)) { 467 tcp_conn->in.datalen = 0; /* force drop */
593 printk(KERN_ERR "iscsi_tcp: received itt %x expected " 468 return 0;
594 "session age (%x)\n", hdr->itt, 469 } else if (rc)
595 session->age & AGE_MASK); 470 return rc;
596 return ISCSI_ERR_BAD_ITT;
597 }
598
599 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
600 printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
601 "CID (%x)\n", hdr->itt, conn->id);
602 return ISCSI_ERR_BAD_ITT;
603 }
604 conn->in.itt = hdr->itt & ITT_MASK;
605 } else
606 conn->in.itt = hdr->itt;
607 471
608 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n", 472 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
609 hdr->opcode, conn->in.offset, conn->in.copy, 473 opcode, tcp_conn->in.offset, tcp_conn->in.copy,
610 conn->in.ahslen, conn->in.datalen); 474 ahslen, tcp_conn->in.datalen);
611
612 if (conn->in.itt < session->cmds_max) {
613 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
614
615 if (!ctask->sc) {
616 printk(KERN_INFO "iscsi_tcp: dropping ctask with "
617 "itt 0x%x\n", ctask->itt);
618 conn->in.datalen = 0; /* force drop */
619 return 0;
620 }
621
622 if (ctask->sc->SCp.phase != session->age) {
623 printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
624 "expected %d\n", ctask->sc->SCp.phase,
625 session->age);
626 return ISCSI_ERR_SESSION_FAILED;
627 }
628
629 conn->in.ctask = ctask;
630
631 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
632 hdr->opcode, conn->id, (long)ctask->sc,
633 ctask->itt, conn->in.datalen);
634
635 switch(conn->in.opcode) {
636 case ISCSI_OP_SCSI_CMD_RSP:
637 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
638 if (!conn->in.datalen)
639 rc = iscsi_cmd_rsp(conn, ctask);
640 else
641 /*
642 * got sense or response data; copying PDU
643 * Header to the connection's header
644 * placeholder
645 */
646 memcpy(&conn->hdr, hdr,
647 sizeof(struct iscsi_hdr));
648 break;
649 case ISCSI_OP_SCSI_DATA_IN:
650 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
651 /* save flags for non-exceptional status */
652 conn->in.flags = hdr->flags;
653 /* save cmd_status for sense data */
654 conn->in.cmd_status =
655 ((struct iscsi_data_rsp*)hdr)->cmd_status;
656 rc = iscsi_data_rsp(conn, ctask);
657 break;
658 case ISCSI_OP_R2T:
659 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
660 if (ctask->sc->sc_data_direction == DMA_TO_DEVICE)
661 rc = iscsi_r2t_rsp(conn, ctask);
662 else
663 rc = ISCSI_ERR_PROTO;
664 break;
665 default:
666 rc = ISCSI_ERR_BAD_OPCODE;
667 break;
668 }
669 } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
670 conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
671 session->mgmtpool_max) {
672 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
673 session->mgmt_cmds[conn->in.itt -
674 ISCSI_MGMT_ITT_OFFSET];
675
676 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
677 conn->in.opcode, conn->id, mtask->itt,
678 conn->in.datalen);
679
680 switch(conn->in.opcode) {
681 case ISCSI_OP_LOGIN_RSP:
682 case ISCSI_OP_TEXT_RSP:
683 case ISCSI_OP_LOGOUT_RSP:
684 rc = iscsi_check_assign_cmdsn(session,
685 (struct iscsi_nopin*)hdr);
686 if (rc)
687 break;
688
689 if (!conn->in.datalen) {
690 rc = iscsi_recv_pdu(conn->cls_conn, hdr,
691 NULL, 0);
692 if (conn->login_mtask != mtask) {
693 spin_lock(&session->lock);
694 __kfifo_put(session->mgmtpool.queue,
695 (void*)&mtask, sizeof(void*));
696 spin_unlock(&session->lock);
697 }
698 }
699 break;
700 case ISCSI_OP_SCSI_TMFUNC_RSP:
701 rc = iscsi_check_assign_cmdsn(session,
702 (struct iscsi_nopin*)hdr);
703 if (rc)
704 break;
705 475
706 if (conn->in.datalen || conn->in.ahslen) { 476 switch(opcode) {
707 rc = ISCSI_ERR_PROTO; 477 case ISCSI_OP_SCSI_DATA_IN:
708 break; 478 tcp_conn->in.ctask = session->cmds[itt];
709 } 479 rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
710 conn->tmfrsp_pdus_cnt++; 480 /* fall through */
711 spin_lock(&session->lock); 481 case ISCSI_OP_SCSI_CMD_RSP:
712 if (conn->tmabort_state == TMABORT_INITIAL) { 482 tcp_conn->in.ctask = session->cmds[itt];
713 __kfifo_put(session->mgmtpool.queue, 483 if (tcp_conn->in.datalen)
714 (void*)&mtask, sizeof(void*)); 484 goto copy_hdr;
715 conn->tmabort_state = 485
716 ((struct iscsi_tm_rsp *)hdr)-> 486 spin_lock(&session->lock);
717 response == ISCSI_TMF_RSP_COMPLETE ? 487 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
718 TMABORT_SUCCESS:TMABORT_FAILED; 488 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
719 /* unblock eh_abort() */ 489 spin_unlock(&session->lock);
720 wake_up(&conn->ehwait); 490 break;
721 } 491 case ISCSI_OP_R2T:
722 spin_unlock(&session->lock); 492 tcp_conn->in.ctask = session->cmds[itt];
723 break; 493 if (ahslen)
724 case ISCSI_OP_NOOP_IN: 494 rc = ISCSI_ERR_AHSLEN;
725 if (hdr->ttt != ISCSI_RESERVED_TAG) { 495 else if (tcp_conn->in.ctask->sc->sc_data_direction ==
726 rc = ISCSI_ERR_PROTO; 496 DMA_TO_DEVICE)
727 break; 497 rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask);
728 } 498 else
729 rc = iscsi_check_assign_cmdsn(session, 499 rc = ISCSI_ERR_PROTO;
730 (struct iscsi_nopin*)hdr); 500 break;
731 if (rc) 501 case ISCSI_OP_LOGIN_RSP:
732 break; 502 case ISCSI_OP_TEXT_RSP:
733 conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1; 503 case ISCSI_OP_LOGOUT_RSP:
734 504 case ISCSI_OP_NOOP_IN:
735 if (!conn->in.datalen) { 505 case ISCSI_OP_REJECT:
736 struct iscsi_mgmt_task *mtask; 506 case ISCSI_OP_ASYNC_EVENT:
737 507 if (tcp_conn->in.datalen)
738 rc = iscsi_recv_pdu(conn->cls_conn, hdr, 508 goto copy_hdr;
739 NULL, 0); 509 /* fall through */
740 mtask = (struct iscsi_mgmt_task *) 510 case ISCSI_OP_SCSI_TMFUNC_RSP:
741 session->mgmt_cmds[conn->in.itt - 511 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
742 ISCSI_MGMT_ITT_OFFSET]; 512 break;
743 if (conn->login_mtask != mtask) { 513 default:
744 spin_lock(&session->lock); 514 rc = ISCSI_ERR_BAD_OPCODE;
745 __kfifo_put(session->mgmtpool.queue, 515 break;
746 (void*)&mtask, sizeof(void*)); 516 }
747 spin_unlock(&session->lock);
748 }
749 }
750 break;
751 default:
752 rc = ISCSI_ERR_BAD_OPCODE;
753 break;
754 }
755 } else if (conn->in.itt == ISCSI_RESERVED_TAG) {
756 switch(conn->in.opcode) {
757 case ISCSI_OP_NOOP_IN:
758 if (!conn->in.datalen) {
759 rc = iscsi_check_assign_cmdsn(session,
760 (struct iscsi_nopin*)hdr);
761 if (!rc && hdr->ttt != ISCSI_RESERVED_TAG)
762 rc = iscsi_recv_pdu(conn->cls_conn,
763 hdr, NULL, 0);
764 } else
765 rc = ISCSI_ERR_PROTO;
766 break;
767 case ISCSI_OP_REJECT:
768 /* we need sth like iscsi_reject_rsp()*/
769 case ISCSI_OP_ASYNC_EVENT:
770 /* we need sth like iscsi_async_event_rsp() */
771 rc = ISCSI_ERR_BAD_OPCODE;
772 break;
773 default:
774 rc = ISCSI_ERR_BAD_OPCODE;
775 break;
776 }
777 } else
778 rc = ISCSI_ERR_BAD_ITT;
779 517
780 return rc; 518 return rc;
519
520copy_hdr:
521 /*
522 * if we did zero copy for the header but we will need multiple
523 * skbs to complete the command then we have to copy the header
524 * for later use
525 */
526 if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <
527 (tcp_conn->in.datalen + tcp_conn->in.padding +
528 (conn->datadgst_en ? 4 : 0))) {
529 debug_tcp("Copying header for later use. in.copy %d in.datalen"
530 " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen);
531 memcpy(&tcp_conn->hdr, tcp_conn->in.hdr,
532 sizeof(struct iscsi_hdr));
533 tcp_conn->in.hdr = &tcp_conn->hdr;
534 tcp_conn->in.zero_copy_hdr = 0;
535 }
536 return 0;
781} 537}
782 538
783/** 539/**
784 * iscsi_ctask_copy - copy skb bits to the destanation cmd task 540 * iscsi_ctask_copy - copy skb bits to the destanation cmd task
785 * @conn: iscsi connection 541 * @conn: iscsi tcp connection
786 * @ctask: scsi command task 542 * @ctask: scsi command task
787 * @buf: buffer to copy to 543 * @buf: buffer to copy to
788 * @buf_size: size of buffer 544 * @buf_size: size of buffer
@@ -804,110 +560,113 @@ iscsi_hdr_recv(struct iscsi_conn *conn)
804 * buf_left left to copy from in progress buffer 560 * buf_left left to copy from in progress buffer
805 **/ 561 **/
806static inline int 562static inline int
807iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 563iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
808 void *buf, int buf_size, int offset) 564 void *buf, int buf_size, int offset)
809{ 565{
810 int buf_left = buf_size - (conn->data_copied + offset); 566 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
811 int size = min(conn->in.copy, buf_left); 567 int buf_left = buf_size - (tcp_conn->data_copied + offset);
568 int size = min(tcp_conn->in.copy, buf_left);
812 int rc; 569 int rc;
813 570
814 size = min(size, ctask->data_count); 571 size = min(size, ctask->data_count);
815 572
816 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n", 573 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
817 size, conn->in.offset, conn->in.copied); 574 size, tcp_conn->in.offset, tcp_conn->in.copied);
818 575
819 BUG_ON(size <= 0); 576 BUG_ON(size <= 0);
820 BUG_ON(ctask->sent + size > ctask->total_length); 577 BUG_ON(tcp_ctask->sent + size > ctask->total_length);
821 578
822 rc = skb_copy_bits(conn->in.skb, conn->in.offset, 579 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
823 (char*)buf + (offset + conn->data_copied), size); 580 (char*)buf + (offset + tcp_conn->data_copied), size);
824 /* must fit into skb->len */ 581 /* must fit into skb->len */
825 BUG_ON(rc); 582 BUG_ON(rc);
826 583
827 conn->in.offset += size; 584 tcp_conn->in.offset += size;
828 conn->in.copy -= size; 585 tcp_conn->in.copy -= size;
829 conn->in.copied += size; 586 tcp_conn->in.copied += size;
830 conn->data_copied += size; 587 tcp_conn->data_copied += size;
831 ctask->sent += size; 588 tcp_ctask->sent += size;
832 ctask->data_count -= size; 589 ctask->data_count -= size;
833 590
834 BUG_ON(conn->in.copy < 0); 591 BUG_ON(tcp_conn->in.copy < 0);
835 BUG_ON(ctask->data_count < 0); 592 BUG_ON(ctask->data_count < 0);
836 593
837 if (buf_size != (conn->data_copied + offset)) { 594 if (buf_size != (tcp_conn->data_copied + offset)) {
838 if (!ctask->data_count) { 595 if (!ctask->data_count) {
839 BUG_ON(buf_size - conn->data_copied < 0); 596 BUG_ON(buf_size - tcp_conn->data_copied < 0);
840 /* done with this PDU */ 597 /* done with this PDU */
841 return buf_size - conn->data_copied; 598 return buf_size - tcp_conn->data_copied;
842 } 599 }
843 return -EAGAIN; 600 return -EAGAIN;
844 } 601 }
845 602
846 /* done with this buffer or with both - PDU and buffer */ 603 /* done with this buffer or with both - PDU and buffer */
847 conn->data_copied = 0; 604 tcp_conn->data_copied = 0;
848 return 0; 605 return 0;
849} 606}
850 607
851/** 608/**
852 * iscsi_tcp_copy - copy skb bits to the destanation buffer 609 * iscsi_tcp_copy - copy skb bits to the destanation buffer
853 * @conn: iscsi connection 610 * @conn: iscsi tcp connection
854 * @buf: buffer to copy to
855 * @buf_size: number of bytes to copy
856 * 611 *
857 * Notes: 612 * Notes:
858 * The function calls skb_copy_bits() and updates per-connection 613 * The function calls skb_copy_bits() and updates per-connection
859 * byte counters. 614 * byte counters.
860 **/ 615 **/
861static inline int 616static inline int
862iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size) 617iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn)
863{ 618{
864 int buf_left = buf_size - conn->data_copied; 619 void *buf = tcp_conn->data;
865 int size = min(conn->in.copy, buf_left); 620 int buf_size = tcp_conn->in.datalen;
621 int buf_left = buf_size - tcp_conn->data_copied;
622 int size = min(tcp_conn->in.copy, buf_left);
866 int rc; 623 int rc;
867 624
868 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n", 625 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
869 size, conn->in.offset, conn->data_copied); 626 size, tcp_conn->in.offset, tcp_conn->data_copied);
870 BUG_ON(size <= 0); 627 BUG_ON(size <= 0);
871 628
872 rc = skb_copy_bits(conn->in.skb, conn->in.offset, 629 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
873 (char*)buf + conn->data_copied, size); 630 (char*)buf + tcp_conn->data_copied, size);
874 BUG_ON(rc); 631 BUG_ON(rc);
875 632
876 conn->in.offset += size; 633 tcp_conn->in.offset += size;
877 conn->in.copy -= size; 634 tcp_conn->in.copy -= size;
878 conn->in.copied += size; 635 tcp_conn->in.copied += size;
879 conn->data_copied += size; 636 tcp_conn->data_copied += size;
880 637
881 if (buf_size != conn->data_copied) 638 if (buf_size != tcp_conn->data_copied)
882 return -EAGAIN; 639 return -EAGAIN;
883 640
884 return 0; 641 return 0;
885} 642}
886 643
887static inline void 644static inline void
888partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg, 645partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn,
889 int offset, int length) 646 struct scatterlist *sg, int offset, int length)
890{ 647{
891 struct scatterlist temp; 648 struct scatterlist temp;
892 649
893 memcpy(&temp, sg, sizeof(struct scatterlist)); 650 memcpy(&temp, sg, sizeof(struct scatterlist));
894 temp.offset = offset; 651 temp.offset = offset;
895 temp.length = length; 652 temp.length = length;
896 crypto_digest_update(conn->data_rx_tfm, &temp, 1); 653 crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1);
897} 654}
898 655
899static void 656static void
900iscsi_recv_digest_update(struct iscsi_conn *conn, char* buf, int len) 657iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
901{ 658{
902 struct scatterlist tmp; 659 struct scatterlist tmp;
903 660
904 sg_init_one(&tmp, buf, len); 661 sg_init_one(&tmp, buf, len);
905 crypto_digest_update(conn->data_rx_tfm, &tmp, 1); 662 crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1);
906} 663}
907 664
908static int iscsi_scsi_data_in(struct iscsi_conn *conn) 665static int iscsi_scsi_data_in(struct iscsi_conn *conn)
909{ 666{
910 struct iscsi_cmd_task *ctask = conn->in.ctask; 667 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
668 struct iscsi_cmd_task *ctask = tcp_conn->in.ctask;
669 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
911 struct scsi_cmnd *sc = ctask->sc; 670 struct scsi_cmnd *sc = ctask->sc;
912 struct scatterlist *sg; 671 struct scatterlist *sg;
913 int i, offset, rc = 0; 672 int i, offset, rc = 0;
@@ -919,31 +678,33 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
919 */ 678 */
920 if (!sc->use_sg) { 679 if (!sc->use_sg) {
921 i = ctask->data_count; 680 i = ctask->data_count;
922 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer, 681 rc = iscsi_ctask_copy(tcp_conn, ctask, sc->request_buffer,
923 sc->request_bufflen, ctask->data_offset); 682 sc->request_bufflen,
683 tcp_ctask->data_offset);
924 if (rc == -EAGAIN) 684 if (rc == -EAGAIN)
925 return rc; 685 return rc;
926 if (conn->datadgst_en) 686 if (conn->datadgst_en)
927 iscsi_recv_digest_update(conn, sc->request_buffer, i); 687 iscsi_recv_digest_update(tcp_conn, sc->request_buffer,
688 i);
928 rc = 0; 689 rc = 0;
929 goto done; 690 goto done;
930 } 691 }
931 692
932 offset = ctask->data_offset; 693 offset = tcp_ctask->data_offset;
933 sg = sc->request_buffer; 694 sg = sc->request_buffer;
934 695
935 if (ctask->data_offset) 696 if (tcp_ctask->data_offset)
936 for (i = 0; i < ctask->sg_count; i++) 697 for (i = 0; i < tcp_ctask->sg_count; i++)
937 offset -= sg[i].length; 698 offset -= sg[i].length;
938 /* we've passed through partial sg*/ 699 /* we've passed through partial sg*/
939 if (offset < 0) 700 if (offset < 0)
940 offset = 0; 701 offset = 0;
941 702
942 for (i = ctask->sg_count; i < sc->use_sg; i++) { 703 for (i = tcp_ctask->sg_count; i < sc->use_sg; i++) {
943 char *dest; 704 char *dest;
944 705
945 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0); 706 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
946 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset, 707 rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset,
947 sg[i].length, offset); 708 sg[i].length, offset);
948 kunmap_atomic(dest, KM_SOFTIRQ0); 709 kunmap_atomic(dest, KM_SOFTIRQ0);
949 if (rc == -EAGAIN) 710 if (rc == -EAGAIN)
@@ -952,15 +713,17 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
952 if (!rc) { 713 if (!rc) {
953 if (conn->datadgst_en) { 714 if (conn->datadgst_en) {
954 if (!offset) 715 if (!offset)
955 crypto_digest_update(conn->data_rx_tfm, 716 crypto_digest_update(
956 &sg[i], 1); 717 tcp_conn->data_rx_tfm,
718 &sg[i], 1);
957 else 719 else
958 partial_sg_digest_update(conn, &sg[i], 720 partial_sg_digest_update(tcp_conn,
721 &sg[i],
959 sg[i].offset + offset, 722 sg[i].offset + offset,
960 sg[i].length - offset); 723 sg[i].length - offset);
961 } 724 }
962 offset = 0; 725 offset = 0;
963 ctask->sg_count++; 726 tcp_ctask->sg_count++;
964 } 727 }
965 728
966 if (!ctask->data_count) { 729 if (!ctask->data_count) {
@@ -968,25 +731,26 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
968 /* 731 /*
969 * data-in is complete, but buffer not... 732 * data-in is complete, but buffer not...
970 */ 733 */
971 partial_sg_digest_update(conn, &sg[i], 734 partial_sg_digest_update(tcp_conn, &sg[i],
972 sg[i].offset, sg[i].length-rc); 735 sg[i].offset, sg[i].length-rc);
973 rc = 0; 736 rc = 0;
974 break; 737 break;
975 } 738 }
976 739
977 if (!conn->in.copy) 740 if (!tcp_conn->in.copy)
978 return -EAGAIN; 741 return -EAGAIN;
979 } 742 }
980 BUG_ON(ctask->data_count); 743 BUG_ON(ctask->data_count);
981 744
982done: 745done:
983 /* check for non-exceptional status */ 746 /* check for non-exceptional status */
984 if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) { 747 if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
985 debug_scsi("done [sc %lx res %d itt 0x%x]\n", 748 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
986 (long)sc, sc->result, ctask->itt); 749 (long)sc, sc->result, ctask->itt);
987 conn->scsirsp_pdus_cnt++; 750 spin_lock(&conn->session->lock);
988 iscsi_ctask_cleanup(conn, ctask); 751 __iscsi_ctask_cleanup(conn, ctask);
989 sc->scsi_done(sc); 752 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
753 spin_unlock(&conn->session->lock);
990 } 754 }
991 755
992 return rc; 756 return rc;
@@ -995,71 +759,38 @@ done:
995static int 759static int
996iscsi_data_recv(struct iscsi_conn *conn) 760iscsi_data_recv(struct iscsi_conn *conn)
997{ 761{
998 struct iscsi_session *session = conn->session; 762 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
999 int rc = 0; 763 int rc = 0, opcode;
1000 764
1001 switch(conn->in.opcode) { 765 opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
766 switch (opcode) {
1002 case ISCSI_OP_SCSI_DATA_IN: 767 case ISCSI_OP_SCSI_DATA_IN:
1003 rc = iscsi_scsi_data_in(conn); 768 rc = iscsi_scsi_data_in(conn);
1004 break; 769 break;
1005 case ISCSI_OP_SCSI_CMD_RSP: { 770 case ISCSI_OP_SCSI_CMD_RSP:
1006 /* 771 spin_lock(&conn->session->lock);
1007 * SCSI Sense Data: 772 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
1008 * copying the entire Data Segment. 773 spin_unlock(&conn->session->lock);
1009 */
1010 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1011 rc = -EAGAIN;
1012 goto exit;
1013 }
1014
1015 /*
1016 * check for sense
1017 */
1018 conn->in.hdr = &conn->hdr;
1019 conn->senselen = (conn->data[0] << 8) | conn->data[1];
1020 rc = iscsi_cmd_rsp(conn, conn->in.ctask);
1021 if (!rc && conn->datadgst_en)
1022 iscsi_recv_digest_update(conn, conn->data,
1023 conn->in.datalen);
1024 }
1025 break;
1026 case ISCSI_OP_TEXT_RSP: 774 case ISCSI_OP_TEXT_RSP:
1027 case ISCSI_OP_LOGIN_RSP: 775 case ISCSI_OP_LOGIN_RSP:
1028 case ISCSI_OP_NOOP_IN: { 776 case ISCSI_OP_NOOP_IN:
1029 struct iscsi_mgmt_task *mtask = NULL; 777 case ISCSI_OP_ASYNC_EVENT:
1030 778 case ISCSI_OP_REJECT:
1031 if (conn->in.itt != ISCSI_RESERVED_TAG)
1032 mtask = (struct iscsi_mgmt_task *)
1033 session->mgmt_cmds[conn->in.itt -
1034 ISCSI_MGMT_ITT_OFFSET];
1035
1036 /* 779 /*
1037 * Collect data segment to the connection's data 780 * Collect data segment to the connection's data
1038 * placeholder 781 * placeholder
1039 */ 782 */
1040 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) { 783 if (iscsi_tcp_copy(tcp_conn)) {
1041 rc = -EAGAIN; 784 rc = -EAGAIN;
1042 goto exit; 785 goto exit;
1043 } 786 }
1044 787
1045 rc = iscsi_recv_pdu(conn->cls_conn, conn->in.hdr, 788 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, tcp_conn->data,
1046 conn->data, conn->in.datalen); 789 tcp_conn->in.datalen);
1047 790 if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP)
1048 if (!rc && conn->datadgst_en && 791 iscsi_recv_digest_update(tcp_conn, tcp_conn->data,
1049 conn->in.opcode != ISCSI_OP_LOGIN_RSP) 792 tcp_conn->in.datalen);
1050 iscsi_recv_digest_update(conn, conn->data, 793 break;
1051 conn->in.datalen);
1052
1053 if (mtask && conn->login_mtask != mtask) {
1054 spin_lock(&session->lock);
1055 __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
1056 sizeof(void*));
1057 spin_unlock(&session->lock);
1058 }
1059 }
1060 break;
1061 case ISCSI_OP_ASYNC_EVENT:
1062 case ISCSI_OP_REJECT:
1063 default: 794 default:
1064 BUG_ON(1); 795 BUG_ON(1);
1065 } 796 }
@@ -1080,6 +811,7 @@ iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1080{ 811{
1081 int rc; 812 int rc;
1082 struct iscsi_conn *conn = rd_desc->arg.data; 813 struct iscsi_conn *conn = rd_desc->arg.data;
814 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1083 int processed; 815 int processed;
1084 char pad[ISCSI_PAD_LEN]; 816 char pad[ISCSI_PAD_LEN];
1085 struct scatterlist sg; 817 struct scatterlist sg;
@@ -1088,15 +820,15 @@ iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1088 * Save current SKB and its offset in the corresponding 820 * Save current SKB and its offset in the corresponding
1089 * connection context. 821 * connection context.
1090 */ 822 */
1091 conn->in.copy = skb->len - offset; 823 tcp_conn->in.copy = skb->len - offset;
1092 conn->in.offset = offset; 824 tcp_conn->in.offset = offset;
1093 conn->in.skb = skb; 825 tcp_conn->in.skb = skb;
1094 conn->in.len = conn->in.copy; 826 tcp_conn->in.len = tcp_conn->in.copy;
1095 BUG_ON(conn->in.copy <= 0); 827 BUG_ON(tcp_conn->in.copy <= 0);
1096 debug_tcp("in %d bytes\n", conn->in.copy); 828 debug_tcp("in %d bytes\n", tcp_conn->in.copy);
1097 829
1098more: 830more:
1099 conn->in.copied = 0; 831 tcp_conn->in.copied = 0;
1100 rc = 0; 832 rc = 0;
1101 833
1102 if (unlikely(conn->suspend_rx)) { 834 if (unlikely(conn->suspend_rx)) {
@@ -1104,9 +836,9 @@ more:
1104 return 0; 836 return 0;
1105 } 837 }
1106 838
1107 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER || 839 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
1108 conn->in_progress == IN_PROGRESS_HEADER_GATHER) { 840 tcp_conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
1109 rc = iscsi_hdr_extract(conn); 841 rc = iscsi_hdr_extract(tcp_conn);
1110 if (rc) { 842 if (rc) {
1111 if (rc == -EAGAIN) 843 if (rc == -EAGAIN)
1112 goto nomore; 844 goto nomore;
@@ -1119,90 +851,91 @@ more:
1119 /* 851 /*
1120 * Verify and process incoming PDU header. 852 * Verify and process incoming PDU header.
1121 */ 853 */
1122 rc = iscsi_hdr_recv(conn); 854 rc = iscsi_tcp_hdr_recv(conn);
1123 if (!rc && conn->in.datalen) { 855 if (!rc && tcp_conn->in.datalen) {
1124 if (conn->datadgst_en) { 856 if (conn->datadgst_en) {
1125 BUG_ON(!conn->data_rx_tfm); 857 BUG_ON(!tcp_conn->data_rx_tfm);
1126 crypto_digest_init(conn->data_rx_tfm); 858 crypto_digest_init(tcp_conn->data_rx_tfm);
1127 } 859 }
1128 conn->in_progress = IN_PROGRESS_DATA_RECV; 860 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
1129 } else if (rc) { 861 } else if (rc) {
1130 iscsi_conn_failure(conn, rc); 862 iscsi_conn_failure(conn, rc);
1131 return 0; 863 return 0;
1132 } 864 }
1133 } 865 }
1134 866
1135 if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) { 867 if (tcp_conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
1136 uint32_t recv_digest; 868 uint32_t recv_digest;
869
1137 debug_tcp("extra data_recv offset %d copy %d\n", 870 debug_tcp("extra data_recv offset %d copy %d\n",
1138 conn->in.offset, conn->in.copy); 871 tcp_conn->in.offset, tcp_conn->in.copy);
1139 skb_copy_bits(conn->in.skb, conn->in.offset, 872 skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
1140 &recv_digest, 4); 873 &recv_digest, 4);
1141 conn->in.offset += 4; 874 tcp_conn->in.offset += 4;
1142 conn->in.copy -= 4; 875 tcp_conn->in.copy -= 4;
1143 if (recv_digest != conn->in.datadgst) { 876 if (recv_digest != tcp_conn->in.datadgst) {
1144 debug_tcp("iscsi_tcp: data digest error!" 877 debug_tcp("iscsi_tcp: data digest error!"
1145 "0x%x != 0x%x\n", recv_digest, 878 "0x%x != 0x%x\n", recv_digest,
1146 conn->in.datadgst); 879 tcp_conn->in.datadgst);
1147 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 880 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1148 return 0; 881 return 0;
1149 } else { 882 } else {
1150 debug_tcp("iscsi_tcp: data digest match!" 883 debug_tcp("iscsi_tcp: data digest match!"
1151 "0x%x == 0x%x\n", recv_digest, 884 "0x%x == 0x%x\n", recv_digest,
1152 conn->in.datadgst); 885 tcp_conn->in.datadgst);
1153 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 886 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1154 } 887 }
1155 } 888 }
1156 889
1157 if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) { 890 if (tcp_conn->in_progress == IN_PROGRESS_DATA_RECV &&
891 tcp_conn->in.copy) {
1158 892
1159 debug_tcp("data_recv offset %d copy %d\n", 893 debug_tcp("data_recv offset %d copy %d\n",
1160 conn->in.offset, conn->in.copy); 894 tcp_conn->in.offset, tcp_conn->in.copy);
1161 895
1162 rc = iscsi_data_recv(conn); 896 rc = iscsi_data_recv(conn);
1163 if (rc) { 897 if (rc) {
1164 if (rc == -EAGAIN) { 898 if (rc == -EAGAIN)
1165 rd_desc->count = conn->in.datalen -
1166 conn->in.ctask->data_count;
1167 goto again; 899 goto again;
1168 }
1169 iscsi_conn_failure(conn, rc); 900 iscsi_conn_failure(conn, rc);
1170 return 0; 901 return 0;
1171 } 902 }
1172 conn->in.copy -= conn->in.padding; 903 tcp_conn->in.copy -= tcp_conn->in.padding;
1173 conn->in.offset += conn->in.padding; 904 tcp_conn->in.offset += tcp_conn->in.padding;
1174 if (conn->datadgst_en) { 905 if (conn->datadgst_en) {
1175 if (conn->in.padding) { 906 if (tcp_conn->in.padding) {
1176 debug_tcp("padding -> %d\n", conn->in.padding); 907 debug_tcp("padding -> %d\n",
1177 memset(pad, 0, conn->in.padding); 908 tcp_conn->in.padding);
1178 sg_init_one(&sg, pad, conn->in.padding); 909 memset(pad, 0, tcp_conn->in.padding);
1179 crypto_digest_update(conn->data_rx_tfm, &sg, 1); 910 sg_init_one(&sg, pad, tcp_conn->in.padding);
911 crypto_digest_update(tcp_conn->data_rx_tfm,
912 &sg, 1);
1180 } 913 }
1181 crypto_digest_final(conn->data_rx_tfm, 914 crypto_digest_final(tcp_conn->data_rx_tfm,
1182 (u8 *) & conn->in.datadgst); 915 (u8 *) & tcp_conn->in.datadgst);
1183 debug_tcp("rx digest 0x%x\n", conn->in.datadgst); 916 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
1184 conn->in_progress = IN_PROGRESS_DDIGEST_RECV; 917 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
1185 } else 918 } else
1186 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 919 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1187 } 920 }
1188 921
1189 debug_tcp("f, processed %d from out of %d padding %d\n", 922 debug_tcp("f, processed %d from out of %d padding %d\n",
1190 conn->in.offset - offset, (int)len, conn->in.padding); 923 tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding);
1191 BUG_ON(conn->in.offset - offset > len); 924 BUG_ON(tcp_conn->in.offset - offset > len);
1192 925
1193 if (conn->in.offset - offset != len) { 926 if (tcp_conn->in.offset - offset != len) {
1194 debug_tcp("continue to process %d bytes\n", 927 debug_tcp("continue to process %d bytes\n",
1195 (int)len - (conn->in.offset - offset)); 928 (int)len - (tcp_conn->in.offset - offset));
1196 goto more; 929 goto more;
1197 } 930 }
1198 931
1199nomore: 932nomore:
1200 processed = conn->in.offset - offset; 933 processed = tcp_conn->in.offset - offset;
1201 BUG_ON(processed == 0); 934 BUG_ON(processed == 0);
1202 return processed; 935 return processed;
1203 936
1204again: 937again:
1205 processed = conn->in.offset - offset; 938 processed = tcp_conn->in.offset - offset;
1206 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n", 939 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
1207 processed, (int)len, (int)rd_desc->count); 940 processed, (int)len, (int)rd_desc->count);
1208 BUG_ON(processed == 0); 941 BUG_ON(processed == 0);
@@ -1220,9 +953,14 @@ iscsi_tcp_data_ready(struct sock *sk, int flag)
1220 953
1221 read_lock(&sk->sk_callback_lock); 954 read_lock(&sk->sk_callback_lock);
1222 955
1223 /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */ 956 /*
957 * Use rd_desc to pass 'conn' to iscsi_tcp_data_recv.
958 * We set count to 1 because we want the network layer to
959 * hand us all the skbs that are available. iscsi_tcp_data_recv
960 * handled pdus that cross buffers or pdus that still need data.
961 */
1224 rd_desc.arg.data = conn; 962 rd_desc.arg.data = conn;
1225 rd_desc.count = 0; 963 rd_desc.count = 1;
1226 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv); 964 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
1227 965
1228 read_unlock(&sk->sk_callback_lock); 966 read_unlock(&sk->sk_callback_lock);
@@ -1231,6 +969,7 @@ iscsi_tcp_data_ready(struct sock *sk, int flag)
1231static void 969static void
1232iscsi_tcp_state_change(struct sock *sk) 970iscsi_tcp_state_change(struct sock *sk)
1233{ 971{
972 struct iscsi_tcp_conn *tcp_conn;
1234 struct iscsi_conn *conn; 973 struct iscsi_conn *conn;
1235 struct iscsi_session *session; 974 struct iscsi_session *session;
1236 void (*old_state_change)(struct sock *); 975 void (*old_state_change)(struct sock *);
@@ -1247,7 +986,8 @@ iscsi_tcp_state_change(struct sock *sk)
1247 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 986 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1248 } 987 }
1249 988
1250 old_state_change = conn->old_state_change; 989 tcp_conn = conn->dd_data;
990 old_state_change = tcp_conn->old_state_change;
1251 991
1252 read_unlock(&sk->sk_callback_lock); 992 read_unlock(&sk->sk_callback_lock);
1253 993
@@ -1262,23 +1002,25 @@ static void
1262iscsi_write_space(struct sock *sk) 1002iscsi_write_space(struct sock *sk)
1263{ 1003{
1264 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data; 1004 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1265 conn->old_write_space(sk); 1005 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1006
1007 tcp_conn->old_write_space(sk);
1266 debug_tcp("iscsi_write_space: cid %d\n", conn->id); 1008 debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1267 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
1268 scsi_queue_work(conn->session->host, &conn->xmitwork); 1009 scsi_queue_work(conn->session->host, &conn->xmitwork);
1269} 1010}
1270 1011
1271static void 1012static void
1272iscsi_conn_set_callbacks(struct iscsi_conn *conn) 1013iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1273{ 1014{
1274 struct sock *sk = conn->sock->sk; 1015 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1016 struct sock *sk = tcp_conn->sock->sk;
1275 1017
1276 /* assign new callbacks */ 1018 /* assign new callbacks */
1277 write_lock_bh(&sk->sk_callback_lock); 1019 write_lock_bh(&sk->sk_callback_lock);
1278 sk->sk_user_data = conn; 1020 sk->sk_user_data = conn;
1279 conn->old_data_ready = sk->sk_data_ready; 1021 tcp_conn->old_data_ready = sk->sk_data_ready;
1280 conn->old_state_change = sk->sk_state_change; 1022 tcp_conn->old_state_change = sk->sk_state_change;
1281 conn->old_write_space = sk->sk_write_space; 1023 tcp_conn->old_write_space = sk->sk_write_space;
1282 sk->sk_data_ready = iscsi_tcp_data_ready; 1024 sk->sk_data_ready = iscsi_tcp_data_ready;
1283 sk->sk_state_change = iscsi_tcp_state_change; 1025 sk->sk_state_change = iscsi_tcp_state_change;
1284 sk->sk_write_space = iscsi_write_space; 1026 sk->sk_write_space = iscsi_write_space;
@@ -1288,14 +1030,15 @@ iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1288static void 1030static void
1289iscsi_conn_restore_callbacks(struct iscsi_conn *conn) 1031iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1290{ 1032{
1291 struct sock *sk = conn->sock->sk; 1033 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1034 struct sock *sk = tcp_conn->sock->sk;
1292 1035
1293 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 1036 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1294 write_lock_bh(&sk->sk_callback_lock); 1037 write_lock_bh(&sk->sk_callback_lock);
1295 sk->sk_user_data = NULL; 1038 sk->sk_user_data = NULL;
1296 sk->sk_data_ready = conn->old_data_ready; 1039 sk->sk_data_ready = tcp_conn->old_data_ready;
1297 sk->sk_state_change = conn->old_state_change; 1040 sk->sk_state_change = tcp_conn->old_state_change;
1298 sk->sk_write_space = conn->old_write_space; 1041 sk->sk_write_space = tcp_conn->old_write_space;
1299 sk->sk_no_check = 0; 1042 sk->sk_no_check = 0;
1300 write_unlock_bh(&sk->sk_callback_lock); 1043 write_unlock_bh(&sk->sk_callback_lock);
1301} 1044}
@@ -1310,8 +1053,9 @@ iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1310static inline int 1053static inline int
1311iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags) 1054iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
1312{ 1055{
1313 struct socket *sk = conn->sock; 1056 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1314 int offset = buf->sg.offset + buf->sent; 1057 struct socket *sk = tcp_conn->sock;
1058 int offset = buf->sg.offset + buf->sent, res;
1315 1059
1316 /* 1060 /*
1317 * if we got use_sg=0 or are sending something we kmallocd 1061 * if we got use_sg=0 or are sending something we kmallocd
@@ -1322,9 +1066,22 @@ iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
1322 * slab case. 1066 * slab case.
1323 */ 1067 */
1324 if (buf->use_sendmsg) 1068 if (buf->use_sendmsg)
1325 return sock_no_sendpage(sk, buf->sg.page, offset, size, flags); 1069 res = sock_no_sendpage(sk, buf->sg.page, offset, size, flags);
1070 else
1071 res = tcp_conn->sendpage(sk, buf->sg.page, offset, size, flags);
1072
1073 if (res >= 0) {
1074 conn->txdata_octets += res;
1075 buf->sent += res;
1076 return res;
1077 }
1078
1079 tcp_conn->sendpage_failures_cnt++;
1080 if (res == -EAGAIN)
1081 res = -ENOBUFS;
1326 else 1082 else
1327 return conn->sendpage(sk, buf->sg.page, offset, size, flags); 1083 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1084 return res;
1328} 1085}
1329 1086
1330/** 1087/**
@@ -1350,16 +1107,10 @@ iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1350 res = iscsi_send(conn, buf, size, flags); 1107 res = iscsi_send(conn, buf, size, flags);
1351 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res); 1108 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1352 if (res >= 0) { 1109 if (res >= 0) {
1353 conn->txdata_octets += res;
1354 buf->sent += res;
1355 if (size != res) 1110 if (size != res)
1356 return -EAGAIN; 1111 return -EAGAIN;
1357 return 0; 1112 return 0;
1358 } else if (res == -EAGAIN) { 1113 }
1359 conn->sendpage_failures_cnt++;
1360 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1361 } else if (res == -EPIPE)
1362 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1363 1114
1364 return res; 1115 return res;
1365} 1116}
@@ -1392,47 +1143,46 @@ iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1392 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n", 1143 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1393 size, buf->sent, *count, *sent, res); 1144 size, buf->sent, *count, *sent, res);
1394 if (res >= 0) { 1145 if (res >= 0) {
1395 conn->txdata_octets += res;
1396 buf->sent += res;
1397 *count -= res; 1146 *count -= res;
1398 *sent += res; 1147 *sent += res;
1399 if (size != res) 1148 if (size != res)
1400 return -EAGAIN; 1149 return -EAGAIN;
1401 return 0; 1150 return 0;
1402 } else if (res == -EAGAIN) { 1151 }
1403 conn->sendpage_failures_cnt++;
1404 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1405 } else if (res == -EPIPE)
1406 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1407 1152
1408 return res; 1153 return res;
1409} 1154}
1410 1155
1411static inline void 1156static inline void
1412iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1157iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
1158 struct iscsi_cmd_task *ctask)
1413{ 1159{
1414 BUG_ON(!conn->data_tx_tfm); 1160 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1415 crypto_digest_init(conn->data_tx_tfm); 1161
1416 ctask->digest_count = 4; 1162 BUG_ON(!tcp_conn->data_tx_tfm);
1163 crypto_digest_init(tcp_conn->data_tx_tfm);
1164 tcp_ctask->digest_count = 4;
1417} 1165}
1418 1166
1419static int 1167static int
1420iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1168iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1421 struct iscsi_buf *buf, uint32_t *digest, int final) 1169 struct iscsi_buf *buf, uint32_t *digest, int final)
1422{ 1170{
1171 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1172 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1423 int rc = 0; 1173 int rc = 0;
1424 int sent = 0; 1174 int sent = 0;
1425 1175
1426 if (final) 1176 if (final)
1427 crypto_digest_final(conn->data_tx_tfm, (u8*)digest); 1177 crypto_digest_final(tcp_conn->data_tx_tfm, (u8*)digest);
1428 1178
1429 iscsi_buf_init_virt(buf, (char*)digest, 4); 1179 iscsi_buf_init_iov(buf, (char*)digest, 4);
1430 rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent); 1180 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
1431 if (rc) { 1181 if (rc) {
1432 ctask->datadigest = *digest; 1182 tcp_ctask->datadigest = *digest;
1433 ctask->xmstate |= XMSTATE_DATA_DIGEST; 1183 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
1434 } else 1184 } else
1435 ctask->digest_count = 4; 1185 tcp_ctask->digest_count = 4;
1436 return rc; 1186 return rc;
1437} 1187}
1438 1188
@@ -1453,21 +1203,19 @@ static void
1453iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1203iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1454 struct iscsi_r2t_info *r2t, int left) 1204 struct iscsi_r2t_info *r2t, int left)
1455{ 1205{
1206 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1456 struct iscsi_data *hdr; 1207 struct iscsi_data *hdr;
1457 struct iscsi_data_task *dtask;
1458 struct scsi_cmnd *sc = ctask->sc; 1208 struct scsi_cmnd *sc = ctask->sc;
1459 int new_offset; 1209 int new_offset;
1460 1210
1461 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 1211 hdr = &r2t->dtask.hdr;
1462 BUG_ON(!dtask);
1463 hdr = &dtask->hdr;
1464 memset(hdr, 0, sizeof(struct iscsi_data)); 1212 memset(hdr, 0, sizeof(struct iscsi_data));
1465 hdr->ttt = r2t->ttt; 1213 hdr->ttt = r2t->ttt;
1466 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 1214 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1467 r2t->solicit_datasn++; 1215 r2t->solicit_datasn++;
1468 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 1216 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1469 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 1217 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
1470 hdr->itt = ctask->hdr.itt; 1218 hdr->itt = ctask->hdr->itt;
1471 hdr->exp_statsn = r2t->exp_statsn; 1219 hdr->exp_statsn = r2t->exp_statsn;
1472 new_offset = r2t->data_offset + r2t->sent; 1220 new_offset = r2t->data_offset + r2t->sent;
1473 hdr->offset = cpu_to_be32(new_offset); 1221 hdr->offset = cpu_to_be32(new_offset);
@@ -1481,181 +1229,98 @@ iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1481 } 1229 }
1482 conn->dataout_pdus_cnt++; 1230 conn->dataout_pdus_cnt++;
1483 1231
1484 iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr, 1232 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
1485 sizeof(struct iscsi_hdr)); 1233 sizeof(struct iscsi_hdr));
1486 1234
1487 r2t->dtask = dtask;
1488
1489 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) { 1235 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1490 BUG_ON(ctask->bad_sg == r2t->sg); 1236 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
1491 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1237 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1492 r2t->sg += 1; 1238 r2t->sg += 1;
1493 } else 1239 } else
1494 iscsi_buf_init_iov(&ctask->sendbuf, 1240 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1495 (char*)sc->request_buffer + new_offset, 1241 (char*)sc->request_buffer + new_offset,
1496 r2t->data_count); 1242 r2t->data_count);
1497
1498 list_add(&dtask->item, &ctask->dataqueue);
1499} 1243}
1500 1244
1501static void 1245static void
1502iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1246iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1503{ 1247{
1504 struct iscsi_data *hdr; 1248 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1505 struct iscsi_data_task *dtask; 1249 struct iscsi_data_task *dtask;
1506 1250
1507 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 1251 dtask = tcp_ctask->dtask = &tcp_ctask->unsol_dtask;
1508 BUG_ON(!dtask); 1252 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr,
1509 hdr = &dtask->hdr; 1253 tcp_ctask->r2t_data_count);
1510 memset(hdr, 0, sizeof(struct iscsi_data)); 1254 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
1511 hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1512 hdr->datasn = cpu_to_be32(ctask->unsol_datasn);
1513 ctask->unsol_datasn++;
1514 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1515 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1516 hdr->itt = ctask->hdr.itt;
1517 hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
1518 hdr->offset = cpu_to_be32(ctask->total_length -
1519 ctask->r2t_data_count -
1520 ctask->unsol_count);
1521 if (ctask->unsol_count > conn->max_xmit_dlength) {
1522 hton24(hdr->dlength, conn->max_xmit_dlength);
1523 ctask->data_count = conn->max_xmit_dlength;
1524 hdr->flags = 0;
1525 } else {
1526 hton24(hdr->dlength, ctask->unsol_count);
1527 ctask->data_count = ctask->unsol_count;
1528 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1529 }
1530
1531 iscsi_buf_init_virt(&ctask->headbuf, (char*)hdr,
1532 sizeof(struct iscsi_hdr)); 1255 sizeof(struct iscsi_hdr));
1533
1534 list_add(&dtask->item, &ctask->dataqueue);
1535
1536 ctask->dtask = dtask;
1537} 1256}
1538 1257
1539/** 1258/**
1540 * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands 1259 * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1541 * @conn: iscsi connection 1260 * @conn: iscsi connection
1542 * @ctask: scsi command task 1261 * @ctask: scsi command task
1543 * @sc: scsi command 1262 * @sc: scsi command
1544 **/ 1263 **/
1545static void 1264static void
1546iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1265iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
1547 struct scsi_cmnd *sc)
1548{ 1266{
1549 struct iscsi_session *session = conn->session; 1267 struct scsi_cmnd *sc = ctask->sc;
1550 1268 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1551 BUG_ON(__kfifo_len(ctask->r2tqueue));
1552
1553 ctask->sc = sc;
1554 ctask->conn = conn;
1555 ctask->hdr.opcode = ISCSI_OP_SCSI_CMD;
1556 ctask->hdr.flags = ISCSI_ATTR_SIMPLE;
1557 int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun);
1558 ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) |
1559 (session->age << AGE_SHIFT);
1560 ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen);
1561 ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++;
1562 ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn);
1563 memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len);
1564 memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
1565 1269
1566 ctask->mtask = NULL; 1270 BUG_ON(__kfifo_len(tcp_ctask->r2tqueue));
1567 ctask->sent = 0;
1568 ctask->sg_count = 0;
1569 1271
1570 ctask->total_length = sc->request_bufflen; 1272 tcp_ctask->sent = 0;
1273 tcp_ctask->sg_count = 0;
1571 1274
1572 if (sc->sc_data_direction == DMA_TO_DEVICE) { 1275 if (sc->sc_data_direction == DMA_TO_DEVICE) {
1573 ctask->exp_r2tsn = 0; 1276 tcp_ctask->xmstate = XMSTATE_W_HDR;
1574 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE; 1277 tcp_ctask->exp_r2tsn = 0;
1575 BUG_ON(ctask->total_length == 0); 1278 BUG_ON(ctask->total_length == 0);
1279
1576 if (sc->use_sg) { 1280 if (sc->use_sg) {
1577 struct scatterlist *sg = sc->request_buffer; 1281 struct scatterlist *sg = sc->request_buffer;
1578 1282
1579 iscsi_buf_init_sg(&ctask->sendbuf, 1283 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1580 &sg[ctask->sg_count++]); 1284 &sg[tcp_ctask->sg_count++]);
1581 ctask->sg = sg; 1285 tcp_ctask->sg = sg;
1582 ctask->bad_sg = sg + sc->use_sg; 1286 tcp_ctask->bad_sg = sg + sc->use_sg;
1583 } else { 1287 } else
1584 iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer, 1288 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1585 sc->request_bufflen); 1289 sc->request_buffer,
1586 } 1290 sc->request_bufflen);
1587 1291
1588 /* 1292 if (ctask->imm_count)
1589 * Write counters: 1293 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1590 * 1294
1591 * imm_count bytes to be sent right after 1295 tcp_ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1592 * SCSI PDU Header 1296 if (tcp_ctask->pad_count) {
1593 * 1297 tcp_ctask->pad_count = ISCSI_PAD_LEN -
1594 * unsol_count bytes(as Data-Out) to be sent 1298 tcp_ctask->pad_count;
1595 * without R2T ack right after
1596 * immediate data
1597 *
1598 * r2t_data_count bytes to be sent via R2T ack's
1599 *
1600 * pad_count bytes to be sent as zero-padding
1601 */
1602 ctask->imm_count = 0;
1603 ctask->unsol_count = 0;
1604 ctask->unsol_datasn = 0;
1605 ctask->xmstate = XMSTATE_W_HDR;
1606 /* calculate write padding */
1607 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1608 if (ctask->pad_count) {
1609 ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count;
1610 debug_scsi("write padding %d bytes\n", 1299 debug_scsi("write padding %d bytes\n",
1611 ctask->pad_count); 1300 tcp_ctask->pad_count);
1612 ctask->xmstate |= XMSTATE_W_PAD; 1301 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1613 } 1302 }
1614 if (session->imm_data_en) {
1615 if (ctask->total_length >= session->first_burst)
1616 ctask->imm_count = min(session->first_burst,
1617 conn->max_xmit_dlength);
1618 else
1619 ctask->imm_count = min(ctask->total_length,
1620 conn->max_xmit_dlength);
1621 hton24(ctask->hdr.dlength, ctask->imm_count);
1622 ctask->xmstate |= XMSTATE_IMM_DATA;
1623 } else
1624 zero_data(ctask->hdr.dlength);
1625
1626 if (!session->initial_r2t_en)
1627 ctask->unsol_count = min(session->first_burst,
1628 ctask->total_length) - ctask->imm_count;
1629 if (!ctask->unsol_count)
1630 /* No unsolicit Data-Out's */
1631 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1632 else
1633 ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1634 1303
1635 ctask->r2t_data_count = ctask->total_length - 1304 if (ctask->unsol_count)
1305 tcp_ctask->xmstate |= XMSTATE_UNS_HDR |
1306 XMSTATE_UNS_INIT;
1307 tcp_ctask->r2t_data_count = ctask->total_length -
1636 ctask->imm_count - 1308 ctask->imm_count -
1637 ctask->unsol_count; 1309 ctask->unsol_count;
1638 1310
1639 debug_scsi("cmd [itt %x total %d imm %d imm_data %d " 1311 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1640 "r2t_data %d]\n", 1312 "r2t_data %d]\n",
1641 ctask->itt, ctask->total_length, ctask->imm_count, 1313 ctask->itt, ctask->total_length, ctask->imm_count,
1642 ctask->unsol_count, ctask->r2t_data_count); 1314 ctask->unsol_count, tcp_ctask->r2t_data_count);
1643 } else { 1315 } else
1644 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL; 1316 tcp_ctask->xmstate = XMSTATE_R_HDR;
1645 if (sc->sc_data_direction == DMA_FROM_DEVICE)
1646 ctask->hdr.flags |= ISCSI_FLAG_CMD_READ;
1647 ctask->datasn = 0;
1648 ctask->xmstate = XMSTATE_R_HDR;
1649 zero_data(ctask->hdr.dlength);
1650 }
1651 1317
1652 iscsi_buf_init_virt(&ctask->headbuf, (char*)&ctask->hdr, 1318 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)ctask->hdr,
1653 sizeof(struct iscsi_hdr)); 1319 sizeof(struct iscsi_hdr));
1654 conn->scsicmd_pdus_cnt++;
1655} 1320}
1656 1321
1657/** 1322/**
1658 * iscsi_mtask_xmit - xmit management(immediate) task 1323 * iscsi_tcp_mtask_xmit - xmit management(immediate) task
1659 * @conn: iscsi connection 1324 * @conn: iscsi connection
1660 * @mtask: task management task 1325 * @mtask: task management task
1661 * 1326 *
@@ -1669,132 +1334,167 @@ iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1669 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress 1334 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1670 **/ 1335 **/
1671static int 1336static int
1672iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask) 1337iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1673{ 1338{
1339 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
1340 int rc;
1674 1341
1675 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n", 1342 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1676 conn->id, mtask->xmstate, mtask->itt); 1343 conn->id, tcp_mtask->xmstate, mtask->itt);
1677 1344
1678 if (mtask->xmstate & XMSTATE_IMM_HDR) { 1345 if (tcp_mtask->xmstate & XMSTATE_IMM_HDR) {
1679 mtask->xmstate &= ~XMSTATE_IMM_HDR; 1346 tcp_mtask->xmstate &= ~XMSTATE_IMM_HDR;
1680 if (mtask->data_count) 1347 if (mtask->data_count)
1681 mtask->xmstate |= XMSTATE_IMM_DATA; 1348 tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
1682 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE && 1349 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
1683 conn->stop_stage != STOP_CONN_RECOVER && 1350 conn->stop_stage != STOP_CONN_RECOVER &&
1684 conn->hdrdgst_en) 1351 conn->hdrdgst_en)
1685 iscsi_hdr_digest(conn, &mtask->headbuf, 1352 iscsi_hdr_digest(conn, &tcp_mtask->headbuf,
1686 (u8*)mtask->hdrext); 1353 (u8*)tcp_mtask->hdrext);
1687 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) { 1354 rc = iscsi_sendhdr(conn, &tcp_mtask->headbuf,
1688 mtask->xmstate |= XMSTATE_IMM_HDR; 1355 mtask->data_count);
1356 if (rc) {
1357 tcp_mtask->xmstate |= XMSTATE_IMM_HDR;
1689 if (mtask->data_count) 1358 if (mtask->data_count)
1690 mtask->xmstate &= ~XMSTATE_IMM_DATA; 1359 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
1691 return -EAGAIN; 1360 return rc;
1692 } 1361 }
1693 } 1362 }
1694 1363
1695 if (mtask->xmstate & XMSTATE_IMM_DATA) { 1364 if (tcp_mtask->xmstate & XMSTATE_IMM_DATA) {
1696 BUG_ON(!mtask->data_count); 1365 BUG_ON(!mtask->data_count);
1697 mtask->xmstate &= ~XMSTATE_IMM_DATA; 1366 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
1698 /* FIXME: implement. 1367 /* FIXME: implement.
1699 * Virtual buffer could be spreaded across multiple pages... 1368 * Virtual buffer could be spreaded across multiple pages...
1700 */ 1369 */
1701 do { 1370 do {
1702 if (iscsi_sendpage(conn, &mtask->sendbuf, 1371 int rc;
1703 &mtask->data_count, &mtask->sent)) { 1372
1704 mtask->xmstate |= XMSTATE_IMM_DATA; 1373 rc = iscsi_sendpage(conn, &tcp_mtask->sendbuf,
1705 return -EAGAIN; 1374 &mtask->data_count, &tcp_mtask->sent);
1375 if (rc) {
1376 tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
1377 return rc;
1706 } 1378 }
1707 } while (mtask->data_count); 1379 } while (mtask->data_count);
1708 } 1380 }
1709 1381
1710 BUG_ON(mtask->xmstate != XMSTATE_IDLE); 1382 BUG_ON(tcp_mtask->xmstate != XMSTATE_IDLE);
1383 if (mtask->hdr->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1384 struct iscsi_session *session = conn->session;
1385
1386 spin_lock_bh(&session->lock);
1387 list_del(&conn->mtask->running);
1388 __kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
1389 sizeof(void*));
1390 spin_unlock_bh(&session->lock);
1391 }
1711 return 0; 1392 return 0;
1712} 1393}
1713 1394
1714static inline int 1395static inline int
1715handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1396handle_xmstate_r_hdr(struct iscsi_conn *conn,
1397 struct iscsi_tcp_cmd_task *tcp_ctask)
1716{ 1398{
1717 ctask->xmstate &= ~XMSTATE_R_HDR; 1399 int rc;
1400
1401 tcp_ctask->xmstate &= ~XMSTATE_R_HDR;
1718 if (conn->hdrdgst_en) 1402 if (conn->hdrdgst_en)
1719 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext); 1403 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1720 if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) { 1404 (u8*)tcp_ctask->hdrext);
1721 BUG_ON(ctask->xmstate != XMSTATE_IDLE); 1405 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, 0);
1406 if (!rc) {
1407 BUG_ON(tcp_ctask->xmstate != XMSTATE_IDLE);
1722 return 0; /* wait for Data-In */ 1408 return 0; /* wait for Data-In */
1723 } 1409 }
1724 ctask->xmstate |= XMSTATE_R_HDR; 1410 tcp_ctask->xmstate |= XMSTATE_R_HDR;
1725 return -EAGAIN; 1411 return rc;
1726} 1412}
1727 1413
1728static inline int 1414static inline int
1729handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1415handle_xmstate_w_hdr(struct iscsi_conn *conn,
1416 struct iscsi_cmd_task *ctask)
1730{ 1417{
1731 ctask->xmstate &= ~XMSTATE_W_HDR; 1418 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1419 int rc;
1420
1421 tcp_ctask->xmstate &= ~XMSTATE_W_HDR;
1732 if (conn->hdrdgst_en) 1422 if (conn->hdrdgst_en)
1733 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext); 1423 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1734 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) { 1424 (u8*)tcp_ctask->hdrext);
1735 ctask->xmstate |= XMSTATE_W_HDR; 1425 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count);
1736 return -EAGAIN; 1426 if (rc)
1737 } 1427 tcp_ctask->xmstate |= XMSTATE_W_HDR;
1738 return 0; 1428 return rc;
1739} 1429}
1740 1430
1741static inline int 1431static inline int
1742handle_xmstate_data_digest(struct iscsi_conn *conn, 1432handle_xmstate_data_digest(struct iscsi_conn *conn,
1743 struct iscsi_cmd_task *ctask) 1433 struct iscsi_cmd_task *ctask)
1744{ 1434{
1745 ctask->xmstate &= ~XMSTATE_DATA_DIGEST; 1435 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1746 debug_tcp("resent data digest 0x%x\n", ctask->datadigest); 1436 int rc;
1747 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1437
1748 &ctask->datadigest, 0)) { 1438 tcp_ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1749 ctask->xmstate |= XMSTATE_DATA_DIGEST; 1439 debug_tcp("resent data digest 0x%x\n", tcp_ctask->datadigest);
1440 rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
1441 &tcp_ctask->datadigest, 0);
1442 if (rc) {
1443 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
1750 debug_tcp("resent data digest 0x%x fail!\n", 1444 debug_tcp("resent data digest 0x%x fail!\n",
1751 ctask->datadigest); 1445 tcp_ctask->datadigest);
1752 return -EAGAIN;
1753 } 1446 }
1754 return 0; 1447
1448 return rc;
1755} 1449}
1756 1450
1757static inline int 1451static inline int
1758handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1452handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1759{ 1453{
1454 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1455 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1456 int rc;
1457
1760 BUG_ON(!ctask->imm_count); 1458 BUG_ON(!ctask->imm_count);
1761 ctask->xmstate &= ~XMSTATE_IMM_DATA; 1459 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
1762 1460
1763 if (conn->datadgst_en) { 1461 if (conn->datadgst_en) {
1764 iscsi_data_digest_init(conn, ctask); 1462 iscsi_data_digest_init(tcp_conn, ctask);
1765 ctask->immdigest = 0; 1463 tcp_ctask->immdigest = 0;
1766 } 1464 }
1767 1465
1768 for (;;) { 1466 for (;;) {
1769 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count, 1467 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf,
1770 &ctask->sent)) { 1468 &ctask->imm_count, &tcp_ctask->sent);
1771 ctask->xmstate |= XMSTATE_IMM_DATA; 1469 if (rc) {
1470 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1772 if (conn->datadgst_en) { 1471 if (conn->datadgst_en) {
1773 crypto_digest_final(conn->data_tx_tfm, 1472 crypto_digest_final(tcp_conn->data_tx_tfm,
1774 (u8*)&ctask->immdigest); 1473 (u8*)&tcp_ctask->immdigest);
1775 debug_tcp("tx imm sendpage fail 0x%x\n", 1474 debug_tcp("tx imm sendpage fail 0x%x\n",
1776 ctask->datadigest); 1475 tcp_ctask->datadigest);
1777 } 1476 }
1778 return -EAGAIN; 1477 return rc;
1779 } 1478 }
1780 if (conn->datadgst_en) 1479 if (conn->datadgst_en)
1781 crypto_digest_update(conn->data_tx_tfm, 1480 crypto_digest_update(tcp_conn->data_tx_tfm,
1782 &ctask->sendbuf.sg, 1); 1481 &tcp_ctask->sendbuf.sg, 1);
1783 1482
1784 if (!ctask->imm_count) 1483 if (!ctask->imm_count)
1785 break; 1484 break;
1786 iscsi_buf_init_sg(&ctask->sendbuf, 1485 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1787 &ctask->sg[ctask->sg_count++]); 1486 &tcp_ctask->sg[tcp_ctask->sg_count++]);
1788 } 1487 }
1789 1488
1790 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) { 1489 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1791 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1490 rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
1792 &ctask->immdigest, 1)) { 1491 &tcp_ctask->immdigest, 1);
1492 if (rc) {
1793 debug_tcp("sending imm digest 0x%x fail!\n", 1493 debug_tcp("sending imm digest 0x%x fail!\n",
1794 ctask->immdigest); 1494 tcp_ctask->immdigest);
1795 return -EAGAIN; 1495 return rc;
1796 } 1496 }
1797 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest); 1497 debug_tcp("sending imm digest 0x%x\n", tcp_ctask->immdigest);
1798 } 1498 }
1799 1499
1800 return 0; 1500 return 0;
@@ -1803,74 +1503,81 @@ handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1803static inline int 1503static inline int
1804handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1504handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1805{ 1505{
1506 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1806 struct iscsi_data_task *dtask; 1507 struct iscsi_data_task *dtask;
1508 int rc;
1807 1509
1808 ctask->xmstate |= XMSTATE_UNS_DATA; 1510 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1809 if (ctask->xmstate & XMSTATE_UNS_INIT) { 1511 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
1810 iscsi_unsolicit_data_init(conn, ctask); 1512 iscsi_unsolicit_data_init(conn, ctask);
1811 BUG_ON(!ctask->dtask); 1513 dtask = tcp_ctask->dtask;
1812 dtask = ctask->dtask;
1813 if (conn->hdrdgst_en) 1514 if (conn->hdrdgst_en)
1814 iscsi_hdr_digest(conn, &ctask->headbuf, 1515 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1815 (u8*)dtask->hdrext); 1516 (u8*)dtask->hdrext);
1816 ctask->xmstate &= ~XMSTATE_UNS_INIT; 1517 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
1817 } 1518 }
1818 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) { 1519
1819 ctask->xmstate &= ~XMSTATE_UNS_DATA; 1520 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count);
1820 ctask->xmstate |= XMSTATE_UNS_HDR; 1521 if (rc) {
1821 return -EAGAIN; 1522 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1523 tcp_ctask->xmstate |= XMSTATE_UNS_HDR;
1524 return rc;
1822 } 1525 }
1823 1526
1824 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n", 1527 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1825 ctask->itt, ctask->unsol_count, ctask->sent); 1528 ctask->itt, ctask->unsol_count, tcp_ctask->sent);
1826 return 0; 1529 return 0;
1827} 1530}
1828 1531
1829static inline int 1532static inline int
1830handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1533handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1831{ 1534{
1832 struct iscsi_data_task *dtask = ctask->dtask; 1535 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1536 struct iscsi_data_task *dtask = tcp_ctask->dtask;
1537 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1538 int rc;
1833 1539
1834 BUG_ON(!ctask->data_count); 1540 BUG_ON(!ctask->data_count);
1835 ctask->xmstate &= ~XMSTATE_UNS_DATA; 1541 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1836 1542
1837 if (conn->datadgst_en) { 1543 if (conn->datadgst_en) {
1838 iscsi_data_digest_init(conn, ctask); 1544 iscsi_data_digest_init(tcp_conn, ctask);
1839 dtask->digest = 0; 1545 dtask->digest = 0;
1840 } 1546 }
1841 1547
1842 for (;;) { 1548 for (;;) {
1843 int start = ctask->sent; 1549 int start = tcp_ctask->sent;
1844 1550
1845 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count, 1551 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf,
1846 &ctask->sent)) { 1552 &ctask->data_count, &tcp_ctask->sent);
1847 ctask->unsol_count -= ctask->sent - start; 1553 if (rc) {
1848 ctask->xmstate |= XMSTATE_UNS_DATA; 1554 ctask->unsol_count -= tcp_ctask->sent - start;
1555 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1849 /* will continue with this ctask later.. */ 1556 /* will continue with this ctask later.. */
1850 if (conn->datadgst_en) { 1557 if (conn->datadgst_en) {
1851 crypto_digest_final(conn->data_tx_tfm, 1558 crypto_digest_final(tcp_conn->data_tx_tfm,
1852 (u8 *)&dtask->digest); 1559 (u8 *)&dtask->digest);
1853 debug_tcp("tx uns data fail 0x%x\n", 1560 debug_tcp("tx uns data fail 0x%x\n",
1854 dtask->digest); 1561 dtask->digest);
1855 } 1562 }
1856 return -EAGAIN; 1563 return rc;
1857 } 1564 }
1858 1565
1859 BUG_ON(ctask->sent > ctask->total_length); 1566 BUG_ON(tcp_ctask->sent > ctask->total_length);
1860 ctask->unsol_count -= ctask->sent - start; 1567 ctask->unsol_count -= tcp_ctask->sent - start;
1861 1568
1862 /* 1569 /*
1863 * XXX:we may run here with un-initial sendbuf. 1570 * XXX:we may run here with un-initial sendbuf.
1864 * so pass it 1571 * so pass it
1865 */ 1572 */
1866 if (conn->datadgst_en && ctask->sent - start > 0) 1573 if (conn->datadgst_en && tcp_ctask->sent - start > 0)
1867 crypto_digest_update(conn->data_tx_tfm, 1574 crypto_digest_update(tcp_conn->data_tx_tfm,
1868 &ctask->sendbuf.sg, 1); 1575 &tcp_ctask->sendbuf.sg, 1);
1869 1576
1870 if (!ctask->data_count) 1577 if (!ctask->data_count)
1871 break; 1578 break;
1872 iscsi_buf_init_sg(&ctask->sendbuf, 1579 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1873 &ctask->sg[ctask->sg_count++]); 1580 &tcp_ctask->sg[tcp_ctask->sg_count++]);
1874 } 1581 }
1875 BUG_ON(ctask->unsol_count < 0); 1582 BUG_ON(ctask->unsol_count < 0);
1876 1583
@@ -1880,27 +1587,29 @@ handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1880 */ 1587 */
1881 if (ctask->unsol_count) { 1588 if (ctask->unsol_count) {
1882 if (conn->datadgst_en) { 1589 if (conn->datadgst_en) {
1883 if (iscsi_digest_final_send(conn, ctask, 1590 rc = iscsi_digest_final_send(conn, ctask,
1884 &dtask->digestbuf, 1591 &dtask->digestbuf,
1885 &dtask->digest, 1)) { 1592 &dtask->digest, 1);
1593 if (rc) {
1886 debug_tcp("send uns digest 0x%x fail\n", 1594 debug_tcp("send uns digest 0x%x fail\n",
1887 dtask->digest); 1595 dtask->digest);
1888 return -EAGAIN; 1596 return rc;
1889 } 1597 }
1890 debug_tcp("sending uns digest 0x%x, more uns\n", 1598 debug_tcp("sending uns digest 0x%x, more uns\n",
1891 dtask->digest); 1599 dtask->digest);
1892 } 1600 }
1893 ctask->xmstate |= XMSTATE_UNS_INIT; 1601 tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
1894 return 1; 1602 return 1;
1895 } 1603 }
1896 1604
1897 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) { 1605 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1898 if (iscsi_digest_final_send(conn, ctask, 1606 rc = iscsi_digest_final_send(conn, ctask,
1899 &dtask->digestbuf, 1607 &dtask->digestbuf,
1900 &dtask->digest, 1)) { 1608 &dtask->digest, 1);
1609 if (rc) {
1901 debug_tcp("send last uns digest 0x%x fail\n", 1610 debug_tcp("send last uns digest 0x%x fail\n",
1902 dtask->digest); 1611 dtask->digest);
1903 return -EAGAIN; 1612 return rc;
1904 } 1613 }
1905 debug_tcp("sending uns digest 0x%x\n",dtask->digest); 1614 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1906 } 1615 }
@@ -1912,15 +1621,17 @@ static inline int
1912handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1621handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1913{ 1622{
1914 struct iscsi_session *session = conn->session; 1623 struct iscsi_session *session = conn->session;
1915 struct iscsi_r2t_info *r2t = ctask->r2t; 1624 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1916 struct iscsi_data_task *dtask = r2t->dtask; 1625 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1917 int left; 1626 struct iscsi_r2t_info *r2t = tcp_ctask->r2t;
1627 struct iscsi_data_task *dtask = &r2t->dtask;
1628 int left, rc;
1918 1629
1919 ctask->xmstate &= ~XMSTATE_SOL_DATA; 1630 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1920 ctask->dtask = dtask; 1631 tcp_ctask->dtask = dtask;
1921 1632
1922 if (conn->datadgst_en) { 1633 if (conn->datadgst_en) {
1923 iscsi_data_digest_init(conn, ctask); 1634 iscsi_data_digest_init(tcp_conn, ctask);
1924 dtask->digest = 0; 1635 dtask->digest = 0;
1925 } 1636 }
1926solicit_again: 1637solicit_again:
@@ -1930,25 +1641,27 @@ solicit_again:
1930 if (!r2t->data_count) 1641 if (!r2t->data_count)
1931 goto data_out_done; 1642 goto data_out_done;
1932 1643
1933 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) { 1644 rc = iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent);
1934 ctask->xmstate |= XMSTATE_SOL_DATA; 1645 if (rc) {
1646 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1935 /* will continue with this ctask later.. */ 1647 /* will continue with this ctask later.. */
1936 if (conn->datadgst_en) { 1648 if (conn->datadgst_en) {
1937 crypto_digest_final(conn->data_tx_tfm, 1649 crypto_digest_final(tcp_conn->data_tx_tfm,
1938 (u8 *)&dtask->digest); 1650 (u8 *)&dtask->digest);
1939 debug_tcp("r2t data send fail 0x%x\n", dtask->digest); 1651 debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1940 } 1652 }
1941 return -EAGAIN; 1653 return rc;
1942 } 1654 }
1943 1655
1944 BUG_ON(r2t->data_count < 0); 1656 BUG_ON(r2t->data_count < 0);
1945 if (conn->datadgst_en) 1657 if (conn->datadgst_en)
1946 crypto_digest_update(conn->data_tx_tfm, &r2t->sendbuf.sg, 1); 1658 crypto_digest_update(tcp_conn->data_tx_tfm, &r2t->sendbuf.sg,
1659 1);
1947 1660
1948 if (r2t->data_count) { 1661 if (r2t->data_count) {
1949 BUG_ON(ctask->sc->use_sg == 0); 1662 BUG_ON(ctask->sc->use_sg == 0);
1950 if (!iscsi_buf_left(&r2t->sendbuf)) { 1663 if (!iscsi_buf_left(&r2t->sendbuf)) {
1951 BUG_ON(ctask->bad_sg == r2t->sg); 1664 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
1952 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1665 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1953 r2t->sg += 1; 1666 r2t->sg += 1;
1954 } 1667 }
@@ -1964,19 +1677,20 @@ data_out_done:
1964 left = r2t->data_length - r2t->sent; 1677 left = r2t->data_length - r2t->sent;
1965 if (left) { 1678 if (left) {
1966 if (conn->datadgst_en) { 1679 if (conn->datadgst_en) {
1967 if (iscsi_digest_final_send(conn, ctask, 1680 rc = iscsi_digest_final_send(conn, ctask,
1968 &dtask->digestbuf, 1681 &dtask->digestbuf,
1969 &dtask->digest, 1)) { 1682 &dtask->digest, 1);
1683 if (rc) {
1970 debug_tcp("send r2t data digest 0x%x" 1684 debug_tcp("send r2t data digest 0x%x"
1971 "fail\n", dtask->digest); 1685 "fail\n", dtask->digest);
1972 return -EAGAIN; 1686 return rc;
1973 } 1687 }
1974 debug_tcp("r2t data send digest 0x%x\n", 1688 debug_tcp("r2t data send digest 0x%x\n",
1975 dtask->digest); 1689 dtask->digest);
1976 } 1690 }
1977 iscsi_solicit_data_cont(conn, ctask, r2t, left); 1691 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1978 ctask->xmstate |= XMSTATE_SOL_DATA; 1692 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1979 ctask->xmstate &= ~XMSTATE_SOL_HDR; 1693 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1980 return 1; 1694 return 1;
1981 } 1695 }
1982 1696
@@ -1984,26 +1698,27 @@ data_out_done:
1984 * Done with this R2T. Check if there are more 1698 * Done with this R2T. Check if there are more
1985 * outstanding R2Ts ready to be processed. 1699 * outstanding R2Ts ready to be processed.
1986 */ 1700 */
1987 BUG_ON(ctask->r2t_data_count - r2t->data_length < 0); 1701 BUG_ON(tcp_ctask->r2t_data_count - r2t->data_length < 0);
1988 if (conn->datadgst_en) { 1702 if (conn->datadgst_en) {
1989 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf, 1703 rc = iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1990 &dtask->digest, 1)) { 1704 &dtask->digest, 1);
1705 if (rc) {
1991 debug_tcp("send last r2t data digest 0x%x" 1706 debug_tcp("send last r2t data digest 0x%x"
1992 "fail\n", dtask->digest); 1707 "fail\n", dtask->digest);
1993 return -EAGAIN; 1708 return rc;
1994 } 1709 }
1995 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest); 1710 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1996 } 1711 }
1997 1712
1998 ctask->r2t_data_count -= r2t->data_length; 1713 tcp_ctask->r2t_data_count -= r2t->data_length;
1999 ctask->r2t = NULL; 1714 tcp_ctask->r2t = NULL;
2000 spin_lock_bh(&session->lock); 1715 spin_lock_bh(&session->lock);
2001 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 1716 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
2002 spin_unlock_bh(&session->lock); 1717 spin_unlock_bh(&session->lock);
2003 if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) { 1718 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
2004 ctask->r2t = r2t; 1719 tcp_ctask->r2t = r2t;
2005 ctask->xmstate |= XMSTATE_SOL_DATA; 1720 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
2006 ctask->xmstate &= ~XMSTATE_SOL_HDR; 1721 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
2007 return 1; 1722 return 1;
2008 } 1723 }
2009 1724
@@ -2013,36 +1728,44 @@ data_out_done:
2013static inline int 1728static inline int
2014handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1729handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2015{ 1730{
2016 struct iscsi_data_task *dtask = ctask->dtask; 1731 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2017 int sent; 1732 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2018 1733 struct iscsi_data_task *dtask = tcp_ctask->dtask;
2019 ctask->xmstate &= ~XMSTATE_W_PAD; 1734 int sent, rc;
2020 iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad, 1735
2021 ctask->pad_count); 1736 tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
2022 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) { 1737 iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
2023 ctask->xmstate |= XMSTATE_W_PAD; 1738 tcp_ctask->pad_count);
2024 return -EAGAIN; 1739 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
1740 &sent);
1741 if (rc) {
1742 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1743 return rc;
2025 } 1744 }
2026 1745
2027 if (conn->datadgst_en) { 1746 if (conn->datadgst_en) {
2028 crypto_digest_update(conn->data_tx_tfm, &ctask->sendbuf.sg, 1); 1747 crypto_digest_update(tcp_conn->data_tx_tfm,
1748 &tcp_ctask->sendbuf.sg, 1);
2029 /* imm data? */ 1749 /* imm data? */
2030 if (!dtask) { 1750 if (!dtask) {
2031 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1751 rc = iscsi_digest_final_send(conn, ctask,
2032 &ctask->immdigest, 1)) { 1752 &tcp_ctask->immbuf,
1753 &tcp_ctask->immdigest, 1);
1754 if (rc) {
2033 debug_tcp("send padding digest 0x%x" 1755 debug_tcp("send padding digest 0x%x"
2034 "fail!\n", ctask->immdigest); 1756 "fail!\n", tcp_ctask->immdigest);
2035 return -EAGAIN; 1757 return rc;
2036 } 1758 }
2037 debug_tcp("done with padding, digest 0x%x\n", 1759 debug_tcp("done with padding, digest 0x%x\n",
2038 ctask->datadigest); 1760 tcp_ctask->datadigest);
2039 } else { 1761 } else {
2040 if (iscsi_digest_final_send(conn, ctask, 1762 rc = iscsi_digest_final_send(conn, ctask,
2041 &dtask->digestbuf, 1763 &dtask->digestbuf,
2042 &dtask->digest, 1)) { 1764 &dtask->digest, 1);
1765 if (rc) {
2043 debug_tcp("send padding digest 0x%x" 1766 debug_tcp("send padding digest 0x%x"
2044 "fail\n", dtask->digest); 1767 "fail\n", dtask->digest);
2045 return -EAGAIN; 1768 return rc;
2046 } 1769 }
2047 debug_tcp("done with padding, digest 0x%x\n", 1770 debug_tcp("done with padding, digest 0x%x\n",
2048 dtask->digest); 1771 dtask->digest);
@@ -2053,12 +1776,13 @@ handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2053} 1776}
2054 1777
2055static int 1778static int
2056iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1779iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2057{ 1780{
1781 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2058 int rc = 0; 1782 int rc = 0;
2059 1783
2060 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n", 1784 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
2061 conn->id, ctask->xmstate, ctask->itt); 1785 conn->id, tcp_ctask->xmstate, ctask->itt);
2062 1786
2063 /* 1787 /*
2064 * serialize with TMF AbortTask 1788 * serialize with TMF AbortTask
@@ -2066,40 +1790,38 @@ iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2066 if (ctask->mtask) 1790 if (ctask->mtask)
2067 return rc; 1791 return rc;
2068 1792
2069 if (ctask->xmstate & XMSTATE_R_HDR) { 1793 if (tcp_ctask->xmstate & XMSTATE_R_HDR)
2070 rc = handle_xmstate_r_hdr(conn, ctask); 1794 return handle_xmstate_r_hdr(conn, tcp_ctask);
2071 return rc;
2072 }
2073 1795
2074 if (ctask->xmstate & XMSTATE_W_HDR) { 1796 if (tcp_ctask->xmstate & XMSTATE_W_HDR) {
2075 rc = handle_xmstate_w_hdr(conn, ctask); 1797 rc = handle_xmstate_w_hdr(conn, ctask);
2076 if (rc) 1798 if (rc)
2077 return rc; 1799 return rc;
2078 } 1800 }
2079 1801
2080 /* XXX: for data digest xmit recover */ 1802 /* XXX: for data digest xmit recover */
2081 if (ctask->xmstate & XMSTATE_DATA_DIGEST) { 1803 if (tcp_ctask->xmstate & XMSTATE_DATA_DIGEST) {
2082 rc = handle_xmstate_data_digest(conn, ctask); 1804 rc = handle_xmstate_data_digest(conn, ctask);
2083 if (rc) 1805 if (rc)
2084 return rc; 1806 return rc;
2085 } 1807 }
2086 1808
2087 if (ctask->xmstate & XMSTATE_IMM_DATA) { 1809 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
2088 rc = handle_xmstate_imm_data(conn, ctask); 1810 rc = handle_xmstate_imm_data(conn, ctask);
2089 if (rc) 1811 if (rc)
2090 return rc; 1812 return rc;
2091 } 1813 }
2092 1814
2093 if (ctask->xmstate & XMSTATE_UNS_HDR) { 1815 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
2094 BUG_ON(!ctask->unsol_count); 1816 BUG_ON(!ctask->unsol_count);
2095 ctask->xmstate &= ~XMSTATE_UNS_HDR; 1817 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
2096unsolicit_head_again: 1818unsolicit_head_again:
2097 rc = handle_xmstate_uns_hdr(conn, ctask); 1819 rc = handle_xmstate_uns_hdr(conn, ctask);
2098 if (rc) 1820 if (rc)
2099 return rc; 1821 return rc;
2100 } 1822 }
2101 1823
2102 if (ctask->xmstate & XMSTATE_UNS_DATA) { 1824 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
2103 rc = handle_xmstate_uns_data(conn, ctask); 1825 rc = handle_xmstate_uns_data(conn, ctask);
2104 if (rc == 1) 1826 if (rc == 1)
2105 goto unsolicit_head_again; 1827 goto unsolicit_head_again;
@@ -2108,23 +1830,24 @@ unsolicit_head_again:
2108 goto done; 1830 goto done;
2109 } 1831 }
2110 1832
2111 if (ctask->xmstate & XMSTATE_SOL_HDR) { 1833 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
2112 struct iscsi_r2t_info *r2t; 1834 struct iscsi_r2t_info *r2t;
2113 1835
2114 ctask->xmstate &= ~XMSTATE_SOL_HDR; 1836 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
2115 ctask->xmstate |= XMSTATE_SOL_DATA; 1837 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
2116 if (!ctask->r2t) 1838 if (!tcp_ctask->r2t)
2117 __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t, 1839 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
2118 sizeof(void*)); 1840 sizeof(void*));
2119solicit_head_again: 1841solicit_head_again:
2120 r2t = ctask->r2t; 1842 r2t = tcp_ctask->r2t;
2121 if (conn->hdrdgst_en) 1843 if (conn->hdrdgst_en)
2122 iscsi_hdr_digest(conn, &r2t->headbuf, 1844 iscsi_hdr_digest(conn, &r2t->headbuf,
2123 (u8*)r2t->dtask->hdrext); 1845 (u8*)r2t->dtask.hdrext);
2124 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) { 1846 rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
2125 ctask->xmstate &= ~XMSTATE_SOL_DATA; 1847 if (rc) {
2126 ctask->xmstate |= XMSTATE_SOL_HDR; 1848 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
2127 return -EAGAIN; 1849 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
1850 return rc;
2128 } 1851 }
2129 1852
2130 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n", 1853 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
@@ -2132,7 +1855,7 @@ solicit_head_again:
2132 r2t->sent); 1855 r2t->sent);
2133 } 1856 }
2134 1857
2135 if (ctask->xmstate & XMSTATE_SOL_DATA) { 1858 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
2136 rc = handle_xmstate_sol_data(conn, ctask); 1859 rc = handle_xmstate_sol_data(conn, ctask);
2137 if (rc == 1) 1860 if (rc == 1)
2138 goto solicit_head_again; 1861 goto solicit_head_again;
@@ -2145,1022 +1868,199 @@ done:
2145 * Last thing to check is whether we need to send write 1868 * Last thing to check is whether we need to send write
2146 * padding. Note that we check for xmstate equality, not just the bit. 1869 * padding. Note that we check for xmstate equality, not just the bit.
2147 */ 1870 */
2148 if (ctask->xmstate == XMSTATE_W_PAD) 1871 if (tcp_ctask->xmstate == XMSTATE_W_PAD)
2149 rc = handle_xmstate_w_pad(conn, ctask); 1872 rc = handle_xmstate_w_pad(conn, ctask);
2150 1873
2151 return rc; 1874 return rc;
2152} 1875}
2153 1876
2154/**
2155 * iscsi_data_xmit - xmit any command into the scheduled connection
2156 * @conn: iscsi connection
2157 *
2158 * Notes:
2159 * The function can return -EAGAIN in which case the caller must
2160 * re-schedule it again later or recover. '0' return code means
2161 * successful xmit.
2162 **/
2163static int
2164iscsi_data_xmit(struct iscsi_conn *conn)
2165{
2166 if (unlikely(conn->suspend_tx)) {
2167 debug_tcp("conn %d Tx suspended!\n", conn->id);
2168 return 0;
2169 }
2170
2171 /*
2172 * Transmit in the following order:
2173 *
2174 * 1) un-finished xmit (ctask or mtask)
2175 * 2) immediate control PDUs
2176 * 3) write data
2177 * 4) SCSI commands
2178 * 5) non-immediate control PDUs
2179 *
2180 * No need to lock around __kfifo_get as long as
2181 * there's one producer and one consumer.
2182 */
2183
2184 BUG_ON(conn->ctask && conn->mtask);
2185
2186 if (conn->ctask) {
2187 if (iscsi_ctask_xmit(conn, conn->ctask))
2188 goto again;
2189 /* done with this in-progress ctask */
2190 conn->ctask = NULL;
2191 }
2192 if (conn->mtask) {
2193 if (iscsi_mtask_xmit(conn, conn->mtask))
2194 goto again;
2195 /* done with this in-progress mtask */
2196 conn->mtask = NULL;
2197 }
2198
2199 /* process immediate first */
2200 if (unlikely(__kfifo_len(conn->immqueue))) {
2201 struct iscsi_session *session = conn->session;
2202 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask,
2203 sizeof(void*))) {
2204 if (iscsi_mtask_xmit(conn, conn->mtask))
2205 goto again;
2206
2207 if (conn->mtask->hdr.itt ==
2208 cpu_to_be32(ISCSI_RESERVED_TAG)) {
2209 spin_lock_bh(&session->lock);
2210 __kfifo_put(session->mgmtpool.queue,
2211 (void*)&conn->mtask, sizeof(void*));
2212 spin_unlock_bh(&session->lock);
2213 }
2214 }
2215 /* done with this mtask */
2216 conn->mtask = NULL;
2217 }
2218
2219 /* process write queue */
2220 while (__kfifo_get(conn->writequeue, (void*)&conn->ctask,
2221 sizeof(void*))) {
2222 if (iscsi_ctask_xmit(conn, conn->ctask))
2223 goto again;
2224 }
2225
2226 /* process command queue */
2227 while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask,
2228 sizeof(void*))) {
2229 if (iscsi_ctask_xmit(conn, conn->ctask))
2230 goto again;
2231 }
2232 /* done with this ctask */
2233 conn->ctask = NULL;
2234
2235 /* process the rest control plane PDUs, if any */
2236 if (unlikely(__kfifo_len(conn->mgmtqueue))) {
2237 struct iscsi_session *session = conn->session;
2238
2239 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask,
2240 sizeof(void*))) {
2241 if (iscsi_mtask_xmit(conn, conn->mtask))
2242 goto again;
2243
2244 if (conn->mtask->hdr.itt ==
2245 cpu_to_be32(ISCSI_RESERVED_TAG)) {
2246 spin_lock_bh(&session->lock);
2247 __kfifo_put(session->mgmtpool.queue,
2248 (void*)&conn->mtask,
2249 sizeof(void*));
2250 spin_unlock_bh(&session->lock);
2251 }
2252 }
2253 /* done with this mtask */
2254 conn->mtask = NULL;
2255 }
2256
2257 return 0;
2258
2259again:
2260 if (unlikely(conn->suspend_tx))
2261 return 0;
2262
2263 return -EAGAIN;
2264}
2265
2266static void
2267iscsi_xmitworker(void *data)
2268{
2269 struct iscsi_conn *conn = data;
2270
2271 /*
2272 * serialize Xmit worker on a per-connection basis.
2273 */
2274 mutex_lock(&conn->xmitmutex);
2275 if (iscsi_data_xmit(conn))
2276 scsi_queue_work(conn->session->host, &conn->xmitwork);
2277 mutex_unlock(&conn->xmitmutex);
2278}
2279
2280#define FAILURE_BAD_HOST 1
2281#define FAILURE_SESSION_FAILED 2
2282#define FAILURE_SESSION_FREED 3
2283#define FAILURE_WINDOW_CLOSED 4
2284#define FAILURE_SESSION_TERMINATE 5
2285
2286static int
2287iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
2288{
2289 struct Scsi_Host *host;
2290 int reason = 0;
2291 struct iscsi_session *session;
2292 struct iscsi_conn *conn = NULL;
2293 struct iscsi_cmd_task *ctask = NULL;
2294
2295 sc->scsi_done = done;
2296 sc->result = 0;
2297
2298 host = sc->device->host;
2299 session = iscsi_hostdata(host->hostdata);
2300 BUG_ON(host != session->host);
2301
2302 spin_lock(&session->lock);
2303
2304 if (session->state != ISCSI_STATE_LOGGED_IN) {
2305 if (session->state == ISCSI_STATE_FAILED) {
2306 reason = FAILURE_SESSION_FAILED;
2307 goto reject;
2308 } else if (session->state == ISCSI_STATE_TERMINATE) {
2309 reason = FAILURE_SESSION_TERMINATE;
2310 goto fault;
2311 }
2312 reason = FAILURE_SESSION_FREED;
2313 goto fault;
2314 }
2315
2316 /*
2317 * Check for iSCSI window and take care of CmdSN wrap-around
2318 */
2319 if ((int)(session->max_cmdsn - session->cmdsn) < 0) {
2320 reason = FAILURE_WINDOW_CLOSED;
2321 goto reject;
2322 }
2323
2324 conn = session->leadconn;
2325
2326 __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
2327 BUG_ON(ctask->sc);
2328
2329 sc->SCp.phase = session->age;
2330 sc->SCp.ptr = (char*)ctask;
2331 iscsi_cmd_init(conn, ctask, sc);
2332
2333 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
2334 debug_scsi(
2335 "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
2336 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
2337 conn->id, (long)sc, ctask->itt, sc->request_bufflen,
2338 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
2339 spin_unlock(&session->lock);
2340
2341 scsi_queue_work(host, &conn->xmitwork);
2342 return 0;
2343
2344reject:
2345 spin_unlock(&session->lock);
2346 debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
2347 return SCSI_MLQUEUE_HOST_BUSY;
2348
2349fault:
2350 spin_unlock(&session->lock);
2351 printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n",
2352 sc->cmnd[0], reason);
2353 sc->sense_buffer[0] = 0x70;
2354 sc->sense_buffer[2] = NOT_READY;
2355 sc->sense_buffer[7] = 0x6;
2356 sc->sense_buffer[12] = 0x08;
2357 sc->sense_buffer[13] = 0x00;
2358 sc->result = (DID_NO_CONNECT << 16);
2359 sc->resid = sc->request_bufflen;
2360 sc->scsi_done(sc);
2361 return 0;
2362}
2363
2364static int
2365iscsi_change_queue_depth(struct scsi_device *sdev, int depth)
2366{
2367 if (depth > ISCSI_MAX_CMD_PER_LUN)
2368 depth = ISCSI_MAX_CMD_PER_LUN;
2369 scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth);
2370 return sdev->queue_depth;
2371}
2372
2373static int
2374iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size)
2375{
2376 int i;
2377
2378 *items = kmalloc(max * sizeof(void*), GFP_KERNEL);
2379 if (*items == NULL)
2380 return -ENOMEM;
2381
2382 q->max = max;
2383 q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL);
2384 if (q->pool == NULL) {
2385 kfree(*items);
2386 return -ENOMEM;
2387 }
2388
2389 q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2390 GFP_KERNEL, NULL);
2391 if (q->queue == ERR_PTR(-ENOMEM)) {
2392 kfree(q->pool);
2393 kfree(*items);
2394 return -ENOMEM;
2395 }
2396
2397 for (i = 0; i < max; i++) {
2398 q->pool[i] = kmalloc(item_size, GFP_KERNEL);
2399 if (q->pool[i] == NULL) {
2400 int j;
2401
2402 for (j = 0; j < i; j++)
2403 kfree(q->pool[j]);
2404
2405 kfifo_free(q->queue);
2406 kfree(q->pool);
2407 kfree(*items);
2408 return -ENOMEM;
2409 }
2410 memset(q->pool[i], 0, item_size);
2411 (*items)[i] = q->pool[i];
2412 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2413 }
2414 return 0;
2415}
2416
2417static void
2418iscsi_pool_free(struct iscsi_queue *q, void **items)
2419{
2420 int i;
2421
2422 for (i = 0; i < q->max; i++)
2423 kfree(items[i]);
2424 kfree(q->pool);
2425 kfree(items);
2426}
2427
2428static struct iscsi_cls_conn * 1877static struct iscsi_cls_conn *
2429iscsi_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx) 1878iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
2430{ 1879{
2431 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
2432 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
2433 struct iscsi_conn *conn; 1880 struct iscsi_conn *conn;
2434 struct iscsi_cls_conn *cls_conn; 1881 struct iscsi_cls_conn *cls_conn;
1882 struct iscsi_tcp_conn *tcp_conn;
2435 1883
2436 cls_conn = iscsi_create_conn(cls_session, conn_idx); 1884 cls_conn = iscsi_conn_setup(cls_session, conn_idx);
2437 if (!cls_conn) 1885 if (!cls_conn)
2438 return NULL; 1886 return NULL;
2439 conn = cls_conn->dd_data; 1887 conn = cls_conn->dd_data;
2440 memset(conn, 0, sizeof(*conn)); 1888 /*
2441 1889 * due to strange issues with iser these are not set
2442 conn->cls_conn = cls_conn; 1890 * in iscsi_conn_setup
2443 conn->c_stage = ISCSI_CONN_INITIAL_STAGE; 1891 */
2444 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2445 conn->id = conn_idx;
2446 conn->exp_statsn = 0;
2447 conn->tmabort_state = TMABORT_INITIAL;
2448
2449 /* initial operational parameters */
2450 conn->hdr_size = sizeof(struct iscsi_hdr);
2451 conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2452 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH; 1892 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2453 1893
2454 /* initialize general xmit PDU commands queue */ 1894 tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL);
2455 conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*), 1895 if (!tcp_conn)
2456 GFP_KERNEL, NULL); 1896 goto tcp_conn_alloc_fail;
2457 if (conn->xmitqueue == ERR_PTR(-ENOMEM))
2458 goto xmitqueue_alloc_fail;
2459
2460 /* initialize write response PDU commands queue */
2461 conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*),
2462 GFP_KERNEL, NULL);
2463 if (conn->writequeue == ERR_PTR(-ENOMEM))
2464 goto writequeue_alloc_fail;
2465
2466 /* initialize general immediate & non-immediate PDU commands queue */
2467 conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2468 GFP_KERNEL, NULL);
2469 if (conn->immqueue == ERR_PTR(-ENOMEM))
2470 goto immqueue_alloc_fail;
2471 1897
2472 conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*), 1898 conn->dd_data = tcp_conn;
2473 GFP_KERNEL, NULL); 1899 tcp_conn->iscsi_conn = conn;
2474 if (conn->mgmtqueue == ERR_PTR(-ENOMEM)) 1900 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2475 goto mgmtqueue_alloc_fail; 1901 /* initial operational parameters */
2476 1902 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
2477 INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn); 1903 tcp_conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2478
2479 /* allocate login_mtask used for the login/text sequences */
2480 spin_lock_bh(&session->lock);
2481 if (!__kfifo_get(session->mgmtpool.queue,
2482 (void*)&conn->login_mtask,
2483 sizeof(void*))) {
2484 spin_unlock_bh(&session->lock);
2485 goto login_mtask_alloc_fail;
2486 }
2487 spin_unlock_bh(&session->lock);
2488 1904
2489 /* allocate initial PDU receive place holder */ 1905 /* allocate initial PDU receive place holder */
2490 if (conn->data_size <= PAGE_SIZE) 1906 if (tcp_conn->data_size <= PAGE_SIZE)
2491 conn->data = kmalloc(conn->data_size, GFP_KERNEL); 1907 tcp_conn->data = kmalloc(tcp_conn->data_size, GFP_KERNEL);
2492 else 1908 else
2493 conn->data = (void*)__get_free_pages(GFP_KERNEL, 1909 tcp_conn->data = (void*)__get_free_pages(GFP_KERNEL,
2494 get_order(conn->data_size)); 1910 get_order(tcp_conn->data_size));
2495 if (!conn->data) 1911 if (!tcp_conn->data)
2496 goto max_recv_dlenght_alloc_fail; 1912 goto max_recv_dlenght_alloc_fail;
2497 1913
2498 init_timer(&conn->tmabort_timer);
2499 mutex_init(&conn->xmitmutex);
2500 init_waitqueue_head(&conn->ehwait);
2501
2502 return cls_conn; 1914 return cls_conn;
2503 1915
2504max_recv_dlenght_alloc_fail: 1916max_recv_dlenght_alloc_fail:
2505 spin_lock_bh(&session->lock); 1917 kfree(tcp_conn);
2506 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask, 1918tcp_conn_alloc_fail:
2507 sizeof(void*)); 1919 iscsi_conn_teardown(cls_conn);
2508 spin_unlock_bh(&session->lock);
2509login_mtask_alloc_fail:
2510 kfifo_free(conn->mgmtqueue);
2511mgmtqueue_alloc_fail:
2512 kfifo_free(conn->immqueue);
2513immqueue_alloc_fail:
2514 kfifo_free(conn->writequeue);
2515writequeue_alloc_fail:
2516 kfifo_free(conn->xmitqueue);
2517xmitqueue_alloc_fail:
2518 iscsi_destroy_conn(cls_conn);
2519 return NULL; 1920 return NULL;
2520} 1921}
2521 1922
2522static void 1923static void
2523iscsi_conn_destroy(struct iscsi_cls_conn *cls_conn) 1924iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
2524{ 1925{
2525 struct iscsi_conn *conn = cls_conn->dd_data; 1926 struct iscsi_conn *conn = cls_conn->dd_data;
2526 struct iscsi_session *session = conn->session; 1927 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2527 unsigned long flags; 1928 int digest = 0;
2528
2529 mutex_lock(&conn->xmitmutex);
2530 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2531 if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) {
2532 struct sock *sk = conn->sock->sk;
2533
2534 /*
2535 * conn_start() has never been called!
2536 * need to cleanup the socket.
2537 */
2538 write_lock_bh(&sk->sk_callback_lock);
2539 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2540 write_unlock_bh(&sk->sk_callback_lock);
2541
2542 sock_hold(conn->sock->sk);
2543 iscsi_conn_restore_callbacks(conn);
2544 sock_put(conn->sock->sk);
2545 sock_release(conn->sock);
2546 conn->sock = NULL;
2547 }
2548 1929
2549 spin_lock_bh(&session->lock); 1930 if (conn->hdrdgst_en || conn->datadgst_en)
2550 conn->c_stage = ISCSI_CONN_CLEANUP_WAIT; 1931 digest = 1;
2551 if (session->leadconn == conn) {
2552 /*
2553 * leading connection? then give up on recovery.
2554 */
2555 session->state = ISCSI_STATE_TERMINATE;
2556 wake_up(&conn->ehwait);
2557 }
2558 spin_unlock_bh(&session->lock);
2559 1932
2560 mutex_unlock(&conn->xmitmutex); 1933 iscsi_conn_teardown(cls_conn);
2561
2562 /*
2563 * Block until all in-progress commands for this connection
2564 * time out or fail.
2565 */
2566 for (;;) {
2567 spin_lock_irqsave(session->host->host_lock, flags);
2568 if (!session->host->host_busy) { /* OK for ERL == 0 */
2569 spin_unlock_irqrestore(session->host->host_lock, flags);
2570 break;
2571 }
2572 spin_unlock_irqrestore(session->host->host_lock, flags);
2573 msleep_interruptible(500);
2574 printk("conn_destroy(): host_busy %d host_failed %d\n",
2575 session->host->host_busy, session->host->host_failed);
2576 /*
2577 * force eh_abort() to unblock
2578 */
2579 wake_up(&conn->ehwait);
2580 }
2581 1934
2582 /* now free crypto */ 1935 /* now free tcp_conn */
2583 if (conn->hdrdgst_en || conn->datadgst_en) { 1936 if (digest) {
2584 if (conn->tx_tfm) 1937 if (tcp_conn->tx_tfm)
2585 crypto_free_tfm(conn->tx_tfm); 1938 crypto_free_tfm(tcp_conn->tx_tfm);
2586 if (conn->rx_tfm) 1939 if (tcp_conn->rx_tfm)
2587 crypto_free_tfm(conn->rx_tfm); 1940 crypto_free_tfm(tcp_conn->rx_tfm);
2588 if (conn->data_tx_tfm) 1941 if (tcp_conn->data_tx_tfm)
2589 crypto_free_tfm(conn->data_tx_tfm); 1942 crypto_free_tfm(tcp_conn->data_tx_tfm);
2590 if (conn->data_rx_tfm) 1943 if (tcp_conn->data_rx_tfm)
2591 crypto_free_tfm(conn->data_rx_tfm); 1944 crypto_free_tfm(tcp_conn->data_rx_tfm);
2592 } 1945 }
2593 1946
2594 /* free conn->data, size = MaxRecvDataSegmentLength */ 1947 /* free conn->data, size = MaxRecvDataSegmentLength */
2595 if (conn->data_size <= PAGE_SIZE) 1948 if (tcp_conn->data_size <= PAGE_SIZE)
2596 kfree(conn->data); 1949 kfree(tcp_conn->data);
2597 else 1950 else
2598 free_pages((unsigned long)conn->data, 1951 free_pages((unsigned long)tcp_conn->data,
2599 get_order(conn->data_size)); 1952 get_order(tcp_conn->data_size));
2600 1953 kfree(tcp_conn);
2601 spin_lock_bh(&session->lock);
2602 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2603 sizeof(void*));
2604 list_del(&conn->item);
2605 if (list_empty(&session->connections))
2606 session->leadconn = NULL;
2607 if (session->leadconn && session->leadconn == conn)
2608 session->leadconn = container_of(session->connections.next,
2609 struct iscsi_conn, item);
2610
2611 if (session->leadconn == NULL)
2612 /* none connections exits.. reset sequencing */
2613 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1;
2614 spin_unlock_bh(&session->lock);
2615
2616 kfifo_free(conn->xmitqueue);
2617 kfifo_free(conn->writequeue);
2618 kfifo_free(conn->immqueue);
2619 kfifo_free(conn->mgmtqueue);
2620
2621 iscsi_destroy_conn(cls_conn);
2622} 1954}
2623 1955
2624static int 1956static int
2625iscsi_conn_bind(struct iscsi_cls_session *cls_session, 1957iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
2626 struct iscsi_cls_conn *cls_conn, uint32_t transport_fd, 1958 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
2627 int is_leading) 1959 int is_leading)
2628{ 1960{
2629 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 1961 struct iscsi_conn *conn = cls_conn->dd_data;
2630 struct iscsi_session *session = iscsi_hostdata(shost->hostdata); 1962 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2631 struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = cls_conn->dd_data;
2632 struct sock *sk; 1963 struct sock *sk;
2633 struct socket *sock; 1964 struct socket *sock;
2634 int err; 1965 int err;
2635 1966
2636 /* lookup for existing socket */ 1967 /* lookup for existing socket */
2637 sock = sockfd_lookup(transport_fd, &err); 1968 sock = sockfd_lookup((int)transport_eph, &err);
2638 if (!sock) { 1969 if (!sock) {
2639 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err); 1970 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
2640 return -EEXIST; 1971 return -EEXIST;
2641 } 1972 }
2642 1973
2643 /* lookup for existing connection */ 1974 err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
2644 spin_lock_bh(&session->lock); 1975 if (err)
2645 list_for_each_entry(tmp, &session->connections, item) { 1976 return err;
2646 if (tmp == conn) {
2647 if (conn->c_stage != ISCSI_CONN_STOPPED ||
2648 conn->stop_stage == STOP_CONN_TERM) {
2649 printk(KERN_ERR "iscsi_tcp: can't bind "
2650 "non-stopped connection (%d:%d)\n",
2651 conn->c_stage, conn->stop_stage);
2652 spin_unlock_bh(&session->lock);
2653 return -EIO;
2654 }
2655 break;
2656 }
2657 }
2658 if (tmp != conn) {
2659 /* bind new iSCSI connection to session */
2660 conn->session = session;
2661 1977
2662 list_add(&conn->item, &session->connections); 1978 /* bind iSCSI connection and socket */
2663 } 1979 tcp_conn->sock = sock;
2664 spin_unlock_bh(&session->lock);
2665
2666 if (conn->stop_stage != STOP_CONN_SUSPEND) {
2667 /* bind iSCSI connection and socket */
2668 conn->sock = sock;
2669
2670 /* setup Socket parameters */
2671 sk = sock->sk;
2672 sk->sk_reuse = 1;
2673 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2674 sk->sk_allocation = GFP_ATOMIC;
2675 1980
2676 /* FIXME: disable Nagle's algorithm */ 1981 /* setup Socket parameters */
1982 sk = sock->sk;
1983 sk->sk_reuse = 1;
1984 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
1985 sk->sk_allocation = GFP_ATOMIC;
2677 1986
2678 /* 1987 /* FIXME: disable Nagle's algorithm */
2679 * Intercept TCP callbacks for sendfile like receive
2680 * processing.
2681 */
2682 iscsi_conn_set_callbacks(conn);
2683
2684 conn->sendpage = conn->sock->ops->sendpage;
2685
2686 /*
2687 * set receive state machine into initial state
2688 */
2689 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2690 }
2691
2692 if (is_leading)
2693 session->leadconn = conn;
2694 1988
2695 /* 1989 /*
2696 * Unblock xmitworker(), Login Phase will pass through. 1990 * Intercept TCP callbacks for sendfile like receive
1991 * processing.
2697 */ 1992 */
2698 clear_bit(SUSPEND_BIT, &conn->suspend_rx); 1993 conn->recv_lock = &sk->sk_callback_lock;
2699 clear_bit(SUSPEND_BIT, &conn->suspend_tx); 1994 iscsi_conn_set_callbacks(conn);
1995 tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
1996 /*
1997 * set receive state machine into initial state
1998 */
1999 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2700 2000
2701 return 0; 2001 return 0;
2702} 2002}
2703 2003
2704static int 2004static void
2705iscsi_conn_start(struct iscsi_cls_conn *cls_conn) 2005iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2706{ 2006{
2707 struct iscsi_conn *conn = cls_conn->dd_data; 2007 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2708 struct iscsi_session *session = conn->session; 2008 struct iscsi_r2t_info *r2t;
2709 struct sock *sk;
2710
2711 /* FF phase warming up... */
2712
2713 if (session == NULL) {
2714 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n");
2715 return -EPERM;
2716 }
2717
2718 sk = conn->sock->sk;
2719
2720 write_lock_bh(&sk->sk_callback_lock);
2721 spin_lock_bh(&session->lock);
2722 conn->c_stage = ISCSI_CONN_STARTED;
2723 session->state = ISCSI_STATE_LOGGED_IN;
2724 2009
2725 switch(conn->stop_stage) { 2010 /* flush ctask's r2t queues */
2726 case STOP_CONN_RECOVER: 2011 while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*)))
2727 /* 2012 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
2728 * unblock eh_abort() if it is blocked. re-try all 2013 sizeof(void*));
2729 * commands after successful recovery
2730 */
2731 session->conn_cnt++;
2732 conn->stop_stage = 0;
2733 conn->tmabort_state = TMABORT_INITIAL;
2734 session->age++;
2735 wake_up(&conn->ehwait);
2736 break;
2737 case STOP_CONN_TERM:
2738 session->conn_cnt++;
2739 conn->stop_stage = 0;
2740 break;
2741 case STOP_CONN_SUSPEND:
2742 conn->stop_stage = 0;
2743 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2744 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2745 break;
2746 default:
2747 break;
2748 }
2749 spin_unlock_bh(&session->lock);
2750 write_unlock_bh(&sk->sk_callback_lock);
2751 2014
2752 return 0; 2015 __iscsi_ctask_cleanup(conn, ctask);
2753} 2016}
2754 2017
2755static void 2018static void
2756iscsi_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) 2019iscsi_tcp_suspend_conn_rx(struct iscsi_conn *conn)
2757{ 2020{
2758 struct iscsi_conn *conn = cls_conn->dd_data; 2021 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2759 struct iscsi_session *session = conn->session;
2760 struct sock *sk; 2022 struct sock *sk;
2761 unsigned long flags;
2762 2023
2763 BUG_ON(!conn->sock); 2024 if (!tcp_conn->sock)
2764 sk = conn->sock->sk; 2025 return;
2026
2027 sk = tcp_conn->sock->sk;
2765 write_lock_bh(&sk->sk_callback_lock); 2028 write_lock_bh(&sk->sk_callback_lock);
2766 set_bit(SUSPEND_BIT, &conn->suspend_rx); 2029 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
2767 write_unlock_bh(&sk->sk_callback_lock); 2030 write_unlock_bh(&sk->sk_callback_lock);
2768
2769 mutex_lock(&conn->xmitmutex);
2770
2771 spin_lock_irqsave(session->host->host_lock, flags);
2772 spin_lock(&session->lock);
2773 conn->stop_stage = flag;
2774 conn->c_stage = ISCSI_CONN_STOPPED;
2775 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2776
2777 if (flag != STOP_CONN_SUSPEND)
2778 session->conn_cnt--;
2779
2780 if (session->conn_cnt == 0 || session->leadconn == conn)
2781 session->state = ISCSI_STATE_FAILED;
2782
2783 spin_unlock(&session->lock);
2784 spin_unlock_irqrestore(session->host->host_lock, flags);
2785
2786 if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) {
2787 struct iscsi_cmd_task *ctask;
2788 struct iscsi_mgmt_task *mtask;
2789
2790 /*
2791 * Socket must go now.
2792 */
2793 sock_hold(conn->sock->sk);
2794 iscsi_conn_restore_callbacks(conn);
2795 sock_put(conn->sock->sk);
2796
2797 /*
2798 * flush xmit queues.
2799 */
2800 spin_lock_bh(&session->lock);
2801 while (__kfifo_get(conn->writequeue, (void*)&ctask,
2802 sizeof(void*)) ||
2803 __kfifo_get(conn->xmitqueue, (void*)&ctask,
2804 sizeof(void*))) {
2805 struct iscsi_r2t_info *r2t;
2806
2807 /*
2808 * flush ctask's r2t queues
2809 */
2810 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
2811 sizeof(void*)))
2812 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
2813 sizeof(void*));
2814
2815 spin_unlock_bh(&session->lock);
2816 local_bh_disable();
2817 iscsi_ctask_cleanup(conn, ctask);
2818 local_bh_enable();
2819 spin_lock_bh(&session->lock);
2820 }
2821 conn->ctask = NULL;
2822 while (__kfifo_get(conn->immqueue, (void*)&mtask,
2823 sizeof(void*)) ||
2824 __kfifo_get(conn->mgmtqueue, (void*)&mtask,
2825 sizeof(void*))) {
2826 __kfifo_put(session->mgmtpool.queue,
2827 (void*)&mtask, sizeof(void*));
2828 }
2829 conn->mtask = NULL;
2830 spin_unlock_bh(&session->lock);
2831
2832 /*
2833 * release socket only after we stopped data_xmit()
2834 * activity and flushed all outstandings
2835 */
2836 sock_release(conn->sock);
2837 conn->sock = NULL;
2838
2839 /*
2840 * for connection level recovery we should not calculate
2841 * header digest. conn->hdr_size used for optimization
2842 * in hdr_extract() and will be re-negotiated at
2843 * set_param() time.
2844 */
2845 if (flag == STOP_CONN_RECOVER) {
2846 conn->hdr_size = sizeof(struct iscsi_hdr);
2847 conn->hdrdgst_en = 0;
2848 conn->datadgst_en = 0;
2849 }
2850 }
2851 mutex_unlock(&conn->xmitmutex);
2852}
2853
2854static int
2855iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
2856 char *data, uint32_t data_size)
2857{
2858 struct iscsi_session *session = conn->session;
2859 struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
2860 struct iscsi_mgmt_task *mtask;
2861
2862 spin_lock_bh(&session->lock);
2863 if (session->state == ISCSI_STATE_TERMINATE) {
2864 spin_unlock_bh(&session->lock);
2865 return -EPERM;
2866 }
2867 if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
2868 hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
2869 /*
2870 * Login and Text are sent serially, in
2871 * request-followed-by-response sequence.
2872 * Same mtask can be used. Same ITT must be used.
2873 * Note that login_mtask is preallocated at conn_create().
2874 */
2875 mtask = conn->login_mtask;
2876 else {
2877 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
2878 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
2879
2880 if (!__kfifo_get(session->mgmtpool.queue,
2881 (void*)&mtask, sizeof(void*))) {
2882 spin_unlock_bh(&session->lock);
2883 return -ENOSPC;
2884 }
2885 }
2886
2887 /*
2888 * pre-format CmdSN and ExpStatSN for outgoing PDU.
2889 */
2890 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2891 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) |
2892 (session->age << AGE_SHIFT);
2893 nop->cmdsn = cpu_to_be32(session->cmdsn);
2894 if (conn->c_stage == ISCSI_CONN_STARTED &&
2895 !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2896 session->cmdsn++;
2897 } else
2898 /* do not advance CmdSN */
2899 nop->cmdsn = cpu_to_be32(session->cmdsn);
2900
2901 nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
2902
2903 memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr));
2904
2905 iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr,
2906 sizeof(struct iscsi_hdr));
2907
2908 spin_unlock_bh(&session->lock);
2909
2910 if (data_size) {
2911 memcpy(mtask->data, data, data_size);
2912 mtask->data_count = data_size;
2913 } else
2914 mtask->data_count = 0;
2915
2916 mtask->xmstate = XMSTATE_IMM_HDR;
2917
2918 if (mtask->data_count) {
2919 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data,
2920 mtask->data_count);
2921 }
2922
2923 debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n",
2924 hdr->opcode, hdr->itt, data_size);
2925
2926 /*
2927 * since send_pdu() could be called at least from two contexts,
2928 * we need to serialize __kfifo_put, so we don't have to take
2929 * additional lock on fast data-path
2930 */
2931 if (hdr->opcode & ISCSI_OP_IMMEDIATE)
2932 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
2933 else
2934 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*));
2935
2936 scsi_queue_work(session->host, &conn->xmitwork);
2937 return 0;
2938} 2031}
2939 2032
2940static int 2033static void
2941iscsi_eh_host_reset(struct scsi_cmnd *sc) 2034iscsi_tcp_terminate_conn(struct iscsi_conn *conn)
2942{ 2035{
2943 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr; 2036 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2944 struct iscsi_conn *conn = ctask->conn;
2945 struct iscsi_session *session = conn->session;
2946 2037
2947 spin_lock_bh(&session->lock); 2038 if (!tcp_conn->sock)
2948 if (session->state == ISCSI_STATE_TERMINATE) { 2039 return;
2949 debug_scsi("failing host reset: session terminated "
2950 "[CID %d age %d]", conn->id, session->age);
2951 spin_unlock_bh(&session->lock);
2952 return FAILED;
2953 }
2954 spin_unlock_bh(&session->lock);
2955 2040
2956 debug_scsi("failing connection CID %d due to SCSI host reset " 2041 sock_hold(tcp_conn->sock->sk);
2957 "[itt 0x%x age %d]", conn->id, ctask->itt, 2042 iscsi_conn_restore_callbacks(conn);
2958 session->age); 2043 sock_put(tcp_conn->sock->sk);
2959 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
2960 2044
2961 return SUCCESS; 2045 sock_release(tcp_conn->sock);
2046 tcp_conn->sock = NULL;
2047 conn->recv_lock = NULL;
2962} 2048}
2963 2049
2050/* called with host lock */
2964static void 2051static void
2965iscsi_tmabort_timedout(unsigned long data) 2052iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask,
2966{ 2053 char *data, uint32_t data_size)
2967 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data;
2968 struct iscsi_conn *conn = ctask->conn;
2969 struct iscsi_session *session = conn->session;
2970
2971 spin_lock(&session->lock);
2972 if (conn->tmabort_state == TMABORT_INITIAL) {
2973 __kfifo_put(session->mgmtpool.queue,
2974 (void*)&ctask->mtask, sizeof(void*));
2975 conn->tmabort_state = TMABORT_TIMEDOUT;
2976 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n",
2977 (long)ctask->sc, ctask->itt);
2978 /* unblock eh_abort() */
2979 wake_up(&conn->ehwait);
2980 }
2981 spin_unlock(&session->lock);
2982}
2983
2984static int
2985iscsi_eh_abort(struct scsi_cmnd *sc)
2986{ 2054{
2987 int rc; 2055 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
2988 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2989 struct iscsi_conn *conn = ctask->conn;
2990 struct iscsi_session *session = conn->session;
2991
2992 conn->eh_abort_cnt++;
2993 debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
2994
2995 /*
2996 * two cases for ERL=0 here:
2997 *
2998 * 1) connection-level failure;
2999 * 2) recovery due protocol error;
3000 */
3001 mutex_lock(&conn->xmitmutex);
3002 spin_lock_bh(&session->lock);
3003 if (session->state != ISCSI_STATE_LOGGED_IN) {
3004 if (session->state == ISCSI_STATE_TERMINATE) {
3005 spin_unlock_bh(&session->lock);
3006 mutex_unlock(&conn->xmitmutex);
3007 goto failed;
3008 }
3009 spin_unlock_bh(&session->lock);
3010 } else {
3011 struct iscsi_tm *hdr = &conn->tmhdr;
3012
3013 /*
3014 * Still LOGGED_IN...
3015 */
3016
3017 if (!ctask->sc || sc->SCp.phase != session->age) {
3018 /*
3019 * 1) ctask completed before time out. But session
3020 * is still ok => Happy Retry.
3021 * 2) session was re-open during time out of ctask.
3022 */
3023 spin_unlock_bh(&session->lock);
3024 mutex_unlock(&conn->xmitmutex);
3025 goto success;
3026 }
3027 conn->tmabort_state = TMABORT_INITIAL;
3028 spin_unlock_bh(&session->lock);
3029
3030 /*
3031 * ctask timed out but session is OK
3032 * ERL=0 requires task mgmt abort to be issued on each
3033 * failed command. requests must be serialized.
3034 */
3035 memset(hdr, 0, sizeof(struct iscsi_tm));
3036 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
3037 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK;
3038 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3039 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
3040 hdr->rtt = ctask->hdr.itt;
3041 hdr->refcmdsn = ctask->hdr.cmdsn;
3042
3043 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr,
3044 NULL, 0);
3045 if (rc) {
3046 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3047 debug_scsi("abort sent failure [itt 0x%x]", ctask->itt);
3048 } else {
3049 struct iscsi_r2t_info *r2t;
3050
3051 /*
3052 * TMF abort vs. TMF response race logic
3053 */
3054 spin_lock_bh(&session->lock);
3055 ctask->mtask = (struct iscsi_mgmt_task *)
3056 session->mgmt_cmds[(hdr->itt & ITT_MASK) -
3057 ISCSI_MGMT_ITT_OFFSET];
3058 /*
3059 * have to flush r2tqueue to avoid r2t leaks
3060 */
3061 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
3062 sizeof(void*))) {
3063 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
3064 sizeof(void*));
3065 }
3066 if (conn->tmabort_state == TMABORT_INITIAL) {
3067 conn->tmfcmd_pdus_cnt++;
3068 conn->tmabort_timer.expires = 3*HZ + jiffies;
3069 conn->tmabort_timer.function =
3070 iscsi_tmabort_timedout;
3071 conn->tmabort_timer.data = (unsigned long)ctask;
3072 add_timer(&conn->tmabort_timer);
3073 debug_scsi("abort sent [itt 0x%x]", ctask->itt);
3074 } else {
3075 if (!ctask->sc ||
3076 conn->tmabort_state == TMABORT_SUCCESS) {
3077 conn->tmabort_state = TMABORT_INITIAL;
3078 spin_unlock_bh(&session->lock);
3079 mutex_unlock(&conn->xmitmutex);
3080 goto success;
3081 }
3082 conn->tmabort_state = TMABORT_INITIAL;
3083 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3084 }
3085 spin_unlock_bh(&session->lock);
3086 }
3087 }
3088 mutex_unlock(&conn->xmitmutex);
3089
3090
3091 /*
3092 * block eh thread until:
3093 *
3094 * 1) abort response;
3095 * 2) abort timeout;
3096 * 3) session re-opened;
3097 * 4) session terminated;
3098 */
3099 for (;;) {
3100 int p_state = session->state;
3101
3102 rc = wait_event_interruptible(conn->ehwait,
3103 (p_state == ISCSI_STATE_LOGGED_IN ?
3104 (session->state == ISCSI_STATE_TERMINATE ||
3105 conn->tmabort_state != TMABORT_INITIAL) :
3106 (session->state == ISCSI_STATE_TERMINATE ||
3107 session->state == ISCSI_STATE_LOGGED_IN)));
3108 if (rc) {
3109 /* shutdown.. */
3110 session->state = ISCSI_STATE_TERMINATE;
3111 goto failed;
3112 }
3113
3114 if (signal_pending(current))
3115 flush_signals(current);
3116
3117 if (session->state == ISCSI_STATE_TERMINATE)
3118 goto failed;
3119
3120 spin_lock_bh(&session->lock);
3121 if (sc->SCp.phase == session->age &&
3122 (conn->tmabort_state == TMABORT_TIMEDOUT ||
3123 conn->tmabort_state == TMABORT_FAILED)) {
3124 conn->tmabort_state = TMABORT_INITIAL;
3125 if (!ctask->sc) {
3126 /*
3127 * ctask completed before tmf abort response or
3128 * time out.
3129 * But session is still ok => Happy Retry.
3130 */
3131 spin_unlock_bh(&session->lock);
3132 break;
3133 }
3134 spin_unlock_bh(&session->lock);
3135 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3136 continue;
3137 }
3138 spin_unlock_bh(&session->lock);
3139 break;
3140 }
3141
3142success:
3143 debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3144 rc = SUCCESS;
3145 goto exit;
3146
3147failed:
3148 debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3149 rc = FAILED;
3150 2056
3151exit: 2057 iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr,
3152 del_timer_sync(&conn->tmabort_timer); 2058 sizeof(struct iscsi_hdr));
3153 2059 tcp_mtask->xmstate = XMSTATE_IMM_HDR;
3154 mutex_lock(&conn->xmitmutex);
3155 if (conn->sock) {
3156 struct sock *sk = conn->sock->sk;
3157 2060
3158 write_lock_bh(&sk->sk_callback_lock); 2061 if (mtask->data_count)
3159 iscsi_ctask_cleanup(conn, ctask); 2062 iscsi_buf_init_iov(&tcp_mtask->sendbuf, (char*)mtask->data,
3160 write_unlock_bh(&sk->sk_callback_lock); 2063 mtask->data_count);
3161 }
3162 mutex_unlock(&conn->xmitmutex);
3163 return rc;
3164} 2064}
3165 2065
3166static int 2066static int
@@ -3174,6 +2074,7 @@ iscsi_r2tpool_alloc(struct iscsi_session *session)
3174 */ 2074 */
3175 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 2075 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3176 struct iscsi_cmd_task *ctask = session->cmds[cmd_i]; 2076 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2077 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
3177 2078
3178 /* 2079 /*
3179 * pre-allocated x4 as much r2ts to handle race when 2080 * pre-allocated x4 as much r2ts to handle race when
@@ -3182,42 +2083,32 @@ iscsi_r2tpool_alloc(struct iscsi_session *session)
3182 */ 2083 */
3183 2084
3184 /* R2T pool */ 2085 /* R2T pool */
3185 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4, 2086 if (iscsi_pool_init(&tcp_ctask->r2tpool, session->max_r2t * 4,
3186 (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) { 2087 (void***)&tcp_ctask->r2ts,
2088 sizeof(struct iscsi_r2t_info))) {
3187 goto r2t_alloc_fail; 2089 goto r2t_alloc_fail;
3188 } 2090 }
3189 2091
3190 /* R2T xmit queue */ 2092 /* R2T xmit queue */
3191 ctask->r2tqueue = kfifo_alloc( 2093 tcp_ctask->r2tqueue = kfifo_alloc(
3192 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL); 2094 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
3193 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) { 2095 if (tcp_ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
3194 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts); 2096 iscsi_pool_free(&tcp_ctask->r2tpool,
2097 (void**)tcp_ctask->r2ts);
3195 goto r2t_alloc_fail; 2098 goto r2t_alloc_fail;
3196 } 2099 }
3197
3198 /*
3199 * number of
3200 * Data-Out PDU's within R2T-sequence can be quite big;
3201 * using mempool
3202 */
3203 ctask->datapool = mempool_create_slab_pool(ISCSI_DTASK_DEFAULT_MAX,
3204 taskcache);
3205 if (ctask->datapool == NULL) {
3206 kfifo_free(ctask->r2tqueue);
3207 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3208 goto r2t_alloc_fail;
3209 }
3210 INIT_LIST_HEAD(&ctask->dataqueue);
3211 } 2100 }
3212 2101
3213 return 0; 2102 return 0;
3214 2103
3215r2t_alloc_fail: 2104r2t_alloc_fail:
3216 for (i = 0; i < cmd_i; i++) { 2105 for (i = 0; i < cmd_i; i++) {
3217 mempool_destroy(session->cmds[i]->datapool); 2106 struct iscsi_cmd_task *ctask = session->cmds[i];
3218 kfifo_free(session->cmds[i]->r2tqueue); 2107 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
3219 iscsi_pool_free(&session->cmds[i]->r2tpool, 2108
3220 (void**)session->cmds[i]->r2ts); 2109 kfifo_free(tcp_ctask->r2tqueue);
2110 iscsi_pool_free(&tcp_ctask->r2tpool,
2111 (void**)tcp_ctask->r2ts);
3221 } 2112 }
3222 return -ENOMEM; 2113 return -ENOMEM;
3223} 2114}
@@ -3228,127 +2119,13 @@ iscsi_r2tpool_free(struct iscsi_session *session)
3228 int i; 2119 int i;
3229 2120
3230 for (i = 0; i < session->cmds_max; i++) { 2121 for (i = 0; i < session->cmds_max; i++) {
3231 mempool_destroy(session->cmds[i]->datapool); 2122 struct iscsi_cmd_task *ctask = session->cmds[i];
3232 kfifo_free(session->cmds[i]->r2tqueue); 2123 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
3233 iscsi_pool_free(&session->cmds[i]->r2tpool,
3234 (void**)session->cmds[i]->r2ts);
3235 }
3236}
3237 2124
3238static struct scsi_host_template iscsi_sht = { 2125 kfifo_free(tcp_ctask->r2tqueue);
3239 .name = "iSCSI Initiator over TCP/IP, v." 2126 iscsi_pool_free(&tcp_ctask->r2tpool,
3240 ISCSI_VERSION_STR, 2127 (void**)tcp_ctask->r2ts);
3241 .queuecommand = iscsi_queuecommand,
3242 .change_queue_depth = iscsi_change_queue_depth,
3243 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
3244 .sg_tablesize = ISCSI_SG_TABLESIZE,
3245 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
3246 .eh_abort_handler = iscsi_eh_abort,
3247 .eh_host_reset_handler = iscsi_eh_host_reset,
3248 .use_clustering = DISABLE_CLUSTERING,
3249 .proc_name = "iscsi_tcp",
3250 .this_id = -1,
3251};
3252
3253static struct iscsi_transport iscsi_tcp_transport;
3254
3255static struct iscsi_cls_session *
3256iscsi_session_create(struct scsi_transport_template *scsit,
3257 uint32_t initial_cmdsn, uint32_t *sid)
3258{
3259 struct Scsi_Host *shost;
3260 struct iscsi_session *session;
3261 int cmd_i;
3262
3263 shost = iscsi_transport_create_session(scsit, &iscsi_tcp_transport);
3264 if (!shost)
3265 return NULL;
3266
3267 session = iscsi_hostdata(shost->hostdata);
3268 memset(session, 0, sizeof(struct iscsi_session));
3269 session->host = shost;
3270 session->state = ISCSI_STATE_FREE;
3271 session->mgmtpool_max = ISCSI_MGMT_CMDS_MAX;
3272 session->cmds_max = ISCSI_XMIT_CMDS_MAX;
3273 session->cmdsn = initial_cmdsn;
3274 session->exp_cmdsn = initial_cmdsn + 1;
3275 session->max_cmdsn = initial_cmdsn + 1;
3276 session->max_r2t = 1;
3277 *sid = shost->host_no;
3278
3279 /* initialize SCSI PDU commands pool */
3280 if (iscsi_pool_init(&session->cmdpool, session->cmds_max,
3281 (void***)&session->cmds, sizeof(struct iscsi_cmd_task)))
3282 goto cmdpool_alloc_fail;
3283
3284 /* pre-format cmds pool with ITT */
3285 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++)
3286 session->cmds[cmd_i]->itt = cmd_i;
3287
3288 spin_lock_init(&session->lock);
3289 INIT_LIST_HEAD(&session->connections);
3290
3291 /* initialize immediate command pool */
3292 if (iscsi_pool_init(&session->mgmtpool, session->mgmtpool_max,
3293 (void***)&session->mgmt_cmds, sizeof(struct iscsi_mgmt_task)))
3294 goto mgmtpool_alloc_fail;
3295
3296
3297 /* pre-format immediate cmds pool with ITT */
3298 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
3299 session->mgmt_cmds[cmd_i]->itt = ISCSI_MGMT_ITT_OFFSET + cmd_i;
3300 session->mgmt_cmds[cmd_i]->data = kmalloc(
3301 DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, GFP_KERNEL);
3302 if (!session->mgmt_cmds[cmd_i]->data) {
3303 int j;
3304
3305 for (j = 0; j < cmd_i; j++)
3306 kfree(session->mgmt_cmds[j]->data);
3307 goto immdata_alloc_fail;
3308 }
3309 } 2128 }
3310
3311 if (iscsi_r2tpool_alloc(session))
3312 goto r2tpool_alloc_fail;
3313
3314 return hostdata_session(shost->hostdata);
3315
3316r2tpool_alloc_fail:
3317 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3318 kfree(session->mgmt_cmds[cmd_i]->data);
3319immdata_alloc_fail:
3320 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3321mgmtpool_alloc_fail:
3322 iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3323cmdpool_alloc_fail:
3324 iscsi_transport_destroy_session(shost);
3325 return NULL;
3326}
3327
3328static void
3329iscsi_session_destroy(struct iscsi_cls_session *cls_session)
3330{
3331 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
3332 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
3333 int cmd_i;
3334 struct iscsi_data_task *dtask, *n;
3335
3336 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3337 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3338 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
3339 list_del(&dtask->item);
3340 mempool_free(dtask, ctask->datapool);
3341 }
3342 }
3343
3344 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3345 kfree(session->mgmt_cmds[cmd_i]->data);
3346
3347 iscsi_r2tpool_free(session);
3348 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3349 iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3350
3351 iscsi_transport_destroy_session(shost);
3352} 2129}
3353 2130
3354static int 2131static int
@@ -3357,23 +2134,14 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
3357{ 2134{
3358 struct iscsi_conn *conn = cls_conn->dd_data; 2135 struct iscsi_conn *conn = cls_conn->dd_data;
3359 struct iscsi_session *session = conn->session; 2136 struct iscsi_session *session = conn->session;
3360 2137 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
3361 spin_lock_bh(&session->lock);
3362 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
3363 conn->stop_stage != STOP_CONN_RECOVER) {
3364 printk(KERN_ERR "iscsi_tcp: can not change parameter [%d]\n",
3365 param);
3366 spin_unlock_bh(&session->lock);
3367 return 0;
3368 }
3369 spin_unlock_bh(&session->lock);
3370 2138
3371 switch(param) { 2139 switch(param) {
3372 case ISCSI_PARAM_MAX_RECV_DLENGTH: { 2140 case ISCSI_PARAM_MAX_RECV_DLENGTH: {
3373 char *saveptr = conn->data; 2141 char *saveptr = tcp_conn->data;
3374 gfp_t flags = GFP_KERNEL; 2142 gfp_t flags = GFP_KERNEL;
3375 2143
3376 if (conn->data_size >= value) { 2144 if (tcp_conn->data_size >= value) {
3377 conn->max_recv_dlength = value; 2145 conn->max_recv_dlength = value;
3378 break; 2146 break;
3379 } 2147 }
@@ -3384,21 +2152,21 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
3384 spin_unlock_bh(&session->lock); 2152 spin_unlock_bh(&session->lock);
3385 2153
3386 if (value <= PAGE_SIZE) 2154 if (value <= PAGE_SIZE)
3387 conn->data = kmalloc(value, flags); 2155 tcp_conn->data = kmalloc(value, flags);
3388 else 2156 else
3389 conn->data = (void*)__get_free_pages(flags, 2157 tcp_conn->data = (void*)__get_free_pages(flags,
3390 get_order(value)); 2158 get_order(value));
3391 if (conn->data == NULL) { 2159 if (tcp_conn->data == NULL) {
3392 conn->data = saveptr; 2160 tcp_conn->data = saveptr;
3393 return -ENOMEM; 2161 return -ENOMEM;
3394 } 2162 }
3395 if (conn->data_size <= PAGE_SIZE) 2163 if (tcp_conn->data_size <= PAGE_SIZE)
3396 kfree(saveptr); 2164 kfree(saveptr);
3397 else 2165 else
3398 free_pages((unsigned long)saveptr, 2166 free_pages((unsigned long)saveptr,
3399 get_order(conn->data_size)); 2167 get_order(tcp_conn->data_size));
3400 conn->max_recv_dlength = value; 2168 conn->max_recv_dlength = value;
3401 conn->data_size = value; 2169 tcp_conn->data_size = value;
3402 } 2170 }
3403 break; 2171 break;
3404 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2172 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
@@ -3406,49 +2174,51 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
3406 break; 2174 break;
3407 case ISCSI_PARAM_HDRDGST_EN: 2175 case ISCSI_PARAM_HDRDGST_EN:
3408 conn->hdrdgst_en = value; 2176 conn->hdrdgst_en = value;
3409 conn->hdr_size = sizeof(struct iscsi_hdr); 2177 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
3410 if (conn->hdrdgst_en) { 2178 if (conn->hdrdgst_en) {
3411 conn->hdr_size += sizeof(__u32); 2179 tcp_conn->hdr_size += sizeof(__u32);
3412 if (!conn->tx_tfm) 2180 if (!tcp_conn->tx_tfm)
3413 conn->tx_tfm = crypto_alloc_tfm("crc32c", 0); 2181 tcp_conn->tx_tfm = crypto_alloc_tfm("crc32c",
3414 if (!conn->tx_tfm) 2182 0);
2183 if (!tcp_conn->tx_tfm)
3415 return -ENOMEM; 2184 return -ENOMEM;
3416 if (!conn->rx_tfm) 2185 if (!tcp_conn->rx_tfm)
3417 conn->rx_tfm = crypto_alloc_tfm("crc32c", 0); 2186 tcp_conn->rx_tfm = crypto_alloc_tfm("crc32c",
3418 if (!conn->rx_tfm) { 2187 0);
3419 crypto_free_tfm(conn->tx_tfm); 2188 if (!tcp_conn->rx_tfm) {
2189 crypto_free_tfm(tcp_conn->tx_tfm);
3420 return -ENOMEM; 2190 return -ENOMEM;
3421 } 2191 }
3422 } else { 2192 } else {
3423 if (conn->tx_tfm) 2193 if (tcp_conn->tx_tfm)
3424 crypto_free_tfm(conn->tx_tfm); 2194 crypto_free_tfm(tcp_conn->tx_tfm);
3425 if (conn->rx_tfm) 2195 if (tcp_conn->rx_tfm)
3426 crypto_free_tfm(conn->rx_tfm); 2196 crypto_free_tfm(tcp_conn->rx_tfm);
3427 } 2197 }
3428 break; 2198 break;
3429 case ISCSI_PARAM_DATADGST_EN: 2199 case ISCSI_PARAM_DATADGST_EN:
3430 conn->datadgst_en = value; 2200 conn->datadgst_en = value;
3431 if (conn->datadgst_en) { 2201 if (conn->datadgst_en) {
3432 if (!conn->data_tx_tfm) 2202 if (!tcp_conn->data_tx_tfm)
3433 conn->data_tx_tfm = 2203 tcp_conn->data_tx_tfm =
3434 crypto_alloc_tfm("crc32c", 0); 2204 crypto_alloc_tfm("crc32c", 0);
3435 if (!conn->data_tx_tfm) 2205 if (!tcp_conn->data_tx_tfm)
3436 return -ENOMEM; 2206 return -ENOMEM;
3437 if (!conn->data_rx_tfm) 2207 if (!tcp_conn->data_rx_tfm)
3438 conn->data_rx_tfm = 2208 tcp_conn->data_rx_tfm =
3439 crypto_alloc_tfm("crc32c", 0); 2209 crypto_alloc_tfm("crc32c", 0);
3440 if (!conn->data_rx_tfm) { 2210 if (!tcp_conn->data_rx_tfm) {
3441 crypto_free_tfm(conn->data_tx_tfm); 2211 crypto_free_tfm(tcp_conn->data_tx_tfm);
3442 return -ENOMEM; 2212 return -ENOMEM;
3443 } 2213 }
3444 } else { 2214 } else {
3445 if (conn->data_tx_tfm) 2215 if (tcp_conn->data_tx_tfm)
3446 crypto_free_tfm(conn->data_tx_tfm); 2216 crypto_free_tfm(tcp_conn->data_tx_tfm);
3447 if (conn->data_rx_tfm) 2217 if (tcp_conn->data_rx_tfm)
3448 crypto_free_tfm(conn->data_rx_tfm); 2218 crypto_free_tfm(tcp_conn->data_rx_tfm);
3449 } 2219 }
3450 conn->sendpage = conn->datadgst_en ? 2220 tcp_conn->sendpage = conn->datadgst_en ?
3451 sock_no_sendpage : conn->sock->ops->sendpage; 2221 sock_no_sendpage : tcp_conn->sock->ops->sendpage;
3452 break; 2222 break;
3453 case ISCSI_PARAM_INITIAL_R2T_EN: 2223 case ISCSI_PARAM_INITIAL_R2T_EN:
3454 session->initial_r2t_en = value; 2224 session->initial_r2t_en = value;
@@ -3489,6 +2259,9 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
3489 BUG_ON(value); 2259 BUG_ON(value);
3490 session->ofmarker_en = value; 2260 session->ofmarker_en = value;
3491 break; 2261 break;
2262 case ISCSI_PARAM_EXP_STATSN:
2263 conn->exp_statsn = value;
2264 break;
3492 default: 2265 default:
3493 break; 2266 break;
3494 } 2267 }
@@ -3535,7 +2308,7 @@ iscsi_session_get_param(struct iscsi_cls_session *cls_session,
3535 *value = session->ofmarker_en; 2308 *value = session->ofmarker_en;
3536 break; 2309 break;
3537 default: 2310 default:
3538 return ISCSI_ERR_PARAM_NOT_FOUND; 2311 return -EINVAL;
3539 } 2312 }
3540 2313
3541 return 0; 2314 return 0;
@@ -3546,6 +2319,8 @@ iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn,
3546 enum iscsi_param param, uint32_t *value) 2319 enum iscsi_param param, uint32_t *value)
3547{ 2320{
3548 struct iscsi_conn *conn = cls_conn->dd_data; 2321 struct iscsi_conn *conn = cls_conn->dd_data;
2322 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2323 struct inet_sock *inet;
3549 2324
3550 switch(param) { 2325 switch(param) {
3551 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2326 case ISCSI_PARAM_MAX_RECV_DLENGTH:
@@ -3560,17 +2335,70 @@ iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn,
3560 case ISCSI_PARAM_DATADGST_EN: 2335 case ISCSI_PARAM_DATADGST_EN:
3561 *value = conn->datadgst_en; 2336 *value = conn->datadgst_en;
3562 break; 2337 break;
2338 case ISCSI_PARAM_CONN_PORT:
2339 mutex_lock(&conn->xmitmutex);
2340 if (!tcp_conn->sock) {
2341 mutex_unlock(&conn->xmitmutex);
2342 return -EINVAL;
2343 }
2344
2345 inet = inet_sk(tcp_conn->sock->sk);
2346 *value = be16_to_cpu(inet->dport);
2347 mutex_unlock(&conn->xmitmutex);
2348 case ISCSI_PARAM_EXP_STATSN:
2349 *value = conn->exp_statsn;
2350 break;
3563 default: 2351 default:
3564 return ISCSI_ERR_PARAM_NOT_FOUND; 2352 return -EINVAL;
3565 } 2353 }
3566 2354
3567 return 0; 2355 return 0;
3568} 2356}
3569 2357
2358static int
2359iscsi_conn_get_str_param(struct iscsi_cls_conn *cls_conn,
2360 enum iscsi_param param, char *buf)
2361{
2362 struct iscsi_conn *conn = cls_conn->dd_data;
2363 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2364 struct sock *sk;
2365 struct inet_sock *inet;
2366 struct ipv6_pinfo *np;
2367 int len = 0;
2368
2369 switch (param) {
2370 case ISCSI_PARAM_CONN_ADDRESS:
2371 mutex_lock(&conn->xmitmutex);
2372 if (!tcp_conn->sock) {
2373 mutex_unlock(&conn->xmitmutex);
2374 return -EINVAL;
2375 }
2376
2377 sk = tcp_conn->sock->sk;
2378 if (sk->sk_family == PF_INET) {
2379 inet = inet_sk(sk);
2380 len = sprintf(buf, "%u.%u.%u.%u\n",
2381 NIPQUAD(inet->daddr));
2382 } else {
2383 np = inet6_sk(sk);
2384 len = sprintf(buf,
2385 "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
2386 NIP6(np->daddr));
2387 }
2388 mutex_unlock(&conn->xmitmutex);
2389 break;
2390 default:
2391 return -EINVAL;
2392 }
2393
2394 return len;
2395}
2396
3570static void 2397static void
3571iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) 2398iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
3572{ 2399{
3573 struct iscsi_conn *conn = cls_conn->dd_data; 2400 struct iscsi_conn *conn = cls_conn->dd_data;
2401 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
3574 2402
3575 stats->txdata_octets = conn->txdata_octets; 2403 stats->txdata_octets = conn->txdata_octets;
3576 stats->rxdata_octets = conn->rxdata_octets; 2404 stats->rxdata_octets = conn->rxdata_octets;
@@ -3583,68 +2411,141 @@ iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
3583 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 2411 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
3584 stats->custom_length = 3; 2412 stats->custom_length = 3;
3585 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 2413 strcpy(stats->custom[0].desc, "tx_sendpage_failures");
3586 stats->custom[0].value = conn->sendpage_failures_cnt; 2414 stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
3587 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 2415 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
3588 stats->custom[1].value = conn->discontiguous_hdr_cnt; 2416 stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
3589 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 2417 strcpy(stats->custom[2].desc, "eh_abort_cnt");
3590 stats->custom[2].value = conn->eh_abort_cnt; 2418 stats->custom[2].value = conn->eh_abort_cnt;
3591} 2419}
3592 2420
3593static int 2421static struct iscsi_cls_session *
3594iscsi_conn_send_pdu(struct iscsi_cls_conn *cls_conn, struct iscsi_hdr *hdr, 2422iscsi_tcp_session_create(struct iscsi_transport *iscsit,
3595 char *data, uint32_t data_size) 2423 struct scsi_transport_template *scsit,
2424 uint32_t initial_cmdsn, uint32_t *hostno)
3596{ 2425{
3597 struct iscsi_conn *conn = cls_conn->dd_data; 2426 struct iscsi_cls_session *cls_session;
3598 int rc; 2427 struct iscsi_session *session;
2428 uint32_t hn;
2429 int cmd_i;
3599 2430
3600 mutex_lock(&conn->xmitmutex); 2431 cls_session = iscsi_session_setup(iscsit, scsit,
3601 rc = iscsi_conn_send_generic(conn, hdr, data, data_size); 2432 sizeof(struct iscsi_tcp_cmd_task),
3602 mutex_unlock(&conn->xmitmutex); 2433 sizeof(struct iscsi_tcp_mgmt_task),
2434 initial_cmdsn, &hn);
2435 if (!cls_session)
2436 return NULL;
2437 *hostno = hn;
3603 2438
3604 return rc; 2439 session = class_to_transport_session(cls_session);
2440 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2441 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2442 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2443
2444 ctask->hdr = &tcp_ctask->hdr;
2445 }
2446
2447 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
2448 struct iscsi_mgmt_task *mtask = session->mgmt_cmds[cmd_i];
2449 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
2450
2451 mtask->hdr = &tcp_mtask->hdr;
2452 }
2453
2454 if (iscsi_r2tpool_alloc(class_to_transport_session(cls_session)))
2455 goto r2tpool_alloc_fail;
2456
2457 return cls_session;
2458
2459r2tpool_alloc_fail:
2460 iscsi_session_teardown(cls_session);
2461 return NULL;
3605} 2462}
3606 2463
2464static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
2465{
2466 iscsi_r2tpool_free(class_to_transport_session(cls_session));
2467 iscsi_session_teardown(cls_session);
2468}
2469
2470static struct scsi_host_template iscsi_sht = {
2471 .name = "iSCSI Initiator over TCP/IP, v"
2472 ISCSI_TCP_VERSION,
2473 .queuecommand = iscsi_queuecommand,
2474 .change_queue_depth = iscsi_change_queue_depth,
2475 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
2476 .sg_tablesize = ISCSI_SG_TABLESIZE,
2477 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
2478 .eh_abort_handler = iscsi_eh_abort,
2479 .eh_host_reset_handler = iscsi_eh_host_reset,
2480 .use_clustering = DISABLE_CLUSTERING,
2481 .proc_name = "iscsi_tcp",
2482 .this_id = -1,
2483};
2484
3607static struct iscsi_transport iscsi_tcp_transport = { 2485static struct iscsi_transport iscsi_tcp_transport = {
3608 .owner = THIS_MODULE, 2486 .owner = THIS_MODULE,
3609 .name = "tcp", 2487 .name = "tcp",
3610 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 2488 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
3611 | CAP_DATADGST, 2489 | CAP_DATADGST,
2490 .param_mask = ISCSI_MAX_RECV_DLENGTH |
2491 ISCSI_MAX_XMIT_DLENGTH |
2492 ISCSI_HDRDGST_EN |
2493 ISCSI_DATADGST_EN |
2494 ISCSI_INITIAL_R2T_EN |
2495 ISCSI_MAX_R2T |
2496 ISCSI_IMM_DATA_EN |
2497 ISCSI_FIRST_BURST |
2498 ISCSI_MAX_BURST |
2499 ISCSI_PDU_INORDER_EN |
2500 ISCSI_DATASEQ_INORDER_EN |
2501 ISCSI_ERL |
2502 ISCSI_CONN_PORT |
2503 ISCSI_CONN_ADDRESS |
2504 ISCSI_EXP_STATSN,
3612 .host_template = &iscsi_sht, 2505 .host_template = &iscsi_sht,
3613 .hostdata_size = sizeof(struct iscsi_session),
3614 .conndata_size = sizeof(struct iscsi_conn), 2506 .conndata_size = sizeof(struct iscsi_conn),
3615 .max_conn = 1, 2507 .max_conn = 1,
3616 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN, 2508 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN,
3617 .create_session = iscsi_session_create, 2509 /* session management */
3618 .destroy_session = iscsi_session_destroy, 2510 .create_session = iscsi_tcp_session_create,
3619 .create_conn = iscsi_conn_create, 2511 .destroy_session = iscsi_tcp_session_destroy,
3620 .bind_conn = iscsi_conn_bind, 2512 /* connection management */
3621 .destroy_conn = iscsi_conn_destroy, 2513 .create_conn = iscsi_tcp_conn_create,
2514 .bind_conn = iscsi_tcp_conn_bind,
2515 .destroy_conn = iscsi_tcp_conn_destroy,
3622 .set_param = iscsi_conn_set_param, 2516 .set_param = iscsi_conn_set_param,
3623 .get_conn_param = iscsi_conn_get_param, 2517 .get_conn_param = iscsi_conn_get_param,
2518 .get_conn_str_param = iscsi_conn_get_str_param,
3624 .get_session_param = iscsi_session_get_param, 2519 .get_session_param = iscsi_session_get_param,
3625 .start_conn = iscsi_conn_start, 2520 .start_conn = iscsi_conn_start,
3626 .stop_conn = iscsi_conn_stop, 2521 .stop_conn = iscsi_conn_stop,
2522 /* these are called as part of conn recovery */
2523 .suspend_conn_recv = iscsi_tcp_suspend_conn_rx,
2524 .terminate_conn = iscsi_tcp_terminate_conn,
2525 /* IO */
3627 .send_pdu = iscsi_conn_send_pdu, 2526 .send_pdu = iscsi_conn_send_pdu,
3628 .get_stats = iscsi_conn_get_stats, 2527 .get_stats = iscsi_conn_get_stats,
2528 .init_cmd_task = iscsi_tcp_cmd_init,
2529 .init_mgmt_task = iscsi_tcp_mgmt_init,
2530 .xmit_cmd_task = iscsi_tcp_ctask_xmit,
2531 .xmit_mgmt_task = iscsi_tcp_mtask_xmit,
2532 .cleanup_cmd_task = iscsi_tcp_cleanup_ctask,
2533 /* recovery */
2534 .session_recovery_timedout = iscsi_session_recovery_timedout,
3629}; 2535};
3630 2536
3631static int __init 2537static int __init
3632iscsi_tcp_init(void) 2538iscsi_tcp_init(void)
3633{ 2539{
3634 if (iscsi_max_lun < 1) { 2540 if (iscsi_max_lun < 1) {
3635 printk(KERN_ERR "Invalid max_lun value of %u\n", iscsi_max_lun); 2541 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
2542 iscsi_max_lun);
3636 return -EINVAL; 2543 return -EINVAL;
3637 } 2544 }
3638 iscsi_tcp_transport.max_lun = iscsi_max_lun; 2545 iscsi_tcp_transport.max_lun = iscsi_max_lun;
3639 2546
3640 taskcache = kmem_cache_create("iscsi_taskcache",
3641 sizeof(struct iscsi_data_task), 0,
3642 SLAB_HWCACHE_ALIGN, NULL, NULL);
3643 if (!taskcache)
3644 return -ENOMEM;
3645
3646 if (!iscsi_register_transport(&iscsi_tcp_transport)) 2547 if (!iscsi_register_transport(&iscsi_tcp_transport))
3647 kmem_cache_destroy(taskcache); 2548 return -ENODEV;
3648 2549
3649 return 0; 2550 return 0;
3650} 2551}
@@ -3653,7 +2554,6 @@ static void __exit
3653iscsi_tcp_exit(void) 2554iscsi_tcp_exit(void)
3654{ 2555{
3655 iscsi_unregister_transport(&iscsi_tcp_transport); 2556 iscsi_unregister_transport(&iscsi_tcp_transport);
3656 kmem_cache_destroy(taskcache);
3657} 2557}
3658 2558
3659module_init(iscsi_tcp_init); 2559module_init(iscsi_tcp_init);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-20 03:10:43 -0500