aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/scsi/cxgb3i.txt11
-rw-r--r--drivers/message/fusion/mptbase.c4
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i.h21
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_ddp.c19
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_ddp.h5
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_init.c4
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_iscsi.c22
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_offload.c146
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_offload.h29
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_pdu.c275
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_pdu.h2
-rw-r--r--drivers/scsi/hptiop.c1
-rw-r--r--drivers/scsi/scsi_lib.c5
-rw-r--r--drivers/scsi/sd.c7
14 files changed, 371 insertions, 180 deletions
diff --git a/Documentation/scsi/cxgb3i.txt b/Documentation/scsi/cxgb3i.txt
index 8141fa01978e..7ac8032ee9b2 100644
--- a/Documentation/scsi/cxgb3i.txt
+++ b/Documentation/scsi/cxgb3i.txt
@@ -4,7 +4,7 @@ Introduction
4============ 4============
5 5
6The Chelsio T3 ASIC based Adapters (S310, S320, S302, S304, Mezz cards, etc. 6The Chelsio T3 ASIC based Adapters (S310, S320, S302, S304, Mezz cards, etc.
7series of products) supports iSCSI acceleration and iSCSI Direct Data Placement 7series of products) support iSCSI acceleration and iSCSI Direct Data Placement
8(DDP) where the hardware handles the expensive byte touching operations, such 8(DDP) where the hardware handles the expensive byte touching operations, such
9as CRC computation and verification, and direct DMA to the final host memory 9as CRC computation and verification, and direct DMA to the final host memory
10destination: 10destination:
@@ -31,9 +31,9 @@ destination:
31 the TCP segments onto the wire. It handles TCP retransmission if 31 the TCP segments onto the wire. It handles TCP retransmission if
32 needed. 32 needed.
33 33
34 On receving, S3 h/w recovers the iSCSI PDU by reassembling TCP 34 On receiving, S3 h/w recovers the iSCSI PDU by reassembling TCP
35 segments, separating the header and data, calculating and verifying 35 segments, separating the header and data, calculating and verifying
36 the digests, then forwards the header to the host. The payload data, 36 the digests, then forwarding the header to the host. The payload data,
37 if possible, will be directly placed into the pre-posted host DDP 37 if possible, will be directly placed into the pre-posted host DDP
38 buffer. Otherwise, the payload data will be sent to the host too. 38 buffer. Otherwise, the payload data will be sent to the host too.
39 39
@@ -68,9 +68,8 @@ The following steps need to be taken to accelerates the open-iscsi initiator:
68 sure the ip address is unique in the network. 68 sure the ip address is unique in the network.
69 69
703. edit /etc/iscsi/iscsid.conf 703. edit /etc/iscsi/iscsid.conf
71 The default setting for MaxRecvDataSegmentLength (131072) is too big, 71 The default setting for MaxRecvDataSegmentLength (131072) is too big;
72 replace "node.conn[0].iscsi.MaxRecvDataSegmentLength" to be a value no 72 replace with a value no bigger than 15360 (for example 8192):
73 bigger than 15360 (for example 8192):
74 73
75 node.conn[0].iscsi.MaxRecvDataSegmentLength = 8192 74 node.conn[0].iscsi.MaxRecvDataSegmentLength = 8192
76 75
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index 96ac88317b8e..ea3aafbbda44 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -91,9 +91,9 @@ MODULE_PARM_DESC(mpt_msi_enable_fc, " Enable MSI Support for FC \
91 controllers (default=0)"); 91 controllers (default=0)");
92 92
93static int mpt_msi_enable_sas; 93static int mpt_msi_enable_sas;
94module_param(mpt_msi_enable_sas, int, 1); 94module_param(mpt_msi_enable_sas, int, 0);
95MODULE_PARM_DESC(mpt_msi_enable_sas, " Enable MSI Support for SAS \ 95MODULE_PARM_DESC(mpt_msi_enable_sas, " Enable MSI Support for SAS \
96 controllers (default=1)"); 96 controllers (default=0)");
97 97
98 98
99static int mpt_channel_mapping; 99static int mpt_channel_mapping;
diff --git a/drivers/scsi/cxgb3i/cxgb3i.h b/drivers/scsi/cxgb3i/cxgb3i.h
index fde6e4c634e7..a7cf550b9cca 100644
--- a/drivers/scsi/cxgb3i/cxgb3i.h
+++ b/drivers/scsi/cxgb3i/cxgb3i.h
@@ -20,6 +20,7 @@
20#include <linux/list.h> 20#include <linux/list.h>
21#include <linux/netdevice.h> 21#include <linux/netdevice.h>
22#include <linux/scatterlist.h> 22#include <linux/scatterlist.h>
23#include <linux/skbuff.h>
23#include <scsi/libiscsi_tcp.h> 24#include <scsi/libiscsi_tcp.h>
24 25
25/* from cxgb3 LLD */ 26/* from cxgb3 LLD */
@@ -113,6 +114,26 @@ struct cxgb3i_endpoint {
113 struct cxgb3i_conn *cconn; 114 struct cxgb3i_conn *cconn;
114}; 115};
115 116
117/**
118 * struct cxgb3i_task_data - private iscsi task data
119 *
120 * @nr_frags: # of coalesced page frags (from scsi sgl)
121 * @frags: coalesced page frags (from scsi sgl)
122 * @skb: tx pdu skb
123 * @offset: data offset for the next pdu
124 * @count: max. possible pdu payload
125 * @sgoffset: offset to the first sg entry for a given offset
126 */
127#define MAX_PDU_FRAGS ((ULP2_MAX_PDU_PAYLOAD + 512 - 1) / 512)
128struct cxgb3i_task_data {
129 unsigned short nr_frags;
130 skb_frag_t frags[MAX_PDU_FRAGS];
131 struct sk_buff *skb;
132 unsigned int offset;
133 unsigned int count;
134 unsigned int sgoffset;
135};
136
116int cxgb3i_iscsi_init(void); 137int cxgb3i_iscsi_init(void);
117void cxgb3i_iscsi_cleanup(void); 138void cxgb3i_iscsi_cleanup(void);
118 139
diff --git a/drivers/scsi/cxgb3i/cxgb3i_ddp.c b/drivers/scsi/cxgb3i/cxgb3i_ddp.c
index 08f3a09d9233..a83d36e4926f 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_ddp.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_ddp.c
@@ -639,10 +639,11 @@ static int ddp_init(struct t3cdev *tdev)
639 write_unlock(&cxgb3i_ddp_rwlock); 639 write_unlock(&cxgb3i_ddp_rwlock);
640 640
641 ddp_log_info("nppods %u (0x%x ~ 0x%x), bits %u, mask 0x%x,0x%x " 641 ddp_log_info("nppods %u (0x%x ~ 0x%x), bits %u, mask 0x%x,0x%x "
642 "pkt %u,%u.\n", 642 "pkt %u/%u, %u/%u.\n",
643 ppmax, ddp->llimit, ddp->ulimit, ddp->idx_bits, 643 ppmax, ddp->llimit, ddp->ulimit, ddp->idx_bits,
644 ddp->idx_mask, ddp->rsvd_tag_mask, 644 ddp->idx_mask, ddp->rsvd_tag_mask,
645 ddp->max_txsz, ddp->max_rxsz); 645 ddp->max_txsz, uinfo.max_txsz,
646 ddp->max_rxsz, uinfo.max_rxsz);
646 return 0; 647 return 0;
647 648
648free_ddp_map: 649free_ddp_map:
@@ -654,8 +655,8 @@ free_ddp_map:
654 * cxgb3i_adapter_ddp_init - initialize the adapter's ddp resource 655 * cxgb3i_adapter_ddp_init - initialize the adapter's ddp resource
655 * @tdev: t3cdev adapter 656 * @tdev: t3cdev adapter
656 * @tformat: tag format 657 * @tformat: tag format
657 * @txsz: max tx pkt size, filled in by this func. 658 * @txsz: max tx pdu payload size, filled in by this func.
658 * @rxsz: max rx pkt size, filled in by this func. 659 * @rxsz: max rx pdu payload size, filled in by this func.
659 * initialize the ddp pagepod manager for a given adapter if needed and 660 * initialize the ddp pagepod manager for a given adapter if needed and
660 * setup the tag format for a given iscsi entity 661 * setup the tag format for a given iscsi entity
661 */ 662 */
@@ -685,10 +686,12 @@ int cxgb3i_adapter_ddp_init(struct t3cdev *tdev,
685 tformat->sw_bits, tformat->rsvd_bits, 686 tformat->sw_bits, tformat->rsvd_bits,
686 tformat->rsvd_shift, tformat->rsvd_mask); 687 tformat->rsvd_shift, tformat->rsvd_mask);
687 688
688 *txsz = ddp->max_txsz; 689 *txsz = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
689 *rxsz = ddp->max_rxsz; 690 ddp->max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
690 ddp_log_info("ddp max pkt size: %u, %u.\n", 691 *rxsz = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
691 ddp->max_txsz, ddp->max_rxsz); 692 ddp->max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
693 ddp_log_info("max payload size: %u/%u, %u/%u.\n",
694 *txsz, ddp->max_txsz, *rxsz, ddp->max_rxsz);
692 return 0; 695 return 0;
693} 696}
694EXPORT_SYMBOL_GPL(cxgb3i_adapter_ddp_init); 697EXPORT_SYMBOL_GPL(cxgb3i_adapter_ddp_init);
diff --git a/drivers/scsi/cxgb3i/cxgb3i_ddp.h b/drivers/scsi/cxgb3i/cxgb3i_ddp.h
index 5c7c4d95c493..3faae7831c83 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_ddp.h
+++ b/drivers/scsi/cxgb3i/cxgb3i_ddp.h
@@ -13,6 +13,8 @@
13#ifndef __CXGB3I_ULP2_DDP_H__ 13#ifndef __CXGB3I_ULP2_DDP_H__
14#define __CXGB3I_ULP2_DDP_H__ 14#define __CXGB3I_ULP2_DDP_H__
15 15
16#include <linux/vmalloc.h>
17
16/** 18/**
17 * struct cxgb3i_tag_format - cxgb3i ulp tag format for an iscsi entity 19 * struct cxgb3i_tag_format - cxgb3i ulp tag format for an iscsi entity
18 * 20 *
@@ -85,8 +87,9 @@ struct cxgb3i_ddp_info {
85 struct sk_buff **gl_skb; 87 struct sk_buff **gl_skb;
86}; 88};
87 89
90#define ISCSI_PDU_NONPAYLOAD_LEN 312 /* bhs(48) + ahs(256) + digest(8) */
88#define ULP2_MAX_PKT_SIZE 16224 91#define ULP2_MAX_PKT_SIZE 16224
89#define ULP2_MAX_PDU_PAYLOAD (ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_MAX) 92#define ULP2_MAX_PDU_PAYLOAD (ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_LEN)
90#define PPOD_PAGES_MAX 4 93#define PPOD_PAGES_MAX 4
91#define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */ 94#define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */
92 95
diff --git a/drivers/scsi/cxgb3i/cxgb3i_init.c b/drivers/scsi/cxgb3i/cxgb3i_init.c
index 091ecb4d9f3d..1ce9f244e46c 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_init.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_init.c
@@ -12,8 +12,8 @@
12#include "cxgb3i.h" 12#include "cxgb3i.h"
13 13
14#define DRV_MODULE_NAME "cxgb3i" 14#define DRV_MODULE_NAME "cxgb3i"
15#define DRV_MODULE_VERSION "1.0.0" 15#define DRV_MODULE_VERSION "1.0.1"
16#define DRV_MODULE_RELDATE "Jun. 1, 2008" 16#define DRV_MODULE_RELDATE "Jan. 2009"
17 17
18static char version[] = 18static char version[] =
19 "Chelsio S3xx iSCSI Driver " DRV_MODULE_NAME 19 "Chelsio S3xx iSCSI Driver " DRV_MODULE_NAME
diff --git a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
index d83464b9b3f9..fa2a44f37b36 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
@@ -364,7 +364,8 @@ cxgb3i_session_create(struct iscsi_endpoint *ep, u16 cmds_max, u16 qdepth,
364 364
365 cls_session = iscsi_session_setup(&cxgb3i_iscsi_transport, shost, 365 cls_session = iscsi_session_setup(&cxgb3i_iscsi_transport, shost,
366 cmds_max, 366 cmds_max,
367 sizeof(struct iscsi_tcp_task), 367 sizeof(struct iscsi_tcp_task) +
368 sizeof(struct cxgb3i_task_data),
368 initial_cmdsn, ISCSI_MAX_TARGET); 369 initial_cmdsn, ISCSI_MAX_TARGET);
369 if (!cls_session) 370 if (!cls_session)
370 return NULL; 371 return NULL;
@@ -402,17 +403,15 @@ static inline int cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
402{ 403{
403 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 404 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
404 struct cxgb3i_conn *cconn = tcp_conn->dd_data; 405 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
405 unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 406 unsigned int max = max(512 * MAX_SKB_FRAGS, SKB_TX_HEADROOM);
406 cconn->hba->snic->tx_max_size -
407 ISCSI_PDU_NONPAYLOAD_MAX);
408 407
408 max = min(cconn->hba->snic->tx_max_size, max);
409 if (conn->max_xmit_dlength) 409 if (conn->max_xmit_dlength)
410 conn->max_xmit_dlength = min_t(unsigned int, 410 conn->max_xmit_dlength = min(conn->max_xmit_dlength, max);
411 conn->max_xmit_dlength, max);
412 else 411 else
413 conn->max_xmit_dlength = max; 412 conn->max_xmit_dlength = max;
414 align_pdu_size(conn->max_xmit_dlength); 413 align_pdu_size(conn->max_xmit_dlength);
415 cxgb3i_log_info("conn 0x%p, max xmit %u.\n", 414 cxgb3i_api_debug("conn 0x%p, max xmit %u.\n",
416 conn, conn->max_xmit_dlength); 415 conn, conn->max_xmit_dlength);
417 return 0; 416 return 0;
418} 417}
@@ -427,9 +426,7 @@ static inline int cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
427{ 426{
428 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 427 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
429 struct cxgb3i_conn *cconn = tcp_conn->dd_data; 428 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
430 unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 429 unsigned int max = cconn->hba->snic->rx_max_size;
431 cconn->hba->snic->rx_max_size -
432 ISCSI_PDU_NONPAYLOAD_MAX);
433 430
434 align_pdu_size(max); 431 align_pdu_size(max);
435 if (conn->max_recv_dlength) { 432 if (conn->max_recv_dlength) {
@@ -439,8 +436,7 @@ static inline int cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
439 conn->max_recv_dlength, max); 436 conn->max_recv_dlength, max);
440 return -EINVAL; 437 return -EINVAL;
441 } 438 }
442 conn->max_recv_dlength = min_t(unsigned int, 439 conn->max_recv_dlength = min(conn->max_recv_dlength, max);
443 conn->max_recv_dlength, max);
444 align_pdu_size(conn->max_recv_dlength); 440 align_pdu_size(conn->max_recv_dlength);
445 } else 441 } else
446 conn->max_recv_dlength = max; 442 conn->max_recv_dlength = max;
@@ -844,7 +840,7 @@ static struct scsi_host_template cxgb3i_host_template = {
844 .proc_name = "cxgb3i", 840 .proc_name = "cxgb3i",
845 .queuecommand = iscsi_queuecommand, 841 .queuecommand = iscsi_queuecommand,
846 .change_queue_depth = iscsi_change_queue_depth, 842 .change_queue_depth = iscsi_change_queue_depth,
847 .can_queue = 128 * (ISCSI_DEF_XMIT_CMDS_MAX - 1), 843 .can_queue = CXGB3I_SCSI_QDEPTH_DFLT - 1,
848 .sg_tablesize = SG_ALL, 844 .sg_tablesize = SG_ALL,
849 .max_sectors = 0xFFFF, 845 .max_sectors = 0xFFFF,
850 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 846 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
diff --git a/drivers/scsi/cxgb3i/cxgb3i_offload.c b/drivers/scsi/cxgb3i/cxgb3i_offload.c
index a865f1fefe8b..de3b3b614cca 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_offload.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
@@ -23,19 +23,19 @@
23#include "cxgb3i_ddp.h" 23#include "cxgb3i_ddp.h"
24 24
25#ifdef __DEBUG_C3CN_CONN__ 25#ifdef __DEBUG_C3CN_CONN__
26#define c3cn_conn_debug cxgb3i_log_info 26#define c3cn_conn_debug cxgb3i_log_debug
27#else 27#else
28#define c3cn_conn_debug(fmt...) 28#define c3cn_conn_debug(fmt...)
29#endif 29#endif
30 30
31#ifdef __DEBUG_C3CN_TX__ 31#ifdef __DEBUG_C3CN_TX__
32#define c3cn_tx_debug cxgb3i_log_debug 32#define c3cn_tx_debug cxgb3i_log_debug
33#else 33#else
34#define c3cn_tx_debug(fmt...) 34#define c3cn_tx_debug(fmt...)
35#endif 35#endif
36 36
37#ifdef __DEBUG_C3CN_RX__ 37#ifdef __DEBUG_C3CN_RX__
38#define c3cn_rx_debug cxgb3i_log_debug 38#define c3cn_rx_debug cxgb3i_log_debug
39#else 39#else
40#define c3cn_rx_debug(fmt...) 40#define c3cn_rx_debug(fmt...)
41#endif 41#endif
@@ -47,9 +47,9 @@ static int cxgb3_rcv_win = 256 * 1024;
47module_param(cxgb3_rcv_win, int, 0644); 47module_param(cxgb3_rcv_win, int, 0644);
48MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)"); 48MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
49 49
50static int cxgb3_snd_win = 64 * 1024; 50static int cxgb3_snd_win = 128 * 1024;
51module_param(cxgb3_snd_win, int, 0644); 51module_param(cxgb3_snd_win, int, 0644);
52MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=64KB)"); 52MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=128KB)");
53 53
54static int cxgb3_rx_credit_thres = 10 * 1024; 54static int cxgb3_rx_credit_thres = 10 * 1024;
55module_param(cxgb3_rx_credit_thres, int, 0644); 55module_param(cxgb3_rx_credit_thres, int, 0644);
@@ -301,8 +301,8 @@ static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
301static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb, 301static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
302 int flags) 302 int flags)
303{ 303{
304 CXGB3_SKB_CB(skb)->seq = c3cn->write_seq; 304 skb_tcp_seq(skb) = c3cn->write_seq;
305 CXGB3_SKB_CB(skb)->flags = flags; 305 skb_flags(skb) = flags;
306 __skb_queue_tail(&c3cn->write_queue, skb); 306 __skb_queue_tail(&c3cn->write_queue, skb);
307} 307}
308 308
@@ -457,12 +457,9 @@ static unsigned int wrlen __read_mostly;
457 * The number of WRs needed for an skb depends on the number of fragments 457 * The number of WRs needed for an skb depends on the number of fragments
458 * in the skb and whether it has any payload in its main body. This maps the 458 * in the skb and whether it has any payload in its main body. This maps the
459 * length of the gather list represented by an skb into the # of necessary WRs. 459 * length of the gather list represented by an skb into the # of necessary WRs.
460 * 460 * The extra two fragments are for iscsi bhs and payload padding.
461 * The max. length of an skb is controlled by the max pdu size which is ~16K.
462 * Also, assume the min. fragment length is the sector size (512), then add
463 * extra fragment counts for iscsi bhs and payload padding.
464 */ 461 */
465#define SKB_WR_LIST_SIZE (16384/512 + 3) 462#define SKB_WR_LIST_SIZE (MAX_SKB_FRAGS + 2)
466static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly; 463static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
467 464
468static void s3_init_wr_tab(unsigned int wr_len) 465static void s3_init_wr_tab(unsigned int wr_len)
@@ -485,7 +482,7 @@ static void s3_init_wr_tab(unsigned int wr_len)
485 482
486static inline void reset_wr_list(struct s3_conn *c3cn) 483static inline void reset_wr_list(struct s3_conn *c3cn)
487{ 484{
488 c3cn->wr_pending_head = NULL; 485 c3cn->wr_pending_head = c3cn->wr_pending_tail = NULL;
489} 486}
490 487
491/* 488/*
@@ -496,7 +493,7 @@ static inline void reset_wr_list(struct s3_conn *c3cn)
496static inline void enqueue_wr(struct s3_conn *c3cn, 493static inline void enqueue_wr(struct s3_conn *c3cn,
497 struct sk_buff *skb) 494 struct sk_buff *skb)
498{ 495{
499 skb_wr_data(skb) = NULL; 496 skb_tx_wr_next(skb) = NULL;
500 497
501 /* 498 /*
502 * We want to take an extra reference since both us and the driver 499 * We want to take an extra reference since both us and the driver
@@ -509,10 +506,22 @@ static inline void enqueue_wr(struct s3_conn *c3cn,
509 if (!c3cn->wr_pending_head) 506 if (!c3cn->wr_pending_head)
510 c3cn->wr_pending_head = skb; 507 c3cn->wr_pending_head = skb;
511 else 508 else
512 skb_wr_data(skb) = skb; 509 skb_tx_wr_next(c3cn->wr_pending_tail) = skb;
513 c3cn->wr_pending_tail = skb; 510 c3cn->wr_pending_tail = skb;
514} 511}
515 512
513static int count_pending_wrs(struct s3_conn *c3cn)
514{
515 int n = 0;
516 const struct sk_buff *skb = c3cn->wr_pending_head;
517
518 while (skb) {
519 n += skb->csum;
520 skb = skb_tx_wr_next(skb);
521 }
522 return n;
523}
524
516static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn) 525static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
517{ 526{
518 return c3cn->wr_pending_head; 527 return c3cn->wr_pending_head;
@@ -529,8 +538,8 @@ static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
529 538
530 if (likely(skb)) { 539 if (likely(skb)) {
531 /* Don't bother clearing the tail */ 540 /* Don't bother clearing the tail */
532 c3cn->wr_pending_head = skb_wr_data(skb); 541 c3cn->wr_pending_head = skb_tx_wr_next(skb);
533 skb_wr_data(skb) = NULL; 542 skb_tx_wr_next(skb) = NULL;
534 } 543 }
535 return skb; 544 return skb;
536} 545}
@@ -543,13 +552,14 @@ static void purge_wr_queue(struct s3_conn *c3cn)
543} 552}
544 553
545static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb, 554static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb,
546 int len) 555 int len, int req_completion)
547{ 556{
548 struct tx_data_wr *req; 557 struct tx_data_wr *req;
549 558
550 skb_reset_transport_header(skb); 559 skb_reset_transport_header(skb);
551 req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req)); 560 req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
552 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)); 561 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
562 (req_completion ? F_WR_COMPL : 0));
553 req->wr_lo = htonl(V_WR_TID(c3cn->tid)); 563 req->wr_lo = htonl(V_WR_TID(c3cn->tid));
554 req->sndseq = htonl(c3cn->snd_nxt); 564 req->sndseq = htonl(c3cn->snd_nxt);
555 /* len includes the length of any HW ULP additions */ 565 /* len includes the length of any HW ULP additions */
@@ -592,7 +602,7 @@ static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion)
592 602
593 if (unlikely(c3cn->state == C3CN_STATE_CONNECTING || 603 if (unlikely(c3cn->state == C3CN_STATE_CONNECTING ||
594 c3cn->state == C3CN_STATE_CLOSE_WAIT_1 || 604 c3cn->state == C3CN_STATE_CLOSE_WAIT_1 ||
595 c3cn->state == C3CN_STATE_ABORTING)) { 605 c3cn->state >= C3CN_STATE_ABORTING)) {
596 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n", 606 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
597 c3cn, c3cn->state); 607 c3cn, c3cn->state);
598 return 0; 608 return 0;
@@ -615,7 +625,7 @@ static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion)
615 if (c3cn->wr_avail < wrs_needed) { 625 if (c3cn->wr_avail < wrs_needed) {
616 c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, " 626 c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, "
617 "wr %d < %u.\n", 627 "wr %d < %u.\n",
618 c3cn, skb->len, skb->datalen, frags, 628 c3cn, skb->len, skb->data_len, frags,
619 wrs_needed, c3cn->wr_avail); 629 wrs_needed, c3cn->wr_avail);
620 break; 630 break;
621 } 631 }
@@ -627,20 +637,24 @@ static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion)
627 c3cn->wr_unacked += wrs_needed; 637 c3cn->wr_unacked += wrs_needed;
628 enqueue_wr(c3cn, skb); 638 enqueue_wr(c3cn, skb);
629 639
630 if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) { 640 c3cn_tx_debug("c3cn 0x%p, enqueue, skb len %u/%u, frag %u, "
631 len += ulp_extra_len(skb); 641 "wr %d, left %u, unack %u.\n",
632 make_tx_data_wr(c3cn, skb, len); 642 c3cn, skb->len, skb->data_len, frags,
633 c3cn->snd_nxt += len; 643 wrs_needed, c3cn->wr_avail, c3cn->wr_unacked);
634 if ((req_completion 644
635 && c3cn->wr_unacked == wrs_needed)
636 || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
637 || c3cn->wr_unacked >= c3cn->wr_max / 2) {
638 struct work_request_hdr *wr = cplhdr(skb);
639 645
640 wr->wr_hi |= htonl(F_WR_COMPL); 646 if (likely(skb_flags(skb) & C3CB_FLAG_NEED_HDR)) {
647 if ((req_completion &&
648 c3cn->wr_unacked == wrs_needed) ||
649 (skb_flags(skb) & C3CB_FLAG_COMPL) ||
650 c3cn->wr_unacked >= c3cn->wr_max / 2) {
651 req_completion = 1;
641 c3cn->wr_unacked = 0; 652 c3cn->wr_unacked = 0;
642 } 653 }
643 CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR; 654 len += ulp_extra_len(skb);
655 make_tx_data_wr(c3cn, skb, len, req_completion);
656 c3cn->snd_nxt += len;
657 skb_flags(skb) &= ~C3CB_FLAG_NEED_HDR;
644 } 658 }
645 659
646 total_size += skb->truesize; 660 total_size += skb->truesize;
@@ -735,8 +749,11 @@ static void process_act_establish(struct s3_conn *c3cn, struct sk_buff *skb)
735 if (unlikely(c3cn_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED))) 749 if (unlikely(c3cn_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED)))
736 /* upper layer has requested closing */ 750 /* upper layer has requested closing */
737 send_abort_req(c3cn); 751 send_abort_req(c3cn);
738 else if (c3cn_push_tx_frames(c3cn, 1)) 752 else {
753 if (skb_queue_len(&c3cn->write_queue))
754 c3cn_push_tx_frames(c3cn, 1);
739 cxgb3i_conn_tx_open(c3cn); 755 cxgb3i_conn_tx_open(c3cn);
756 }
740} 757}
741 758
742static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb, 759static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
@@ -1082,8 +1099,8 @@ static void process_rx_iscsi_hdr(struct s3_conn *c3cn, struct sk_buff *skb)
1082 return; 1099 return;
1083 } 1100 }
1084 1101
1085 CXGB3_SKB_CB(skb)->seq = ntohl(hdr_cpl->seq); 1102 skb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
1086 CXGB3_SKB_CB(skb)->flags = 0; 1103 skb_flags(skb) = 0;
1087 1104
1088 skb_reset_transport_header(skb); 1105 skb_reset_transport_header(skb);
1089 __skb_pull(skb, sizeof(struct cpl_iscsi_hdr)); 1106 __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
@@ -1103,12 +1120,12 @@ static void process_rx_iscsi_hdr(struct s3_conn *c3cn, struct sk_buff *skb)
1103 goto abort_conn; 1120 goto abort_conn;
1104 1121
1105 skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY; 1122 skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY;
1106 skb_ulp_pdulen(skb) = ntohs(ddp_cpl.len); 1123 skb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
1107 skb_ulp_ddigest(skb) = ntohl(ddp_cpl.ulp_crc); 1124 skb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
1108 status = ntohl(ddp_cpl.ddp_status); 1125 status = ntohl(ddp_cpl.ddp_status);
1109 1126
1110 c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n", 1127 c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
1111 skb, skb->len, skb_ulp_pdulen(skb), status); 1128 skb, skb->len, skb_rx_pdulen(skb), status);
1112 1129
1113 if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT)) 1130 if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
1114 skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR; 1131 skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
@@ -1126,7 +1143,7 @@ static void process_rx_iscsi_hdr(struct s3_conn *c3cn, struct sk_buff *skb)
1126 } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT)) 1143 } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT))
1127 skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED; 1144 skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED;
1128 1145
1129 c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_ulp_pdulen(skb); 1146 c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_rx_pdulen(skb);
1130 __pskb_trim(skb, len); 1147 __pskb_trim(skb, len);
1131 __skb_queue_tail(&c3cn->receive_queue, skb); 1148 __skb_queue_tail(&c3cn->receive_queue, skb);
1132 cxgb3i_conn_pdu_ready(c3cn); 1149 cxgb3i_conn_pdu_ready(c3cn);
@@ -1151,12 +1168,27 @@ static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
1151 * Process an acknowledgment of WR completion. Advance snd_una and send the 1168 * Process an acknowledgment of WR completion. Advance snd_una and send the
1152 * next batch of work requests from the write queue. 1169 * next batch of work requests from the write queue.
1153 */ 1170 */
1171static void check_wr_invariants(struct s3_conn *c3cn)
1172{
1173 int pending = count_pending_wrs(c3cn);
1174
1175 if (unlikely(c3cn->wr_avail + pending != c3cn->wr_max))
1176 cxgb3i_log_error("TID %u: credit imbalance: avail %u, "
1177 "pending %u, total should be %u\n",
1178 c3cn->tid, c3cn->wr_avail, pending,
1179 c3cn->wr_max);
1180}
1181
1154static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb) 1182static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
1155{ 1183{
1156 struct cpl_wr_ack *hdr = cplhdr(skb); 1184 struct cpl_wr_ack *hdr = cplhdr(skb);
1157 unsigned int credits = ntohs(hdr->credits); 1185 unsigned int credits = ntohs(hdr->credits);
1158 u32 snd_una = ntohl(hdr->snd_una); 1186 u32 snd_una = ntohl(hdr->snd_una);
1159 1187
1188 c3cn_tx_debug("%u WR credits, avail %u, unack %u, TID %u, state %u.\n",
1189 credits, c3cn->wr_avail, c3cn->wr_unacked,
1190 c3cn->tid, c3cn->state);
1191
1160 c3cn->wr_avail += credits; 1192 c3cn->wr_avail += credits;
1161 if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail) 1193 if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
1162 c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail; 1194 c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
@@ -1171,6 +1203,17 @@ static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
1171 break; 1203 break;
1172 } 1204 }
1173 if (unlikely(credits < p->csum)) { 1205 if (unlikely(credits < p->csum)) {
1206 struct tx_data_wr *w = cplhdr(p);
1207 cxgb3i_log_error("TID %u got %u WR credits need %u, "
1208 "len %u, main body %u, frags %u, "
1209 "seq # %u, ACK una %u, ACK nxt %u, "
1210 "WR_AVAIL %u, WRs pending %u\n",
1211 c3cn->tid, credits, p->csum, p->len,
1212 p->len - p->data_len,
1213 skb_shinfo(p)->nr_frags,
1214 ntohl(w->sndseq), snd_una,
1215 ntohl(hdr->snd_nxt), c3cn->wr_avail,
1216 count_pending_wrs(c3cn) - credits);
1174 p->csum -= credits; 1217 p->csum -= credits;
1175 break; 1218 break;
1176 } else { 1219 } else {
@@ -1180,15 +1223,24 @@ static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
1180 } 1223 }
1181 } 1224 }
1182 1225
1183 if (unlikely(before(snd_una, c3cn->snd_una))) 1226 check_wr_invariants(c3cn);
1227
1228 if (unlikely(before(snd_una, c3cn->snd_una))) {
1229 cxgb3i_log_error("TID %u, unexpected sequence # %u in WR_ACK "
1230 "snd_una %u\n",
1231 c3cn->tid, snd_una, c3cn->snd_una);
1184 goto out_free; 1232 goto out_free;
1233 }
1185 1234
1186 if (c3cn->snd_una != snd_una) { 1235 if (c3cn->snd_una != snd_una) {
1187 c3cn->snd_una = snd_una; 1236 c3cn->snd_una = snd_una;
1188 dst_confirm(c3cn->dst_cache); 1237 dst_confirm(c3cn->dst_cache);
1189 } 1238 }
1190 1239
1191 if (skb_queue_len(&c3cn->write_queue) && c3cn_push_tx_frames(c3cn, 0)) 1240 if (skb_queue_len(&c3cn->write_queue)) {
1241 if (c3cn_push_tx_frames(c3cn, 0))
1242 cxgb3i_conn_tx_open(c3cn);
1243 } else
1192 cxgb3i_conn_tx_open(c3cn); 1244 cxgb3i_conn_tx_open(c3cn);
1193out_free: 1245out_free:
1194 __kfree_skb(skb); 1246 __kfree_skb(skb);
@@ -1452,7 +1504,7 @@ static void init_offload_conn(struct s3_conn *c3cn,
1452 struct dst_entry *dst) 1504 struct dst_entry *dst)
1453{ 1505{
1454 BUG_ON(c3cn->cdev != cdev); 1506 BUG_ON(c3cn->cdev != cdev);
1455 c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs; 1507 c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs - 1;
1456 c3cn->wr_unacked = 0; 1508 c3cn->wr_unacked = 0;
1457 c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst)); 1509 c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
1458 1510
@@ -1671,9 +1723,17 @@ int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb)
1671 goto out_err; 1723 goto out_err;
1672 } 1724 }
1673 1725
1674 err = -EPIPE;
1675 if (c3cn->err) { 1726 if (c3cn->err) {
1676 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err); 1727 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err);
1728 err = -EPIPE;
1729 goto out_err;
1730 }
1731
1732 if (c3cn->write_seq - c3cn->snd_una >= cxgb3_snd_win) {
1733 c3cn_tx_debug("c3cn 0x%p, snd %u - %u > %u.\n",
1734 c3cn, c3cn->write_seq, c3cn->snd_una,
1735 cxgb3_snd_win);
1736 err = -EAGAIN;
1677 goto out_err; 1737 goto out_err;
1678 } 1738 }
1679 1739
diff --git a/drivers/scsi/cxgb3i/cxgb3i_offload.h b/drivers/scsi/cxgb3i/cxgb3i_offload.h
index d23156907ffd..6344b9eb2589 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_offload.h
+++ b/drivers/scsi/cxgb3i/cxgb3i_offload.h
@@ -178,25 +178,33 @@ void cxgb3i_c3cn_release(struct s3_conn *);
178 * @flag: see C3CB_FLAG_* below 178 * @flag: see C3CB_FLAG_* below
179 * @ulp_mode: ULP mode/submode of sk_buff 179 * @ulp_mode: ULP mode/submode of sk_buff
180 * @seq: tcp sequence number 180 * @seq: tcp sequence number
181 * @ddigest: pdu data digest
182 * @pdulen: recovered pdu length
183 * @wr_data: scratch area for tx wr
184 */ 181 */
182struct cxgb3_skb_rx_cb {
183 __u32 ddigest; /* data digest */
184 __u32 pdulen; /* recovered pdu length */
185};
186
187struct cxgb3_skb_tx_cb {
188 struct sk_buff *wr_next; /* next wr */
189};
190
185struct cxgb3_skb_cb { 191struct cxgb3_skb_cb {
186 __u8 flags; 192 __u8 flags;
187 __u8 ulp_mode; 193 __u8 ulp_mode;
188 __u32 seq; 194 __u32 seq;
189 __u32 ddigest; 195 union {
190 __u32 pdulen; 196 struct cxgb3_skb_rx_cb rx;
191 struct sk_buff *wr_data; 197 struct cxgb3_skb_tx_cb tx;
198 };
192}; 199};
193 200
194#define CXGB3_SKB_CB(skb) ((struct cxgb3_skb_cb *)&((skb)->cb[0])) 201#define CXGB3_SKB_CB(skb) ((struct cxgb3_skb_cb *)&((skb)->cb[0]))
195 202#define skb_flags(skb) (CXGB3_SKB_CB(skb)->flags)
196#define skb_ulp_mode(skb) (CXGB3_SKB_CB(skb)->ulp_mode) 203#define skb_ulp_mode(skb) (CXGB3_SKB_CB(skb)->ulp_mode)
197#define skb_ulp_ddigest(skb) (CXGB3_SKB_CB(skb)->ddigest) 204#define skb_tcp_seq(skb) (CXGB3_SKB_CB(skb)->seq)
198#define skb_ulp_pdulen(skb) (CXGB3_SKB_CB(skb)->pdulen) 205#define skb_rx_ddigest(skb) (CXGB3_SKB_CB(skb)->rx.ddigest)
199#define skb_wr_data(skb) (CXGB3_SKB_CB(skb)->wr_data) 206#define skb_rx_pdulen(skb) (CXGB3_SKB_CB(skb)->rx.pdulen)
207#define skb_tx_wr_next(skb) (CXGB3_SKB_CB(skb)->tx.wr_next)
200 208
201enum c3cb_flags { 209enum c3cb_flags {
202 C3CB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */ 210 C3CB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
@@ -217,6 +225,7 @@ struct sge_opaque_hdr {
217/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */ 225/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
218#define TX_HEADER_LEN \ 226#define TX_HEADER_LEN \
219 (sizeof(struct tx_data_wr) + sizeof(struct sge_opaque_hdr)) 227 (sizeof(struct tx_data_wr) + sizeof(struct sge_opaque_hdr))
228#define SKB_TX_HEADROOM SKB_MAX_HEAD(TX_HEADER_LEN)
220 229
221/* 230/*
222 * get and set private ip for iscsi traffic 231 * get and set private ip for iscsi traffic
diff --git a/drivers/scsi/cxgb3i/cxgb3i_pdu.c b/drivers/scsi/cxgb3i/cxgb3i_pdu.c
index ce7ce8c6094c..17115c230d65 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_pdu.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_pdu.c
@@ -32,6 +32,10 @@
32#define cxgb3i_tx_debug(fmt...) 32#define cxgb3i_tx_debug(fmt...)
33#endif 33#endif
34 34
35/* always allocate rooms for AHS */
36#define SKB_TX_PDU_HEADER_LEN \
37 (sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE)
38static unsigned int skb_extra_headroom;
35static struct page *pad_page; 39static struct page *pad_page;
36 40
37/* 41/*
@@ -146,12 +150,13 @@ static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
146 150
147void cxgb3i_conn_cleanup_task(struct iscsi_task *task) 151void cxgb3i_conn_cleanup_task(struct iscsi_task *task)
148{ 152{
149 struct iscsi_tcp_task *tcp_task = task->dd_data; 153 struct cxgb3i_task_data *tdata = task->dd_data +
154 sizeof(struct iscsi_tcp_task);
150 155
151 /* never reached the xmit task callout */ 156 /* never reached the xmit task callout */
152 if (tcp_task->dd_data) 157 if (tdata->skb)
153 kfree_skb(tcp_task->dd_data); 158 __kfree_skb(tdata->skb);
154 tcp_task->dd_data = NULL; 159 memset(tdata, 0, sizeof(struct cxgb3i_task_data));
155 160
156 /* MNC - Do we need a check in case this is called but 161 /* MNC - Do we need a check in case this is called but
157 * cxgb3i_conn_alloc_pdu has never been called on the task */ 162 * cxgb3i_conn_alloc_pdu has never been called on the task */
@@ -159,28 +164,102 @@ void cxgb3i_conn_cleanup_task(struct iscsi_task *task)
159 iscsi_tcp_cleanup_task(task); 164 iscsi_tcp_cleanup_task(task);
160} 165}
161 166
162/* 167static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt,
163 * We do not support ahs yet 168 unsigned int offset, unsigned int *off,
164 */ 169 struct scatterlist **sgp)
170{
171 int i;
172 struct scatterlist *sg;
173
174 for_each_sg(sgl, sg, sgcnt, i) {
175 if (offset < sg->length) {
176 *off = offset;
177 *sgp = sg;
178 return 0;
179 }
180 offset -= sg->length;
181 }
182 return -EFAULT;
183}
184
185static int sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
186 unsigned int dlen, skb_frag_t *frags,
187 int frag_max)
188{
189 unsigned int datalen = dlen;
190 unsigned int sglen = sg->length - sgoffset;
191 struct page *page = sg_page(sg);
192 int i;
193
194 i = 0;
195 do {
196 unsigned int copy;
197
198 if (!sglen) {
199 sg = sg_next(sg);
200 if (!sg) {
201 cxgb3i_log_error("%s, sg NULL, len %u/%u.\n",
202 __func__, datalen, dlen);
203 return -EINVAL;
204 }
205 sgoffset = 0;
206 sglen = sg->length;
207 page = sg_page(sg);
208
209 }
210 copy = min(datalen, sglen);
211 if (i && page == frags[i - 1].page &&
212 sgoffset + sg->offset ==
213 frags[i - 1].page_offset + frags[i - 1].size) {
214 frags[i - 1].size += copy;
215 } else {
216 if (i >= frag_max) {
217 cxgb3i_log_error("%s, too many pages %u, "
218 "dlen %u.\n", __func__,
219 frag_max, dlen);
220 return -EINVAL;
221 }
222
223 frags[i].page = page;
224 frags[i].page_offset = sg->offset + sgoffset;
225 frags[i].size = copy;
226 i++;
227 }
228 datalen -= copy;
229 sgoffset += copy;
230 sglen -= copy;
231 } while (datalen);
232
233 return i;
234}
235
165int cxgb3i_conn_alloc_pdu(struct iscsi_task *task, u8 opcode) 236int cxgb3i_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
166{ 237{
238 struct iscsi_conn *conn = task->conn;
167 struct iscsi_tcp_task *tcp_task = task->dd_data; 239 struct iscsi_tcp_task *tcp_task = task->dd_data;
168 struct sk_buff *skb; 240 struct cxgb3i_task_data *tdata = task->dd_data + sizeof(*tcp_task);
241 struct scsi_cmnd *sc = task->sc;
242 int headroom = SKB_TX_PDU_HEADER_LEN;
169 243
244 tcp_task->dd_data = tdata;
170 task->hdr = NULL; 245 task->hdr = NULL;
171 /* always allocate rooms for AHS */ 246
172 skb = alloc_skb(sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE + 247 /* write command, need to send data pdus */
173 TX_HEADER_LEN, GFP_ATOMIC); 248 if (skb_extra_headroom && (opcode == ISCSI_OP_SCSI_DATA_OUT ||
174 if (!skb) 249 (opcode == ISCSI_OP_SCSI_CMD &&
250 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_TO_DEVICE))))
251 headroom += min(skb_extra_headroom, conn->max_xmit_dlength);
252
253 tdata->skb = alloc_skb(TX_HEADER_LEN + headroom, GFP_ATOMIC);
254 if (!tdata->skb)
175 return -ENOMEM; 255 return -ENOMEM;
256 skb_reserve(tdata->skb, TX_HEADER_LEN);
176 257
177 cxgb3i_tx_debug("task 0x%p, opcode 0x%x, skb 0x%p.\n", 258 cxgb3i_tx_debug("task 0x%p, opcode 0x%x, skb 0x%p.\n",
178 task, opcode, skb); 259 task, opcode, tdata->skb);
179 260
180 tcp_task->dd_data = skb; 261 task->hdr = (struct iscsi_hdr *)tdata->skb->data;
181 skb_reserve(skb, TX_HEADER_LEN); 262 task->hdr_max = SKB_TX_PDU_HEADER_LEN;
182 task->hdr = (struct iscsi_hdr *)skb->data;
183 task->hdr_max = sizeof(struct iscsi_hdr);
184 263
185 /* data_out uses scsi_cmd's itt */ 264 /* data_out uses scsi_cmd's itt */
186 if (opcode != ISCSI_OP_SCSI_DATA_OUT) 265 if (opcode != ISCSI_OP_SCSI_DATA_OUT)
@@ -192,13 +271,13 @@ int cxgb3i_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
192int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset, 271int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
193 unsigned int count) 272 unsigned int count)
194{ 273{
195 struct iscsi_tcp_task *tcp_task = task->dd_data;
196 struct sk_buff *skb = tcp_task->dd_data;
197 struct iscsi_conn *conn = task->conn; 274 struct iscsi_conn *conn = task->conn;
198 struct page *pg; 275 struct iscsi_tcp_task *tcp_task = task->dd_data;
276 struct cxgb3i_task_data *tdata = tcp_task->dd_data;
277 struct sk_buff *skb = tdata->skb;
199 unsigned int datalen = count; 278 unsigned int datalen = count;
200 int i, padlen = iscsi_padding(count); 279 int i, padlen = iscsi_padding(count);
201 skb_frag_t *frag; 280 struct page *pg;
202 281
203 cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n", 282 cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n",
204 task, task->sc, offset, count, skb); 283 task, task->sc, offset, count, skb);
@@ -209,90 +288,94 @@ int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
209 return 0; 288 return 0;
210 289
211 if (task->sc) { 290 if (task->sc) {
212 struct scatterlist *sg; 291 struct scsi_data_buffer *sdb = scsi_out(task->sc);
213 struct scsi_data_buffer *sdb; 292 struct scatterlist *sg = NULL;
214 unsigned int sgoffset = offset; 293 int err;
215 struct page *sgpg; 294
216 unsigned int sglen; 295 tdata->offset = offset;
217 296 tdata->count = count;
218 sdb = scsi_out(task->sc); 297 err = sgl_seek_offset(sdb->table.sgl, sdb->table.nents,
219 sg = sdb->table.sgl; 298 tdata->offset, &tdata->sgoffset, &sg);
220 299 if (err < 0) {
221 for_each_sg(sdb->table.sgl, sg, sdb->table.nents, i) { 300 cxgb3i_log_warn("tpdu, sgl %u, bad offset %u/%u.\n",
222 cxgb3i_tx_debug("sg %d, page 0x%p, len %u offset %u\n", 301 sdb->table.nents, tdata->offset,
223 i, sg_page(sg), sg->length, sg->offset); 302 sdb->length);
224 303 return err;
225 if (sgoffset < sg->length)
226 break;
227 sgoffset -= sg->length;
228 } 304 }
229 sgpg = sg_page(sg); 305 err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count,
230 sglen = sg->length - sgoffset; 306 tdata->frags, MAX_PDU_FRAGS);
231 307 if (err < 0) {
232 do { 308 cxgb3i_log_warn("tpdu, sgl %u, bad offset %u + %u.\n",
233 int j = skb_shinfo(skb)->nr_frags; 309 sdb->table.nents, tdata->offset,
234 unsigned int copy; 310 tdata->count);
235 311 return err;
236 if (!sglen) { 312 }
237 sg = sg_next(sg); 313 tdata->nr_frags = err;
238 sgpg = sg_page(sg); 314
239 sgoffset = 0; 315 if (tdata->nr_frags > MAX_SKB_FRAGS ||
240 sglen = sg->length; 316 (padlen && tdata->nr_frags == MAX_SKB_FRAGS)) {
241 ++i; 317 char *dst = skb->data + task->hdr_len;
318 skb_frag_t *frag = tdata->frags;
319
320 /* data fits in the skb's headroom */
321 for (i = 0; i < tdata->nr_frags; i++, frag++) {
322 char *src = kmap_atomic(frag->page,
323 KM_SOFTIRQ0);
324
325 memcpy(dst, src+frag->page_offset, frag->size);
326 dst += frag->size;
327 kunmap_atomic(src, KM_SOFTIRQ0);
242 } 328 }
243 copy = min(sglen, datalen); 329 if (padlen) {
244 if (j && skb_can_coalesce(skb, j, sgpg, 330 memset(dst, 0, padlen);
245 sg->offset + sgoffset)) { 331 padlen = 0;
246 skb_shinfo(skb)->frags[j - 1].size += copy;
247 } else {
248 get_page(sgpg);
249 skb_fill_page_desc(skb, j, sgpg,
250 sg->offset + sgoffset, copy);
251 } 332 }
252 sgoffset += copy; 333 skb_put(skb, count + padlen);
253 sglen -= copy; 334 } else {
254 datalen -= copy; 335 /* data fit into frag_list */
255 } while (datalen); 336 for (i = 0; i < tdata->nr_frags; i++)
337 get_page(tdata->frags[i].page);
338
339 memcpy(skb_shinfo(skb)->frags, tdata->frags,
340 sizeof(skb_frag_t) * tdata->nr_frags);
341 skb_shinfo(skb)->nr_frags = tdata->nr_frags;
342 skb->len += count;
343 skb->data_len += count;
344 skb->truesize += count;
345 }
346
256 } else { 347 } else {
257 pg = virt_to_page(task->data); 348 pg = virt_to_page(task->data);
258 349
259 while (datalen) { 350 get_page(pg);
260 i = skb_shinfo(skb)->nr_frags; 351 skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data),
261 frag = &skb_shinfo(skb)->frags[i]; 352 count);
262 353 skb->len += count;
263 get_page(pg); 354 skb->data_len += count;
264 frag->page = pg; 355 skb->truesize += count;
265 frag->page_offset = 0;
266 frag->size = min((unsigned int)PAGE_SIZE, datalen);
267
268 skb_shinfo(skb)->nr_frags++;
269 datalen -= frag->size;
270 pg++;
271 }
272 } 356 }
273 357
274 if (padlen) { 358 if (padlen) {
275 i = skb_shinfo(skb)->nr_frags; 359 i = skb_shinfo(skb)->nr_frags;
276 frag = &skb_shinfo(skb)->frags[i]; 360 get_page(pad_page);
277 frag->page = pad_page; 361 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, pad_page, 0,
278 frag->page_offset = 0; 362 padlen);
279 frag->size = padlen; 363
280 skb_shinfo(skb)->nr_frags++; 364 skb->data_len += padlen;
365 skb->truesize += padlen;
366 skb->len += padlen;
281 } 367 }
282 368
283 datalen = count + padlen;
284 skb->data_len += datalen;
285 skb->truesize += datalen;
286 skb->len += datalen;
287 return 0; 369 return 0;
288} 370}
289 371
290int cxgb3i_conn_xmit_pdu(struct iscsi_task *task) 372int cxgb3i_conn_xmit_pdu(struct iscsi_task *task)
291{ 373{
292 struct iscsi_tcp_task *tcp_task = task->dd_data;
293 struct sk_buff *skb = tcp_task->dd_data;
294 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 374 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
295 struct cxgb3i_conn *cconn = tcp_conn->dd_data; 375 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
376 struct iscsi_tcp_task *tcp_task = task->dd_data;
377 struct cxgb3i_task_data *tdata = tcp_task->dd_data;
378 struct sk_buff *skb = tdata->skb;
296 unsigned int datalen; 379 unsigned int datalen;
297 int err; 380 int err;
298 381
@@ -300,13 +383,14 @@ int cxgb3i_conn_xmit_pdu(struct iscsi_task *task)
300 return 0; 383 return 0;
301 384
302 datalen = skb->data_len; 385 datalen = skb->data_len;
303 tcp_task->dd_data = NULL; 386 tdata->skb = NULL;
304 err = cxgb3i_c3cn_send_pdus(cconn->cep->c3cn, skb); 387 err = cxgb3i_c3cn_send_pdus(cconn->cep->c3cn, skb);
305 cxgb3i_tx_debug("task 0x%p, skb 0x%p, len %u/%u, rv %d.\n",
306 task, skb, skb->len, skb->data_len, err);
307 if (err > 0) { 388 if (err > 0) {
308 int pdulen = err; 389 int pdulen = err;
309 390
391 cxgb3i_tx_debug("task 0x%p, skb 0x%p, len %u/%u, rv %d.\n",
392 task, skb, skb->len, skb->data_len, err);
393
310 if (task->conn->hdrdgst_en) 394 if (task->conn->hdrdgst_en)
311 pdulen += ISCSI_DIGEST_SIZE; 395 pdulen += ISCSI_DIGEST_SIZE;
312 if (datalen && task->conn->datadgst_en) 396 if (datalen && task->conn->datadgst_en)
@@ -325,12 +409,14 @@ int cxgb3i_conn_xmit_pdu(struct iscsi_task *task)
325 return err; 409 return err;
326 } 410 }
327 /* reset skb to send when we are called again */ 411 /* reset skb to send when we are called again */
328 tcp_task->dd_data = skb; 412 tdata->skb = skb;
329 return -EAGAIN; 413 return -EAGAIN;
330} 414}
331 415
332int cxgb3i_pdu_init(void) 416int cxgb3i_pdu_init(void)
333{ 417{
418 if (SKB_TX_HEADROOM > (512 * MAX_SKB_FRAGS))
419 skb_extra_headroom = SKB_TX_HEADROOM;
334 pad_page = alloc_page(GFP_KERNEL); 420 pad_page = alloc_page(GFP_KERNEL);
335 if (!pad_page) 421 if (!pad_page)
336 return -ENOMEM; 422 return -ENOMEM;
@@ -366,7 +452,9 @@ void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
366 skb = skb_peek(&c3cn->receive_queue); 452 skb = skb_peek(&c3cn->receive_queue);
367 while (!err && skb) { 453 while (!err && skb) {
368 __skb_unlink(skb, &c3cn->receive_queue); 454 __skb_unlink(skb, &c3cn->receive_queue);
369 read += skb_ulp_pdulen(skb); 455 read += skb_rx_pdulen(skb);
456 cxgb3i_rx_debug("conn 0x%p, cn 0x%p, rx skb 0x%p, pdulen %u.\n",
457 conn, c3cn, skb, skb_rx_pdulen(skb));
370 err = cxgb3i_conn_read_pdu_skb(conn, skb); 458 err = cxgb3i_conn_read_pdu_skb(conn, skb);
371 __kfree_skb(skb); 459 __kfree_skb(skb);
372 skb = skb_peek(&c3cn->receive_queue); 460 skb = skb_peek(&c3cn->receive_queue);
@@ -377,6 +465,11 @@ void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
377 cxgb3i_c3cn_rx_credits(c3cn, read); 465 cxgb3i_c3cn_rx_credits(c3cn, read);
378 } 466 }
379 conn->rxdata_octets += read; 467 conn->rxdata_octets += read;
468
469 if (err) {
470 cxgb3i_log_info("conn 0x%p rx failed err %d.\n", conn, err);
471 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
472 }
380} 473}
381 474
382void cxgb3i_conn_tx_open(struct s3_conn *c3cn) 475void cxgb3i_conn_tx_open(struct s3_conn *c3cn)
diff --git a/drivers/scsi/cxgb3i/cxgb3i_pdu.h b/drivers/scsi/cxgb3i/cxgb3i_pdu.h
index a3f685cc2362..0770b23d90da 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_pdu.h
+++ b/drivers/scsi/cxgb3i/cxgb3i_pdu.h
@@ -53,7 +53,7 @@ struct cpl_rx_data_ddp_norss {
53#define ULP2_FLAG_DCRC_ERROR 0x20 53#define ULP2_FLAG_DCRC_ERROR 0x20
54#define ULP2_FLAG_PAD_ERROR 0x40 54#define ULP2_FLAG_PAD_ERROR 0x40
55 55
56void cxgb3i_conn_closing(struct s3_conn *); 56void cxgb3i_conn_closing(struct s3_conn *c3cn);
57void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn); 57void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);
58void cxgb3i_conn_tx_open(struct s3_conn *c3cn); 58void cxgb3i_conn_tx_open(struct s3_conn *c3cn);
59#endif 59#endif
diff --git a/drivers/scsi/hptiop.c b/drivers/scsi/hptiop.c
index a48e4990fe12..34be88d7afa5 100644
--- a/drivers/scsi/hptiop.c
+++ b/drivers/scsi/hptiop.c
@@ -1251,6 +1251,7 @@ static struct pci_device_id hptiop_id_table[] = {
1251 { PCI_VDEVICE(TTI, 0x3530), (kernel_ulong_t)&hptiop_itl_ops }, 1251 { PCI_VDEVICE(TTI, 0x3530), (kernel_ulong_t)&hptiop_itl_ops },
1252 { PCI_VDEVICE(TTI, 0x3560), (kernel_ulong_t)&hptiop_itl_ops }, 1252 { PCI_VDEVICE(TTI, 0x3560), (kernel_ulong_t)&hptiop_itl_ops },
1253 { PCI_VDEVICE(TTI, 0x4322), (kernel_ulong_t)&hptiop_itl_ops }, 1253 { PCI_VDEVICE(TTI, 0x4322), (kernel_ulong_t)&hptiop_itl_ops },
1254 { PCI_VDEVICE(TTI, 0x4321), (kernel_ulong_t)&hptiop_itl_ops },
1254 { PCI_VDEVICE(TTI, 0x4210), (kernel_ulong_t)&hptiop_itl_ops }, 1255 { PCI_VDEVICE(TTI, 0x4210), (kernel_ulong_t)&hptiop_itl_ops },
1255 { PCI_VDEVICE(TTI, 0x4211), (kernel_ulong_t)&hptiop_itl_ops }, 1256 { PCI_VDEVICE(TTI, 0x4211), (kernel_ulong_t)&hptiop_itl_ops },
1256 { PCI_VDEVICE(TTI, 0x4310), (kernel_ulong_t)&hptiop_itl_ops }, 1257 { PCI_VDEVICE(TTI, 0x4310), (kernel_ulong_t)&hptiop_itl_ops },
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 940dc32ff0dc..b82ffd90632e 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -1040,12 +1040,11 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
1040 action = ACTION_FAIL; 1040 action = ACTION_FAIL;
1041 break; 1041 break;
1042 case ABORTED_COMMAND: 1042 case ABORTED_COMMAND:
1043 action = ACTION_FAIL;
1043 if (sshdr.asc == 0x10) { /* DIF */ 1044 if (sshdr.asc == 0x10) { /* DIF */
1044 description = "Target Data Integrity Failure"; 1045 description = "Target Data Integrity Failure";
1045 action = ACTION_FAIL;
1046 error = -EILSEQ; 1046 error = -EILSEQ;
1047 } else 1047 }
1048 action = ACTION_RETRY;
1049 break; 1048 break;
1050 case NOT_READY: 1049 case NOT_READY:
1051 /* If the device is in the process of becoming 1050 /* If the device is in the process of becoming
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index d57566b8be0a..55310dbc10a6 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -107,6 +107,7 @@ static void scsi_disk_release(struct device *cdev);
107static void sd_print_sense_hdr(struct scsi_disk *, struct scsi_sense_hdr *); 107static void sd_print_sense_hdr(struct scsi_disk *, struct scsi_sense_hdr *);
108static void sd_print_result(struct scsi_disk *, int); 108static void sd_print_result(struct scsi_disk *, int);
109 109
110static DEFINE_SPINLOCK(sd_index_lock);
110static DEFINE_IDA(sd_index_ida); 111static DEFINE_IDA(sd_index_ida);
111 112
112/* This semaphore is used to mediate the 0->1 reference get in the 113/* This semaphore is used to mediate the 0->1 reference get in the
@@ -1914,7 +1915,9 @@ static int sd_probe(struct device *dev)
1914 if (!ida_pre_get(&sd_index_ida, GFP_KERNEL)) 1915 if (!ida_pre_get(&sd_index_ida, GFP_KERNEL))
1915 goto out_put; 1916 goto out_put;
1916 1917
1918 spin_lock(&sd_index_lock);
1917 error = ida_get_new(&sd_index_ida, &index); 1919 error = ida_get_new(&sd_index_ida, &index);
1920 spin_unlock(&sd_index_lock);
1918 } while (error == -EAGAIN); 1921 } while (error == -EAGAIN);
1919 1922
1920 if (error) 1923 if (error)
@@ -1936,7 +1939,9 @@ static int sd_probe(struct device *dev)
1936 return 0; 1939 return 0;
1937 1940
1938 out_free_index: 1941 out_free_index:
1942 spin_lock(&sd_index_lock);
1939 ida_remove(&sd_index_ida, index); 1943 ida_remove(&sd_index_ida, index);
1944 spin_unlock(&sd_index_lock);
1940 out_put: 1945 out_put:
1941 put_disk(gd); 1946 put_disk(gd);
1942 out_free: 1947 out_free:
@@ -1986,7 +1991,9 @@ static void scsi_disk_release(struct device *dev)
1986 struct scsi_disk *sdkp = to_scsi_disk(dev); 1991 struct scsi_disk *sdkp = to_scsi_disk(dev);
1987 struct gendisk *disk = sdkp->disk; 1992 struct gendisk *disk = sdkp->disk;
1988 1993
1994 spin_lock(&sd_index_lock);
1989 ida_remove(&sd_index_ida, sdkp->index); 1995 ida_remove(&sd_index_ida, sdkp->index);
1996 spin_unlock(&sd_index_lock);
1990 1997
1991 disk->private_data = NULL; 1998 disk->private_data = NULL;
1992 put_disk(disk); 1999 put_disk(disk);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-06 19:39:36 -0500