aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKaren Xie <kxie@chelsio.com>2008-12-09 17:15:32 -0500
committerJames Bottomley <James.Bottomley@HansenPartnership.com>2008-12-30 11:45:33 -0500
commitc3673464ebc004a3d82063cd41b9cf74d1b55db2 (patch)
treeb061ecd04da7dd3ddddad8f39a4922f437493311
parentb632ade282895562924d18b8eedd11a825f4b08c (diff)
[SCSI] cxgb3i: Add cxgb3i iSCSI driver.
This patch implements the cxgb3i iscsi connection acceleration for the open-iscsi initiator. The cxgb3i driver offers the iscsi PDU based offload: - digest insertion and verification - payload direct-placement into host memory buffer. Signed-off-by: Karen Xie <kxie@chelsio.com> Signed-off-by: James Bottomley <James.Bottomley@HansenPartnership.com>
Diffstat
-rw-r--r--Documentation/scsi/cxgb3i.txt85
-rw-r--r--drivers/scsi/Kconfig2
-rw-r--r--drivers/scsi/Makefile1
-rw-r--r--drivers/scsi/cxgb3i/Kbuild4
-rw-r--r--drivers/scsi/cxgb3i/Kconfig6
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i.h139
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_ddp.c770
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_ddp.h306
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_init.c107
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_iscsi.c951
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_offload.c1810
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_offload.h231
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_pdu.c402
-rw-r--r--drivers/scsi/cxgb3i/cxgb3i_pdu.h59
14 files changed, 4873 insertions, 0 deletions
diff --git a/Documentation/scsi/cxgb3i.txt b/Documentation/scsi/cxgb3i.txt
new file mode 100644
index 000000000000..8141fa01978e
--- /dev/null
+++ b/Documentation/scsi/cxgb3i.txt
@@ -0,0 +1,85 @@
1Chelsio S3 iSCSI Driver for Linux
2
3Introduction
4============
5
6The Chelsio T3 ASIC based Adapters (S310, S320, S302, S304, Mezz cards, etc.
7series of products) supports iSCSI acceleration and iSCSI Direct Data Placement
8(DDP) where the hardware handles the expensive byte touching operations, such
9as CRC computation and verification, and direct DMA to the final host memory
10destination:
11
12 - iSCSI PDU digest generation and verification
13
14 On transmitting, Chelsio S3 h/w computes and inserts the Header and
15 Data digest into the PDUs.
16 On receiving, Chelsio S3 h/w computes and verifies the Header and
17 Data digest of the PDUs.
18
19 - Direct Data Placement (DDP)
20
21 S3 h/w can directly place the iSCSI Data-In or Data-Out PDU's
22 payload into pre-posted final destination host-memory buffers based
23 on the Initiator Task Tag (ITT) in Data-In or Target Task Tag (TTT)
24 in Data-Out PDUs.
25
26 - PDU Transmit and Recovery
27
28 On transmitting, S3 h/w accepts the complete PDU (header + data)
29 from the host driver, computes and inserts the digests, decomposes
30 the PDU into multiple TCP segments if necessary, and transmit all
31 the TCP segments onto the wire. It handles TCP retransmission if
32 needed.
33
34 On receving, S3 h/w recovers the iSCSI PDU by reassembling TCP
35 segments, separating the header and data, calculating and verifying
36 the digests, then forwards the header to the host. The payload data,
37 if possible, will be directly placed into the pre-posted host DDP
38 buffer. Otherwise, the payload data will be sent to the host too.
39
40The cxgb3i driver interfaces with open-iscsi initiator and provides the iSCSI
41acceleration through Chelsio hardware wherever applicable.
42
43Using the cxgb3i Driver
44=======================
45
46The following steps need to be taken to accelerates the open-iscsi initiator:
47
481. Load the cxgb3i driver: "modprobe cxgb3i"
49
50 The cxgb3i module registers a new transport class "cxgb3i" with open-iscsi.
51
52 * in the case of recompiling the kernel, the cxgb3i selection is located at
53 Device Drivers
54 SCSI device support --->
55 [*] SCSI low-level drivers --->
56 <M> Chelsio S3xx iSCSI support
57
582. Create an interface file located under /etc/iscsi/ifaces/ for the new
59 transport class "cxgb3i".
60
61 The content of the file should be in the following format:
62 iface.transport_name = cxgb3i
63 iface.net_ifacename = <ethX>
64 iface.ipaddress = <iscsi ip address>
65
66 * if iface.ipaddress is specified, <iscsi ip address> needs to be either the
67 same as the ethX's ip address or an address on the same subnet. Make
68 sure the ip address is unique in the network.
69
703. edit /etc/iscsi/iscsid.conf
71 The default setting for MaxRecvDataSegmentLength (131072) is too big,
72 replace "node.conn[0].iscsi.MaxRecvDataSegmentLength" to be a value no
73 bigger than 15360 (for example 8192):
74
75 node.conn[0].iscsi.MaxRecvDataSegmentLength = 8192
76
77 * The login would fail for a normal session if MaxRecvDataSegmentLength is
78 too big. A error message in the format of
79 "cxgb3i: ERR! MaxRecvSegmentLength <X> too big. Need to be <= <Y>."
80 would be logged to dmesg.
81
824. To direct open-iscsi traffic to go through cxgb3i's accelerated path,
83 "-I <iface file name>" option needs to be specified with most of the
84 iscsiadm command. <iface file name> is the transport interface file created
85 in step 2.
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 673463e4bbf0..0e5e084dfb4c 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -352,6 +352,8 @@ config ISCSI_TCP
352 352
353 http://open-iscsi.org 353 http://open-iscsi.org
354 354
355source "drivers/scsi/cxgb3i/Kconfig"
356
355config SGIWD93_SCSI 357config SGIWD93_SCSI
356 tristate "SGI WD93C93 SCSI Driver" 358 tristate "SGI WD93C93 SCSI Driver"
357 depends on SGI_HAS_WD93 && SCSI 359 depends on SGI_HAS_WD93 && SCSI
diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile
index 07d0f58de9bb..1410697257cb 100644
--- a/drivers/scsi/Makefile
+++ b/drivers/scsi/Makefile
@@ -126,6 +126,7 @@ obj-$(CONFIG_SCSI_HPTIOP) += hptiop.o
126obj-$(CONFIG_SCSI_STEX) += stex.o 126obj-$(CONFIG_SCSI_STEX) += stex.o
127obj-$(CONFIG_SCSI_MVSAS) += mvsas.o 127obj-$(CONFIG_SCSI_MVSAS) += mvsas.o
128obj-$(CONFIG_PS3_ROM) += ps3rom.o 128obj-$(CONFIG_PS3_ROM) += ps3rom.o
129obj-$(CONFIG_SCSI_CXGB3_ISCSI) += libiscsi.o libiscsi_tcp.o cxgb3i/
129 130
130obj-$(CONFIG_ARM) += arm/ 131obj-$(CONFIG_ARM) += arm/
131 132
diff --git a/drivers/scsi/cxgb3i/Kbuild b/drivers/scsi/cxgb3i/Kbuild
new file mode 100644
index 000000000000..ee7d6d2f9c3b
--- /dev/null
+++ b/drivers/scsi/cxgb3i/Kbuild
@@ -0,0 +1,4 @@
1EXTRA_CFLAGS += -I$(TOPDIR)/drivers/net/cxgb3
2
3cxgb3i-y := cxgb3i_init.o cxgb3i_iscsi.o cxgb3i_pdu.o cxgb3i_offload.o
4obj-$(CONFIG_SCSI_CXGB3_ISCSI) += cxgb3i_ddp.o cxgb3i.o
diff --git a/drivers/scsi/cxgb3i/Kconfig b/drivers/scsi/cxgb3i/Kconfig
new file mode 100644
index 000000000000..276281460ece
--- /dev/null
+++ b/drivers/scsi/cxgb3i/Kconfig
@@ -0,0 +1,6 @@
1config SCSI_CXGB3_ISCSI
2 tristate "Chelsio S3xx iSCSI support"
3 select CHELSIO_T3
4 select SCSI_ISCSI_ATTRS
5 ---help---
6 This driver supports iSCSI offload for the Chelsio S3 series devices.
diff --git a/drivers/scsi/cxgb3i/cxgb3i.h b/drivers/scsi/cxgb3i/cxgb3i.h
new file mode 100644
index 000000000000..fde6e4c634e7
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i.h
@@ -0,0 +1,139 @@
1/*
2 * cxgb3i.h: Chelsio S3xx iSCSI driver.
3 *
4 * Copyright (c) 2008 Chelsio Communications, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 *
10 * Written by: Karen Xie (kxie@chelsio.com)
11 */
12
13#ifndef __CXGB3I_H__
14#define __CXGB3I_H__
15
16#include <linux/module.h>
17#include <linux/moduleparam.h>
18#include <linux/errno.h>
19#include <linux/types.h>
20#include <linux/list.h>
21#include <linux/netdevice.h>
22#include <linux/scatterlist.h>
23#include <scsi/libiscsi_tcp.h>
24
25/* from cxgb3 LLD */
26#include "common.h"
27#include "t3_cpl.h"
28#include "t3cdev.h"
29#include "cxgb3_ctl_defs.h"
30#include "cxgb3_offload.h"
31#include "firmware_exports.h"
32
33#include "cxgb3i_offload.h"
34#include "cxgb3i_ddp.h"
35
36#define CXGB3I_SCSI_QDEPTH_DFLT 128
37#define CXGB3I_MAX_TARGET CXGB3I_MAX_CONN
38#define CXGB3I_MAX_LUN 512
39#define ISCSI_PDU_NONPAYLOAD_MAX \
40 (sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE + 2*ISCSI_DIGEST_SIZE)
41
42struct cxgb3i_adapter;
43struct cxgb3i_hba;
44struct cxgb3i_endpoint;
45
46/**
47 * struct cxgb3i_hba - cxgb3i iscsi structure (per port)
48 *
49 * @snic: cxgb3i adapter containing this port
50 * @ndev: pointer to netdev structure
51 * @shost: pointer to scsi host structure
52 */
53struct cxgb3i_hba {
54 struct cxgb3i_adapter *snic;
55 struct net_device *ndev;
56 struct Scsi_Host *shost;
57};
58
59/**
60 * struct cxgb3i_adapter - cxgb3i adapter structure (per pci)
61 *
62 * @listhead: list head to link elements
63 * @lock: lock for this structure
64 * @tdev: pointer to t3cdev used by cxgb3 driver
65 * @pdev: pointer to pci dev
66 * @hba_cnt: # of hbas (the same as # of ports)
67 * @hba: all the hbas on this adapter
68 * @tx_max_size: max. tx packet size supported
69 * @rx_max_size: max. rx packet size supported
70 * @tag_format: ddp tag format settings
71 */
72struct cxgb3i_adapter {
73 struct list_head list_head;
74 spinlock_t lock;
75 struct t3cdev *tdev;
76 struct pci_dev *pdev;
77 unsigned char hba_cnt;
78 struct cxgb3i_hba *hba[MAX_NPORTS];
79
80 unsigned int tx_max_size;
81 unsigned int rx_max_size;
82
83 struct cxgb3i_tag_format tag_format;
84};
85
86/**
87 * struct cxgb3i_conn - cxgb3i iscsi connection
88 *
89 * @listhead: list head to link elements
90 * @cep: pointer to iscsi_endpoint structure
91 * @conn: pointer to iscsi_conn structure
92 * @hba: pointer to the hba this conn. is going through
93 * @task_idx_bits: # of bits needed for session->cmds_max
94 */
95struct cxgb3i_conn {
96 struct list_head list_head;
97 struct cxgb3i_endpoint *cep;
98 struct iscsi_conn *conn;
99 struct cxgb3i_hba *hba;
100 unsigned int task_idx_bits;
101};
102
103/**
104 * struct cxgb3i_endpoint - iscsi tcp endpoint
105 *
106 * @c3cn: the h/w tcp connection representation
107 * @hba: pointer to the hba this conn. is going through
108 * @cconn: pointer to the associated cxgb3i iscsi connection
109 */
110struct cxgb3i_endpoint {
111 struct s3_conn *c3cn;
112 struct cxgb3i_hba *hba;
113 struct cxgb3i_conn *cconn;
114};
115
116int cxgb3i_iscsi_init(void);
117void cxgb3i_iscsi_cleanup(void);
118
119struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *);
120void cxgb3i_adapter_remove(struct t3cdev *);
121int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *);
122void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *);
123
124struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *);
125struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *,
126 struct net_device *);
127void cxgb3i_hba_host_remove(struct cxgb3i_hba *);
128
129int cxgb3i_pdu_init(void);
130void cxgb3i_pdu_cleanup(void);
131void cxgb3i_conn_cleanup_task(struct iscsi_task *);
132int cxgb3i_conn_alloc_pdu(struct iscsi_task *, u8);
133int cxgb3i_conn_init_pdu(struct iscsi_task *, unsigned int, unsigned int);
134int cxgb3i_conn_xmit_pdu(struct iscsi_task *);
135
136void cxgb3i_release_itt(struct iscsi_task *task, itt_t hdr_itt);
137int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt);
138
139#endif
diff --git a/drivers/scsi/cxgb3i/cxgb3i_ddp.c b/drivers/scsi/cxgb3i/cxgb3i_ddp.c
new file mode 100644
index 000000000000..1a41f04264f7
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i_ddp.c
@@ -0,0 +1,770 @@
1/*
2 * cxgb3i_ddp.c: Chelsio S3xx iSCSI DDP Manager.
3 *
4 * Copyright (c) 2008 Chelsio Communications, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 *
10 * Written by: Karen Xie (kxie@chelsio.com)
11 */
12
13#include <linux/skbuff.h>
14
15/* from cxgb3 LLD */
16#include "common.h"
17#include "t3_cpl.h"
18#include "t3cdev.h"
19#include "cxgb3_ctl_defs.h"
20#include "cxgb3_offload.h"
21#include "firmware_exports.h"
22
23#include "cxgb3i_ddp.h"
24
25#define DRV_MODULE_NAME "cxgb3i_ddp"
26#define DRV_MODULE_VERSION "1.0.0"
27#define DRV_MODULE_RELDATE "Dec. 1, 2008"
28
29static char version[] =
30 "Chelsio S3xx iSCSI DDP " DRV_MODULE_NAME
31 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
32
33MODULE_AUTHOR("Karen Xie <kxie@chelsio.com>");
34MODULE_DESCRIPTION("cxgb3i ddp pagepod manager");
35MODULE_LICENSE("GPL");
36MODULE_VERSION(DRV_MODULE_VERSION);
37
38#define ddp_log_error(fmt...) printk(KERN_ERR "cxgb3i_ddp: ERR! " fmt)
39#define ddp_log_warn(fmt...) printk(KERN_WARNING "cxgb3i_ddp: WARN! " fmt)
40#define ddp_log_info(fmt...) printk(KERN_INFO "cxgb3i_ddp: " fmt)
41
42#ifdef __DEBUG_CXGB3I_DDP__
43#define ddp_log_debug(fmt, args...) \
44 printk(KERN_INFO "cxgb3i_ddp: %s - " fmt, __func__ , ## args)
45#else
46#define ddp_log_debug(fmt...)
47#endif
48
49/*
50 * iSCSI Direct Data Placement
51 *
52 * T3 h/w can directly place the iSCSI Data-In or Data-Out PDU's payload into
53 * pre-posted final destination host-memory buffers based on the Initiator
54 * Task Tag (ITT) in Data-In or Target Task Tag (TTT) in Data-Out PDUs.
55 *
56 * The host memory address is programmed into h/w in the format of pagepod
57 * entries.
58 * The location of the pagepod entry is encoded into ddp tag which is used or
59 * is the base for ITT/TTT.
60 */
61
62#define DDP_PGIDX_MAX 4
63#define DDP_THRESHOLD 2048
64static unsigned char ddp_page_order[DDP_PGIDX_MAX] = {0, 1, 2, 4};
65static unsigned char ddp_page_shift[DDP_PGIDX_MAX] = {12, 13, 14, 16};
66static unsigned char page_idx = DDP_PGIDX_MAX;
67
68static LIST_HEAD(cxgb3i_ddp_list);
69static DEFINE_RWLOCK(cxgb3i_ddp_rwlock);
70
71/*
72 * functions to program the pagepod in h/w
73 */
74static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
75{
76 struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
77
78 req->wr.wr_lo = 0;
79 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
80 req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
81 V_ULPTX_CMD(ULP_MEM_WRITE));
82 req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) |
83 V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1));
84}
85
86static int set_ddp_map(struct cxgb3i_ddp_info *ddp, struct pagepod_hdr *hdr,
87 unsigned int idx, unsigned int npods,
88 struct cxgb3i_gather_list *gl)
89{
90 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
91 int i;
92
93 for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
94 struct sk_buff *skb = ddp->gl_skb[idx];
95 struct pagepod *ppod;
96 int j, pidx;
97
98 /* hold on to the skb until we clear the ddp mapping */
99 skb_get(skb);
100
101 ulp_mem_io_set_hdr(skb, pm_addr);
102 ppod = (struct pagepod *)
103 (skb->head + sizeof(struct ulp_mem_io));
104 memcpy(&(ppod->hdr), hdr, sizeof(struct pagepod));
105 for (pidx = 4 * i, j = 0; j < 5; ++j, ++pidx)
106 ppod->addr[j] = pidx < gl->nelem ?
107 cpu_to_be64(gl->phys_addr[pidx]) : 0UL;
108
109 skb->priority = CPL_PRIORITY_CONTROL;
110 cxgb3_ofld_send(ddp->tdev, skb);
111 }
112 return 0;
113}
114
115static int clear_ddp_map(struct cxgb3i_ddp_info *ddp, unsigned int idx,
116 unsigned int npods)
117{
118 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
119 int i;
120
121 for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
122 struct sk_buff *skb = ddp->gl_skb[idx];
123
124 ddp->gl_skb[idx] = NULL;
125 memset((skb->head + sizeof(struct ulp_mem_io)), 0, PPOD_SIZE);
126 ulp_mem_io_set_hdr(skb, pm_addr);
127 skb->priority = CPL_PRIORITY_CONTROL;
128 cxgb3_ofld_send(ddp->tdev, skb);
129 }
130 return 0;
131}
132
133static inline int ddp_find_unused_entries(struct cxgb3i_ddp_info *ddp,
134 int start, int max, int count,
135 struct cxgb3i_gather_list *gl)
136{
137 unsigned int i, j;
138
139 spin_lock(&ddp->map_lock);
140 for (i = start; i <= max;) {
141 for (j = 0; j < count; j++) {
142 if (ddp->gl_map[i + j])
143 break;
144 }
145 if (j == count) {
146 for (j = 0; j < count; j++)
147 ddp->gl_map[i + j] = gl;
148 spin_unlock(&ddp->map_lock);
149 return i;
150 }
151 i += j + 1;
152 }
153 spin_unlock(&ddp->map_lock);
154 return -EBUSY;
155}
156
157static inline void ddp_unmark_entries(struct cxgb3i_ddp_info *ddp,
158 int start, int count)
159{
160 spin_lock(&ddp->map_lock);
161 memset(&ddp->gl_map[start], 0,
162 count * sizeof(struct cxgb3i_gather_list *));
163 spin_unlock(&ddp->map_lock);
164}
165
166static inline void ddp_free_gl_skb(struct cxgb3i_ddp_info *ddp,
167 int idx, int count)
168{
169 int i;
170
171 for (i = 0; i < count; i++, idx++)
172 if (ddp->gl_skb[idx]) {
173 kfree_skb(ddp->gl_skb[idx]);
174 ddp->gl_skb[idx] = NULL;
175 }
176}
177
178static inline int ddp_alloc_gl_skb(struct cxgb3i_ddp_info *ddp, int idx,
179 int count, gfp_t gfp)
180{
181 int i;
182
183 for (i = 0; i < count; i++) {
184 struct sk_buff *skb = alloc_skb(sizeof(struct ulp_mem_io) +
185 PPOD_SIZE, gfp);
186 if (skb) {
187 ddp->gl_skb[idx + i] = skb;
188 skb_put(skb, sizeof(struct ulp_mem_io) + PPOD_SIZE);
189 } else {
190 ddp_free_gl_skb(ddp, idx, i);
191 return -ENOMEM;
192 }
193 }
194 return 0;
195}
196
197/**
198 * cxgb3i_ddp_find_page_index - return ddp page index for a given page size.
199 * @pgsz: page size
200 * return the ddp page index, if no match is found return DDP_PGIDX_MAX.
201 */
202int cxgb3i_ddp_find_page_index(unsigned long pgsz)
203{
204 int i;
205
206 for (i = 0; i < DDP_PGIDX_MAX; i++) {
207 if (pgsz == (1UL << ddp_page_shift[i]))
208 return i;
209 }
210 ddp_log_debug("ddp page size 0x%lx not supported.\n", pgsz);
211 return DDP_PGIDX_MAX;
212}
213EXPORT_SYMBOL_GPL(cxgb3i_ddp_find_page_index);
214
215static inline void ddp_gl_unmap(struct pci_dev *pdev,
216 struct cxgb3i_gather_list *gl)
217{
218 int i;
219
220 for (i = 0; i < gl->nelem; i++)
221 pci_unmap_page(pdev, gl->phys_addr[i], PAGE_SIZE,
222 PCI_DMA_FROMDEVICE);
223}
224
225static inline int ddp_gl_map(struct pci_dev *pdev,
226 struct cxgb3i_gather_list *gl)
227{
228 int i;
229
230 for (i = 0; i < gl->nelem; i++) {
231 gl->phys_addr[i] = pci_map_page(pdev, gl->pages[i], 0,
232 PAGE_SIZE,
233 PCI_DMA_FROMDEVICE);
234 if (unlikely(pci_dma_mapping_error(pdev, gl->phys_addr[i])))
235 goto unmap;
236 }
237
238 return i;
239
240unmap:
241 if (i) {
242 unsigned int nelem = gl->nelem;
243
244 gl->nelem = i;
245 ddp_gl_unmap(pdev, gl);
246 gl->nelem = nelem;
247 }
248 return -ENOMEM;
249}
250
251/**
252 * cxgb3i_ddp_make_gl - build ddp page buffer list
253 * @xferlen: total buffer length
254 * @sgl: page buffer scatter-gather list
255 * @sgcnt: # of page buffers
256 * @pdev: pci_dev, used for pci map
257 * @gfp: allocation mode
258 *
259 * construct a ddp page buffer list from the scsi scattergather list.
260 * coalesce buffers as much as possible, and obtain dma addresses for
261 * each page.
262 *
263 * Return the cxgb3i_gather_list constructed from the page buffers if the
264 * memory can be used for ddp. Return NULL otherwise.
265 */
266struct cxgb3i_gather_list *cxgb3i_ddp_make_gl(unsigned int xferlen,
267 struct scatterlist *sgl,
268 unsigned int sgcnt,
269 struct pci_dev *pdev,
270 gfp_t gfp)
271{
272 struct cxgb3i_gather_list *gl;
273 struct scatterlist *sg = sgl;
274 struct page *sgpage = sg_page(sg);
275 unsigned int sglen = sg->length;
276 unsigned int sgoffset = sg->offset;
277 unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >>
278 PAGE_SHIFT;
279 int i = 1, j = 0;
280
281 if (xferlen < DDP_THRESHOLD) {
282 ddp_log_debug("xfer %u < threshold %u, no ddp.\n",
283 xferlen, DDP_THRESHOLD);
284 return NULL;
285 }
286
287 gl = kzalloc(sizeof(struct cxgb3i_gather_list) +
288 npages * (sizeof(dma_addr_t) + sizeof(struct page *)),
289 gfp);
290 if (!gl)
291 return NULL;
292
293 gl->pages = (struct page **)&gl->phys_addr[npages];
294 gl->length = xferlen;
295 gl->offset = sgoffset;
296 gl->pages[0] = sgpage;
297
298 sg = sg_next(sg);
299 while (sg) {
300 struct page *page = sg_page(sg);
301
302 if (sgpage == page && sg->offset == sgoffset + sglen)
303 sglen += sg->length;
304 else {
305 /* make sure the sgl is fit for ddp:
306 * each has the same page size, and
307 * all of the middle pages are used completely
308 */
309 if ((j && sgoffset) ||
310 ((i != sgcnt - 1) &&
311 ((sglen + sgoffset) & ~PAGE_MASK)))
312 goto error_out;
313
314 j++;
315 if (j == gl->nelem || sg->offset)
316 goto error_out;
317 gl->pages[j] = page;
318 sglen = sg->length;
319 sgoffset = sg->offset;
320 sgpage = page;
321 }
322 i++;
323 sg = sg_next(sg);
324 }
325 gl->nelem = ++j;
326
327 if (ddp_gl_map(pdev, gl) < 0)
328 goto error_out;
329
330 return gl;
331
332error_out:
333 kfree(gl);
334 return NULL;
335}
336EXPORT_SYMBOL_GPL(cxgb3i_ddp_make_gl);
337
338/**
339 * cxgb3i_ddp_release_gl - release a page buffer list
340 * @gl: a ddp page buffer list
341 * @pdev: pci_dev used for pci_unmap
342 * free a ddp page buffer list resulted from cxgb3i_ddp_make_gl().
343 */
344void cxgb3i_ddp_release_gl(struct cxgb3i_gather_list *gl,
345 struct pci_dev *pdev)
346{
347 ddp_gl_unmap(pdev, gl);
348 kfree(gl);
349}
350EXPORT_SYMBOL_GPL(cxgb3i_ddp_release_gl);
351
352/**
353 * cxgb3i_ddp_tag_reserve - set up ddp for a data transfer
354 * @tdev: t3cdev adapter
355 * @tid: connection id
356 * @tformat: tag format
357 * @tagp: the s/w tag, if ddp setup is successful, it will be updated with
358 * ddp/hw tag
359 * @gl: the page momory list
360 * @gfp: allocation mode
361 *
362 * ddp setup for a given page buffer list and construct the ddp tag.
363 * return 0 if success, < 0 otherwise.
364 */
365int cxgb3i_ddp_tag_reserve(struct t3cdev *tdev, unsigned int tid,
366 struct cxgb3i_tag_format *tformat, u32 *tagp,
367 struct cxgb3i_gather_list *gl, gfp_t gfp)
368{
369 struct cxgb3i_ddp_info *ddp = tdev->ulp_iscsi;
370 struct pagepod_hdr hdr;
371 unsigned int npods;
372 int idx = -1, idx_max;
373 int err = -ENOMEM;
374 u32 sw_tag = *tagp;
375 u32 tag;
376
377 if (page_idx >= DDP_PGIDX_MAX || !ddp || !gl || !gl->nelem ||
378 gl->length < DDP_THRESHOLD) {
379 ddp_log_debug("pgidx %u, xfer %u/%u, NO ddp.\n",
380 page_idx, gl->length, DDP_THRESHOLD);
381 return -EINVAL;
382 }
383
384 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
385 idx_max = ddp->nppods - npods + 1;
386
387 if (ddp->idx_last == ddp->nppods)
388 idx = ddp_find_unused_entries(ddp, 0, idx_max, npods, gl);
389 else {
390 idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1,
391 idx_max, npods, gl);
392 if (idx < 0 && ddp->idx_last >= npods)
393 idx = ddp_find_unused_entries(ddp, 0,
394 ddp->idx_last - npods + 1,
395 npods, gl);
396 }
397 if (idx < 0) {
398 ddp_log_debug("xferlen %u, gl %u, npods %u NO DDP.\n",
399 gl->length, gl->nelem, npods);
400 return idx;
401 }
402
403 err = ddp_alloc_gl_skb(ddp, idx, npods, gfp);
404 if (err < 0)
405 goto unmark_entries;
406
407 tag = cxgb3i_ddp_tag_base(tformat, sw_tag);
408 tag |= idx << PPOD_IDX_SHIFT;
409
410 hdr.rsvd = 0;
411 hdr.vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
412 hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask);
413 hdr.maxoffset = htonl(gl->length);
414 hdr.pgoffset = htonl(gl->offset);
415
416 err = set_ddp_map(ddp, &hdr, idx, npods, gl);
417 if (err < 0)
418 goto free_gl_skb;
419
420 ddp->idx_last = idx;
421 ddp_log_debug("xfer %u, gl %u,%u, tid 0x%x, 0x%x -> 0x%x(%u,%u).\n",
422 gl->length, gl->nelem, gl->offset, tid, sw_tag, tag,
423 idx, npods);
424 *tagp = tag;
425 return 0;
426
427free_gl_skb:
428 ddp_free_gl_skb(ddp, idx, npods);
429unmark_entries:
430 ddp_unmark_entries(ddp, idx, npods);
431 return err;
432}
433EXPORT_SYMBOL_GPL(cxgb3i_ddp_tag_reserve);
434
435/**
436 * cxgb3i_ddp_tag_release - release a ddp tag
437 * @tdev: t3cdev adapter
438 * @tag: ddp tag
439 * ddp cleanup for a given ddp tag and release all the resources held
440 */
441void cxgb3i_ddp_tag_release(struct t3cdev *tdev, u32 tag)
442{
443 struct cxgb3i_ddp_info *ddp = tdev->ulp_iscsi;
444 u32 idx;
445
446 if (!ddp) {
447 ddp_log_error("release ddp tag 0x%x, ddp NULL.\n", tag);
448 return;
449 }
450
451 idx = (tag >> PPOD_IDX_SHIFT) & ddp->idx_mask;
452 if (idx < ddp->nppods) {
453 struct cxgb3i_gather_list *gl = ddp->gl_map[idx];
454 unsigned int npods;
455
456 if (!gl) {
457 ddp_log_error("release ddp 0x%x, idx 0x%x, gl NULL.\n",
458 tag, idx);
459 return;
460 }
461 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
462 ddp_log_debug("ddp tag 0x%x, release idx 0x%x, npods %u.\n",
463 tag, idx, npods);
464 clear_ddp_map(ddp, idx, npods);
465 ddp_unmark_entries(ddp, idx, npods);
466 cxgb3i_ddp_release_gl(gl, ddp->pdev);
467 } else
468 ddp_log_error("ddp tag 0x%x, idx 0x%x > max 0x%x.\n",
469 tag, idx, ddp->nppods);
470}
471EXPORT_SYMBOL_GPL(cxgb3i_ddp_tag_release);
472
473static int setup_conn_pgidx(struct t3cdev *tdev, unsigned int tid, int pg_idx,
474 int reply)
475{
476 struct sk_buff *skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
477 GFP_KERNEL);
478 struct cpl_set_tcb_field *req;
479 u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;
480
481 if (!skb)
482 return -ENOMEM;
483
484 /* set up ulp submode and page size */
485 req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
486 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
487 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
488 req->reply = V_NO_REPLY(reply ? 0 : 1);
489 req->cpu_idx = 0;
490 req->word = htons(31);
491 req->mask = cpu_to_be64(0xF0000000);
492 req->val = cpu_to_be64(val << 28);
493 skb->priority = CPL_PRIORITY_CONTROL;
494
495 cxgb3_ofld_send(tdev, skb);
496 return 0;
497}
498
499/**
500 * cxgb3i_setup_conn_host_pagesize - setup the conn.'s ddp page size
501 * @tdev: t3cdev adapter
502 * @tid: connection id
503 * @reply: request reply from h/w
504 * set up the ddp page size based on the host PAGE_SIZE for a connection
505 * identified by tid
506 */
507int cxgb3i_setup_conn_host_pagesize(struct t3cdev *tdev, unsigned int tid,
508 int reply)
509{
510 return setup_conn_pgidx(tdev, tid, page_idx, reply);
511}
512EXPORT_SYMBOL_GPL(cxgb3i_setup_conn_host_pagesize);
513
514/**
515 * cxgb3i_setup_conn_pagesize - setup the conn.'s ddp page size
516 * @tdev: t3cdev adapter
517 * @tid: connection id
518 * @reply: request reply from h/w
519 * @pgsz: ddp page size
520 * set up the ddp page size for a connection identified by tid
521 */
522int cxgb3i_setup_conn_pagesize(struct t3cdev *tdev, unsigned int tid,
523 int reply, unsigned long pgsz)
524{
525 int pgidx = cxgb3i_ddp_find_page_index(pgsz);
526
527 return setup_conn_pgidx(tdev, tid, pgidx, reply);
528}
529EXPORT_SYMBOL_GPL(cxgb3i_setup_conn_pagesize);
530
531/**
532 * cxgb3i_setup_conn_digest - setup conn. digest setting
533 * @tdev: t3cdev adapter
534 * @tid: connection id
535 * @hcrc: header digest enabled
536 * @dcrc: data digest enabled
537 * @reply: request reply from h/w
538 * set up the iscsi digest settings for a connection identified by tid
539 */
540int cxgb3i_setup_conn_digest(struct t3cdev *tdev, unsigned int tid,
541 int hcrc, int dcrc, int reply)
542{
543 struct sk_buff *skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
544 GFP_KERNEL);
545 struct cpl_set_tcb_field *req;
546 u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
547
548 if (!skb)
549 return -ENOMEM;
550
551 /* set up ulp submode and page size */
552 req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req));
553 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
554 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
555 req->reply = V_NO_REPLY(reply ? 0 : 1);
556 req->cpu_idx = 0;
557 req->word = htons(31);
558 req->mask = cpu_to_be64(0x0F000000);
559 req->val = cpu_to_be64(val << 24);
560 skb->priority = CPL_PRIORITY_CONTROL;
561
562 cxgb3_ofld_send(tdev, skb);
563 return 0;
564}
565EXPORT_SYMBOL_GPL(cxgb3i_setup_conn_digest);
566
567static int ddp_init(struct t3cdev *tdev)
568{
569 struct cxgb3i_ddp_info *ddp;
570 struct ulp_iscsi_info uinfo;
571 unsigned int ppmax, bits;
572 int i, err;
573 static int vers_printed;
574
575 if (!vers_printed) {
576 printk(KERN_INFO "%s", version);
577 vers_printed = 1;
578 }
579
580 err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
581 if (err < 0) {
582 ddp_log_error("%s, failed to get iscsi param err=%d.\n",
583 tdev->name, err);
584 return err;
585 }
586
587 ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
588 bits = __ilog2_u32(ppmax) + 1;
589 if (bits > PPOD_IDX_MAX_SIZE)
590 bits = PPOD_IDX_MAX_SIZE;
591 ppmax = (1 << (bits - 1)) - 1;
592
593 ddp = cxgb3i_alloc_big_mem(sizeof(struct cxgb3i_ddp_info) +
594 ppmax *
595 (sizeof(struct cxgb3i_gather_list *) +
596 sizeof(struct sk_buff *)),
597 GFP_KERNEL);
598 if (!ddp) {
599 ddp_log_warn("%s unable to alloc ddp 0x%d, ddp disabled.\n",
600 tdev->name, ppmax);
601 return 0;
602 }
603 ddp->gl_map = (struct cxgb3i_gather_list **)(ddp + 1);
604 ddp->gl_skb = (struct sk_buff **)(((char *)ddp->gl_map) +
605 ppmax *
606 sizeof(struct cxgb3i_gather_list *));
607 spin_lock_init(&ddp->map_lock);
608
609 ddp->tdev = tdev;
610 ddp->pdev = uinfo.pdev;
611 ddp->max_txsz = min_t(unsigned int, uinfo.max_txsz, ULP2_MAX_PKT_SIZE);
612 ddp->max_rxsz = min_t(unsigned int, uinfo.max_rxsz, ULP2_MAX_PKT_SIZE);
613 ddp->llimit = uinfo.llimit;
614 ddp->ulimit = uinfo.ulimit;
615 ddp->nppods = ppmax;
616 ddp->idx_last = ppmax;
617 ddp->idx_bits = bits;
618 ddp->idx_mask = (1 << bits) - 1;
619 ddp->rsvd_tag_mask = (1 << (bits + PPOD_IDX_SHIFT)) - 1;
620
621 uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
622 for (i = 0; i < DDP_PGIDX_MAX; i++)
623 uinfo.pgsz_factor[i] = ddp_page_order[i];
624 uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
625
626 err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
627 if (err < 0) {
628 ddp_log_warn("%s unable to set iscsi param err=%d, "
629 "ddp disabled.\n", tdev->name, err);
630 goto free_ddp_map;
631 }
632
633 tdev->ulp_iscsi = ddp;
634
635 /* add to the list */
636 write_lock(&cxgb3i_ddp_rwlock);
637 list_add_tail(&ddp->list, &cxgb3i_ddp_list);
638 write_unlock(&cxgb3i_ddp_rwlock);
639
640 ddp_log_info("nppods %u (0x%x ~ 0x%x), bits %u, mask 0x%x,0x%x "
641 "pkt %u,%u.\n",
642 ppmax, ddp->llimit, ddp->ulimit, ddp->idx_bits,
643 ddp->idx_mask, ddp->rsvd_tag_mask,
644 ddp->max_txsz, ddp->max_rxsz);
645 return 0;
646
647free_ddp_map:
648 cxgb3i_free_big_mem(ddp);
649 return err;
650}
651
652/**
653 * cxgb3i_adapter_ddp_init - initialize the adapter's ddp resource
654 * @tdev: t3cdev adapter
655 * @tformat: tag format
656 * @txsz: max tx pkt size, filled in by this func.
657 * @rxsz: max rx pkt size, filled in by this func.
658 * initialize the ddp pagepod manager for a given adapter if needed and
659 * setup the tag format for a given iscsi entity
660 */
661int cxgb3i_adapter_ddp_init(struct t3cdev *tdev,
662 struct cxgb3i_tag_format *tformat,
663 unsigned int *txsz, unsigned int *rxsz)
664{
665 struct cxgb3i_ddp_info *ddp;
666 unsigned char idx_bits;
667
668 if (!tformat)
669 return -EINVAL;
670
671 if (!tdev->ulp_iscsi) {
672 int err = ddp_init(tdev);
673 if (err < 0)
674 return err;
675 }
676 ddp = (struct cxgb3i_ddp_info *)tdev->ulp_iscsi;
677
678 idx_bits = 32 - tformat->sw_bits;
679 tformat->rsvd_bits = ddp->idx_bits;
680 tformat->rsvd_shift = PPOD_IDX_SHIFT;
681 tformat->rsvd_mask = (1 << tformat->rsvd_bits) - 1;
682
683 ddp_log_info("tag format: sw %u, rsvd %u,%u, mask 0x%x.\n",
684 tformat->sw_bits, tformat->rsvd_bits,
685 tformat->rsvd_shift, tformat->rsvd_mask);
686
687 *txsz = ddp->max_txsz;
688 *rxsz = ddp->max_rxsz;
689 ddp_log_info("ddp max pkt size: %u, %u.\n",
690 ddp->max_txsz, ddp->max_rxsz);
691 return 0;
692}
693EXPORT_SYMBOL_GPL(cxgb3i_adapter_ddp_init);
694
695static void ddp_release(struct cxgb3i_ddp_info *ddp)
696{
697 int i = 0;
698 struct t3cdev *tdev = ddp->tdev;
699
700 tdev->ulp_iscsi = NULL;
701 while (i < ddp->nppods) {
702 struct cxgb3i_gather_list *gl = ddp->gl_map[i];
703 if (gl) {
704 int npods = (gl->nelem + PPOD_PAGES_MAX - 1)
705 >> PPOD_PAGES_SHIFT;
706
707 kfree(gl);
708 ddp_free_gl_skb(ddp, i, npods);
709 } else
710 i++;
711 }
712 cxgb3i_free_big_mem(ddp);
713}
714
715/**
716 * cxgb3i_adapter_ddp_cleanup - release the adapter's ddp resource
717 * @tdev: t3cdev adapter
718 * release all the resource held by the ddp pagepod manager for a given
719 * adapter if needed
720 */
721void cxgb3i_adapter_ddp_cleanup(struct t3cdev *tdev)
722{
723 struct cxgb3i_ddp_info *ddp;
724
725 /* remove from the list */
726 write_lock(&cxgb3i_ddp_rwlock);
727 list_for_each_entry(ddp, &cxgb3i_ddp_list, list) {
728 if (ddp->tdev == tdev) {
729 list_del(&ddp->list);
730 break;
731 }
732 }
733 write_unlock(&cxgb3i_ddp_rwlock);
734
735 if (ddp)
736 ddp_release(ddp);
737}
738EXPORT_SYMBOL_GPL(cxgb3i_adapter_ddp_cleanup);
739
740/**
741 * cxgb3i_ddp_init_module - module init entry point
742 * initialize any driver wide global data structures
743 */
744static int __init cxgb3i_ddp_init_module(void)
745{
746 page_idx = cxgb3i_ddp_find_page_index(PAGE_SIZE);
747 ddp_log_info("system PAGE_SIZE %lu, ddp idx %u.\n",
748 PAGE_SIZE, page_idx);
749 return 0;
750}
751
752/**
753 * cxgb3i_ddp_exit_module - module cleanup/exit entry point
754 * go through the ddp list and release any resource held.
755 */
756static void __exit cxgb3i_ddp_exit_module(void)
757{
758 struct cxgb3i_ddp_info *ddp;
759
760 /* release all ddp manager if there is any */
761 write_lock(&cxgb3i_ddp_rwlock);
762 list_for_each_entry(ddp, &cxgb3i_ddp_list, list) {
763 list_del(&ddp->list);
764 ddp_release(ddp);
765 }
766 write_unlock(&cxgb3i_ddp_rwlock);
767}
768
769module_init(cxgb3i_ddp_init_module);
770module_exit(cxgb3i_ddp_exit_module);
diff --git a/drivers/scsi/cxgb3i/cxgb3i_ddp.h b/drivers/scsi/cxgb3i/cxgb3i_ddp.h
new file mode 100644
index 000000000000..5c7c4d95c493
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i_ddp.h
@@ -0,0 +1,306 @@
1/*
2 * cxgb3i_ddp.h: Chelsio S3xx iSCSI DDP Manager.
3 *
4 * Copyright (c) 2008 Chelsio Communications, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 *
10 * Written by: Karen Xie (kxie@chelsio.com)
11 */
12
13#ifndef __CXGB3I_ULP2_DDP_H__
14#define __CXGB3I_ULP2_DDP_H__
15
16/**
17 * struct cxgb3i_tag_format - cxgb3i ulp tag format for an iscsi entity
18 *
19 * @sw_bits: # of bits used by iscsi software layer
20 * @rsvd_bits: # of bits used by h/w
21 * @rsvd_shift: h/w bits shift left
22 * @rsvd_mask: reserved bit mask
23 */
24struct cxgb3i_tag_format {
25 unsigned char sw_bits;
26 unsigned char rsvd_bits;
27 unsigned char rsvd_shift;
28 unsigned char filler[1];
29 u32 rsvd_mask;
30};
31
32/**
33 * struct cxgb3i_gather_list - cxgb3i direct data placement memory
34 *
35 * @tag: ddp tag
36 * @length: total data buffer length
37 * @offset: initial offset to the 1st page
38 * @nelem: # of pages
39 * @pages: page pointers
40 * @phys_addr: physical address
41 */
42struct cxgb3i_gather_list {
43 u32 tag;
44 unsigned int length;
45 unsigned int offset;
46 unsigned int nelem;
47 struct page **pages;
48 dma_addr_t phys_addr[0];
49};
50
51/**
52 * struct cxgb3i_ddp_info - cxgb3i direct data placement for pdu payload
53 *
54 * @list: list head to link elements
55 * @tdev: pointer to t3cdev used by cxgb3 driver
56 * @max_txsz: max tx packet size for ddp
57 * @max_rxsz: max rx packet size for ddp
58 * @llimit: lower bound of the page pod memory
59 * @ulimit: upper bound of the page pod memory
60 * @nppods: # of page pod entries
61 * @idx_last: page pod entry last used
62 * @idx_bits: # of bits the pagepod index would take
63 * @idx_mask: pagepod index mask
64 * @rsvd_tag_mask: tag mask
65 * @map_lock: lock to synchonize access to the page pod map
66 * @gl_map: ddp memory gather list
67 * @gl_skb: skb used to program the pagepod
68 */
69struct cxgb3i_ddp_info {
70 struct list_head list;
71 struct t3cdev *tdev;
72 struct pci_dev *pdev;
73 unsigned int max_txsz;
74 unsigned int max_rxsz;
75 unsigned int llimit;
76 unsigned int ulimit;
77 unsigned int nppods;
78 unsigned int idx_last;
79 unsigned char idx_bits;
80 unsigned char filler[3];
81 u32 idx_mask;
82 u32 rsvd_tag_mask;
83 spinlock_t map_lock;
84 struct cxgb3i_gather_list **gl_map;
85 struct sk_buff **gl_skb;
86};
87
88#define ULP2_MAX_PKT_SIZE 16224
89#define ULP2_MAX_PDU_PAYLOAD (ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_MAX)
90#define PPOD_PAGES_MAX 4
91#define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */
92
93/*
94 * struct pagepod_hdr, pagepod - pagepod format
95 */
96struct pagepod_hdr {
97 u32 vld_tid;
98 u32 pgsz_tag_clr;
99 u32 maxoffset;
100 u32 pgoffset;
101 u64 rsvd;
102};
103
104struct pagepod {
105 struct pagepod_hdr hdr;
106 u64 addr[PPOD_PAGES_MAX + 1];
107};
108
109#define PPOD_SIZE sizeof(struct pagepod) /* 64 */
110#define PPOD_SIZE_SHIFT 6
111
112#define PPOD_COLOR_SHIFT 0
113#define PPOD_COLOR_SIZE 6
114#define PPOD_COLOR_MASK ((1 << PPOD_COLOR_SIZE) - 1)
115
116#define PPOD_IDX_SHIFT PPOD_COLOR_SIZE
117#define PPOD_IDX_MAX_SIZE 24
118
119#define S_PPOD_TID 0
120#define M_PPOD_TID 0xFFFFFF
121#define V_PPOD_TID(x) ((x) << S_PPOD_TID)
122
123#define S_PPOD_VALID 24
124#define V_PPOD_VALID(x) ((x) << S_PPOD_VALID)
125#define F_PPOD_VALID V_PPOD_VALID(1U)
126
127#define S_PPOD_COLOR 0
128#define M_PPOD_COLOR 0x3F
129#define V_PPOD_COLOR(x) ((x) << S_PPOD_COLOR)
130
131#define S_PPOD_TAG 6
132#define M_PPOD_TAG 0xFFFFFF
133#define V_PPOD_TAG(x) ((x) << S_PPOD_TAG)
134
135#define S_PPOD_PGSZ 30
136#define M_PPOD_PGSZ 0x3
137#define V_PPOD_PGSZ(x) ((x) << S_PPOD_PGSZ)
138
139/*
140 * large memory chunk allocation/release
141 * use vmalloc() if kmalloc() fails
142 */
143static inline void *cxgb3i_alloc_big_mem(unsigned int size,
144 gfp_t gfp)
145{
146 void *p = kmalloc(size, gfp);
147 if (!p)
148 p = vmalloc(size);
149 if (p)
150 memset(p, 0, size);
151 return p;
152}
153
154static inline void cxgb3i_free_big_mem(void *addr)
155{
156 if (is_vmalloc_addr(addr))
157 vfree(addr);
158 else
159 kfree(addr);
160}
161
162/*
163 * cxgb3i ddp tag are 32 bits, it consists of reserved bits used by h/w and
164 * non-reserved bits that can be used by the iscsi s/w.
165 * The reserved bits are identified by the rsvd_bits and rsvd_shift fields
166 * in struct cxgb3i_tag_format.
167 *
168 * The upper most reserved bit can be used to check if a tag is ddp tag or not:
169 * if the bit is 0, the tag is a valid ddp tag
170 */
171
172/**
173 * cxgb3i_is_ddp_tag - check if a given tag is a hw/ddp tag
174 * @tformat: tag format information
175 * @tag: tag to be checked
176 *
177 * return true if the tag is a ddp tag, false otherwise.
178 */
179static inline int cxgb3i_is_ddp_tag(struct cxgb3i_tag_format *tformat, u32 tag)
180{
181 return !(tag & (1 << (tformat->rsvd_bits + tformat->rsvd_shift - 1)));
182}
183
184/**
185 * cxgb3i_sw_tag_usable - check if a given s/w tag has enough bits left for
186 * the reserved/hw bits
187 * @tformat: tag format information
188 * @sw_tag: s/w tag to be checked
189 *
190 * return true if the tag is a ddp tag, false otherwise.
191 */
192static inline int cxgb3i_sw_tag_usable(struct cxgb3i_tag_format *tformat,
193 u32 sw_tag)
194{
195 sw_tag >>= (32 - tformat->rsvd_bits);
196 return !sw_tag;
197}
198
199/**
200 * cxgb3i_set_non_ddp_tag - mark a given s/w tag as an invalid ddp tag
201 * @tformat: tag format information
202 * @sw_tag: s/w tag to be checked
203 *
204 * insert 1 at the upper most reserved bit to mark it as an invalid ddp tag.
205 */
206static inline u32 cxgb3i_set_non_ddp_tag(struct cxgb3i_tag_format *tformat,
207 u32 sw_tag)
208{
209 unsigned char shift = tformat->rsvd_bits + tformat->rsvd_shift - 1;
210 u32 mask = (1 << shift) - 1;
211
212 if (sw_tag && (sw_tag & ~mask)) {
213 u32 v1 = sw_tag & ((1 << shift) - 1);
214 u32 v2 = (sw_tag >> (shift - 1)) << shift;
215
216 return v2 | v1 | 1 << shift;
217 }
218 return sw_tag | 1 << shift;
219}
220
221/**
222 * cxgb3i_ddp_tag_base - shift the s/w tag bits so that reserved bits are not
223 * used.
224 * @tformat: tag format information
225 * @sw_tag: s/w tag to be checked
226 */
227static inline u32 cxgb3i_ddp_tag_base(struct cxgb3i_tag_format *tformat,
228 u32 sw_tag)
229{
230 u32 mask = (1 << tformat->rsvd_shift) - 1;
231
232 if (sw_tag && (sw_tag & ~mask)) {
233 u32 v1 = sw_tag & mask;
234 u32 v2 = sw_tag >> tformat->rsvd_shift;
235
236 v2 <<= tformat->rsvd_shift + tformat->rsvd_bits;
237 return v2 | v1;
238 }
239 return sw_tag;
240}
241
242/**
243 * cxgb3i_tag_rsvd_bits - get the reserved bits used by the h/w
244 * @tformat: tag format information
245 * @tag: tag to be checked
246 *
247 * return the reserved bits in the tag
248 */
249static inline u32 cxgb3i_tag_rsvd_bits(struct cxgb3i_tag_format *tformat,
250 u32 tag)
251{
252 if (cxgb3i_is_ddp_tag(tformat, tag))
253 return (tag >> tformat->rsvd_shift) & tformat->rsvd_mask;
254 return 0;
255}
256
257/**
258 * cxgb3i_tag_nonrsvd_bits - get the non-reserved bits used by the s/w
259 * @tformat: tag format information
260 * @tag: tag to be checked
261 *
262 * return the non-reserved bits in the tag.
263 */
264static inline u32 cxgb3i_tag_nonrsvd_bits(struct cxgb3i_tag_format *tformat,
265 u32 tag)
266{
267 unsigned char shift = tformat->rsvd_bits + tformat->rsvd_shift - 1;
268 u32 v1, v2;
269
270 if (cxgb3i_is_ddp_tag(tformat, tag)) {
271 v1 = tag & ((1 << tformat->rsvd_shift) - 1);
272 v2 = (tag >> (shift + 1)) << tformat->rsvd_shift;
273 } else {
274 u32 mask = (1 << shift) - 1;
275
276 tag &= ~(1 << shift);
277 v1 = tag & mask;
278 v2 = (tag >> 1) & ~mask;
279 }
280 return v1 | v2;
281}
282
283int cxgb3i_ddp_tag_reserve(struct t3cdev *, unsigned int tid,
284 struct cxgb3i_tag_format *, u32 *tag,
285 struct cxgb3i_gather_list *, gfp_t gfp);
286void cxgb3i_ddp_tag_release(struct t3cdev *, u32 tag);
287
288struct cxgb3i_gather_list *cxgb3i_ddp_make_gl(unsigned int xferlen,
289 struct scatterlist *sgl,
290 unsigned int sgcnt,
291 struct pci_dev *pdev,
292 gfp_t gfp);
293void cxgb3i_ddp_release_gl(struct cxgb3i_gather_list *gl,
294 struct pci_dev *pdev);
295
296int cxgb3i_setup_conn_host_pagesize(struct t3cdev *, unsigned int tid,
297 int reply);
298int cxgb3i_setup_conn_pagesize(struct t3cdev *, unsigned int tid, int reply,
299 unsigned long pgsz);
300int cxgb3i_setup_conn_digest(struct t3cdev *, unsigned int tid,
301 int hcrc, int dcrc, int reply);
302int cxgb3i_ddp_find_page_index(unsigned long pgsz);
303int cxgb3i_adapter_ddp_init(struct t3cdev *, struct cxgb3i_tag_format *,
304 unsigned int *txsz, unsigned int *rxsz);
305void cxgb3i_adapter_ddp_cleanup(struct t3cdev *);
306#endif
diff --git a/drivers/scsi/cxgb3i/cxgb3i_init.c b/drivers/scsi/cxgb3i/cxgb3i_init.c
new file mode 100644
index 000000000000..091ecb4d9f3d
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i_init.c
@@ -0,0 +1,107 @@
1/* cxgb3i_init.c: Chelsio S3xx iSCSI driver.
2 *
3 * Copyright (c) 2008 Chelsio Communications, Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Written by: Karen Xie (kxie@chelsio.com)
10 */
11
12#include "cxgb3i.h"
13
14#define DRV_MODULE_NAME "cxgb3i"
15#define DRV_MODULE_VERSION "1.0.0"
16#define DRV_MODULE_RELDATE "Jun. 1, 2008"
17
18static char version[] =
19 "Chelsio S3xx iSCSI Driver " DRV_MODULE_NAME
20 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
21
22MODULE_AUTHOR("Karen Xie <kxie@chelsio.com>");
23MODULE_DESCRIPTION("Chelsio S3xx iSCSI Driver");
24MODULE_LICENSE("GPL");
25MODULE_VERSION(DRV_MODULE_VERSION);
26
27static void open_s3_dev(struct t3cdev *);
28static void close_s3_dev(struct t3cdev *);
29
30static cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS];
31static struct cxgb3_client t3c_client = {
32 .name = "iscsi_cxgb3",
33 .handlers = cxgb3i_cpl_handlers,
34 .add = open_s3_dev,
35 .remove = close_s3_dev,
36};
37
38/**
39 * open_s3_dev - register with cxgb3 LLD
40 * @t3dev: cxgb3 adapter instance
41 */
42static void open_s3_dev(struct t3cdev *t3dev)
43{
44 static int vers_printed;
45
46 if (!vers_printed) {
47 printk(KERN_INFO "%s", version);
48 vers_printed = 1;
49 }
50
51 cxgb3i_sdev_add(t3dev, &t3c_client);
52 cxgb3i_adapter_add(t3dev);
53}
54
55/**
56 * close_s3_dev - de-register with cxgb3 LLD
57 * @t3dev: cxgb3 adapter instance
58 */
59static void close_s3_dev(struct t3cdev *t3dev)
60{
61 cxgb3i_adapter_remove(t3dev);
62 cxgb3i_sdev_remove(t3dev);
63}
64
65/**
66 * cxgb3i_init_module - module init entry point
67 *
68 * initialize any driver wide global data structures and register itself
69 * with the cxgb3 module
70 */
71static int __init cxgb3i_init_module(void)
72{
73 int err;
74
75 err = cxgb3i_sdev_init(cxgb3i_cpl_handlers);
76 if (err < 0)
77 return err;
78
79 err = cxgb3i_iscsi_init();
80 if (err < 0)
81 return err;
82
83 err = cxgb3i_pdu_init();
84 if (err < 0)
85 return err;
86
87 cxgb3_register_client(&t3c_client);
88
89 return 0;
90}
91
92/**
93 * cxgb3i_exit_module - module cleanup/exit entry point
94 *
95 * go through the driver hba list and for each hba, release any resource held.
96 * and unregisters iscsi transport and the cxgb3 module
97 */
98static void __exit cxgb3i_exit_module(void)
99{
100 cxgb3_unregister_client(&t3c_client);
101 cxgb3i_pdu_cleanup();
102 cxgb3i_iscsi_cleanup();
103 cxgb3i_sdev_cleanup();
104}
105
106module_init(cxgb3i_init_module);
107module_exit(cxgb3i_exit_module);
diff --git a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
new file mode 100644
index 000000000000..d83464b9b3f9
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c
@@ -0,0 +1,951 @@
1/* cxgb3i_iscsi.c: Chelsio S3xx iSCSI driver.
2 *
3 * Copyright (c) 2008 Chelsio Communications, Inc.
4 * Copyright (c) 2008 Mike Christie
5 * Copyright (c) 2008 Red Hat, Inc. All rights reserved.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation.
10 *
11 * Written by: Karen Xie (kxie@chelsio.com)
12 */
13
14#include <linux/inet.h>
15#include <linux/crypto.h>
16#include <net/tcp.h>
17#include <scsi/scsi_cmnd.h>
18#include <scsi/scsi_device.h>
19#include <scsi/scsi_eh.h>
20#include <scsi/scsi_host.h>
21#include <scsi/scsi.h>
22#include <scsi/iscsi_proto.h>
23#include <scsi/libiscsi.h>
24#include <scsi/scsi_transport_iscsi.h>
25
26#include "cxgb3i.h"
27#include "cxgb3i_pdu.h"
28
29#ifdef __DEBUG_CXGB3I_TAG__
30#define cxgb3i_tag_debug cxgb3i_log_debug
31#else
32#define cxgb3i_tag_debug(fmt...)
33#endif
34
35#ifdef __DEBUG_CXGB3I_API__
36#define cxgb3i_api_debug cxgb3i_log_debug
37#else
38#define cxgb3i_api_debug(fmt...)
39#endif
40
41/*
42 * align pdu size to multiple of 512 for better performance
43 */
44#define align_pdu_size(n) do { n = (n) & (~511); } while (0)
45
46static struct scsi_transport_template *cxgb3i_scsi_transport;
47static struct scsi_host_template cxgb3i_host_template;
48static struct iscsi_transport cxgb3i_iscsi_transport;
49static unsigned char sw_tag_idx_bits;
50static unsigned char sw_tag_age_bits;
51
52static LIST_HEAD(cxgb3i_snic_list);
53static DEFINE_RWLOCK(cxgb3i_snic_rwlock);
54
55/**
56 * cxgb3i_adapter_add - init a s3 adapter structure and any h/w settings
57 * @t3dev: t3cdev adapter
58 * return the resulting cxgb3i_adapter struct
59 */
60struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *t3dev)
61{
62 struct cxgb3i_adapter *snic;
63 struct adapter *adapter = tdev2adap(t3dev);
64 int i;
65
66 snic = kzalloc(sizeof(*snic), GFP_KERNEL);
67 if (!snic) {
68 cxgb3i_api_debug("cxgb3 %s, OOM.\n", t3dev->name);
69 return NULL;
70 }
71 spin_lock_init(&snic->lock);
72
73 snic->tdev = t3dev;
74 snic->pdev = adapter->pdev;
75 snic->tag_format.sw_bits = sw_tag_idx_bits + sw_tag_age_bits;
76
77 if (cxgb3i_adapter_ddp_init(t3dev, &snic->tag_format,
78 &snic->tx_max_size,
79 &snic->rx_max_size) < 0)
80 goto free_snic;
81
82 for_each_port(adapter, i) {
83 snic->hba[i] = cxgb3i_hba_host_add(snic, adapter->port[i]);
84 if (!snic->hba[i])
85 goto ulp_cleanup;
86 }
87 snic->hba_cnt = adapter->params.nports;
88
89 /* add to the list */
90 write_lock(&cxgb3i_snic_rwlock);
91 list_add_tail(&snic->list_head, &cxgb3i_snic_list);
92 write_unlock(&cxgb3i_snic_rwlock);
93
94 return snic;
95
96ulp_cleanup:
97 cxgb3i_adapter_ddp_cleanup(t3dev);
98free_snic:
99 kfree(snic);
100 return NULL;
101}
102
103/**
104 * cxgb3i_adapter_remove - release all the resources held and cleanup any
105 * h/w settings
106 * @t3dev: t3cdev adapter
107 */
108void cxgb3i_adapter_remove(struct t3cdev *t3dev)
109{
110 int i;
111 struct cxgb3i_adapter *snic;
112
113 /* remove from the list */
114 write_lock(&cxgb3i_snic_rwlock);
115 list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
116 if (snic->tdev == t3dev) {
117 list_del(&snic->list_head);
118 break;
119 }
120 }
121 write_unlock(&cxgb3i_snic_rwlock);
122
123 if (snic) {
124 for (i = 0; i < snic->hba_cnt; i++) {
125 if (snic->hba[i]) {
126 cxgb3i_hba_host_remove(snic->hba[i]);
127 snic->hba[i] = NULL;
128 }
129 }
130
131 /* release ddp resources */
132 cxgb3i_adapter_ddp_cleanup(snic->tdev);
133 kfree(snic);
134 }
135}
136
137/**
138 * cxgb3i_hba_find_by_netdev - find the cxgb3i_hba structure with a given
139 * net_device
140 * @t3dev: t3cdev adapter
141 */
142struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *ndev)
143{
144 struct cxgb3i_adapter *snic;
145 int i;
146
147 read_lock(&cxgb3i_snic_rwlock);
148 list_for_each_entry(snic, &cxgb3i_snic_list, list_head) {
149 for (i = 0; i < snic->hba_cnt; i++) {
150 if (snic->hba[i]->ndev == ndev) {
151 read_unlock(&cxgb3i_snic_rwlock);
152 return snic->hba[i];
153 }
154 }
155 }
156 read_unlock(&cxgb3i_snic_rwlock);
157 return NULL;
158}
159
160/**
161 * cxgb3i_hba_host_add - register a new host with scsi/iscsi
162 * @snic: the cxgb3i adapter
163 * @ndev: associated net_device
164 */
165struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *snic,
166 struct net_device *ndev)
167{
168 struct cxgb3i_hba *hba;
169 struct Scsi_Host *shost;
170 int err;
171
172 shost = iscsi_host_alloc(&cxgb3i_host_template,
173 sizeof(struct cxgb3i_hba),
174 CXGB3I_SCSI_QDEPTH_DFLT);
175 if (!shost) {
176 cxgb3i_log_info("iscsi_host_alloc failed.\n");
177 return NULL;
178 }
179
180 shost->transportt = cxgb3i_scsi_transport;
181 shost->max_lun = CXGB3I_MAX_LUN;
182 shost->max_id = CXGB3I_MAX_TARGET;
183 shost->max_channel = 0;
184 shost->max_cmd_len = 16;
185
186 hba = iscsi_host_priv(shost);
187 hba->snic = snic;
188 hba->ndev = ndev;
189 hba->shost = shost;
190
191 pci_dev_get(snic->pdev);
192 err = iscsi_host_add(shost, &snic->pdev->dev);
193 if (err) {
194 cxgb3i_log_info("iscsi_host_add failed.\n");
195 goto pci_dev_put;
196 }
197
198 cxgb3i_api_debug("shost 0x%p, hba 0x%p, no %u.\n",
199 shost, hba, shost->host_no);
200
201 return hba;
202
203pci_dev_put:
204 pci_dev_put(snic->pdev);
205 scsi_host_put(shost);
206 return NULL;
207}
208
209/**
210 * cxgb3i_hba_host_remove - de-register the host with scsi/iscsi
211 * @hba: the cxgb3i hba
212 */
213void cxgb3i_hba_host_remove(struct cxgb3i_hba *hba)
214{
215 cxgb3i_api_debug("shost 0x%p, hba 0x%p, no %u.\n",
216 hba->shost, hba, hba->shost->host_no);
217 iscsi_host_remove(hba->shost);
218 pci_dev_put(hba->snic->pdev);
219 iscsi_host_free(hba->shost);
220}
221
222/**
223 * cxgb3i_ep_connect - establish TCP connection to target portal
224 * @dst_addr: target IP address
225 * @non_blocking: blocking or non-blocking call
226 *
227 * Initiates a TCP/IP connection to the dst_addr
228 */
229static struct iscsi_endpoint *cxgb3i_ep_connect(struct sockaddr *dst_addr,
230 int non_blocking)
231{
232 struct iscsi_endpoint *ep;
233 struct cxgb3i_endpoint *cep;
234 struct cxgb3i_hba *hba;
235 struct s3_conn *c3cn = NULL;
236 int err = 0;
237
238 c3cn = cxgb3i_c3cn_create();
239 if (!c3cn) {
240 cxgb3i_log_info("ep connect OOM.\n");
241 err = -ENOMEM;
242 goto release_conn;
243 }
244
245 err = cxgb3i_c3cn_connect(c3cn, (struct sockaddr_in *)dst_addr);
246 if (err < 0) {
247 cxgb3i_log_info("ep connect failed.\n");
248 goto release_conn;
249 }
250 hba = cxgb3i_hba_find_by_netdev(c3cn->dst_cache->dev);
251 if (!hba) {
252 err = -ENOSPC;
253 cxgb3i_log_info("NOT going through cxgbi device.\n");
254 goto release_conn;
255 }
256 if (c3cn_is_closing(c3cn)) {
257 err = -ENOSPC;
258 cxgb3i_log_info("ep connect unable to connect.\n");
259 goto release_conn;
260 }
261
262 ep = iscsi_create_endpoint(sizeof(*cep));
263 if (!ep) {
264 err = -ENOMEM;
265 cxgb3i_log_info("iscsi alloc ep, OOM.\n");
266 goto release_conn;
267 }
268 cep = ep->dd_data;
269 cep->c3cn = c3cn;
270 cep->hba = hba;
271
272 cxgb3i_api_debug("ep 0x%p, 0x%p, c3cn 0x%p, hba 0x%p.\n",
273 ep, cep, c3cn, hba);
274 return ep;
275
276release_conn:
277 cxgb3i_api_debug("conn 0x%p failed, release.\n", c3cn);
278 if (c3cn)
279 cxgb3i_c3cn_release(c3cn);
280 return ERR_PTR(err);
281}
282
283/**
284 * cxgb3i_ep_poll - polls for TCP connection establishement
285 * @ep: TCP connection (endpoint) handle
286 * @timeout_ms: timeout value in milli secs
287 *
288 * polls for TCP connect request to complete
289 */
290static int cxgb3i_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
291{
292 struct cxgb3i_endpoint *cep = ep->dd_data;
293 struct s3_conn *c3cn = cep->c3cn;
294
295 if (!c3cn_is_established(c3cn))
296 return 0;
297 cxgb3i_api_debug("ep 0x%p, c3cn 0x%p established.\n", ep, c3cn);
298 return 1;
299}
300
301/**
302 * cxgb3i_ep_disconnect - teardown TCP connection
303 * @ep: TCP connection (endpoint) handle
304 *
305 * teardown TCP connection
306 */
307static void cxgb3i_ep_disconnect(struct iscsi_endpoint *ep)
308{
309 struct cxgb3i_endpoint *cep = ep->dd_data;
310 struct cxgb3i_conn *cconn = cep->cconn;
311
312 cxgb3i_api_debug("ep 0x%p, cep 0x%p.\n", ep, cep);
313
314 if (cconn && cconn->conn) {
315 /*
316 * stop the xmit path so the xmit_pdu function is
317 * not being called
318 */
319 iscsi_suspend_tx(cconn->conn);
320
321 write_lock_bh(&cep->c3cn->callback_lock);
322 cep->c3cn->user_data = NULL;
323 cconn->cep = NULL;
324 write_unlock_bh(&cep->c3cn->callback_lock);
325 }
326
327 cxgb3i_api_debug("ep 0x%p, cep 0x%p, release c3cn 0x%p.\n",
328 ep, cep, cep->c3cn);
329 cxgb3i_c3cn_release(cep->c3cn);
330 iscsi_destroy_endpoint(ep);
331}
332
333/**
334 * cxgb3i_session_create - create a new iscsi session
335 * @cmds_max: max # of commands
336 * @qdepth: scsi queue depth
337 * @initial_cmdsn: initial iscsi CMDSN for this session
338 * @host_no: pointer to return host no
339 *
340 * Creates a new iSCSI session
341 */
342static struct iscsi_cls_session *
343cxgb3i_session_create(struct iscsi_endpoint *ep, u16 cmds_max, u16 qdepth,
344 u32 initial_cmdsn, u32 *host_no)
345{
346 struct cxgb3i_endpoint *cep;
347 struct cxgb3i_hba *hba;
348 struct Scsi_Host *shost;
349 struct iscsi_cls_session *cls_session;
350 struct iscsi_session *session;
351
352 if (!ep) {
353 cxgb3i_log_error("%s, missing endpoint.\n", __func__);
354 return NULL;
355 }
356
357 cep = ep->dd_data;
358 hba = cep->hba;
359 shost = hba->shost;
360 cxgb3i_api_debug("ep 0x%p, cep 0x%p, hba 0x%p.\n", ep, cep, hba);
361 BUG_ON(hba != iscsi_host_priv(shost));
362
363 *host_no = shost->host_no;
364
365 cls_session = iscsi_session_setup(&cxgb3i_iscsi_transport, shost,
366 cmds_max,
367 sizeof(struct iscsi_tcp_task),
368 initial_cmdsn, ISCSI_MAX_TARGET);
369 if (!cls_session)
370 return NULL;
371 session = cls_session->dd_data;
372 if (iscsi_tcp_r2tpool_alloc(session))
373 goto remove_session;
374
375 return cls_session;
376
377remove_session:
378 iscsi_session_teardown(cls_session);
379 return NULL;
380}
381
382/**
383 * cxgb3i_session_destroy - destroys iscsi session
384 * @cls_session: pointer to iscsi cls session
385 *
386 * Destroys an iSCSI session instance and releases its all resources held
387 */
388static void cxgb3i_session_destroy(struct iscsi_cls_session *cls_session)
389{
390 cxgb3i_api_debug("sess 0x%p.\n", cls_session);
391 iscsi_tcp_r2tpool_free(cls_session->dd_data);
392 iscsi_session_teardown(cls_session);
393}
394
395/**
396 * cxgb3i_conn_max_xmit_dlength -- check the max. xmit pdu segment size,
397 * reduce it to be within the hardware limit if needed
398 * @conn: iscsi connection
399 */
400static inline int cxgb3i_conn_max_xmit_dlength(struct iscsi_conn *conn)
401
402{
403 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
404 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
405 unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
406 cconn->hba->snic->tx_max_size -
407 ISCSI_PDU_NONPAYLOAD_MAX);
408
409 if (conn->max_xmit_dlength)
410 conn->max_xmit_dlength = min_t(unsigned int,
411 conn->max_xmit_dlength, max);
412 else
413 conn->max_xmit_dlength = max;
414 align_pdu_size(conn->max_xmit_dlength);
415 cxgb3i_log_info("conn 0x%p, max xmit %u.\n",
416 conn, conn->max_xmit_dlength);
417 return 0;
418}
419
420/**
421 * cxgb3i_conn_max_recv_dlength -- check the max. recv pdu segment size against
422 * the hardware limit
423 * @conn: iscsi connection
424 * return 0 if the value is valid, < 0 otherwise.
425 */
426static inline int cxgb3i_conn_max_recv_dlength(struct iscsi_conn *conn)
427{
428 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
429 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
430 unsigned int max = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
431 cconn->hba->snic->rx_max_size -
432 ISCSI_PDU_NONPAYLOAD_MAX);
433
434 align_pdu_size(max);
435 if (conn->max_recv_dlength) {
436 if (conn->max_recv_dlength > max) {
437 cxgb3i_log_error("MaxRecvDataSegmentLength %u too big."
438 " Need to be <= %u.\n",
439 conn->max_recv_dlength, max);
440 return -EINVAL;
441 }
442 conn->max_recv_dlength = min_t(unsigned int,
443 conn->max_recv_dlength, max);
444 align_pdu_size(conn->max_recv_dlength);
445 } else
446 conn->max_recv_dlength = max;
447 cxgb3i_api_debug("conn 0x%p, max recv %u.\n",
448 conn, conn->max_recv_dlength);
449 return 0;
450}
451
452/**
453 * cxgb3i_conn_create - create iscsi connection instance
454 * @cls_session: pointer to iscsi cls session
455 * @cid: iscsi cid
456 *
457 * Creates a new iSCSI connection instance for a given session
458 */
459static struct iscsi_cls_conn *cxgb3i_conn_create(struct iscsi_cls_session
460 *cls_session, u32 cid)
461{
462 struct iscsi_cls_conn *cls_conn;
463 struct iscsi_conn *conn;
464 struct iscsi_tcp_conn *tcp_conn;
465 struct cxgb3i_conn *cconn;
466
467 cxgb3i_api_debug("sess 0x%p, cid %u.\n", cls_session, cid);
468
469 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid);
470 if (!cls_conn)
471 return NULL;
472 conn = cls_conn->dd_data;
473 tcp_conn = conn->dd_data;
474 cconn = tcp_conn->dd_data;
475
476 cconn->conn = conn;
477 return cls_conn;
478}
479
480/**
481 * cxgb3i_conn_bind - binds iscsi sess, conn and endpoint together
482 * @cls_session: pointer to iscsi cls session
483 * @cls_conn: pointer to iscsi cls conn
484 * @transport_eph: 64-bit EP handle
485 * @is_leading: leading connection on this session?
486 *
487 * Binds together an iSCSI session, an iSCSI connection and a
488 * TCP connection. This routine returns error code if the TCP
489 * connection does not belong on the device iSCSI sess/conn is bound
490 */
491
492static int cxgb3i_conn_bind(struct iscsi_cls_session *cls_session,
493 struct iscsi_cls_conn *cls_conn,
494 u64 transport_eph, int is_leading)
495{
496 struct iscsi_conn *conn = cls_conn->dd_data;
497 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
498 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
499 struct cxgb3i_adapter *snic;
500 struct iscsi_endpoint *ep;
501 struct cxgb3i_endpoint *cep;
502 struct s3_conn *c3cn;
503 int err;
504
505 ep = iscsi_lookup_endpoint(transport_eph);
506 if (!ep)
507 return -EINVAL;
508
509 /* setup ddp pagesize */
510 cep = ep->dd_data;
511 c3cn = cep->c3cn;
512 snic = cep->hba->snic;
513 err = cxgb3i_setup_conn_host_pagesize(snic->tdev, c3cn->tid, 0);
514 if (err < 0)
515 return err;
516
517 cxgb3i_api_debug("ep 0x%p, cls sess 0x%p, cls conn 0x%p.\n",
518 ep, cls_session, cls_conn);
519
520 err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
521 if (err)
522 return -EINVAL;
523
524 /* calculate the tag idx bits needed for this conn based on cmds_max */
525 cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1;
526 cxgb3i_api_debug("session cmds_max 0x%x, bits %u.\n",
527 conn->session->cmds_max, cconn->task_idx_bits);
528
529 read_lock(&c3cn->callback_lock);
530 c3cn->user_data = conn;
531 cconn->hba = cep->hba;
532 cconn->cep = cep;
533 cep->cconn = cconn;
534 read_unlock(&c3cn->callback_lock);
535
536 cxgb3i_conn_max_xmit_dlength(conn);
537 cxgb3i_conn_max_recv_dlength(conn);
538
539 spin_lock_bh(&conn->session->lock);
540 sprintf(conn->portal_address, NIPQUAD_FMT,
541 NIPQUAD(c3cn->daddr.sin_addr.s_addr));
542 conn->portal_port = ntohs(c3cn->daddr.sin_port);
543 spin_unlock_bh(&conn->session->lock);
544
545 /* init recv engine */
546 iscsi_tcp_hdr_recv_prep(tcp_conn);
547
548 return 0;
549}
550
551/**
552 * cxgb3i_conn_get_param - return iscsi connection parameter to caller
553 * @cls_conn: pointer to iscsi cls conn
554 * @param: parameter type identifier
555 * @buf: buffer pointer
556 *
557 * returns iSCSI connection parameters
558 */
559static int cxgb3i_conn_get_param(struct iscsi_cls_conn *cls_conn,
560 enum iscsi_param param, char *buf)
561{
562 struct iscsi_conn *conn = cls_conn->dd_data;
563 int len;
564
565 cxgb3i_api_debug("cls_conn 0x%p, param %d.\n", cls_conn, param);
566
567 switch (param) {
568 case ISCSI_PARAM_CONN_PORT:
569 spin_lock_bh(&conn->session->lock);
570 len = sprintf(buf, "%hu\n", conn->portal_port);
571 spin_unlock_bh(&conn->session->lock);
572 break;
573 case ISCSI_PARAM_CONN_ADDRESS:
574 spin_lock_bh(&conn->session->lock);
575 len = sprintf(buf, "%s\n", conn->portal_address);
576 spin_unlock_bh(&conn->session->lock);
577 break;
578 default:
579 return iscsi_conn_get_param(cls_conn, param, buf);
580 }
581
582 return len;
583}
584
585/**
586 * cxgb3i_conn_set_param - set iscsi connection parameter
587 * @cls_conn: pointer to iscsi cls conn
588 * @param: parameter type identifier
589 * @buf: buffer pointer
590 * @buflen: buffer length
591 *
592 * set iSCSI connection parameters
593 */
594static int cxgb3i_conn_set_param(struct iscsi_cls_conn *cls_conn,
595 enum iscsi_param param, char *buf, int buflen)
596{
597 struct iscsi_conn *conn = cls_conn->dd_data;
598 struct iscsi_session *session = conn->session;
599 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
600 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
601 struct cxgb3i_adapter *snic = cconn->hba->snic;
602 struct s3_conn *c3cn = cconn->cep->c3cn;
603 int value, err = 0;
604
605 switch (param) {
606 case ISCSI_PARAM_HDRDGST_EN:
607 err = iscsi_set_param(cls_conn, param, buf, buflen);
608 if (!err && conn->hdrdgst_en)
609 err = cxgb3i_setup_conn_digest(snic->tdev, c3cn->tid,
610 conn->hdrdgst_en,
611 conn->datadgst_en, 0);
612 break;
613 case ISCSI_PARAM_DATADGST_EN:
614 err = iscsi_set_param(cls_conn, param, buf, buflen);
615 if (!err && conn->datadgst_en)
616 err = cxgb3i_setup_conn_digest(snic->tdev, c3cn->tid,
617 conn->hdrdgst_en,
618 conn->datadgst_en, 0);
619 break;
620 case ISCSI_PARAM_MAX_R2T:
621 sscanf(buf, "%d", &value);
622 if (value <= 0 || !is_power_of_2(value))
623 return -EINVAL;
624 if (session->max_r2t == value)
625 break;
626 iscsi_tcp_r2tpool_free(session);
627 err = iscsi_set_param(cls_conn, param, buf, buflen);
628 if (!err && iscsi_tcp_r2tpool_alloc(session))
629 return -ENOMEM;
630 case ISCSI_PARAM_MAX_RECV_DLENGTH:
631 err = iscsi_set_param(cls_conn, param, buf, buflen);
632 if (!err)
633 err = cxgb3i_conn_max_recv_dlength(conn);
634 break;
635 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
636 err = iscsi_set_param(cls_conn, param, buf, buflen);
637 if (!err)
638 err = cxgb3i_conn_max_xmit_dlength(conn);
639 break;
640 default:
641 return iscsi_set_param(cls_conn, param, buf, buflen);
642 }
643 return err;
644}
645
646/**
647 * cxgb3i_host_set_param - configure host (adapter) related parameters
648 * @shost: scsi host pointer
649 * @param: parameter type identifier
650 * @buf: buffer pointer
651 */
652static int cxgb3i_host_set_param(struct Scsi_Host *shost,
653 enum iscsi_host_param param,
654 char *buf, int buflen)
655{
656 struct cxgb3i_hba *hba = iscsi_host_priv(shost);
657
658 cxgb3i_api_debug("param %d, buf %s.\n", param, buf);
659
660 switch (param) {
661 case ISCSI_HOST_PARAM_IPADDRESS:
662 {
663 __be32 addr = in_aton(buf);
664 cxgb3i_set_private_ipv4addr(hba->ndev, addr);
665 return 0;
666 }
667 case ISCSI_HOST_PARAM_HWADDRESS:
668 case ISCSI_HOST_PARAM_NETDEV_NAME:
669 /* ignore */
670 return 0;
671 default:
672 return iscsi_host_set_param(shost, param, buf, buflen);
673 }
674}
675
676/**
677 * cxgb3i_host_get_param - returns host (adapter) related parameters
678 * @shost: scsi host pointer
679 * @param: parameter type identifier
680 * @buf: buffer pointer
681 */
682static int cxgb3i_host_get_param(struct Scsi_Host *shost,
683 enum iscsi_host_param param, char *buf)
684{
685 struct cxgb3i_hba *hba = iscsi_host_priv(shost);
686 int len = 0;
687
688 cxgb3i_api_debug("hba %s, param %d.\n", hba->ndev->name, param);
689
690 switch (param) {
691 case ISCSI_HOST_PARAM_HWADDRESS:
692 len = sysfs_format_mac(buf, hba->ndev->dev_addr, 6);
693 break;
694 case ISCSI_HOST_PARAM_NETDEV_NAME:
695 len = sprintf(buf, "%s\n", hba->ndev->name);
696 break;
697 case ISCSI_HOST_PARAM_IPADDRESS:
698 {
699 __be32 addr;
700
701 addr = cxgb3i_get_private_ipv4addr(hba->ndev);
702 len = sprintf(buf, NIPQUAD_FMT, NIPQUAD(addr));
703 break;
704 }
705 default:
706 return iscsi_host_get_param(shost, param, buf);
707 }
708 return len;
709}
710
711/**
712 * cxgb3i_conn_get_stats - returns iSCSI stats
713 * @cls_conn: pointer to iscsi cls conn
714 * @stats: pointer to iscsi statistic struct
715 */
716static void cxgb3i_conn_get_stats(struct iscsi_cls_conn *cls_conn,
717 struct iscsi_stats *stats)
718{
719 struct iscsi_conn *conn = cls_conn->dd_data;
720
721 stats->txdata_octets = conn->txdata_octets;
722 stats->rxdata_octets = conn->rxdata_octets;
723 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
724 stats->dataout_pdus = conn->dataout_pdus_cnt;
725 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
726 stats->datain_pdus = conn->datain_pdus_cnt;
727 stats->r2t_pdus = conn->r2t_pdus_cnt;
728 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
729 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
730 stats->digest_err = 0;
731 stats->timeout_err = 0;
732 stats->custom_length = 1;
733 strcpy(stats->custom[0].desc, "eh_abort_cnt");
734 stats->custom[0].value = conn->eh_abort_cnt;
735}
736
737/**
738 * cxgb3i_parse_itt - get the idx and age bits from a given tag
739 * @conn: iscsi connection
740 * @itt: itt tag
741 * @idx: task index, filled in by this function
742 * @age: session age, filled in by this function
743 */
744static void cxgb3i_parse_itt(struct iscsi_conn *conn, itt_t itt,
745 int *idx, int *age)
746{
747 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
748 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
749 struct cxgb3i_adapter *snic = cconn->hba->snic;
750 u32 tag = ntohl((__force u32) itt);
751 u32 sw_bits;
752
753 sw_bits = cxgb3i_tag_nonrsvd_bits(&snic->tag_format, tag);
754 if (idx)
755 *idx = sw_bits & ((1 << cconn->task_idx_bits) - 1);
756 if (age)
757 *age = (sw_bits >> cconn->task_idx_bits) & ISCSI_AGE_MASK;
758
759 cxgb3i_tag_debug("parse tag 0x%x/0x%x, sw 0x%x, itt 0x%x, age 0x%x.\n",
760 tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
761 age ? *age : 0xFF);
762}
763
764/**
765 * cxgb3i_reserve_itt - generate tag for a give task
766 * Try to set up ddp for a scsi read task.
767 * @task: iscsi task
768 * @hdr_itt: tag, filled in by this function
769 */
770int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
771{
772 struct scsi_cmnd *sc = task->sc;
773 struct iscsi_conn *conn = task->conn;
774 struct iscsi_session *sess = conn->session;
775 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
776 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
777 struct cxgb3i_adapter *snic = cconn->hba->snic;
778 struct cxgb3i_tag_format *tformat = &snic->tag_format;
779 u32 sw_tag = (sess->age << cconn->task_idx_bits) | task->itt;
780 u32 tag;
781 int err = -EINVAL;
782
783 if (sc &&
784 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) &&
785 cxgb3i_sw_tag_usable(tformat, sw_tag)) {
786 struct s3_conn *c3cn = cconn->cep->c3cn;
787 struct cxgb3i_gather_list *gl;
788
789 gl = cxgb3i_ddp_make_gl(scsi_in(sc)->length,
790 scsi_in(sc)->table.sgl,
791 scsi_in(sc)->table.nents,
792 snic->pdev,
793 GFP_ATOMIC);
794 if (gl) {
795 tag = sw_tag;
796 err = cxgb3i_ddp_tag_reserve(snic->tdev, c3cn->tid,
797 tformat, &tag,
798 gl, GFP_ATOMIC);
799 if (err < 0)
800 cxgb3i_ddp_release_gl(gl, snic->pdev);
801 }
802 }
803
804 if (err < 0)
805 tag = cxgb3i_set_non_ddp_tag(tformat, sw_tag);
806 /* the itt need to sent in big-endian order */
807 *hdr_itt = (__force itt_t)htonl(tag);
808
809 cxgb3i_tag_debug("new tag 0x%x/0x%x (itt 0x%x, age 0x%x).\n",
810 tag, *hdr_itt, task->itt, sess->age);
811 return 0;
812}
813
814/**
815 * cxgb3i_release_itt - release the tag for a given task
816 * if the tag is a ddp tag, release the ddp setup
817 * @task: iscsi task
818 * @hdr_itt: tag
819 */
820void cxgb3i_release_itt(struct iscsi_task *task, itt_t hdr_itt)
821{
822 struct scsi_cmnd *sc = task->sc;
823 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
824 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
825 struct cxgb3i_adapter *snic = cconn->hba->snic;
826 struct cxgb3i_tag_format *tformat = &snic->tag_format;
827 u32 tag = ntohl((__force u32)hdr_itt);
828
829 cxgb3i_tag_debug("release tag 0x%x.\n", tag);
830
831 if (sc &&
832 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) &&
833 cxgb3i_is_ddp_tag(tformat, tag))
834 cxgb3i_ddp_tag_release(snic->tdev, tag);
835}
836
837/**
838 * cxgb3i_host_template -- Scsi_Host_Template structure
839 * used when registering with the scsi mid layer
840 */
841static struct scsi_host_template cxgb3i_host_template = {
842 .module = THIS_MODULE,
843 .name = "Chelsio S3xx iSCSI Initiator",
844 .proc_name = "cxgb3i",
845 .queuecommand = iscsi_queuecommand,
846 .change_queue_depth = iscsi_change_queue_depth,
847 .can_queue = 128 * (ISCSI_DEF_XMIT_CMDS_MAX - 1),
848 .sg_tablesize = SG_ALL,
849 .max_sectors = 0xFFFF,
850 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
851 .eh_abort_handler = iscsi_eh_abort,
852 .eh_device_reset_handler = iscsi_eh_device_reset,
853 .eh_target_reset_handler = iscsi_eh_target_reset,
854 .use_clustering = DISABLE_CLUSTERING,
855 .this_id = -1,
856};
857
858static struct iscsi_transport cxgb3i_iscsi_transport = {
859 .owner = THIS_MODULE,
860 .name = "cxgb3i",
861 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
862 | CAP_DATADGST | CAP_DIGEST_OFFLOAD |
863 CAP_PADDING_OFFLOAD,
864 .param_mask = ISCSI_MAX_RECV_DLENGTH |
865 ISCSI_MAX_XMIT_DLENGTH |
866 ISCSI_HDRDGST_EN |
867 ISCSI_DATADGST_EN |
868 ISCSI_INITIAL_R2T_EN |
869 ISCSI_MAX_R2T |
870 ISCSI_IMM_DATA_EN |
871 ISCSI_FIRST_BURST |
872 ISCSI_MAX_BURST |
873 ISCSI_PDU_INORDER_EN |
874 ISCSI_DATASEQ_INORDER_EN |
875 ISCSI_ERL |
876 ISCSI_CONN_PORT |
877 ISCSI_CONN_ADDRESS |
878 ISCSI_EXP_STATSN |
879 ISCSI_PERSISTENT_PORT |
880 ISCSI_PERSISTENT_ADDRESS |
881 ISCSI_TARGET_NAME | ISCSI_TPGT |
882 ISCSI_USERNAME | ISCSI_PASSWORD |
883 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
884 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
885 ISCSI_LU_RESET_TMO |
886 ISCSI_PING_TMO | ISCSI_RECV_TMO |
887 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
888 .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
889 ISCSI_HOST_INITIATOR_NAME | ISCSI_HOST_NETDEV_NAME,
890 .get_host_param = cxgb3i_host_get_param,
891 .set_host_param = cxgb3i_host_set_param,
892 /* session management */
893 .create_session = cxgb3i_session_create,
894 .destroy_session = cxgb3i_session_destroy,
895 .get_session_param = iscsi_session_get_param,
896 /* connection management */
897 .create_conn = cxgb3i_conn_create,
898 .bind_conn = cxgb3i_conn_bind,
899 .destroy_conn = iscsi_tcp_conn_teardown,
900 .start_conn = iscsi_conn_start,
901 .stop_conn = iscsi_conn_stop,
902 .get_conn_param = cxgb3i_conn_get_param,
903 .set_param = cxgb3i_conn_set_param,
904 .get_stats = cxgb3i_conn_get_stats,
905 /* pdu xmit req. from user space */
906 .send_pdu = iscsi_conn_send_pdu,
907 /* task */
908 .init_task = iscsi_tcp_task_init,
909 .xmit_task = iscsi_tcp_task_xmit,
910 .cleanup_task = cxgb3i_conn_cleanup_task,
911
912 /* pdu */
913 .alloc_pdu = cxgb3i_conn_alloc_pdu,
914 .init_pdu = cxgb3i_conn_init_pdu,
915 .xmit_pdu = cxgb3i_conn_xmit_pdu,
916 .parse_pdu_itt = cxgb3i_parse_itt,
917
918 /* TCP connect/disconnect */
919 .ep_connect = cxgb3i_ep_connect,
920 .ep_poll = cxgb3i_ep_poll,
921 .ep_disconnect = cxgb3i_ep_disconnect,
922 /* Error recovery timeout call */
923 .session_recovery_timedout = iscsi_session_recovery_timedout,
924};
925
926int cxgb3i_iscsi_init(void)
927{
928 sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
929 sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
930 cxgb3i_log_info("tag itt 0x%x, %u bits, age 0x%x, %u bits.\n",
931 ISCSI_ITT_MASK, sw_tag_idx_bits,
932 ISCSI_AGE_MASK, sw_tag_age_bits);
933
934 cxgb3i_scsi_transport =
935 iscsi_register_transport(&cxgb3i_iscsi_transport);
936 if (!cxgb3i_scsi_transport) {
937 cxgb3i_log_error("Could not register cxgb3i transport.\n");
938 return -ENODEV;
939 }
940 cxgb3i_api_debug("cxgb3i transport 0x%p.\n", cxgb3i_scsi_transport);
941 return 0;
942}
943
944void cxgb3i_iscsi_cleanup(void)
945{
946 if (cxgb3i_scsi_transport) {
947 cxgb3i_api_debug("cxgb3i transport 0x%p.\n",
948 cxgb3i_scsi_transport);
949 iscsi_unregister_transport(&cxgb3i_iscsi_transport);
950 }
951}
diff --git a/drivers/scsi/cxgb3i/cxgb3i_offload.c b/drivers/scsi/cxgb3i/cxgb3i_offload.c
new file mode 100644
index 000000000000..5f16081b68d5
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c
@@ -0,0 +1,1810 @@
1/*
2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
3 *
4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
5 *
6 * This program is distributed in the hope that it will be useful, but WITHOUT
7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
9 * release for licensing terms and conditions.
10 *
11 * Written by: Dimitris Michailidis (dm@chelsio.com)
12 * Karen Xie (kxie@chelsio.com)
13 */
14
15#include <linux/if_vlan.h>
16#include <linux/version.h>
17
18#include "cxgb3_defs.h"
19#include "cxgb3_ctl_defs.h"
20#include "firmware_exports.h"
21#include "cxgb3i_offload.h"
22#include "cxgb3i_pdu.h"
23#include "cxgb3i_ddp.h"
24
25#ifdef __DEBUG_C3CN_CONN__
26#define c3cn_conn_debug cxgb3i_log_info
27#else
28#define c3cn_conn_debug(fmt...)
29#endif
30
31#ifdef __DEBUG_C3CN_TX__
32#define c3cn_tx_debug cxgb3i_log_debug
33#else
34#define c3cn_tx_debug(fmt...)
35#endif
36
37#ifdef __DEBUG_C3CN_RX__
38#define c3cn_rx_debug cxgb3i_log_debug
39#else
40#define c3cn_rx_debug(fmt...)
41#endif
42
43/*
44 * module parameters releated to offloaded iscsi connection
45 */
46static int cxgb3_rcv_win = 256 * 1024;
47module_param(cxgb3_rcv_win, int, 0644);
48MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
49
50static int cxgb3_snd_win = 64 * 1024;
51module_param(cxgb3_snd_win, int, 0644);
52MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=64KB)");
53
54static int cxgb3_rx_credit_thres = 10 * 1024;
55module_param(cxgb3_rx_credit_thres, int, 0644);
56MODULE_PARM_DESC(rx_credit_thres,
57 "RX credits return threshold in bytes (default=10KB)");
58
59static unsigned int cxgb3_max_connect = 8 * 1024;
60module_param(cxgb3_max_connect, uint, 0644);
61MODULE_PARM_DESC(cxgb3_max_connect, "Max. # of connections (default=8092)");
62
63static unsigned int cxgb3_sport_base = 20000;
64module_param(cxgb3_sport_base, uint, 0644);
65MODULE_PARM_DESC(cxgb3_sport_base, "starting port number (default=20000)");
66
67/*
68 * cxgb3i tcp connection data(per adapter) list
69 */
70static LIST_HEAD(cdata_list);
71static DEFINE_RWLOCK(cdata_rwlock);
72
73static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion);
74static void c3cn_release_offload_resources(struct s3_conn *c3cn);
75
76/*
77 * iscsi source port management
78 *
79 * Find a free source port in the port allocation map. We use a very simple
80 * rotor scheme to look for the next free port.
81 *
82 * If a source port has been specified make sure that it doesn't collide with
83 * our normal source port allocation map. If it's outside the range of our
84 * allocation/deallocation scheme just let them use it.
85 *
86 * If the source port is outside our allocation range, the caller is
87 * responsible for keeping track of their port usage.
88 */
89static int c3cn_get_port(struct s3_conn *c3cn, struct cxgb3i_sdev_data *cdata)
90{
91 unsigned int start;
92 int idx;
93
94 if (!cdata)
95 goto error_out;
96
97 if (c3cn->saddr.sin_port != 0) {
98 idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
99 if (idx < 0 || idx >= cxgb3_max_connect)
100 return 0;
101 if (!test_and_set_bit(idx, cdata->sport_map))
102 return -EADDRINUSE;
103 }
104
105 /* the sport_map_next may not be accurate but that is okay, sport_map
106 should be */
107 start = idx = cdata->sport_map_next;
108 do {
109 if (++idx >= cxgb3_max_connect)
110 idx = 0;
111 if (!(test_and_set_bit(idx, cdata->sport_map))) {
112 c3cn->saddr.sin_port = htons(cxgb3_sport_base + idx);
113 cdata->sport_map_next = idx;
114 c3cn_conn_debug("%s reserve port %u.\n",
115 cdata->cdev->name,
116 cxgb3_sport_base + idx);
117 return 0;
118 }
119 } while (idx != start);
120
121error_out:
122 return -EADDRNOTAVAIL;
123}
124
125static void c3cn_put_port(struct s3_conn *c3cn)
126{
127 struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(c3cn->cdev);
128
129 if (c3cn->saddr.sin_port) {
130 int idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
131
132 c3cn->saddr.sin_port = 0;
133 if (idx < 0 || idx >= cxgb3_max_connect)
134 return;
135 clear_bit(idx, cdata->sport_map);
136 c3cn_conn_debug("%s, release port %u.\n",
137 cdata->cdev->name, cxgb3_sport_base + idx);
138 }
139}
140
141static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
142{
143 __set_bit(flag, &c3cn->flags);
144 c3cn_conn_debug("c3cn 0x%p, set %d, s %u, f 0x%lx.\n",
145 c3cn, flag, c3cn->state, c3cn->flags);
146}
147
148static inline void c3cn_clear_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
149{
150 __clear_bit(flag, &c3cn->flags);
151 c3cn_conn_debug("c3cn 0x%p, clear %d, s %u, f 0x%lx.\n",
152 c3cn, flag, c3cn->state, c3cn->flags);
153}
154
155static inline int c3cn_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
156{
157 if (c3cn == NULL)
158 return 0;
159 return test_bit(flag, &c3cn->flags);
160}
161
162static void c3cn_set_state(struct s3_conn *c3cn, int state)
163{
164 c3cn_conn_debug("c3cn 0x%p state -> %u.\n", c3cn, state);
165 c3cn->state = state;
166}
167
168static inline void c3cn_hold(struct s3_conn *c3cn)
169{
170 atomic_inc(&c3cn->refcnt);
171}
172
173static inline void c3cn_put(struct s3_conn *c3cn)
174{
175 if (atomic_dec_and_test(&c3cn->refcnt)) {
176 c3cn_conn_debug("free c3cn 0x%p, s %u, f 0x%lx.\n",
177 c3cn, c3cn->state, c3cn->flags);
178 kfree(c3cn);
179 }
180}
181
182static void c3cn_closed(struct s3_conn *c3cn)
183{
184 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
185 c3cn, c3cn->state, c3cn->flags);
186
187 c3cn_put_port(c3cn);
188 c3cn_release_offload_resources(c3cn);
189 c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
190 cxgb3i_conn_closing(c3cn);
191}
192
193/*
194 * CPL (Chelsio Protocol Language) defines a message passing interface between
195 * the host driver and T3 asic.
196 * The section below implments CPLs that related to iscsi tcp connection
197 * open/close/abort and data send/receive.
198 */
199
200/*
201 * CPL connection active open request: host ->
202 */
203static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
204{
205 int i = 0;
206
207 while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
208 ++i;
209 return i;
210}
211
212static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
213{
214 unsigned int idx;
215 struct dst_entry *dst = c3cn->dst_cache;
216 struct t3cdev *cdev = c3cn->cdev;
217 const struct t3c_data *td = T3C_DATA(cdev);
218 u16 advmss = dst_metric(dst, RTAX_ADVMSS);
219
220 if (advmss > pmtu - 40)
221 advmss = pmtu - 40;
222 if (advmss < td->mtus[0] - 40)
223 advmss = td->mtus[0] - 40;
224 idx = find_best_mtu(td, advmss + 40);
225 return idx;
226}
227
228static inline int compute_wscale(int win)
229{
230 int wscale = 0;
231 while (wscale < 14 && (65535<<wscale) < win)
232 wscale++;
233 return wscale;
234}
235
236static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
237{
238 int wscale = compute_wscale(cxgb3_rcv_win);
239 return V_KEEP_ALIVE(1) |
240 F_TCAM_BYPASS |
241 V_WND_SCALE(wscale) |
242 V_MSS_IDX(c3cn->mss_idx);
243}
244
245static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
246{
247 return V_ULP_MODE(ULP_MODE_ISCSI) |
248 V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
249}
250
251static void make_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
252 unsigned int atid, const struct l2t_entry *e)
253{
254 struct cpl_act_open_req *req;
255
256 c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
257
258 skb->priority = CPL_PRIORITY_SETUP;
259 req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
260 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
261 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
262 req->local_port = c3cn->saddr.sin_port;
263 req->peer_port = c3cn->daddr.sin_port;
264 req->local_ip = c3cn->saddr.sin_addr.s_addr;
265 req->peer_ip = c3cn->daddr.sin_addr.s_addr;
266 req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
267 V_TX_CHANNEL(e->smt_idx));
268 req->opt0l = htonl(calc_opt0l(c3cn));
269 req->params = 0;
270}
271
272static void fail_act_open(struct s3_conn *c3cn, int errno)
273{
274 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
275 c3cn, c3cn->state, c3cn->flags);
276 c3cn->err = errno;
277 c3cn_closed(c3cn);
278}
279
280static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
281{
282 struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
283
284 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
285
286 c3cn_hold(c3cn);
287 spin_lock_bh(&c3cn->lock);
288 if (c3cn->state == C3CN_STATE_CONNECTING)
289 fail_act_open(c3cn, EHOSTUNREACH);
290 spin_unlock_bh(&c3cn->lock);
291 c3cn_put(c3cn);
292 __kfree_skb(skb);
293}
294
295/*
296 * CPL connection close request: host ->
297 *
298 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
299 * the write queue (i.e., after any unsent txt data).
300 */
301static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
302 int flags)
303{
304 CXGB3_SKB_CB(skb)->seq = c3cn->write_seq;
305 CXGB3_SKB_CB(skb)->flags = flags;
306 __skb_queue_tail(&c3cn->write_queue, skb);
307}
308
309static void send_close_req(struct s3_conn *c3cn)
310{
311 struct sk_buff *skb = c3cn->cpl_close;
312 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
313 unsigned int tid = c3cn->tid;
314
315 c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
316 c3cn, c3cn->state, c3cn->flags);
317
318 c3cn->cpl_close = NULL;
319
320 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
321 req->wr.wr_lo = htonl(V_WR_TID(tid));
322 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
323 req->rsvd = htonl(c3cn->write_seq);
324
325 skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
326 if (c3cn->state != C3CN_STATE_CONNECTING)
327 c3cn_push_tx_frames(c3cn, 1);
328}
329
330/*
331 * CPL connection abort request: host ->
332 *
333 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
334 * for the same connection and also that we do not try to send a message
335 * after the connection has closed.
336 */
337static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
338{
339 struct cpl_abort_req *req = cplhdr(skb);
340
341 c3cn_conn_debug("tdev 0x%p.\n", cdev);
342
343 req->cmd = CPL_ABORT_NO_RST;
344 cxgb3_ofld_send(cdev, skb);
345}
346
347static inline void c3cn_purge_write_queue(struct s3_conn *c3cn)
348{
349 struct sk_buff *skb;
350
351 while ((skb = __skb_dequeue(&c3cn->write_queue)))
352 __kfree_skb(skb);
353}
354
355static void send_abort_req(struct s3_conn *c3cn)
356{
357 struct sk_buff *skb = c3cn->cpl_abort_req;
358 struct cpl_abort_req *req;
359 unsigned int tid = c3cn->tid;
360
361 if (unlikely(c3cn->state == C3CN_STATE_ABORTING) || !skb ||
362 !c3cn->cdev)
363 return;
364
365 c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
366
367 c3cn_conn_debug("c3cn 0x%p, flag ABORT_RPL + ABORT_SHUT.\n", c3cn);
368
369 c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
370
371 /* Purge the send queue so we don't send anything after an abort. */
372 c3cn_purge_write_queue(c3cn);
373
374 c3cn->cpl_abort_req = NULL;
375 req = (struct cpl_abort_req *)skb->head;
376
377 skb->priority = CPL_PRIORITY_DATA;
378 set_arp_failure_handler(skb, abort_arp_failure);
379
380 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
381 req->wr.wr_lo = htonl(V_WR_TID(tid));
382 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
383 req->rsvd0 = htonl(c3cn->snd_nxt);
384 req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
385 req->cmd = CPL_ABORT_SEND_RST;
386
387 l2t_send(c3cn->cdev, skb, c3cn->l2t);
388}
389
390/*
391 * CPL connection abort reply: host ->
392 *
393 * Send an ABORT_RPL message in response of the ABORT_REQ received.
394 */
395static void send_abort_rpl(struct s3_conn *c3cn, int rst_status)
396{
397 struct sk_buff *skb = c3cn->cpl_abort_rpl;
398 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
399
400 c3cn->cpl_abort_rpl = NULL;
401
402 skb->priority = CPL_PRIORITY_DATA;
403 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
404 rpl->wr.wr_lo = htonl(V_WR_TID(c3cn->tid));
405 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, c3cn->tid));
406 rpl->cmd = rst_status;
407
408 cxgb3_ofld_send(c3cn->cdev, skb);
409}
410
411/*
412 * CPL connection rx data ack: host ->
413 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
414 * credits sent.
415 */
416static u32 send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack)
417{
418 struct sk_buff *skb;
419 struct cpl_rx_data_ack *req;
420
421 skb = alloc_skb(sizeof(*req), GFP_ATOMIC);
422 if (!skb)
423 return 0;
424
425 req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
426 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
427 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
428 req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
429 skb->priority = CPL_PRIORITY_ACK;
430 cxgb3_ofld_send(c3cn->cdev, skb);
431 return credits;
432}
433
434/*
435 * CPL connection tx data: host ->
436 *
437 * Send iscsi PDU via TX_DATA CPL message. Returns the number of
438 * credits sent.
439 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
440 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
441 */
442
443/*
444 * For ULP connections HW may inserts digest bytes into the pdu. Those digest
445 * bytes are not sent by the host but are part of the TCP payload and therefore
446 * consume TCP sequence space.
447 */
448static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
449static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
450{
451 return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
452}
453
454static unsigned int wrlen __read_mostly;
455
456/*
457 * The number of WRs needed for an skb depends on the number of fragments
458 * in the skb and whether it has any payload in its main body. This maps the
459 * length of the gather list represented by an skb into the # of necessary WRs.
460 *
461 * The max. length of an skb is controlled by the max pdu size which is ~16K.
462 * Also, assume the min. fragment length is the sector size (512), then add
463 * extra fragment counts for iscsi bhs and payload padding.
464 */
465#define SKB_WR_LIST_SIZE (16384/512 + 3)
466static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
467
468static void s3_init_wr_tab(unsigned int wr_len)
469{
470 int i;
471
472 if (skb_wrs[1]) /* already initialized */
473 return;
474
475 for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
476 int sgl_len = (3 * i) / 2 + (i & 1);
477
478 sgl_len += 3;
479 skb_wrs[i] = (sgl_len <= wr_len
480 ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
481 }
482
483 wrlen = wr_len * 8;
484}
485
486static inline void reset_wr_list(struct s3_conn *c3cn)
487{
488 c3cn->wr_pending_head = NULL;
489}
490
491/*
492 * Add a WR to a connections's list of pending WRs. This is a singly-linked
493 * list of sk_buffs operating as a FIFO. The head is kept in wr_pending_head
494 * and the tail in wr_pending_tail.
495 */
496static inline void enqueue_wr(struct s3_conn *c3cn,
497 struct sk_buff *skb)
498{
499 skb->sp = NULL;
500
501 /*
502 * We want to take an extra reference since both us and the driver
503 * need to free the packet before it's really freed. We know there's
504 * just one user currently so we use atomic_set rather than skb_get
505 * to avoid the atomic op.
506 */
507 atomic_set(&skb->users, 2);
508
509 if (!c3cn->wr_pending_head)
510 c3cn->wr_pending_head = skb;
511 else
512 c3cn->wr_pending_tail->sp = (void *)skb;
513 c3cn->wr_pending_tail = skb;
514}
515
516static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
517{
518 return c3cn->wr_pending_head;
519}
520
521static inline void free_wr_skb(struct sk_buff *skb)
522{
523 kfree_skb(skb);
524}
525
526static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
527{
528 struct sk_buff *skb = c3cn->wr_pending_head;
529
530 if (likely(skb)) {
531 /* Don't bother clearing the tail */
532 c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
533 skb->sp = NULL;
534 }
535 return skb;
536}
537
538static void purge_wr_queue(struct s3_conn *c3cn)
539{
540 struct sk_buff *skb;
541 while ((skb = dequeue_wr(c3cn)) != NULL)
542 free_wr_skb(skb);
543}
544
545static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb,
546 int len)
547{
548 struct tx_data_wr *req;
549
550 skb_reset_transport_header(skb);
551 req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
552 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
553 req->wr_lo = htonl(V_WR_TID(c3cn->tid));
554 req->sndseq = htonl(c3cn->snd_nxt);
555 /* len includes the length of any HW ULP additions */
556 req->len = htonl(len);
557 req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
558 /* V_TX_ULP_SUBMODE sets both the mode and submode */
559 req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
560 V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
561
562 if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
563 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
564 V_TX_CPU_IDX(c3cn->qset));
565 /* Sendbuffer is in units of 32KB. */
566 req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
567 c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
568 }
569}
570
571/**
572 * c3cn_push_tx_frames -- start transmit
573 * @c3cn: the offloaded connection
574 * @req_completion: request wr_ack or not
575 *
576 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
577 * connection's send queue and sends them on to T3. Must be called with the
578 * connection's lock held. Returns the amount of send buffer space that was
579 * freed as a result of sending queued data to T3.
580 */
581static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
582{
583 kfree_skb(skb);
584}
585
586static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion)
587{
588 int total_size = 0;
589 struct sk_buff *skb;
590 struct t3cdev *cdev;
591 struct cxgb3i_sdev_data *cdata;
592
593 if (unlikely(c3cn->state == C3CN_STATE_CONNECTING ||
594 c3cn->state == C3CN_STATE_CLOSE_WAIT_1 ||
595 c3cn->state == C3CN_STATE_ABORTING)) {
596 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
597 c3cn, c3cn->state);
598 return 0;
599 }
600
601 cdev = c3cn->cdev;
602 cdata = CXGB3_SDEV_DATA(cdev);
603
604 while (c3cn->wr_avail
605 && (skb = skb_peek(&c3cn->write_queue)) != NULL) {
606 int len = skb->len; /* length before skb_push */
607 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
608 int wrs_needed = skb_wrs[frags];
609
610 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
611 wrs_needed = 1;
612
613 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
614
615 if (c3cn->wr_avail < wrs_needed) {
616 c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, "
617 "wr %d < %u.\n",
618 c3cn, skb->len, skb->datalen, frags,
619 wrs_needed, c3cn->wr_avail);
620 break;
621 }
622
623 __skb_unlink(skb, &c3cn->write_queue);
624 skb->priority = CPL_PRIORITY_DATA;
625 skb->csum = wrs_needed; /* remember this until the WR_ACK */
626 c3cn->wr_avail -= wrs_needed;
627 c3cn->wr_unacked += wrs_needed;
628 enqueue_wr(c3cn, skb);
629
630 if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
631 len += ulp_extra_len(skb);
632 make_tx_data_wr(c3cn, skb, len);
633 c3cn->snd_nxt += len;
634 if ((req_completion
635 && c3cn->wr_unacked == wrs_needed)
636 || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
637 || c3cn->wr_unacked >= c3cn->wr_max / 2) {
638 struct work_request_hdr *wr = cplhdr(skb);
639
640 wr->wr_hi |= htonl(F_WR_COMPL);
641 c3cn->wr_unacked = 0;
642 }
643 CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
644 }
645
646 total_size += skb->truesize;
647 set_arp_failure_handler(skb, arp_failure_discard);
648 l2t_send(cdev, skb, c3cn->l2t);
649 }
650 return total_size;
651}
652
653/*
654 * process_cpl_msg: -> host
655 * Top-level CPL message processing used by most CPL messages that
656 * pertain to connections.
657 */
658static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
659 struct sk_buff *),
660 struct s3_conn *c3cn,
661 struct sk_buff *skb)
662{
663 spin_lock_bh(&c3cn->lock);
664 fn(c3cn, skb);
665 spin_unlock_bh(&c3cn->lock);
666}
667
668/*
669 * process_cpl_msg_ref: -> host
670 * Similar to process_cpl_msg() but takes an extra connection reference around
671 * the call to the handler. Should be used if the handler may drop a
672 * connection reference.
673 */
674static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
675 struct sk_buff *),
676 struct s3_conn *c3cn,
677 struct sk_buff *skb)
678{
679 c3cn_hold(c3cn);
680 process_cpl_msg(fn, c3cn, skb);
681 c3cn_put(c3cn);
682}
683
684/*
685 * Process a CPL_ACT_ESTABLISH message: -> host
686 * Updates connection state from an active establish CPL message. Runs with
687 * the connection lock held.
688 */
689
690static inline void s3_free_atid(struct t3cdev *cdev, unsigned int tid)
691{
692 struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
693 if (c3cn)
694 c3cn_put(c3cn);
695}
696
697static void c3cn_established(struct s3_conn *c3cn, u32 snd_isn,
698 unsigned int opt)
699{
700 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
701
702 c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
703
704 /*
705 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
706 * pass through opt0.
707 */
708 if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
709 c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
710
711 dst_confirm(c3cn->dst_cache);
712
713 smp_mb();
714
715 c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
716}
717
718static void process_act_establish(struct s3_conn *c3cn, struct sk_buff *skb)
719{
720 struct cpl_act_establish *req = cplhdr(skb);
721 u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
722
723 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
724 c3cn, c3cn->state, c3cn->flags);
725
726 if (unlikely(c3cn->state != C3CN_STATE_CONNECTING))
727 cxgb3i_log_error("TID %u expected SYN_SENT, got EST., s %u\n",
728 c3cn->tid, c3cn->state);
729
730 c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
731 c3cn_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
732
733 __kfree_skb(skb);
734
735 if (unlikely(c3cn_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED)))
736 /* upper layer has requested closing */
737 send_abort_req(c3cn);
738 else if (c3cn_push_tx_frames(c3cn, 1))
739 cxgb3i_conn_tx_open(c3cn);
740}
741
742static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
743 void *ctx)
744{
745 struct cpl_act_establish *req = cplhdr(skb);
746 unsigned int tid = GET_TID(req);
747 unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
748 struct s3_conn *c3cn = ctx;
749 struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
750
751 c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
752 tid, c3cn, c3cn->state, c3cn->flags);
753
754 c3cn->tid = tid;
755 c3cn_hold(c3cn);
756 cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
757 s3_free_atid(cdev, atid);
758
759 c3cn->qset = G_QNUM(ntohl(skb->csum));
760
761 process_cpl_msg(process_act_establish, c3cn, skb);
762 return 0;
763}
764
765/*
766 * Process a CPL_ACT_OPEN_RPL message: -> host
767 * Handle active open failures.
768 */
769static int act_open_rpl_status_to_errno(int status)
770{
771 switch (status) {
772 case CPL_ERR_CONN_RESET:
773 return ECONNREFUSED;
774 case CPL_ERR_ARP_MISS:
775 return EHOSTUNREACH;
776 case CPL_ERR_CONN_TIMEDOUT:
777 return ETIMEDOUT;
778 case CPL_ERR_TCAM_FULL:
779 return ENOMEM;
780 case CPL_ERR_CONN_EXIST:
781 cxgb3i_log_error("ACTIVE_OPEN_RPL: 4-tuple in use\n");
782 return EADDRINUSE;
783 default:
784 return EIO;
785 }
786}
787
788static void act_open_retry_timer(unsigned long data)
789{
790 struct sk_buff *skb;
791 struct s3_conn *c3cn = (struct s3_conn *)data;
792
793 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
794
795 spin_lock_bh(&c3cn->lock);
796 skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
797 if (!skb)
798 fail_act_open(c3cn, ENOMEM);
799 else {
800 skb->sk = (struct sock *)c3cn;
801 set_arp_failure_handler(skb, act_open_req_arp_failure);
802 make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
803 l2t_send(c3cn->cdev, skb, c3cn->l2t);
804 }
805 spin_unlock_bh(&c3cn->lock);
806 c3cn_put(c3cn);
807}
808
809static void process_act_open_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
810{
811 struct cpl_act_open_rpl *rpl = cplhdr(skb);
812
813 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
814 c3cn, c3cn->state, c3cn->flags);
815
816 if (rpl->status == CPL_ERR_CONN_EXIST &&
817 c3cn->retry_timer.function != act_open_retry_timer) {
818 c3cn->retry_timer.function = act_open_retry_timer;
819 if (!mod_timer(&c3cn->retry_timer, jiffies + HZ / 2))
820 c3cn_hold(c3cn);
821 } else
822 fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
823 __kfree_skb(skb);
824}
825
826static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
827{
828 struct s3_conn *c3cn = ctx;
829 struct cpl_act_open_rpl *rpl = cplhdr(skb);
830
831 c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
832 rpl->status, c3cn, c3cn->state, c3cn->flags);
833
834 if (rpl->status != CPL_ERR_TCAM_FULL &&
835 rpl->status != CPL_ERR_CONN_EXIST &&
836 rpl->status != CPL_ERR_ARP_MISS)
837 cxgb3_queue_tid_release(cdev, GET_TID(rpl));
838
839 process_cpl_msg_ref(process_act_open_rpl, c3cn, skb);
840 return 0;
841}
842
843/*
844 * Process PEER_CLOSE CPL messages: -> host
845 * Handle peer FIN.
846 */
847static void process_peer_close(struct s3_conn *c3cn, struct sk_buff *skb)
848{
849 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
850 c3cn, c3cn->state, c3cn->flags);
851
852 if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
853 goto out;
854
855 switch (c3cn->state) {
856 case C3CN_STATE_ESTABLISHED:
857 c3cn_set_state(c3cn, C3CN_STATE_PASSIVE_CLOSE);
858 break;
859 case C3CN_STATE_ACTIVE_CLOSE:
860 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
861 break;
862 case C3CN_STATE_CLOSE_WAIT_1:
863 c3cn_closed(c3cn);
864 break;
865 case C3CN_STATE_ABORTING:
866 break;
867 default:
868 cxgb3i_log_error("%s: peer close, TID %u in bad state %u\n",
869 c3cn->cdev->name, c3cn->tid, c3cn->state);
870 }
871
872 cxgb3i_conn_closing(c3cn);
873out:
874 __kfree_skb(skb);
875}
876
877static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
878{
879 struct s3_conn *c3cn = ctx;
880
881 c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
882 c3cn, c3cn->state, c3cn->flags);
883 process_cpl_msg_ref(process_peer_close, c3cn, skb);
884 return 0;
885}
886
887/*
888 * Process CLOSE_CONN_RPL CPL message: -> host
889 * Process a peer ACK to our FIN.
890 */
891static void process_close_con_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
892{
893 struct cpl_close_con_rpl *rpl = cplhdr(skb);
894
895 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
896 c3cn, c3cn->state, c3cn->flags);
897
898 c3cn->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
899
900 if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
901 goto out;
902
903 switch (c3cn->state) {
904 case C3CN_STATE_ACTIVE_CLOSE:
905 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_1);
906 break;
907 case C3CN_STATE_CLOSE_WAIT_1:
908 case C3CN_STATE_CLOSE_WAIT_2:
909 c3cn_closed(c3cn);
910 break;
911 case C3CN_STATE_ABORTING:
912 break;
913 default:
914 cxgb3i_log_error("%s: close_rpl, TID %u in bad state %u\n",
915 c3cn->cdev->name, c3cn->tid, c3cn->state);
916 }
917
918out:
919 kfree_skb(skb);
920}
921
922static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
923 void *ctx)
924{
925 struct s3_conn *c3cn = ctx;
926
927 c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
928 c3cn, c3cn->state, c3cn->flags);
929
930 process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
931 return 0;
932}
933
934/*
935 * Process ABORT_REQ_RSS CPL message: -> host
936 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
937 * request except that we need to reply to it.
938 */
939
940static int abort_status_to_errno(struct s3_conn *c3cn, int abort_reason,
941 int *need_rst)
942{
943 switch (abort_reason) {
944 case CPL_ERR_BAD_SYN: /* fall through */
945 case CPL_ERR_CONN_RESET:
946 return c3cn->state > C3CN_STATE_ESTABLISHED ?
947 EPIPE : ECONNRESET;
948 case CPL_ERR_XMIT_TIMEDOUT:
949 case CPL_ERR_PERSIST_TIMEDOUT:
950 case CPL_ERR_FINWAIT2_TIMEDOUT:
951 case CPL_ERR_KEEPALIVE_TIMEDOUT:
952 return ETIMEDOUT;
953 default:
954 return EIO;
955 }
956}
957
958static void process_abort_req(struct s3_conn *c3cn, struct sk_buff *skb)
959{
960 int rst_status = CPL_ABORT_NO_RST;
961 const struct cpl_abort_req_rss *req = cplhdr(skb);
962
963 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
964 c3cn, c3cn->state, c3cn->flags);
965
966 if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
967 c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
968 c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
969 __kfree_skb(skb);
970 return;
971 }
972
973 c3cn_clear_flag(c3cn, C3CN_ABORT_REQ_RCVD);
974 send_abort_rpl(c3cn, rst_status);
975
976 if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
977 c3cn->err =
978 abort_status_to_errno(c3cn, req->status, &rst_status);
979 c3cn_closed(c3cn);
980 }
981}
982
983static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
984{
985 const struct cpl_abort_req_rss *req = cplhdr(skb);
986 struct s3_conn *c3cn = ctx;
987
988 c3cn_conn_debug("rcv, c3cn 0x%p, s 0x%x, f 0x%lx.\n",
989 c3cn, c3cn->state, c3cn->flags);
990
991 if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
992 req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
993 __kfree_skb(skb);
994 return 0;
995 }
996
997 process_cpl_msg_ref(process_abort_req, c3cn, skb);
998 return 0;
999}
1000
1001/*
1002 * Process ABORT_RPL_RSS CPL message: -> host
1003 * Process abort replies. We only process these messages if we anticipate
1004 * them as the coordination between SW and HW in this area is somewhat lacking
1005 * and sometimes we get ABORT_RPLs after we are done with the connection that
1006 * originated the ABORT_REQ.
1007 */
1008static void process_abort_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
1009{
1010 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1011 c3cn, c3cn->state, c3cn->flags);
1012
1013 if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
1014 if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
1015 c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
1016 else {
1017 c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_RCVD);
1018 c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_PENDING);
1019 if (c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD))
1020 cxgb3i_log_error("%s tid %u, ABORT_RPL_RSS\n",
1021 c3cn->cdev->name, c3cn->tid);
1022 c3cn_closed(c3cn);
1023 }
1024 }
1025 __kfree_skb(skb);
1026}
1027
1028static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1029{
1030 struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
1031 struct s3_conn *c3cn = ctx;
1032
1033 c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, 0x%lx.\n",
1034 rpl->status, c3cn, c3cn ? c3cn->state : 0,
1035 c3cn ? c3cn->flags : 0UL);
1036
1037 /*
1038 * Ignore replies to post-close aborts indicating that the abort was
1039 * requested too late. These connections are terminated when we get
1040 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
1041 * arrives the TID is either no longer used or it has been recycled.
1042 */
1043 if (rpl->status == CPL_ERR_ABORT_FAILED)
1044 goto discard;
1045
1046 /*
1047 * Sometimes we've already closed the connection, e.g., a post-close
1048 * abort races with ABORT_REQ_RSS, the latter frees the connection
1049 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
1050 * but FW turns the ABORT_REQ into a regular one and so we get
1051 * ABORT_RPL_RSS with status 0 and no connection.
1052 */
1053 if (!c3cn)
1054 goto discard;
1055
1056 process_cpl_msg_ref(process_abort_rpl, c3cn, skb);
1057 return 0;
1058
1059discard:
1060 __kfree_skb(skb);
1061 return 0;
1062}
1063
1064/*
1065 * Process RX_ISCSI_HDR CPL message: -> host
1066 * Handle received PDUs, the payload could be DDP'ed. If not, the payload
1067 * follow after the bhs.
1068 */
1069static void process_rx_iscsi_hdr(struct s3_conn *c3cn, struct sk_buff *skb)
1070{
1071 struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
1072 struct cpl_iscsi_hdr_norss data_cpl;
1073 struct cpl_rx_data_ddp_norss ddp_cpl;
1074 unsigned int hdr_len, data_len, status;
1075 unsigned int len;
1076 int err;
1077
1078 if (unlikely(c3cn->state >= C3CN_STATE_PASSIVE_CLOSE)) {
1079 if (c3cn->state != C3CN_STATE_ABORTING)
1080 send_abort_req(c3cn);
1081 __kfree_skb(skb);
1082 return;
1083 }
1084
1085 CXGB3_SKB_CB(skb)->seq = ntohl(hdr_cpl->seq);
1086 CXGB3_SKB_CB(skb)->flags = 0;
1087
1088 skb_reset_transport_header(skb);
1089 __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
1090
1091 len = hdr_len = ntohs(hdr_cpl->len);
1092 /* msg coalesce is off or not enough data received */
1093 if (skb->len <= hdr_len) {
1094 cxgb3i_log_error("%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
1095 c3cn->cdev->name, c3cn->tid,
1096 skb->len, hdr_len);
1097 goto abort_conn;
1098 }
1099
1100 err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
1101 sizeof(ddp_cpl));
1102 if (err < 0)
1103 goto abort_conn;
1104
1105 skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY;
1106 skb_ulp_pdulen(skb) = ntohs(ddp_cpl.len);
1107 skb_ulp_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
1108 status = ntohl(ddp_cpl.ddp_status);
1109
1110 c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
1111 skb, skb->len, skb_ulp_pdulen(skb), status);
1112
1113 if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
1114 skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
1115 if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
1116 skb_ulp_mode(skb) |= ULP2_FLAG_DCRC_ERROR;
1117 if (status & (1 << RX_DDP_STATUS_PAD_SHIFT))
1118 skb_ulp_mode(skb) |= ULP2_FLAG_PAD_ERROR;
1119
1120 if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
1121 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
1122 if (err < 0)
1123 goto abort_conn;
1124 data_len = ntohs(data_cpl.len);
1125 len += sizeof(data_cpl) + data_len;
1126 } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT))
1127 skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED;
1128
1129 c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_ulp_pdulen(skb);
1130 __pskb_trim(skb, len);
1131 __skb_queue_tail(&c3cn->receive_queue, skb);
1132 cxgb3i_conn_pdu_ready(c3cn);
1133
1134 return;
1135
1136abort_conn:
1137 send_abort_req(c3cn);
1138 __kfree_skb(skb);
1139}
1140
1141static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
1142{
1143 struct s3_conn *c3cn = ctx;
1144
1145 process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
1146 return 0;
1147}
1148
1149/*
1150 * Process TX_DATA_ACK CPL messages: -> host
1151 * Process an acknowledgment of WR completion. Advance snd_una and send the
1152 * next batch of work requests from the write queue.
1153 */
1154static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
1155{
1156 struct cpl_wr_ack *hdr = cplhdr(skb);
1157 unsigned int credits = ntohs(hdr->credits);
1158 u32 snd_una = ntohl(hdr->snd_una);
1159
1160 c3cn->wr_avail += credits;
1161 if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
1162 c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
1163
1164 while (credits) {
1165 struct sk_buff *p = peek_wr(c3cn);
1166
1167 if (unlikely(!p)) {
1168 cxgb3i_log_error("%u WR_ACK credits for TID %u with "
1169 "nothing pending, state %u\n",
1170 credits, c3cn->tid, c3cn->state);
1171 break;
1172 }
1173 if (unlikely(credits < p->csum)) {
1174 p->csum -= credits;
1175 break;
1176 } else {
1177 dequeue_wr(c3cn);
1178 credits -= p->csum;
1179 free_wr_skb(p);
1180 }
1181 }
1182
1183 if (unlikely(before(snd_una, c3cn->snd_una)))
1184 goto out_free;
1185
1186 if (c3cn->snd_una != snd_una) {
1187 c3cn->snd_una = snd_una;
1188 dst_confirm(c3cn->dst_cache);
1189 }
1190
1191 if (skb_queue_len(&c3cn->write_queue) && c3cn_push_tx_frames(c3cn, 0))
1192 cxgb3i_conn_tx_open(c3cn);
1193out_free:
1194 __kfree_skb(skb);
1195}
1196
1197static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1198{
1199 struct s3_conn *c3cn = ctx;
1200
1201 process_cpl_msg(process_wr_ack, c3cn, skb);
1202 return 0;
1203}
1204
1205/*
1206 * for each connection, pre-allocate skbs needed for close/abort requests. So
1207 * that we can service the request right away.
1208 */
1209static void c3cn_free_cpl_skbs(struct s3_conn *c3cn)
1210{
1211 if (c3cn->cpl_close)
1212 kfree_skb(c3cn->cpl_close);
1213 if (c3cn->cpl_abort_req)
1214 kfree_skb(c3cn->cpl_abort_req);
1215 if (c3cn->cpl_abort_rpl)
1216 kfree_skb(c3cn->cpl_abort_rpl);
1217}
1218
1219static int c3cn_alloc_cpl_skbs(struct s3_conn *c3cn)
1220{
1221 c3cn->cpl_close = alloc_skb(sizeof(struct cpl_close_con_req),
1222 GFP_KERNEL);
1223 if (!c3cn->cpl_close)
1224 return -ENOMEM;
1225 skb_put(c3cn->cpl_close, sizeof(struct cpl_close_con_req));
1226
1227 c3cn->cpl_abort_req = alloc_skb(sizeof(struct cpl_abort_req),
1228 GFP_KERNEL);
1229 if (!c3cn->cpl_abort_req)
1230 goto free_cpl_skbs;
1231 skb_put(c3cn->cpl_abort_req, sizeof(struct cpl_abort_req));
1232
1233 c3cn->cpl_abort_rpl = alloc_skb(sizeof(struct cpl_abort_rpl),
1234 GFP_KERNEL);
1235 if (!c3cn->cpl_abort_rpl)
1236 goto free_cpl_skbs;
1237 skb_put(c3cn->cpl_abort_rpl, sizeof(struct cpl_abort_rpl));
1238
1239 return 0;
1240
1241free_cpl_skbs:
1242 c3cn_free_cpl_skbs(c3cn);
1243 return -ENOMEM;
1244}
1245
1246/**
1247 * c3cn_release_offload_resources - release offload resource
1248 * @c3cn: the offloaded iscsi tcp connection.
1249 * Release resources held by an offload connection (TID, L2T entry, etc.)
1250 */
1251static void c3cn_release_offload_resources(struct s3_conn *c3cn)
1252{
1253 struct t3cdev *cdev = c3cn->cdev;
1254 unsigned int tid = c3cn->tid;
1255
1256 if (!cdev)
1257 return;
1258
1259 c3cn->qset = 0;
1260
1261 c3cn_free_cpl_skbs(c3cn);
1262
1263 if (c3cn->wr_avail != c3cn->wr_max) {
1264 purge_wr_queue(c3cn);
1265 reset_wr_list(c3cn);
1266 }
1267
1268 if (c3cn->l2t) {
1269 l2t_release(L2DATA(cdev), c3cn->l2t);
1270 c3cn->l2t = NULL;
1271 }
1272
1273 if (c3cn->state == C3CN_STATE_CONNECTING) /* we have ATID */
1274 s3_free_atid(cdev, tid);
1275 else { /* we have TID */
1276 cxgb3_remove_tid(cdev, (void *)c3cn, tid);
1277 c3cn_put(c3cn);
1278 }
1279
1280 c3cn->cdev = NULL;
1281}
1282
1283/**
1284 * cxgb3i_c3cn_create - allocate and initialize an s3_conn structure
1285 * returns the s3_conn structure allocated.
1286 */
1287struct s3_conn *cxgb3i_c3cn_create(void)
1288{
1289 struct s3_conn *c3cn;
1290
1291 c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
1292 if (!c3cn)
1293 return NULL;
1294
1295 /* pre-allocate close/abort cpl, so we don't need to wait for memory
1296 when close/abort is requested. */
1297 if (c3cn_alloc_cpl_skbs(c3cn) < 0)
1298 goto free_c3cn;
1299
1300 c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
1301
1302 c3cn->flags = 0;
1303 spin_lock_init(&c3cn->lock);
1304 atomic_set(&c3cn->refcnt, 1);
1305 skb_queue_head_init(&c3cn->receive_queue);
1306 skb_queue_head_init(&c3cn->write_queue);
1307 setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
1308 rwlock_init(&c3cn->callback_lock);
1309
1310 return c3cn;
1311
1312free_c3cn:
1313 kfree(c3cn);
1314 return NULL;
1315}
1316
1317static void c3cn_active_close(struct s3_conn *c3cn)
1318{
1319 int data_lost;
1320 int close_req = 0;
1321
1322 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1323 c3cn, c3cn->state, c3cn->flags);
1324
1325 dst_confirm(c3cn->dst_cache);
1326
1327 c3cn_hold(c3cn);
1328 spin_lock_bh(&c3cn->lock);
1329
1330 data_lost = skb_queue_len(&c3cn->receive_queue);
1331 __skb_queue_purge(&c3cn->receive_queue);
1332
1333 switch (c3cn->state) {
1334 case C3CN_STATE_CLOSED:
1335 case C3CN_STATE_ACTIVE_CLOSE:
1336 case C3CN_STATE_CLOSE_WAIT_1:
1337 case C3CN_STATE_CLOSE_WAIT_2:
1338 case C3CN_STATE_ABORTING:
1339 /* nothing need to be done */
1340 break;
1341 case C3CN_STATE_CONNECTING:
1342 /* defer until cpl_act_open_rpl or cpl_act_establish */
1343 c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
1344 break;
1345 case C3CN_STATE_ESTABLISHED:
1346 close_req = 1;
1347 c3cn_set_state(c3cn, C3CN_STATE_ACTIVE_CLOSE);
1348 break;
1349 case C3CN_STATE_PASSIVE_CLOSE:
1350 close_req = 1;
1351 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
1352 break;
1353 }
1354
1355 if (close_req) {
1356 if (data_lost)
1357 /* Unread data was tossed, zap the connection. */
1358 send_abort_req(c3cn);
1359 else
1360 send_close_req(c3cn);
1361 }
1362
1363 spin_unlock_bh(&c3cn->lock);
1364 c3cn_put(c3cn);
1365}
1366
1367/**
1368 * cxgb3i_c3cn_release - close and release an iscsi tcp connection and any
1369 * resource held
1370 * @c3cn: the iscsi tcp connection
1371 */
1372void cxgb3i_c3cn_release(struct s3_conn *c3cn)
1373{
1374 c3cn_conn_debug("c3cn 0x%p, s %u, f 0x%lx.\n",
1375 c3cn, c3cn->state, c3cn->flags);
1376 if (likely(c3cn->state != C3CN_STATE_CONNECTING))
1377 c3cn_active_close(c3cn);
1378 else
1379 c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
1380 c3cn_put(c3cn);
1381}
1382
1383static int is_cxgb3_dev(struct net_device *dev)
1384{
1385 struct cxgb3i_sdev_data *cdata;
1386
1387 write_lock(&cdata_rwlock);
1388 list_for_each_entry(cdata, &cdata_list, list) {
1389 struct adap_ports *ports = &cdata->ports;
1390 int i;
1391
1392 for (i = 0; i < ports->nports; i++)
1393 if (dev == ports->lldevs[i]) {
1394 write_unlock(&cdata_rwlock);
1395 return 1;
1396 }
1397 }
1398 write_unlock(&cdata_rwlock);
1399 return 0;
1400}
1401
1402/**
1403 * cxgb3_egress_dev - return the cxgb3 egress device
1404 * @root_dev: the root device anchoring the search
1405 * @c3cn: the connection used to determine egress port in bonding mode
1406 * @context: in bonding mode, indicates a connection set up or failover
1407 *
1408 * Return egress device or NULL if the egress device isn't one of our ports.
1409 */
1410static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
1411 struct s3_conn *c3cn,
1412 int context)
1413{
1414 while (root_dev) {
1415 if (root_dev->priv_flags & IFF_802_1Q_VLAN)
1416 root_dev = vlan_dev_real_dev(root_dev);
1417 else if (is_cxgb3_dev(root_dev))
1418 return root_dev;
1419 else
1420 return NULL;
1421 }
1422 return NULL;
1423}
1424
1425static struct rtable *find_route(__be32 saddr, __be32 daddr,
1426 __be16 sport, __be16 dport)
1427{
1428 struct rtable *rt;
1429 struct flowi fl = {
1430 .oif = 0,
1431 .nl_u = {
1432 .ip4_u = {
1433 .daddr = daddr,
1434 .saddr = saddr,
1435 .tos = 0 } },
1436 .proto = IPPROTO_TCP,
1437 .uli_u = {
1438 .ports = {
1439 .sport = sport,
1440 .dport = dport } } };
1441
1442 if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
1443 return NULL;
1444 return rt;
1445}
1446
1447/*
1448 * Assign offload parameters to some connection fields.
1449 */
1450static void init_offload_conn(struct s3_conn *c3cn,
1451 struct t3cdev *cdev,
1452 struct dst_entry *dst)
1453{
1454 BUG_ON(c3cn->cdev != cdev);
1455 c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs;
1456 c3cn->wr_unacked = 0;
1457 c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
1458
1459 reset_wr_list(c3cn);
1460}
1461
1462static int initiate_act_open(struct s3_conn *c3cn, struct net_device *dev)
1463{
1464 struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
1465 struct t3cdev *cdev = cdata->cdev;
1466 struct dst_entry *dst = c3cn->dst_cache;
1467 struct sk_buff *skb;
1468
1469 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1470 c3cn, c3cn->state, c3cn->flags);
1471 /*
1472 * Initialize connection data. Note that the flags and ULP mode are
1473 * initialized higher up ...
1474 */
1475 c3cn->dev = dev;
1476 c3cn->cdev = cdev;
1477 c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
1478 if (c3cn->tid < 0)
1479 goto out_err;
1480
1481 c3cn->qset = 0;
1482 c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
1483 if (!c3cn->l2t)
1484 goto free_tid;
1485
1486 skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
1487 if (!skb)
1488 goto free_l2t;
1489
1490 skb->sk = (struct sock *)c3cn;
1491 set_arp_failure_handler(skb, act_open_req_arp_failure);
1492
1493 c3cn_hold(c3cn);
1494
1495 init_offload_conn(c3cn, cdev, dst);
1496 c3cn->err = 0;
1497
1498 make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
1499 l2t_send(cdev, skb, c3cn->l2t);
1500 return 0;
1501
1502free_l2t:
1503 l2t_release(L2DATA(cdev), c3cn->l2t);
1504free_tid:
1505 s3_free_atid(cdev, c3cn->tid);
1506 c3cn->tid = 0;
1507out_err:
1508 return -1;
1509}
1510
1511
1512/**
1513 * cxgb3i_c3cn_connect - initiates an iscsi tcp connection to a given address
1514 * @c3cn: the iscsi tcp connection
1515 * @usin: destination address
1516 *
1517 * return 0 if active open request is sent, < 0 otherwise.
1518 */
1519int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin)
1520{
1521 struct rtable *rt;
1522 struct net_device *dev;
1523 struct cxgb3i_sdev_data *cdata;
1524 struct t3cdev *cdev;
1525 __be32 sipv4;
1526 int err;
1527
1528 if (usin->sin_family != AF_INET)
1529 return -EAFNOSUPPORT;
1530
1531 c3cn->daddr.sin_port = usin->sin_port;
1532 c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
1533
1534 rt = find_route(c3cn->saddr.sin_addr.s_addr,
1535 c3cn->daddr.sin_addr.s_addr,
1536 c3cn->saddr.sin_port,
1537 c3cn->daddr.sin_port);
1538 if (rt == NULL) {
1539 c3cn_conn_debug("NO route to 0x%x, port %u.\n",
1540 c3cn->daddr.sin_addr.s_addr,
1541 ntohs(c3cn->daddr.sin_port));
1542 return -ENETUNREACH;
1543 }
1544
1545 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
1546 c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
1547 c3cn->daddr.sin_addr.s_addr,
1548 ntohs(c3cn->daddr.sin_port));
1549 ip_rt_put(rt);
1550 return -ENETUNREACH;
1551 }
1552
1553 if (!c3cn->saddr.sin_addr.s_addr)
1554 c3cn->saddr.sin_addr.s_addr = rt->rt_src;
1555
1556 /* now commit destination to connection */
1557 c3cn->dst_cache = &rt->u.dst;
1558
1559 /* try to establish an offloaded connection */
1560 dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
1561 if (dev == NULL) {
1562 c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
1563 return -ENETUNREACH;
1564 }
1565 cdata = NDEV2CDATA(dev);
1566 cdev = cdata->cdev;
1567
1568 /* get a source port if one hasn't been provided */
1569 err = c3cn_get_port(c3cn, cdata);
1570 if (err)
1571 return err;
1572
1573 c3cn_conn_debug("c3cn 0x%p get port %u.\n",
1574 c3cn, ntohs(c3cn->saddr.sin_port));
1575
1576 sipv4 = cxgb3i_get_private_ipv4addr(dev);
1577 if (!sipv4) {
1578 c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
1579 sipv4 = c3cn->saddr.sin_addr.s_addr;
1580 cxgb3i_set_private_ipv4addr(dev, sipv4);
1581 } else
1582 c3cn->saddr.sin_addr.s_addr = sipv4;
1583
1584 c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
1585 c3cn, NIPQUAD(c3cn->saddr.sin_addr.s_addr),
1586 ntohs(c3cn->saddr.sin_port),
1587 NIPQUAD(c3cn->daddr.sin_addr.s_addr),
1588 ntohs(c3cn->daddr.sin_port));
1589
1590 c3cn_set_state(c3cn, C3CN_STATE_CONNECTING);
1591 if (!initiate_act_open(c3cn, dev))
1592 return 0;
1593
1594 /*
1595 * If we get here, we don't have an offload connection so simply
1596 * return a failure.
1597 */
1598 err = -ENOTSUPP;
1599
1600 /*
1601 * This trashes the connection and releases the local port,
1602 * if necessary.
1603 */
1604 c3cn_conn_debug("c3cn 0x%p -> CLOSED.\n", c3cn);
1605 c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
1606 ip_rt_put(rt);
1607 c3cn_put_port(c3cn);
1608 c3cn->daddr.sin_port = 0;
1609 return err;
1610}
1611
1612/**
1613 * cxgb3i_c3cn_rx_credits - ack received tcp data.
1614 * @c3cn: iscsi tcp connection
1615 * @copied: # of bytes processed
1616 *
1617 * Called after some received data has been read. It returns RX credits
1618 * to the HW for the amount of data processed.
1619 */
1620void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
1621{
1622 struct t3cdev *cdev;
1623 int must_send;
1624 u32 credits, dack = 0;
1625
1626 if (c3cn->state != C3CN_STATE_ESTABLISHED)
1627 return;
1628
1629 credits = c3cn->copied_seq - c3cn->rcv_wup;
1630 if (unlikely(!credits))
1631 return;
1632
1633 cdev = c3cn->cdev;
1634
1635 if (unlikely(cxgb3_rx_credit_thres == 0))
1636 return;
1637
1638 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
1639
1640 /*
1641 * For coalescing to work effectively ensure the receive window has
1642 * at least 16KB left.
1643 */
1644 must_send = credits + 16384 >= cxgb3_rcv_win;
1645
1646 if (must_send || credits >= cxgb3_rx_credit_thres)
1647 c3cn->rcv_wup += send_rx_credits(c3cn, credits, dack);
1648}
1649
1650/**
1651 * cxgb3i_c3cn_send_pdus - send the skbs containing iscsi pdus
1652 * @c3cn: iscsi tcp connection
1653 * @skb: skb contains the iscsi pdu
1654 *
1655 * Add a list of skbs to a connection send queue. The skbs must comply with
1656 * the max size limit of the device and have a headroom of at least
1657 * TX_HEADER_LEN bytes.
1658 * Return # of bytes queued.
1659 */
1660int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb)
1661{
1662 struct sk_buff *next;
1663 int err, copied = 0;
1664
1665 spin_lock_bh(&c3cn->lock);
1666
1667 if (c3cn->state != C3CN_STATE_ESTABLISHED) {
1668 c3cn_tx_debug("c3cn 0x%p, not in est. state %u.\n",
1669 c3cn, c3cn->state);
1670 err = -EAGAIN;
1671 goto out_err;
1672 }
1673
1674 err = -EPIPE;
1675 if (c3cn->err) {
1676 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err);
1677 goto out_err;
1678 }
1679
1680 while (skb) {
1681 int frags = skb_shinfo(skb)->nr_frags +
1682 (skb->len != skb->data_len);
1683
1684 if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
1685 c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
1686 err = -EINVAL;
1687 goto out_err;
1688 }
1689
1690 if (frags >= SKB_WR_LIST_SIZE) {
1691 cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
1692 c3cn, skb_shinfo(skb)->nr_frags,
1693 skb->len, skb->data_len);
1694 err = -EINVAL;
1695 goto out_err;
1696 }
1697
1698 next = skb->next;
1699 skb->next = NULL;
1700 skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
1701 copied += skb->len;
1702 c3cn->write_seq += skb->len + ulp_extra_len(skb);
1703 skb = next;
1704 }
1705done:
1706 if (likely(skb_queue_len(&c3cn->write_queue)))
1707 c3cn_push_tx_frames(c3cn, 1);
1708 spin_unlock_bh(&c3cn->lock);
1709 return copied;
1710
1711out_err:
1712 if (copied == 0 && err == -EPIPE)
1713 copied = c3cn->err ? c3cn->err : -EPIPE;
1714 goto done;
1715}
1716
1717static void sdev_data_cleanup(struct cxgb3i_sdev_data *cdata)
1718{
1719 struct adap_ports *ports = &cdata->ports;
1720 int i;
1721
1722 for (i = 0; i < ports->nports; i++)
1723 NDEV2CDATA(ports->lldevs[i]) = NULL;
1724 cxgb3i_free_big_mem(cdata);
1725}
1726
1727void cxgb3i_sdev_cleanup(void)
1728{
1729 struct cxgb3i_sdev_data *cdata;
1730
1731 write_lock(&cdata_rwlock);
1732 list_for_each_entry(cdata, &cdata_list, list) {
1733 list_del(&cdata->list);
1734 sdev_data_cleanup(cdata);
1735 }
1736 write_unlock(&cdata_rwlock);
1737}
1738
1739int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
1740{
1741 cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
1742 cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
1743 cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
1744 cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
1745 cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
1746 cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
1747 cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
1748 cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
1749
1750 if (cxgb3_max_connect > CXGB3I_MAX_CONN)
1751 cxgb3_max_connect = CXGB3I_MAX_CONN;
1752 return 0;
1753}
1754
1755/**
1756 * cxgb3i_sdev_add - allocate and initialize resources for each adapter found
1757 * @cdev: t3cdev adapter
1758 * @client: cxgb3 driver client
1759 */
1760void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
1761{
1762 struct cxgb3i_sdev_data *cdata;
1763 struct ofld_page_info rx_page_info;
1764 unsigned int wr_len;
1765 int mapsize = DIV_ROUND_UP(cxgb3_max_connect,
1766 8 * sizeof(unsigned long));
1767 int i;
1768
1769 cdata = cxgb3i_alloc_big_mem(sizeof(*cdata) + mapsize, GFP_KERNEL);
1770 if (!cdata)
1771 return;
1772
1773 if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
1774 cdev->ctl(cdev, GET_PORTS, &cdata->ports) < 0 ||
1775 cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0)
1776 goto free_cdata;
1777
1778 s3_init_wr_tab(wr_len);
1779
1780 INIT_LIST_HEAD(&cdata->list);
1781 cdata->cdev = cdev;
1782 cdata->client = client;
1783
1784 for (i = 0; i < cdata->ports.nports; i++)
1785 NDEV2CDATA(cdata->ports.lldevs[i]) = cdata;
1786
1787 write_lock(&cdata_rwlock);
1788 list_add_tail(&cdata->list, &cdata_list);
1789 write_unlock(&cdata_rwlock);
1790
1791 return;
1792
1793free_cdata:
1794 cxgb3i_free_big_mem(cdata);
1795}
1796
1797/**
1798 * cxgb3i_sdev_remove - free the allocated resources for the adapter
1799 * @cdev: t3cdev adapter
1800 */
1801void cxgb3i_sdev_remove(struct t3cdev *cdev)
1802{
1803 struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
1804
1805 write_lock(&cdata_rwlock);
1806 list_del(&cdata->list);
1807 write_unlock(&cdata_rwlock);
1808
1809 sdev_data_cleanup(cdata);
1810}
diff --git a/drivers/scsi/cxgb3i/cxgb3i_offload.h b/drivers/scsi/cxgb3i/cxgb3i_offload.h
new file mode 100644
index 000000000000..5b93d629e5c1
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i_offload.h
@@ -0,0 +1,231 @@
1/*
2 * cxgb3i_offload.h: Chelsio S3xx iscsi offloaded tcp connection management
3 *
4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
5 *
6 * This program is distributed in the hope that it will be useful, but WITHOUT
7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
9 * release for licensing terms and conditions.
10 *
11 * Written by: Dimitris Michailidis (dm@chelsio.com)
12 * Karen Xie (kxie@chelsio.com)
13 */
14
15#ifndef _CXGB3I_OFFLOAD_H
16#define _CXGB3I_OFFLOAD_H
17
18#include <linux/skbuff.h>
19#include <net/tcp.h>
20
21#include "common.h"
22#include "adapter.h"
23#include "t3cdev.h"
24#include "cxgb3_offload.h"
25
26#define cxgb3i_log_error(fmt...) printk(KERN_ERR "cxgb3i: ERR! " fmt)
27#define cxgb3i_log_warn(fmt...) printk(KERN_WARNING "cxgb3i: WARN! " fmt)
28#define cxgb3i_log_info(fmt...) printk(KERN_INFO "cxgb3i: " fmt)
29#define cxgb3i_log_debug(fmt, args...) \
30 printk(KERN_INFO "cxgb3i: %s - " fmt, __func__ , ## args)
31
32/**
33 * struct s3_conn - an iscsi tcp connection structure
34 *
35 * @dev: net device of with connection
36 * @cdev: adapter t3cdev for net device
37 * @flags: see c3cn_flags below
38 * @tid: connection id assigned by the h/w
39 * @qset: queue set used by connection
40 * @mss_idx: Maximum Segment Size table index
41 * @l2t: ARP resolution entry for offload packets
42 * @wr_max: maximum in-flight writes
43 * @wr_avail: number of writes available
44 * @wr_unacked: writes since last request for completion notification
45 * @wr_pending_head: head of pending write queue
46 * @wr_pending_tail: tail of pending write queue
47 * @cpl_close: skb for cpl_close_req
48 * @cpl_abort_req: skb for cpl_abort_req
49 * @cpl_abort_rpl: skb for cpl_abort_rpl
50 * @lock: connection status lock
51 * @refcnt: reference count on connection
52 * @state: connection state
53 * @saddr: source ip/port address
54 * @daddr: destination ip/port address
55 * @dst_cache: reference to destination route
56 * @receive_queue: received PDUs
57 * @write_queue: un-pushed pending writes
58 * @retry_timer: retry timer for various operations
59 * @err: connection error status
60 * @callback_lock: lock for opaque user context
61 * @user_data: opaque user context
62 * @rcv_nxt: next receive seq. #
63 * @copied_seq: head of yet unread data
64 * @rcv_wup: rcv_nxt on last window update sent
65 * @snd_nxt: next sequence we send
66 * @snd_una: first byte we want an ack for
67 * @write_seq: tail+1 of data held in send buffer
68 */
69struct s3_conn {
70 struct net_device *dev;
71 struct t3cdev *cdev;
72 unsigned long flags;
73 int tid;
74 int qset;
75 int mss_idx;
76 struct l2t_entry *l2t;
77 int wr_max;
78 int wr_avail;
79 int wr_unacked;
80 struct sk_buff *wr_pending_head;
81 struct sk_buff *wr_pending_tail;
82 struct sk_buff *cpl_close;
83 struct sk_buff *cpl_abort_req;
84 struct sk_buff *cpl_abort_rpl;
85 spinlock_t lock;
86 atomic_t refcnt;
87 volatile unsigned int state;
88 struct sockaddr_in saddr;
89 struct sockaddr_in daddr;
90 struct dst_entry *dst_cache;
91 struct sk_buff_head receive_queue;
92 struct sk_buff_head write_queue;
93 struct timer_list retry_timer;
94 int err;
95 rwlock_t callback_lock;
96 void *user_data;
97
98 u32 rcv_nxt;
99 u32 copied_seq;
100 u32 rcv_wup;
101 u32 snd_nxt;
102 u32 snd_una;
103 u32 write_seq;
104};
105
106/*
107 * connection state
108 */
109enum conn_states {
110 C3CN_STATE_CONNECTING = 1,
111 C3CN_STATE_ESTABLISHED,
112 C3CN_STATE_ACTIVE_CLOSE,
113 C3CN_STATE_PASSIVE_CLOSE,
114 C3CN_STATE_CLOSE_WAIT_1,
115 C3CN_STATE_CLOSE_WAIT_2,
116 C3CN_STATE_ABORTING,
117 C3CN_STATE_CLOSED,
118};
119
120static inline unsigned int c3cn_is_closing(const struct s3_conn *c3cn)
121{
122 return c3cn->state >= C3CN_STATE_ACTIVE_CLOSE;
123}
124static inline unsigned int c3cn_is_established(const struct s3_conn *c3cn)
125{
126 return c3cn->state == C3CN_STATE_ESTABLISHED;
127}
128
129/*
130 * Connection flags -- many to track some close related events.
131 */
132enum c3cn_flags {
133 C3CN_ABORT_RPL_RCVD, /* received one ABORT_RPL_RSS message */
134 C3CN_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
135 C3CN_ABORT_RPL_PENDING, /* expecting an abort reply */
136 C3CN_TX_DATA_SENT, /* already sent a TX_DATA WR */
137 C3CN_ACTIVE_CLOSE_NEEDED, /* need to be closed */
138};
139
140/**
141 * cxgb3i_sdev_data - Per adapter data.
142 * Linked off of each Ethernet device port on the adapter.
143 * Also available via the t3cdev structure since we have pointers to our port
144 * net_device's there ...
145 *
146 * @list: list head to link elements
147 * @cdev: t3cdev adapter
148 * @client: CPL client pointer
149 * @ports: array of adapter ports
150 * @sport_map_next: next index into the port map
151 * @sport_map: source port map
152 */
153struct cxgb3i_sdev_data {
154 struct list_head list;
155 struct t3cdev *cdev;
156 struct cxgb3_client *client;
157 struct adap_ports ports;
158 unsigned int sport_map_next;
159 unsigned long sport_map[0];
160};
161#define NDEV2CDATA(ndev) (*(struct cxgb3i_sdev_data **)&(ndev)->ec_ptr)
162#define CXGB3_SDEV_DATA(cdev) NDEV2CDATA((cdev)->lldev)
163
164void cxgb3i_sdev_cleanup(void);
165int cxgb3i_sdev_init(cxgb3_cpl_handler_func *);
166void cxgb3i_sdev_add(struct t3cdev *, struct cxgb3_client *);
167void cxgb3i_sdev_remove(struct t3cdev *);
168
169struct s3_conn *cxgb3i_c3cn_create(void);
170int cxgb3i_c3cn_connect(struct s3_conn *, struct sockaddr_in *);
171void cxgb3i_c3cn_rx_credits(struct s3_conn *, int);
172int cxgb3i_c3cn_send_pdus(struct s3_conn *, struct sk_buff *);
173void cxgb3i_c3cn_release(struct s3_conn *);
174
175/**
176 * cxgb3_skb_cb - control block for received pdu state and ULP mode management.
177 *
178 * @flag: see C3CB_FLAG_* below
179 * @ulp_mode: ULP mode/submode of sk_buff
180 * @seq: tcp sequence number
181 * @ddigest: pdu data digest
182 * @pdulen: recovered pdu length
183 * @ulp_data: scratch area for ULP
184 */
185struct cxgb3_skb_cb {
186 __u8 flags;
187 __u8 ulp_mode;
188 __u32 seq;
189 __u32 ddigest;
190 __u32 pdulen;
191 __u8 ulp_data[16];
192};
193
194#define CXGB3_SKB_CB(skb) ((struct cxgb3_skb_cb *)&((skb)->cb[0]))
195
196#define skb_ulp_mode(skb) (CXGB3_SKB_CB(skb)->ulp_mode)
197#define skb_ulp_ddigest(skb) (CXGB3_SKB_CB(skb)->ddigest)
198#define skb_ulp_pdulen(skb) (CXGB3_SKB_CB(skb)->pdulen)
199#define skb_ulp_data(skb) (CXGB3_SKB_CB(skb)->ulp_data)
200
201enum c3cb_flags {
202 C3CB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
203 C3CB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
204 C3CB_FLAG_COMPL = 1 << 2, /* request WR completion */
205};
206
207/**
208 * sge_opaque_hdr -
209 * Opaque version of structure the SGE stores at skb->head of TX_DATA packets
210 * and for which we must reserve space.
211 */
212struct sge_opaque_hdr {
213 void *dev;
214 dma_addr_t addr[MAX_SKB_FRAGS + 1];
215};
216
217/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
218#define TX_HEADER_LEN \
219 (sizeof(struct tx_data_wr) + sizeof(struct sge_opaque_hdr))
220
221/*
222 * get and set private ip for iscsi traffic
223 */
224#define cxgb3i_get_private_ipv4addr(ndev) \
225 (((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr)
226#define cxgb3i_set_private_ipv4addr(ndev, addr) \
227 (((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr) = addr
228
229/* max. connections per adapter */
230#define CXGB3I_MAX_CONN 16384
231#endif /* _CXGB3_OFFLOAD_H */
diff --git a/drivers/scsi/cxgb3i/cxgb3i_pdu.c b/drivers/scsi/cxgb3i/cxgb3i_pdu.c
new file mode 100644
index 000000000000..ce7ce8c6094c
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i_pdu.c
@@ -0,0 +1,402 @@
1/*
2 * cxgb3i_pdu.c: Chelsio S3xx iSCSI driver.
3 *
4 * Copyright (c) 2008 Chelsio Communications, Inc.
5 * Copyright (c) 2008 Mike Christie
6 * Copyright (c) 2008 Red Hat, Inc. All rights reserved.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation.
11 *
12 * Written by: Karen Xie (kxie@chelsio.com)
13 */
14
15#include <linux/skbuff.h>
16#include <linux/crypto.h>
17#include <scsi/scsi_cmnd.h>
18#include <scsi/scsi_host.h>
19
20#include "cxgb3i.h"
21#include "cxgb3i_pdu.h"
22
23#ifdef __DEBUG_CXGB3I_RX__
24#define cxgb3i_rx_debug cxgb3i_log_debug
25#else
26#define cxgb3i_rx_debug(fmt...)
27#endif
28
29#ifdef __DEBUG_CXGB3I_TX__
30#define cxgb3i_tx_debug cxgb3i_log_debug
31#else
32#define cxgb3i_tx_debug(fmt...)
33#endif
34
35static struct page *pad_page;
36
37/*
38 * pdu receive, interact with libiscsi_tcp
39 */
40static inline int read_pdu_skb(struct iscsi_conn *conn, struct sk_buff *skb,
41 unsigned int offset, int offloaded)
42{
43 int status = 0;
44 int bytes_read;
45
46 bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
47 switch (status) {
48 case ISCSI_TCP_CONN_ERR:
49 return -EIO;
50 case ISCSI_TCP_SUSPENDED:
51 /* no transfer - just have caller flush queue */
52 return bytes_read;
53 case ISCSI_TCP_SKB_DONE:
54 /*
55 * pdus should always fit in the skb and we should get
56 * segment done notifcation.
57 */
58 iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
59 return -EFAULT;
60 case ISCSI_TCP_SEGMENT_DONE:
61 return bytes_read;
62 default:
63 iscsi_conn_printk(KERN_ERR, conn, "Invalid iscsi_tcp_recv_skb "
64 "status %d\n", status);
65 return -EINVAL;
66 }
67}
68
69static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
70 struct sk_buff *skb)
71{
72 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
73 bool offloaded = 0;
74 unsigned int offset;
75 int rc;
76
77 cxgb3i_rx_debug("conn 0x%p, skb 0x%p, len %u, flag 0x%x.\n",
78 conn, skb, skb->len, skb_ulp_mode(skb));
79
80 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
81 iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
82 return -EIO;
83 }
84
85 if (conn->hdrdgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_HCRC_ERROR)) {
86 iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
87 return -EIO;
88 }
89
90 if (conn->datadgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) {
91 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
92 return -EIO;
93 }
94
95 /* iscsi hdr */
96 rc = read_pdu_skb(conn, skb, 0, 0);
97 if (rc <= 0)
98 return rc;
99
100 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
101 return 0;
102
103 offset = rc;
104 if (conn->hdrdgst_en)
105 offset += ISCSI_DIGEST_SIZE;
106
107 /* iscsi data */
108 if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
109 cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, ddp'ed, "
110 "itt 0x%x.\n",
111 skb,
112 tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK,
113 tcp_conn->in.datalen,
114 ntohl(tcp_conn->in.hdr->itt));
115 offloaded = 1;
116 } else {
117 cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, NOT ddp'ed, "
118 "itt 0x%x.\n",
119 skb,
120 tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK,
121 tcp_conn->in.datalen,
122 ntohl(tcp_conn->in.hdr->itt));
123 offset += sizeof(struct cpl_iscsi_hdr_norss);
124 }
125
126 rc = read_pdu_skb(conn, skb, offset, offloaded);
127 if (rc < 0)
128 return rc;
129 else
130 return 0;
131}
132
133/*
134 * pdu transmit, interact with libiscsi_tcp
135 */
136static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
137{
138 u8 submode = 0;
139
140 if (hcrc)
141 submode |= 1;
142 if (dcrc)
143 submode |= 2;
144 skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode;
145}
146
147void cxgb3i_conn_cleanup_task(struct iscsi_task *task)
148{
149 struct iscsi_tcp_task *tcp_task = task->dd_data;
150
151 /* never reached the xmit task callout */
152 if (tcp_task->dd_data)
153 kfree_skb(tcp_task->dd_data);
154 tcp_task->dd_data = NULL;
155
156 /* MNC - Do we need a check in case this is called but
157 * cxgb3i_conn_alloc_pdu has never been called on the task */
158 cxgb3i_release_itt(task, task->hdr_itt);
159 iscsi_tcp_cleanup_task(task);
160}
161
162/*
163 * We do not support ahs yet
164 */
165int cxgb3i_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
166{
167 struct iscsi_tcp_task *tcp_task = task->dd_data;
168 struct sk_buff *skb;
169
170 task->hdr = NULL;
171 /* always allocate rooms for AHS */
172 skb = alloc_skb(sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE +
173 TX_HEADER_LEN, GFP_ATOMIC);
174 if (!skb)
175 return -ENOMEM;
176
177 cxgb3i_tx_debug("task 0x%p, opcode 0x%x, skb 0x%p.\n",
178 task, opcode, skb);
179
180 tcp_task->dd_data = skb;
181 skb_reserve(skb, TX_HEADER_LEN);
182 task->hdr = (struct iscsi_hdr *)skb->data;
183 task->hdr_max = sizeof(struct iscsi_hdr);
184
185 /* data_out uses scsi_cmd's itt */
186 if (opcode != ISCSI_OP_SCSI_DATA_OUT)
187 cxgb3i_reserve_itt(task, &task->hdr->itt);
188
189 return 0;
190}
191
192int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
193 unsigned int count)
194{
195 struct iscsi_tcp_task *tcp_task = task->dd_data;
196 struct sk_buff *skb = tcp_task->dd_data;
197 struct iscsi_conn *conn = task->conn;
198 struct page *pg;
199 unsigned int datalen = count;
200 int i, padlen = iscsi_padding(count);
201 skb_frag_t *frag;
202
203 cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n",
204 task, task->sc, offset, count, skb);
205
206 skb_put(skb, task->hdr_len);
207 tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
208 if (!count)
209 return 0;
210
211 if (task->sc) {
212 struct scatterlist *sg;
213 struct scsi_data_buffer *sdb;
214 unsigned int sgoffset = offset;
215 struct page *sgpg;
216 unsigned int sglen;
217
218 sdb = scsi_out(task->sc);
219 sg = sdb->table.sgl;
220
221 for_each_sg(sdb->table.sgl, sg, sdb->table.nents, i) {
222 cxgb3i_tx_debug("sg %d, page 0x%p, len %u offset %u\n",
223 i, sg_page(sg), sg->length, sg->offset);
224
225 if (sgoffset < sg->length)
226 break;
227 sgoffset -= sg->length;
228 }
229 sgpg = sg_page(sg);
230 sglen = sg->length - sgoffset;
231
232 do {
233 int j = skb_shinfo(skb)->nr_frags;
234 unsigned int copy;
235
236 if (!sglen) {
237 sg = sg_next(sg);
238 sgpg = sg_page(sg);
239 sgoffset = 0;
240 sglen = sg->length;
241 ++i;
242 }
243 copy = min(sglen, datalen);
244 if (j && skb_can_coalesce(skb, j, sgpg,
245 sg->offset + sgoffset)) {
246 skb_shinfo(skb)->frags[j - 1].size += copy;
247 } else {
248 get_page(sgpg);
249 skb_fill_page_desc(skb, j, sgpg,
250 sg->offset + sgoffset, copy);
251 }
252 sgoffset += copy;
253 sglen -= copy;
254 datalen -= copy;
255 } while (datalen);
256 } else {
257 pg = virt_to_page(task->data);
258
259 while (datalen) {
260 i = skb_shinfo(skb)->nr_frags;
261 frag = &skb_shinfo(skb)->frags[i];
262
263 get_page(pg);
264 frag->page = pg;
265 frag->page_offset = 0;
266 frag->size = min((unsigned int)PAGE_SIZE, datalen);
267
268 skb_shinfo(skb)->nr_frags++;
269 datalen -= frag->size;
270 pg++;
271 }
272 }
273
274 if (padlen) {
275 i = skb_shinfo(skb)->nr_frags;
276 frag = &skb_shinfo(skb)->frags[i];
277 frag->page = pad_page;
278 frag->page_offset = 0;
279 frag->size = padlen;
280 skb_shinfo(skb)->nr_frags++;
281 }
282
283 datalen = count + padlen;
284 skb->data_len += datalen;
285 skb->truesize += datalen;
286 skb->len += datalen;
287 return 0;
288}
289
290int cxgb3i_conn_xmit_pdu(struct iscsi_task *task)
291{
292 struct iscsi_tcp_task *tcp_task = task->dd_data;
293 struct sk_buff *skb = tcp_task->dd_data;
294 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
295 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
296 unsigned int datalen;
297 int err;
298
299 if (!skb)
300 return 0;
301
302 datalen = skb->data_len;
303 tcp_task->dd_data = NULL;
304 err = cxgb3i_c3cn_send_pdus(cconn->cep->c3cn, skb);
305 cxgb3i_tx_debug("task 0x%p, skb 0x%p, len %u/%u, rv %d.\n",
306 task, skb, skb->len, skb->data_len, err);
307 if (err > 0) {
308 int pdulen = err;
309
310 if (task->conn->hdrdgst_en)
311 pdulen += ISCSI_DIGEST_SIZE;
312 if (datalen && task->conn->datadgst_en)
313 pdulen += ISCSI_DIGEST_SIZE;
314
315 task->conn->txdata_octets += pdulen;
316 return 0;
317 }
318
319 if (err < 0 && err != -EAGAIN) {
320 kfree_skb(skb);
321 cxgb3i_tx_debug("itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
322 task->itt, skb, skb->len, skb->data_len, err);
323 iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
324 iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
325 return err;
326 }
327 /* reset skb to send when we are called again */
328 tcp_task->dd_data = skb;
329 return -EAGAIN;
330}
331
332int cxgb3i_pdu_init(void)
333{
334 pad_page = alloc_page(GFP_KERNEL);
335 if (!pad_page)
336 return -ENOMEM;
337 memset(page_address(pad_page), 0, PAGE_SIZE);
338 return 0;
339}
340
341void cxgb3i_pdu_cleanup(void)
342{
343 if (pad_page) {
344 __free_page(pad_page);
345 pad_page = NULL;
346 }
347}
348
349void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
350{
351 struct sk_buff *skb;
352 unsigned int read = 0;
353 struct iscsi_conn *conn = c3cn->user_data;
354 int err = 0;
355
356 cxgb3i_rx_debug("cn 0x%p.\n", c3cn);
357
358 read_lock(&c3cn->callback_lock);
359 if (unlikely(!conn || conn->suspend_rx)) {
360 cxgb3i_rx_debug("conn 0x%p, id %d, suspend_rx %lu!\n",
361 conn, conn ? conn->id : 0xFF,
362 conn ? conn->suspend_rx : 0xFF);
363 read_unlock(&c3cn->callback_lock);
364 return;
365 }
366 skb = skb_peek(&c3cn->receive_queue);
367 while (!err && skb) {
368 __skb_unlink(skb, &c3cn->receive_queue);
369 read += skb_ulp_pdulen(skb);
370 err = cxgb3i_conn_read_pdu_skb(conn, skb);
371 __kfree_skb(skb);
372 skb = skb_peek(&c3cn->receive_queue);
373 }
374 read_unlock(&c3cn->callback_lock);
375 if (c3cn) {
376 c3cn->copied_seq += read;
377 cxgb3i_c3cn_rx_credits(c3cn, read);
378 }
379 conn->rxdata_octets += read;
380}
381
382void cxgb3i_conn_tx_open(struct s3_conn *c3cn)
383{
384 struct iscsi_conn *conn = c3cn->user_data;
385
386 cxgb3i_tx_debug("cn 0x%p.\n", c3cn);
387 if (conn) {
388 cxgb3i_tx_debug("cn 0x%p, cid %d.\n", c3cn, conn->id);
389 scsi_queue_work(conn->session->host, &conn->xmitwork);
390 }
391}
392
393void cxgb3i_conn_closing(struct s3_conn *c3cn)
394{
395 struct iscsi_conn *conn;
396
397 read_lock(&c3cn->callback_lock);
398 conn = c3cn->user_data;
399 if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
400 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
401 read_unlock(&c3cn->callback_lock);
402}
diff --git a/drivers/scsi/cxgb3i/cxgb3i_pdu.h b/drivers/scsi/cxgb3i/cxgb3i_pdu.h
new file mode 100644
index 000000000000..a3f685cc2362
--- /dev/null
+++ b/drivers/scsi/cxgb3i/cxgb3i_pdu.h
@@ -0,0 +1,59 @@
1/*
2 * cxgb3i_ulp2.h: Chelsio S3xx iSCSI driver.
3 *
4 * Copyright (c) 2008 Chelsio Communications, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 *
10 * Written by: Karen Xie (kxie@chelsio.com)
11 */
12
13#ifndef __CXGB3I_ULP2_PDU_H__
14#define __CXGB3I_ULP2_PDU_H__
15
16struct cpl_iscsi_hdr_norss {
17 union opcode_tid ot;
18 u16 pdu_len_ddp;
19 u16 len;
20 u32 seq;
21 u16 urg;
22 u8 rsvd;
23 u8 status;
24};
25
26struct cpl_rx_data_ddp_norss {
27 union opcode_tid ot;
28 u16 urg;
29 u16 len;
30 u32 seq;
31 u32 nxt_seq;
32 u32 ulp_crc;
33 u32 ddp_status;
34};
35
36#define RX_DDP_STATUS_IPP_SHIFT 27 /* invalid pagepod */
37#define RX_DDP_STATUS_TID_SHIFT 26 /* tid mismatch */
38#define RX_DDP_STATUS_COLOR_SHIFT 25 /* color mismatch */
39#define RX_DDP_STATUS_OFFSET_SHIFT 24 /* offset mismatch */
40#define RX_DDP_STATUS_ULIMIT_SHIFT 23 /* ulimit error */
41#define RX_DDP_STATUS_TAG_SHIFT 22 /* tag mismatch */
42#define RX_DDP_STATUS_DCRC_SHIFT 21 /* dcrc error */
43#define RX_DDP_STATUS_HCRC_SHIFT 20 /* hcrc error */
44#define RX_DDP_STATUS_PAD_SHIFT 19 /* pad error */
45#define RX_DDP_STATUS_PPP_SHIFT 18 /* pagepod parity error */
46#define RX_DDP_STATUS_LLIMIT_SHIFT 17 /* llimit error */
47#define RX_DDP_STATUS_DDP_SHIFT 16 /* ddp'able */
48#define RX_DDP_STATUS_PMM_SHIFT 15 /* pagepod mismatch */
49
50#define ULP2_FLAG_DATA_READY 0x1
51#define ULP2_FLAG_DATA_DDPED 0x2
52#define ULP2_FLAG_HCRC_ERROR 0x10
53#define ULP2_FLAG_DCRC_ERROR 0x20
54#define ULP2_FLAG_PAD_ERROR 0x40
55
56void cxgb3i_conn_closing(struct s3_conn *);
57void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn);
58void cxgb3i_conn_tx_open(struct s3_conn *c3cn);
59#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-28 00:55:34 -0500