aboutsummaryrefslogtreecommitdiffstats
path: root/sound/pci/echoaudio/echoaudio_gml.c
diff options
context:
space:
mode:
authorGiuliano Pochini <pochini@shiny.it>2006-06-28 07:53:41 -0400
committerJaroslav Kysela <perex@suse.cz>2006-06-28 13:31:20 -0400
commitdd7b254d8dd3a9528f423ac3bf875e6f0c8da561 (patch)
tree923ac13451c796b730f21c7260beba9f74acff63 /sound/pci/echoaudio/echoaudio_gml.c
parentcb9d24e4349013628259b5fee97e692173731b07 (diff)
[ALSA] Add echoaudio sound drivers
From: Giuliano Pochini <pochini@shiny.it>Add echoaudio sound drivers (darla20, darla24, echo3g, gina20, gina24, indigo, indigodj, indigoio, layla20, lala24, mia, mona) Signed-off-by: Takashi Iwai <tiwai@suse.de> Signed-off-by: Jaroslav Kysela <perex@suse.cz>
Diffstat (limited to 'sound/pci/echoaudio/echoaudio_gml.c')
-rw-r--r--sound/pci/echoaudio/echoaudio_gml.c198
1 files changed, 198 insertions, 0 deletions
diff --git a/sound/pci/echoaudio/echoaudio_gml.c b/sound/pci/echoaudio/echoaudio_gml.c
new file mode 100644
index 000000000000..3aa37e76ebab
--- /dev/null
+++ b/sound/pci/echoaudio/echoaudio_gml.c
@@ -0,0 +1,198 @@
1/****************************************************************************
2
3 Copyright Echo Digital Audio Corporation (c) 1998 - 2004
4 All rights reserved
5 www.echoaudio.com
6
7 This file is part of Echo Digital Audio's generic driver library.
8
9 Echo Digital Audio's generic driver library is free software;
10 you can redistribute it and/or modify it under the terms of
11 the GNU General Public License as published by the Free Software
12 Foundation.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 59 Temple Place - Suite 330, Boston,
22 MA 02111-1307, USA.
23
24 *************************************************************************
25
26 Translation from C++ and adaptation for use in ALSA-Driver
27 were made by Giuliano Pochini <pochini@shiny.it>
28
29****************************************************************************/
30
31
32/* These functions are common for Gina24, Layla24 and Mona cards */
33
34
35/* ASIC status check - some cards have one or two ASICs that need to be
36loaded. Once that load is complete, this function is called to see if
37the load was successful.
38If this load fails, it does not necessarily mean that the hardware is
39defective - the external box may be disconnected or turned off. */
40static int check_asic_status(struct echoaudio *chip)
41{
42 u32 asic_status;
43
44 send_vector(chip, DSP_VC_TEST_ASIC);
45
46 /* The DSP will return a value to indicate whether or not the
47 ASIC is currently loaded */
48 if (read_dsp(chip, &asic_status) < 0) {
49 DE_INIT(("check_asic_status: failed on read_dsp\n"));
50 chip->asic_loaded = FALSE;
51 return -EIO;
52 }
53
54 chip->asic_loaded = (asic_status == ASIC_ALREADY_LOADED);
55 return chip->asic_loaded ? 0 : -EIO;
56}
57
58
59
60/* Most configuration of Gina24, Layla24, or Mona is accomplished by writing
61the control register. write_control_reg sends the new control register
62value to the DSP. */
63static int write_control_reg(struct echoaudio *chip, u32 value, char force)
64{
65 /* Handle the digital input auto-mute */
66 if (chip->digital_in_automute)
67 value |= GML_DIGITAL_IN_AUTO_MUTE;
68 else
69 value &= ~GML_DIGITAL_IN_AUTO_MUTE;
70
71 DE_ACT(("write_control_reg: 0x%x\n", value));
72
73 /* Write the control register */
74 value = cpu_to_le32(value);
75 if (value != chip->comm_page->control_register || force) {
76 if (wait_handshake(chip))
77 return -EIO;
78 chip->comm_page->control_register = value;
79 clear_handshake(chip);
80 return send_vector(chip, DSP_VC_WRITE_CONTROL_REG);
81 }
82 return 0;
83}
84
85
86
87/* Gina24, Layla24, and Mona support digital input auto-mute. If the digital
88input auto-mute is enabled, the DSP will only enable the digital inputs if
89the card is syncing to a valid clock on the ADAT or S/PDIF inputs.
90If the auto-mute is disabled, the digital inputs are enabled regardless of
91what the input clock is set or what is connected. */
92static int set_input_auto_mute(struct echoaudio *chip, int automute)
93{
94 DE_ACT(("set_input_auto_mute %d\n", automute));
95
96 chip->digital_in_automute = automute;
97
98 /* Re-set the input clock to the current value - indirectly causes
99 the auto-mute flag to be sent to the DSP */
100 return set_input_clock(chip, chip->input_clock);
101}
102
103
104
105/* S/PDIF coax / S/PDIF optical / ADAT - switch */
106static int set_digital_mode(struct echoaudio *chip, u8 mode)
107{
108 u8 previous_mode;
109 int err, i, o;
110
111 if (chip->bad_board)
112 return -EIO;
113
114 /* All audio channels must be closed before changing the digital mode */
115 snd_assert(!chip->pipe_alloc_mask, return -EAGAIN);
116
117 snd_assert(chip->digital_modes & (1 << mode), return -EINVAL);
118
119 previous_mode = chip->digital_mode;
120 err = dsp_set_digital_mode(chip, mode);
121
122 /* If we successfully changed the digital mode from or to ADAT,
123 then make sure all output, input and monitor levels are
124 updated by the DSP comm object. */
125 if (err >= 0 && previous_mode != mode &&
126 (previous_mode == DIGITAL_MODE_ADAT || mode == DIGITAL_MODE_ADAT)) {
127 spin_lock_irq(&chip->lock);
128 for (o = 0; o < num_busses_out(chip); o++)
129 for (i = 0; i < num_busses_in(chip); i++)
130 set_monitor_gain(chip, o, i,
131 chip->monitor_gain[o][i]);
132
133#ifdef ECHOCARD_HAS_INPUT_GAIN
134 for (i = 0; i < num_busses_in(chip); i++)
135 set_input_gain(chip, i, chip->input_gain[i]);
136 update_input_line_level(chip);
137#endif
138
139 for (o = 0; o < num_busses_out(chip); o++)
140 set_output_gain(chip, o, chip->output_gain[o]);
141 update_output_line_level(chip);
142 spin_unlock_irq(&chip->lock);
143 }
144
145 return err;
146}
147
148
149
150/* Set the S/PDIF output format */
151static int set_professional_spdif(struct echoaudio *chip, char prof)
152{
153 u32 control_reg;
154 int err;
155
156 /* Clear the current S/PDIF flags */
157 control_reg = le32_to_cpu(chip->comm_page->control_register);
158 control_reg &= GML_SPDIF_FORMAT_CLEAR_MASK;
159
160 /* Set the new S/PDIF flags depending on the mode */
161 control_reg |= GML_SPDIF_TWO_CHANNEL | GML_SPDIF_24_BIT |
162 GML_SPDIF_COPY_PERMIT;
163 if (prof) {
164 /* Professional mode */
165 control_reg |= GML_SPDIF_PRO_MODE;
166
167 switch (chip->sample_rate) {
168 case 32000:
169 control_reg |= GML_SPDIF_SAMPLE_RATE0 |
170 GML_SPDIF_SAMPLE_RATE1;
171 break;
172 case 44100:
173 control_reg |= GML_SPDIF_SAMPLE_RATE0;
174 break;
175 case 48000:
176 control_reg |= GML_SPDIF_SAMPLE_RATE1;
177 break;
178 }
179 } else {
180 /* Consumer mode */
181 switch (chip->sample_rate) {
182 case 32000:
183 control_reg |= GML_SPDIF_SAMPLE_RATE0 |
184 GML_SPDIF_SAMPLE_RATE1;
185 break;
186 case 48000:
187 control_reg |= GML_SPDIF_SAMPLE_RATE1;
188 break;
189 }
190 }
191
192 if ((err = write_control_reg(chip, control_reg, FALSE)))
193 return err;
194 chip->professional_spdif = prof;
195 DE_ACT(("set_professional_spdif to %s\n",
196 prof ? "Professional" : "Consumer"));
197 return 0;
198}
5 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365
/*******************************************************************************
 * This file contains iSCSI extentions for RDMA (iSER) Verbs
 *
 * (c) Copyright 2013 RisingTide Systems LLC.
 *
 * Nicholas A. Bellinger <nab@linux-iscsi.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 ****************************************************************************/

#include <linux/string.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <rdma/ib_verbs.h>
#include <rdma/rdma_cm.h>
#include <target/target_core_base.h>
#include <target/target_core_fabric.h>
#include <target/iscsi/iscsi_transport.h>

#include "isert_proto.h"
#include "ib_isert.h"

#define	ISERT_MAX_CONN		8
#define ISER_MAX_RX_CQ_LEN	(ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
#define ISER_MAX_TX_CQ_LEN	(ISERT_QP_MAX_REQ_DTOS  * ISERT_MAX_CONN)

static DEFINE_MUTEX(device_list_mutex);
static LIST_HEAD(device_list);
static struct workqueue_struct *isert_rx_wq;
static struct workqueue_struct *isert_comp_wq;
static struct kmem_cache *isert_cmd_cache;

static void
isert_qp_event_callback(struct ib_event *e, void *context)
{
	struct isert_conn *isert_conn = (struct isert_conn *)context;

	pr_err("isert_qp_event_callback event: %d\n", e->event);
	switch (e->event) {
	case IB_EVENT_COMM_EST:
		rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST);
		break;
	case IB_EVENT_QP_LAST_WQE_REACHED:
		pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n");
		break;
	default:
		break;
	}
}

static int
isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
{
	int ret;

	ret = ib_query_device(ib_dev, devattr);
	if (ret) {
		pr_err("ib_query_device() failed: %d\n", ret);
		return ret;
	}
	pr_debug("devattr->max_sge: %d\n", devattr->max_sge);
	pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);

	return 0;
}

static int
isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
{
	struct isert_device *device = isert_conn->conn_device;
	struct ib_qp_init_attr attr;
	struct ib_device_attr devattr;
	int ret, index, min_index = 0;

	memset(&devattr, 0, sizeof(struct ib_device_attr));
	ret = isert_query_device(cma_id->device, &devattr);
	if (ret)
		return ret;

	mutex_lock(&device_list_mutex);
	for (index = 0; index < device->cqs_used; index++)
		if (device->cq_active_qps[index] <
		    device->cq_active_qps[min_index])
			min_index = index;
	device->cq_active_qps[min_index]++;
	pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index);
	mutex_unlock(&device_list_mutex);

	memset(&attr, 0, sizeof(struct ib_qp_init_attr));
	attr.event_handler = isert_qp_event_callback;
	attr.qp_context = isert_conn;
	attr.send_cq = device->dev_tx_cq[min_index];
	attr.recv_cq = device->dev_rx_cq[min_index];
	attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
	attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS;
	/*
	 * FIXME: Use devattr.max_sge - 2 for max_send_sge as
	 * work-around for RDMA_READ..
	 */
	attr.cap.max_send_sge = devattr.max_sge - 2;
	isert_conn->max_sge = attr.cap.max_send_sge;

	attr.cap.max_recv_sge = 1;
	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
	attr.qp_type = IB_QPT_RC;

	pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
		 cma_id->device);
	pr_debug("isert_conn_setup_qp conn_pd->device: %p\n",
		 isert_conn->conn_pd->device);

	ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr);
	if (ret) {
		pr_err("rdma_create_qp failed for cma_id %d\n", ret);
		return ret;
	}
	isert_conn->conn_qp = cma_id->qp;
	pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n");

	return 0;
}

static void
isert_cq_event_callback(struct ib_event *e, void *context)
{
	pr_debug("isert_cq_event_callback event: %d\n", e->event);
}

static int
isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct iser_rx_desc *rx_desc;
	struct ib_sge *rx_sg;
	u64 dma_addr;
	int i, j;

	isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
				sizeof(struct iser_rx_desc), GFP_KERNEL);
	if (!isert_conn->conn_rx_descs)
		goto fail;

	rx_desc = isert_conn->conn_rx_descs;

	for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
		dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
					ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
		if (ib_dma_mapping_error(ib_dev, dma_addr))
			goto dma_map_fail;

		rx_desc->dma_addr = dma_addr;

		rx_sg = &rx_desc->rx_sg;
		rx_sg->addr = rx_desc->dma_addr;
		rx_sg->length = ISER_RX_PAYLOAD_SIZE;
		rx_sg->lkey = isert_conn->conn_mr->lkey;
	}

	isert_conn->conn_rx_desc_head = 0;
	return 0;

dma_map_fail:
	rx_desc = isert_conn->conn_rx_descs;
	for (j = 0; j < i; j++, rx_desc++) {
		ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
	}
	kfree(isert_conn->conn_rx_descs);
	isert_conn->conn_rx_descs = NULL;
fail:
	return -ENOMEM;
}

static void
isert_free_rx_descriptors(struct isert_conn *isert_conn)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct iser_rx_desc *rx_desc;
	int i;

	if (!isert_conn->conn_rx_descs)
		return;

	rx_desc = isert_conn->conn_rx_descs;
	for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
		ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
	}

	kfree(isert_conn->conn_rx_descs);
	isert_conn->conn_rx_descs = NULL;
}

static void isert_cq_tx_callback(struct ib_cq *, void *);
static void isert_cq_rx_callback(struct ib_cq *, void *);

static int
isert_create_device_ib_res(struct isert_device *device)
{
	struct ib_device *ib_dev = device->ib_device;
	struct isert_cq_desc *cq_desc;
	int ret = 0, i, j;

	device->cqs_used = min_t(int, num_online_cpus(),
				 device->ib_device->num_comp_vectors);
	device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used);
	pr_debug("Using %d CQs, device %s supports %d vectors\n",
		 device->cqs_used, device->ib_device->name,
		 device->ib_device->num_comp_vectors);
	device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) *
				device->cqs_used, GFP_KERNEL);
	if (!device->cq_desc) {
		pr_err("Unable to allocate device->cq_desc\n");
		return -ENOMEM;
	}
	cq_desc = device->cq_desc;

	device->dev_pd = ib_alloc_pd(ib_dev);
	if (IS_ERR(device->dev_pd)) {
		ret = PTR_ERR(device->dev_pd);
		pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret);
		goto out_cq_desc;
	}

	for (i = 0; i < device->cqs_used; i++) {
		cq_desc[i].device = device;
		cq_desc[i].cq_index = i;

		device->dev_rx_cq[i] = ib_create_cq(device->ib_device,
						isert_cq_rx_callback,
						isert_cq_event_callback,
						(void *)&cq_desc[i],
						ISER_MAX_RX_CQ_LEN, i);
		if (IS_ERR(device->dev_rx_cq[i]))
			goto out_cq;

		device->dev_tx_cq[i] = ib_create_cq(device->ib_device,
						isert_cq_tx_callback,
						isert_cq_event_callback,
						(void *)&cq_desc[i],
						ISER_MAX_TX_CQ_LEN, i);
		if (IS_ERR(device->dev_tx_cq[i]))
			goto out_cq;

		if (ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP))
			goto out_cq;

		if (ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP))
			goto out_cq;
	}

	device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE);
	if (IS_ERR(device->dev_mr)) {
		ret = PTR_ERR(device->dev_mr);
		pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret);
		goto out_cq;
	}

	return 0;

out_cq:
	for (j = 0; j < i; j++) {
		cq_desc = &device->cq_desc[j];

		if (device->dev_rx_cq[j]) {
			cancel_work_sync(&cq_desc->cq_rx_work);
			ib_destroy_cq(device->dev_rx_cq[j]);
		}
		if (device->dev_tx_cq[j]) {
			cancel_work_sync(&cq_desc->cq_tx_work);
			ib_destroy_cq(device->dev_tx_cq[j]);
		}
	}
	ib_dealloc_pd(device->dev_pd);

out_cq_desc:
	kfree(device->cq_desc);

	return ret;
}

static void
isert_free_device_ib_res(struct isert_device *device)
{
	struct isert_cq_desc *cq_desc;
	int i;

	for (i = 0; i < device->cqs_used; i++) {
		cq_desc = &device->cq_desc[i];

		cancel_work_sync(&cq_desc->cq_rx_work);
		cancel_work_sync(&cq_desc->cq_tx_work);
		ib_destroy_cq(device->dev_rx_cq[i]);
		ib_destroy_cq(device->dev_tx_cq[i]);
		device->dev_rx_cq[i] = NULL;
		device->dev_tx_cq[i] = NULL;
	}

	ib_dereg_mr(device->dev_mr);
	ib_dealloc_pd(device->dev_pd);
	kfree(device->cq_desc);
}

static void
isert_device_try_release(struct isert_device *device)
{
	mutex_lock(&device_list_mutex);
	device->refcount--;
	if (!device->refcount) {
		isert_free_device_ib_res(device);
		list_del(&device->dev_node);
		kfree(device);
	}
	mutex_unlock(&device_list_mutex);
}

static struct isert_device *
isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id)
{
	struct isert_device *device;
	int ret;

	mutex_lock(&device_list_mutex);
	list_for_each_entry(device, &device_list, dev_node) {
		if (device->ib_device->node_guid == cma_id->device->node_guid) {
			device->refcount++;
			mutex_unlock(&device_list_mutex);
			return device;
		}
	}

	device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
	if (!device) {
		mutex_unlock(&device_list_mutex);
		return ERR_PTR(-ENOMEM);
	}

	INIT_LIST_HEAD(&device->dev_node);

	device->ib_device = cma_id->device;
	ret = isert_create_device_ib_res(device);
	if (ret) {
		kfree(device);
		mutex_unlock(&device_list_mutex);
		return ERR_PTR(ret);
	}

	device->refcount++;
	list_add_tail(&device->dev_node, &device_list);
	mutex_unlock(&device_list_mutex);

	return device;
}

static int
isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
	struct iscsi_np *np = cma_id->context;
	struct isert_np *isert_np = np->np_context;
	struct isert_conn *isert_conn;
	struct isert_device *device;
	struct ib_device *ib_dev = cma_id->device;
	int ret = 0;

	pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
		 cma_id, cma_id->context);

	isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
	if (!isert_conn) {
		pr_err("Unable to allocate isert_conn\n");
		return -ENOMEM;
	}
	isert_conn->state = ISER_CONN_INIT;
	INIT_LIST_HEAD(&isert_conn->conn_accept_node);
	init_completion(&isert_conn->conn_login_comp);
	init_waitqueue_head(&isert_conn->conn_wait);
	init_waitqueue_head(&isert_conn->conn_wait_comp_err);
	kref_init(&isert_conn->conn_kref);
	kref_get(&isert_conn->conn_kref);
	mutex_init(&isert_conn->conn_mutex);

	cma_id->context = isert_conn;
	isert_conn->conn_cm_id = cma_id;
	isert_conn->responder_resources = event->param.conn.responder_resources;
	isert_conn->initiator_depth = event->param.conn.initiator_depth;
	pr_debug("Using responder_resources: %u initiator_depth: %u\n",
		 isert_conn->responder_resources, isert_conn->initiator_depth);

	isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
					ISER_RX_LOGIN_SIZE, GFP_KERNEL);
	if (!isert_conn->login_buf) {
		pr_err("Unable to allocate isert_conn->login_buf\n");
		ret = -ENOMEM;
		goto out;
	}

	isert_conn->login_req_buf = isert_conn->login_buf;
	isert_conn->login_rsp_buf = isert_conn->login_buf +
				    ISCSI_DEF_MAX_RECV_SEG_LEN;
	pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
		 isert_conn->login_buf, isert_conn->login_req_buf,
		 isert_conn->login_rsp_buf);

	isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
				(void *)isert_conn->login_req_buf,
				ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);

	ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
	if (ret) {
		pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
		       ret);
		isert_conn->login_req_dma = 0;
		goto out_login_buf;
	}

	isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
					(void *)isert_conn->login_rsp_buf,
					ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);

	ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
	if (ret) {
		pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
		       ret);
		isert_conn->login_rsp_dma = 0;
		goto out_req_dma_map;
	}

	device = isert_device_find_by_ib_dev(cma_id);
	if (IS_ERR(device)) {
		ret = PTR_ERR(device);
		goto out_rsp_dma_map;
	}

	isert_conn->conn_device = device;
	isert_conn->conn_pd = device->dev_pd;
	isert_conn->conn_mr = device->dev_mr;

	ret = isert_conn_setup_qp(isert_conn, cma_id);
	if (ret)
		goto out_conn_dev;

	mutex_lock(&isert_np->np_accept_mutex);
	list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node);
	mutex_unlock(&isert_np->np_accept_mutex);

	pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np);
	wake_up(&isert_np->np_accept_wq);
	return 0;

out_conn_dev:
	isert_device_try_release(device);
out_rsp_dma_map:
	ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
			    ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
out_req_dma_map:
	ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
			    ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
out_login_buf:
	kfree(isert_conn->login_buf);
out:
	kfree(isert_conn);
	return ret;
}

static void
isert_connect_release(struct isert_conn *isert_conn)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct isert_device *device = isert_conn->conn_device;
	int cq_index;

	pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");

	if (isert_conn->conn_qp) {
		cq_index = ((struct isert_cq_desc *)
			isert_conn->conn_qp->recv_cq->cq_context)->cq_index;
		pr_debug("isert_connect_release: cq_index: %d\n", cq_index);
		isert_conn->conn_device->cq_active_qps[cq_index]--;

		rdma_destroy_qp(isert_conn->conn_cm_id);
	}

	isert_free_rx_descriptors(isert_conn);
	rdma_destroy_id(isert_conn->conn_cm_id);

	if (isert_conn->login_buf) {
		ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
				    ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
		ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
				    ISCSI_DEF_MAX_RECV_SEG_LEN,
				    DMA_FROM_DEVICE);
		kfree(isert_conn->login_buf);
	}
	kfree(isert_conn);

	if (device)
		isert_device_try_release(device);

	pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n");
}

static void
isert_connected_handler(struct rdma_cm_id *cma_id)
{
	return;
}

static void
isert_release_conn_kref(struct kref *kref)
{
	struct isert_conn *isert_conn = container_of(kref,
				struct isert_conn, conn_kref);

	pr_debug("Calling isert_connect_release for final kref %s/%d\n",
		 current->comm, current->pid);

	isert_connect_release(isert_conn);
}

static void
isert_put_conn(struct isert_conn *isert_conn)
{
	kref_put(&isert_conn->conn_kref, isert_release_conn_kref);
}

static void
isert_disconnect_work(struct work_struct *work)
{
	struct isert_conn *isert_conn = container_of(work,
				struct isert_conn, conn_logout_work);

	pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
	mutex_lock(&isert_conn->conn_mutex);
	isert_conn->state = ISER_CONN_DOWN;

	if (isert_conn->post_recv_buf_count == 0 &&
	    atomic_read(&isert_conn->post_send_buf_count) == 0) {
		pr_debug("Calling wake_up(&isert_conn->conn_wait);\n");
		mutex_unlock(&isert_conn->conn_mutex);
		goto wake_up;
	}
	if (!isert_conn->conn_cm_id) {
		mutex_unlock(&isert_conn->conn_mutex);
		isert_put_conn(isert_conn);
		return;
	}
	if (!isert_conn->logout_posted) {
		pr_debug("Calling rdma_disconnect for !logout_posted from"
			 " isert_disconnect_work\n");
		rdma_disconnect(isert_conn->conn_cm_id);
		mutex_unlock(&isert_conn->conn_mutex);
		iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
		goto wake_up;
	}
	mutex_unlock(&isert_conn->conn_mutex);

wake_up:
	wake_up(&isert_conn->conn_wait);
	isert_put_conn(isert_conn);
}

static void
isert_disconnected_handler(struct rdma_cm_id *cma_id)
{
	struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context;

	INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work);
	schedule_work(&isert_conn->conn_logout_work);
}

static int
isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
	int ret = 0;

	pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n",
		 event->event, event->status, cma_id->context, cma_id);

	switch (event->event) {
	case RDMA_CM_EVENT_CONNECT_REQUEST:
		pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n");
		ret = isert_connect_request(cma_id, event);
		break;
	case RDMA_CM_EVENT_ESTABLISHED:
		pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n");
		isert_connected_handler(cma_id);
		break;
	case RDMA_CM_EVENT_DISCONNECTED:
		pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n");
		isert_disconnected_handler(cma_id);
		break;
	case RDMA_CM_EVENT_DEVICE_REMOVAL:
	case RDMA_CM_EVENT_ADDR_CHANGE:
		break;
	case RDMA_CM_EVENT_CONNECT_ERROR:
	default:
		pr_err("Unknown RDMA CMA event: %d\n", event->event);
		break;
	}

	if (ret != 0) {
		pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n",
		       event->event, ret);
		dump_stack();
	}

	return ret;
}

static int
isert_post_recv(struct isert_conn *isert_conn, u32 count)
{
	struct ib_recv_wr *rx_wr, *rx_wr_failed;
	int i, ret;
	unsigned int rx_head = isert_conn->conn_rx_desc_head;
	struct iser_rx_desc *rx_desc;

	for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) {
		rx_desc		= &isert_conn->conn_rx_descs[rx_head];
		rx_wr->wr_id	= (unsigned long)rx_desc;
		rx_wr->sg_list	= &rx_desc->rx_sg;
		rx_wr->num_sge	= 1;
		rx_wr->next	= rx_wr + 1;
		rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1);
	}

	rx_wr--;
	rx_wr->next = NULL; /* mark end of work requests list */

	isert_conn->post_recv_buf_count += count;
	ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr,
				&rx_wr_failed);
	if (ret) {
		pr_err("ib_post_recv() failed with ret: %d\n", ret);
		isert_conn->post_recv_buf_count -= count;
	} else {
		pr_debug("isert_post_recv(): Posted %d RX buffers\n", count);
		isert_conn->conn_rx_desc_head = rx_head;
	}
	return ret;
}

static int
isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct ib_send_wr send_wr, *send_wr_failed;
	int ret;

	ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
				      ISER_HEADERS_LEN, DMA_TO_DEVICE);

	send_wr.next	= NULL;
	send_wr.wr_id	= (unsigned long)tx_desc;
	send_wr.sg_list	= tx_desc->tx_sg;
	send_wr.num_sge	= tx_desc->num_sge;
	send_wr.opcode	= IB_WR_SEND;
	send_wr.send_flags = IB_SEND_SIGNALED;

	atomic_inc(&isert_conn->post_send_buf_count);

	ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed);
	if (ret) {
		pr_err("ib_post_send() failed, ret: %d\n", ret);
		atomic_dec(&isert_conn->post_send_buf_count);
	}

	return ret;
}

static void
isert_create_send_desc(struct isert_conn *isert_conn,
		       struct isert_cmd *isert_cmd,
		       struct iser_tx_desc *tx_desc)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;

	ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
				   ISER_HEADERS_LEN, DMA_TO_DEVICE);

	memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
	tx_desc->iser_header.flags = ISER_VER;

	tx_desc->num_sge = 1;
	tx_desc->isert_cmd = isert_cmd;

	if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) {
		tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
		pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc);
	}
}

static int
isert_init_tx_hdrs(struct isert_conn *isert_conn,
		   struct iser_tx_desc *tx_desc)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	u64 dma_addr;

	dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
			ISER_HEADERS_LEN, DMA_TO_DEVICE);
	if (ib_dma_mapping_error(ib_dev, dma_addr)) {
		pr_err("ib_dma_mapping_error() failed\n");
		return -ENOMEM;
	}

	tx_desc->dma_addr = dma_addr;
	tx_desc->tx_sg[0].addr	= tx_desc->dma_addr;
	tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
	tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;

	pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u"
		 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr,
		 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey);

	return 0;
}

static void
isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr)
{
	isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
	send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
	send_wr->opcode = IB_WR_SEND;
	send_wr->send_flags = IB_SEND_SIGNALED;
	send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0];
	send_wr->num_sge = isert_cmd->tx_desc.num_sge;
}

static int
isert_rdma_post_recvl(struct isert_conn *isert_conn)
{
	struct ib_recv_wr rx_wr, *rx_wr_fail;
	struct ib_sge sge;
	int ret;

	memset(&sge, 0, sizeof(struct ib_sge));
	sge.addr = isert_conn->login_req_dma;
	sge.length = ISER_RX_LOGIN_SIZE;
	sge.lkey = isert_conn->conn_mr->lkey;

	pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n",
		sge.addr, sge.length, sge.lkey);

	memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
	rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf;
	rx_wr.sg_list = &sge;
	rx_wr.num_sge = 1;

	isert_conn->post_recv_buf_count++;
	ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail);
	if (ret) {
		pr_err("ib_post_recv() failed: %d\n", ret);
		isert_conn->post_recv_buf_count--;
	}

	pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n");
	return ret;
}

static int
isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
		   u32 length)
{
	struct isert_conn *isert_conn = conn->context;
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc;
	int ret;

	isert_create_send_desc(isert_conn, NULL, tx_desc);

	memcpy(&tx_desc->iscsi_header, &login->rsp[0],
	       sizeof(struct iscsi_hdr));

	isert_init_tx_hdrs(isert_conn, tx_desc);

	if (length > 0) {
		struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];

		ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
					   length, DMA_TO_DEVICE);

		memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);

		ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
					      length, DMA_TO_DEVICE);

		tx_dsg->addr	= isert_conn->login_rsp_dma;
		tx_dsg->length	= length;
		tx_dsg->lkey	= isert_conn->conn_mr->lkey;
		tx_desc->num_sge = 2;
	}
	if (!login->login_failed) {
		if (login->login_complete) {
			ret = isert_alloc_rx_descriptors(isert_conn);
			if (ret)
				return ret;

			ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX);
			if (ret)
				return ret;

			isert_conn->state = ISER_CONN_UP;
			goto post_send;
		}

		ret = isert_rdma_post_recvl(isert_conn);
		if (ret)
			return ret;
	}
post_send:
	ret = isert_post_send(isert_conn, tx_desc);
	if (ret)
		return ret;

	return 0;
}

static void
isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen,
		   struct isert_conn *isert_conn)
{
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_login *login = conn->conn_login;
	int size;

	if (!login) {
		pr_err("conn->conn_login is NULL\n");
		dump_stack();
		return;
	}

	if (login->first_request) {
		struct iscsi_login_req *login_req =
			(struct iscsi_login_req *)&rx_desc->iscsi_header;
		/*
		 * Setup the initial iscsi_login values from the leading
		 * login request PDU.
		 */
		login->leading_connection = (!login_req->tsih) ? 1 : 0;
		login->current_stage =
			(login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
			 >> 2;
		login->version_min	= login_req->min_version;
		login->version_max	= login_req->max_version;
		memcpy(login->isid, login_req->isid, 6);
		login->cmd_sn		= be32_to_cpu(login_req->cmdsn);
		login->init_task_tag	= login_req->itt;
		login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
		login->cid		= be16_to_cpu(login_req->cid);
		login->tsih		= be16_to_cpu(login_req->tsih);
	}

	memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);

	size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
	pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n",
		 size, rx_buflen, MAX_KEY_VALUE_PAIRS);
	memcpy(login->req_buf, &rx_desc->data[0], size);

	complete(&isert_conn->conn_login_comp);
}

static void
isert_release_cmd(struct iscsi_cmd *cmd)
{
	struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd,
						   iscsi_cmd);

	pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd);

	kfree(cmd->buf_ptr);
	kfree(cmd->tmr_req);

	kmem_cache_free(isert_cmd_cache, isert_cmd);
}

static struct iscsi_cmd
*isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp)
{
	struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
	struct isert_cmd *isert_cmd;

	isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp);
	if (!isert_cmd) {
		pr_err("Unable to allocate isert_cmd\n");
		return NULL;
	}
	isert_cmd->conn = isert_conn;
	isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd;

	return &isert_cmd->iscsi_cmd;
}

static int
isert_handle_scsi_cmd(struct isert_conn *isert_conn,
		      struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc,
		      unsigned char *buf)
{
	struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
	struct scatterlist *sg;
	int imm_data, imm_data_len, unsol_data, sg_nents, rc;
	bool dump_payload = false;

	rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
	if (rc < 0)
		return rc;

	imm_data = cmd->immediate_data;
	imm_data_len = cmd->first_burst_len;
	unsol_data = cmd->unsolicited_data;

	rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
	if (rc < 0) {
		return 0;
	} else if (rc > 0) {
		dump_payload = true;
		goto sequence_cmd;
	}

	if (!imm_data)
		return 0;

	sg = &cmd->se_cmd.t_data_sg[0];
	sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));

	pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
		 sg, sg_nents, &rx_desc->data[0], imm_data_len);

	sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len);

	cmd->write_data_done += imm_data_len;

	if (cmd->write_data_done == cmd->se_cmd.data_length) {
		spin_lock_bh(&cmd->istate_lock);
		cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
		cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
		spin_unlock_bh(&cmd->istate_lock);
	}

sequence_cmd:
	rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);

	if (!rc && dump_payload == false && unsol_data)
		iscsit_set_unsoliticed_dataout(cmd);

	return 0;
}

static int
isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
			   struct iser_rx_desc *rx_desc, unsigned char *buf)
{
	struct scatterlist *sg_start;
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_cmd *cmd = NULL;
	struct iscsi_data *hdr = (struct iscsi_data *)buf;
	u32 unsol_data_len = ntoh24(hdr->dlength);
	int rc, sg_nents, sg_off, page_off;

	rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
	if (rc < 0)
		return rc;
	else if (!cmd)
		return 0;
	/*
	 * FIXME: Unexpected unsolicited_data out
	 */
	if (!cmd->unsolicited_data) {
		pr_err("Received unexpected solicited data payload\n");
		dump_stack();
		return -1;
	}

	pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n",
		 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length);

	sg_off = cmd->write_data_done / PAGE_SIZE;
	sg_start = &cmd->se_cmd.t_data_sg[sg_off];
	sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
	page_off = cmd->write_data_done % PAGE_SIZE;
	/*
	 * FIXME: Non page-aligned unsolicited_data out
	 */
	if (page_off) {
		pr_err("Received unexpected non-page aligned data payload\n");
		dump_stack();
		return -1;
	}
	pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n",
		 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len);

	sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
			    unsol_data_len);

	rc = iscsit_check_dataout_payload(cmd, hdr, false);
	if (rc < 0)
		return rc;

	return 0;
}

static int
isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
		     struct iser_rx_desc *rx_desc, unsigned char *buf)
{
	struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
	int rc;

	rc = iscsit_setup_nop_out(conn, cmd, hdr);
	if (rc < 0)
		return rc;
	/*
	 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
	 */

	return iscsit_process_nop_out(conn, cmd, hdr);
}

static int
isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
		uint32_t read_stag, uint64_t read_va,
		uint32_t write_stag, uint64_t write_va)
{
	struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_cmd *cmd;
	struct isert_cmd *isert_cmd;
	int ret = -EINVAL;
	u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);

	switch (opcode) {
	case ISCSI_OP_SCSI_CMD:
		cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
		if (!cmd)
			break;

		isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd);
		isert_cmd->read_stag = read_stag;
		isert_cmd->read_va = read_va;
		isert_cmd->write_stag = write_stag;
		isert_cmd->write_va = write_va;

		ret = isert_handle_scsi_cmd(isert_conn, isert_cmd,
					rx_desc, (unsigned char *)hdr);
		break;
	case ISCSI_OP_NOOP_OUT:
		cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
		if (!cmd)
			break;

		isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd);
		ret = isert_handle_nop_out(isert_conn, isert_cmd,
					   rx_desc, (unsigned char *)hdr);
		break;
	case ISCSI_OP_SCSI_DATA_OUT:
		ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
						(unsigned char *)hdr);
		break;
	case ISCSI_OP_SCSI_TMFUNC:
		cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
		if (!cmd)
			break;

		ret = iscsit_handle_task_mgt_cmd(conn, cmd,
						(unsigned char *)hdr);
		break;
	case ISCSI_OP_LOGOUT:
		cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
		if (!cmd)
			break;

		ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
		if (ret > 0)
			wait_for_completion_timeout(&conn->conn_logout_comp,
						    SECONDS_FOR_LOGOUT_COMP *
						    HZ);
		break;
	default:
		pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
		dump_stack();
		break;
	}

	return ret;
}

static void
isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
{
	struct iser_hdr *iser_hdr = &rx_desc->iser_header;
	uint64_t read_va = 0, write_va = 0;
	uint32_t read_stag = 0, write_stag = 0;
	int rc;

	switch (iser_hdr->flags & 0xF0) {
	case ISCSI_CTRL:
		if (iser_hdr->flags & ISER_RSV) {
			read_stag = be32_to_cpu(iser_hdr->read_stag);
			read_va = be64_to_cpu(iser_hdr->read_va);
			pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n",
				 read_stag, (unsigned long long)read_va);
		}
		if (iser_hdr->flags & ISER_WSV) {
			write_stag = be32_to_cpu(iser_hdr->write_stag);
			write_va = be64_to_cpu(iser_hdr->write_va);
			pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n",
				 write_stag, (unsigned long long)write_va);
		}

		pr_debug("ISER ISCSI_CTRL PDU\n");
		break;
	case ISER_HELLO:
		pr_err("iSER Hello message\n");
		break;
	default:
		pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
		break;
	}

	rc = isert_rx_opcode(isert_conn, rx_desc,
			     read_stag, read_va, write_stag, write_va);
}

static void
isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn,
		    unsigned long xfer_len)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct iscsi_hdr *hdr;
	u64 rx_dma;
	int rx_buflen, outstanding;

	if ((char *)desc == isert_conn->login_req_buf) {
		rx_dma = isert_conn->login_req_dma;
		rx_buflen = ISER_RX_LOGIN_SIZE;
		pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
			 rx_dma, rx_buflen);
	} else {
		rx_dma = desc->dma_addr;
		rx_buflen = ISER_RX_PAYLOAD_SIZE;
		pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
			 rx_dma, rx_buflen);
	}

	ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE);

	hdr = &desc->iscsi_header;
	pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
		 hdr->opcode, hdr->itt, hdr->flags,
		 (int)(xfer_len - ISER_HEADERS_LEN));

	if ((char *)desc == isert_conn->login_req_buf)
		isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN,
				   isert_conn);
	else
		isert_rx_do_work(desc, isert_conn);

	ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen,
				      DMA_FROM_DEVICE);

	isert_conn->post_recv_buf_count--;
	pr_debug("iSERT: Decremented post_recv_buf_count: %d\n",
		 isert_conn->post_recv_buf_count);

	if ((char *)desc == isert_conn->login_req_buf)
		return;

	outstanding = isert_conn->post_recv_buf_count;
	if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) {
		int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding,
				ISERT_MIN_POSTED_RX);
		err = isert_post_recv(isert_conn, count);
		if (err) {
			pr_err("isert_post_recv() count: %d failed, %d\n",
			       count, err);
		}
	}
}

static void
isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
{
	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;

	pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n");

	if (wr->sge) {
		ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
		wr->sge = NULL;
	}

	kfree(wr->send_wr);
	wr->send_wr = NULL;

	kfree(isert_cmd->ib_sge);
	isert_cmd->ib_sge = NULL;
}

static void
isert_put_cmd(struct isert_cmd *isert_cmd)
{
	struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
	struct isert_conn *isert_conn = isert_cmd->conn;
	struct iscsi_conn *conn = isert_conn->conn;

	pr_debug("Entering isert_put_cmd: %p\n", isert_cmd);

	switch (cmd->iscsi_opcode) {
	case ISCSI_OP_SCSI_CMD:
		spin_lock_bh(&conn->cmd_lock);
		if (!list_empty(&cmd->i_conn_node))
			list_del(&cmd->i_conn_node);
		spin_unlock_bh(&conn->cmd_lock);

		if (cmd->data_direction == DMA_TO_DEVICE)
			iscsit_stop_dataout_timer(cmd);

		isert_unmap_cmd(isert_cmd, isert_conn);
		transport_generic_free_cmd(&cmd->se_cmd, 0);
		break;
	case ISCSI_OP_SCSI_TMFUNC:
		spin_lock_bh(&conn->cmd_lock);
		if (!list_empty(&cmd->i_conn_node))
			list_del(&cmd->i_conn_node);
		spin_unlock_bh(&conn->cmd_lock);

		transport_generic_free_cmd(&cmd->se_cmd, 0);
		break;
	case ISCSI_OP_REJECT:
	case ISCSI_OP_NOOP_OUT:
		spin_lock_bh(&conn->cmd_lock);
		if (!list_empty(&cmd->i_conn_node))
			list_del(&cmd->i_conn_node);
		spin_unlock_bh(&conn->cmd_lock);

		/*
		 * Handle special case for REJECT when iscsi_add_reject*() has
		 * overwritten the original iscsi_opcode assignment, and the
		 * associated cmd->se_cmd needs to be released.
		 */
		if (cmd->se_cmd.se_tfo != NULL) {
			pr_debug("Calling transport_generic_free_cmd from"
				 " isert_put_cmd for 0x%02x\n",
				 cmd->iscsi_opcode);
			transport_generic_free_cmd(&cmd->se_cmd, 0);
			break;
		}
		/*
		 * Fall-through
		 */
	default:
		isert_release_cmd(cmd);
		break;
	}
}

static void
isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
{
	if (tx_desc->dma_addr != 0) {
		pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n");
		ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
				    ISER_HEADERS_LEN, DMA_TO_DEVICE);
		tx_desc->dma_addr = 0;
	}
}

static void
isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
		     struct ib_device *ib_dev)
{
	if (isert_cmd->sense_buf_dma != 0) {
		pr_debug("Calling ib_dma_unmap_single for isert_cmd->sense_buf_dma\n");
		ib_dma_unmap_single(ib_dev, isert_cmd->sense_buf_dma,
				    isert_cmd->sense_buf_len, DMA_TO_DEVICE);
		isert_cmd->sense_buf_dma = 0;
	}

	isert_unmap_tx_desc(tx_desc, ib_dev);
	isert_put_cmd(isert_cmd);
}

static void
isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
			   struct isert_cmd *isert_cmd)
{
	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
	struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
	struct se_cmd *se_cmd = &cmd->se_cmd;
	struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device;

	iscsit_stop_dataout_timer(cmd);

	if (wr->sge) {
		pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n");
		ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
		wr->sge = NULL;
	}

	if (isert_cmd->ib_sge) {
		pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n");
		kfree(isert_cmd->ib_sge);
		isert_cmd->ib_sge = NULL;
	}

	cmd->write_data_done = se_cmd->data_length;

	pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n");
	spin_lock_bh(&cmd->istate_lock);
	cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
	cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
	spin_unlock_bh(&cmd->istate_lock);

	target_execute_cmd(se_cmd);
}

static void
isert_do_control_comp(struct work_struct *work)
{
	struct isert_cmd *isert_cmd = container_of(work,
			struct isert_cmd, comp_work);
	struct isert_conn *isert_conn = isert_cmd->conn;
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;

	switch (cmd->i_state) {
	case ISTATE_SEND_TASKMGTRSP:
		pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n");

		atomic_dec(&isert_conn->post_send_buf_count);
		iscsit_tmr_post_handler(cmd, cmd->conn);

		cmd->i_state = ISTATE_SENT_STATUS;
		isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev);
		break;
	case ISTATE_SEND_REJECT:
		pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n");
		atomic_dec(&isert_conn->post_send_buf_count);

		cmd->i_state = ISTATE_SENT_STATUS;
		isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev);
		break;
	case ISTATE_SEND_LOGOUTRSP:
		pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n");
		/*
		 * Call atomic_dec(&isert_conn->post_send_buf_count)
		 * from isert_free_conn()
		 */
		isert_conn->logout_posted = true;
		iscsit_logout_post_handler(cmd, cmd->conn);
		break;
	default:
		pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state);
		dump_stack();
		break;
	}
}

static void
isert_response_completion(struct iser_tx_desc *tx_desc,
			  struct isert_cmd *isert_cmd,
			  struct isert_conn *isert_conn,
			  struct ib_device *ib_dev)
{
	struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;

	if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||
	    cmd->i_state == ISTATE_SEND_LOGOUTRSP ||
	    cmd->i_state == ISTATE_SEND_REJECT) {
		isert_unmap_tx_desc(tx_desc, ib_dev);

		INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
		queue_work(isert_comp_wq, &isert_cmd->comp_work);
		return;
	}
	atomic_dec(&isert_conn->post_send_buf_count);

	cmd->i_state = ISTATE_SENT_STATUS;
	isert_completion_put(tx_desc, isert_cmd, ib_dev);
}

static void
isert_send_completion(struct iser_tx_desc *tx_desc,
		      struct isert_conn *isert_conn)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
	struct isert_rdma_wr *wr;

	if (!isert_cmd) {
		atomic_dec(&isert_conn->post_send_buf_count);
		isert_unmap_tx_desc(tx_desc, ib_dev);
		return;
	}
	wr = &isert_cmd->rdma_wr;

	switch (wr->iser_ib_op) {
	case ISER_IB_RECV:
		pr_err("isert_send_completion: Got ISER_IB_RECV\n");
		dump_stack();
		break;
	case ISER_IB_SEND:
		pr_debug("isert_send_completion: Got ISER_IB_SEND\n");
		isert_response_completion(tx_desc, isert_cmd,
					  isert_conn, ib_dev);
		break;
	case ISER_IB_RDMA_WRITE:
		pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n");
		dump_stack();
		break;
	case ISER_IB_RDMA_READ:
		pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n");

		atomic_dec(&isert_conn->post_send_buf_count);
		isert_completion_rdma_read(tx_desc, isert_cmd);
		break;
	default:
		pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op);
		dump_stack();
		break;
	}
}

static void
isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn)
{
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;

	if (tx_desc) {
		struct isert_cmd *isert_cmd = tx_desc->isert_cmd;

		if (!isert_cmd)
			isert_unmap_tx_desc(tx_desc, ib_dev);
		else
			isert_completion_put(tx_desc, isert_cmd, ib_dev);
	}

	if (isert_conn->post_recv_buf_count == 0 &&
	    atomic_read(&isert_conn->post_send_buf_count) == 0) {
		pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
		pr_debug("Calling wake_up from isert_cq_comp_err\n");

		mutex_lock(&isert_conn->conn_mutex);
		if (isert_conn->state != ISER_CONN_DOWN)
			isert_conn->state = ISER_CONN_TERMINATING;
		mutex_unlock(&isert_conn->conn_mutex);

		wake_up(&isert_conn->conn_wait_comp_err);
	}
}

static void
isert_cq_tx_work(struct work_struct *work)
{
	struct isert_cq_desc *cq_desc = container_of(work,
				struct isert_cq_desc, cq_tx_work);
	struct isert_device *device = cq_desc->device;
	int cq_index = cq_desc->cq_index;
	struct ib_cq *tx_cq = device->dev_tx_cq[cq_index];
	struct isert_conn *isert_conn;
	struct iser_tx_desc *tx_desc;
	struct ib_wc wc;

	while (ib_poll_cq(tx_cq, 1, &wc) == 1) {
		tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id;
		isert_conn = wc.qp->qp_context;

		if (wc.status == IB_WC_SUCCESS) {
			isert_send_completion(tx_desc, isert_conn);
		} else {
			pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
			pr_debug("TX wc.status: 0x%08x\n", wc.status);
			atomic_dec(&isert_conn->post_send_buf_count);
			isert_cq_comp_err(tx_desc, isert_conn);
		}
	}

	ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP);
}

static void
isert_cq_tx_callback(struct ib_cq *cq, void *context)
{
	struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;

	INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work);
	queue_work(isert_comp_wq, &cq_desc->cq_tx_work);
}

static void
isert_cq_rx_work(struct work_struct *work)
{
	struct isert_cq_desc *cq_desc = container_of(work,
			struct isert_cq_desc, cq_rx_work);
	struct isert_device *device = cq_desc->device;
	int cq_index = cq_desc->cq_index;
	struct ib_cq *rx_cq = device->dev_rx_cq[cq_index];
	struct isert_conn *isert_conn;
	struct iser_rx_desc *rx_desc;
	struct ib_wc wc;
	unsigned long xfer_len;

	while (ib_poll_cq(rx_cq, 1, &wc) == 1) {
		rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id;
		isert_conn = wc.qp->qp_context;

		if (wc.status == IB_WC_SUCCESS) {
			xfer_len = (unsigned long)wc.byte_len;
			isert_rx_completion(rx_desc, isert_conn, xfer_len);
		} else {
			pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
			if (wc.status != IB_WC_WR_FLUSH_ERR)
				pr_debug("RX wc.status: 0x%08x\n", wc.status);

			isert_conn->post_recv_buf_count--;
			isert_cq_comp_err(NULL, isert_conn);
		}
	}

	ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP);
}

static void
isert_cq_rx_callback(struct ib_cq *cq, void *context)
{
	struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;

	INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work);
	queue_work(isert_rx_wq, &cq_desc->cq_rx_work);
}

static int
isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
{
	struct ib_send_wr *wr_failed;
	int ret;

	atomic_inc(&isert_conn->post_send_buf_count);

	ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr,
			   &wr_failed);
	if (ret) {
		pr_err("ib_post_send failed with %d\n", ret);
		atomic_dec(&isert_conn->post_send_buf_count);
		return ret;
	}
	return ret;
}

static int
isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
{
	struct isert_cmd *isert_cmd = container_of(cmd,
					struct isert_cmd, iscsi_cmd);
	struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
	struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
				&isert_cmd->tx_desc.iscsi_header;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_rsp_pdu(cmd, conn, true, hdr);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
	/*
	 * Attach SENSE DATA payload to iSCSI Response PDU
	 */
	if (cmd->se_cmd.sense_buffer &&
	    ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
	    (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
		struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
		u32 padding, sense_len;

		put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
				   cmd->sense_buffer);
		cmd->se_cmd.scsi_sense_length += sizeof(__be16);

		padding = -(cmd->se_cmd.scsi_sense_length) & 3;
		hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
		sense_len = cmd->se_cmd.scsi_sense_length + padding;

		isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
				(void *)cmd->sense_buffer, sense_len,
				DMA_TO_DEVICE);

		isert_cmd->sense_buf_len = sense_len;
		tx_dsg->addr	= isert_cmd->sense_buf_dma;
		tx_dsg->length	= sense_len;
		tx_dsg->lkey	= isert_conn->conn_mr->lkey;
		isert_cmd->tx_desc.num_sge = 2;
	}

	isert_init_send_wr(isert_cmd, send_wr);

	pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");

	return isert_post_response(isert_conn, isert_cmd);
}

static int
isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
		bool nopout_response)
{
	struct isert_cmd *isert_cmd = container_of(cmd,
				struct isert_cmd, iscsi_cmd);
	struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
			       &isert_cmd->tx_desc.iscsi_header,
			       nopout_response);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
	isert_init_send_wr(isert_cmd, send_wr);

	pr_debug("Posting NOPIN Reponse IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");

	return isert_post_response(isert_conn, isert_cmd);
}

static int
isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
{
	struct isert_cmd *isert_cmd = container_of(cmd,
				struct isert_cmd, iscsi_cmd);
	struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
				&isert_cmd->tx_desc.iscsi_header);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
	isert_init_send_wr(isert_cmd, send_wr);

	pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");

	return isert_post_response(isert_conn, isert_cmd);
}

static int
isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
{
	struct isert_cmd *isert_cmd = container_of(cmd,
				struct isert_cmd, iscsi_cmd);
	struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
				  &isert_cmd->tx_desc.iscsi_header);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
	isert_init_send_wr(isert_cmd, send_wr);

	pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");

	return isert_post_response(isert_conn, isert_cmd);
}

static int
isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
{
	struct isert_cmd *isert_cmd = container_of(cmd,
				struct isert_cmd, iscsi_cmd);
	struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
	struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
	struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
	struct iscsi_reject *hdr =
		(struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_reject(cmd, conn, hdr);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);