Merge branch 'for-next-merge' of git://git.kernel.org/pub/scm/linux/kernel/git/nab/target-pending

Pull SCSI target update from Nicholas Bellinger:
 "The highlights this round include:

   - Add fileio support for WRITE_SAME w/ UNMAP=1 discard (asias)
   - Add fileio support for UNMAP discard (asias)
   - Add tcm_vhost hotplug support to work with upstream QEMU
     vhost-scsi-pci code (asias + mst)
   - Check for aborted sequence in tcm_fc response path (mdr)
   - Add initial iscsit_transport support into iscsi-target code (nab)
   - Refactor iscsi-target RX PDU logic + export request PDU handling
     (nab)
   - Refactor iscsi-target TX queue logic + export response PDU creation
     (nab)
   - Add new iSCSI Extentions for RDMA (ISER) target driver (Or + nab)

  The biggest changes revolve around iscsi-target refactoring in order
  to support the iser-target driver.  This includes the conversion of
  the iscsi-target data-path to use modern se_cmd->cmd_kref counting,
  and allowing transport independent aspects of RX/TX PDU
  request/response handling be shared across existing traditional
  iscsi-target code, and the new iser-target code.

  Thanks to Or Gerlitz + Mellanox for supporting the iser-target
  development effort!"

* 'for-next-merge' of git://git.kernel.org/pub/scm/linux/kernel/git/nab/target-pending: (25 commits)
  iser-target: Add iSCSI Extensions for RDMA (iSER) target driver
  tcm_vhost: Enable VIRTIO_SCSI_F_HOTPLUG
  tcm_vhost: Add ioctl to get and set events missed flag
  tcm_vhost: Add hotplug/hotunplug support
  tcm_vhost: Refactor the lock nesting rule
  tcm_fc: Check for aborted sequence
  iscsi-target: Add iser network portal attribute
  iscsi-target: Refactor TX queue logic + export response PDU creation
  iscsi-target: Refactor RX PDU logic + export request PDU handling
  iscsi-target: Add per transport iscsi_cmd alloc/free
  iscsi-target: Add iser-target parameter keys + setup during login
  iscsi-target: Initial traditional TCP conversion to iscsit_transport
  iscsi-target: Add iscsit_transport API template
  target: Add export of target_get_sess_cmd symbol
  target: Change default sense key of NOT_READY
  target/file: Set is_nonrot attribute
  target: Add sbc_execute_unmap() helper
  target/iblock: Add iblock_do_unmap() helper
  target/file: Add fd_do_unmap() helper
  target/file: Add UNMAP emulation support
  ...

39 files changed, 4469 insertions, 1001 deletions

diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index a0f29c1d03bc..c85b56c28099 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -59,5 +59,6 @@ source "drivers/infiniband/ulp/srp/Kconfig"
 source "drivers/infiniband/ulp/srpt/Kconfig"
 
 source "drivers/infiniband/ulp/iser/Kconfig"
+source "drivers/infiniband/ulp/isert/Kconfig"
 
 endif # INFINIBAND
diff --git a/drivers/infiniband/Makefile b/drivers/infiniband/Makefile
index bf846a14b9d3..b126fefe0b1c 100644
--- a/drivers/infiniband/Makefile
+++ b/drivers/infiniband/Makefile
@@ -13,3 +13,4 @@ obj-$(CONFIG_INFINIBAND_IPOIB)		+= ulp/ipoib/
 obj-$(CONFIG_INFINIBAND_SRP)            += ulp/srp/
 obj-$(CONFIG_INFINIBAND_SRPT)           += ulp/srpt/
 obj-$(CONFIG_INFINIBAND_ISER)           += ulp/iser/
+obj-$(CONFIG_INFINIBAND_ISERT)          += ulp/isert/
diff --git a/drivers/infiniband/ulp/isert/Kconfig b/drivers/infiniband/ulp/isert/Kconfig
new file mode 100644
index 000000000000..ce3fd32167dc
--- /dev/null
+++ b/drivers/infiniband/ulp/isert/Kconfig
@@ -0,0 +1,5 @@
+config INFINIBAND_ISERT
+        tristate "iSCSI Extentions for RDMA (iSER) target support"
+        depends on INET && INFINIBAND_ADDR_TRANS && TARGET_CORE && ISCSI_TARGET
+        ---help---
+        Support for iSCSI Extentions for RDMA (iSER) Target on Infiniband fabrics.
diff --git a/drivers/infiniband/ulp/isert/Makefile b/drivers/infiniband/ulp/isert/Makefile
new file mode 100644
index 000000000000..c8bf2421f5bc
--- /dev/null
+++ b/drivers/infiniband/ulp/isert/Makefile
@@ -0,0 +1,2 @@
+ccflags-y               := -Idrivers/target -Idrivers/target/iscsi
+obj-$(CONFIG_INFINIBAND_ISERT)  += ib_isert.o
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
new file mode 100644
index 000000000000..41712f096515
--- /dev/null
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -0,0 +1,2281 @@
+/*******************************************************************************
+ * This file contains iSCSI extentions for RDMA (iSER) Verbs
+ *
+ * (c) Copyright 2013 RisingTide Systems LLC.
+ *
+ * Nicholas A. Bellinger <nab@linux-iscsi.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ ****************************************************************************/
+
+#include <linux/string.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/socket.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/rdma_cm.h>
+#include <target/target_core_base.h>
+#include <target/target_core_fabric.h>
+#include <target/iscsi/iscsi_transport.h>
+
+#include "isert_proto.h"
+#include "ib_isert.h"
+
+#define ISERT_MAX_CONN          8
+#define ISER_MAX_RX_CQ_LEN      (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
+#define ISER_MAX_TX_CQ_LEN      (ISERT_QP_MAX_REQ_DTOS  * ISERT_MAX_CONN)
+
+static DEFINE_MUTEX(device_list_mutex);
+static LIST_HEAD(device_list);
+static struct workqueue_struct *isert_rx_wq;
+static struct workqueue_struct *isert_comp_wq;
+static struct kmem_cache *isert_cmd_cache;
+
+static void
+isert_qp_event_callback(struct ib_event *e, void *context)
+{
+        struct isert_conn *isert_conn = (struct isert_conn *)context;
+
+        pr_err("isert_qp_event_callback event: %d\n", e->event);
+        switch (e->event) {
+        case IB_EVENT_COMM_EST:
+                rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST);
+                break;
+        case IB_EVENT_QP_LAST_WQE_REACHED:
+                pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n");
+                break;
+        default:
+                break;
+        }
+}
+
+static int
+isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
+{
+        int ret;
+
+        ret = ib_query_device(ib_dev, devattr);
+        if (ret) {
+                pr_err("ib_query_device() failed: %d\n", ret);
+                return ret;
+        }
+        pr_debug("devattr->max_sge: %d\n", devattr->max_sge);
+        pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);
+
+        return 0;
+}
+
+static int
+isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
+{
+        struct isert_device *device = isert_conn->conn_device;
+        struct ib_qp_init_attr attr;
+        struct ib_device_attr devattr;
+        int ret, index, min_index = 0;
+
+        memset(&devattr, 0, sizeof(struct ib_device_attr));
+        ret = isert_query_device(cma_id->device, &devattr);
+        if (ret)
+                return ret;
+
+        mutex_lock(&device_list_mutex);
+        for (index = 0; index < device->cqs_used; index++)
+                if (device->cq_active_qps[index] <
+                    device->cq_active_qps[min_index])
+                        min_index = index;
+        device->cq_active_qps[min_index]++;
+        pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index);
+        mutex_unlock(&device_list_mutex);
+
+        memset(&attr, 0, sizeof(struct ib_qp_init_attr));
+        attr.event_handler = isert_qp_event_callback;
+        attr.qp_context = isert_conn;
+        attr.send_cq = device->dev_tx_cq[min_index];
+        attr.recv_cq = device->dev_rx_cq[min_index];
+        attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
+        attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS;
+        /*
+         * FIXME: Use devattr.max_sge - 2 for max_send_sge as
+         * work-around for RDMA_READ..
+         */
+        attr.cap.max_send_sge = devattr.max_sge - 2;
+        isert_conn->max_sge = attr.cap.max_send_sge;
+
+        attr.cap.max_recv_sge = 1;
+        attr.sq_sig_type = IB_SIGNAL_REQ_WR;
+        attr.qp_type = IB_QPT_RC;
+
+        pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
+                 cma_id->device);
+        pr_debug("isert_conn_setup_qp conn_pd->device: %p\n",
+                 isert_conn->conn_pd->device);
+
+        ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr);
+        if (ret) {
+                pr_err("rdma_create_qp failed for cma_id %d\n", ret);
+                return ret;
+        }
+        isert_conn->conn_qp = cma_id->qp;
+        pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n");
+
+        return 0;
+}
+
+static void
+isert_cq_event_callback(struct ib_event *e, void *context)
+{
+        pr_debug("isert_cq_event_callback event: %d\n", e->event);
+}
+
+static int
+isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct iser_rx_desc *rx_desc;
+        struct ib_sge *rx_sg;
+        u64 dma_addr;
+        int i, j;
+
+        isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
+                                sizeof(struct iser_rx_desc), GFP_KERNEL);
+        if (!isert_conn->conn_rx_descs)
+                goto fail;
+
+        rx_desc = isert_conn->conn_rx_descs;
+
+        for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
+                dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
+                                        ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
+                if (ib_dma_mapping_error(ib_dev, dma_addr))
+                        goto dma_map_fail;
+
+                rx_desc->dma_addr = dma_addr;
+
+                rx_sg = &rx_desc->rx_sg;
+                rx_sg->addr = rx_desc->dma_addr;
+                rx_sg->length = ISER_RX_PAYLOAD_SIZE;
+                rx_sg->lkey = isert_conn->conn_mr->lkey;
+        }
+
+        isert_conn->conn_rx_desc_head = 0;
+        return 0;
+
+dma_map_fail:
+        rx_desc = isert_conn->conn_rx_descs;
+        for (j = 0; j < i; j++, rx_desc++) {
+                ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
+                                    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
+        }
+        kfree(isert_conn->conn_rx_descs);
+        isert_conn->conn_rx_descs = NULL;
+fail:
+        return -ENOMEM;
+}
+
+static void
+isert_free_rx_descriptors(struct isert_conn *isert_conn)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct iser_rx_desc *rx_desc;
+        int i;
+
+        if (!isert_conn->conn_rx_descs)
+                return;
+
+        rx_desc = isert_conn->conn_rx_descs;
+        for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
+                ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
+                                    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
+        }
+
+        kfree(isert_conn->conn_rx_descs);
+        isert_conn->conn_rx_descs = NULL;
+}
+
+static void isert_cq_tx_callback(struct ib_cq *, void *);
+static void isert_cq_rx_callback(struct ib_cq *, void *);
+
+static int
+isert_create_device_ib_res(struct isert_device *device)
+{
+        struct ib_device *ib_dev = device->ib_device;
+        struct isert_cq_desc *cq_desc;
+        int ret = 0, i, j;
+
+        device->cqs_used = min_t(int, num_online_cpus(),
+                                 device->ib_device->num_comp_vectors);
+        device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used);
+        pr_debug("Using %d CQs, device %s supports %d vectors\n",
+                 device->cqs_used, device->ib_device->name,
+                 device->ib_device->num_comp_vectors);
+        device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) *
+                                device->cqs_used, GFP_KERNEL);
+        if (!device->cq_desc) {
+                pr_err("Unable to allocate device->cq_desc\n");
+                return -ENOMEM;
+        }
+        cq_desc = device->cq_desc;
+
+        device->dev_pd = ib_alloc_pd(ib_dev);
+        if (IS_ERR(device->dev_pd)) {
+                ret = PTR_ERR(device->dev_pd);
+                pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret);
+                goto out_cq_desc;
+        }
+
+        for (i = 0; i < device->cqs_used; i++) {
+                cq_desc[i].device = device;
+                cq_desc[i].cq_index = i;
+
+                device->dev_rx_cq[i] = ib_create_cq(device->ib_device,
+                                                isert_cq_rx_callback,
+                                                isert_cq_event_callback,
+                                                (void *)&cq_desc[i],
+                                                ISER_MAX_RX_CQ_LEN, i);
+                if (IS_ERR(device->dev_rx_cq[i]))
+                        goto out_cq;
+
+                device->dev_tx_cq[i] = ib_create_cq(device->ib_device,
+                                                isert_cq_tx_callback,
+                                                isert_cq_event_callback,
+                                                (void *)&cq_desc[i],
+                                                ISER_MAX_TX_CQ_LEN, i);
+                if (IS_ERR(device->dev_tx_cq[i]))
+                        goto out_cq;
+
+                if (ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP))
+                        goto out_cq;
+
+                if (ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP))
+                        goto out_cq;
+        }
+
+        device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE);
+        if (IS_ERR(device->dev_mr)) {
+                ret = PTR_ERR(device->dev_mr);
+                pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret);
+                goto out_cq;
+        }
+
+        return 0;
+
+out_cq:
+        for (j = 0; j < i; j++) {
+                cq_desc = &device->cq_desc[j];
+
+                if (device->dev_rx_cq[j]) {
+                        cancel_work_sync(&cq_desc->cq_rx_work);
+                        ib_destroy_cq(device->dev_rx_cq[j]);
+                }
+                if (device->dev_tx_cq[j]) {
+                        cancel_work_sync(&cq_desc->cq_tx_work);
+                        ib_destroy_cq(device->dev_tx_cq[j]);
+                }
+        }
+        ib_dealloc_pd(device->dev_pd);
+
+out_cq_desc:
+        kfree(device->cq_desc);
+
+        return ret;
+}
+
+static void
+isert_free_device_ib_res(struct isert_device *device)
+{
+        struct isert_cq_desc *cq_desc;
+        int i;
+
+        for (i = 0; i < device->cqs_used; i++) {
+                cq_desc = &device->cq_desc[i];
+
+                cancel_work_sync(&cq_desc->cq_rx_work);
+                cancel_work_sync(&cq_desc->cq_tx_work);
+                ib_destroy_cq(device->dev_rx_cq[i]);
+                ib_destroy_cq(device->dev_tx_cq[i]);
+                device->dev_rx_cq[i] = NULL;
+                device->dev_tx_cq[i] = NULL;
+        }
+
+        ib_dereg_mr(device->dev_mr);
+        ib_dealloc_pd(device->dev_pd);
+        kfree(device->cq_desc);
+}
+
+static void
+isert_device_try_release(struct isert_device *device)
+{
+        mutex_lock(&device_list_mutex);
+        device->refcount--;
+        if (!device->refcount) {
+                isert_free_device_ib_res(device);
+                list_del(&device->dev_node);
+                kfree(device);
+        }
+        mutex_unlock(&device_list_mutex);
+}
+
+static struct isert_device *
+isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id)
+{
+        struct isert_device *device;
+        int ret;
+
+        mutex_lock(&device_list_mutex);
+        list_for_each_entry(device, &device_list, dev_node) {
+                if (device->ib_device->node_guid == cma_id->device->node_guid) {
+                        device->refcount++;
+                        mutex_unlock(&device_list_mutex);
+                        return device;
+                }
+        }
+
+        device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
+        if (!device) {
+                mutex_unlock(&device_list_mutex);
+                return ERR_PTR(-ENOMEM);
+        }
+
+        INIT_LIST_HEAD(&device->dev_node);
+
+        device->ib_device = cma_id->device;
+        ret = isert_create_device_ib_res(device);
+        if (ret) {
+                kfree(device);
+                mutex_unlock(&device_list_mutex);
+                return ERR_PTR(ret);
+        }
+
+        device->refcount++;
+        list_add_tail(&device->dev_node, &device_list);
+        mutex_unlock(&device_list_mutex);
+
+        return device;
+}
+
+static int
+isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
+{
+        struct iscsi_np *np = cma_id->context;
+        struct isert_np *isert_np = np->np_context;
+        struct isert_conn *isert_conn;
+        struct isert_device *device;
+        struct ib_device *ib_dev = cma_id->device;
+        int ret = 0;
+
+        pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
+                 cma_id, cma_id->context);
+
+        isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
+        if (!isert_conn) {
+                pr_err("Unable to allocate isert_conn\n");
+                return -ENOMEM;
+        }
+        isert_conn->state = ISER_CONN_INIT;
+        INIT_LIST_HEAD(&isert_conn->conn_accept_node);
+        init_completion(&isert_conn->conn_login_comp);
+        init_waitqueue_head(&isert_conn->conn_wait);
+        init_waitqueue_head(&isert_conn->conn_wait_comp_err);
+        kref_init(&isert_conn->conn_kref);
+        kref_get(&isert_conn->conn_kref);
+
+        cma_id->context = isert_conn;
+        isert_conn->conn_cm_id = cma_id;
+        isert_conn->responder_resources = event->param.conn.responder_resources;
+        isert_conn->initiator_depth = event->param.conn.initiator_depth;
+        pr_debug("Using responder_resources: %u initiator_depth: %u\n",
+                 isert_conn->responder_resources, isert_conn->initiator_depth);
+
+        isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
+                                        ISER_RX_LOGIN_SIZE, GFP_KERNEL);
+        if (!isert_conn->login_buf) {
+                pr_err("Unable to allocate isert_conn->login_buf\n");
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        isert_conn->login_req_buf = isert_conn->login_buf;
+        isert_conn->login_rsp_buf = isert_conn->login_buf +
+                                    ISCSI_DEF_MAX_RECV_SEG_LEN;
+        pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
+                 isert_conn->login_buf, isert_conn->login_req_buf,
+                 isert_conn->login_rsp_buf);
+
+        isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
+                                (void *)isert_conn->login_req_buf,
+                                ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
+
+        ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
+        if (ret) {
+                pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
+                       ret);
+                isert_conn->login_req_dma = 0;
+                goto out_login_buf;
+        }
+
+        isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
+                                        (void *)isert_conn->login_rsp_buf,
+                                        ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
+
+        ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
+        if (ret) {
+                pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
+                       ret);
+                isert_conn->login_rsp_dma = 0;
+                goto out_req_dma_map;
+        }
+
+        device = isert_device_find_by_ib_dev(cma_id);
+        if (IS_ERR(device)) {
+                ret = PTR_ERR(device);
+                goto out_rsp_dma_map;
+        }
+
+        isert_conn->conn_device = device;
+        isert_conn->conn_pd = device->dev_pd;
+        isert_conn->conn_mr = device->dev_mr;
+
+        ret = isert_conn_setup_qp(isert_conn, cma_id);
+        if (ret)
+                goto out_conn_dev;
+
+        mutex_lock(&isert_np->np_accept_mutex);
+        list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node);
+        mutex_unlock(&isert_np->np_accept_mutex);
+
+        pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np);
+        wake_up(&isert_np->np_accept_wq);
+        return 0;
+
+out_conn_dev:
+        isert_device_try_release(device);
+out_rsp_dma_map:
+        ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
+                            ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
+out_req_dma_map:
+        ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
+                            ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
+out_login_buf:
+        kfree(isert_conn->login_buf);
+out:
+        kfree(isert_conn);
+        return ret;
+}
+
+static void
+isert_connect_release(struct isert_conn *isert_conn)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct isert_device *device = isert_conn->conn_device;
+        int cq_index;
+
+        pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
+
+        if (isert_conn->conn_qp) {
+                cq_index = ((struct isert_cq_desc *)
+                        isert_conn->conn_qp->recv_cq->cq_context)->cq_index;
+                pr_debug("isert_connect_release: cq_index: %d\n", cq_index);
+                isert_conn->conn_device->cq_active_qps[cq_index]--;
+
+                rdma_destroy_qp(isert_conn->conn_cm_id);
+        }
+
+        isert_free_rx_descriptors(isert_conn);
+        rdma_destroy_id(isert_conn->conn_cm_id);
+
+        if (isert_conn->login_buf) {
+                ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
+                                    ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
+                ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
+                                    ISCSI_DEF_MAX_RECV_SEG_LEN,
+                                    DMA_FROM_DEVICE);
+                kfree(isert_conn->login_buf);
+        }
+        kfree(isert_conn);
+
+        if (device)
+                isert_device_try_release(device);
+
+        pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n");
+}
+
+static void
+isert_connected_handler(struct rdma_cm_id *cma_id)
+{
+        return;
+}
+
+static void
+isert_release_conn_kref(struct kref *kref)
+{
+        struct isert_conn *isert_conn = container_of(kref,
+                                struct isert_conn, conn_kref);
+
+        pr_debug("Calling isert_connect_release for final kref %s/%d\n",
+                 current->comm, current->pid);
+
+        isert_connect_release(isert_conn);
+}
+
+static void
+isert_put_conn(struct isert_conn *isert_conn)
+{
+        kref_put(&isert_conn->conn_kref, isert_release_conn_kref);
+}
+
+static void
+isert_disconnect_work(struct work_struct *work)
+{
+        struct isert_conn *isert_conn = container_of(work,
+                                struct isert_conn, conn_logout_work);
+
+        pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
+
+        isert_conn->state = ISER_CONN_DOWN;
+
+        if (isert_conn->post_recv_buf_count == 0 &&
+            atomic_read(&isert_conn->post_send_buf_count) == 0) {
+                pr_debug("Calling wake_up(&isert_conn->conn_wait);\n");
+                wake_up(&isert_conn->conn_wait);
+        }
+
+        isert_put_conn(isert_conn);
+}
+
+static void
+isert_disconnected_handler(struct rdma_cm_id *cma_id)
+{
+        struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context;
+
+        INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work);
+        schedule_work(&isert_conn->conn_logout_work);
+}
+
+static int
+isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
+{
+        int ret = 0;
+
+        pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n",
+                 event->event, event->status, cma_id->context, cma_id);
+
+        switch (event->event) {
+        case RDMA_CM_EVENT_CONNECT_REQUEST:
+                pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n");
+                ret = isert_connect_request(cma_id, event);
+                break;
+        case RDMA_CM_EVENT_ESTABLISHED:
+                pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n");
+                isert_connected_handler(cma_id);
+                break;
+        case RDMA_CM_EVENT_DISCONNECTED:
+                pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n");
+                isert_disconnected_handler(cma_id);
+                break;
+        case RDMA_CM_EVENT_DEVICE_REMOVAL:
+        case RDMA_CM_EVENT_ADDR_CHANGE:
+                break;
+        case RDMA_CM_EVENT_CONNECT_ERROR:
+        default:
+                pr_err("Unknown RDMA CMA event: %d\n", event->event);
+                break;
+        }
+
+        if (ret != 0) {
+                pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n",
+                       event->event, ret);
+                dump_stack();
+        }
+
+        return ret;
+}
+
+static int
+isert_post_recv(struct isert_conn *isert_conn, u32 count)
+{
+        struct ib_recv_wr *rx_wr, *rx_wr_failed;
+        int i, ret;
+        unsigned int rx_head = isert_conn->conn_rx_desc_head;
+        struct iser_rx_desc *rx_desc;
+
+        for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) {
+                rx_desc         = &isert_conn->conn_rx_descs[rx_head];
+                rx_wr->wr_id    = (unsigned long)rx_desc;
+                rx_wr->sg_list  = &rx_desc->rx_sg;
+                rx_wr->num_sge  = 1;
+                rx_wr->next     = rx_wr + 1;
+                rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1);
+        }
+
+        rx_wr--;
+        rx_wr->next = NULL; /* mark end of work requests list */
+
+        isert_conn->post_recv_buf_count += count;
+        ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr,
+                                &rx_wr_failed);
+        if (ret) {
+                pr_err("ib_post_recv() failed with ret: %d\n", ret);
+                isert_conn->post_recv_buf_count -= count;
+        } else {
+                pr_debug("isert_post_recv(): Posted %d RX buffers\n", count);
+                isert_conn->conn_rx_desc_head = rx_head;
+        }
+        return ret;
+}
+
+static int
+isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct ib_send_wr send_wr, *send_wr_failed;
+        int ret;
+
+        ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
+                                      ISER_HEADERS_LEN, DMA_TO_DEVICE);
+
+        send_wr.next    = NULL;
+        send_wr.wr_id   = (unsigned long)tx_desc;
+        send_wr.sg_list = tx_desc->tx_sg;
+        send_wr.num_sge = tx_desc->num_sge;
+        send_wr.opcode  = IB_WR_SEND;
+        send_wr.send_flags = IB_SEND_SIGNALED;
+
+        atomic_inc(&isert_conn->post_send_buf_count);
+
+        ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed);
+        if (ret) {
+                pr_err("ib_post_send() failed, ret: %d\n", ret);
+                atomic_dec(&isert_conn->post_send_buf_count);
+        }
+
+        return ret;
+}
+
+static void
+isert_create_send_desc(struct isert_conn *isert_conn,
+                       struct isert_cmd *isert_cmd,
+                       struct iser_tx_desc *tx_desc)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+
+        ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
+                                   ISER_HEADERS_LEN, DMA_TO_DEVICE);
+
+        memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
+        tx_desc->iser_header.flags = ISER_VER;
+
+        tx_desc->num_sge = 1;
+        tx_desc->isert_cmd = isert_cmd;
+
+        if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) {
+                tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
+                pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc);
+        }
+}
+
+static int
+isert_init_tx_hdrs(struct isert_conn *isert_conn,
+                   struct iser_tx_desc *tx_desc)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        u64 dma_addr;
+
+        dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
+                        ISER_HEADERS_LEN, DMA_TO_DEVICE);
+        if (ib_dma_mapping_error(ib_dev, dma_addr)) {
+                pr_err("ib_dma_mapping_error() failed\n");
+                return -ENOMEM;
+        }
+
+        tx_desc->dma_addr = dma_addr;
+        tx_desc->tx_sg[0].addr  = tx_desc->dma_addr;
+        tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
+        tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
+
+        pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u"
+                 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr,
+                 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey);
+
+        return 0;
+}
+
+static void
+isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr)
+{
+        isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
+        send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
+        send_wr->opcode = IB_WR_SEND;
+        send_wr->send_flags = IB_SEND_SIGNALED;
+        send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0];
+        send_wr->num_sge = isert_cmd->tx_desc.num_sge;
+}
+
+static int
+isert_rdma_post_recvl(struct isert_conn *isert_conn)
+{
+        struct ib_recv_wr rx_wr, *rx_wr_fail;
+        struct ib_sge sge;
+        int ret;
+
+        memset(&sge, 0, sizeof(struct ib_sge));
+        sge.addr = isert_conn->login_req_dma;
+        sge.length = ISER_RX_LOGIN_SIZE;
+        sge.lkey = isert_conn->conn_mr->lkey;
+
+        pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n",
+                sge.addr, sge.length, sge.lkey);
+
+        memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
+        rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf;
+        rx_wr.sg_list = &sge;
+        rx_wr.num_sge = 1;
+
+        isert_conn->post_recv_buf_count++;
+        ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail);
+        if (ret) {
+                pr_err("ib_post_recv() failed: %d\n", ret);
+                isert_conn->post_recv_buf_count--;
+        }
+
+        pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n");
+        return ret;
+}
+
+static int
+isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
+                   u32 length)
+{
+        struct isert_conn *isert_conn = conn->context;
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc;
+        int ret;
+
+        isert_create_send_desc(isert_conn, NULL, tx_desc);
+
+        memcpy(&tx_desc->iscsi_header, &login->rsp[0],
+               sizeof(struct iscsi_hdr));
+
+        isert_init_tx_hdrs(isert_conn, tx_desc);
+
+        if (length > 0) {
+                struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
+
+                ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
+                                           length, DMA_TO_DEVICE);
+
+                memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
+
+                ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
+                                              length, DMA_TO_DEVICE);
+
+                tx_dsg->addr    = isert_conn->login_rsp_dma;
+                tx_dsg->length  = length;
+                tx_dsg->lkey    = isert_conn->conn_mr->lkey;
+                tx_desc->num_sge = 2;
+        }
+        if (!login->login_failed) {
+                if (login->login_complete) {
+                        ret = isert_alloc_rx_descriptors(isert_conn);
+                        if (ret)
+                                return ret;
+
+                        ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX);
+                        if (ret)
+                                return ret;
+
+                        isert_conn->state = ISER_CONN_UP;
+                        goto post_send;
+                }
+
+                ret = isert_rdma_post_recvl(isert_conn);
+                if (ret)
+                        return ret;
+        }
+post_send:
+        ret = isert_post_send(isert_conn, tx_desc);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+static void
+isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen,
+                   struct isert_conn *isert_conn)
+{
+        struct iscsi_conn *conn = isert_conn->conn;
+        struct iscsi_login *login = conn->conn_login;
+        int size;
+
+        if (!login) {
+                pr_err("conn->conn_login is NULL\n");
+                dump_stack();
+                return;
+        }
+
+        if (login->first_request) {
+                struct iscsi_login_req *login_req =
+                        (struct iscsi_login_req *)&rx_desc->iscsi_header;
+                /*
+                 * Setup the initial iscsi_login values from the leading
+                 * login request PDU.
+                 */
+                login->leading_connection = (!login_req->tsih) ? 1 : 0;
+                login->current_stage =
+                        (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
+                         >> 2;
+                login->version_min      = login_req->min_version;
+                login->version_max      = login_req->max_version;
+                memcpy(login->isid, login_req->isid, 6);
+                login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
+                login->init_task_tag    = login_req->itt;
+                login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
+                login->cid              = be16_to_cpu(login_req->cid);
+                login->tsih             = be16_to_cpu(login_req->tsih);
+        }
+
+        memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
+
+        size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
+        pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n",
+                 size, rx_buflen, MAX_KEY_VALUE_PAIRS);
+        memcpy(login->req_buf, &rx_desc->data[0], size);
+
+        complete(&isert_conn->conn_login_comp);
+}
+
+static void
+isert_release_cmd(struct iscsi_cmd *cmd)
+{
+        struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd,
+                                                   iscsi_cmd);
+
+        pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd);
+
+        kfree(cmd->buf_ptr);
+        kfree(cmd->tmr_req);
+
+        kmem_cache_free(isert_cmd_cache, isert_cmd);
+}
+
+static struct iscsi_cmd
+*isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp)
+{
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        struct isert_cmd *isert_cmd;
+
+        isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp);
+        if (!isert_cmd) {
+                pr_err("Unable to allocate isert_cmd\n");
+                return NULL;
+        }
+        isert_cmd->conn = isert_conn;
+        isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd;
+
+        return &isert_cmd->iscsi_cmd;
+}
+
+static int
+isert_handle_scsi_cmd(struct isert_conn *isert_conn,
+                      struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc,
+                      unsigned char *buf)
+{
+        struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
+        struct iscsi_conn *conn = isert_conn->conn;
+        struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
+        struct scatterlist *sg;
+        int imm_data, imm_data_len, unsol_data, sg_nents, rc;
+        bool dump_payload = false;
+
+        rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
+        if (rc < 0)
+                return rc;
+
+        imm_data = cmd->immediate_data;
+        imm_data_len = cmd->first_burst_len;
+        unsol_data = cmd->unsolicited_data;
+
+        rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
+        if (rc < 0) {
+                return 0;
+        } else if (rc > 0) {
+                dump_payload = true;
+                goto sequence_cmd;
+        }
+
+        if (!imm_data)
+                return 0;
+
+        sg = &cmd->se_cmd.t_data_sg[0];
+        sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
+
+        pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
+                 sg, sg_nents, &rx_desc->data[0], imm_data_len);
+
+        sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len);
+
+        cmd->write_data_done += imm_data_len;
+
+        if (cmd->write_data_done == cmd->se_cmd.data_length) {
+                spin_lock_bh(&cmd->istate_lock);
+                cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
+                cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
+                spin_unlock_bh(&cmd->istate_lock);
+        }
+
+sequence_cmd:
+        rc = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
+
+        if (!rc && dump_payload == false && unsol_data)
+                iscsit_set_unsoliticed_dataout(cmd);
+
+        if (rc == CMDSN_ERROR_CANNOT_RECOVER)
+                return iscsit_add_reject_from_cmd(
+                           ISCSI_REASON_PROTOCOL_ERROR,
+                           1, 0, (unsigned char *)hdr, cmd);
+
+        return 0;
+}
+
+static int
+isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
+                           struct iser_rx_desc *rx_desc, unsigned char *buf)
+{
+        struct scatterlist *sg_start;
+        struct iscsi_conn *conn = isert_conn->conn;
+        struct iscsi_cmd *cmd = NULL;
+        struct iscsi_data *hdr = (struct iscsi_data *)buf;
+        u32 unsol_data_len = ntoh24(hdr->dlength);
+        int rc, sg_nents, sg_off, page_off;
+
+        rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
+        if (rc < 0)
+                return rc;
+        else if (!cmd)
+                return 0;
+        /*
+         * FIXME: Unexpected unsolicited_data out
+         */
+        if (!cmd->unsolicited_data) {
+                pr_err("Received unexpected solicited data payload\n");
+                dump_stack();
+                return -1;
+        }
+
+        pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n",
+                 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length);
+
+        sg_off = cmd->write_data_done / PAGE_SIZE;
+        sg_start = &cmd->se_cmd.t_data_sg[sg_off];
+        sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
+        page_off = cmd->write_data_done % PAGE_SIZE;
+        /*
+         * FIXME: Non page-aligned unsolicited_data out
+         */
+        if (page_off) {
+                pr_err("Received unexpected non-page aligned data payload\n");
+                dump_stack();
+                return -1;
+        }
+        pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n",
+                 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len);
+
+        sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
+                            unsol_data_len);
+
+        rc = iscsit_check_dataout_payload(cmd, hdr, false);
+        if (rc < 0)
+                return rc;
+
+        return 0;
+}
+
+static int
+isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
+                uint32_t read_stag, uint64_t read_va,
+                uint32_t write_stag, uint64_t write_va)
+{
+        struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
+        struct iscsi_conn *conn = isert_conn->conn;
+        struct iscsi_cmd *cmd;
+        struct isert_cmd *isert_cmd;
+        int ret = -EINVAL;
+        u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
+
+        switch (opcode) {
+        case ISCSI_OP_SCSI_CMD:
+                cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
+                if (!cmd)
+                        break;
+
+                isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd);
+                isert_cmd->read_stag = read_stag;
+                isert_cmd->read_va = read_va;
+                isert_cmd->write_stag = write_stag;
+                isert_cmd->write_va = write_va;
+
+                ret = isert_handle_scsi_cmd(isert_conn, isert_cmd,
+                                        rx_desc, (unsigned char *)hdr);
+                break;
+        case ISCSI_OP_NOOP_OUT:
+                cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
+                if (!cmd)
+                        break;
+
+                ret = iscsit_handle_nop_out(conn, cmd, (unsigned char *)hdr);
+                break;
+        case ISCSI_OP_SCSI_DATA_OUT:
+                ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
+                                                (unsigned char *)hdr);
+                break;
+        case ISCSI_OP_SCSI_TMFUNC:
+                cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
+                if (!cmd)
+                        break;
+
+                ret = iscsit_handle_task_mgt_cmd(conn, cmd,
+                                                (unsigned char *)hdr);
+                break;
+        case ISCSI_OP_LOGOUT:
+                cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
+                if (!cmd)
+                        break;
+
+                ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
+                if (ret > 0)
+                        wait_for_completion_timeout(&conn->conn_logout_comp,
+                                                    SECONDS_FOR_LOGOUT_COMP *
+                                                    HZ);
+                break;
+        default:
+                pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
+                dump_stack();
+                break;
+        }
+
+        return ret;
+}
+
+static void
+isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
+{
+        struct iser_hdr *iser_hdr = &rx_desc->iser_header;
+        uint64_t read_va = 0, write_va = 0;
+        uint32_t read_stag = 0, write_stag = 0;
+        int rc;
+
+        switch (iser_hdr->flags & 0xF0) {
+        case ISCSI_CTRL:
+                if (iser_hdr->flags & ISER_RSV) {
+                        read_stag = be32_to_cpu(iser_hdr->read_stag);
+                        read_va = be64_to_cpu(iser_hdr->read_va);
+                        pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n",
+                                 read_stag, (unsigned long long)read_va);
+                }
+                if (iser_hdr->flags & ISER_WSV) {
+                        write_stag = be32_to_cpu(iser_hdr->write_stag);
+                        write_va = be64_to_cpu(iser_hdr->write_va);
+                        pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n",
+                                 write_stag, (unsigned long long)write_va);
+                }
+
+                pr_debug("ISER ISCSI_CTRL PDU\n");
+                break;
+        case ISER_HELLO:
+                pr_err("iSER Hello message\n");
+                break;
+        default:
+                pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
+                break;
+        }
+
+        rc = isert_rx_opcode(isert_conn, rx_desc,
+                             read_stag, read_va, write_stag, write_va);
+}
+
+static void
+isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn,
+                    unsigned long xfer_len)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct iscsi_hdr *hdr;
+        u64 rx_dma;
+        int rx_buflen, outstanding;
+
+        if ((char *)desc == isert_conn->login_req_buf) {
+                rx_dma = isert_conn->login_req_dma;
+                rx_buflen = ISER_RX_LOGIN_SIZE;
+                pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
+                         rx_dma, rx_buflen);
+        } else {
+                rx_dma = desc->dma_addr;
+                rx_buflen = ISER_RX_PAYLOAD_SIZE;
+                pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
+                         rx_dma, rx_buflen);
+        }
+
+        ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE);
+
+        hdr = &desc->iscsi_header;
+        pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
+                 hdr->opcode, hdr->itt, hdr->flags,
+                 (int)(xfer_len - ISER_HEADERS_LEN));
+
+        if ((char *)desc == isert_conn->login_req_buf)
+                isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN,
+                                   isert_conn);
+        else
+                isert_rx_do_work(desc, isert_conn);
+
+        ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen,
+                                      DMA_FROM_DEVICE);
+
+        isert_conn->post_recv_buf_count--;
+        pr_debug("iSERT: Decremented post_recv_buf_count: %d\n",
+                 isert_conn->post_recv_buf_count);
+
+        if ((char *)desc == isert_conn->login_req_buf)
+                return;
+
+        outstanding = isert_conn->post_recv_buf_count;
+        if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) {
+                int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding,
+                                ISERT_MIN_POSTED_RX);
+                err = isert_post_recv(isert_conn, count);
+                if (err) {
+                        pr_err("isert_post_recv() count: %d failed, %d\n",
+                               count, err);
+                }
+        }
+}
+
+static void
+isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
+{
+        struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+
+        pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n");
+
+        if (wr->sge) {
+                ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
+                wr->sge = NULL;
+        }
+
+        kfree(wr->send_wr);
+        wr->send_wr = NULL;
+
+        kfree(isert_cmd->ib_sge);
+        isert_cmd->ib_sge = NULL;
+}
+
+static void
+isert_put_cmd(struct isert_cmd *isert_cmd)
+{
+        struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
+        struct isert_conn *isert_conn = isert_cmd->conn;
+        struct iscsi_conn *conn;
+
+        pr_debug("Entering isert_put_cmd: %p\n", isert_cmd);
+
+        switch (cmd->iscsi_opcode) {
+        case ISCSI_OP_SCSI_CMD:
+                conn = isert_conn->conn;
+
+                spin_lock_bh(&conn->cmd_lock);
+                if (!list_empty(&cmd->i_conn_node))
+                        list_del(&cmd->i_conn_node);
+                spin_unlock_bh(&conn->cmd_lock);
+
+                if (cmd->data_direction == DMA_TO_DEVICE)
+                        iscsit_stop_dataout_timer(cmd);
+
+                isert_unmap_cmd(isert_cmd, isert_conn);
+                /*
+                 * Fall-through
+                 */
+        case ISCSI_OP_SCSI_TMFUNC:
+                transport_generic_free_cmd(&cmd->se_cmd, 0);
+                break;
+        case ISCSI_OP_REJECT:
+        case ISCSI_OP_NOOP_OUT:
+                conn = isert_conn->conn;
+
+                spin_lock_bh(&conn->cmd_lock);
+                if (!list_empty(&cmd->i_conn_node))
+                        list_del(&cmd->i_conn_node);
+                spin_unlock_bh(&conn->cmd_lock);
+
+                /*
+                 * Handle special case for REJECT when iscsi_add_reject*() has
+                 * overwritten the original iscsi_opcode assignment, and the
+                 * associated cmd->se_cmd needs to be released.
+                 */
+                if (cmd->se_cmd.se_tfo != NULL) {
+                        transport_generic_free_cmd(&cmd->se_cmd, 0);
+                        break;
+                }
+                /*
+                 * Fall-through
+                 */
+        default:
+                isert_release_cmd(cmd);
+                break;
+        }
+}
+
+static void
+isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
+{
+        if (tx_desc->dma_addr != 0) {
+                pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n");
+                ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
+                                    ISER_HEADERS_LEN, DMA_TO_DEVICE);
+                tx_desc->dma_addr = 0;
+        }
+}
+
+static void
+isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
+                     struct ib_device *ib_dev)
+{
+        if (isert_cmd->sense_buf_dma != 0) {
+                pr_debug("Calling ib_dma_unmap_single for isert_cmd->sense_buf_dma\n");
+                ib_dma_unmap_single(ib_dev, isert_cmd->sense_buf_dma,
+                                    isert_cmd->sense_buf_len, DMA_TO_DEVICE);
+                isert_cmd->sense_buf_dma = 0;
+        }
+
+        isert_unmap_tx_desc(tx_desc, ib_dev);
+        isert_put_cmd(isert_cmd);
+}
+
+static void
+isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
+                           struct isert_cmd *isert_cmd)
+{
+        struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
+        struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
+        struct se_cmd *se_cmd = &cmd->se_cmd;
+        struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device;
+
+        iscsit_stop_dataout_timer(cmd);
+
+        if (wr->sge) {
+                pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n");
+                ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
+                wr->sge = NULL;
+        }
+
+        if (isert_cmd->ib_sge) {
+                pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n");
+                kfree(isert_cmd->ib_sge);
+                isert_cmd->ib_sge = NULL;
+        }
+
+        cmd->write_data_done = se_cmd->data_length;
+
+        pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n");
+        spin_lock_bh(&cmd->istate_lock);
+        cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
+        cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
+        spin_unlock_bh(&cmd->istate_lock);
+
+        target_execute_cmd(se_cmd);
+}
+
+static void
+isert_do_control_comp(struct work_struct *work)
+{
+        struct isert_cmd *isert_cmd = container_of(work,
+                        struct isert_cmd, comp_work);
+        struct isert_conn *isert_conn = isert_cmd->conn;
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
+
+        switch (cmd->i_state) {
+        case ISTATE_SEND_TASKMGTRSP:
+                pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n");
+
+                atomic_dec(&isert_conn->post_send_buf_count);
+                iscsit_tmr_post_handler(cmd, cmd->conn);
+
+                cmd->i_state = ISTATE_SENT_STATUS;
+                isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev);
+                break;
+        case ISTATE_SEND_REJECT:
+                pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n");
+                atomic_dec(&isert_conn->post_send_buf_count);
+
+                cmd->i_state = ISTATE_SENT_STATUS;
+                complete(&cmd->reject_comp);
+                isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev);
+        case ISTATE_SEND_LOGOUTRSP:
+                pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n");
+                /*
+                 * Call atomic_dec(&isert_conn->post_send_buf_count)
+                 * from isert_free_conn()
+                 */
+                isert_conn->logout_posted = true;
+                iscsit_logout_post_handler(cmd, cmd->conn);
+                break;
+        default:
+                pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state);
+                dump_stack();
+                break;
+        }
+}
+
+static void
+isert_response_completion(struct iser_tx_desc *tx_desc,
+                          struct isert_cmd *isert_cmd,
+                          struct isert_conn *isert_conn,
+                          struct ib_device *ib_dev)
+{
+        struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
+
+        if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||
+            cmd->i_state == ISTATE_SEND_LOGOUTRSP) {
+                isert_unmap_tx_desc(tx_desc, ib_dev);
+
+                INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
+                queue_work(isert_comp_wq, &isert_cmd->comp_work);
+                return;
+        }
+        atomic_dec(&isert_conn->post_send_buf_count);
+
+        cmd->i_state = ISTATE_SENT_STATUS;
+        isert_completion_put(tx_desc, isert_cmd, ib_dev);
+}
+
+static void
+isert_send_completion(struct iser_tx_desc *tx_desc,
+                      struct isert_conn *isert_conn)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
+        struct isert_rdma_wr *wr;
+
+        if (!isert_cmd) {
+                atomic_dec(&isert_conn->post_send_buf_count);
+                isert_unmap_tx_desc(tx_desc, ib_dev);
+                return;
+        }
+        wr = &isert_cmd->rdma_wr;
+
+        switch (wr->iser_ib_op) {
+        case ISER_IB_RECV:
+                pr_err("isert_send_completion: Got ISER_IB_RECV\n");
+                dump_stack();
+                break;
+        case ISER_IB_SEND:
+                pr_debug("isert_send_completion: Got ISER_IB_SEND\n");
+                isert_response_completion(tx_desc, isert_cmd,
+                                          isert_conn, ib_dev);
+                break;
+        case ISER_IB_RDMA_WRITE:
+                pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n");
+                dump_stack();
+                break;
+        case ISER_IB_RDMA_READ:
+                pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n");
+
+                atomic_dec(&isert_conn->post_send_buf_count);
+                isert_completion_rdma_read(tx_desc, isert_cmd);
+                break;
+        default:
+                pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op);
+                dump_stack();
+                break;
+        }
+}
+
+static void
+isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn)
+{
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+
+        if (tx_desc) {
+                struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
+
+                if (!isert_cmd)
+                        isert_unmap_tx_desc(tx_desc, ib_dev);
+                else
+                        isert_completion_put(tx_desc, isert_cmd, ib_dev);
+        }
+
+        if (isert_conn->post_recv_buf_count == 0 &&
+            atomic_read(&isert_conn->post_send_buf_count) == 0) {
+                pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
+                pr_debug("Calling wake_up from isert_cq_comp_err\n");
+
+                isert_conn->state = ISER_CONN_TERMINATING;
+                wake_up(&isert_conn->conn_wait_comp_err);
+        }
+}
+
+static void
+isert_cq_tx_work(struct work_struct *work)
+{
+        struct isert_cq_desc *cq_desc = container_of(work,
+                                struct isert_cq_desc, cq_tx_work);
+        struct isert_device *device = cq_desc->device;
+        int cq_index = cq_desc->cq_index;
+        struct ib_cq *tx_cq = device->dev_tx_cq[cq_index];
+        struct isert_conn *isert_conn;
+        struct iser_tx_desc *tx_desc;
+        struct ib_wc wc;
+
+        while (ib_poll_cq(tx_cq, 1, &wc) == 1) {
+                tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id;
+                isert_conn = wc.qp->qp_context;
+
+                if (wc.status == IB_WC_SUCCESS) {
+                        isert_send_completion(tx_desc, isert_conn);
+                } else {
+                        pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
+                        pr_debug("TX wc.status: 0x%08x\n", wc.status);
+                        atomic_dec(&isert_conn->post_send_buf_count);
+                        isert_cq_comp_err(tx_desc, isert_conn);
+                }
+        }
+
+        ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP);
+}
+
+static void
+isert_cq_tx_callback(struct ib_cq *cq, void *context)
+{
+        struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
+
+        INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work);
+        queue_work(isert_comp_wq, &cq_desc->cq_tx_work);
+}
+
+static void
+isert_cq_rx_work(struct work_struct *work)
+{
+        struct isert_cq_desc *cq_desc = container_of(work,
+                        struct isert_cq_desc, cq_rx_work);
+        struct isert_device *device = cq_desc->device;
+        int cq_index = cq_desc->cq_index;
+        struct ib_cq *rx_cq = device->dev_rx_cq[cq_index];
+        struct isert_conn *isert_conn;
+        struct iser_rx_desc *rx_desc;
+        struct ib_wc wc;
+        unsigned long xfer_len;
+
+        while (ib_poll_cq(rx_cq, 1, &wc) == 1) {
+                rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id;
+                isert_conn = wc.qp->qp_context;
+
+                if (wc.status == IB_WC_SUCCESS) {
+                        xfer_len = (unsigned long)wc.byte_len;
+                        isert_rx_completion(rx_desc, isert_conn, xfer_len);
+                } else {
+                        pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
+                        if (wc.status != IB_WC_WR_FLUSH_ERR)
+                                pr_debug("RX wc.status: 0x%08x\n", wc.status);
+
+                        isert_conn->post_recv_buf_count--;
+                        isert_cq_comp_err(NULL, isert_conn);
+                }
+        }
+
+        ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP);
+}
+
+static void
+isert_cq_rx_callback(struct ib_cq *cq, void *context)
+{
+        struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
+
+        INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work);
+        queue_work(isert_rx_wq, &cq_desc->cq_rx_work);
+}
+
+static int
+isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
+{
+        struct ib_send_wr *wr_failed;
+        int ret;
+
+        atomic_inc(&isert_conn->post_send_buf_count);
+
+        ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr,
+                           &wr_failed);
+        if (ret) {
+                pr_err("ib_post_send failed with %d\n", ret);
+                atomic_dec(&isert_conn->post_send_buf_count);
+                return ret;
+        }
+        return ret;
+}
+
+static int
+isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
+{
+        struct isert_cmd *isert_cmd = container_of(cmd,
+                                        struct isert_cmd, iscsi_cmd);
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
+        struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
+                                &isert_cmd->tx_desc.iscsi_header;
+
+        isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
+        iscsit_build_rsp_pdu(cmd, conn, true, hdr);
+        isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
+        /*
+         * Attach SENSE DATA payload to iSCSI Response PDU
+         */
+        if (cmd->se_cmd.sense_buffer &&
+            ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
+            (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
+                struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+                struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
+                u32 padding, sense_len;
+
+                put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
+                                   cmd->sense_buffer);
+                cmd->se_cmd.scsi_sense_length += sizeof(__be16);
+
+                padding = -(cmd->se_cmd.scsi_sense_length) & 3;
+                hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
+                sense_len = cmd->se_cmd.scsi_sense_length + padding;
+
+                isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
+                                (void *)cmd->sense_buffer, sense_len,
+                                DMA_TO_DEVICE);
+
+                isert_cmd->sense_buf_len = sense_len;
+                tx_dsg->addr    = isert_cmd->sense_buf_dma;
+                tx_dsg->length  = sense_len;
+                tx_dsg->lkey    = isert_conn->conn_mr->lkey;
+                isert_cmd->tx_desc.num_sge = 2;
+        }
+
+        isert_init_send_wr(isert_cmd, send_wr);
+
+        pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
+
+        return isert_post_response(isert_conn, isert_cmd);
+}
+
+static int
+isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
+                bool nopout_response)
+{
+        struct isert_cmd *isert_cmd = container_of(cmd,
+                                struct isert_cmd, iscsi_cmd);
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
+
+        isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
+        iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
+                               &isert_cmd->tx_desc.iscsi_header,
+                               nopout_response);
+        isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
+        isert_init_send_wr(isert_cmd, send_wr);
+
+        pr_debug("Posting NOPIN Reponse IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
+
+        return isert_post_response(isert_conn, isert_cmd);
+}
+
+static int
+isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
+{
+        struct isert_cmd *isert_cmd = container_of(cmd,
+                                struct isert_cmd, iscsi_cmd);
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
+
+        isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
+        iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
+                                &isert_cmd->tx_desc.iscsi_header);
+        isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
+        isert_init_send_wr(isert_cmd, send_wr);
+
+        pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
+
+        return isert_post_response(isert_conn, isert_cmd);
+}
+
+static int
+isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
+{
+        struct isert_cmd *isert_cmd = container_of(cmd,
+                                struct isert_cmd, iscsi_cmd);
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
+
+        isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
+        iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
+                                  &isert_cmd->tx_desc.iscsi_header);
+        isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
+        isert_init_send_wr(isert_cmd, send_wr);
+
+        pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
+
+        return isert_post_response(isert_conn, isert_cmd);
+}
+
+static int
+isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
+{
+        struct isert_cmd *isert_cmd = container_of(cmd,
+                                struct isert_cmd, iscsi_cmd);
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
+
+        isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
+        iscsit_build_reject(cmd, conn, (struct iscsi_reject *)
+                                &isert_cmd->tx_desc.iscsi_header);
+        isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
+        isert_init_send_wr(isert_cmd, send_wr);
+
+        pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
+
+        return isert_post_response(isert_conn, isert_cmd);
+}
+
+static int
+isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
+                    struct ib_sge *ib_sge, struct ib_send_wr *send_wr,
+                    u32 data_left, u32 offset)
+{
+        struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
+        struct scatterlist *sg_start, *tmp_sg;
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        u32 sg_off, page_off;
+        int i = 0, sg_nents;
+
+        sg_off = offset / PAGE_SIZE;
+        sg_start = &cmd->se_cmd.t_data_sg[sg_off];
+        sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);
+        page_off = offset % PAGE_SIZE;
+
+        send_wr->sg_list = ib_sge;
+        send_wr->num_sge = sg_nents;
+        send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
+        /*
+         * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
+         */
+        for_each_sg(sg_start, tmp_sg, sg_nents, i) {
+                pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n",
+                         (unsigned long long)tmp_sg->dma_address,
+                         tmp_sg->length, page_off);
+
+                ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off;
+                ib_sge->length = min_t(u32, data_left,
+                                ib_sg_dma_len(ib_dev, tmp_sg) - page_off);
+                ib_sge->lkey = isert_conn->conn_mr->lkey;
+
+                pr_debug("RDMA ib_sge: addr: 0x%16llx  length: %u\n",
+                         ib_sge->addr, ib_sge->length);
+                page_off = 0;
+                data_left -= ib_sge->length;
+                ib_sge++;
+                pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge);
+        }
+
+        pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
+                 send_wr->sg_list, send_wr->num_sge);
+
+        return sg_nents;
+}
+
+static int
+isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
+{
+        struct se_cmd *se_cmd = &cmd->se_cmd;
+        struct isert_cmd *isert_cmd = container_of(cmd,
+                                        struct isert_cmd, iscsi_cmd);
+        struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        struct ib_send_wr *wr_failed, *send_wr;
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct ib_sge *ib_sge;
+        struct scatterlist *sg;
+        u32 offset = 0, data_len, data_left, rdma_write_max;
+        int rc, ret = 0, count, sg_nents, i, ib_sge_cnt;
+
+        pr_debug("RDMA_WRITE: data_length: %u\n", se_cmd->data_length);
+
+        sg = &se_cmd->t_data_sg[0];
+        sg_nents = se_cmd->t_data_nents;
+
+        count = ib_dma_map_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
+        if (unlikely(!count)) {
+                pr_err("Unable to map put_datain SGs\n");
+                return -EINVAL;
+        }
+        wr->sge = sg;
+        wr->num_sge = sg_nents;
+        pr_debug("Mapped IB count: %u sg: %p sg_nents: %u for RDMA_WRITE\n",
+                 count, sg, sg_nents);
+
+        ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
+        if (!ib_sge) {
+                pr_warn("Unable to allocate datain ib_sge\n");
+                ret = -ENOMEM;
+                goto unmap_sg;
+        }
+        isert_cmd->ib_sge = ib_sge;
+
+        pr_debug("Allocated ib_sge: %p from t_data_ents: %d for RDMA_WRITE\n",
+                 ib_sge, se_cmd->t_data_nents);
+
+        wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
+        wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
+                                GFP_KERNEL);
+        if (!wr->send_wr) {
+                pr_err("Unable to allocate wr->send_wr\n");
+                ret = -ENOMEM;
+                goto unmap_sg;
+        }
+        pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
+                 wr->send_wr, wr->send_wr_num);
+
+        iscsit_increment_maxcmdsn(cmd, conn->sess);
+        cmd->stat_sn = conn->stat_sn++;
+
+        wr->isert_cmd = isert_cmd;
+        rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
+        data_left = se_cmd->data_length;
+
+        for (i = 0; i < wr->send_wr_num; i++) {
+                send_wr = &isert_cmd->rdma_wr.send_wr[i];
+                data_len = min(data_left, rdma_write_max);
+
+                send_wr->opcode = IB_WR_RDMA_WRITE;
+                send_wr->send_flags = 0;
+                send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset;
+                send_wr->wr.rdma.rkey = isert_cmd->read_stag;
+
+                ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
+                                        send_wr, data_len, offset);
+                ib_sge += ib_sge_cnt;
+
+                if (i + 1 == wr->send_wr_num)
+                        send_wr->next = &isert_cmd->tx_desc.send_wr;
+                else
+                        send_wr->next = &wr->send_wr[i + 1];
+
+                offset += data_len;
+                data_left -= data_len;
+        }
+        /*
+         * Build isert_conn->tx_desc for iSCSI response PDU and attach
+         */
+        isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
+        iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *)
+                             &isert_cmd->tx_desc.iscsi_header);
+        isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
+        isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr);
+
+        atomic_inc(&isert_conn->post_send_buf_count);
+
+        rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
+        if (rc) {
+                pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
+                atomic_dec(&isert_conn->post_send_buf_count);
+        }
+        pr_debug("Posted RDMA_WRITE + Response for iSER Data READ\n");
+        return 1;
+
+unmap_sg:
+        ib_dma_unmap_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
+        return ret;
+}
+
+static int
+isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
+{
+        struct se_cmd *se_cmd = &cmd->se_cmd;
+        struct isert_cmd *isert_cmd = container_of(cmd,
+                                        struct isert_cmd, iscsi_cmd);
+        struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        struct ib_send_wr *wr_failed, *send_wr;
+        struct ib_sge *ib_sge;
+        struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
+        struct scatterlist *sg_start;
+        u32 sg_off, sg_nents, page_off, va_offset = 0;
+        u32 offset = 0, data_len, data_left, rdma_write_max;
+        int rc, ret = 0, count, i, ib_sge_cnt;
+
+        pr_debug("RDMA_READ: data_length: %u write_data_done: %u\n",
+                 se_cmd->data_length, cmd->write_data_done);
+
+        sg_off = cmd->write_data_done / PAGE_SIZE;
+        sg_start = &cmd->se_cmd.t_data_sg[sg_off];
+        page_off = cmd->write_data_done % PAGE_SIZE;
+
+        pr_debug("RDMA_READ: sg_off: %d, sg_start: %p page_off: %d\n",
+                 sg_off, sg_start, page_off);
+
+        data_left = se_cmd->data_length - cmd->write_data_done;
+        sg_nents = se_cmd->t_data_nents - sg_off;
+
+        pr_debug("RDMA_READ: data_left: %d, sg_nents: %d\n",
+                 data_left, sg_nents);
+
+        count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
+        if (unlikely(!count)) {
+                pr_err("Unable to map get_dataout SGs\n");
+                return -EINVAL;
+        }
+        wr->sge = sg_start;
+        wr->num_sge = sg_nents;
+        pr_debug("Mapped IB count: %u sg_start: %p sg_nents: %u for RDMA_READ\n",
+                 count, sg_start, sg_nents);
+
+        ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
+        if (!ib_sge) {
+                pr_warn("Unable to allocate dataout ib_sge\n");
+                ret = -ENOMEM;
+                goto unmap_sg;
+        }
+        isert_cmd->ib_sge = ib_sge;
+
+        pr_debug("Using ib_sge: %p from sg_ents: %d for RDMA_READ\n",
+                 ib_sge, sg_nents);
+
+        wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
+        wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
+                                GFP_KERNEL);
+        if (!wr->send_wr) {
+                pr_debug("Unable to allocate wr->send_wr\n");
+                ret = -ENOMEM;
+                goto unmap_sg;
+        }
+        pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
+                 wr->send_wr, wr->send_wr_num);
+
+        isert_cmd->tx_desc.isert_cmd = isert_cmd;
+
+        wr->iser_ib_op = ISER_IB_RDMA_READ;
+        wr->isert_cmd = isert_cmd;
+        rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
+        offset = cmd->write_data_done;
+
+        for (i = 0; i < wr->send_wr_num; i++) {
+                send_wr = &isert_cmd->rdma_wr.send_wr[i];
+                data_len = min(data_left, rdma_write_max);
+
+                send_wr->opcode = IB_WR_RDMA_READ;
+                send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset;
+                send_wr->wr.rdma.rkey = isert_cmd->write_stag;
+
+                ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
+                                        send_wr, data_len, offset);
+                ib_sge += ib_sge_cnt;
+
+                if (i + 1 == wr->send_wr_num)
+                        send_wr->send_flags = IB_SEND_SIGNALED;
+                else
+                        send_wr->next = &wr->send_wr[i + 1];
+
+                offset += data_len;
+                va_offset += data_len;
+                data_left -= data_len;
+        }
+
+        atomic_inc(&isert_conn->post_send_buf_count);
+
+        rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
+        if (rc) {
+                pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
+                atomic_dec(&isert_conn->post_send_buf_count);
+        }
+        pr_debug("Posted RDMA_READ memory for ISER Data WRITE\n");
+        return 0;
+
+unmap_sg:
+        ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
+        return ret;
+}
+
+static int
+isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
+{
+        int ret;
+
+        switch (state) {
+        case ISTATE_SEND_NOPIN_WANT_RESPONSE:
+                ret = isert_put_nopin(cmd, conn, false);
+                break;
+        default:
+                pr_err("Unknown immediate state: 0x%02x\n", state);
+                ret = -EINVAL;
+                break;
+        }
+
+        return ret;
+}
+
+static int
+isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
+{
+        int ret;
+
+        switch (state) {
+        case ISTATE_SEND_LOGOUTRSP:
+                ret = isert_put_logout_rsp(cmd, conn);
+                if (!ret) {
+                        pr_debug("Returning iSER Logout -EAGAIN\n");
+                        ret = -EAGAIN;
+                }
+                break;
+        case ISTATE_SEND_NOPIN:
+                ret = isert_put_nopin(cmd, conn, true);
+                break;
+        case ISTATE_SEND_TASKMGTRSP:
+                ret = isert_put_tm_rsp(cmd, conn);
+                break;
+        case ISTATE_SEND_REJECT:
+                ret = isert_put_reject(cmd, conn);
+                break;
+        case ISTATE_SEND_STATUS:
+                /*
+                 * Special case for sending non GOOD SCSI status from TX thread
+                 * context during pre se_cmd excecution failure.
+                 */
+                ret = isert_put_response(conn, cmd);
+                break;
+        default:
+                pr_err("Unknown response state: 0x%02x\n", state);
+                ret = -EINVAL;
+                break;
+        }
+
+        return ret;
+}
+
+static int
+isert_setup_np(struct iscsi_np *np,
+               struct __kernel_sockaddr_storage *ksockaddr)
+{
+        struct isert_np *isert_np;
+        struct rdma_cm_id *isert_lid;
+        struct sockaddr *sa;
+        int ret;
+
+        isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
+        if (!isert_np) {
+                pr_err("Unable to allocate struct isert_np\n");
+                return -ENOMEM;
+        }
+        init_waitqueue_head(&isert_np->np_accept_wq);
+        mutex_init(&isert_np->np_accept_mutex);
+        INIT_LIST_HEAD(&isert_np->np_accept_list);
+        init_completion(&isert_np->np_login_comp);
+
+        sa = (struct sockaddr *)ksockaddr;
+        pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa);
+        /*
+         * Setup the np->np_sockaddr from the passed sockaddr setup
+         * in iscsi_target_configfs.c code..
+         */
+        memcpy(&np->np_sockaddr, ksockaddr,
+               sizeof(struct __kernel_sockaddr_storage));
+
+        isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP,
+                                IB_QPT_RC);
+        if (IS_ERR(isert_lid)) {
+                pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n",
+                       PTR_ERR(isert_lid));
+                ret = PTR_ERR(isert_lid);
+                goto out;
+        }
+
+        ret = rdma_bind_addr(isert_lid, sa);
+        if (ret) {
+                pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret);
+                goto out_lid;
+        }
+
+        ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG);
+        if (ret) {
+                pr_err("rdma_listen() for isert_lid failed: %d\n", ret);
+                goto out_lid;
+        }
+
+        isert_np->np_cm_id = isert_lid;
+        np->np_context = isert_np;
+        pr_debug("Setup isert_lid->context: %p\n", isert_lid->context);
+
+        return 0;
+
+out_lid:
+        rdma_destroy_id(isert_lid);
+out:
+        kfree(isert_np);
+        return ret;
+}
+
+static int
+isert_check_accept_queue(struct isert_np *isert_np)
+{
+        int empty;
+
+        mutex_lock(&isert_np->np_accept_mutex);
+        empty = list_empty(&isert_np->np_accept_list);
+        mutex_unlock(&isert_np->np_accept_mutex);
+
+        return empty;
+}
+
+static int
+isert_rdma_accept(struct isert_conn *isert_conn)
+{
+        struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
+        struct rdma_conn_param cp;
+        int ret;
+
+        memset(&cp, 0, sizeof(struct rdma_conn_param));
+        cp.responder_resources = isert_conn->responder_resources;
+        cp.initiator_depth = isert_conn->initiator_depth;
+        cp.retry_count = 7;
+        cp.rnr_retry_count = 7;
+
+        pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n");
+
+        ret = rdma_accept(cm_id, &cp);
+        if (ret) {
+                pr_err("rdma_accept() failed with: %d\n", ret);
+                return ret;
+        }
+
+        pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n");
+
+        return 0;
+}
+
+static int
+isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
+{
+        struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
+        int ret;
+
+        pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn);
+
+        ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp);
+        if (ret)
+                return ret;
+
+        pr_debug("isert_get_login_rx processing login->req: %p\n", login->req);
+        return 0;
+}
+
+static void
+isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
+                    struct isert_conn *isert_conn)
+{
+        struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
+        struct rdma_route *cm_route = &cm_id->route;
+        struct sockaddr_in *sock_in;
+        struct sockaddr_in6 *sock_in6;
+
+        conn->login_family = np->np_sockaddr.ss_family;
+
+        if (np->np_sockaddr.ss_family == AF_INET6) {
+                sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr;
+                snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c",
+                         &sock_in6->sin6_addr.in6_u);
+                conn->login_port = ntohs(sock_in6->sin6_port);
+
+                sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr;
+                snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c",
+                         &sock_in6->sin6_addr.in6_u);
+                conn->local_port = ntohs(sock_in6->sin6_port);
+        } else {
+                sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr;
+                sprintf(conn->login_ip, "%pI4",
+                        &sock_in->sin_addr.s_addr);
+                conn->login_port = ntohs(sock_in->sin_port);
+
+                sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr;
+                sprintf(conn->local_ip, "%pI4",
+                        &sock_in->sin_addr.s_addr);
+                conn->local_port = ntohs(sock_in->sin_port);
+        }
+}
+
+static int
+isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
+{
+        struct isert_np *isert_np = (struct isert_np *)np->np_context;
+        struct isert_conn *isert_conn;
+        int max_accept = 0, ret;
+
+accept_wait:
+        ret = wait_event_interruptible(isert_np->np_accept_wq,
+                        !isert_check_accept_queue(isert_np) ||
+                        np->np_thread_state == ISCSI_NP_THREAD_RESET);
+        if (max_accept > 5)
+                return -ENODEV;
+
+        spin_lock_bh(&np->np_thread_lock);
+        if (np->np_thread_state == ISCSI_NP_THREAD_RESET) {
+                spin_unlock_bh(&np->np_thread_lock);
+                pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n");
+                return -ENODEV;
+        }
+        spin_unlock_bh(&np->np_thread_lock);
+
+        mutex_lock(&isert_np->np_accept_mutex);
+        if (list_empty(&isert_np->np_accept_list)) {
+                mutex_unlock(&isert_np->np_accept_mutex);
+                max_accept++;
+                goto accept_wait;
+        }
+        isert_conn = list_first_entry(&isert_np->np_accept_list,
+                        struct isert_conn, conn_accept_node);
+        list_del_init(&isert_conn->conn_accept_node);
+        mutex_unlock(&isert_np->np_accept_mutex);
+
+        conn->context = isert_conn;
+        isert_conn->conn = conn;
+        max_accept = 0;
+
+        ret = isert_rdma_post_recvl(isert_conn);
+        if (ret)
+                return ret;
+
+        ret = isert_rdma_accept(isert_conn);
+        if (ret)
+                return ret;
+
+        isert_set_conn_info(np, conn, isert_conn);
+
+        pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn);
+        return 0;
+}
+
+static void
+isert_free_np(struct iscsi_np *np)
+{
+        struct isert_np *isert_np = (struct isert_np *)np->np_context;
+
+        rdma_destroy_id(isert_np->np_cm_id);
+
+        np->np_context = NULL;
+        kfree(isert_np);
+}
+
+static void isert_free_conn(struct iscsi_conn *conn)
+{
+        struct isert_conn *isert_conn = conn->context;
+
+        pr_debug("isert_free_conn: Starting \n");
+        /*
+         * Decrement post_send_buf_count for special case when called
+         * from isert_do_control_comp() -> iscsit_logout_post_handler()
+         */
+        if (isert_conn->logout_posted)
+                atomic_dec(&isert_conn->post_send_buf_count);
+
+        if (isert_conn->conn_cm_id)
+                rdma_disconnect(isert_conn->conn_cm_id);
+        /*
+         * Only wait for conn_wait_comp_err if the isert_conn made it
+         * into full feature phase..
+         */
+        if (isert_conn->state > ISER_CONN_INIT) {
+                pr_debug("isert_free_conn: Before wait_event comp_err %d\n",
+                         isert_conn->state);
+                wait_event(isert_conn->conn_wait_comp_err,
+                           isert_conn->state == ISER_CONN_TERMINATING);
+                pr_debug("isert_free_conn: After wait_event #1 >>>>>>>>>>>>\n");
+        }
+
+        pr_debug("isert_free_conn: wait_event conn_wait %d\n", isert_conn->state);
+        wait_event(isert_conn->conn_wait, isert_conn->state == ISER_CONN_DOWN);
+        pr_debug("isert_free_conn: After wait_event #2 >>>>>>>>>>>>>>>>>>>>\n");
+
+        isert_put_conn(isert_conn);
+}
+
+static struct iscsit_transport iser_target_transport = {
+        .name                   = "IB/iSER",
+        .transport_type         = ISCSI_INFINIBAND,
+        .owner                  = THIS_MODULE,
+        .iscsit_setup_np        = isert_setup_np,
+        .iscsit_accept_np       = isert_accept_np,
+        .iscsit_free_np         = isert_free_np,
+        .iscsit_free_conn       = isert_free_conn,
+        .iscsit_alloc_cmd       = isert_alloc_cmd,
+        .iscsit_get_login_rx    = isert_get_login_rx,
+        .iscsit_put_login_tx    = isert_put_login_tx,
+        .iscsit_immediate_queue = isert_immediate_queue,
+        .iscsit_response_queue  = isert_response_queue,
+        .iscsit_get_dataout     = isert_get_dataout,
+        .iscsit_queue_data_in   = isert_put_datain,
+        .iscsit_queue_status    = isert_put_response,
+};
+
+static int __init isert_init(void)
+{
+        int ret;
+
+        isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0);
+        if (!isert_rx_wq) {
+                pr_err("Unable to allocate isert_rx_wq\n");
+                return -ENOMEM;
+        }
+
+        isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0);
+        if (!isert_comp_wq) {
+                pr_err("Unable to allocate isert_comp_wq\n");
+                ret = -ENOMEM;
+                goto destroy_rx_wq;
+        }
+
+        isert_cmd_cache = kmem_cache_create("isert_cmd_cache",
+                        sizeof(struct isert_cmd), __alignof__(struct isert_cmd),
+                        0, NULL);
+        if (!isert_cmd_cache) {
+                pr_err("Unable to create isert_cmd_cache\n");
+                ret = -ENOMEM;
+                goto destroy_tx_cq;
+        }
+
+        iscsit_register_transport(&iser_target_transport);
+        pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n");
+        return 0;
+
+destroy_tx_cq:
+        destroy_workqueue(isert_comp_wq);
+destroy_rx_wq:
+        destroy_workqueue(isert_rx_wq);
+        return ret;
+}
+
+static void __exit isert_exit(void)
+{
+        kmem_cache_destroy(isert_cmd_cache);
+        destroy_workqueue(isert_comp_wq);
+        destroy_workqueue(isert_rx_wq);
+        iscsit_unregister_transport(&iser_target_transport);
+        pr_debug("iSER_TARGET[0] - Released iser_target_transport\n");
+}
+
+MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
+MODULE_VERSION("0.1");
+MODULE_AUTHOR("nab@Linux-iSCSI.org");
+MODULE_LICENSE("GPL");
+
+module_init(isert_init);
+module_exit(isert_exit);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
new file mode 100644
index 000000000000..b104f4c2cd38
--- /dev/null
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -0,0 +1,138 @@
+#include <linux/socket.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/rdma_cm.h>
+
+#define ISERT_RDMA_LISTEN_BACKLOG       10
+
+enum isert_desc_type {
+        ISCSI_TX_CONTROL,
+        ISCSI_TX_DATAIN
+};
+
+enum iser_ib_op_code {
+        ISER_IB_RECV,
+        ISER_IB_SEND,
+        ISER_IB_RDMA_WRITE,
+        ISER_IB_RDMA_READ,
+};
+
+enum iser_conn_state {
+        ISER_CONN_INIT,
+        ISER_CONN_UP,
+        ISER_CONN_TERMINATING,
+        ISER_CONN_DOWN,
+};
+
+struct iser_rx_desc {
+        struct iser_hdr iser_header;
+        struct iscsi_hdr iscsi_header;
+        char            data[ISER_RECV_DATA_SEG_LEN];
+        u64             dma_addr;
+        struct ib_sge   rx_sg;
+        char            pad[ISER_RX_PAD_SIZE];
+} __packed;
+
+struct iser_tx_desc {
+        struct iser_hdr iser_header;
+        struct iscsi_hdr iscsi_header;
+        enum isert_desc_type type;
+        u64             dma_addr;
+        struct ib_sge   tx_sg[2];
+        int             num_sge;
+        struct isert_cmd *isert_cmd;
+        struct ib_send_wr send_wr;
+} __packed;
+
+struct isert_rdma_wr {
+        struct list_head        wr_list;
+        struct isert_cmd        *isert_cmd;
+        enum iser_ib_op_code    iser_ib_op;
+        struct ib_sge           *ib_sge;
+        int                     num_sge;
+        struct scatterlist      *sge;
+        int                     send_wr_num;
+        struct ib_send_wr       *send_wr;
+};
+
+struct isert_cmd {
+        uint32_t                read_stag;
+        uint32_t                write_stag;
+        uint64_t                read_va;
+        uint64_t                write_va;
+        u64                     sense_buf_dma;
+        u32                     sense_buf_len;
+        u32                     read_va_off;
+        u32                     write_va_off;
+        u32                     rdma_wr_num;
+        struct isert_conn       *conn;
+        struct iscsi_cmd        iscsi_cmd;
+        struct ib_sge           *ib_sge;
+        struct iser_tx_desc     tx_desc;
+        struct isert_rdma_wr    rdma_wr;
+        struct work_struct      comp_work;
+};
+
+struct isert_device;
+
+struct isert_conn {
+        enum iser_conn_state    state;
+        bool                    logout_posted;
+        int                     post_recv_buf_count;
+        atomic_t                post_send_buf_count;
+        u32                     responder_resources;
+        u32                     initiator_depth;
+        u32                     max_sge;
+        char                    *login_buf;
+        char                    *login_req_buf;
+        char                    *login_rsp_buf;
+        u64                     login_req_dma;
+        u64                     login_rsp_dma;
+        unsigned int            conn_rx_desc_head;
+        struct iser_rx_desc     *conn_rx_descs;
+        struct ib_recv_wr       conn_rx_wr[ISERT_MIN_POSTED_RX];
+        struct iscsi_conn       *conn;
+        struct list_head        conn_accept_node;
+        struct completion       conn_login_comp;
+        struct iser_tx_desc     conn_login_tx_desc;
+        struct rdma_cm_id       *conn_cm_id;
+        struct ib_pd            *conn_pd;
+        struct ib_mr            *conn_mr;
+        struct ib_qp            *conn_qp;
+        struct isert_device     *conn_device;
+        struct work_struct      conn_logout_work;
+        wait_queue_head_t       conn_wait;
+        wait_queue_head_t       conn_wait_comp_err;
+        struct kref             conn_kref;
+};
+
+#define ISERT_MAX_CQ 64
+
+struct isert_cq_desc {
+        struct isert_device     *device;
+        int                     cq_index;
+        struct work_struct      cq_rx_work;
+        struct work_struct      cq_tx_work;
+};
+
+struct isert_device {
+        int                     cqs_used;
+        int                     refcount;
+        int                     cq_active_qps[ISERT_MAX_CQ];
+        struct ib_device        *ib_device;
+        struct ib_pd            *dev_pd;
+        struct ib_mr            *dev_mr;
+        struct ib_cq            *dev_rx_cq[ISERT_MAX_CQ];
+        struct ib_cq            *dev_tx_cq[ISERT_MAX_CQ];
+        struct isert_cq_desc    *cq_desc;
+        struct list_head        dev_node;
+};
+
+struct isert_np {
+        wait_queue_head_t       np_accept_wq;
+        struct rdma_cm_id       *np_cm_id;
+        struct mutex            np_accept_mutex;
+        struct list_head        np_accept_list;
+        struct completion       np_login_comp;
+};
diff --git a/drivers/infiniband/ulp/isert/isert_proto.h b/drivers/infiniband/ulp/isert/isert_proto.h
new file mode 100644
index 000000000000..4dccd313b777
--- /dev/null
+++ b/drivers/infiniband/ulp/isert/isert_proto.h
@@ -0,0 +1,47 @@
+/* From iscsi_iser.h */
+
+struct iser_hdr {
+        u8      flags;
+        u8      rsvd[3];
+        __be32  write_stag; /* write rkey */
+        __be64  write_va;
+        __be32  read_stag;  /* read rkey */
+        __be64  read_va;
+} __packed;
+
+/*Constant PDU lengths calculations */
+#define ISER_HEADERS_LEN  (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr))
+
+#define ISER_RECV_DATA_SEG_LEN  8192
+#define ISER_RX_PAYLOAD_SIZE    (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
+#define ISER_RX_LOGIN_SIZE      (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
+
+/* QP settings */
+/* Maximal bounds on received asynchronous PDUs */
+#define ISERT_MAX_TX_MISC_PDUS  4 /* NOOP_IN(2) , ASYNC_EVENT(2)   */
+
+#define ISERT_MAX_RX_MISC_PDUS  6 /* NOOP_OUT(2), TEXT(1),         *
+                                   * SCSI_TMFUNC(2), LOGOUT(1) */
+
+#define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */
+
+#define ISERT_QP_MAX_RECV_DTOS  (ISCSI_DEF_XMIT_CMDS_MAX)
+
+#define ISERT_MIN_POSTED_RX     (ISCSI_DEF_XMIT_CMDS_MAX >> 2)
+
+#define ISERT_INFLIGHT_DATAOUTS 8
+
+#define ISERT_QP_MAX_REQ_DTOS   (ISCSI_DEF_XMIT_CMDS_MAX *    \
+                                (1 + ISERT_INFLIGHT_DATAOUTS) + \
+                                ISERT_MAX_TX_MISC_PDUS  + \
+                                ISERT_MAX_RX_MISC_PDUS)
+
+#define ISER_RX_PAD_SIZE        (ISER_RECV_DATA_SEG_LEN + 4096 - \
+                (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge)))
+
+#define ISER_VER        0x10
+#define ISER_WSV        0x08
+#define ISER_RSV        0x04
+#define ISCSI_CTRL      0x10
+#define ISER_HELLO      0x20
+#define ISER_HELLORPLY  0x30
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index 61b5d8c2b5da..fcdc22306cab 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -2585,25 +2585,6 @@ static void qlt_do_ctio_completion(struct scsi_qla_host *vha, uint32_t handle,
         ha->tgt.tgt_ops->free_cmd(cmd);
 }
 
-/* ha->hardware_lock supposed to be held on entry */
-/* called via callback from qla2xxx */
-void qlt_ctio_completion(struct scsi_qla_host *vha, uint32_t handle)
-{
-        struct qla_hw_data *ha = vha->hw;
-        struct qla_tgt *tgt = ha->tgt.qla_tgt;
-
-        if (likely(tgt == NULL)) {
-                ql_dbg(ql_dbg_tgt, vha, 0xe021,
-                    "CTIO, but target mode not enabled"
-                    " (ha %d %p handle %#x)", vha->vp_idx, ha, handle);
-                return;
-        }
-
-        tgt->irq_cmd_count++;
-        qlt_do_ctio_completion(vha, handle, CTIO_SUCCESS, NULL);
-        tgt->irq_cmd_count--;
-}
-
 static inline int qlt_get_fcp_task_attr(struct scsi_qla_host *vha,
         uint8_t task_codes)
 {
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index ff9ccb9fd036..b33e411f28a0 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -980,7 +980,6 @@ extern int qlt_xmit_response(struct qla_tgt_cmd *, int, uint8_t);
 extern void qlt_xmit_tm_rsp(struct qla_tgt_mgmt_cmd *);
 extern void qlt_free_mcmd(struct qla_tgt_mgmt_cmd *);
 extern void qlt_free_cmd(struct qla_tgt_cmd *cmd);
-extern void qlt_ctio_completion(struct scsi_qla_host *, uint32_t);
 extern void qlt_async_event(uint16_t, struct scsi_qla_host *, uint16_t *);
 extern void qlt_enable_vha(struct scsi_qla_host *);
 extern void qlt_vport_create(struct scsi_qla_host *, struct qla_hw_data *);
diff --git a/drivers/target/iscsi/Makefile b/drivers/target/iscsi/Makefile
index 5b9a2cf7f0a9..13a92403fe3e 100644
--- a/drivers/target/iscsi/Makefile
+++ b/drivers/target/iscsi/Makefile
@@ -15,6 +15,7 @@ iscsi_target_mod-y +=		iscsi_target_parameters.o \
                                 iscsi_target_util.o \
                                 iscsi_target.o \
                                 iscsi_target_configfs.o \
-                                iscsi_target_stat.o
+                                iscsi_target_stat.o \
+                                iscsi_target_transport.o
 
 obj-$(CONFIG_ISCSI_TARGET)      += iscsi_target_mod.o
diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
index 7ea246a07731..ffbc6a94be52 100644
--- a/drivers/target/iscsi/iscsi_target.c
+++ b/drivers/target/iscsi/iscsi_target.c
@@ -49,6 +49,8 @@
 #include "iscsi_target_device.h"
 #include "iscsi_target_stat.h"
 
+#include <target/iscsi/iscsi_transport.h>
+
 static LIST_HEAD(g_tiqn_list);
 static LIST_HEAD(g_np_list);
 static DEFINE_SPINLOCK(tiqn_lock);
@@ -68,8 +70,7 @@ struct kmem_cache *lio_ooo_cache;
 struct kmem_cache *lio_r2t_cache;
 
 static int iscsit_handle_immediate_data(struct iscsi_cmd *,
-                        unsigned char *buf, u32);
-static int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *);
+                        struct iscsi_scsi_req *, u32);
 
 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
 {
@@ -401,8 +402,7 @@ struct iscsi_np *iscsit_add_np(
         spin_unlock_bh(&np_lock);
 
         pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n",
-                np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ?
-                "TCP" : "SCTP");
+                np->np_ip, np->np_port, np->np_transport->name);
 
         return np;
 }
@@ -441,11 +441,10 @@ int iscsit_reset_np_thread(
         return 0;
 }
 
-static int iscsit_del_np_comm(struct iscsi_np *np)
+static void iscsit_free_np(struct iscsi_np *np)
 {
         if (np->np_socket)
                 sock_release(np->np_socket);
-        return 0;
 }
 
 int iscsit_del_np(struct iscsi_np *np)
@@ -467,20 +466,47 @@ int iscsit_del_np(struct iscsi_np *np)
                 send_sig(SIGINT, np->np_thread, 1);
                 kthread_stop(np->np_thread);
         }
-        iscsit_del_np_comm(np);
+
+        np->np_transport->iscsit_free_np(np);
 
         spin_lock_bh(&np_lock);
         list_del(&np->np_list);
         spin_unlock_bh(&np_lock);
 
         pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n",
-                np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ?
-                "TCP" : "SCTP");
+                np->np_ip, np->np_port, np->np_transport->name);
 
+        iscsit_put_transport(np->np_transport);
         kfree(np);
         return 0;
 }
 
+static int iscsit_immediate_queue(struct iscsi_conn *, struct iscsi_cmd *, int);
+static int iscsit_response_queue(struct iscsi_conn *, struct iscsi_cmd *, int);
+
+static int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
+{
+        iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
+        return 0;
+}
+
+static struct iscsit_transport iscsi_target_transport = {
+        .name                   = "iSCSI/TCP",
+        .transport_type         = ISCSI_TCP,
+        .owner                  = NULL,
+        .iscsit_setup_np        = iscsit_setup_np,
+        .iscsit_accept_np       = iscsit_accept_np,
+        .iscsit_free_np         = iscsit_free_np,
+        .iscsit_alloc_cmd       = iscsit_alloc_cmd,
+        .iscsit_get_login_rx    = iscsit_get_login_rx,
+        .iscsit_put_login_tx    = iscsit_put_login_tx,
+        .iscsit_get_dataout     = iscsit_build_r2ts_for_cmd,
+        .iscsit_immediate_queue = iscsit_immediate_queue,
+        .iscsit_response_queue  = iscsit_response_queue,
+        .iscsit_queue_data_in   = iscsit_queue_rsp,
+        .iscsit_queue_status    = iscsit_queue_rsp,
+};
+
 static int __init iscsi_target_init_module(void)
 {
         int ret = 0;
@@ -557,6 +583,8 @@ static int __init iscsi_target_init_module(void)
                 goto ooo_out;
         }
 
+        iscsit_register_transport(&iscsi_target_transport);
+
         if (iscsit_load_discovery_tpg() < 0)
                 goto r2t_out;
 
@@ -587,6 +615,7 @@ static void __exit iscsi_target_cleanup_module(void)
         iscsi_deallocate_thread_sets();
         iscsi_thread_set_free();
         iscsit_release_discovery_tpg();
+        iscsit_unregister_transport(&iscsi_target_transport);
         kmem_cache_destroy(lio_cmd_cache);
         kmem_cache_destroy(lio_qr_cache);
         kmem_cache_destroy(lio_dr_cache);
@@ -682,11 +711,20 @@ int iscsit_add_reject_from_cmd(
         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 
         ret = wait_for_completion_interruptible(&cmd->reject_comp);
+        /*
+         * Perform the kref_put now if se_cmd has already been setup by
+         * scsit_setup_scsi_cmd()
+         */
+        if (cmd->se_cmd.se_tfo != NULL) {
+                pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n");
+                target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
+        }
         if (ret != 0)
                 return -1;
 
         return (!fail_conn) ? 0 : -1;
 }
+EXPORT_SYMBOL(iscsit_add_reject_from_cmd);
 
 /*
  * Map some portion of the allocated scatterlist to an iovec, suitable for
@@ -745,6 +783,9 @@ static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
 
         conn->exp_statsn = exp_statsn;
 
+        if (conn->sess->sess_ops->RDMAExtensions)
+                return;
+
         spin_lock_bh(&conn->cmd_lock);
         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
                 spin_lock(&cmd->istate_lock);
@@ -777,12 +818,10 @@ static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
         return 0;
 }
 
-static int iscsit_handle_scsi_cmd(
-        struct iscsi_conn *conn,
-        unsigned char *buf)
+int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
+                          unsigned char *buf)
 {
-        int data_direction, payload_length, cmdsn_ret = 0, immed_ret;
-        struct iscsi_cmd *cmd = NULL;
+        int data_direction, payload_length;
         struct iscsi_scsi_req *hdr;
         int iscsi_task_attr;
         int sam_task_attr;
@@ -805,8 +844,8 @@ static int iscsit_handle_scsi_cmd(
             !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
                 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
                                 " not set. Bad iSCSI Initiator.\n");
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
+                                1, 1, buf, cmd);
         }
 
         if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
@@ -826,8 +865,8 @@ static int iscsit_handle_scsi_cmd(
                 pr_err("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
                         " set when Expected Data Transfer Length is 0 for"
                         " CDB: 0x%02x. Bad iSCSI Initiator.\n", hdr->cdb[0]);
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
+                                1, 1, buf, cmd);
         }
 done:
 
@@ -836,29 +875,29 @@ done:
                 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
                         " MUST be set if Expected Data Transfer Length is not 0."
                         " Bad iSCSI Initiator\n");
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
+                                1, 1, buf, cmd);
         }
 
         if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
             (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
                 pr_err("Bidirectional operations not supported!\n");
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
+                                1, 1, buf, cmd);
         }
 
         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
                 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
                                 " Scsi Command PDU.\n");
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
+                                1, 1, buf, cmd);
         }
 
         if (payload_length && !conn->sess->sess_ops->ImmediateData) {
                 pr_err("ImmediateData=No but DataSegmentLength=%u,"
                         " protocol error.\n", payload_length);
-                return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
+                                1, 1, buf, cmd);
         }
 
         if ((be32_to_cpu(hdr->data_length )== payload_length) &&
@@ -866,43 +905,38 @@ done:
                 pr_err("Expected Data Transfer Length and Length of"
                         " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
                         " bit is not set protocol error\n");
-                return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
+                                1, 1, buf, cmd);
         }
 
         if (payload_length > be32_to_cpu(hdr->data_length)) {
                 pr_err("DataSegmentLength: %u is greater than"
                         " EDTL: %u, protocol error.\n", payload_length,
                                 hdr->data_length);
-                return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
+                                1, 1, buf, cmd);
         }
 
         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
                 pr_err("DataSegmentLength: %u is greater than"
                         " MaxXmitDataSegmentLength: %u, protocol error.\n",
                         payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
-                return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
-                                buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
+                                1, 1, buf, cmd);
         }
 
         if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
                 pr_err("DataSegmentLength: %u is greater than"
                         " FirstBurstLength: %u, protocol error.\n",
                         payload_length, conn->sess->sess_ops->FirstBurstLength);
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
-                                        buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
+                                1, 1, buf, cmd);
         }
 
         data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
                          (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
                           DMA_NONE;
 
-        cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
-        if (!cmd)
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1,
-                                         buf, conn);
-
         cmd->data_direction = data_direction;
         iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK;
         /*
@@ -945,7 +979,8 @@ done:
         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
         cmd->first_burst_len    = payload_length;
 
-        if (cmd->data_direction == DMA_FROM_DEVICE) {
+        if (!conn->sess->sess_ops->RDMAExtensions &&
+             cmd->data_direction == DMA_FROM_DEVICE) {
                 struct iscsi_datain_req *dr;
 
                 dr = iscsit_allocate_datain_req();
@@ -967,7 +1002,10 @@ done:
 
         pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
                 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
-                hdr->cmdsn, hdr->data_length, payload_length, conn->cid);
+                hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length,
+                conn->cid);
+
+        target_get_sess_cmd(conn->sess->se_sess, &cmd->se_cmd, true);
 
         cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd,
                                                      scsilun_to_int(&hdr->lun));
@@ -1001,12 +1039,24 @@ attach_cmd:
          */
         core_alua_check_nonop_delay(&cmd->se_cmd);
 
-        if (iscsit_allocate_iovecs(cmd) < 0) {
-                return iscsit_add_reject_from_cmd(
-                                ISCSI_REASON_BOOKMARK_NO_RESOURCES,
-                                1, 0, buf, cmd);
-        }
+        return 0;
+}
+EXPORT_SYMBOL(iscsit_setup_scsi_cmd);
+
+void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *cmd)
+{
+        iscsit_set_dataout_sequence_values(cmd);
+
+        spin_lock_bh(&cmd->dataout_timeout_lock);
+        iscsit_start_dataout_timer(cmd, cmd->conn);
+        spin_unlock_bh(&cmd->dataout_timeout_lock);
+}
+EXPORT_SYMBOL(iscsit_set_unsoliticed_dataout);
 
+int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
+                            struct iscsi_scsi_req *hdr)
+{
+        int cmdsn_ret = 0;
         /*
          * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
          * the Immediate Bit is not set, and no Immediate
@@ -1019,12 +1069,17 @@ attach_cmd:
          */
         if (!cmd->immediate_data) {
                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
-                if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
+                if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
+                        if (!cmd->sense_reason)
+                                return 0;
+
+                        target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
                         return 0;
-                else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
+                } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
                         return iscsit_add_reject_from_cmd(
                                 ISCSI_REASON_PROTOCOL_ERROR,
-                                1, 0, buf, cmd);
+                                1, 0, (unsigned char *)hdr, cmd);
+                }
         }
 
         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
@@ -1033,25 +1088,23 @@ attach_cmd:
          * If no Immediate Data is attached, it's OK to return now.
          */
         if (!cmd->immediate_data) {
-                if (!cmd->sense_reason && cmd->unsolicited_data) {
-                        iscsit_set_dataout_sequence_values(cmd);
-
-                        spin_lock_bh(&cmd->dataout_timeout_lock);
-                        iscsit_start_dataout_timer(cmd, cmd->conn);
-                        spin_unlock_bh(&cmd->dataout_timeout_lock);
-                }
+                if (!cmd->sense_reason && cmd->unsolicited_data)
+                        iscsit_set_unsoliticed_dataout(cmd);
+                if (!cmd->sense_reason)
+                        return 0;
 
+                target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
                 return 0;
         }
 
         /*
-         * Early CHECK_CONDITIONs never make it to the transport processing
-         * thread.  They are processed in CmdSN order by
-         * iscsit_check_received_cmdsn() below.
+         * Early CHECK_CONDITIONs with ImmediateData never make it to command
+         * execution.  These exceptions are processed in CmdSN order using
+         * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below.
          */
         if (cmd->sense_reason) {
-                immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
-                goto after_immediate_data;
+                target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
+                return 1;
         }
         /*
          * Call directly into transport_generic_new_cmd() to perform
@@ -1059,11 +1112,27 @@ attach_cmd:
          */
         cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd);
         if (cmd->sense_reason) {
-                immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
-                goto after_immediate_data;
+                target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
+                return 1;
         }
 
-        immed_ret = iscsit_handle_immediate_data(cmd, buf, payload_length);
+        return 0;
+}
+EXPORT_SYMBOL(iscsit_process_scsi_cmd);
+
+static int
+iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
+                          bool dump_payload)
+{
+        int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
+        /*
+         * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
+         */
+        if (dump_payload == true)
+                goto after_immediate_data;
+
+        immed_ret = iscsit_handle_immediate_data(cmd, hdr,
+                                        cmd->first_burst_len);
 after_immediate_data:
         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
                 /*
@@ -1071,26 +1140,19 @@ after_immediate_data:
                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
                  * Immediate Bit is not set.
                  */
-                cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
-                /*
-                 * Special case for Unsupported SAM WRITE Opcodes
-                 * and ImmediateData=Yes.
-                 */
+                cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd, hdr->cmdsn);
+
                 if (cmd->sense_reason) {
-                        if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
+                        if (iscsit_dump_data_payload(cmd->conn,
+                                        cmd->first_burst_len, 1) < 0)
                                 return -1;
-                } else if (cmd->unsolicited_data) {
-                        iscsit_set_dataout_sequence_values(cmd);
-
-                        spin_lock_bh(&cmd->dataout_timeout_lock);
-                        iscsit_start_dataout_timer(cmd, cmd->conn);
-                        spin_unlock_bh(&cmd->dataout_timeout_lock);
-                }
+                } else if (cmd->unsolicited_data)
+                        iscsit_set_unsoliticed_dataout(cmd);
 
                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
                         return iscsit_add_reject_from_cmd(
                                 ISCSI_REASON_PROTOCOL_ERROR,
-                                1, 0, buf, cmd);
+                                1, 0, (unsigned char *)hdr, cmd);
 
         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
                 /*
@@ -1105,13 +1167,47 @@ after_immediate_data:
                  * CmdSN and issue a retry to plug the sequence.
                  */
                 cmd->i_state = ISTATE_REMOVE;
-                iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
+                iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state);
         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
                 return -1;
 
         return 0;
 }
 
+static int
+iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
+                           unsigned char *buf)
+{
+        struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
+        int rc, immed_data;
+        bool dump_payload = false;
+
+        rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
+        if (rc < 0)
+                return rc;
+        /*
+         * Allocation iovecs needed for struct socket operations for
+         * traditional iSCSI block I/O.
+         */
+        if (iscsit_allocate_iovecs(cmd) < 0) {
+                return iscsit_add_reject_from_cmd(
+                                ISCSI_REASON_BOOKMARK_NO_RESOURCES,
+                                1, 0, buf, cmd);
+        }
+        immed_data = cmd->immediate_data;
+
+        rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
+        if (rc < 0)
+                return rc;
+        else if (rc > 0)
+                dump_payload = true;
+
+        if (!immed_data)
+                return 0;
+
+        return iscsit_get_immediate_data(cmd, hdr, dump_payload);
+}
+
 static u32 iscsit_do_crypto_hash_sg(
         struct hash_desc *hash,
         struct iscsi_cmd *cmd,
@@ -1174,20 +1270,16 @@ static void iscsit_do_crypto_hash_buf(
         crypto_hash_final(hash, data_crc);
 }
 
-static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
+int
+iscsit_check_dataout_hdr(struct iscsi_conn *conn, unsigned char *buf,
+                          struct iscsi_cmd **out_cmd)
 {
-        int iov_ret, ooo_cmdsn = 0, ret;
-        u8 data_crc_failed = 0;
-        u32 checksum, iov_count = 0, padding = 0, rx_got = 0;
-        u32 rx_size = 0, payload_length;
+        struct iscsi_data *hdr = (struct iscsi_data *)buf;
         struct iscsi_cmd *cmd = NULL;
         struct se_cmd *se_cmd;
-        struct iscsi_data *hdr;
-        struct kvec *iov;
         unsigned long flags;
-
-        hdr                     = (struct iscsi_data *) buf;
-        payload_length          = ntoh24(hdr->dlength);
+        u32 payload_length = ntoh24(hdr->dlength);
+        int rc;
 
         if (!payload_length) {
                 pr_err("DataOUT payload is ZERO, protocol error.\n");
@@ -1220,7 +1312,7 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
 
         pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
-                hdr->itt, hdr->ttt, hdr->datasn, hdr->offset,
+                hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset),
                 payload_length, conn->cid);
 
         if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
@@ -1312,12 +1404,26 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
          * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and
          * within-command recovery checks before receiving the payload.
          */
-        ret = iscsit_check_pre_dataout(cmd, buf);
-        if (ret == DATAOUT_WITHIN_COMMAND_RECOVERY)
+        rc = iscsit_check_pre_dataout(cmd, buf);
+        if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)
                 return 0;
-        else if (ret == DATAOUT_CANNOT_RECOVER)
+        else if (rc == DATAOUT_CANNOT_RECOVER)
                 return -1;
 
+        *out_cmd = cmd;
+        return 0;
+}
+EXPORT_SYMBOL(iscsit_check_dataout_hdr);
+
+static int
+iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
+                   struct iscsi_data *hdr)
+{
+        struct kvec *iov;
+        u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0;
+        u32 payload_length = ntoh24(hdr->dlength);
+        int iov_ret, data_crc_failed = 0;
+
         rx_size += payload_length;
         iov = &cmd->iov_data[0];
 
@@ -1370,17 +1476,27 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
                                 payload_length);
                 }
         }
+
+        return data_crc_failed;
+}
+
+int
+iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr,
+                             bool data_crc_failed)
+{
+        struct iscsi_conn *conn = cmd->conn;
+        int rc, ooo_cmdsn;
         /*
          * Increment post receive data and CRC values or perform
          * within-command recovery.
          */
-        ret = iscsit_check_post_dataout(cmd, buf, data_crc_failed);
-        if ((ret == DATAOUT_NORMAL) || (ret == DATAOUT_WITHIN_COMMAND_RECOVERY))
+        rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed);
+        if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY))
                 return 0;
-        else if (ret == DATAOUT_SEND_R2T) {
+        else if (rc == DATAOUT_SEND_R2T) {
                 iscsit_set_dataout_sequence_values(cmd);
-                iscsit_build_r2ts_for_cmd(cmd, conn, false);
-        } else if (ret == DATAOUT_SEND_TO_TRANSPORT) {
+                conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
+        } else if (rc == DATAOUT_SEND_TO_TRANSPORT) {
                 /*
                  * Handle extra special case for out of order
                  * Unsolicited Data Out.
@@ -1401,15 +1517,37 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
 
         return 0;
 }
+EXPORT_SYMBOL(iscsit_check_dataout_payload);
 
-static int iscsit_handle_nop_out(
-        struct iscsi_conn *conn,
-        unsigned char *buf)
+static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
+{
+        struct iscsi_cmd *cmd;
+        struct iscsi_data *hdr = (struct iscsi_data *)buf;
+        int rc;
+        bool data_crc_failed = false;
+
+        rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
+        if (rc < 0)
+                return rc;
+        else if (!cmd)
+                return 0;
+
+        rc = iscsit_get_dataout(conn, cmd, hdr);
+        if (rc < 0)
+                return rc;
+        else if (rc > 0)
+                data_crc_failed = true;
+
+        return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed);
+}
+
+int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
+                        unsigned char *buf)
 {
         unsigned char *ping_data = NULL;
         int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size;
         u32 checksum, data_crc, padding = 0, payload_length;
-        struct iscsi_cmd *cmd = NULL;
+        struct iscsi_cmd *cmd_p = NULL;
         struct kvec *iov = NULL;
         struct iscsi_nopout *hdr;
 
@@ -1432,7 +1570,7 @@ static int iscsit_handle_nop_out(
                                         buf, conn);
         }
 
-        pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%09x,"
+        pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x,"
                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
                 hdr->itt == RESERVED_ITT ? "Response" : "Request",
                 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
@@ -1445,7 +1583,6 @@ static int iscsit_handle_nop_out(
          * can contain ping data.
          */
         if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
-                cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
                 if (!cmd)
                         return iscsit_add_reject(
                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES,
@@ -1580,14 +1717,14 @@ static int iscsit_handle_nop_out(
                 /*
                  * This was a response to a unsolicited NOPIN ping.
                  */
-                cmd = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt));
-                if (!cmd)
+                cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt));
+                if (!cmd_p)
                         return -1;
 
                 iscsit_stop_nopin_response_timer(conn);
 
-                cmd->i_state = ISTATE_REMOVE;
-                iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
+                cmd_p->i_state = ISTATE_REMOVE;
+                iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state);
                 iscsit_start_nopin_timer(conn);
         } else {
                 /*
@@ -1611,12 +1748,12 @@ ping_out:
         kfree(ping_data);
         return ret;
 }
+EXPORT_SYMBOL(iscsit_handle_nop_out);
 
-static int iscsit_handle_task_mgt_cmd(
-        struct iscsi_conn *conn,
-        unsigned char *buf)
+int
+iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
+                           unsigned char *buf)
 {
-        struct iscsi_cmd *cmd;
         struct se_tmr_req *se_tmr;
         struct iscsi_tmr_req *tmr_req;
         struct iscsi_tm *hdr;
@@ -1645,18 +1782,13 @@ static int iscsit_handle_task_mgt_cmd(
                 pr_err("Task Management Request TASK_REASSIGN not"
                         " issued as immediate command, bad iSCSI Initiator"
                                 "implementation\n");
-                return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
-                                        buf, conn);
+                return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
+                                        1, 1, buf, cmd);
         }
         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
             be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG)
                 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG);
 
-        cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
-        if (!cmd)
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
-                                         1, buf, conn);
-
         cmd->data_direction = DMA_NONE;
 
         cmd->tmr_req = kzalloc(sizeof(struct iscsi_tmr_req), GFP_KERNEL);
@@ -1827,6 +1959,7 @@ attach:
         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
         return 0;
 }
+EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd);
 
 /* #warning FIXME: Support Text Command parameters besides SendTargets */
 static int iscsit_handle_text_cmd(
@@ -2089,13 +2222,12 @@ int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn
         return 0;
 }
 
-static int iscsit_handle_logout_cmd(
-        struct iscsi_conn *conn,
-        unsigned char *buf)
+int
+iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
+                        unsigned char *buf)
 {
         int cmdsn_ret, logout_remove = 0;
         u8 reason_code = 0;
-        struct iscsi_cmd *cmd;
         struct iscsi_logout *hdr;
         struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
 
@@ -2119,14 +2251,10 @@ static int iscsit_handle_logout_cmd(
         if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
                 pr_err("Received logout request on connection that"
                         " is not in logged in state, ignoring request.\n");
+                iscsit_release_cmd(cmd);
                 return 0;
         }
 
-        cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
-        if (!cmd)
-                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1,
-                                        buf, conn);
-
         cmd->iscsi_opcode       = ISCSI_OP_LOGOUT;
         cmd->i_state            = ISTATE_SEND_LOGOUTRSP;
         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
@@ -2176,6 +2304,7 @@ static int iscsit_handle_logout_cmd(
 
         return logout_remove;
 }
+EXPORT_SYMBOL(iscsit_handle_logout_cmd);
 
 static int iscsit_handle_snack(
         struct iscsi_conn *conn,
@@ -2243,7 +2372,7 @@ static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
 
 static int iscsit_handle_immediate_data(
         struct iscsi_cmd *cmd,
-        unsigned char *buf,
+        struct iscsi_scsi_req *hdr,
         u32 length)
 {
         int iov_ret, rx_got = 0, rx_size = 0;
@@ -2299,12 +2428,12 @@ static int iscsit_handle_immediate_data(
                                         " in ERL=0.\n");
                                 iscsit_add_reject_from_cmd(
                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
-                                                1, 0, buf, cmd);
+                                                1, 0, (unsigned char *)hdr, cmd);
                                 return IMMEDIATE_DATA_CANNOT_RECOVER;
                         } else {
                                 iscsit_add_reject_from_cmd(
                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
-                                                0, 0, buf, cmd);
+                                                0, 0, (unsigned char *)hdr, cmd);
                                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
                         }
                 } else {
@@ -2424,18 +2553,60 @@ static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
         }
 }
 
-static int iscsit_send_data_in(
-        struct iscsi_cmd *cmd,
-        struct iscsi_conn *conn)
+static void
+iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
+                        struct iscsi_datain *datain, struct iscsi_data_rsp *hdr,
+                        bool set_statsn)
 {
-        int iov_ret = 0, set_statsn = 0;
-        u32 iov_count = 0, tx_size = 0;
+        hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
+        hdr->flags              = datain->flags;
+        if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
+                if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
+                        hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
+                        hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
+                } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
+                        hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
+                        hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
+                }
+        }
+        hton24(hdr->dlength, datain->length);
+        if (hdr->flags & ISCSI_FLAG_DATA_ACK)
+                int_to_scsilun(cmd->se_cmd.orig_fe_lun,
+                                (struct scsi_lun *)&hdr->lun);
+        else
+                put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
+
+        hdr->itt                = cmd->init_task_tag;
+
+        if (hdr->flags & ISCSI_FLAG_DATA_ACK)
+                hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
+        else
+                hdr->ttt                = cpu_to_be32(0xFFFFFFFF);
+        if (set_statsn)
+                hdr->statsn             = cpu_to_be32(cmd->stat_sn);
+        else
+                hdr->statsn             = cpu_to_be32(0xFFFFFFFF);
+
+        hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
+        hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
+        hdr->datasn             = cpu_to_be32(datain->data_sn);
+        hdr->offset             = cpu_to_be32(datain->offset);
+
+        pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
+                " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
+                cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
+                ntohl(hdr->offset), datain->length, conn->cid);
+}
+
+static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
+{
+        struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0];
         struct iscsi_datain datain;
         struct iscsi_datain_req *dr;
-        struct iscsi_data_rsp *hdr;
         struct kvec *iov;
-        int eodr = 0;
-        int ret;
+        u32 iov_count = 0, tx_size = 0;
+        int eodr = 0, ret, iov_ret;
+        bool set_statsn = false;
 
         memset(&datain, 0, sizeof(struct iscsi_datain));
         dr = iscsit_get_datain_values(cmd, &datain);
@@ -2444,7 +2615,6 @@ static int iscsit_send_data_in(
                                 cmd->init_task_tag);
                 return -1;
         }
-
         /*
          * Be paranoid and double check the logic for now.
          */
@@ -2452,7 +2622,7 @@ static int iscsit_send_data_in(
                 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
                         " datain.length: %u exceeds cmd->data_length: %u\n",
                         cmd->init_task_tag, datain.offset, datain.length,
-                                cmd->se_cmd.data_length);
+                        cmd->se_cmd.data_length);
                 return -1;
         }
 
@@ -2476,47 +2646,13 @@ static int iscsit_send_data_in(
                     (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
                         iscsit_increment_maxcmdsn(cmd, conn->sess);
                         cmd->stat_sn = conn->stat_sn++;
-                        set_statsn = 1;
+                        set_statsn = true;
                 } else if (dr->dr_complete ==
-                                DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
-                        set_statsn = 1;
+                           DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
+                        set_statsn = true;
         }
 
-        hdr     = (struct iscsi_data_rsp *) cmd->pdu;
-        memset(hdr, 0, ISCSI_HDR_LEN);
-        hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
-        hdr->flags              = datain.flags;
-        if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
-                if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
-                        hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
-                        hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
-                } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
-                        hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
-                        hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
-                }
-        }
-        hton24(hdr->dlength, datain.length);
-        if (hdr->flags & ISCSI_FLAG_DATA_ACK)
-                int_to_scsilun(cmd->se_cmd.orig_fe_lun,
-                                (struct scsi_lun *)&hdr->lun);
-        else
-                put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
-
-        hdr->itt                = cmd->init_task_tag;
-
-        if (hdr->flags & ISCSI_FLAG_DATA_ACK)
-                hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
-        else
-                hdr->ttt                = cpu_to_be32(0xFFFFFFFF);
-        if (set_statsn)
-                hdr->statsn             = cpu_to_be32(cmd->stat_sn);
-        else
-                hdr->statsn             = cpu_to_be32(0xFFFFFFFF);
-
-        hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
-        hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
-        hdr->datasn             = cpu_to_be32(datain.data_sn);
-        hdr->offset             = cpu_to_be32(datain.offset);
+        iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn);
 
         iov = &cmd->iov_data[0];
         iov[iov_count].iov_base = cmd->pdu;
@@ -2527,7 +2663,7 @@ static int iscsit_send_data_in(
                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 
                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
-                                (unsigned char *)hdr, ISCSI_HDR_LEN,
+                                (unsigned char *)cmd->pdu, ISCSI_HDR_LEN,
                                 0, NULL, (u8 *)header_digest);
 
                 iov[0].iov_len += ISCSI_CRC_LEN;
@@ -2537,7 +2673,8 @@ static int iscsit_send_data_in(
                         " for DataIN PDU 0x%08x\n", *header_digest);
         }
 
-        iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], datain.offset, datain.length);
+        iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1],
+                                datain.offset, datain.length);
         if (iov_ret < 0)
                 return -1;
 
@@ -2568,11 +2705,6 @@ static int iscsit_send_data_in(
         cmd->iov_data_count = iov_count;
         cmd->tx_size = tx_size;
 
-        pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
-                " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
-                cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
-                ntohl(hdr->offset), datain.length, conn->cid);
-
         /* sendpage is preferred but can't insert markers */
         if (!conn->conn_ops->IFMarker)
                 ret = iscsit_fe_sendpage_sg(cmd, conn);
@@ -2595,16 +2727,13 @@ static int iscsit_send_data_in(
         return eodr;
 }
 
-static int iscsit_send_logout_response(
-        struct iscsi_cmd *cmd,
-        struct iscsi_conn *conn)
+int
+iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
+                        struct iscsi_logout_rsp *hdr)
 {
-        int niov = 0, tx_size;
         struct iscsi_conn *logout_conn = NULL;
         struct iscsi_conn_recovery *cr = NULL;
         struct iscsi_session *sess = conn->sess;
-        struct kvec *iov;
-        struct iscsi_logout_rsp *hdr;
         /*
          * The actual shutting down of Sessions and/or Connections
          * for CLOSESESSION and CLOSECONNECTION Logout Requests
@@ -2673,9 +2802,6 @@ static int iscsit_send_logout_response(
                 return -1;
         }
 
-        tx_size = ISCSI_HDR_LEN;
-        hdr                     = (struct iscsi_logout_rsp *)cmd->pdu;
-        memset(hdr, 0, ISCSI_HDR_LEN);
         hdr->opcode             = ISCSI_OP_LOGOUT_RSP;
         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
         hdr->response           = cmd->logout_response;
@@ -2687,6 +2813,27 @@ static int iscsit_send_logout_response(
         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
 
+        pr_debug("Built Logout Response ITT: 0x%08x StatSN:"
+                " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
+                cmd->init_task_tag, cmd->stat_sn, hdr->response,
+                cmd->logout_cid, conn->cid);
+
+        return 0;
+}
+EXPORT_SYMBOL(iscsit_build_logout_rsp);
+
+static int
+iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
+{
+        struct kvec *iov;
+        int niov = 0, tx_size, rc;
+
+        rc = iscsit_build_logout_rsp(cmd, conn,
+                        (struct iscsi_logout_rsp *)&cmd->pdu[0]);
+        if (rc < 0)
+                return rc;
+
+        tx_size = ISCSI_HDR_LEN;
         iov = &cmd->iov_misc[0];
         iov[niov].iov_base      = cmd->pdu;
         iov[niov++].iov_len     = ISCSI_HDR_LEN;
@@ -2695,7 +2842,7 @@ static int iscsit_send_logout_response(
                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 
                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
-                                (unsigned char *)hdr, ISCSI_HDR_LEN,
+                                (unsigned char *)&cmd->pdu[0], ISCSI_HDR_LEN,
                                 0, NULL, (u8 *)header_digest);
 
                 iov[0].iov_len += ISCSI_CRC_LEN;
@@ -2706,14 +2853,37 @@ static int iscsit_send_logout_response(
         cmd->iov_misc_count = niov;
         cmd->tx_size = tx_size;
 
-        pr_debug("Sending Logout Response ITT: 0x%08x StatSN:"
-                " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
-                cmd->init_task_tag, cmd->stat_sn, hdr->response,
-                cmd->logout_cid, conn->cid);
-
         return 0;
 }
 
+void
+iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
+                       struct iscsi_nopin *hdr, bool nopout_response)
+{
+        hdr->opcode             = ISCSI_OP_NOOP_IN;
+        hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
+        hton24(hdr->dlength, cmd->buf_ptr_size);
+        if (nopout_response)
+                put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
+        hdr->itt                = cmd->init_task_tag;
+        hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
+        cmd->stat_sn            = (nopout_response) ? conn->stat_sn++ :
+                                  conn->stat_sn;
+        hdr->statsn             = cpu_to_be32(cmd->stat_sn);
+
+        if (nopout_response)
+                iscsit_increment_maxcmdsn(cmd, conn->sess);
+
+        hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
+        hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
+
+        pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x,"
+                " StatSN: 0x%08x, Length %u\n", (nopout_response) ?
+                "Solicitied" : "Unsolicitied", cmd->init_task_tag,
+                cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
+}
+EXPORT_SYMBOL(iscsit_build_nopin_rsp);
+
 /*
  *      Unsolicited NOPIN, either requesting a response or not.
  */
@@ -2722,20 +2892,10 @@ static int iscsit_send_unsolicited_nopin(
         struct iscsi_conn *conn,
         int want_response)
 {
-        int tx_size = ISCSI_HDR_LEN;
-        struct iscsi_nopin *hdr;
-        int ret;
+        struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
+        int tx_size = ISCSI_HDR_LEN, ret;
 
-        hdr                     = (struct iscsi_nopin *) cmd->pdu;
-        memset(hdr, 0, ISCSI_HDR_LEN);
-        hdr->opcode             = ISCSI_OP_NOOP_IN;
-        hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
-        hdr->itt                = cmd->init_task_tag;
-        hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
-        cmd->stat_sn            = conn->stat_sn;
-        hdr->statsn             = cpu_to_be32(cmd->stat_sn);
-        hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
-        hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
+        iscsit_build_nopin_rsp(cmd, conn, hdr, false);
 
         if (conn->conn_ops->HeaderDigest) {
                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
@@ -2771,31 +2931,17 @@ static int iscsit_send_unsolicited_nopin(
         return 0;
 }
 
-static int iscsit_send_nopin_response(
-        struct iscsi_cmd *cmd,
-        struct iscsi_conn *conn)
+static int
+iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
 {
-        int niov = 0, tx_size;
-        u32 padding = 0;
+        struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
         struct kvec *iov;
-        struct iscsi_nopin *hdr;
-
-        tx_size = ISCSI_HDR_LEN;
-        hdr                     = (struct iscsi_nopin *) cmd->pdu;
-        memset(hdr, 0, ISCSI_HDR_LEN);
-        hdr->opcode             = ISCSI_OP_NOOP_IN;
-        hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
-        hton24(hdr->dlength, cmd->buf_ptr_size);
-        put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
-        hdr->itt                = cmd->init_task_tag;
-        hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
-        cmd->stat_sn            = conn->stat_sn++;
-        hdr->statsn             = cpu_to_be32(cmd->stat_sn);
+        u32 padding = 0;
+        int niov = 0, tx_size;
 
-        iscsit_increment_maxcmdsn(cmd, conn->sess);
-        hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
-        hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
+        iscsit_build_nopin_rsp(cmd, conn, hdr, true);
 
+        tx_size = ISCSI_HDR_LEN;
         iov = &cmd->iov_misc[0];
         iov[niov].iov_base      = cmd->pdu;
         iov[niov++].iov_len     = ISCSI_HDR_LEN;
@@ -2851,10 +2997,6 @@ static int iscsit_send_nopin_response(
         cmd->iov_misc_count = niov;
         cmd->tx_size = tx_size;
 
-        pr_debug("Sending NOPIN Response ITT: 0x%08x, TTT:"
-                " 0x%08x, StatSN: 0x%08x, Length %u\n", cmd->init_task_tag,
-                cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
-
         return 0;
 }
 
@@ -2939,8 +3081,8 @@ static int iscsit_send_r2t(
  *              connection recovery.
  */
 int iscsit_build_r2ts_for_cmd(
-        struct iscsi_cmd *cmd,
         struct iscsi_conn *conn,
+        struct iscsi_cmd *cmd,
         bool recovery)
 {
         int first_r2t = 1;
@@ -3015,24 +3157,16 @@ int iscsit_build_r2ts_for_cmd(
         return 0;
 }
 
-static int iscsit_send_status(
-        struct iscsi_cmd *cmd,
-        struct iscsi_conn *conn)
+void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
+                        bool inc_stat_sn, struct iscsi_scsi_rsp *hdr)
 {
-        u8 iov_count = 0, recovery;
-        u32 padding = 0, tx_size = 0;
-        struct iscsi_scsi_rsp *hdr;
-        struct kvec *iov;
-
-        recovery = (cmd->i_state != ISTATE_SEND_STATUS);
-        if (!recovery)
+        if (inc_stat_sn)
                 cmd->stat_sn = conn->stat_sn++;
 
         spin_lock_bh(&conn->sess->session_stats_lock);
         conn->sess->rsp_pdus++;
         spin_unlock_bh(&conn->sess->session_stats_lock);
 
-        hdr                     = (struct iscsi_scsi_rsp *) cmd->pdu;
         memset(hdr, 0, ISCSI_HDR_LEN);
         hdr->opcode             = ISCSI_OP_SCSI_CMD_RSP;
         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
@@ -3052,6 +3186,23 @@ static int iscsit_send_status(
         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
 
+        pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
+                " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
+                cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status,
+                cmd->se_cmd.scsi_status, conn->cid);
+}
+EXPORT_SYMBOL(iscsit_build_rsp_pdu);
+
+static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
+{
+        struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0];
+        struct kvec *iov;
+        u32 padding = 0, tx_size = 0;
+        int iov_count = 0;
+        bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS);
+
+        iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr);
+
         iov = &cmd->iov_misc[0];
         iov[iov_count].iov_base = cmd->pdu;
         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
@@ -3106,7 +3257,7 @@ static int iscsit_send_status(
                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 
                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
-                                (unsigned char *)hdr, ISCSI_HDR_LEN,
+                                (unsigned char *)cmd->pdu, ISCSI_HDR_LEN,
                                 0, NULL, (u8 *)header_digest);
 
                 iov[0].iov_len += ISCSI_CRC_LEN;
@@ -3118,11 +3269,6 @@ static int iscsit_send_status(
         cmd->iov_misc_count = iov_count;
         cmd->tx_size = tx_size;
 
-        pr_debug("Built %sSCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
-                " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
-                (!recovery) ? "" : "Recovery ", cmd->init_task_tag,
-                cmd->stat_sn, 0x00, cmd->se_cmd.scsi_status, conn->cid);
-
         return 0;
 }
 
@@ -3145,16 +3291,12 @@ static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
         }
 }
 
-static int iscsit_send_task_mgt_rsp(
-        struct iscsi_cmd *cmd,
-        struct iscsi_conn *conn)
+void
+iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
+                          struct iscsi_tm_rsp *hdr)
 {
         struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
-        struct iscsi_tm_rsp *hdr;
-        u32 tx_size = 0;
 
-        hdr                     = (struct iscsi_tm_rsp *) cmd->pdu;
-        memset(hdr, 0, ISCSI_HDR_LEN);
         hdr->opcode             = ISCSI_OP_SCSI_TMFUNC_RSP;
         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
         hdr->response           = iscsit_convert_tcm_tmr_rsp(se_tmr);
@@ -3166,6 +3308,20 @@ static int iscsit_send_task_mgt_rsp(
         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
 
+        pr_debug("Built Task Management Response ITT: 0x%08x,"
+                " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
+                cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
+}
+EXPORT_SYMBOL(iscsit_build_task_mgt_rsp);
+
+static int
+iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
+{
+        struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0];
+        u32 tx_size = 0;
+
+        iscsit_build_task_mgt_rsp(cmd, conn, hdr);
+
         cmd->iov_misc[0].iov_base       = cmd->pdu;
         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
         tx_size += ISCSI_HDR_LEN;
@@ -3186,10 +3342,6 @@ static int iscsit_send_task_mgt_rsp(
         cmd->iov_misc_count = 1;
         cmd->tx_size = tx_size;
 
-        pr_debug("Built Task Management Response ITT: 0x%08x,"
-                " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
-                cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
-
         return 0;
 }
 
@@ -3385,6 +3537,22 @@ static int iscsit_send_text_rsp(
         return 0;
 }
 
+void
+iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
+                    struct iscsi_reject *hdr)
+{
+        hdr->opcode             = ISCSI_OP_REJECT;
+        hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
+        hton24(hdr->dlength, ISCSI_HDR_LEN);
+        hdr->ffffffff           = cpu_to_be32(0xffffffff);
+        cmd->stat_sn            = conn->stat_sn++;
+        hdr->statsn             = cpu_to_be32(cmd->stat_sn);
+        hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
+        hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
+
+}
+EXPORT_SYMBOL(iscsit_build_reject);
+
 static int iscsit_send_reject(
         struct iscsi_cmd *cmd,
         struct iscsi_conn *conn)
@@ -3393,18 +3561,9 @@ static int iscsit_send_reject(
         struct iscsi_reject *hdr;
         struct kvec *iov;
 
-        hdr                     = (struct iscsi_reject *) cmd->pdu;
-        hdr->opcode             = ISCSI_OP_REJECT;
-        hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
-        hton24(hdr->dlength, ISCSI_HDR_LEN);
-        hdr->ffffffff           = cpu_to_be32(0xffffffff);
-        cmd->stat_sn            = conn->stat_sn++;
-        hdr->statsn             = cpu_to_be32(cmd->stat_sn);
-        hdr->exp_cmdsn  = cpu_to_be32(conn->sess->exp_cmd_sn);
-        hdr->max_cmdsn  = cpu_to_be32(conn->sess->max_cmd_sn);
+        iscsit_build_reject(cmd, conn, (struct iscsi_reject *)&cmd->pdu[0]);
 
         iov = &cmd->iov_misc[0];
-
         iov[iov_count].iov_base = cmd->pdu;
         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
         iov[iov_count].iov_base = cmd->buf_ptr;
@@ -3501,55 +3660,41 @@ static inline void iscsit_thread_check_cpumask(
         set_cpus_allowed_ptr(p, conn->conn_cpumask);
 }
 
-static int handle_immediate_queue(struct iscsi_conn *conn)
+static int
+iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
 {
-        struct iscsi_queue_req *qr;
-        struct iscsi_cmd *cmd;
-        u8 state;
         int ret;
 
-        while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
-                atomic_set(&conn->check_immediate_queue, 0);
-                cmd = qr->cmd;
-                state = qr->state;
-                kmem_cache_free(lio_qr_cache, qr);
-
-                switch (state) {
-                case ISTATE_SEND_R2T:
-                        ret = iscsit_send_r2t(cmd, conn);
-                        if (ret < 0)
-                                goto err;
-                        break;
-                case ISTATE_REMOVE:
-                        if (cmd->data_direction == DMA_TO_DEVICE)
-                                iscsit_stop_dataout_timer(cmd);
-
-                        spin_lock_bh(&conn->cmd_lock);
-                        list_del(&cmd->i_conn_node);
-                        spin_unlock_bh(&conn->cmd_lock);
+        switch (state) {
+        case ISTATE_SEND_R2T:
+                ret = iscsit_send_r2t(cmd, conn);
+                if (ret < 0)
+                        goto err;
+                break;
+        case ISTATE_REMOVE:
+                spin_lock_bh(&conn->cmd_lock);
+                list_del(&cmd->i_conn_node);
+                spin_unlock_bh(&conn->cmd_lock);
 
-                        iscsit_free_cmd(cmd);
-                        continue;
-                case ISTATE_SEND_NOPIN_WANT_RESPONSE:
-                        iscsit_mod_nopin_response_timer(conn);
-                        ret = iscsit_send_unsolicited_nopin(cmd,
-                                                            conn, 1);
-                        if (ret < 0)
-                                goto err;
-                        break;
-                case ISTATE_SEND_NOPIN_NO_RESPONSE:
-                        ret = iscsit_send_unsolicited_nopin(cmd,
-                                                            conn, 0);
-                        if (ret < 0)
-                                goto err;
-                        break;
-                default:
-                        pr_err("Unknown Opcode: 0x%02x ITT:"
-                               " 0x%08x, i_state: %d on CID: %hu\n",
-                               cmd->iscsi_opcode, cmd->init_task_tag, state,
-                               conn->cid);
+                iscsit_free_cmd(cmd);
+                break;
+        case ISTATE_SEND_NOPIN_WANT_RESPONSE:
+                iscsit_mod_nopin_response_timer(conn);
+                ret = iscsit_send_unsolicited_nopin(cmd, conn, 1);
+                if (ret < 0)
                         goto err;
-                }
+                break;
+        case ISTATE_SEND_NOPIN_NO_RESPONSE:
+                ret = iscsit_send_unsolicited_nopin(cmd, conn, 0);
+                if (ret < 0)
+                        goto err;
+                break;
+        default:
+                pr_err("Unknown Opcode: 0x%02x ITT:"
+                       " 0x%08x, i_state: %d on CID: %hu\n",
+                       cmd->iscsi_opcode, cmd->init_task_tag, state,
+                       conn->cid);
+                goto err;
         }
 
         return 0;
@@ -3558,128 +3703,143 @@ err:
         return -1;
 }
 
-static int handle_response_queue(struct iscsi_conn *conn)
+static int
+iscsit_handle_immediate_queue(struct iscsi_conn *conn)
 {
+        struct iscsit_transport *t = conn->conn_transport;
         struct iscsi_queue_req *qr;
         struct iscsi_cmd *cmd;
         u8 state;
         int ret;
 
-        while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
+        while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
+                atomic_set(&conn->check_immediate_queue, 0);
                 cmd = qr->cmd;
                 state = qr->state;
                 kmem_cache_free(lio_qr_cache, qr);
 
-check_rsp_state:
-                switch (state) {
-                case ISTATE_SEND_DATAIN:
-                        ret = iscsit_send_data_in(cmd, conn);
-                        if (ret < 0)
-                                goto err;
-                        else if (!ret)
-                                /* more drs */
-                                goto check_rsp_state;
-                        else if (ret == 1) {
-                                /* all done */
-                                spin_lock_bh(&cmd->istate_lock);
-                                cmd->i_state = ISTATE_SENT_STATUS;
-                                spin_unlock_bh(&cmd->istate_lock);
-
-                                if (atomic_read(&conn->check_immediate_queue))
-                                        return 1;
+                ret = t->iscsit_immediate_queue(conn, cmd, state);
+                if (ret < 0)
+                        return ret;
+        }
 
-                                continue;
-                        } else if (ret == 2) {
-                                /* Still must send status,
-                                   SCF_TRANSPORT_TASK_SENSE was set */
-                                spin_lock_bh(&cmd->istate_lock);
-                                cmd->i_state = ISTATE_SEND_STATUS;
-                                spin_unlock_bh(&cmd->istate_lock);
-                                state = ISTATE_SEND_STATUS;
-                                goto check_rsp_state;
-                        }
+        return 0;
+}
 
-                        break;
-                case ISTATE_SEND_STATUS:
-                case ISTATE_SEND_STATUS_RECOVERY:
-                        ret = iscsit_send_status(cmd, conn);
-                        break;
-                case ISTATE_SEND_LOGOUTRSP:
-                        ret = iscsit_send_logout_response(cmd, conn);
-                        break;
-                case ISTATE_SEND_ASYNCMSG:
-                        ret = iscsit_send_conn_drop_async_message(
-                                cmd, conn);
-                        break;
-                case ISTATE_SEND_NOPIN:
-                        ret = iscsit_send_nopin_response(cmd, conn);
-                        break;
-                case ISTATE_SEND_REJECT:
-                        ret = iscsit_send_reject(cmd, conn);
-                        break;
-                case ISTATE_SEND_TASKMGTRSP:
-                        ret = iscsit_send_task_mgt_rsp(cmd, conn);
-                        if (ret != 0)
-                                break;
-                        ret = iscsit_tmr_post_handler(cmd, conn);
-                        if (ret != 0)
-                                iscsit_fall_back_to_erl0(conn->sess);
-                        break;
-                case ISTATE_SEND_TEXTRSP:
-                        ret = iscsit_send_text_rsp(cmd, conn);
-                        break;
-                default:
-                        pr_err("Unknown Opcode: 0x%02x ITT:"
-                               " 0x%08x, i_state: %d on CID: %hu\n",
-                               cmd->iscsi_opcode, cmd->init_task_tag,
-                               state, conn->cid);
-                        goto err;
-                }
+static int
+iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
+{
+        int ret;
+
+check_rsp_state:
+        switch (state) {
+        case ISTATE_SEND_DATAIN:
+                ret = iscsit_send_datain(cmd, conn);
                 if (ret < 0)
                         goto err;
+                else if (!ret)
+                        /* more drs */
+                        goto check_rsp_state;
+                else if (ret == 1) {
+                        /* all done */
+                        spin_lock_bh(&cmd->istate_lock);
+                        cmd->i_state = ISTATE_SENT_STATUS;
+                        spin_unlock_bh(&cmd->istate_lock);
 
-                if (iscsit_send_tx_data(cmd, conn, 1) < 0) {
-                        iscsit_tx_thread_wait_for_tcp(conn);
-                        iscsit_unmap_iovec(cmd);
-                        goto err;
-                }
-                iscsit_unmap_iovec(cmd);
+                        if (atomic_read(&conn->check_immediate_queue))
+                                return 1;
 
-                switch (state) {
-                case ISTATE_SEND_LOGOUTRSP:
-                        if (!iscsit_logout_post_handler(cmd, conn))
-                                goto restart;
-                        /* fall through */
-                case ISTATE_SEND_STATUS:
-                case ISTATE_SEND_ASYNCMSG:
-                case ISTATE_SEND_NOPIN:
-                case ISTATE_SEND_STATUS_RECOVERY:
-                case ISTATE_SEND_TEXTRSP:
-                case ISTATE_SEND_TASKMGTRSP:
+                        return 0;
+                } else if (ret == 2) {
+                        /* Still must send status,
+                           SCF_TRANSPORT_TASK_SENSE was set */
                         spin_lock_bh(&cmd->istate_lock);
-                        cmd->i_state = ISTATE_SENT_STATUS;
+                        cmd->i_state = ISTATE_SEND_STATUS;
                         spin_unlock_bh(&cmd->istate_lock);
+                        state = ISTATE_SEND_STATUS;
+                        goto check_rsp_state;
+                }
+
+                break;
+        case ISTATE_SEND_STATUS:
+        case ISTATE_SEND_STATUS_RECOVERY:
+                ret = iscsit_send_response(cmd, conn);
+                break;
+        case ISTATE_SEND_LOGOUTRSP:
+                ret = iscsit_send_logout(cmd, conn);
+                break;
+        case ISTATE_SEND_ASYNCMSG:
+                ret = iscsit_send_conn_drop_async_message(
+                        cmd, conn);
+                break;
+        case ISTATE_SEND_NOPIN:
+                ret = iscsit_send_nopin(cmd, conn);
+                break;
+        case ISTATE_SEND_REJECT:
+                ret = iscsit_send_reject(cmd, conn);
+                break;
+        case ISTATE_SEND_TASKMGTRSP:
+                ret = iscsit_send_task_mgt_rsp(cmd, conn);
+                if (ret != 0)
                         break;
-                case ISTATE_SEND_REJECT:
-                        if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) {
-                                cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN;
-                                complete(&cmd->reject_comp);
-                                goto err;
-                        }
+                ret = iscsit_tmr_post_handler(cmd, conn);
+                if (ret != 0)
+                        iscsit_fall_back_to_erl0(conn->sess);
+                break;
+        case ISTATE_SEND_TEXTRSP:
+                ret = iscsit_send_text_rsp(cmd, conn);
+                break;
+        default:
+                pr_err("Unknown Opcode: 0x%02x ITT:"
+                       " 0x%08x, i_state: %d on CID: %hu\n",
+                       cmd->iscsi_opcode, cmd->init_task_tag,
+                       state, conn->cid);
+                goto err;
+        }
+        if (ret < 0)
+                goto err;
+
+        if (iscsit_send_tx_data(cmd, conn, 1) < 0) {
+                iscsit_tx_thread_wait_for_tcp(conn);
+                iscsit_unmap_iovec(cmd);
+                goto err;
+        }
+        iscsit_unmap_iovec(cmd);
+
+        switch (state) {
+        case ISTATE_SEND_LOGOUTRSP:
+                if (!iscsit_logout_post_handler(cmd, conn))
+                        goto restart;
+                /* fall through */
+        case ISTATE_SEND_STATUS:
+        case ISTATE_SEND_ASYNCMSG:
+        case ISTATE_SEND_NOPIN:
+        case ISTATE_SEND_STATUS_RECOVERY:
+        case ISTATE_SEND_TEXTRSP:
+        case ISTATE_SEND_TASKMGTRSP:
+                spin_lock_bh(&cmd->istate_lock);
+                cmd->i_state = ISTATE_SENT_STATUS;
+                spin_unlock_bh(&cmd->istate_lock);
+                break;
+        case ISTATE_SEND_REJECT:
+                if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) {
+                        cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN;
                         complete(&cmd->reject_comp);
-                        break;
-                default:
-                        pr_err("Unknown Opcode: 0x%02x ITT:"
-                               " 0x%08x, i_state: %d on CID: %hu\n",
-                               cmd->iscsi_opcode, cmd->init_task_tag,
-                               cmd->i_state, conn->cid);
                         goto err;
                 }
-
-                if (atomic_read(&conn->check_immediate_queue))
-                        return 1;
+                complete(&cmd->reject_comp);
+                break;
+        default:
+                pr_err("Unknown Opcode: 0x%02x ITT:"
+                       " 0x%08x, i_state: %d on CID: %hu\n",
+                       cmd->iscsi_opcode, cmd->init_task_tag,
+                       cmd->i_state, conn->cid);
+                goto err;
         }
 
+        if (atomic_read(&conn->check_immediate_queue))
+                return 1;
+
         return 0;
 
 err:
@@ -3688,6 +3848,27 @@ restart:
         return -EAGAIN;
 }
 
+static int iscsit_handle_response_queue(struct iscsi_conn *conn)
+{
+        struct iscsit_transport *t = conn->conn_transport;
+        struct iscsi_queue_req *qr;
+        struct iscsi_cmd *cmd;
+        u8 state;
+        int ret;
+
+        while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
+                cmd = qr->cmd;
+                state = qr->state;
+                kmem_cache_free(lio_qr_cache, qr);
+
+                ret = t->iscsit_response_queue(conn, cmd, state);
+                if (ret == 1 || ret < 0)
+                        return ret;
+        }
+
+        return 0;
+}
+
 int iscsi_target_tx_thread(void *arg)
 {
         int ret = 0;
@@ -3722,11 +3903,11 @@ restart:
                         goto transport_err;
 
 get_immediate:
-                ret = handle_immediate_queue(conn);
+                ret = iscsit_handle_immediate_queue(conn);
                 if (ret < 0)
                         goto transport_err;
 
-                ret = handle_response_queue(conn);
+                ret = iscsit_handle_response_queue(conn);
                 if (ret == 1)
                         goto get_immediate;
                 else if (ret == -EAGAIN)
@@ -3742,6 +3923,83 @@ out:
         return 0;
 }
 
+static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf)
+{
+        struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf;
+        struct iscsi_cmd *cmd;
+        int ret = 0;
+
+        switch (hdr->opcode & ISCSI_OPCODE_MASK) {
+        case ISCSI_OP_SCSI_CMD:
+                cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
+                if (!cmd)
+                        return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
+                                                1, buf, conn);
+
+                ret = iscsit_handle_scsi_cmd(conn, cmd, buf);
+                break;
+        case ISCSI_OP_SCSI_DATA_OUT:
+                ret = iscsit_handle_data_out(conn, buf);
+                break;
+        case ISCSI_OP_NOOP_OUT:
+                cmd = NULL;
+                if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
+                        cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
+                        if (!cmd)
+                                return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
+                                                1, buf, conn);
+                }
+                ret = iscsit_handle_nop_out(conn, cmd, buf);
+                break;
+        case ISCSI_OP_SCSI_TMFUNC:
+                cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
+                if (!cmd)
+                        return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
+                                                1, buf, conn);
+
+                ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf);
+                break;
+        case ISCSI_OP_TEXT:
+                ret = iscsit_handle_text_cmd(conn, buf);
+                break;
+        case ISCSI_OP_LOGOUT:
+                cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
+                if (!cmd)
+                        return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
+                                                1, buf, conn);
+
+                ret = iscsit_handle_logout_cmd(conn, cmd, buf);
+                if (ret > 0)
+                        wait_for_completion_timeout(&conn->conn_logout_comp,
+                                        SECONDS_FOR_LOGOUT_COMP * HZ);
+                break;
+        case ISCSI_OP_SNACK:
+                ret = iscsit_handle_snack(conn, buf);
+                break;
+        default:
+                pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode);
+                if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
+                        pr_err("Cannot recover from unknown"
+                        " opcode while ERL=0, closing iSCSI connection.\n");
+                        return -1;
+                }
+                if (!conn->conn_ops->OFMarker) {
+                        pr_err("Unable to recover from unknown"
+                        " opcode while OFMarker=No, closing iSCSI"
+                                " connection.\n");
+                        return -1;
+                }
+                if (iscsit_recover_from_unknown_opcode(conn) < 0) {
+                        pr_err("Unable to recover from unknown"
+                                " opcode, closing iSCSI connection.\n");
+                        return -1;
+                }
+                break;
+        }
+
+        return ret;
+}
+
 int iscsi_target_rx_thread(void *arg)
 {
         int ret;
@@ -3761,6 +4019,18 @@ restart:
         if (!conn)
                 goto out;
 
+        if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) {
+                struct completion comp;
+                int rc;
+
+                init_completion(&comp);
+                rc = wait_for_completion_interruptible(&comp);
+                if (rc < 0)
+                        goto transport_err;
+
+                goto out;
+        }
+
         while (!kthread_should_stop()) {
                 /*
                  * Ensure that both TX and RX per connection kthreads
@@ -3832,62 +4102,9 @@ restart:
                         goto transport_err;
                 }
 
-                switch (opcode) {
-                case ISCSI_OP_SCSI_CMD:
-                        if (iscsit_handle_scsi_cmd(conn, buffer) < 0)
-                                goto transport_err;
-                        break;
-                case ISCSI_OP_SCSI_DATA_OUT:
-                        if (iscsit_handle_data_out(conn, buffer) < 0)
-                                goto transport_err;
-                        break;
-                case ISCSI_OP_NOOP_OUT:
-                        if (iscsit_handle_nop_out(conn, buffer) < 0)
-                                goto transport_err;
-                        break;
-                case ISCSI_OP_SCSI_TMFUNC:
-                        if (iscsit_handle_task_mgt_cmd(conn, buffer) < 0)
-                                goto transport_err;
-                        break;
-                case ISCSI_OP_TEXT:
-                        if (iscsit_handle_text_cmd(conn, buffer) < 0)
-                                goto transport_err;
-                        break;
-                case ISCSI_OP_LOGOUT:
-                        ret = iscsit_handle_logout_cmd(conn, buffer);
-                        if (ret > 0) {
-                                wait_for_completion_timeout(&conn->conn_logout_comp,
-                                                SECONDS_FOR_LOGOUT_COMP * HZ);
-                                goto transport_err;
-                        } else if (ret < 0)
-                                goto transport_err;
-                        break;
-                case ISCSI_OP_SNACK:
-                        if (iscsit_handle_snack(conn, buffer) < 0)
-                                goto transport_err;
-                        break;
-                default:
-                        pr_err("Got unknown iSCSI OpCode: 0x%02x\n",
-                                        opcode);
-                        if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
-                                pr_err("Cannot recover from unknown"
-                                " opcode while ERL=0, closing iSCSI connection"
-                                ".\n");
-                                goto transport_err;
-                        }
-                        if (!conn->conn_ops->OFMarker) {
-                                pr_err("Unable to recover from unknown"
-                                " opcode while OFMarker=No, closing iSCSI"
-                                        " connection.\n");
-                                goto transport_err;
-                        }
-                        if (iscsit_recover_from_unknown_opcode(conn) < 0) {
-                                pr_err("Unable to recover from unknown"
-                                        " opcode, closing iSCSI connection.\n");
-                                goto transport_err;
-                        }
-                        break;
-                }
+                ret = iscsi_target_rx_opcode(conn, buffer);
+                if (ret < 0)
+                        goto transport_err;
         }
 
 transport_err:
@@ -4053,6 +4270,12 @@ int iscsit_close_connection(
 
         if (conn->sock)
                 sock_release(conn->sock);
+
+        if (conn->conn_transport->iscsit_free_conn)
+                conn->conn_transport->iscsit_free_conn(conn);
+
+        iscsit_put_transport(conn->conn_transport);
+
         conn->thread_set = NULL;
 
         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
@@ -4284,7 +4507,7 @@ static void iscsit_logout_post_handler_diffcid(
 /*
  *      Return of 0 causes the TX thread to restart.
  */
-static int iscsit_logout_post_handler(
+int iscsit_logout_post_handler(
         struct iscsi_cmd *cmd,
         struct iscsi_conn *conn)
 {
@@ -4342,6 +4565,7 @@ static int iscsit_logout_post_handler(
         }
         return ret;
 }
+EXPORT_SYMBOL(iscsit_logout_post_handler);
 
 void iscsit_fail_session(struct iscsi_session *sess)
 {
diff --git a/drivers/target/iscsi/iscsi_target.h b/drivers/target/iscsi/iscsi_target.h
index b1a1e6350707..a0050b2f294e 100644
--- a/drivers/target/iscsi/iscsi_target.h
+++ b/drivers/target/iscsi/iscsi_target.h
@@ -16,11 +16,12 @@ extern int iscsit_reset_np_thread(struct iscsi_np *, struct iscsi_tpg_np *,
                                 struct iscsi_portal_group *);
 extern int iscsit_del_np(struct iscsi_np *);
 extern int iscsit_add_reject_from_cmd(u8, int, int, unsigned char *, struct iscsi_cmd *);
+extern void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *);
 extern int iscsit_logout_closesession(struct iscsi_cmd *, struct iscsi_conn *);
 extern int iscsit_logout_closeconnection(struct iscsi_cmd *, struct iscsi_conn *);
 extern int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *, struct iscsi_conn *);
 extern int iscsit_send_async_msg(struct iscsi_conn *, u16, u8, u8);
-extern int iscsit_build_r2ts_for_cmd(struct iscsi_cmd *, struct iscsi_conn *, bool recovery);
+extern int iscsit_build_r2ts_for_cmd(struct iscsi_conn *, struct iscsi_cmd *, bool recovery);
 extern void iscsit_thread_get_cpumask(struct iscsi_conn *);
 extern int iscsi_target_tx_thread(void *);
 extern int iscsi_target_rx_thread(void *);
diff --git a/drivers/target/iscsi/iscsi_target_auth.c b/drivers/target/iscsi/iscsi_target_auth.c
index a0fc7b9eea65..cee17543278c 100644
--- a/drivers/target/iscsi/iscsi_target_auth.c
+++ b/drivers/target/iscsi/iscsi_target_auth.c
@@ -49,32 +49,6 @@ static void chap_binaryhex_to_asciihex(char *dst, char *src, int src_len)
         }
 }
 
-static void chap_set_random(char *data, int length)
-{
-        long r;
-        unsigned n;
-
-        while (length > 0) {
-                get_random_bytes(&r, sizeof(long));
-                r = r ^ (r >> 8);
-                r = r ^ (r >> 4);
-                n = r & 0x7;
-
-                get_random_bytes(&r, sizeof(long));
-                r = r ^ (r >> 8);
-                r = r ^ (r >> 5);
-                n = (n << 3) | (r & 0x7);
-
-                get_random_bytes(&r, sizeof(long));
-                r = r ^ (r >> 8);
-                r = r ^ (r >> 5);
-                n = (n << 2) | (r & 0x3);
-
-                *data++ = n;
-                length--;
-        }
-}
-
 static void chap_gen_challenge(
         struct iscsi_conn *conn,
         int caller,
@@ -86,7 +60,7 @@ static void chap_gen_challenge(
 
         memset(challenge_asciihex, 0, CHAP_CHALLENGE_LENGTH * 2 + 1);
 
-        chap_set_random(chap->challenge, CHAP_CHALLENGE_LENGTH);
+        get_random_bytes(chap->challenge, CHAP_CHALLENGE_LENGTH);
         chap_binaryhex_to_asciihex(challenge_asciihex, chap->challenge,
                                 CHAP_CHALLENGE_LENGTH);
         /*
diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c
index 78d75c8567d0..13e9e715ad2e 100644
--- a/drivers/target/iscsi/iscsi_target_configfs.c
+++ b/drivers/target/iscsi/iscsi_target_configfs.c
@@ -27,6 +27,7 @@
 #include <target/target_core_fabric_configfs.h>
 #include <target/target_core_configfs.h>
 #include <target/configfs_macros.h>
+#include <target/iscsi/iscsi_transport.h>
 
 #include "iscsi_target_core.h"
 #include "iscsi_target_parameters.h"
@@ -124,8 +125,87 @@ out:
 
 TF_NP_BASE_ATTR(lio_target, sctp, S_IRUGO | S_IWUSR);
 
+static ssize_t lio_target_np_show_iser(
+        struct se_tpg_np *se_tpg_np,
+        char *page)
+{
+        struct iscsi_tpg_np *tpg_np = container_of(se_tpg_np,
+                                struct iscsi_tpg_np, se_tpg_np);
+        struct iscsi_tpg_np *tpg_np_iser;
+        ssize_t rb;
+
+        tpg_np_iser = iscsit_tpg_locate_child_np(tpg_np, ISCSI_INFINIBAND);
+        if (tpg_np_iser)
+                rb = sprintf(page, "1\n");
+        else
+                rb = sprintf(page, "0\n");
+
+        return rb;
+}
+
+static ssize_t lio_target_np_store_iser(
+        struct se_tpg_np *se_tpg_np,
+        const char *page,
+        size_t count)
+{
+        struct iscsi_np *np;
+        struct iscsi_portal_group *tpg;
+        struct iscsi_tpg_np *tpg_np = container_of(se_tpg_np,
+                                struct iscsi_tpg_np, se_tpg_np);
+        struct iscsi_tpg_np *tpg_np_iser = NULL;
+        char *endptr;
+        u32 op;
+        int rc;
+
+        op = simple_strtoul(page, &endptr, 0);
+        if ((op != 1) && (op != 0)) {
+                pr_err("Illegal value for tpg_enable: %u\n", op);
+                return -EINVAL;
+        }
+        np = tpg_np->tpg_np;
+        if (!np) {
+                pr_err("Unable to locate struct iscsi_np from"
+                                " struct iscsi_tpg_np\n");
+                return -EINVAL;
+        }
+
+        tpg = tpg_np->tpg;
+        if (iscsit_get_tpg(tpg) < 0)
+                return -EINVAL;
+
+        if (op) {
+                int rc = request_module("ib_isert");
+                if (rc != 0)
+                        pr_warn("Unable to request_module for ib_isert\n");
+
+                tpg_np_iser = iscsit_tpg_add_network_portal(tpg, &np->np_sockaddr,
+                                np->np_ip, tpg_np, ISCSI_INFINIBAND);
+                if (!tpg_np_iser || IS_ERR(tpg_np_iser))
+                        goto out;
+        } else {
+                tpg_np_iser = iscsit_tpg_locate_child_np(tpg_np, ISCSI_INFINIBAND);
+                if (!tpg_np_iser)
+                        goto out;
+
+                rc = iscsit_tpg_del_network_portal(tpg, tpg_np_iser);
+                if (rc < 0)
+                        goto out;
+        }
+
+        printk("lio_target_np_store_iser() done, op: %d\n", op);
+
+        iscsit_put_tpg(tpg);
+        return count;
+out:
+        iscsit_put_tpg(tpg);
+        return -EINVAL;
+}
+
+TF_NP_BASE_ATTR(lio_target, iser, S_IRUGO | S_IWUSR);
+
 static struct configfs_attribute *lio_target_portal_attrs[] = {
         &lio_target_np_sctp.attr,
+        &lio_target_np_iser.attr,
         NULL,
 };
 
@@ -1536,16 +1616,18 @@ static int lio_queue_data_in(struct se_cmd *se_cmd)
         struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd);
 
         cmd->i_state = ISTATE_SEND_DATAIN;
-        iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
+        cmd->conn->conn_transport->iscsit_queue_data_in(cmd->conn, cmd);
+
         return 0;
 }
 
 static int lio_write_pending(struct se_cmd *se_cmd)
 {
         struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd);
+        struct iscsi_conn *conn = cmd->conn;
 
         if (!cmd->immediate_data && !cmd->unsolicited_data)
-                return iscsit_build_r2ts_for_cmd(cmd, cmd->conn, false);
+                return conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
 
         return 0;
 }
@@ -1567,7 +1649,8 @@ static int lio_queue_status(struct se_cmd *se_cmd)
         struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd);
 
         cmd->i_state = ISTATE_SEND_STATUS;
-        iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
+        cmd->conn->conn_transport->iscsit_queue_status(cmd->conn, cmd);
+
         return 0;
 }
 
@@ -1696,11 +1779,17 @@ static void lio_set_default_node_attributes(struct se_node_acl *se_acl)
         iscsit_set_default_node_attribues(acl);
 }
 
+static int lio_check_stop_free(struct se_cmd *se_cmd)
+{
+        return target_put_sess_cmd(se_cmd->se_sess, se_cmd);
+}
+
 static void lio_release_cmd(struct se_cmd *se_cmd)
 {
         struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd);
 
-        iscsit_release_cmd(cmd);
+        pr_debug("Entering lio_release_cmd for se_cmd: %p\n", se_cmd);
+        cmd->release_cmd(cmd);
 }
 
 /* End functions for target_core_fabric_ops */
@@ -1740,6 +1829,7 @@ int iscsi_target_register_configfs(void)
         fabric->tf_ops.tpg_alloc_fabric_acl = &lio_tpg_alloc_fabric_acl;
         fabric->tf_ops.tpg_release_fabric_acl = &lio_tpg_release_fabric_acl;
         fabric->tf_ops.tpg_get_inst_index = &lio_tpg_get_inst_index;
+        fabric->tf_ops.check_stop_free = &lio_check_stop_free,
         fabric->tf_ops.release_cmd = &lio_release_cmd;
         fabric->tf_ops.shutdown_session = &lio_tpg_shutdown_session;
         fabric->tf_ops.close_session = &lio_tpg_close_session;
diff --git a/drivers/target/iscsi/iscsi_target_core.h b/drivers/target/iscsi/iscsi_target_core.h
index 7a333d28d9a2..60ec4b92be03 100644
--- a/drivers/target/iscsi/iscsi_target_core.h
+++ b/drivers/target/iscsi/iscsi_target_core.h
@@ -60,7 +60,7 @@
 
 #define ISCSI_IOV_DATA_BUFFER           5
 
-enum tpg_np_network_transport_table {
+enum iscsit_transport_type {
         ISCSI_TCP                               = 0,
         ISCSI_SCTP_TCP                          = 1,
         ISCSI_SCTP_UDP                          = 2,
@@ -244,6 +244,11 @@ struct iscsi_conn_ops {
         u8      IFMarker;                       /* [0,1] == [No,Yes] */
         u32     OFMarkInt;                      /* [1..65535] */
         u32     IFMarkInt;                      /* [1..65535] */
+        /*
+         * iSER specific connection parameters
+         */
+        u32     InitiatorRecvDataSegmentLength; /* [512..2**24-1] */
+        u32     TargetRecvDataSegmentLength;    /* [512..2**24-1] */
 };
 
 struct iscsi_sess_ops {
@@ -265,6 +270,10 @@ struct iscsi_sess_ops {
         u8      DataSequenceInOrder;            /* [0,1] == [No,Yes] */
         u8      ErrorRecoveryLevel;             /* [0..2] */
         u8      SessionType;                    /* [0,1] == [Normal,Discovery]*/
+        /*
+         * iSER specific session parameters
+         */
+        u8      RDMAExtensions;                 /* [0,1] == [No,Yes] */
 };
 
 struct iscsi_queue_req {
@@ -284,6 +293,7 @@ struct iscsi_data_count {
 };
 
 struct iscsi_param_list {
+        bool                    iser;
         struct list_head        param_list;
         struct list_head        extra_response_list;
 };
@@ -475,6 +485,7 @@ struct iscsi_cmd {
         u32                     first_data_sg_off;
         u32                     kmapped_nents;
         sense_reason_t          sense_reason;
+        void (*release_cmd)(struct iscsi_cmd *);
 }  ____cacheline_aligned;
 
 struct iscsi_tmr_req {
@@ -503,6 +514,7 @@ struct iscsi_conn {
         u16                     login_port;
         u16                     local_port;
         int                     net_size;
+        int                     login_family;
         u32                     auth_id;
         u32                     conn_flags;
         /* Used for iscsi_tx_login_rsp() */
@@ -562,9 +574,12 @@ struct iscsi_conn {
         struct list_head        immed_queue_list;
         struct list_head        response_queue_list;
         struct iscsi_conn_ops   *conn_ops;
+        struct iscsi_login      *conn_login;
+        struct iscsit_transport *conn_transport;
         struct iscsi_param_list *param_list;
         /* Used for per connection auth state machine */
         void                    *auth_protocol;
+        void                    *context;
         struct iscsi_login_thread_s *login_thread;
         struct iscsi_portal_group *tpg;
         /* Pointer to parent session */
@@ -663,6 +678,8 @@ struct iscsi_login {
         u8 first_request;
         u8 version_min;
         u8 version_max;
+        u8 login_complete;
+        u8 login_failed;
         char isid[6];
         u32 cmd_sn;
         itt_t init_task_tag;
@@ -670,10 +687,11 @@ struct iscsi_login {
         u32 rsp_length;
         u16 cid;
         u16 tsih;
-        char *req;
-        char *rsp;
+        char req[ISCSI_HDR_LEN];
+        char rsp[ISCSI_HDR_LEN];
         char *req_buf;
         char *rsp_buf;
+        struct iscsi_conn *conn;
 } ____cacheline_aligned;
 
 struct iscsi_node_attrib {
@@ -754,6 +772,8 @@ struct iscsi_np {
         struct task_struct      *np_thread;
         struct timer_list       np_login_timer;
         struct iscsi_portal_group *np_login_tpg;
+        void                    *np_context;
+        struct iscsit_transport *np_transport;
         struct list_head        np_list;
 } ____cacheline_aligned;
 
diff --git a/drivers/target/iscsi/iscsi_target_device.c b/drivers/target/iscsi/iscsi_target_device.c
index bcc409853a67..1b74033510a0 100644
--- a/drivers/target/iscsi/iscsi_target_device.c
+++ b/drivers/target/iscsi/iscsi_target_device.c
@@ -60,8 +60,13 @@ void iscsit_increment_maxcmdsn(struct iscsi_cmd *cmd, struct iscsi_session *sess
 
         cmd->maxcmdsn_inc = 1;
 
-        mutex_lock(&sess->cmdsn_mutex);
+        if (!mutex_trylock(&sess->cmdsn_mutex)) {
+                sess->max_cmd_sn += 1;
+                pr_debug("Updated MaxCmdSN to 0x%08x\n", sess->max_cmd_sn);
+                return;
+        }
         sess->max_cmd_sn += 1;
         pr_debug("Updated MaxCmdSN to 0x%08x\n", sess->max_cmd_sn);
         mutex_unlock(&sess->cmdsn_mutex);
 }
+EXPORT_SYMBOL(iscsit_increment_maxcmdsn);
diff --git a/drivers/target/iscsi/iscsi_target_erl1.c b/drivers/target/iscsi/iscsi_target_erl1.c
index 0b52a2371305..7816af6cdd12 100644
--- a/drivers/target/iscsi/iscsi_target_erl1.c
+++ b/drivers/target/iscsi/iscsi_target_erl1.c
@@ -22,6 +22,7 @@
 #include <scsi/iscsi_proto.h>
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
+#include <target/iscsi/iscsi_transport.h>
 
 #include "iscsi_target_core.h"
 #include "iscsi_target_seq_pdu_list.h"
@@ -53,6 +54,9 @@ int iscsit_dump_data_payload(
         u32 length, padding, offset = 0, size;
         struct kvec iov;
 
+        if (conn->sess->sess_ops->RDMAExtensions)
+                return 0;
+
         length = (buf_len > OFFLOAD_BUF_SIZE) ? OFFLOAD_BUF_SIZE : buf_len;
 
         buf = kzalloc(length, GFP_ATOMIC);
@@ -919,6 +923,7 @@ int iscsit_execute_ooo_cmdsns(struct iscsi_session *sess)
 int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo)
 {
         struct se_cmd *se_cmd = &cmd->se_cmd;
+        struct iscsi_conn *conn = cmd->conn;
         int lr = 0;
 
         spin_lock_bh(&cmd->istate_lock);
@@ -981,7 +986,7 @@ int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo)
                                         return 0;
 
                                 iscsit_set_dataout_sequence_values(cmd);
-                                iscsit_build_r2ts_for_cmd(cmd, cmd->conn, false);
+                                conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
                         }
                         return 0;
                 }
@@ -999,10 +1004,7 @@ int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo)
                         if (transport_check_aborted_status(se_cmd, 1) != 0)
                                 return 0;
 
-                        iscsit_set_dataout_sequence_values(cmd);
-                        spin_lock_bh(&cmd->dataout_timeout_lock);
-                        iscsit_start_dataout_timer(cmd, cmd->conn);
-                        spin_unlock_bh(&cmd->dataout_timeout_lock);
+                        iscsit_set_unsoliticed_dataout(cmd);
                 }
                 return transport_handle_cdb_direct(&cmd->se_cmd);
 
@@ -1290,3 +1292,4 @@ void iscsit_stop_dataout_timer(struct iscsi_cmd *cmd)
                         cmd->init_task_tag);
         spin_unlock_bh(&cmd->dataout_timeout_lock);
 }
+EXPORT_SYMBOL(iscsit_stop_dataout_timer);
diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
index 2535d4d46c0e..bb5d5c5bce65 100644
--- a/drivers/target/iscsi/iscsi_target_login.c
+++ b/drivers/target/iscsi/iscsi_target_login.c
@@ -39,8 +39,39 @@
 #include "iscsi_target.h"
 #include "iscsi_target_parameters.h"
 
-static int iscsi_login_init_conn(struct iscsi_conn *conn)
+#include <target/iscsi/iscsi_transport.h>
+
+static struct iscsi_login *iscsi_login_init_conn(struct iscsi_conn *conn)
 {
+        struct iscsi_login *login;
+
+        login = kzalloc(sizeof(struct iscsi_login), GFP_KERNEL);
+        if (!login) {
+                pr_err("Unable to allocate memory for struct iscsi_login.\n");
+                return NULL;
+        }
+        login->conn = conn;
+        login->first_request = 1;
+
+        login->req_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL);
+        if (!login->req_buf) {
+                pr_err("Unable to allocate memory for response buffer.\n");
+                goto out_login;
+        }
+
+        login->rsp_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL);
+        if (!login->rsp_buf) {
+                pr_err("Unable to allocate memory for request buffer.\n");
+                goto out_req_buf;
+        }
+
+        conn->conn_ops = kzalloc(sizeof(struct iscsi_conn_ops), GFP_KERNEL);
+        if (!conn->conn_ops) {
+                pr_err("Unable to allocate memory for"
+                        " struct iscsi_conn_ops.\n");
+                goto out_rsp_buf;
+        }
+
         init_waitqueue_head(&conn->queues_wq);
         INIT_LIST_HEAD(&conn->conn_list);
         INIT_LIST_HEAD(&conn->conn_cmd_list);
@@ -62,10 +93,21 @@ static int iscsi_login_init_conn(struct iscsi_conn *conn)
 
         if (!zalloc_cpumask_var(&conn->conn_cpumask, GFP_KERNEL)) {
                 pr_err("Unable to allocate conn->conn_cpumask\n");
-                return -ENOMEM;
+                goto out_conn_ops;
         }
+        conn->conn_login = login;
 
-        return 0;
+        return login;
+
+out_conn_ops:
+        kfree(conn->conn_ops);
+out_rsp_buf:
+        kfree(login->rsp_buf);
+out_req_buf:
+        kfree(login->req_buf);
+out_login:
+        kfree(login);
+        return NULL;
 }
 
 /*
@@ -298,6 +340,7 @@ static int iscsi_login_zero_tsih_s2(
         struct iscsi_node_attrib *na;
         struct iscsi_session *sess = conn->sess;
         unsigned char buf[32];
+        bool iser = false;
 
         sess->tpg = conn->tpg;
 
@@ -319,7 +362,10 @@ static int iscsi_login_zero_tsih_s2(
                 return -1;
         }
 
-        iscsi_set_keys_to_negotiate(0, conn->param_list);
+        if (conn->conn_transport->transport_type == ISCSI_INFINIBAND)
+                iser = true;
+
+        iscsi_set_keys_to_negotiate(conn->param_list, iser);
 
         if (sess->sess_ops->SessionType)
                 return iscsi_set_keys_irrelevant_for_discovery(
@@ -357,6 +403,56 @@ static int iscsi_login_zero_tsih_s2(
 
         if (iscsi_login_disable_FIM_keys(conn->param_list, conn) < 0)
                 return -1;
+        /*
+         * Set RDMAExtensions=Yes by default for iSER enabled network portals
+         */
+        if (iser) {
+                struct iscsi_param *param;
+                unsigned long mrdsl, off;
+                int rc;
+
+                sprintf(buf, "RDMAExtensions=Yes");
+                if (iscsi_change_param_value(buf, conn->param_list, 0) < 0) {
+                        iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
+                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
+                        return -1;
+                }
+                /*
+                 * Make MaxRecvDataSegmentLength PAGE_SIZE aligned for
+                 * Immediate Data + Unsolicitied Data-OUT if necessary..
+                 */
+                param = iscsi_find_param_from_key("MaxRecvDataSegmentLength",
+                                                  conn->param_list);
+                if (!param) {
+                        iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
+                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
+                        return -1;
+                }
+                rc = strict_strtoul(param->value, 0, &mrdsl);
+                if (rc < 0) {
+                        iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
+                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
+                        return -1;
+                }
+                off = mrdsl % PAGE_SIZE;
+                if (!off)
+                        return 0;
+
+                if (mrdsl < PAGE_SIZE)
+                        mrdsl = PAGE_SIZE;
+                else
+                        mrdsl -= off;
+
+                pr_warn("Aligning ISER MaxRecvDataSegmentLength: %lu down"
+                        " to PAGE_SIZE\n", mrdsl);
+
+                sprintf(buf, "MaxRecvDataSegmentLength=%lu\n", mrdsl);
+                if (iscsi_change_param_value(buf, conn->param_list, 0) < 0) {
+                        iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
+                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
+                        return -1;
+                }
+        }
 
         return 0;
 }
@@ -436,6 +532,7 @@ static int iscsi_login_non_zero_tsih_s2(
         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
         struct se_session *se_sess, *se_sess_tmp;
         struct iscsi_login_req *pdu = (struct iscsi_login_req *)buf;
+        bool iser = false;
 
         spin_lock_bh(&se_tpg->session_lock);
         list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
@@ -485,7 +582,10 @@ static int iscsi_login_non_zero_tsih_s2(
                 return -1;
         }
 
-        iscsi_set_keys_to_negotiate(0, conn->param_list);
+        if (conn->conn_transport->transport_type == ISCSI_INFINIBAND)
+                iser = true;
+
+        iscsi_set_keys_to_negotiate(conn->param_list, iser);
         /*
          * Need to send TargetPortalGroupTag back in first login response
          * on any iSCSI connection where the Initiator provides TargetName.
@@ -574,6 +674,11 @@ int iscsi_login_post_auth_non_zero_tsih(
 static void iscsi_post_login_start_timers(struct iscsi_conn *conn)
 {
         struct iscsi_session *sess = conn->sess;
+        /*
+         * FIXME: Unsolicitied NopIN support for ISER
+         */
+        if (conn->conn_transport->transport_type == ISCSI_INFINIBAND)
+                return;
 
         if (!sess->sess_ops->SessionType)
                 iscsit_start_nopin_timer(conn);
@@ -632,6 +737,7 @@ static int iscsi_post_login_handler(
                 spin_unlock_bh(&sess->conn_lock);
 
                 iscsi_post_login_start_timers(conn);
+
                 iscsi_activate_thread_set(conn, ts);
                 /*
                  * Determine CPU mask to ensure connection's RX and TX kthreads
@@ -761,11 +867,11 @@ static void iscsi_stop_login_thread_timer(struct iscsi_np *np)
         spin_unlock_bh(&np->np_thread_lock);
 }
 
-int iscsi_target_setup_login_socket(
+int iscsit_setup_np(
         struct iscsi_np *np,
         struct __kernel_sockaddr_storage *sockaddr)
 {
-        struct socket *sock;
+        struct socket *sock = NULL;
         int backlog = 5, ret, opt = 0, len;
 
         switch (np->np_network_transport) {
@@ -781,15 +887,15 @@ int iscsi_target_setup_login_socket(
                 np->np_ip_proto = IPPROTO_SCTP;
                 np->np_sock_type = SOCK_SEQPACKET;
                 break;
-        case ISCSI_IWARP_TCP:
-        case ISCSI_IWARP_SCTP:
-        case ISCSI_INFINIBAND:
         default:
                 pr_err("Unsupported network_transport: %d\n",
                                 np->np_network_transport);
                 return -EINVAL;
         }
 
+        np->np_ip_proto = IPPROTO_TCP;
+        np->np_sock_type = SOCK_STREAM;
+
         ret = sock_create(sockaddr->ss_family, np->np_sock_type,
                         np->np_ip_proto, &sock);
         if (ret < 0) {
@@ -853,7 +959,6 @@ int iscsi_target_setup_login_socket(
         }
 
         return 0;
-
 fail:
         np->np_socket = NULL;
         if (sock)
@@ -861,21 +966,169 @@ fail:
         return ret;
 }
 
+int iscsi_target_setup_login_socket(
+        struct iscsi_np *np,
+        struct __kernel_sockaddr_storage *sockaddr)
+{
+        struct iscsit_transport *t;
+        int rc;
+
+        t = iscsit_get_transport(np->np_network_transport);
+        if (!t)
+                return -EINVAL;
+
+        rc = t->iscsit_setup_np(np, sockaddr);
+        if (rc < 0) {
+                iscsit_put_transport(t);
+                return rc;
+        }
+
+        np->np_transport = t;
+        printk("Set np->np_transport to %p -> %s\n", np->np_transport,
+                                np->np_transport->name);
+        return 0;
+}
+
+int iscsit_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
+{
+        struct socket *new_sock, *sock = np->np_socket;
+        struct sockaddr_in sock_in;
+        struct sockaddr_in6 sock_in6;
+        int rc, err;
+
+        rc = kernel_accept(sock, &new_sock, 0);
+        if (rc < 0)
+                return rc;
+
+        conn->sock = new_sock;
+        conn->login_family = np->np_sockaddr.ss_family;
+        printk("iSCSI/TCP: Setup conn->sock from new_sock: %p\n", new_sock);
+
+        if (np->np_sockaddr.ss_family == AF_INET6) {
+                memset(&sock_in6, 0, sizeof(struct sockaddr_in6));
+
+                rc = conn->sock->ops->getname(conn->sock,
+                                (struct sockaddr *)&sock_in6, &err, 1);
+                if (!rc) {
+                        snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c",
+                                &sock_in6.sin6_addr.in6_u);
+                        conn->login_port = ntohs(sock_in6.sin6_port);
+                }
+
+                rc = conn->sock->ops->getname(conn->sock,
+                                (struct sockaddr *)&sock_in6, &err, 0);
+                if (!rc) {
+                        snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c",
+                                &sock_in6.sin6_addr.in6_u);
+                        conn->local_port = ntohs(sock_in6.sin6_port);
+                }
+        } else {
+                memset(&sock_in, 0, sizeof(struct sockaddr_in));
+
+                rc = conn->sock->ops->getname(conn->sock,
+                                (struct sockaddr *)&sock_in, &err, 1);
+                if (!rc) {
+                        sprintf(conn->login_ip, "%pI4",
+                                        &sock_in.sin_addr.s_addr);
+                        conn->login_port = ntohs(sock_in.sin_port);
+                }
+
+                rc = conn->sock->ops->getname(conn->sock,
+                                (struct sockaddr *)&sock_in, &err, 0);
+                if (!rc) {
+                        sprintf(conn->local_ip, "%pI4",
+                                        &sock_in.sin_addr.s_addr);
+                        conn->local_port = ntohs(sock_in.sin_port);
+                }
+        }
+
+        return 0;
+}
+
+int iscsit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
+{
+        struct iscsi_login_req *login_req;
+        u32 padding = 0, payload_length;
+
+        if (iscsi_login_rx_data(conn, login->req, ISCSI_HDR_LEN) < 0)
+                return -1;
+
+        login_req = (struct iscsi_login_req *)login->req;
+        payload_length  = ntoh24(login_req->dlength);
+        padding = ((-payload_length) & 3);
+
+        pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
+                " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
+                login_req->flags, login_req->itt, login_req->cmdsn,
+                login_req->exp_statsn, login_req->cid, payload_length);
+        /*
+         * Setup the initial iscsi_login values from the leading
+         * login request PDU.
+         */
+        if (login->first_request) {
+                login_req = (struct iscsi_login_req *)login->req;
+                login->leading_connection = (!login_req->tsih) ? 1 : 0;
+                login->current_stage    = ISCSI_LOGIN_CURRENT_STAGE(login_req->flags);
+                login->version_min      = login_req->min_version;
+                login->version_max      = login_req->max_version;
+                memcpy(login->isid, login_req->isid, 6);
+                login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
+                login->init_task_tag    = login_req->itt;
+                login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
+                login->cid              = be16_to_cpu(login_req->cid);
+                login->tsih             = be16_to_cpu(login_req->tsih);
+        }
+
+        if (iscsi_target_check_login_request(conn, login) < 0)
+                return -1;
+
+        memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
+        if (iscsi_login_rx_data(conn, login->req_buf,
+                                payload_length + padding) < 0)
+                return -1;
+
+        return 0;
+}
+
+int iscsit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
+                        u32 length)
+{
+        if (iscsi_login_tx_data(conn, login->rsp, login->rsp_buf, length) < 0)
+                return -1;
+
+        return 0;
+}
+
+static int
+iscsit_conn_set_transport(struct iscsi_conn *conn, struct iscsit_transport *t)
+{
+        int rc;
+
+        if (!t->owner) {
+                conn->conn_transport = t;
+                return 0;
+        }
+
+        rc = try_module_get(t->owner);
+        if (!rc) {
+                pr_err("try_module_get() failed for %s\n", t->name);
+                return -EINVAL;
+        }
+
+        conn->conn_transport = t;
+        return 0;
+}
+
 static int __iscsi_target_login_thread(struct iscsi_np *np)
 {
-        u8 buffer[ISCSI_HDR_LEN], iscsi_opcode, zero_tsih = 0;
-        int err, ret = 0, stop;
+        u8 *buffer, zero_tsih = 0;
+        int ret = 0, rc, stop;
         struct iscsi_conn *conn = NULL;
         struct iscsi_login *login;
         struct iscsi_portal_group *tpg = NULL;
-        struct socket *new_sock, *sock;
-        struct kvec iov;
         struct iscsi_login_req *pdu;
-        struct sockaddr_in sock_in;
-        struct sockaddr_in6 sock_in6;
 
         flush_signals(current);
-        sock = np->np_socket;
 
         spin_lock_bh(&np->np_thread_lock);
         if (np->np_thread_state == ISCSI_NP_THREAD_RESET) {
@@ -886,75 +1139,76 @@ static int __iscsi_target_login_thread(struct iscsi_np *np)
         }
         spin_unlock_bh(&np->np_thread_lock);
 
-        if (kernel_accept(sock, &new_sock, 0) < 0) {
-                spin_lock_bh(&np->np_thread_lock);
-                if (np->np_thread_state == ISCSI_NP_THREAD_RESET) {
-                        spin_unlock_bh(&np->np_thread_lock);
-                        complete(&np->np_restart_comp);
-                        /* Get another socket */
-                        return 1;
-                }
-                spin_unlock_bh(&np->np_thread_lock);
-                goto out;
-        }
-        iscsi_start_login_thread_timer(np);
-
         conn = kzalloc(sizeof(struct iscsi_conn), GFP_KERNEL);
         if (!conn) {
                 pr_err("Could not allocate memory for"
                         " new connection\n");
-                sock_release(new_sock);
                 /* Get another socket */
                 return 1;
         }
-
         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
         conn->conn_state = TARG_CONN_STATE_FREE;
-        conn->sock = new_sock;
 
-        pr_debug("Moving to TARG_CONN_STATE_XPT_UP.\n");
-        conn->conn_state = TARG_CONN_STATE_XPT_UP;
+        if (iscsit_conn_set_transport(conn, np->np_transport) < 0) {
+                kfree(conn);
+                return 1;
+        }
 
-        /*
-         * Allocate conn->conn_ops early as a failure calling
-         * iscsit_tx_login_rsp() below will call tx_data().
-         */
-        conn->conn_ops = kzalloc(sizeof(struct iscsi_conn_ops), GFP_KERNEL);
-        if (!conn->conn_ops) {
-                pr_err("Unable to allocate memory for"
-                        " struct iscsi_conn_ops.\n");
-                goto new_sess_out;
+        rc = np->np_transport->iscsit_accept_np(np, conn);
+        if (rc == -ENOSYS) {
+                complete(&np->np_restart_comp);
+                iscsit_put_transport(conn->conn_transport);
+                kfree(conn);
+                conn = NULL;
+                goto exit;
+        } else if (rc < 0) {
+                spin_lock_bh(&np->np_thread_lock);
+                if (np->np_thread_state == ISCSI_NP_THREAD_RESET) {
+                        spin_unlock_bh(&np->np_thread_lock);
+                        complete(&np->np_restart_comp);
+                        if (ret == -ENODEV) {
+                                iscsit_put_transport(conn->conn_transport);
+                                kfree(conn);
+                                conn = NULL;
+                                goto out;
+                        }
+                        /* Get another socket */
+                        return 1;
+                }
+                spin_unlock_bh(&np->np_thread_lock);
+                iscsit_put_transport(conn->conn_transport);
+                kfree(conn);
+                conn = NULL;
+                goto out;
         }
         /*
          * Perform the remaining iSCSI connection initialization items..
          */
-        if (iscsi_login_init_conn(conn) < 0)
-                goto new_sess_out;
-
-        memset(buffer, 0, ISCSI_HDR_LEN);
-        memset(&iov, 0, sizeof(struct kvec));
-        iov.iov_base    = buffer;
-        iov.iov_len     = ISCSI_HDR_LEN;
-
-        if (rx_data(conn, &iov, 1, ISCSI_HDR_LEN) <= 0) {
-                pr_err("rx_data() returned an error.\n");
+        login = iscsi_login_init_conn(conn);
+        if (!login) {
                 goto new_sess_out;
         }
 
-        iscsi_opcode = (buffer[0] & ISCSI_OPCODE_MASK);
-        if (!(iscsi_opcode & ISCSI_OP_LOGIN)) {
-                pr_err("First opcode is not login request,"
-                        " failing login request.\n");
-                goto new_sess_out;
-        }
+        iscsi_start_login_thread_timer(np);
 
-        pdu                     = (struct iscsi_login_req *) buffer;
+        pr_debug("Moving to TARG_CONN_STATE_XPT_UP.\n");
+        conn->conn_state = TARG_CONN_STATE_XPT_UP;
+        /*
+         * This will process the first login request + payload..
+         */
+        rc = np->np_transport->iscsit_get_login_rx(conn, login);
+        if (rc == 1)
+                return 1;
+        else if (rc < 0)
+                goto new_sess_out;
 
+        buffer = &login->req[0];
+        pdu = (struct iscsi_login_req *)buffer;
         /*
          * Used by iscsit_tx_login_rsp() for Login Resonses PDUs
          * when Status-Class != 0.
         */
-        conn->login_itt         = pdu->itt;
+        conn->login_itt = pdu->itt;
 
         spin_lock_bh(&np->np_thread_lock);
         if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
@@ -967,61 +1221,11 @@ static int __iscsi_target_login_thread(struct iscsi_np *np)
         }
         spin_unlock_bh(&np->np_thread_lock);
 
-        if (np->np_sockaddr.ss_family == AF_INET6) {
-                memset(&sock_in6, 0, sizeof(struct sockaddr_in6));
-
-                if (conn->sock->ops->getname(conn->sock,
-                                (struct sockaddr *)&sock_in6, &err, 1) < 0) {
-                        pr_err("sock_ops->getname() failed.\n");
-                        iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                        ISCSI_LOGIN_STATUS_TARGET_ERROR);
-                        goto new_sess_out;
-                }
-                snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c",
-                                &sock_in6.sin6_addr.in6_u);
-                conn->login_port = ntohs(sock_in6.sin6_port);
-
-                if (conn->sock->ops->getname(conn->sock,
-                                (struct sockaddr *)&sock_in6, &err, 0) < 0) {
-                        pr_err("sock_ops->getname() failed.\n");
-                        iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                        ISCSI_LOGIN_STATUS_TARGET_ERROR);
-                        goto new_sess_out;
-                }
-                snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c",
-                                &sock_in6.sin6_addr.in6_u);
-                conn->local_port = ntohs(sock_in6.sin6_port);
-
-        } else {
-                memset(&sock_in, 0, sizeof(struct sockaddr_in));
-
-                if (conn->sock->ops->getname(conn->sock,
-                                (struct sockaddr *)&sock_in, &err, 1) < 0) {
-                        pr_err("sock_ops->getname() failed.\n");
-                        iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                        ISCSI_LOGIN_STATUS_TARGET_ERROR);
-                        goto new_sess_out;
-                }
-                sprintf(conn->login_ip, "%pI4", &sock_in.sin_addr.s_addr);
-                conn->login_port = ntohs(sock_in.sin_port);
-
-                if (conn->sock->ops->getname(conn->sock,
-                                (struct sockaddr *)&sock_in, &err, 0) < 0) {
-                        pr_err("sock_ops->getname() failed.\n");
-                        iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                        ISCSI_LOGIN_STATUS_TARGET_ERROR);
-                        goto new_sess_out;
-                }
-                sprintf(conn->local_ip, "%pI4", &sock_in.sin_addr.s_addr);
-                conn->local_port = ntohs(sock_in.sin_port);
-        }
-
         conn->network_transport = np->np_network_transport;
 
         pr_debug("Received iSCSI login request from %s on %s Network"
-                        " Portal %s:%hu\n", conn->login_ip,
-                (conn->network_transport == ISCSI_TCP) ? "TCP" : "SCTP",
-                        conn->local_ip, conn->local_port);
+                " Portal %s:%hu\n", conn->login_ip, np->np_transport->name,
+                conn->local_ip, conn->local_port);
 
         pr_debug("Moving to TARG_CONN_STATE_IN_LOGIN.\n");
         conn->conn_state        = TARG_CONN_STATE_IN_LOGIN;
@@ -1050,13 +1254,17 @@ static int __iscsi_target_login_thread(struct iscsi_np *np)
                 if (iscsi_login_non_zero_tsih_s1(conn, buffer) < 0)
                         goto new_sess_out;
         }
-
         /*
-         * This will process the first login request, and call
-         * iscsi_target_locate_portal(), and return a valid struct iscsi_login.
+         * SessionType: Discovery
+         *
+         *      Locates Default Portal
+         *
+         * SessionType: Normal
+         *
+         *      Locates Target Portal from NP -> Target IQN
          */
-        login = iscsi_target_init_negotiation(np, conn, buffer);
-        if (!login) {
+        rc = iscsi_target_locate_portal(np, conn, login);
+        if (rc < 0) {
                 tpg = conn->tpg;
                 goto new_sess_out;
         }
@@ -1068,15 +1276,11 @@ static int __iscsi_target_login_thread(struct iscsi_np *np)
         }
 
         if (zero_tsih) {
-                if (iscsi_login_zero_tsih_s2(conn) < 0) {
-                        iscsi_target_nego_release(login, conn);
+                if (iscsi_login_zero_tsih_s2(conn) < 0)
                         goto new_sess_out;
-                }
         } else {
-                if (iscsi_login_non_zero_tsih_s2(conn, buffer) < 0) {
-                        iscsi_target_nego_release(login, conn);
+                if (iscsi_login_non_zero_tsih_s2(conn, buffer) < 0)
                         goto old_sess_out;
-                }
         }
 
         if (iscsi_target_start_negotiation(login, conn) < 0)
@@ -1153,8 +1357,18 @@ old_sess_out:
                 iscsi_release_param_list(conn->param_list);
                 conn->param_list = NULL;
         }
-        if (conn->sock)
+        iscsi_target_nego_release(conn);
+
+        if (conn->sock) {
                 sock_release(conn->sock);
+                conn->sock = NULL;
+        }
+
+        if (conn->conn_transport->iscsit_free_conn)
+                conn->conn_transport->iscsit_free_conn(conn);
+
+        iscsit_put_transport(conn->conn_transport);
+
         kfree(conn);
 
         if (tpg) {
@@ -1172,11 +1386,13 @@ out:
         /* Wait for another socket.. */
         if (!stop)
                 return 1;
-
+exit:
         iscsi_stop_login_thread_timer(np);
         spin_lock_bh(&np->np_thread_lock);
         np->np_thread_state = ISCSI_NP_THREAD_EXIT;
+        np->np_thread = NULL;
         spin_unlock_bh(&np->np_thread_lock);
+
         return 0;
 }
 
diff --git a/drivers/target/iscsi/iscsi_target_login.h b/drivers/target/iscsi/iscsi_target_login.h
index 091dcae2532b..63efd2878451 100644
--- a/drivers/target/iscsi/iscsi_target_login.h
+++ b/drivers/target/iscsi/iscsi_target_login.h
@@ -4,8 +4,14 @@
 extern int iscsi_login_setup_crypto(struct iscsi_conn *);
 extern int iscsi_check_for_session_reinstatement(struct iscsi_conn *);
 extern int iscsi_login_post_auth_non_zero_tsih(struct iscsi_conn *, u16, u32);
+extern int iscsit_setup_np(struct iscsi_np *,
+                                struct __kernel_sockaddr_storage *);
 extern int iscsi_target_setup_login_socket(struct iscsi_np *,
                                 struct __kernel_sockaddr_storage *);
+extern int iscsit_accept_np(struct iscsi_np *, struct iscsi_conn *);
+extern int iscsit_get_login_rx(struct iscsi_conn *, struct iscsi_login *);
+extern int iscsit_put_login_tx(struct iscsi_conn *, struct iscsi_login *, u32);
+extern void iscsit_free_conn(struct iscsi_np *, struct iscsi_conn *);
 extern int iscsi_target_login_thread(void *);
 extern int iscsi_login_disable_FIM_keys(struct iscsi_param_list *, struct iscsi_conn *);
 
diff --git a/drivers/target/iscsi/iscsi_target_nego.c b/drivers/target/iscsi/iscsi_target_nego.c
index 9d902aefe01a..7ad912060e21 100644
--- a/drivers/target/iscsi/iscsi_target_nego.c
+++ b/drivers/target/iscsi/iscsi_target_nego.c
@@ -22,6 +22,7 @@
 #include <scsi/iscsi_proto.h>
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
+#include <target/iscsi/iscsi_transport.h>
 
 #include "iscsi_target_core.h"
 #include "iscsi_target_parameters.h"
@@ -169,7 +170,7 @@ static void iscsi_remove_failed_auth_entry(struct iscsi_conn *conn)
         kfree(conn->auth_protocol);
 }
 
-static int iscsi_target_check_login_request(
+int iscsi_target_check_login_request(
         struct iscsi_conn *conn,
         struct iscsi_login *login)
 {
@@ -200,8 +201,8 @@ static int iscsi_target_check_login_request(
                 return -1;
         }
 
-        req_csg = (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) >> 2;
-        req_nsg = (login_req->flags & ISCSI_FLAG_LOGIN_NEXT_STAGE_MASK);
+        req_csg = ISCSI_LOGIN_CURRENT_STAGE(login_req->flags);
+        req_nsg = ISCSI_LOGIN_NEXT_STAGE(login_req->flags);
 
         if (req_csg != login->current_stage) {
                 pr_err("Initiator unexpectedly changed login stage"
@@ -352,11 +353,8 @@ static int iscsi_target_do_tx_login_io(struct iscsi_conn *conn, struct iscsi_log
 
         padding = ((-login->rsp_length) & 3);
 
-        if (iscsi_login_tx_data(
-                        conn,
-                        login->rsp,
-                        login->rsp_buf,
-                        login->rsp_length + padding) < 0)
+        if (conn->conn_transport->iscsit_put_login_tx(conn, login,
+                                        login->rsp_length + padding) < 0)
                 return -1;
 
         login->rsp_length               = 0;
@@ -368,72 +366,12 @@ static int iscsi_target_do_tx_login_io(struct iscsi_conn *conn, struct iscsi_log
         return 0;
 }
 
-static int iscsi_target_do_rx_login_io(struct iscsi_conn *conn, struct iscsi_login *login)
-{
-        u32 padding = 0, payload_length;
-        struct iscsi_login_req *login_req;
-
-        if (iscsi_login_rx_data(conn, login->req, ISCSI_HDR_LEN) < 0)
-                return -1;
-
-        login_req = (struct iscsi_login_req *) login->req;
-        payload_length                  = ntoh24(login_req->dlength);
-
-        pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
-                " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
-                 login_req->flags, login_req->itt, login_req->cmdsn,
-                 login_req->exp_statsn, login_req->cid, payload_length);
-
-        if (iscsi_target_check_login_request(conn, login) < 0)
-                return -1;
-
-        padding = ((-payload_length) & 3);
-        memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
-
-        if (iscsi_login_rx_data(
-                        conn,
-                        login->req_buf,
-                        payload_length + padding) < 0)
-                return -1;
-
-        return 0;
-}
-
 static int iscsi_target_do_login_io(struct iscsi_conn *conn, struct iscsi_login *login)
 {
         if (iscsi_target_do_tx_login_io(conn, login) < 0)
                 return -1;
 
-        if (iscsi_target_do_rx_login_io(conn, login) < 0)
-                return -1;
-
-        return 0;
-}
-
-static int iscsi_target_get_initial_payload(
-        struct iscsi_conn *conn,
-        struct iscsi_login *login)
-{
-        u32 padding = 0, payload_length;
-        struct iscsi_login_req *login_req;
-
-        login_req = (struct iscsi_login_req *) login->req;
-        payload_length = ntoh24(login_req->dlength);
-
-        pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
-                " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
-                login_req->flags, login_req->itt, login_req->cmdsn,
-                login_req->exp_statsn, payload_length);
-
-        if (iscsi_target_check_login_request(conn, login) < 0)
-                return -1;
-
-        padding = ((-payload_length) & 3);
-
-        if (iscsi_login_rx_data(
-                        conn,
-                        login->req_buf,
-                        payload_length + padding) < 0)
+        if (conn->conn_transport->iscsit_get_login_rx(conn, login) < 0)
                 return -1;
 
         return 0;
@@ -681,9 +619,9 @@ static int iscsi_target_do_login(struct iscsi_conn *conn, struct iscsi_login *lo
                         return -1;
                 }
 
-                switch ((login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) >> 2) {
+                switch (ISCSI_LOGIN_CURRENT_STAGE(login_req->flags)) {
                 case 0:
-                        login_rsp->flags |= (0 & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK);
+                        login_rsp->flags &= ~ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK;
                         if (iscsi_target_handle_csg_zero(conn, login) < 0)
                                 return -1;
                         break;
@@ -693,6 +631,7 @@ static int iscsi_target_do_login(struct iscsi_conn *conn, struct iscsi_login *lo
                                 return -1;
                         if (login_rsp->flags & ISCSI_FLAG_LOGIN_TRANSIT) {
                                 login->tsih = conn->sess->tsih;
+                                login->login_complete = 1;
                                 if (iscsi_target_do_tx_login_io(conn,
                                                 login) < 0)
                                         return -1;
@@ -702,8 +641,7 @@ static int iscsi_target_do_login(struct iscsi_conn *conn, struct iscsi_login *lo
                 default:
                         pr_err("Illegal CSG: %d received from"
                                 " Initiator, protocol error.\n",
-                                (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
-                                >> 2);
+                                ISCSI_LOGIN_CURRENT_STAGE(login_req->flags));
                         break;
                 }
 
@@ -737,7 +675,7 @@ static void iscsi_initiatorname_tolower(
 /*
  * Processes the first Login Request..
  */
-static int iscsi_target_locate_portal(
+int iscsi_target_locate_portal(
         struct iscsi_np *np,
         struct iscsi_conn *conn,
         struct iscsi_login *login)
@@ -753,22 +691,6 @@ static int iscsi_target_locate_portal(
         login_req = (struct iscsi_login_req *) login->req;
         payload_length = ntoh24(login_req->dlength);
 
-        login->first_request    = 1;
-        login->leading_connection = (!login_req->tsih) ? 1 : 0;
-        login->current_stage    =
-                (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) >> 2;
-        login->version_min      = login_req->min_version;
-        login->version_max      = login_req->max_version;
-        memcpy(login->isid, login_req->isid, 6);
-        login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
-        login->init_task_tag    = login_req->itt;
-        login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
-        login->cid              = be16_to_cpu(login_req->cid);
-        login->tsih             = be16_to_cpu(login_req->tsih);
-
-        if (iscsi_target_get_initial_payload(conn, login) < 0)
-                return -1;
-
         tmpbuf = kzalloc(payload_length + 1, GFP_KERNEL);
         if (!tmpbuf) {
                 pr_err("Unable to allocate memory for tmpbuf.\n");
@@ -800,6 +722,8 @@ static int iscsi_target_locate_portal(
                 start += strlen(key) + strlen(value) + 2;
         }
 
+        printk("i_buf: %s, s_buf: %s, t_buf: %s\n", i_buf, s_buf, t_buf);
+
         /*
          * See 5.3.  Login Phase.
          */
@@ -958,100 +882,30 @@ out:
         return ret;
 }
 
-struct iscsi_login *iscsi_target_init_negotiation(
-        struct iscsi_np *np,
-        struct iscsi_conn *conn,
-        char *login_pdu)
-{
-        struct iscsi_login *login;
-
-        login = kzalloc(sizeof(struct iscsi_login), GFP_KERNEL);
-        if (!login) {
-                pr_err("Unable to allocate memory for struct iscsi_login.\n");
-                iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
-                return NULL;
-        }
-
-        login->req = kmemdup(login_pdu, ISCSI_HDR_LEN, GFP_KERNEL);
-        if (!login->req) {
-                pr_err("Unable to allocate memory for Login Request.\n");
-                iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
-                goto out;
-        }
-
-        login->req_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL);
-        if (!login->req_buf) {
-                pr_err("Unable to allocate memory for response buffer.\n");
-                iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
-                goto out;
-        }
-        /*
-         * SessionType: Discovery
-         *
-         *      Locates Default Portal
-         *
-         * SessionType: Normal
-         *
-         *      Locates Target Portal from NP -> Target IQN
-         */
-        if (iscsi_target_locate_portal(np, conn, login) < 0) {
-                goto out;
-        }
-
-        return login;
-out:
-        kfree(login->req);
-        kfree(login->req_buf);
-        kfree(login);
-
-        return NULL;
-}
-
 int iscsi_target_start_negotiation(
         struct iscsi_login *login,
         struct iscsi_conn *conn)
 {
-        int ret = -1;
-
-        login->rsp = kzalloc(ISCSI_HDR_LEN, GFP_KERNEL);
-        if (!login->rsp) {
-                pr_err("Unable to allocate memory for"
-                                " Login Response.\n");
-                iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
-                ret = -1;
-                goto out;
-        }
-
-        login->rsp_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL);
-        if (!login->rsp_buf) {
-                pr_err("Unable to allocate memory for"
-                        " request buffer.\n");
-                iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR,
-                                ISCSI_LOGIN_STATUS_NO_RESOURCES);
-                ret = -1;
-                goto out;
-        }
+        int ret;
 
         ret = iscsi_target_do_login(conn, login);
-out:
         if (ret != 0)
                 iscsi_remove_failed_auth_entry(conn);
 
-        iscsi_target_nego_release(login, conn);
+        iscsi_target_nego_release(conn);
         return ret;
 }
 
-void iscsi_target_nego_release(
-        struct iscsi_login *login,
-        struct iscsi_conn *conn)
+void iscsi_target_nego_release(struct iscsi_conn *conn)
 {
-        kfree(login->req);
-        kfree(login->rsp);
+        struct iscsi_login *login = conn->conn_login;
+
+        if (!login)
+                return;
+
         kfree(login->req_buf);
         kfree(login->rsp_buf);
         kfree(login);
+
+        conn->conn_login = NULL;
 }
diff --git a/drivers/target/iscsi/iscsi_target_nego.h b/drivers/target/iscsi/iscsi_target_nego.h
index 92e133a5158f..f021cbd330e5 100644
--- a/drivers/target/iscsi/iscsi_target_nego.h
+++ b/drivers/target/iscsi/iscsi_target_nego.h
@@ -7,11 +7,14 @@
 extern void convert_null_to_semi(char *, int);
 extern int extract_param(const char *, const char *, unsigned int, char *,
                 unsigned char *);
-extern struct iscsi_login *iscsi_target_init_negotiation(
-                struct iscsi_np *, struct iscsi_conn *, char *);
+extern int iscsi_target_check_login_request(struct iscsi_conn *,
+                struct iscsi_login *);
+extern int iscsi_target_get_initial_payload(struct iscsi_conn *,
+                struct iscsi_login *);
+extern int iscsi_target_locate_portal(struct iscsi_np *, struct iscsi_conn *,
+                struct iscsi_login *);
 extern int iscsi_target_start_negotiation(
                 struct iscsi_login *, struct iscsi_conn *);
-extern void iscsi_target_nego_release(
-                struct iscsi_login *, struct iscsi_conn *);
+extern void iscsi_target_nego_release(struct iscsi_conn *);
 
 #endif /* ISCSI_TARGET_NEGO_H */
diff --git a/drivers/target/iscsi/iscsi_target_parameters.c b/drivers/target/iscsi/iscsi_target_parameters.c
index ca2be406f141..f690be9e5293 100644
--- a/drivers/target/iscsi/iscsi_target_parameters.c
+++ b/drivers/target/iscsi/iscsi_target_parameters.c
@@ -59,7 +59,7 @@ int iscsi_login_tx_data(
         char *text_buf,
         int text_length)
 {
-        int length, tx_sent;
+        int length, tx_sent, iov_cnt = 1;
         struct kvec iov[2];
 
         length = (ISCSI_HDR_LEN + text_length);
@@ -67,8 +67,12 @@ int iscsi_login_tx_data(
         memset(&iov[0], 0, 2 * sizeof(struct kvec));
         iov[0].iov_len          = ISCSI_HDR_LEN;
         iov[0].iov_base         = pdu_buf;
-        iov[1].iov_len          = text_length;
-        iov[1].iov_base         = text_buf;
+
+        if (text_buf && text_length) {
+                iov[1].iov_len  = text_length;
+                iov[1].iov_base = text_buf;
+                iov_cnt++;
+        }
 
         /*
          * Initial Marker-less Interval.
@@ -77,7 +81,7 @@ int iscsi_login_tx_data(
          */
         conn->if_marker += length;
 
-        tx_sent = tx_data(conn, &iov[0], 2, length);
+        tx_sent = tx_data(conn, &iov[0], iov_cnt, length);
         if (tx_sent != length) {
                 pr_err("tx_data returned %d, expecting %d.\n",
                                 tx_sent, length);
@@ -429,6 +433,28 @@ int iscsi_create_default_params(struct iscsi_param_list **param_list_ptr)
                         TYPERANGE_MARKINT, USE_INITIAL_ONLY);
         if (!param)
                 goto out;
+        /*
+         * Extra parameters for ISER from RFC-5046
+         */
+        param = iscsi_set_default_param(pl, RDMAEXTENTIONS, INITIAL_RDMAEXTENTIONS,
+                        PHASE_OPERATIONAL, SCOPE_SESSION_WIDE, SENDER_BOTH,
+                        TYPERANGE_BOOL_AND, USE_LEADING_ONLY);
+        if (!param)
+                goto out;
+
+        param = iscsi_set_default_param(pl, INITIATORRECVDATASEGMENTLENGTH,
+                        INITIAL_INITIATORRECVDATASEGMENTLENGTH,
+                        PHASE_OPERATIONAL, SCOPE_CONNECTION_ONLY, SENDER_BOTH,
+                        TYPERANGE_512_TO_16777215, USE_ALL);
+        if (!param)
+                goto out;
+
+        param = iscsi_set_default_param(pl, TARGETRECVDATASEGMENTLENGTH,
+                        INITIAL_TARGETRECVDATASEGMENTLENGTH,
+                        PHASE_OPERATIONAL, SCOPE_CONNECTION_ONLY, SENDER_BOTH,
+                        TYPERANGE_512_TO_16777215, USE_ALL);
+        if (!param)
+                goto out;
 
         *param_list_ptr = pl;
         return 0;
@@ -438,19 +464,23 @@ out:
 }
 
 int iscsi_set_keys_to_negotiate(
-        int sessiontype,
-        struct iscsi_param_list *param_list)
+        struct iscsi_param_list *param_list,
+        bool iser)
 {
         struct iscsi_param *param;
 
+        param_list->iser = iser;
+
         list_for_each_entry(param, &param_list->param_list, p_list) {
                 param->state = 0;
                 if (!strcmp(param->name, AUTHMETHOD)) {
                         SET_PSTATE_NEGOTIATE(param);
                 } else if (!strcmp(param->name, HEADERDIGEST)) {
-                        SET_PSTATE_NEGOTIATE(param);
+                        if (iser == false)
+                                SET_PSTATE_NEGOTIATE(param);
                 } else if (!strcmp(param->name, DATADIGEST)) {
-                        SET_PSTATE_NEGOTIATE(param);
+                        if (iser == false)
+                                SET_PSTATE_NEGOTIATE(param);
                 } else if (!strcmp(param->name, MAXCONNECTIONS)) {
                         SET_PSTATE_NEGOTIATE(param);
                 } else if (!strcmp(param->name, TARGETNAME)) {
@@ -469,7 +499,8 @@ int iscsi_set_keys_to_negotiate(
                 } else if (!strcmp(param->name, IMMEDIATEDATA)) {
                         SET_PSTATE_NEGOTIATE(param);
                 } else if (!strcmp(param->name, MAXRECVDATASEGMENTLENGTH)) {
-                        SET_PSTATE_NEGOTIATE(param);
+                        if (iser == false)
+                                SET_PSTATE_NEGOTIATE(param);
                 } else if (!strcmp(param->name, MAXXMITDATASEGMENTLENGTH)) {
                         continue;
                 } else if (!strcmp(param->name, MAXBURSTLENGTH)) {
@@ -498,6 +529,15 @@ int iscsi_set_keys_to_negotiate(
                         SET_PSTATE_NEGOTIATE(param);
                 } else if (!strcmp(param->name, OFMARKINT)) {
                         SET_PSTATE_NEGOTIATE(param);
+                } else if (!strcmp(param->name, RDMAEXTENTIONS)) {
+                        if (iser == true)
+                                SET_PSTATE_NEGOTIATE(param);
+                } else if (!strcmp(param->name, INITIATORRECVDATASEGMENTLENGTH)) {
+                        if (iser == true)
+                                SET_PSTATE_NEGOTIATE(param);
+                } else if (!strcmp(param->name, TARGETRECVDATASEGMENTLENGTH)) {
+                        if (iser == true)
+                                SET_PSTATE_NEGOTIATE(param);
                 }
         }
 
@@ -540,6 +580,12 @@ int iscsi_set_keys_irrelevant_for_discovery(
                         param->state &= ~PSTATE_NEGOTIATE;
                 else if (!strcmp(param->name, OFMARKINT))
                         param->state &= ~PSTATE_NEGOTIATE;
+                else if (!strcmp(param->name, RDMAEXTENTIONS))
+                        param->state &= ~PSTATE_NEGOTIATE;
+                else if (!strcmp(param->name, INITIATORRECVDATASEGMENTLENGTH))
+                        param->state &= ~PSTATE_NEGOTIATE;
+                else if (!strcmp(param->name, TARGETRECVDATASEGMENTLENGTH))
+                        param->state &= ~PSTATE_NEGOTIATE;
         }
 
         return 0;
@@ -1755,6 +1801,9 @@ void iscsi_set_connection_parameters(
                  * this key is not sent over the wire.
                  */
                 if (!strcmp(param->name, MAXXMITDATASEGMENTLENGTH)) {
+                        if (param_list->iser == true)
+                                continue;
+
                         ops->MaxXmitDataSegmentLength =
                                 simple_strtoul(param->value, &tmpptr, 0);
                         pr_debug("MaxXmitDataSegmentLength:     %s\n",
@@ -1800,6 +1849,22 @@ void iscsi_set_connection_parameters(
                                 simple_strtoul(param->value, &tmpptr, 0);
                         pr_debug("IFMarkInt:                    %s\n",
                                 param->value);
+                } else if (!strcmp(param->name, INITIATORRECVDATASEGMENTLENGTH)) {
+                        ops->InitiatorRecvDataSegmentLength =
+                                simple_strtoul(param->value, &tmpptr, 0);
+                        pr_debug("InitiatorRecvDataSegmentLength: %s\n",
+                                param->value);
+                        ops->MaxRecvDataSegmentLength =
+                                        ops->InitiatorRecvDataSegmentLength;
+                        pr_debug("Set MRDSL from InitiatorRecvDataSegmentLength\n");
+                } else if (!strcmp(param->name, TARGETRECVDATASEGMENTLENGTH)) {
+                        ops->TargetRecvDataSegmentLength =
+                                simple_strtoul(param->value, &tmpptr, 0);
+                        pr_debug("TargetRecvDataSegmentLength:  %s\n",
+                                param->value);
+                        ops->MaxXmitDataSegmentLength =
+                                        ops->TargetRecvDataSegmentLength;
+                        pr_debug("Set MXDSL from TargetRecvDataSegmentLength\n");
                 }
         }
         pr_debug("----------------------------------------------------"
@@ -1912,6 +1977,10 @@ void iscsi_set_session_parameters(
                         ops->SessionType = !strcmp(param->value, DISCOVERY);
                         pr_debug("SessionType:                  %s\n",
                                 param->value);
+                } else if (!strcmp(param->name, RDMAEXTENTIONS)) {
+                        ops->RDMAExtensions = !strcmp(param->value, YES);
+                        pr_debug("RDMAExtensions:               %s\n",
+                                param->value);
                 }
         }
         pr_debug("----------------------------------------------------"
diff --git a/drivers/target/iscsi/iscsi_target_parameters.h b/drivers/target/iscsi/iscsi_target_parameters.h
index 1e1b7504a76b..f31b9c4b83f2 100644
--- a/drivers/target/iscsi/iscsi_target_parameters.h
+++ b/drivers/target/iscsi/iscsi_target_parameters.h
@@ -27,7 +27,7 @@ extern void iscsi_dump_conn_ops(struct iscsi_conn_ops *);
 extern void iscsi_dump_sess_ops(struct iscsi_sess_ops *);
 extern void iscsi_print_params(struct iscsi_param_list *);
 extern int iscsi_create_default_params(struct iscsi_param_list **);
-extern int iscsi_set_keys_to_negotiate(int, struct iscsi_param_list *);
+extern int iscsi_set_keys_to_negotiate(struct iscsi_param_list *, bool);
 extern int iscsi_set_keys_irrelevant_for_discovery(struct iscsi_param_list *);
 extern int iscsi_copy_param_list(struct iscsi_param_list **,
                         struct iscsi_param_list *, int);
@@ -89,6 +89,13 @@ extern void iscsi_set_session_parameters(struct iscsi_sess_ops *,
 #define X_EXTENSIONKEY_CISCO_OLD        "X-com.cisco.iscsi.draft"
 
 /*
+ * Parameter names of iSCSI Extentions for RDMA (iSER).  See RFC-5046
+ */
+#define RDMAEXTENTIONS                  "RDMAExtensions"
+#define INITIATORRECVDATASEGMENTLENGTH  "InitiatorRecvDataSegmentLength"
+#define TARGETRECVDATASEGMENTLENGTH     "TargetRecvDataSegmentLength"
+
+/*
  * For AuthMethod.
  */
 #define KRB5                            "KRB5"
@@ -133,6 +140,13 @@ extern void iscsi_set_session_parameters(struct iscsi_sess_ops *,
 #define INITIAL_OFMARKINT                       "2048~65535"
 
 /*
+ * Initial values for iSER parameters following RFC-5046 Section 6
+ */
+#define INITIAL_RDMAEXTENTIONS                  NO
+#define INITIAL_INITIATORRECVDATASEGMENTLENGTH  "262144"
+#define INITIAL_TARGETRECVDATASEGMENTLENGTH     "8192"
+
+/*
  * For [Header,Data]Digests.
  */
 #define CRC32C                          "CRC32C"
diff --git a/drivers/target/iscsi/iscsi_target_tmr.c b/drivers/target/iscsi/iscsi_target_tmr.c
index 9d4417aae921..b997e5da47d3 100644
--- a/drivers/target/iscsi/iscsi_target_tmr.c
+++ b/drivers/target/iscsi/iscsi_target_tmr.c
@@ -23,6 +23,7 @@
 #include <scsi/iscsi_proto.h>
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
+#include <target/iscsi/iscsi_transport.h>
 
 #include "iscsi_target_core.h"
 #include "iscsi_target_seq_pdu_list.h"
@@ -301,7 +302,7 @@ static int iscsit_task_reassign_complete_write(
         /*
          * iscsit_build_r2ts_for_cmd() can handle the rest from here.
          */
-        return iscsit_build_r2ts_for_cmd(cmd, conn, true);
+        return conn->conn_transport->iscsit_get_dataout(conn, cmd, true);
 }
 
 static int iscsit_task_reassign_complete_read(
@@ -471,6 +472,7 @@ int iscsit_tmr_post_handler(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
 
         return 0;
 }
+EXPORT_SYMBOL(iscsit_tmr_post_handler);
 
 /*
  *      Nothing to do here, but leave it for good measure. :-)
diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c
index ee8f8c66248d..439260b7d87f 100644
--- a/drivers/target/iscsi/iscsi_target_tpg.c
+++ b/drivers/target/iscsi/iscsi_target_tpg.c
@@ -31,6 +31,8 @@
 #include "iscsi_target.h"
 #include "iscsi_target_parameters.h"
 
+#include <target/iscsi/iscsi_transport.h>
+
 struct iscsi_portal_group *iscsit_alloc_portal_group(struct iscsi_tiqn *tiqn, u16 tpgt)
 {
         struct iscsi_portal_group *tpg;
@@ -508,7 +510,7 @@ struct iscsi_tpg_np *iscsit_tpg_add_network_portal(
 
         pr_debug("CORE[%s] - Added Network Portal: %s:%hu,%hu on %s\n",
                 tpg->tpg_tiqn->tiqn, np->np_ip, np->np_port, tpg->tpgt,
-                (np->np_network_transport == ISCSI_TCP) ? "TCP" : "SCTP");
+                np->np_transport->name);
 
         return tpg_np;
 }
@@ -522,7 +524,7 @@ static int iscsit_tpg_release_np(
 
         pr_debug("CORE[%s] - Removed Network Portal: %s:%hu,%hu on %s\n",
                 tpg->tpg_tiqn->tiqn, np->np_ip, np->np_port, tpg->tpgt,
-                (np->np_network_transport == ISCSI_TCP) ? "TCP" : "SCTP");
+                np->np_transport->name);
 
         tpg_np->tpg_np = NULL;
         tpg_np->tpg = NULL;
diff --git a/drivers/target/iscsi/iscsi_target_transport.c b/drivers/target/iscsi/iscsi_target_transport.c
new file mode 100644
index 000000000000..882728fac30c
--- /dev/null
+++ b/drivers/target/iscsi/iscsi_target_transport.c
@@ -0,0 +1,55 @@
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <target/iscsi/iscsi_transport.h>
+
+static LIST_HEAD(g_transport_list);
+static DEFINE_MUTEX(transport_mutex);
+
+struct iscsit_transport *iscsit_get_transport(int type)
+{
+        struct iscsit_transport *t;
+
+        mutex_lock(&transport_mutex);
+        list_for_each_entry(t, &g_transport_list, t_node) {
+                if (t->transport_type == type) {
+                        if (t->owner && !try_module_get(t->owner)) {
+                                t = NULL;
+                        }
+                        mutex_unlock(&transport_mutex);
+                        return t;
+                }
+        }
+        mutex_unlock(&transport_mutex);
+
+        return NULL;
+}
+
+void iscsit_put_transport(struct iscsit_transport *t)
+{
+        if (t->owner)
+                module_put(t->owner);
+}
+
+int iscsit_register_transport(struct iscsit_transport *t)
+{
+        INIT_LIST_HEAD(&t->t_node);
+
+        mutex_lock(&transport_mutex);
+        list_add_tail(&t->t_node, &g_transport_list);
+        mutex_unlock(&transport_mutex);
+
+        pr_debug("Registered iSCSI transport: %s\n", t->name);
+
+        return 0;
+}
+EXPORT_SYMBOL(iscsit_register_transport);
+
+void iscsit_unregister_transport(struct iscsit_transport *t)
+{
+        mutex_lock(&transport_mutex);
+        list_del(&t->t_node);
+        mutex_unlock(&transport_mutex);
+
+        pr_debug("Unregistered iSCSI transport: %s\n", t->name);
+}
+EXPORT_SYMBOL(iscsit_unregister_transport);
diff --git a/drivers/target/iscsi/iscsi_target_util.c b/drivers/target/iscsi/iscsi_target_util.c
index 7ce350578c82..2cc6c9a3ffb8 100644
--- a/drivers/target/iscsi/iscsi_target_util.c
+++ b/drivers/target/iscsi/iscsi_target_util.c
@@ -24,6 +24,7 @@
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
 #include <target/target_core_configfs.h>
+#include <target/iscsi/iscsi_transport.h>
 
 #include "iscsi_target_core.h"
 #include "iscsi_target_parameters.h"
@@ -148,6 +149,18 @@ void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd)
         spin_unlock_bh(&cmd->r2t_lock);
 }
 
+struct iscsi_cmd *iscsit_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp_mask)
+{
+        struct iscsi_cmd *cmd;
+
+        cmd = kmem_cache_zalloc(lio_cmd_cache, gfp_mask);
+        if (!cmd)
+                return NULL;
+
+        cmd->release_cmd = &iscsit_release_cmd;
+        return cmd;
+}
+
 /*
  * May be called from software interrupt (timer) context for allocating
  * iSCSI NopINs.
@@ -156,13 +169,12 @@ struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, gfp_t gfp_mask)
 {
         struct iscsi_cmd *cmd;
 
-        cmd = kmem_cache_zalloc(lio_cmd_cache, gfp_mask);
+        cmd = conn->conn_transport->iscsit_alloc_cmd(conn, gfp_mask);
         if (!cmd) {
                 pr_err("Unable to allocate memory for struct iscsi_cmd.\n");
                 return NULL;
         }
-
-        cmd->conn       = conn;
+        cmd->conn = conn;
         INIT_LIST_HEAD(&cmd->i_conn_node);
         INIT_LIST_HEAD(&cmd->datain_list);
         INIT_LIST_HEAD(&cmd->cmd_r2t_list);
@@ -175,6 +187,7 @@ struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, gfp_t gfp_mask)
 
         return cmd;
 }
+EXPORT_SYMBOL(iscsit_allocate_cmd);
 
 struct iscsi_seq *iscsit_get_seq_holder_for_datain(
         struct iscsi_cmd *cmd,
@@ -304,6 +317,7 @@ int iscsit_sequence_cmd(
 
         return ret;
 }
+EXPORT_SYMBOL(iscsit_sequence_cmd);
 
 int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf)
 {
@@ -689,6 +703,11 @@ void iscsit_free_cmd(struct iscsi_cmd *cmd)
          */
         switch (cmd->iscsi_opcode) {
         case ISCSI_OP_SCSI_CMD:
+                if (cmd->data_direction == DMA_TO_DEVICE)
+                        iscsit_stop_dataout_timer(cmd);
+                /*
+                 * Fallthrough
+                 */
         case ISCSI_OP_SCSI_TMFUNC:
                 transport_generic_free_cmd(&cmd->se_cmd, 1);
                 break;
@@ -704,7 +723,7 @@ void iscsit_free_cmd(struct iscsi_cmd *cmd)
                 }
                 /* Fall-through */
         default:
-                iscsit_release_cmd(cmd);
+                cmd->release_cmd(cmd);
                 break;
         }
 }
@@ -1226,34 +1245,19 @@ send_datacrc:
  */
 int iscsit_tx_login_rsp(struct iscsi_conn *conn, u8 status_class, u8 status_detail)
 {
-        u8 iscsi_hdr[ISCSI_HDR_LEN];
-        int err;
-        struct kvec iov;
         struct iscsi_login_rsp *hdr;
+        struct iscsi_login *login = conn->conn_login;
 
+        login->login_failed = 1;
         iscsit_collect_login_stats(conn, status_class, status_detail);
 
-        memset(&iov, 0, sizeof(struct kvec));
-        memset(&iscsi_hdr, 0x0, ISCSI_HDR_LEN);
-
-        hdr     = (struct iscsi_login_rsp *)&iscsi_hdr;
+        hdr     = (struct iscsi_login_rsp *)&login->rsp[0];
         hdr->opcode             = ISCSI_OP_LOGIN_RSP;
         hdr->status_class       = status_class;
         hdr->status_detail      = status_detail;
         hdr->itt                = conn->login_itt;
 
-        iov.iov_base            = &iscsi_hdr;
-        iov.iov_len             = ISCSI_HDR_LEN;
-
-        PRINT_BUFF(iscsi_hdr, ISCSI_HDR_LEN);
-
-        err = tx_data(conn, &iov, 1, ISCSI_HDR_LEN);
-        if (err != ISCSI_HDR_LEN) {
-                pr_err("tx_data returned less than expected\n");
-                return -1;
-        }
-
-        return 0;
+        return conn->conn_transport->iscsit_put_login_tx(conn, login, 0);
 }
 
 void iscsit_print_session_params(struct iscsi_session *sess)
@@ -1432,7 +1436,8 @@ void iscsit_collect_login_stats(
                 strcpy(ls->last_intr_fail_name,
                        (intrname ? intrname->value : "Unknown"));
 
-                ls->last_intr_fail_ip_family = conn->sock->sk->sk_family;
+                ls->last_intr_fail_ip_family = conn->login_family;
+
                 snprintf(ls->last_intr_fail_ip_addr, IPV6_ADDRESS_SPACE,
                                 "%s", conn->login_ip);
                 ls->last_fail_time = get_jiffies_64();
diff --git a/drivers/target/iscsi/iscsi_target_util.h b/drivers/target/iscsi/iscsi_target_util.h
index 894d0f837924..4f8e01a47081 100644
--- a/drivers/target/iscsi/iscsi_target_util.h
+++ b/drivers/target/iscsi/iscsi_target_util.h
@@ -8,6 +8,7 @@ extern struct iscsi_r2t *iscsit_get_r2t_for_eos(struct iscsi_cmd *, u32, u32);
 extern struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *);
 extern void iscsit_free_r2t(struct iscsi_r2t *, struct iscsi_cmd *);
 extern void iscsit_free_r2ts_from_list(struct iscsi_cmd *);
+extern struct iscsi_cmd *iscsit_alloc_cmd(struct iscsi_conn *, gfp_t);
 extern struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *, gfp_t);
 extern struct iscsi_seq *iscsit_get_seq_holder_for_datain(struct iscsi_cmd *, u32);
 extern struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *);
diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c
index 17a6acbc3ab0..58ed683e04ae 100644
--- a/drivers/target/target_core_file.c
+++ b/drivers/target/target_core_file.c
@@ -30,8 +30,10 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/module.h>
+#include <linux/falloc.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_host.h>
+#include <asm/unaligned.h>
 
 #include <target/target_core_base.h>
 #include <target/target_core_backend.h>
@@ -166,6 +168,33 @@ static int fd_configure_device(struct se_device *dev)
                         " block_device blocks: %llu logical_block_size: %d\n",
                         dev_size, div_u64(dev_size, fd_dev->fd_block_size),
                         fd_dev->fd_block_size);
+                /*
+                 * Check if the underlying struct block_device request_queue supports
+                 * the QUEUE_FLAG_DISCARD bit for UNMAP/WRITE_SAME in SCSI + TRIM
+                 * in ATA and we need to set TPE=1
+                 */
+                if (blk_queue_discard(q)) {
+                        dev->dev_attrib.max_unmap_lba_count =
+                                q->limits.max_discard_sectors;
+                        /*
+                         * Currently hardcoded to 1 in Linux/SCSI code..
+                         */
+                        dev->dev_attrib.max_unmap_block_desc_count = 1;
+                        dev->dev_attrib.unmap_granularity =
+                                q->limits.discard_granularity >> 9;
+                        dev->dev_attrib.unmap_granularity_alignment =
+                                q->limits.discard_alignment;
+                        pr_debug("IFILE: BLOCK Discard support available,"
+                                        " disabled by default\n");
+                }
+                /*
+                 * Enable write same emulation for IBLOCK and use 0xFFFF as
+                 * the smaller WRITE_SAME(10) only has a two-byte block count.
+                 */
+                dev->dev_attrib.max_write_same_len = 0xFFFF;
+
+                if (blk_queue_nonrot(q))
+                        dev->dev_attrib.is_nonrot = 1;
         } else {
                 if (!(fd_dev->fbd_flags & FBDF_HAS_SIZE)) {
                         pr_err("FILEIO: Missing fd_dev_size="
@@ -176,6 +205,23 @@ static int fd_configure_device(struct se_device *dev)
 
                 dev->dev_attrib.hw_block_size = FD_BLOCKSIZE;
                 dev->dev_attrib.hw_max_sectors = FD_MAX_SECTORS;
+
+                /*
+                 * Limit UNMAP emulation to 8k Number of LBAs (NoLB)
+                 */
+                dev->dev_attrib.max_unmap_lba_count = 0x2000;
+                /*
+                 * Currently hardcoded to 1 in Linux/SCSI code..
+                 */
+                dev->dev_attrib.max_unmap_block_desc_count = 1;
+                dev->dev_attrib.unmap_granularity = 1;
+                dev->dev_attrib.unmap_granularity_alignment = 0;
+
+                /*
+                 * Limit WRITE_SAME w/ UNMAP=0 emulation to 8k Number of LBAs (NoLB)
+                 * based upon struct iovec limit for vfs_writev()
+                 */
+                dev->dev_attrib.max_write_same_len = 0x1000;
         }
 
         fd_dev->fd_block_size = dev->dev_attrib.hw_block_size;
@@ -190,11 +236,6 @@ static int fd_configure_device(struct se_device *dev)
 
         fd_dev->fd_dev_id = fd_host->fd_host_dev_id_count++;
         fd_dev->fd_queue_depth = dev->queue_depth;
-        /*
-         * Limit WRITE_SAME w/ UNMAP=0 emulation to 8k Number of LBAs (NoLB)
-         * based upon struct iovec limit for vfs_writev()
-         */
-        dev->dev_attrib.max_write_same_len = 0x1000;
 
         pr_debug("CORE_FILE[%u] - Added TCM FILEIO Device ID: %u at %s,"
                 " %llu total bytes\n", fd_host->fd_host_id, fd_dev->fd_dev_id,
@@ -442,6 +483,75 @@ fd_execute_write_same(struct se_cmd *cmd)
 }
 
 static sense_reason_t
+fd_do_unmap(struct se_cmd *cmd, void *priv, sector_t lba, sector_t nolb)
+{
+        struct file *file = priv;
+        struct inode *inode = file->f_mapping->host;
+        int ret;
+
+        if (S_ISBLK(inode->i_mode)) {
+                /* The backend is block device, use discard */
+                struct block_device *bdev = inode->i_bdev;
+
+                ret = blkdev_issue_discard(bdev, lba,
+                                nolb, GFP_KERNEL, 0);
+                if (ret < 0) {
+                        pr_warn("FILEIO: blkdev_issue_discard() failed: %d\n",
+                                ret);
+                        return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+                }
+        } else {
+                /* The backend is normal file, use fallocate */
+                struct se_device *se_dev = cmd->se_dev;
+                loff_t pos = lba * se_dev->dev_attrib.block_size;
+                unsigned int len = nolb * se_dev->dev_attrib.block_size;
+                int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
+
+                if (!file->f_op->fallocate)
+                        return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+
+                ret = file->f_op->fallocate(file, mode, pos, len);
+                if (ret < 0) {
+                        pr_warn("FILEIO: fallocate() failed: %d\n", ret);
+                        return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+                }
+        }
+
+        return 0;
+}
+
+static sense_reason_t
+fd_execute_write_same_unmap(struct se_cmd *cmd)
+{
+        struct se_device *se_dev = cmd->se_dev;
+        struct fd_dev *fd_dev = FD_DEV(se_dev);
+        struct file *file = fd_dev->fd_file;
+        sector_t lba = cmd->t_task_lba;
+        sector_t nolb = sbc_get_write_same_sectors(cmd);
+        int ret;
+
+        if (!nolb) {
+                target_complete_cmd(cmd, SAM_STAT_GOOD);
+                return 0;
+        }
+
+        ret = fd_do_unmap(cmd, file, lba, nolb);
+        if (ret)
+                return ret;
+
+        target_complete_cmd(cmd, GOOD);
+        return 0;
+}
+
+static sense_reason_t
+fd_execute_unmap(struct se_cmd *cmd)
+{
+        struct file *file = FD_DEV(cmd->se_dev)->fd_file;
+
+        return sbc_execute_unmap(cmd, fd_do_unmap, file);
+}
+
+static sense_reason_t
 fd_execute_rw(struct se_cmd *cmd)
 {
         struct scatterlist *sgl = cmd->t_data_sg;
@@ -600,6 +710,8 @@ static struct sbc_ops fd_sbc_ops = {
         .execute_rw             = fd_execute_rw,
         .execute_sync_cache     = fd_execute_sync_cache,
         .execute_write_same     = fd_execute_write_same,
+        .execute_write_same_unmap = fd_execute_write_same_unmap,
+        .execute_unmap          = fd_execute_unmap,
 };
 
 static sense_reason_t
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c
index 8bcc514ec8b6..07f5f94634bb 100644
--- a/drivers/target/target_core_iblock.c
+++ b/drivers/target/target_core_iblock.c
@@ -380,104 +380,40 @@ iblock_execute_sync_cache(struct se_cmd *cmd)
 }
 
 static sense_reason_t
-iblock_execute_unmap(struct se_cmd *cmd)
+iblock_do_unmap(struct se_cmd *cmd, void *priv,
+                sector_t lba, sector_t nolb)
 {
-        struct se_device *dev = cmd->se_dev;
-        struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
-        unsigned char *buf, *ptr = NULL;
-        sector_t lba;
-        int size;
-        u32 range;
-        sense_reason_t ret = 0;
-        int dl, bd_dl, err;
-
-        /* We never set ANC_SUP */
-        if (cmd->t_task_cdb[1])
-                return TCM_INVALID_CDB_FIELD;
-
-        if (cmd->data_length == 0) {
-                target_complete_cmd(cmd, SAM_STAT_GOOD);
-                return 0;
-        }
+        struct block_device *bdev = priv;
+        int ret;
 
-        if (cmd->data_length < 8) {
-                pr_warn("UNMAP parameter list length %u too small\n",
-                        cmd->data_length);
-                return TCM_PARAMETER_LIST_LENGTH_ERROR;
-        }
-
-        buf = transport_kmap_data_sg(cmd);
-        if (!buf)
+        ret = blkdev_issue_discard(bdev, lba, nolb, GFP_KERNEL, 0);
+        if (ret < 0) {
+                pr_err("blkdev_issue_discard() failed: %d\n", ret);
                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
-
-        dl = get_unaligned_be16(&buf[0]);
-        bd_dl = get_unaligned_be16(&buf[2]);
-
-        size = cmd->data_length - 8;
-        if (bd_dl > size)
-                pr_warn("UNMAP parameter list length %u too small, ignoring bd_dl %u\n",
-                        cmd->data_length, bd_dl);
-        else
-                size = bd_dl;
-
-        if (size / 16 > dev->dev_attrib.max_unmap_block_desc_count) {
-                ret = TCM_INVALID_PARAMETER_LIST;
-                goto err;
         }
 
-        /* First UNMAP block descriptor starts at 8 byte offset */
-        ptr = &buf[8];
-        pr_debug("UNMAP: Sub: %s Using dl: %u bd_dl: %u size: %u"
-                " ptr: %p\n", dev->transport->name, dl, bd_dl, size, ptr);
-
-        while (size >= 16) {
-                lba = get_unaligned_be64(&ptr[0]);
-                range = get_unaligned_be32(&ptr[8]);
-                pr_debug("UNMAP: Using lba: %llu and range: %u\n",
-                                 (unsigned long long)lba, range);
-
-                if (range > dev->dev_attrib.max_unmap_lba_count) {
-                        ret = TCM_INVALID_PARAMETER_LIST;
-                        goto err;
-                }
-
-                if (lba + range > dev->transport->get_blocks(dev) + 1) {
-                        ret = TCM_ADDRESS_OUT_OF_RANGE;
-                        goto err;
-                }
-
-                err = blkdev_issue_discard(ib_dev->ibd_bd, lba, range,
-                                           GFP_KERNEL, 0);
-                if (err < 0) {
-                        pr_err("blkdev_issue_discard() failed: %d\n",
-                                        err);
-                        ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
-                        goto err;
-                }
+        return 0;
+}
 
-                ptr += 16;
-                size -= 16;
-        }
+static sense_reason_t
+iblock_execute_unmap(struct se_cmd *cmd)
+{
+        struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
 
-err:
-        transport_kunmap_data_sg(cmd);
-        if (!ret)
-                target_complete_cmd(cmd, GOOD);
-        return ret;
+        return sbc_execute_unmap(cmd, iblock_do_unmap, bdev);
 }
 
 static sense_reason_t
 iblock_execute_write_same_unmap(struct se_cmd *cmd)
 {
-        struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
-        int rc;
-
-        rc = blkdev_issue_discard(ib_dev->ibd_bd, cmd->t_task_lba,
-                        sbc_get_write_same_sectors(cmd), GFP_KERNEL, 0);
-        if (rc < 0) {
-                pr_warn("blkdev_issue_discard() failed: %d\n", rc);
-                return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
-        }
+        struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
+        sector_t lba = cmd->t_task_lba;
+        sector_t nolb = sbc_get_write_same_sectors(cmd);
+        int ret;
+
+        ret = iblock_do_unmap(cmd, bdev, lba, nolb);
+        if (ret)
+                return ret;
 
         target_complete_cmd(cmd, GOOD);
         return 0;
diff --git a/drivers/target/target_core_sbc.c b/drivers/target/target_core_sbc.c
index 60d4b5185f32..bbc5b0ee2bdc 100644
--- a/drivers/target/target_core_sbc.c
+++ b/drivers/target/target_core_sbc.c
@@ -596,3 +596,88 @@ u32 sbc_get_device_type(struct se_device *dev)
         return TYPE_DISK;
 }
 EXPORT_SYMBOL(sbc_get_device_type);
+
+sense_reason_t
+sbc_execute_unmap(struct se_cmd *cmd,
+        sense_reason_t (*do_unmap_fn)(struct se_cmd *, void *,
+                                      sector_t, sector_t),
+        void *priv)
+{
+        struct se_device *dev = cmd->se_dev;
+        unsigned char *buf, *ptr = NULL;
+        sector_t lba;
+        int size;
+        u32 range;
+        sense_reason_t ret = 0;
+        int dl, bd_dl;
+
+        /* We never set ANC_SUP */
+        if (cmd->t_task_cdb[1])
+                return TCM_INVALID_CDB_FIELD;
+
+        if (cmd->data_length == 0) {
+                target_complete_cmd(cmd, SAM_STAT_GOOD);
+                return 0;
+        }
+
+        if (cmd->data_length < 8) {
+                pr_warn("UNMAP parameter list length %u too small\n",
+                        cmd->data_length);
+                return TCM_PARAMETER_LIST_LENGTH_ERROR;
+        }
+
+        buf = transport_kmap_data_sg(cmd);
+        if (!buf)
+                return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+
+        dl = get_unaligned_be16(&buf[0]);
+        bd_dl = get_unaligned_be16(&buf[2]);
+
+        size = cmd->data_length - 8;
+        if (bd_dl > size)
+                pr_warn("UNMAP parameter list length %u too small, ignoring bd_dl %u\n",
+                        cmd->data_length, bd_dl);
+        else
+                size = bd_dl;
+
+        if (size / 16 > dev->dev_attrib.max_unmap_block_desc_count) {
+                ret = TCM_INVALID_PARAMETER_LIST;
+                goto err;
+        }
+
+        /* First UNMAP block descriptor starts at 8 byte offset */
+        ptr = &buf[8];
+        pr_debug("UNMAP: Sub: %s Using dl: %u bd_dl: %u size: %u"
+                " ptr: %p\n", dev->transport->name, dl, bd_dl, size, ptr);
+
+        while (size >= 16) {
+                lba = get_unaligned_be64(&ptr[0]);
+                range = get_unaligned_be32(&ptr[8]);
+                pr_debug("UNMAP: Using lba: %llu and range: %u\n",
+                                 (unsigned long long)lba, range);
+
+                if (range > dev->dev_attrib.max_unmap_lba_count) {
+                        ret = TCM_INVALID_PARAMETER_LIST;
+                        goto err;
+                }
+
+                if (lba + range > dev->transport->get_blocks(dev) + 1) {
+                        ret = TCM_ADDRESS_OUT_OF_RANGE;
+                        goto err;
+                }
+
+                ret = do_unmap_fn(cmd, priv, lba, range);
+                if (ret)
+                        goto err;
+
+                ptr += 16;
+                size -= 16;
+        }
+
+err:
+        transport_kunmap_data_sg(cmd);
+        if (!ret)
+                target_complete_cmd(cmd, GOOD);
+        return ret;
+}
+EXPORT_SYMBOL(sbc_execute_unmap);
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 3243ea790eab..f8388b4024aa 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -65,7 +65,6 @@ static void transport_complete_task_attr(struct se_cmd *cmd);
 static void transport_handle_queue_full(struct se_cmd *cmd,
                 struct se_device *dev);
 static int transport_generic_get_mem(struct se_cmd *cmd);
-static int target_get_sess_cmd(struct se_session *, struct se_cmd *, bool);
 static void transport_put_cmd(struct se_cmd *cmd);
 static void target_complete_ok_work(struct work_struct *work);
 
@@ -2179,7 +2178,7 @@ EXPORT_SYMBOL(transport_generic_free_cmd);
  * @se_cmd:     command descriptor to add
  * @ack_kref:   Signal that fabric will perform an ack target_put_sess_cmd()
  */
-static int target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd,
+int target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd,
                                bool ack_kref)
 {
         unsigned long flags;
@@ -2208,6 +2207,7 @@ out:
         spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
         return ret;
 }
+EXPORT_SYMBOL(target_get_sess_cmd);
 
 static void target_release_cmd_kref(struct kref *kref)
 {
@@ -2765,8 +2765,13 @@ transport_send_check_condition_and_sense(struct se_cmd *cmd,
                 /* CURRENT ERROR */
                 buffer[0] = 0x70;
                 buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
-                /* ILLEGAL REQUEST */
-                buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
+                /*
+                 * Returning ILLEGAL REQUEST would cause immediate IO errors on
+                 * Solaris initiators.  Returning NOT READY instead means the
+                 * operations will be retried a finite number of times and we
+                 * can survive intermittent errors.
+                 */
+                buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY;
                 /* LOGICAL UNIT COMMUNICATION FAILURE */
                 buffer[SPC_ASC_KEY_OFFSET] = 0x08;
                 break;
diff --git a/drivers/target/tcm_fc/tfc_io.c b/drivers/target/tcm_fc/tfc_io.c
index b6fd4cf42840..e415af32115a 100644
--- a/drivers/target/tcm_fc/tfc_io.c
+++ b/drivers/target/tcm_fc/tfc_io.c
@@ -103,6 +103,13 @@ int ft_queue_data_in(struct se_cmd *se_cmd)
         use_sg = !(remaining % 4);
 
         while (remaining) {
+                struct fc_seq *seq = cmd->seq;
+
+                if (!seq) {
+                        pr_debug("%s: Command aborted, xid 0x%x\n",
+                                 __func__, ep->xid);
+                        break;
+                }
                 if (!mem_len) {
                         sg = sg_next(sg);
                         mem_len = min((size_t)sg->length, remaining);
@@ -169,7 +176,7 @@ int ft_queue_data_in(struct se_cmd *se_cmd)
                         f_ctl |= FC_FC_END_SEQ;
                 fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid,
                                FC_TYPE_FCP, f_ctl, fh_off);
-                error = lport->tt.seq_send(lport, cmd->seq, fp);
+                error = lport->tt.seq_send(lport, seq, fp);
                 if (error) {
                         /* XXX For now, initiator will retry */
                         pr_err_ratelimited("%s: Failed to send frame %p, "
diff --git a/drivers/target/tcm_fc/tfc_sess.c b/drivers/target/tcm_fc/tfc_sess.c
index 113f33598b9f..4859505ae2ed 100644
--- a/drivers/target/tcm_fc/tfc_sess.c
+++ b/drivers/target/tcm_fc/tfc_sess.c
@@ -428,19 +428,12 @@ static int ft_prli(struct fc_rport_priv *rdata, u32 spp_len,
         return ret;
 }
 
-static void ft_sess_rcu_free(struct rcu_head *rcu)
-{
-        struct ft_sess *sess = container_of(rcu, struct ft_sess, rcu);
-
-        kfree(sess);
-}
-
 static void ft_sess_free(struct kref *kref)
 {
         struct ft_sess *sess = container_of(kref, struct ft_sess, kref);
 
         transport_deregister_session(sess->se_sess);
-        call_rcu(&sess->rcu, ft_sess_rcu_free);
+        kfree_rcu(sess, rcu);
 }
 
 void ft_sess_put(struct ft_sess *sess)
diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
index 957a0b98a5d9..1677238d281f 100644
--- a/drivers/vhost/tcm_vhost.c
+++ b/drivers/vhost/tcm_vhost.c
@@ -66,11 +66,13 @@ enum {
  * TODO: debug and remove the workaround.
  */
 enum {
-        VHOST_SCSI_FEATURES = VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX)
+        VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX)) |
+                              (1ULL << VIRTIO_SCSI_F_HOTPLUG)
 };
 
 #define VHOST_SCSI_MAX_TARGET   256
 #define VHOST_SCSI_MAX_VQ       128
+#define VHOST_SCSI_MAX_EVENT    128
 
 struct vhost_scsi {
         /* Protected by vhost_scsi->dev.mutex */
@@ -82,6 +84,12 @@ struct vhost_scsi {
 
         struct vhost_work vs_completion_work; /* cmd completion work item */
         struct llist_head vs_completion_list; /* cmd completion queue */
+
+        struct vhost_work vs_event_work; /* evt injection work item */
+        struct llist_head vs_event_list; /* evt injection queue */
+
+        bool vs_events_missed; /* any missed events, protected by vq->mutex */
+        int vs_events_nr; /* num of pending events, protected by vq->mutex */
 };
 
 /* Local pointer to allocated TCM configfs fabric module */
@@ -349,6 +357,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)
         return 0;
 }
 
+static void tcm_vhost_free_evt(struct vhost_scsi *vs, struct tcm_vhost_evt *evt)
+{
+        vs->vs_events_nr--;
+        kfree(evt);
+}
+
+static struct tcm_vhost_evt *tcm_vhost_allocate_evt(struct vhost_scsi *vs,
+        u32 event, u32 reason)
+{
+        struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT];
+        struct tcm_vhost_evt *evt;
+
+        if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
+                vs->vs_events_missed = true;
+                return NULL;
+        }
+
+        evt = kzalloc(sizeof(*evt), GFP_KERNEL);
+        if (!evt) {
+                vq_err(vq, "Failed to allocate tcm_vhost_evt\n");
+                vs->vs_events_missed = true;
+                return NULL;
+        }
+
+        evt->event.event = event;
+        evt->event.reason = reason;
+        vs->vs_events_nr++;
+
+        return evt;
+}
+
 static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)
 {
         struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;
@@ -367,6 +406,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)
         kfree(tv_cmd);
 }
 
+static void tcm_vhost_do_evt_work(struct vhost_scsi *vs,
+        struct tcm_vhost_evt *evt)
+{
+        struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT];
+        struct virtio_scsi_event *event = &evt->event;
+        struct virtio_scsi_event __user *eventp;
+        unsigned out, in;
+        int head, ret;
+
+        if (!vq->private_data) {
+                vs->vs_events_missed = true;
+                return;
+        }
+
+again:
+        vhost_disable_notify(&vs->dev, vq);
+        head = vhost_get_vq_desc(&vs->dev, vq, vq->iov,
+                        ARRAY_SIZE(vq->iov), &out, &in,
+                        NULL, NULL);
+        if (head < 0) {
+                vs->vs_events_missed = true;
+                return;
+        }
+        if (head == vq->num) {
+                if (vhost_enable_notify(&vs->dev, vq))
+                        goto again;
+                vs->vs_events_missed = true;
+                return;
+        }
+
+        if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
+                vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
+                                vq->iov[out].iov_len);
+                vs->vs_events_missed = true;
+                return;
+        }
+
+        if (vs->vs_events_missed) {
+                event->event |= VIRTIO_SCSI_T_EVENTS_MISSED;
+                vs->vs_events_missed = false;
+        }
+
+        eventp = vq->iov[out].iov_base;
+        ret = __copy_to_user(eventp, event, sizeof(*event));
+        if (!ret)
+                vhost_add_used_and_signal(&vs->dev, vq, head, 0);
+        else
+                vq_err(vq, "Faulted on tcm_vhost_send_event\n");
+}
+
+static void tcm_vhost_evt_work(struct vhost_work *work)
+{
+        struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
+                                        vs_event_work);
+        struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT];
+        struct tcm_vhost_evt *evt;
+        struct llist_node *llnode;
+
+        mutex_lock(&vq->mutex);
+        llnode = llist_del_all(&vs->vs_event_list);
+        while (llnode) {
+                evt = llist_entry(llnode, struct tcm_vhost_evt, list);
+                llnode = llist_next(llnode);
+                tcm_vhost_do_evt_work(vs, evt);
+                tcm_vhost_free_evt(vs, evt);
+        }
+        mutex_unlock(&vq->mutex);
+}
+
 /* Fill in status and signal that we are done processing this command
  *
  * This is scheduled in the vhost work queue so we are called with the owner
@@ -777,9 +885,46 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
         pr_debug("%s: The handling func for control queue.\n", __func__);
 }
 
+static void tcm_vhost_send_evt(struct vhost_scsi *vs, struct tcm_vhost_tpg *tpg,
+        struct se_lun *lun, u32 event, u32 reason)
+{
+        struct tcm_vhost_evt *evt;
+
+        evt = tcm_vhost_allocate_evt(vs, event, reason);
+        if (!evt)
+                return;
+
+        if (tpg && lun) {
+                /* TODO: share lun setup code with virtio-scsi.ko */
+                /*
+                 * Note: evt->event is zeroed when we allocate it and
+                 * lun[4-7] need to be zero according to virtio-scsi spec.
+                 */
+                evt->event.lun[0] = 0x01;
+                evt->event.lun[1] = tpg->tport_tpgt & 0xFF;
+                if (lun->unpacked_lun >= 256)
+                        evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;
+                evt->event.lun[3] = lun->unpacked_lun & 0xFF;
+        }
+
+        llist_add(&evt->list, &vs->vs_event_list);
+        vhost_work_queue(&vs->dev, &vs->vs_event_work);
+}
+
 static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
 {
-        pr_debug("%s: The handling func for event queue.\n", __func__);
+        struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
+                                                poll.work);
+        struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
+
+        mutex_lock(&vq->mutex);
+        if (!vq->private_data)
+                goto out;
+
+        if (vs->vs_events_missed)
+                tcm_vhost_send_evt(vs, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
+out:
+        mutex_unlock(&vq->mutex);
 }
 
 static void vhost_scsi_handle_kick(struct vhost_work *work)
@@ -803,11 +948,15 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)
         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
                 vhost_scsi_flush_vq(vs, i);
         vhost_work_flush(&vs->dev, &vs->vs_completion_work);
+        vhost_work_flush(&vs->dev, &vs->vs_event_work);
 }
 
 /*
  * Called from vhost_scsi_ioctl() context to walk the list of available
  * tcm_vhost_tpg with an active struct tcm_vhost_nexus
+ *
+ *  The lock nesting rule is:
+ *    tcm_vhost_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex
  */
 static int vhost_scsi_set_endpoint(
         struct vhost_scsi *vs,
@@ -820,26 +969,27 @@ static int vhost_scsi_set_endpoint(
         int index, ret, i, len;
         bool match = false;
 
+        mutex_lock(&tcm_vhost_mutex);
         mutex_lock(&vs->dev.mutex);
+
         /* Verify that ring has been setup correctly. */
         for (index = 0; index < vs->dev.nvqs; ++index) {
                 /* Verify that ring has been setup correctly. */
                 if (!vhost_vq_access_ok(&vs->vqs[index])) {
-                        mutex_unlock(&vs->dev.mutex);
-                        return -EFAULT;
+                        ret = -EFAULT;
+                        goto out;
                 }
         }
 
         len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
         vs_tpg = kzalloc(len, GFP_KERNEL);
         if (!vs_tpg) {
-                mutex_unlock(&vs->dev.mutex);
-                return -ENOMEM;
+                ret = -ENOMEM;
+                goto out;
         }
         if (vs->vs_tpg)
                 memcpy(vs_tpg, vs->vs_tpg, len);
 
-        mutex_lock(&tcm_vhost_mutex);
         list_for_each_entry(tv_tpg, &tcm_vhost_list, tv_tpg_list) {
                 mutex_lock(&tv_tpg->tv_tpg_mutex);
                 if (!tv_tpg->tpg_nexus) {
@@ -854,20 +1004,19 @@ static int vhost_scsi_set_endpoint(
 
                 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
                         if (vs->vs_tpg && vs->vs_tpg[tv_tpg->tport_tpgt]) {
-                                mutex_unlock(&tv_tpg->tv_tpg_mutex);
-                                mutex_unlock(&tcm_vhost_mutex);
-                                mutex_unlock(&vs->dev.mutex);
                                 kfree(vs_tpg);
-                                return -EEXIST;
+                                mutex_unlock(&tv_tpg->tv_tpg_mutex);
+                                ret = -EEXIST;
+                                goto out;
                         }
                         tv_tpg->tv_tpg_vhost_count++;
+                        tv_tpg->vhost_scsi = vs;
                         vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;
                         smp_mb__after_atomic_inc();
                         match = true;
                 }
                 mutex_unlock(&tv_tpg->tv_tpg_mutex);
         }
-        mutex_unlock(&tcm_vhost_mutex);
 
         if (match) {
                 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
@@ -893,7 +1042,9 @@ static int vhost_scsi_set_endpoint(
         kfree(vs->vs_tpg);
         vs->vs_tpg = vs_tpg;
 
+out:
         mutex_unlock(&vs->dev.mutex);
+        mutex_unlock(&tcm_vhost_mutex);
         return ret;
 }
 
@@ -908,6 +1059,7 @@ static int vhost_scsi_clear_endpoint(
         int index, ret, i;
         u8 target;
 
+        mutex_lock(&tcm_vhost_mutex);
         mutex_lock(&vs->dev.mutex);
         /* Verify that ring has been setup correctly. */
         for (index = 0; index < vs->dev.nvqs; ++index) {
@@ -918,8 +1070,8 @@ static int vhost_scsi_clear_endpoint(
         }
 
         if (!vs->vs_tpg) {
-                mutex_unlock(&vs->dev.mutex);
-                return 0;
+                ret = 0;
+                goto err_dev;
         }
 
         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
@@ -944,6 +1096,7 @@ static int vhost_scsi_clear_endpoint(
                         goto err_tpg;
                 }
                 tv_tpg->tv_tpg_vhost_count--;
+                tv_tpg->vhost_scsi = NULL;
                 vs->vs_tpg[target] = NULL;
                 match = true;
                 mutex_unlock(&tv_tpg->tv_tpg_mutex);
@@ -964,14 +1117,16 @@ static int vhost_scsi_clear_endpoint(
         vhost_scsi_flush(vs);
         kfree(vs->vs_tpg);
         vs->vs_tpg = NULL;
+        WARN_ON(vs->vs_events_nr);
         mutex_unlock(&vs->dev.mutex);
-
+        mutex_unlock(&tcm_vhost_mutex);
         return 0;
 
 err_tpg:
         mutex_unlock(&tv_tpg->tv_tpg_mutex);
 err_dev:
         mutex_unlock(&vs->dev.mutex);
+        mutex_unlock(&tcm_vhost_mutex);
         return ret;
 }
 
@@ -1003,6 +1158,10 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)
                 return -ENOMEM;
 
         vhost_work_init(&s->vs_completion_work, vhost_scsi_complete_cmd_work);
+        vhost_work_init(&s->vs_event_work, tcm_vhost_evt_work);
+
+        s->vs_events_nr = 0;
+        s->vs_events_missed = false;
 
         s->vqs[VHOST_SCSI_VQ_CTL].handle_kick = vhost_scsi_ctl_handle_kick;
         s->vqs[VHOST_SCSI_VQ_EVT].handle_kick = vhost_scsi_evt_handle_kick;
@@ -1029,6 +1188,8 @@ static int vhost_scsi_release(struct inode *inode, struct file *f)
         vhost_scsi_clear_endpoint(s, &t);
         vhost_dev_stop(&s->dev);
         vhost_dev_cleanup(&s->dev, false);
+        /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */
+        vhost_scsi_flush(s);
         kfree(s);
         return 0;
 }
@@ -1040,8 +1201,11 @@ static long vhost_scsi_ioctl(struct file *f, unsigned int ioctl,
         struct vhost_scsi_target backend;
         void __user *argp = (void __user *)arg;
         u64 __user *featurep = argp;
+        u32 __user *eventsp = argp;
+        u32 events_missed;
         u64 features;
         int r, abi_version = VHOST_SCSI_ABI_VERSION;
+        struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT];
 
         switch (ioctl) {
         case VHOST_SCSI_SET_ENDPOINT:
@@ -1062,6 +1226,20 @@ static long vhost_scsi_ioctl(struct file *f, unsigned int ioctl,
                 if (copy_to_user(argp, &abi_version, sizeof abi_version))
                         return -EFAULT;
                 return 0;
+        case VHOST_SCSI_SET_EVENTS_MISSED:
+                if (get_user(events_missed, eventsp))
+                        return -EFAULT;
+                mutex_lock(&vq->mutex);
+                vs->vs_events_missed = events_missed;
+                mutex_unlock(&vq->mutex);
+                return 0;
+        case VHOST_SCSI_GET_EVENTS_MISSED:
+                mutex_lock(&vq->mutex);
+                events_missed = vs->vs_events_missed;
+                mutex_unlock(&vq->mutex);
+                if (put_user(events_missed, eventsp))
+                        return -EFAULT;
+                return 0;
         case VHOST_GET_FEATURES:
                 features = VHOST_SCSI_FEATURES;
                 if (copy_to_user(featurep, &features, sizeof features))
@@ -1133,28 +1311,80 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)
         return "Unknown";
 }
 
+static void tcm_vhost_do_plug(struct tcm_vhost_tpg *tpg,
+        struct se_lun *lun, bool plug)
+{
+
+        struct vhost_scsi *vs = tpg->vhost_scsi;
+        struct vhost_virtqueue *vq;
+        u32 reason;
+
+        if (!vs)
+                return;
+
+        mutex_lock(&vs->dev.mutex);
+        if (!vhost_has_feature(&vs->dev, VIRTIO_SCSI_F_HOTPLUG)) {
+                mutex_unlock(&vs->dev.mutex);
+                return;
+        }
+
+        if (plug)
+                reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
+        else
+                reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
+
+        vq = &vs->vqs[VHOST_SCSI_VQ_EVT];
+        mutex_lock(&vq->mutex);
+        tcm_vhost_send_evt(vs, tpg, lun,
+                        VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
+        mutex_unlock(&vq->mutex);
+        mutex_unlock(&vs->dev.mutex);
+}
+
+static void tcm_vhost_hotplug(struct tcm_vhost_tpg *tpg, struct se_lun *lun)
+{
+        tcm_vhost_do_plug(tpg, lun, true);
+}
+
+static void tcm_vhost_hotunplug(struct tcm_vhost_tpg *tpg, struct se_lun *lun)
+{
+        tcm_vhost_do_plug(tpg, lun, false);
+}
+
 static int tcm_vhost_port_link(struct se_portal_group *se_tpg,
         struct se_lun *lun)
 {
         struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg,
                                 struct tcm_vhost_tpg, se_tpg);
 
+        mutex_lock(&tcm_vhost_mutex);
+
         mutex_lock(&tv_tpg->tv_tpg_mutex);
         tv_tpg->tv_tpg_port_count++;
         mutex_unlock(&tv_tpg->tv_tpg_mutex);
 
+        tcm_vhost_hotplug(tv_tpg, lun);
+
+        mutex_unlock(&tcm_vhost_mutex);
+
         return 0;
 }
 
 static void tcm_vhost_port_unlink(struct se_portal_group *se_tpg,
-        struct se_lun *se_lun)
+        struct se_lun *lun)
 {
         struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg,
                                 struct tcm_vhost_tpg, se_tpg);
 
+        mutex_lock(&tcm_vhost_mutex);
+
         mutex_lock(&tv_tpg->tv_tpg_mutex);
         tv_tpg->tv_tpg_port_count--;
         mutex_unlock(&tv_tpg->tv_tpg_mutex);
+
+        tcm_vhost_hotunplug(tv_tpg, lun);
+
+        mutex_unlock(&tcm_vhost_mutex);
 }
 
 static struct se_node_acl *tcm_vhost_make_nodeacl(
diff --git a/drivers/vhost/tcm_vhost.h b/drivers/vhost/tcm_vhost.h
index 1d2ae7a60e11..514b9fda230e 100644
--- a/drivers/vhost/tcm_vhost.h
+++ b/drivers/vhost/tcm_vhost.h
@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {
         struct se_node_acl se_node_acl;
 };
 
+struct vhost_scsi;
 struct tcm_vhost_tpg {
         /* Vhost port target portal group tag for TCM */
         u16 tport_tpgt;
@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {
         struct tcm_vhost_tport *tport;
         /* Returned by tcm_vhost_make_tpg() */
         struct se_portal_group se_tpg;
+        /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
+        struct vhost_scsi *vhost_scsi;
 };
 
 struct tcm_vhost_tport {
@@ -83,6 +86,13 @@ struct tcm_vhost_tport {
         struct se_wwn tport_wwn;
 };
 
+struct tcm_vhost_evt {
+        /* event to be sent to guest */
+        struct virtio_scsi_event event;
+        /* event list, serviced from vhost worker thread */
+        struct llist_node list;
+};
+
 /*
  * As per request from MST, keep TCM_VHOST related ioctl defines out of
  * linux/vhost.h (user-space) for now..
@@ -113,3 +123,6 @@ struct vhost_scsi_target {
 #define VHOST_SCSI_CLEAR_ENDPOINT _IOW(VHOST_VIRTIO, 0x41, struct vhost_scsi_target)
 /* Changing this breaks userspace. */
 #define VHOST_SCSI_GET_ABI_VERSION _IOW(VHOST_VIRTIO, 0x42, int)
+/* Set and get the events missed flag */
+#define VHOST_SCSI_SET_EVENTS_MISSED _IOW(VHOST_VIRTIO, 0x43, __u32)
+#define VHOST_SCSI_GET_EVENTS_MISSED _IOW(VHOST_VIRTIO, 0x44, __u32)
diff --git a/include/target/iscsi/iscsi_transport.h b/include/target/iscsi/iscsi_transport.h
new file mode 100644
index 000000000000..23a87d0cd72c
--- /dev/null
+++ b/include/target/iscsi/iscsi_transport.h
@@ -0,0 +1,83 @@
+#include <linux/module.h>
+#include <linux/list.h>
+#include "../../../drivers/target/iscsi/iscsi_target_core.h"
+
+struct iscsit_transport {
+#define ISCSIT_TRANSPORT_NAME   16
+        char name[ISCSIT_TRANSPORT_NAME];
+        int transport_type;
+        struct module *owner;
+        struct list_head t_node;
+        int (*iscsit_setup_np)(struct iscsi_np *, struct __kernel_sockaddr_storage *);
+        int (*iscsit_accept_np)(struct iscsi_np *, struct iscsi_conn *);
+        void (*iscsit_free_np)(struct iscsi_np *);
+        void (*iscsit_free_conn)(struct iscsi_conn *);
+        struct iscsi_cmd *(*iscsit_alloc_cmd)(struct iscsi_conn *, gfp_t);
+        int (*iscsit_get_login_rx)(struct iscsi_conn *, struct iscsi_login *);
+        int (*iscsit_put_login_tx)(struct iscsi_conn *, struct iscsi_login *, u32);
+        int (*iscsit_immediate_queue)(struct iscsi_conn *, struct iscsi_cmd *, int);
+        int (*iscsit_response_queue)(struct iscsi_conn *, struct iscsi_cmd *, int);
+        int (*iscsit_get_dataout)(struct iscsi_conn *, struct iscsi_cmd *, bool);
+        int (*iscsit_queue_data_in)(struct iscsi_conn *, struct iscsi_cmd *);
+        int (*iscsit_queue_status)(struct iscsi_conn *, struct iscsi_cmd *);
+};
+
+/*
+ * From iscsi_target_transport.c
+ */
+
+extern int iscsit_register_transport(struct iscsit_transport *);
+extern void iscsit_unregister_transport(struct iscsit_transport *);
+extern struct iscsit_transport *iscsit_get_transport(int);
+extern void iscsit_put_transport(struct iscsit_transport *);
+
+/*
+ * From iscsi_target.c
+ */
+extern int iscsit_add_reject_from_cmd(u8, int, int, unsigned char *,
+                                struct iscsi_cmd *);
+extern int iscsit_setup_scsi_cmd(struct iscsi_conn *, struct iscsi_cmd *,
+                                unsigned char *);
+extern void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *);
+extern int iscsit_process_scsi_cmd(struct iscsi_conn *, struct iscsi_cmd *,
+                                struct iscsi_scsi_req *);
+extern int iscsit_check_dataout_hdr(struct iscsi_conn *, unsigned char *,
+                                struct iscsi_cmd **);
+extern int iscsit_check_dataout_payload(struct iscsi_cmd *, struct iscsi_data *,
+                                bool);
+extern int iscsit_handle_nop_out(struct iscsi_conn *, struct iscsi_cmd *,
+                                unsigned char *);
+extern int iscsit_handle_logout_cmd(struct iscsi_conn *, struct iscsi_cmd *,
+                                unsigned char *);
+extern int iscsit_handle_task_mgt_cmd(struct iscsi_conn *, struct iscsi_cmd *,
+                                unsigned char *);
+extern void iscsit_build_rsp_pdu(struct iscsi_cmd *, struct iscsi_conn *,
+                                bool, struct iscsi_scsi_rsp *);
+extern void iscsit_build_nopin_rsp(struct iscsi_cmd *, struct iscsi_conn *,
+                                struct iscsi_nopin *, bool);
+extern void iscsit_build_task_mgt_rsp(struct iscsi_cmd *, struct iscsi_conn *,
+                                struct iscsi_tm_rsp *);
+extern void iscsit_build_reject(struct iscsi_cmd *, struct iscsi_conn *,
+                                struct iscsi_reject *);
+extern int iscsit_build_logout_rsp(struct iscsi_cmd *, struct iscsi_conn *,
+                                struct iscsi_logout_rsp *);
+extern int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *);
+/*
+ * From iscsi_target_device.c
+ */
+extern void iscsit_increment_maxcmdsn(struct iscsi_cmd *, struct iscsi_session *);
+/*
+ * From iscsi_target_erl1.c
+ */
+extern void iscsit_stop_dataout_timer(struct iscsi_cmd *);
+
+/*
+ * From iscsi_target_tmr.c
+ */
+extern int iscsit_tmr_post_handler(struct iscsi_cmd *, struct iscsi_conn *);
+
+/*
+ * From iscsi_target_util.c
+ */
+extern struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *, gfp_t);
+extern int iscsit_sequence_cmd(struct iscsi_conn *, struct iscsi_cmd *, __be32);
diff --git a/include/target/target_core_backend.h b/include/target/target_core_backend.h
index b128c20770bc..ffa2696d64dc 100644
--- a/include/target/target_core_backend.h
+++ b/include/target/target_core_backend.h
@@ -60,6 +60,10 @@ sense_reason_t	sbc_parse_cdb(struct se_cmd *cmd, struct sbc_ops *ops);
 u32     sbc_get_device_rev(struct se_device *dev);
 u32     sbc_get_device_type(struct se_device *dev);
 sector_t        sbc_get_write_same_sectors(struct se_cmd *cmd);
+sense_reason_t sbc_execute_unmap(struct se_cmd *cmd,
+        sense_reason_t (*do_unmap_fn)(struct se_cmd *cmd, void *priv,
+                                      sector_t lba, sector_t nolb),
+        void *priv);
 
 void    transport_set_vpd_proto_id(struct t10_vpd *, unsigned char *);
 int     transport_set_vpd_assoc(struct t10_vpd *, unsigned char *);
diff --git a/include/target/target_core_fabric.h b/include/target/target_core_fabric.h
index aaa1ee6ab391..ba3471b73c07 100644
--- a/include/target/target_core_fabric.h
+++ b/include/target/target_core_fabric.h
@@ -120,7 +120,7 @@ bool	transport_wait_for_tasks(struct se_cmd *);
 int     transport_check_aborted_status(struct se_cmd *, int);
 int     transport_send_check_condition_and_sense(struct se_cmd *,
                 sense_reason_t, int);
-
+int     target_get_sess_cmd(struct se_session *, struct se_cmd *, bool);
 int     target_put_sess_cmd(struct se_session *, struct se_cmd *);
 void    target_sess_cmd_list_set_waiting(struct se_session *);
 void    target_wait_for_sess_cmds(struct se_session *, int);